From fc7be6708f1c670436148fd59dec72a7e68e2b96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= Date: Sat, 6 Sep 2025 11:15:11 +0200 Subject: [PATCH] feat(api): nuq (#1984) --- .github/k8s/playwright.yml | 62 ++ .github/k8s/searxng.yml | 59 ++ .github/k8s/server-nsh.yml | 114 ++++ .github/k8s/server.yml | 305 ++++++++++ .github/k8s/tailscale-proxy/Dockerfile | 28 + .github/k8s/tailscale-proxy/entrypoint.sh | 48 ++ .github/k8s/tailscale-proxy/nginx.conf | 24 + .github/workflows/test-server-self-host.yml | 145 ----- .github/workflows/test-server.yml | 462 +++++++++++--- apps/api/Dockerfile | 6 +- apps/api/docker-entrypoint.sh | 24 - apps/api/package.json | 18 +- apps/api/pnpm-lock.yaml | 93 ++- apps/api/requests.http | 4 +- apps/api/sharedLibs/go-html-to-md/.gitignore | 3 +- .../queue-concurrency-integration.test.ts | 269 --------- apps/api/src/__tests__/snips/lib.ts | 2 +- apps/api/src/__tests__/snips/v1/crawl.test.ts | 107 ---- apps/api/src/__tests__/snips/v1/zdr.test.ts | 41 +- apps/api/src/__tests__/snips/v2/crawl.test.ts | 107 ---- apps/api/src/__tests__/snips/v2/zdr.test.ts | 41 +- apps/api/src/controllers/v0/admin/metrics.ts | 5 + apps/api/src/controllers/v0/admin/queue.ts | 201 ------- apps/api/src/controllers/v0/crawl-status.ts | 41 +- apps/api/src/controllers/v0/crawl.ts | 17 +- apps/api/src/controllers/v0/scrape.ts | 11 +- apps/api/src/controllers/v0/search.ts | 19 +- apps/api/src/controllers/v1/batch-scrape.ts | 8 +- apps/api/src/controllers/v1/crawl-errors.ts | 57 +- .../api/src/controllers/v1/crawl-status-ws.ts | 58 +- apps/api/src/controllers/v1/crawl-status.ts | 103 ++-- apps/api/src/controllers/v1/crawl.ts | 2 - apps/api/src/controllers/v1/extract-status.ts | 19 +- apps/api/src/controllers/v1/scrape.ts | 17 +- apps/api/src/controllers/v1/search.ts | 23 +- apps/api/src/controllers/v1/x402-search.ts | 22 +- apps/api/src/controllers/v2/batch-scrape.ts | 8 +- apps/api/src/controllers/v2/crawl-errors.ts | 57 +- .../api/src/controllers/v2/crawl-status-ws.ts | 58 +- apps/api/src/controllers/v2/crawl-status.ts | 109 ++-- apps/api/src/controllers/v2/crawl.ts | 2 - apps/api/src/controllers/v2/extract-status.ts | 19 +- apps/api/src/controllers/v2/scrape.ts | 22 +- apps/api/src/controllers/v2/search.ts | 20 +- apps/api/src/harness.ts | 137 +++-- apps/api/src/index.ts | 101 +--- apps/api/src/lib/concurrency-limit.ts | 16 +- apps/api/src/lib/extract/document-scraper.ts | 12 +- .../lib/extract/fire-0/document-scraper-f0.ts | 11 +- apps/api/src/lib/logger.ts | 12 + apps/api/src/lib/scrape-events.ts | 109 ---- apps/api/src/main/runWebScraper.ts | 46 +- apps/api/src/routes/admin.ts | 23 - apps/api/src/services/alerts/index.ts | 62 -- .../api/src/services/billing/batch_billing.ts | 7 +- .../api/src/services/indexing/index-worker.ts | 2 - apps/api/src/services/logging/log_job.ts | 8 +- apps/api/src/services/queue-jobs.ts | 161 ++--- apps/api/src/services/queue-service.ts | 33 +- apps/api/src/services/queue-worker.ts | 179 +----- apps/api/src/services/worker/crawl-logic.ts | 20 +- apps/api/src/services/worker/nuq-worker.ts | 119 ++++ apps/api/src/services/worker/nuq.ts | 568 ++++++++++++++++++ apps/api/src/services/worker/scrape-worker.ts | 57 +- apps/nuq-postgres/Dockerfile | 24 + apps/nuq-postgres/nuq.sql | 51 ++ docker-compose.yaml | 28 +- 67 files changed, 2529 insertions(+), 2117 deletions(-) create mode 100644 .github/k8s/playwright.yml create mode 100644 .github/k8s/searxng.yml create mode 100644 .github/k8s/server-nsh.yml create mode 100644 .github/k8s/server.yml create mode 100644 .github/k8s/tailscale-proxy/Dockerfile create mode 100644 .github/k8s/tailscale-proxy/entrypoint.sh create mode 100644 .github/k8s/tailscale-proxy/nginx.conf delete mode 100644 .github/workflows/test-server-self-host.yml delete mode 100755 apps/api/docker-entrypoint.sh delete mode 100644 apps/api/src/__tests__/queue-concurrency-integration.test.ts delete mode 100644 apps/api/src/controllers/v0/admin/queue.ts delete mode 100644 apps/api/src/lib/scrape-events.ts delete mode 100644 apps/api/src/services/alerts/index.ts create mode 100644 apps/api/src/services/worker/nuq-worker.ts create mode 100644 apps/api/src/services/worker/nuq.ts create mode 100644 apps/nuq-postgres/Dockerfile create mode 100644 apps/nuq-postgres/nuq.sql diff --git a/.github/k8s/playwright.yml b/.github/k8s/playwright.yml new file mode 100644 index 000000000..cd33ec6b5 --- /dev/null +++ b/.github/k8s/playwright.yml @@ -0,0 +1,62 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: firecrawl-playwright-service +spec: + replicas: 1 + selector: + matchLabels: + app: firecrawl-playwright-service + template: + metadata: + labels: + app: firecrawl-playwright-service + spec: + containers: + - name: playwright-service + image: firecrawl/playwright-service:latest + imagePullPolicy: Never + envFrom: + - configMapRef: + name: firecrawl-config + - configMapRef: + name: firecrawl-config-dynamic + - secretRef: + name: firecrawl-secret-dynamic + env: + - name: PORT + value: "3003" + ports: + - containerPort: 3003 + resources: + requests: + memory: "1G" + cpu: "500m" + limits: + memory: "3G" + cpu: "1000m" + livenessProbe: + httpGet: + path: /health + port: 3003 + initialDelaySeconds: 30 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /health + port: 3003 + initialDelaySeconds: 30 + periodSeconds: 10 +--- +apiVersion: v1 +kind: Service +metadata: + name: firecrawl-playwright-service +spec: + selector: + app: firecrawl-playwright-service + ports: + - name: http + protocol: TCP + port: 3003 + targetPort: 3003 \ No newline at end of file diff --git a/.github/k8s/searxng.yml b/.github/k8s/searxng.yml new file mode 100644 index 000000000..03c9f88e6 --- /dev/null +++ b/.github/k8s/searxng.yml @@ -0,0 +1,59 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: searxng-settings +data: + settings.yml: | + use_default_settings: true + search: + formats: [html, json, csv] + server: + secret_key: 'fcsecret' +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: firecrawl-searxng +spec: + replicas: 1 + selector: + matchLabels: + app: firecrawl-searxng + template: + metadata: + labels: + app: firecrawl-searxng + spec: + containers: + - name: searxng + image: searxng/searxng:latest + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8080 + volumeMounts: + - name: settings + mountPath: /etc/searxng + resources: + requests: + memory: "256Mi" + cpu: "100m" + limits: + memory: "512Mi" + cpu: "200m" + volumes: + - name: settings + configMap: + name: searxng-settings +--- +apiVersion: v1 +kind: Service +metadata: + name: firecrawl-searxng +spec: + selector: + app: firecrawl-searxng + ports: + - name: http + protocol: TCP + port: 8080 + targetPort: 8080 diff --git a/.github/k8s/server-nsh.yml b/.github/k8s/server-nsh.yml new file mode 100644 index 000000000..7778ed32a --- /dev/null +++ b/.github/k8s/server-nsh.yml @@ -0,0 +1,114 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tailscale-proxy +spec: + replicas: 1 + selector: + matchLabels: + app: tailscale-proxy + template: + metadata: + labels: + app: tailscale-proxy + spec: + containers: + - name: proxy + image: firecrawl/tailscale-proxy:latest + imagePullPolicy: Never + ports: + - containerPort: 8080 + securityContext: + capabilities: + add: ["NET_ADMIN", "NET_RAW"] + allowPrivilegeEscalation: false + volumeMounts: + - name: dev-net-tun + mountPath: /dev/net/tun + readinessProbe: + tcpSocket: + port: 8080 + initialDelaySeconds: 3 + periodSeconds: 5 + timeoutSeconds: 3 + successThreshold: 1 + failureThreshold: 3 + env: + - name: TS_AUTHKEY + valueFrom: + secretKeyRef: + name: tailscale + key: TS_AUTHKEY + - name: TS_ADVERTISE_TAGS + valueFrom: + secretKeyRef: + name: tailscale + key: TS_ADVERTISE_TAGS + - name: FIRE_ENGINE_BETA_URL + valueFrom: + secretKeyRef: + name: tailscale + key: FIRE_ENGINE_BETA_URL + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "256Mi" + cpu: "200m" + volumes: + - name: dev-net-tun + hostPath: + path: /dev/net/tun + type: CharDevice +--- +apiVersion: v1 +kind: Service +metadata: + name: tailscale-proxy +spec: + selector: + app: tailscale-proxy + ports: + - name: http + protocol: TCP + port: 8080 + targetPort: 8080 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: firecrawl-index-worker +spec: + replicas: 1 + selector: + matchLabels: + app: firecrawl-index-worker + template: + metadata: + labels: + app: firecrawl-index-worker + spec: + containers: + - name: index-worker + image: firecrawl/firecrawl:latest + imagePullPolicy: Never + command: [ "node" ] + args: [ "--max-old-space-size=2048", "dist/src/services/indexing/index-worker.js" ] + envFrom: + - configMapRef: + name: firecrawl-config + - configMapRef: + name: firecrawl-config-dynamic + - secretRef: + name: firecrawl-secret-dynamic + env: + - name: FLY_PROCESS_GROUP + value: "index-worker" + resources: + requests: + memory: "1G" + cpu: "500m" + limits: + memory: "3G" + cpu: "1000m" \ No newline at end of file diff --git a/.github/k8s/server.yml b/.github/k8s/server.yml new file mode 100644 index 000000000..5d55bb080 --- /dev/null +++ b/.github/k8s/server.yml @@ -0,0 +1,305 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: firecrawl-config +data: + HOST: "0.0.0.0" + REDIS_URL: "redis://firecrawl-dragonfly:6379" + REDIS_RATE_LIMIT_URL: "redis://firecrawl-dragonfly:6379" + REDIS_EVICT_URL: "redis://firecrawl-dragonfly:6379" + SENTRY_ENVIRONMENT: "dev" + ENV: "production" + LOGGING_LEVEL: "DEBUG" + IS_KUBERNETES: "true" + USE_GO_MARKDOWN_PARSER: "true" + NUQ_DATABASE_URL: "postgres://postgres:postgres@nuq-postgres:5432/postgres" + NUQ_DATABASE_URL_LISTEN: "postgres://postgres:postgres@nuq-postgres:5432/postgres" +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nuq-postgres +spec: + replicas: 1 + selector: + matchLabels: + app: nuq-postgres + template: + metadata: + labels: + app: nuq-postgres + spec: + containers: + - name: postgres + image: firecrawl/nuq-postgres:latest + imagePullPolicy: Never + ports: + - containerPort: 5432 + env: + - name: POSTGRES_USER + value: "postgres" + - name: POSTGRES_PASSWORD + value: "postgres" + - name: POSTGRES_DB + value: "postgres" + resources: + requests: + memory: "512Mi" + cpu: "250m" + limits: + memory: "1Gi" + cpu: "500m" + +--- +apiVersion: v1 +kind: Service +metadata: + name: nuq-postgres +spec: + selector: + app: nuq-postgres + ports: + - name: postgres + protocol: TCP + port: 5432 + targetPort: 5432 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: firecrawl-dragonfly +spec: + replicas: 1 + selector: + matchLabels: + app: firecrawl-dragonfly + template: + metadata: + labels: + app: firecrawl-dragonfly + spec: + containers: + - name: dragonfly + image: ghcr.io/dragonflydb/dragonfly:v1.31.2 + args: + - "--logtostderr" + - "--cluster_mode=emulated" + - "--lock_on_hashtags" + - "--bind=::" + - "--maxmemory=4gb" + - "--dir=/data" + ports: + - containerPort: 6379 + readinessProbe: + tcpSocket: + port: 6379 + initialDelaySeconds: 3 + periodSeconds: 5 + timeoutSeconds: 3 + successThreshold: 1 + failureThreshold: 3 + resources: + requests: + memory: "1Gi" + cpu: "500m" + limits: + memory: "4Gi" + cpu: "1000m" +--- +apiVersion: v1 +kind: Service +metadata: + name: firecrawl-dragonfly +spec: + selector: + app: firecrawl-dragonfly + ports: + - name: redis + protocol: TCP + port: 6379 + targetPort: 6379 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: firecrawl-app +spec: + replicas: 1 + selector: + matchLabels: + app: firecrawl-app + template: + metadata: + labels: + app: firecrawl-app + spec: + containers: + - name: firecrawl-app + image: firecrawl/firecrawl:latest + imagePullPolicy: Never + command: ["node"] + args: ["dist/src/index.js"] + ports: + - containerPort: 3002 + envFrom: + - configMapRef: + name: firecrawl-config + - configMapRef: + name: firecrawl-config-dynamic + - secretRef: + name: firecrawl-secret-dynamic + env: + - name: FLY_PROCESS_GROUP + value: "app" + - name: PORT + value: "3002" + - name: NODE_OPTIONS + value: "--no-node-snapshot --max-old-space-size=6144" + resources: + requests: + memory: "1G" + cpu: "500m" + limits: + memory: "6G" + cpu: "1000m" + livenessProbe: + httpGet: + path: /v0/health/liveness + port: 3002 + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /v0/health/readiness + port: 3002 + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 +--- +apiVersion: v1 +kind: Service +metadata: + name: firecrawl-app + namespace: default + labels: + app: firecrawl-app +spec: + selector: + app: firecrawl-app + ports: + - name: http + protocol: TCP + port: 3002 + targetPort: 3002 + type: NodePort +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: firecrawl-worker +spec: + replicas: 1 + selector: + matchLabels: + app: firecrawl-worker + template: + metadata: + labels: + app: firecrawl-worker + spec: + containers: + - name: worker + image: firecrawl/firecrawl:latest + imagePullPolicy: Never + command: [ "node" ] + args: [ "dist/src/services/queue-worker.js", "--max-old-space-size=2048" ] + envFrom: + - configMapRef: + name: firecrawl-config + - configMapRef: + name: firecrawl-config-dynamic + - secretRef: + name: firecrawl-secret-dynamic + env: + - name: FLY_PROCESS_GROUP + value: "worker" + - name: PORT + value: "3005" + - name: NODE_OPTIONS + value: "--no-node-snapshot" + - name: GOMEMLIMIT + value: "1024MiB" + - name: SCRAPE_WORKER_MAX_OLD_SPACE_SIZE + value: "768" + resources: + requests: + memory: "2G" + cpu: "500m" + limits: + memory: "4G" + cpu: "1000m" + livenessProbe: + httpGet: + path: /liveness + port: 3005 + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: firecrawl-nuq-worker +spec: + replicas: 5 + selector: + matchLabels: + app: firecrawl-nuq-worker + template: + metadata: + labels: + app: firecrawl-nuq-worker + spec: + containers: + - name: nuq-worker + image: firecrawl/firecrawl:latest + imagePullPolicy: Never + command: [ "node" ] + args: [ "dist/src/services/worker/nuq-worker.js", "--max-old-space-size=2048" ] + envFrom: + - configMapRef: + name: firecrawl-config + - configMapRef: + name: firecrawl-config-dynamic + - secretRef: + name: firecrawl-secret-dynamic + env: + - name: PORT + value: "3005" + - name: NODE_OPTIONS + value: "--no-node-snapshot" + - name: GOMEMLIMIT + value: "1024MiB" + resources: + requests: + memory: "2G" + cpu: "500m" + limits: + memory: "4G" + cpu: "1000m" + livenessProbe: + httpGet: + path: /health + port: 3005 + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 10 + successThreshold: 1 + failureThreshold: 3 \ No newline at end of file diff --git a/.github/k8s/tailscale-proxy/Dockerfile b/.github/k8s/tailscale-proxy/Dockerfile new file mode 100644 index 000000000..77bd48e43 --- /dev/null +++ b/.github/k8s/tailscale-proxy/Dockerfile @@ -0,0 +1,28 @@ +FROM debian:bookworm-slim + +# Install prerequisites, iptables, nginx, and Tailscale +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + gnupg \ + iptables \ + iproute2 \ + nginx \ + && curl -fsSL https://pkgs.tailscale.com/stable/debian/bookworm.noarmor.gpg \ + -o /usr/share/keyrings/tailscale-archive-keyring.gpg \ + && curl -fsSL https://pkgs.tailscale.com/stable/debian/bookworm.tailscale-keyring.list \ + -o /etc/apt/sources.list.d/tailscale.list \ + && apt-get update \ + && apt-get install -y --no-install-recommends tailscale \ + && rm -rf /var/lib/apt/lists/* + +# Configure nginx as a reverse proxy +COPY nginx.conf /etc/nginx/nginx.conf.template + +# Start script that sets up tailscale and nginx +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +EXPOSE 8080 +ENTRYPOINT ["/entrypoint.sh"] \ No newline at end of file diff --git a/.github/k8s/tailscale-proxy/entrypoint.sh b/.github/k8s/tailscale-proxy/entrypoint.sh new file mode 100644 index 000000000..a1a81639f --- /dev/null +++ b/.github/k8s/tailscale-proxy/entrypoint.sh @@ -0,0 +1,48 @@ +#!/bin/sh +set -eu + +# Require auth key +if [ -z "${TS_AUTHKEY:-}" ]; then + echo "[tailscale-proxy] TS_AUTHKEY is not set; refusing to start." >&2 + exit 1 +fi + +# Compute hostname: allow override via TS_HOSTNAME, else default to pod name +TS_HOSTNAME=${TS_HOSTNAME:-} +if [ -z "${TS_HOSTNAME}" ]; then + # Kubernetes sets HOSTNAME to the pod name + TS_HOSTNAME="firecrawl-proxy-${HOSTNAME:-container}" +fi + +# Start tailscaled +tailscaled --state=mem: & + +# Wait for tailscaled control socket to be ready (up to ~15s) +i=0 +until tailscale version >/dev/null 2>&1 || [ $i -ge 30 ]; do + i=$((i+1)) + sleep 0.5 +done + +# Prepare optional flags +ADVERTISE_FLAGS="" +if [ -n "${TS_ADVERTISE_TAGS:-}" ]; then + ADVERTISE_FLAGS="--advertise-tags=${TS_ADVERTISE_TAGS}" +fi + +set -x +tailscale up \ + --authkey="${TS_AUTHKEY}" \ + --hostname="${TS_HOSTNAME}" \ + --accept-routes=true \ + --accept-dns=true \ + --ssh=false \ + ${ADVERTISE_FLAGS} \ + --reset +set +x + +# Replace FIRE_ENGINE_BETA_URL in nginx config +sed "s|\$FIRE_ENGINE_BETA_URL|${FIRE_ENGINE_BETA_URL}|g" /etc/nginx/nginx.conf.template > /etc/nginx/nginx.conf + +# Start nginx +exec nginx -g "daemon off;" \ No newline at end of file diff --git a/.github/k8s/tailscale-proxy/nginx.conf b/.github/k8s/tailscale-proxy/nginx.conf new file mode 100644 index 000000000..ca48512ac --- /dev/null +++ b/.github/k8s/tailscale-proxy/nginx.conf @@ -0,0 +1,24 @@ +user www-data; +worker_processes auto; +error_log /dev/stderr notice; +pid /run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + access_log /dev/stdout combined; + + server { + listen 8080; + + location / { + proxy_pass $FIRE_ENGINE_BETA_URL; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + } +} \ No newline at end of file diff --git a/.github/workflows/test-server-self-host.yml b/.github/workflows/test-server-self-host.yml deleted file mode 100644 index 9c97c8f00..000000000 --- a/.github/workflows/test-server-self-host.yml +++ /dev/null @@ -1,145 +0,0 @@ -name: Self-hosted Server Test Suite - -on: - pull_request: - branches: - - main - - nsc/v2 - -env: - REDIS_URL: redis://localhost:6379 - ENV: ${{ secrets.ENV }} - TEST_SUITE_SELF_HOSTED: true - USE_GO_MARKDOWN_PARSER: true - FIRECRAWL_DEBUG_FILTER_LINKS: true - SENTRY_ENVIRONMENT: dev - -jobs: - test: - name: Run tests - strategy: - matrix: - ai: ['openai', 'no-ai'] - search: ['searxng', 'google'] - engine: ['playwright', 'fetch'] - # proxy: ["proxy", "no-proxy"] - proxy: ['proxy'] - fail-fast: false - runs-on: ubuntu-latest - services: - redis: - image: redis - ports: - - 6379:6379 - env: - OPENAI_API_KEY: ${{ matrix.ai == 'openai' && secrets.OPENAI_API_KEY || '' }} - SEARXNG_ENDPOINT: ${{ matrix.search == 'searxng' && 'http://localhost:3434' || '' }} - PLAYWRIGHT_MICROSERVICE_URL: ${{ matrix.engine == 'playwright' && 'http://localhost:3003/scrape' || '' }} - PROXY_SERVER: ${{ matrix.proxy == 'proxy' && secrets.PROXY_SERVER || '' }} - PROXY_USERNAME: ${{ matrix.proxy == 'proxy' && secrets.PROXY_USERNAME || '' }} - PROXY_PASSWORD: ${{ matrix.proxy == 'proxy' && secrets.PROXY_PASSWORD || '' }} - steps: - - uses: actions/checkout@v5 - - name: Install pnpm - uses: pnpm/action-setup@v4 - with: - version: 10 - - name: Set up Node.js - uses: actions/setup-node@v4 - with: - node-version: '20' - cache: 'pnpm' - cache-dependency-path: './apps/api/pnpm-lock.yaml' - - name: Install dependencies - run: pnpm install - working-directory: ./apps/api - - name: Install Playwright dependencies - if: matrix.engine == 'playwright' - run: | - pnpm install - pnpm exec playwright install-deps - pnpm exec playwright install - working-directory: ./apps/playwright-service-ts - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version: '1.24' - cache-dependency-path: ./apps/api/sharedLibs/go-html-to-md/go.sum - - name: Build go-html-to-md - run: | - go mod tidy - go build -o libhtml-to-markdown.so -buildmode=c-shared html-to-markdown.go - chmod +x libhtml-to-markdown.so - working-directory: ./apps/api/sharedLibs/go-html-to-md - - name: Set up SearXNG - if: matrix.search == 'searxng' - run: | - mkdir searxng - - echo "use_default_settings: true - search: - formats: [html, json, csv] - server: - secret_key: 'fcsecret'" > searxng/settings.yml - - docker run -d -p 3434:8080 -v "${PWD}/searxng:/etc/searxng" --name searxng searxng/searxng - pnpx wait-on tcp:3434 -t 30s - working-directory: ./ - - name: Build the application - run: pnpm run build - working-directory: ./apps/api - - name: Start server - run: pnpm run start:production:nobuild > api.log 2>&1 & - env: - PORT: 3002 - HOST: 0.0.0.0 - working-directory: ./apps/api - - name: Start worker - run: pnpm run worker:production > worker.log 2>&1 & - env: - PORT: 3005 - HOST: 0.0.0.0 - working-directory: ./apps/api - - name: Start playwright - if: matrix.engine == 'playwright' - run: pnpm run dev > playwright.log 2>&1 & - working-directory: ./apps/playwright-service-ts - env: - PORT: 3003 - - name: Wait for server - run: pnpx wait-on tcp:3002 -t 60s - - name: Wait for playwright - if: matrix.engine == 'playwright' - run: pnpx wait-on tcp:3003 -t 15s - - name: Run snippet tests - run: | - npm run test:snips - working-directory: ./apps/api - - name: Kill instances - if: always() - run: pkill -9 node - - name: Kill SearXNG - if: always() && matrix.search == 'searxng' - run: | - docker logs searxng > searxng.log 2>&1 - docker kill searxng - working-directory: ./ - - uses: actions/upload-artifact@v4 - if: always() - with: - name: Logs (${{ matrix.ai }}, ${{ matrix.search }}, ${{ matrix.engine }}, ${{ matrix.proxy }}) - path: | - ./apps/api/api.log - ./apps/api/worker.log - - uses: actions/upload-artifact@v4 - if: always() && matrix.playwright - with: - name: Playwright Logs (${{ matrix.ai }}, ${{ matrix.search }}, ${{ matrix.proxy }}) - path: | - ./apps/playwright-service-ts/playwright.log - - uses: actions/upload-artifact@v4 - if: always() && matrix.search == 'searxng' - with: - name: SearXNG (${{ matrix.ai }}, ${{ matrix.engine }}, ${{ matrix.proxy }}) - path: | - ./searxng.log diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.yml index 6278db813..66b9093b6 100644 --- a/.github/workflows/test-server.yml +++ b/.github/workflows/test-server.yml @@ -6,50 +6,74 @@ on: - main - nsc/v2 -env: - BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }} - POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }} - POSTHOG_HOST: ${{ secrets.POSTHOG_HOST }} - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - REDIS_URL: ${{ secrets.REDIS_URL }} - SUPABASE_ANON_TOKEN: ${{ secrets.SUPABASE_ANON_TOKEN }} - SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }} - SUPABASE_URL: ${{ secrets.SUPABASE_URL }} - SUPABASE_REPLICA_URL: ${{ secrets.SUPABASE_REPLICA_URL }} - INDEX_SUPABASE_SERVICE_TOKEN: ${{ secrets.INDEX_SUPABASE_SERVICE_TOKEN }} - INDEX_SUPABASE_ANON_TOKEN: ${{ secrets.INDEX_SUPABASE_ANON_TOKEN }} - INDEX_SUPABASE_URL: ${{ secrets.INDEX_SUPABASE_URL }} - TEST_API_KEY: ${{ secrets.TEST_API_KEY }} - TEST_TEAM_ID: ${{ secrets.TEST_TEAM_ID }} - TEST_API_KEY_CONCURRENCY: ${{ secrets.TEST_API_KEY_CONCURRENCY }} - TEST_TEAM_ID_CONCURRENCY: ${{ secrets.TEST_TEAM_ID_CONCURRENCY }} - TEST_API_KEY_ZDR: ${{ secrets.TEST_API_KEY_ZDR }} - TEST_TEAM_ID_ZDR: ${{ secrets.TEST_TEAM_ID_ZDR }} - FIRE_ENGINE_BETA_URL: ${{ secrets.FIRE_ENGINE_BETA_URL }} - FIRE_ENGINE_STAGING_URL: ${{ secrets.FIRE_ENGINE_STAGING_URL }} - USE_DB_AUTHENTICATION: true - SERPER_API_KEY: ${{ secrets.SERPER_API_KEY }} - ENV: ${{ secrets.ENV }} - RUNPOD_MU_POD_ID: ${{ secrets.RUNPOD_MU_POD_ID }} - RUNPOD_MUV2_POD_ID: ${{ secrets.RUNPOD_MUV2_POD_ID }} - RUNPOD_MU_API_KEY: ${{ secrets.RUNPOD_MU_API_KEY }} - GCS_CREDENTIALS: ${{ secrets.GCS_CREDENTIALS }} - GCS_BUCKET_NAME: ${{ secrets.GCS_BUCKET_NAME }} - GCS_INDEX_BUCKET_NAME: ${{ secrets.GCS_INDEX_BUCKET_NAME }} - GCS_MEDIA_BUCKET_NAME: ${{ secrets.GCS_MEDIA_BUCKET_NAME }} - GOOGLE_GENERATIVE_AI_API_KEY: ${{ secrets.GOOGLE_GENERATIVE_AI_API_KEY }} - GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - VERTEX_CREDENTIALS: ${{ secrets.VERTEX_CREDENTIALS }} - USE_GO_MARKDOWN_PARSER: true - SENTRY_ENVIRONMENT: dev - IDMUX_URL: ${{ secrets.IDMUX_URL }} - LOG_ENCRYPTION_KEY: ${{ secrets.LOG_ENCRYPTION_KEY }} - jobs: + build-images: + name: Build images + runs-on: big-runner + steps: + - uses: actions/checkout@v5 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build API image + run: | + docker buildx build \ + -t firecrawl/firecrawl:latest \ + --cache-from type=gha \ + --cache-to type=gha,mode=max \ + --load \ + ./apps/api + - name: Build nuq-postgres image + run: | + docker buildx build \ + -t firecrawl/nuq-postgres:latest \ + --cache-from type=gha \ + --cache-to type=gha,mode=max \ + --load \ + ./apps/nuq-postgres + - name: Build tailscale-proxy image + run: | + docker buildx build \ + -t firecrawl/tailscale-proxy:latest \ + --cache-from type=gha \ + --cache-to type=gha,mode=max \ + --load \ + .github/k8s/tailscale-proxy + - name: Build Playwright image + run: | + docker buildx build \ + -t firecrawl/playwright-service:latest \ + --cache-from type=gha \ + --cache-to type=gha,mode=max \ + --load \ + ./apps/playwright-service-ts + - name: Export images as tarballs + run: | + docker save -o firecrawl-firecrawl.tar firecrawl/firecrawl:latest + docker save -o firecrawl-nuq-postgres.tar firecrawl/nuq-postgres:latest + docker save -o firecrawl-tailscale-proxy.tar firecrawl/tailscale-proxy:latest + docker save -o firecrawl-playwright-service.tar firecrawl/playwright-service:latest + - name: Upload images artifact + uses: actions/upload-artifact@v4 + with: + name: built-images + path: | + firecrawl-firecrawl.tar + firecrawl-nuq-postgres.tar + firecrawl-tailscale-proxy.tar + firecrawl-playwright-service.tar + test: - name: Run tests - runs-on: ubuntu-latest + name: Self-hosted (Kubernetes) environment tests + needs: build-images + strategy: + matrix: + ai: ["openai", "no-ai"] + search: ["searxng", "google"] + engine: ["playwright", "fetch"] + # proxy: ["proxy", "no-proxy"] + proxy: ["proxy"] + fail-fast: false + runs-on: big-runner services: redis: image: redis @@ -57,13 +81,43 @@ jobs: - 6379:6379 steps: - uses: actions/checkout@v5 - - name: Tailscale - uses: tailscale/github-action@v3 + - name: Start minikube + uses: medyagh/setup-minikube@v0.0.20 + - name: Download images artifact + uses: actions/download-artifact@v4 with: - oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }} - oauth-secret: ${{ secrets.TS_OAUTH_SECRET }} - tags: tag:ci - use-cache: 'true' + name: built-images + path: ./images + - name: Load images into minikube + run: | + eval $(minikube docker-env) + docker load -i ./images/firecrawl-firecrawl.tar + docker load -i ./images/firecrawl-nuq-postgres.tar + - name: Load Playwright image into minikube + if: matrix.engine == 'playwright' + run: | + eval $(minikube docker-env) + docker load -i ./images/firecrawl-playwright-service.tar + - name: Create configmap + run: | + kubectl create configmap firecrawl-config-dynamic -n default \ + --from-literal=SEARXNG_ENDPOINT=${{ matrix.search == 'searxng' && 'http://firecrawl-searxng:8080/search' || '' }} \ + --from-literal=PLAYWRIGHT_MICROSERVICE_URL=${{ matrix.engine == 'playwright' && 'http://firecrawl-playwright-service:3003/scrape' || '' }} \ + --from-literal=PROXY_SERVER=${{ matrix.proxy == 'proxy' && secrets.PROXY_SERVER || '' }} \ + --from-literal=PROXY_USERNAME=${{ matrix.proxy == 'proxy' && secrets.PROXY_USERNAME || '' }} + - name: Create secret + run: | + kubectl create secret generic firecrawl-secret-dynamic \ + --from-literal=OPENAI_API_KEY=${{ matrix.ai == 'openai' && secrets.OPENAI_API_KEY || '' }} \ + --from-literal=PROXY_PASSWORD=${{ matrix.proxy == 'proxy' && secrets.PROXY_PASSWORD || '' }} + - name: Deploy server + run: kubectl apply -f .github/k8s/server.yml + - name: Deploy playwright + if: matrix.engine == 'playwright' + run: kubectl apply -f .github/k8s/playwright.yml + - name: Deploy searxng + if: matrix.search == 'searxng' + run: kubectl apply -f .github/k8s/searxng.yml - name: Install pnpm uses: pnpm/action-setup@v4 with: @@ -77,43 +131,277 @@ jobs: - name: Install dependencies run: pnpm install working-directory: ./apps/api - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version: '1.24' - cache-dependency-path: ./apps/api/sharedLibs/go-html-to-md/go.sum - - name: Build go-html-to-md + - name: Wait for server + run: kubectl wait --for=condition=ready pod -l app=firecrawl-app --timeout=60s -o json + - name: Debug wait failure (server) + if: failure() run: | - go mod tidy - go build -o libhtml-to-markdown.so -buildmode=c-shared html-to-markdown.go - chmod +x libhtml-to-markdown.so - working-directory: ./apps/api/sharedLibs/go-html-to-md - - name: Build the application - run: pnpm run build - working-directory: ./apps/api - - name: Start the application - run: pnpm run start:production:nobuild > api.log 2>&1 & - env: - PORT: 3002 - HOST: 0.0.0.0 - working-directory: ./apps/api - id: start_app - - name: Start worker - run: pnpm run worker:production > worker.log 2>&1 & - env: - PORT: 3005 - HOST: 0.0.0.0 - working-directory: ./apps/api - id: start_workers - - name: Start index worker - run: pnpm run index-worker:production > index-worker.log 2>&1 & - working-directory: ./apps/api - id: start_index_worker - - name: Wait for API - run: pnpx wait-on tcp:3002 -t 60s + echo "Pods with label app=firecrawl-app:" || true + kubectl get pods -l app=firecrawl-app -o wide || true + echo "\nDescribe pods:" || true + kubectl describe pods -l app=firecrawl-app || true + echo "\nRecent events:" || true + kubectl get events --sort-by=.lastTimestamp | tail -n 100 || true + echo "\nLogs from firecrawl-app deployment (if any):" || true + kubectl logs deployment/firecrawl-app --all-containers=true --tail=200 || true + - name: Wait for playwright + if: matrix.engine == 'playwright' + run: kubectl wait --for=condition=ready pod -l app=firecrawl-playwright-service --timeout=60s -o json + - name: Debug wait failure (playwright) + if: failure() && matrix.engine == 'playwright' + run: | + echo "Pods with label app=firecrawl-playwright-service:" || true + kubectl get pods -l app=firecrawl-playwright-service -o wide || true + echo "\nDescribe pods:" || true + kubectl describe pods -l app=firecrawl-playwright-service || true + echo "\nRecent events:" || true + kubectl get events --sort-by=.lastTimestamp | tail -n 100 || true + echo "\nLogs from playwright deployment (if any):" || true + kubectl logs deployment/firecrawl-playwright-service --all-containers=true --tail=200 || true + - name: Wait for searxng + if: matrix.search == 'searxng' + run: kubectl wait --for=condition=ready pod -l app=firecrawl-searxng --timeout=60s -o json + - name: Debug wait failure (searxng) + if: failure() && matrix.search == 'searxng' + run: | + echo "Pods with label app=firecrawl-searxng:" || true + kubectl get pods -l app=firecrawl-searxng -o wide || true + echo "\nDescribe pods:" || true + kubectl describe pods -l app=firecrawl-searxng || true + echo "\nRecent events:" || true + kubectl get events --sort-by=.lastTimestamp | tail -n 100 || true + echo "\nLogs from searxng deployment (if any):" || true + kubectl logs deployment/firecrawl-searxng --all-containers=true --tail=200 || true - name: Run snippet tests + env: + TEST_SUITE_SELF_HOSTED: true + OPENAI_API_KEY: ${{ matrix.ai == 'openai' && '' || '' }} + PROXY_SERVER: ${{ matrix.proxy == 'proxy' && secrets.PROXY_SERVER || '' }} + PLAYWRIGHT_MICROSERVICE_URL: ${{ matrix.engine == 'playwright' && '' || '' }} + run: TEST_API_URL=$(minikube service firecrawl-app --url) pnpm run test:snips + working-directory: ./apps/api + - name: Create logs directory + if: always() + run: mkdir -p logs + - name: Copy log files + if: always() run: | - npm run test:snips + kubectl logs deployment/firecrawl-app > logs/api.log || true + kubectl logs deployment/firecrawl-worker > logs/worker.log || true + kubectl logs deployment/firecrawl-nuq-worker > logs/nuq-worker.log || true + kubectl logs deployment/firecrawl-dragonfly > logs/dragonfly.log || true + kubectl logs deployment/nuq-postgres > logs/postgres.log || true + - name: Copy SearXNG logs + if: always() && matrix.search == 'searxng' + run: kubectl logs deployment/firecrawl-searxng > logs/searxng.log || true + - name: Copy Playwright logs + if: always() && matrix.engine == 'playwright' + run: kubectl logs deployment/firecrawl-playwright-service > logs/playwright.log || true + - name: Zip logs + if: always() + run: | + cd logs + zip -r logs.zip ./* + - uses: actions/upload-artifact@v4 + if: always() + with: + name: Logs (kubernetes, ${{ matrix.ai }}, ${{ matrix.search }}, ${{ matrix.engine }}, ${{ matrix.proxy }}) + path: logs/logs.zip + + docker-test: + name: Self-hosted (Docker) environment tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - name: Compose + env: + PROXY_SERVER: ${{ secrets.PROXY_SERVER }} + PROXY_USERNAME: ${{ secrets.PROXY_USERNAME }} + PROXY_PASSWORD: ${{ secrets.PROXY_PASSWORD }} + run: | + docker compose up -d + - name: Install pnpm + uses: pnpm/action-setup@v4 + with: + version: 10 + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: "20" + cache: "pnpm" + cache-dependency-path: './apps/api/pnpm-lock.yaml' + - name: Install dependencies + run: pnpm install + working-directory: ./apps/api + - name: Wait for server + run: pnpx wait-on http://localhost:3002 + - name: Run snippet tests + env: + PROXY_SERVER: ${{ secrets.PROXY_SERVER }} + PLAYWRIGHT_MICROSERVICE_URL: ${{ secrets.PLAYWRIGHT_MICROSERVICE_URL }} + run: pnpm run test:snips + working-directory: ./apps/api + - name: Create logs directory + if: always() + run: mkdir -p logs + - name: Copy log files + if: always() + run: | + docker compose logs api > logs/api.log || true + docker compose logs playwright-service > logs/playwright.log || true + docker compose logs nuq-postgres > logs/postgres.log || true + - name: Zip logs + if: always() + run: | + cd logs + zip -r logs.zip ./* + - uses: actions/upload-artifact@v4 + if: always() + with: + name: Logs (docker, no-ai, google, playwright, proxy) + path: logs/logs.zip + + prod-test: + name: Production environment tests + needs: build-images + runs-on: big-runner + steps: + - uses: actions/checkout@v5 + - name: Start minikube + uses: medyagh/setup-minikube@v0.0.20 + - name: Download images artifact + uses: actions/download-artifact@v4 + with: + name: built-images + path: ./images + - name: Tailscale + uses: tailscale/github-action@v3 + with: + oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }} + oauth-secret: ${{ secrets.TS_OAUTH_SECRET }} + tags: tag:ci + - name: Load prebuilt images into minikube + run: | + eval $(minikube docker-env) + docker load -i ./images/firecrawl-firecrawl.tar + docker load -i ./images/firecrawl-nuq-postgres.tar + docker load -i ./images/firecrawl-tailscale-proxy.tar + - name: Create Tailscale secret + run: | + if [ -z "${{ secrets.TS_AUTHKEY }}" ]; then + echo "secrets.TS_AUTHKEY is empty; set a reusable/preauthorized or ephemeral key" >&2 + exit 1 + fi + kubectl create secret generic tailscale \ + --from-literal=TS_AUTHKEY=${{ secrets.TS_AUTHKEY }} \ + --from-literal=TS_ADVERTISE_TAGS=${{ secrets.TS_ADVERTISE_TAGS }} \ + --from-literal=FIRE_ENGINE_BETA_URL=${{ secrets.FIRE_ENGINE_BETA_URL }} \ + --dry-run=client -o yaml | kubectl apply -f - + - name: Create configmap + run: | + kubectl create configmap firecrawl-config-dynamic -n default \ + --from-literal=USE_DB_AUTHENTICATION=true \ + --from-literal=SUPABASE_ANON_TOKEN=${{ secrets.SUPABASE_ANON_TOKEN }} \ + --from-literal=SUPABASE_URL=${{ secrets.SUPABASE_URL }} \ + --from-literal=SUPABASE_REPLICA_URL=${{ secrets.SUPABASE_REPLICA_URL }} \ + --from-literal=INDEX_SUPABASE_ANON_TOKEN=${{ secrets.INDEX_SUPABASE_ANON_TOKEN }} \ + --from-literal=INDEX_SUPABASE_URL=${{ secrets.INDEX_SUPABASE_URL }} \ + --from-literal=FIRE_ENGINE_BETA_URL=http://tailscale-proxy:8080 \ + --from-literal=RUNPOD_MU_POD_ID=${{ secrets.RUNPOD_MU_POD_ID }} \ + --from-literal=RUNPOD_MUV2_POD_ID=${{ secrets.RUNPOD_MUV2_POD_ID }} \ + --from-literal=GCS_BUCKET_NAME=${{ secrets.GCS_BUCKET_NAME }} \ + --from-literal=GCS_MEDIA_BUCKET_NAME=${{ secrets.GCS_MEDIA_BUCKET_NAME }} \ + --from-literal=GCS_INDEX_BUCKET_NAME=${{ secrets.GCS_INDEX_BUCKET_NAME }} + - name: Create secret + run: | + kubectl create secret generic firecrawl-secret-dynamic \ + --from-literal=BULL_AUTH_KEY=${{ secrets.BULL_AUTH_KEY }} \ + --from-literal=OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }} \ + --from-literal=SUPABASE_SERVICE_TOKEN=${{ secrets.SUPABASE_SERVICE_TOKEN }} \ + --from-literal=INDEX_SUPABASE_SERVICE_TOKEN=${{ secrets.INDEX_SUPABASE_SERVICE_TOKEN }} \ + --from-literal=SERPER_API_KEY=${{ secrets.SERPER_API_KEY }} \ + --from-literal=RUNPOD_MU_API_KEY=${{ secrets.RUNPOD_MU_API_KEY }} \ + --from-literal=GCS_CREDENTIALS=${{ secrets.GCS_CREDENTIALS }} \ + --from-literal=GOOGLE_GENERATIVE_AI_API_KEY=${{ secrets.GOOGLE_GENERATIVE_AI_API_KEY }} \ + --from-literal=GROQ_API_KEY=${{ secrets.GROQ_API_KEY }} \ + --from-literal=ANTHROPIC_API_KEY=${{ secrets.ANTHROPIC_API_KEY }} \ + --from-literal=VERTEX_CREDENTIALS=${{ secrets.VERTEX_CREDENTIALS }} \ + --from-literal=LOG_ENCRYPTION_KEY=${{ secrets.LOG_ENCRYPTION_KEY }} + - name: Deploy server + run: | + kubectl apply -f .github/k8s/server.yml + kubectl apply -f .github/k8s/server-nsh.yml + - name: Install pnpm + uses: pnpm/action-setup@v4 + with: + version: 10 + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: "20" + cache: "pnpm" + cache-dependency-path: './apps/api/pnpm-lock.yaml' + - name: Install dependencies + run: pnpm install + working-directory: ./apps/api + - name: Wait for API + run: kubectl wait --for=condition=ready pod -l app=firecrawl-app --timeout=60s -o json + - name: Debug wait failure (API) + if: failure() + run: | + echo "Pods with label app=firecrawl-app:" || true + kubectl get pods -l app=firecrawl-app -o wide || true + echo "\nDescribe pods:" || true + kubectl describe pods -l app=firecrawl-app || true + echo "\nRecent events:" || true + kubectl get events --sort-by=.lastTimestamp | tail -n 100 || true + - name: Wait for nuq-postgres + run: kubectl wait --for=condition=ready pod -l app=nuq-postgres --timeout=60s -o json + - name: Debug wait failure (nuq-postgres) + if: failure() + run: | + echo "Pods with label app=nuq-postgres:" || true + kubectl get pods -l app=nuq-postgres -o wide || true + echo "\nDescribe pods:" || true + kubectl describe pods -l app=nuq-postgres || true + echo "\nRecent events:" || true + kubectl get events --sort-by=.lastTimestamp | tail -n 100 || true + echo "\nLogs from nuq-postgres deployment (if any):" || true + kubectl logs deployment/nuq-postgres --all-containers=true --tail=200 || true + - name: Wait for tailscale-proxy + run: kubectl wait --for=condition=ready pod -l app=tailscale-proxy --timeout=60s -o json + - name: Debug wait failure (tailscale-proxy) + if: failure() + run: | + echo "Pods with label app=tailscale-proxy:" || true + kubectl get pods -l app=tailscale-proxy -o wide || true + echo "\nDescribe pods:" || true + kubectl describe pods -l app=tailscale-proxy || true + echo "\nRecent events:" || true + kubectl get events --sort-by=.lastTimestamp | tail -n 100 || true + - name: Wait for dragonfly (redis) + run: kubectl wait --for=condition=ready pod -l app=firecrawl-dragonfly --timeout=60s -o json + - name: Debug wait failure (dragonfly) + if: failure() + run: | + echo "Pods with label app=firecrawl-dragonfly:" || true + kubectl get pods -l app=firecrawl-dragonfly -o wide || true + echo "\nDescribe pods:" || true + kubectl describe pods -l app=firecrawl-dragonfly || true + echo "\nRecent events:" || true + kubectl get events --sort-by=.lastTimestamp | tail -n 100 || true + - name: Run snippet tests + env: + IDMUX_URL: ${{ secrets.IDMUX_URL }} + USE_DB_AUTHENTICATION: true + SUPABASE_ANON_TOKEN: ${{ secrets.SUPABASE_ANON_TOKEN }} + SUPABASE_URL: ${{ secrets.SUPABASE_URL }} + SUPABASE_REPLICA_URL: ${{ secrets.SUPABASE_REPLICA_URL }} + SUPABASE_SERVICE_TOKEN: ${{ secrets.SUPABASE_SERVICE_TOKEN }} + GCS_BUCKET_NAME: ${{ secrets.GCS_BUCKET_NAME }} + GCS_CREDENTIALS: ${{ secrets.GCS_CREDENTIALS }} + run: | + TEST_API_URL=$(minikube service firecrawl-app --url) pnpm run test:snips working-directory: ./apps/api - name: Kill instances if: always() @@ -124,9 +412,13 @@ jobs: - name: Copy log files if: always() run: | - cp ./apps/api/api.log logs/ || true - cp ./apps/api/worker.log logs/ || true - cp ./apps/api/index-worker.log logs/ || true + kubectl logs deployment/firecrawl-app > logs/api.log || true + kubectl logs deployment/firecrawl-worker > logs/worker.log || true + kubectl logs deployment/firecrawl-index-worker > logs/index-worker.log || true + kubectl logs deployment/firecrawl-nuq-worker > logs/nuq-worker.log || true + kubectl logs deployment/firecrawl-dragonfly > logs/dragonfly.log || true + kubectl logs deployment/nuq-postgres > logs/postgres.log || true + kubectl logs deployment/tailscale-proxy > logs/tailscale-proxy.log || true - name: Zip and encrypt logs if: always() run: | diff --git a/apps/api/Dockerfile b/apps/api/Dockerfile index bf96a1615..5a2a9f62e 100644 --- a/apps/api/Dockerfile +++ b/apps/api/Dockerfile @@ -70,8 +70,4 @@ COPY --from=build /app/native ./native # Copy Go shared library COPY --from=go-build /app/sharedLibs/go-html-to-md/libhtml-to-markdown.so ./sharedLibs/go-html-to-md/ -# Copy and prepare entrypoint -COPY docker-entrypoint.sh ./ -RUN sed -i 's/\r$//' ./docker-entrypoint.sh && chmod +x ./docker-entrypoint.sh - -ENTRYPOINT ["./docker-entrypoint.sh"] \ No newline at end of file +CMD ["node", "dist/src/harness.js", "--start-docker"] diff --git a/apps/api/docker-entrypoint.sh b/apps/api/docker-entrypoint.sh deleted file mode 100755 index 6b09e7053..000000000 --- a/apps/api/docker-entrypoint.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -e - -if [ "$UID" -eq 0 ]; then - set +e # disable failing on errror - ulimit -n 65535 - echo "NEW ULIMIT: $(ulimit -n)" - set -e # enable failing on error -else - echo ENTRYPOINT DID NOT RUN AS ROOT -fi - -if [ "$FLY_PROCESS_GROUP" = "app" ]; then - echo "RUNNING app" - node --max-old-space-size=8192 dist/src/index.js -elif [ "$FLY_PROCESS_GROUP" = "worker" ]; then - echo "RUNNING worker" - node --max-old-space-size=8192 dist/src/services/queue-worker.js -elif [ "$FLY_PROCESS_GROUP" = "index-worker" ]; then - echo "RUNNING index worker" - node --max-old-space-size=8192 dist/src/services/indexing/index-worker.js -else - echo "NO FLY PROCESS GROUP" - node --max-old-space-size=8192 dist/src/index.js -fi diff --git a/apps/api/package.json b/apps/api/package.json index be164054e..1716cd45a 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -4,9 +4,12 @@ "description": "", "main": "index.js", "scripts": { - "start": "tsc-watch --onSuccess \"node dist/src/index.js\"", - "start:production": "tsc && node dist/src/index.js", - "start:production:nobuild": "node dist/src/index.js", + "start": "tsc-watch --onSuccess \"node dist/src/harness.js --start-built\"", + "start:production": "tsx src/harness.ts --start", + "server": "tsc-watch --onSuccess \"node dist/src/index.js\"", + "server:production": "tsc && node dist/src/index.js", + "server:production:nobuild": "node dist/src/index.js", + "format": "prettier --write \"src/**/*.(js|ts)\"", "flyio": "node dist/src/index.js", "start:dev": "tsc-watch --onSuccess \"node dist/src/index.js\"", "build": "tsc", @@ -19,6 +22,8 @@ "harness": "tsx src/harness.ts", "workers": "tsc-watch --onSuccess \"node dist/src/services/queue-worker.js\"", "worker:production": "node dist/src/services/queue-worker.js", + "nuq-worker": "tsc-watch --onSuccess \"node dist/src/services/worker/nuq-worker.js\"", + "nuq-worker:production": "node dist/src/services/worker/nuq-worker.js", "index-worker": "tsc-watch --onSuccess \"node dist/src/services/indexing/index-worker.js\"", "index-worker:production": "node dist/src/services/indexing/index-worker.js", "mongo-docker": "docker run -d -p 2717:27017 -v ./mongo-data:/data/db --name mongodb mongo:latest", @@ -43,6 +48,7 @@ "@types/lodash": "^4.17.14", "@types/node": "^20.14.1", "@types/pdf-parse": "^1.1.4", + "@types/pg": "^8.15.5", "@types/supertest": "^6.0.2", "@types/tough-cookie": "^4.0.5", "husky": "^9.1.7", @@ -136,6 +142,7 @@ "openai": "^5.12.0", "parse-diff": "^0.11.1", "pdf-parse": "^1.1.1", + "pg": "^8.16.3", "pos": "^0.4.2", "posthog-node": "^4.0.1", "prettier": "^3.6.2", @@ -172,6 +179,11 @@ "pnpm": { "onlyBuiltDependencies": [ "@sentry-internal/node-cpu-profiler", + "bigint-buffer", + "bufferutil", + "keccak", + "libpq", + "utf-8-validate", "supabase" ], "overrides": { diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yaml index b04f60409..4642bd0ce 100644 --- a/apps/api/pnpm-lock.yaml +++ b/apps/api/pnpm-lock.yaml @@ -249,6 +249,9 @@ importers: pdf-parse: specifier: ^1.1.1 version: 1.1.1 + pg: + specifier: ^8.16.3 + version: 8.16.3(pg-native@3.5.2) pos: specifier: ^0.4.2 version: 0.4.2 @@ -355,6 +358,9 @@ importers: '@types/pdf-parse': specifier: ^1.1.4 version: 1.1.4 + '@types/pg': + specifier: ^8.15.5 + version: 8.15.5 '@types/supertest': specifier: ^6.0.2 version: 6.0.2 @@ -3476,6 +3482,9 @@ packages: '@types/pg@8.15.4': resolution: {integrity: sha512-I6UNVBAoYbvuWkkU3oosC8yxqH21f4/Jc4DK71JLG3dT2mdlGe1z+ep/LQGXaKaOgcvUrsQoPRqfgtMcvZiJhg==} + '@types/pg@8.15.5': + resolution: {integrity: sha512-LF7lF6zWEKxuT3/OR8wAZGzkg4ENGXFNyiV/JeOt9z5B+0ZVwbql9McqX5c/WStFq1GaGso7H1AzP/qSzmlCKQ==} + '@types/pg@8.6.1': resolution: {integrity: sha512-1Kc4oAGzAl7uqUStZCDvaLFqZrW9qWSjXOmBfdgyBP5La7Us6Mg4GBvRlSoaZMhQF/zSj1C8CtKMBkoiT8eL8w==} @@ -5417,6 +5426,9 @@ packages: resolution: {integrity: sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==} engines: {node: '>=6'} + libpq@1.8.15: + resolution: {integrity: sha512-4lSWmly2Nsj3LaTxxtFmJWuP3Kx+0hYHEd+aNrcXEWT0nKWaPd9/QZPiMkkC680zeALFGHQdQWjBvnilL+vgWA==} + lie@3.3.0: resolution: {integrity: sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==} @@ -5684,6 +5696,9 @@ packages: resolution: {integrity: sha512-WWdIxpyjEn+FhQJQQv9aQAYlHoNVdzIzUySNV1gHUPDSdZJ3yZn7pAAbQcV7B56Mvu881q9FZV+0Vx2xC44VWA==} engines: {node: ^18.17.0 || >=20.5.0} + nan@2.22.2: + resolution: {integrity: sha512-DANghxFkS1plDdRsX0X9pm0Z6SJNN6gBdtXfanwoZ8hooC5gosGFSBGRYHUVPz1asKA/kMRqDRdHrluZ61SpBQ==} + nano-spawn@1.0.2: resolution: {integrity: sha512-21t+ozMQDAL/UGgQVBbZ/xXvNO10++ZPuTmKRO8k9V3AClVRht49ahtDjfY8l1q6nSHOrE5ASfthzH3ol6R/hg==} engines: {node: '>=20.17'} @@ -5975,10 +5990,27 @@ packages: pend@1.2.0: resolution: {integrity: sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==} + pg-cloudflare@1.2.7: + resolution: {integrity: sha512-YgCtzMH0ptvZJslLM1ffsY4EuGaU0cx4XSdXLRFae8bPP4dS5xL1tNB3k2o/N64cHJpwU7dxKli/nZ2lUa5fLg==} + + pg-connection-string@2.9.1: + resolution: {integrity: sha512-nkc6NpDcvPVpZXxrreI/FOtX3XemeLl8E0qFr6F2Lrm/I8WOnaWNhIPK2Z7OHpw7gh5XJThi6j6ppgNoaT1w4w==} + pg-int8@1.0.1: resolution: {integrity: sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==} engines: {node: '>=4.0.0'} + pg-native@3.5.2: + resolution: {integrity: sha512-3oi+KVil86Vngo4H0IlhBaYSJWdcu8t2f1Y4TkQoQi5oZ9bNeYECGqW3oSGx69mjSZYHoC3h+3jYtqzRgndn5A==} + + pg-pool@3.10.1: + resolution: {integrity: sha512-Tu8jMlcX+9d8+QVzKIvM/uJtp07PKr82IUOYEphaWcoBhIYkoHpLXN3qO59nAI11ripznDsEzEv8nUxBVWajGg==} + peerDependencies: + pg: '>=8.0' + + pg-protocol@1.10.3: + resolution: {integrity: sha512-6DIBgBQaTKDJyxnXaLiLR8wBpQQcGWuAESkRBX/t6OwA8YsqP+iVSiond2EDy6Y/dsGk8rh/jtax3js5NeV7JQ==} + pg-protocol@1.6.1: resolution: {integrity: sha512-jPIlvgoD63hrEuihvIg+tJhoGjUsLPn6poJY9N5CnlPd91c2T18T/9zBtLxZSb1EhYxBRoZJtzScCaWlYLtktg==} @@ -5986,6 +6018,18 @@ packages: resolution: {integrity: sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==} engines: {node: '>=4'} + pg@8.16.3: + resolution: {integrity: sha512-enxc1h0jA/aq5oSDMvqyW3q89ra6XIIDZgCX9vkMrnz5DFTw/Ny3Li2lFQ+pt3L6MCgm/5o2o8HW9hiJji+xvw==} + engines: {node: '>= 16.0.0'} + peerDependencies: + pg-native: '>=3.0.1' + peerDependenciesMeta: + pg-native: + optional: true + + pgpass@1.0.5: + resolution: {integrity: sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==} + picocolors@1.0.1: resolution: {integrity: sha512-anP1Z8qwhkbmu7MFP5iTt+wQKXgwzf7zTyGlcdzabySa9vd0Xt392U0rVmz9poOaBj0uHJKyyo9/upk0HrEQew==} @@ -11585,7 +11629,7 @@ snapshots: '@types/pg-pool@2.0.6': dependencies: - '@types/pg': 8.6.1 + '@types/pg': 8.15.5 '@types/pg@8.15.4': dependencies: @@ -11593,6 +11637,12 @@ snapshots: pg-protocol: 1.6.1 pg-types: 2.2.0 + '@types/pg@8.15.5': + dependencies: + '@types/node': 20.14.1 + pg-protocol: 1.10.3 + pg-types: 2.2.0 + '@types/pg@8.6.1': dependencies: '@types/node': 20.14.1 @@ -14356,6 +14406,12 @@ snapshots: leven@3.1.0: {} + libpq@1.8.15: + dependencies: + bindings: 1.5.0 + nan: 2.22.2 + optional: true + lie@3.3.0: dependencies: immediate: 3.0.6 @@ -14618,6 +14674,9 @@ snapshots: mute-stream@2.0.0: {} + nan@2.22.2: + optional: true + nano-spawn@1.0.2: {} nanoid@3.3.8: {} @@ -14931,8 +14990,25 @@ snapshots: pend@1.2.0: {} + pg-cloudflare@1.2.7: + optional: true + + pg-connection-string@2.9.1: {} + pg-int8@1.0.1: {} + pg-native@3.5.2: + dependencies: + libpq: 1.8.15 + pg-types: 2.2.0 + optional: true + + pg-pool@3.10.1(pg@8.16.3(pg-native@3.5.2)): + dependencies: + pg: 8.16.3(pg-native@3.5.2) + + pg-protocol@1.10.3: {} + pg-protocol@1.6.1: {} pg-types@2.2.0: @@ -14943,6 +15019,21 @@ snapshots: postgres-date: 1.0.7 postgres-interval: 1.2.0 + pg@8.16.3(pg-native@3.5.2): + dependencies: + pg-connection-string: 2.9.1 + pg-pool: 3.10.1(pg@8.16.3(pg-native@3.5.2)) + pg-protocol: 1.10.3 + pg-types: 2.2.0 + pgpass: 1.0.5 + optionalDependencies: + pg-cloudflare: 1.2.7 + pg-native: 3.5.2 + + pgpass@1.0.5: + dependencies: + split2: 4.2.0 + picocolors@1.0.1: {} picocolors@1.1.1: {} diff --git a/apps/api/requests.http b/apps/api/requests.http index 1507fd99a..5ffc11708 100644 --- a/apps/api/requests.http +++ b/apps/api/requests.http @@ -1,6 +1,6 @@ # Pick your baseUrl here: -# @baseUrl = http://localhost:3002 -@baseUrl = https://api.firecrawl.dev +@baseUrl = http://localhost:3002 +# @baseUrl = https://api.firecrawl.dev ### Scrape Website # @name scrape diff --git a/apps/api/sharedLibs/go-html-to-md/.gitignore b/apps/api/sharedLibs/go-html-to-md/.gitignore index 9e40cf424..909db1866 100644 --- a/apps/api/sharedLibs/go-html-to-md/.gitignore +++ b/apps/api/sharedLibs/go-html-to-md/.gitignore @@ -1,2 +1,3 @@ html-to-markdown.* -!html-to-markdown.go \ No newline at end of file +!html-to-markdown.go +libhtml-to-markdown.* \ No newline at end of file diff --git a/apps/api/src/__tests__/queue-concurrency-integration.test.ts b/apps/api/src/__tests__/queue-concurrency-integration.test.ts deleted file mode 100644 index 7e2141d07..000000000 --- a/apps/api/src/__tests__/queue-concurrency-integration.test.ts +++ /dev/null @@ -1,269 +0,0 @@ -import { redisEvictConnection } from "../services/redis"; -import { addScrapeJob, addScrapeJobs } from "../services/queue-jobs"; -import { - cleanOldConcurrencyLimitEntries, - pushConcurrencyLimitActiveJob, - takeConcurrencyLimitedJob, - removeConcurrencyLimitActiveJob, -} from "../lib/concurrency-limit"; -import { WebScraperOptions } from "../types"; -import { getACUCTeam } from "../controllers/auth"; - -// Mock all the dependencies -const mockAdd = jest.fn(); -jest.mock("../services/queue-service", () => ({ - redisConnection: { - zremrangebyscore: jest.fn(), - zrangebyscore: jest.fn(), - zadd: jest.fn(), - zrem: jest.fn(), - zmpop: jest.fn(), - zcard: jest.fn(), - smembers: jest.fn(), - }, - getScrapeQueue: jest.fn(() => ({ - add: mockAdd, - })), -})); - -jest.mock("uuid", () => ({ - v4: jest.fn(() => "mock-uuid"), -})); - -describe("Queue Concurrency Integration", () => { - const mockTeamId = "test-team-id"; - const mockNow = Date.now(); - - const defaultScrapeOptions = { - formats: [{ type: "markdown" }], - onlyMainContent: true, - waitFor: 0, - mobile: false, - parsePDF: false, - timeout: 30000, - javascript: true, - headers: {}, - cookies: [], - blockResources: true, - skipTlsVerification: false, - removeBase64Images: true, - fastMode: false, - blockAds: true, - maxAge: 0, - storeInCache: true, - proxy: "basic", - }; - - beforeEach(() => { - jest.clearAllMocks(); - jest.spyOn(Date, "now").mockImplementation(() => mockNow); - }); - - describe("Single Job Addition", () => { - const mockWebScraperOptions: WebScraperOptions = { - url: "https://test.com", - mode: "single_urls", - team_id: mockTeamId, - scrapeOptions: defaultScrapeOptions, - crawlerOptions: null, - zeroDataRetention: false, - apiKeyId: null, - } as WebScraperOptions; - - it("should add job directly to BullMQ when under concurrency limit", async () => { - // Mock current active jobs to be under limit - (redisEvictConnection.zrangebyscore as jest.Mock).mockResolvedValue([]); - - await addScrapeJob(mockWebScraperOptions); - - // Should have checked concurrency - expect(redisEvictConnection.zrangebyscore).toHaveBeenCalled(); - - // Should have added to BullMQ - expect(mockAdd).toHaveBeenCalled(); - - // Should have added to active jobs - expect(redisEvictConnection.zadd).toHaveBeenCalledWith( - expect.stringContaining("concurrency-limiter"), - expect.any(Number), - expect.any(String), - ); - }); - - it("should add job to concurrency queue when at concurrency limit", async () => { - // Mock current active jobs to be at limit - (getACUCTeam as jest.Mock).mockResolvedValue({ - concurrency: 15, - } as any); - const activeJobs = Array(15).fill("active-job"); - (redisEvictConnection.zrangebyscore as jest.Mock).mockResolvedValue( - activeJobs, - ); - - await addScrapeJob(mockWebScraperOptions); - - // Should have checked concurrency - expect(redisEvictConnection.zrangebyscore).toHaveBeenCalled(); - - // Should NOT have added to BullMQ - expect(mockAdd).not.toHaveBeenCalled(); - - // Should have added to concurrency queue - expect(redisEvictConnection.zadd).toHaveBeenCalledWith( - expect.stringContaining("concurrency-limit-queue"), - expect.any(Number), - expect.stringContaining("mock-uuid"), - ); - }); - }); - - describe("Batch Job Addition", () => { - const createMockJobs = (count: number) => - Array(count) - .fill(null) - .map((_, i) => ({ - data: { - url: `https://test${i}.com`, - mode: "single_urls", - team_id: mockTeamId, - scrapeOptions: defaultScrapeOptions, - zeroDataRetention: false, - } as any, - opts: { - jobId: `job-${i}`, - priority: 1, - }, - })); - - it("should handle batch jobs respecting concurrency limits", async () => { - const maxConcurrency = 15; - (getACUCTeam as jest.Mock).mockResolvedValue({ - concurrency: maxConcurrency, - } as any); - const totalJobs = maxConcurrency + 5; // Some jobs should go to queue - const mockJobs = createMockJobs(totalJobs); - - // Mock current active jobs to be empty - (redisEvictConnection.zrangebyscore as jest.Mock).mockResolvedValue([]); - - await addScrapeJobs(mockJobs); - - // Should have added maxConcurrency jobs to BullMQ - expect(mockAdd).toHaveBeenCalledTimes(maxConcurrency); - - // Should have added remaining jobs to concurrency queue - expect(redisEvictConnection.zadd).toHaveBeenCalledWith( - expect.stringContaining("concurrency-limit-queue"), - expect.any(Number), - expect.any(String), - ); - }); - - it("should handle empty job array", async () => { - const result = await addScrapeJobs([]); - expect(result).toBe(true); - expect(mockAdd).not.toHaveBeenCalled(); - expect(redisEvictConnection.zadd).not.toHaveBeenCalled(); - }); - }); - - describe("Queue Worker Integration", () => { - it("should process next queued job when active job completes", async () => { - const mockJob = { - id: "test-job", - data: { - team_id: mockTeamId, - zeroDataRetention: false, - }, - }; - - // Mock a queued job - const queuedJob = { - id: "queued-job", - data: { test: "data" }, - opts: {}, - }; - (redisEvictConnection.zmpop as jest.Mock).mockResolvedValueOnce([ - "key", - [[JSON.stringify(queuedJob)]], - ]); - - // Simulate job completion in worker - await removeConcurrencyLimitActiveJob(mockTeamId, mockJob.id); - await cleanOldConcurrencyLimitEntries(mockTeamId); - - const nextJob = await takeConcurrencyLimitedJob(mockTeamId); - - // Should have taken next job from queue - expect(nextJob).toEqual(queuedJob); - - // Should have added new job to active jobs - await pushConcurrencyLimitActiveJob( - mockTeamId, - nextJob!.id, - 2 * 60 * 1000, - ); - expect(redisEvictConnection.zadd).toHaveBeenCalledWith( - expect.stringContaining("concurrency-limiter"), - expect.any(Number), - nextJob!.id, - ); - }); - - it("should handle job failure and cleanup", async () => { - const mockJob = { - id: "failing-job", - data: { - team_id: mockTeamId, - }, - }; - - // Add job to active jobs - await pushConcurrencyLimitActiveJob( - mockTeamId, - mockJob.id, - 2 * 60 * 1000, - ); - - // Simulate job failure and cleanup - await removeConcurrencyLimitActiveJob(mockTeamId, mockJob.id); - await cleanOldConcurrencyLimitEntries(mockTeamId); - - // Verify job was removed from active jobs - expect(redisEvictConnection.zrem).toHaveBeenCalledWith( - expect.stringContaining("concurrency-limiter"), - mockJob.id, - ); - }); - }); - - describe("Edge Cases", () => { - it("should handle stalled jobs cleanup", async () => { - const stalledTime = mockNow - 3 * 60 * 1000; // 3 minutes ago - - // Mock stalled jobs in Redis - (redisEvictConnection.zrangebyscore as jest.Mock).mockResolvedValueOnce([ - "stalled-job", - ]); - - await cleanOldConcurrencyLimitEntries(mockTeamId, mockNow); - - // Should have cleaned up stalled jobs - expect(redisEvictConnection.zremrangebyscore).toHaveBeenCalledWith( - expect.stringContaining("concurrency-limiter"), - -Infinity, - mockNow, - ); - }); - - it("should handle race conditions in job queue processing", async () => { - // Mock a race condition where job is taken by another worker - (redisEvictConnection.zmpop as jest.Mock).mockResolvedValueOnce(null); - - const nextJob = await takeConcurrencyLimitedJob(mockTeamId); - - // Should handle gracefully when no job is available - expect(nextJob).toBeNull(); - }); - }); -}); diff --git a/apps/api/src/__tests__/snips/lib.ts b/apps/api/src/__tests__/snips/lib.ts index 43ef79ed7..ba9fec277 100644 --- a/apps/api/src/__tests__/snips/lib.ts +++ b/apps/api/src/__tests__/snips/lib.ts @@ -7,7 +7,7 @@ import { TeamFlags } from "../../controllers/v1/types"; // Configuration // ========================================= -export const TEST_URL = "http://127.0.0.1:3002"; +export const TEST_URL = process.env.TEST_API_URL || "http://127.0.0.1:3002"; // Due to the limited resources of the CI runner, we need to set a longer timeout for the many many scrape tests export const scrapeTimeout = 90000; diff --git a/apps/api/src/__tests__/snips/v1/crawl.test.ts b/apps/api/src/__tests__/snips/v1/crawl.test.ts index bb033d5cf..ce0550c66 100644 --- a/apps/api/src/__tests__/snips/v1/crawl.test.ts +++ b/apps/api/src/__tests__/snips/v1/crawl.test.ts @@ -387,110 +387,3 @@ describe("Crawl tests", () => { }, ); }); - -describe("Robots.txt FFI Integration tests", () => { - it.concurrent( - "handles normal robots.txt parsing via FFI", - async () => { - const result = await filterLinks({ - links: [ - "https://example.com/allowed", - "https://example.com/disallowed", - ], - limit: 10, - maxDepth: 10, - baseUrl: "https://example.com", - initialUrl: "https://example.com", - regexOnFullUrl: false, - excludes: [], - includes: [], - allowBackwardCrawling: true, - ignoreRobotsTxt: false, - robotsTxt: "User-agent: *\nDisallow: /disallowed", - }); - - expect(result.links).toHaveLength(1); - expect(result.links[0]).toBe("https://example.com/allowed"); - expect("https://example.com/disallowed" in result.denialReasons).toBe( - true, - ); - expect(result.denialReasons["https://example.com/disallowed"]).toBe( - "ROBOTS_TXT", - ); - }, - 10000, - ); - - it.concurrent( - "handles malformed robots.txt without crashing via FFI", - async () => { - const result = await filterLinks({ - links: ["https://example.com/test"], - limit: 10, - maxDepth: 10, - baseUrl: "https://example.com", - initialUrl: "https://example.com", - regexOnFullUrl: false, - excludes: [], - includes: [], - allowBackwardCrawling: true, - ignoreRobotsTxt: false, - robotsTxt: - "Invalid robots.txt content with \x00 null bytes and malformed syntax", - }); - - expect(result.links).toHaveLength(1); - expect(result.links[0]).toBe("https://example.com/test"); - }, - 10000, - ); - - it.concurrent( - "handles non-UTF8 robots.txt content without crashing via FFI", - async () => { - const nonUtf8Content = - String.fromCharCode(0xff, 0xfe) + "User-agent: *\nDisallow: /blocked"; - const result = await filterLinks({ - links: ["https://example.com/allowed"], - limit: 10, - maxDepth: 10, - baseUrl: "https://example.com", - initialUrl: "https://example.com", - regexOnFullUrl: false, - excludes: [], - includes: [], - allowBackwardCrawling: true, - ignoreRobotsTxt: false, - robotsTxt: nonUtf8Content, - }); - - expect(result.links).toHaveLength(1); - expect(result.links[0]).toBe("https://example.com/allowed"); - }, - 10000, - ); - - it.concurrent( - "handles char boundary issues without crashing via FFI", - async () => { - const problematicContent = "User-agent: *\nDisallow: /\u{a0}test"; - const result = await filterLinks({ - links: ["https://example.com/safe"], - limit: 10, - maxDepth: 10, - baseUrl: "https://example.com", - initialUrl: "https://example.com", - regexOnFullUrl: false, - excludes: [], - includes: [], - allowBackwardCrawling: true, - ignoreRobotsTxt: false, - robotsTxt: problematicContent, - }); - - expect(result.links).toHaveLength(1); - expect(result.links[0]).toBe("https://example.com/safe"); - }, - 10000, - ); -}); diff --git a/apps/api/src/__tests__/snips/v1/zdr.test.ts b/apps/api/src/__tests__/snips/v1/zdr.test.ts index b23c49a02..27d7b81db 100644 --- a/apps/api/src/__tests__/snips/v1/zdr.test.ts +++ b/apps/api/src/__tests__/snips/v1/zdr.test.ts @@ -1,3 +1,4 @@ +import { exec, spawn } from "node:child_process"; import { getJobFromGCS } from "../../../lib/gcs-jobs"; import { supabase_service } from "../../../services/supabase"; import { @@ -20,6 +21,9 @@ const logIgnoreList = [ "Billing batch processing", "Processing batch of", "Billing team", + "No jobs to process", + "nuqHealthCheck metrics", + "nuqGetJobToProcess metrics", ]; if (process.env.TEST_SUITE_SELF_HOSTED) { @@ -27,7 +31,35 @@ if (process.env.TEST_SUITE_SELF_HOSTED) { expect(true).toBe(true); }); } else { + function getOutput(command: string): Promise { + return new Promise((resolve, reject) => { + try { + const cmd = spawn(command, { shell: true }); + let out = ""; + cmd.stdout?.on("data", data => { + out += data; + }); + cmd.stderr?.on("data", data => { + out += data; + }); + cmd.on("error", e => { + reject(e); + }); + cmd.on("close", code => { + if (code !== 0) { + reject(new Error(`Command ${command} failed with code ${code}`)); + } else { + resolve(out); + } + }); + } catch (e) { + reject(e); + } + }); + } + async function getServerLogs() { + let logs: string; if (!process.env.GITHUB_ACTIONS) { try { await stat("api.log"); @@ -35,8 +67,10 @@ if (process.env.TEST_SUITE_SELF_HOSTED) { console.warn("No api.log file found"); return []; } + logs = await readFile("api.log", "utf8"); + } else { + logs = await getOutput("kubectl logs deployment/firecrawl-app"); } - const logs = await readFile("api.log", "utf8"); return logs .split("\n") .filter( @@ -45,6 +79,7 @@ if (process.env.TEST_SUITE_SELF_HOSTED) { } async function getWorkerLogs() { + let logs: string; if (!process.env.GITHUB_ACTIONS) { try { await stat("worker.log"); @@ -52,8 +87,10 @@ if (process.env.TEST_SUITE_SELF_HOSTED) { console.warn("No worker.log file found"); return []; } + logs = await readFile("worker.log", "utf8"); + } else { + logs = await getOutput("kubectl logs deployment/firecrawl-nuq-worker"); } - const logs = await readFile("worker.log", "utf8"); return logs .split("\n") .filter( diff --git a/apps/api/src/__tests__/snips/v2/crawl.test.ts b/apps/api/src/__tests__/snips/v2/crawl.test.ts index 70c077abc..d81e7a74e 100644 --- a/apps/api/src/__tests__/snips/v2/crawl.test.ts +++ b/apps/api/src/__tests__/snips/v2/crawl.test.ts @@ -389,110 +389,3 @@ describe("Crawl tests", () => { }); } }); - -describe("Robots.txt FFI Integration tests", () => { - it.concurrent( - "handles normal robots.txt parsing via FFI", - async () => { - const result = await filterLinks({ - links: [ - "https://example.com/allowed", - "https://example.com/disallowed", - ], - limit: 10, - maxDepth: 10, - baseUrl: "https://example.com", - initialUrl: "https://example.com", - regexOnFullUrl: false, - excludes: [], - includes: [], - allowBackwardCrawling: true, - ignoreRobotsTxt: false, - robotsTxt: "User-agent: *\nDisallow: /disallowed", - }); - - expect(result.links).toHaveLength(1); - expect(result.links[0]).toBe("https://example.com/allowed"); - expect("https://example.com/disallowed" in result.denialReasons).toBe( - true, - ); - expect(result.denialReasons["https://example.com/disallowed"]).toBe( - "ROBOTS_TXT", - ); - }, - 10000, - ); - - it.concurrent( - "handles malformed robots.txt without crashing via FFI", - async () => { - const result = await filterLinks({ - links: ["https://example.com/test"], - limit: 10, - maxDepth: 10, - baseUrl: "https://example.com", - initialUrl: "https://example.com", - regexOnFullUrl: false, - excludes: [], - includes: [], - allowBackwardCrawling: true, - ignoreRobotsTxt: false, - robotsTxt: - "Invalid robots.txt content with \x00 null bytes and malformed syntax", - }); - - expect(result.links).toHaveLength(1); - expect(result.links[0]).toBe("https://example.com/test"); - }, - 10000, - ); - - it.concurrent( - "handles non-UTF8 robots.txt content without crashing via FFI", - async () => { - const nonUtf8Content = - String.fromCharCode(0xff, 0xfe) + "User-agent: *\nDisallow: /blocked"; - const result = await filterLinks({ - links: ["https://example.com/allowed"], - limit: 10, - maxDepth: 10, - baseUrl: "https://example.com", - initialUrl: "https://example.com", - regexOnFullUrl: false, - excludes: [], - includes: [], - allowBackwardCrawling: true, - ignoreRobotsTxt: false, - robotsTxt: nonUtf8Content, - }); - - expect(result.links).toHaveLength(1); - expect(result.links[0]).toBe("https://example.com/allowed"); - }, - 10000, - ); - - it.concurrent( - "handles char boundary issues without crashing via FFI", - async () => { - const problematicContent = "User-agent: *\nDisallow: /\u{a0}test"; - const result = await filterLinks({ - links: ["https://example.com/safe"], - limit: 10, - maxDepth: 10, - baseUrl: "https://example.com", - initialUrl: "https://example.com", - regexOnFullUrl: false, - excludes: [], - includes: [], - allowBackwardCrawling: true, - ignoreRobotsTxt: false, - robotsTxt: problematicContent, - }); - - expect(result.links).toHaveLength(1); - expect(result.links[0]).toBe("https://example.com/safe"); - }, - 10000, - ); -}); diff --git a/apps/api/src/__tests__/snips/v2/zdr.test.ts b/apps/api/src/__tests__/snips/v2/zdr.test.ts index 889b14e76..61937aa7c 100644 --- a/apps/api/src/__tests__/snips/v2/zdr.test.ts +++ b/apps/api/src/__tests__/snips/v2/zdr.test.ts @@ -9,6 +9,7 @@ import { idmux, } from "./lib"; import { readFile, stat } from "node:fs/promises"; +import { spawn } from "node:child_process"; const logIgnoreList = [ "Billing queue created", @@ -20,6 +21,9 @@ const logIgnoreList = [ "Billing batch processing", "Processing batch of", "Billing team", + "No jobs to process", + "nuqHealthCheck metrics", + "nuqGetJobToProcess metrics", ]; if (process.env.TEST_SUITE_SELF_HOSTED) { @@ -27,7 +31,35 @@ if (process.env.TEST_SUITE_SELF_HOSTED) { expect(true).toBe(true); }); } else { + function getOutput(command: string): Promise { + return new Promise((resolve, reject) => { + try { + const cmd = spawn(command, { shell: true }); + let out = ""; + cmd.stdout?.on("data", data => { + out += data; + }); + cmd.stderr?.on("data", data => { + out += data; + }); + cmd.on("error", e => { + reject(e); + }); + cmd.on("close", code => { + if (code !== 0) { + reject(new Error(`Command ${command} failed with code ${code}`)); + } else { + resolve(out); + } + }); + } catch (e) { + reject(e); + } + }); + } + async function getServerLogs() { + let logs: string; if (!process.env.GITHUB_ACTIONS) { try { await stat("api.log"); @@ -35,8 +67,10 @@ if (process.env.TEST_SUITE_SELF_HOSTED) { console.warn("No api.log file found"); return []; } + logs = await readFile("api.log", "utf8"); + } else { + logs = await getOutput("kubectl logs deployment/firecrawl-app"); } - const logs = await readFile("api.log", "utf8"); return logs .split("\n") .filter( @@ -45,6 +79,7 @@ if (process.env.TEST_SUITE_SELF_HOSTED) { } async function getWorkerLogs() { + let logs: string; if (!process.env.GITHUB_ACTIONS) { try { await stat("worker.log"); @@ -52,8 +87,10 @@ if (process.env.TEST_SUITE_SELF_HOSTED) { console.warn("No worker.log file found"); return []; } + logs = await readFile("worker.log", "utf8"); + } else { + logs = await getOutput("kubectl logs deployment/firecrawl-nuq-worker"); } - const logs = await readFile("worker.log", "utf8"); return logs .split("\n") .filter( diff --git a/apps/api/src/controllers/v0/admin/metrics.ts b/apps/api/src/controllers/v0/admin/metrics.ts index afe0c055b..54b12cdc9 100644 --- a/apps/api/src/controllers/v0/admin/metrics.ts +++ b/apps/api/src/controllers/v0/admin/metrics.ts @@ -1,5 +1,6 @@ import type { Request, Response } from "express"; import { redisEvictConnection } from "../../../services/redis"; +import { nuqGetLocalMetrics, scrapeQueue } from "../../../services/worker/nuq"; export async function metricsController(_: Request, res: Response) { let cursor: string = "0"; @@ -34,5 +35,9 @@ ${Object.entries(metrics) `concurrency_limit_queue_job_count{team_id="${key}"} ${value}`, ) .join("\n")} + +${await scrapeQueue.getMetrics()} + +${nuqGetLocalMetrics()} `); } diff --git a/apps/api/src/controllers/v0/admin/queue.ts b/apps/api/src/controllers/v0/admin/queue.ts deleted file mode 100644 index c5a87cb61..000000000 --- a/apps/api/src/controllers/v0/admin/queue.ts +++ /dev/null @@ -1,201 +0,0 @@ -import { Request, Response } from "express"; - -import { Job } from "bullmq"; -import { logger } from "../../../lib/logger"; -import { getScrapeQueue } from "../../../services/queue-service"; -import { checkAlerts } from "../../../services/alerts"; -import { sendSlackWebhook } from "../../../services/alerts/slack"; - -export async function cleanBefore24hCompleteJobsController( - req: Request, - res: Response, -) { - logger.info("🐂 Cleaning jobs older than 24h"); - try { - const scrapeQueue = getScrapeQueue(); - const batchSize = 10; - const numberOfBatches = 9; // Adjust based on your needs - const completedJobsPromises: Promise[] = []; - for (let i = 0; i < numberOfBatches; i++) { - completedJobsPromises.push( - scrapeQueue.getJobs( - ["completed"], - i * batchSize, - i * batchSize + batchSize, - true, - ), - ); - } - const completedJobs: Job[] = ( - await Promise.all(completedJobsPromises) - ).flat(); - const before24hJobs = - completedJobs.filter( - job => - job.finishedOn !== undefined && - job.finishedOn < Date.now() - 24 * 60 * 60 * 1000, - ) || []; - - let count = 0; - - if (!before24hJobs) { - return res.status(200).send(`No jobs to remove.`); - } - - for (const job of before24hJobs) { - try { - await job.remove(); - count++; - } catch (jobError) { - logger.error(`🐂 Failed to remove job with ID ${job.id}: ${jobError}`); - } - } - return res.status(200).send(`Removed ${count} completed jobs.`); - } catch (error) { - logger.error(`🐂 Failed to clean last 24h complete jobs: ${error}`); - return res.status(500).send("Failed to clean jobs"); - } -} - -export async function checkQueuesController(req: Request, res: Response) { - try { - await checkAlerts(); - return res.status(200).send("Alerts initialized"); - } catch (error) { - logger.debug(`Failed to initialize alerts: ${error}`); - return res.status(500).send("Failed to initialize alerts"); - } -} - -// Use this as a "health check" that way we dont destroy the server -export async function queuesController(req: Request, res: Response) { - try { - const scrapeQueue = getScrapeQueue(); - - const [webScraperActive] = await Promise.all([ - scrapeQueue.getActiveCount(), - ]); - - const noActiveJobs = webScraperActive === 0; - // 200 if no active jobs, 503 if there are active jobs - return res.status(noActiveJobs ? 200 : 500).json({ - webScraperActive, - noActiveJobs, - }); - } catch (error) { - logger.error(error); - return res.status(500).json({ error: error.message }); - } -} - -export async function autoscalerController(req: Request, res: Response) { - try { - const maxNumberOfMachines = 80; - const minNumberOfMachines = 20; - - const scrapeQueue = getScrapeQueue(); - - const [webScraperActive, webScraperWaiting, webScraperPriority] = - await Promise.all([ - scrapeQueue.getActiveCount(), - scrapeQueue.getWaitingCount(), - scrapeQueue.getPrioritizedCount(), - ]); - - let waitingAndPriorityCount = webScraperWaiting + webScraperPriority; - - // get number of machines active - const request = await fetch( - "https://api.machines.dev/v1/apps/firecrawl-scraper-js/machines", - { - headers: { - Authorization: `Bearer ${process.env.FLY_API_TOKEN}`, - }, - }, - ); - const machines = await request.json(); - - // Only worker machines - const activeMachines = machines.filter( - machine => - (machine.state === "started" || - machine.state === "starting" || - machine.state === "replacing") && - machine.config.env["FLY_PROCESS_GROUP"] === "worker", - ).length; - - let targetMachineCount = activeMachines; - - const baseScaleUp = 10; - // Slow scale down - const baseScaleDown = 2; - - // Scale up logic - if (webScraperActive > 9000 || waitingAndPriorityCount > 2000) { - targetMachineCount = Math.min( - maxNumberOfMachines, - activeMachines + baseScaleUp * 3, - ); - } else if (webScraperActive > 5000 || waitingAndPriorityCount > 1000) { - targetMachineCount = Math.min( - maxNumberOfMachines, - activeMachines + baseScaleUp * 2, - ); - } else if (webScraperActive > 1000 || waitingAndPriorityCount > 500) { - targetMachineCount = Math.min( - maxNumberOfMachines, - activeMachines + baseScaleUp, - ); - } - - // Scale down logic - if (webScraperActive < 100 && waitingAndPriorityCount < 50) { - targetMachineCount = Math.max( - minNumberOfMachines, - activeMachines - baseScaleDown * 3, - ); - } else if (webScraperActive < 500 && waitingAndPriorityCount < 200) { - targetMachineCount = Math.max( - minNumberOfMachines, - activeMachines - baseScaleDown * 2, - ); - } else if (webScraperActive < 1000 && waitingAndPriorityCount < 500) { - targetMachineCount = Math.max( - minNumberOfMachines, - activeMachines - baseScaleDown, - ); - } - - if (targetMachineCount !== activeMachines) { - logger.info( - `🐂 Scaling from ${activeMachines} to ${targetMachineCount} - ${webScraperActive} active, ${webScraperWaiting} waiting`, - ); - - if (targetMachineCount > activeMachines) { - sendSlackWebhook( - `🐂 Scaling from ${activeMachines} to ${targetMachineCount} - ${webScraperActive} active, ${webScraperWaiting} waiting - Current DateTime: ${new Date().toISOString()}`, - false, - process.env.SLACK_AUTOSCALER ?? "", - ); - } else { - sendSlackWebhook( - `🐂 Scaling from ${activeMachines} to ${targetMachineCount} - ${webScraperActive} active, ${webScraperWaiting} waiting - Current DateTime: ${new Date().toISOString()}`, - false, - process.env.SLACK_AUTOSCALER ?? "", - ); - } - return res.status(200).json({ - mode: "scale-descale", - count: targetMachineCount, - }); - } - - return res.status(200).json({ - mode: "normal", - count: activeMachines, - }); - } catch (error) { - logger.error(error); - return res.status(500).send("Failed to initialize autoscaler"); - } -} diff --git a/apps/api/src/controllers/v0/crawl-status.ts b/apps/api/src/controllers/v0/crawl-status.ts index d2fa6f30f..446e20f0c 100644 --- a/apps/api/src/controllers/v0/crawl-status.ts +++ b/apps/api/src/controllers/v0/crawl-status.ts @@ -1,27 +1,24 @@ import { Request, Response } from "express"; import { authenticateUser } from "../auth"; import { RateLimiterMode } from "../../../src/types"; -import { getScrapeQueue } from "../../../src/services/queue-service"; import { redisEvictConnection } from "../../../src/services/redis"; import { logger } from "../../../src/lib/logger"; import { getCrawl, getCrawlJobs } from "../../../src/lib/crawl-redis"; import { supabaseGetJobsByCrawlId } from "../../../src/lib/supabase-jobs"; import * as Sentry from "@sentry/node"; import { configDotenv } from "dotenv"; -import { Job } from "bullmq"; import { toLegacyDocument } from "../v1/types"; import type { DBJob, PseudoJob } from "../v1/crawl-status"; import { getJobFromGCS } from "../../lib/gcs-jobs"; +import { scrapeQueue, NuQJob } from "../../services/worker/nuq"; configDotenv(); export async function getJobs( crawlId: string, ids: string[], ): Promise[]> { - const [bullJobs, dbJobs, gcsJobs] = await Promise.all([ - Promise.all(ids.map(x => getScrapeQueue().getJob(x))).then(x => - x.filter(x => x), - ) as Promise<(Job & { id: string })[]>, + const [nuqJobs, dbJobs, gcsJobs] = await Promise.all([ + scrapeQueue.getJobs(ids), process.env.USE_DB_AUTHENTICATION === "true" ? await supabaseGetJobsByCrawlId(crawlId) : [], @@ -34,12 +31,12 @@ export async function getJobs( : [], ]); - const bullJobMap = new Map>(); + const nuqJobMap = new Map>(); const dbJobMap = new Map(); const gcsJobMap = new Map(); - for (const job of bullJobs) { - bullJobMap.set(job.id, job); + for (const job of nuqJobs) { + nuqJobMap.set(job.id, job); } for (const job of dbJobs) { @@ -53,13 +50,13 @@ export async function getJobs( const jobs: PseudoJob[] = []; for (const id of ids) { - const bullJob = bullJobMap.get(id); + const nuqJob = nuqJobMap.get(id); const dbJob = dbJobMap.get(id); const gcsJob = gcsJobMap.get(id); - if (!bullJob && !dbJob) continue; + if (!nuqJob && !dbJob) continue; - const data = gcsJob ?? dbJob?.docs ?? bullJob?.returnvalue; + const data = gcsJob ?? dbJob?.docs ?? nuqJob?.returnvalue; if (gcsJob === null && data) { logger.warn("GCS Job not found", { jobId: id, @@ -68,20 +65,16 @@ export async function getJobs( const job: PseudoJob = { id, - getState: dbJob - ? () => (dbJob.success ? "completed" : "failed") - : () => bullJob!.getState(), + status: dbJob ? (dbJob.success ? "completed" : "failed") : nuqJob!.status, returnvalue: Array.isArray(data) ? data[0] : data, data: { - scrapeOptions: bullJob - ? bullJob.data.scrapeOptions - : dbJob!.page_options, + scrapeOptions: nuqJob ? nuqJob.data.scrapeOptions : dbJob!.page_options, }, - timestamp: bullJob - ? bullJob.timestamp + timestamp: nuqJob + ? nuqJob.createdAt.valueOf() : new Date(dbJob!.date_added).valueOf(), failedReason: - (bullJob ? bullJob.failedReason : dbJob!.message) || undefined, + (nuqJob ? nuqJob.failedReason : dbJob!.message) || undefined, }; jobs.push(job); @@ -135,7 +128,7 @@ export async function crawlStatusController(req: Request, res: Response) { } let jobIDs = await getCrawlJobs(req.params.jobId); let jobs = await getJobs(req.params.jobId, jobIDs); - let jobStatuses = await Promise.all(jobs.map(x => x.getState())); + let jobStatuses = jobs.map(x => x.status); // Combine jobs and jobStatuses into a single array of objects let jobsWithStatuses = jobs.map((job, index) => ({ @@ -144,9 +137,7 @@ export async function crawlStatusController(req: Request, res: Response) { })); // Filter out failed jobs - jobsWithStatuses = jobsWithStatuses.filter( - x => x.status !== "failed" && x.status !== "unknown", - ); + jobsWithStatuses = jobsWithStatuses.filter(x => x.status !== "failed"); // Sort jobs by timestamp jobsWithStatuses.sort((a, b) => a.job.timestamp - b.job.timestamp); diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.ts index 8cdcde057..287cd5a3e 100644 --- a/apps/api/src/controllers/v0/crawl.ts +++ b/apps/api/src/controllers/v0/crawl.ts @@ -27,7 +27,7 @@ import { redisEvictConnection } from "../../../src/services/redis"; import { checkAndUpdateURL } from "../../../src/lib/validateUrl"; import * as Sentry from "@sentry/node"; import { getJobPriority } from "../../lib/job-priority"; -import { fromLegacyScrapeOptions, url as urlSchema } from "../v1/types"; +import { url as urlSchema } from "../v1/types"; import { ZodError } from "zod"; import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings"; import { fromV0ScrapeOptions } from "../v2/types"; @@ -216,7 +216,7 @@ export async function crawlController(req: Request, res: Response) { const jobs = urls.map(url => { const uuid = uuidv4(); return { - name: uuid, + jobId: uuid, data: { url, mode: "single_urls" as const, @@ -231,10 +231,7 @@ export async function crawlController(req: Request, res: Response) { zeroDataRetention: false, // not supported on v0 apiKeyId: chunk?.api_key_id ?? null, }, - opts: { - jobId: uuid, - priority: jobPriority, - }, + priority: jobPriority, }; }); @@ -246,12 +243,12 @@ export async function crawlController(req: Request, res: Response) { ); await addCrawlJobs( id, - jobs.map(x => x.opts.jobId), + jobs.map(x => x.jobId), logger, ); for (const job of jobs) { // add with sentry instrumentation - await addScrapeJob(job.data, {}, job.opts.jobId); + await addScrapeJob(job.data, job.jobId, job.priority); } }); @@ -276,10 +273,8 @@ export async function crawlController(req: Request, res: Response) { zeroDataRetention: false, // not supported on v0 apiKeyId: chunk?.api_key_id ?? null, }, - { - priority: 15, // prioritize request 0 of crawl jobs same as scrape jobs - }, jobId, + await getJobPriority({ team_id, basePriority: 15 }), ); await addCrawlJob(id, jobId, logger); } diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.ts index be901c96b..3d48d832b 100644 --- a/apps/api/src/controllers/v0/scrape.ts +++ b/apps/api/src/controllers/v0/scrape.ts @@ -12,7 +12,6 @@ import { defaultOrigin, } from "../../lib/default-values"; import { addScrapeJob, waitForJob } from "../../services/queue-jobs"; -import { getScrapeQueue } from "../../services/queue-service"; import { redisEvictConnection } from "../../../src/services/redis"; import { v4 as uuidv4 } from "uuid"; import { logger } from "../../lib/logger"; @@ -23,6 +22,7 @@ import { Document as V0Document } from "./../../lib/entities"; import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings"; import { fromV0Combo } from "../v2/types"; import { ScrapeJobTimeoutError } from "../../lib/error"; +import { scrapeQueue } from "../../services/worker/nuq"; async function scrapeHelper( jobId: string, @@ -53,8 +53,6 @@ async function scrapeHelper( }; } - const jobPriority = await getJobPriority({ team_id, basePriority: 10 }); - const { scrapeOptions, internalOptions } = fromV0Combo( pageOptions, extractorOptions, @@ -81,15 +79,14 @@ async function scrapeHelper( zeroDataRetention: false, // not supported on v0 apiKeyId, }, - {}, jobId, - jobPriority, + await getJobPriority({ team_id, basePriority: 10 }), ); let doc; try { - doc = await waitForJob(jobId, timeout); + doc = await waitForJob(jobId, timeout, false); } catch (e) { if (e instanceof ScrapeJobTimeoutError) { return { @@ -120,7 +117,7 @@ async function scrapeHelper( return err; } - await getScrapeQueue().remove(jobId); + await scrapeQueue.removeJob(jobId); if (!doc) { console.error("!!! PANIC DOC IS", doc); diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.ts index 086742a53..065c54c54 100644 --- a/apps/api/src/controllers/v0/search.ts +++ b/apps/api/src/controllers/v0/search.ts @@ -11,22 +11,18 @@ import { search } from "../../search"; import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist"; import { v4 as uuidv4 } from "uuid"; import { logger } from "../../lib/logger"; -import { getScrapeQueue } from "../../services/queue-service"; import { redisEvictConnection } from "../../../src/services/redis"; import { addScrapeJob, waitForJob } from "../../services/queue-jobs"; import * as Sentry from "@sentry/node"; import { getJobPriority } from "../../lib/job-priority"; -import { Job } from "bullmq"; import { - Document, - fromLegacyCombo, fromLegacyScrapeOptions, TeamFlags, toLegacyDocument, } from "../v1/types"; -import { getJobFromGCS } from "../../lib/gcs-jobs"; import { fromV0Combo } from "../v2/types"; import { ScrapeJobTimeoutError } from "../../lib/error"; +import { scrapeQueue } from "../../services/worker/nuq"; export async function searchHelper( jobId: string, @@ -111,7 +107,7 @@ export async function searchHelper( const url = x.url; const uuid = uuidv4(); return { - name: uuid, + jobId: uuid, data: { url, mode: "single_urls" as const, @@ -122,28 +118,23 @@ export async function searchHelper( zeroDataRetention: false, // not supported on v0 apiKeyId: api_key_id, }, - opts: { - jobId: uuid, - priority: jobPriority, - }, }; }); // TODO: addScrapeJobs for (const job of jobDatas) { - await addScrapeJob(job.data, {}, job.opts.jobId, job.opts.priority); + await addScrapeJob(job.data, job.jobId, jobPriority); } const docs = ( - await Promise.all(jobDatas.map(x => waitForJob(x.opts.jobId, 60000))) + await Promise.all(jobDatas.map(x => waitForJob(x.jobId, 60000, false))) ).map(x => toLegacyDocument(x, internalOptions)); if (docs.length === 0) { return { success: true, error: "No search results found", returnCode: 200 }; } - const sq = getScrapeQueue(); - await Promise.all(jobDatas.map(x => sq.remove(x.opts.jobId))); + await scrapeQueue.removeJobs(jobDatas.map(x => x.jobId)); // make sure doc.content is not empty const filteredDocs = docs.filter( diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.ts index 9cbd93716..bfad81b0b 100644 --- a/apps/api/src/controllers/v1/batch-scrape.ts +++ b/apps/api/src/controllers/v1/batch-scrape.ts @@ -145,6 +145,7 @@ export async function batchScrapeController( logger.debug("Using job priority " + jobPriority, { jobPriority }); const jobs = urls.map(x => ({ + jobId: uuidv4(), data: { url: x, mode: "single_urls" as const, @@ -161,10 +162,7 @@ export async function batchScrapeController( zeroDataRetention: zeroDataRetention ?? false, apiKeyId: req.acuc?.api_key_id ?? null, }, - opts: { - jobId: uuidv4(), - priority: 20, - }, + priority: jobPriority, })); await finishCrawlKickoff(id); @@ -179,7 +177,7 @@ export async function batchScrapeController( logger.debug("Adding scrape jobs to Redis..."); await addCrawlJobs( id, - jobs.map(x => x.opts.jobId), + jobs.map(x => x.jobId), logger, ); logger.debug("Adding scrape jobs to BullMQ..."); diff --git a/apps/api/src/controllers/v1/crawl-errors.ts b/apps/api/src/controllers/v1/crawl-errors.ts index e084310ea..a4b5cd2bb 100644 --- a/apps/api/src/controllers/v1/crawl-errors.ts +++ b/apps/api/src/controllers/v1/crawl-errors.ts @@ -5,31 +5,15 @@ import { RequestWithAuth, } from "./types"; import { getCrawl, getCrawlJobs } from "../../lib/crawl-redis"; -import { getScrapeQueue } from "../../services/queue-service"; import { redisEvictConnection } from "../../../src/services/redis"; import { configDotenv } from "dotenv"; -import { Job } from "bullmq"; import { supabase_rr_service } from "../../services/supabase"; -import { logger } from "../../lib/logger"; +import { logger as _logger } from "../../lib/logger"; import { deserializeTransportableError } from "../../lib/error-serde"; import { TransportableError } from "../../lib/error"; +import { scrapeQueue } from "../../services/worker/nuq"; configDotenv(); -export async function getJob(id: string) { - const job = await getScrapeQueue().getJob(id); - if (!job) return job; - - return job; -} - -export async function getJobs(ids: string[]) { - const jobs: (Job & { id: string })[] = ( - await Promise.all(ids.map(x => getScrapeQueue().getJob(x))) - ).filter(x => x) as (Job & { id: string })[]; - - return jobs; -} - export async function crawlErrorsController( req: RequestWithAuth, res: Response, @@ -41,25 +25,24 @@ export async function crawlErrorsController( return res.status(403).json({ success: false, error: "Forbidden" }); } - let jobStatuses = await Promise.all( - (await getCrawlJobs(req.params.jobId)).map( - async x => [x, await getScrapeQueue().getJobState(x)] as const, - ), - ); + const logger = _logger.child({ + crawlId: req.params.jobId, + zeroDataRetention: sc.zeroDataRetention ?? false, + }); - const failedJobIDs: string[] = []; - - for (const [id, status] of jobStatuses) { - if (status === "failed") { - failedJobIDs.push(id); - } - } + const failedJobs = ( + await scrapeQueue.getJobsWithStatus( + await getCrawlJobs(req.params.jobId), + "failed", + logger, + ) + ).filter(x => x.failedReason); res.status(200).json({ - errors: (await getJobs(failedJobIDs)) + errors: failedJobs .map(x => { const error = deserializeTransportableError( - x.failedReason, + x.failedReason!, ) as TransportableError | null; if (error?.code === "SCRAPE_RACED_REDIRECT_ERROR") { return null; @@ -67,8 +50,8 @@ export async function crawlErrorsController( return { id: x.id, timestamp: - x.finishedOn !== undefined - ? new Date(x.finishedOn).toISOString() + x.finishedAt !== undefined + ? new Date(x.finishedAt).toISOString() : undefined, url: x.data.url, ...(error @@ -77,7 +60,7 @@ export async function crawlErrorsController( error: error.message, } : { - error: x.failedReason, + error: x.failedReason!, }), }; }) @@ -95,7 +78,7 @@ export async function crawlErrorsController( .throwOnError(); if (crawlJobError) { - logger.error("Error getting crawl job", { error: crawlJobError }); + _logger.error("Error getting crawl job", { error: crawlJobError }); throw crawlJobError; } @@ -130,7 +113,7 @@ export async function crawlErrorsController( .throwOnError(); if (failedJobsError) { - logger.error("Error getting failed jobs", { error: failedJobsError }); + _logger.error("Error getting failed jobs", { error: failedJobsError }); throw failedJobsError; } diff --git a/apps/api/src/controllers/v1/crawl-status-ws.ts b/apps/api/src/controllers/v1/crawl-status-ws.ts index 2313566fb..f1ec30606 100644 --- a/apps/api/src/controllers/v1/crawl-status-ws.ts +++ b/apps/api/src/controllers/v1/crawl-status-ws.ts @@ -15,15 +15,11 @@ import { getCrawlExpiry, getCrawlJobs, getDoneJobsOrdered, - getDoneJobsOrderedLength, - isCrawlFinished, - isCrawlFinishedLocked, } from "../../lib/crawl-redis"; -import { getScrapeQueue } from "../../services/queue-service"; -import { getJob, getJobs } from "./crawl-status"; +import { getJobs, PseudoJob } from "./crawl-status"; import * as Sentry from "@sentry/node"; -import { Job, JobState } from "bullmq"; import { getConcurrencyLimitedJobs } from "../../lib/concurrency-limit"; +import { scrapeQueue, NuQJobStatus } from "../../services/worker/nuq"; type ErrorMessage = { type: "error"; @@ -88,18 +84,14 @@ async function crawlStatusWS( const notDoneJobIDs = jobIDs.filter(x => !doneJobIDs.includes(x)); - const queue = getScrapeQueue(); + const newlyDoneJobIDs: string[] = ( + await scrapeQueue.getJobsWithStatuses(notDoneJobIDs, [ + "completed", + "failed", + ]) + ).map(x => x.id); - const jobStatuses = await Promise.all( - notDoneJobIDs.map(async x => [x, await queue.getJobState(x)]), - ); - const newlyDoneJobIDs: string[] = jobStatuses - .filter(x => x[1] === "completed" || x[1] === "failed") - .map(x => x[0]); - - const newlyDoneJobs: Job[] = ( - await Promise.all(newlyDoneJobIDs.map(x => getJob(x))) - ).filter(x => x !== undefined) as Job[]; + const newlyDoneJobs: PseudoJob[] = await getJobs(newlyDoneJobIDs); for (const job of newlyDoneJobs) { if (job.returnvalue) { @@ -118,28 +110,28 @@ async function crawlStatusWS( setTimeout(loop, 1000); - doneJobIDs = await getDoneJobsOrdered(req.params.jobId); + let [_doneJobIDs, jobIDs, throttledJobsSet] = await Promise.all([ + getDoneJobsOrdered(req.params.jobId), + getCrawlJobs(req.params.jobId), + getConcurrencyLimitedJobs(req.auth.team_id), + ]); - let jobIDs = await getCrawlJobs(req.params.jobId); + doneJobIDs = _doneJobIDs; + const jobs = new Map((await scrapeQueue.getJobs(jobIDs)).map(x => [x.id, x])); - const queue = getScrapeQueue(); - - let jobStatuses = await Promise.all( - jobIDs.map(async x => [x, await queue.getJobState(x)] as const), - ); - - const throttledJobsSet = await getConcurrencyLimitedJobs(req.auth.team_id); - - const validJobStatuses: [string, JobState | "unknown"][] = []; + const validJobStatuses: [string, NuQJobStatus][] = []; const validJobIDs: string[] = []; - for (const [id, status] of jobStatuses) { + for (const id of jobIDs) { if (throttledJobsSet.has(id)) { - validJobStatuses.push([id, "prioritized"]); - validJobIDs.push(id); - } else if (status !== "failed" && status !== "unknown") { - validJobStatuses.push([id, status]); + validJobStatuses.push([id, "queued"]); validJobIDs.push(id); + } else { + const job = jobs.get(id); + if (job && job.status !== "failed") { + validJobStatuses.push([id, job.status]); + validJobIDs.push(id); + } } } diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.ts index 8380681ee..d856de2cf 100644 --- a/apps/api/src/controllers/v1/crawl-status.ts +++ b/apps/api/src/controllers/v1/crawl-status.ts @@ -13,21 +13,20 @@ import { getCrawlQualifiedJobCount, getDoneJobsOrderedUntil, } from "../../lib/crawl-redis"; -import { getScrapeQueue } from "../../services/queue-service"; import { supabaseGetJobById, supabaseGetJobsById, } from "../../lib/supabase-jobs"; import { configDotenv } from "dotenv"; -import type { Job, JobState } from "bullmq"; import { logger } from "../../lib/logger"; import { supabase_rr_service, supabase_service } from "../../services/supabase"; import { getJobFromGCS } from "../../lib/gcs-jobs"; +import { scrapeQueue, NuQJob, NuQJobStatus } from "../../services/worker/nuq"; configDotenv(); export type PseudoJob = { id: string; - getState(): Promise | JobState | "unknown"; + status: NuQJobStatus; returnvalue: T | null; timestamp: number; data: { @@ -47,8 +46,8 @@ export type DBJob = { }; export async function getJob(id: string): Promise | null> { - const [bullJob, dbJob, gcsJob] = await Promise.all([ - getScrapeQueue().getJob(id), + const [nuqJob, dbJob, gcsJob] = await Promise.all([ + scrapeQueue.getJob(id), (process.env.USE_DB_AUTHENTICATION === "true" ? supabaseGetJobById(id) : null) as Promise, @@ -57,9 +56,9 @@ export async function getJob(id: string): Promise | null> { >, ]); - if (!bullJob && !dbJob) return null; + if (!nuqJob && !dbJob) return null; - const data = gcsJob ?? dbJob?.docs ?? bullJob?.returnvalue; + const data = gcsJob ?? dbJob?.docs ?? nuqJob?.returnvalue; if (gcsJob === null && data) { logger.warn("GCS Job not found", { jobId: id, @@ -68,28 +67,23 @@ export async function getJob(id: string): Promise | null> { const job: PseudoJob = { id, - getState: dbJob - ? () => (dbJob.success ? "completed" : "failed") - : bullJob!.getState, + status: dbJob ? (dbJob.success ? "completed" : "failed") : nuqJob!.status, returnvalue: Array.isArray(data) ? data[0] : data, data: { - scrapeOptions: bullJob ? bullJob.data.scrapeOptions : dbJob!.page_options, + scrapeOptions: nuqJob ? nuqJob.data.scrapeOptions : dbJob!.page_options, }, - timestamp: bullJob - ? bullJob.timestamp + timestamp: nuqJob + ? nuqJob.createdAt.valueOf() : new Date(dbJob!.date_added).valueOf(), - failedReason: - (bullJob ? bullJob.failedReason : dbJob!.message) || undefined, + failedReason: (nuqJob ? nuqJob.failedReason : dbJob!.message) || undefined, }; return job; } export async function getJobs(ids: string[]): Promise[]> { - const [bullJobs, dbJobs, gcsJobs] = await Promise.all([ - Promise.all(ids.map(x => getScrapeQueue().getJob(x))).then(x => - x.filter(x => x), - ) as Promise<(Job & { id: string })[]>, + const [nuqJobs, dbJobs, gcsJobs] = await Promise.all([ + scrapeQueue.getJobs(ids), process.env.USE_DB_AUTHENTICATION === "true" ? supabaseGetJobsById(ids) : [], @@ -102,12 +96,12 @@ export async function getJobs(ids: string[]): Promise[]> { : [], ]); - const bullJobMap = new Map>(); + const nuqJobMap = new Map>(); const dbJobMap = new Map(); const gcsJobMap = new Map(); - for (const job of bullJobs) { - bullJobMap.set(job.id, job); + for (const job of nuqJobs) { + nuqJobMap.set(job.id, job); } for (const job of dbJobs) { @@ -121,13 +115,13 @@ export async function getJobs(ids: string[]): Promise[]> { const jobs: PseudoJob[] = []; for (const id of ids) { - const bullJob = bullJobMap.get(id); + const nuqJob = nuqJobMap.get(id); const dbJob = dbJobMap.get(id); const gcsJob = gcsJobMap.get(id); - if (!bullJob && !dbJob) continue; + if (!nuqJob && !dbJob) continue; - const data = gcsJob ?? dbJob?.docs ?? bullJob?.returnvalue; + const data = gcsJob ?? dbJob?.docs ?? nuqJob?.returnvalue; if (gcsJob === null && data) { logger.warn("GCS Job not found", { jobId: id, @@ -136,20 +130,16 @@ export async function getJobs(ids: string[]): Promise[]> { const job: PseudoJob = { id, - getState: dbJob - ? () => (dbJob.success ? "completed" : "failed") - : () => bullJob!.getState(), + status: dbJob ? (dbJob.success ? "completed" : "failed") : nuqJob!.status, returnvalue: Array.isArray(data) ? data[0] : data, data: { - scrapeOptions: bullJob - ? bullJob.data.scrapeOptions - : dbJob!.page_options, + scrapeOptions: nuqJob ? nuqJob.data.scrapeOptions : dbJob!.page_options, }, - timestamp: bullJob - ? bullJob.timestamp + timestamp: nuqJob + ? nuqJob.createdAt.valueOf() : new Date(dbJob!.date_added).valueOf(), failedReason: - (bullJob ? bullJob.failedReason : dbJob!.message) || undefined, + (nuqJob ? nuqJob.failedReason : dbJob!.message) || undefined, }; jobs.push(job); @@ -390,30 +380,35 @@ export async function crawlStatusController( let bytes = 0; const bytesLimit = 10485760; // 10 MiB in bytes - for (const jobId of doneJobs) { - const job = await getScrapeQueue().getJob(jobId); - const state = await job?.getState(); + for (let i = 0; i < Math.ceil(doneJobs.length / 50); i++) { + const jobIds = doneJobs.slice(i * 50, (i + 1) * 50); + const jobs = await getJobs(jobIds); - if (state === "failed") { - // no iterated over, just ignore - continue; - } else { - if (job?.returnvalue) { - scrapes.push(job.returnvalue); - bytes += JSON.stringify(job.returnvalue).length; + for (const job of jobs) { + if (job.status === "failed") { + continue; } else { - logger.warn( - "Job was considered done, but returnvalue is undefined!", - { - scrapeId: jobId, - crawlId: req.params.jobId, - state, - returnvalue: job?.returnvalue, - }, - ); + if (job?.returnvalue) { + scrapes.push(job.returnvalue); + bytes += JSON.stringify(job.returnvalue).length; + } else { + logger.warn( + "Job was considered done, but returnvalue is undefined!", + { + scrapeId: job.id, + crawlId: req.params.jobId, + state: job.status, + returnvalue: job?.returnvalue, + }, + ); + } + + iteratedOver++; } - iteratedOver++; + if (bytes > bytesLimit) { + break; + } } if (bytes > bytesLimit) { diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.ts index 3e4399927..c04236470 100644 --- a/apps/api/src/controllers/v1/crawl.ts +++ b/apps/api/src/controllers/v1/crawl.ts @@ -148,9 +148,7 @@ export async function crawlController( zeroDataRetention: zeroDataRetention || false, apiKeyId: req.acuc?.api_key_id ?? null, }, - {}, crypto.randomUUID(), - 10, ); const protocol = process.env.ENV === "local" ? req.protocol : "https"; diff --git a/apps/api/src/controllers/v1/extract-status.ts b/apps/api/src/controllers/v1/extract-status.ts index d5bf6575b..cea0cc3bf 100644 --- a/apps/api/src/controllers/v1/extract-status.ts +++ b/apps/api/src/controllers/v1/extract-status.ts @@ -1,14 +1,27 @@ import { Response } from "express"; import { RequestWithAuth } from "./types"; import { getExtract, getExtractExpiry } from "../../lib/extract/extract-redis"; -import { DBJob, PseudoJob } from "./crawl-status"; +import { DBJob } from "./crawl-status"; import { getExtractQueue } from "../../services/queue-service"; import { ExtractResult } from "../../lib/extract/extraction-service"; import { supabaseGetJobByIdDirect } from "../../lib/supabase-jobs"; +import { JobState } from "bullmq"; + +type ExtractPseudoJob = { + id: string; + getState: () => Promise | JobState | "unknown"; + returnvalue: T | null; + timestamp: number; + data: { + scrapeOptions: any; + teamId?: string; + }; + failedReason?: string; +}; export async function getExtractJob( id: string, -): Promise | null> { +): Promise | null> { const [bullJob, dbJob] = await Promise.all([ getExtractQueue().getJob(id), (process.env.USE_DB_AUTHENTICATION === "true" @@ -20,7 +33,7 @@ export async function getExtractJob( const data = dbJob?.docs ?? bullJob?.returnvalue?.data; - const job: PseudoJob = { + const job: ExtractPseudoJob = { id, getState: dbJob ? () => (dbJob.success ? "completed" : "failed") diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.ts index a8a85447e..baee96800 100644 --- a/apps/api/src/controllers/v1/scrape.ts +++ b/apps/api/src/controllers/v1/scrape.ts @@ -10,9 +10,9 @@ import { import { v4 as uuidv4 } from "uuid"; import { addScrapeJob, waitForJob } from "../../services/queue-jobs"; import { getJobPriority } from "../../lib/job-priority"; -import { getScrapeQueue } from "../../services/queue-service"; import { fromV1ScrapeOptions } from "../v2/types"; import { TransportableError } from "../../lib/error"; +import { scrapeQueue } from "../../services/worker/nuq"; import { checkPermissions } from "../../lib/permissions"; export async function scrapeController( @@ -54,10 +54,6 @@ export async function scrapeController( const timeout = req.body.timeout; const startTime = new Date().getTime(); - const jobPriority = await getJobPriority({ - team_id: req.auth.team_id, - basePriority: 10, - }); const isDirectToBullMQ = process.env.SEARCH_PREVIEW_TOKEN !== undefined && @@ -69,6 +65,11 @@ export async function scrapeController( req.auth.team_id, ); + const jobPriority = await getJobPriority({ + team_id: req.auth.team_id, + basePriority: 10, + }); + const bullJob = await addScrapeJob( { url: req.body.url, @@ -92,7 +93,6 @@ export async function scrapeController( zeroDataRetention: zeroDataRetention ?? false, apiKeyId: req.acuc?.api_key_id ?? null, }, - {}, jobId, jobPriority, isDirectToBullMQ, @@ -113,6 +113,7 @@ export async function scrapeController( doc = await waitForJob( bullJob ? bullJob : jobId, timeout + totalWait, + zeroDataRetention ?? false, logger, ); } catch (e) { @@ -122,7 +123,7 @@ export async function scrapeController( }); if (zeroDataRetention) { - await getScrapeQueue().remove(jobId); + await scrapeQueue.removeJob(jobId); } if (e instanceof TransportableError) { @@ -142,7 +143,7 @@ export async function scrapeController( logger.info("Done with waitForJob"); - await getScrapeQueue().remove(jobId); + await scrapeQueue.removeJob(jobId); logger.info("Removed job from queue"); diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.ts index 89a8a8e11..00b51ed26 100644 --- a/apps/api/src/controllers/v1/search.ts +++ b/apps/api/src/controllers/v1/search.ts @@ -7,26 +7,25 @@ import { searchRequestSchema, ScrapeOptions, TeamFlags, - scrapeOptions, } from "./types"; import { billTeam } from "../../services/billing/credit_billing"; import { v4 as uuidv4 } from "uuid"; import { addScrapeJob, waitForJob } from "../../services/queue-jobs"; import { logJob } from "../../services/logging/log_job"; -import { getJobPriority } from "../../lib/job-priority"; import { Mode } from "../../types"; -import { getScrapeQueue } from "../../services/queue-service"; import { search } from "../../search"; import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist"; import * as Sentry from "@sentry/node"; import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings"; import { logger as _logger } from "../../lib/logger"; import type { Logger } from "winston"; +import { getJobPriority } from "../../lib/job-priority"; import { CostTracking } from "../../lib/cost-tracking"; import { calculateCreditsToBeBilled } from "../../lib/scrape-billing"; import { supabase_service } from "../../services/supabase"; import { fromV1ScrapeOptions } from "../v2/types"; import { ScrapeJobTimeoutError } from "../../lib/error"; +import { scrapeQueue } from "../../services/worker/nuq"; interface DocumentWithCostTracking { document: Document; @@ -89,10 +88,6 @@ async function scrapeSearchResult( isSearchPreview: boolean = false, ): Promise { const jobId = uuidv4(); - const jobPriority = await getJobPriority({ - team_id: options.teamId, - basePriority: 10, - }); const costTracking = new CostTracking(); @@ -116,6 +111,11 @@ async function scrapeSearchResult( options.teamId, ); + const jobPriority = await getJobPriority({ + team_id: options.teamId, + basePriority: 10, + }); + await addScrapeJob( { url: searchResult.url, @@ -140,13 +140,16 @@ async function scrapeSearchResult( zeroDataRetention, apiKeyId: options.apiKeyId, }, - {}, jobId, jobPriority, directToBullMQ, ); - const doc: Document = await waitForJob(jobId, options.timeout); + const doc: Document = await waitForJob( + jobId, + options.timeout, + zeroDataRetention, + ); logger.info("Scrape job completed", { scrapeId: jobId, @@ -154,7 +157,7 @@ async function scrapeSearchResult( teamId: options.teamId, origin: options.origin, }); - await getScrapeQueue().remove(jobId); + await scrapeQueue.removeJob(jobId); const document = { title: searchResult.title, diff --git a/apps/api/src/controllers/v1/x402-search.ts b/apps/api/src/controllers/v1/x402-search.ts index 706fed9de..0e5aed0e0 100644 --- a/apps/api/src/controllers/v1/x402-search.ts +++ b/apps/api/src/controllers/v1/x402-search.ts @@ -11,9 +11,7 @@ import { import { v4 as uuidv4 } from "uuid"; import { addScrapeJob, waitForJob } from "../../services/queue-jobs"; import { logJob } from "../../services/logging/log_job"; -import { getJobPriority } from "../../lib/job-priority"; import { Mode } from "../../types"; -import { getScrapeQueue } from "../../services/queue-service"; import { search } from "../../search"; import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist"; import * as Sentry from "@sentry/node"; @@ -24,6 +22,8 @@ import { CostTracking } from "../../lib/cost-tracking"; import { supabase_service } from "../../services/supabase"; import { fromV1ScrapeOptions } from "../v2/types"; import { ScrapeJobTimeoutError } from "../../lib/error"; +import { scrapeQueue } from "../../services/worker/nuq"; +import { getJobPriority } from "../../lib/job-priority"; interface DocumentWithCostTracking { document: Document; @@ -45,10 +45,6 @@ async function scrapeX402SearchResult( isSearchPreview: boolean = false, ): Promise { const jobId = uuidv4(); - const jobPriority = await getJobPriority({ - team_id: options.teamId, - basePriority: 10, - }); const costTracking = new CostTracking(); @@ -94,13 +90,19 @@ async function scrapeX402SearchResult( zeroDataRetention, apiKeyId: options.apiKeyId, }, - {}, jobId, - jobPriority, + await getJobPriority({ + team_id: options.teamId, + basePriority: 10, + }), directToBullMQ, ); - const doc: Document = await waitForJob(jobId, options.timeout); + const doc: Document = await waitForJob( + jobId, + options.timeout, + zeroDataRetention, + ); logger.info("Scrape job [x402] completed", { scrapeId: jobId, @@ -108,7 +110,7 @@ async function scrapeX402SearchResult( teamId: options.teamId, origin: options.origin, }); - await getScrapeQueue().remove(jobId); + await scrapeQueue.removeJob(jobId); const document = { title: searchResult.title, diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.ts index fe17023de..bdb704877 100644 --- a/apps/api/src/controllers/v2/batch-scrape.ts +++ b/apps/api/src/controllers/v2/batch-scrape.ts @@ -142,6 +142,7 @@ export async function batchScrapeController( delete (scrapeOptions as any).appendToId; const jobs = urls.map(x => ({ + jobId: uuidv4(), data: { url: x, mode: "single_urls" as const, @@ -158,10 +159,7 @@ export async function batchScrapeController( zeroDataRetention, apiKeyId: req.acuc?.api_key_id ?? null, }, - opts: { - jobId: uuidv4(), - priority: 20, - }, + priority: jobPriority, })); await finishCrawlKickoff(id); @@ -176,7 +174,7 @@ export async function batchScrapeController( logger.debug("Adding scrape jobs to Redis..."); await addCrawlJobs( id, - jobs.map(x => x.opts.jobId), + jobs.map(x => x.jobId), logger, ); logger.debug("Adding scrape jobs to BullMQ..."); diff --git a/apps/api/src/controllers/v2/crawl-errors.ts b/apps/api/src/controllers/v2/crawl-errors.ts index e084310ea..a4b5cd2bb 100644 --- a/apps/api/src/controllers/v2/crawl-errors.ts +++ b/apps/api/src/controllers/v2/crawl-errors.ts @@ -5,31 +5,15 @@ import { RequestWithAuth, } from "./types"; import { getCrawl, getCrawlJobs } from "../../lib/crawl-redis"; -import { getScrapeQueue } from "../../services/queue-service"; import { redisEvictConnection } from "../../../src/services/redis"; import { configDotenv } from "dotenv"; -import { Job } from "bullmq"; import { supabase_rr_service } from "../../services/supabase"; -import { logger } from "../../lib/logger"; +import { logger as _logger } from "../../lib/logger"; import { deserializeTransportableError } from "../../lib/error-serde"; import { TransportableError } from "../../lib/error"; +import { scrapeQueue } from "../../services/worker/nuq"; configDotenv(); -export async function getJob(id: string) { - const job = await getScrapeQueue().getJob(id); - if (!job) return job; - - return job; -} - -export async function getJobs(ids: string[]) { - const jobs: (Job & { id: string })[] = ( - await Promise.all(ids.map(x => getScrapeQueue().getJob(x))) - ).filter(x => x) as (Job & { id: string })[]; - - return jobs; -} - export async function crawlErrorsController( req: RequestWithAuth, res: Response, @@ -41,25 +25,24 @@ export async function crawlErrorsController( return res.status(403).json({ success: false, error: "Forbidden" }); } - let jobStatuses = await Promise.all( - (await getCrawlJobs(req.params.jobId)).map( - async x => [x, await getScrapeQueue().getJobState(x)] as const, - ), - ); + const logger = _logger.child({ + crawlId: req.params.jobId, + zeroDataRetention: sc.zeroDataRetention ?? false, + }); - const failedJobIDs: string[] = []; - - for (const [id, status] of jobStatuses) { - if (status === "failed") { - failedJobIDs.push(id); - } - } + const failedJobs = ( + await scrapeQueue.getJobsWithStatus( + await getCrawlJobs(req.params.jobId), + "failed", + logger, + ) + ).filter(x => x.failedReason); res.status(200).json({ - errors: (await getJobs(failedJobIDs)) + errors: failedJobs .map(x => { const error = deserializeTransportableError( - x.failedReason, + x.failedReason!, ) as TransportableError | null; if (error?.code === "SCRAPE_RACED_REDIRECT_ERROR") { return null; @@ -67,8 +50,8 @@ export async function crawlErrorsController( return { id: x.id, timestamp: - x.finishedOn !== undefined - ? new Date(x.finishedOn).toISOString() + x.finishedAt !== undefined + ? new Date(x.finishedAt).toISOString() : undefined, url: x.data.url, ...(error @@ -77,7 +60,7 @@ export async function crawlErrorsController( error: error.message, } : { - error: x.failedReason, + error: x.failedReason!, }), }; }) @@ -95,7 +78,7 @@ export async function crawlErrorsController( .throwOnError(); if (crawlJobError) { - logger.error("Error getting crawl job", { error: crawlJobError }); + _logger.error("Error getting crawl job", { error: crawlJobError }); throw crawlJobError; } @@ -130,7 +113,7 @@ export async function crawlErrorsController( .throwOnError(); if (failedJobsError) { - logger.error("Error getting failed jobs", { error: failedJobsError }); + _logger.error("Error getting failed jobs", { error: failedJobsError }); throw failedJobsError; } diff --git a/apps/api/src/controllers/v2/crawl-status-ws.ts b/apps/api/src/controllers/v2/crawl-status-ws.ts index 2313566fb..f1ec30606 100644 --- a/apps/api/src/controllers/v2/crawl-status-ws.ts +++ b/apps/api/src/controllers/v2/crawl-status-ws.ts @@ -15,15 +15,11 @@ import { getCrawlExpiry, getCrawlJobs, getDoneJobsOrdered, - getDoneJobsOrderedLength, - isCrawlFinished, - isCrawlFinishedLocked, } from "../../lib/crawl-redis"; -import { getScrapeQueue } from "../../services/queue-service"; -import { getJob, getJobs } from "./crawl-status"; +import { getJobs, PseudoJob } from "./crawl-status"; import * as Sentry from "@sentry/node"; -import { Job, JobState } from "bullmq"; import { getConcurrencyLimitedJobs } from "../../lib/concurrency-limit"; +import { scrapeQueue, NuQJobStatus } from "../../services/worker/nuq"; type ErrorMessage = { type: "error"; @@ -88,18 +84,14 @@ async function crawlStatusWS( const notDoneJobIDs = jobIDs.filter(x => !doneJobIDs.includes(x)); - const queue = getScrapeQueue(); + const newlyDoneJobIDs: string[] = ( + await scrapeQueue.getJobsWithStatuses(notDoneJobIDs, [ + "completed", + "failed", + ]) + ).map(x => x.id); - const jobStatuses = await Promise.all( - notDoneJobIDs.map(async x => [x, await queue.getJobState(x)]), - ); - const newlyDoneJobIDs: string[] = jobStatuses - .filter(x => x[1] === "completed" || x[1] === "failed") - .map(x => x[0]); - - const newlyDoneJobs: Job[] = ( - await Promise.all(newlyDoneJobIDs.map(x => getJob(x))) - ).filter(x => x !== undefined) as Job[]; + const newlyDoneJobs: PseudoJob[] = await getJobs(newlyDoneJobIDs); for (const job of newlyDoneJobs) { if (job.returnvalue) { @@ -118,28 +110,28 @@ async function crawlStatusWS( setTimeout(loop, 1000); - doneJobIDs = await getDoneJobsOrdered(req.params.jobId); + let [_doneJobIDs, jobIDs, throttledJobsSet] = await Promise.all([ + getDoneJobsOrdered(req.params.jobId), + getCrawlJobs(req.params.jobId), + getConcurrencyLimitedJobs(req.auth.team_id), + ]); - let jobIDs = await getCrawlJobs(req.params.jobId); + doneJobIDs = _doneJobIDs; + const jobs = new Map((await scrapeQueue.getJobs(jobIDs)).map(x => [x.id, x])); - const queue = getScrapeQueue(); - - let jobStatuses = await Promise.all( - jobIDs.map(async x => [x, await queue.getJobState(x)] as const), - ); - - const throttledJobsSet = await getConcurrencyLimitedJobs(req.auth.team_id); - - const validJobStatuses: [string, JobState | "unknown"][] = []; + const validJobStatuses: [string, NuQJobStatus][] = []; const validJobIDs: string[] = []; - for (const [id, status] of jobStatuses) { + for (const id of jobIDs) { if (throttledJobsSet.has(id)) { - validJobStatuses.push([id, "prioritized"]); - validJobIDs.push(id); - } else if (status !== "failed" && status !== "unknown") { - validJobStatuses.push([id, status]); + validJobStatuses.push([id, "queued"]); validJobIDs.push(id); + } else { + const job = jobs.get(id); + if (job && job.status !== "failed") { + validJobStatuses.push([id, job.status]); + validJobIDs.push(id); + } } } diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.ts index 9c421d116..3520e2063 100644 --- a/apps/api/src/controllers/v2/crawl-status.ts +++ b/apps/api/src/controllers/v2/crawl-status.ts @@ -13,21 +13,20 @@ import { getDoneJobsOrderedUntil, isCrawlKickoffFinished, } from "../../lib/crawl-redis"; -import { getScrapeQueue } from "../../services/queue-service"; import { supabaseGetJobById, supabaseGetJobsById, } from "../../lib/supabase-jobs"; import { configDotenv } from "dotenv"; -import type { Job, JobState, Queue } from "bullmq"; import { logger } from "../../lib/logger"; import { supabase_rr_service, supabase_service } from "../../services/supabase"; import { getJobFromGCS } from "../../lib/gcs-jobs"; +import { scrapeQueue, NuQJob, NuQJobStatus } from "../../services/worker/nuq"; configDotenv(); export type PseudoJob = { id: string; - getState(): Promise | JobState | "unknown"; + status: NuQJobStatus; returnvalue: T | null; timestamp: number; data: { @@ -47,8 +46,8 @@ export type DBJob = { }; export async function getJob(id: string): Promise | null> { - const [bullJob, dbJob, gcsJob] = await Promise.all([ - getScrapeQueue().getJob(id), + const [nuqJob, dbJob, gcsJob] = await Promise.all([ + scrapeQueue.getJob(id), (process.env.USE_DB_AUTHENTICATION === "true" ? supabaseGetJobById(id) : null) as Promise, @@ -57,9 +56,9 @@ export async function getJob(id: string): Promise | null> { >, ]); - if (!bullJob && !dbJob) return null; + if (!nuqJob && !dbJob) return null; - const data = gcsJob ?? dbJob?.docs ?? bullJob?.returnvalue; + const data = gcsJob ?? dbJob?.docs ?? nuqJob?.returnvalue; if (gcsJob === null && data) { logger.warn("GCS Job not found", { jobId: id, @@ -68,28 +67,23 @@ export async function getJob(id: string): Promise | null> { const job: PseudoJob = { id, - getState: bullJob - ? bullJob.getState - : () => (dbJob!.success ? "completed" : "failed"), + status: dbJob ? (dbJob.success ? "completed" : "failed") : nuqJob!.status, returnvalue: Array.isArray(data) ? data[0] : data, data: { - scrapeOptions: bullJob ? bullJob.data.scrapeOptions : dbJob!.page_options, + scrapeOptions: nuqJob ? nuqJob.data.scrapeOptions : dbJob!.page_options, }, - timestamp: bullJob - ? bullJob.timestamp + timestamp: nuqJob + ? nuqJob.createdAt.valueOf() : new Date(dbJob!.date_added).valueOf(), - failedReason: - (bullJob ? bullJob.failedReason : dbJob!.message) || undefined, + failedReason: (nuqJob ? nuqJob.failedReason : dbJob!.message) || undefined, }; return job; } export async function getJobs(ids: string[]): Promise[]> { - const [bullJobs, dbJobs, gcsJobs] = await Promise.all([ - Promise.all(ids.map(x => getScrapeQueue().getJob(x))).then(x => - x.filter(x => x), - ) as Promise<(Job & { id: string })[]>, + const [nuqJobs, dbJobs, gcsJobs] = await Promise.all([ + scrapeQueue.getJobs(ids), process.env.USE_DB_AUTHENTICATION === "true" ? supabaseGetJobsById(ids) : [], @@ -102,12 +96,12 @@ export async function getJobs(ids: string[]): Promise[]> { : [], ]); - const bullJobMap = new Map>(); + const nuqJobMap = new Map>(); const dbJobMap = new Map(); const gcsJobMap = new Map(); - for (const job of bullJobs) { - bullJobMap.set(job.id, job); + for (const job of nuqJobs) { + nuqJobMap.set(job.id, job); } for (const job of dbJobs) { @@ -121,13 +115,13 @@ export async function getJobs(ids: string[]): Promise[]> { const jobs: PseudoJob[] = []; for (const id of ids) { - const bullJob = bullJobMap.get(id); + const nuqJob = nuqJobMap.get(id); const dbJob = dbJobMap.get(id); const gcsJob = gcsJobMap.get(id); - if (!bullJob && !dbJob) continue; + if (!nuqJob && !dbJob) continue; - const data = gcsJob ?? dbJob?.docs ?? bullJob?.returnvalue; + const data = gcsJob ?? dbJob?.docs ?? nuqJob?.returnvalue; if (gcsJob === null && data) { logger.warn("GCS Job not found", { jobId: id, @@ -136,20 +130,16 @@ export async function getJobs(ids: string[]): Promise[]> { const job: PseudoJob = { id, - getState: bullJob - ? () => bullJob.getState() - : () => (dbJob!.success ? "completed" : "failed"), + status: dbJob ? (dbJob.success ? "completed" : "failed") : nuqJob!.status, returnvalue: Array.isArray(data) ? data[0] : data, data: { - scrapeOptions: bullJob - ? bullJob.data.scrapeOptions - : dbJob!.page_options, + scrapeOptions: nuqJob ? nuqJob.data.scrapeOptions : dbJob!.page_options, }, - timestamp: bullJob - ? bullJob.timestamp + timestamp: nuqJob + ? nuqJob.createdAt.valueOf() : new Date(dbJob!.date_added).valueOf(), failedReason: - (bullJob ? bullJob.failedReason : dbJob!.message) || undefined, + (nuqJob ? nuqJob.failedReason : dbJob!.message) || undefined, }; jobs.push(job); @@ -390,7 +380,6 @@ export async function crawlStatusController( : undefined, }; } else { - // old BullMQ-based path const doneJobs = await getDoneJobsOrderedUntil( req.params.jobId, djoCutoff, @@ -403,33 +392,35 @@ export async function crawlStatusController( let bytes = 0; const bytesLimit = 10485760; // 10 MiB in bytes - for (const jobId of doneJobs) { - const job = await getScrapeQueue().getJob(jobId); - const state = await job?.getState(); + for (let i = 0; i < Math.ceil(doneJobs.length / 50); i++) { + const jobIds = doneJobs.slice(i * 50, (i + 1) * 50); + const jobs = await getJobs(jobIds); - if (state === "failed") { - continue; - } - - let doc: Document | undefined; - if (job?.returnvalue) { - doc = job.returnvalue as Document; - } else { - const gcsDocs = await getJobFromGCS(jobId); - if (gcsDocs && gcsDocs.length > 0) { - doc = gcsDocs[0] as Document; + for (const job of jobs) { + if (job.status === "failed") { + continue; } else { - logger.warn( - "Job was considered done, but neither BullMQ returnvalue nor GCS blob is available!", - { scrapeId: jobId, crawlId: req.params.jobId, state }, - ); - } - } + if (job?.returnvalue) { + scrapes.push(job.returnvalue); + bytes += JSON.stringify(job.returnvalue).length; + } else { + logger.warn( + "Job was considered done, but returnvalue is undefined!", + { + scrapeId: job.id, + crawlId: req.params.jobId, + state: job.status, + returnvalue: job?.returnvalue, + }, + ); + } - if (doc) { - scrapes.push(doc); - bytes += JSON.stringify(doc).length; - iteratedOver++; + iteratedOver++; + } + + if (bytes > bytesLimit) { + break; + } } if (bytes > bytesLimit) { diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.ts index ec0ff91b8..1982ad8b4 100644 --- a/apps/api/src/controllers/v2/crawl.ts +++ b/apps/api/src/controllers/v2/crawl.ts @@ -196,9 +196,7 @@ export async function crawlController( zeroDataRetention: zeroDataRetention || false, apiKeyId: req.acuc?.api_key_id ?? null, }, - {}, crypto.randomUUID(), - 10, ); const protocol = process.env.ENV === "local" ? req.protocol : "https"; diff --git a/apps/api/src/controllers/v2/extract-status.ts b/apps/api/src/controllers/v2/extract-status.ts index 08f24c784..bec3b62bc 100644 --- a/apps/api/src/controllers/v2/extract-status.ts +++ b/apps/api/src/controllers/v2/extract-status.ts @@ -1,14 +1,27 @@ import { Response } from "express"; import { RequestWithAuth } from "./types"; import { getExtract, getExtractExpiry } from "../../lib/extract/extract-redis"; -import { DBJob, PseudoJob } from "./crawl-status"; +import { DBJob } from "./crawl-status"; import { getExtractQueue } from "../../services/queue-service"; import { ExtractResult } from "../../lib/extract/extraction-service"; import { supabaseGetJobByIdDirect } from "../../lib/supabase-jobs"; +import { JobState } from "bullmq"; + +type ExtractPseudoJob = { + id: string; + getState: () => Promise | JobState | "unknown"; + returnvalue: T | null; + timestamp: number; + data: { + scrapeOptions: any; + teamId?: string; + }; + failedReason?: string; +}; export async function getExtractJob( id: string, -): Promise | null> { +): Promise | null> { const [bullJob, dbJob] = await Promise.all([ getExtractQueue().getJob(id), (process.env.USE_DB_AUTHENTICATION === "true" @@ -20,7 +33,7 @@ export async function getExtractJob( const data = dbJob?.docs ?? bullJob?.returnvalue?.data; - const job: PseudoJob = { + const job: ExtractPseudoJob = { id, getState: bullJob ? bullJob.getState.bind(bullJob) diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.ts index 7200f0865..3663f0135 100644 --- a/apps/api/src/controllers/v2/scrape.ts +++ b/apps/api/src/controllers/v2/scrape.ts @@ -10,9 +10,9 @@ import { import { v4 as uuidv4 } from "uuid"; import { addScrapeJob, waitForJob } from "../../services/queue-jobs"; import { getJobPriority } from "../../lib/job-priority"; -import { getScrapeQueue } from "../../services/queue-service"; import { hasFormatOfType } from "../../lib/format-utils"; import { TransportableError } from "../../lib/error"; +import { scrapeQueue } from "../../services/worker/nuq"; import { checkPermissions } from "../../lib/permissions"; export async function scrapeController( @@ -54,16 +54,17 @@ export async function scrapeController( const timeout = req.body.timeout; const startTime = new Date().getTime(); - const jobPriority = await getJobPriority({ - team_id: req.auth.team_id, - basePriority: 10, - }); const isDirectToBullMQ = process.env.SEARCH_PREVIEW_TOKEN !== undefined && process.env.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken; - await addScrapeJob( + const jobPriority = await getJobPriority({ + team_id: req.auth.team_id, + basePriority: 10, + }); + + const job = await addScrapeJob( { url: req.body.url, mode: "single_urls", @@ -92,7 +93,6 @@ export async function scrapeController( zeroDataRetention, apiKeyId: req.acuc?.api_key_id ?? null, }, - {}, jobId, jobPriority, isDirectToBullMQ, @@ -108,8 +108,10 @@ export async function scrapeController( let doc: Document; try { doc = await waitForJob( - jobId, + job ?? jobId, timeout !== undefined ? timeout + totalWait : null, + zeroDataRetention, + logger, ); } catch (e) { logger.error(`Error in scrapeController`, { @@ -118,7 +120,7 @@ export async function scrapeController( }); if (zeroDataRetention) { - await getScrapeQueue().remove(jobId); + await scrapeQueue.removeJob(jobId); } if (e instanceof TransportableError) { @@ -135,7 +137,7 @@ export async function scrapeController( } } - await getScrapeQueue().remove(jobId); + await scrapeQueue.removeJob(jobId); if (!hasFormatOfType(req.body.formats, "rawHtml")) { if (doc && doc.rawHtml) { diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.ts index c958cf63b..8989abfa6 100644 --- a/apps/api/src/controllers/v2/search.ts +++ b/apps/api/src/controllers/v2/search.ts @@ -12,19 +12,19 @@ import { billTeam } from "../../services/billing/credit_billing"; import { v4 as uuidv4 } from "uuid"; import { addScrapeJob, waitForJob } from "../../services/queue-jobs"; import { logJob } from "../../services/logging/log_job"; -import { getJobPriority } from "../../lib/job-priority"; import { Mode } from "../../types"; -import { getScrapeQueue } from "../../services/queue-service"; import { search } from "../../search/v2"; import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist"; import * as Sentry from "@sentry/node"; import { logger as _logger } from "../../lib/logger"; import type { Logger } from "winston"; +import { getJobPriority } from "../../lib/job-priority"; import { CostTracking } from "../../lib/cost-tracking"; import { calculateCreditsToBeBilled } from "../../lib/scrape-billing"; import { supabase_service } from "../../services/supabase"; import { SearchResult, SearchV2Response } from "../../lib/entities"; import { ScrapeJobTimeoutError } from "../../lib/error"; +import { scrapeQueue } from "../../services/worker/nuq"; import { z } from "zod"; import { buildSearchQuery, @@ -59,10 +59,6 @@ async function startScrapeJob( isSearchPreview: boolean = false, ): Promise { const jobId = uuidv4(); - const jobPriority = await getJobPriority({ - team_id: options.teamId, - basePriority: 10, - }); const zeroDataRetention = flags?.forceZDR ?? false; @@ -74,6 +70,11 @@ async function startScrapeJob( zeroDataRetention, }); + const jobPriority = await getJobPriority({ + team_id: options.teamId, + basePriority: 10, + }); + await addScrapeJob( { url: searchResult.url, @@ -96,7 +97,6 @@ async function startScrapeJob( zeroDataRetention, apiKeyId: options.apiKeyId, }, - {}, jobId, jobPriority, directToBullMQ, @@ -131,8 +131,7 @@ async function scrapeSearchResult( isSearchPreview, ); - // Wait for the job to complete - const doc: Document = await waitForJob(jobId, options.timeout); + const doc: Document = await waitForJob(jobId, options.timeout, false); logger.info("Scrape job completed", { scrapeId: jobId, @@ -140,7 +139,8 @@ async function scrapeSearchResult( teamId: options.teamId, origin: options.origin, }); - await getScrapeQueue().remove(jobId); + + await scrapeQueue.removeJob(jobId); const document = { title: searchResult.title, diff --git a/apps/api/src/harness.ts b/apps/api/src/harness.ts index c59f4edbb..cc3b1d4b5 100644 --- a/apps/api/src/harness.ts +++ b/apps/api/src/harness.ts @@ -1,3 +1,4 @@ +import "dotenv/config"; import { exec } from "child_process"; import * as net from "net"; import { basename } from "path"; @@ -77,50 +78,64 @@ function execForward(fancyName: string, command: string): Promise { } (async () => { - console.log("=== Installing dependencies and building all components..."); - await Promise.all([ - (async () => { - const install = execForward("api@install", "pnpm install"); - await install; + if (process.argv[2] !== "--start-docker") { + console.log("=== Installing dependencies and building all components..."); + await Promise.all([ + (async () => { + if (process.argv[2] !== "--start-built") { + const install = execForward("api@install", "pnpm install"); + await install; - const build = execForward("api@build", "pnpm build"); - await build; - })(), - execForward( - "sharedLibs/crawler@build", - "cd sharedLibs/crawler && cargo build --release", - ), - execForward( - "sharedLibs/html-transformer@build", - "cd sharedLibs/html-transformer && cargo build --release", - ), - execForward( - "sharedLibs/pdf-parser@build", - "cd sharedLibs/pdf-parser && cargo build --release", - ), - (async () => { - const install = execForward( - "sharedLibs/go-html-to-md@install", - "cd sharedLibs/go-html-to-md && go mod tidy", - ); - await install; + const build = execForward("api@build", "pnpm build"); + await build; + } else { + console.log("=== Skipping install and build, using built files..."); + } + })(), + execForward( + "sharedLibs/crawler@build", + "cd sharedLibs/crawler && cargo build --release", + ), + execForward( + "sharedLibs/html-transformer@build", + "cd sharedLibs/html-transformer && cargo build --release", + ), + execForward( + "sharedLibs/pdf-parser@build", + "cd sharedLibs/pdf-parser && cargo build --release", + ), + (async () => { + const install = execForward( + "sharedLibs/go-html-to-md@install", + "cd sharedLibs/go-html-to-md && go mod tidy", + ); + await install; - const build = execForward( - "sharedLibs/go-html-to-md@build", - `cd sharedLibs/go-html-to-md && go build -o ${basename(HTML_TO_MARKDOWN_PATH)} -buildmode=c-shared html-to-markdown.go`, - ); - await build; - })(), - ]); + const build = execForward( + "sharedLibs/go-html-to-md@build", + `cd sharedLibs/go-html-to-md && go build -o ${basename(HTML_TO_MARKDOWN_PATH)} -buildmode=c-shared html-to-markdown.go`, + ); + await build; + })(), + ]); + } console.log("=== Starting API, Worker, and Index Worker..."); - const api = execForward("api", "pnpm start:production:nobuild"); + const api = execForward("api", "pnpm server:production:nobuild"); const worker = execForward("worker", "pnpm worker:production"); - const indexWorker = execForward( - "index-worker", - "pnpm index-worker:production", - ); + const nuqWorkers = new Array(5) + .fill(0) + .map((_, i) => + execForward( + `nuq-worker-${i}`, + `NUQ_WORKER_PORT=${3006 + i} NUQ_REDUCE_NOISE=true pnpm nuq-worker:production`, + ), + ); + const indexWorker = + process.env.USE_DB_AUTHENTICATION === "true" + ? execForward("index-worker", "pnpm index-worker:production") + : null; try { await Promise.race([ @@ -130,17 +145,49 @@ function execForward(fancyName: string, command: string): Promise { ), ]); - console.log("=== Running command..."); - - const cmd = execForward("command", command.join(" ")); - - await Promise.race([cmd, api, worker, indexWorker]); + if ( + process.argv[2] === "--start" || + process.argv[2] === "--start-built" || + process.argv[2] === "--start-docker" + ) { + console.log( + "=== Everything is up and running, waiting for termination or failure...", + ); + await Promise.race([ + new Promise(resolve => { + process.on("SIGINT", resolve); + process.on("SIGTERM", resolve); + }), + api, + worker, + ...nuqWorkers, + ...(indexWorker ? [indexWorker] : []), + ]); + } else { + console.log("=== Running command..."); + const cmd = execForward("command", command.join(" ")); + await Promise.race([ + cmd, + api, + worker, + ...nuqWorkers, + ...(indexWorker ? [indexWorker] : []), + ]); + } } finally { console.log("=== Tearing down API, Worker, and Index Worker..."); exec("pkill -f 'queue-worker.js'"); exec("pkill -f 'index.js'"); - exec("pkill -f 'index-worker.js'"); - await Promise.all([api, worker, indexWorker]); + if (indexWorker) { + exec("pkill -f 'index-worker.js'"); + } + exec("pkill -f 'nuq-worker.js'"); + await Promise.all([ + api, + worker, + ...nuqWorkers, + ...(indexWorker ? [indexWorker] : []), + ]); } console.log("=== Goodbye!"); diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index db0848b8d..c8ca7fd54 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -6,7 +6,6 @@ import bodyParser from "body-parser"; import cors from "cors"; import { getExtractQueue, - getScrapeQueue, getGenerateLlmsTxtQueue, getDeepResearchQueue, getBillingQueue, @@ -27,7 +26,6 @@ import { } from "./controllers/v1/types"; import { ZodError } from "zod"; import { v4 as uuidv4 } from "uuid"; -import { RateLimiterMode } from "./types"; import { attachWsProxy } from "./services/agentLivecastWS"; import { cacheableLookup } from "./scraper/scrapeURL/lib/cacheableLookup"; import { v2Router } from "./routes/v2"; @@ -39,6 +37,7 @@ import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions"; import { resourceFromAttributes } from "@opentelemetry/resources"; import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-node"; import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-grpc"; +import { nuqShutdown } from "./services/worker/nuq"; const { createBullBoard } = require("@bull-board/api"); const { BullMQAdapter } = require("@bull-board/api/bullMQAdapter"); @@ -105,7 +104,6 @@ serverAdapter.setBasePath(`/admin/${process.env.BULL_AUTH_KEY}/queues`); const { addQueue, removeQueue, setQueues, replaceQueues } = createBullBoard({ queues: [ - new BullMQAdapter(getScrapeQueue()), new BullMQAdapter(getExtractQueue()), new BullMQAdapter(getGenerateLlmsTxtQueue()), new BullMQAdapter(getDeepResearchQueue()), @@ -156,14 +154,17 @@ function startServer(port = DEFAULT_PORT) { } server.close(() => { logger.info("Server closed."); - if (otelSdk) { - otelSdk.shutdown().then(() => { - logger.info("OTEL shutdown"); + nuqShutdown().finally(() => { + logger.info("NUQ shutdown complete"); + if (otelSdk) { + otelSdk.shutdown().finally(() => { + logger.info("OTEL shutdown"); + process.exit(0); + }); + } else { process.exit(0); - }); - } else { - process.exit(0); - } + } + }); }); }; @@ -176,77 +177,6 @@ if (require.main === module) { startServer(); } -app.get(`/serverHealthCheck`, async (req, res) => { - try { - const scrapeQueue = getScrapeQueue(); - const [waitingJobs] = await Promise.all([scrapeQueue.getWaitingCount()]); - const noWaitingJobs = waitingJobs === 0; - // 200 if no active jobs, 503 if there are active jobs - return res.status(noWaitingJobs ? 200 : 500).json({ - waitingJobs, - }); - } catch (error) { - Sentry.captureException(error); - logger.error(error); - return res.status(500).json({ error: error.message }); - } -}); - -app.get("/serverHealthCheck/notify", async (req, res) => { - if (process.env.SLACK_WEBHOOK_URL) { - const treshold = 1; // The treshold value for the active jobs - const timeout = 60000; // 1 minute // The timeout value for the check in milliseconds - - const getWaitingJobsCount = async () => { - const scrapeQueue = getScrapeQueue(); - const [waitingJobsCount] = await Promise.all([ - scrapeQueue.getWaitingCount(), - ]); - - return waitingJobsCount; - }; - - res.status(200).json({ message: "Check initiated" }); - - const checkWaitingJobs = async () => { - try { - let waitingJobsCount = await getWaitingJobsCount(); - if (waitingJobsCount >= treshold) { - setTimeout(async () => { - // Re-check the waiting jobs count after the timeout - waitingJobsCount = await getWaitingJobsCount(); - if (waitingJobsCount >= treshold) { - const slackWebhookUrl = process.env.SLACK_WEBHOOK_URL!; - const message = { - text: `⚠️ Warning: The number of active jobs (${waitingJobsCount}) has exceeded the threshold (${treshold}) for more than ${ - timeout / 60000 - } minute(s).`, - }; - - const response = await fetch(slackWebhookUrl, { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify(message), - }); - - if (!response.ok) { - logger.error("Failed to send Slack notification"); - } - } - }, timeout); - } - } catch (error) { - Sentry.captureException(error); - logger.debug(error); - } - }; - - checkWaitingJobs(); - } -}); - app.get("/is-production", (req, res) => { res.send({ isProduction: global.isProduction }); }); @@ -323,12 +253,3 @@ app.use( ); logger.info(`Worker ${process.pid} started`); -// const sq = getScrapeQueue(); - -// sq.on("waiting", j => ScrapeEvents.logJobEvent(j, "waiting")); -// sq.on("active", j => ScrapeEvents.logJobEvent(j, "active")); -// sq.on("completed", j => ScrapeEvents.logJobEvent(j, "completed")); -// sq.on("paused", j => ScrapeEvents.logJobEvent(j, "paused")); -// sq.on("resumed", j => ScrapeEvents.logJobEvent(j, "resumed")); -// sq.on("removed", j => ScrapeEvents.logJobEvent(j, "removed")); -// diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.ts index 42527634b..e56721769 100644 --- a/apps/api/src/lib/concurrency-limit.ts +++ b/apps/api/src/lib/concurrency-limit.ts @@ -1,11 +1,10 @@ import { RateLimiterMode } from "../types"; import { redisEvictConnection } from "../services/redis"; -import type { Job, JobsOptions } from "bullmq"; import { getACUCTeam } from "../controllers/auth"; import { getCrawl, StoredCrawl } from "./crawl-redis"; -import { getScrapeQueue } from "../services/queue-service"; import { logger } from "./logger"; import { abTestJob } from "../services/ab-test"; +import { scrapeQueue, type NuQJob } from "../services/worker/nuq"; const constructKey = (team_id: string) => "concurrency-limiter:" + team_id; const constructQueueKey = (team_id: string) => @@ -65,8 +64,7 @@ export async function removeConcurrencyLimitActiveJob( export type ConcurrencyLimitedJob = { id: string; data: any; - opts: JobsOptions; - priority?: number; + priority: number; }; export async function cleanOldConcurrencyLimitedJobs( @@ -311,7 +309,7 @@ async function getNextConcurrentJob( * * @param job The BullMQ job that is done. */ -export async function concurrentJobDone(job: Job) { +export async function concurrentJobDone(job: NuQJob) { if (job.id && job.data && job.data.team_id) { await removeConcurrencyLimitActiveJob(job.data.team_id, job.id); await cleanOldConcurrencyLimitEntries(job.data.team_id); @@ -360,17 +358,13 @@ export async function concurrentJobDone(job: Job) { abTestJob(nextJob.job.data); - (await getScrapeQueue()).add( + await scrapeQueue.addJob( nextJob.job.id, { ...nextJob.job.data, concurrencyLimitHit: true, }, - { - ...nextJob.job.opts, - jobId: nextJob.job.id, - priority: nextJob.job.priority, - }, + nextJob.job.priority, ); } } diff --git a/apps/api/src/lib/extract/document-scraper.ts b/apps/api/src/lib/extract/document-scraper.ts index becd6ea9e..d7b70618e 100644 --- a/apps/api/src/lib/extract/document-scraper.ts +++ b/apps/api/src/lib/extract/document-scraper.ts @@ -5,12 +5,12 @@ import { URLTrace, scrapeOptions as scrapeOptionsSchema, } from "../../controllers/v2/types"; -import { getScrapeQueue } from "../../services/queue-service"; import { waitForJob } from "../../services/queue-jobs"; import { addScrapeJob } from "../../services/queue-jobs"; import { getJobPriority } from "../job-priority"; import type { Logger } from "winston"; import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist"; +import { scrapeQueue } from "../../services/worker/nuq"; interface ScrapeDocumentOptions { url: string; @@ -69,14 +69,16 @@ export async function scrapeDocument( zeroDataRetention: false, // not supported apiKeyId: options.apiKeyId, }, - {}, jobId, jobPriority, ); - const doc = await waitForJob(jobId, timeout); - - await getScrapeQueue().remove(jobId); + const doc = await waitForJob(jobId, timeout, false, logger); + try { + await scrapeQueue.removeJob(jobId); + } catch (error) { + logger.warn("Error removing job from queue", { error, scrapeId: jobId }); + } if (trace) { trace.timing.completedAt = new Date().toISOString(); diff --git a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts index cf9d04661..df7319b8e 100644 --- a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts +++ b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts @@ -5,12 +5,12 @@ import { URLTrace, scrapeOptions as scrapeOptionsSchema, } from "../../../controllers/v2/types"; -import { getScrapeQueue } from "../../../services/queue-service"; import { waitForJob } from "../../../services/queue-jobs"; import { addScrapeJob } from "../../../services/queue-jobs"; import { getJobPriority } from "../../job-priority"; import type { Logger } from "winston"; import { isUrlBlocked } from "../../../scraper/WebScraper/utils/blocklist"; +import { scrapeQueue } from "../../../services/worker/nuq"; interface ScrapeDocumentOptions { url: string; @@ -68,13 +68,16 @@ export async function scrapeDocument_F0( zeroDataRetention: false, // not supported apiKeyId: options.apiKeyId, }, - {}, jobId, jobPriority, ); - const doc = await waitForJob(jobId, timeout); - await getScrapeQueue().remove(jobId); + const doc = await waitForJob(jobId, timeout, false, logger); + try { + await scrapeQueue.removeJob(jobId); + } catch (error) { + logger.warn("Error removing job from queue", { error, scrapeId: jobId }); + } if (trace) { trace.timing.completedAt = new Date().toISOString(); diff --git a/apps/api/src/lib/logger.ts b/apps/api/src/lib/logger.ts index 10a1e11a5..29d78e2d5 100644 --- a/apps/api/src/lib/logger.ts +++ b/apps/api/src/lib/logger.ts @@ -35,6 +35,17 @@ const zeroDataRetentionFilter = winston.format(info => { return info; })(); +const reduceNoiseFilter = winston.format(info => { + if ( + (info.metadata?.module === "nuq/metrics" || + info.module === "nuq/metrics") && + process.env.NUQ_REDUCE_NOISE === "true" + ) { + return false; // Don't log this message + } + return info; +})(); + export const logger = winston.createLogger({ level: process.env.LOGGING_LEVEL?.toLowerCase() ?? "debug", format: winston.format.json({ @@ -71,6 +82,7 @@ export const logger = winston.createLogger({ : []), new winston.transports.Console({ format: winston.format.combine( + reduceNoiseFilter, zeroDataRetentionFilter, winston.format.timestamp({ format: "YYYY-MM-DD HH:mm:ss" }), winston.format.metadata({ diff --git a/apps/api/src/lib/scrape-events.ts b/apps/api/src/lib/scrape-events.ts deleted file mode 100644 index a50f8fba1..000000000 --- a/apps/api/src/lib/scrape-events.ts +++ /dev/null @@ -1,109 +0,0 @@ -// import { Job } from "bullmq"; -// import { supabase_service as supabase } from "../services/supabase"; -// import { logger } from "./logger"; -// import { configDotenv } from "dotenv"; -// import { Engine } from "../scraper/scrapeURL/engines"; -// configDotenv(); - -// export type ScrapeErrorEvent = { -// type: "error"; -// message: string; -// stack?: string; -// }; - -// export type ScrapeScrapeEvent = { -// type: "scrape"; -// url: string; -// worker?: string; -// method: Engine; -// result: null | { -// success: boolean; -// response_code?: number; -// response_size?: number; -// error?: string | object; -// // proxy?: string, -// time_taken: number; -// }; -// }; - -// export type ScrapeQueueEvent = { -// type: "queue"; -// event: -// | "waiting" -// | "active" -// | "completed" -// | "paused" -// | "resumed" -// | "removed" -// | "failed"; -// worker?: string; -// }; - -// export type ScrapeEvent = -// | ScrapeErrorEvent -// | ScrapeScrapeEvent -// | ScrapeQueueEvent; - -// export class ScrapeEvents { -// static async insert(jobId: string, content: ScrapeEvent) { -// if (jobId === "TEST") return null; - -// const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true"; -// if (useDbAuthentication) { -// try { -// const result = await supabase -// .from("scrape_events") -// .insert({ -// job_id: jobId, -// type: content.type, -// content: content, -// // created_at -// }) -// .select() -// .single(); -// return (result.data as any).id; -// } catch (error) { -// // logger.error(`Error inserting scrape event: ${error}`); -// return null; -// } -// } - -// return null; -// } - -// static async updateScrapeResult( -// logId: number | null, -// result: ScrapeScrapeEvent["result"], -// ) { -// if (logId === null) return; - -// try { -// const previousLog = ( -// await supabase.from("scrape_events").select().eq("id", logId).single() -// ).data as any; -// await supabase -// .from("scrape_events") -// .update({ -// content: { -// ...previousLog.content, -// result, -// }, -// }) -// .eq("id", logId); -// } catch (error) { -// logger.error(`Error updating scrape result: ${error}`); -// } -// } - -// static async logJobEvent(job: Job | any, event: ScrapeQueueEvent["event"]) { -// try { -// await this.insert(((job as any).id ? (job as any).id : job) as string, { -// type: "queue", -// event, -// worker: process.env.FLY_MACHINE_ID, -// }); -// } catch (error) { -// logger.error(`Error logging job event: ${error}`); -// } -// } -// } diff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.ts index e500ebd4f..7d90faffe 100644 --- a/apps/api/src/main/runWebScraper.ts +++ b/apps/api/src/main/runWebScraper.ts @@ -1,9 +1,8 @@ -import { Job } from "bullmq"; import { WebScraperOptions, RunWebScraperParams } from "../types"; -import { supabase_service } from "../services/supabase"; import { logger as _logger } from "../lib/logger"; import { configDotenv } from "dotenv"; import { scrapeURL, ScrapeUrlResponse } from "../scraper/scrapeURL"; +import type { NuQJob } from "../services/worker/nuq"; import { CostTracking } from "../lib/cost-tracking"; configDotenv(); @@ -11,7 +10,7 @@ export async function startWebScraperPipeline({ job, costTracking, }: { - job: Job & { id: string }; + job: NuQJob; costTracking: CostTracking; }) { return await runWebScraper({ @@ -33,8 +32,8 @@ export async function startWebScraperPipeline({ ...job.data.internalOptions, }, team_id: job.data.team_id, - bull_job_id: job.id.toString(), - priority: job.opts.priority, + bull_job_id: job.id, + priority: job.priority, is_scrape: job.data.is_scrape ?? false, is_crawl: !!(job.data.crawl_id && job.data.crawlerOptions !== null), urlInvisibleInCurrentCrawl: @@ -172,40 +171,3 @@ export async function runWebScraper({ } } } - -const saveJob = async (job: Job, result: any, mode: string) => { - try { - const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true"; - if (useDbAuthentication) { - const { data, error } = await supabase_service - .from("firecrawl_jobs") - .update({ docs: result }) - .eq("job_id", job.id); - - if (error) throw new Error(error.message); - // try { - // if (mode === "crawl") { - // await job.moveToCompleted(null, token, false); - // } else { - // await job.moveToCompleted(result, token, false); - // } - // } catch (error) { - // // I think the job won't exist here anymore - // } - // } else { - // try { - // await job.moveToCompleted(result, token, false); - // } catch (error) { - // // I think the job won't exist here anymore - // } - } - // ScrapeEvents.logJobEvent(job, "completed"); - } catch (error) { - _logger.error(`🐂 Failed to update job status`, { - module: "runWebScraper", - method: "saveJob", - jobId: job.id, - scrapeId: job.id, - }); - } -}; diff --git a/apps/api/src/routes/admin.ts b/apps/api/src/routes/admin.ts index 5a8a83b8f..c455aa549 100644 --- a/apps/api/src/routes/admin.ts +++ b/apps/api/src/routes/admin.ts @@ -1,11 +1,5 @@ import express from "express"; import { redisHealthController } from "../controllers/v0/admin/redis-health"; -import { - autoscalerController, - checkQueuesController, - cleanBefore24hCompleteJobsController, - queuesController, -} from "../controllers/v0/admin/queue"; import { wrap } from "./shared"; import { acucCacheClearController } from "../controllers/v0/admin/acuc-cache-clear"; import { checkFireEngine } from "../controllers/v0/admin/check-fire-engine"; @@ -22,23 +16,6 @@ adminRouter.get( redisHealthController, ); -adminRouter.get( - `/admin/${process.env.BULL_AUTH_KEY}/clean-before-24h-complete-jobs`, - cleanBefore24hCompleteJobsController, -); - -adminRouter.get( - `/admin/${process.env.BULL_AUTH_KEY}/check-queues`, - checkQueuesController, -); - -adminRouter.get(`/admin/${process.env.BULL_AUTH_KEY}/queues`, queuesController); - -adminRouter.get( - `/admin/${process.env.BULL_AUTH_KEY}/autoscaler`, - autoscalerController, -); - adminRouter.post( `/admin/${process.env.BULL_AUTH_KEY}/acuc-cache-clear`, wrap(acucCacheClearController), diff --git a/apps/api/src/services/alerts/index.ts b/apps/api/src/services/alerts/index.ts deleted file mode 100644 index 44f2b8a04..000000000 --- a/apps/api/src/services/alerts/index.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { logger } from "../../../src/lib/logger"; -import { getScrapeQueue } from "../queue-service"; -import { sendSlackWebhook } from "./slack"; - -export async function checkAlerts() { - try { - if ( - process.env.SLACK_WEBHOOK_URL && - process.env.ENV === "production" && - process.env.ALERT_NUM_ACTIVE_JOBS && - process.env.ALERT_NUM_WAITING_JOBS - ) { - logger.info("Initializing alerts"); - const checkActiveJobs = async () => { - try { - const scrapeQueue = getScrapeQueue(); - const activeJobs = await scrapeQueue.getActiveCount(); - if (activeJobs > Number(process.env.ALERT_NUM_ACTIVE_JOBS)) { - logger.warn( - `Alert: Number of active jobs is over ${process.env.ALERT_NUM_ACTIVE_JOBS}. Current active jobs: ${activeJobs}.`, - ); - sendSlackWebhook( - `Alert: Number of active jobs is over ${process.env.ALERT_NUM_ACTIVE_JOBS}. Current active jobs: ${activeJobs}`, - true, - ); - } else { - logger.info( - `Number of active jobs is under ${process.env.ALERT_NUM_ACTIVE_JOBS}. Current active jobs: ${activeJobs}`, - ); - } - } catch (error) { - logger.error(`Failed to check active jobs: ${error}`); - } - }; - - const checkWaitingQueue = async () => { - const scrapeQueue = getScrapeQueue(); - const waitingJobs = await scrapeQueue.getWaitingCount(); - - if (waitingJobs > Number(process.env.ALERT_NUM_WAITING_JOBS)) { - logger.warn( - `Alert: Number of waiting jobs is over ${process.env.ALERT_NUM_WAITING_JOBS}. Current waiting jobs: ${waitingJobs}.`, - ); - sendSlackWebhook( - `Alert: Number of waiting jobs is over ${process.env.ALERT_NUM_WAITING_JOBS}. Current waiting jobs: ${waitingJobs}. Scale up the number of workers with fly scale count worker=20`, - true, - ); - } - }; - - const checkAll = async () => { - await checkActiveJobs(); - await checkWaitingQueue(); - }; - - await checkAll(); - // setInterval(checkAll, 10000); // Run every - } - } catch (error) { - logger.error(`Failed to initialize alerts: ${error}`); - } -} diff --git a/apps/api/src/services/billing/batch_billing.ts b/apps/api/src/services/billing/batch_billing.ts index 5962f192a..8e675129b 100644 --- a/apps/api/src/services/billing/batch_billing.ts +++ b/apps/api/src/services/billing/batch_billing.ts @@ -2,13 +2,8 @@ import { logger } from "../../lib/logger"; import { getRedisConnection } from "../queue-service"; import { supabase_service } from "../supabase"; import * as Sentry from "@sentry/node"; -import { Queue } from "bullmq"; import { withAuth } from "../../lib/withAuth"; -import { - getACUC, - setCachedACUC, - setCachedACUCTeam, -} from "../../controllers/auth"; +import { setCachedACUC, setCachedACUCTeam } from "../../controllers/auth"; // Configuration constants const BATCH_KEY = "billing_batch"; diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.ts index c85f101ff..a70100f7a 100644 --- a/apps/api/src/services/indexing/index-worker.ts +++ b/apps/api/src/services/indexing/index-worker.ts @@ -218,9 +218,7 @@ const processPrecrawlJobInternal = async (token: string, job: Job) => { zeroDataRetention: false, apiKeyId: null, }, - {}, crypto.randomUUID(), - 10, ); } catch (e) { logger.error("Error processing one cycle of the precrawl job", { diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.ts index 8fda3d39e..cd0907205 100644 --- a/apps/api/src/services/logging/log_job.ts +++ b/apps/api/src/services/logging/log_job.ts @@ -55,6 +55,10 @@ export async function logJob( }); try { + if (process.env.GCS_BUCKET_NAME) { + await saveJobToGCS(job); + } + const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true"; if (!useDbAuthentication) { return; @@ -116,10 +120,6 @@ export async function logJob( : null, }; - if (process.env.GCS_BUCKET_NAME) { - await saveJobToGCS(job); - } - if (bypassLogging) { return; } diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.ts index 986ee1818..67141224a 100644 --- a/apps/api/src/services/queue-jobs.ts +++ b/apps/api/src/services/queue-jobs.ts @@ -1,7 +1,5 @@ -import { getScrapeQueue, getScrapeQueueEvents } from "./queue-service"; import { v4 as uuidv4 } from "uuid"; import { NotificationType, RateLimiterMode, WebScraperOptions } from "../types"; -import * as Sentry from "@sentry/node"; import { cleanOldConcurrencyLimitEntries, getConcurrencyLimitActiveJobs, @@ -14,16 +12,15 @@ import { import { logger as _logger } from "../lib/logger"; import { sendNotificationWithCustomDays } from "./notification/email_notification"; import { shouldSendConcurrencyLimitNotification } from "./notification/notification-check"; -import { getACUC, getACUCTeam } from "../controllers/auth"; +import { getACUCTeam } from "../controllers/auth"; import { getJobFromGCS, removeJobFromGCS } from "../lib/gcs-jobs"; import { Document } from "../controllers/v1/types"; import { getCrawl } from "../lib/crawl-redis"; import { Logger } from "winston"; -import { Job } from "bullmq"; import { ScrapeJobTimeoutError, TransportableError } from "../lib/error"; import { deserializeTransportableError } from "../lib/error-serde"; -import { robustFetch } from "../scraper/scrapeURL/lib/fetch"; import { abTestJob } from "./ab-test"; +import { NuQJob, scrapeQueue } from "./worker/nuq"; /** * Checks if a job is a crawl or batch scrape based on its options @@ -41,21 +38,15 @@ function isCrawlOrBatchScrape(options: { async function _addScrapeJobToConcurrencyQueue( webScraperOptions: any, - options: any, jobId: string, - jobPriority: number, + priority: number = 0, ) { await pushConcurrencyLimitedJob( webScraperOptions.team_id, { id: jobId, data: webScraperOptions, - opts: { - ...options, - priority: jobPriority, - jobId: jobId, - }, - priority: jobPriority, + priority, }, webScraperOptions.crawl_id ? Infinity @@ -65,10 +56,9 @@ async function _addScrapeJobToConcurrencyQueue( export async function _addScrapeJobToBullMQ( webScraperOptions: WebScraperOptions, - options: any, jobId: string, - jobPriority: number, -): Promise { + priority: number = 0, +): Promise> { abTestJob(webScraperOptions); if (webScraperOptions && webScraperOptions.team_id) { @@ -90,20 +80,15 @@ export async function _addScrapeJobToBullMQ( } } - return await getScrapeQueue().add(jobId, webScraperOptions, { - ...options, - priority: jobPriority, - jobId, - }); + return await scrapeQueue.addJob(jobId, webScraperOptions, priority); } async function addScrapeJobRaw( webScraperOptions: WebScraperOptions, - options: any, jobId: string, - jobPriority: number, + priority: number = 0, directToBullMQ: boolean = false, -): Promise { +): Promise | null> { let concurrencyLimited: "yes" | "yes-crawl" | "no" | null = null; let currentActiveConcurrency = 0; let maxConcurrency = 0; @@ -188,46 +173,32 @@ async function addScrapeJobRaw( webScraperOptions.concurrencyLimited = true; - await _addScrapeJobToConcurrencyQueue( - webScraperOptions, - options, - jobId, - jobPriority, - ); + await _addScrapeJobToConcurrencyQueue(webScraperOptions, jobId); return null; } else { - return await _addScrapeJobToBullMQ( - webScraperOptions, - options, - jobId, - jobPriority, - ); + return await _addScrapeJobToBullMQ(webScraperOptions, jobId, priority); } } export async function addScrapeJob( webScraperOptions: WebScraperOptions, - options: any = {}, jobId: string = uuidv4(), - jobPriority: number = 10, + priority: number = 0, directToBullMQ: boolean = false, -): Promise { +): Promise | null> { return await addScrapeJobRaw( webScraperOptions, - options, jobId, - jobPriority, + priority, directToBullMQ, ); } export async function addScrapeJobs( jobs: { + jobId: string; data: WebScraperOptions; - opts: { - jobId: string; - priority: number; - }; + priority: number; }[], ) { if (jobs.length === 0) return true; @@ -235,11 +206,9 @@ export async function addScrapeJobs( const jobsByTeam = new Map< string, { + jobId: string; data: WebScraperOptions; - opts: { - jobId: string; - priority: number; - }; + priority: number; }[] >(); @@ -254,18 +223,14 @@ export async function addScrapeJobs( // == Buckets for jobs == let jobsForcedToCQ: { data: WebScraperOptions; - opts: { - jobId: string; - priority: number; - }; + jobId: string; + priority: number; }[] = []; let jobsPotentiallyInCQ: { data: WebScraperOptions; - opts: { - jobId: string; - priority: number; - }; + jobId: string; + priority: number; }[] = []; // == Select jobs by crawl ID == @@ -273,19 +238,15 @@ export async function addScrapeJobs( string, { data: WebScraperOptions; - opts: { - jobId: string; - priority: number; - }; + jobId: string; + priority: number; }[] >(); const jobsWithoutCrawlID: { data: WebScraperOptions; - opts: { - jobId: string; - priority: number; - }; + jobId: string; + priority: number; }[] = []; for (const job of teamJobs) { @@ -386,66 +347,46 @@ export async function addScrapeJobs( const size = JSON.stringify(job.data).length; await _addScrapeJobToConcurrencyQueue( job.data, - job.opts, - job.opts.jobId, - job.opts.priority, + job.jobId, + job.priority, ); }), ); await Promise.all( addToBull.map(async job => { - const size = JSON.stringify(job.data).length; - await _addScrapeJobToBullMQ( - job.data, - job.opts, - job.opts.jobId, - job.opts.priority, - ); + await _addScrapeJobToBullMQ(job.data, job.jobId, job.priority); }), ); } } export async function waitForJob( - _job: Job | string, + job: NuQJob | string, timeout: number | null, + zeroDataRetention: boolean, logger: Logger = _logger, ): Promise { - const start = Date.now(); - const queue = getScrapeQueue(); - let job: Job | undefined = - typeof _job == "string" ? await queue.getJob(_job) : _job; - while (job === undefined) { - logger.debug("Waiting for job to be created"); - await new Promise(resolve => setTimeout(resolve, 500)); - job = await queue.getJob(_job as string); - if (Date.now() - start > (timeout ?? 180000)) { - throw new ScrapeJobTimeoutError( - "Scrape timed out while waiting in the concurrency limit queue", - ); - } - } - let doc: Document; + const jobId = typeof job == "string" ? job : job.id; + const isConcurrencyLimited = !!(typeof job === "string"); + + let doc: Document | null = null; try { doc = await Promise.race( [ - job.waitUntilFinished(getScrapeQueueEvents(), timeout ?? 180000), + scrapeQueue.waitForJob(jobId, timeout !== null ? timeout + 100 : null), timeout !== null - ? new Promise((resolve, reject) => { - setTimeout( - () => { - reject( - new ScrapeJobTimeoutError( - "Scrape timed out" + - (typeof _job === "string" - ? " after waiting in the concurrency limit queue" - : ""), - ), - ); - }, - Math.max(0, timeout - (Date.now() - start)), - ); + ? new Promise((_resolve, reject) => { + setTimeout(() => { + reject( + new ScrapeJobTimeoutError( + "Scrape timed out" + + (isConcurrencyLimited + ? " after waiting in the concurrency limit queue" + : ""), + ), + ); + }, timeout); }) : null, ].filter(x => x !== null), @@ -467,15 +408,15 @@ export async function waitForJob( logger.debug("Got job"); if (!doc) { - const docs = await getJobFromGCS(job.id!); + const docs = await getJobFromGCS(jobId); logger.debug("Got job from GCS"); if (!docs || docs.length === 0) { throw new Error("Job not found in GCS"); } - doc = docs[0]; + doc = docs[0]!; - if (job.data?.internalOptions?.zeroDataRetention) { - await removeJobFromGCS(job.id!); + if (zeroDataRetention) { + await removeJobFromGCS(jobId); } } diff --git a/apps/api/src/services/queue-service.ts b/apps/api/src/services/queue-service.ts index 10d7c6474..5e0f68f47 100644 --- a/apps/api/src/services/queue-service.ts +++ b/apps/api/src/services/queue-service.ts @@ -1,4 +1,4 @@ -import { Queue, QueueEvents } from "bullmq"; +import { Queue } from "bullmq"; import { logger } from "../lib/logger"; import IORedis from "ioredis"; import { BullMQOtel } from "bullmq-otel"; @@ -6,8 +6,6 @@ import type { DeepResearchServiceOptions } from "../lib/deep-research/deep-resea export type QueueFunction = () => Queue; -let scrapeQueue: Queue; -let scrapeQueueEvents: QueueEvents; let extractQueue: Queue; let loggingQueue: Queue; let indexQueue: Queue; @@ -29,7 +27,6 @@ export function getRedisConnection(): IORedis { return redisConnection; } -export const scrapeQueueName = "{scrapeQueue}"; export const extractQueueName = "{extractQueue}"; export const loggingQueueName = "{loggingQueue}"; export const indexQueueName = "{indexQueue}"; @@ -38,34 +35,6 @@ export const deepResearchQueueName = "{deepResearchQueue}"; export const billingQueueName = "{billingQueue}"; export const precrawlQueueName = "{precrawlQueue}"; -export function getScrapeQueue() { - if (!scrapeQueue) { - scrapeQueue = new Queue(scrapeQueueName, { - connection: getRedisConnection(), - defaultJobOptions: { - removeOnComplete: { - age: 3600, // 1 hour - }, - removeOnFail: { - age: 3600, // 1 hour - }, - }, - telemetry: new BullMQOtel("firecrawl-bullmq"), - }); - } - return scrapeQueue; -} - -export function getScrapeQueueEvents() { - if (!scrapeQueueEvents) { - scrapeQueueEvents = new QueueEvents(scrapeQueueName, { - connection: getRedisConnection(), - }); - } - - return scrapeQueueEvents; -} - export function getExtractQueue() { if (!extractQueue) { extractQueue = new Queue(extractQueueName, { diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts index 6b3fae215..83494420f 100644 --- a/apps/api/src/services/queue-worker.ts +++ b/apps/api/src/services/queue-worker.ts @@ -2,27 +2,16 @@ import "dotenv/config"; import "./sentry"; import * as Sentry from "@sentry/node"; import { - getScrapeQueue, getExtractQueue, getDeepResearchQueue, getGenerateLlmsTxtQueue, - scrapeQueueName, getRedisConnection, } from "./queue-service"; -import { Job, Queue, QueueEvents } from "bullmq"; +import { Job, Queue, Worker } from "bullmq"; import { logger as _logger } from "../lib/logger"; -import { Worker } from "bullmq"; import systemMonitor from "./system-monitor"; import { v4 as uuidv4 } from "uuid"; -import { - addCrawlJobDone, - finishCrawlKickoff, - getCrawl, - normalizeURL, -} from "../lib/crawl-redis"; -import { StoredCrawl } from "../lib/crawl-redis"; import { configDotenv } from "dotenv"; -import { concurrentJobDone } from "../lib/concurrency-limit"; import { ExtractResult, performExtraction, @@ -39,9 +28,6 @@ import http from "http"; import https from "https"; import { cacheableLookup } from "../scraper/scrapeURL/lib/cacheableLookup"; import { robustFetch } from "../scraper/scrapeURL/lib/fetch"; -import { redisEvictConnection } from "./redis"; -import path from "path"; -import { finishCrawlIfNeeded } from "./worker/crawl-logic"; import { NodeSDK } from "@opentelemetry/sdk-node"; import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node"; import { LangfuseExporter } from "langfuse-vercel"; @@ -414,62 +400,6 @@ process.on("SIGTERM", () => { let cantAcceptConnectionCount = 0; -/** - * Converts a file path to a proper URL for cross-platform compatibility. - * On Windows, absolute paths need to be converted to file:// URLs for ESM imports. - * @param filePath - The file path to convert - * @returns A properly formatted path/URL for the current platform - */ -function getWorkerPath(filePath: string): string | URL { - if (process.platform === "win32" && path.isAbsolute(filePath)) { - // On Windows, convert absolute paths to file:// URLs for ESM compatibility - return pathToFileURL(filePath); - } - return filePath; -} - -const separateWorkerFun = (queue: Queue, workerPath: string): Worker => { - // Extract memory size from --max-old-space-size flag if present - const maxOldSpaceSize = - process.env.SCRAPE_WORKER_MAX_OLD_SPACE_SIZE || - process.execArgv - .find(arg => arg.startsWith("--max-old-space-size=")) - ?.split("=")[1]; - - // Filter out the invalid flag for worker threads - const filteredExecArgv = process.execArgv.filter( - arg => !arg.startsWith("--max-old-space-size"), - ); - - // Convert path to proper format for the current platform - const platformWorkerPath = getWorkerPath(workerPath); - - const worker = new Worker(queue.name, platformWorkerPath, { - connection: getRedisConnection(), - lockDuration: 60 * 1000, // 60 seconds - stalledInterval: 60 * 1000, // 60 seconds - maxStalledCount: 10, // 10 times - concurrency: 8, - useWorkerThreads: false, - workerForkOptions: { - execArgv: filteredExecArgv.concat( - maxOldSpaceSize ? ["--max-old-space-size=" + maxOldSpaceSize] : [], - ), - }, - workerThreadsOptions: { - execArgv: filteredExecArgv, - resourceLimits: maxOldSpaceSize - ? { - maxOldGenerationSizeMb: parseInt(maxOldSpaceSize), - } - : undefined, - }, - telemetry: new BullMQOtel("firecrawl-bullmq"), - }); - - return worker; -}; - const workerFun = async ( queue: Queue, processJobInternal: (token: string, job: Job) => Promise, @@ -526,17 +456,11 @@ const workerFun = async ( runningJobs.add(job.id); } - async function afterJobDone(job: Job) { - try { - await concurrentJobDone(job); - } finally { - if (job.id) { - runningJobs.delete(job.id); - } + processJobInternal(token, job).finally(() => { + if (job.id) { + runningJobs.delete(job.id); } - } - - processJobInternal(token, job).finally(() => afterJobDone(job)); + }); await sleep(gotJobInterval); } else { @@ -588,82 +512,7 @@ app.listen(workerPort, () => { }); (async () => { - async function failedListener(args: { - jobId: string; - failedReason: string; - prev?: string | undefined; - }) { - const job = await getScrapeQueue().getJob(args.jobId); - - if (job && job.data.crawl_id) { - await redisEvictConnection.srem( - "crawl:" + job.data.crawl_id + ":jobs_qualified", - args.jobId, - ); - await redisEvictConnection.expire( - "crawl:" + job.data.crawl_id + ":jobs_qualified", - 24 * 60 * 60, - ); - } - - if (args.failedReason === "job stalled more than allowable limit") { - const set = await redisEvictConnection.set( - "stalled-job-cleaner:" + args.jobId, - "1", - "EX", - 60 * 60 * 24, - "NX", - ); - if (!set) { - return; - } - - let logger = _logger.child({ - jobId: args.jobId, - scrapeId: args.jobId, - module: "queue-worker", - method: "failedListener", - zeroDataRetention: job?.data.zeroDataRetention, - }); - if (job && job.data.crawl_id) { - logger = logger.child({ crawlId: job.data.crawl_id }); - logger.warn("Job stalled more than allowable limit"); - - const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl; - - if (job.data.mode === "kickoff") { - await finishCrawlKickoff(job.data.crawl_id); - if (sc) { - await finishCrawlIfNeeded(job, sc); - } - } else { - const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl; - - logger.debug("Declaring job as done..."); - await addCrawlJobDone(job.data.crawl_id, job.id, false, logger); - await redisEvictConnection.srem( - "crawl:" + job.data.crawl_id + ":visited_unique", - normalizeURL(job.data.url, sc), - ); - - await finishCrawlIfNeeded(job, sc); - } - } else { - logger.warn("Job stalled more than allowable limit"); - } - } - } - - const scrapeQueueEvents = new QueueEvents(scrapeQueueName, { - connection: getRedisConnection(), - }); - scrapeQueueEvents.on("failed", failedListener); - - const results = await Promise.all([ - separateWorkerFun( - getScrapeQueue(), - path.join(__dirname, "worker", "scrape-worker.js"), - ), + await Promise.all([ workerFun(getExtractQueue(), processExtractJobInternal), workerFun(getDeepResearchQueue(), processDeepResearchJobInternal), workerFun(getGenerateLlmsTxtQueue(), processGenerateLlmsTxtJobInternal), @@ -671,26 +520,10 @@ app.listen(workerPort, () => { console.log("All workers exited. Waiting for all jobs to finish..."); - const workerResults = results.filter(x => x instanceof Worker); - await Promise.all(workerResults.map(x => x.close())); - while (runningJobs.size > 0) { await new Promise(resolve => setTimeout(resolve, 500)); } - setInterval(async () => { - _logger.debug("Currently running jobs", { - jobs: ( - await Promise.all( - [...runningJobs].map(async jobId => { - return await getScrapeQueue().getJob(jobId); - }), - ) - ).filter(x => x && !x.data?.zeroDataRetention), - }); - }, 1000); - - await scrapeQueueEvents.close(); console.log("All jobs finished. Worker out!"); if (otelSdk) { await otelSdk.shutdown(); diff --git a/apps/api/src/services/worker/crawl-logic.ts b/apps/api/src/services/worker/crawl-logic.ts index c3971e76d..ff1e7db0c 100644 --- a/apps/api/src/services/worker/crawl-logic.ts +++ b/apps/api/src/services/worker/crawl-logic.ts @@ -1,5 +1,4 @@ import { logger as _logger } from "../../lib/logger"; -import { Job } from "bullmq"; import { addCrawlJobs, finishCrawl, @@ -21,11 +20,9 @@ import { getJobs } from "../../controllers/v1/crawl-status"; import { logJob } from "../logging/log_job"; import { createWebhookSender, WebhookEvent } from "../webhook"; import { hasFormatOfType } from "../../lib/format-utils"; +import type { NuQJob } from "./nuq"; -export async function finishCrawlIfNeeded( - job: Job & { id: string }, - sc: StoredCrawl, -) { +export async function finishCrawlIfNeeded(job: NuQJob, sc: StoredCrawl) { const logger = _logger.child({ module: "queue-worker", method: "finishCrawlIfNeeded", @@ -113,7 +110,7 @@ export async function finishCrawlIfNeeded( const jobs = univistedUrls.links.slice(0, addableJobCount).map(url => { const uuid = uuidv4(); return { - name: uuid, + jobId: uuid, data: { url, mode: "single_urls" as const, @@ -133,24 +130,21 @@ export async function finishCrawlIfNeeded( zeroDataRetention: job.data.zeroDataRetention, apiKeyId: job.data.apiKeyId, }, - opts: { - jobId: uuid, - priority: 20, - }, + priority: 20, // TODO: make this dynamic }; }); const lockedIds = await lockURLsIndividually( job.data.crawl_id, sc, - jobs.map(x => ({ id: x.opts.jobId, url: x.data.url })), + jobs.map(x => ({ id: x.jobId, url: x.data.url })), ); const lockedJobs = jobs.filter(x => - lockedIds.find(y => y.id === x.opts.jobId), + lockedIds.find(y => y.id === x.jobId), ); await addCrawlJobs( job.data.crawl_id, - lockedJobs.map(x => x.opts.jobId), + lockedJobs.map(x => x.jobId), logger, ); await addScrapeJobs(lockedJobs); diff --git a/apps/api/src/services/worker/nuq-worker.ts b/apps/api/src/services/worker/nuq-worker.ts new file mode 100644 index 000000000..e5fc91d01 --- /dev/null +++ b/apps/api/src/services/worker/nuq-worker.ts @@ -0,0 +1,119 @@ +import "dotenv/config"; +import { logger as _logger } from "../../lib/logger"; +import { processJobInternal } from "./scrape-worker"; +import { scrapeQueue, nuqGetLocalMetrics, nuqHealthCheck } from "./nuq"; +import Express from "express"; +import { _ } from "ajv"; + +(async () => { + let isShuttingDown = false; + const myLock = crypto.randomUUID(); + + const app = Express(); + + app.get("/metrics", (_, res) => + res.contentType("text/plain").send(nuqGetLocalMetrics()), + ); + app.get("/health", async (_, res) => { + if (await nuqHealthCheck()) { + res.status(200).send("OK"); + } else { + res.status(500).send("Not OK"); + } + }); + + const server = app.listen( + Number(process.env.NUQ_WORKER_PORT ?? process.env.PORT ?? 3000), + () => { + _logger.info("NuQ worker metrics server started"); + }, + ); + + function shutdown() { + isShuttingDown = true; + } + + process.on("SIGINT", shutdown); + process.on("SIGTERM", shutdown); + + let noJobTimeout = 500; + + while (!isShuttingDown) { + const job = await scrapeQueue.getJobToProcess(myLock); + + if (job === null) { + _logger.info("No jobs to process", { module: "nuq/metrics" }); + await new Promise(resolve => setTimeout(resolve, noJobTimeout)); + noJobTimeout = Math.min(noJobTimeout * 2, 10000); + continue; + } + + noJobTimeout = 500; + + const logger = _logger.child({ + module: "nuq-worker", + scrapeId: job.id, + zeroDataRetention: job.data?.zeroDataRetention ?? false, + }); + + logger.info("Acquired job"); + + const lockRenewInterval = setInterval(async () => { + logger.info("Renewing lock"); + if (!(await scrapeQueue.renewLock(job.id, myLock, logger))) { + logger.warn("Failed to renew lock"); + clearInterval(lockRenewInterval); + return; + } + logger.info("Renewed lock"); + }, 15000); + + let processResult: + | { ok: true; data: Awaited> } + | { ok: false; error: any }; + + try { + processResult = { ok: true, data: await processJobInternal(job) }; + } catch (error) { + processResult = { ok: false, error }; + } + + clearInterval(lockRenewInterval); + + if (processResult.ok) { + if ( + !(await scrapeQueue.jobFinish( + job.id, + myLock, + processResult.data, + logger, + )) + ) { + logger.warn("Could not update job status"); + } + } else { + if ( + !(await scrapeQueue.jobFail( + job.id, + myLock, + processResult.error instanceof Error + ? processResult.error.message + : typeof processResult.error === "string" + ? processResult.error + : JSON.stringify(processResult.error), + logger, + )) + ) { + logger.warn("Could not update job status"); + } + } + } + + _logger.info("NuQ worker shutting down"); + + server.close(async () => { + await scrapeQueue.shutdown(); + _logger.info("NuQ worker shut down"); + process.exit(0); + }); +})(); diff --git a/apps/api/src/services/worker/nuq.ts b/apps/api/src/services/worker/nuq.ts new file mode 100644 index 000000000..993603a7a --- /dev/null +++ b/apps/api/src/services/worker/nuq.ts @@ -0,0 +1,568 @@ +import { Logger } from "winston"; +import { logger } from "../../lib/logger"; +import { Client, Pool } from "pg"; + +// === Basics + +const nuqPool = new Pool({ + connectionString: process.env.NUQ_DATABASE_URL, // may be a pgbouncer transaction pooler URL + application_name: "nuq", +}); + +nuqPool.on("error", err => + logger.error("Error in NuQ idle client", { err, module: "nuq" }), +); + +export type NuQJobStatus = "queued" | "active" | "completed" | "failed"; // must match nuq.job_status enum +export type NuQJob = { + id: string; + status: NuQJobStatus; + createdAt: Date; + priority: number; + data: Data; + finishedAt?: Date; + returnvalue?: ReturnValue; + failedReason?: string; +}; + +// === Queue + +class NuQ { + constructor(public readonly queueName: string) {} + + // === Listener + + private listener: Client | null = null; + private listens: { + [key: string]: ((status: "completed" | "failed") => void)[]; + } = {}; + private shuttingDown = false; + + async startListener() { + if (this.listener || this.shuttingDown) return; + + this.listener = new Client({ + connectionString: + process.env.NUQ_DATABASE_URL_LISTEN ?? process.env.NUQ_DATABASE_URL, // will always be a direct connection + application_name: "nuq_listener", + }); + + this.listener.on("notification", msg => { + const tok = (msg.payload ?? "unknown|unknown").split("|"); + if (tok[0] in this.listens) { + this.listens[tok[0]].forEach(listener => + listener(tok[1] as "completed" | "failed"), + ); + delete this.listens[tok[0]]; + } + }); + + this.listener.on("error", err => + logger.error("Error in NuQ listener", { err, module: "nuq" }), + ); + + this.listener.on("end", () => { + logger.info("NuQ listener disconnected", { module: "nuq" }); + this.listener = null; + setTimeout( + (() => { + this.startListener().catch(err => + logger.error("Error in NuQ listener reconnect", { + err, + module: "nuq", + }), + ); + }).bind(this), + 250, + ); + }); + + await this.listener.connect(); + await this.listener.query(`LISTEN "${this.queueName}";`); + + (async () => { + const backedUpJobs = ( + await this.getJobs(Object.keys(this.listens)) + ).filter(job => ["completed", "failed"].includes(job.status)); + for (const job of backedUpJobs) { + this.listens[job.id].forEach(listener => + listener(job.status as "completed" | "failed"), + ); + delete this.listens[job.id]; + } + })(); + } + + async addListener( + id: string, + listener: (status: "completed" | "failed") => void, + ) { + await this.startListener(); + + if (!(id in this.listens)) this.listens[id] = [listener]; + else this.listens[id].push(listener); + } + + async removeListener( + id: string, + listener: (status: "completed" | "failed") => void, + ) { + if (id in this.listens) { + this.listens[id] = this.listens[id].filter(l => l !== listener); + if (this.listens[id].length === 0) delete this.listens[id]; + } + } + + // === Job management + + private readonly jobReturning = [ + "id", + "status", + "created_at", + "priority", + "data", + "finished_at", + "returnvalue", + "failedreason", + ]; + + private rowToJob(row: any): NuQJob | null { + if (!row) return null; + return { + id: row.id, + status: row.status, + createdAt: new Date(row.created_at), + priority: row.priority, + data: row.data, + finishedAt: row.finished_at ? new Date(row.finished_at) : undefined, + returnvalue: row.returnvalue ?? undefined, + failedReason: row.failedreason ?? undefined, + }; + } + + public async getJob( + id: string, + ): Promise | null> { + const start = Date.now(); + try { + return this.rowToJob( + ( + await nuqPool.query( + `SELECT ${this.jobReturning.join(", ")} FROM ${this.queueName} WHERE ${this.queueName}.id = $1;`, + [id], + ) + ).rows[0], + ); + } finally { + logger.info("nuqGetJob metrics", { + module: "nuq/metrics", + method: "nuqGetJob", + duration: Date.now() - start, + scrapeId: id, + }); + } + } + + public async getJobs( + ids: string[], + ): Promise[]> { + if (ids.length === 0) return []; + + const start = Date.now(); + try { + return ( + await nuqPool.query( + `SELECT ${this.jobReturning.join(", ")} FROM ${this.queueName} WHERE ${this.queueName}.id = ANY($1::uuid[]);`, + [ids], + ) + ).rows.map(row => this.rowToJob(row)!); + } finally { + logger.info("nuqGetJobs metrics", { + module: "nuq/metrics", + method: "nuqGetJobs", + duration: Date.now() - start, + scrapeIds: ids.length, + }); + } + } + + public async getJobsWithStatus( + ids: string[], + status: NuQJobStatus, + _logger: Logger = logger, + ): Promise[]> { + if (ids.length === 0) return []; + + const start = Date.now(); + try { + return ( + await nuqPool.query( + `SELECT ${this.jobReturning.join(", ")} FROM ${this.queueName} WHERE ${this.queueName}.id = ANY($1::uuid[]) AND ${this.queueName}.status = $2::nuq.job_status;`, + [ids, status], + ) + ).rows.map(row => this.rowToJob(row)!); + } finally { + _logger.info("nuqGetJobsWithStatus metrics", { + module: "nuq/metrics", + method: "nuqGetJobsWithStatus", + duration: Date.now() - start, + scrapeIds: ids.length, + status, + }); + } + } + + public async getJobsWithStatuses( + ids: string[], + statuses: NuQJobStatus[], + ): Promise[]> { + if (ids.length === 0) return []; + + const start = Date.now(); + try { + return ( + await nuqPool.query( + `SELECT ${this.jobReturning.join(", ")} FROM ${this.queueName} WHERE ${this.queueName}.id = ANY($1::uuid[]) AND ${this.queueName}.status = ANY($2::nuq.job_status[]);`, + [ids, statuses], + ) + ).rows.map(row => this.rowToJob(row)!); + } finally { + logger.info("nuqGetJobsWithStatuses metrics", { + module: "nuq/metrics", + method: "nuqGetJobsWithStatuses", + duration: Date.now() - start, + scrapeIds: ids.length, + statuses, + }); + } + } + + public async removeJob(id: string): Promise { + const start = Date.now(); + try { + return ( + ( + await nuqPool.query(`DELETE FROM ${this.queueName} WHERE id = $1;`, [ + id, + ]) + ).rowCount !== 0 + ); + } finally { + logger.info("nuqRemoveJob metrics", { + module: "nuq/metrics", + method: "nuqRemoveJob", + duration: Date.now() - start, + scrapeId: id, + }); + } + } + + public async removeJobs(ids: string[]): Promise { + if (ids.length === 0) return 0; + + const start = Date.now(); + try { + return ( + ( + await nuqPool.query( + `DELETE FROM ${this.queueName} WHERE id = ANY($1::uuid[]);`, + [ids], + ) + ).rowCount ?? 0 + ); + } finally { + logger.info("nuqRemoveJobs metrics", { + module: "nuq/metrics", + method: "nuqRemoveJobs", + duration: Date.now() - start, + scrapeIds: ids.length, + }); + } + } + + // === Producer + public async addJob( + id: string, + data: JobData, + priority: number = 0, + ): Promise> { + const start = Date.now(); + try { + return this.rowToJob( + ( + await nuqPool.query( + `INSERT INTO ${this.queueName} (id, data, priority) VALUES ($1, $2, $3) RETURNING ${this.jobReturning.join(", ")};`, + [id, data, priority], + ) + ).rows[0], + )!; + } finally { + logger.info("nuqAddJob metrics", { + module: "nuq/metrics", + method: "nuqAddJob", + duration: Date.now() - start, + scrapeId: id, + zeroDataRetention: (data as any)?.zeroDataRetention ?? false, + }); + } + } + + private readonly nuqWaitMode = + process.env.NUQ_WAIT_MODE === "listen" + ? ("listen" as const) + : ("poll" as const); + + public waitForJob( + id: string, + timeout: number | null, + ): Promise { + const done = new Promise( + (async (resolve, reject) => { + if (this.nuqWaitMode === "listen") { + let timer: NodeJS.Timeout | null = null; + if (timeout !== null) { + timer = setTimeout( + (() => { + this.removeListener(id, listener); + reject(new Error("Timed out")); + }).bind(this), + timeout, + ); + } + + const listener = async function (_msg: "completed" | "failed") { + if (timer) clearTimeout(timer); + const job = await this.getJob(id); + if (!job) { + reject(new Error("Job raced out while waiting for it")); + } else { + if (job.status === "completed") { + resolve(job.returnvalue!); + } else { + reject(new Error(job.failedReason!)); + } + } + }.bind(this); + + try { + await this.addListener(id, listener); + } catch (e) { + reject(e); + } + + try { + const job = await this.getJob(id); + if (job && ["completed", "failed"].includes(job.status)) { + this.removeListener(id, listener); + if (timer) clearTimeout(timer); + if (job.status === "completed") { + resolve(job.returnvalue!); + } else { + reject(new Error(job.failedReason!)); + } + return; + } + } catch (e) { + logger.warn("nuqGetJob ensure check failed", { + module: "nuq", + method: "nuqWaitForJob", + error: e, + scrapeId: id, + }); + } + } else { + const timeoutAt = timeout !== null ? Date.now() + timeout : null; + const poll = async function poll() { + try { + const job = await this.getJob(id); + if (job && ["completed", "failed"].includes(job.status)) { + if (job.status === "completed") { + return resolve(job.returnvalue!); + } else { + return reject(new Error(job.failedReason!)); + } + } + } catch (e) { + return reject(e); + } + + if (timeoutAt && Date.now() > timeoutAt) { + return reject(new Error("Timed out")); + } + + setTimeout(poll.bind(this), 250); + }.bind(this); + + poll(); + } + }).bind(this), + ); + + return done; + } + + // === Consumer + + public async getJobToProcess(lock: string): Promise | null> { + const start = Date.now(); + try { + return this.rowToJob( + ( + await nuqPool.query( + ` + WITH next AS (SELECT ${this.jobReturning.join(", ")} FROM ${this.queueName} WHERE ${this.queueName}.status = 'queued'::nuq.job_status ORDER BY ${this.queueName}.priority ASC, ${this.queueName}.created_at ASC FOR UPDATE SKIP LOCKED LIMIT 1) + UPDATE ${this.queueName} q SET status = 'active'::nuq.job_status, lock = $1, locked_at = now() FROM next WHERE q.id = next.id RETURNING ${this.jobReturning.map(x => `q.${x}`).join(", ")}; + `, + [lock], + ) + ).rows[0], + )!; + } finally { + logger.info("nuqGetJobToProcess metrics", { + module: "nuq/metrics", + method: "nuqGetJobToProcess", + duration: Date.now() - start, + }); + } + } + + public async renewLock( + id: string, + lock: string, + _logger: Logger = logger, + ): Promise { + const start = Date.now(); + try { + return ( + ( + await nuqPool.query( + `UPDATE ${this.queueName} SET locked_at = now() WHERE id = $1 AND lock = $2 AND status = 'active'::nuq.job_status;`, + [id, lock], + ) + ).rowCount !== 0 + ); + } finally { + _logger.info("nuqRenewLock metrics", { + module: "nuq/metrics", + method: "nuqRenewLock", + duration: Date.now() - start, + scrapeId: id, + }); + } + } + + public async jobFinish( + id: string, + lock: string, + returnvalue: any | null, + _logger: Logger = logger, + ): Promise { + const start = Date.now(); + try { + return ( + ( + await nuqPool.query( + ` + WITH updated AS (UPDATE ${this.queueName} SET status = 'completed'::nuq.job_status, lock = null, locked_at = null, finished_at = now(), returnvalue = $3 WHERE id = $1 AND lock = $2 RETURNING id) + SELECT pg_notify('${this.queueName}', (id::text || '|completed')) FROM updated; + `, + [id, lock, returnvalue], + ) + ).rowCount !== 0 + ); + } finally { + _logger.info("nuqJobFinish metrics", { + module: "nuq/metrics", + method: "nuqJobFinish", + duration: Date.now() - start, + scrapeId: id, + }); + } + } + + public async jobFail( + id: string, + lock: string, + failedReason: string, + _logger: Logger = logger, + ): Promise { + const start = Date.now(); + try { + return ( + ( + await nuqPool.query( + ` + WITH updated AS (UPDATE ${this.queueName} SET status = 'failed'::nuq.job_status, lock = null, locked_at = null, finished_at = now(), failedreason = $3 WHERE id = $1 AND lock = $2 RETURNING id) + SELECT pg_notify('${this.queueName}', (id::text || '|failed')) FROM updated; + `, + [id, lock, failedReason], + ) + ).rowCount !== 0 + ); + } finally { + _logger.info("nuqJobFail metrics", { + module: "nuq/metrics", + method: "nuqJobFail", + duration: Date.now() - start, + scrapeId: id, + }); + } + } + + // === Metrics + public async getMetrics(): Promise { + const start = Date.now(); + const result = await nuqPool.query( + `SELECT status, COUNT(id) as count FROM ${this.queueName} GROUP BY status ORDER BY count DESC;`, + ); + logger.info("nuqGetMetrics metrics", { + module: "nuq/metrics", + method: "nuqGetMetrics", + duration: Date.now() - start, + }); + const prometheusQueueName = this.queueName.replace(".", "_"); + return `# HELP ${prometheusQueueName}_job_count Number of jobs in each status\n# TYPE ${prometheusQueueName}_job_count gauge\n${result.rows.map(x => `${prometheusQueueName}_job_count{status="${x.status}"} ${x.count}`).join("\n")}\n`; + } + + // === Cleanup + public async shutdown() { + this.shuttingDown = true; + if (this.listener) { + const nl = this.listener; + this.listener = null; + this.listens = {}; + await nl.query(`UNLISTEN "${this.queueName}";`); + await nl.end(); + } + } +} + +export function nuqGetLocalMetrics(): string { + return `# HELP nuq_pool_waiting_count Number of requests waiting in the pool\n# TYPE nuq_pool_waiting_count gauge\nnuq_pool_waiting_count ${nuqPool.waitingCount}\n +# HELP nuq_pool_idle_count Number of connections idle in the pool\n# TYPE nuq_pool_idle_count gauge\nnuq_pool_idle_count ${nuqPool.idleCount}\n +# HELP nuq_pool_total_count Number of connections in the pool\n# TYPE nuq_pool_total_count gauge\nnuq_pool_total_count ${nuqPool.totalCount}\n`; +} + +export async function nuqHealthCheck(): Promise { + const start = Date.now(); + try { + return (await nuqPool.query("SELECT 1;")).rowCount !== 0; + } finally { + logger.info("nuqHealthCheck metrics", { + module: "nuq/metrics", + method: "nuqHealthCheck", + duration: Date.now() - start, + }); + } +} + +// === Instances + +export const scrapeQueue = new NuQ("nuq.queue_scrape"); + +// === Cleanup + +export async function nuqShutdown() { + await scrapeQueue.shutdown(); + await nuqPool.end(); +} diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts index 29ded1a92..83a69d95c 100644 --- a/apps/api/src/services/worker/scrape-worker.ts +++ b/apps/api/src/services/worker/scrape-worker.ts @@ -1,5 +1,4 @@ import { configDotenv } from "dotenv"; -import { Job } from "bullmq"; import * as Sentry from "@sentry/node"; import http from "http"; import https from "https"; @@ -60,6 +59,7 @@ import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-node"; import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-grpc"; import { resourceFromAttributes } from "@opentelemetry/resources"; import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions"; +import type { NuQJob } from "./nuq"; const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); @@ -74,7 +74,7 @@ cacheableLookup.install(http.globalAgent); cacheableLookup.install(https.globalAgent); async function billScrapeJob( - job: Job & { id: string }, + job: NuQJob, document: Document | null, logger: Logger, costTracking: CostTracking, @@ -138,7 +138,7 @@ async function billScrapeJob( return creditsToBeBilled; } -async function processJob(job: Job & { id: string }) { +async function processJob(job: NuQJob) { const logger = _logger.child({ module: "queue-worker", method: "processJob", @@ -157,13 +157,6 @@ async function processJob(job: Job & { id: string }) { const costTracking = new CostTracking(); try { - job.updateProgress({ - current: 1, - total: 100, - current_step: "SCRAPING", - current_url: "", - }); - if (remainingTime !== undefined && remainingTime < 0) { throw new ScrapeJobTimeoutError("Scrape timed out"); } @@ -368,7 +361,6 @@ async function processJob(job: Job & { id: string }) { zeroDataRetention: job.data.zeroDataRetention, apiKeyId: job.data.apiKeyId, }, - {}, jobId, jobPriority, ); @@ -699,7 +691,7 @@ async function kickoffGetIndexLinks( return validIndexLinks; } -async function processKickoffJob(job: Job & { id: string }) { +async function processKickoffJob(job: NuQJob) { const logger = _logger.child({ module: "queue-worker", method: "processKickoffJob", @@ -739,10 +731,8 @@ async function processKickoffJob(job: Job & { id: string }) { zeroDataRetention: job.data.zeroDataRetention, apiKeyId: job.data.apiKeyId, }, - { - priority: 15, - }, jobId, + await getJobPriority({ team_id: job.data.team_id, basePriority: 15 }), ); logger.debug("Adding scrape job to BullMQ...", { jobId }); await addCrawlJob(job.data.crawl_id, jobId, logger); @@ -780,7 +770,6 @@ async function processKickoffJob(job: Job & { id: string }) { const jobs = urls.map(url => { const uuid = uuidv4(); return { - name: uuid, data: { url, mode: "single_urls" as const, @@ -797,10 +786,8 @@ async function processKickoffJob(job: Job & { id: string }) { zeroDataRetention: job.data.zeroDataRetention, apiKeyId: job.data.apiKeyId, }, - opts: { - jobId: uuid, - priority: 20, - }, + jobId: uuid, + priority: jobPriority, }; }); @@ -808,15 +795,15 @@ async function processKickoffJob(job: Job & { id: string }) { const lockedIds = await lockURLsIndividually( job.data.crawl_id, sc, - jobs.map(x => ({ id: x.opts.jobId, url: x.data.url })), + jobs.map(x => ({ id: x.jobId, url: x.data.url })), ); const lockedJobs = jobs.filter(x => - lockedIds.find(y => y.id === x.opts.jobId), + lockedIds.find(y => y.id === x.jobId), ); logger.debug("Adding scrape jobs to Redis..."); await addCrawlJobs( job.data.crawl_id, - lockedJobs.map(x => x.opts.jobId), + lockedJobs.map(x => x.jobId), logger, ); logger.debug("Adding scrape jobs to BullMQ..."); @@ -845,7 +832,7 @@ async function processKickoffJob(job: Job & { id: string }) { const jobs = indexLinks.map(url => { const uuid = uuidv4(); return { - name: uuid, + jobId: uuid, data: { url, mode: "single_urls" as const, @@ -862,10 +849,7 @@ async function processKickoffJob(job: Job & { id: string }) { zeroDataRetention: job.data.zeroDataRetention, apiKeyId: job.data.apiKeyId, }, - opts: { - jobId: uuid, - priority: 20, - }, + priority: jobPriority, }; }); @@ -873,15 +857,15 @@ async function processKickoffJob(job: Job & { id: string }) { const lockedIds = await lockURLsIndividually( job.data.crawl_id, sc, - jobs.map(x => ({ id: x.opts.jobId, url: x.data.url })), + jobs.map(x => ({ id: x.jobId, url: x.data.url })), ); const lockedJobs = jobs.filter(x => - lockedIds.find(y => y.id === x.opts.jobId), + lockedIds.find(y => y.id === x.jobId), ); logger.debug("Adding scrape jobs to Redis..."); await addCrawlJobs( job.data.crawl_id, - lockedJobs.map(x => x.opts.jobId), + lockedJobs.map(x => x.jobId), logger, ); logger.debug("Adding scrape jobs to BullMQ..."); @@ -905,7 +889,7 @@ async function processKickoffJob(job: Job & { id: string }) { } } -export const processJobInternal = async (job: Job & { id: string }) => { +export const processJobInternal = async (job: NuQJob) => { const logger = _logger.child({ module: "queue-worker", method: "processJobInternal", @@ -954,10 +938,7 @@ export const processJobInternal = async (job: Job & { id: string }) => { } try { - if ( - process.env.USE_DB_AUTHENTICATION === "true" && - (job.data.crawl_id || process.env.GCS_BUCKET_NAME) - ) { + if (process.env.GCS_BUCKET_NAME) { logger.debug("Job succeeded -- putting null in Redis"); return null; } else { @@ -1018,11 +999,9 @@ if (otelSdk) { otelSdk.start(); } -module.exports = processJobInternal; - const exitHandler = () => { if (otelSdk) { - otelSdk.shutdown().then(() => { + otelSdk.shutdown().finally(() => { _logger.debug("OTEL shutdown"); process.exit(0); }); diff --git a/apps/nuq-postgres/Dockerfile b/apps/nuq-postgres/Dockerfile new file mode 100644 index 000000000..942b2fbe1 --- /dev/null +++ b/apps/nuq-postgres/Dockerfile @@ -0,0 +1,24 @@ +# Build a Postgres image that runs nuq.sql during initdb + +ARG PG_MAJOR=17 +FROM postgres:${PG_MAJOR} + +# Install pg_cron for the specified Postgres major version +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + postgresql-${PG_MAJOR}-cron; \ + rm -rf /var/lib/apt/lists/* + +# Ensure pg_cron is preloaded on first startup by modifying the initdb template +# This must be set before the first server start (init scripts run after start) +RUN set -eux; \ + conf_sample="/usr/share/postgresql/${PG_MAJOR}/postgresql.conf.sample"; \ + sed -ri "s/^#?shared_preload_libraries\s*=.*/shared_preload_libraries = 'pg_cron'/" "$conf_sample"; \ + printf "\n# Added for pg_cron\ncron.database_name = 'postgres'\n" >> "$conf_sample" + +# Create required extensions before executing our SQL +RUN printf 'CREATE EXTENSION IF NOT EXISTS pgcrypto;\nCREATE EXTENSION IF NOT EXISTS pg_cron;\n' > /docker-entrypoint-initdb.d/010-extensions.sql + +# Copy nuq.sql so it is executed as part of the initdb sequence +COPY nuq.sql /docker-entrypoint-initdb.d/020-nuq.sql \ No newline at end of file diff --git a/apps/nuq-postgres/nuq.sql b/apps/nuq-postgres/nuq.sql new file mode 100644 index 000000000..5d3ad041c --- /dev/null +++ b/apps/nuq-postgres/nuq.sql @@ -0,0 +1,51 @@ +CREATE SCHEMA IF NOT EXISTS nuq; + +DO $$ BEGIN + CREATE TYPE nuq.job_status AS ENUM ('queued', 'active', 'completed', 'failed'); +EXCEPTION + WHEN duplicate_object THEN null; +END $$; + +CREATE TABLE IF NOT EXISTS nuq.queue_scrape ( + id uuid NOT NULL DEFAULT gen_random_uuid(), + status nuq.job_status NOT NULL DEFAULT 'queued'::nuq.job_status, + data jsonb, + created_at timestamp with time zone NOT NULL DEFAULT now(), + priority int NOT NULL DEFAULT 0, + lock uuid, + locked_at timestamp with time zone, + stalls integer, + finished_at timestamp with time zone, + returnvalue jsonb, -- only for selfhost + failedreason text, -- only for selfhost + CONSTRAINT queue_scrape_pkey PRIMARY KEY (id) +); + +ALTER TABLE nuq.queue_scrape +SET (autovacuum_vacuum_scale_factor = 0.01, + autovacuum_analyze_scale_factor = 0.01, + autovacuum_vacuum_cost_limit = 2000, + autovacuum_vacuum_cost_delay = 2); + +CREATE INDEX IF NOT EXISTS queue_scrape_active_locked_at_idx ON nuq.queue_scrape USING btree (locked_at) WHERE (status = 'active'::nuq.job_status); +CREATE INDEX IF NOT EXISTS nuq_queue_scrape_queued_optimal_2_idx ON nuq.queue_scrape (priority ASC, created_at ASC, id) WHERE (status = 'queued'::nuq.job_status); +CREATE INDEX IF NOT EXISTS nuq_queue_scrape_failed_created_at_idx ON nuq.queue_scrape USING btree (created_at) WHERE (status = 'failed'::nuq.job_status); +CREATE INDEX IF NOT EXISTS nuq_queue_scrape_completed_created_at_idx ON nuq.queue_scrape USING btree (created_at) WHERE (status = 'completed'::nuq.job_status); + +SELECT cron.schedule('nuq_queue_scrape_clean_completed', '*/5 * * * *', $$ + DELETE FROM nuq.queue_scrape WHERE nuq.queue_scrape.status = 'completed'::nuq.job_status AND nuq.queue_scrape.created_at < now() - interval '1 hour'; +$$); + +SELECT cron.schedule('nuq_queue_scrape_clean_failed', '*/5 * * * *', $$ + DELETE FROM nuq.queue_scrape WHERE nuq.queue_scrape.status = 'failed'::nuq.job_status AND nuq.queue_scrape.created_at < now() - interval '6 hours'; +$$); + +SELECT cron.schedule('nuq_queue_scrape_lock_reaper', '15 seconds', $$ + UPDATE nuq.queue_scrape SET status = 'queued'::nuq.job_status, lock = null, locked_at = null, stalls = COALESCE(stalls, 0) + 1 WHERE nuq.queue_scrape.locked_at <= now() - interval '1 minute' AND nuq.queue_scrape.status = 'active'::nuq.job_status AND COALESCE(nuq.queue_scrape.stalls, 0) < 9; + WITH stallfail AS (UPDATE nuq.queue_scrape SET status = 'failed'::nuq.job_status, lock = null, locked_at = null, stalls = COALESCE(stalls, 0) + 1 WHERE nuq.queue_scrape.locked_at <= now() - interval '1 minute' AND nuq.queue_scrape.status = 'active'::nuq.job_status AND COALESCE(nuq.queue_scrape.stalls, 0) >= 9 RETURNING id) + SELECT pg_notify('nuq.queue_scrape', (id::text || '|' || 'failed'::text)) FROM stallfail; +$$); + +SELECT cron.schedule('nuq_queue_scrape_reindex', '0 9 * * *', $$ + REINDEX TABLE CONCURRENTLY nuq.queue_scrape; +$$); diff --git a/docker-compose.yaml b/docker-compose.yaml index c6714dd5a..5892fa0db 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -19,6 +19,7 @@ x-common-env: &common-env REDIS_URL: ${REDIS_URL:-redis://redis:6379} REDIS_RATE_LIMIT_URL: ${REDIS_URL:-redis://redis:6379} PLAYWRIGHT_MICROSERVICE_URL: ${PLAYWRIGHT_MICROSERVICE_URL:-http://playwright-service:3000/scrape} + NUQ_DATABASE_URL: postgres://postgres:postgres@nuq-postgres:5432/postgres USE_DB_AUTHENTICATION: ${USE_DB_AUTHENTICATION} OPENAI_API_KEY: ${OPENAI_API_KEY} OPENAI_BASE_URL: ${OPENAI_BASE_URL} @@ -65,26 +66,14 @@ services: <<: *common-env HOST: "0.0.0.0" PORT: ${INTERNAL_PORT:-3002} - FLY_PROCESS_GROUP: app + WORKER_PORT: ${WORKER_PORT:-3005} ENV: local depends_on: - redis - playwright-service ports: - "${PORT:-3002}:${INTERNAL_PORT:-3002}" - command: [ "pnpm", "run", "start:production" ] - - worker: - <<: *common-service - environment: - <<: *common-env - FLY_PROCESS_GROUP: worker - ENV: local - depends_on: - - redis - - playwright-service - - api - command: [ "pnpm", "run", "workers" ] + command: node dist/src/harness.js --start-docker redis: # NOTE: If you want to use Valkey (open source) instead of Redis (source available), @@ -96,6 +85,17 @@ services: networks: - backend command: redis-server --bind 0.0.0.0 + + nuq-postgres: + build: apps/nuq-postgres + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + networks: + - backend + ports: + - "5432:5432" networks: backend: