diff --git a/.github/workflows/test-server-self-host.yml b/.github/workflows/test-server-self-host.yml index f404e928d..6209f2369 100644 --- a/.github/workflows/test-server-self-host.yml +++ b/.github/workflows/test-server-self-host.yml @@ -64,7 +64,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: '1.19' + go-version: '1.24' cache-dependency-path: ./apps/api/sharedLibs/go-html-to-md/go.sum - name: Set up Rust uses: actions-rust-lang/setup-rust-toolchain@v1 diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.yml index 8cba7ce51..8672c00c1 100644 --- a/.github/workflows/test-server.yml +++ b/.github/workflows/test-server.yml @@ -21,7 +21,11 @@ env: INDEX_SUPABASE_ANON_TOKEN: ${{ secrets.INDEX_SUPABASE_ANON_TOKEN }} INDEX_SUPABASE_URL: ${{ secrets.INDEX_SUPABASE_URL }} TEST_API_KEY: ${{ secrets.TEST_API_KEY }} + TEST_TEAM_ID: ${{ secrets.TEST_TEAM_ID }} TEST_API_KEY_CONCURRENCY: ${{ secrets.TEST_API_KEY_CONCURRENCY }} + TEST_TEAM_ID_CONCURRENCY: ${{ secrets.TEST_TEAM_ID_CONCURRENCY }} + TEST_API_KEY_ZDR: ${{ secrets.TEST_API_KEY_ZDR }} + TEST_TEAM_ID_ZDR: ${{ secrets.TEST_TEAM_ID_ZDR }} FIRE_ENGINE_BETA_URL: ${{ secrets.FIRE_ENGINE_BETA_URL }} USE_DB_AUTHENTICATION: true SERPER_API_KEY: ${{ secrets.SERPER_API_KEY }} @@ -74,7 +78,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: '1.19' + go-version: '1.24' cache-dependency-path: ./apps/api/sharedLibs/go-html-to-md/go.sum - name: Set up Rust uses: actions-rust-lang/setup-rust-toolchain@v1 diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yaml index 8614d62d8..fdc17447a 100644 --- a/apps/api/pnpm-lock.yaml +++ b/apps/api/pnpm-lock.yaml @@ -2095,6 +2095,7 @@ packages: bignumber.js@9.2.0: resolution: {integrity: sha512-JocpCSOixzy5XFJi2ub6IMmV/G9i8Lrm2lZvwBv9xPdglmZM0ufDVBbjbrfU/zuLvBfD7Bv2eYxz9i+OHTgkew==} + deprecated: pkg version number incorrect bin-links@4.0.4: resolution: {integrity: sha512-cMtq4W5ZsEwcutJrVId+a/tjt8GSbS+h0oNkdl6+6rBuEv8Ot33Bevj5KPm40t309zuhVic8NjpuL42QCiJWWA==} diff --git a/apps/api/sharedLibs/go-html-to-md/go.mod b/apps/api/sharedLibs/go-html-to-md/go.mod index 5689be6f9..ae30cf152 100644 --- a/apps/api/sharedLibs/go-html-to-md/go.mod +++ b/apps/api/sharedLibs/go-html-to-md/go.mod @@ -1,14 +1,16 @@ module html-to-markdown.go -go 1.19 +go 1.23.0 + +toolchain go1.24.0 require github.com/tomkosm/html-to-markdown v0.0.0-20250128162844-2f19490e042d require ( - github.com/PuerkitoBio/goquery v1.9.2 // indirect - github.com/andybalholm/cascadia v1.3.2 // indirect + github.com/PuerkitoBio/goquery v1.10.3 // indirect + github.com/andybalholm/cascadia v1.3.3 // indirect github.com/kr/pretty v0.3.0 // indirect - golang.org/x/net v0.25.0 // indirect + golang.org/x/net v0.41.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/apps/api/sharedLibs/go-html-to-md/go.sum b/apps/api/sharedLibs/go-html-to-md/go.sum index 01d2ce894..80024f323 100644 --- a/apps/api/sharedLibs/go-html-to-md/go.sum +++ b/apps/api/sharedLibs/go-html-to-md/go.sum @@ -1,8 +1,9 @@ -github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4yPeE= -github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk= -github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= -github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= +github.com/PuerkitoBio/goquery v1.10.3 h1:pFYcNSqHxBD06Fpj/KsbStFRsgRATgnf3LeXiUkhzPo= +github.com/PuerkitoBio/goquery v1.10.3/go.mod h1:tMUX0zDMHXYlAQk6p35XxQMqMweEKB7iK7iLNd4RH4Y= +github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= +github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= @@ -12,48 +13,82 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/sebdah/goldie/v2 v2.5.3 h1:9ES/mNN+HNUbNWpVAlrzuZ7jE+Nrczbj8uFRjM7624Y= +github.com/sebdah/goldie/v2 v2.5.3/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= +github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= github.com/tomkosm/html-to-markdown v0.0.0-20250128162844-2f19490e042d h1:NBs5X/qGdcYalsplADJxPR5CjhMWo4PxcjJeIjXm2Ww= github.com/tomkosm/html-to-markdown v0.0.0-20250128162844-2f19490e042d/go.mod h1:I2mfsDlV0RelCsTjeYh9mdXdwD2M70rA7LT/y2girik= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/goldmark v1.7.1 h1:3bajkSilaCbjdKVsKdZjZCLBNPL9pYzrCakKaf4U49U= +github.com/yuin/goldmark v1.7.1/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= +golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/apps/api/src/__tests__/queue-concurrency-integration.test.ts b/apps/api/src/__tests__/queue-concurrency-integration.test.ts index 0f427e482..c2b75befd 100644 --- a/apps/api/src/__tests__/queue-concurrency-integration.test.ts +++ b/apps/api/src/__tests__/queue-concurrency-integration.test.ts @@ -80,6 +80,7 @@ describe("Queue Concurrency Integration", () => { team_id: mockTeamId, scrapeOptions: defaultScrapeOptions, crawlerOptions: null, + zeroDataRetention: false, }; it("should add job directly to BullMQ when under concurrency limit", async () => { @@ -139,6 +140,7 @@ describe("Queue Concurrency Integration", () => { mode: "single_urls", team_id: mockTeamId, scrapeOptions: defaultScrapeOptions, + zeroDataRetention: false, } as WebScraperOptions, opts: { jobId: `job-${i}`, @@ -184,6 +186,7 @@ describe("Queue Concurrency Integration", () => { id: "test-job", data: { team_id: mockTeamId, + zeroDataRetention: false, }, }; diff --git a/apps/api/src/__tests__/snips/billing.test.ts b/apps/api/src/__tests__/snips/billing.test.ts index d1e004438..c0fee2025 100644 --- a/apps/api/src/__tests__/snips/billing.test.ts +++ b/apps/api/src/__tests__/snips/billing.test.ts @@ -247,5 +247,229 @@ describe("Billing tests", () => { expect(rc1 - rc2).toBe(305); }, 300000); + + it.concurrent("bills ZDR scrape correctly", async () => { + const identity = await idmux({ + name: "billing/bills ZDR scrape correctly", + credits: 100, + flags: { + allowZDR: true, + } + }); + + const rc1 = (await creditUsage(identity)).remaining_credits; + + // Run all scrape operations in parallel with Promise.all + const [scrape1, scrape2, scrape3] = await Promise.all([ + // scrape 1: regular fc.dev scrape (1 credit) + scrape({ + url: "https://firecrawl.dev", + zeroDataRetention: true, + }, identity), + + // scrape 1.1: regular fc.dev scrape (1 credit) + scrape({ + url: "https://firecrawl.dev", + zeroDataRetention: true, + }, identity), + + // scrape 2: fc.dev with json (5 credits) + scrape({ + url: "https://firecrawl.dev", + formats: ["json"], + jsonOptions: { + schema: { + type: "object", + properties: { + is_open_source: { type: "boolean" }, + }, + required: ["is_open_source"], + }, + }, + zeroDataRetention: true, + }, identity) + ]); + + expect(scrape1.metadata.creditsUsed).toBe(2); + expect(scrape2.metadata.creditsUsed).toBe(2); + expect(scrape3.metadata.creditsUsed).toBe(6); + + // sum: 10 credits + + await sleepForBatchBilling(); + + const rc2 = (await creditUsage(identity)).remaining_credits; + + expect(rc1 - rc2).toBe(10); + }, 120000); + + it.concurrent("bills ZDR batch scrape correctly", async () => { + const identity = await idmux({ + name: "billing/bills ZDR batch scrape correctly", + credits: 100, + flags: { + allowZDR: true, + } + }); + + const rc1 = (await creditUsage(identity)).remaining_credits; + + // Run both scrape operations in parallel with Promise.all + const [scrape1, scrape2] = await Promise.all([ + // scrape 1: regular batch scrape with failing domain (2 credits) + batchScrape({ + urls: [ + "https://firecrawl.dev", + "https://mendable.ai", + "https://thisdomaindoesnotexistandwillfail.fcr", + ], + zeroDataRetention: true, + }, identity), + + // scrape 2: batch scrape with json (10 credits) + batchScrape({ + urls: [ + "https://firecrawl.dev", + "https://mendable.ai", + "https://thisdomaindoesnotexistandwillfail.fcr", + ], + formats: ["json"], + jsonOptions: { + schema: { + type: "object", + properties: { + four_word_summary: { type: "string" }, + }, + required: ["four_word_summary"], + }, + }, + zeroDataRetention: true, + }, identity) + ]); + + expect(scrape1.data[0].metadata.creditsUsed).toBe(2); + expect(scrape1.data[1].metadata.creditsUsed).toBe(2); + + expect(scrape2.data[0].metadata.creditsUsed).toBe(6); + expect(scrape2.data[1].metadata.creditsUsed).toBe(6); + + // sum: 16 credits + + await sleepForBatchBilling(); + + const rc2 = (await creditUsage(identity)).remaining_credits; + + expect(rc1 - rc2).toBe(16); + }, 600000); + + it.concurrent("bills ZDR crawl correctly", async () => { + const identity = await idmux({ + name: "billing/bills ZDR crawl correctly", + credits: 200, + flags: { + allowZDR: true, + } + }); + + const rc1 = (await creditUsage(identity)).remaining_credits; + + // Run both crawl operations in parallel with Promise.all + const [crawl1, crawl2] = await Promise.all([ + // crawl 1: regular fc.dev crawl (x credits) + crawl({ + url: "https://firecrawl.dev", + limit: 10, + zeroDataRetention: true, + }, identity), + + // crawl 2: fc.dev crawl with json (5y credits) + crawl({ + url: "https://firecrawl.dev", + scrapeOptions: { + formats: ["json"], + jsonOptions: { + schema: { + type: "object", + properties: { + four_word_summary: { type: "string" }, + }, + required: ["four_word_summary"], + }, + }, + }, + limit: 10, + zeroDataRetention: true, + }, identity) + ]); + + expect(crawl1.success).toBe(true); + expect(crawl2.success).toBe(true); + + // sum: 2x+6y credits + + await sleepForBatchBilling(); + + const rc2 = (await creditUsage(identity)).remaining_credits; + + if (crawl1.success && crawl2.success) { + expect(rc1 - rc2).toBe(crawl1.completed * 2 + crawl2.completed * 6); + } + }, 600000); + + it.concurrent("bills custom-cost ZDR scrape correctly", async () => { + const identity = await idmux({ + name: "billing/bills ZDR scrape correctly", + credits: 100, + flags: { + allowZDR: true, + zdrCost: 0, + } + }); + + const rc1 = (await creditUsage(identity)).remaining_credits; + + // Run all scrape operations in parallel with Promise.all + const [scrape1, scrape2, scrape3] = await Promise.all([ + // scrape 1: regular fc.dev scrape (1 credit) + scrape({ + url: "https://firecrawl.dev", + zeroDataRetention: true, + }, identity), + + // scrape 1.1: regular fc.dev scrape (1 credit) + scrape({ + url: "https://firecrawl.dev", + zeroDataRetention: true, + }, identity), + + // scrape 2: fc.dev with json (5 credits) + scrape({ + url: "https://firecrawl.dev", + formats: ["json"], + jsonOptions: { + schema: { + type: "object", + properties: { + is_open_source: { type: "boolean" }, + }, + required: ["is_open_source"], + }, + }, + zeroDataRetention: true, + }, identity) + ]); + + expect(scrape1.metadata.creditsUsed).toBe(1); + expect(scrape2.metadata.creditsUsed).toBe(1); + expect(scrape3.metadata.creditsUsed).toBe(5); + + // sum: 7 credits + + await sleepForBatchBilling(); + + const rc2 = (await creditUsage(identity)).remaining_credits; + + expect(rc1 - rc2).toBe(7); + }, 120000); } }); diff --git a/apps/api/src/__tests__/snips/lib.ts b/apps/api/src/__tests__/snips/lib.ts index d39db3fc4..ebd34f822 100644 --- a/apps/api/src/__tests__/snips/lib.ts +++ b/apps/api/src/__tests__/snips/lib.ts @@ -1,7 +1,7 @@ import { configDotenv } from "dotenv"; configDotenv(); -import { ScrapeRequestInput, Document, ExtractRequestInput, ExtractResponse, CrawlRequestInput, MapRequestInput, BatchScrapeRequestInput, SearchRequestInput, CrawlStatusResponse, CrawlResponse, OngoingCrawlsResponse, ErrorResponse, CrawlErrorsResponse } from "../../controllers/v1/types"; +import { ScrapeRequestInput, Document, ExtractRequestInput, ExtractResponse, CrawlRequestInput, MapRequestInput, BatchScrapeRequestInput, SearchRequestInput, CrawlStatusResponse, CrawlResponse, OngoingCrawlsResponse, ErrorResponse, CrawlErrorsResponse, TeamFlags } from "../../controllers/v1/types"; import request from "supertest"; // ========================================= @@ -24,7 +24,8 @@ export type IdmuxRequest = { concurrency?: number, credits?: number, tokens?: number, - flags?: any, + flags?: TeamFlags, + teamId?: string; } export async function idmux(req: IdmuxRequest): Promise { @@ -228,7 +229,10 @@ export async function crawl(body: CrawlRequestInput, identity: Identity): Promis } expectCrawlToSucceed(x); - return x.body; + return { + ...x.body, + id: cs.body.id, + }; } // ========================================= @@ -279,7 +283,10 @@ export async function batchScrape(body: BatchScrapeRequestInput, identity: Ident } while (x.body.status === "scraping"); expectBatchScrapeToSucceed(x); - return x.body; + return { + ...x.body, + id: bss.body.id, + }; } // ========================================= @@ -462,6 +469,19 @@ export async function batchScrapeWithConcurrencyTracking(body: BatchScrapeReques }; } +// ========================================= +// ZDR API +// ========================================= + +export async function zdrcleaner(teamId: string) { + const res = await request(TEST_URL) + .get(`/admin/${process.env.BULL_AUTH_KEY}/zdrcleaner`) + .query({ teamId }); + + expect(res.statusCode).toBe(200); + expect(res.body.ok).toBe(true); +} + // ========================================= // ========================================= diff --git a/apps/api/src/__tests__/snips/zdr.test.ts b/apps/api/src/__tests__/snips/zdr.test.ts new file mode 100644 index 000000000..70239862a --- /dev/null +++ b/apps/api/src/__tests__/snips/zdr.test.ts @@ -0,0 +1,245 @@ +import { supabase_service } from "../../services/supabase"; +import { getJobFromGCS } from "../../lib/gcs-jobs"; +import { scrape, crawl, batchScrape, scrapeStatusRaw, zdrcleaner, idmux } from "./lib"; +import { readFile, stat } from "node:fs/promises"; + +const logIgnoreList = ["Billing queue created", "No billing operations to process in batch", "billing batch queue", "billing batch processing lock", "Batch billing team", "Successfully billed team", "Billing batch processing", "Processing batch of", "Billing team"]; + +if (process.env.TEST_SUITE_SELF_HOSTED) { + it("mocked", () => { + expect(true).toBe(true); + }); +} else { + async function getServerLogs() { + if (!process.env.GITHUB_ACTIONS) { + try { + await stat("api.log"); + } catch (e) { + console.warn("No api.log file found"); + return []; + } + } + const logs = await readFile("api.log", "utf8"); + return logs.split("\n").filter(x => x.trim().length > 0 && !logIgnoreList.some(y => x.includes(y))); + } + + async function getWorkerLogs() { + if (!process.env.GITHUB_ACTIONS) { + try { + await stat("worker.log"); + } catch (e) { + console.warn("No worker.log file found"); + return []; + } + } + const logs = await readFile("worker.log", "utf8"); + return logs.split("\n").filter(x => x.trim().length > 0 && !logIgnoreList.some(y => x.includes(y))); + } + + describe("Zero Data Retention", () => { + describe.each(["Team-scoped", "Request-scoped"] as const)("%s", (scope) => { + it("should clean up a scrape immediately", async () => { + let identity = await idmux({ + name: `zdr/${scope}/scrape`, + credits: 10000, + flags: { + allowZDR: true, + ...(scope === "Team-scoped" ? { + forceZDR: true, + } : {}), + }, + }); + + const testId = crypto.randomUUID(); + const scrape1 = await scrape({ + url: "https://firecrawl.dev/?test=" + testId, + zeroDataRetention: scope === "Request-scoped" ? true : undefined, + }, identity); + + const gcsJob = await getJobFromGCS(scrape1.metadata.scrapeId!); + expect(gcsJob).toBeNull(); + + const { data, error } = await supabase_service.from("firecrawl_jobs") + .select("*") + .eq("job_id", scrape1.metadata.scrapeId!) + .limit(1); + + expect(error).toBeFalsy(); + expect(data).toHaveLength(1); + + if (data && data.length === 1) { + const record = data[0]; + expect(record.url).not.toContain("://"); // no url stored + expect(record.docs).toBeNull(); + expect(record.page_options).toBeNull(); + expect(record.crawler_options).toBeNull(); + } + + if (scope === "Request-scoped") { + const status = await scrapeStatusRaw(scrape1.metadata.scrapeId!, identity); + + expect(status.statusCode).toBe(404); + } + }, 60000); + + it("should clean up a crawl", async () => { + const preServerLogs = await getServerLogs(); + const preWorkerLogs = await getWorkerLogs(); + + let identity = await idmux({ + name: `zdr/${scope}/crawl`, + credits: 10000, + flags: { + allowZDR: true, + ...(scope === "Team-scoped" ? { + forceZDR: true, + } : {}), + }, + }); + + const crawl1 = await crawl({ + url: "https://firecrawl.dev", + limit: 10, + zeroDataRetention: scope === "Request-scoped" ? true : undefined, + }, identity); + + const postServerLogs = (await getServerLogs()).slice(preServerLogs.length); + const postWorkerLogs = (await getWorkerLogs()).slice(preWorkerLogs.length); + + if (postWorkerLogs.length > 0 || postServerLogs.length > 0) { + console.warn("Logs changed during crawl", postServerLogs, postWorkerLogs); + } + + expect(postServerLogs).toHaveLength(0); + expect(postWorkerLogs).toHaveLength(0); + + const { data, error } = await supabase_service.from("firecrawl_jobs") + .select("*") + .eq("job_id", crawl1.id) + .limit(1); + + expect(error).toBeFalsy(); + expect(data).toHaveLength(1); + + if (data && data.length === 1) { + const record = data[0]; + expect(record.url).not.toContain("://"); // no url stored + expect(record.docs).toBeNull(); + expect(record.page_options).toBeNull(); + expect(record.crawler_options).toBeNull(); + } + + const { data: jobs, error: jobsError } = await supabase_service.from("firecrawl_jobs") + .select("*") + .eq("crawl_id", crawl1.id); + + expect(jobsError).toBeFalsy(); + expect((jobs ?? []).length).toBeGreaterThanOrEqual(1); + + for (const job of jobs ?? []) { + expect(job.url).not.toContain("://"); // no url stored + expect(job.docs).toBeNull(); + expect(job.page_options).toBeNull(); + expect(job.crawler_options).toBeNull(); + expect(typeof job.dr_clean_by).toBe("string"); // clean up happens async on a worker after expiry + + if (job.success) { + const gcsJob = await getJobFromGCS(job.job_id); + expect(gcsJob).not.toBeNull(); // clean up happens async on a worker after expiry + } + } + + await zdrcleaner(identity.teamId!); + + for (const job of jobs ?? []) { + const gcsJob = await getJobFromGCS(job.job_id); + expect(gcsJob).toBeNull(); + + if (scope === "Request-scoped") { + const status = await scrapeStatusRaw(job.job_id, identity); + expect(status.statusCode).toBe(404); + } + } + }, 600000); + + it("should clean up a batch scrape", async () => { + const preServerLogs = await getServerLogs(); + const preWorkerLogs = await getWorkerLogs(); + + let identity = await idmux({ + name: `zdr/${scope}/batch-scrape`, + credits: 10000, + flags: { + allowZDR: true, + ...(scope === "Team-scoped" ? { + forceZDR: true, + } : {}), + }, + }); + + const crawl1 = await batchScrape({ + urls: ["https://firecrawl.dev", "https://mendable.ai"], + zeroDataRetention: scope === "Request-scoped" ? true : undefined, + }, identity); + + const postServerLogs = (await getServerLogs()).slice(preServerLogs.length); + const postWorkerLogs = (await getWorkerLogs()).slice(preWorkerLogs.length); + + if (postWorkerLogs.length > 0 || postServerLogs.length > 0) { + console.warn("Logs changed during batch scrape", postServerLogs, postWorkerLogs); + } + + expect(postServerLogs).toHaveLength(0); + expect(postWorkerLogs).toHaveLength(0); + + const { data, error } = await supabase_service.from("firecrawl_jobs") + .select("*") + .eq("job_id", crawl1.id) + .limit(1); + + expect(error).toBeFalsy(); + expect(data).toHaveLength(1); + + if (data && data.length === 1) { + const record = data[0]; + expect(record.url).not.toContain("://"); // no url stored + expect(record.docs).toBeNull(); + expect(record.page_options).toBeNull(); + expect(record.crawler_options).toBeNull(); + } + + const { data: jobs, error: jobsError } = await supabase_service.from("firecrawl_jobs") + .select("*") + .eq("crawl_id", crawl1.id); + + expect(jobsError).toBeFalsy(); + expect((jobs ?? []).length).toBe(2); + + for (const job of jobs ?? []) { + expect(job.url).not.toContain("://"); // no url stored + expect(job.docs).toBeNull(); + expect(job.page_options).toBeNull(); + expect(job.crawler_options).toBeNull(); + expect(typeof job.dr_clean_by).toBe("string"); // clean up happens async on a worker after expiry + + if (job.success) { + const gcsJob = await getJobFromGCS(job.job_id); + expect(gcsJob).not.toBeNull(); // clean up happens async on a worker after expiry + } + } + + await zdrcleaner(identity.teamId!); + + for (const job of jobs ?? []) { + const gcsJob = await getJobFromGCS(job.job_id); + expect(gcsJob).toBeNull(); + + if (scope === "Request-scoped") { + const status = await scrapeStatusRaw(job.job_id, identity); + expect(status.statusCode).toBe(404); + } + } + }, 600000); + }); + }); +} diff --git a/apps/api/src/controllers/v0/admin/zdrcleaner.ts b/apps/api/src/controllers/v0/admin/zdrcleaner.ts new file mode 100644 index 000000000..70f999e40 --- /dev/null +++ b/apps/api/src/controllers/v0/admin/zdrcleaner.ts @@ -0,0 +1,102 @@ +import "dotenv/config"; +import { supabase_service } from "../../../services/supabase"; +import { removeJobFromGCS } from "../../../lib/gcs-jobs"; +import { Request, Response } from "express"; +import { logger as _logger } from "../../../lib/logger"; +import type { Logger } from "winston"; + +async function cleanUpJob(jobId: string) { + await removeJobFromGCS(jobId); +} + +async function cleanUp(specificTeamId: string | null, _logger: Logger) { + const logger = _logger.child({ + ...(specificTeamId ? { teamId: specificTeamId } : {}), + method: "cleanUp", + }); + + const cleanedUp: number[] = []; + + try { + for (let i = 0; ; i++) { + let selector = supabase_service.from("firecrawl_jobs") + .select("id, job_id"); + + if (specificTeamId) { + selector = selector.eq("team_id", specificTeamId).not("dr_clean_by", "is", null); + } else { + selector = selector + .lte("dr_clean_by", new Date().toISOString()) + .gte("dr_clean_by", new Date(Date.now() - 1000 * 60 * 60 * 24 * 7).toISOString()); + // Explanation for the gte: since the cleaner should run every 5 minutes, it is very unlikely that + // the cleaner will be down for 7 days without anyone noticing. + // Since the firecrawl_jobs table is incredibly large, even with the index on dr_clean_by, + // not giving the select a lower bound guarantees that the select will not run with an empty result + // in reasonable time. + // Therefore, we give it a lower bound which should never cause problems. + } + + const { data: jobs } = await selector + .range(i * 1000, (i + 1) * 1000) + .throwOnError(); + + if (jobs?.length === 0) { + break; + } + + for (let i = 0; i < Math.ceil((jobs?.length ?? 0) / 50); i++) { + const theseJobs = (jobs ?? []).slice(i * 50, (i + 1) * 50); + await Promise.allSettled(theseJobs.map(async (job) => { + try { + await cleanUpJob(job.job_id); + cleanedUp.push(job.id); + } catch (error) { + logger.error(`Error cleaning up job`, { + method: "cleanUpJob", + jobId: job.job_id, + scrapeId: job.job_id, + error, + }); + throw error; + } + }) ?? []); + } + + if ((jobs ?? []).length < 1000) { + break; + } + } + } catch (error) { + logger.error(`Error looping through jobs`, { + error, + }); + } + + if (cleanedUp.length > 0) { + try { + await supabase_service.from("firecrawl_jobs") + .update({ + dr_clean_by: null, + }) + .in("id", cleanedUp) + .throwOnError(); + } catch (error) { + logger.error(`Error setting cleanup value on team`, { + error, + }); + } + } +} + +export async function zdrcleanerController(req: Request, res: Response) { + const logger = _logger.child({ + module: "zdrcleaner", + method: "zdrcleanerController", + }); + + await cleanUp((req.query.teamId as string | undefined) ?? null, logger); + + logger.info("ZDR Cleaner finished!"); + + res.json({ ok: true }) +} \ No newline at end of file diff --git a/apps/api/src/controllers/v0/crawl-cancel.ts b/apps/api/src/controllers/v0/crawl-cancel.ts index a9c07b8d3..a454f700c 100644 --- a/apps/api/src/controllers/v0/crawl-cancel.ts +++ b/apps/api/src/controllers/v0/crawl-cancel.ts @@ -20,6 +20,10 @@ export async function crawlCancelController(req: Request, res: Response) { const { team_id } = auth; + if (auth.chunk?.flags?.forceZDR) { + return res.status(400).json({ error: "Your team has zero data retention enabled. This is not supported on the v0 API. Please update your code to use the v1 API." }); + } + redisEvictConnection.sadd("teams_using_v0", team_id) .catch(error => logger.error("Failed to add team to teams_using_v0", { error, team_id })); diff --git a/apps/api/src/controllers/v0/crawl-status.ts b/apps/api/src/controllers/v0/crawl-status.ts index 924828172..9f6e5b7bb 100644 --- a/apps/api/src/controllers/v0/crawl-status.ts +++ b/apps/api/src/controllers/v0/crawl-status.ts @@ -79,6 +79,10 @@ export async function crawlStatusController(req: Request, res: Response) { return res.status(auth.status).json({ error: auth.error }); } + if (auth.chunk?.flags?.forceZDR) { + return res.status(400).json({ error: "Your team has zero data retention enabled. This is not supported on the v0 API. Please update your code to use the v1 API." }); + } + const { team_id } = auth; redisEvictConnection.sadd("teams_using_v0", team_id) diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.ts index 92b452108..abd858a91 100644 --- a/apps/api/src/controllers/v0/crawl.ts +++ b/apps/api/src/controllers/v0/crawl.ts @@ -41,6 +41,10 @@ export async function crawlController(req: Request, res: Response) { const { team_id, chunk } = auth; + if (chunk?.flags?.forceZDR) { + return res.status(400).json({ error: "Your team has zero data retention enabled. This is not supported on the v0 API. Please update your code to use the v1 API." }); + } + redisEvictConnection.sadd("teams_using_v0", team_id) .catch(error => logger.error("Failed to add team to teams_using_v0", { error, team_id })); @@ -198,7 +202,7 @@ export async function crawlController(req: Request, res: Response) { name: uuid, data: { url, - mode: "single_urls", + mode: "single_urls" as const, crawlerOptions, scrapeOptions, internalOptions, @@ -207,6 +211,7 @@ export async function crawlController(req: Request, res: Response) { integration: req.body.integration, crawl_id: id, sitemapped: true, + zeroDataRetention: false, // not supported on v0 }, opts: { jobId: uuid, @@ -219,14 +224,16 @@ export async function crawlController(req: Request, res: Response) { id, sc, jobs.map((x) => x.data.url), + logger, ); await addCrawlJobs( id, jobs.map((x) => x.opts.jobId), + logger, ); for (const job of jobs) { // add with sentry instrumentation - await addScrapeJob(job.data as any, {}, job.opts.jobId); + await addScrapeJob(job.data, {}, job.opts.jobId); } }); @@ -248,13 +255,14 @@ export async function crawlController(req: Request, res: Response) { origin: req.body.origin ?? defaultOrigin, integration: req.body.integration, crawl_id: id, + zeroDataRetention: false, // not supported on v0 }, { priority: 15, // prioritize request 0 of crawl jobs same as scrape jobs }, jobId, ); - await addCrawlJob(id, jobId); + await addCrawlJob(id, jobId, logger); } res.json({ jobId: id }); diff --git a/apps/api/src/controllers/v0/crawlPreview.ts b/apps/api/src/controllers/v0/crawlPreview.ts index 3f9940a2d..cea249335 100644 --- a/apps/api/src/controllers/v0/crawlPreview.ts +++ b/apps/api/src/controllers/v0/crawlPreview.ts @@ -30,6 +30,10 @@ export async function crawlPreviewController(req: Request, res: Response) { if (!auth.success) { return res.status(auth.status).json({ error: auth.error }); } + + if (auth.chunk?.flags?.forceZDR) { + return res.status(400).json({ error: "Your team has zero data retention enabled. This is not supported on the v0 API. Please update your code to use the v1 API." }); + } let url = req.body.url; if (!url) { @@ -133,11 +137,12 @@ export async function crawlPreviewController(req: Request, res: Response) { origin: "website-preview", crawl_id: id, sitemapped: true, + zeroDataRetention: false, // not supported on v0 }, {}, jobId, ); - await addCrawlJob(id, jobId); + await addCrawlJob(id, jobId, logger); } }); @@ -154,11 +159,12 @@ export async function crawlPreviewController(req: Request, res: Response) { internalOptions, origin: "website-preview", crawl_id: id, + zeroDataRetention: false, // not supported on v0 }, {}, jobId, ); - await addCrawlJob(id, jobId); + await addCrawlJob(id, jobId, logger); } res.json({ jobId: id }); diff --git a/apps/api/src/controllers/v0/keyAuth.ts b/apps/api/src/controllers/v0/keyAuth.ts index baf4427e9..febac4115 100644 --- a/apps/api/src/controllers/v0/keyAuth.ts +++ b/apps/api/src/controllers/v0/keyAuth.ts @@ -13,6 +13,10 @@ export const keyAuthController = async (req: Request, res: Response) => { return res.status(auth.status).json({ error: auth.error }); } + if (auth.chunk?.flags?.forceZDR) { + return res.status(400).json({ error: "Your team has zero data retention enabled. This is not supported on the v0 API. Please update your code to use the v1 API." }); + } + redisEvictConnection.sadd("teams_using_v0", auth.team_id) .catch(error => logger.error("Failed to add team to teams_using_v0", { error, team_id: auth.team_id })); diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.ts index 9d1ce86dd..a5ec432f1 100644 --- a/apps/api/src/controllers/v0/scrape.ts +++ b/apps/api/src/controllers/v0/scrape.ts @@ -84,6 +84,7 @@ export async function scrapeHelper( integration: req.body.integration, is_scrape: true, startTime: Date.now(), + zeroDataRetention: false, // not supported on v0 }, {}, jobId, @@ -187,6 +188,10 @@ export async function scrapeController(req: Request, res: Response) { const { team_id, chunk } = auth; + if (chunk?.flags?.forceZDR) { + return res.status(400).json({ error: "Your team has zero data retention enabled. This is not supported on the v0 API. Please update your code to use the v1 API." }); + } + redisEvictConnection.sadd("teams_using_v0", team_id) .catch(error => logger.error("Failed to add team to teams_using_v0", { error, team_id })); diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.ts index 3bbb45ce5..aaff178c2 100644 --- a/apps/api/src/controllers/v0/search.ts +++ b/apps/api/src/controllers/v0/search.ts @@ -108,11 +108,12 @@ export async function searchHelper( name: uuid, data: { url, - mode: "single_urls", + mode: "single_urls" as const, team_id: team_id, scrapeOptions, internalOptions, startTime: Date.now(), + zeroDataRetention: false, // not supported on v0 }, opts: { jobId: uuid, @@ -123,7 +124,7 @@ export async function searchHelper( // TODO: addScrapeJobs for (const job of jobDatas) { - await addScrapeJob(job.data as any, {}, job.opts.jobId, job.opts.priority); + await addScrapeJob(job.data, {}, job.opts.jobId, job.opts.priority); } const docs = ( @@ -169,6 +170,10 @@ export async function searchController(req: Request, res: Response) { } const { team_id, chunk } = auth; + if (chunk?.flags?.forceZDR) { + return res.status(400).json({ error: "Your team has zero data retention enabled. This is not supported on the v0 API. Please update your code to use the v1 API." }); + } + redisEvictConnection.sadd("teams_using_v0", team_id) .catch(error => logger.error("Failed to add team to teams_using_v0", { error, team_id })); @@ -224,6 +229,7 @@ export async function searchController(req: Request, res: Response) { crawlerOptions: crawlerOptions, origin, integration: req.body.integration, + zeroDataRetention: false, // not supported }); return res.status(result.returnCode).json(result); } catch (error) { diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.ts index 0a270aa6e..9a034c150 100644 --- a/apps/api/src/controllers/v1/batch-scrape.ts +++ b/apps/api/src/controllers/v1/batch-scrape.ts @@ -31,6 +31,15 @@ export async function batchScrapeController( ) { const preNormalizedBody = { ...req.body }; + if (req.body.zeroDataRetention && !req.acuc?.flags?.allowZDR) { + return res.status(400).json({ + success: false, + error: "Zero data retention is enabled for this team. If you're interested in ZDR, please contact support@firecrawl.com", + }); + } + + const zeroDataRetention = req.acuc?.flags?.forceZDR || req.body.zeroDataRetention; + if (req.body?.ignoreInvalidURLs === true) { req.body = batchScrapeRequestSchemaNoURLValidation.parse(req.body); } else { @@ -44,9 +53,10 @@ export async function batchScrapeController( module: "api/v1", method: "batchScrapeController", teamId: req.auth.team_id, + zeroDataRetention, }); - let urls = req.body.urls; + let urls: string[] = req.body.urls; let unnormalizedURLs = preNormalizedBody.urls; let invalidURLs: string[] | undefined = undefined; @@ -99,10 +109,12 @@ export async function batchScrapeController( disableSmartWaitCache: true, teamId: req.auth.team_id, saveScrapeResultToGCS: process.env.GCS_FIRE_ENGINE_BUCKET_NAME ? true : false, + zeroDataRetention, }, // NOTE: smart wait disabled for batch scrapes to ensure contentful scrape, speed does not matter team_id: req.auth.team_id, createdAt: Date.now(), maxConcurrency: req.body.maxConcurrency, + zeroDataRetention, }; if (!req.body.appendToId) { @@ -126,8 +138,7 @@ export async function batchScrapeController( delete (scrapeOptions as any).urls; delete (scrapeOptions as any).appendToId; - const jobs = urls.map((x, i) => { - return { + const jobs = urls.map(x => ({ data: { url: x, mode: "single_urls" as const, @@ -140,17 +151,14 @@ export async function batchScrapeController( sitemapped: true, v1: true, webhook: req.body.webhook, - internalOptions: { - saveScrapeResultToGCS: process.env.GCS_FIRE_ENGINE_BUCKET_NAME ? true : false, - unnormalizedSourceURL: unnormalizedURLs[i], - }, + internalOptions: sc.internalOptions, + zeroDataRetention, }, opts: { jobId: uuidv4(), priority: 20, }, - }; - }); + })); await finishCrawlKickoff(id); @@ -159,11 +167,13 @@ export async function batchScrapeController( id, sc, jobs.map((x) => x.data.url), + logger, ); logger.debug("Adding scrape jobs to Redis..."); await addCrawlJobs( id, jobs.map((x) => x.opts.jobId), + logger, ); logger.debug("Adding scrape jobs to BullMQ..."); await addScrapeJobs(jobs); diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.ts index 66bf9ad17..d1ae218f1 100644 --- a/apps/api/src/controllers/v1/crawl.ts +++ b/apps/api/src/controllers/v1/crawl.ts @@ -19,13 +19,24 @@ export async function crawlController( const preNormalizedBody = req.body; req.body = crawlRequestSchema.parse(req.body); + if (req.body.zeroDataRetention && !req.acuc?.flags?.allowZDR) { + return res.status(400).json({ + success: false, + error: "Zero data retention is enabled for this team. If you're interested in ZDR, please contact support@firecrawl.com", + }); + } + + const zeroDataRetention = req.acuc?.flags?.forceZDR || req.body.zeroDataRetention; + const id = uuidv4(); const logger = _logger.child({ crawlId: id, module: "api/v1", method: "crawlController", teamId: req.auth.team_id, + zeroDataRetention, }); + logger.debug("Crawl " + id + " starting", { request: req.body, originalRequest: preNormalizedBody, @@ -84,10 +95,12 @@ export async function crawlController( disableSmartWaitCache: true, teamId: req.auth.team_id, saveScrapeResultToGCS: process.env.GCS_FIRE_ENGINE_BUCKET_NAME ? true : false, + zeroDataRetention, }, // NOTE: smart wait disabled for crawls to ensure contentful scrape, speed does not matter team_id: req.auth.team_id, createdAt: Date.now(), maxConcurrency: req.body.maxConcurrency !== undefined ? Math.min(req.body.maxConcurrency, req.acuc.concurrency) : undefined, + zeroDataRetention, }; const crawler = crawlToCrawler(id, sc, req.acuc?.flags ?? null); @@ -119,6 +132,7 @@ export async function crawlController( crawl_id: id, webhook: req.body.webhook, v1: true, + zeroDataRetention: zeroDataRetention || false, }, {}, crypto.randomUUID(), diff --git a/apps/api/src/controllers/v1/deep-research.ts b/apps/api/src/controllers/v1/deep-research.ts index b9baa7ca7..472e208eb 100644 --- a/apps/api/src/controllers/v1/deep-research.ts +++ b/apps/api/src/controllers/v1/deep-research.ts @@ -1,5 +1,5 @@ import { Request, Response } from "express"; -import { extractOptions, RequestWithAuth } from "./types"; +import { ErrorResponse, extractOptions, RequestWithAuth } from "./types"; import { getDeepResearchQueue } from "../../services/queue-service"; import * as Sentry from "@sentry/node"; import { saveDeepResearch } from "../../lib/deep-research/deep-research-redis"; @@ -31,7 +31,7 @@ export const deepResearchRequestSchema = z.object({ export type DeepResearchRequest = z.infer; -export type DeepResearchResponse = { +export type DeepResearchResponse = ErrorResponse | { success: boolean; id: string; }; @@ -46,6 +46,10 @@ export async function deepResearchController( req: RequestWithAuth<{}, DeepResearchResponse, DeepResearchRequest>, res: Response, ) { + if (req.acuc?.flags?.forceZDR) { + return res.status(400).json({ success: false, error: "Your team has zero data retention enabled. This is not supported on deep research. Please contact support@firecrawl.com to unblock this feature." }); + } + req.body = deepResearchRequestSchema.parse(req.body); const researchId = crypto.randomUUID(); diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.ts index 4ffaddfe6..e6d356b85 100644 --- a/apps/api/src/controllers/v1/extract.ts +++ b/apps/api/src/controllers/v1/extract.ts @@ -62,6 +62,10 @@ export async function extractController( const originalRequest = { ...req.body }; req.body = extractRequestSchema.parse(req.body); + if (req.acuc?.flags?.forceZDR) { + return res.status(400).json({ success: false, error: "Your team has zero data retention enabled. This is not supported on extract. Please contact support@firecrawl.com to unblock this feature." }); + } + const invalidURLs: string[] = req.body.urls?.filter((url: string) => isUrlBlocked(url, req.acuc?.flags ?? null)) ?? []; if (invalidURLs.length > 0 && !req.body.ignoreInvalidURLs) { @@ -82,6 +86,7 @@ export async function extractController( team_id: req.auth.team_id, subId: req.acuc?.sub_id, extractId, + zeroDataRetention: req.acuc?.flags?.forceZDR, }); const jobData = { @@ -111,6 +116,7 @@ export async function extractController( showLLMUsage: req.body.__experimental_llmUsage, showSources: req.body.__experimental_showSources || req.body.showSources, showCostTracking: req.body.__experimental_showCostTracking, + zeroDataRetention: req.acuc?.flags?.forceZDR, }); if (Sentry.isInitialized()) { diff --git a/apps/api/src/controllers/v1/generate-llmstxt.ts b/apps/api/src/controllers/v1/generate-llmstxt.ts index aaf34c3bb..4177af00d 100644 --- a/apps/api/src/controllers/v1/generate-llmstxt.ts +++ b/apps/api/src/controllers/v1/generate-llmstxt.ts @@ -1,5 +1,6 @@ import { Response } from "express"; import { + ErrorResponse, GenerateLLMsTextRequest, generateLLMsTextRequestSchema, RequestWithAuth, @@ -7,9 +8,8 @@ import { import { getGenerateLlmsTxtQueue } from "../../services/queue-service"; import * as Sentry from "@sentry/node"; import { saveGeneratedLlmsTxt } from "../../lib/generate-llmstxt/generate-llmstxt-redis"; -import { z } from "zod"; -export type GenerateLLMsTextResponse = { +export type GenerateLLMsTextResponse = ErrorResponse | { success: boolean; id: string; }; @@ -24,6 +24,10 @@ export async function generateLLMsTextController( req: RequestWithAuth<{}, GenerateLLMsTextResponse, GenerateLLMsTextRequest>, res: Response, ) { + if (req.acuc?.flags?.forceZDR) { + return res.status(400).json({ success: false, error: "Your team has zero data retention enabled. This is not supported on llmstxt. Please contact support@firecrawl.com to unblock this feature." }); + } + req.body = generateLLMsTextRequestSchema.parse(req.body); const generationId = crypto.randomUUID(); diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.ts index 1e6738317..6145beced 100644 --- a/apps/api/src/controllers/v1/map.ts +++ b/apps/api/src/controllers/v1/map.ts @@ -101,6 +101,8 @@ export async function getMapResults({ let links: string[] = [url]; let mapResults: MapDocument[] = []; + const zeroDataRetention = flags?.forceZDR ?? false; + const sc: StoredCrawl = { originUrl: url, crawlerOptions: { @@ -184,7 +186,9 @@ export async function getMapResults({ ); allResults = await Promise.all(pagePromises); - await redis.set(cacheKey, JSON.stringify(allResults), "EX", 48 * 60 * 60); // Cache for 48 hours + if (!zeroDataRetention) { + await redis.set(cacheKey, JSON.stringify(allResults), "EX", 48 * 60 * 60); // Cache for 48 hours + } } // Parallelize sitemap index query with search results @@ -316,6 +320,10 @@ export async function mapController( ) { const originalRequest = req.body; req.body = mapRequestSchema.parse(req.body); + + if (req.acuc?.flags?.forceZDR) { + return res.status(400).json({ success: false, error: "Your team has zero data retention enabled. This is not supported on map. Please contact support@firecrawl.com to unblock this feature." }); + } logger.info("Map request", { request: req.body, @@ -385,6 +393,7 @@ export async function mapController( integration: req.body.integration, num_tokens: 0, credits_billed: 1, + zeroDataRetention: false, // not supported }); const response = { diff --git a/apps/api/src/controllers/v1/scrape-status.ts b/apps/api/src/controllers/v1/scrape-status.ts index 4f076a347..78ded113a 100644 --- a/apps/api/src/controllers/v1/scrape-status.ts +++ b/apps/api/src/controllers/v1/scrape-status.ts @@ -10,8 +10,13 @@ export async function scrapeStatusController(req: any, res: any) { teamId: req.auth.team_id, jobId: req.params.jobId, scrapeId: req.params.jobId, + zeroDataRetention: req.acuc?.flags?.forceZDR, }); + if (req.acuc?.flags?.forceZDR) { + return res.status(400).json({ success: false, error: "Your team has zero data retention enabled. This is not supported on scrape status. Please contact support@firecrawl.com to unblock this feature." }); + } + const job = await supabaseGetJobByIdOnlyData(req.params.jobId, logger); if (!job) { @@ -34,6 +39,13 @@ export async function scrapeStatusController(req: any, res: any) { const data = Array.isArray(jobData?.returnvalue) ? jobData?.returnvalue[0] : jobData?.returnvalue; + + if (!data) { + return res.status(404).json({ + success: false, + error: "Job not found.", + }); + } return res.status(200).json({ success: true, diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.ts index 993f3d4f6..f3861a61b 100644 --- a/apps/api/src/controllers/v1/scrape.ts +++ b/apps/api/src/controllers/v1/scrape.ts @@ -18,12 +18,23 @@ export async function scrapeController( ) { const jobId = uuidv4(); const preNormalizedBody = { ...req.body }; + + if (req.body.zeroDataRetention && !req.acuc?.flags?.allowZDR) { + return res.status(400).json({ + success: false, + error: "Zero data retention is enabled for this team. If you're interested in ZDR, please contact support@firecrawl.com", + }); + } + + const zeroDataRetention = req.acuc?.flags?.forceZDR || req.body.zeroDataRetention; + const logger = _logger.child({ method: "scrapeController", jobId, scrapeId: jobId, teamId: req.auth.team_id, team_id: req.auth.team_id, + zeroDataRetention, }); logger.debug("Scrape " + jobId + " starting", { @@ -62,10 +73,12 @@ export async function scrapeController( saveScrapeResultToGCS: process.env.GCS_FIRE_ENGINE_BUCKET_NAME ? true : false, unnormalizedSourceURL: preNormalizedBody.url, bypassBilling: isDirectToBullMQ, + zeroDataRetention, }, origin, integration: req.body.integration, startTime, + zeroDataRetention, }, {}, jobId, @@ -88,6 +101,10 @@ export async function scrapeController( startTime, }); + if (zeroDataRetention) { + await getScrapeQueue().remove(jobId); + } + if ( e instanceof Error && (e.message.startsWith("Job wait") || e.message === "timeout") diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.ts index 68aab2c3b..2fdaca2c6 100644 --- a/apps/api/src/controllers/v1/search.ts +++ b/apps/api/src/controllers/v1/search.ts @@ -84,6 +84,8 @@ async function scrapeSearchResult( basePriority: 10, }); + const zeroDataRetention = flags?.forceZDR ?? false; + try { if (isUrlBlocked(searchResult.url, flags)) { throw new Error("Could not scrape url: " + BLOCKLISTED_URL_MESSAGE); @@ -93,6 +95,7 @@ async function scrapeSearchResult( url: searchResult.url, teamId: options.teamId, origin: options.origin, + zeroDataRetention, }); await addScrapeJob( { @@ -103,10 +106,11 @@ async function scrapeSearchResult( ...options.scrapeOptions, maxAge: 3 * 24 * 60 * 60 * 1000, // 3 days }, - internalOptions: { teamId: options.teamId, bypassBilling: true }, + internalOptions: { teamId: options.teamId, bypassBilling: true, zeroDataRetention }, origin: options.origin, is_scrape: true, startTime: Date.now(), + zeroDataRetention, }, {}, jobId, @@ -166,8 +170,13 @@ export async function searchController( teamId: req.auth.team_id, module: "search", method: "searchController", + zeroDataRetention: req.acuc?.flags?.forceZDR, }); + if (req.acuc?.flags?.forceZDR) { + return res.status(400).json({ success: false, error: "Your team has zero data retention enabled. This is not supported on search. Please contact support@firecrawl.com to unblock this feature." }); + } + let responseData: SearchResponse = { success: true, data: [], @@ -315,6 +324,7 @@ export async function searchController( integration: req.body.integration, cost_tracking: costTracking, credits_billed, + zeroDataRetention: false, // not supported }, false, isSearchPreview, diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts index d2f5706c2..f918430a8 100644 --- a/apps/api/src/controllers/v1/types.ts +++ b/apps/api/src/controllers/v1/types.ts @@ -559,6 +559,7 @@ export const scrapeRequestSchema = baseScrapeOptions origin: z.string().optional().default("api"), integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null), timeout: z.number().int().positive().finite().safe().default(30000), + zeroDataRetention: z.boolean().optional(), }) .strict(strictMessage) .refine(extractRefine, extractRefineOpts) @@ -597,6 +598,7 @@ export const batchScrapeRequestSchema = baseScrapeOptions appendToId: z.string().uuid().optional(), ignoreInvalidURLs: z.boolean().default(false), maxConcurrency: z.number().positive().int().optional(), + zeroDataRetention: z.boolean().optional(), }) .strict(strictMessage) .refine(extractRefine, extractRefineOpts) @@ -612,6 +614,7 @@ export const batchScrapeRequestSchemaNoURLValidation = baseScrapeOptions appendToId: z.string().uuid().optional(), ignoreInvalidURLs: z.boolean().default(false), maxConcurrency: z.number().positive().int().optional(), + zeroDataRetention: z.boolean().optional(), }) .strict(strictMessage) .refine(extractRefine, extractRefineOpts) @@ -663,6 +666,7 @@ export const crawlRequestSchema = crawlerOptions webhook: webhookSchema.optional(), limit: z.number().default(10000), maxConcurrency: z.number().positive().int().optional(), + zeroDataRetention: z.boolean().optional(), }) .strict(strictMessage) .refine((x) => extractRefine(x.scrapeOptions), extractRefineOpts) @@ -1002,6 +1006,9 @@ export type AuthCreditUsageChunk = { export type TeamFlags = { ignoreRobots?: boolean; unblockedDomains?: string[]; + forceZDR?: boolean; + allowZDR?: boolean; + zdrCost?: number; } | null; export type AuthCreditUsageChunkFromTeam = Omit; diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.ts index d9b396d80..022f3523b 100644 --- a/apps/api/src/lib/concurrency-limit.ts +++ b/apps/api/src/lib/concurrency-limit.ts @@ -215,6 +215,7 @@ async function getNextConcurrentJob(teamId: string, i = 0): Promise<{ logger.warn("Failed to remove job from concurrency limit queue", { teamId, jobId: finalJob.job.id, + zeroDataRetention: finalJob.job.data?.zeroDataRetention, i }); } diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.ts index cb64789b5..0fb3fec0f 100644 --- a/apps/api/src/lib/crawl-redis.ts +++ b/apps/api/src/lib/crawl-redis.ts @@ -4,6 +4,7 @@ import { WebCrawler } from "../scraper/WebScraper/crawler"; import { redisEvictConnection } from "../services/redis"; import { logger as _logger } from "./logger"; import { getAdjustedMaxDepth } from "../scraper/WebScraper/utils/maxDepthUtils"; +import type { Logger } from "winston"; export type StoredCrawl = { originUrl?: string; @@ -15,6 +16,7 @@ export type StoredCrawl = { cancelled?: boolean; createdAt: number; maxConcurrency?: number; + zeroDataRetention?: boolean; }; export async function saveCrawl(id: string, crawl: StoredCrawl) { @@ -24,6 +26,7 @@ export async function saveCrawl(id: string, crawl: StoredCrawl) { method: "saveCrawl", crawlId: id, teamId: crawl.team_id, + zeroDataRetention: crawl.zeroDataRetention, }); await redisEvictConnection.set("crawl:" + id, JSON.stringify(crawl)); await redisEvictConnection.expire("crawl:" + id, 24 * 60 * 60); @@ -55,8 +58,8 @@ export async function getCrawlExpiry(id: string): Promise { return d; } -export async function addCrawlJob(id: string, job_id: string) { - _logger.debug("Adding crawl job " + job_id + " to Redis...", { +export async function addCrawlJob(id: string, job_id: string, __logger: Logger = _logger) { + __logger.debug("Adding crawl job " + job_id + " to Redis...", { jobId: job_id, module: "crawl-redis", method: "addCrawlJob", @@ -66,10 +69,10 @@ export async function addCrawlJob(id: string, job_id: string) { await redisEvictConnection.expire("crawl:" + id + ":jobs", 24 * 60 * 60); } -export async function addCrawlJobs(id: string, job_ids: string[]) { +export async function addCrawlJobs(id: string, job_ids: string[], __logger: Logger = _logger) { if (job_ids.length === 0) return true; - _logger.debug("Adding crawl jobs to Redis...", { + __logger.debug("Adding crawl jobs to Redis...", { jobIds: job_ids, module: "crawl-redis", method: "addCrawlJobs", @@ -83,8 +86,9 @@ export async function addCrawlJobDone( id: string, job_id: string, success: boolean, + __logger: Logger = _logger, ) { - _logger.debug("Adding done crawl job to Redis...", { + __logger.debug("Adding done crawl job to Redis...", { jobId: job_id, module: "crawl-redis", method: "addCrawlJobDone", @@ -160,9 +164,9 @@ export async function finishCrawlKickoff(id: string) { ); } -export async function finishCrawlPre(id: string) { +export async function finishCrawlPre(id: string, __logger: Logger = _logger) { if (await isCrawlFinished(id)) { - _logger.debug("Marking crawl as pre-finished.", { + __logger.debug("Marking crawl as pre-finished.", { module: "crawl-redis", method: "finishCrawlPre", crawlId: id, @@ -182,8 +186,8 @@ export async function unPreFinishCrawl(id: string) { await redisEvictConnection.del("crawl:" + id + ":finished_pre"); } -export async function finishCrawl(id: string) { - _logger.debug("Marking crawl as finished.", { +export async function finishCrawl(id: string, __logger: Logger = _logger) { + __logger.debug("Marking crawl as finished.", { module: "crawl-redis", method: "finishCrawl", crawlId: id, @@ -279,35 +283,20 @@ export async function lockURL( sc: StoredCrawl, url: string, ): Promise { - let logger = _logger.child({ - crawlId: id, - module: "crawl-redis", - method: "lockURL", - preNormalizedURL: url, - teamId: sc.team_id, - }); - if (typeof sc.crawlerOptions?.limit === "number") { if ( (await redisEvictConnection.scard("crawl:" + id + ":visited_unique")) >= sc.crawlerOptions.limit ) { - // logger.debug( - // "Crawl has already hit visited_unique limit, not locking URL.", - // ); return false; } } - url = normalizeURL(url, sc); - logger = logger.child({ url }); - let res: boolean; if (!sc.crawlerOptions?.deduplicateSimilarURLs) { res = (await redisEvictConnection.sadd("crawl:" + id + ":visited", url)) !== 0; } else { const permutations = generateURLPermutations(url).map((x) => x.href); - // logger.debug("Adding URL permutations for URL " + JSON.stringify(url) + "...", { permutations }); const x = await redisEvictConnection.sadd( "crawl:" + id + ":visited", ...permutations, @@ -325,9 +314,6 @@ export async function lockURL( ); } - // logger.debug("Locking URL " + JSON.stringify(url) + "... result: " + res, { - // res, - // }); return res; } @@ -336,11 +322,12 @@ export async function lockURLs( id: string, sc: StoredCrawl, urls: string[], + __logger: Logger = _logger, ): Promise { if (urls.length === 0) return true; urls = urls.map((url) => normalizeURL(url, sc)); - const logger = _logger.child({ + const logger = __logger.child({ crawlId: id, module: "crawl-redis", method: "lockURL", @@ -421,6 +408,7 @@ export function crawlToCrawler( regexOnFullURL: sc.crawlerOptions?.regexOnFullURL ?? false, maxDiscoveryDepth: sc.crawlerOptions?.maxDiscoveryDepth, currentDiscoveryDepth: crawlerOptions?.currentDiscoveryDepth ?? 0, + zeroDataRetention: (teamFlags?.forceZDR || sc.zeroDataRetention) ?? false, }); if (sc.robots !== undefined) { diff --git a/apps/api/src/lib/deep-research/deep-research-service.ts b/apps/api/src/lib/deep-research/deep-research-service.ts index 1a8238f0b..da55ada86 100644 --- a/apps/api/src/lib/deep-research/deep-research-service.ts +++ b/apps/api/src/lib/deep-research/deep-research-service.ts @@ -365,6 +365,7 @@ export async function performDeepResearch(options: DeepResearchServiceOptions) { tokens_billed: 0, cost_tracking: costTracking, credits_billed, + zeroDataRetention: false, // not supported }); await updateDeepResearch(researchId, { status: "completed", diff --git a/apps/api/src/lib/extract/document-scraper.ts b/apps/api/src/lib/extract/document-scraper.ts index f6ae72c2a..b6eecf320 100644 --- a/apps/api/src/lib/extract/document-scraper.ts +++ b/apps/api/src/lib/extract/document-scraper.ts @@ -53,6 +53,7 @@ export async function scrapeDocument( is_scrape: true, from_extract: true, startTime: Date.now(), + zeroDataRetention: false, // not supported }, {}, jobId, diff --git a/apps/api/src/lib/extract/extract-redis.ts b/apps/api/src/lib/extract/extract-redis.ts index 5d82adef0..f96446d3f 100644 --- a/apps/api/src/lib/extract/extract-redis.ts +++ b/apps/api/src/lib/extract/extract-redis.ts @@ -41,6 +41,7 @@ export type StoredExtract = { }; sessionIds?: string[]; tokensBilled?: number; + zeroDataRetention?: boolean; }; // Reduce TTL to 6 hours instead of 24 diff --git a/apps/api/src/lib/extract/extraction-service.ts b/apps/api/src/lib/extract/extraction-service.ts index c25689d5d..304cde911 100644 --- a/apps/api/src/lib/extract/extraction-service.ts +++ b/apps/api/src/lib/extract/extraction-service.ts @@ -190,6 +190,7 @@ export async function performExtraction( tokens_billed, sources, cost_tracking: costTracking, + zeroDataRetention: false, // not supported }); await billTeam(teamId, subId, tokens_billed, logger, true).catch((error) => { @@ -686,6 +687,7 @@ export async function performExtraction( tokens_billed, sources, cost_tracking: costTracking, + zeroDataRetention: false, // not supported }); await billTeam(teamId, subId, tokens_billed, logger, true).catch((error) => { logger.error( @@ -794,6 +796,7 @@ export async function performExtraction( tokens_billed, sources, cost_tracking: costTracking, + zeroDataRetention: false, // not supported }); await billTeam(teamId, subId, tokens_billed, logger, true).catch((error) => { logger.error( @@ -835,6 +838,7 @@ export async function performExtraction( tokens_billed, sources, cost_tracking: costTracking, + zeroDataRetention: false, // not supported }); return { success: false, @@ -1020,6 +1024,7 @@ export async function performExtraction( tokens_billed: tokensToBill, sources, cost_tracking: costTracking, + zeroDataRetention: false, // not supported }).then(() => { updateExtract(extractId, { status: "completed", @@ -1090,6 +1095,7 @@ export async function performExtraction( tokens_billed, sources, cost_tracking: costTracking, + zeroDataRetention: false, // not supported }); throw error; diff --git a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts index f00038b6c..824440b1d 100644 --- a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts +++ b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts @@ -51,6 +51,7 @@ export async function scrapeDocument_F0( is_scrape: true, from_extract: true, startTime: Date.now(), + zeroDataRetention: false, // not supported }, {}, jobId, diff --git a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts index e096740f3..76b6cdb5f 100644 --- a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts +++ b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts @@ -122,6 +122,7 @@ import { getACUCTeam } from "../../../controllers/auth"; num_tokens: 0, tokens_billed: 0, sources, + zeroDataRetention: false, // not supported }); return { success: false, @@ -224,6 +225,7 @@ import { getACUCTeam } from "../../../controllers/auth"; num_tokens: 0, tokens_billed: 0, sources, + zeroDataRetention: false, // not supported }); return { success: false, @@ -574,6 +576,7 @@ import { getACUCTeam } from "../../../controllers/auth"; num_tokens: 0, tokens_billed: 0, sources, + zeroDataRetention: false, // not supported }); return { success: false, @@ -670,6 +673,7 @@ import { getACUCTeam } from "../../../controllers/auth"; num_tokens: 0, tokens_billed: 0, sources, + zeroDataRetention: false, // not supported }); return { success: false, @@ -699,6 +703,7 @@ import { getACUCTeam } from "../../../controllers/auth"; num_tokens: 0, tokens_billed: 0, sources, + zeroDataRetention: false, // not supported }); return { success: false, @@ -861,6 +866,7 @@ import { getACUCTeam } from "../../../controllers/auth"; num_tokens: totalTokensUsed, tokens_billed: tokensToBill, sources, + zeroDataRetention: false, // not supported }).then(() => { updateExtract(extractId, { status: "completed", diff --git a/apps/api/src/lib/gcs-jobs.ts b/apps/api/src/lib/gcs-jobs.ts index 834be2f95..79e02a2f0 100644 --- a/apps/api/src/lib/gcs-jobs.ts +++ b/apps/api/src/lib/gcs-jobs.ts @@ -104,6 +104,31 @@ export async function getJobFromGCS(jobId: string): Promise { } } +export async function removeJobFromGCS(jobId: string): Promise { + try { + if (!process.env.GCS_BUCKET_NAME) { + return; + } + + const bucket = storage.bucket(process.env.GCS_BUCKET_NAME); + const blob = bucket.file(`${jobId}.json`); + await blob.delete({ + ignoreNotFound: true, + }); + } catch (error) { + if (error instanceof ApiError && error.code === 404 && error.message.includes("No such object:")) { + // Object does not exist + return; + } + + logger.error(`Error removing job from GCS`, { + error, + jobId, + scrapeId: jobId, + }); + } +} + // TODO: fix the any type (we have multiple Document types in the codebase) export async function getDocFromGCS(url: string): Promise { // logger.info(`Getting f-engine document from GCS`, { diff --git a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts index d9f953d6a..9f2fb4487 100644 --- a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts +++ b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts @@ -243,6 +243,7 @@ export async function performGenerateLlmsTxt( sources: {}, cost_tracking: costTracking, credits_billed: urls.length, + zeroDataRetention: false, }); // Bill team for usage diff --git a/apps/api/src/lib/logger.ts b/apps/api/src/lib/logger.ts index 76b66180e..ede4be1da 100644 --- a/apps/api/src/lib/logger.ts +++ b/apps/api/src/lib/logger.ts @@ -24,6 +24,14 @@ const logFormat = winston.format.printf( }`, ); +// Filter function to prevent logging when zeroDataRetention is true +const zeroDataRetentionFilter = winston.format((info) => { + if (info.metadata?.zeroDataRetention === true || info.zeroDataRetention === true) { + return false; // Don't log this message + } + return info; +})(); + export const logger = winston.createLogger({ level: process.env.LOGGING_LEVEL?.toLowerCase() ?? "debug", format: winston.format.json({ @@ -51,11 +59,16 @@ export const logger = winston.createLogger({ "-" + crypto.randomUUID() + ".log", + format: winston.format.combine( + zeroDataRetentionFilter, + winston.format.json() + ), }), ] : []), new winston.transports.Console({ format: winston.format.combine( + zeroDataRetentionFilter, winston.format.timestamp({ format: "YYYY-MM-DD HH:mm:ss" }), winston.format.metadata({ fillExcept: ["message", "level", "timestamp"], diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.ts index 45209aaf3..7030734b2 100644 --- a/apps/api/src/lib/scrape-billing.ts +++ b/apps/api/src/lib/scrape-billing.ts @@ -1,10 +1,11 @@ -import { Document, ScrapeOptions } from "../controllers/v1/types"; +import { InternalOptions } from "src/scraper/scrapeURL"; +import { Document, ScrapeOptions, TeamFlags } from "../controllers/v1/types"; import { CostTracking } from "./extract/extraction-service"; const creditsPerPDFPage = 1; const stealthProxyCostBonus = 4; -export async function calculateCreditsToBeBilled(options: ScrapeOptions, document: Document | null, costTracking: CostTracking) { +export async function calculateCreditsToBeBilled(options: ScrapeOptions, internalOptions: InternalOptions, document: Document | null, costTracking: CostTracking, flags: TeamFlags) { if (document === null) { // Failure -- check cost tracking if FIRE-1 let creditsToBeBilled = 0; @@ -24,6 +25,10 @@ export async function calculateCreditsToBeBilled(options: ScrapeOptions, documen if (options.agent?.model?.toLowerCase() === "fire-1" || options.extract?.agent?.model?.toLowerCase() === "fire-1" || options.jsonOptions?.agent?.model?.toLowerCase() === "fire-1") { creditsToBeBilled = Math.ceil((costTracking.toJSON().totalCost ?? 1) * 1800); } + + if (internalOptions.zeroDataRetention) { + creditsToBeBilled += (flags?.zdrCost ?? 1); + } if (document.metadata.numPages !== undefined && document.metadata.numPages > 1) { creditsToBeBilled += creditsPerPDFPage * (document.metadata.numPages - 1); diff --git a/apps/api/src/routes/admin.ts b/apps/api/src/routes/admin.ts index ea1fcf0f0..2c38f501d 100644 --- a/apps/api/src/routes/admin.ts +++ b/apps/api/src/routes/admin.ts @@ -11,6 +11,7 @@ import { acucCacheClearController } from "../controllers/v0/admin/acuc-cache-cle import { checkFireEngine } from "../controllers/v0/admin/check-fire-engine"; import { cclogController } from "../controllers/v0/admin/cclog"; import { indexQueuePrometheus } from "../controllers/v0/admin/index-queue-prometheus"; +import { zdrcleanerController } from "../controllers/v0/admin/zdrcleaner"; export const adminRouter = express.Router(); @@ -51,6 +52,12 @@ adminRouter.get( wrap(cclogController), ); +adminRouter.get( + `/admin/${process.env.BULL_AUTH_KEY}/zdrcleaner`, + wrap(zdrcleanerController), +); + + adminRouter.get( `/admin/${process.env.BULL_AUTH_KEY}/index-queue-prometheus`, wrap(indexQueuePrometheus), diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts index d5cf8f1f5..3fc7e9a3b 100644 --- a/apps/api/src/scraper/WebScraper/crawler.ts +++ b/apps/api/src/scraper/WebScraper/crawler.ts @@ -58,6 +58,7 @@ export class WebCrawler { private sitemapsHit: Set = new Set(); private maxDiscoveryDepth: number | undefined; private currentDiscoveryDepth: number; + private zeroDataRetention: boolean; constructor({ jobId, @@ -76,6 +77,7 @@ export class WebCrawler { regexOnFullURL = false, maxDiscoveryDepth, currentDiscoveryDepth, + zeroDataRetention, }: { jobId: string; initialUrl: string; @@ -93,6 +95,7 @@ export class WebCrawler { regexOnFullURL?: boolean; maxDiscoveryDepth?: number; currentDiscoveryDepth?: number; + zeroDataRetention?: boolean; }) { this.jobId = jobId; this.initialUrl = initialUrl; @@ -111,7 +114,8 @@ export class WebCrawler { this.allowSubdomains = allowSubdomains ?? false; this.ignoreRobotsTxt = ignoreRobotsTxt ?? false; this.regexOnFullURL = regexOnFullURL ?? false; - this.logger = _logger.child({ crawlId: this.jobId, module: "WebCrawler" }); + this.zeroDataRetention = zeroDataRetention ?? false; + this.logger = _logger.child({ crawlId: this.jobId, module: "WebCrawler", zeroDataRetention: this.zeroDataRetention }); this.maxDiscoveryDepth = maxDiscoveryDepth; this.currentDiscoveryDepth = currentDiscoveryDepth ?? 0; } @@ -682,7 +686,7 @@ export class WebCrawler { // Try to get sitemap from the provided URL first try { sitemapCount = await getLinksFromSitemap( - { sitemapUrl, urlsHandler, mode: "fire-engine", maxAge }, + { sitemapUrl, urlsHandler, mode: "fire-engine", maxAge, zeroDataRetention: this.zeroDataRetention }, this.logger, this.jobId, this.sitemapsHit, @@ -731,6 +735,7 @@ export class WebCrawler { }, mode: "fire-engine", maxAge, + zeroDataRetention: this.zeroDataRetention, }, this.logger, this.jobId, @@ -766,7 +771,7 @@ export class WebCrawler { const baseUrlSitemap = `${this.baseUrl}/sitemap.xml`; try { sitemapCount += await getLinksFromSitemap( - { sitemapUrl: baseUrlSitemap, urlsHandler, mode: "fire-engine", maxAge }, + { sitemapUrl: baseUrlSitemap, urlsHandler, mode: "fire-engine", maxAge, zeroDataRetention: this.zeroDataRetention }, this.logger, this.jobId, this.sitemapsHit, @@ -786,7 +791,7 @@ export class WebCrawler { // ignore 404 } else { sitemapCount += await getLinksFromSitemap( - { sitemapUrl: baseUrlSitemap, urlsHandler, mode: "fire-engine", maxAge }, + { sitemapUrl: baseUrlSitemap, urlsHandler, mode: "fire-engine", maxAge, zeroDataRetention: this.zeroDataRetention }, this.logger, this.jobId, this.sitemapsHit, diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.ts index a89c1e0fc..2cec2562d 100644 --- a/apps/api/src/scraper/WebScraper/sitemap.ts +++ b/apps/api/src/scraper/WebScraper/sitemap.ts @@ -13,11 +13,13 @@ export async function getLinksFromSitemap( urlsHandler, mode = "axios", maxAge = 0, + zeroDataRetention, }: { sitemapUrl: string; urlsHandler(urls: string[]): unknown; mode?: "axios" | "fire-engine"; maxAge?: number; + zeroDataRetention: boolean; }, logger: Logger, crawlId: string, @@ -52,6 +54,7 @@ export async function getLinksFromSitemap( v0DisableJsDom: true, abort, teamId: "sitemap", + zeroDataRetention, }, new CostTracking(), ); @@ -102,7 +105,7 @@ export async function getLinksFromSitemap( .map((sitemap) => sitemap.loc[0].trim()); const sitemapPromises: Promise[] = sitemapUrls.map((sitemapUrl) => - getLinksFromSitemap({ sitemapUrl, urlsHandler, mode }, logger, crawlId, sitemapsHit, abort, mock), + getLinksFromSitemap({ sitemapUrl, urlsHandler, mode, zeroDataRetention }, logger, crawlId, sitemapsHit, abort, mock), ); const results = await Promise.all(sitemapPromises); @@ -122,7 +125,7 @@ export async function getLinksFromSitemap( // Recursively fetch links from additional sitemaps const sitemapPromises = xmlSitemaps.map((sitemapUrl) => getLinksFromSitemap( - { sitemapUrl: sitemapUrl, urlsHandler, mode }, + { sitemapUrl: sitemapUrl, urlsHandler, mode, zeroDataRetention }, logger, crawlId, sitemapsHit, diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts index 66349fbc7..91fcda2ea 100644 --- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts +++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts @@ -230,8 +230,8 @@ export async function scrapeURLWithFireEngineChromeCDP( timeout, // TODO: better timeout logic disableSmartWaitCache: meta.internalOptions.disableSmartWaitCache, mobileProxy: meta.featureFlags.has("stealthProxy"), - saveScrapeResultToGCS: meta.internalOptions.saveScrapeResultToGCS, - // TODO: scrollXPaths + saveScrapeResultToGCS: !meta.internalOptions.zeroDataRetention && meta.internalOptions.saveScrapeResultToGCS, + zeroDataRetention: meta.internalOptions.zeroDataRetention, }; let response = await performFireEngineScrape( @@ -321,6 +321,8 @@ export async function scrapeURLWithFireEnginePlaywright( mobileProxy: meta.featureFlags.has("stealthProxy"), timeout, + saveScrapeResultToGCS: !meta.internalOptions.zeroDataRetention && meta.internalOptions.saveScrapeResultToGCS, + zeroDataRetention: meta.internalOptions.zeroDataRetention, }; let response = await performFireEngineScrape( @@ -384,6 +386,8 @@ export async function scrapeURLWithFireEngineTLSClient( mobileProxy: meta.featureFlags.has("stealthProxy"), timeout, + saveScrapeResultToGCS: !meta.internalOptions.zeroDataRetention && meta.internalOptions.saveScrapeResultToGCS, + zeroDataRetention: meta.internalOptions.zeroDataRetention, }; let response = await performFireEngineScrape( diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts index accee57e9..b43989360 100644 --- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts +++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts @@ -29,6 +29,8 @@ export type FireEngineScrapeRequestCommon = { mobileProxy?: boolean; // leave it undefined if user doesn't specify timeout?: number; + saveScrapeResultToGCS?: boolean; + zeroDataRetention?: boolean; }; export type FireEngineScrapeRequestChromeCDP = { @@ -38,7 +40,6 @@ export type FireEngineScrapeRequestChromeCDP = { blockMedia?: true; // cannot be false mobile?: boolean; disableSmartWaitCache?: boolean; - saveScrapeResultToGCS?: boolean; }; export type FireEngineScrapeRequestPlaywright = { diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts index 1f423f424..b286a5ea9 100644 --- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts +++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts @@ -7,6 +7,7 @@ import crypto from "crypto"; export async function sendDocumentToIndex(meta: Meta, document: Document) { const shouldCache = meta.options.storeInCache + && !meta.internalOptions.zeroDataRetention && meta.winnerEngine !== "index" && meta.winnerEngine !== "index;documents" && ( diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts index 7d14bd0dc..6649c735d 100644 --- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts +++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts @@ -140,13 +140,15 @@ async function scrapePDFWithRunPodMU( html: await marked.parse(result.markdown, { async: true }), }; - try { - await savePdfResultToCache(base64Content, processorResult); - } catch (error) { - meta.logger.warn("Error saving PDF to cache", { - error, - tempFilePath, - }); + if (!meta.internalOptions.zeroDataRetention) { + try { + await savePdfResultToCache(base64Content, processorResult); + } catch (error) { + meta.logger.warn("Error saving PDF to cache", { + error, + tempFilePath, + }); + } } return processorResult; diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.ts index 5afcca69c..b2ac60aaa 100644 --- a/apps/api/src/scraper/scrapeURL/error.ts +++ b/apps/api/src/scraper/scrapeURL/error.ts @@ -110,6 +110,12 @@ export class IndexMissError extends Error { } } +export class ZDRViolationError extends Error { + constructor(feature: string) { + super(`${feature} is not supported when using zeroDataRetention. Please contact support@firecrawl.com to unblock this feature.`); + } +} + export class PDFPrefetchFailed extends Error { constructor() { super("Failed to prefetch PDF that is protected by anti-bot. Please contact help@firecrawl.com"); diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts index 3493c015b..03391042b 100644 --- a/apps/api/src/scraper/scrapeURL/index.ts +++ b/apps/api/src/scraper/scrapeURL/index.ts @@ -25,6 +25,7 @@ import { PDFInsufficientTimeError, IndexMissError, DNSResolutionError, + ZDRViolationError, PDFPrefetchFailed, FEPageLoadFailed, } from "./error"; @@ -177,6 +178,7 @@ async function buildMetaObject( module: "ScrapeURL", scrapeId: id, scrapeURL: url, + zeroDataRetention: internalOptions.zeroDataRetention, }); const logs: any[] = []; @@ -216,6 +218,7 @@ export type InternalOptions = { saveScrapeResultToGCS?: boolean; // Passed along to fire-engine bypassBilling?: boolean; + zeroDataRetention?: boolean; }; export type EngineResultsTracker = { @@ -259,6 +262,24 @@ function safeguardCircularError(error: T): T { async function scrapeURLLoop(meta: Meta): Promise { meta.logger.info(`Scraping URL ${JSON.stringify(meta.rewrittenUrl ?? meta.url)}...`); + if (meta.internalOptions.zeroDataRetention) { + if (meta.featureFlags.has("screenshot")) { + throw new ZDRViolationError("screenshot"); + } + + if (meta.featureFlags.has("screenshot@fullScreen")) { + throw new ZDRViolationError("screenshot@fullScreen"); + } + + if (meta.options.actions && meta.options.actions.find(x => x.type === "screenshot")) { + throw new ZDRViolationError("screenshot action"); + } + + if (meta.options.actions && meta.options.actions.find(x => x.type === "pdf")) { + throw new ZDRViolationError("pdf action"); + } + } + // TODO: handle sitemap data, see WebScraper/index.ts:280 // TODO: ScrapeEvents diff --git a/apps/api/src/scraper/scrapeURL/transformers/agent.ts b/apps/api/src/scraper/scrapeURL/transformers/agent.ts index 7f98bee1d..0b8474056 100644 --- a/apps/api/src/scraper/scrapeURL/transformers/agent.ts +++ b/apps/api/src/scraper/scrapeURL/transformers/agent.ts @@ -12,6 +12,11 @@ export async function performAgent( document: Document, ): Promise { if (meta.options.agent?.prompt) { + if (meta.internalOptions.zeroDataRetention) { + document.warning = "Agent is not supported with zero data retention." + (document.warning ? " " + document.warning : "") + return document; + } + const url: string | undefined = document.url || document.metadata.sourceURL if (!url) { diff --git a/apps/api/src/scraper/scrapeURL/transformers/diff.ts b/apps/api/src/scraper/scrapeURL/transformers/diff.ts index 1d313ce94..e56f9b206 100644 --- a/apps/api/src/scraper/scrapeURL/transformers/diff.ts +++ b/apps/api/src/scraper/scrapeURL/transformers/diff.ts @@ -59,6 +59,11 @@ function compareExtractedData(previousData: any, currentData: any): any { export async function deriveDiff(meta: Meta, document: Document): Promise { if (meta.options.formats.includes("changeTracking")) { + if (meta.internalOptions.zeroDataRetention) { + document.warning = "Change tracking is not supported with zero data retention." + (document.warning ? " " + document.warning : "") + return document; + } + const start = Date.now(); const res = await supabase_service .rpc("diff_get_last_scrape_4", { diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts index 4ca536965..1b0983240 100644 --- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts +++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts @@ -653,6 +653,11 @@ export async function performLLMExtract( document: Document, ): Promise { if (meta.options.formats.includes("extract")) { + if (meta.internalOptions.zeroDataRetention) { + document.warning = "JSON mode is not supported with zero data retention." + (document.warning ? " " + document.warning : "") + return document; + } + // const originalOptions = meta.options.extract!; // let generationOptions = { ...originalOptions }; // Start with original options diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.ts index 5c46cfde1..cce85dfd2 100644 --- a/apps/api/src/services/logging/log_job.ts +++ b/apps/api/src/services/logging/log_job.ts @@ -22,7 +22,7 @@ function cleanOfNull(x: T): T { } export async function logJob(job: FirecrawlJob, force: boolean = false, bypassLogging: boolean = false) { - const logger = _logger.child({ + let logger = _logger.child({ module: "log_job", method: "logJob", ...(job.mode === "scrape" || job.mode === "single_urls" || job.mode === "single_url" ? ({ @@ -36,6 +36,12 @@ export async function logJob(job: FirecrawlJob, force: boolean = false, bypassLo }) : {}), }); + const zeroDataRetention = job.zeroDataRetention ?? false; + + logger = logger.child({ + zeroDataRetention, + }); + try { const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true"; if (!useDbAuthentication) { @@ -60,26 +66,27 @@ export async function logJob(job: FirecrawlJob, force: boolean = false, bypassLo const jobColumn = { job_id: job.job_id ? job.job_id : null, success: job.success, - message: job.message, + message: zeroDataRetention ? null : job.message, num_docs: job.num_docs, - docs: ((job.mode === "single_urls" || job.mode === "scrape") && process.env.GCS_BUCKET_NAME) ? null : cleanOfNull(job.docs), + docs: zeroDataRetention ? null : ((job.mode === "single_urls" || job.mode === "scrape") && process.env.GCS_BUCKET_NAME) ? null : cleanOfNull(job.docs), time_taken: job.time_taken, team_id: (job.team_id === "preview" || job.team_id?.startsWith("preview_"))? null : job.team_id, mode: job.mode, - url: job.url, - crawler_options: job.crawlerOptions, - page_options: job.scrapeOptions, - origin: job.origin, - integration: job.integration ?? null, + url: zeroDataRetention ? "" : job.url, + crawler_options: zeroDataRetention ? null : job.crawlerOptions, + page_options: zeroDataRetention ? null : job.scrapeOptions, + origin: zeroDataRetention ? null : job.origin, + integration: zeroDataRetention ? null : job.integration ?? null, num_tokens: job.num_tokens, retry: !!job.retry, crawl_id: job.crawl_id, tokens_billed: job.tokens_billed, is_migrated: true, - cost_tracking: job.cost_tracking, - pdf_num_pages: job.pdf_num_pages ?? null, + cost_tracking: zeroDataRetention ? null : job.cost_tracking, + pdf_num_pages: zeroDataRetention ? null : job.pdf_num_pages ?? null, credits_billed: job.credits_billed ?? null, - change_tracking_tag: job.change_tracking_tag ?? null, + change_tracking_tag: zeroDataRetention ? null : job.change_tracking_tag ?? null, + dr_clean_by: zeroDataRetention && job.crawl_id ? new Date(Date.now() + 1000 * 60 * 60 * 24).toISOString() : null, }; if (process.env.GCS_BUCKET_NAME) { @@ -145,21 +152,21 @@ export async function logJob(job: FirecrawlJob, force: boolean = false, bypassLo event: "job-logged", properties: { success: job.success, - message: job.message, + message: zeroDataRetention ? null: job.message, num_docs: job.num_docs, time_taken: job.time_taken, team_id: (job.team_id === "preview" || job.team_id?.startsWith("preview_"))? null : job.team_id, mode: job.mode, - url: job.url, - crawler_options: job.crawlerOptions, - page_options: job.scrapeOptions, - origin: job.origin, + url: zeroDataRetention ? "" : job.url, + crawler_options: zeroDataRetention ? null : job.crawlerOptions, + page_options: zeroDataRetention ? null : job.scrapeOptions, + origin: zeroDataRetention ? null : job.origin, num_tokens: job.num_tokens, retry: job.retry, tokens_billed: job.tokens_billed, - cost_tracking: job.cost_tracking, - pdf_num_pages: job.pdf_num_pages, - change_tracking_tag: job.change_tracking_tag ?? null, + cost_tracking: zeroDataRetention ? null : job.cost_tracking, + pdf_num_pages: zeroDataRetention ? null : job.pdf_num_pages, + change_tracking_tag: zeroDataRetention ? null : job.change_tracking_tag ?? null, }, }; if (job.mode !== "single_urls") { diff --git a/apps/api/src/services/logging/scrape_log.ts b/apps/api/src/services/logging/scrape_log.ts deleted file mode 100644 index 6e076330b..000000000 --- a/apps/api/src/services/logging/scrape_log.ts +++ /dev/null @@ -1,55 +0,0 @@ -import "dotenv/config"; -import { ScrapeLog } from "../../types"; -import { supabase_service } from "../supabase"; -import { PageOptions } from "../../lib/entities"; -import { logger } from "../../lib/logger"; -import { configDotenv } from "dotenv"; -configDotenv(); - -export async function logScrape( - scrapeLog: ScrapeLog, - pageOptions?: PageOptions, -) { - const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true"; - if (!useDbAuthentication) { - logger.debug("Skipping logging scrape to Supabase"); - return; - } - try { - // Only log jobs in production - // if (process.env.ENV !== "production") { - // return; - // } - // Redact any pages that have an authorization header - if ( - pageOptions && - pageOptions.headers && - pageOptions.headers["Authorization"] - ) { - scrapeLog.html = "REDACTED DUE TO AUTHORIZATION HEADER"; - } - - const { data, error } = await supabase_service.from("scrape_logs").insert([ - { - url: scrapeLog.url, - scraper: scrapeLog.scraper, - success: scrapeLog.success, - response_code: scrapeLog.response_code, - time_taken_seconds: scrapeLog.time_taken_seconds, - proxy: scrapeLog.proxy, - retried: scrapeLog.retried, - error_message: scrapeLog.error_message, - date_added: new Date().toISOString(), - html: "Removed to save db space", - ipv4_support: scrapeLog.ipv4_support, - ipv6_support: scrapeLog.ipv6_support, - }, - ]); - - if (error) { - logger.error(`Error logging proxy:\n${JSON.stringify(error)}`); - } - } catch (error) { - logger.error(`Error logging proxy:\n${JSON.stringify(error)}`); - } -} diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.ts index b7cef75f4..053ea61d9 100644 --- a/apps/api/src/services/queue-jobs.ts +++ b/apps/api/src/services/queue-jobs.ts @@ -15,7 +15,7 @@ import { logger } from "../lib/logger"; import { sendNotificationWithCustomDays } from './notification/email_notification'; import { shouldSendConcurrencyLimitNotification } from './notification/notification-check'; import { getACUC, getACUCTeam } from "../controllers/auth"; -import { getJobFromGCS } from "../lib/gcs-jobs"; +import { getJobFromGCS, removeJobFromGCS } from "../lib/gcs-jobs"; import { Document } from "../controllers/v1/types"; import { getCrawl } from "../lib/crawl-redis"; @@ -49,7 +49,7 @@ async function _addScrapeJobToConcurrencyQueue( } export async function _addScrapeJobToBullMQ( - webScraperOptions: any, + webScraperOptions: WebScraperOptions, options: any, jobId: string, jobPriority: number, @@ -76,7 +76,7 @@ export async function _addScrapeJobToBullMQ( } async function addScrapeJobRaw( - webScraperOptions: any, + webScraperOptions: WebScraperOptions, options: any, jobId: string, jobPriority: number, @@ -399,7 +399,8 @@ export function waitForJob( if (state === "completed") { clearInterval(int); let doc: Document; - doc = (await getScrapeQueue().getJob(jobId))!.returnvalue; + const job = (await getScrapeQueue().getJob(jobId))!; + doc = job.returnvalue; if (!doc) { const docs = await getJobFromGCS(jobId); @@ -407,6 +408,10 @@ export function waitForJob( throw new Error("Job not found in GCS"); } doc = docs[0]; + + if (job.data?.internalOptions?.zeroDataRetention) { + await removeJobFromGCS(jobId); + } } resolve(doc); diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts index c8aacdbab..8c6b74cbc 100644 --- a/apps/api/src/services/queue-worker.ts +++ b/apps/api/src/services/queue-worker.ts @@ -29,11 +29,9 @@ import { finishCrawlKickoff, generateURLPermutations, getCrawl, - getCrawlJobCount, getCrawlJobs, getDoneJobsOrderedLength, lockURL, - lockURLs, lockURLsIndividually, normalizeURL, saveCrawl, @@ -54,7 +52,7 @@ import { } from "../lib/concurrency-limit"; import { isUrlBlocked } from "../scraper/WebScraper/utils/blocklist"; import { BLOCKLISTED_URL_MESSAGE } from "../lib/strings"; -import { Document } from "../controllers/v1/types"; +import { Document, TeamFlags } from "../controllers/v1/types"; import { ExtractResult, performExtraction, @@ -119,9 +117,10 @@ async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) { jobId: job.id, scrapeId: job.id, crawlId: job.data.crawl_id, + zeroDataRetention: sc.internalOptions.zeroDataRetention, }); - if (await finishCrawlPre(job.data.crawl_id)) { + if (await finishCrawlPre(job.data.crawl_id, logger)) { logger.info("Crawl is pre-finished, checking if we need to add more jobs"); if ( job.data.crawlerOptions && @@ -213,6 +212,7 @@ async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) { sitemapped: true, webhook: job.data.webhook, v1: job.data.v1, + zeroDataRetention: job.data.zeroDataRetention, }, opts: { jobId: uuid, @@ -232,6 +232,7 @@ async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) { await addCrawlJobs( job.data.crawl_id, lockedJobs.map((x) => x.opts.jobId), + logger, ); await addScrapeJobs(lockedJobs); @@ -249,7 +250,7 @@ async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) { } logger.info("Finishing crawl"); - await finishCrawl(job.data.crawl_id); + await finishCrawl(job.data.crawl_id, logger); if (!job.data.v1) { const jobIDs = await getCrawlJobs(job.data.crawl_id); @@ -286,6 +287,7 @@ async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) { crawlerOptions: sc.crawlerOptions, origin: job.data.origin, integration: job.data.integration, + zeroDataRetention: job.data.zeroDataRetention, }, false, job.data.internalOptions?.bypassBilling ?? false); const data = { @@ -354,6 +356,7 @@ async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) { origin: job.data.origin, integration: job.data.integration, credits_billed, + zeroDataRetention: job.data.zeroDataRetention, }, true, job.data.internalOptions?.bypassBilling ?? false, @@ -384,6 +387,7 @@ const processJobInternal = async (token: string, job: Job & { id: string }) => { jobId: job.id, scrapeId: job.id, crawlId: job.data?.crawl_id ?? undefined, + zeroDataRetention: job.data?.zeroDataRetention ?? false, }); const extendLockInterval = setInterval(async () => { @@ -874,6 +878,7 @@ async function processKickoffJob(job: Job & { id: string }, token: string) { scrapeId: job.id, crawlId: job.data?.crawl_id ?? undefined, teamId: job.data?.team_id ?? undefined, + zeroDataRetention: job.data.zeroDataRetention ?? false, }); try { @@ -898,6 +903,7 @@ async function processKickoffJob(job: Job & { id: string }, token: string) { webhook: job.data.webhook, v1: job.data.v1, isCrawlSourceScrape: true, + zeroDataRetention: job.data.zeroDataRetention, }, { priority: 15, @@ -905,7 +911,7 @@ async function processKickoffJob(job: Job & { id: string }, token: string) { jobId, ); logger.debug("Adding scrape job to BullMQ...", { jobId }); - await addCrawlJob(job.data.crawl_id, jobId); + await addCrawlJob(job.data.crawl_id, jobId, logger); if (job.data.webhook) { logger.debug("Calling webhook with crawl.started...", { @@ -953,6 +959,7 @@ async function processKickoffJob(job: Job & { id: string }, token: string) { sitemapped: true, webhook: job.data.webhook, v1: job.data.v1, + zeroDataRetention: job.data.zeroDataRetention, }, opts: { jobId: uuid, @@ -974,6 +981,7 @@ async function processKickoffJob(job: Job & { id: string }, token: string) { await addCrawlJobs( job.data.crawl_id, lockedJobs.map((x) => x.opts.jobId), + logger, ); logger.debug("Adding scrape jobs to BullMQ..."); await addScrapeJobs(lockedJobs); @@ -1015,6 +1023,7 @@ async function processKickoffJob(job: Job & { id: string }, token: string) { sitemapped: true, webhook: job.data.webhook, v1: job.data.v1, + zeroDataRetention: job.data.zeroDataRetention, }, opts: { jobId: uuid, @@ -1036,6 +1045,7 @@ async function processKickoffJob(job: Job & { id: string }, token: string) { await addCrawlJobs( job.data.crawl_id, lockedJobs.map((x) => x.opts.jobId), + logger, ); logger.debug("Adding scrape jobs to BullMQ..."); await addScrapeJobs(lockedJobs); @@ -1058,11 +1068,11 @@ async function processKickoffJob(job: Job & { id: string }, token: string) { } } -async function billScrapeJob(job: Job & { id: string }, document: Document | null, logger: Logger, costTracking: CostTracking) { +async function billScrapeJob(job: Job & { id: string }, document: Document | null, logger: Logger, costTracking: CostTracking, flags: TeamFlags) { let creditsToBeBilled: number | null = null; if (job.data.is_scrape !== true && !job.data.internalOptions?.bypassBilling) { - creditsToBeBilled = await calculateCreditsToBeBilled(job.data.scrapeOptions, document, costTracking); + creditsToBeBilled = await calculateCreditsToBeBilled(job.data.scrapeOptions, job.data.internalOptions, document, costTracking, flags); if ( job.data.team_id !== process.env.BACKGROUND_INDEX_TEAM_ID! && @@ -1119,6 +1129,7 @@ async function processJob(job: Job & { id: string }, token: string) { scrapeId: job.id, crawlId: job.data?.crawl_id ?? undefined, teamId: job.data?.team_id ?? undefined, + zeroDataRetention: job.data?.zeroDataRetention ?? false, }); logger.info(`🐂 Worker taking job ${job.id}`, { url: job.data.url }); const start = job.data.startTime ?? Date.now(); @@ -1324,13 +1335,14 @@ async function processJob(job: Job & { id: string }, token: string) { crawl_id: job.data.crawl_id, webhook: job.data.webhook, v1: job.data.v1, + zeroDataRetention: job.data.zeroDataRetention, }, {}, jobId, jobPriority, ); - await addCrawlJob(job.data.crawl_id, jobId); + await addCrawlJob(job.data.crawl_id, jobId, logger); logger.debug("Added job for URL " + JSON.stringify(link), { jobPriority, url: link, @@ -1366,7 +1378,7 @@ async function processJob(job: Job & { id: string }, token: string) { throw new Error("timeout"); } - const credits_billed = await billScrapeJob(job, doc, logger, costTracking); + const credits_billed = await billScrapeJob(job, doc, logger, costTracking, (await getACUCTeam(job.data.team_id))?.flags ?? null); doc.metadata.creditsUsed = credits_billed ?? undefined; @@ -1390,6 +1402,7 @@ async function processJob(job: Job & { id: string }, token: string) { pdf_num_pages: doc.metadata.numPages, credits_billed, change_tracking_tag: job.data.scrapeOptions.changeTrackingOptions?.tag ?? null, + zeroDataRetention: job.data.zeroDataRetention, }, true, job.data.internalOptions?.bypassBilling ?? false, @@ -1411,7 +1424,7 @@ async function processJob(job: Job & { id: string }, token: string) { } logger.debug("Declaring job as done..."); - await addCrawlJobDone(job.data.crawl_id, job.id, true); + await addCrawlJobDone(job.data.crawl_id, job.id, true, logger); await finishCrawlIfNeeded(job, sc); } else { @@ -1421,7 +1434,7 @@ async function processJob(job: Job & { id: string }, token: string) { throw new Error("timeout"); } - const credits_billed = await billScrapeJob(job, doc, logger, costTracking); + const credits_billed = await billScrapeJob(job, doc, logger, costTracking, (await getACUCTeam(job.data.team_id))?.flags ?? null); doc.metadata.creditsUsed = credits_billed ?? undefined; @@ -1443,6 +1456,7 @@ async function processJob(job: Job & { id: string }, token: string) { pdf_num_pages: doc.metadata.numPages, credits_billed, change_tracking_tag: job.data.scrapeOptions.changeTrackingOptions?.tag ?? null, + zeroDataRetention: job.data.zeroDataRetention, }, false, job.data.internalOptions?.bypassBilling ?? false); } @@ -1453,7 +1467,7 @@ async function processJob(job: Job & { id: string }, token: string) { const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl; logger.debug("Declaring job as done..."); - await addCrawlJobDone(job.data.crawl_id, job.id, false); + await addCrawlJobDone(job.data.crawl_id, job.id, false, logger); await redisEvictConnection.srem( "crawl:" + job.data.crawl_id + ":visited_unique", normalizeURL(job.data.url, sc), @@ -1520,7 +1534,7 @@ async function processJob(job: Job & { id: string }, token: string) { const end = Date.now(); const timeTakenInSeconds = (end - start) / 1000; - const credits_billed = await billScrapeJob(job, null, logger, costTracking); + const credits_billed = await billScrapeJob(job, null, logger, costTracking, (await getACUCTeam(job.data.team_id))?.flags ?? null); logger.debug("Logging job to DB..."); await logJob( @@ -1545,6 +1559,7 @@ async function processJob(job: Job & { id: string }, token: string) { crawl_id: job.data.crawl_id, cost_tracking: costTracking, credits_billed, + zeroDataRetention: job.data.zeroDataRetention, }, true, job.data.internalOptions?.bypassBilling ?? false, diff --git a/apps/api/src/types.ts b/apps/api/src/types.ts index f1625ee78..c42720c1c 100644 --- a/apps/api/src/types.ts +++ b/apps/api/src/types.ts @@ -10,7 +10,7 @@ import { ExtractorOptions, Document } from "./lib/entities"; import { InternalOptions } from "./scraper/scrapeURL"; import type { CostTracking } from "./lib/extract/extraction-service"; -type Mode = "crawl" | "single_urls" | "sitemap"; +type Mode = "crawl" | "single_urls" | "sitemap" | "kickoff"; export { Mode }; @@ -54,6 +54,11 @@ export interface WebScraperOptions { isCrawlSourceScrape?: boolean; from_extract?: boolean; startTime?: number; + + zeroDataRetention: boolean; + sentry?: any; + is_extract?: boolean; + concurrencyLimited?: boolean; } export interface RunWebScraperParams { @@ -105,6 +110,9 @@ export interface FirecrawlJob { pdf_num_pages?: number; credits_billed?: number | null; change_tracking_tag?: string | null; + dr_clean_by?: string | null; + + zeroDataRetention: boolean; } export interface FirecrawlScrapeResponse { diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.ts index 3bebe6424..139f7a919 100644 --- a/apps/js-sdk/firecrawl/src/index.ts +++ b/apps/js-sdk/firecrawl/src/index.ts @@ -176,6 +176,7 @@ export interface ScrapeParams ScrapeResponse[Any]: """ Scrape and extract content from a URL. @@ -504,6 +505,7 @@ class FirecrawlApp: json_options (Optional[JsonConfig]): JSON extraction settings actions (Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction, PDFAction]]]): Actions to perform change_tracking_options (Optional[ChangeTrackingOptions]): Change tracking settings + zero_data_retention (Optional[bool]): Whether to delete data after scrape is done Returns: @@ -711,6 +713,7 @@ class FirecrawlApp: delay: Optional[int] = None, allow_subdomains: Optional[bool] = None, max_concurrency: Optional[int] = None, + zero_data_retention: Optional[bool] = None, poll_interval: Optional[int] = 2, idempotency_key: Optional[str] = None, **kwargs @@ -737,6 +740,7 @@ class FirecrawlApp: delay (Optional[int]): Delay in seconds between scrapes allow_subdomains (Optional[bool]): Follow subdomains max_concurrency (Optional[int]): Maximum number of concurrent scrapes + zero_data_retention (Optional[bool]): Whether to delete data after 24 hours poll_interval (Optional[int]): Seconds between status checks (default: 2) idempotency_key (Optional[str]): Unique key to prevent duplicate requests **kwargs: Additional parameters to pass to the API @@ -790,7 +794,8 @@ class FirecrawlApp: crawl_params['allowSubdomains'] = allow_subdomains if max_concurrency is not None: crawl_params['maxConcurrency'] = max_concurrency - + if zero_data_retention is not None: + crawl_params['zeroDataRetention'] = zero_data_retention # Add any additional kwargs crawl_params.update(kwargs) @@ -834,6 +839,7 @@ class FirecrawlApp: delay: Optional[int] = None, allow_subdomains: Optional[bool] = None, max_concurrency: Optional[int] = None, + zero_data_retention: Optional[bool] = None, idempotency_key: Optional[str] = None, **kwargs ) -> CrawlResponse: @@ -859,6 +865,7 @@ class FirecrawlApp: delay (Optional[int]): Delay in seconds between scrapes allow_subdomains (Optional[bool]): Follow subdomains max_concurrency (Optional[int]): Maximum number of concurrent scrapes + zero_data_retention (Optional[bool]): Whether to delete data after 24 hours idempotency_key (Optional[str]): Unique key to prevent duplicate requests **kwargs: Additional parameters to pass to the API @@ -912,7 +919,8 @@ class FirecrawlApp: crawl_params['allowSubdomains'] = allow_subdomains if max_concurrency is not None: crawl_params['maxConcurrency'] = max_concurrency - + if zero_data_retention is not None: + crawl_params['zeroDataRetention'] = zero_data_retention # Add any additional kwargs crawl_params.update(kwargs) @@ -1092,6 +1100,7 @@ class FirecrawlApp: delay: Optional[int] = None, allow_subdomains: Optional[bool] = None, max_concurrency: Optional[int] = None, + zero_data_retention: Optional[bool] = None, idempotency_key: Optional[str] = None, **kwargs ) -> 'CrawlWatcher': @@ -1117,6 +1126,7 @@ class FirecrawlApp: delay (Optional[int]): Delay in seconds between scrapes allow_subdomains (Optional[bool]): Follow subdomains max_concurrency (Optional[int]): Maximum number of concurrent scrapes + zero_data_retention (Optional[bool]): Whether to delete data after 24 hours idempotency_key (Optional[str]): Unique key to prevent duplicate requests **kwargs: Additional parameters to pass to the API @@ -1144,6 +1154,7 @@ class FirecrawlApp: delay=delay, allow_subdomains=allow_subdomains, max_concurrency=max_concurrency, + zero_data_retention=zero_data_retention, idempotency_key=idempotency_key, **kwargs ) @@ -1261,6 +1272,7 @@ class FirecrawlApp: agent: Optional[AgentOptions] = None, poll_interval: Optional[int] = 2, max_concurrency: Optional[int] = None, + zero_data_retention: Optional[bool] = None, idempotency_key: Optional[str] = None, **kwargs ) -> BatchScrapeStatusResponse: @@ -1424,6 +1436,7 @@ class FirecrawlApp: actions (Optional[List[Union]]): Actions to perform agent (Optional[AgentOptions]): Agent configuration max_concurrency (Optional[int]): Maximum number of concurrent scrapes + zero_data_retention (Optional[bool]): Whether to delete data after 24 hours idempotency_key (Optional[str]): Unique key to prevent duplicate requests **kwargs: Additional parameters to pass to the API @@ -1485,6 +1498,8 @@ class FirecrawlApp: scrape_params['agent'] = agent.dict(exclude_none=True) if max_concurrency is not None: scrape_params['maxConcurrency'] = max_concurrency + if zero_data_retention is not None: + scrape_params['zeroDataRetention'] = zero_data_retention # Add any additional kwargs scrape_params.update(kwargs) @@ -1534,6 +1549,7 @@ class FirecrawlApp: actions: Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction, PDFAction]]] = None, agent: Optional[AgentOptions] = None, max_concurrency: Optional[int] = None, + zero_data_retention: Optional[bool] = None, idempotency_key: Optional[str] = None, **kwargs ) -> 'CrawlWatcher': @@ -1560,6 +1576,7 @@ class FirecrawlApp: actions (Optional[List[Union]]): Actions to perform agent (Optional[AgentOptions]): Agent configuration max_concurrency (Optional[int]): Maximum number of concurrent scrapes + zero_data_retention (Optional[bool]): Whether to delete data after 24 hours idempotency_key (Optional[str]): Unique key to prevent duplicate requests **kwargs: Additional parameters to pass to the API @@ -1617,6 +1634,8 @@ class FirecrawlApp: scrape_params['agent'] = agent.dict(exclude_none=True) if max_concurrency is not None: scrape_params['maxConcurrency'] = max_concurrency + if zero_data_retention is not None: + scrape_params['zeroDataRetention'] = zero_data_retention # Add any additional kwargs scrape_params.update(kwargs) @@ -3204,6 +3223,7 @@ class AsyncFirecrawlApp(FirecrawlApp): json_options: Optional[JsonConfig] = None, actions: Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction, PDFAction]]] = None, agent: Optional[AgentOptions] = None, + zero_data_retention: Optional[bool] = None, idempotency_key: Optional[str] = None, **kwargs ) -> BatchScrapeResponse: @@ -3229,6 +3249,7 @@ class AsyncFirecrawlApp(FirecrawlApp): json_options (Optional[JsonConfig]): JSON extraction config actions (Optional[List[Union]]): Actions to perform agent (Optional[AgentOptions]): Agent configuration + zero_data_retention (Optional[bool]): Whether to delete data after 24 hours idempotency_key (Optional[str]): Unique key to prevent duplicate requests **kwargs: Additional parameters to pass to the API @@ -3288,7 +3309,9 @@ class AsyncFirecrawlApp(FirecrawlApp): scrape_params['actions'] = [action.dict(exclude_none=True) for action in actions] if agent is not None: scrape_params['agent'] = agent.dict(exclude_none=True) - + if zero_data_retention is not None: + scrape_params['zeroDataRetention'] = zero_data_retention + # Add any additional kwargs scrape_params.update(kwargs)