feat(api/scrapeURL/formats/json): support ZDR (#4444)

apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts

@@ -39,7 +39,7 @@ const commonReasoningPromptProperties = {
   },
   smartscrape_prompt: {
     type: ["string", "null"],
-    description: `A clear, outcome-focused prompt describing what information to find on the page. 
+    description: `A clear, outcome-focused prompt describing what information to find on the page.
       Example: "Find the product specifications in the expandable section" rather than "Click the button to reveal product specs".
       Used by the smart scraping agent to determine what actions to take.
       Dont mention anything about extraction, smartscrape just returns page content.`,
@@ -289,6 +289,7 @@ export async function extractData({
           ? metadata.functionId + "/extractData"
           : "extractData",
       },
+      extractOptions.zeroDataRetention,
     );
     schema = genRes.extract;
   }
@@ -372,6 +373,7 @@ export async function extractData({
           logger,
           costTracking: extractOptions.costTrackingOptions.costTracking,
           metadata,
+          zeroDataRetention: !!extractOptions.zeroDataRetention,
         })
       : Promise.resolve(),
     generateCompletions({
@@ -429,6 +431,13 @@ export async function extractData({
     });
 
     if (useAgent && extract?.shouldUseSmartscrape) {
+      // technically this should be checked upstream but might as well add another guard - Mogery
+      if (extractOptions.zeroDataRetention) {
+        throw new Error(
+          "JSON mode with agent is not supported with Zero Data Retention.",
+        );
+      }
+
       let smartscrapeResults: SmartScrapeResult[];
       if (isSingleUrl) {
         smartscrapeResults = [
@@ -495,6 +504,7 @@ export async function extractData({
               logger,
               costTracking: extractOptions.costTrackingOptions.costTracking,
               metadata,
+              zeroDataRetention: !!extractOptions.zeroDataRetention,
             });
           }
 

apps/api/src/scraper/scrapeURL/lib/promptInjectionGuard.ts

@@ -11,7 +11,10 @@ import {
   UnsupportedFunctionalityError,
 } from "ai";
 import { getModel } from "../../../lib/generic-ai";
-import { CostLimitExceededError, CostTracking } from "../../../lib/cost-tracking";
+import {
+  CostLimitExceededError,
+  CostTracking,
+} from "../../../lib/cost-tracking";
 import { calculateCost } from "../transformers/llmExtract";
 import { PromptInjectionDetectedError } from "../error";
 import { captureExceptionWithZdrCheck } from "../../../services/sentry";
@@ -88,6 +91,7 @@ async function classifyChunk(
   logger: Logger,
   costTracking: CostTracking,
   metadata: { teamId: string; functionId?: string },
+  zeroDataRetention: boolean,
 ): Promise<void> {
   const tagName = `untrusted_page_content_${crypto.randomUUID()}`;
 
@@ -104,7 +108,7 @@ async function classifyChunk(
         },
       },
       experimental_telemetry: {
-        isEnabled: true,
+        isEnabled: !zeroDataRetention,
         functionId: metadata.functionId
           ? metadata.functionId + "/promptInjectionGuard"
           : "promptInjectionGuard",
@@ -176,11 +180,13 @@ export async function checkForPromptInjection({
   logger,
   costTracking,
   metadata,
+  zeroDataRetention,
 }: {
   markdown: string | undefined;
   logger: Logger;
   costTracking: CostTracking;
   metadata: { teamId: string; functionId?: string };
+  zeroDataRetention: boolean;
 }): Promise<void> {
   if (!markdown || markdown.trim().length === 0) {
     return;
@@ -201,7 +207,15 @@ export async function checkForPromptInjection({
     const batch = chunks.slice(i, i + GUARD_CONCURRENCY_LIMIT);
     await Promise.all(
       batch.map(chunk =>
-        classifyChunk(chunk, model, modelId, logger, costTracking, metadata),
+        classifyChunk(
+          chunk,
+          model,
+          modelId,
+          logger,
+          costTracking,
+          metadata,
+          zeroDataRetention,
+        ),
       ),
     );
   }

apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts

@@ -308,6 +308,7 @@ export type GenerateCompletionsOptions = {
     deepResearchId?: string;
     llmsTxtId?: string;
   };
+  zeroDataRetention?: boolean;
 };
 export async function generateCompletions({
   logger,
@@ -321,6 +322,7 @@ export async function generateCompletions({
   retryModel = getModel("gpt-4.1-mini", "openai"),
   costTrackingOptions,
   metadata,
+  zeroDataRetention = false,
 }: GenerateCompletionsOptions): Promise<{
   extract: any;
   numTokens: number;
@@ -371,7 +373,7 @@ export async function generateCompletions({
             },
           },
           experimental_telemetry: {
-            isEnabled: true,
+            isEnabled: !zeroDataRetention,
             functionId: metadata.functionId
               ? metadata.functionId + "/generateText"
               : "generateText",
@@ -477,7 +479,7 @@ export async function generateCompletions({
                 },
               },
               experimental_telemetry: {
-                isEnabled: true,
+                isEnabled: !zeroDataRetention,
                 functionId: metadata.functionId
                   ? metadata.functionId + "/generateText"
                   : "generateText",
@@ -648,7 +650,7 @@ export async function generateCompletions({
               },
             },
             experimental_telemetry: {
-              isEnabled: true,
+              isEnabled: !zeroDataRetention,
               functionId: metadata.functionId
                 ? metadata.functionId + "/repairText"
                 : "repairText",
@@ -744,7 +746,7 @@ export async function generateCompletions({
         },
       }),
       experimental_telemetry: {
-        isEnabled: true,
+        isEnabled: !zeroDataRetention,
         functionId: metadata.functionId,
         metadata: {
           teamId: metadata.teamId,
@@ -970,9 +972,17 @@ export async function performLLMExtract(
   }
 
   if (jsonFormat) {
-    if (meta.internalOptions.zeroDataRetention) {
+    const useAgent = isAgentExtractModelValid(
+      meta.internalOptions.v1JSONAgent?.model,
+    );
+
+    // NOTE: ZDR deny policy on JSON mode has been lightened to only
+    // disallow agent/smart scrape (deprecated). This now binds us to
+    // only use model providers in JSON mode that we have ZDR agreements
+    // with. (openai certified yes.) WE MUST OBEY THIS! - Mogery
+    if (useAgent && meta.internalOptions.zeroDataRetention) {
       document.warning =
-        "JSON mode is not supported with zero data retention." +
+        "JSON mode with agent is not supported with zero data retention." +
         (document.warning ? " " + document.warning : "");
       return document;
     }
@@ -1004,15 +1014,14 @@ export async function performLLMExtract(
         functionId: "performLLMExtract",
         scrapeId: meta.id,
       },
+      zeroDataRetention: meta.internalOptions.zeroDataRetention,
     };
 
     const { extractedDataArray, warning, costLimitExceededTokenUsage } =
       await extractData({
         extractOptions: generationOptions,
         urls: [meta.rewrittenUrl ?? meta.url],
-        useAgent: isAgentExtractModelValid(
-          meta.internalOptions.v1JSONAgent?.model,
-        ),
+        useAgent,
         scrapeId: meta.id,
         metadata: {
           teamId: meta.internalOptions.teamId,
@@ -1454,6 +1463,7 @@ export async function generateSchemaFromPrompt(
     extractId?: string;
     scrapeId?: string;
   },
+  zeroDataRetention = false,
 ): Promise<{ extract: any }> {
   const model = getModel("gpt-4o-mini", "openai");
   const retryModel = getModel("gpt-4.1-mini", "openai");
@@ -1511,6 +1521,7 @@ Return a valid JSON schema object with properties that would capture the informa
             ? metadata.functionId + "/generateSchemaFromPrompt"
             : "generateSchemaFromPrompt",
         },
+        zeroDataRetention,
       });
 
       return { extract };