diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts index dd7c0c168..8f4a75762 100644 --- a/apps/api/src/controllers/v2/types.ts +++ b/apps/api/src/controllers/v2/types.ts @@ -72,13 +72,39 @@ function normalizeSchemaForOpenAI(schema: any): any { function normalizeObject(obj: any): any { if (typeof obj !== "object" || obj === null) return obj; - if (Array.isArray(obj)) return obj; + if (Array.isArray(obj)) { + return obj.map(item => normalizeObject(item)); + } if (visited.has(obj)) return obj; visited.add(obj); const normalized = { ...obj }; + // Handle $ref recursion - preserve as-is for OpenAI compatibility + if (normalized.hasOwnProperty("$ref")) { + return normalized; + } + + if (normalized.hasOwnProperty("$defs")) { + const { $defs, ...rest } = normalized; + const processedRest = {}; + + for (const [key, value] of Object.entries(rest)) { + if ( + typeof value === "object" && + value !== null && + !value.hasOwnProperty("$ref") + ) { + processedRest[key] = normalizeObject(value); + } else { + processedRest[key] = value; + } + } + + return { ...processedRest, $defs }; + } + if ( normalized.type === "object" && normalized.hasOwnProperty("properties") && @@ -111,7 +137,11 @@ function normalizeSchemaForOpenAI(schema: any): any { } for (const [key, value] of Object.entries(normalized)) { - if (typeof value === "object" && value !== null) { + if ( + typeof value === "object" && + value !== null && + !value.hasOwnProperty("$ref") + ) { normalized[key] = normalizeObject(value); } } @@ -135,6 +165,10 @@ function validateSchemaForOpenAI(schema: any): boolean { if (visited.has(obj)) return false; visited.add(obj); + if (obj.hasOwnProperty("$ref")) { + return false; + } + if ( obj.type === "object" && !obj.hasOwnProperty("properties") && @@ -145,7 +179,11 @@ function validateSchemaForOpenAI(schema: any): boolean { } for (const value of Object.values(obj)) { - if (typeof value === "object" && value !== null) { + if ( + typeof value === "object" && + value !== null && + !value.hasOwnProperty("$ref") + ) { if (hasInvalidStructure(value)) return true; } } @@ -156,7 +194,7 @@ function validateSchemaForOpenAI(schema: any): boolean { } const OPENAI_SCHEMA_ERROR_MESSAGE = - "Schema contains invalid structure for OpenAI: object type with no 'properties' defined but 'additionalProperties: true' (schema-less dictionary not supported by OpenAI). Please define specific properties for your object."; + "Schema contains invalid structure for OpenAI: object type with no 'properties' defined but 'additionalProperties: true' (schema-less dictionary not supported by OpenAI). Please define specific properties for your object. Note: Recursive schemas using '$ref' are supported."; const ACTIONS_MAX_WAIT_TIME = 60; const MAX_ACTIONS = 50; diff --git a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts index 3088a46c2..d4b23c8bd 100644 --- a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts +++ b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts @@ -120,29 +120,114 @@ function prepareSmartScrapeSchema( return { schemaToUse: wrappedSchema }; } +const hasRecursiveRefs = (schema: any, defs: any): boolean => { + if (!defs || typeof defs !== "object") return false; + + for (const [defName, defValue] of Object.entries(defs)) { + if (containsRecursiveRef(defValue, defName, defs)) { + return true; + } + } + + return false; +}; + +const containsRecursiveRef = ( + obj: any, + targetDefName: string, + defs: any, + visited = new Set(), +): boolean => { + if (!obj || typeof obj !== "object") return false; + + const objKey = JSON.stringify(obj); + if (visited.has(objKey)) return false; + visited.add(objKey); + + if (obj.$ref && typeof obj.$ref === "string") { + const refPath = obj.$ref.split("/"); + if (refPath[0] === "#" && refPath[1] === "$defs") { + const defName = refPath[refPath.length - 1]; + if (defName === targetDefName) { + visited.delete(objKey); + return true; + } + if (defs[defName]) { + const isRecursive = containsRecursiveRef( + defs[defName], + targetDefName, + defs, + visited, + ); + visited.delete(objKey); + return isRecursive; + } + } + } + + if (Array.isArray(obj)) { + for (const item of obj) { + if (containsRecursiveRef(item, targetDefName, defs, visited)) { + visited.delete(objKey); + return true; + } + } + } else { + for (const value of Object.values(obj)) { + if (containsRecursiveRef(value, targetDefName, defs, visited)) { + visited.delete(objKey); + return true; + } + } + } + + visited.delete(objKey); + return false; +}; + // Resolve all $defs references in the schema -const resolveRefs = (obj: any, defs: any): any => { - if (!obj || typeof obj !== "object") return obj; +const resolveRefs = ( + obj: any, + defs: any, + visited = new WeakSet(), + depth = 0, +): any => { + if (!obj || typeof obj !== "object" || depth > 10) return obj; + + const objString = JSON.stringify(obj); + if (objString.includes("#/$defs/") && objString.includes('"$ref"')) { + console.warn( + "resolveRefs: Detected recursive schema pattern, aborting to prevent infinite recursion", + ); + return obj; + } + + // Prevent infinite recursion + if (visited.has(obj)) return obj; + visited.add(obj); if (obj.$ref && typeof obj.$ref === "string") { // Handle $ref references const refPath = obj.$ref.split("/"); if (refPath[0] === "#" && refPath[1] === "$defs") { const defName = refPath[refPath.length - 1]; - return resolveRefs({ ...defs[defName] }, defs); + if (defs[defName]) { + return resolveRefs({ ...defs[defName] }, defs, visited, depth + 1); + } } + return obj; // Return original if ref can't be resolved } // Handle arrays if (Array.isArray(obj)) { - return obj.map(item => resolveRefs(item, defs)); + return obj.map(item => resolveRefs(item, defs, visited, depth + 1)); } // Handle objects const resolved: any = {}; for (const [key, value] of Object.entries(obj)) { if (key === "$defs") continue; - resolved[key] = resolveRefs(value, defs); + resolved[key] = resolveRefs(value, defs, visited, depth + 1); } return resolved; }; @@ -193,11 +278,31 @@ export async function extractData({ if (schema) { const defs = schema.$defs || {}; - schema = resolveRefs(schema, defs); - delete schema.$defs; - logger.info("Resolved schema refs", { - schema, - }); + const schemaString = JSON.stringify(schema); + const hasAnyRefs = + schema.$defs || + schemaString.includes('"$ref"') || + schemaString.includes("#/$defs/"); + + if (hasAnyRefs) { + logger.info( + "Detected schema with references, preserving as-is to avoid recursion", + { + hasDefsProperty: !!schema.$defs, + hasRefInString: schemaString.includes('"$ref"'), + hasRefPathInString: schemaString.includes("#/$defs/"), + }, + ); + } else { + logger.info("No recursive references detected, resolving refs", { + schema, + }); + schema = resolveRefs(schema, defs); + delete schema.$defs; + logger.info("Resolved schema refs", { + schema, + }); + } } const { schemaToUse } = prepareSmartScrapeSchema(schema, logger, isSingleUrl); diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts index e57963882..27cb616ef 100644 --- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts +++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts @@ -25,6 +25,46 @@ import { extractData } from "../lib/extractSmartScrape"; import { CostTracking } from "../../../lib/cost-tracking"; import { isAgentExtractModelValid } from "../../../controllers/v1/types"; import { hasFormatOfType } from "../../../lib/format-utils"; + +// Smart model selection based on schema +function detectRecursiveSchema(schema: any): boolean { + if (!schema || typeof schema !== "object") return false; + + const schemaString = JSON.stringify(schema); + const hasRefs = + schemaString.includes('"$ref"') || + schemaString.includes("#/$defs/") || + schemaString.includes("#/definitions/"); + const hasDefs = !!(schema.$defs || schema.definitions); + + return hasRefs || hasDefs; +} + +function selectModelForSchema(schema?: any): { + modelName: string; + reason: string; +} { + if (!schema) { + return { modelName: "gpt-4o-mini", reason: "no_schema" }; + } + + const isRecursive = detectRecursiveSchema(schema); + + if (isRecursive) { + logger.info(`Model: gpt-4o | hasRef: true`); + return { + modelName: "gpt-4o", + reason: "recursive_schema_detected", + }; + } + + logger.info(`Model: gpt-4o-mini | hasRef: false`); + return { + modelName: "gpt-4o-mini", + reason: "simple_schema", + }; +} + // TODO: fix this, it's horrible type LanguageModelV1ProviderMetadata = { anthropic?: { @@ -890,6 +930,8 @@ export async function performLLMExtract( // let generationOptions = { ...originalOptions }; // Start with original options + const modelSelection = selectModelForSchema(jsonFormat.schema); + const generationOptions: GenerateCompletionsOptions = { logger: meta.logger.child({ method: "performLLMExtract/generateCompletions", @@ -897,7 +939,7 @@ export async function performLLMExtract( options: jsonFormat, markdown: document.markdown, previousWarning: document.warning, - model: getModel("gpt-4o-mini", "openai"), + model: getModel(modelSelection.modelName, "openai"), retryModel: getModel("gpt-4o", "openai"), costTrackingOptions: { costTracking: meta.costTracking, @@ -1089,7 +1131,15 @@ export async function performSummary( }, markdown: trimOutput.text, previousWarning: document.warning, - model: getModel("gpt-4o-mini", "openai"), + model: (() => { + const inlineSchema = { + type: "object", + properties: { summary: { type: "string" } }, + required: ["summary"], + }; + const selection = selectModelForSchema(inlineSchema); + return getModel(selection.modelName, "openai"); + })(), retryModel: getModel("gpt-4o", "openai"), costTrackingOptions: { costTracking: meta.costTracking,