From a54081e5b25d0c00c8c9991b94ae2ec7d62baa66 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 15 May 2025 21:56:09 +0000 Subject: [PATCH 1/3] Update LLMs.txt test file to use helper functions and concurrent tests Co-Authored-By: hello@sideguide.dev --- apps/api/src/__tests__/snips/llmstxt.test.ts | 86 ++++++++------------ 1 file changed, 34 insertions(+), 52 deletions(-) diff --git a/apps/api/src/__tests__/snips/llmstxt.test.ts b/apps/api/src/__tests__/snips/llmstxt.test.ts index f1bfe6c78..fb320a0fe 100644 --- a/apps/api/src/__tests__/snips/llmstxt.test.ts +++ b/apps/api/src/__tests__/snips/llmstxt.test.ts @@ -5,31 +5,37 @@ import request from "supertest"; const TEST_URL = "http://127.0.0.1:3002"; -describe("LLMs.txt cache tests", () => { - it("should generate different results for different subdomains", async () => { - const response1 = await request(TEST_URL) - .post("/v1/llmstxt") - .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) - .set("Content-Type", "application/json") - .send({ - url: "https://domain1.example.com", - maxUrls: 1, - showFullText: false - }); +async function generateLLMsText(url, options = {}) { + const defaultOptions = { + maxUrls: 1, + showFullText: false, + bypassCache: false + }; + + const requestOptions = { ...defaultOptions, ...options, url }; + + const response = await request(TEST_URL) + .post("/v1/llmstxt") + .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) + .set("Content-Type", "application/json") + .send(requestOptions); + + return response; +} +async function getLLMsTextStatus(id) { + return await request(TEST_URL) + .get(`/v1/llmstxt/${id}`) + .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`); +} + +describe("LLMs.txt cache tests", () => { + it.concurrent("should generate different results for different subdomains", async () => { + const response1 = await generateLLMsText("https://domain1.example.com"); expect(response1.statusCode).toBe(200); expect(response1.body.success).toBe(true); - const response2 = await request(TEST_URL) - .post("/v1/llmstxt") - .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) - .set("Content-Type", "application/json") - .send({ - url: "https://domain2.example.com", - maxUrls: 1, - showFullText: false - }); - + const response2 = await generateLLMsText("https://domain2.example.com"); expect(response2.statusCode).toBe(200); expect(response2.body.success).toBe(true); @@ -38,43 +44,19 @@ describe("LLMs.txt cache tests", () => { await new Promise(resolve => setTimeout(resolve, 5000)); - const statusResponse1 = await request(TEST_URL) - .get(`/v1/llmstxt/${id1}`) - .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`); - - const statusResponse2 = await request(TEST_URL) - .get(`/v1/llmstxt/${id2}`) - .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`); + const statusResponse1 = await getLLMsTextStatus(id1); + const statusResponse2 = await getLLMsTextStatus(id2); expect(statusResponse1.body.data.generatedText).not.toEqual(statusResponse2.body.data.generatedText); - }, 15000); + }, 30000); - it("should bypass cache when bypassCache=true is specified", async () => { - const response1 = await request(TEST_URL) - .post("/v1/llmstxt") - .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) - .set("Content-Type", "application/json") - .send({ - url: "https://example.com", - maxUrls: 1, - showFullText: false - }); - + it.concurrent("should bypass cache when bypassCache=true is specified", async () => { + const response1 = await generateLLMsText("https://example.com"); expect(response1.statusCode).toBe(200); - const response2 = await request(TEST_URL) - .post("/v1/llmstxt") - .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) - .set("Content-Type", "application/json") - .send({ - url: "https://example.com", - maxUrls: 1, - showFullText: false, - bypassCache: true - }); - + const response2 = await generateLLMsText("https://example.com", { bypassCache: true }); expect(response2.statusCode).toBe(200); expect(response1.body.id).not.toEqual(response2.body.id); - }, 15000); + }, 30000); }); From 673e8f02c2f197457663a1d1dd9d0df597a93b9b Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 15 May 2025 21:57:45 +0000 Subject: [PATCH 2/3] Remove LLMs.txt test file as requested Co-Authored-By: hello@sideguide.dev --- apps/api/src/__tests__/snips/llmstxt.test.ts | 62 -------------------- 1 file changed, 62 deletions(-) delete mode 100644 apps/api/src/__tests__/snips/llmstxt.test.ts diff --git a/apps/api/src/__tests__/snips/llmstxt.test.ts b/apps/api/src/__tests__/snips/llmstxt.test.ts deleted file mode 100644 index fb320a0fe..000000000 --- a/apps/api/src/__tests__/snips/llmstxt.test.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { configDotenv } from "dotenv"; -configDotenv(); - -import request from "supertest"; - -const TEST_URL = "http://127.0.0.1:3002"; - -async function generateLLMsText(url, options = {}) { - const defaultOptions = { - maxUrls: 1, - showFullText: false, - bypassCache: false - }; - - const requestOptions = { ...defaultOptions, ...options, url }; - - const response = await request(TEST_URL) - .post("/v1/llmstxt") - .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`) - .set("Content-Type", "application/json") - .send(requestOptions); - - return response; -} - -async function getLLMsTextStatus(id) { - return await request(TEST_URL) - .get(`/v1/llmstxt/${id}`) - .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`); -} - -describe("LLMs.txt cache tests", () => { - it.concurrent("should generate different results for different subdomains", async () => { - const response1 = await generateLLMsText("https://domain1.example.com"); - expect(response1.statusCode).toBe(200); - expect(response1.body.success).toBe(true); - - const response2 = await generateLLMsText("https://domain2.example.com"); - expect(response2.statusCode).toBe(200); - expect(response2.body.success).toBe(true); - - const id1 = response1.body.id; - const id2 = response2.body.id; - - await new Promise(resolve => setTimeout(resolve, 5000)); - - const statusResponse1 = await getLLMsTextStatus(id1); - const statusResponse2 = await getLLMsTextStatus(id2); - - expect(statusResponse1.body.data.generatedText).not.toEqual(statusResponse2.body.data.generatedText); - }, 30000); - - it.concurrent("should bypass cache when bypassCache=true is specified", async () => { - const response1 = await generateLLMsText("https://example.com"); - expect(response1.statusCode).toBe(200); - - const response2 = await generateLLMsText("https://example.com", { bypassCache: true }); - expect(response2.statusCode).toBe(200); - - expect(response1.body.id).not.toEqual(response2.body.id); - }, 30000); -}); From c5cc80dca1d23ed7be693bfc8d01ed58194e3b65 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 15 May 2025 22:01:55 +0000 Subject: [PATCH 3/3] Change parameter name to 'cache' and keep 7-day expiration Co-Authored-By: hello@sideguide.dev --- apps/api/requests.http | 2 +- apps/api/src/controllers/v1/generate-llmstxt.ts | 2 +- apps/api/src/controllers/v1/types.ts | 6 +++--- .../src/lib/generate-llmstxt/generate-llmstxt-redis.ts | 4 ++-- .../src/lib/generate-llmstxt/generate-llmstxt-service.ts | 8 ++++---- .../src/lib/generate-llmstxt/generate-llmstxt-supabase.ts | 8 ++++---- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/apps/api/requests.http b/apps/api/requests.http index b8c125e41..55d315d1b 100644 --- a/apps/api/requests.http +++ b/apps/api/requests.http @@ -105,7 +105,7 @@ content-type: application/json "url": "https://firecrawl.dev", "maxUrls": 1, "showFullText": false, - "bypassCache": false + "cache": true } diff --git a/apps/api/src/controllers/v1/generate-llmstxt.ts b/apps/api/src/controllers/v1/generate-llmstxt.ts index df67b8661..aaf34c3bb 100644 --- a/apps/api/src/controllers/v1/generate-llmstxt.ts +++ b/apps/api/src/controllers/v1/generate-llmstxt.ts @@ -42,7 +42,7 @@ export async function generateLLMsTextController( url: req.body.url, maxUrls: req.body.maxUrls, showFullText: req.body.showFullText, - bypassCache: req.body.bypassCache, + cache: req.body.cache, generatedText: "", fullText: "", }); diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts index 950102bb6..0ae2acc35 100644 --- a/apps/api/src/controllers/v1/types.ts +++ b/apps/api/src/controllers/v1/types.ts @@ -1211,10 +1211,10 @@ export const generateLLMsTextRequestSchema = z.object({ .boolean() .default(false) .describe("Whether to show the full LLMs-full.txt in the response"), - bypassCache: z + cache: z .boolean() - .default(false) - .describe("Whether to bypass the cache and generate new content"), + .default(true) + .describe("Whether to use cached content if available"), __experimental_stream: z.boolean().optional(), }); diff --git a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-redis.ts b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-redis.ts index e5c78f26b..c5bf6479e 100644 --- a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-redis.ts +++ b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-redis.ts @@ -9,7 +9,7 @@ export interface GenerationData { url: string; maxUrls: number; showFullText: boolean; - bypassCache?: boolean; + cache?: boolean; generatedText: string; fullText: string; error?: string; @@ -67,4 +67,4 @@ export async function updateGeneratedLlmsTxtStatus( if (error !== undefined) updates.error = error; await updateGeneratedLlmsTxt(id, updates); -} \ No newline at end of file +} \ No newline at end of file diff --git a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts index b4ff804ac..497cf734c 100644 --- a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts +++ b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts @@ -19,7 +19,7 @@ interface GenerateLLMsTextServiceOptions { url: string; maxUrls: number; showFullText: boolean; - bypassCache?: boolean; + cache?: boolean; subId?: string; } @@ -64,7 +64,7 @@ function limitLlmsTxtEntries(llmstxt: string, maxEntries: number): string { export async function performGenerateLlmsTxt( options: GenerateLLMsTextServiceOptions, ) { - const { generationId, teamId, url, maxUrls = 100, showFullText, bypassCache = false, subId } = + const { generationId, teamId, url, maxUrls = 100, showFullText, cache = true, subId } = options; const startTime = Date.now(); const logger = _logger.child({ @@ -80,8 +80,8 @@ export async function performGenerateLlmsTxt( // Enforce max URL limit const effectiveMaxUrls = Math.min(maxUrls, 5000); - // Check cache first, unless bypass is requested - const cachedResult = !bypassCache ? await getLlmsTextFromCache(url, effectiveMaxUrls) : null; + // Check cache first, unless cache is set to false + const cachedResult = cache ? await getLlmsTextFromCache(url, effectiveMaxUrls) : null; if (cachedResult) { logger.info("Found cached LLMs text", { url }); diff --git a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-supabase.ts b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-supabase.ts index 6b42adf01..5fb9c3a88 100644 --- a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-supabase.ts +++ b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-supabase.ts @@ -33,11 +33,11 @@ export async function getLlmsTextFromCache( return null; } - // Check if data is older than 24 hours - const oneDayAgo = new Date(); - oneDayAgo.setDate(oneDayAgo.getDate() - 1); + // Check if data is older than 1 week + const oneWeekAgo = new Date(); + oneWeekAgo.setDate(oneWeekAgo.getDate() - 7); - if (!data || new Date(data.updated_at) < oneDayAgo) { + if (!data || new Date(data.updated_at) < oneWeekAgo) { return null; }