From 822b95f0ba863853c9cc1434c9de486a715e12f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= Date: Wed, 27 Aug 2025 15:56:06 +0200 Subject: [PATCH] feat(scrapeURL/rewrite): scrape Google Drive TXT/PDF files and sheets (#2053) --- .../api/src/__tests__/snips/v1/scrape.test.ts | 34 +++++++++++++++--- .../api/src/__tests__/snips/v2/scrape.test.ts | 36 +++++++++++++++++-- .../engines/utils/specialtyHandler.ts | 3 +- apps/api/src/scraper/scrapeURL/index.ts | 10 ++++++ 4 files changed, 76 insertions(+), 7 deletions(-) diff --git a/apps/api/src/__tests__/snips/v1/scrape.test.ts b/apps/api/src/__tests__/snips/v1/scrape.test.ts index 44236a10a..c49581e1f 100644 --- a/apps/api/src/__tests__/snips/v1/scrape.test.ts +++ b/apps/api/src/__tests__/snips/v1/scrape.test.ts @@ -937,11 +937,14 @@ describe("Scrape tests", () => { // text on the last page expect(response.markdown).toContain("Redistribution and use in source and binary forms, with or without modification"); }, scrapeTimeout * 5); + }); + } + describe("URL rewriting", () => { + if (!process.env.TEST_SUITE_SELF_HOSTED) { it.concurrent("scrapes Google Docs links as PDFs", async () => { const response = await scrape({ url: "https://docs.google.com/document/d/1H-hOLYssS8xXl2o5hxj4ipE7yyhZAX1s7ADYM1Hdlzo/view", - timeout: scrapeTimeout * 5, }, identity); expect(response.markdown).toContain("This is a test to confirm Google Docs scraping abilities."); @@ -950,13 +953,36 @@ describe("Scrape tests", () => { it.concurrent("scrapes Google Slides links as PDFs", async () => { const response = await scrape({ url: "https://docs.google.com/presentation/d/1pDKL1UULpr6siq_eVWE1hjqt5MKCgSSuKS_MWahnHAQ/view", - timeout: scrapeTimeout * 5, }, identity); expect(response.markdown).toContain("This is a test to confirm Google Slides scraping abilities."); }, scrapeTimeout * 5); - }); - } + + it.concurrent("scrapes Google Drive PDF files as PDFs", async () => { + const response = await scrape({ + url: "https://drive.google.com/file/d/1QrgvXM2F7sgSdrhoBfdp9IMBVhUk-Ueu/view?usp=drive_link", + }, identity); + + expect(response.markdown).toContain("This is a simple PDF file."); + }, scrapeTimeout * 5); + } + + it.concurrent("scrapes Google Drive text files correctly", async () => { + const response = await scrape({ + url: "https://drive.google.com/file/d/14m3ZVDnWJwwPSDHX6U6jkL7FXxOf6cHB/view?usp=sharing", + }, identity); + + expect(response.markdown).toContain("This is a simple TXT file."); + }, scrapeTimeout * 5); + + it.concurrent("scrapes Google Sheets links correctly", async () => { + const response = await scrape({ + url: "https://docs.google.com/spreadsheets/d/1DTpw_bbsf3OY17ZqEYEpW6lAmLdCRC2WfLrV0isG9ac/edit?usp=sharing", + }, identity); + + expect(response.markdown).toContain("This is a test sheet."); + }, scrapeTimeout * 5); + }); if (!process.env.TEST_SUITE_SELF_HOSTED || process.env.OPENAI_API_KEY || process.env.OLLAMA_BASE_URL) { describe("JSON format", () => { diff --git a/apps/api/src/__tests__/snips/v2/scrape.test.ts b/apps/api/src/__tests__/snips/v2/scrape.test.ts index 315f93cca..9de6f38e6 100644 --- a/apps/api/src/__tests__/snips/v2/scrape.test.ts +++ b/apps/api/src/__tests__/snips/v2/scrape.test.ts @@ -851,10 +851,15 @@ describe("Scrape tests", () => { // text on the last page expect(response.markdown).toContain("Redistribution and use in source and binary forms, with or without modification"); }, scrapeTimeout * 5); + }); + } + describe("URL rewriting", () => { + if (!process.env.TEST_SUITE_SELF_HOSTED) { it.concurrent("scrapes Google Docs links as PDFs", async () => { const response = await scrape({ url: "https://docs.google.com/document/d/1H-hOLYssS8xXl2o5hxj4ipE7yyhZAX1s7ADYM1Hdlzo/view", + maxAge: 0, }, identity); expect(response.markdown).toContain("This is a test to confirm Google Docs scraping abilities."); @@ -868,8 +873,35 @@ describe("Scrape tests", () => { expect(response.markdown).toContain("This is a test to confirm Google Slides scraping abilities."); }, scrapeTimeout * 5); - }); - } + + it.concurrent("scrapes Google Drive PDF files as PDFs", async () => { + const response = await scrape({ + url: "https://drive.google.com/file/d/1QrgvXM2F7sgSdrhoBfdp9IMBVhUk-Ueu/view?usp=drive_link", + maxAge: 0, + }, identity); + + expect(response.markdown).toContain("This is a simple PDF file."); + }, scrapeTimeout * 5); + } + + it.concurrent("scrapes Google Drive text files correctly", async () => { + const response = await scrape({ + url: "https://drive.google.com/file/d/14m3ZVDnWJwwPSDHX6U6jkL7FXxOf6cHB/view?usp=sharing", + maxAge: 0, + }, identity); + + expect(response.markdown).toContain("This is a simple TXT file."); + }, scrapeTimeout * 5); + + it.concurrent("scrapes Google Sheets links correctly", async () => { + const response = await scrape({ + url: "https://docs.google.com/spreadsheets/d/1DTpw_bbsf3OY17ZqEYEpW6lAmLdCRC2WfLrV0isG9ac/edit?usp=sharing", + maxAge: 0, + }, identity); + + expect(response.markdown).toContain("This is a test sheet."); + }, scrapeTimeout * 5); + }); if (!process.env.TEST_SUITE_SELF_HOSTED || process.env.OPENAI_API_KEY || process.env.OLLAMA_BASE_URL) { describe("JSON format", () => { diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts index c67581cd4..6b782f999 100644 --- a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts +++ b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts @@ -38,7 +38,8 @@ export async function specialtyScrapeCheck( }); } else if ( contentType === "application/pdf" || - contentType.startsWith("application/pdf;") + contentType.startsWith("application/pdf;") || + (contentType === "application/octet-stream" && (feRes?.file?.content.startsWith("JVBERi0") || feRes?.content.startsWith("%PDF-"))) ) { // .pdf throw new AddFeatureError(["pdf"], await feResToPdfPrefetch(logger, feRes)); diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts index b98448a86..6f811a049 100644 --- a/apps/api/src/scraper/scrapeURL/index.ts +++ b/apps/api/src/scraper/scrapeURL/index.ts @@ -154,6 +154,16 @@ function rewriteUrl(url: string): string | undefined { if (id) { return `https://docs.google.com/presentation/d/${id}/export?format=pdf`; } + } else if (url.startsWith("https://drive.google.com/file/d/") || url.startsWith("http://drive.google.com/file/d/")) { + const id = url.match(/\/file\/d\/([-\w]+)/)?.[1]; + if (id) { + return `https://drive.google.com/uc?export=download&id=${id}`; + } + } else if (url.startsWith("https://docs.google.com/spreadsheets/d/") || url.startsWith("http://docs.google.com/spreadsheets/d/")) { + const id = url.match(/\/spreadsheets\/d\/([-\w]+)/)?.[1]; + if (id) { + return `https://docs.google.com/spreadsheets/d/${id}/gviz/tq?tqx=out:html`; + } } return undefined;