From d657a3f88f38a97d95e01bbb947a8faf2c145b3d Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 17 Jan 2026 07:13:30 +0000 Subject: [PATCH] fix: preserve gid parameter when rewriting Google Sheets URLs When scraping Google Sheets URLs with a gid parameter (which specifies a specific tab/sheet), the URL rewriting was losing this parameter, causing Firecrawl to always scrape the first tab instead of the selected one. This fix extracts the gid parameter from either the query string or hash fragment and includes it in the rewritten URL. Fixes issue where scraping: https://docs.google.com/spreadsheets/d/.../edit?gid=89683736 would not return the correct tab content. Co-Authored-By: himanshu@sideguide.dev --- .../lib/__tests__/rewriteUrl.test.ts | 24 +++++++++++++++++++ .../src/scraper/scrapeURL/lib/rewriteUrl.ts | 5 +++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/apps/api/src/scraper/scrapeURL/lib/__tests__/rewriteUrl.test.ts b/apps/api/src/scraper/scrapeURL/lib/__tests__/rewriteUrl.test.ts index c115c0091..08c5fe600 100644 --- a/apps/api/src/scraper/scrapeURL/lib/__tests__/rewriteUrl.test.ts +++ b/apps/api/src/scraper/scrapeURL/lib/__tests__/rewriteUrl.test.ts @@ -64,6 +64,30 @@ describe("rewriteUrl", () => { ); }); + it("should preserve gid parameter from query string for specific sheet tab", () => { + const url = + "https://docs.google.com/spreadsheets/d/1dhyxGttUbI2RlTxPXF4CQcY4TD2k6Jp7-hcqS9PP5uc/edit?gid=89683736"; + expect(rewriteUrl(url)).toBe( + "https://docs.google.com/spreadsheets/d/1dhyxGttUbI2RlTxPXF4CQcY4TD2k6Jp7-hcqS9PP5uc/gviz/tq?tqx=out:html&gid=89683736", + ); + }); + + it("should preserve gid parameter from hash fragment for specific sheet tab", () => { + const url = + "https://docs.google.com/spreadsheets/d/1dhyxGttUbI2RlTxPXF4CQcY4TD2k6Jp7-hcqS9PP5uc/edit#gid=89683736"; + expect(rewriteUrl(url)).toBe( + "https://docs.google.com/spreadsheets/d/1dhyxGttUbI2RlTxPXF4CQcY4TD2k6Jp7-hcqS9PP5uc/gviz/tq?tqx=out:html&gid=89683736", + ); + }); + + it("should preserve gid parameter when both query and hash have gid (uses first match)", () => { + const url = + "https://docs.google.com/spreadsheets/d/1dhyxGttUbI2RlTxPXF4CQcY4TD2k6Jp7-hcqS9PP5uc/edit?gid=89683736#gid=89683736"; + expect(rewriteUrl(url)).toBe( + "https://docs.google.com/spreadsheets/d/1dhyxGttUbI2RlTxPXF4CQcY4TD2k6Jp7-hcqS9PP5uc/gviz/tq?tqx=out:html&gid=89683736", + ); + }); + it("should NOT rewrite published Google Sheets URLs (/d/e/)", () => { const url = "https://docs.google.com/spreadsheets/d/e/2PACX-1vSomePublishId/pubhtml"; diff --git a/apps/api/src/scraper/scrapeURL/lib/rewriteUrl.ts b/apps/api/src/scraper/scrapeURL/lib/rewriteUrl.ts index 1a56429f7..0b817f801 100644 --- a/apps/api/src/scraper/scrapeURL/lib/rewriteUrl.ts +++ b/apps/api/src/scraper/scrapeURL/lib/rewriteUrl.ts @@ -43,7 +43,10 @@ export function rewriteUrl(url: string): string | undefined { } const id = url.match(/\/spreadsheets\/d\/([-\w]+)/)?.[1]; if (id) { - return `https://docs.google.com/spreadsheets/d/${id}/gviz/tq?tqx=out:html`; + // Extract gid parameter from query string or hash fragment to preserve the selected tab + const gidMatch = url.match(/[?&#]gid=(\d+)/); + const gidParam = gidMatch ? `&gid=${gidMatch[1]}` : ""; + return `https://docs.google.com/spreadsheets/d/${id}/gviz/tq?tqx=out:html${gidParam}`; } }