feat(api): make ad blocking configurable for browser sessions (#4977)

apps/api/src/controllers/v2/browser.ts

@@ -44,6 +44,7 @@ export const browserCreateRequestSchema = z.object({
   recordSession: z.boolean().default(false),
   zeroDataRetention: z.boolean().optional(),
   integration: integrationSchema.optional().transform(value => value || null),
+  blockAds: z.boolean().default(true),
   profile: z
     .object({
       name: browserProfileNameSchema,

apps/api/src/controllers/v2/scrape-browser.ts

@@ -52,6 +52,10 @@ import { integrationSchema } from "../../utils/integration";
 import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";
 import { getScrapeJobAccess } from "../../lib/operational-job-access";
 import { readScrapeJobState } from "../../lib/job-state-store";
+import {
+  type BrowserOptions,
+  browserOptionsFromScrape,
+} from "../../lib/browser-options";
 import { scrapeQueue } from "../../services/worker/nuq-router";
 
 const browserExecuteRequestSchema = z
@@ -264,17 +268,19 @@ async function scrapeInteractInternal(
           "Replay context is unavailable. Supply url or an existingSessionId to continue without retained scrape context.",
       });
     }
-    const profile =
-      state?.profile ??
-      (nuqJob?.data.mode === "single_urls"
-        ? nuqJob.data.scrapeOptions.profile
-        : undefined);
+    const browserOptions =
+      state?.browser ??
+      browserOptionsFromScrape(
+        nuqJob?.data.mode === "single_urls"
+          ? nuqJob.data.scrapeOptions
+          : { profile: state?.profile },
+      );
     const created = await createSessionForScrape(
       req,
       scrapeId,
       replayContext,
       logger,
-      profile,
+      browserOptions,
       zeroDataRetention,
     );
     if (created.error === true) {
@@ -470,14 +476,14 @@ async function createSessionForScrape(
     ReturnType<typeof buildReplayContextFromScrape>["context"]
   >,
   logger: typeof _logger,
-  profile: { name: string; saveChanges: boolean } | undefined,
+  browserOptions: BrowserOptions,
   zeroDataRetention: boolean,
 ) {
   try {
     const { session } = await createBrowserSession(req, {
       ...browserCreateRequestSchema.parse({}),
+      ...browserOptions,
       scrapeId,
-      profile,
       zeroDataRetention,
       initialize: async browserId => {
         const replay = await executeHangarBrowser(browserId, {

apps/api/src/lib/browser-lifecycle.ts

@@ -124,6 +124,7 @@ export async function createBrowserSession(
     streamWebView: boolean;
     recordSession: boolean;
     zeroDataRetention?: boolean;
+    blockAds: boolean;
     profile?: { name: string; saveChanges: boolean };
     scrapeId?: string;
     shouldBill?: boolean;

apps/api/src/lib/browser-options.test.ts

@@ -0,0 +1,12 @@
+import { hangarBrowserOptions } from "./browser-options";
+
+describe("hangarBrowserOptions", () => {
+  it("maps blockAds to Hangar's uBlock extension", () => {
+    expect(hangarBrowserOptions({ blockAds: true })).toEqual({
+      extensions: ["ublock"],
+    });
+    expect(hangarBrowserOptions({ blockAds: false })).toEqual({
+      extensions: [],
+    });
+  });
+});

apps/api/src/lib/browser-options.ts

@@ -0,0 +1,48 @@
+/**
+ * Browser behavior shared by /v2/browser sessions and sessions created from a
+ * scrape. Option names and defaults follow the scrape API so an interact
+ * session behaves like the page the scrape saw.
+ */
+export type BrowserOptions = {
+  blockAds: boolean;
+  profile?: { name: string; saveChanges: boolean };
+};
+
+export function browserOptionsFromScrape(options: {
+  blockAds?: boolean;
+  profile?: { name: string; saveChanges: boolean };
+}): BrowserOptions {
+  return {
+    blockAds: options.blockAds ?? true,
+    ...(options.profile ? { profile: options.profile } : {}),
+  };
+}
+
+export function isBrowserOptions(value: unknown): value is BrowserOptions {
+  if (!value || typeof value !== "object") return false;
+  const { blockAds, profile } = value as Record<string, unknown>;
+  return (
+    typeof blockAds === "boolean" &&
+    (profile === undefined ||
+      (typeof profile === "object" &&
+        profile !== null &&
+        typeof (profile as any).name === "string" &&
+        typeof (profile as any).saveChanges === "boolean"))
+  );
+}
+
+/** The Hangar create-request fields that carry these options. */
+export function hangarBrowserOptions(options: BrowserOptions) {
+  return {
+    // uBlock Origin Lite covers ads, trackers and cookie notices.
+    extensions: options.blockAds ? ["ublock"] : [],
+    ...(options.profile
+      ? {
+          profile: {
+            name: options.profile.name,
+            save_changes: options.profile.saveChanges,
+          },
+        }
+      : {}),
+  };
+}

apps/api/src/lib/hangar.ts

@@ -1,4 +1,5 @@
 import { config } from "../config";
+import { type BrowserOptions, hangarBrowserOptions } from "./browser-options";
 
 export interface HangarBrowser {
   id: string;
@@ -106,12 +107,11 @@ export const stopHangarBrowser = (id: string) =>
 export async function createHangarBrowser(
   key: string,
   teamId: string,
-  options: {
+  options: BrowserOptions & {
     ttl: number;
     activityTtl: number;
     streamWebView: boolean;
     recordSession: boolean;
-    profile?: { name: string; saveChanges: boolean };
   },
 ): Promise<HangarCreated> {
   const body = {
@@ -124,14 +124,7 @@ export async function createHangarBrowser(
     },
     recording: { enabled: options.recordSession },
     execution: { enabled: true },
-    ...(options.profile
-      ? {
-          profile: {
-            name: options.profile.name,
-            save_changes: options.profile.saveChanges,
-          },
-        }
-      : {}),
+    ...hangarBrowserOptions(options),
   };
   let created: HangarCreated | undefined;
   for (let attempt = 0; attempt < 3; attempt++) {

apps/api/src/lib/job-state-store.ts

@@ -3,6 +3,7 @@ import { getBigtableTable } from "./bigtable-client";
 import { saltedUuidV7RowKey } from "./bigtable-row-key";
 import { setSpanAttributes, withSpan } from "./otel-tracer";
 import type { ScrapeReplayContext } from "./scrape-interact/scrape-replay";
+import { type BrowserOptions, isBrowserOptions } from "./browser-options";
 
 const QUALIFIER = "v";
 const FAMILY = "s";
@@ -17,6 +18,8 @@ type ScrapeJobState = {
   creditsBilled: number;
   error?: string;
   replay?: ScrapeReplayContext;
+  browser?: BrowserOptions;
+  /** Written before `browser`; unused once that state has expired. */
   profile?: { name: string; saveChanges: boolean };
   origin?: string;
 };
@@ -75,6 +78,8 @@ function parseScrapeState(value: Buffer | string): ScrapeJobState {
           typeof (candidate.replay as any).targetUrl === "string" &&
           isFiniteNumber((candidate.replay as any).waitForMs) &&
           Array.isArray((candidate.replay as any).actions))) &&
+      (candidate.browser === undefined ||
+        isBrowserOptions(candidate.browser)) &&
       (candidate.profile === undefined ||
         (typeof candidate.profile === "object" &&
           candidate.profile !== null &&

apps/api/src/services/logging/log_job.ts

@@ -39,6 +39,7 @@ import {
   writeScrapeJobState,
 } from "../../lib/job-state-store";
 import { buildReplayContextFromScrape } from "../../lib/scrape-interact/scrape-replay";
+import { browserOptionsFromScrape } from "../../lib/browser-options";
 import {
   initializeRequestCredits,
   recordRequestCredits,
@@ -770,9 +771,7 @@ async function logScrapeInternal(
           : {
               ...(scrape.error ? { error: scrape.error } : {}),
               ...(replay ? { replay } : {}),
-              ...(scrape.options.profile
-                ? { profile: scrape.options.profile }
-                : {}),
+              browser: browserOptionsFromScrape(scrape.options),
               ...(typeof (scrape.options as any).origin === "string"
                 ? { origin: (scrape.options as any).origin }
                 : {}),

apps/js-sdk/firecrawl/src/v2/client.ts

@@ -641,7 +641,7 @@ export class FirecrawlClient {
   // Browser
   /**
    * Create a new browser session.
-   * @param args Session options (ttl, activityTtl, streamWebView, profile).
+   * @param args Session options (ttl, activityTtl, streamWebView, blockAds, profile).
    * @returns Session id, CDP URL, live view URL, and expiration time.
    */
   async browser(

apps/js-sdk/firecrawl/src/v2/methods/browser.ts

@@ -16,6 +16,8 @@ export async function browser(
     ttl?: number;
     activityTtl?: number;
     streamWebView?: boolean;
+    /** Block ads, trackers and cookie notices, as in scrape (default true). */
+    blockAds?: boolean;
     profile?: {
       name: string;
       saveChanges?: boolean;
@@ -28,6 +30,7 @@ export async function browser(
   if (args.ttl != null) body.ttl = args.ttl;
   if (args.activityTtl != null) body.activityTtl = args.activityTtl;
   if (args.streamWebView != null) body.streamWebView = args.streamWebView;
+  if (args.blockAds != null) body.blockAds = args.blockAds;
   if (args.profile != null) body.profile = args.profile;
   if (args.integration != null) body.integration = args.integration;
   if (args.origin) body.origin = args.origin;

apps/python-sdk/firecrawl/v2/client.py

@@ -1741,6 +1741,7 @@ class FirecrawlClient:
         ttl: Optional[int] = None,
         activity_ttl: Optional[int] = None,
         stream_web_view: Optional[bool] = None,
+        block_ads: Optional[bool] = None,
         profile: Optional[Dict[str, Any]] = None,
     ):
         """Create a new browser session.
@@ -1749,6 +1750,7 @@ class FirecrawlClient:
             ttl: Total time-to-live in seconds (30-3600, default 300)
             activity_ttl: Inactivity TTL in seconds (10-3600)
             stream_web_view: Whether to enable webview streaming
+            block_ads: Block ads, trackers and cookie notices (default ``True``)
             profile: Profile config with ``name`` (str) and
                 optional ``save_changes`` (bool, default ``True``)
 
@@ -1760,6 +1762,7 @@ class FirecrawlClient:
             ttl=ttl,
             activity_ttl=activity_ttl,
             stream_web_view=stream_web_view,
+            block_ads=block_ads,
             profile=profile,
         )
 

apps/python-sdk/firecrawl/v2/client_async.py

@@ -949,6 +949,7 @@ class AsyncFirecrawlClient:
         ttl: Optional[int] = None,
         activity_ttl: Optional[int] = None,
         stream_web_view: Optional[bool] = None,
+        block_ads: Optional[bool] = None,
         profile: Optional[Dict[str, Any]] = None,
     ):
         """Create a new browser session.
@@ -957,6 +958,7 @@ class AsyncFirecrawlClient:
             ttl: Total time-to-live in seconds (30-3600, default 300)
             activity_ttl: Inactivity TTL in seconds (10-3600)
             stream_web_view: Whether to enable webview streaming
+            block_ads: Block ads, trackers and cookie notices (default ``True``)
             profile: Profile config with ``name`` (str) and
                 optional ``save_changes`` (bool, default ``True``)
 
@@ -968,6 +970,7 @@ class AsyncFirecrawlClient:
             ttl=ttl,
             activity_ttl=activity_ttl,
             stream_web_view=stream_web_view,
+            block_ads=block_ads,
             profile=profile,
         )
 

apps/python-sdk/firecrawl/v2/methods/aio/browser.py

@@ -57,6 +57,7 @@ async def browser(
     ttl: Optional[int] = None,
     activity_ttl: Optional[int] = None,
     stream_web_view: Optional[bool] = None,
+    block_ads: Optional[bool] = None,
     profile: Optional[Dict[str, Any]] = None,
 ) -> BrowserCreateResponse:
     """Create a new browser session.
@@ -66,6 +67,7 @@ async def browser(
         ttl: Total time-to-live in seconds (30-3600, default 300)
         activity_ttl: Inactivity TTL in seconds (10-3600)
         stream_web_view: Whether to enable webview streaming
+        block_ads: Block ads, trackers and cookie notices (default ``True``)
         profile: Profile config with ``name`` (str) and
             optional ``save_changes`` (bool, default ``True``)
 
@@ -79,6 +81,8 @@ async def browser(
         body["activityTtl"] = activity_ttl
     if stream_web_view is not None:
         body["streamWebView"] = stream_web_view
+    if block_ads is not None:
+        body["blockAds"] = block_ads
     if profile is not None:
         body["profile"] = {
             "name": profile["name"],

apps/python-sdk/firecrawl/v2/methods/browser.py

@@ -58,6 +58,7 @@ def browser(
     ttl: Optional[int] = None,
     activity_ttl: Optional[int] = None,
     stream_web_view: Optional[bool] = None,
+    block_ads: Optional[bool] = None,
     profile: Optional[Dict[str, Any]] = None,
 ) -> BrowserCreateResponse:
     """Create a new browser session.
@@ -67,6 +68,7 @@ def browser(
         ttl: Total time-to-live in seconds (30-3600, default 300)
         activity_ttl: Inactivity TTL in seconds (10-3600)
         stream_web_view: Whether to enable webview streaming
+        block_ads: Block ads, trackers and cookie notices (default ``True``)
         profile: Profile config with ``name`` (str) and
             optional ``save_changes`` (bool, default ``True``)
 
@@ -80,6 +82,8 @@ def browser(
         body["activityTtl"] = activity_ttl
     if stream_web_view is not None:
         body["streamWebView"] = stream_web_view
+    if block_ads is not None:
+        body["blockAds"] = block_ads
     if profile is not None:
         body["profile"] = {
             "name": profile["name"],