diff --git a/apps/api/src/lib/robots-txt.ts b/apps/api/src/lib/robots-txt.ts
index 1a67051d2..6a53018d3 100644
--- a/apps/api/src/lib/robots-txt.ts
+++ b/apps/api/src/lib/robots-txt.ts
@@ -84,6 +84,11 @@ export async function fetchRobotsTxt(
   ) {
     content = response.document.rawHtml!;
   } else {
+    if (response.success && response.document.metadata.statusCode === 404) {
+      logger.warn("Robots.txt not found", { robotsTxtUrl }); // should probably index 404 robots.txt
+      return { content: "", url: robotsTxtUrl };
+    }
+
     logger.error(`Request failed for robots.txt fetch`, {
       method: "fetchRobotsTxt",
       robotsTxtUrl,
diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.ts
index 348358121..7c28a867e 100644
--- a/apps/api/src/scraper/WebScraper/sitemap.ts
+++ b/apps/api/src/scraper/WebScraper/sitemap.ts
@@ -131,6 +131,14 @@ export async function getLinksFromSitemap(
         ) {
           content = response.document.rawHtml!;
         } else {
+          if (
+            response.success &&
+            response.document.metadata.statusCode === 404
+          ) {
+            logger.warn("Sitemap not found", { sitemapUrl }); // should probably index 404 sitemaps
+            return 0;
+          }
+
           logger.error(`Request failed for sitemap fetch`, {
             method: "getLinksFromSitemap",
             mode,
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 72b012a47..3967437c1 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -384,7 +384,10 @@ async function scrapeURLLoopIter(
     );
 
     let checkMarkdown: string;
-    if (meta.internalOptions.teamId === "sitemap") {
+    if (
+      meta.internalOptions.teamId === "sitemap" ||
+      meta.internalOptions.teamId === "robots-txt"
+    ) {
       checkMarkdown = engineResult.html?.trim() ?? "";
     } else {
       checkMarkdown = await parseMarkdown(