improvement(seo): restore explicit AI/search bot allow-list and add link-preview rules (#4480)

* improvement(seo): restore explicit AI/search bot allow-list and add link-preview rules

* fix(seo): correct xAI UA strings, drop Bravebot, block /playground/ and /w/ from link-preview bots

* fix(seo): drop unverified Grok UAs, correct DeepSeekBot and ImagesiftBot tokens

* fix(seo): re-add Bravebot to allow-list per Brave Search docs

* improvement(seo): drop redundant named AI/search bot allow-list

* chore(seo): trim verbose comments in robots.ts

apps/sim/app/robots.ts

@@ -1,29 +1,59 @@
 import type { MetadataRoute } from 'next'
 import { getBaseUrl } from '@/lib/core/utils/urls'
 
+const DISALLOWED_PATHS = [
+  '/api/',
+  '/workspace/',
+  '/chat/',
+  '/playground/',
+  '/resume/',
+  '/invite/',
+  '/unsubscribe/',
+  '/w/',
+  '/form/',
+  '/credential-account/',
+  '/_next/',
+  '/private/',
+  '/blog*tag=',
+]
+
+/** Looser disallow than the wildcard so OG previews can fetch /chat/ and /form/. */
+const LINK_PREVIEW_DISALLOWED_PATHS = [
+  '/api/',
+  '/workspace/',
+  '/w/',
+  '/playground/',
+  '/resume/',
+  '/invite/',
+  '/unsubscribe/',
+  '/credential-account/',
+  '/_next/',
+  '/private/',
+]
+
+const LINK_PREVIEW_BOTS = [
+  'Twitterbot',
+  'LinkedInBot',
+  'Slackbot',
+  'Slack-ImgProxy',
+  'Discordbot',
+  'TelegramBot',
+  'WhatsApp',
+  'facebookexternalhit',
+  'Pinterestbot',
+  'redditbot',
+]
+
 export default function robots(): MetadataRoute.Robots {
   const baseUrl = getBaseUrl()
 
   return {
     rules: [
+      { userAgent: '*', allow: '/', disallow: DISALLOWED_PATHS },
       {
-        userAgent: '*',
+        userAgent: LINK_PREVIEW_BOTS,
         allow: '/',
-        disallow: [
-          '/api/',
-          '/workspace/',
-          '/chat/',
-          '/playground/',
-          '/resume/',
-          '/invite/',
-          '/unsubscribe/',
-          '/w/',
-          '/form/',
-          '/credential-account/',
-          '/_next/',
-          '/private/',
-          '/blog*tag=',
-        ],
+        disallow: LINK_PREVIEW_DISALLOWED_PATHS,
       },
     ],
     sitemap: [`${baseUrl}/sitemap.xml`, `${baseUrl}/blog/sitemap-images.xml`],