fix(seo): noindex non-production sim.ai hosts, redirect indexed 404s (#5988)

* fix(seo): noindex non-production sim.ai hosts, redirect indexed 404s

Non-production deployments (dev.sim.ai, staging.sim.ai) serve the same
build as www.sim.ai and were fully crawlable. Send X-Robots-Tag noindex
for those hosts, and 301 seven marketing paths that an external SEO
audit found returning 404.

* fix(seo): parse only the first entry of a comma-joined forwarded host

A multi-value x-forwarded-host survives the port split intact, so
endsWith('.sim.ai') matched on the trailing entry and could apply
noindex to the canonical site. Normalize with the same
split(',')[0].trim() the rest of the repo uses for forwarded headers.

* fix(seo): drop the /security redirect, complete the urls test mock

security.txt advertises /security as its RFC 9116 Policy URI, so a
permanent redirect to marketing would mislead that link and shadow a
real policy page added later.

urlsMock is installed globally and documents itself as carrying every
real export, but was missing CANONICAL_SITE_HOST and
isNonCanonicalSimHost — any test loading proxy.ts would have called
undefined in track().

packages/testing/src/mocks/urls.mock.ts

@@ -10,6 +10,9 @@ export const LOCALHOST_HOSTNAMES_MOCK: ReadonlySet<string> = new Set([
   '::1',
 ])
 
+/** Mirrors the real `CANONICAL_SITE_HOST` from `@/lib/core/utils/urls`. */
+export const CANONICAL_SITE_HOST_MOCK = 'www.sim.ai'
+
 const DEFAULT_SOCKET_URL = 'http://localhost:3002'
 const DEFAULT_OLLAMA_URL = 'http://localhost:11434'
 
@@ -72,6 +75,18 @@ function isLoopbackHostnameImpl(hostname: string): boolean {
   return LOCALHOST_HOSTNAMES_MOCK.has(hostname)
 }
 
+/** Mirrors the real `stripWwwPrefix` from `@/lib/core/utils/urls`. */
+function stripWwwPrefix(host: string): string {
+  return host.startsWith('www.') ? host.slice(4) : host
+}
+
+function isNonCanonicalSimHostImpl(host: string): boolean {
+  const first = host.split(',')[0]?.trim() ?? ''
+  const hostname = stripWwwPrefix(first.toLowerCase().split(':')[0])
+  const canonical = stripWwwPrefix(CANONICAL_SITE_HOST_MOCK)
+  return hostname !== canonical && hostname.endsWith(`.${canonical}`)
+}
+
 function parseOriginListImpl(
   raw: string | undefined | null,
   onInvalid?: (value: string) => void
@@ -156,6 +171,7 @@ export const urlsMockFns = {
   mockGetBaseDomain: vi.fn(getBaseDomainImpl),
   mockGetEmailDomain: vi.fn(getEmailDomainImpl),
   mockIsLoopbackHostname: vi.fn(isLoopbackHostnameImpl),
+  mockIsNonCanonicalSimHost: vi.fn(isNonCanonicalSimHostImpl),
   mockParseOriginList: vi.fn(parseOriginListImpl),
   mockIsLocalhostUrl: vi.fn(isLocalhostUrlImpl),
   mockGetBrowserOrigin: vi.fn(getBrowserOriginImpl),
@@ -176,6 +192,7 @@ export function resetUrlsMock(): void {
   urlsMockFns.mockGetBaseDomain.mockReset().mockImplementation(getBaseDomainImpl)
   urlsMockFns.mockGetEmailDomain.mockReset().mockImplementation(getEmailDomainImpl)
   urlsMockFns.mockIsLoopbackHostname.mockReset().mockImplementation(isLoopbackHostnameImpl)
+  urlsMockFns.mockIsNonCanonicalSimHost.mockReset().mockImplementation(isNonCanonicalSimHostImpl)
   urlsMockFns.mockParseOriginList.mockReset().mockImplementation(parseOriginListImpl)
   urlsMockFns.mockIsLocalhostUrl.mockReset().mockImplementation(isLocalhostUrlImpl)
   urlsMockFns.mockGetBrowserOrigin.mockReset().mockImplementation(getBrowserOriginImpl)
@@ -197,12 +214,14 @@ export function resetUrlsMock(): void {
 export const urlsMock = {
   SITE_URL: 'https://www.sim.ai',
   LOCALHOST_HOSTNAMES: LOCALHOST_HOSTNAMES_MOCK,
+  CANONICAL_SITE_HOST: CANONICAL_SITE_HOST_MOCK,
   getBaseUrl: urlsMockFns.mockGetBaseUrl,
   getInternalApiBaseUrl: urlsMockFns.mockGetInternalApiBaseUrl,
   ensureAbsoluteUrl: urlsMockFns.mockEnsureAbsoluteUrl,
   getBaseDomain: urlsMockFns.mockGetBaseDomain,
   getEmailDomain: urlsMockFns.mockGetEmailDomain,
   isLoopbackHostname: urlsMockFns.mockIsLoopbackHostname,
+  isNonCanonicalSimHost: urlsMockFns.mockIsNonCanonicalSimHost,
   parseOriginList: urlsMockFns.mockParseOriginList,
   isLocalhostUrl: urlsMockFns.mockIsLocalhostUrl,
   getBrowserOrigin: urlsMockFns.mockGetBrowserOrigin,