mirror of
https://github.com/simstudioai/sim.git
synced 2026-09-24 15:45:35 +08:00
improvement(seo): optimize sitemaps, robots.txt, and core web vitals across sim and docs (#4170)
* improvement(seo): optimize sitemaps and robots.txt across sim and docs - Add missing pages to sim sitemap: blog author pages, academy catalog and course pages - Fix 6x duplicate URL bug in docs sitemap by deduplicating with source.getLanguages() - Convert docs sitemap from route handler to Next.js metadata convention with native hreflang - Add x-default hreflang alternate for docs multi-language pages - Remove changeFrequency and priority fields (Google ignores both) - Fix inaccurate lastModified timestamps — derive from real content dates, omit when unknown - Consolidate 20+ redundant per-bot robots rules into single wildcard entry - Add /form/ and /credential-account/ to sim robots disallow list - Reference image sitemap in sim robots.txt - Remove deprecated host directive from sim robots - Move disallow rules before allow in docs robots for crawler compatibility - Extract hardcoded docs baseUrl to env variable with production fallback * fix(seo): remove homepage new Date(), guard latestModelDate empty array * improvement(seo): consolidate DOCS_BASE_URL, optimize core web vitals Extract hardcoded https://docs.sim.ai into shared DOCS_BASE_URL constant in lib/urls.ts and replace all 20+ instances across layouts, metadata, structured data, LLM manifest, sitemap, and robots files. Remove OneDollarStats analytics script and tighten CSP for improved core web vitals. * fix: removed onedollarstats from bun lock * fix(seo): guard per-provider Math.max, consolidate docs robots to single wildcard
This commit is contained in:
Emir Karabeg
@@ -17,9 +17,10 @@ import { ResponseSection } from '@/components/ui/response-section'
|
||||
import { i18n } from '@/lib/i18n'
|
||||
import { getApiSpecContent, openapi } from '@/lib/openapi'
|
||||
import { type PageData, source } from '@/lib/source'
|
||||
import { DOCS_BASE_URL } from '@/lib/urls'
|
||||
|
||||
const SUPPORTED_LANGUAGES: Set<string> = new Set(i18n.languages)
|
||||
const BASE_URL = 'https://docs.sim.ai'
|
||||
const BASE_URL = DOCS_BASE_URL
|
||||
|
||||
const OG_LOCALE_MAP: Record<string, string> = {
|
||||
en: 'en_US',
|
||||
|
||||
@@ -3,7 +3,6 @@ import { defineI18nUI } from 'fumadocs-ui/i18n'
|
||||
import { DocsLayout } from 'fumadocs-ui/layouts/docs'
|
||||
import { RootProvider } from 'fumadocs-ui/provider/next'
|
||||
import { Geist_Mono, Inter } from 'next/font/google'
|
||||
import Script from 'next/script'
|
||||
import {
|
||||
SidebarFolder,
|
||||
SidebarItem,
|
||||
@@ -13,6 +12,7 @@ import { Navbar } from '@/components/navbar/navbar'
|
||||
import { SimLogoFull } from '@/components/ui/sim-logo'
|
||||
import { i18n } from '@/lib/i18n'
|
||||
import { source } from '@/lib/source'
|
||||
import { DOCS_BASE_URL } from '@/lib/urls'
|
||||
import '../global.css'
|
||||
|
||||
const inter = Inter({
|
||||
@@ -67,14 +67,14 @@ export default async function Layout({ children, params }: LayoutProps) {
|
||||
name: 'Sim Documentation',
|
||||
description:
|
||||
'Documentation for Sim — the open-source AI workspace where teams build, deploy, and manage AI agents. Connect 1,000+ integrations and every major LLM.',
|
||||
url: 'https://docs.sim.ai',
|
||||
url: DOCS_BASE_URL,
|
||||
publisher: {
|
||||
'@type': 'Organization',
|
||||
name: 'Sim',
|
||||
url: 'https://sim.ai',
|
||||
logo: {
|
||||
'@type': 'ImageObject',
|
||||
url: 'https://docs.sim.ai/static/logo.png',
|
||||
url: `${DOCS_BASE_URL}/static/logo.png`,
|
||||
},
|
||||
},
|
||||
inLanguage: lang,
|
||||
@@ -82,7 +82,7 @@ export default async function Layout({ children, params }: LayoutProps) {
|
||||
'@type': 'SearchAction',
|
||||
target: {
|
||||
'@type': 'EntryPoint',
|
||||
urlTemplate: 'https://docs.sim.ai/api/search?q={search_term_string}',
|
||||
urlTemplate: `${DOCS_BASE_URL}/api/search?q={search_term_string}`,
|
||||
},
|
||||
'query-input': 'required name=search_term_string',
|
||||
},
|
||||
@@ -101,7 +101,6 @@ export default async function Layout({ children, params }: LayoutProps) {
|
||||
/>
|
||||
</head>
|
||||
<body className='flex min-h-screen flex-col font-sans'>
|
||||
<Script src='https://assets.onedollarstats.com/stonks.js' strategy='lazyOnload' />
|
||||
<RootProvider i18n={provider(lang)}>
|
||||
<Navbar />
|
||||
<DocsLayout
|
||||
|
||||
+13
-12
@@ -1,5 +1,6 @@
|
||||
import type { ReactNode } from 'react'
|
||||
import type { Viewport } from 'next'
|
||||
import { DOCS_BASE_URL } from '@/lib/urls'
|
||||
|
||||
export default function RootLayout({ children }: { children: ReactNode }) {
|
||||
return children
|
||||
@@ -12,7 +13,7 @@ export const viewport: Viewport = {
|
||||
}
|
||||
|
||||
export const metadata = {
|
||||
metadataBase: new URL('https://docs.sim.ai'),
|
||||
metadataBase: new URL(DOCS_BASE_URL),
|
||||
title: {
|
||||
default: 'Sim Documentation — The AI Workspace for Teams',
|
||||
template: '%s | Sim Docs',
|
||||
@@ -61,14 +62,14 @@ export const metadata = {
|
||||
type: 'website',
|
||||
locale: 'en_US',
|
||||
alternateLocale: ['es_ES', 'fr_FR', 'de_DE', 'ja_JP', 'zh_CN'],
|
||||
url: 'https://docs.sim.ai',
|
||||
url: DOCS_BASE_URL,
|
||||
siteName: 'Sim Documentation',
|
||||
title: 'Sim Documentation — The AI Workspace for Teams',
|
||||
description:
|
||||
'Documentation for Sim — the open-source AI workspace where teams build, deploy, and manage AI agents. Connect 1,000+ integrations and every major LLM.',
|
||||
images: [
|
||||
{
|
||||
url: 'https://docs.sim.ai/api/og?title=Sim%20Documentation',
|
||||
url: `${DOCS_BASE_URL}/api/og?title=Sim%20Documentation`,
|
||||
width: 1200,
|
||||
height: 630,
|
||||
alt: 'Sim Documentation',
|
||||
@@ -82,7 +83,7 @@ export const metadata = {
|
||||
'Documentation for Sim — the open-source AI workspace where teams build, deploy, and manage AI agents. Connect 1,000+ integrations and every major LLM.',
|
||||
creator: '@simdotai',
|
||||
site: '@simdotai',
|
||||
images: ['https://docs.sim.ai/api/og?title=Sim%20Documentation'],
|
||||
images: [`${DOCS_BASE_URL}/api/og?title=Sim%20Documentation`],
|
||||
},
|
||||
robots: {
|
||||
index: true,
|
||||
@@ -96,15 +97,15 @@ export const metadata = {
|
||||
},
|
||||
},
|
||||
alternates: {
|
||||
canonical: 'https://docs.sim.ai',
|
||||
canonical: DOCS_BASE_URL,
|
||||
languages: {
|
||||
'x-default': 'https://docs.sim.ai',
|
||||
en: 'https://docs.sim.ai',
|
||||
es: 'https://docs.sim.ai/es',
|
||||
fr: 'https://docs.sim.ai/fr',
|
||||
de: 'https://docs.sim.ai/de',
|
||||
ja: 'https://docs.sim.ai/ja',
|
||||
zh: 'https://docs.sim.ai/zh',
|
||||
'x-default': DOCS_BASE_URL,
|
||||
en: DOCS_BASE_URL,
|
||||
es: `${DOCS_BASE_URL}/es`,
|
||||
fr: `${DOCS_BASE_URL}/fr`,
|
||||
de: `${DOCS_BASE_URL}/de`,
|
||||
ja: `${DOCS_BASE_URL}/ja`,
|
||||
zh: `${DOCS_BASE_URL}/zh`,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
import { source } from '@/lib/source'
|
||||
import { DOCS_BASE_URL } from '@/lib/urls'
|
||||
|
||||
export const revalidate = false
|
||||
|
||||
export async function GET() {
|
||||
const baseUrl = 'https://docs.sim.ai'
|
||||
const baseUrl = DOCS_BASE_URL
|
||||
|
||||
try {
|
||||
const pages = source.getPages().filter((page) => {
|
||||
|
||||
@@ -1,70 +1,18 @@
|
||||
import { DOCS_BASE_URL } from '@/lib/urls'
|
||||
|
||||
export const revalidate = false
|
||||
|
||||
export async function GET() {
|
||||
const baseUrl = 'https://docs.sim.ai'
|
||||
const baseUrl = DOCS_BASE_URL
|
||||
|
||||
const robotsTxt = `# Robots.txt for Sim Documentation
|
||||
|
||||
User-agent: *
|
||||
Allow: /
|
||||
|
||||
# Search engine crawlers
|
||||
User-agent: Googlebot
|
||||
Allow: /
|
||||
|
||||
User-agent: Bingbot
|
||||
Allow: /
|
||||
|
||||
User-agent: Slurp
|
||||
Allow: /
|
||||
|
||||
User-agent: DuckDuckBot
|
||||
Allow: /
|
||||
|
||||
User-agent: Baiduspider
|
||||
Allow: /
|
||||
|
||||
User-agent: YandexBot
|
||||
Allow: /
|
||||
|
||||
# AI and LLM crawlers - explicitly allowed for documentation indexing
|
||||
User-agent: GPTBot
|
||||
Allow: /
|
||||
|
||||
User-agent: ChatGPT-User
|
||||
Allow: /
|
||||
|
||||
User-agent: CCBot
|
||||
Allow: /
|
||||
|
||||
User-agent: anthropic-ai
|
||||
Allow: /
|
||||
|
||||
User-agent: Claude-Web
|
||||
Allow: /
|
||||
|
||||
User-agent: Applebot
|
||||
Allow: /
|
||||
|
||||
User-agent: PerplexityBot
|
||||
Allow: /
|
||||
|
||||
User-agent: Diffbot
|
||||
Allow: /
|
||||
|
||||
User-agent: FacebookBot
|
||||
Allow: /
|
||||
|
||||
User-agent: cohere-ai
|
||||
Allow: /
|
||||
|
||||
# Disallow admin and internal paths (if any exist)
|
||||
Disallow: /.next/
|
||||
Disallow: /api/internal/
|
||||
Disallow: /_next/static/
|
||||
Disallow: /admin/
|
||||
|
||||
# Allow but don't prioritize these
|
||||
Allow: /
|
||||
Allow: /api/search
|
||||
Allow: /llms.txt
|
||||
Allow: /llms-full.txt
|
||||
@@ -73,23 +21,12 @@ Allow: /llms.mdx/
|
||||
# Sitemaps
|
||||
Sitemap: ${baseUrl}/sitemap.xml
|
||||
|
||||
# Crawl delay for aggressive bots (optional)
|
||||
# Crawl-delay: 1
|
||||
|
||||
# Additional resources for AI indexing
|
||||
# See https://github.com/AnswerDotAI/llms-txt for more info
|
||||
# LLM-friendly content:
|
||||
# Manifest: ${baseUrl}/llms.txt
|
||||
# Full content: ${baseUrl}/llms-full.txt
|
||||
# Individual pages: ${baseUrl}/llms.mdx/[page-path]
|
||||
|
||||
# Multi-language documentation available at:
|
||||
# ${baseUrl}/en - English
|
||||
# ${baseUrl}/es - Español
|
||||
# ${baseUrl}/fr - Français
|
||||
# ${baseUrl}/de - Deutsch
|
||||
# ${baseUrl}/ja - 日本語
|
||||
# ${baseUrl}/zh - 简体中文`
|
||||
# Individual pages: ${baseUrl}/llms.mdx/[page-path]`
|
||||
|
||||
return new Response(robotsTxt, {
|
||||
headers: {
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
import type { MetadataRoute } from 'next'
|
||||
import { i18n } from '@/lib/i18n'
|
||||
import { source } from '@/lib/source'
|
||||
import { DOCS_BASE_URL } from '@/lib/urls'
|
||||
|
||||
export const revalidate = 3600
|
||||
|
||||
export default function sitemap(): MetadataRoute.Sitemap {
|
||||
const baseUrl = DOCS_BASE_URL
|
||||
const languages = source.getLanguages()
|
||||
|
||||
const pagesBySlug = new Map<string, Map<string, string>>()
|
||||
for (const { language, pages } of languages) {
|
||||
for (const page of pages) {
|
||||
const key = page.slugs.join('/')
|
||||
if (!pagesBySlug.has(key)) {
|
||||
pagesBySlug.set(key, new Map())
|
||||
}
|
||||
pagesBySlug.get(key)!.set(language, `${baseUrl}${page.url}`)
|
||||
}
|
||||
}
|
||||
|
||||
const entries: MetadataRoute.Sitemap = []
|
||||
for (const [, localeMap] of pagesBySlug) {
|
||||
const defaultUrl = localeMap.get(i18n.defaultLanguage)
|
||||
if (!defaultUrl) continue
|
||||
|
||||
const langAlternates: Record<string, string> = {}
|
||||
for (const [lang, url] of localeMap) {
|
||||
langAlternates[lang] = url
|
||||
}
|
||||
|
||||
langAlternates['x-default'] = defaultUrl
|
||||
|
||||
entries.push({
|
||||
url: defaultUrl,
|
||||
alternates: { languages: langAlternates },
|
||||
})
|
||||
}
|
||||
|
||||
return entries
|
||||
}
|
||||
@@ -1,62 +0,0 @@
|
||||
import { i18n } from '@/lib/i18n'
|
||||
import { source } from '@/lib/source'
|
||||
|
||||
export const revalidate = 3600
|
||||
|
||||
export async function GET() {
|
||||
const baseUrl = 'https://docs.sim.ai'
|
||||
|
||||
const allPages = source.getPages()
|
||||
|
||||
const getPriority = (url: string): string => {
|
||||
if (url === '/introduction' || url === '/') return '1.0'
|
||||
if (url === '/getting-started') return '0.9'
|
||||
if (url.match(/^\/[^/]+$/)) return '0.8'
|
||||
if (url.includes('/sdks/') || url.includes('/tools/')) return '0.7'
|
||||
return '0.6'
|
||||
}
|
||||
|
||||
const urls = allPages
|
||||
.flatMap((page) => {
|
||||
const urlWithoutLang = page.url.replace(/^\/[a-z]{2}\//, '/')
|
||||
|
||||
return i18n.languages.map((lang) => {
|
||||
const url =
|
||||
lang === i18n.defaultLanguage
|
||||
? `${baseUrl}${urlWithoutLang}`
|
||||
: `${baseUrl}/${lang}${urlWithoutLang}`
|
||||
|
||||
return ` <url>
|
||||
<loc>${url}</loc>
|
||||
<priority>${getPriority(urlWithoutLang)}</priority>
|
||||
${i18n.languages.length > 1 ? generateAlternateLinks(baseUrl, urlWithoutLang) : ''}
|
||||
</url>`
|
||||
})
|
||||
})
|
||||
.join('\n')
|
||||
|
||||
const sitemap = `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:xhtml="http://www.w3.org/1999/xhtml">
|
||||
${urls}
|
||||
</urlset>`
|
||||
|
||||
return new Response(sitemap, {
|
||||
headers: {
|
||||
'Content-Type': 'application/xml',
|
||||
'Cache-Control': 'public, max-age=3600, s-maxage=3600',
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
function generateAlternateLinks(baseUrl: string, urlWithoutLang: string): string {
|
||||
const langLinks = i18n.languages
|
||||
.map((lang) => {
|
||||
const url =
|
||||
lang === i18n.defaultLanguage
|
||||
? `${baseUrl}${urlWithoutLang}`
|
||||
: `${baseUrl}/${lang}${urlWithoutLang}`
|
||||
return ` <xhtml:link rel="alternate" hreflang="${lang}" href="${url}" />`
|
||||
})
|
||||
.join('\n')
|
||||
return `${langLinks}\n <xhtml:link rel="alternate" hreflang="x-default" href="${baseUrl}${urlWithoutLang}" />`
|
||||
}
|
||||
@@ -1,3 +1,5 @@
|
||||
import { DOCS_BASE_URL } from '@/lib/urls'
|
||||
|
||||
interface StructuredDataProps {
|
||||
title: string
|
||||
description: string
|
||||
@@ -15,7 +17,7 @@ export function StructuredData({
|
||||
dateModified,
|
||||
breadcrumb,
|
||||
}: StructuredDataProps) {
|
||||
const baseUrl = 'https://docs.sim.ai'
|
||||
const baseUrl = DOCS_BASE_URL
|
||||
|
||||
const articleStructuredData = {
|
||||
'@context': 'https://schema.org',
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
export const DOCS_BASE_URL = process.env.NEXT_PUBLIC_DOCS_URL ?? 'https://docs.sim.ai'
|
||||
@@ -5,7 +5,6 @@ import { BrandedLayout } from '@/components/branded-layout'
|
||||
import { PostHogProvider } from '@/app/_shell/providers/posthog-provider'
|
||||
import { generateBrandedMetadata, generateThemeCSS } from '@/ee/whitelabeling'
|
||||
import '@/app/_styles/globals.css'
|
||||
import { OneDollarStats } from '@/components/analytics/onedollarstats'
|
||||
import { isHosted, isReactGrabEnabled, isReactScanEnabled } from '@/lib/core/config/feature-flags'
|
||||
import { HydrationErrorHandler } from '@/app/_shell/hydration-error-handler'
|
||||
import { QueryProvider } from '@/app/_shell/providers/query-provider'
|
||||
@@ -207,10 +206,6 @@ export default function RootLayout({ children }: { children: React.ReactNode })
|
||||
<meta name='format-detection' content='telephone=no' />
|
||||
<meta httpEquiv='x-ua-compatible' content='ie=edge' />
|
||||
|
||||
{/* OneDollarStats Analytics */}
|
||||
<link rel='dns-prefetch' href='https://assets.onedollarstats.com' />
|
||||
<script defer src='https://assets.onedollarstats.com/stonks.js' />
|
||||
|
||||
{/* Google Tag Manager — hosted only */}
|
||||
{isHosted && (
|
||||
<Script
|
||||
@@ -260,7 +255,6 @@ j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
|
||||
</noscript>
|
||||
)}
|
||||
<HydrationErrorHandler />
|
||||
<OneDollarStats />
|
||||
<PostHogProvider>
|
||||
<ThemeProvider>
|
||||
<QueryProvider>
|
||||
|
||||
+15
-121
@@ -4,133 +4,27 @@ import { getBaseUrl } from '@/lib/core/utils/urls'
|
||||
export default function robots(): MetadataRoute.Robots {
|
||||
const baseUrl = getBaseUrl()
|
||||
|
||||
const disallowedPaths = [
|
||||
'/api/',
|
||||
'/workspace/',
|
||||
'/chat/',
|
||||
'/playground/',
|
||||
'/resume/',
|
||||
'/invite/',
|
||||
'/unsubscribe/',
|
||||
'/w/',
|
||||
'/_next/',
|
||||
'/private/',
|
||||
]
|
||||
|
||||
return {
|
||||
rules: [
|
||||
{
|
||||
userAgent: '*',
|
||||
allow: '/',
|
||||
disallow: disallowedPaths,
|
||||
},
|
||||
{
|
||||
userAgent: 'Googlebot',
|
||||
allow: '/',
|
||||
disallow: disallowedPaths,
|
||||
},
|
||||
{
|
||||
userAgent: 'Bingbot',
|
||||
allow: '/',
|
||||
disallow: disallowedPaths,
|
||||
},
|
||||
{
|
||||
userAgent: 'YandexBot',
|
||||
allow: '/',
|
||||
disallow: disallowedPaths,
|
||||
},
|
||||
{
|
||||
userAgent: 'Baiduspider',
|
||||
allow: '/',
|
||||
disallow: disallowedPaths,
|
||||
},
|
||||
{
|
||||
userAgent: 'GPTBot',
|
||||
allow: '/',
|
||||
disallow: disallowedPaths,
|
||||
},
|
||||
{
|
||||
userAgent: 'ChatGPT-User',
|
||||
allow: '/',
|
||||
disallow: disallowedPaths,
|
||||
},
|
||||
{
|
||||
userAgent: 'OAI-SearchBot',
|
||||
allow: '/',
|
||||
disallow: disallowedPaths,
|
||||
},
|
||||
{
|
||||
userAgent: 'ClaudeBot',
|
||||
allow: '/',
|
||||
disallow: disallowedPaths,
|
||||
},
|
||||
{
|
||||
userAgent: 'Claude-SearchBot',
|
||||
allow: '/',
|
||||
disallow: disallowedPaths,
|
||||
},
|
||||
{
|
||||
userAgent: 'Google-Extended',
|
||||
allow: '/',
|
||||
disallow: disallowedPaths,
|
||||
},
|
||||
{
|
||||
userAgent: 'PerplexityBot',
|
||||
allow: '/',
|
||||
disallow: disallowedPaths,
|
||||
},
|
||||
{
|
||||
userAgent: 'Meta-ExternalAgent',
|
||||
allow: '/',
|
||||
disallow: disallowedPaths,
|
||||
},
|
||||
{
|
||||
userAgent: 'FacebookBot',
|
||||
allow: '/',
|
||||
disallow: disallowedPaths,
|
||||
},
|
||||
{
|
||||
userAgent: 'Applebot',
|
||||
allow: '/',
|
||||
disallow: disallowedPaths,
|
||||
},
|
||||
{
|
||||
userAgent: 'Applebot-Extended',
|
||||
allow: '/',
|
||||
disallow: disallowedPaths,
|
||||
},
|
||||
{
|
||||
userAgent: 'Amazonbot',
|
||||
allow: '/',
|
||||
disallow: disallowedPaths,
|
||||
},
|
||||
{
|
||||
userAgent: 'Bytespider',
|
||||
allow: '/',
|
||||
disallow: disallowedPaths,
|
||||
},
|
||||
{
|
||||
userAgent: 'CCBot',
|
||||
allow: '/',
|
||||
disallow: disallowedPaths,
|
||||
},
|
||||
{
|
||||
userAgent: 'cohere-ai',
|
||||
allow: '/',
|
||||
disallow: disallowedPaths,
|
||||
},
|
||||
{
|
||||
userAgent: 'Grok-web-crawl',
|
||||
allow: '/',
|
||||
disallow: disallowedPaths,
|
||||
},
|
||||
{
|
||||
userAgent: 'DeepSeek-AI',
|
||||
allow: '/',
|
||||
disallow: disallowedPaths,
|
||||
disallow: [
|
||||
'/api/',
|
||||
'/workspace/',
|
||||
'/chat/',
|
||||
'/playground/',
|
||||
'/resume/',
|
||||
'/invite/',
|
||||
'/unsubscribe/',
|
||||
'/w/',
|
||||
'/form/',
|
||||
'/credential-account/',
|
||||
'/_next/',
|
||||
'/private/',
|
||||
],
|
||||
},
|
||||
],
|
||||
sitemap: `${baseUrl}/sitemap.xml`,
|
||||
host: baseUrl,
|
||||
sitemap: [`${baseUrl}/sitemap.xml`, `${baseUrl}/blog/sitemap-images.xml`],
|
||||
}
|
||||
}
|
||||
|
||||
+72
-55
@@ -1,4 +1,5 @@
|
||||
import type { MetadataRoute } from 'next'
|
||||
import { COURSES } from '@/lib/academy/content'
|
||||
import { getAllPostMeta } from '@/lib/blog/registry'
|
||||
import { getBaseUrl } from '@/lib/core/utils/urls'
|
||||
import integrations from '@/app/(landing)/integrations/data/integrations.json'
|
||||
@@ -6,69 +7,44 @@ import { ALL_CATALOG_MODELS, MODEL_PROVIDERS_WITH_CATALOGS } from '@/app/(landin
|
||||
|
||||
export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
|
||||
const baseUrl = getBaseUrl()
|
||||
const posts = await getAllPostMeta()
|
||||
|
||||
const now = new Date()
|
||||
const integrationPages: MetadataRoute.Sitemap = integrations.map((integration) => ({
|
||||
url: `${baseUrl}/integrations/${integration.slug}`,
|
||||
lastModified: now,
|
||||
changeFrequency: 'monthly',
|
||||
priority: 0.6,
|
||||
}))
|
||||
const modelHubPages: MetadataRoute.Sitemap = [
|
||||
{
|
||||
url: `${baseUrl}/integrations`,
|
||||
lastModified: now,
|
||||
changeFrequency: 'weekly',
|
||||
priority: 0.8,
|
||||
},
|
||||
{
|
||||
url: `${baseUrl}/models`,
|
||||
lastModified: now,
|
||||
changeFrequency: 'weekly',
|
||||
priority: 0.8,
|
||||
},
|
||||
{
|
||||
url: `${baseUrl}/partners`,
|
||||
lastModified: now,
|
||||
changeFrequency: 'monthly',
|
||||
priority: 0.5,
|
||||
},
|
||||
]
|
||||
const providerPages: MetadataRoute.Sitemap = MODEL_PROVIDERS_WITH_CATALOGS.map((provider) => ({
|
||||
url: `${baseUrl}${provider.href}`,
|
||||
lastModified: new Date(
|
||||
Math.max(...provider.models.map((model) => new Date(model.pricing.updatedAt).getTime()))
|
||||
),
|
||||
changeFrequency: 'weekly',
|
||||
priority: 0.7,
|
||||
}))
|
||||
const modelPages: MetadataRoute.Sitemap = ALL_CATALOG_MODELS.map((model) => ({
|
||||
url: `${baseUrl}${model.href}`,
|
||||
lastModified: new Date(model.pricing.updatedAt),
|
||||
changeFrequency: 'monthly',
|
||||
priority: 0.6,
|
||||
}))
|
||||
const latestPostDate =
|
||||
posts.length > 0
|
||||
? new Date(Math.max(...posts.map((p) => new Date(p.updated ?? p.date).getTime())))
|
||||
: undefined
|
||||
|
||||
const modelTimes = MODEL_PROVIDERS_WITH_CATALOGS.flatMap((provider) =>
|
||||
provider.models.map((model) => new Date(model.pricing.updatedAt).getTime())
|
||||
)
|
||||
const latestModelDate = modelTimes.length > 0 ? new Date(Math.max(...modelTimes)) : undefined
|
||||
|
||||
const staticPages: MetadataRoute.Sitemap = [
|
||||
{
|
||||
url: baseUrl,
|
||||
lastModified: now,
|
||||
changeFrequency: 'daily',
|
||||
priority: 1.0,
|
||||
},
|
||||
{
|
||||
url: `${baseUrl}/blog`,
|
||||
lastModified: now,
|
||||
changeFrequency: 'daily',
|
||||
priority: 0.8,
|
||||
lastModified: latestPostDate,
|
||||
},
|
||||
{
|
||||
url: `${baseUrl}/blog/tags`,
|
||||
lastModified: now,
|
||||
lastModified: latestPostDate,
|
||||
},
|
||||
{
|
||||
url: `${baseUrl}/changelog`,
|
||||
lastModified: now,
|
||||
lastModified: latestPostDate,
|
||||
},
|
||||
{
|
||||
url: `${baseUrl}/integrations`,
|
||||
lastModified: latestModelDate,
|
||||
},
|
||||
{
|
||||
url: `${baseUrl}/models`,
|
||||
lastModified: latestModelDate,
|
||||
},
|
||||
{
|
||||
url: `${baseUrl}/partners`,
|
||||
},
|
||||
{
|
||||
url: `${baseUrl}/terms`,
|
||||
@@ -80,20 +56,61 @@ export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
|
||||
},
|
||||
]
|
||||
|
||||
const posts = await getAllPostMeta()
|
||||
const blogPages: MetadataRoute.Sitemap = posts.map((p) => ({
|
||||
url: p.canonical,
|
||||
lastModified: new Date(p.updated ?? p.date),
|
||||
changeFrequency: 'weekly',
|
||||
priority: 0.7,
|
||||
}))
|
||||
|
||||
const authorsMap = new Map<string, Date>()
|
||||
for (const p of posts) {
|
||||
for (const author of p.authors ?? [p.author]) {
|
||||
const postDate = new Date(p.updated ?? p.date)
|
||||
const existing = authorsMap.get(author.id)
|
||||
if (!existing || postDate > existing) {
|
||||
authorsMap.set(author.id, postDate)
|
||||
}
|
||||
}
|
||||
}
|
||||
const authorPages: MetadataRoute.Sitemap = [...authorsMap.entries()].map(([id, date]) => ({
|
||||
url: `${baseUrl}/blog/authors/${id}`,
|
||||
lastModified: date,
|
||||
}))
|
||||
|
||||
const integrationPages: MetadataRoute.Sitemap = integrations.map((integration) => ({
|
||||
url: `${baseUrl}/integrations/${integration.slug}`,
|
||||
}))
|
||||
|
||||
const providerPages: MetadataRoute.Sitemap = MODEL_PROVIDERS_WITH_CATALOGS.flatMap((provider) => {
|
||||
if (provider.models.length === 0) return []
|
||||
return [
|
||||
{
|
||||
url: `${baseUrl}${provider.href}`,
|
||||
lastModified: new Date(
|
||||
Math.max(...provider.models.map((model) => new Date(model.pricing.updatedAt).getTime()))
|
||||
),
|
||||
},
|
||||
]
|
||||
})
|
||||
|
||||
const modelEntries: MetadataRoute.Sitemap = ALL_CATALOG_MODELS.map((model) => ({
|
||||
url: `${baseUrl}${model.href}`,
|
||||
lastModified: new Date(model.pricing.updatedAt),
|
||||
}))
|
||||
|
||||
const academyPages: MetadataRoute.Sitemap = [
|
||||
{ url: `${baseUrl}/academy` },
|
||||
...COURSES.map((course) => ({
|
||||
url: `${baseUrl}/academy/${course.slug}`,
|
||||
})),
|
||||
]
|
||||
|
||||
return [
|
||||
...staticPages,
|
||||
...modelHubPages,
|
||||
...blogPages,
|
||||
...authorPages,
|
||||
...integrationPages,
|
||||
...providerPages,
|
||||
...modelPages,
|
||||
...blogPages,
|
||||
...modelEntries,
|
||||
...academyPages,
|
||||
]
|
||||
}
|
||||
|
||||
@@ -1,26 +0,0 @@
|
||||
'use client'
|
||||
|
||||
import { useEffect } from 'react'
|
||||
import { env } from '@/lib/core/config/env'
|
||||
|
||||
export function OneDollarStats() {
|
||||
useEffect(() => {
|
||||
const shouldInitialize = !!env.DRIZZLE_ODS_API_KEY
|
||||
|
||||
if (!shouldInitialize) {
|
||||
return
|
||||
}
|
||||
|
||||
import('onedollarstats')
|
||||
.then(({ configure }) => {
|
||||
configure({
|
||||
collectorUrl: 'https://collector.onedollarstats.com/events',
|
||||
autocollect: true,
|
||||
hashRouting: true,
|
||||
})
|
||||
})
|
||||
.catch(() => {})
|
||||
}, [])
|
||||
|
||||
return null
|
||||
}
|
||||
@@ -132,7 +132,6 @@ export const env = createEnv({
|
||||
TELEMETRY_ENDPOINT: z.string().url().optional(), // Custom telemetry/analytics endpoint
|
||||
COST_MULTIPLIER: z.number().optional(), // Multiplier for cost calculations
|
||||
LOG_LEVEL: z.enum(['DEBUG', 'INFO', 'WARN', 'ERROR']).optional(), // Minimum log level to display (defaults to ERROR in production, DEBUG in development)
|
||||
DRIZZLE_ODS_API_KEY: z.string().min(1).optional(), // OneDollarStats API key for analytics tracking
|
||||
PROFOUND_API_KEY: z.string().min(1).optional(), // Profound analytics API key
|
||||
PROFOUND_ENDPOINT: z.string().url().optional(), // Profound analytics endpoint
|
||||
|
||||
|
||||
@@ -38,7 +38,6 @@ const STATIC_SCRIPT_SRC = [
|
||||
"'unsafe-inline'",
|
||||
'https://*.google.com',
|
||||
'https://apis.google.com',
|
||||
'https://assets.onedollarstats.com',
|
||||
'https://challenges.cloudflare.com',
|
||||
...(isReactGrabEnabled ? ['https://unpkg.com'] : []),
|
||||
...(isHosted
|
||||
@@ -64,7 +63,6 @@ const STATIC_IMG_SRC = [
|
||||
'https://*.amazonaws.com',
|
||||
'https://*.blob.core.windows.net',
|
||||
'https://github.com/*',
|
||||
'https://collector.onedollarstats.com',
|
||||
'https://cursor.com',
|
||||
...(isHosted ? ['https://www.googletagmanager.com', 'https://www.google-analytics.com'] : []),
|
||||
] as const
|
||||
@@ -85,7 +83,6 @@ const STATIC_CONNECT_SRC = [
|
||||
'https://api.github.com',
|
||||
'https://github.com/*',
|
||||
'https://challenges.cloudflare.com',
|
||||
'https://collector.onedollarstats.com',
|
||||
...(isHosted
|
||||
? [
|
||||
'https://www.googletagmanager.com',
|
||||
|
||||
@@ -155,7 +155,6 @@
|
||||
"next-themes": "^0.4.6",
|
||||
"nodemailer": "7.0.11",
|
||||
"officeparser": "^5.2.0",
|
||||
"onedollarstats": "0.0.10",
|
||||
"openai": "^4.91.1",
|
||||
"papaparse": "5.5.3",
|
||||
"pdf-lib": "1.17.1",
|
||||
|
||||
@@ -179,7 +179,6 @@
|
||||
"next-themes": "^0.4.6",
|
||||
"nodemailer": "7.0.11",
|
||||
"officeparser": "^5.2.0",
|
||||
"onedollarstats": "0.0.10",
|
||||
"openai": "^4.91.1",
|
||||
"papaparse": "5.5.3",
|
||||
"pdf-lib": "1.17.1",
|
||||
@@ -3123,8 +3122,6 @@
|
||||
|
||||
"once": ["once@1.4.0", "", { "dependencies": { "wrappy": "1" } }, "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w=="],
|
||||
|
||||
"onedollarstats": ["onedollarstats@0.0.10", "", {}, "sha512-+s2o5qBuKej2BrbJDqVRZr9U7F0ERBsNjXIJs1DSy2yK4yNk8z5iM0nHuwhelbNgqyVEwckCV7BJ9MsP/c8kQw=="],
|
||||
|
||||
"onetime": ["onetime@6.0.0", "", { "dependencies": { "mimic-fn": "^4.0.0" } }, "sha512-1FlR+gjXK7X+AsAHso35MnyN5KqGwJRi/31ft6x0M194ht7S+rWAvd7PHss9xSKMzE0asv1pyIHaJYq+BbacAQ=="],
|
||||
|
||||
"oniguruma-parser": ["oniguruma-parser@0.12.1", "", {}, "sha512-8Unqkvk1RYc6yq2WBYRj4hdnsAxVze8i7iPfQr8e4uSP3tRv0rpZcbGUDvxfQQcdwHt/e9PrMvGCsa8OqG9X3w=="],
|
||||
|
||||
Reference in New Issue
Block a user