opensource/kilocode
3
0
Fork 0
mirror of https://github.com/Kilo-Org/kilocode.git synced 2026-08-28 11:05:31 +08:00
Code Issues Packages Projects Releases Wiki Activity

chore(monorepo): remove stats package and align upstream skip rules

Browse Source 
Drop the entire packages/stats workspace (app, core, server) and add
packages/stats/** to the upstream merge skip list so future syncs no
longer pull it in. Update skip-files tests to cover the new glob.

Dependency changes for opencode v1.15.13:
- add ws 8.21.0 and @types/ws for WebSocket transport
- bump @lydell/node-pty to 1.2.0-beta.12
- add proxy-env and googleapis source links
This commit is contained in:
Imanol Maiztegui
2026-06-19 11:01:30 +02:00
parent 229068188b
commit 351c2e4fc7
73 changed files with 131 additions and 18046 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bun.lock
+55 -13
View File
@@ -34,15 +34,15 @@
},
"dependencies": {
"@ai-sdk/alibaba": "1.0.17",
"@ai-sdk/amazon-bedrock": "4.0.96",
"@ai-sdk/amazon-bedrock": "4.0.107",
"@ai-sdk/anthropic": "3.0.71",
"@ai-sdk/azure": "3.0.49",
"@ai-sdk/cerebras": "2.0.54",
"@ai-sdk/cohere": "3.0.27",
"@ai-sdk/deepinfra": "2.0.41",
"@ai-sdk/gateway": "3.0.104",
"@ai-sdk/google": "3.0.63",
"@ai-sdk/google-vertex": "4.0.112",
"@ai-sdk/google": "3.0.73",
"@ai-sdk/google-vertex": "4.0.128",
"@ai-sdk/groq": "3.0.31",
"@ai-sdk/mistral": "3.0.27",
"@ai-sdk/openai": "3.0.53",
@@ -67,16 +67,17 @@
"ai-gateway-provider": "3.1.2",
"cross-spawn": "catalog:",
"effect": "catalog:",
"gitlab-ai-provider": "6.7.0",
"gitlab-ai-provider": "6.8.0",
"glob": "13.0.5",
"google-auth-library": "10.5.0",
"immer": "11.1.4",
"jsonc-parser": "3.3.1",
"mime-types": "3.0.2",
"minimatch": "10.2.5",
"npm-package-arg": "13.0.2",
"rotating-file-stream": "3.2.9",
"semver": "^7.6.3",
"venice-ai-sdk-provider": "2.0.1",
"venice-ai-sdk-provider": "2.0.2",
"xdg-basedir": "5.1.0",
"zod": "catalog:",
},
@@ -86,6 +87,7 @@
"@types/cross-spawn": "catalog:",
"@types/npm-package-arg": "6.1.4",
"@types/npmcli__arborist": "6.3.3",
"@types/semver": "catalog:",
},
},
"packages/effect-drizzle-sqlite": {
@@ -466,6 +468,7 @@
"@silvia-odwyer/photon-node": "0.3.4",
"@solid-primitives/event-bus": "1.1.2",
"@solid-primitives/scheduled": "1.5.2",
"@types/ws": "8.18.1",
"@zip.js/zip.js": "2.7.62",
"ai": "catalog:",
"ai-gateway-provider": "3.1.2",
@@ -510,6 +513,7 @@
"vscode-jsonrpc": "8.2.1",
"web-tree-sitter": "0.25.10",
"which": "6.0.1",
"ws": "8.21.0",
"xlsx": "https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz",
"yargs": "18.0.0",
"zod": "catalog:",
@@ -723,7 +727,7 @@
"@effect/platform-node": "4.0.0-beta.66",
"@effect/sql-sqlite-bun": "4.0.0-beta.66",
"@kobalte/core": "0.13.11",
"@lydell/node-pty": "1.2.0-beta.10",
"@lydell/node-pty": "1.2.0-beta.12",
"@npmcli/arborist": "9.4.0",
"@octokit/rest": "22.0.0",
"@openauthjs/openauth": "0.0.0-20250322224806",
@@ -1338,19 +1342,19 @@
"@lukeed/ms": ["@lukeed/ms@2.0.2", "", {}, "sha512-9I2Zn6+NJLfaGoz9jN3lpwDgAYvfGeNYdbAIjJOqzs4Tpc+VU3Jqq4IofSUBKajiDS8k9fZIg18/z13mpk1bsA=="],
"@lydell/node-pty": ["@lydell/node-pty@1.2.0-beta.10", "", { "optionalDependencies": { "@lydell/node-pty-darwin-arm64": "1.2.0-beta.10", "@lydell/node-pty-darwin-x64": "1.2.0-beta.10", "@lydell/node-pty-linux-arm64": "1.2.0-beta.10", "@lydell/node-pty-linux-x64": "1.2.0-beta.10", "@lydell/node-pty-win32-arm64": "1.2.0-beta.10", "@lydell/node-pty-win32-x64": "1.2.0-beta.10" } }, "sha512-Fv+A3+MZVA8qhkBIZsM1E6dCdHNMyXXz22mAYiMWd03LlyK///F3OH6CKPX9mj4id7LUlxpr45yPzyBVy9aDPw=="],
"@lydell/node-pty": ["@lydell/node-pty@1.2.0-beta.12", "", { "optionalDependencies": { "@lydell/node-pty-darwin-arm64": "1.2.0-beta.12", "@lydell/node-pty-darwin-x64": "1.2.0-beta.12", "@lydell/node-pty-linux-arm64": "1.2.0-beta.12", "@lydell/node-pty-linux-x64": "1.2.0-beta.12", "@lydell/node-pty-win32-arm64": "1.2.0-beta.12", "@lydell/node-pty-win32-x64": "1.2.0-beta.12" } }, "sha512-qIK890UwPupoj07osVvgOIa++1mxeHbcGry4PKRHhNVNs81V2SCG34eJr46GybiOmBtc8Sj5PB1/GGM5PL549g=="],
"@lydell/node-pty-darwin-arm64": ["@lydell/node-pty-darwin-arm64@1.2.0-beta.10", "", { "os": "darwin", "cpu": "arm64" }, "sha512-C+eqDyRNHRYvx7RaHj6VVCx6nCpRBPuuxhTcc3JH3GuBMoxTsYeY4GkWH2XOktrgbAq1BG8e/Y8bu/wNQreCEw=="],
"@lydell/node-pty-darwin-arm64": ["@lydell/node-pty-darwin-arm64@1.2.0-beta.12", "", { "os": "darwin", "cpu": "arm64" }, "sha512-tqaifcY9Cr41SblO1+FLzh8oxxtkNhuW9Dhl22lKme9BreYvKvxEZcdPIXTuqkJc5tagOEC4QHShKmJjLyLXLQ=="],
"@lydell/node-pty-darwin-x64": ["@lydell/node-pty-darwin-x64@1.2.0-beta.10", "", { "os": "darwin", "cpu": "x64" }, "sha512-aZoIK6HtJO5BiT4ELm683U4dyHtt8b7wNgq3NJqYAQwSXrcPv576Z8vY3BIulVxfcFkht/SPLKou9TtdFXdNpg=="],
"@lydell/node-pty-darwin-x64": ["@lydell/node-pty-darwin-x64@1.2.0-beta.12", "", { "os": "darwin", "cpu": "x64" }, "sha512-4LrS5pCJwqHKDVf1zS2gyNV0m4hKAXch+XZNhbZ6LY8uwVL8BhchzQBO40Os5anuRxRCWzHpw4Sp64Ie8q7E4Q=="],
"@lydell/node-pty-linux-arm64": ["@lydell/node-pty-linux-arm64@1.2.0-beta.10", "", { "os": "linux", "cpu": "arm64" }, "sha512-0cKX2iMyXFNBE4fGtGK6B7IkdXcDMZajyEDoGMOgQQs/DDtoI5tSPcBcqNY9VitVrsRQA8+gFt6eKYU9Ye/lUA=="],
"@lydell/node-pty-linux-arm64": ["@lydell/node-pty-linux-arm64@1.2.0-beta.12", "", { "os": "linux", "cpu": "arm64" }, "sha512-Sx+A71x5BDGHt9ansfrtGxwq2VFVDWvJUAdlUL0Hv0qeiJUfts+hgopx+CgT4PSwahKjdEgtu0+FAfY9rICKRw=="],
"@lydell/node-pty-linux-x64": ["@lydell/node-pty-linux-x64@1.2.0-beta.10", "", { "os": "linux", "cpu": "x64" }, "sha512-J9HnxvSzEeMH748+Ul1VrmCLWMo7iCVJy9EGijRR62+YO/Yk5GaCydUTZ+KzlH0/X5aTrgt5cfiof4vx45tRRg=="],
"@lydell/node-pty-linux-x64": ["@lydell/node-pty-linux-x64@1.2.0-beta.12", "", { "os": "linux", "cpu": "x64" }, "sha512-bJzs94njofYhGg/UDqW1nj0dtvvu+2OvxMY+RlLS1T17VgcktKoIR6PuenTwE5HJ/D6StCPADmXcT0nNsCKmIQ=="],
"@lydell/node-pty-win32-arm64": ["@lydell/node-pty-win32-arm64@1.2.0-beta.10", "", { "os": "win32", "cpu": "arm64" }, "sha512-PlDJpJX/pnKyy6OmADKzhf+INZDDnzTBGaI0LT4laVNc6NblZNqUSkCMjLFWbeakeuQp0VG37M49WQSN9FDfeA=="],
"@lydell/node-pty-win32-arm64": ["@lydell/node-pty-win32-arm64@1.2.0-beta.12", "", { "os": "win32", "cpu": "arm64" }, "sha512-p7POgjVEiFaBC3/y+AKuV1FzePCsJ6HmZDv2XK+jBZSfwP8+uBAw181ZiKYN1YuRa/XpmBGaWezcI8hZkbW++g=="],
"@lydell/node-pty-win32-x64": ["@lydell/node-pty-win32-x64@1.2.0-beta.10", "", { "os": "win32", "cpu": "x64" }, "sha512-ExFgWrzyldNAMi45U9PLIOu+g/RatP+f0c/dZxaooifME6yLW32BoHveH26/TtoAjZyJrc2iL0u48pgnR1fzmg=="],
"@lydell/node-pty-win32-x64": ["@lydell/node-pty-win32-x64@1.2.0-beta.12", "", { "os": "win32", "cpu": "x64" }, "sha512-IDFa00g7qUDGUYgByrUBJtC+mOjYVt/8KYyWivCg5JjGOHbBUACUQZLl0jTWmnr+tld/UyTpX90a2PY6oTVtRw=="],
"@manypkg/find-root": ["@manypkg/find-root@1.1.0", "", { "dependencies": { "@babel/runtime": "^7.5.5", "@types/node": "^12.7.1", "find-up": "^4.1.0", "fs-extra": "^8.1.0" } }, "sha512-mki5uBvhHzO8kYYix/WRy2WX8S3B5wdVSc9D6KcU5lQNglP2yt58/VfLuAK49glRXChosY8ap2oJ1qgma3GUVA=="],
@@ -4498,8 +4502,18 @@
"@octokit/rest/@octokit/plugin-rest-endpoint-methods": ["@octokit/plugin-rest-endpoint-methods@16.1.1", "", { "dependencies": { "@octokit/types": "^15.0.1" }, "peerDependencies": { "@octokit/core": ">=6" } }, "sha512-VztDkhM0ketQYSh5Im3IcKWFZl7VIrrsCaHbDINkdYeiiAsJzjhS2xRFCSJgfN6VOcsoW4laMtsmf3HcNqIimg=="],
"@opencode-ai/core/@ai-sdk/amazon-bedrock": ["@ai-sdk/amazon-bedrock@4.0.107", "", { "dependencies": { "@ai-sdk/anthropic": "3.0.78", "@ai-sdk/provider": "3.0.10", "@ai-sdk/provider-utils": "4.0.27", "@smithy/eventstream-codec": "^4.0.1", "@smithy/util-utf8": "^4.0.0", "aws4fetch": "^1.0.20" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-8nT08pGPy25rleJNk56ep00UHK6kCtCmu+ZNqVVSSPDieADlIZqcaN1iRXAFBoCH0Fb9F6C2EjFDaySdsargfQ=="],
"@opencode-ai/core/@ai-sdk/google": ["@ai-sdk/google@3.0.73", "", { "dependencies": { "@ai-sdk/provider": "3.0.10", "@ai-sdk/provider-utils": "4.0.27" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-o2MuIeyvZrFIeIbnbA8Thrr63irdyUBh0uWBZ2lY6yFeXuE/tcwyXF74bDKS4KvTu84uFpQfpbS/LXHGKKXz+g=="],
"@opencode-ai/core/@ai-sdk/google-vertex": ["@ai-sdk/google-vertex@4.0.128", "", { "dependencies": { "@ai-sdk/anthropic": "3.0.77", "@ai-sdk/google": "3.0.73", "@ai-sdk/openai-compatible": "2.0.47", "@ai-sdk/provider": "3.0.10", "@ai-sdk/provider-utils": "4.0.27", "google-auth-library": "^10.5.0" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-jK8fixb4km2yfgvb9DUFQRpV/jiDB0v9gyxHoHfPydaQvz+CpAz8DTt1quyaM+Wg9G2R8Zo68CYmHbIkUqW2AA=="],
"@opencode-ai/core/@aws-sdk/credential-providers": ["@aws-sdk/credential-providers@3.993.0", "", { "dependencies": { "@aws-sdk/client-cognito-identity": "3.993.0", "@aws-sdk/core": "^3.973.11", "@aws-sdk/credential-provider-cognito-identity": "^3.972.3", "@aws-sdk/credential-provider-env": "^3.972.9", "@aws-sdk/credential-provider-http": "^3.972.11", "@aws-sdk/credential-provider-ini": "^3.972.9", "@aws-sdk/credential-provider-login": "^3.972.9", "@aws-sdk/credential-provider-node": "^3.972.10", "@aws-sdk/credential-provider-process": "^3.972.9", "@aws-sdk/credential-provider-sso": "^3.972.9", "@aws-sdk/credential-provider-web-identity": "^3.972.9", "@aws-sdk/nested-clients": "3.993.0", "@aws-sdk/types": "^3.973.1", "@smithy/config-resolver": "^4.4.6", "@smithy/core": "^3.23.2", "@smithy/credential-provider-imds": "^4.2.8", "@smithy/node-config-provider": "^4.3.8", "@smithy/property-provider": "^4.2.8", "@smithy/types": "^4.12.0", "tslib": "^2.6.2" } }, "sha512-1M/nukgPSLqe9krzOKHnE8OylUaKAiokAV3xRLdeExVHcRE7WG5uzCTKWTj1imKvPjDqXq/FWhlbbdWIn7xIwA=="],
"@opencode-ai/core/gitlab-ai-provider": ["gitlab-ai-provider@6.8.0", "", { "dependencies": { "@anthropic-ai/sdk": "^0.71.0", "@anycable/core": "^0.9.2", "graphql-request": "^6.1.0", "isomorphic-ws": "^5.0.0", "openai": "^6.16.0", "socket.io-client": "^4.8.1", "vscode-jsonrpc": "^8.2.1", "zod": "^3.25.76" }, "peerDependencies": { "@ai-sdk/provider": ">=3.0.0", "@ai-sdk/provider-utils": ">=4.0.0" } }, "sha512-KwHASXkHtDcgrzTXZVp9Dyx6t8m9nK0R2fCm47MWcxxQ1kOBt3f2LZugtu1kOby8i4Sbd+kvBSYM66PGkDclng=="],
"@opencode-ai/core/venice-ai-sdk-provider": ["venice-ai-sdk-provider@2.0.2", "", { "dependencies": { "@ai-sdk/openai-compatible": "^2.0.47", "@ai-sdk/provider": "^3.0.10", "@ai-sdk/provider-utils": "^4.0.27" }, "peerDependencies": { "ai": "^6.0.90" } }, "sha512-aoa05nI3BTK5aGbjBflq+Gfln2AHAkwNbWuGGvCzUIsOfp5Y3iPD4O4PUGDAEiWVJWbjpPn0KfDa0H/HebwsaA=="],
"@opencode-ai/storybook/@storybook/addon-a11y": ["@storybook/addon-a11y@10.4.4", "", { "dependencies": { "@storybook/global": "^5.0.0", "axe-core": "^4.2.0" }, "peerDependencies": { "storybook": "^10.4.4" } }, "sha512-/eUCx/6Ozq5grauwm/NqKtlW0oJ26b6GNesXrMuFID8WLg/qLEKf79Awfz9XrmyWxe7loD40K952r7AA5Oc23A=="],
"@opencode-ai/storybook/@storybook/addon-docs": ["@storybook/addon-docs@10.4.4", "", { "dependencies": { "@mdx-js/react": "^3.0.0", "@storybook/csf-plugin": "10.4.4", "@storybook/icons": "^2.0.2", "@storybook/react-dom-shim": "10.4.4", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "ts-dedent": "^2.0.0" }, "peerDependencies": { "@types/react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "storybook": "^10.4.4" }, "optionalPeers": ["@types/react"] }, "sha512-yPshCvtmQTq52T2sXuXgjy7B/QbhA/WIZxLYggptNjBL8BJMvbOfp9bAfCKh7+KpRWGqDZ6Y6tWL1Q48Wj3vtw=="],
@@ -4958,10 +4972,38 @@
"@octokit/rest/@octokit/core/before-after-hook": ["before-after-hook@4.0.0", "", {}, "sha512-q6tR3RPqIB1pMiTRMFcZwuG5T8vwp+vUvEG0vuI6B+Rikh5BfPp2fQ82c925FOs+b0lcFQ8CFrL+KbilfZFhOQ=="],
"@opencode-ai/core/@ai-sdk/amazon-bedrock/@ai-sdk/anthropic": ["@ai-sdk/anthropic@3.0.78", "", { "dependencies": { "@ai-sdk/provider": "3.0.10", "@ai-sdk/provider-utils": "4.0.27" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-0OY12G20cUt6iU6htpEA1491Oz++NVxZxlmWGX4B7rSbeZ5pnDmOu6YtW9BKzdZlNx5Gn23i6WMxyZFoMKNcgA=="],
"@opencode-ai/core/@ai-sdk/amazon-bedrock/@ai-sdk/provider": ["@ai-sdk/provider@3.0.10", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-Q3BZ27qfpYqnCYGvE3vt+Qi6LGOF9R5Nmzn+9JoM1lCRsD9mYaIhfJLkSunN48nfGXJ6n+XNV0J/XVpqGQl7Dw=="],
"@opencode-ai/core/@ai-sdk/amazon-bedrock/@ai-sdk/provider-utils": ["@ai-sdk/provider-utils@4.0.27", "", { "dependencies": { "@ai-sdk/provider": "3.0.10", "@standard-schema/spec": "^1.1.0", "eventsource-parser": "^3.0.8" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-ubkAJ+xODouwtmN1tYlvTPphH1hPOBfZaEQe8U7skGvFAnIRs9PPpsq57bC2+Ky/MB4yzhd6YOsxTAx9sGpazw=="],
"@opencode-ai/core/@ai-sdk/google/@ai-sdk/provider": ["@ai-sdk/provider@3.0.10", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-Q3BZ27qfpYqnCYGvE3vt+Qi6LGOF9R5Nmzn+9JoM1lCRsD9mYaIhfJLkSunN48nfGXJ6n+XNV0J/XVpqGQl7Dw=="],
"@opencode-ai/core/@ai-sdk/google/@ai-sdk/provider-utils": ["@ai-sdk/provider-utils@4.0.27", "", { "dependencies": { "@ai-sdk/provider": "3.0.10", "@standard-schema/spec": "^1.1.0", "eventsource-parser": "^3.0.8" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-ubkAJ+xODouwtmN1tYlvTPphH1hPOBfZaEQe8U7skGvFAnIRs9PPpsq57bC2+Ky/MB4yzhd6YOsxTAx9sGpazw=="],
"@opencode-ai/core/@ai-sdk/google-vertex/@ai-sdk/anthropic": ["@ai-sdk/anthropic@3.0.77", "", { "dependencies": { "@ai-sdk/provider": "3.0.10", "@ai-sdk/provider-utils": "4.0.27" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-ML8C2M1YvPA1ulEx4TiyF0k1xvC2ikEiPBIC1PPQ0a5xELUGrO2lAaEzsTEoJ+eCeDd8PSBuFJjs+r+9yIwQXA=="],
"@opencode-ai/core/@ai-sdk/google-vertex/@ai-sdk/openai-compatible": ["@ai-sdk/openai-compatible@2.0.47", "", { "dependencies": { "@ai-sdk/provider": "3.0.10", "@ai-sdk/provider-utils": "4.0.27" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-Enm5UlL0zUCrW3792opk5h7hRWxZOZzDe6eQYVFqX9LUOGGCe1h8MZWAGim765nwzgnjlpeYOsuzZmLtRsTPlg=="],
"@opencode-ai/core/@ai-sdk/google-vertex/@ai-sdk/provider": ["@ai-sdk/provider@3.0.10", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-Q3BZ27qfpYqnCYGvE3vt+Qi6LGOF9R5Nmzn+9JoM1lCRsD9mYaIhfJLkSunN48nfGXJ6n+XNV0J/XVpqGQl7Dw=="],
"@opencode-ai/core/@ai-sdk/google-vertex/@ai-sdk/provider-utils": ["@ai-sdk/provider-utils@4.0.27", "", { "dependencies": { "@ai-sdk/provider": "3.0.10", "@standard-schema/spec": "^1.1.0", "eventsource-parser": "^3.0.8" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-ubkAJ+xODouwtmN1tYlvTPphH1hPOBfZaEQe8U7skGvFAnIRs9PPpsq57bC2+Ky/MB4yzhd6YOsxTAx9sGpazw=="],
"@opencode-ai/core/@aws-sdk/credential-providers/@aws-sdk/client-cognito-identity": ["@aws-sdk/client-cognito-identity@3.993.0", "", { "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "^3.973.11", "@aws-sdk/credential-provider-node": "^3.972.10", "@aws-sdk/middleware-host-header": "^3.972.3", "@aws-sdk/middleware-logger": "^3.972.3", "@aws-sdk/middleware-recursion-detection": "^3.972.3", "@aws-sdk/middleware-user-agent": "^3.972.11", "@aws-sdk/region-config-resolver": "^3.972.3", "@aws-sdk/types": "^3.973.1", "@aws-sdk/util-endpoints": "3.993.0", "@aws-sdk/util-user-agent-browser": "^3.972.3", "@aws-sdk/util-user-agent-node": "^3.972.9", "@smithy/config-resolver": "^4.4.6", "@smithy/core": "^3.23.2", "@smithy/fetch-http-handler": "^5.3.9", "@smithy/hash-node": "^4.2.8", "@smithy/invalid-dependency": "^4.2.8", "@smithy/middleware-content-length": "^4.2.8", "@smithy/middleware-endpoint": "^4.4.16", "@smithy/middleware-retry": "^4.4.33", "@smithy/middleware-serde": "^4.2.9", "@smithy/middleware-stack": "^4.2.8", "@smithy/node-config-provider": "^4.3.8", "@smithy/node-http-handler": "^4.4.10", "@smithy/protocol-http": "^5.3.8", "@smithy/smithy-client": "^4.11.5", "@smithy/types": "^4.12.0", "@smithy/url-parser": "^4.2.8", "@smithy/util-base64": "^4.3.0", "@smithy/util-body-length-browser": "^4.2.0", "@smithy/util-body-length-node": "^4.2.1", "@smithy/util-defaults-mode-browser": "^4.3.32", "@smithy/util-defaults-mode-node": "^4.2.35", "@smithy/util-endpoints": "^3.2.8", "@smithy/util-middleware": "^4.2.8", "@smithy/util-retry": "^4.2.8", "@smithy/util-utf8": "^4.2.0", "tslib": "^2.6.2" } }, "sha512-7Ne3Yk/bgQPVebAkv7W+RfhiwTRSbfER9BtbhOa2w/+dIr902LrJf6vrZlxiqaJbGj2ALx8M+ZK1YIHVxSwu9A=="],
"@opencode-ai/core/@aws-sdk/credential-providers/@aws-sdk/nested-clients": ["@aws-sdk/nested-clients@3.993.0", "", { "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "^3.973.11", "@aws-sdk/middleware-host-header": "^3.972.3", "@aws-sdk/middleware-logger": "^3.972.3", "@aws-sdk/middleware-recursion-detection": "^3.972.3", "@aws-sdk/middleware-user-agent": "^3.972.11", "@aws-sdk/region-config-resolver": "^3.972.3", "@aws-sdk/types": "^3.973.1", "@aws-sdk/util-endpoints": "3.993.0", "@aws-sdk/util-user-agent-browser": "^3.972.3", "@aws-sdk/util-user-agent-node": "^3.972.9", "@smithy/config-resolver": "^4.4.6", "@smithy/core": "^3.23.2", "@smithy/fetch-http-handler": "^5.3.9", "@smithy/hash-node": "^4.2.8", "@smithy/invalid-dependency": "^4.2.8", "@smithy/middleware-content-length": "^4.2.8", "@smithy/middleware-endpoint": "^4.4.16", "@smithy/middleware-retry": "^4.4.33", "@smithy/middleware-serde": "^4.2.9", "@smithy/middleware-stack": "^4.2.8", "@smithy/node-config-provider": "^4.3.8", "@smithy/node-http-handler": "^4.4.10", "@smithy/protocol-http": "^5.3.8", "@smithy/smithy-client": "^4.11.5", "@smithy/types": "^4.12.0", "@smithy/url-parser": "^4.2.8", "@smithy/util-base64": "^4.3.0", "@smithy/util-body-length-browser": "^4.2.0", "@smithy/util-body-length-node": "^4.2.1", "@smithy/util-defaults-mode-browser": "^4.3.32", "@smithy/util-defaults-mode-node": "^4.2.35", "@smithy/util-endpoints": "^3.2.8", "@smithy/util-middleware": "^4.2.8", "@smithy/util-retry": "^4.2.8", "@smithy/util-utf8": "^4.2.0", "tslib": "^2.6.2" } }, "sha512-iOq86f2H67924kQUIPOAvlmMaOAvOLoDOIb66I2YqSUpMYB6ufiuJW3RlREgskxv86S5qKzMnfy/X6CqMjK6XQ=="],
"@opencode-ai/core/gitlab-ai-provider/@anthropic-ai/sdk": ["@anthropic-ai/sdk@0.71.2", "", { "dependencies": { "json-schema-to-ts": "^3.1.1" }, "peerDependencies": { "zod": "^3.25.0 || ^4.0.0" }, "optionalPeers": ["zod"], "bin": { "anthropic-ai-sdk": "bin/cli" } }, "sha512-TGNDEUuEstk/DKu0/TflXAEt+p+p/WhTlFzEnoosvbaDU2LTjm42igSdlL0VijrKpWejtOKxX0b8A7uc+XiSAQ=="],
"@opencode-ai/core/gitlab-ai-provider/zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="],
"@opencode-ai/core/venice-ai-sdk-provider/@ai-sdk/openai-compatible": ["@ai-sdk/openai-compatible@2.0.50", "", { "dependencies": { "@ai-sdk/provider": "3.0.10", "@ai-sdk/provider-utils": "4.0.29" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-HyuxddF2Yv5G8qxK/0uksAINjQ4h6TpwOqHuqzsCM0u78/JWAW2OXcIplQeB44PIAORgPjbMzrw9DhnPYHMskA=="],
"@opencode-ai/core/venice-ai-sdk-provider/@ai-sdk/provider": ["@ai-sdk/provider@3.0.10", "", { "dependencies": { "json-schema": "^0.4.0" } }, "sha512-Q3BZ27qfpYqnCYGvE3vt+Qi6LGOF9R5Nmzn+9JoM1lCRsD9mYaIhfJLkSunN48nfGXJ6n+XNV0J/XVpqGQl7Dw=="],
"@opencode-ai/core/venice-ai-sdk-provider/@ai-sdk/provider-utils": ["@ai-sdk/provider-utils@4.0.29", "", { "dependencies": { "@ai-sdk/provider": "3.0.10", "@standard-schema/spec": "^1.1.0", "eventsource-parser": "^3.0.8" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-uhukHaCBvqkwBHkT8C2PrnqKTCoLn3pdHXqtcR9I8ErH+flbzgW4o7VHSNIup9LRu+WBvZIZDQLsx6rwl2tiOA=="],
"@opencode-ai/storybook/@storybook/addon-docs/@storybook/csf-plugin": ["@storybook/csf-plugin@10.4.4", "", { "dependencies": { "unplugin": "^2.3.5" }, "peerDependencies": { "esbuild": "*", "rollup": "*", "storybook": "^10.4.4", "vite": "*", "webpack": "*" }, "optionalPeers": ["esbuild", "rollup", "vite", "webpack"] }, "sha512-1mzZyAwVUmAcw4WEUsJDVdSupkJf+Kf/f5uNAs4RzlBXA75P8YRkDKAb2EoMwsB5URiXFi9XoeAN/vWke0G6+w=="],
"@opencode-ai/storybook/@storybook/addon-docs/@storybook/react-dom-shim": ["@storybook/react-dom-shim@10.4.4", "", { "peerDependencies": { "@types/react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "@types/react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "storybook": "^10.4.4" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-y6SObmoW78AydE6VfKQSUmCkuqiaMPy9LgMpMdMEyWfJ/pSxBDMIKycr9dlRMJP1cvNgByaJgrusWtA46ndSQw=="],
package.json
+1 -1
View File
@@ -76,7 +76,7 @@
"@solidjs/start": "https://pkg.pr.new/@solidjs/start@dfb2020",
"solid-js": "1.9.12",
"vite-plugin-solid": "2.11.10",
"@lydell/node-pty": "1.2.0-beta.10",
"@lydell/node-pty": "1.2.0-beta.12",
"@opentui/keymap": "0.2.15",
"@effect/sql-sqlite-bun": "4.0.0-beta.66"
}
packages/kilo-docs/source-links.md
+4
View File
@@ -95,6 +95,8 @@
<!-- packages/opencode/src/lsp/server.ts -->
- <https://github.com/oven-sh/bun/issues/16682>
<!-- packages/opencode/src/provider/provider.ts -->
- <https://github.com/Rob--W/proxy-from-env>
<!-- packages/opencode/src/util/proxy-env.ts -->
- <https://github.com/vercel/ai/blob/2dc67e0ef538307f21368db32d5a12345d98831b/packages/ai/src/logger/log-warnings.ts#L85>
<!-- packages/opencode/src/server/server.ts -->
- <https://gitlab.com>
@@ -188,6 +190,8 @@
<!-- packages/opencode/src/cli/cmd/providers.ts -->
- <https://www.eclipse.org/downloads/download.php?file=/jdtls/snapshots/jdt-language-server-latest.tar.gz>
<!-- packages/opencode/src/lsp/server.ts -->
- <https://www.googleapis.com/auth/cloud-platform>
<!-- packages/opencode/src/provider/provider.ts -->
- <https://www.morphllm.com/>
<!-- packages/opencode/src/tool/warpgrep.ts -->
- <https://www.rfc-editor.org/rfc/rfc8628#section-3.5>
packages/opencode/package.json
+2
View File
@@ -128,6 +128,7 @@
"@silvia-odwyer/photon-node": "0.3.4",
"@solid-primitives/event-bus": "1.1.2",
"@solid-primitives/scheduled": "1.5.2",
"@types/ws": "8.18.1",
"@zip.js/zip.js": "2.7.62",
"ai": "catalog:",
"ai-gateway-provider": "3.1.2",
@@ -172,6 +173,7 @@
"vscode-jsonrpc": "8.2.1",
"web-tree-sitter": "0.25.10",
"which": "6.0.1",
"ws": "8.21.0",
"xlsx": "https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz",
"yargs": "18.0.0",
"zod": "catalog:"
packages/stats/AGENTS.md
-1
View File
@@ -1 +0,0 @@
To start the stats site locally, run `bun dev:stats` from the repo root.
packages/stats/README.md
-16
View File
@@ -1,16 +0,0 @@
# OpenCode Stats
Stats is a separate site from the console. Runtime, database, and domain services live in `core`; the SolidStart website lives in `app`; deployable Lambda entrypoints live in `function`.
## Packages
- `app`: SolidStart frontend/site.
- `core`: Effect services, app config, Drizzle schema/migrations, and stats domains.
- `function`: Lambda handlers that call into `core` services.
## Commands
- `bun run dev:stats` from the repo root starts the SolidStart app.
- `bun run --cwd packages/stats/app typecheck` typechecks the site.
- `bun run --cwd packages/stats/core typecheck` typechecks the Effect/database package.
- `bun run --cwd packages/stats/function typecheck` typechecks Lambda entrypoints.
packages/stats/app/.gitignore
-17
View File
@@ -1,17 +0,0 @@
dist
.wrangler
.output
.vercel
.netlify
app.config.timestamp_*.js
# Environment
.env
.env*.local
# dependencies
/node_modules
# System Files
.DS_Store
Thumbs.db
packages/stats/app/app.config.ts
-5
View File
@@ -1,5 +0,0 @@
export default {
server: {
preset: "cloudflare-module",
},
}
packages/stats/app/package.json
-38
View File
@@ -1,38 +0,0 @@
{
"$schema": "https://json.schemastore.org/package.json",
"name": "@opencode-ai/stats-app",
"version": "7.3.46",
"private": true,
"type": "module",
"license": "MIT",
"scripts": {
"typecheck": "tsgo --noEmit",
"dev": "vite dev --host 0.0.0.0",
"build": "vite build",
"start": "vite start"
},
"dependencies": {
"@ibm/plex": "6.4.1",
"@opencode-ai/stats-core": "workspace:*",
"@opencode-ai/ui": "workspace:*",
"@solidjs/meta": "catalog:",
"@solidjs/router": "catalog:",
"@solidjs/start": "catalog:",
"d3-scale": "4.0.2",
"effect": "catalog:",
"nitro": "3.0.1-alpha.1",
"sst": "catalog:",
"solid-js": "catalog:",
"vite": "catalog:"
},
"devDependencies": {
"@cloudflare/workers-types": "catalog:",
"@types/bun": "catalog:",
"@types/d3-scale": "4.0.9",
"@typescript/native-preview": "catalog:",
"typescript": "catalog:"
},
"engines": {
"node": ">=22"
}
}
packages/stats/app/src/app.css
-123
View File
@@ -1,123 +0,0 @@
:root {
color-scheme: light dark;
--stats-bg: #f8f5ee;
--stats-ink: #16110d;
--stats-muted: #6d6257;
--stats-line: #ded5c9;
--stats-panel: #fffaf1;
--stats-accent: #2357ff;
}
@media (prefers-color-scheme: dark) {
:root {
--stats-bg: #11100e;
--stats-ink: #f7efe4;
--stats-muted: #b8aa99;
--stats-line: #322d27;
--stats-panel: #1a1714;
--stats-accent: #86a2ff;
}
}
html {
line-height: 1;
background: var(--stats-bg);
}
body {
margin: 0;
min-width: 320px;
background:
radial-gradient(circle at top left, color-mix(in srgb, var(--stats-accent) 16%, transparent), transparent 32rem),
var(--stats-bg);
color: var(--stats-ink);
font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace;
-webkit-font-smoothing: antialiased;
}
a {
color: inherit;
}
.shell {
box-sizing: border-box;
min-height: 100vh;
padding: 2rem clamp(1rem, 4vw, 4rem);
}
.panel {
display: grid;
gap: clamp(2rem, 8vw, 5rem);
box-sizing: border-box;
width: min(100%, 72rem);
margin: 0 auto;
padding: clamp(1.25rem, 4vw, 3rem);
border: 1px solid var(--stats-line);
border-radius: 1.5rem;
background: color-mix(in srgb, var(--stats-panel) 88%, transparent);
}
.eyebrow {
margin: 0 0 1rem;
color: var(--stats-muted);
font-size: 0.75rem;
letter-spacing: 0.14em;
text-transform: uppercase;
}
h1 {
max-width: 11ch;
margin: 0;
font-size: clamp(3rem, 14vw, 9rem);
line-height: 0.85;
letter-spacing: -0.08em;
}
.summary {
max-width: 42rem;
margin: 1.5rem 0 0;
color: var(--stats-muted);
font-size: clamp(1rem, 2vw, 1.25rem);
line-height: 1.6;
}
.grid {
display: grid;
grid-template-columns: repeat(3, minmax(0, 1fr));
gap: 1px;
overflow: hidden;
border: 1px solid var(--stats-line);
border-radius: 1rem;
background: var(--stats-line);
}
.metric {
padding: 1rem;
background: var(--stats-panel);
}
.metric b {
display: block;
margin-bottom: 0.5rem;
font-size: clamp(1.5rem, 4vw, 3rem);
letter-spacing: -0.05em;
}
.metric span {
color: var(--stats-muted);
font-size: 0.8125rem;
}
.link {
display: inline-flex;
width: fit-content;
margin-top: 1.5rem;
color: var(--stats-accent);
text-decoration: none;
}
@media (max-width: 720px) {
.grid {
grid-template-columns: 1fr;
}
}
packages/stats/app/src/app.tsx
-31
View File
@@ -1,31 +0,0 @@
import { MetaProvider, Meta, Title } from "@solidjs/meta"
import { Router } from "@solidjs/router"
import { FileRoutes } from "@solidjs/start/router"
import { Suspense } from "solid-js"
import "./app.css"
function AppMeta() {
return (
<>
<Title>OpenCode Stats</Title>
<Meta name="description" content="OpenCode usage, market share, token cost, and session cost stats." />
</>
)
}
export default function App() {
return (
<Router
base={import.meta.env.BASE_URL.replace(/\/$/, "")}
explicitLinks={true}
root={(props) => (
<MetaProvider>
<AppMeta />
<Suspense>{props.children}</Suspense>
</MetaProvider>
)}
>
<FileRoutes />
</Router>
)
}
packages/stats/app/src/asset/logo-ornate-dark.svg
-18
View File
@@ -1,18 +0,0 @@
<svg width="234" height="42" viewBox="0 0 234 42" fill="none" xmlns="http://www.w3.org/2000/svg">
<path d="M18 30H6V18H18V30Z" fill="#4B4646"/>
<path d="M18 12H6V30H18V12ZM24 36H0V6H24V36Z" fill="#B7B1B1"/>
<path d="M48 30H36V18H48V30Z" fill="#4B4646"/>
<path d="M36 30H48V12H36V30ZM54 36H36V42H30V6H54V36Z" fill="#B7B1B1"/>
<path d="M84 24V30H66V24H84Z" fill="#4B4646"/>
<path d="M84 24H66V30H84V36H60V6H84V24ZM66 18H78V12H66V18Z" fill="#B7B1B1"/>
<path d="M108 36H96V18H108V36Z" fill="#4B4646"/>
<path d="M108 12H96V36H90V6H108V12ZM114 36H108V12H114V36Z" fill="#B7B1B1"/>
<path d="M144 30H126V18H144V30Z" fill="#4B4646"/>
<path d="M144 12H126V30H144V36H120V6H144V12Z" fill="#F1ECEC"/>
<path d="M168 30H156V18H168V30Z" fill="#4B4646"/>
<path d="M168 12H156V30H168V12ZM174 36H150V6H174V36Z" fill="#F1ECEC"/>
<path d="M198 30H186V18H198V30Z" fill="#4B4646"/>
<path d="M198 12H186V30H198V12ZM204 36H180V6H198V0H204V36Z" fill="#F1ECEC"/>
<path d="M234 24V30H216V24H234Z" fill="#4B4646"/>
<path d="M216 12V18H228V12H216ZM234 24H216V30H234V36H210V6H234V24Z" fill="#F1ECEC"/>
</svg>
Side by Side

Before

Width:  |  Height:  |  Size: 1.1 KiB

packages/stats/app/src/asset/logo-ornate-light.svg
-18
View File
@@ -1,18 +0,0 @@
<svg width="234" height="42" viewBox="0 0 234 42" fill="none" xmlns="http://www.w3.org/2000/svg">
<path d="M18 30H6V18H18V30Z" fill="#CFCECD"/>
<path d="M18 12H6V30H18V12ZM24 36H0V6H24V36Z" fill="#656363"/>
<path d="M48 30H36V18H48V30Z" fill="#CFCECD"/>
<path d="M36 30H48V12H36V30ZM54 36H36V42H30V6H54V36Z" fill="#656363"/>
<path d="M84 24V30H66V24H84Z" fill="#CFCECD"/>
<path d="M84 24H66V30H84V36H60V6H84V24ZM66 18H78V12H66V18Z" fill="#656363"/>
<path d="M108 36H96V18H108V36Z" fill="#CFCECD"/>
<path d="M108 12H96V36H90V6H108V12ZM114 36H108V12H114V36Z" fill="#656363"/>
<path d="M144 30H126V18H144V30Z" fill="#CFCECD"/>
<path d="M144 12H126V30H144V36H120V6H144V12Z" fill="#211E1E"/>
<path d="M168 30H156V18H168V30Z" fill="#CFCECD"/>
<path d="M168 12H156V30H168V12ZM174 36H150V6H174V36Z" fill="#211E1E"/>
<path d="M198 30H186V18H198V30Z" fill="#CFCECD"/>
<path d="M198 12H186V30H198V12ZM204 36H180V6H198V0H204V36Z" fill="#211E1E"/>
<path d="M234 24V30H216V24H234Z" fill="#CFCECD"/>
<path d="M216 12V18H228V12H216ZM234 24H216V30H234V36H210V6H234V24Z" fill="#211E1E"/>
</svg>
Side by Side

Before

Width:  |  Height:  |  Size: 1.1 KiB

packages/stats/app/src/asset/unfurl-rankings.png
BIN
View File
Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 12 KiB

packages/stats/app/src/entry-client.tsx
-7
View File
@@ -1,7 +0,0 @@
// @refresh reload
import { mount, StartClient } from "@solidjs/start/client"
const root = document.getElementById("app")
if (!root) throw new Error("Root element #app not found")
mount(() => <StartClient />, root)
packages/stats/app/src/entry-server.tsx
-37
View File
@@ -1,37 +0,0 @@
// @refresh reload
import { createHandler, StartServer } from "@solidjs/start/server"
const statsThemePreloadScript = `;(function () {
var preference = "system"
try {
var stored = localStorage.getItem("opencode:stats-theme")
if (stored === "dark" || stored === "light" || stored === "system") preference = stored
} catch (_) {}
document.documentElement.dataset.statsTheme = preference
if (preference === "system") document.documentElement.style.removeProperty("color-scheme")
else document.documentElement.style.setProperty("color-scheme", preference)
})()`
export default createHandler(
() => (
<StartServer
document={({ assets, children, scripts }) => (
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<script id="stats-theme-preload-script">{statsThemePreloadScript}</script>
{assets}
</head>
<body>
<div id="app">{children}</div>
{scripts}
</body>
</html>
)}
/>
),
{
mode: "async",
},
)
packages/stats/app/src/global.d.ts
Vendored
-1
View File
@@ -1 +0,0 @@
/// <reference types="@solidjs/start/env" />
packages/stats/app/src/resource.d.ts
Vendored
-10
View File
@@ -1,10 +0,0 @@
import "sst/resource"
declare module "sst/resource" {
export interface Resource {
EMAILOCTOPUS_API_KEY: {
type: "sst.sst.Secret"
value: string
}
}
}
packages/stats/app/src/routes/api/health.ts
-19
View File
@@ -1,19 +0,0 @@
import { AppConfig } from "@opencode-ai/stats-core/config"
import { runtime } from "@opencode-ai/stats-core/runtime"
import { Effect } from "effect"
export async function GET() {
return Response.json(
await runtime.runPromise(
Effect.gen(function* () {
const config = yield* AppConfig
return {
ok: true,
app: "stats",
stage: config.stage,
publicUrl: config.publicUrl,
}
}),
),
)
}
packages/stats/app/src/routes/api/newsletter.ts
-29
View File
@@ -1,29 +0,0 @@
import { Resource } from "sst/resource"
const listId = "8b9bb82c-9d5f-11f0-975f-0df6fd1e4945"
export async function POST(event: { request: Request }) {
const contentType = event.request.headers.get("content-type") ?? ""
if (!contentType.includes("multipart/form-data") && !contentType.includes("application/x-www-form-urlencoded")) {
return Response.json({ error: "Email address is required" }, { status: 400 })
}
const form = await event.request.formData()
const emailAddress = form.get("email")
if (typeof emailAddress !== "string" || emailAddress.trim().length === 0) {
return Response.json({ error: "Email address is required" }, { status: 400 })
}
const response = await fetch(`https://api.emailoctopus.com/lists/${listId}/contacts`, {
method: "PUT",
headers: {
Authorization: `Bearer ${Resource.EMAILOCTOPUS_API_KEY.value}`,
"Content-Type": "application/json",
},
body: JSON.stringify({
email_address: emailAddress.trim(),
}),
})
if (!response.ok) return Response.json({ error: "Failed to subscribe" }, { status: 502 })
return Response.json({ success: true })
}
packages/stats/app/src/routes/index.css
-2971
View File
File diff suppressed because it is too large Load Diff
packages/stats/app/src/routes/index.tsx
-1830
View File
File diff suppressed because it is too large Load Diff
packages/stats/app/src/routes/stats/api/health.ts
-1
View File
@@ -1 +0,0 @@
export { GET } from "../../api/health"
packages/stats/app/src/routes/stats/api/newsletter.ts
-1
View File
@@ -1 +0,0 @@
export { POST } from "../../api/newsletter"
packages/stats/app/sst-env.d.ts
Vendored
-10
View File
@@ -1,10 +0,0 @@
/* This file is auto-generated by SST. Do not edit. */
/* tslint:disable */
/* eslint-disable */
/* deno-fmt-ignore-file */
/* biome-ignore-all lint: auto-generated */
/// <reference path="../../../sst-env.d.ts" />
import "sst"
export {}
packages/stats/app/tsconfig.json
-22
View File
@@ -1,22 +0,0 @@
{
"$schema": "https://json.schemastore.org/tsconfig",
"compilerOptions": {
"target": "ESNext",
"module": "ESNext",
"skipLibCheck": true,
"moduleResolution": "bundler",
"allowSyntheticDefaultImports": true,
"esModuleInterop": true,
"jsx": "preserve",
"jsxImportSource": "solid-js",
"allowJs": true,
"strict": true,
"noEmit": true,
"types": ["vite/client", "bun"],
"isolatedModules": true,
"paths": {
"~/*": ["./src/*"]
}
},
"include": ["*.ts", "src", "../core/src/resource.d.ts"]
}
packages/stats/app/vite.config.ts
-23
View File
@@ -1,23 +0,0 @@
import { solidStart } from "@solidjs/start/config"
import { nitro } from "nitro/vite"
import { defineConfig, type PluginOption } from "vite"
export default defineConfig({
base: "/stats/",
plugins: [
solidStart() as PluginOption,
nitro({
compatibilityDate: "2024-09-19",
preset: "cloudflare-module",
cloudflare: {
nodeCompat: true,
},
}),
],
server: {
allowedHosts: true,
},
build: {
minify: false,
},
})
packages/stats/core/drizzle.config.ts
-21
View File
@@ -1,21 +0,0 @@
import { Resource } from "sst/resource"
import { defineConfig } from "drizzle-kit"
export default defineConfig({
dialect: "mysql",
schema: ["./src/database/schema.ts"],
// schema: ["./src/**/*.sql.ts"],
out: "./migrations/",
strict: true,
verbose: true,
dbCredentials: {
database: Resource.StatsDatabase.database,
host: Resource.StatsDatabase.host,
user: Resource.StatsDatabase.username,
password: Resource.StatsDatabase.password,
port: Resource.StatsDatabase.port,
ssl: {
rejectUnauthorized: false,
},
},
})
packages/stats/core/migrations/20260522121617_common_dust/migration.sql
-42
View File
@@ -1,42 +0,0 @@
CREATE TABLE `stat` (
`id` bigint AUTO_INCREMENT PRIMARY KEY,
`grain` varchar(16) NOT NULL,
`period_start` datetime NOT NULL,
`period_end` datetime NOT NULL,
`dataset` varchar(64) NOT NULL DEFAULT 'all',
`tier` varchar(64) NOT NULL DEFAULT 'all',
`client` varchar(64) NOT NULL DEFAULT 'all',
`source` varchar(64) NOT NULL DEFAULT 'all',
`provider` varchar(128) NOT NULL,
`model` varchar(256) NOT NULL,
`provider_model` varchar(256) NOT NULL DEFAULT '',
`sessions` bigint NOT NULL DEFAULT 0,
`requests` bigint NOT NULL DEFAULT 0,
`input_tokens` bigint NOT NULL DEFAULT 0,
`output_tokens` bigint NOT NULL DEFAULT 0,
`reasoning_tokens` bigint NOT NULL DEFAULT 0,
`cache_read_tokens` bigint NOT NULL DEFAULT 0,
`total_tokens` bigint NOT NULL DEFAULT 0,
`input_cost_microcents` bigint NOT NULL DEFAULT 0,
`output_cost_microcents` bigint NOT NULL DEFAULT 0,
`total_cost_microcents` bigint NOT NULL DEFAULT 0,
`avg_duration_ms` decimal(12,2),
`p50_duration_ms` int,
`p95_duration_ms` int,
`avg_ttfb_ms` decimal(12,2),
`p50_ttfb_ms` int,
`p95_ttfb_ms` int,
`avg_output_tps` decimal(12,4),
`success_count` bigint NOT NULL DEFAULT 0,
`error_count` bigint NOT NULL DEFAULT 0,
`sample_count` bigint NOT NULL DEFAULT 0,
`rank_by_tokens` int,
`rank_by_requests` int,
`rank_by_cost` int,
`created_at` datetime NOT NULL DEFAULT (now()),
`updated_at` datetime NOT NULL DEFAULT (now()) ON UPDATE CURRENT_TIMESTAMP,
CONSTRAINT `uniq_model_period` UNIQUE INDEX(`grain`,`period_start`,`dataset`,`tier`,`client`,`source`,`provider`,`model`)
);
--> statement-breakpoint
CREATE INDEX `idx_leaderboard_tokens` ON `stat` (`grain`,`period_start`,`dataset`,`tier`,`total_tokens`);--> statement-breakpoint
CREATE INDEX `idx_model` ON `stat` (`model`,`grain`,`period_start`);
packages/stats/core/migrations/20260522121617_common_dust/snapshot.json
-623
View File
@@ -1,623 +0,0 @@
{
"version": "6",
"dialect": "mysql",
"id": "72655266-65da-408e-bfd8-9f3a4ad817a5",
"prevIds": ["00000000-0000-0000-0000-000000000000"],
"ddl": [
{
"name": "stat",
"entityType": "tables"
},
{
"type": "bigint",
"notNull": true,
"autoIncrement": true,
"default": null,
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "id",
"entityType": "columns",
"table": "stat"
},
{
"type": "varchar(16)",
"notNull": true,
"autoIncrement": false,
"default": null,
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "grain",
"entityType": "columns",
"table": "stat"
},
{
"type": "datetime",
"notNull": true,
"autoIncrement": false,
"default": null,
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "period_start",
"entityType": "columns",
"table": "stat"
},
{
"type": "datetime",
"notNull": true,
"autoIncrement": false,
"default": null,
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "period_end",
"entityType": "columns",
"table": "stat"
},
{
"type": "varchar(64)",
"notNull": true,
"autoIncrement": false,
"default": "'all'",
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "dataset",
"entityType": "columns",
"table": "stat"
},
{
"type": "varchar(64)",
"notNull": true,
"autoIncrement": false,
"default": "'all'",
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "tier",
"entityType": "columns",
"table": "stat"
},
{
"type": "varchar(64)",
"notNull": true,
"autoIncrement": false,
"default": "'all'",
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "client",
"entityType": "columns",
"table": "stat"
},
{
"type": "varchar(64)",
"notNull": true,
"autoIncrement": false,
"default": "'all'",
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "source",
"entityType": "columns",
"table": "stat"
},
{
"type": "varchar(128)",
"notNull": true,
"autoIncrement": false,
"default": null,
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "provider",
"entityType": "columns",
"table": "stat"
},
{
"type": "varchar(256)",
"notNull": true,
"autoIncrement": false,
"default": null,
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "model",
"entityType": "columns",
"table": "stat"
},
{
"type": "varchar(256)",
"notNull": true,
"autoIncrement": false,
"default": "''",
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "provider_model",
"entityType": "columns",
"table": "stat"
},
{
"type": "bigint",
"notNull": true,
"autoIncrement": false,
"default": "0",
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "sessions",
"entityType": "columns",
"table": "stat"
},
{
"type": "bigint",
"notNull": true,
"autoIncrement": false,
"default": "0",
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "requests",
"entityType": "columns",
"table": "stat"
},
{
"type": "bigint",
"notNull": true,
"autoIncrement": false,
"default": "0",
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "input_tokens",
"entityType": "columns",
"table": "stat"
},
{
"type": "bigint",
"notNull": true,
"autoIncrement": false,
"default": "0",
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "output_tokens",
"entityType": "columns",
"table": "stat"
},
{
"type": "bigint",
"notNull": true,
"autoIncrement": false,
"default": "0",
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "reasoning_tokens",
"entityType": "columns",
"table": "stat"
},
{
"type": "bigint",
"notNull": true,
"autoIncrement": false,
"default": "0",
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "cache_read_tokens",
"entityType": "columns",
"table": "stat"
},
{
"type": "bigint",
"notNull": true,
"autoIncrement": false,
"default": "0",
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "total_tokens",
"entityType": "columns",
"table": "stat"
},
{
"type": "bigint",
"notNull": true,
"autoIncrement": false,
"default": "0",
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "input_cost_microcents",
"entityType": "columns",
"table": "stat"
},
{
"type": "bigint",
"notNull": true,
"autoIncrement": false,
"default": "0",
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "output_cost_microcents",
"entityType": "columns",
"table": "stat"
},
{
"type": "bigint",
"notNull": true,
"autoIncrement": false,
"default": "0",
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "total_cost_microcents",
"entityType": "columns",
"table": "stat"
},
{
"type": "decimal(12,2)",
"notNull": false,
"autoIncrement": false,
"default": null,
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "avg_duration_ms",
"entityType": "columns",
"table": "stat"
},
{
"type": "int",
"notNull": false,
"autoIncrement": false,
"default": null,
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "p50_duration_ms",
"entityType": "columns",
"table": "stat"
},
{
"type": "int",
"notNull": false,
"autoIncrement": false,
"default": null,
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "p95_duration_ms",
"entityType": "columns",
"table": "stat"
},
{
"type": "decimal(12,2)",
"notNull": false,
"autoIncrement": false,
"default": null,
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "avg_ttfb_ms",
"entityType": "columns",
"table": "stat"
},
{
"type": "int",
"notNull": false,
"autoIncrement": false,
"default": null,
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "p50_ttfb_ms",
"entityType": "columns",
"table": "stat"
},
{
"type": "int",
"notNull": false,
"autoIncrement": false,
"default": null,
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "p95_ttfb_ms",
"entityType": "columns",
"table": "stat"
},
{
"type": "decimal(12,4)",
"notNull": false,
"autoIncrement": false,
"default": null,
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "avg_output_tps",
"entityType": "columns",
"table": "stat"
},
{
"type": "bigint",
"notNull": true,
"autoIncrement": false,
"default": "0",
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "success_count",
"entityType": "columns",
"table": "stat"
},
{
"type": "bigint",
"notNull": true,
"autoIncrement": false,
"default": "0",
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "error_count",
"entityType": "columns",
"table": "stat"
},
{
"type": "bigint",
"notNull": true,
"autoIncrement": false,
"default": "0",
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "sample_count",
"entityType": "columns",
"table": "stat"
},
{
"type": "int",
"notNull": false,
"autoIncrement": false,
"default": null,
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "rank_by_tokens",
"entityType": "columns",
"table": "stat"
},
{
"type": "int",
"notNull": false,
"autoIncrement": false,
"default": null,
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "rank_by_requests",
"entityType": "columns",
"table": "stat"
},
{
"type": "int",
"notNull": false,
"autoIncrement": false,
"default": null,
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "rank_by_cost",
"entityType": "columns",
"table": "stat"
},
{
"type": "datetime",
"notNull": true,
"autoIncrement": false,
"default": "(now())",
"onUpdateNow": false,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "created_at",
"entityType": "columns",
"table": "stat"
},
{
"type": "datetime",
"notNull": true,
"autoIncrement": false,
"default": "(now())",
"onUpdateNow": true,
"onUpdateNowFsp": null,
"charSet": null,
"collation": null,
"generated": null,
"name": "updated_at",
"entityType": "columns",
"table": "stat"
},
{
"columns": ["id"],
"name": "PRIMARY",
"table": "stat",
"entityType": "pks"
},
{
"columns": [
{
"value": "grain",
"isExpression": false
},
{
"value": "period_start",
"isExpression": false
},
{
"value": "dataset",
"isExpression": false
},
{
"value": "tier",
"isExpression": false
},
{
"value": "client",
"isExpression": false
},
{
"value": "source",
"isExpression": false
},
{
"value": "provider",
"isExpression": false
},
{
"value": "model",
"isExpression": false
}
],
"isUnique": true,
"using": null,
"algorithm": null,
"lock": null,
"nameExplicit": true,
"name": "uniq_model_period",
"entityType": "indexes",
"table": "stat"
},
{
"columns": [
{
"value": "grain",
"isExpression": false
},
{
"value": "period_start",
"isExpression": false
},
{
"value": "dataset",
"isExpression": false
},
{
"value": "tier",
"isExpression": false
},
{
"value": "total_tokens",
"isExpression": false
}
],
"isUnique": false,
"using": null,
"algorithm": null,
"lock": null,
"nameExplicit": true,
"name": "idx_leaderboard_tokens",
"entityType": "indexes",
"table": "stat"
},
{
"columns": [
{
"value": "model",
"isExpression": false
},
{
"value": "grain",
"isExpression": false
},
{
"value": "period_start",
"isExpression": false
}
],
"isUnique": false,
"using": null,
"algorithm": null,
"lock": null,
"nameExplicit": true,
"name": "idx_model",
"entityType": "indexes",
"table": "stat"
}
],
"renames": []
}
packages/stats/core/migrations/20260523110335_cool_vin_gonzales/migration.sql
-94
View File
@@ -1,94 +0,0 @@
CREATE TABLE `geo_stat` (
`id` bigint AUTO_INCREMENT PRIMARY KEY,
`grain` varchar(16) NOT NULL,
`period_start` datetime NOT NULL,
`period_end` datetime NOT NULL,
`dataset` varchar(64) NOT NULL DEFAULT 'all',
`tier` varchar(64) NOT NULL DEFAULT 'all',
`client` varchar(64) NOT NULL DEFAULT 'all',
`source` varchar(64) NOT NULL DEFAULT 'all',
`country` char(2) NOT NULL,
`continent` varchar(8) NOT NULL DEFAULT '',
`sessions` bigint NOT NULL DEFAULT 0,
`requests` bigint NOT NULL DEFAULT 0,
`input_tokens` bigint NOT NULL DEFAULT 0,
`output_tokens` bigint NOT NULL DEFAULT 0,
`reasoning_tokens` bigint NOT NULL DEFAULT 0,
`cache_read_tokens` bigint NOT NULL DEFAULT 0,
`total_tokens` bigint NOT NULL DEFAULT 0,
`input_cost_microcents` bigint NOT NULL DEFAULT 0,
`output_cost_microcents` bigint NOT NULL DEFAULT 0,
`total_cost_microcents` bigint NOT NULL DEFAULT 0,
`avg_duration_ms` decimal(12,2),
`p50_duration_ms` int,
`p95_duration_ms` int,
`avg_ttfb_ms` decimal(12,2),
`p50_ttfb_ms` int,
`p95_ttfb_ms` int,
`avg_output_tps` decimal(12,4),
`success_count` bigint NOT NULL DEFAULT 0,
`error_count` bigint NOT NULL DEFAULT 0,
`sample_count` bigint NOT NULL DEFAULT 0,
`market_share_tokens` decimal(10,6),
`market_share_requests` decimal(10,6),
`market_share_sessions` decimal(10,6),
`rank_by_tokens` int,
`rank_by_requests` int,
`rank_by_sessions` int,
`rank_by_cost` int,
`created_at` datetime NOT NULL DEFAULT (now()),
`updated_at` datetime NOT NULL DEFAULT (now()) ON UPDATE CURRENT_TIMESTAMP,
CONSTRAINT `uniq_country_period` UNIQUE INDEX(`grain`,`period_start`,`dataset`,`tier`,`client`,`source`,`country`)
);
--> statement-breakpoint
CREATE TABLE `provider_stat` (
`id` bigint AUTO_INCREMENT PRIMARY KEY,
`grain` varchar(16) NOT NULL,
`period_start` datetime NOT NULL,
`period_end` datetime NOT NULL,
`dataset` varchar(64) NOT NULL DEFAULT 'all',
`tier` varchar(64) NOT NULL DEFAULT 'all',
`client` varchar(64) NOT NULL DEFAULT 'all',
`source` varchar(64) NOT NULL DEFAULT 'all',
`provider` varchar(128) NOT NULL,
`sessions` bigint NOT NULL DEFAULT 0,
`requests` bigint NOT NULL DEFAULT 0,
`input_tokens` bigint NOT NULL DEFAULT 0,
`output_tokens` bigint NOT NULL DEFAULT 0,
`reasoning_tokens` bigint NOT NULL DEFAULT 0,
`cache_read_tokens` bigint NOT NULL DEFAULT 0,
`total_tokens` bigint NOT NULL DEFAULT 0,
`input_cost_microcents` bigint NOT NULL DEFAULT 0,
`output_cost_microcents` bigint NOT NULL DEFAULT 0,
`total_cost_microcents` bigint NOT NULL DEFAULT 0,
`avg_duration_ms` decimal(12,2),
`p50_duration_ms` int,
`p95_duration_ms` int,
`avg_ttfb_ms` decimal(12,2),
`p50_ttfb_ms` int,
`p95_ttfb_ms` int,
`avg_output_tps` decimal(12,4),
`success_count` bigint NOT NULL DEFAULT 0,
`error_count` bigint NOT NULL DEFAULT 0,
`sample_count` bigint NOT NULL DEFAULT 0,
`market_share_tokens` decimal(10,6),
`market_share_requests` decimal(10,6),
`market_share_sessions` decimal(10,6),
`rank_by_tokens` int,
`rank_by_requests` int,
`rank_by_sessions` int,
`rank_by_cost` int,
`created_at` datetime NOT NULL DEFAULT (now()),
`updated_at` datetime NOT NULL DEFAULT (now()) ON UPDATE CURRENT_TIMESTAMP,
CONSTRAINT `uniq_provider_period` UNIQUE INDEX(`grain`,`period_start`,`dataset`,`tier`,`client`,`source`,`provider`)
);
--> statement-breakpoint
RENAME TABLE `stat` TO `model_stat`;--> statement-breakpoint
CREATE INDEX `idx_country_map_tokens` ON `geo_stat` (`grain`,`period_start`,`dataset`,`tier`,`total_tokens`);--> statement-breakpoint
CREATE INDEX `idx_country_rank` ON `geo_stat` (`grain`,`period_start`,`dataset`,`tier`,`rank_by_tokens`);--> statement-breakpoint
CREATE INDEX `idx_country` ON `geo_stat` (`country`,`grain`,`period_start`);--> statement-breakpoint
CREATE INDEX `idx_continent` ON `geo_stat` (`continent`,`grain`,`period_start`);--> statement-breakpoint
CREATE INDEX `idx_provider_leaderboard_tokens` ON `provider_stat` (`grain`,`period_start`,`dataset`,`tier`,`total_tokens`);--> statement-breakpoint
CREATE INDEX `idx_provider_market_share` ON `provider_stat` (`grain`,`period_start`,`dataset`,`tier`,`market_share_tokens`);--> statement-breakpoint
CREATE INDEX `idx_provider_rank` ON `provider_stat` (`grain`,`period_start`,`dataset`,`tier`,`rank_by_tokens`);--> statement-breakpoint
CREATE INDEX `idx_provider` ON `provider_stat` (`provider`,`grain`,`period_start`);
packages/stats/core/migrations/20260523110335_cool_vin_gonzales/snapshot.json
-2033
View File
File diff suppressed because it is too large Load Diff
packages/stats/core/migrations/20260528012726_worthless_ultimo/migration.sql
-5
View File
@@ -1,5 +0,0 @@
ALTER TABLE `geo_stat` ADD `provider` varchar(128) DEFAULT 'all' NOT NULL;--> statement-breakpoint
ALTER TABLE `geo_stat` ADD `model` varchar(256) DEFAULT 'all' NOT NULL;--> statement-breakpoint
ALTER TABLE `geo_stat` DROP INDEX `uniq_country_period`;--> statement-breakpoint
CREATE UNIQUE INDEX `uniq_country_period` ON `geo_stat` (`grain`,`period_start`,`dataset`,`tier`,`client`,`source`,`provider`,`model`,`country`);--> statement-breakpoint
CREATE INDEX `idx_country_model` ON `geo_stat` (`model`,`country`,`grain`,`period_start`);
packages/stats/core/migrations/20260528012726_worthless_ultimo/snapshot.json
-2097
View File
File diff suppressed because it is too large Load Diff
packages/stats/core/migrations/20260528114447_misty_black_knight/migration.sql
-3
View File
@@ -1,3 +0,0 @@
ALTER TABLE `geo_stat` ADD `period_key` varchar(32) NOT NULL;--> statement-breakpoint
ALTER TABLE `model_stat` ADD `period_key` varchar(32) NOT NULL;--> statement-breakpoint
ALTER TABLE `provider_stat` ADD `period_key` varchar(32) NOT NULL;
packages/stats/core/migrations/20260528114447_misty_black_knight/snapshot.json
-2139
View File
File diff suppressed because it is too large Load Diff
packages/stats/core/migrations/20260528121120_easy_multiple_man/migration.sql
-6
View File
@@ -1,6 +0,0 @@
ALTER TABLE `geo_stat` DROP COLUMN `period_start`;--> statement-breakpoint
ALTER TABLE `geo_stat` DROP COLUMN `period_end`;--> statement-breakpoint
ALTER TABLE `model_stat` DROP COLUMN `period_start`;--> statement-breakpoint
ALTER TABLE `model_stat` DROP COLUMN `period_end`;--> statement-breakpoint
ALTER TABLE `provider_stat` DROP COLUMN `period_start`;--> statement-breakpoint
ALTER TABLE `provider_stat` DROP COLUMN `period_end`;
packages/stats/core/migrations/20260528121120_easy_multiple_man/snapshot.json
-2055
View File
File diff suppressed because it is too large Load Diff
packages/stats/core/package.json
-44
View File
@@ -1,44 +0,0 @@
{
"$schema": "https://json.schemastore.org/package.json",
"name": "@opencode-ai/stats-core",
"version": "7.3.46",
"private": true,
"type": "module",
"license": "MIT",
"exports": {
".": "./src/index.ts",
"./athena": "./src/athena.ts",
"./config": "./src/config.ts",
"./database": "./src/database.ts",
"./database/*": "./src/database/*.ts",
"./domain/*": "./src/domain/*.ts",
"./runtime": "./src/runtime.ts",
"./stat-sync": "./src/stat-sync.ts"
},
"scripts": {
"db:generate": "drizzle-kit generate --config=drizzle.config.ts",
"db:migrate": "bun src/migrate.ts",
"db:push": "drizzle-kit push --config=drizzle.config.ts",
"db:studio": "drizzle-kit studio --config=drizzle.config.ts",
"honeycomb:backfill": "bun src/honeycomb-backfill.ts",
"typecheck": "tsgo --noEmit"
},
"dependencies": {
"@aws-sdk/client-athena": "3.933.0",
"@planetscale/database": "1.19.0",
"drizzle-orm": "catalog:",
"effect": "catalog:",
"sst": "catalog:"
},
"devDependencies": {
"@tsconfig/node22": "catalog:",
"@types/bun": "catalog:",
"@types/node": "catalog:",
"@typescript/native-preview": "catalog:",
"drizzle-kit": "catalog:",
"typescript": "catalog:"
},
"engines": {
"node": ">=22"
}
}
packages/stats/core/src/athena.ts
-139
View File
@@ -1,139 +0,0 @@
import {
AthenaClient as AwsAthenaClient,
GetQueryExecutionCommand,
GetQueryResultsCommand,
StartQueryExecutionCommand,
type Row,
} from "@aws-sdk/client-athena"
import { Effect, Layer, Schema } from "effect"
import * as Context from "effect/Context"
import { Resource } from "sst/resource"
const ATHENA_MAX_POLL_ATTEMPTS = 60
const ATHENA_PAGE_SIZE = 1000
export type AthenaData = Record<string, string>
export class AthenaQueryError extends Schema.TaggedErrorClass<AthenaQueryError>()("AthenaQueryError", {
message: Schema.String,
queryExecutionId: Schema.optional(Schema.String),
cause: Schema.optional(Schema.Defect),
}) {}
export class AthenaQueryTimeoutError extends Schema.TaggedErrorClass<AthenaQueryTimeoutError>()(
"AthenaQueryTimeoutError",
{
message: Schema.String,
queryExecutionId: Schema.String,
},
) {}
export declare namespace Athena {
export interface Service {
readonly query: (query: string) => Effect.Effect<AthenaData[], AthenaQueryError | AthenaQueryTimeoutError>
}
}
export class Athena extends Context.Service<Athena, Athena.Service>()("@opencode/stats/Athena") {
static readonly layer: Layer.Layer<Athena> = Layer.effect(
Athena,
Effect.sync(() => {
const client = new AwsAthenaClient({ region: Resource.InferenceEvent.region })
const query = Effect.fn("Athena.query")(function* (query: string) {
const started = yield* Effect.tryPromise({
try: () =>
client.send(
new StartQueryExecutionCommand({
QueryString: query,
WorkGroup: Resource.InferenceEvent.workgroup,
QueryExecutionContext: {
Catalog: Resource.InferenceEvent.catalog,
Database: Resource.InferenceEvent.database,
},
}),
),
catch: (cause) => new AthenaQueryError({ message: "Failed to start Athena stats query", cause }),
})
const queryExecutionId = started.QueryExecutionId
if (!queryExecutionId)
return yield* new AthenaQueryError({ message: "Athena did not return a query execution id" })
yield* poll(client, queryExecutionId)
return yield* results(client, queryExecutionId)
})
return Athena.of({ query })
}),
)
}
const poll: (
client: AwsAthenaClient,
queryExecutionId: string,
attempt?: number,
) => Effect.Effect<void, AthenaQueryError | AthenaQueryTimeoutError> = Effect.fn("Athena.poll")(function* (
client: AwsAthenaClient,
queryExecutionId: string,
attempt = 0,
) {
if (attempt > 0) yield* Effect.sleep("2 seconds")
const result = yield* Effect.tryPromise({
try: () => client.send(new GetQueryExecutionCommand({ QueryExecutionId: queryExecutionId })),
catch: (cause) => new AthenaQueryError({ message: "Failed to poll Athena stats query", queryExecutionId, cause }),
})
const status = result.QueryExecution?.Status
if (status?.State === "SUCCEEDED") return
if (status?.State === "FAILED" || status?.State === "CANCELLED")
return yield* new AthenaQueryError({
message: `Athena stats query ${status.State.toLowerCase()}: ${status.StateChangeReason ?? "unknown reason"}`,
queryExecutionId,
})
if (attempt >= ATHENA_MAX_POLL_ATTEMPTS - 1)
return yield* new AthenaQueryTimeoutError({
message: `Athena stats query ${queryExecutionId} did not complete`,
queryExecutionId,
})
return yield* poll(client, queryExecutionId, attempt + 1)
})
const results: (
client: AwsAthenaClient,
queryExecutionId: string,
nextToken?: string,
) => Effect.Effect<AthenaData[], AthenaQueryError> = Effect.fn("Athena.results")(function* (
client: AwsAthenaClient,
queryExecutionId: string,
nextToken?: string,
) {
const result = yield* Effect.tryPromise({
try: () =>
client.send(
new GetQueryResultsCommand({
QueryExecutionId: queryExecutionId,
NextToken: nextToken,
MaxResults: ATHENA_PAGE_SIZE,
}),
),
catch: (cause) => new AthenaQueryError({ message: "Failed to read Athena stats results", queryExecutionId, cause }),
})
const columns = result.ResultSet?.ResultSetMetadata?.ColumnInfo?.map((item) => item.Name ?? "") ?? []
const rows = (result.ResultSet?.Rows ?? []).slice(nextToken ? 0 : 1).map((row) => rowData(columns, row))
if (!result.NextToken) return rows
return [...rows, ...(yield* results(client, queryExecutionId, result.NextToken))]
})
function rowData(columns: string[], row: Row): AthenaData {
return Object.fromEntries(
columns.flatMap((column, index) => {
const value = row.Data?.[index]?.VarCharValue
if (!column || value === undefined) return []
return [[column, value]]
}),
)
}
packages/stats/core/src/config.ts
-23
View File
@@ -1,23 +0,0 @@
import { Config, ConfigProvider, Effect, Layer, Schema } from "effect"
import * as Context from "effect/Context"
import { Resource } from "sst/resource"
export class AppConfigValue extends Schema.Class<AppConfigValue>("AppConfigValue")({
stage: Schema.NonEmptyString,
publicUrl: Schema.NonEmptyString,
}) {}
const decodeAppConfigValue = Schema.decodeUnknownSync(AppConfigValue)
const config = Config.all({
stage: Config.succeed(Resource.App.stage),
publicUrl: Config.string("PUBLIC_URL").pipe(Config.withDefault("http://localhost:3000")),
}).pipe(Config.map(decodeAppConfigValue))
export class AppConfig extends Context.Service<AppConfig, AppConfigValue>()("@opencode/stats/AppConfig") {
static readonly config = config
static readonly layer: Layer.Layer<AppConfig, never, never> = Layer.effect(
AppConfig,
config.parse(ConfigProvider.fromEnv()).pipe(Effect.orDie),
)
}
packages/stats/core/src/database.ts
-79
View File
@@ -1,79 +0,0 @@
import { Client } from "@planetscale/database"
import { drizzle } from "drizzle-orm/planetscale-serverless"
import { migrate as drizzleMigrate } from "drizzle-orm/planetscale-serverless/migrator"
import { Config, ConfigProvider, Effect, Layer, Schema } from "effect"
import * as Context from "effect/Context"
import * as schema from "./database/schema"
import { Resource } from "sst/resource"
export const DatabaseUrl = Schema.NonEmptyString.pipe(Schema.brand("DatabaseUrl"))
export type DatabaseUrl = typeof DatabaseUrl.Type
export class DatabaseSettings extends Schema.Class<DatabaseSettings>("DatabaseSettings")({
url: DatabaseUrl,
migrationsDir: Schema.NonEmptyString,
}) {}
const decodeDatabaseSettings = Schema.decodeUnknownSync(DatabaseSettings)
const config = Config.all({
url: Config.nonEmptyString("DATABASE_URL").pipe(Config.withDefault(Resource.StatsDatabase.url)),
migrationsDir: Config.nonEmptyString("DATABASE_MIGRATIONS_DIR").pipe(Config.withDefault("./migrations")),
}).pipe(Config.map(decodeDatabaseSettings))
export class DatabaseConfig extends Context.Service<DatabaseConfig, DatabaseSettings>()(
"@opencode/stats/DatabaseConfig",
) {
static readonly config = config
static readonly layer: Layer.Layer<DatabaseConfig, never, never> = Layer.effect(
DatabaseConfig,
config.parse(ConfigProvider.fromEnv()).pipe(Effect.orDie),
)
}
function makeDrizzle(settings: DatabaseSettings) {
return drizzle({ client: new Client({ url: settings.url }), schema })
}
export type Drizzle = ReturnType<typeof makeDrizzle>
export class DrizzleClient extends Context.Service<DrizzleClient, Drizzle>()("@opencode/stats/DrizzleClient") {
static readonly layer: Layer.Layer<DrizzleClient, never, DatabaseConfig> = Layer.effect(
DrizzleClient,
Effect.map(DatabaseConfig, makeDrizzle),
)
}
export class DatabaseError extends Schema.TaggedErrorClass<DatabaseError>()("DatabaseError", {
cause: Schema.Defect,
}) {}
export const catchDbError = Effect.mapError((cause) => DatabaseError.make({ cause }))
export class MigrationError extends Schema.TaggedErrorClass<MigrationError>()("MigrationError", {
message: Schema.String,
cause: Schema.optional(Schema.Defect),
}) {}
export const migrate = Effect.fn("Database.migrate")(function* () {
const settings = yield* DatabaseConfig
yield* Effect.logInfo("applying database migrations").pipe(
Effect.annotateLogs({ migrationsDir: settings.migrationsDir }),
)
const result = yield* Effect.tryPromise({
try: () =>
drizzleMigrate(drizzle({ client: new Client({ url: settings.url }) }), {
migrationsFolder: settings.migrationsDir,
}),
catch: (cause) => new MigrationError({ message: "Failed to apply database migrations", cause }),
})
if (result)
return yield* new MigrationError({
message: `Failed to initialize database migrations: ${result.exitCode}`,
})
yield* Effect.logInfo("database migrations complete").pipe(
Effect.annotateLogs({ migrationsDir: settings.migrationsDir }),
)
})
export const layer = Layer.mergeAll(DatabaseConfig.layer, DrizzleClient.layer.pipe(Layer.provide(DatabaseConfig.layer)))
packages/stats/core/src/database/schema.ts
-160
View File
@@ -1,160 +0,0 @@
import { bigint, char, datetime, decimal, index, int, mysqlTable, uniqueIndex, varchar } from "drizzle-orm/mysql-core"
export const modelStat = mysqlTable(
"model_stat",
{
...periodColumns(),
provider: varchar({ length: 128 }).notNull(),
model: varchar({ length: 256 }).notNull(),
provider_model: varchar({ length: 256 }).notNull().default(""),
...metricColumns(),
rank_by_tokens: int(),
rank_by_requests: int(),
rank_by_cost: int(),
...timestampColumns(),
},
(table) => [
uniqueIndex("uniq_model_period").on(
table.grain,
table.period_key,
table.dataset,
table.tier,
table.client,
table.source,
table.provider,
table.model,
),
index("idx_leaderboard_tokens").on(table.grain, table.period_key, table.dataset, table.tier, table.total_tokens),
index("idx_model").on(table.model, table.grain, table.period_key),
],
)
export const providerStat = mysqlTable(
"provider_stat",
{
...periodColumns(),
provider: varchar({ length: 128 }).notNull(),
...metricColumns(),
...marketShareColumns(),
rank_by_tokens: int(),
rank_by_requests: int(),
rank_by_sessions: int(),
rank_by_cost: int(),
...timestampColumns(),
},
(table) => [
uniqueIndex("uniq_provider_period").on(
table.grain,
table.period_key,
table.dataset,
table.tier,
table.client,
table.source,
table.provider,
),
index("idx_provider_leaderboard_tokens").on(
table.grain,
table.period_key,
table.dataset,
table.tier,
table.total_tokens,
),
index("idx_provider_market_share").on(
table.grain,
table.period_key,
table.dataset,
table.tier,
table.market_share_tokens,
),
index("idx_provider_rank").on(table.grain, table.period_key, table.dataset, table.tier, table.rank_by_tokens),
index("idx_provider").on(table.provider, table.grain, table.period_key),
],
)
export const geoStat = mysqlTable(
"geo_stat",
{
...periodColumns(),
provider: varchar({ length: 128 }).notNull().default("all"),
model: varchar({ length: 256 }).notNull().default("all"),
country: char({ length: 2 }).notNull(),
continent: varchar({ length: 8 }).notNull().default(""),
...metricColumns(),
...marketShareColumns(),
rank_by_tokens: int(),
rank_by_requests: int(),
rank_by_sessions: int(),
rank_by_cost: int(),
...timestampColumns(),
},
(table) => [
uniqueIndex("uniq_country_period").on(
table.grain,
table.period_key,
table.dataset,
table.tier,
table.client,
table.source,
table.provider,
table.model,
table.country,
),
index("idx_country_map_tokens").on(table.grain, table.period_key, table.dataset, table.tier, table.total_tokens),
index("idx_country_rank").on(table.grain, table.period_key, table.dataset, table.tier, table.rank_by_tokens),
index("idx_country").on(table.country, table.grain, table.period_key),
index("idx_continent").on(table.continent, table.grain, table.period_key),
index("idx_country_model").on(table.model, table.country, table.grain, table.period_key),
],
)
function periodColumns() {
return {
id: bigint({ mode: "number" }).autoincrement().primaryKey(),
grain: varchar({ length: 16 }).notNull(),
period_key: varchar({ length: 32 }).notNull(),
dataset: varchar({ length: 64 }).notNull().default("all"),
tier: varchar({ length: 64 }).notNull().default("all"),
client: varchar({ length: 64 }).notNull().default("all"),
source: varchar({ length: 64 }).notNull().default("all"),
}
}
function metricColumns() {
return {
sessions: bigint({ mode: "number" }).notNull().default(0),
requests: bigint({ mode: "number" }).notNull().default(0),
input_tokens: bigint({ mode: "number" }).notNull().default(0),
output_tokens: bigint({ mode: "number" }).notNull().default(0),
reasoning_tokens: bigint({ mode: "number" }).notNull().default(0),
cache_read_tokens: bigint({ mode: "number" }).notNull().default(0),
total_tokens: bigint({ mode: "number" }).notNull().default(0),
input_cost_microcents: bigint({ mode: "number" }).notNull().default(0),
output_cost_microcents: bigint({ mode: "number" }).notNull().default(0),
total_cost_microcents: bigint({ mode: "number" }).notNull().default(0),
avg_duration_ms: decimal({ precision: 12, scale: 2, mode: "number" }),
p50_duration_ms: int(),
p95_duration_ms: int(),
avg_ttfb_ms: decimal({ precision: 12, scale: 2, mode: "number" }),
p50_ttfb_ms: int(),
p95_ttfb_ms: int(),
avg_output_tps: decimal({ precision: 12, scale: 4, mode: "number" }),
success_count: bigint({ mode: "number" }).notNull().default(0),
error_count: bigint({ mode: "number" }).notNull().default(0),
sample_count: bigint({ mode: "number" }).notNull().default(0),
}
}
function marketShareColumns() {
return {
market_share_tokens: decimal({ precision: 10, scale: 6, mode: "number" }),
market_share_requests: decimal({ precision: 10, scale: 6, mode: "number" }),
market_share_sessions: decimal({ precision: 10, scale: 6, mode: "number" }),
}
}
function timestampColumns() {
return {
created_at: datetime({ mode: "date" }).notNull().defaultNow(),
updated_at: datetime({ mode: "date" }).notNull().defaultNow().onUpdateNow(),
}
}
packages/stats/core/src/domain/geo.ts
-203
View File
@@ -1,203 +0,0 @@
import { and, asc, eq } from "drizzle-orm"
import { Effect, Layer } from "effect"
import * as Context from "effect/Context"
import { DatabaseError, DrizzleClient } from "../database"
import { geoStat } from "../database/schema"
import {
chunks,
collapseRows,
inserted,
rankRowsWithMarketShare,
statPeriodKey,
synthesizeAllTierRows,
toStatBaseRow,
UPSERT_CHUNK_SIZE,
type StatBaseAggregate,
} from "./stat"
export type GeoStatRow = typeof geoStat.$inferInsert
export type GeoStatAggregate = StatBaseAggregate & {
provider: string
model: string
country: string
continent: string
}
export type GeoStatMetric = {
periodKey: string
updatedAt: Date
tier: string
provider: string
model: string
country: string
continent: string
totalTokens: number
}
export declare namespace GeoStatRepo {
export interface Service {
readonly listDaily: () => Effect.Effect<GeoStatMetric[], DatabaseError>
readonly listByPeriod: (opts: {
readonly grain: string
readonly periodKey: string
readonly dataset?: string
readonly tier?: string
readonly client?: string
readonly source?: string
readonly provider?: string
readonly model?: string
}) => Effect.Effect<GeoStatRow[], DatabaseError>
readonly upsert: (rows: GeoStatRow[]) => Effect.Effect<void, DatabaseError>
}
}
export class GeoStatRepo extends Context.Service<GeoStatRepo, GeoStatRepo.Service>()("@opencode/stats/GeoStatRepo") {
static readonly layer: Layer.Layer<GeoStatRepo, never, DrizzleClient> = Layer.effect(
GeoStatRepo,
Effect.gen(function* () {
const db = yield* DrizzleClient
const listDaily = Effect.fn("GeoStatRepo.listDaily")(function* () {
return yield* Effect.tryPromise({
try: () =>
db
.select({
periodKey: geoStat.period_key,
updatedAt: geoStat.updated_at,
tier: geoStat.tier,
provider: geoStat.provider,
model: geoStat.model,
country: geoStat.country,
continent: geoStat.continent,
totalTokens: geoStat.total_tokens,
})
.from(geoStat)
.where(
and(
eq(geoStat.grain, "day"),
eq(geoStat.client, "all"),
eq(geoStat.source, "all"),
eq(geoStat.provider, "all"),
eq(geoStat.model, "all"),
),
)
.orderBy(asc(geoStat.period_key)),
catch: (cause) => DatabaseError.make({ cause }),
})
})
const listByPeriod = Effect.fn("GeoStatRepo.listByPeriod")(function* (opts: {
readonly grain: string
readonly periodKey: string
readonly dataset?: string
readonly tier?: string
readonly client?: string
readonly source?: string
readonly provider?: string
readonly model?: string
}) {
return yield* Effect.tryPromise({
try: () =>
db
.select()
.from(geoStat)
.where(
and(
eq(geoStat.grain, opts.grain),
eq(geoStat.period_key, opts.periodKey),
eq(geoStat.dataset, opts.dataset ?? "zen"),
eq(geoStat.tier, opts.tier ?? "all"),
eq(geoStat.client, opts.client ?? "all"),
eq(geoStat.source, opts.source ?? "all"),
eq(geoStat.provider, opts.provider ?? "all"),
eq(geoStat.model, opts.model ?? "all"),
),
),
catch: (cause) => DatabaseError.make({ cause }),
})
})
const upsert = Effect.fn("GeoStatRepo.upsert")(function* (rows: GeoStatRow[]) {
yield* Effect.forEach(
chunks(rows, UPSERT_CHUNK_SIZE),
(chunk) =>
Effect.tryPromise({
try: () =>
db
.insert(geoStat)
.values(chunk)
.onDuplicateKeyUpdate({
set: {
continent: inserted("continent"),
sessions: inserted("sessions"),
requests: inserted("requests"),
input_tokens: inserted("input_tokens"),
output_tokens: inserted("output_tokens"),
reasoning_tokens: inserted("reasoning_tokens"),
cache_read_tokens: inserted("cache_read_tokens"),
total_tokens: inserted("total_tokens"),
input_cost_microcents: inserted("input_cost_microcents"),
output_cost_microcents: inserted("output_cost_microcents"),
total_cost_microcents: inserted("total_cost_microcents"),
avg_duration_ms: inserted("avg_duration_ms"),
p50_duration_ms: inserted("p50_duration_ms"),
p95_duration_ms: inserted("p95_duration_ms"),
avg_ttfb_ms: inserted("avg_ttfb_ms"),
p50_ttfb_ms: inserted("p50_ttfb_ms"),
p95_ttfb_ms: inserted("p95_ttfb_ms"),
avg_output_tps: inserted("avg_output_tps"),
success_count: inserted("success_count"),
error_count: inserted("error_count"),
sample_count: inserted("sample_count"),
market_share_tokens: inserted("market_share_tokens"),
market_share_requests: inserted("market_share_requests"),
market_share_sessions: inserted("market_share_sessions"),
rank_by_tokens: inserted("rank_by_tokens"),
rank_by_requests: inserted("rank_by_requests"),
rank_by_sessions: inserted("rank_by_sessions"),
rank_by_cost: inserted("rank_by_cost"),
},
}),
catch: (cause) => DatabaseError.make({ cause }),
}),
{ discard: true },
)
})
return GeoStatRepo.of({ listDaily, listByPeriod, upsert })
}),
)
}
export function rowsFromAggregates(aggregates: GeoStatAggregate[]) {
return rankRowsWithMarketShare(
[
...synthesizeAllTierRows(
collapseRows(aggregates.filter((item) => item.grain === "week").map(toRow), dimensionKey),
dimensionKey,
),
...synthesizeAllTierRows(
collapseRows(aggregates.filter((item) => item.grain === "day").map(toRow), dimensionKey),
dimensionKey,
),
],
marketShareKey,
)
}
function toRow(data: GeoStatAggregate): GeoStatRow {
return {
...toStatBaseRow(data),
provider: data.provider,
model: data.model,
country: data.country,
continent: data.continent,
}
}
function dimensionKey(row: GeoStatRow) {
return [row.provider, row.model, row.country].join("\u0000")
}
function marketShareKey(row: GeoStatRow) {
return [statPeriodKey(row), row.provider, row.model].join("\u0000")
}
packages/stats/core/src/domain/home.ts
-499
View File
@@ -1,499 +0,0 @@
import { Effect } from "effect"
import { DatabaseError } from "../database"
import { GeoStatRepo, type GeoStatMetric } from "./geo"
import { ModelStatRepo, type ModelStatMetric } from "./model"
import { ProviderStatRepo, type ProviderStatMetric } from "./provider"
export type UsageProduct = "All Users" | "Zen" | "Go" | "Enterprise"
export type TokenProduct = "Zen" | "Go" | "Enterprise"
export type UsageRange = "1D" | "1W" | "2W" | "1M" | "2M" | "3M" | "YTD" | "ALL"
export type UsagePoint = { date: string; segments: { model: string; value: number }[] }
export type MarketDay = { date: string; total: number; authors: { author: string; share: number; tokens: number }[] }
export type LeaderboardEntry = { model: string; author: string; tokens: number; change: number; rank: number }
export type TokenCostEntry = { model: string; total: number; input: number; output: number; cached: number }
export type SessionCostEntry = { model: string; cost: number; tokens: number }
export type CountryEntry = { country: string; continent: string; tokens: number; share: number; rank: number }
export type StatsHomeData = {
updatedAt: string | null
usage: Record<UsageProduct, Record<UsageRange, UsagePoint[]>>
leaderboard: Record<UsageProduct, Record<UsageRange, LeaderboardEntry[]>>
market: Record<UsageRange, MarketDay[]>
tokenCost: Record<TokenProduct, TokenCostEntry[]>
sessionCost: Record<TokenProduct, SessionCostEntry[]>
country: Record<UsageRange, CountryEntry[]>
}
const DAY_MS = 86_400_000
const TOKEN_SCALE = 1_000_000
const DOLLARS_PER_MICROCENT = 1 / 100_000_000
const months = ["JAN", "FEB", "MAR", "APR", "MAY", "JUN", "JUL", "AUG", "SEP", "OCT", "NOV", "DEC"] as const
type StatMetricRow = Omit<ModelStatMetric, "updatedAt"> & {
periodStart: number
updatedAt: number
}
type ProviderMetricRow = Omit<ProviderStatMetric, "updatedAt"> & {
periodStart: number
updatedAt: number
}
type GeoMetricRow = Omit<GeoStatMetric, "updatedAt"> & {
periodStart: number
updatedAt: number
}
type DateWindow = { start: number; end: number; previousStart: number; previousEnd: number }
type Bucket = { start: number; end: number; label: string }
type ModelAggregate = {
model: string
provider: string
sessions: number
inputTokens: number
outputTokens: number
reasoningTokens: number
cacheReadTokens: number
totalTokens: number
inputCostMicrocents: number
outputCostMicrocents: number
totalCostMicrocents: number
}
export const getStatsHomeData: () => Effect.Effect<
StatsHomeData,
DatabaseError,
ModelStatRepo | ProviderStatRepo | GeoStatRepo
> = Effect.fn("StatsHome.getData")(function* () {
const modelStats = yield* ModelStatRepo
const providerStats = yield* ProviderStatRepo
const geoStats = yield* GeoStatRepo
const [modelRows, providerRows, geoRows] = yield* Effect.all(
[modelStats.listDaily(), providerStats.listDaily(), geoStats.listDaily()],
{ concurrency: "unbounded" },
)
return buildStatsHomeData(modelRows, providerRows, geoRows)
})
function buildStatsHomeData(
modelRows: ModelStatMetric[],
providerRows: ProviderStatMetric[],
geoRows: GeoStatMetric[],
): StatsHomeData {
const normalized = modelRows.flatMap(normalizeStatRow)
const providers = providerRows.flatMap(normalizeProviderRow)
const geo = geoRows.flatMap(normalizeGeoRow)
const periods = [...normalized, ...providers, ...geo]
if (periods.length === 0) return emptyStatsHomeData()
const earliest = Math.min(...periods.map((row) => row.periodStart))
const latest = Math.max(...periods.map((row) => row.periodStart))
const latestUpdate = Math.max(...periods.map((row) => row.updatedAt))
return {
updatedAt: new Date(latestUpdate).toISOString(),
usage: createUsageProductRecord((product) =>
createRangeRecord((range) => buildUsagePoints(normalized, product, range, getWindow(range, earliest, latest))),
),
leaderboard: createUsageProductRecord((product) =>
createRangeRecord((range) => buildLeaderboard(normalized, product, getWindow(range, earliest, latest))),
),
market: createRangeRecord((range) => buildMarketShare(providers, range, getWindow(range, earliest, latest))),
tokenCost: createTokenProductRecord((product) =>
buildTokenCost(normalized, product, getWindow("1W", earliest, latest)),
),
sessionCost: createTokenProductRecord((product) =>
buildSessionCost(normalized, product, getWindow("1W", earliest, latest)),
),
country: createRangeRecord((range) => buildCountryStats(geo, getWindow(range, earliest, latest))),
}
}
function emptyStatsHomeData(): StatsHomeData {
return {
updatedAt: null,
usage: createUsageProductRecord(() => createRangeRecord(() => [])),
leaderboard: createUsageProductRecord(() => createRangeRecord(() => [])),
market: createRangeRecord(() => []),
tokenCost: createTokenProductRecord(() => []),
sessionCost: createTokenProductRecord(() => []),
country: createRangeRecord(() => []),
}
}
function buildUsagePoints(rows: StatMetricRow[], product: UsageProduct, range: UsageRange, window: DateWindow) {
const windowRows = rowsForProduct(rows, product, window.start, window.end)
const modelOrder = aggregateByModel(windowRows)
.toSorted((a, b) => b.totalTokens - a.totalTokens)
.slice(0, 6)
.map((item) => ({ key: modelKey(item.provider, item.model), model: item.model }))
return createBuckets(window, range).map((bucket) => {
const bucketRows = aggregateByModel(rowsForProduct(rows, product, bucket.start, bucket.end))
const byModel = new Map(bucketRows.map((item) => [modelKey(item.provider, item.model), item.totalTokens]))
const segmentTokens = modelOrder.map((model) => ({ model: model.model, tokens: byModel.get(model.key) ?? 0 }))
const knownTokens = segmentTokens.reduce((sum, item) => sum + item.tokens, 0)
const totalTokens = bucketRows.reduce((sum, item) => sum + item.totalTokens, 0)
return {
date: bucket.label,
segments: [
...segmentTokens.map((item) => ({ model: item.model, value: round(item.tokens / 1_000_000_000_000, 4) })),
{ model: "Other", value: round(Math.max(totalTokens - knownTokens, 0) / 1_000_000_000_000, 4) },
],
}
})
}
function buildLeaderboard(rows: StatMetricRow[], product: UsageProduct, window: DateWindow) {
const previous = new Map(
aggregateByModel(rowsForProduct(rows, product, window.previousStart, window.previousEnd)).map((item) => [
modelKey(item.provider, item.model),
item.totalTokens,
]),
)
return aggregateByModel(rowsForProduct(rows, product, window.start, window.end))
.toSorted((a, b) => b.totalTokens - a.totalTokens)
.slice(0, 18)
.map((item, index) => ({
model: item.model,
author: formatProvider(item.provider),
tokens: Math.round(item.totalTokens / 1_000_000_000),
change: percentChange(item.totalTokens, previous.get(modelKey(item.provider, item.model)) ?? 0),
rank: index + 1,
}))
}
function buildMarketShare(rows: ProviderMetricRow[], range: UsageRange, window: DateWindow) {
return createBuckets(window, range).flatMap((bucket) => {
const total = aggregateByProvider(rowsForProduct(rows, "All Users", bucket.start, bucket.end)).toSorted(
(a, b) => b.tokens - a.tokens,
)
const totalTokens = total.reduce((sum, item) => sum + item.tokens, 0)
if (totalTokens === 0) return []
const authors = total.slice(0, 8)
const knownTokens = authors.reduce((sum, item) => sum + item.tokens, 0)
const withOther = [...authors, { provider: "Other", tokens: Math.max(totalTokens - knownTokens, 0) }].filter(
(item) => item.tokens > 0,
)
return [
{
date: bucket.label,
total: round(totalTokens / 1_000_000_000_000, 2),
authors: withOther.map((item) => ({
author: item.provider === "Other" ? "Other" : formatProvider(item.provider),
share: round((item.tokens / totalTokens) * 100, 1),
tokens: round(item.tokens / 1_000_000_000_000, 2),
})),
},
]
})
}
function buildCountryStats(rows: GeoMetricRow[], window: DateWindow) {
const countries = aggregateByCountry(rowsForProduct(rows, "All Users", window.start, window.end))
.filter((item) => item.tokens > 0)
.toSorted((a, b) => b.tokens - a.tokens)
const totalTokens = countries.reduce((sum, item) => sum + item.tokens, 0)
if (totalTokens === 0) return []
return countries.slice(0, 16).map((item, index) => ({
country: item.country,
continent: item.continent,
tokens: round(item.tokens / 1_000_000_000_000, 4),
share: round((item.tokens / totalTokens) * 100, 1),
rank: index + 1,
}))
}
function buildTokenCost(rows: StatMetricRow[], product: TokenProduct, window: DateWindow) {
return aggregateByModel(rowsForProduct(rows, product, window.start, window.end))
.flatMap((item) => {
const total = costPerMillion(item.totalCostMicrocents, item.totalTokens)
if (total === 0) return []
return [
{
model: item.model,
total,
input: costPerMillion(item.inputCostMicrocents, item.inputTokens),
output: costPerMillion(item.outputCostMicrocents, item.outputTokens + item.reasoningTokens),
cached: costPerMillion(item.inputCostMicrocents, item.inputTokens + item.cacheReadTokens),
},
]
})
.toSorted((a, b) => a.total - b.total)
.slice(0, 17)
}
function buildSessionCost(rows: StatMetricRow[], product: TokenProduct, window: DateWindow) {
return aggregateByModel(rowsForProduct(rows, product, window.start, window.end))
.flatMap((item) => {
if (item.sessions === 0) return []
const cost = round(microcentsToDollars(item.totalCostMicrocents) / item.sessions, 4)
if (cost === 0) return []
return [{ model: item.model, cost, tokens: Math.round(item.totalTokens / item.sessions) }]
})
.toSorted((a, b) => a.cost - b.cost)
.slice(0, 17)
}
function rowsForProduct<T extends { periodStart: number; tier: string }>(
rows: T[],
product: UsageProduct,
start: number,
end: number,
) {
const windowRows = rows.filter((row) => row.periodStart >= start && row.periodStart < end)
if (product !== "All Users") return windowRows.filter((row) => row.tier === product)
const allRows = windowRows.filter((row) => row.tier === "all")
if (allRows.length > 0) return allRows
return windowRows.filter((row) => row.tier !== "all")
}
function aggregateByModel(rows: StatMetricRow[]) {
return Object.values(
rows.reduce<Record<string, ModelAggregate>>((result, row) => {
const key = modelKey(row.provider, row.model)
result[key] = combineModelAggregate(result[key], row)
return result
}, {}),
)
}
function aggregateByProvider(rows: ProviderMetricRow[]) {
return Object.values(
rows.reduce<Record<string, { provider: string; tokens: number }>>((result, row) => {
result[row.provider] = {
provider: row.provider,
tokens: (result[row.provider]?.tokens ?? 0) + row.totalTokens,
}
return result
}, {}),
)
}
function aggregateByCountry(rows: GeoMetricRow[]) {
return Object.values(
rows.reduce<Record<string, { country: string; continent: string; tokens: number }>>((result, row) => {
result[row.country] = {
country: row.country,
continent: result[row.country]?.continent || row.continent,
tokens: (result[row.country]?.tokens ?? 0) + row.totalTokens,
}
return result
}, {}),
)
}
function combineModelAggregate(current: ModelAggregate | undefined, row: StatMetricRow): ModelAggregate {
return {
model: row.model,
provider: row.provider,
sessions: (current?.sessions ?? 0) + row.sessions,
inputTokens: (current?.inputTokens ?? 0) + row.inputTokens,
outputTokens: (current?.outputTokens ?? 0) + row.outputTokens,
reasoningTokens: (current?.reasoningTokens ?? 0) + row.reasoningTokens,
cacheReadTokens: (current?.cacheReadTokens ?? 0) + row.cacheReadTokens,
totalTokens: (current?.totalTokens ?? 0) + row.totalTokens,
inputCostMicrocents: (current?.inputCostMicrocents ?? 0) + row.inputCostMicrocents,
outputCostMicrocents: (current?.outputCostMicrocents ?? 0) + row.outputCostMicrocents,
totalCostMicrocents: (current?.totalCostMicrocents ?? 0) + row.totalCostMicrocents,
}
}
function getWindow(range: UsageRange, earliest: number, latest: number): DateWindow {
const end = latest + DAY_MS
const start = Math.max(
earliest,
range === "1D"
? latest
: range === "1W"
? latest - 6 * DAY_MS
: range === "2W"
? latest - 13 * DAY_MS
: range === "1M"
? latest - 27 * DAY_MS
: range === "2M"
? latest - 55 * DAY_MS
: range === "3M"
? latest - 89 * DAY_MS
: range === "YTD"
? Date.UTC(new Date(latest).getUTCFullYear(), 0, 1)
: earliest,
)
const duration = end - start
return { start, end, previousStart: start - duration, previousEnd: start }
}
function createBuckets(window: DateWindow, range: UsageRange): Bucket[] {
const span = Math.max(window.end - window.start, DAY_MS)
const count =
range === "1D"
? 1
: range === "1W" || range === "2W" || range === "1M" || range === "2M" || range === "3M"
? Math.ceil(span / DAY_MS)
: Math.max(1, Math.min(7, Math.ceil(span / DAY_MS)))
const size = span / count
return Array.from({ length: count }, (_, index) => {
const start = window.start + index * size
const end = index === count - 1 ? window.end : window.start + (index + 1) * size
return { start, end, label: formatBucketLabel(start, end, range) }
})
}
function createUsageProductRecord<T>(value: (product: UsageProduct) => T): Record<UsageProduct, T> {
return {
"All Users": value("All Users"),
Zen: value("Zen"),
Go: value("Go"),
Enterprise: value("Enterprise"),
}
}
function createTokenProductRecord<T>(value: (product: TokenProduct) => T): Record<TokenProduct, T> {
return {
Zen: value("Zen"),
Go: value("Go"),
Enterprise: value("Enterprise"),
}
}
function createRangeRecord<T>(value: (range: UsageRange) => T): Record<UsageRange, T> {
return {
"1D": value("1D"),
"1W": value("1W"),
"2W": value("2W"),
"1M": value("1M"),
"2M": value("2M"),
"3M": value("3M"),
YTD: value("YTD"),
ALL: value("ALL"),
}
}
function normalizeStatRow(row: ModelStatMetric): StatMetricRow[] {
const periodStart = periodKeyTime(row.periodKey)
const updatedAt = dateTime(row.updatedAt)
if (!Number.isFinite(periodStart) || !Number.isFinite(updatedAt)) return []
return [
{
...row,
periodStart,
updatedAt,
tier: normalizeTier(row.tier),
provider: row.provider || "unknown",
model: row.model || "unknown",
},
]
}
function normalizeProviderRow(row: ProviderStatMetric): ProviderMetricRow[] {
const periodStart = periodKeyTime(row.periodKey)
const updatedAt = dateTime(row.updatedAt)
if (!Number.isFinite(periodStart) || !Number.isFinite(updatedAt)) return []
return [
{
...row,
periodStart,
updatedAt,
tier: normalizeTier(row.tier),
provider: row.provider || "unknown",
},
]
}
function normalizeGeoRow(row: GeoStatMetric): GeoMetricRow[] {
const periodStart = periodKeyTime(row.periodKey)
const updatedAt = dateTime(row.updatedAt)
if (!Number.isFinite(periodStart) || !Number.isFinite(updatedAt)) return []
return [
{
...row,
periodStart,
updatedAt,
tier: normalizeTier(row.tier),
provider: row.provider || "all",
model: row.model || "all",
country: row.country || "ZZ",
continent: row.continent || "",
},
]
}
function normalizeTier(value: string) {
const normalized = value.toLowerCase()
if (normalized === "paid" || normalized === "zen") return "Zen"
if (normalized === "go") return "Go"
if (normalized === "enterprise") return "Enterprise"
if (normalized === "all") return "all"
return value
}
function dateTime(value: Date | string) {
return (value instanceof Date ? value : new Date(value)).getTime()
}
function periodKeyTime(value: string) {
const match = /^(\d{4})-(\d{2})-(\d{2})$/.exec(value)
if (!match) return Number.NaN
return Date.UTC(Number(match[1]), Number(match[2]) - 1, Number(match[3]))
}
function formatBucketLabel(start: number, _end: number, range: UsageRange) {
const date = new Date(start)
if (range === "YTD") return months[date.getUTCMonth()]
if (range === "ALL")
return date.getUTCFullYear() === new Date().getUTCFullYear()
? months[date.getUTCMonth()]
: String(date.getUTCFullYear())
return formatDay(start)
}
function formatDay(value: number) {
const date = new Date(value)
return `${months[date.getUTCMonth()]} ${date.getUTCDate()}`
}
function formatProvider(provider: string) {
const known: Record<string, string> = {
anthropic: "Anthropic",
deepseek: "DeepSeek",
google: "Google",
minimax: "MiniMax",
moonshot: "Moonshot",
moonshotai: "Moonshot",
nvidia: "NVIDIA",
opencode: "opencode",
openai: "OpenAI",
qwen: "Qwen",
tencent: "Tencent",
xai: "xAI",
xiaomi: "Xiaomi",
zhipu: "Zhipu",
zhipuai: "Zhipu",
}
const normalized = provider.toLowerCase().replace(/[^a-z0-9]/g, "")
return known[normalized] ?? provider.replace(/[-_]/g, " ").replace(/\b\w/g, (letter) => letter.toUpperCase())
}
function modelKey(provider: string, model: string) {
return `${provider}\u0000${model}`
}
function costPerMillion(costMicrocents: number, tokens: number) {
if (tokens <= 0 || costMicrocents <= 0) return 0
return round((microcentsToDollars(costMicrocents) / tokens) * TOKEN_SCALE, 2)
}
function microcentsToDollars(value: number) {
return value * DOLLARS_PER_MICROCENT
}
function percentChange(current: number, previous: number) {
if (previous <= 0) return current > 0 ? 100 : 0
return Math.round(((current - previous) / previous) * 100)
}
function round(value: number, digits: number) {
return Number(value.toFixed(digits))
}
packages/stats/core/src/domain/inference.test.ts
-66
View File
@@ -1,66 +0,0 @@
import { describe, expect, test } from "bun:test"
import { toModelAggregate } from "./inference"
import { modelAuthor, normalizeInferenceModel } from "./model-normalization"
describe("inference stat normalization", () => {
test("normalizes model suffixes used by router/provider variants", () => {
expect(normalizeInferenceModel("deepseek-v4-flash-free")).toBe("deepseek-v4-flash")
expect(normalizeInferenceModel("deepseek-v4-flash:global")).toBe("deepseek-v4-flash")
expect(normalizeInferenceModel("mimo-v2.5-free")).toBe("mimo-v2.5")
expect(normalizeInferenceModel("nemotron-3-super-free")).toBe("nemotron-3-super")
expect(normalizeInferenceModel("mimo-v2.5-free:global")).toBe("mimo-v2.5")
})
test("maps normalized model ids to public authors", () => {
expect(modelAuthor("big-pickle")).toBe("opencode")
expect(modelAuthor("claude-sonnet-4-5")).toBe("anthropic")
expect(modelAuthor("deepseek-v4-pro")).toBe("deepseek")
expect(modelAuthor("gemini-3.5-flash")).toBe("google")
expect(modelAuthor("glm-5.1")).toBe("zhipu")
expect(modelAuthor("gpt-5.5-pro")).toBe("openai")
expect(modelAuthor("grok-build-0.1")).toBe("xai")
expect(modelAuthor("hy3-preview")).toBe("tencent")
expect(modelAuthor("kimi-k2.6")).toBe("moonshot")
expect(modelAuthor("mimo-v2-omni")).toBe("xiaomi")
expect(modelAuthor("minimax-m2.7")).toBe("minimax")
expect(modelAuthor("nemotron-3-super-free")).toBe("nvidia")
expect(modelAuthor("qwen3.7-max")).toBe("qwen")
expect(modelAuthor("alpha-gpt-next")).toBeUndefined()
})
test("model aggregates ignore datalake provider and use normalized author/model", () => {
expect(toModelAggregate(aggregate("alpha-gpt-next", "openai"))).toEqual([])
expect(toModelAggregate(aggregate("deepseek-v4-flash-free", "not-public-provider"))).toMatchObject([
{
period_key: "2026-05-20",
provider: "deepseek",
model: "deepseek-v4-flash",
},
])
})
test("model aggregates use ISO week period keys", () => {
expect(
toModelAggregate({
...aggregate("gpt-5.5-pro", "openai"),
grain: "week",
period_key: "2026-W20",
}),
).toMatchObject([{ period_key: "2026-W20" }])
})
})
function aggregate(model: string, provider: string) {
return {
grain: "day",
period_key: "2026-05-20",
dataset: "zen",
tier: "Paid",
provider,
model,
sessions: "1",
requests: "1",
sample_count: "1",
}
}
packages/stats/core/src/domain/inference.ts
-252
View File
@@ -1,252 +0,0 @@
import { Resource } from "sst/resource"
import type { AthenaData } from "../athena"
import type { GeoStatAggregate } from "./geo"
import type { ModelStatAggregate } from "./model"
import {
EXCLUDED_MODELS,
MODEL_AUTHOR_OVERRIDES,
MODEL_AUTHOR_RULES,
modelAuthor,
normalizeInferenceModel,
} from "./model-normalization"
import type { ProviderStatAggregate } from "./provider"
import { normalizeCountry, normalizeTier, type StatBaseAggregate } from "./stat"
export type StatDimension = "model" | "provider" | "geo" | "geo_model"
export function buildStatsQuery(periodStart: Date, periodEnd: Date, dimension: StatDimension) {
const periodStartValue = sqlString(periodStart.toISOString())
const periodEndValue = sqlString(periodEnd.toISOString())
const sourceTable = [Resource.InferenceEvent.catalog, Resource.InferenceEvent.database, Resource.InferenceEvent.table]
.map(sqlIdentifier)
.join(".")
const dimensionSql = (() => {
if (dimension === "model")
return {
select: "provider, model, COALESCE(MAX(NULLIF(provider_model, '')), '') AS provider_model",
groupBy: "provider, model",
}
if (dimension === "provider") return { select: "provider", groupBy: "provider" }
if (dimension === "geo_model")
return {
select: "provider, model, country, COALESCE(MAX(NULLIF(continent, '')), '') AS continent",
groupBy: "provider, model, country",
}
return {
select: "'all' AS provider, 'all' AS model, country, COALESCE(MAX(NULLIF(continent, '')), '') AS continent",
groupBy: "country",
}
})()
const aggregateColumns = `
COUNT(DISTINCT session) AS sessions,
COUNT(*) AS requests,
COALESCE(SUM(tokens_input), 0) AS input_tokens,
COALESCE(SUM(tokens_output), 0) AS output_tokens,
COALESCE(SUM(tokens_reasoning), 0) AS reasoning_tokens,
COALESCE(SUM(tokens_cache_read), 0) AS cache_read_tokens,
COALESCE(SUM(tokens_total), 0) AS total_tokens,
COALESCE(SUM(cost_input_microcents), 0) AS input_cost_microcents,
COALESCE(SUM(cost_output_microcents), 0) AS output_cost_microcents,
COALESCE(SUM(cost_total_microcents), 0) AS total_cost_microcents,
AVG(duration_ms) AS avg_duration_ms,
approx_percentile(CAST(duration_ms AS double), 0.5) AS p50_duration_ms,
approx_percentile(CAST(duration_ms AS double), 0.95) AS p95_duration_ms,
AVG(ttfb_ms) AS avg_ttfb_ms,
approx_percentile(CAST(ttfb_ms AS double), 0.5) AS p50_ttfb_ms,
approx_percentile(CAST(ttfb_ms AS double), 0.95) AS p95_ttfb_ms,
AVG(output_tps) AS avg_output_tps,
SUM(CASE WHEN status >= 200 AND status < 400 THEN 1 ELSE 0 END) AS success_count,
SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END) AS error_count,
COUNT(*) AS sample_count`
return `
WITH normalized AS (
SELECT
from_iso8601_timestamp(event_timestamp) AS event_time,
model AS raw_model,
COALESCE(NULLIF(regexp_replace(model, '(-free|:global)+$', ''), ''), 'unknown') AS model,
COALESCE(NULLIF(provider_model, ''), '') AS provider_model,
UPPER(COALESCE(NULLIF(cf_country, ''), 'ZZ')) AS country,
COALESCE(NULLIF(cf_continent, ''), '') AS continent,
session,
status,
duration AS duration_ms,
time_to_first_byte AS ttfb_ms,
timestamp_first_byte,
timestamp_last_byte,
tokens_input,
tokens_output,
tokens_reasoning,
tokens_cache_read,
tokens_cache_write_5m,
cost_input_microcents,
cost_output_microcents,
cost_total_microcents,
cost_input,
cost_output,
cost_total,
source
FROM ${sourceTable}
WHERE event_type = 'completions'
AND model IS NOT NULL
AND model <> ''
AND (strpos(COALESCE(user_agent, ''), 'ai-sdk') > 0 OR strpos(COALESCE(user_agent, ''), 'opencode') > 0)
AND event_timestamp >= ${periodStartValue}
AND event_timestamp < ${periodEndValue}
), filtered AS (
SELECT
event_time,
CASE
WHEN source = 'lite' THEN 'Go'
WHEN model IN ('gpt-5-nano', 'grok-code', 'big-pickle') OR regexp_like(raw_model, '-free(:global)?$') THEN 'Free'
ELSE 'Paid'
END AS tier,
${modelAuthorSql("model")} AS provider,
provider_model,
model,
country,
continent,
session,
status,
duration_ms,
ttfb_ms,
CASE
WHEN timestamp_last_byte - timestamp_first_byte < 100 THEN null
ELSE CAST(tokens_output AS double) / (timestamp_last_byte - timestamp_first_byte) * 1000
END AS output_tps,
tokens_input,
tokens_output,
tokens_reasoning,
tokens_cache_read,
COALESCE(tokens_cache_read, 0) + COALESCE(tokens_cache_write_5m, 0) + COALESCE(tokens_input, 0) + COALESCE(tokens_output, 0) AS tokens_total,
COALESCE(cost_input_microcents, cost_input * 1000000) AS cost_input_microcents,
COALESCE(cost_output_microcents, cost_output * 1000000) AS cost_output_microcents,
COALESCE(cost_total_microcents, cost_total * 1000000) AS cost_total_microcents
FROM normalized
WHERE lower(model) NOT IN (${[...EXCLUDED_MODELS].map(sqlString).join(", ")})
), weekly AS (
SELECT
concat(CAST(year_of_week(event_time) AS varchar), '-W', lpad(CAST(week(event_time) AS varchar), 2, '0')) AS week_key,
*
FROM filtered
), daily AS (
SELECT substr(to_iso8601(date_trunc('day', event_time)), 1, 10) AS day_key, *
FROM filtered
)
SELECT
'week' AS grain,
week_key AS period_key,
${sqlString(Resource.StatsSyncConfig.dataset)} AS dataset,
tier,
${dimensionSql.select},
${aggregateColumns}
FROM weekly
GROUP BY week_key, tier, ${dimensionSql.groupBy}
UNION ALL
SELECT
'day' AS grain,
day_key AS period_key,
${sqlString(Resource.StatsSyncConfig.dataset)} AS dataset,
tier,
${dimensionSql.select},
${aggregateColumns}
FROM daily
GROUP BY day_key, tier, ${dimensionSql.groupBy}
ORDER BY grain, period_key, total_tokens DESC
`
}
export function toModelAggregate(data: AthenaData): ModelStatAggregate[] {
const model = normalizeInferenceModel(data.model)
const author = modelAuthor(model)
if (!author) return []
return toStatBaseAggregate(data).flatMap((base) => [
{ ...base, provider: author, model, provider_model: data.provider_model || "" },
])
}
export function toProviderAggregate(data: AthenaData): ProviderStatAggregate[] {
return toStatBaseAggregate(data).flatMap((base) => [{ ...base, provider: data.provider || "unknown" }])
}
export function toGeoAggregate(data: AthenaData): GeoStatAggregate[] {
return toStatBaseAggregate(data).flatMap((base) => [
{
...base,
provider: data.provider || "all",
model: normalizeInferenceModel(data.model || "all"),
country: normalizeCountry(data.country),
continent: data.continent || "",
},
])
}
function toStatBaseAggregate(data: AthenaData): StatBaseAggregate[] {
const grain = data.grain === "day" || data.grain === "week" ? data.grain : undefined
if (!grain || !data.period_key) return []
return [
{
grain,
period_key: data.period_key,
dataset: data.dataset || Resource.StatsSyncConfig.dataset,
tier: normalizeTier(data.tier || "unknown"),
sessions: integer(data, "sessions"),
requests: integer(data, "requests"),
input_tokens: integer(data, "input_tokens"),
output_tokens: integer(data, "output_tokens"),
reasoning_tokens: integer(data, "reasoning_tokens"),
cache_read_tokens: integer(data, "cache_read_tokens"),
total_tokens: integer(data, "total_tokens"),
input_cost_microcents: integer(data, "input_cost_microcents"),
output_cost_microcents: integer(data, "output_cost_microcents"),
total_cost_microcents: integer(data, "total_cost_microcents"),
avg_duration_ms: nullableNumber(data, "avg_duration_ms"),
p50_duration_ms: nullableInteger(data, "p50_duration_ms"),
p95_duration_ms: nullableInteger(data, "p95_duration_ms"),
avg_ttfb_ms: nullableNumber(data, "avg_ttfb_ms"),
p50_ttfb_ms: nullableInteger(data, "p50_ttfb_ms"),
p95_ttfb_ms: nullableInteger(data, "p95_ttfb_ms"),
avg_output_tps: nullableNumber(data, "avg_output_tps"),
success_count: integer(data, "success_count"),
error_count: integer(data, "error_count"),
sample_count: integer(data, "sample_count"),
},
]
}
function integer(data: AthenaData, key: string) {
return Math.round(number(data, key))
}
function nullableNumber(data: AthenaData, key: string) {
if (data[key] === undefined || data[key] === "") return null
return Number(number(data, key).toFixed(2))
}
function nullableInteger(data: AthenaData, key: string) {
if (data[key] === undefined || data[key] === "") return null
return Math.round(number(data, key))
}
function number(data: AthenaData, key: string) {
const value = Number(data[key])
return Number.isFinite(value) ? value : 0
}
function sqlIdentifier(value: string) {
return `"${value.replace(/"/g, '""')}"`
}
function sqlString(value: string) {
return `'${value.replace(/'/g, "''")}'`
}
function modelAuthorSql(model: string) {
return `CASE
${MODEL_AUTHOR_OVERRIDES.map((item) => ` WHEN lower(${model}) = ${sqlString(item.model)} THEN ${sqlString(item.author)}`).join("\n")}
${MODEL_AUTHOR_RULES.map((item) => ` WHEN strpos(lower(${model}), ${sqlString(item.match)}) > 0 THEN ${sqlString(item.author)}`).join("\n")}
ELSE 'unknown'
END`
}
packages/stats/core/src/domain/model-normalization.ts
-30
View File
@@ -1,30 +0,0 @@
export const MODEL_AUTHOR_OVERRIDES = [{ model: "big-pickle", author: "opencode" }] as const
export const MODEL_AUTHOR_RULES = [
{ match: "claude", author: "anthropic" },
{ match: "gemini", author: "google" },
{ match: "deepseek", author: "deepseek" },
{ match: "glm", author: "zhipu" },
{ match: "gpt", author: "openai" },
{ match: "grok", author: "xai" },
{ match: "hy3", author: "tencent" },
{ match: "kimi", author: "moonshot" },
{ match: "mimo", author: "xiaomi" },
{ match: "minimax", author: "minimax" },
{ match: "nemotron", author: "nvidia" },
{ match: "qwen", author: "qwen" },
] as const
export const EXCLUDED_MODELS = new Set(["alpha-gpt-next"])
export function normalizeInferenceModel(value: string | undefined) {
return (value || "unknown").replace(/(-free|:global)+$/, "") || "unknown"
}
export function modelAuthor(value: string | undefined) {
const model = normalizeInferenceModel(value).toLowerCase()
if (EXCLUDED_MODELS.has(model)) return undefined
const override = MODEL_AUTHOR_OVERRIDES.find((item) => item.model === model)
if (override) return override.author
return MODEL_AUTHOR_RULES.find((item) => model.includes(item.match))?.author ?? "unknown"
}
packages/stats/core/src/domain/model.ts
-172
View File
@@ -1,172 +0,0 @@
import { and, asc, eq } from "drizzle-orm"
import { Effect, Layer } from "effect"
import * as Context from "effect/Context"
import { DatabaseError, DrizzleClient } from "../database"
import { modelStat } from "../database/schema"
import {
chunks,
collapseRows,
inserted,
rankBy,
statPeriodKey,
synthesizeAllTierRows,
toStatBaseRow,
UPSERT_CHUNK_SIZE,
type StatBaseAggregate,
} from "./stat"
export type ModelStatRow = typeof modelStat.$inferInsert
export type ModelStatAggregate = StatBaseAggregate & { provider: string; model: string; provider_model: string }
export type ModelStatMetric = {
periodKey: string
updatedAt: Date
tier: string
provider: string
model: string
sessions: number
inputTokens: number
outputTokens: number
reasoningTokens: number
cacheReadTokens: number
totalTokens: number
inputCostMicrocents: number
outputCostMicrocents: number
totalCostMicrocents: number
}
export declare namespace ModelStatRepo {
export interface Service {
readonly listDaily: () => Effect.Effect<ModelStatMetric[], DatabaseError>
readonly upsert: (rows: ModelStatRow[]) => Effect.Effect<void, DatabaseError>
}
}
export class ModelStatRepo extends Context.Service<ModelStatRepo, ModelStatRepo.Service>()(
"@opencode/stats/ModelStatRepo",
) {
static readonly layer: Layer.Layer<ModelStatRepo, never, DrizzleClient> = Layer.effect(
ModelStatRepo,
Effect.gen(function* () {
const db = yield* DrizzleClient
const listDaily = Effect.fn("ModelStatRepo.listDaily")(function* () {
return yield* Effect.tryPromise({
try: () =>
db
.select({
periodKey: modelStat.period_key,
updatedAt: modelStat.updated_at,
tier: modelStat.tier,
provider: modelStat.provider,
model: modelStat.model,
sessions: modelStat.sessions,
inputTokens: modelStat.input_tokens,
outputTokens: modelStat.output_tokens,
reasoningTokens: modelStat.reasoning_tokens,
cacheReadTokens: modelStat.cache_read_tokens,
totalTokens: modelStat.total_tokens,
inputCostMicrocents: modelStat.input_cost_microcents,
outputCostMicrocents: modelStat.output_cost_microcents,
totalCostMicrocents: modelStat.total_cost_microcents,
})
.from(modelStat)
.where(and(eq(modelStat.grain, "day"), eq(modelStat.client, "all"), eq(modelStat.source, "all")))
.orderBy(asc(modelStat.period_key)),
catch: (cause) => DatabaseError.make({ cause }),
})
})
const upsert = Effect.fn("ModelStatRepo.upsert")(function* (rows: ModelStatRow[]) {
yield* Effect.forEach(
chunks(rows, UPSERT_CHUNK_SIZE),
(chunk) =>
Effect.tryPromise({
try: () =>
db
.insert(modelStat)
.values(chunk)
.onDuplicateKeyUpdate({
set: {
provider_model: inserted("provider_model"),
sessions: inserted("sessions"),
requests: inserted("requests"),
input_tokens: inserted("input_tokens"),
output_tokens: inserted("output_tokens"),
reasoning_tokens: inserted("reasoning_tokens"),
cache_read_tokens: inserted("cache_read_tokens"),
total_tokens: inserted("total_tokens"),
input_cost_microcents: inserted("input_cost_microcents"),
output_cost_microcents: inserted("output_cost_microcents"),
total_cost_microcents: inserted("total_cost_microcents"),
avg_duration_ms: inserted("avg_duration_ms"),
p50_duration_ms: inserted("p50_duration_ms"),
p95_duration_ms: inserted("p95_duration_ms"),
avg_ttfb_ms: inserted("avg_ttfb_ms"),
p50_ttfb_ms: inserted("p50_ttfb_ms"),
p95_ttfb_ms: inserted("p95_ttfb_ms"),
avg_output_tps: inserted("avg_output_tps"),
success_count: inserted("success_count"),
error_count: inserted("error_count"),
sample_count: inserted("sample_count"),
rank_by_tokens: inserted("rank_by_tokens"),
rank_by_requests: inserted("rank_by_requests"),
rank_by_cost: inserted("rank_by_cost"),
},
}),
catch: (cause) => DatabaseError.make({ cause }),
}),
{ discard: true },
)
})
return ModelStatRepo.of({ listDaily, upsert })
}),
)
}
export function rowsFromAggregates(aggregates: ModelStatAggregate[]) {
return rankRows([
...synthesizeAllTierRows(
collapseRows(aggregates.filter((item) => item.grain === "week").map(toRow), dimensionKey),
dimensionKey,
),
...synthesizeAllTierRows(
collapseRows(aggregates.filter((item) => item.grain === "day").map(toRow), dimensionKey),
dimensionKey,
),
])
}
function toRow(data: ModelStatAggregate): ModelStatRow {
return {
...toStatBaseRow(data),
provider: data.provider,
model: data.model,
provider_model: data.provider_model,
}
}
function rankRows(rows: ModelStatRow[]) {
return Object.values(
rows.reduce<Record<string, ModelStatRow[]>>((result, row) => {
const key = statPeriodKey(row)
result[key] = [...(result[key] ?? []), row]
return result
}, {}),
).flatMap((group) => {
const tokenRanks = rankBy(group, (row) => row.total_tokens ?? 0)
const requestRanks = rankBy(group, (row) => row.requests ?? 0)
const costRanks = rankBy(group, (row) => row.total_cost_microcents ?? 0)
return group.map((row) => ({
...row,
rank_by_tokens: tokenRanks.get(row) ?? null,
rank_by_requests: requestRanks.get(row) ?? null,
rank_by_cost: costRanks.get(row) ?? null,
}))
})
}
function dimensionKey(row: ModelStatRow) {
return [row.provider, row.model].join("\u0000")
}
packages/stats/core/src/domain/provider.ts
-168
View File
@@ -1,168 +0,0 @@
import { and, asc, eq } from "drizzle-orm"
import { Effect, Layer } from "effect"
import * as Context from "effect/Context"
import { DatabaseError, DrizzleClient } from "../database"
import { providerStat } from "../database/schema"
import {
chunks,
collapseRows,
inserted,
rankRowsWithMarketShare,
synthesizeAllTierRows,
toStatBaseRow,
UPSERT_CHUNK_SIZE,
type StatBaseAggregate,
} from "./stat"
export type ProviderStatRow = typeof providerStat.$inferInsert
export type ProviderStatAggregate = StatBaseAggregate & { provider: string }
export type ProviderStatMetric = {
periodKey: string
updatedAt: Date
tier: string
provider: string
totalTokens: number
}
export declare namespace ProviderStatRepo {
export interface Service {
readonly listDaily: () => Effect.Effect<ProviderStatMetric[], DatabaseError>
readonly listByPeriod: (opts: {
readonly grain: string
readonly periodKey: string
readonly dataset?: string
readonly tier?: string
readonly client?: string
readonly source?: string
}) => Effect.Effect<ProviderStatRow[], DatabaseError>
readonly upsert: (rows: ProviderStatRow[]) => Effect.Effect<void, DatabaseError>
}
}
export class ProviderStatRepo extends Context.Service<ProviderStatRepo, ProviderStatRepo.Service>()(
"@opencode/stats/ProviderStatRepo",
) {
static readonly layer: Layer.Layer<ProviderStatRepo, never, DrizzleClient> = Layer.effect(
ProviderStatRepo,
Effect.gen(function* () {
const db = yield* DrizzleClient
const listDaily = Effect.fn("ProviderStatRepo.listDaily")(function* () {
return yield* Effect.tryPromise({
try: () =>
db
.select({
periodKey: providerStat.period_key,
updatedAt: providerStat.updated_at,
tier: providerStat.tier,
provider: providerStat.provider,
totalTokens: providerStat.total_tokens,
})
.from(providerStat)
.where(and(eq(providerStat.grain, "day"), eq(providerStat.client, "all"), eq(providerStat.source, "all")))
.orderBy(asc(providerStat.period_key)),
catch: (cause) => DatabaseError.make({ cause }),
})
})
const listByPeriod = Effect.fn("ProviderStatRepo.listByPeriod")(function* (opts: {
readonly grain: string
readonly periodKey: string
readonly dataset?: string
readonly tier?: string
readonly client?: string
readonly source?: string
}) {
return yield* Effect.tryPromise({
try: () =>
db
.select()
.from(providerStat)
.where(
and(
eq(providerStat.grain, opts.grain),
eq(providerStat.period_key, opts.periodKey),
eq(providerStat.dataset, opts.dataset ?? "zen"),
eq(providerStat.tier, opts.tier ?? "all"),
eq(providerStat.client, opts.client ?? "all"),
eq(providerStat.source, opts.source ?? "all"),
),
),
catch: (cause) => DatabaseError.make({ cause }),
})
})
const upsert = Effect.fn("ProviderStatRepo.upsert")(function* (rows: ProviderStatRow[]) {
yield* Effect.forEach(
chunks(rows, UPSERT_CHUNK_SIZE),
(chunk) =>
Effect.tryPromise({
try: () =>
db
.insert(providerStat)
.values(chunk)
.onDuplicateKeyUpdate({
set: {
sessions: inserted("sessions"),
requests: inserted("requests"),
input_tokens: inserted("input_tokens"),
output_tokens: inserted("output_tokens"),
reasoning_tokens: inserted("reasoning_tokens"),
cache_read_tokens: inserted("cache_read_tokens"),
total_tokens: inserted("total_tokens"),
input_cost_microcents: inserted("input_cost_microcents"),
output_cost_microcents: inserted("output_cost_microcents"),
total_cost_microcents: inserted("total_cost_microcents"),
avg_duration_ms: inserted("avg_duration_ms"),
p50_duration_ms: inserted("p50_duration_ms"),
p95_duration_ms: inserted("p95_duration_ms"),
avg_ttfb_ms: inserted("avg_ttfb_ms"),
p50_ttfb_ms: inserted("p50_ttfb_ms"),
p95_ttfb_ms: inserted("p95_ttfb_ms"),
avg_output_tps: inserted("avg_output_tps"),
success_count: inserted("success_count"),
error_count: inserted("error_count"),
sample_count: inserted("sample_count"),
market_share_tokens: inserted("market_share_tokens"),
market_share_requests: inserted("market_share_requests"),
market_share_sessions: inserted("market_share_sessions"),
rank_by_tokens: inserted("rank_by_tokens"),
rank_by_requests: inserted("rank_by_requests"),
rank_by_sessions: inserted("rank_by_sessions"),
rank_by_cost: inserted("rank_by_cost"),
},
}),
catch: (cause) => DatabaseError.make({ cause }),
}),
{ discard: true },
)
})
return ProviderStatRepo.of({ listDaily, listByPeriod, upsert })
}),
)
}
export function rowsFromAggregates(aggregates: ProviderStatAggregate[]) {
return rankRowsWithMarketShare([
...synthesizeAllTierRows(
collapseRows(aggregates.filter((item) => item.grain === "week").map(toRow), dimensionKey),
dimensionKey,
),
...synthesizeAllTierRows(
collapseRows(aggregates.filter((item) => item.grain === "day").map(toRow), dimensionKey),
dimensionKey,
),
])
}
function toRow(data: ProviderStatAggregate): ProviderStatRow {
return {
...toStatBaseRow(data),
provider: data.provider,
}
}
function dimensionKey(row: ProviderStatRow) {
return row.provider
}
packages/stats/core/src/domain/stat.ts
-246
View File
@@ -1,246 +0,0 @@
import { sql } from "drizzle-orm"
export const UPSERT_CHUNK_SIZE = 500
const DAY_MS = 86_400_000
export type StatGrain = "day" | "week"
export type StatBaseAggregate = {
grain: StatGrain
period_key: string
dataset: string
tier: string
sessions: number
requests: number
input_tokens: number
output_tokens: number
reasoning_tokens: number
cache_read_tokens: number
total_tokens: number
input_cost_microcents: number
output_cost_microcents: number
total_cost_microcents: number
avg_duration_ms: number | null
p50_duration_ms: number | null
p95_duration_ms: number | null
avg_ttfb_ms: number | null
p50_ttfb_ms: number | null
p95_ttfb_ms: number | null
avg_output_tps: number | null
success_count: number
error_count: number
sample_count: number
}
export type StatBaseRow = {
grain: string
period_key: string
dataset?: string
tier?: string
client?: string
source?: string
sessions?: number
requests?: number
input_tokens?: number
output_tokens?: number
reasoning_tokens?: number
cache_read_tokens?: number
total_tokens?: number
input_cost_microcents?: number
output_cost_microcents?: number
total_cost_microcents?: number
avg_duration_ms?: number | null
p50_duration_ms?: number | null
p95_duration_ms?: number | null
avg_ttfb_ms?: number | null
p50_ttfb_ms?: number | null
p95_ttfb_ms?: number | null
avg_output_tps?: number | null
success_count?: number
error_count?: number
sample_count?: number
}
export function toStatBaseRow(data: StatBaseAggregate) {
return {
grain: data.grain,
period_key: data.period_key,
dataset: data.dataset,
tier: data.tier,
client: "all",
source: "all",
sessions: data.sessions,
requests: data.requests,
input_tokens: data.input_tokens,
output_tokens: data.output_tokens,
reasoning_tokens: data.reasoning_tokens,
cache_read_tokens: data.cache_read_tokens,
total_tokens: data.total_tokens,
input_cost_microcents: data.input_cost_microcents,
output_cost_microcents: data.output_cost_microcents,
total_cost_microcents: data.total_cost_microcents,
avg_duration_ms: data.avg_duration_ms,
p50_duration_ms: data.p50_duration_ms,
p95_duration_ms: data.p95_duration_ms,
avg_ttfb_ms: data.avg_ttfb_ms,
p50_ttfb_ms: data.p50_ttfb_ms,
p95_ttfb_ms: data.p95_ttfb_ms,
avg_output_tps: data.avg_output_tps,
success_count: data.success_count,
error_count: data.error_count,
sample_count: data.sample_count,
}
}
export function synthesizeAllTierRows<T extends StatBaseRow>(rows: T[], dimensionKey: (row: T) => string) {
return [
...rows,
...Object.values(
rows.reduce<Record<string, T>>((result, row) => {
const key = [row.grain, row.period_key, row.dataset, row.client, row.source, dimensionKey(row)].join("\u0000")
result[key] = result[key] ? combineRows(result[key], row) : { ...row, tier: "all" }
return result
}, {}),
),
]
}
export function collapseRows<T extends StatBaseRow>(rows: T[], dimensionKey: (row: T) => string) {
return Object.values(
rows.reduce<Record<string, T>>((result, row) => {
const key = [row.grain, row.period_key, row.dataset, row.tier, row.client, row.source, dimensionKey(row)].join(
"\u0000",
)
result[key] = result[key] ? combineRows(result[key], row) : row
return result
}, {}),
)
}
export function combineRows<T extends StatBaseRow>(left: T, right: T): T {
return {
...left,
sessions: (left.sessions ?? 0) + (right.sessions ?? 0),
requests: (left.requests ?? 0) + (right.requests ?? 0),
input_tokens: (left.input_tokens ?? 0) + (right.input_tokens ?? 0),
output_tokens: (left.output_tokens ?? 0) + (right.output_tokens ?? 0),
reasoning_tokens: (left.reasoning_tokens ?? 0) + (right.reasoning_tokens ?? 0),
cache_read_tokens: (left.cache_read_tokens ?? 0) + (right.cache_read_tokens ?? 0),
total_tokens: (left.total_tokens ?? 0) + (right.total_tokens ?? 0),
input_cost_microcents: (left.input_cost_microcents ?? 0) + (right.input_cost_microcents ?? 0),
output_cost_microcents: (left.output_cost_microcents ?? 0) + (right.output_cost_microcents ?? 0),
total_cost_microcents: (left.total_cost_microcents ?? 0) + (right.total_cost_microcents ?? 0),
avg_duration_ms: weightedAverage(left.avg_duration_ms, left.requests, right.avg_duration_ms, right.requests),
p50_duration_ms: null,
p95_duration_ms: null,
avg_ttfb_ms: weightedAverage(left.avg_ttfb_ms, left.requests, right.avg_ttfb_ms, right.requests),
p50_ttfb_ms: null,
p95_ttfb_ms: null,
avg_output_tps: weightedAverage(left.avg_output_tps, left.requests, right.avg_output_tps, right.requests),
success_count: (left.success_count ?? 0) + (right.success_count ?? 0),
error_count: (left.error_count ?? 0) + (right.error_count ?? 0),
sample_count: (left.sample_count ?? 0) + (right.sample_count ?? 0),
}
}
export function statPeriodKey(row: StatBaseRow) {
return [row.grain, row.period_key, row.dataset, row.tier, row.client, row.source].join("\u0000")
}
export function periodKeyFor(grain: StatGrain, periodStart: Date) {
if (grain === "week") return isoWeekId(periodStart)
return utcDateId(periodStart)
}
export function startOfUtcDay(value: Date) {
return new Date(Date.UTC(value.getUTCFullYear(), value.getUTCMonth(), value.getUTCDate()))
}
export function startOfIsoWeek(value: Date) {
return new Date(
Date.UTC(value.getUTCFullYear(), value.getUTCMonth(), value.getUTCDate() - (value.getUTCDay() || 7) + 1),
)
}
export function isoWeekId(value: Date) {
const thursday = new Date(
Date.UTC(value.getUTCFullYear(), value.getUTCMonth(), value.getUTCDate() + 4 - (value.getUTCDay() || 7)),
)
return `${thursday.getUTCFullYear()}-W${String(Math.ceil(((thursday.getTime() - Date.UTC(thursday.getUTCFullYear(), 0, 1)) / DAY_MS + 1) / 7)).padStart(2, "0")}`
}
function utcDateId(value: Date) {
return `${value.getUTCFullYear()}-${String(value.getUTCMonth() + 1).padStart(2, "0")}-${String(value.getUTCDate()).padStart(2, "0")}`
}
export function rankBy<T extends StatBaseRow>(rows: T[], value: (row: T) => number) {
return new Map(rows.toSorted((a, b) => value(b) - value(a)).map((row, index) => [row, index + 1]))
}
export function rankRowsWithMarketShare<T extends StatBaseRow>(
rows: T[],
groupKey: (row: T) => string = statPeriodKey,
) {
return Object.values(
rows.reduce<Record<string, T[]>>((result, row) => {
const key = groupKey(row)
result[key] = [...(result[key] ?? []), row]
return result
}, {}),
).flatMap((group) => {
const tokens = group.reduce((sum, row) => sum + (row.total_tokens ?? 0), 0)
const requests = group.reduce((sum, row) => sum + (row.requests ?? 0), 0)
const sessions = group.reduce((sum, row) => sum + (row.sessions ?? 0), 0)
const tokenRanks = rankBy(group, (row) => row.total_tokens ?? 0)
const requestRanks = rankBy(group, (row) => row.requests ?? 0)
const sessionRanks = rankBy(group, (row) => row.sessions ?? 0)
const costRanks = rankBy(group, (row) => row.total_cost_microcents ?? 0)
return group.map((row) => ({
...row,
market_share_tokens: share(row.total_tokens, tokens),
market_share_requests: share(row.requests, requests),
market_share_sessions: share(row.sessions, sessions),
rank_by_tokens: tokenRanks.get(row) ?? null,
rank_by_requests: requestRanks.get(row) ?? null,
rank_by_sessions: sessionRanks.get(row) ?? null,
rank_by_cost: costRanks.get(row) ?? null,
}))
})
}
export function share(value: number | null | undefined, total: number) {
if (total <= 0) return null
return Number(((value ?? 0) / total).toFixed(6))
}
export function chunks<T>(items: T[], size: number) {
return Array.from({ length: Math.ceil(items.length / size) }, (_, index) =>
items.slice(index * size, (index + 1) * size),
)
}
export function inserted(column: string) {
return sql.raw(`values(\`${column}\`)`)
}
export function weightedAverage(
left: number | null | undefined,
leftWeight = 0,
right: number | null | undefined,
rightWeight = 0,
) {
const totalWeight =
(left === null || left === undefined ? 0 : leftWeight) + (right === null || right === undefined ? 0 : rightWeight)
if (totalWeight === 0) return null
return Number((((left ?? 0) * leftWeight + (right ?? 0) * rightWeight) / totalWeight).toFixed(2))
}
export function normalizeTier(value: string) {
if (value === "Paid") return "Zen"
return value
}
export function normalizeCountry(value: string | undefined) {
if (!value || value.length !== 2) return "ZZ"
return value.toUpperCase()
}
packages/stats/core/src/honeycomb-backfill.ts
-974
View File
@@ -1,974 +0,0 @@
import { Client } from "@planetscale/database"
import { readdir } from "node:fs/promises"
import path from "node:path"
import { drizzle } from "drizzle-orm/planetscale-serverless"
import { geoStat, modelStat, providerStat } from "./database/schema"
import { modelAuthor, normalizeInferenceModel } from "./domain/model-normalization"
import {
chunks,
collapseRows,
inserted,
isoWeekId,
normalizeCountry,
normalizeTier,
periodKeyFor,
rankBy,
rankRowsWithMarketShare,
startOfIsoWeek,
startOfUtcDay,
statPeriodKey,
synthesizeAllTierRows,
toStatBaseRow,
type StatBaseAggregate,
} from "./domain/stat"
const DAY_MS = 86_400_000
const DEFAULT_UPSERT_CHUNK_SIZE = 100
const DEFAULT_TIERS = ["Go", "Free", "Paid"]
const FREE_MODELS = new Set(["gpt-5-nano", "grok-code", "big-pickle"])
type Grain = "day" | "week"
type MetricDimension = "model" | "provider" | "geo" | "geo-model"
type LookupDimension = "model-provider-model" | "geo-continent"
type ImportKey = `${MetricDimension | LookupDimension}-${Grain}`
type QuerySpec = {
name: string
importKey: ImportKey
importFlag: `--${ImportKey}`
query: ReturnType<typeof metricQuery>
}
type RawRow = Record<string, string>
type ImportOptions = {
dataset: string
databaseUrl: string | undefined
directories: string[]
dryRun: boolean
periodStart: Date | undefined
upsertChunkSize: number
files: Partial<Record<ImportKey, string[]>>
}
type ModelAggregate = StatBaseAggregate & { provider: string; model: string; provider_model: string }
type ProviderAggregate = StatBaseAggregate & { provider: string }
type GeoAggregate = StatBaseAggregate & { provider: string; model: string; country: string; continent: string }
type ModelStatRow = typeof modelStat.$inferInsert
type ProviderStatRow = typeof providerStat.$inferInsert
type GeoStatRow = typeof geoStat.$inferInsert
const inputKeys = [
"model-day",
"model-week",
"model-provider-model-day",
"model-provider-model-week",
"provider-day",
"provider-week",
"geo-day",
"geo-week",
"geo-model-day",
"geo-model-week",
"geo-continent-day",
"geo-continent-week",
] as const satisfies ImportKey[]
if (import.meta.main) await main()
async function main() {
const command = process.argv[2]
if (command === "queries") return printQueries(process.argv.slice(3))
if (command === "import") return importFiles(process.argv.slice(3))
usage()
}
function printQueries(args: string[]) {
const flags = parseFlags(args)
const limit = parseIntegerFlag(flags, "limit") ?? 1000
const tiers = parseListFlag(flags, "tiers") ?? DEFAULT_TIERS
const queries = buildQueries(limit, tiers)
const only = flags.get("only")?.[0]
if (only) {
const item = queries.find((query) => query.name === only)
if (!item) fail(`Unknown --only ${only}. Expected one of: ${queries.map((query) => query.name).join(", ")}`)
console.log(JSON.stringify(item.query, null, 2))
return
}
console.log(
JSON.stringify(
{
tiers,
import_hint: "bun src/honeycomb-backfill.ts import --dir downloads",
queries,
},
null,
2,
),
)
}
async function importFiles(args: string[]) {
const parsed = parseImportOptions(args)
const opts = { ...parsed, files: mergeFiles(parsed.files, await discoverFiles(parsed.directories)) }
if (!inputKeys.some((key) => opts.files[key]?.length)) fail("No CSV or JSON import files were provided or discovered")
const providerModelLookup = new Map([
...(await lookupRows(opts.files["model-provider-model-day"], "day", opts, modelProviderModelLookup)),
...(await lookupRows(opts.files["model-provider-model-week"], "week", opts, modelProviderModelLookup)),
])
const continentLookup = new Map([
...(await lookupRows(opts.files["geo-continent-day"], "day", opts, geoContinentLookup)),
...(await lookupRows(opts.files["geo-continent-week"], "week", opts, geoContinentLookup)),
])
const modelAggregates = [
...(await metricRows(opts.files["model-day"], "day", opts, (row, base) =>
modelAggregate(row, base, providerModelLookup),
)),
...(await metricRows(opts.files["model-week"], "week", opts, (row, base) =>
modelAggregate(row, base, providerModelLookup),
)),
]
const modelRows = modelRowsFromAggregates(modelAggregates)
const providerRows = providerRowsFromAggregates([
...(await metricRows(opts.files["provider-day"], "day", opts, (row, base) => ({
...base,
provider: provider(row) ?? "unknown",
}))),
...(await metricRows(opts.files["provider-week"], "week", opts, (row, base) => ({
...base,
provider: provider(row) ?? "unknown",
}))),
])
const geoRows = geoRowsFromAggregates([
...(await metricRows(opts.files["geo-day"], "day", opts, (row, base) => ({
...base,
provider: "all",
model: "all",
country: country(row),
continent: continentLookup.get(lookupKey(base, country(row))) ?? continent(row),
}))),
...(await metricRows(opts.files["geo-week"], "week", opts, (row, base) => ({
...base,
provider: "all",
model: "all",
country: country(row),
continent: continentLookup.get(lookupKey(base, country(row))) ?? continent(row),
}))),
...(await metricRows(opts.files["geo-model-day"], "day", opts, (row, base) =>
geoModelAggregate(row, base, continentLookup),
)),
...(await metricRows(opts.files["geo-model-week"], "week", opts, (row, base) =>
geoModelAggregate(row, base, continentLookup),
)),
])
console.log(
JSON.stringify(
{
inputs: Object.fromEntries(
inputKeys.flatMap((key) => (opts.files[key]?.length ? [[key, opts.files[key].length]] : [])),
),
modelRows: modelRows.length,
providerRows: providerRows.length,
geoRows: geoRows.length,
dryRun: opts.dryRun,
upsertChunkSize: opts.upsertChunkSize,
},
null,
2,
),
)
if (opts.dryRun) return
if (!opts.databaseUrl) fail("DATABASE_URL is required unless --dry-run is set")
const db = drizzle({ client: new Client({ url: opts.databaseUrl }) })
await upsertModelRows(db, modelRows, opts.upsertChunkSize)
await upsertProviderRows(db, providerRows, opts.upsertChunkSize)
await upsertGeoRows(db, geoRows, opts.upsertChunkSize)
}
function buildQueries(limit: number, tiers: string[]): QuerySpec[] {
const daily = tiers.flatMap((tier) => [
querySpec(
"model-day",
tier,
metricQuery(["date", "tier", "stat_provider", "stat_model"], limit, tierFilters(tier)),
),
querySpec("provider-day", tier, metricQuery(["date", "tier", "stat_provider"], limit, tierFilters(tier))),
querySpec("geo-day", tier, metricQuery(["date", "tier", "country", "continent"], limit, tierFilters(tier))),
querySpec(
"geo-model-day",
tier,
metricQuery(["date", "tier", "stat_provider", "stat_model", "country", "continent"], limit, tierFilters(tier)),
),
])
const weekly = tiers.flatMap((tier) => [
querySpec(
"model-week",
tier,
metricQuery(["week", "tier", "stat_provider", "stat_model"], limit, tierFilters(tier)),
),
querySpec("provider-week", tier, metricQuery(["week", "tier", "stat_provider"], limit, tierFilters(tier))),
querySpec("geo-week", tier, metricQuery(["week", "tier", "country", "continent"], limit, tierFilters(tier))),
querySpec(
"geo-model-week",
tier,
metricQuery(["week", "tier", "stat_provider", "stat_model", "country", "continent"], limit, tierFilters(tier)),
),
])
return [...daily, ...weekly]
}
function querySpec(importKey: ImportKey, tier: string, query: ReturnType<typeof metricQuery>) {
return {
name: `${importKey}-${queryNameSegment(tier)}`,
importKey,
importFlag: `--${importKey}` as const,
query,
}
}
function metricQuery(breakdowns: string[], limit: number, filters: ReturnType<typeof commonFilters> = []) {
return {
granularity: 0,
breakdowns,
calculations: [
{ op: "COUNT_DISTINCT", column: "session" },
{ op: "COUNT" },
{ op: "SUM", column: "tokens.input" },
{ op: "SUM", column: "tokens.output" },
{ op: "SUM", column: "tokens.reasoning" },
{ op: "SUM", column: "tokens.cache_read" },
{ op: "SUM", column: "tokens" },
{ op: "SUM", column: "cost.input.microcents" },
{ op: "SUM", column: "cost.output.microcents" },
{ op: "SUM", column: "cost.total.microcents" },
{ op: "AVG", column: "duration" },
{ op: "P50", column: "duration" },
{ op: "P95", column: "duration" },
{ op: "AVG", column: "time_to_first_byte" },
{ op: "P50", column: "time_to_first_byte" },
{ op: "P95", column: "time_to_first_byte" },
{ op: "AVG", column: "tps.output" },
],
filters: [...commonFilters(), ...filters],
filter_combination: "AND",
orders: [{ column: "tokens", op: "SUM", order: "descending" }],
havings: [],
limit,
formulas: [],
}
}
function tierFilters(tier: string) {
if (tier === "all") return []
return [{ column: "tier", op: "=", value: tier }]
}
function queryNameSegment(value: string) {
return (
value
.toLowerCase()
.replace(/[^a-z0-9]+/g, "-")
.replace(/^-|-$/g, "") || "all"
)
}
function commonFilters() {
return [
{ column: "event_type", op: "=", value: "completions" },
{ column: "model", op: "exists" },
{ column: "model", op: "!=", value: "" },
{ column: "model", op: "!=", value: "alpha-gpt-next" },
]
}
function metricRows<T extends StatBaseAggregate>(
files: string[] | undefined,
grain: Grain,
opts: ImportOptions,
map: (row: RawRow, base: StatBaseAggregate) => T | T[],
) {
if (!files) return Promise.resolve([])
return readFiles(files).then((rows) => rows.flatMap((row) => map(row, baseAggregate(row, grain, opts))))
}
function lookupRows(
files: string[] | undefined,
grain: Grain,
opts: ImportOptions,
map: (row: RawRow, grain: Grain, opts: ImportOptions) => readonly (readonly [string, string])[],
) {
if (!files) return Promise.resolve([])
return readFiles(files).then((rows) =>
Array.from(
rows
.flatMap((row) => map(row, grain, opts))
.reduce((result, [key, value]) => {
if (value && value > (result.get(key) ?? "")) result.set(key, value)
return result
}, new Map<string, string>()),
),
)
}
async function readFiles(files: string[]) {
return (await Promise.all(files.map(readRows))).flat()
}
async function discoverFiles(directories: string[]) {
const classified = await Promise.all(
(await Promise.all(directories.map(filesInDirectory))).flat().map(async (file) => ({
file,
key: classifyRows(file, await readRows(file)),
})),
)
return classified.reduce<Partial<Record<ImportKey, string[]>>>((result, item) => {
return { ...result, [item.key]: [...(result[item.key] ?? []), item.file] }
}, {})
}
async function filesInDirectory(directory: string): Promise<string[]> {
return (
await Promise.all(
(await readdir(directory, { withFileTypes: true })).map((entry) => {
const file = path.join(directory, entry.name)
if (entry.isDirectory()) return filesInDirectory(file)
if (entry.isFile() && /\.(csv|json)$/i.test(entry.name)) return Promise.resolve([file])
return Promise.resolve([])
}),
)
).flat()
}
function classifyRows(file: string, rows: RawRow[]): ImportKey {
if (rows.length === 0) fail(`Cannot classify empty export: ${file}`)
const headers = new Set(rows.flatMap((row) => Object.keys(row).map(normalizeHeader)))
const grain: Grain = headers.has("date") ? "day" : "week"
if (hasHeader(headers, ["country", "cf.country"])) {
if (hasHeader(headers, ["model", "stat_model"]) && hasMetricHeaders(headers)) return `geo-model-${grain}`
return hasMetricHeaders(headers) ? `geo-${grain}` : `geo-continent-${grain}`
}
if (hasHeader(headers, ["model", "stat_model"]))
return hasMetricHeaders(headers) ? `model-${grain}` : `model-provider-model-${grain}`
if (hasHeader(headers, ["provider", "provider.normalized", "stat_provider"])) return `provider-${grain}`
fail(`Cannot classify export from columns in ${file}`)
}
function hasMetricHeaders(headers: Set<string>) {
return ["sumtokens", "sumtokensinput", "inputtokens", "totaltokens", "avgduration", "countdistinctsession"].some(
(header) => headers.has(header),
)
}
function hasHeader(headers: Set<string>, names: string[]) {
return names.some((name) => headers.has(normalizeHeader(name)))
}
function mergeFiles(left: Partial<Record<ImportKey, string[]>>, right: Partial<Record<ImportKey, string[]>>) {
return inputKeys.reduce<Partial<Record<ImportKey, string[]>>>((result, key) => {
const files = [...(left[key] ?? []), ...(right[key] ?? [])]
if (files.length === 0) return result
return { ...result, [key]: files }
}, {})
}
function modelProviderModelLookup(row: RawRow, grain: Grain, opts: ImportOptions): [string, string][] {
const base = basePeriod(row, grain, opts)
const value = providerModel(row)
const author = provider(row)
if (!value || !author) return []
return [[lookupKey({ ...base, dataset: opts.dataset, tier: tier(row), grain }, author, model(row)), value]]
}
function modelAggregate(
row: RawRow,
base: StatBaseAggregate,
providerModelLookup: Map<string, string>,
): ModelAggregate[] {
const author = provider(row)
if (!author) return []
return [
{
...base,
provider: author,
model: model(row),
provider_model: providerModelLookup.get(lookupKey(base, author, model(row))) ?? providerModel(row),
},
]
}
function geoContinentLookup(row: RawRow, grain: Grain, opts: ImportOptions): [string, string][] {
const base = basePeriod(row, grain, opts)
const value = continent(row)
if (!value) return []
return [[lookupKey({ ...base, dataset: opts.dataset, tier: tier(row), grain }, country(row)), value]]
}
function geoModelAggregate(row: RawRow, base: StatBaseAggregate, continentLookup: Map<string, string>): GeoAggregate[] {
const author = provider(row)
if (!author) return []
return [
{
...base,
provider: author,
model: model(row),
country: country(row),
continent: continentLookup.get(lookupKey(base, country(row))) ?? continent(row),
},
]
}
function baseAggregate(row: RawRow, grain: Grain, opts: ImportOptions): StatBaseAggregate {
return {
...basePeriod(row, grain, opts),
grain,
dataset: opts.dataset,
tier: tier(row),
sessions: integer(row, "sessions", ["COUNT_DISTINCT(session)"]),
requests: integer(row, "requests", ["COUNT", "COUNT()"]),
input_tokens: integer(row, "input_tokens", ["SUM(tokens.input)", "SUM(tokens_input)"]),
output_tokens: integer(row, "output_tokens", ["SUM(tokens.output)", "SUM(tokens_output)"]),
reasoning_tokens: integer(row, "reasoning_tokens", ["SUM(tokens.reasoning)", "SUM(tokens_reasoning)"]),
cache_read_tokens: integer(row, "cache_read_tokens", ["SUM(tokens.cache_read)", "SUM(tokens_cache_read)"]),
total_tokens: integer(row, "total_tokens", ["SUM(stat_tokens_total)", "SUM(tokens)", "SUM(tokens_total)"]),
input_cost_microcents: integer(row, "input_cost_microcents", [
"SUM(cost.input.microcents)",
"SUM(stat_cost_input_microcents)",
]),
output_cost_microcents: integer(row, "output_cost_microcents", [
"SUM(cost.output.microcents)",
"SUM(stat_cost_output_microcents)",
]),
total_cost_microcents: integer(row, "total_cost_microcents", [
"SUM(cost.total.microcents)",
"SUM(stat_cost_total_microcents)",
]),
avg_duration_ms: nullableNumber(row, "avg_duration_ms", ["AVG(duration)", "AVG(duration_ms)"]),
p50_duration_ms: nullableInteger(row, "p50_duration_ms", ["P50(duration)", "P50(duration_ms)"]),
p95_duration_ms: nullableInteger(row, "p95_duration_ms", ["P95(duration)", "P95(duration_ms)"]),
avg_ttfb_ms: nullableNumber(row, "avg_ttfb_ms", ["AVG(time_to_first_byte)", "AVG(ttfb_ms)"]),
p50_ttfb_ms: nullableInteger(row, "p50_ttfb_ms", ["P50(time_to_first_byte)", "P50(ttfb_ms)"]),
p95_ttfb_ms: nullableInteger(row, "p95_ttfb_ms", ["P95(time_to_first_byte)", "P95(ttfb_ms)"]),
avg_output_tps: nullableNumber(row, "avg_output_tps", ["AVG(tps.output)", "AVG(stat_output_tps)"]),
success_count: integer(row, "success_count", ["SUM(success)", "SUM(is_success)", "SUM(stat_success)"]),
error_count: integer(row, "error_count", ["SUM(error)", "SUM(is_error)", "SUM(stat_error)"]),
sample_count: integer(row, "sample_count", ["COUNT", "COUNT()"]),
}
}
function basePeriod(row: RawRow, grain: Grain, opts: ImportOptions) {
return { period_key: periodKey(row, grain, opts) }
}
function periodKey(row: RawRow, grain: Grain, opts: ImportOptions) {
if (grain === "week") {
const week = parseWeek(row)
if (week) return week
fail("weekly imports require a week or period_key column")
}
const time = parseTime(row)
const start = time ? startOfUtcDay(time) : opts.periodStart
if (!start) fail("daily imports require a time column or --period-start")
return periodKeyFor("day", start)
}
function modelRowsFromAggregates(aggregates: ModelAggregate[]) {
return rankModelRows([
...synthesizeAllTierRows(
collapseRows(aggregates.filter((item) => item.grain === "week").map(toModelRow), modelDimensionKey),
modelDimensionKey,
),
...synthesizeAllTierRows(
collapseRows(aggregates.filter((item) => item.grain === "day").map(toModelRow), modelDimensionKey),
modelDimensionKey,
),
])
}
function providerRowsFromAggregates(aggregates: ProviderAggregate[]) {
return rankRowsWithMarketShare([
...synthesizeAllTierRows(
collapseRows(aggregates.filter((item) => item.grain === "week").map(toProviderRow), providerDimensionKey),
providerDimensionKey,
),
...synthesizeAllTierRows(
collapseRows(aggregates.filter((item) => item.grain === "day").map(toProviderRow), providerDimensionKey),
providerDimensionKey,
),
])
}
function geoRowsFromAggregates(aggregates: GeoAggregate[]) {
return rankRowsWithMarketShare(
[
...synthesizeAllTierRows(
collapseRows(aggregates.filter((item) => item.grain === "week").map(toGeoRow), geoDimensionKey),
geoDimensionKey,
),
...synthesizeAllTierRows(
collapseRows(aggregates.filter((item) => item.grain === "day").map(toGeoRow), geoDimensionKey),
geoDimensionKey,
),
],
geoMarketShareKey,
)
}
function toModelRow(data: ModelAggregate): ModelStatRow {
return { ...toStatBaseRow(data), provider: data.provider, model: data.model, provider_model: data.provider_model }
}
function toProviderRow(data: ProviderAggregate): ProviderStatRow {
return { ...toStatBaseRow(data), provider: data.provider }
}
function toGeoRow(data: GeoAggregate): GeoStatRow {
return {
...toStatBaseRow(data),
provider: data.provider,
model: data.model,
country: data.country,
continent: data.continent,
}
}
function rankModelRows(rows: ModelStatRow[]) {
return Object.values(
rows.reduce<Record<string, ModelStatRow[]>>((result, row) => {
const key = statPeriodKey(row)
result[key] = [...(result[key] ?? []), row]
return result
}, {}),
).flatMap((group) => {
const tokenRanks = rankBy(group, (row) => row.total_tokens ?? 0)
const requestRanks = rankBy(group, (row) => row.requests ?? 0)
const costRanks = rankBy(group, (row) => row.total_cost_microcents ?? 0)
return group.map((row) => ({
...row,
rank_by_tokens: tokenRanks.get(row) ?? null,
rank_by_requests: requestRanks.get(row) ?? null,
rank_by_cost: costRanks.get(row) ?? null,
}))
})
}
function modelDimensionKey(row: ModelStatRow) {
return [row.provider, row.model].join("\u0000")
}
function providerDimensionKey(row: ProviderStatRow) {
return row.provider
}
function geoDimensionKey(row: GeoStatRow) {
return [row.provider, row.model, row.country].join("\u0000")
}
function geoMarketShareKey(row: GeoStatRow) {
return [statPeriodKey(row), row.provider, row.model].join("\u0000")
}
function lookupKey(base: { grain: string; period_key: string; dataset: string; tier: string }, ...dimension: string[]) {
return [base.grain, base.period_key, base.dataset, base.tier, ...dimension].join("\u0000")
}
function tier(row: RawRow) {
return normalizeTier(cell(row, ["stat_tier", "tier"]) || deriveTier(row))
}
function deriveTier(row: RawRow) {
const source = cell(row, ["source"])
const value = model(row)
if (source === "lite") return "Go"
if (FREE_MODELS.has(value) || /-free(:global)?$/.test(rawModel(row))) return "Free"
return "Zen"
}
function provider(row: RawRow) {
return cell(row, ["stat_provider"]) || modelAuthor(model(row))
}
function model(row: RawRow) {
return normalizeInferenceModel(cell(row, ["stat_model"]) || rawModel(row))
}
function rawModel(row: RawRow) {
return cell(row, ["model"]) || "unknown"
}
function providerModel(row: RawRow) {
return cell(row, ["provider.model", "provider_model"]) || ""
}
function country(row: RawRow) {
return normalizeCountry(cell(row, ["stat_country", "cf.country", "cf_country", "country"]))
}
function continent(row: RawRow) {
return cell(row, ["cf.continent", "cf_continent", "continent"]) || ""
}
function integer(row: RawRow, name: string, aliases: string[] = []) {
return Math.round(number(row, name, aliases))
}
function nullableInteger(row: RawRow, name: string, aliases: string[] = []) {
if (!hasCell(row, [name, ...aliases])) return null
return Math.round(number(row, name, aliases))
}
function nullableNumber(row: RawRow, name: string, aliases: string[] = []) {
if (!hasCell(row, [name, ...aliases])) return null
return Number(number(row, name, aliases).toFixed(2))
}
function number(row: RawRow, name: string, aliases: string[] = []) {
const value = Number(cell(row, [name, ...aliases]).replace(/,/g, ""))
return Number.isFinite(value) ? value : 0
}
function hasCell(row: RawRow, names: string[]) {
return names.some((name) => row[name] !== undefined && row[name] !== "")
}
function cell(row: RawRow, names: string[]) {
const normalized = normalizedCells(row)
return (
names.flatMap((name) => [row[name], normalized.get(normalizeHeader(name))]).find((value) => value !== undefined) ??
""
)
}
function normalizedCells(row: RawRow) {
return new Map(Object.entries(row).map(([key, value]) => [normalizeHeader(key), value]))
}
function normalizeHeader(value: string) {
return value.toLowerCase().replace(/[^a-z0-9]+/g, "")
}
function parseTime(row: RawRow) {
const value = cell(row, ["date", "time", "timestamp", "datetime", "bucket"])
if (!value) return undefined
const numeric = Number(value)
const date = Number.isFinite(numeric)
? new Date(numeric > 10_000_000_000 ? numeric : numeric * 1000)
: new Date(value)
if (Number.isNaN(date.getTime())) fail(`Invalid time value: ${value}`)
return date
}
function parseWeek(row: RawRow) {
const value = cell(row, ["period_key", "week", "stat_week"])
if (!value) return undefined
const match = /^(\d{4})-W(\d{1,2})$/.exec(value)
if (!match) fail(`Invalid week value: ${value}`)
const year = Number(match[1])
const week = Number(match[2])
if (week < 1 || week > 53) fail(`Invalid week value: ${value}`)
const start = new Date(startOfIsoWeek(new Date(Date.UTC(year, 0, 4))).getTime() + (week - 1) * 7 * DAY_MS)
const id = `${year}-W${String(week).padStart(2, "0")}`
if (isoWeekId(start) !== id) fail(`Invalid week value: ${value}`)
return id
}
async function readRows(file: string) {
const text = await Bun.file(file).text()
if (file.toLowerCase().endsWith(".json")) {
const parsed: unknown = JSON.parse(text)
return rowsFromJson(parsed)
}
return rowsFromCsv(text)
}
function rowsFromJson(value: unknown): RawRow[] {
if (Array.isArray(value)) return value.flatMap(rowFromUnknown)
if (!isRecord(value)) fail("JSON imports must be an array of rows or an object with results/data/rows")
const rows = [value.results, value.data, value.rows].flatMap((candidate) =>
Array.isArray(candidate) ? candidate.flatMap(rowFromUnknown) : [],
)
if (rows.length === 0) fail("JSON import did not contain rows")
return rows
}
function rowFromUnknown(value: unknown): RawRow[] {
if (!isRecord(value)) return []
const nested = isRecord(value.data) ? value.data : {}
return [
Object.fromEntries(
Object.entries({ ...value, ...nested }).flatMap(([key, item]) => {
if (key === "data") return []
return [[key, cellValue(item)]]
}),
),
]
}
function rowsFromCsv(text: string): RawRow[] {
const [headers, ...rows] = csvRecords(text).filter((row) => row.some((value) => value.trim() !== ""))
if (!headers) return []
return rows.map((row) =>
Object.fromEntries(headers.map((header, index) => [header.trim(), row[index]?.trim() ?? ""])),
)
}
function csvRecords(text: string) {
const rows: string[][] = []
let row: string[] = []
let field = ""
let quoted = false
for (let index = 0; index < text.length; index++) {
const char = text[index]
const next = text[index + 1]
if (quoted) {
if (char === '"' && next === '"') {
field += '"'
index++
continue
}
if (char === '"') {
quoted = false
continue
}
field += char
continue
}
if (char === '"') {
quoted = true
continue
}
if (char === ",") {
row.push(field)
field = ""
continue
}
if (char === "\n") {
row.push(field)
rows.push(row)
row = []
field = ""
continue
}
if (char === "\r") continue
field += char
}
row.push(field)
rows.push(row)
return rows
}
function cellValue(value: unknown) {
if (value === null || value === undefined) return ""
if (typeof value === "string") return value
if (typeof value === "number" || typeof value === "boolean" || typeof value === "bigint") return String(value)
return JSON.stringify(value) ?? ""
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value)
}
async function upsertModelRows(db: ReturnType<typeof drizzle>, rows: ModelStatRow[], chunkSize: number) {
const batches = chunks(rows, chunkSize)
console.log(JSON.stringify({ table: "model_stat", batches: batches.length, chunkSize }))
for (const chunk of batches) {
await db
.insert(modelStat)
.values(chunk)
.onDuplicateKeyUpdate({
set: {
provider_model: inserted("provider_model"),
sessions: inserted("sessions"),
requests: inserted("requests"),
input_tokens: inserted("input_tokens"),
output_tokens: inserted("output_tokens"),
reasoning_tokens: inserted("reasoning_tokens"),
cache_read_tokens: inserted("cache_read_tokens"),
total_tokens: inserted("total_tokens"),
input_cost_microcents: inserted("input_cost_microcents"),
output_cost_microcents: inserted("output_cost_microcents"),
total_cost_microcents: inserted("total_cost_microcents"),
avg_duration_ms: inserted("avg_duration_ms"),
p50_duration_ms: inserted("p50_duration_ms"),
p95_duration_ms: inserted("p95_duration_ms"),
avg_ttfb_ms: inserted("avg_ttfb_ms"),
p50_ttfb_ms: inserted("p50_ttfb_ms"),
p95_ttfb_ms: inserted("p95_ttfb_ms"),
avg_output_tps: inserted("avg_output_tps"),
success_count: inserted("success_count"),
error_count: inserted("error_count"),
sample_count: inserted("sample_count"),
rank_by_tokens: inserted("rank_by_tokens"),
rank_by_requests: inserted("rank_by_requests"),
rank_by_cost: inserted("rank_by_cost"),
},
})
}
}
async function upsertProviderRows(db: ReturnType<typeof drizzle>, rows: ProviderStatRow[], chunkSize: number) {
const batches = chunks(rows, chunkSize)
console.log(JSON.stringify({ table: "provider_stat", batches: batches.length, chunkSize }))
for (const chunk of batches) {
await db
.insert(providerStat)
.values(chunk)
.onDuplicateKeyUpdate({
set: {
sessions: inserted("sessions"),
requests: inserted("requests"),
input_tokens: inserted("input_tokens"),
output_tokens: inserted("output_tokens"),
reasoning_tokens: inserted("reasoning_tokens"),
cache_read_tokens: inserted("cache_read_tokens"),
total_tokens: inserted("total_tokens"),
input_cost_microcents: inserted("input_cost_microcents"),
output_cost_microcents: inserted("output_cost_microcents"),
total_cost_microcents: inserted("total_cost_microcents"),
avg_duration_ms: inserted("avg_duration_ms"),
p50_duration_ms: inserted("p50_duration_ms"),
p95_duration_ms: inserted("p95_duration_ms"),
avg_ttfb_ms: inserted("avg_ttfb_ms"),
p50_ttfb_ms: inserted("p50_ttfb_ms"),
p95_ttfb_ms: inserted("p95_ttfb_ms"),
avg_output_tps: inserted("avg_output_tps"),
success_count: inserted("success_count"),
error_count: inserted("error_count"),
sample_count: inserted("sample_count"),
market_share_tokens: inserted("market_share_tokens"),
market_share_requests: inserted("market_share_requests"),
market_share_sessions: inserted("market_share_sessions"),
rank_by_tokens: inserted("rank_by_tokens"),
rank_by_requests: inserted("rank_by_requests"),
rank_by_sessions: inserted("rank_by_sessions"),
rank_by_cost: inserted("rank_by_cost"),
},
})
}
}
async function upsertGeoRows(db: ReturnType<typeof drizzle>, rows: GeoStatRow[], chunkSize: number) {
const batches = chunks(rows, chunkSize)
console.log(JSON.stringify({ table: "geo_stat", batches: batches.length, chunkSize }))
for (const chunk of batches) {
await db
.insert(geoStat)
.values(chunk)
.onDuplicateKeyUpdate({
set: {
continent: inserted("continent"),
sessions: inserted("sessions"),
requests: inserted("requests"),
input_tokens: inserted("input_tokens"),
output_tokens: inserted("output_tokens"),
reasoning_tokens: inserted("reasoning_tokens"),
cache_read_tokens: inserted("cache_read_tokens"),
total_tokens: inserted("total_tokens"),
input_cost_microcents: inserted("input_cost_microcents"),
output_cost_microcents: inserted("output_cost_microcents"),
total_cost_microcents: inserted("total_cost_microcents"),
avg_duration_ms: inserted("avg_duration_ms"),
p50_duration_ms: inserted("p50_duration_ms"),
p95_duration_ms: inserted("p95_duration_ms"),
avg_ttfb_ms: inserted("avg_ttfb_ms"),
p50_ttfb_ms: inserted("p50_ttfb_ms"),
p95_ttfb_ms: inserted("p95_ttfb_ms"),
avg_output_tps: inserted("avg_output_tps"),
success_count: inserted("success_count"),
error_count: inserted("error_count"),
sample_count: inserted("sample_count"),
market_share_tokens: inserted("market_share_tokens"),
market_share_requests: inserted("market_share_requests"),
market_share_sessions: inserted("market_share_sessions"),
rank_by_tokens: inserted("rank_by_tokens"),
rank_by_requests: inserted("rank_by_requests"),
rank_by_sessions: inserted("rank_by_sessions"),
rank_by_cost: inserted("rank_by_cost"),
},
})
}
}
function parseImportOptions(args: string[]): ImportOptions {
const flags = parseFlags(args)
const files = inputKeys.reduce<Partial<Record<ImportKey, string[]>>>((result, key) => {
const values = flags.get(key)
if (!values) return result
return { ...result, [key]: values }
}, {})
return {
dataset: flags.get("dataset")?.[0] ?? "zen",
databaseUrl: flags.get("database-url")?.[0] ?? process.env.DATABASE_URL,
directories: flags.get("dir") ?? flags.get("directory") ?? [],
dryRun: flags.has("dry-run"),
periodStart: parseDateFlag(flags, "period-start"),
upsertChunkSize: parseIntegerFlag(flags, "upsert-chunk-size") ?? DEFAULT_UPSERT_CHUNK_SIZE,
files,
}
}
function parseFlags(args: string[]) {
const result = new Map<string, string[]>()
for (let index = 0; index < args.length; index++) {
const arg = args[index]
if (!arg.startsWith("--")) fail(`Unexpected argument: ${arg}`)
const name = arg.slice(2)
if (name === "dry-run" || name === "include-weekly") {
result.set(name, ["true"])
continue
}
const nextFlag = args.findIndex((value, valueIndex) => valueIndex > index && value.startsWith("--"))
const values = args.slice(index + 1, nextFlag === -1 ? args.length : nextFlag)
if (values.length === 0) fail(`Missing value for --${name}`)
result.set(name, [...(result.get(name) ?? []), ...values])
index += values.length
}
return result
}
function parseDateFlag(flags: Map<string, string[]>, name: string) {
const value = flags.get(name)?.[0]
if (!value) return undefined
const date = new Date(value)
if (Number.isNaN(date.getTime())) fail(`Invalid --${name}: ${value}`)
return date
}
function parseIntegerFlag(flags: Map<string, string[]>, name: string) {
const value = flags.get(name)?.[0]
if (!value) return undefined
const parsed = Number(value)
if (!Number.isInteger(parsed) || parsed <= 0) fail(`Invalid --${name}: ${value}`)
return parsed
}
function parseListFlag(flags: Map<string, string[]>, name: string) {
const value = flags.get(name)?.[0]
if (!value) return undefined
if (value === "all") return ["all"]
return value
.split(",")
.map((item) => item.trim())
.filter(Boolean)
}
function usage(): never {
fail(`Usage:
bun src/honeycomb-backfill.ts queries [--tiers Go,Free,Paid] [--limit 1000]
bun src/honeycomb-backfill.ts import [--dry-run] [--upsert-chunk-size 100] [--database-url URL] --dir downloads
bun src/honeycomb-backfill.ts import [--dry-run] [--upsert-chunk-size 100] [--database-url URL] --model-day file.csv [--model-day more.csv] ...`)
}
function fail(message: string): never {
console.error(message)
process.exit(1)
}
packages/stats/core/src/index.ts
-11
View File
@@ -1,11 +0,0 @@
export * as Athena from "./athena"
export * as AppConfig from "./config"
export * as Database from "./database"
export * as GeoStat from "./domain/geo"
export * as StatsHome from "./domain/home"
export * as Inference from "./domain/inference"
export * as ModelStat from "./domain/model"
export * as ProviderStat from "./domain/provider"
export * as Stat from "./domain/stat"
export * as Runtime from "./runtime"
export * as StatSync from "./stat-sync"
packages/stats/core/src/migrate.ts
-4
View File
@@ -1,4 +0,0 @@
import { Effect } from "effect"
import { layer, migrate } from "./database"
await Effect.runPromise(migrate().pipe(Effect.provide(layer)))
packages/stats/core/src/resource.d.ts
Vendored
-28
View File
@@ -1,28 +0,0 @@
import "sst/resource"
declare module "sst/resource" {
export interface Resource {
InferenceEvent: {
catalog: string
database: string
region: string
table: string
tableBucket: string
type: "sst.sst.Linkable"
workgroup: string
}
StatsSyncConfig: {
dataset: string
type: "sst.sst.Linkable"
}
StatsDatabase: {
database: string
host: string
password: string
port: number
type: "sst.sst.Linkable"
url: string
username: string
}
}
}
packages/stats/core/src/runtime.ts
-14
View File
@@ -1,14 +0,0 @@
import { Layer, ManagedRuntime } from "effect"
import { AppConfig } from "./config"
import { layer as databaseLayer } from "./database"
import { GeoStatRepo } from "./domain/geo"
import { ModelStatRepo } from "./domain/model"
import { ProviderStatRepo } from "./domain/provider"
const repoLayer = Layer.mergeAll(ModelStatRepo.layer, ProviderStatRepo.layer, GeoStatRepo.layer).pipe(
Layer.provide(databaseLayer),
)
export const layer = Layer.mergeAll(AppConfig.layer, databaseLayer, repoLayer)
export const runtime = ManagedRuntime.make(layer)
export type RuntimeServices = ManagedRuntime.ManagedRuntime.Services<typeof runtime>
packages/stats/core/src/stat-sync.ts
-93
View File
@@ -1,93 +0,0 @@
import { DateTime, Effect } from "effect"
import { Resource } from "sst/resource"
import { Athena, AthenaQueryError, AthenaQueryTimeoutError } from "./athena"
import { DatabaseError } from "./database"
import { GeoStatRepo, rowsFromAggregates as geoRowsFromAggregates } from "./domain/geo"
import { buildStatsQuery, toGeoAggregate, toModelAggregate, toProviderAggregate } from "./domain/inference"
import { ModelStatRepo, rowsFromAggregates as modelRowsFromAggregates } from "./domain/model"
import { ProviderStatRepo, rowsFromAggregates as providerRowsFromAggregates } from "./domain/provider"
import { startOfIsoWeek } from "./domain/stat"
const DATALAKE_INGESTION_LAG_MS = 5 * 60_000
const STATS_DATA_START_MS = new Date("2026-05-28T00:00:00.000Z").getTime()
const WEEK_MS = 7 * 86_400_000
export type SyncStatsResult = { ok: true; rows: number; startedAt: string; periodStart: string; periodEnd: string }
export type SyncStatsError = AthenaQueryError | AthenaQueryTimeoutError | DatabaseError
export const syncStats: () => Effect.Effect<
SyncStatsResult,
SyncStatsError,
Athena | ModelStatRepo | ProviderStatRepo | GeoStatRepo
> = Effect.fn("StatSync.sync")(function* () {
const startedAt = yield* DateTime.nowAsDate
const periodEnd = new Date(Math.floor((startedAt.getTime() - DATALAKE_INGESTION_LAG_MS) / 60_000) * 60_000)
// May 27 was partial, so keep Athena stats anchored at the first complete day.
const periodStart = new Date(Math.max(startOfIsoWeek(periodEnd).getTime() - WEEK_MS, STATS_DATA_START_MS))
const athena = yield* Athena
const modelStats = yield* ModelStatRepo
const providerStats = yield* ProviderStatRepo
const geoStats = yield* GeoStatRepo
yield* logRuntimeCheck()
const [modelAggregates, providerAggregates, geoAggregates, geoModelAggregates] = yield* Effect.all(
[
athena
.query(buildStatsQuery(periodStart, periodEnd, "model"))
.pipe(Effect.map((rows) => rows.flatMap(toModelAggregate))),
athena
.query(buildStatsQuery(periodStart, periodEnd, "provider"))
.pipe(Effect.map((rows) => rows.flatMap(toProviderAggregate))),
athena
.query(buildStatsQuery(periodStart, periodEnd, "geo"))
.pipe(Effect.map((rows) => rows.flatMap(toGeoAggregate))),
athena
.query(buildStatsQuery(periodStart, periodEnd, "geo_model"))
.pipe(Effect.map((rows) => rows.flatMap(toGeoAggregate))),
],
{ concurrency: "unbounded" },
)
const modelRows = modelRowsFromAggregates(modelAggregates)
const providerRows = providerRowsFromAggregates(providerAggregates)
const geoRows = geoRowsFromAggregates([...geoAggregates, ...geoModelAggregates])
yield* Effect.all([modelStats.upsert(modelRows), providerStats.upsert(providerRows), geoStats.upsert(geoRows)], {
concurrency: "unbounded",
discard: true,
})
yield* Effect.logInfo(
`stats sync complete ${JSON.stringify({
startedAt: startedAt.toISOString(),
periodStart: periodStart.toISOString(),
periodEnd: periodEnd.toISOString(),
rows: modelRows.length,
providerRows: providerRows.length,
geoRows: geoRows.length,
stage: Resource.App.stage,
})}`,
)
return {
ok: true,
rows: modelRows.length,
startedAt: startedAt.toISOString(),
periodStart: periodStart.toISOString(),
periodEnd: periodEnd.toISOString(),
}
})
function logRuntimeCheck() {
return Effect.logInfo(
`athena stats runtime check ${JSON.stringify({
catalog: Resource.InferenceEvent.catalog,
database: Resource.InferenceEvent.database,
dataset: Resource.StatsSyncConfig.dataset,
table: Resource.InferenceEvent.table,
workgroup: Resource.InferenceEvent.workgroup,
region: Resource.InferenceEvent.region,
stage: Resource.App.stage,
})}`,
)
}
packages/stats/core/sst-env.d.ts
Vendored
-10
View File
@@ -1,10 +0,0 @@
/* This file is auto-generated by SST. Do not edit. */
/* tslint:disable */
/* eslint-disable */
/* deno-fmt-ignore-file */
/* biome-ignore-all lint: auto-generated */
/// <reference path="../../../sst-env.d.ts" />
import "sst"
export {}
packages/stats/core/tsconfig.json
-11
View File
@@ -1,11 +0,0 @@
{
"$schema": "https://json.schemastore.org/tsconfig",
"extends": "@tsconfig/node22/tsconfig.json",
"compilerOptions": {
"module": "ESNext",
"moduleResolution": "bundler",
"strict": true,
"noEmit": true,
"types": ["bun", "node"]
}
}
packages/stats/server/Dockerfile
-32
View File
@@ -1,32 +0,0 @@
FROM oven/bun:1.3.14-alpine AS base
WORKDIR /app
ENV NODE_ENV=production
ENV BUN_RUNTIME_TRANSPILER_CACHE_PATH=0
FROM base AS pruner
COPY . .
RUN bunx turbo@2.8.13 prune @opencode-ai/stats-server --docker --no-update-notifier --no-color
FROM base AS installer
COPY --from=pruner /app/out/json/ ./
# Bun 1.3.x needs the pruned workspace globs and lockfile metadata refreshed before the frozen production install.
RUN bun -e 'const packageJson = await Bun.file("package.json").json(); packageJson.workspaces.packages = Array.from(new Bun.Glob("packages/**/package.json").scanSync(".")).map((file) => file.slice(0, -"/package.json".length)).sort(); await Bun.write("package.json", JSON.stringify(packageJson, null, 2) + "\n")'
RUN rm -f bun.lock && bun install --filter @opencode-ai/stats-server --lockfile-only --ignore-scripts
RUN bun install --filter @opencode-ai/stats-server --frozen-lockfile --production --ignore-scripts
FROM base AS runner
COPY --from=installer /app ./
COPY --from=pruner /app/out/full/ ./
WORKDIR /app/packages/stats/server
EXPOSE 3000
CMD ["bun", "src/server.ts"]
packages/stats/server/package.json
-33
View File
@@ -1,33 +0,0 @@
{
"$schema": "https://json.schemastore.org/package.json",
"name": "@opencode-ai/stats-server",
"version": "7.3.46",
"private": true,
"type": "module",
"license": "MIT",
"main": "./src/server.ts",
"exports": {
".": "./src/server.ts"
},
"scripts": {
"start": "bun src/server.ts",
"typecheck": "tsgo --noEmit"
},
"dependencies": {
"@aws-sdk/client-firehose": "3.933.0",
"@effect/platform-node": "catalog:",
"@opencode-ai/stats-core": "workspace:*",
"effect": "catalog:",
"sst": "catalog:"
},
"devDependencies": {
"@tsconfig/node22": "catalog:",
"@types/bun": "catalog:",
"@types/node": "catalog:",
"@typescript/native-preview": "catalog:",
"typescript": "catalog:"
},
"engines": {
"node": ">=22"
}
}
packages/stats/server/src/ingest.ts
-159
View File
@@ -1,159 +0,0 @@
import { Buffer } from "node:buffer"
import { FirehoseClient, PutRecordBatchCommand } from "@aws-sdk/client-firehose"
import { Effect, Layer, Schema } from "effect"
import * as Context from "effect/Context"
import { Resource } from "sst/resource"
const MAX_FIREHOSE_BATCH_SIZE = 500
const MAX_FIREHOSE_ATTEMPTS = 3
const LAKE_TYPE = /^([A-Za-z0-9_]+)\.([A-Za-z0-9_]+)$/
type IngestEvent = Record<string, unknown>
type LakeRoute = { database: string; table: string }
type FirehoseRecord = { Data: Uint8Array }
export class IngestError extends Schema.TaggedErrorClass<IngestError>()("IngestError", {
message: Schema.String,
failed: Schema.Number,
cause: Schema.optional(Schema.Defect),
}) {}
export declare namespace Ingest {
export interface Service {
readonly write: (events: unknown[]) => Effect.Effect<{ records: number }, IngestError>
}
}
export class Ingest extends Context.Service<Ingest, Ingest.Service>()("@opencode/stats/Ingest") {
static readonly layer: Layer.Layer<Ingest> = Layer.effect(
Ingest,
Effect.sync(() => {
const client = new FirehoseClient({})
const write = Effect.fn("Ingest.write")(function* (events: unknown[]) {
if (events.length === 0) return { records: 0 }
const counts = countRoutedEvents(events)
if (counts.unsupported > 0) {
yield* Effect.logWarning(
`lake ingest rejected ${JSON.stringify({ records: counts.records, unsupported: counts.unsupported })}`,
)
return yield* new IngestError({
message: "Unsupported lake event type",
failed: counts.unsupported,
})
}
if (counts.records === 0) return { records: 0 }
let batch: FirehoseRecord[] = []
let batches = 0
let failed = 0
for (const event of events) {
if (!isRecord(event)) continue
const route = routeEvent(event)
if (!route) continue
batch.push(toFirehoseRecord(event, route))
if (batch.length < MAX_FIREHOSE_BATCH_SIZE) continue
failed += yield* putRecords(client, Resource.LakeIngestConfig.streamName, batch)
batches++
batch = []
}
if (batch.length > 0) {
failed += yield* putRecords(client, Resource.LakeIngestConfig.streamName, batch)
batches++
}
if (failed > 0) {
yield* Effect.logWarning(`lake ingest incomplete ${JSON.stringify({ records: counts.records, failed })}`)
return yield* new IngestError({ message: "Failed to ingest all lake records", failed })
}
yield* Effect.logInfo(`lake ingest complete ${JSON.stringify({ records: counts.records, batches })}`)
return { records: counts.records }
})
return Ingest.of({ write })
}),
)
}
const putRecords: (
client: FirehoseClient,
streamName: string,
records: FirehoseRecord[],
attempt?: number,
) => Effect.Effect<number, IngestError> = Effect.fn("Ingest.putRecords")(function* (
client,
streamName,
records,
attempt = 1,
) {
const result = yield* Effect.tryPromise({
try: () => client.send(new PutRecordBatchCommand({ DeliveryStreamName: streamName, Records: records })),
catch: (cause) =>
new IngestError({ message: "Failed to write lake records to Firehose", failed: records.length, cause }),
}).pipe(
Effect.tapError(() =>
Effect.logWarning(`firehose batch write failed ${JSON.stringify({ records: records.length, attempt })}`),
),
)
const failed =
result.RequestResponses?.flatMap((item, index) => {
const record = records[index]
if (!item.ErrorCode || !record) return []
return [record]
}) ?? []
if (failed.length === 0) return 0
if (attempt >= MAX_FIREHOSE_ATTEMPTS) {
yield* Effect.logWarning(
`firehose batch failed ${JSON.stringify({ records: failed.length, attempts: MAX_FIREHOSE_ATTEMPTS })}`,
)
return failed.length
}
yield* Effect.logWarning(
`firehose batch retrying ${JSON.stringify({ records: failed.length, attempt: attempt + 1 })}`,
)
yield* Effect.sleep(`${250 * 2 ** (attempt - 1)} millis`)
return yield* putRecords(client, streamName, failed, attempt + 1)
})
function countRoutedEvents(events: unknown[]) {
let records = 0
let unsupported = 0
for (const event of events) {
if (!isRecord(event)) continue
if (routeEvent(event)) records++
else unsupported++
}
return { records, unsupported }
}
function isRecord(item: unknown): item is IngestEvent {
return Boolean(item) && typeof item === "object" && !Array.isArray(item)
}
function routeEvent(event: IngestEvent): LakeRoute | undefined {
if (typeof event._datalake_key !== "string") return
const match = event._datalake_key.match(LAKE_TYPE)
if (!match?.[1] || !match[2]) return
return {
database: match[1],
table: match[2],
}
}
function toFirehoseRecord(event: IngestEvent, route: LakeRoute): FirehoseRecord {
return {
Data: Buffer.from(
JSON.stringify({
...Object.fromEntries(Object.entries(event).filter(([key]) => key !== "_datalake_key")),
_lake_database: route.database,
_lake_table: route.table,
_lake_operation: "insert" as const,
}),
),
}
}
packages/stats/server/src/resource.d.ts
Vendored
-11
View File
@@ -1,11 +0,0 @@
import "sst/resource"
declare module "sst/resource" {
export interface Resource {
LakeIngestConfig: {
secret: string
streamName: string
type: "sst.sst.Linkable"
}
}
}
packages/stats/server/src/router.ts
-73
View File
@@ -1,73 +0,0 @@
import { Buffer } from "node:buffer"
import { timingSafeEqual } from "node:crypto"
import { Effect, Schema } from "effect"
import * as Semaphore from "effect/Semaphore"
import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/unstable/http"
import { Resource } from "sst/resource"
import { Ingest } from "./ingest"
import { isShuttingDown } from "./shutdown"
const MAX_CONCURRENT_INGEST_REQUESTS = 8
const IngestPayload = Schema.Struct({
events: Schema.optional(Schema.Unknown),
})
export const Routes = HttpRouter.use((router) =>
Effect.gen(function* () {
const ingestService = yield* Ingest
const ingestRequests = yield* Semaphore.make(MAX_CONCURRENT_INGEST_REQUESTS)
yield* Effect.all(
[
router.add("GET", "/health", () => json(200, { ok: true })),
router.add("GET", "/ready", () => json(isShuttingDown() ? 503 : 200, { ok: !isShuttingDown() })),
router.add("POST", "/", ingestRequests.withPermit(ingest(ingestService))),
],
{ discard: true },
)
}),
)
const ingest = (ingestService: Ingest.Service) =>
Effect.gen(function* () {
const request = yield* HttpServerRequest.HttpServerRequest
if (!isAuthorized(request.headers)) return yield* json(401, { ok: false, error: "Unauthorized" })
const payload = yield* HttpServerRequest.schemaBodyJson(IngestPayload).pipe(
Effect.match({
onFailure: () => undefined,
onSuccess: (value) => value,
}),
)
if (!payload) return yield* json(400, { ok: false, error: "Invalid JSON body" })
const events = Array.isArray(payload.events) ? payload.events : []
if (events.length === 0) return yield* json(202, { ok: true, records: 0 })
return yield* ingestService.write(events).pipe(
Effect.flatMap((result) => json(202, { ok: true, records: result.records })),
Effect.catchTag("IngestError", (error) =>
json(502, { ok: false, records: countRecords(events), failed: error.failed }),
),
)
})
function isAuthorized(headers: Record<string, string | undefined>) {
const actual = Buffer.from(headers.authorization ?? headers.Authorization ?? "")
const expected = Buffer.from(`Bearer ${Resource.LakeIngestConfig.secret}`)
if (actual.length !== expected.length) return false
return timingSafeEqual(actual, expected)
}
function countRecords(items: unknown[]) {
let records = 0
for (const item of items) {
if (Boolean(item) && typeof item === "object" && !Array.isArray(item)) records++
}
return records
}
function json(status: number, body: Record<string, unknown>) {
return HttpServerResponse.json(body, { status }).pipe(Effect.orDie)
}
packages/stats/server/src/server.ts
-28
View File
@@ -1,28 +0,0 @@
import * as NodeHttpServer from "@effect/platform-node/NodeHttpServer"
import * as NodeRuntime from "@effect/platform-node/NodeRuntime"
import { Config, Layer } from "effect"
import { HttpRouter } from "effect/unstable/http"
import { createServer } from "node:http"
import { Ingest } from "./ingest"
import { Routes } from "./router"
import { registerShutdownSignalHandlers } from "./shutdown"
registerShutdownSignalHandlers()
const ServerLive = NodeHttpServer.layerConfig(
() => createServer(),
Config.all({
port: Config.number("PORT").pipe(Config.withDefault(3000)),
host: Config.string("HOST").pipe(Config.withDefault("0.0.0.0")),
}),
)
const runtimeLayer = Ingest.layer
const programLayer = Routes.pipe(Layer.provide(runtimeLayer))
const main = Layer.launch(
HttpRouter.serve(programLayer, {
disableLogger: true,
}).pipe(Layer.provideMerge(ServerLive)),
)
NodeRuntime.runMain(main, { disableErrorReporting: true })
packages/stats/server/src/shutdown.ts
-17
View File
@@ -1,17 +0,0 @@
let shuttingDown = false
let signalHandlersRegistered = false
export function isShuttingDown() {
return shuttingDown
}
export function registerShutdownSignalHandlers() {
if (signalHandlersRegistered) return
signalHandlersRegistered = true
process.once("SIGTERM", markShuttingDown)
process.once("SIGINT", markShuttingDown)
}
function markShuttingDown() {
shuttingDown = true
}
packages/stats/server/src/stat-sync.ts
-22
View File
@@ -1,22 +0,0 @@
import * as NodeRuntime from "@effect/platform-node/NodeRuntime"
import { Athena } from "@opencode-ai/stats-core/athena"
import { layer as statsLayer } from "@opencode-ai/stats-core/runtime"
import { syncStats } from "@opencode-ai/stats-core/stat-sync"
import { Cause, Effect, Layer, Schedule } from "effect"
const SYNC_INTERVAL = "1 hour"
const runtimeLayer = Layer.mergeAll(statsLayer, Athena.layer)
const syncPass = syncStats().pipe(
Effect.catchCause((cause) =>
Effect.logWarning(`stats sync failed ${JSON.stringify({ cause: Cause.pretty(cause) })}`),
),
)
const daemon = Effect.logInfo("stats sync daemon started").pipe(
Effect.andThen(syncPass.pipe(Effect.repeat(Schedule.fixed(SYNC_INTERVAL)))),
Effect.forkScoped,
)
NodeRuntime.runMain(Layer.launch(Layer.effectDiscard(daemon).pipe(Layer.provide(runtimeLayer))), {
disableErrorReporting: true,
})
packages/stats/server/sst-env.d.ts
Vendored
-10
View File
@@ -1,10 +0,0 @@
/* This file is auto-generated by SST. Do not edit. */
/* tslint:disable */
/* eslint-disable */
/* deno-fmt-ignore-file */
/* biome-ignore-all lint: auto-generated */
/// <reference path="../../../sst-env.d.ts" />
import "sst"
export {}
packages/stats/server/tsconfig.json
-12
View File
@@ -1,12 +0,0 @@
{
"$schema": "https://json.schemastore.org/tsconfig",
"extends": "@tsconfig/node22/tsconfig.json",
"compilerOptions": {
"module": "ESNext",
"moduleResolution": "bundler",
"strict": true,
"noEmit": true,
"types": ["bun", "node"]
},
"include": ["src", "../core/src/resource.d.ts"]
}
script/upstream/transforms/skip-files.test.ts
+5
View File
@@ -16,6 +16,11 @@ test("matches upstream CLI scaffold glob paths", () => {
expect(shouldSkip("packages/cli/src/index.ts", ["packages/cli/**"])).toBe(true)
})
test("matches upstream stats package glob paths", () => {
expect(shouldSkip("packages/stats/app/package.json", ["packages/stats/**"])).toBe(true)
expect(shouldSkip("packages/stats/core/src/index.ts", ["packages/stats/**"])).toBe(true)
})
test("matches removed vscode sdk glob paths", () => {
expect(shouldSkip("sdks/vscode/package.json", ["sdks/vscode/**"])).toBe(true)
expect(shouldSkip("sdks/vscode/src/extension.ts", ["sdks/vscode/**"])).toBe(true)
script/upstream/utils/config.ts
+1
View File
@@ -160,6 +160,7 @@ export const defaultConfig: MergeConfig = {
"packages/desktop/**",
"packages/desktop-electron/**",
"packages/cli/**",
"packages/stats/**",
"sdks/vscode/**",
// GitHub Action - Kilo version is fully ported and complete
"github/index.ts",
specs/v2/config.md
+11 -11
View File
@@ -18,7 +18,7 @@ Use one v2 config schema for now. Some fields, such as `autoupdate`, are intende
Small fields describing the config file itself rather than application behavior.
| Field | Current Purpose | Status | Notes |
| --------- | ---------------------------------------------------------- | ------ | ------------------------------------------------------------------------------------- |
|---|---|---|---|
| `$schema` | JSON schema reference for editor validation and completion | keep | Keep as read-only metadata; loading config must not insert it or create files for it. |
## Group 2: Process And Server Settings
@@ -26,7 +26,7 @@ Small fields describing the config file itself rather than application behavior.
Settings that affect process startup, shell execution, or network serving. Review global-only versus location-specific scope carefully.
| Field | Current Purpose | Status | Notes |
| ------------ | --------------------------------------------------- | ------ | ------------------------------------------------------------------------------ |
|---|---|---|---|
| `shell` | Default shell for terminal and shell tool execution | keep | Port as effective config; shared shell choice is used throughout opencode. |
| `logLevel` | Intended logging level configuration | remove | Do not port: no config consumer exists and logging initializes from CLI input. |
| `server` | Hostname, port, mDNS, and CORS settings | remove | Do not port: location config is loaded after the server is already running. |
@@ -37,7 +37,7 @@ Settings that affect process startup, shell execution, or network serving. Revie
Configuration that introduces location-scoped project resources or discoverable content.
| Field | Current Purpose | Status | Notes |
| -------------- | --------------------------------------- | -------- | --------------------------------------------------------------------------------------------------------- |
|---|---|---|---|
| `command` | User-defined commands | remove | Do not port as v2 config; named reusable user workflows belong to skills. |
| `skills` | Additional skill locations | redesign | Replace `{ paths?, urls? }` with a single array of local path or remote URL discovery sources. |
| `reference` | Named git or local directory references | redesign | Rename to plural `references`; retain named local path and Git repository external-context entries. |
@@ -86,7 +86,7 @@ Retain the compact string entry form as well: values starting with `.`, `/`, or
Plugin loading has source-path and scope-sensitive behavior, so it should be reviewed separately from other project resources.
| Field | Current Purpose | Status | Notes |
| -------- | ----------------------------- | -------- | ----------------------------------------------------------------------------------------------------------- |
|---|---|---|---|
| `plugin` | User-specified plugin modules | redesign | Rename to plural `plugins`; retain ordered loading with package strings or `{ package, options? }` entries. |
Plugin order remains part of the v2 configuration contract because hook registration and execution can depend on load order. Replace legacy option tuples with readable object entries:
@@ -112,7 +112,7 @@ The configured `plugins` list represents package-loaded plugins only. Local plug
Settings controlling local file observation, snapshots, language tooling, and tool output behavior.
| Field | Current Purpose | Status | Notes |
| ------------- | --------------------------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------- |
|---|---|---|---|
| `watcher` | Ignore patterns for filesystem watching | keep | Keep `{ ignore?: string[] }`; this configures the filesystem watcher subsystem. |
| `snapshot` | Enable filesystem snapshot tracking | redesign | Rename to plural `snapshots`; controls creation of snapshots used for undo and revert behavior. |
| `formatter` | Configure formatters | keep | Keep singular `boolean \| Record<string, entry>` shape; it configures built-in enablement and named formatter overrides. |
@@ -146,7 +146,7 @@ Rename legacy `attachment` to `attachments` in v2. This setting controls process
Settings affecting sharing behavior or user/account identity rather than model execution.
| Field | Current Purpose | Status | Notes |
| ------------ | ----------------------------------------------- | ------ | ---------------------------------------------------------------------------------------------------------------------- |
|---|---|---|---|
| `share` | Session sharing behavior | keep | Keep `"manual" \| "auto" \| "disabled"`; it controls manual sharing permission and automatic sharing of new sessions. |
| `autoshare` | Legacy automatic sharing flag | remove | Do not port deprecated alias; use `share: "auto"`. |
| `enterprise` | Enterprise URL configuration | keep | Keep `{ url?: string }`; currently selects the legacy sharing service endpoint when no organization account is active. |
@@ -169,7 +169,7 @@ Retain `enterprise.url` for legacy enterprise share hosting selection and `usern
Provider catalog customization and model-choice configuration. The new core work has started here.
| Field | Current Purpose | Status | Notes |
| -------------------- | ------------------------------------------------- | -------- | ---------------------------------------------------------------------------------------------------------------------------- |
|---|---|---|---|
| `provider` | Custom provider configuration and model overrides | redesign | Rename to plural `providers` in v2; do not preserve the legacy singular key. Review nested provider/model fields separately. |
| `disabled_providers` | Disable automatically loaded providers | redesign | Replace with `experimental.policies: [{ effect: "deny", action: "provider.use", resource: "..." }]`. |
| `enabled_providers` | Restrict enabled providers to an allowlist | redesign | Replace with ordered `provider.use` allow/deny statements and wildcard resources. |
@@ -239,7 +239,7 @@ Do not port legacy provider model `reasoning`, `temperature`, or `interleaved` f
Agent behavior and tool-access policy. Review together because agent configuration can contain permissions and model choices.
| Field | Current Purpose | Status | Notes |
| --------------- | --------------------------------------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------ |
|---|---|---|---|
| `default_agent` | Choose default primary agent | remove | Do not retain a separate top-level selector; default choice should be designed with the v2 agent configuration model. |
| `mode` | Legacy agent configuration alias | remove | Do not port deprecated alias; configure agents through the v2 agent surface only. |
| `agent` | Configure primary, subagent, and specialized agents | redesign | Rename to plural `agents`; retain a named map of built-in overrides and custom agent definitions. |
@@ -305,7 +305,7 @@ Rename legacy `permission` to `permissions` and expose the normalized ordered ru
External protocol and server integration configuration.
| Field | Current Purpose | Status | Notes |
| ----- | ------------------------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------- |
|---|---|---|---|
| `mcp` | MCP server definitions and enablement | redesign | Keep opencode's explicit local/remote server entry format, nested under `mcp.servers`; use `disabled` for inactive entries and move timeout here. |
Keep the opencode MCP server entry format instead of adopting the common `mcpServers` copy/paste shape. Local servers remain explicit `type: "local"` entries with command arrays and `environment`; remote servers remain explicit `type: "remote"` entries with `url`, `headers`, and optional `oauth`. Nest the server map under `mcp.servers` so protocol-wide settings such as default timeout can live under the same subsystem.
@@ -345,7 +345,7 @@ Keep the opencode MCP server entry format instead of adopting the common `mcpSer
Behavior affecting long-running conversations and context management.
| Field | Current Purpose | Status | Notes |
| ------------ | ----------------------------------------------------------- | -------- | ------------------------------------------------------------------------------------- |
|---|---|---|---|
| `compaction` | Automatic compaction, pruning, and context reserve settings | redesign | Group retained verbatim history under `keep` and rename context headroom to `buffer`. |
Retain the compaction capability but redesign the less clear limits. `keep.turns` is the maximum number of recent user turns to preserve verbatim after compaction, and `keep.tokens` is the token budget for those retained turns. `buffer` is the token headroom reserved so automatic compaction triggers before the input window is exhausted.
@@ -369,7 +369,7 @@ Retain the compaction capability but redesign the less clear limits. `keep.turns
Fields that should not be ported by inertia; each needs an explicit justification.
| Field | Current Purpose | Status | Notes |
| ------------------------------------ | --------------------------------------- | -------- | --------------------------------------------------------------------------------------------------------------------------- |
|---|---|---|---|
| `layout` | Legacy layout selection | remove | Do not port deprecated option; stretch layout is always used. |
| `experimental.disable_paste_summary` | Disable pasted-content summary behavior | remove | Do not port; pasted-input presentation behavior belongs to the client/UI surface. |
| `experimental.batch_tool` | Enable batch tool | remove | Do not port; batch tool is no longer a supported feature. |
specs/v2/provider-policy.md
+1 -1
View File
@@ -67,7 +67,7 @@ Both `action` and `resource` use opencode's existing wildcard matching behavior.
Examples:
| Action | Resource | Matches |
| -------------- | ----------- | ---------------------------------------------------------------------------- |
|---|---|---|
| `provider.use` | `openai` | Only use of provider ID `openai` |
| `provider.use` | `company-*` | Use of provider IDs such as `company-us` and `company-eu` |
| `provider.*` | `*` | Any provider operation on any provider, if more actions are introduced later |