From 6902129eb0fcfcb620c50f9c80a5965dea4d04ac Mon Sep 17 00:00:00 2001 From: DIYgod Date: Fri, 8 Mar 2024 23:31:50 +0800 Subject: [PATCH] feat: new build script --- .github/workflows/build-assets.yml | 2 +- .github/workflows/test.yml | 2 +- lib/registry.ts | 11 +- lib/routes-new/bilibili/bangumi.ts | 4 + lib/routes/bilibili/radar.ts | 2 +- lib/types.ts | 2 +- package.json | 5 +- pnpm-lock.yaml | 14 +++ scripts/workflow/build-maintainer.ts | 75 ------------ scripts/workflow/build-radar.ts | 108 ------------------ scripts/workflow/build-routes.ts | 56 +++++++++ .../docs/joinus/advanced/script-standard.md | 4 +- .../joinus/advanced/script-standard.md | 4 +- 13 files changed, 94 insertions(+), 195 deletions(-) delete mode 100644 scripts/workflow/build-maintainer.ts delete mode 100644 scripts/workflow/build-radar.ts create mode 100644 scripts/workflow/build-routes.ts diff --git a/.github/workflows/build-assets.yml b/.github/workflows/build-assets.yml index 44b896103a..e862f5fadf 100644 --- a/.github/workflows/build-assets.yml +++ b/.github/workflows/build-assets.yml @@ -34,7 +34,7 @@ jobs: - name: Install dependencies (yarn) run: pnpm i - name: Build assets - run: npm run build:all + run: npm run build - name: Deploy uses: peaceiris/actions-gh-pages@v3 with: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1793bb0bb8..2f768191ea 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -149,7 +149,7 @@ jobs: cache: 'pnpm' - run: pnpm i - name: Build radar and maintainer - run: npm run build:all + run: npm run build automerge: if: github.triggering_actor == 'dependabot[bot]' && github.event_name == 'pull_request' diff --git a/lib/registry.ts b/lib/registry.ts index c555cdc1f0..e36dcbfb74 100644 --- a/lib/registry.ts +++ b/lib/registry.ts @@ -14,6 +14,7 @@ const namespaces: Record< routes: Record; } > = {}; + for (const module in modules) { const content = modules[module] as | { @@ -38,10 +39,18 @@ for (const module in modules) { routes: {}, }; } - namespaces[namespace].routes[content.route.path] = content.route; + if (Array.isArray(content.route.path)) { + for (const path of content.route.path) { + namespaces[namespace].routes[path] = content.route; + } + } else { + namespaces[namespace].routes[content.route.path] = content.route; + } } } +export { namespaces }; + export default function (app: Hono) { for (const namespace in namespaces) { const subApp = app.basePath(`/${namespace}`); diff --git a/lib/routes-new/bilibili/bangumi.ts b/lib/routes-new/bilibili/bangumi.ts index 9aeafcb2c4..6c425737e6 100644 --- a/lib/routes-new/bilibili/bangumi.ts +++ b/lib/routes-new/bilibili/bangumi.ts @@ -19,6 +19,10 @@ export const route: Route = { }, name: '番剧', maintainers: ['DIYgod'], + radar: { + source: ['www.bilibili.com/bangumi/media/:bid'], + target: '/bangumi/media/:bid', + }, handler: async (ctx) => { let seasonid = ctx.req.param('seasonid'); const mediaid = ctx.req.param('mediaid'); diff --git a/lib/routes/bilibili/radar.ts b/lib/routes/bilibili/radar.ts index 43edc5c653..6e8f424e9c 100644 --- a/lib/routes/bilibili/radar.ts +++ b/lib/routes/bilibili/radar.ts @@ -26,7 +26,7 @@ export default { title: '番剧', docs: 'https://docs.rsshub.app/routes/social-media#bilibili', source: '/bangumi/media/:bid', - target: (params) => `/bilibili/bangumi/media/${params.bid.replace('md', '')}`, + target: `/bilibili/bangumi/media/:bid`, }, { title: '当前在线', diff --git a/lib/types.ts b/lib/types.ts index d72716d6b7..d59aefaa84 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -62,7 +62,7 @@ interface Namespace extends NamespaceItem { export type { Namespace }; interface RouteItem { - path: string; + path: string | string[]; name: string; url?: string; maintainers: string[]; diff --git a/package.json b/package.json index a55c5868a1..a53e3880e2 100644 --- a/package.json +++ b/package.json @@ -20,9 +20,7 @@ "lib" ], "scripts": { - "build:all": "npm run build:radar && npm run build:maintainer", - "build:maintainer": "tsx scripts/workflow/build-maintainer.ts", - "build:radar": "tsx scripts/workflow/build-radar.ts", + "build": "tsx scripts/workflow/build-routes.ts", "dev": "cross-env NODE_ENV=dev tsx watch --no-cache lib/index.ts", "format": "eslint --cache --fix \"**/*.{ts,js,yml}\" && node website/docs/.format/format.mjs && prettier \"**/*.{ts,js,json}\" --write", "format:check": "eslint --cache \"**/*.{ts,js,yml}\" && prettier \"**/*.{ts,js,json}\" --check", @@ -175,6 +173,7 @@ "staged-git-files": "1.3.0", "string-width": "7.1.0", "supertest": "6.3.4", + "tldts": "6.1.13", "to-vfile": "8.0.0", "tosource": "2.0.0-alpha.3", "typescript": "5.3.3", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 892efac8d3..4f35ed51cb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -379,6 +379,9 @@ devDependencies: supertest: specifier: 6.3.4 version: 6.3.4 + tldts: + specifier: 6.1.13 + version: 6.1.13 to-vfile: specifier: 8.0.0 version: 8.0.0 @@ -9437,6 +9440,17 @@ packages: hasBin: true dev: false + /tldts-core@6.1.13: + resolution: {integrity: sha512-M1XP4D13YtXARKroULnLsKKuI1NCRAbJmUGGoXqWinajIDOhTeJf/trYUyBoLVx1/Nx1KBKxCrlW57ZW9cMHAA==} + dev: true + + /tldts@6.1.13: + resolution: {integrity: sha512-+GxHFKVHvUTg2ieNPTx3b/NpZbgJSTZEDdI4cJzTjVYDuxijeHi1tt7CHHsMjLqyc+T50VVgWs3LIb2LrXOzxw==} + hasBin: true + dependencies: + tldts-core: 6.1.13 + dev: true + /tmp@0.0.33: resolution: {integrity: sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==} engines: {node: '>=0.6.0'} diff --git a/scripts/workflow/build-maintainer.ts b/scripts/workflow/build-maintainer.ts deleted file mode 100644 index 108763ec92..0000000000 --- a/scripts/workflow/build-maintainer.ts +++ /dev/null @@ -1,75 +0,0 @@ -import * as fs from 'node:fs'; -import * as path from 'node:path'; -import { directoryImport } from 'directory-import'; - -const target = path.join(__dirname, '../../assets/build/maintainer.json'); -const dirname = path.join(__dirname + '../../../lib/routes'); - -// Presence Check -for (const dir of fs.readdirSync(dirname)) { - const dirPath = path.join(dirname, dir); - if (fs.existsSync(path.join(dirPath, 'router.ts')) && !fs.existsSync(path.join(dirPath, 'maintainer.ts'))) { - throw new Error(`No maintainer.ts in "${dirPath}".`); - } -} - -// 遍历整个 routes 文件夹，收集模块 maintainer.ts -// const maintainerPath = require('require-all')({ -// dirname, -// filter: /maintainer\.ts$/, -// }); -const imports = directoryImport({ - targetDirectoryPath: dirname, - importPattern: /maintainer\.ts$/, -}); - -const maintainers = {}; - -// 将收集到的自定义模块进行合并 -for (const dir in imports) { - const routes = imports[dir].default; // Do not merge other file - - // typo check e.g., ✘ module.export, ✔ module.exports - if (!Object.keys(routes).length) { - throw new Error(`No maintainer in "${dir}".`); - } - for (const author of Object.values(routes)) { - if (!Array.isArray(author)) { - throw new TypeError(`Maintainers' name should be an array in "${dir}".`); - } - // check for [], [''] or ['Someone', ''] - if (author.length < 1 || author.includes('')) { - throw new Error(`Empty maintainer in "${dir}".`); - } - } - - for (const key in routes) { - maintainers[dir.replace('/maintainer.ts', '') + (key.endsWith('/') ? key.substring(0, key.length - 1) : key)] = routes[key]; - } -} - -// 兼容旧版路由 -// const router = require('../../lib/router.js'); -// for (const e of router.stack) { -// if (!maintainers[e.path]) { -// maintainers[e.path] = []; -// } -// } - -const maintainer = Object.keys(maintainers) - .sort() - .reduce((obj, path) => { - obj[path] = maintainers[path]; - return obj; - }, {}); - -const count = Object.keys(maintainer).length; -const uniqueMaintainer = new Set(); -for (const e of Object.values(maintainer).flat()) { - uniqueMaintainer.add(e); -} - -// eslint-disable-next-line no-console -console.log(`We have ${count} routes and maintained by ${uniqueMaintainer.size} contributors!`); - -fs.writeFileSync(target, JSON.stringify(maintainer, null, 4)); diff --git a/scripts/workflow/build-radar.ts b/scripts/workflow/build-radar.ts deleted file mode 100644 index 36325907c7..0000000000 --- a/scripts/workflow/build-radar.ts +++ /dev/null @@ -1,108 +0,0 @@ -import * as fs from 'node:fs'; -import * as path from 'node:path'; -import { directoryImport } from 'directory-import'; -import toSource from 'tosource'; - -const targetJs = path.join(__dirname, '../../assets/build/radar-rules.js'); -const targetJson = path.join(__dirname, '../../assets/build/radar-rules.json'); -const dirname = path.join(__dirname + '../../../lib/routes'); - -// Namespaces that do not require radar.ts -const allowNamespace = new Set(['discourse', 'discuz', 'ehentai', 'lemmy', 'mail', 'test', 'index.tsx', 'robots.txt.ts']); -// Check if a radar.ts file is exist under each folder of dirname -for (const dir of fs.readdirSync(dirname)) { - const dirPath = path.join(dirname, dir); - if (!fs.existsSync(path.join(dirPath, 'radar.ts')) && !allowNamespace.has(dir)) { - throw new Error(`No radar.ts in "${dirPath}".`); - } -} - -const validateRadarRules = (rule, dir) => { - const allowDomains = new Set(['www.gov.cn']); - const blockWords = ['/', 'http', 'www']; - const domain = Object.keys(rule); - if (!domain.length && !allowNamespace.has(dir)) { - // typo check e.g., ✘ module.export, ✔ module.exports - throw new Error(`No Radar rule in "${dir}".`); - } - for (const [d, r] of Object.entries(rule)) { - if (blockWords.some((word) => d.startsWith(word)) && !allowDomains.has(d)) { - throw new Error(`Domain name "${d}" should not contain any of ${blockWords.join(', ')}.`); - } - if (!Object.hasOwn(r, '_name')) { - throw new Error(`No _name in "${dir}".`); - } - // property check - for (const [host, items] of Object.entries(r)) { - if (host !== '_name') { - if (!Array.isArray(items)) { - throw new TypeError(`Radar rules for domain "${host}" in "${dir}" should be placed in an array.`); - } - for (const item of items) { - if (!Object.hasOwn(item, 'title') || !Object.hasOwn(item, 'docs')) { - throw new Error(`Radar rules for "${host}" in "${dir}" should have at least "title" and "docs".`); - } - if (!item.title || !item.docs) { - throw new Error(`Radar rules for "${host}" in "${dir}" should not be empty.`); - } - if (!item.docs.startsWith('https://docs.rsshub.app/')) { - throw new Error(`Radar rules for "${host}" in "${dir}" should start with 'https://docs.rsshub.app/'.`); - } - if (Array.isArray(item.source)) { - if (!item.source.length) { - // check for [] - throw new Error(`Radar rule of "${item.title}" for subdomain "${host}" in "${dir}" should not be empty.`); - } - if (item.source.some((s) => s.includes('#') || s.includes('='))) { - // Some will try to match '/some/path?a=1' which is not supported - throw new Error(`Radar rule of "${item.title}" for subdomain "${host}" in "${dir}" cannot match URL hash or URL search parameters.`); - } - if (item.source.some((s) => !s.length)) { - // check for ['/some/thing', ''] and [''] - throw new Error(`Radar rule of "${item.title}" for subdomain "${host}" in "${dir}" should not be empty.`); - } - } - if (typeof item.source === 'string') { - if (!item.source.length) { - // check for '' - throw new Error(`Radar rule of "${item.title}" for subdomain "${host}" in "${dir}" should not be empty.`); - } - if (item.source.includes('#') || item.source.includes('=')) { - throw new Error(`Radar rule of "${item.title}" for subdomain "${host}" in "${dir}" cannot match URL hash or URL search parameters.`); - } - } - for (const key in item) { - if (key !== 'title' && key !== 'docs' && key !== 'source' && key !== 'target') { - throw new Error(`Radar rules for "${host}" in "${dir}" should not have property "${key}".`); - } - } - } - } - } - } -}; - -// const radarRules = require('require-all')({ -// dirname, -// filter: /radar\.ts$/, -// }); -const imports = directoryImport({ - targetDirectoryPath: dirname, - importPattern: /radar\.ts$/, -}); - -let rules = {}; - -for (const dir in imports) { - const rule = imports[dir].default; // Do not merge other file - - validateRadarRules(rule, dir.replace('/radar.ts', '').replace(/^\//, '')); - - rules = { ...rules, ...rule }; -} - -const oldRules = require('./radar-rules.js'); // Match old rules -rules = { ...rules, ...oldRules }; - -fs.writeFileSync(targetJs, `(${toSource(rules)})`); -fs.writeFileSync(targetJson, JSON.stringify(rules, undefined, 2)); diff --git a/scripts/workflow/build-routes.ts b/scripts/workflow/build-routes.ts new file mode 100644 index 0000000000..cdbe77870f --- /dev/null +++ b/scripts/workflow/build-routes.ts @@ -0,0 +1,56 @@ +import { namespaces } from '../../lib/registry'; +import { parse } from 'tldts'; +import fs from 'node:fs'; +import * as path from 'node:path'; + +const maintainers: Record = {}; +const radar: { + [domain: string]: { + _name: string; + [subdomain: string]: + | { + title: string; + docs: string; + source: string[]; + target: string | ((params: any, url: string) => string); + }[] + | string; + }; +} = {}; + +for (const namespace in namespaces) { + for (const path in namespaces[namespace].routes) { + const realPath = `/${namespace}${path}`; + const data = namespaces[namespace].routes[path]; + if (data.maintainers) { + maintainers[realPath] = data.maintainers; + } + if (data.radar && data.radar.source) { + const parsedDomain = parse(new URL('https://' + data.radar.source[0]).hostname); + const subdomain = parsedDomain.subdomain || '.'; + const domain = parsedDomain.domain; + if (domain) { + if (!radar[domain]) { + radar[domain] = { + _name: namespaces[namespace].name, + }; + } + if (!radar[domain][subdomain]) { + radar[domain][subdomain] = []; + } + radar[domain][subdomain].push({ + title: data.name, + docs: `https://docs.rsshub.app/routes/${data.categories[0] || 'other'}`, + source: data.radar.source.map((source) => { + const sourceURL = new URL('https://' + source); + return sourceURL.pathname + sourceURL.search + sourceURL.hash; + }), + target: data.radar.target || realPath, + }); + } + } + } +} + +fs.writeFileSync(path.join(__dirname, '../../assets/build/radar-rules.json'), JSON.stringify(radar, null, 2)); +fs.writeFileSync(path.join(__dirname, '../../assets/build/maintainers.json'), JSON.stringify(maintainers, null, 2)); diff --git a/website/docs/joinus/advanced/script-standard.md b/website/docs/joinus/advanced/script-standard.md index 2729b86d7e..b4c66ae0a9 100644 --- a/website/docs/joinus/advanced/script-standard.md +++ b/website/docs/joinus/advanced/script-standard.md @@ -109,7 +109,7 @@ The `maintainer.ts` file should export an object that provides maintainer inform - Key: Corresponding route path - Value: Array of string, including all maintainers' GitHub ID. -To generate a list of maintainers, use the following command: `pnpm run build:maintainer`, which will create the list under `assets/build/`. +To generate a list of maintainers, use the following command: `pnpm run build`, which will create the list under `assets/build/`. :::danger @@ -121,7 +121,7 @@ The path should be the same as the `path` in the corresponding documentation bef All routes are required to include the `radar.ts` file, which includes the corresponding domain name. The minimum requirement for a successful match is for the rule to show up on the corresponding site which requires filling in the `title` and `docs` fields. -To generate a complete `radar-rules.ts` file, use the following command: `yarn build:radar`, which will create the file under `assets/build/`. +To generate a complete `radar-rules.ts` file, use the following command: `yarn build`, which will create the file under `assets/build/`. :::tip diff --git a/website/i18n/zh/docusaurus-plugin-content-docs/current/joinus/advanced/script-standard.md b/website/i18n/zh/docusaurus-plugin-content-docs/current/joinus/advanced/script-standard.md index 3b27b37aab..f8a3dbf59d 100644 --- a/website/i18n/zh/docusaurus-plugin-content-docs/current/joinus/advanced/script-standard.md +++ b/website/i18n/zh/docusaurus-plugin-content-docs/current/joinus/advanced/script-standard.md @@ -107,7 +107,7 @@ RSSHub 会将所有路由命名空间的文件夹名附加到路由前面。路 - 键: 对应的路由 - 值：一个字符串数组，包括所有维护者的 GitHub ID。 -要生成维护者列表，可使用以下命令：`pnpm run build:maintainer`，它将在 `assets/build/` 目录下一份维护者列表。 +要生成维护者列表，可使用以下命令：`pnpm run build`，它将在 `assets/build/` 目录下一份维护者列表。 :::danger @@ -119,7 +119,7 @@ RSSHub 会将所有路由命名空间的文件夹名附加到路由前面。路 所有路由都需要包含 `radar.ts` 文件，其中包括相应的域名。最低要求是规则出现在相应的站点上，即需要填写 `title` 和 `docs` 字段。 -要生成完整的 `radar-rules.js` 文件，可使用以下命令：`yarn build:radar`，它将在 `assets/build/` 目录下创建文件。 +要生成完整的 `radar-rules.js` 文件，可使用以下命令：`yarn build`，它将在 `assets/build/` 目录下创建文件。 :::tip