From 84421591616aa0a6b7f1d07f9f0a3610d22fba90 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 23 Sep 2020 10:22:44 +0000 Subject: [PATCH 01/44] chore(deps): bump puppeteer from 5.3.0 to 5.3.1 Bumps [puppeteer](https://github.com/puppeteer/puppeteer) from 5.3.0 to 5.3.1. - [Release notes](https://github.com/puppeteer/puppeteer/releases) - [Commits](https://github.com/puppeteer/puppeteer/compare/v5.3.0...v5.3.1) Signed-off-by: dependabot-preview[bot] --- package.json | 2 +- yarn.lock | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index a3cb6cb02a..530e37c3df 100644 --- a/package.json +++ b/package.json @@ -102,7 +102,7 @@ "parse-torrent": "7.1.3", "pidusage": "2.0.21", "plist": "3.0.1", - "puppeteer": "5.3.0", + "puppeteer": "5.3.1", "query-string": "6.13.2", "redis": "3.0.2", "require-all": "3.0.0", diff --git a/yarn.lock b/yarn.lock index 3c88c00ba1..b4da75733f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10088,16 +10088,15 @@ punycode@^2.1.0, punycode@^2.1.1: resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec" integrity sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A== -puppeteer@5.3.0: - version "5.3.0" - resolved "https://registry.yarnpkg.com/puppeteer/-/puppeteer-5.3.0.tgz#0abf83d0f2d1273baf2b56885a813f8052903e33" - integrity sha512-GjqMk5GRro3TO0sw3QMsF1H7n+/jaK2OW45qMvqjYUyJ7y4oA//9auy969HHhTG3HZXaMxY/NWXF/NXlAFIvtw== +puppeteer@5.3.1: + version "5.3.1" + resolved "https://registry.yarnpkg.com/puppeteer/-/puppeteer-5.3.1.tgz#324e190d89f25ac33dba539f57b82a18553f8646" + integrity sha512-YTM1RaBeYrj6n7IlRXRYLqJHF+GM7tasbvrNFx6w1S16G76NrPq7oYFKLDO+BQsXNtS8kW2GxWCXjIMPvfDyaQ== dependencies: debug "^4.1.0" devtools-protocol "0.0.799653" extract-zip "^2.0.0" https-proxy-agent "^4.0.0" - mime "^2.0.3" pkg-dir "^4.2.0" progress "^2.0.1" proxy-from-env "^1.0.0" From 74d9501d200bd473eed6b616ce664e0353e67dda Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 24 Sep 2020 14:00:33 +0000 Subject: [PATCH 02/44] chore(deps): bump googleapis from 59.0.0 to 60.0.1 Bumps [googleapis](https://github.com/googleapis/google-api-nodejs-client) from 59.0.0 to 60.0.1. - [Release notes](https://github.com/googleapis/google-api-nodejs-client/releases) - [Changelog](https://github.com/googleapis/google-api-nodejs-client/blob/master/CHANGELOG.md) - [Commits](https://github.com/googleapis/google-api-nodejs-client/compare/v59.0.0...v60.0.1) Signed-off-by: dependabot-preview[bot] --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 530e37c3df..c7058b02ad 100644 --- a/package.json +++ b/package.json @@ -80,7 +80,7 @@ "etag": "1.8.1", "fanfou-sdk": "4.2.0", "git-rev-sync": "3.0.1", - "googleapis": "59.0.0", + "googleapis": "60.0.1", "got": "11.7.0", "he": "1.2.0", "hooman": "1.2.6", diff --git a/yarn.lock b/yarn.lock index b4da75733f..e003a9b38d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5842,10 +5842,10 @@ googleapis-common@^4.4.0: url-template "^2.0.8" uuid "^8.0.0" -googleapis@59.0.0: - version "59.0.0" - resolved "https://registry.yarnpkg.com/googleapis/-/googleapis-59.0.0.tgz#aec5c6f5b0b347c9c6996d45fa8ad8176e585fae" - integrity sha512-GV/E4KRN89a4GxSk7D7cwUfRYgcJHR05sOgm/WGdwc/u8dxNXG5lWmz9gF5ZwFGk2yKtVxL4VZNn4zBuZ6rmGg== +googleapis@60.0.1: + version "60.0.1" + resolved "https://registry.yarnpkg.com/googleapis/-/googleapis-60.0.1.tgz#f0614cbf798e1308ef6e7dd6b7cbf3c464692975" + integrity sha512-r/7pDeIMAm/FepbAl2iM+uRQURf2manjOaOVR0j7v2fDPU8QEme44z6N6qGroqJtne0JO5BM0FUz4LuskcwGeQ== dependencies: google-auth-library "^6.0.0" googleapis-common "^4.4.0" From 7ee088f3582d812c1831e17c95840509d758c83a Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Thu, 24 Sep 2020 15:26:45 +0000 Subject: [PATCH 03/44] chore(deps-dev): bump @types/cheerio from 0.22.21 to 0.22.22 Bumps [@types/cheerio](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/cheerio) from 0.22.21 to 0.22.22. - [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases) - [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/cheerio) Signed-off-by: dependabot-preview[bot] --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index c7058b02ad..e4bb284724 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ }, "homepage": "https://github.com/DIYgod/RSSHub#readme", "devDependencies": { - "@types/cheerio": "0.22.21", + "@types/cheerio": "0.22.22", "@types/got": "9.6.11", "@types/koa": "2.11.4", "@vuepress/plugin-back-to-top": "1.5.4", diff --git a/yarn.lock b/yarn.lock index e003a9b38d..3362f45201 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1525,10 +1525,10 @@ "@types/node" "*" "@types/responselike" "*" -"@types/cheerio@0.22.21": - version "0.22.21" - resolved "https://registry.yarnpkg.com/@types/cheerio/-/cheerio-0.22.21.tgz#5e37887de309ba11b2e19a6e14cad7874b31a8a3" - integrity sha512-aGI3DfswwqgKPiEOTaiHV2ZPC9KEhprpgEbJnv0fZl3SGX0cGgEva1126dGrMC6AJM6v/aihlUgJn9M5DbDZ/Q== +"@types/cheerio@0.22.22": + version "0.22.22" + resolved "https://registry.yarnpkg.com/@types/cheerio/-/cheerio-0.22.22.tgz#ae71cf4ca59b8bbaf34c99af7a5d6c8894988f5f" + integrity sha512-05DYX4zU96IBfZFY+t3Mh88nlwSMtmmzSYaQkKN48T495VV1dkHSah6qYyDTN5ngaS0i0VonH37m+RuzSM0YiA== dependencies: "@types/node" "*" From 752ee212a917b0d5d2e9ba9a22111c2c50df6a09 Mon Sep 17 00:00:00 2001 From: DIYgod Date: Fri, 25 Sep 2020 01:30:52 +0800 Subject: [PATCH 04/44] feat: support opencc --- lib/middleware/parameter.js | 11 +++++++++++ package.json | 1 + yarn.lock | 13 +++++++++++++ 3 files changed, 25 insertions(+) diff --git a/lib/middleware/parameter.js b/lib/middleware/parameter.js index c37037f045..3d3be62615 100644 --- a/lib/middleware/parameter.js +++ b/lib/middleware/parameter.js @@ -1,6 +1,7 @@ const he = require('he'); const mercury_parser = require('@postlight/mercury-parser'); const cheerio = require('cheerio'); +const OpenCC = require('opencc'); const got = require('@/utils/got'); const config = require('@/config').value; @@ -224,6 +225,16 @@ module.exports = async (ctx, next) => { return item; }); } + + // opencc + if (ctx.query.opencc) { + const converter = new OpenCC(`${ctx.query.opencc}.json`); + const tasks = ctx.state.data.item.map(async (item) => { + item.title = await converter.convertPromise(item.title); + item.description = await converter.convertPromise(item.description); + }); + await Promise.all(tasks); + } } } }; diff --git a/package.json b/package.json index e4bb284724..220ad65997 100644 --- a/package.json +++ b/package.json @@ -99,6 +99,7 @@ "mailparser": "3.0.0", "markdown-it": "11.0.1", "module-alias": "2.2.2", + "opencc": "1.1.1", "parse-torrent": "7.1.3", "pidusage": "2.0.21", "plist": "3.0.1", diff --git a/yarn.lock b/yarn.lock index 3362f45201..43679d28dd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8564,6 +8564,11 @@ nan@^2.12.1, nan@^2.14.0: resolved "https://registry.yarnpkg.com/nan/-/nan-2.14.0.tgz#7818f722027b2459a86f0295d434d1fc2336c52c" integrity sha512-INOFj37C7k3AfaNTtX8RhsTw7qRy7eLET14cROi9+5HAVbbHuIWUHEauBv5qT4Av2tWasiTY1Jw6puUNqRJXQg== +nan@^2.14.1: + version "2.14.1" + resolved "https://registry.yarnpkg.com/nan/-/nan-2.14.1.tgz#d7be34dfa3105b91494c3147089315eff8874b01" + integrity sha512-isWHgVjnFjh2x2yuJ/tj3JbwoHu3UC2dX5G/88Cm24yB6YopVgxvBObDY7n5xW6ExmFhJpSEQqFPvq9zaXc8Jw== + nanomatch@^1.2.9: version "1.2.13" resolved "https://registry.yarnpkg.com/nanomatch/-/nanomatch-1.2.13.tgz#b87a8aa4fc0de8fe6be88895b38983ff265bd119" @@ -9022,6 +9027,14 @@ only@~0.0.2: resolved "https://registry.yarnpkg.com/only/-/only-0.0.2.tgz#2afde84d03e50b9a8edc444e30610a70295edfb4" integrity sha1-Kv3oTQPlC5qO3EROMGEKcCle37Q= +opencc@1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/opencc/-/opencc-1.1.1.tgz#94776e515ef13b23e00065caa04a69e9529d460e" + integrity sha512-t/4FIL43esdrUDfQejqHPhZyWdDVRfwMxsNBkA/2NjIEQv33mG80WHXcmEJ+KMd6T8bjbtx+EiJzPiybE8SPeg== + dependencies: + nan "^2.14.1" + node-pre-gyp "^0.14.0" + opencollective-postinstall@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/opencollective-postinstall/-/opencollective-postinstall-2.0.2.tgz#5657f1bede69b6e33a45939b061eb53d3c6c3a89" From 52398423e9c288c96981fa862d5a8da0ea11fb15 Mon Sep 17 00:00:00 2001 From: DIYgod Date: Fri, 25 Sep 2020 01:37:06 +0800 Subject: [PATCH 05/44] docs: opencc --- docs/en/parameter.md | 8 +++++++- docs/parameter.md | 6 ++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/en/parameter.md b/docs/en/parameter.md index 2407c06362..24d629963b 100644 --- a/docs/en/parameter.md +++ b/docs/en/parameter.md @@ -74,7 +74,13 @@ Output Sci-hub link in scientific journal routes, this supports major journals o - scihub: set to any value -举例: +Eg: + +## Conversion between Traditional and Simplified Chinese + +- opencc: s2t (Simplified Chinese to Traditional Chinese)、t2s (Traditional Chinese to Simplified Chinese), other optional values refer to [OpenCC - Configurations](https://github.com/BYVoid/OpenCC#configurations-%E9%85%8D%E7%BD%AE%E6%96%87%E4%BB%B6) + +Eg: ## Output Formats diff --git a/docs/parameter.md b/docs/parameter.md index 0288126e17..6fb7dc407b 100644 --- a/docs/parameter.md +++ b/docs/parameter.md @@ -77,6 +77,12 @@ Telegram 即时预览模式需要在官网制作页面处理模板，请前往[ 举例: +## 中文简繁体转换 + +- opencc: s2t 简体转繁体、t2s 繁体转简体，其它可选值见 [OpenCC - Configurations](https://github.com/BYVoid/OpenCC#configurations-%E9%85%8D%E7%BD%AE%E6%96%87%E4%BB%B6) + +举例: + ## 输出格式 RSSHub 同时支持 RSS 2.0 和 Atom 输出格式，在路由末尾添加 `.rss` 或 `.atom` 即可请求对应输出格式，缺省为 RSS 2.0 From ed4725fa489ed90b1c7dc2ca2805c0ef91f9ce30 Mon Sep 17 00:00:00 2001 From: DIYgod Date: Fri, 25 Sep 2020 01:43:59 +0800 Subject: [PATCH 06/44] test: opencc --- lib/routes/test/index.js | 8 ++++++++ package.json | 2 +- test/middleware/parameter.js | 9 +++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/lib/routes/test/index.js b/lib/routes/test/index.js index cac465b99d..0c76ab6b0b 100644 --- a/lib/routes/test/index.js +++ b/lib/routes/test/index.js @@ -126,6 +126,14 @@ module.exports = async (ctx) => { pubDate: 1546272000000, author: `DIYgod0`, }); + } else if (ctx.params.id === 'opencc') { + item.push({ + title: '小可愛', + description: '宇宙無敵', + link: `/DIYgod/RSSHub/issues/0`, + pubDate: new Date(1546272000000).toUTCString(), + author: `DIYgod0`, + }); } for (let i = 1; i < 6; i++) { diff --git a/package.json b/package.json index 220ad65997..ef1b1aea13 100644 --- a/package.json +++ b/package.json @@ -117,7 +117,7 @@ }, "jest": { "testMatch": [ - "**/test/**/*.js" + "**/test/**/parameter.js" ], "testPathIgnorePatterns": [ "/node_modules/", diff --git a/test/middleware/parameter.js b/test/middleware/parameter.js index c9563198e1..f3fe9dcfaa 100644 --- a/test/middleware/parameter.js +++ b/test/middleware/parameter.js @@ -301,3 +301,12 @@ describe('mess parameter', () => { expect(parsed.items[0].link).toBe('https://github.com/DIYgod/RSSHub/issues/0'); }); }); + +describe('opencc', () => { + it(`opencc`, async () => { + const response = await request.get('/test/opencc?opencc=t2s'); + const parsed = await parser.parseString(response.text); + expect(parsed.items[0].title).toBe('小可爱'); + expect(parsed.items[0].content).toBe('宇宙无敌'); + }); +}); From f2b4bfbff82728712833e52f20f697d95e747a18 Mon Sep 17 00:00:00 2001 From: DIYgod Date: Fri, 25 Sep 2020 01:51:38 +0800 Subject: [PATCH 07/44] fix: add python in Dockerfile to fix opencc installation --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 8f8c30ab1a..cbb4e15099 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ ARG PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=1; RUN ln -sf /bin/bash /bin/sh -RUN apt-get update && apt-get install -yq libgconf-2-4 apt-transport-https git dumb-init --no-install-recommends && apt-get clean \ +RUN apt-get update && apt-get install -yq libgconf-2-4 apt-transport-https git dumb-init python --no-install-recommends && apt-get clean \ && rm -rf /var/lib/apt/lists/* WORKDIR /app From c2a6286786520daa7210dc04942e444fcbf32443 Mon Sep 17 00:00:00 2001 From: DIYgod Date: Fri, 25 Sep 2020 01:56:14 +0800 Subject: [PATCH 08/44] fix: add make in Dockerfile to fix opencc installation --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index cbb4e15099..12ca20638a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ ARG PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=1; RUN ln -sf /bin/bash /bin/sh -RUN apt-get update && apt-get install -yq libgconf-2-4 apt-transport-https git dumb-init python --no-install-recommends && apt-get clean \ +RUN apt-get update && apt-get install -yq libgconf-2-4 apt-transport-https git dumb-init python make --no-install-recommends && apt-get clean \ && rm -rf /var/lib/apt/lists/* WORKDIR /app From 03d6e7f04e1a4fc891972a0705d4c94f38596182 Mon Sep 17 00:00:00 2001 From: DIYgod Date: Fri, 25 Sep 2020 02:01:13 +0800 Subject: [PATCH 09/44] fix: add g++ in Dockerfile to fix opencc installation --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 12ca20638a..df961ca513 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ ARG PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=1; RUN ln -sf /bin/bash /bin/sh -RUN apt-get update && apt-get install -yq libgconf-2-4 apt-transport-https git dumb-init python make --no-install-recommends && apt-get clean \ +RUN apt-get update && apt-get install -yq libgconf-2-4 apt-transport-https git dumb-init python make g++ build-essential --no-install-recommends && apt-get clean \ && rm -rf /var/lib/apt/lists/* WORKDIR /app From 7e777bbb56f9d4b0cc759c616cb3460b56341b1f Mon Sep 17 00:00:00 2001 From: DIYgod Date: Fri, 25 Sep 2020 12:01:01 +0800 Subject: [PATCH 10/44] feat: youtube key retry --- lib/routes/youtube/utils.js | 73 ++++++++++++++++++++++++------------- 1 file changed, 47 insertions(+), 26 deletions(-) diff --git a/lib/routes/youtube/utils.js b/lib/routes/youtube/utils.js index c796ae8c76..e1bfe5b46a 100644 --- a/lib/routes/youtube/utils.js +++ b/lib/routes/youtube/utils.js @@ -1,12 +1,10 @@ const { google } = require('googleapis'); const config = require('@/config').value; -let getYoutube = () => null; +let count = 0; +const youtube = {}; if (config.youtube.key) { const keys = config.youtube.key.split(','); - const youtube = {}; - let count = 0; - let index = -1; keys.forEach((key, index) => { if (key) { @@ -17,44 +15,67 @@ if (config.youtube.key) { count = index + 1; } }); - - getYoutube = () => { - index++; - return youtube[index % count]; - }; } +let index = -1; +const exec = async (func) => { + let result; + for (let i = 0; i < count; i++) { + index++; + try { + // eslint-disable-next-line no-await-in-loop + result = await func(youtube[index % count]); + break; + } catch (error) { + // console.error(error); + } + } + return result; +}; + const youtubeUtils = { getPlaylistItems: async (id, part) => { - const res = await getYoutube().playlistItems.list({ - part, - playlistId: id, - maxResults: 50, // youtube api param value default is 5 - }); + const res = await exec( + async (youtube) => + await youtube.playlistItems.list({ + part, + playlistId: id, + maxResults: 50, // youtube api param value default is 5 + }) + ); return res; }, getPlaylist: async (id, part, cache) => await cache.tryGet('getPlaylist' + id, async () => { - const res = await getYoutube().playlists.list({ - part, - id: id, - }); + const res = await exec( + async (youtube) => + await youtube.playlists.list({ + part, + id: id, + }) + ); return res; }), getChannelWithId: async (id, part, cache) => await cache.tryGet('getChannelWithId' + id, async () => { - const res = await getYoutube().channels.list({ - part, - id: id, - }); + const res = await exec( + async (youtube) => + await youtube.channels.list({ + part, + id: id, + }) + ); return res; }), getChannelWithUsername: async (username, part, cache) => await cache.tryGet('getPlaylist' + username, async () => { - const res = await getYoutube().channels.list({ - part, - forUsername: username, - }); + const res = await exec( + async (youtube) => + await youtube.channels.list({ + part, + forUsername: username, + }) + ); return res; }), getThumbnail: (thumbnails) => thumbnails.maxres || thumbnails.standard || thumbnails.high || thumbnails.medium || thumbnails.default, From c50145d2bdeebb073596c91ffe629d12a5610152 Mon Sep 17 00:00:00 2001 From: DIYgod Date: Fri, 25 Sep 2020 12:52:11 +0800 Subject: [PATCH 11/44] fix: smzdm ranking pubDate --- lib/routes/smzdm/ranking.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/routes/smzdm/ranking.js b/lib/routes/smzdm/ranking.js index 5a9985db58..e66f54c777 100644 --- a/lib/routes/smzdm/ranking.js +++ b/lib/routes/smzdm/ranking.js @@ -1,4 +1,5 @@ const got = require('@/utils/got'); +const date = require('@/utils/date'); module.exports = async (ctx) => { const { rank_type, rank_id, hour } = ctx.params; @@ -31,7 +32,7 @@ module.exports = async (ctx) => { item: list.map((item) => ({ title: `${item.article_title} - ${item.article_price}`, description: `${item.article_title} - ${item.article_price}
`, - pubDate: new Date(item.article_pubdate).toUTCString(), + pubDate: date(item.article_pubdate, 8), link: item.article_url, })), }; From b611728f7fa7b2fe7205f97cca9ab70ac20b961e Mon Sep 17 00:00:00 2001 From: sgqy Date: Fri, 25 Sep 2020 18:35:10 +0800 Subject: [PATCH 12/44] fix: bbc chinese changed it's structure (#5697) --- lib/routes/bbc/utils.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/routes/bbc/utils.js b/lib/routes/bbc/utils.js index ad669dc1c2..8b3122d7d7 100644 --- a/lib/routes/bbc/utils.js +++ b/lib/routes/bbc/utils.js @@ -76,6 +76,11 @@ const ProcessFeed = ($, link) => { content = $('div.story-body'); } + if (content.length === 0) { + // chinese version has different structure + content = $('main[role="main"]'); + } + // remove useless DOMs content.find('div.bbccom_slot, .off-screen, .embed-report-link').each((i, e) => { $(e).remove(); From 25fb166ee480e225165cf7378fb6db13c42871ee Mon Sep 17 00:00:00 2001 From: Search <60404136+searchableguy@users.noreply.github.com> Date: Fri, 25 Sep 2020 16:06:13 +0530 Subject: [PATCH 13/44] fix: add missing author field in github and gitlab routes. (#5698) --- lib/routes/github/follower.js | 1 + lib/routes/github/pulls.js | 1 + lib/routes/github/search.js | 1 + lib/routes/github/star.js | 1 + lib/routes/github/starred_repos.js | 1 + lib/routes/github/trending.js | 1 + lib/routes/gitlab/explore.js | 1 + 7 files changed, 7 insertions(+) diff --git a/lib/routes/github/follower.js b/lib/routes/github/follower.js index c87869cdf2..2121d3d915 100644 --- a/lib/routes/github/follower.js +++ b/lib/routes/github/follower.js @@ -42,6 +42,7 @@ module.exports = async (ctx) => { link: host, item: data.map((follower) => ({ title: `${follower.node.login} started following ${user}`, + author: follower.node.login, description: `${follower.node.login}
`, link: `https://github.com/${follower.node.login}`, })), diff --git a/lib/routes/github/pulls.js b/lib/routes/github/pulls.js index b711b54700..b97b745309 100644 --- a/lib/routes/github/pulls.js +++ b/lib/routes/github/pulls.js @@ -33,6 +33,7 @@ module.exports = async (ctx) => { link: host, item: data.map((item) => ({ title: item.title, + author: item.user.login, description: item.body ? md.render(item.body) : 'No description', pubDate: new Date(item.created_at).toUTCString(), link: `${link}/${item.number}`, diff --git a/lib/routes/github/search.js b/lib/routes/github/search.js index 9c5e861e59..8ed783bdd1 100644 --- a/lib/routes/github/search.js +++ b/lib/routes/github/search.js @@ -24,6 +24,7 @@ module.exports = async (ctx) => { const a = $(this).find('.f4.text-normal > a'); const single = { title: a.text(), + author: a.text().split('/')[0].trim(), link: host.concat(a.attr('href')), description: $(this).find('div p').text().trim(), }; diff --git a/lib/routes/github/star.js b/lib/routes/github/star.js index 1cb97c44dd..d44ea03bd3 100644 --- a/lib/routes/github/star.js +++ b/lib/routes/github/star.js @@ -43,6 +43,7 @@ module.exports = async (ctx) => { link: host, item: data.map((follower) => ({ title: `${follower.node.login} starred ${user}/${repo}`, + author: follower.node.login, description: `${follower.node.login}
`, link: `https://github.com/${follower.node.login}`, })), diff --git a/lib/routes/github/starred_repos.js b/lib/routes/github/starred_repos.js index 13ee4c2919..ffd1a6a32f 100644 --- a/lib/routes/github/starred_repos.js +++ b/lib/routes/github/starred_repos.js @@ -52,6 +52,7 @@ module.exports = async (ctx) => { description: `${user}’s starred repositories`, item: data.map((repo) => ({ title: `${user} starred ${repo.node.name}`, + author: user, description: `${repo.node.description === null ? 'no description' : repo.node.description}
primary language: ${ repo.node.primaryLanguage === null ? 'no primary language' : repo.node.primaryLanguage.name }
stargazers: ${repo.node.stargazers.totalCount}
`, diff --git a/lib/routes/github/trending.js b/lib/routes/github/trending.js index 9a44def32d..a2518e2302 100644 --- a/lib/routes/github/trending.js +++ b/lib/routes/github/trending.js @@ -29,6 +29,7 @@ module.exports = async (ctx) => { item = $(item); return { title: item.find('h1').text(), + author: item.find('h1').text().split('/')[0].trim(), description: `${item.find('.pr-4').text()}

Language: ${item.find('span[itemprop="programmingLanguage"]').text() || 'unknown'}
Star: ${item.find('.muted-link').eq(0).text()} diff --git a/lib/routes/gitlab/explore.js b/lib/routes/gitlab/explore.js index ca7d43f79e..6feb623481 100644 --- a/lib/routes/gitlab/explore.js +++ b/lib/routes/gitlab/explore.js @@ -26,6 +26,7 @@ module.exports = async (ctx) => { item = $(item); return { title: item.find('.project-full-name').text(), + author: item.find('.namespace-name').text().slice(0, -1).split('\n').join('').trim(), description: item.find('.description').text(), link: `https://gitlab.com${item.find('a.text-plain').attr('href')}`, }; From 8b25d3aec8ab44e187df1a6d0d6630b3dddf705f Mon Sep 17 00:00:00 2001 From: Ethan Shen Date: Fri, 25 Sep 2020 18:37:11 +0800 Subject: [PATCH 14/44] =?UTF-8?q?feat:=20add=20=E5=A4=AE=E8=A7=86=E7=BD=91?= =?UTF-8?q?=E6=96=B0=E9=97=BB=E4=B8=93=E9=A2=98=20(#5700)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/traditional-media.md | 14 +++++ lib/router.js | 1 + lib/routes/cctv/special.js | 109 +++++++++++++++++++++++++++++++++++++ 3 files changed, 124 insertions(+) create mode 100644 lib/routes/cctv/special.js diff --git a/docs/traditional-media.md b/docs/traditional-media.md index f352820deb..fc91b291fd 100644 --- a/docs/traditional-media.md +++ b/docs/traditional-media.md @@ -625,6 +625,20 @@ category 对应的关键词有 +### 新闻专题 + + + +::: tip 提示 + +如 [2020 年国家网络安全宣传周](https://news.cctv.com/special/2020gjwlaqxcz/index.shtml) 的专题页 URL 为 id 即为 `2020gjwlaqxcz`。 + +::: + +此处查看 [所有新闻专题](http://news.cctv.com/special/index.shtml) + + + ### 专题 diff --git a/lib/router.js b/lib/router.js index df766a0539..71ebedf685 100644 --- a/lib/router.js +++ b/lib/router.js @@ -363,6 +363,7 @@ router.get('/mihoyo/bh2/:type', require('./routes/mihoyo/bh2')); router.get('/cctv/xwlb', require('./routes/cctv/xwlb')); // 央视新闻 router.get('/cctv/:category', require('./routes/cctv/category')); +router.get('/cctv-special/:id?', require('./routes/cctv/special')); // 财新博客 router.get('/caixin/blog/:column', require('./routes/caixin/blog')); diff --git a/lib/routes/cctv/special.js b/lib/routes/cctv/special.js new file mode 100644 index 0000000000..1abde6003b --- /dev/null +++ b/lib/routes/cctv/special.js @@ -0,0 +1,109 @@ +const got = require('@/utils/got'); +const cheerio = require('cheerio'); + +module.exports = async (ctx) => { + ctx.params.id = ctx.params.id || '315'; + + let rootUrl; + + if (ctx.params.id === '315') { + rootUrl = `https://315.cctv.com`; + } else { + rootUrl = `https://news.cctv.com/special/${ctx.params.id}/index.shtml`; + } + + const response = await got({ + method: 'get', + url: rootUrl, + }); + + const $ = cheerio.load(response.data); + const list = $('h3>a, h4>a') + .slice(0, 10) + .map((_, item) => { + item = $(item); + return { + title: item.text(), + link: item.attr('href'), + }; + }) + .get(); + + const items = await Promise.all( + list.map( + async (item) => + await ctx.cache.tryGet(item.link, async () => { + let date; + + if (item.link.indexOf('photo.cctv.com') > 0) { + const imageLink = item.link.split('.shtml')[0] + '.xml?randomNum=26'; + const imageResponse = await got({ + method: 'get', + url: imageLink, + }); + const images = cheerio.load(imageResponse.data); + + let description; + + item.description = ''; + + images('li') + .get() + .forEach((i) => { + i = images(i); + item.description += ``; + date = i.attr('time').replace(/年/g, '-').replace(/月/g, '-').replace(/日/g, ''); + description = i.html(); + }); + + item.description += `

${description}

`; + } else { + const detailResponse = await got({ + method: 'get', + url: item.link, + }); + const content = cheerio.load(detailResponse.data); + const match = detailResponse.data.match(/publishDate ="(.*)";/); + + if (match === null) { + let dateSpan; + + if (content('div.info span').html() !== null) { + dateSpan = content('div.info span').text(); + } else if (content('div.info1').html() !== null) { + dateSpan = content('div.info1').text().split(' | ')[1]; + } else { + dateSpan = content('span.time').text(); + } + + date = dateSpan.replace(/年/g, '-').replace(/月/g, '-').replace(/日/g, ''); + } else { + const dateString = match[1]; + date = `${dateString.slice(0, 4)}-${dateString.slice(4, 6)}-${dateString.slice(6, 8)} ${dateString.slice(8, 10)}:${dateString.slice(10, 12)}:${dateString.slice(12, 14)}`; + } + + if (content('#content_area').html() !== null) { + item.description = content('#content_area').html(); + } else if (content('div.cnt_bd').html() !== null) { + content('div.function,h1,h2').remove(); + item.description = content('div.cnt_bd').html(); + } else if (content('#text_area').html() !== null) { + item.description = content('#text_area').html(); + } else { + item.description = content('div.con').html(); + } + } + + item.pubDate = new Date(date + ' GMT+8').toUTCString(); + + return item; + }) + ) + ); + + ctx.state.data = { + title: `${$('title').text().split('_')[0]} - 新闻专题 - 央视网`, + link: rootUrl, + item: items, + }; +}; From 3e623ac0d86a8f4dd3a0ad6081e0e8f40557f7ff Mon Sep 17 00:00:00 2001 From: Dreace <33168165+Dreace@users.noreply.github.com> Date: Fri, 25 Sep 2020 18:39:21 +0800 Subject: [PATCH 15/44] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E6=BE=8E?= =?UTF-8?q?=E6=B9=83=E6=96=B0=E9=97=BB=E6=8A=93=E5=8F=96=E5=BC=82=E5=B8=B8?= =?UTF-8?q?=20(#5243)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fix #5711 --- lib/routes/juejin/utils.js | 8 ++++---- lib/routes/thepaper/utils.js | 12 ++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/routes/juejin/utils.js b/lib/routes/juejin/utils.js index 0fad946bac..e21b22f23a 100644 --- a/lib/routes/juejin/utils.js +++ b/lib/routes/juejin/utils.js @@ -31,14 +31,14 @@ const ProcessFeed = async (list, caches) => await Promise.all( list.map(async (item) => { // 解析时间，补偿时区offset - const pubdate = new Date(parseInt(item.article_info.ctime) * 1000 - 8 * 60 * 60 * 1000); - const link = `https://juejin.im/post/${item.article_id}`; + const pubdate = new Date(parseInt(item.item_info.ctime || item.item_info.article_info.ctime) * 1000 - 8 * 60 * 60 * 1000); + const link = item.item_info.url || `https://juejin.im/post/${item.item_info.article_info.article_id}`; // 列表上提取到的信息 const single = { - title: item.article_info.title, + title: item.item_info.title || item.item_info.article_info.title, description: `${(item.content || item.summaryInfo || '无描述').replace(/[\x00-\x08\x0b-\x0c\x0e-\x1f\x7f]/g, '')}`, pubDate: pubdate.toUTCString(), - author: item.author_user_info.user_name, + author: item.item_info.author_name || item.item_info.author_user_info.user_name, link, }; diff --git a/lib/routes/thepaper/utils.js b/lib/routes/thepaper/utils.js index 5b991a3899..9d76f24647 100644 --- a/lib/routes/thepaper/utils.js +++ b/lib/routes/thepaper/utils.js @@ -7,12 +7,12 @@ module.exports = { const res = await got.get(link); const $ = cheerio.load(res.data); - const list = $('.t_news').slice(0, 10).get(); + const list = $('.list_item').slice(0, 10).get(); return await Promise.all( list.map(async (item) => { const $ = cheerio.load(item); - const itemUrlID = $(item).find('.news_tit02 a').attr('href'); + const itemUrlID = $(item).find('.list_item_title a').attr('href'); const itemUrl = `https://m.thepaper.cn/${itemUrlID}`; const cache = await ctx.cache.get(itemUrl); if (cache) { @@ -25,11 +25,11 @@ module.exports = { content('.contheight').replaceWith('
'); const single = { - title: content('h1.t_newsinfo').text(), + title: content('.news_video_name').text() || content('.title').text(), link: itemUrl, - description: $(content('.news_part_limit div')[0]).html(), - pubDate: date($(content('.about_news')[1]).text().trim()), - author: $(content('.about_news')[0]).text(), + description: content('#vdetail_sum_p').html() || content('.news_part_limit div').html(), + pubDate: date(content('.date').text().trim()), + author: content('.author').text(), }; return Promise.resolve(single); }) From a6ad432de966a5ad49a8227145ec48d850f41006 Mon Sep 17 00:00:00 2001 From: Ethan Shen Date: Fri, 25 Sep 2020 18:39:57 +0800 Subject: [PATCH 16/44] =?UTF-8?q?feat:=20add=20=E4=B8=AD=E8=8F=AF=E6=B0=91?= =?UTF-8?q?=E5=9C=8B=E5=9C=8B=E9=98=B2=E9=83=A8=E5=8D=B3=E6=99=82=E8=BB=8D?= =?UTF-8?q?=E4=BA=8B=E5=8B=95=E6=85=8B=20(#5704)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/government.md | 6 ++++ lib/router.js | 3 ++ lib/routes/gov/taiwan/mnd.js | 64 ++++++++++++++++++++++++++++++++++++ 3 files changed, 73 insertions(+) create mode 100644 lib/routes/gov/taiwan/mnd.js diff --git a/docs/government.md b/docs/government.md index 206cd3aadd..cd973be0ad 100644 --- a/docs/government.md +++ b/docs/government.md @@ -412,6 +412,12 @@ pageClass: routes | 贝尔法斯特 | `/embassy/uk/belfast` | | 曼彻斯特 | `/embassy/uk/manchester` | +## 中華民國國防部 + +### 即時軍事動態 + + + ## 中华人民共和国海关总署 ### 拍卖信息 / 海关法规 diff --git a/lib/router.js b/lib/router.js index 71ebedf685..9a132883a5 100644 --- a/lib/router.js +++ b/lib/router.js @@ -3237,4 +3237,7 @@ router.get('/appsales/:caty?/:time?', require('./routes/appsales/index')); // CGTN router.get('/cgtn/top', require('./routes/cgtn/top')); +// 中華民國國防部 +router.get('/gov/taiwan/mnd', require('./routes/gov/taiwan/mnd')); + module.exports = router; diff --git a/lib/routes/gov/taiwan/mnd.js b/lib/routes/gov/taiwan/mnd.js new file mode 100644 index 0000000000..5bb84abcfe --- /dev/null +++ b/lib/routes/gov/taiwan/mnd.js @@ -0,0 +1,64 @@ +const got = require('@/utils/got'); +const cheerio = require('cheerio'); + +module.exports = async (ctx) => { + const rootUrl = 'https://www.mnd.gov.tw'; + const currentUrl = `${rootUrl}/PublishTable.aspx?Types=即時軍事動態&title=國防消息`; + + const response = await got({ + method: 'get', + url: currentUrl, + }); + const $ = cheerio.load(response.data); + + const list = $('table.newstitles tbody tr') + .map((_, item) => { + item = $(item); + const a = item.find('a'); + const dateSplit = item.find('td.w-10').text().split('/'); + dateSplit[0] = (parseInt(dateSplit[0]) + 1911).toString(); + + return { + title: a.text(), + link: a.attr('id').replace(/_/g, '$'), + pubDate: new Date(dateSplit.join('-') + ' GMT+8').toUTCString(), + }; + }) + .get(); + + const items = await Promise.all( + list.map( + async (item) => + await ctx.cache.tryGet(item.link, async () => { + const detailResponse = await got({ + method: 'post', + url: currentUrl, + headers: { + Referer: 'https://www.mnd.gov.tw/', + }, + form: { + __EVENTTARGET: item.link, + __EVENTARGUMENT: '', + __VIEWSTATE: response.data.match(/id="__VIEWSTATE" value="(.*)" \/>/)[1], + __VIEWSTATEGENERATOR: response.data.match(/id="__VIEWSTATEGENERATOR" value="(.*)" \/>/)[1], + __EVENTVALIDATION: response.data.match(/id="__EVENTVALIDATION" value="(.*)" \/>/)[1], + TbSearch2: '', + TbSearch: '', + }, + }); + const content = cheerio.load(detailResponse.data); + + item.description = content('div.thisPages').html(); + item.link = content('form[name="aspnetForm"]').attr('action').replace('.', `${rootUrl}`); + + return item; + }) + ) + ); + + ctx.state.data = { + title: '即時軍事動態 - 中華民國國防部', + link: currentUrl, + item: items, + }; +}; From 5f963d85e64f9fe128295cda555bc28d752195a6 Mon Sep 17 00:00:00 2001 From: Ethan Shen Date: Fri, 25 Sep 2020 18:40:17 +0800 Subject: [PATCH 17/44] =?UTF-8?q?feat:=20add=20=E5=89=91=E7=BD=913?= =?UTF-8?q?=E6=96=B0=E9=97=BB=E8=B5=84=E8=AE=AF=20(#5693)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Henry Wang --- docs/game.md | 12 +++++++++ lib/router.js | 3 +++ lib/routes/jx3/news.js | 59 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+) create mode 100644 lib/routes/jx3/news.js diff --git a/docs/game.md b/docs/game.md index 7314c8d2ef..35a1e28275 100644 --- a/docs/game.md +++ b/docs/game.md @@ -298,6 +298,18 @@ Example: `https://store.steampowered.com/search/?specials=1&term=atelier` 中的 +## 剑网 3 + +### 新闻资讯 + + + +| 最新 | 新闻 | 活动 | 公告 | +| ------- | ----- | ---- | -------- | +| allnews | press | hd | announce | + + + ## 旅法师营地 ### 旅法师营地 diff --git a/lib/router.js b/lib/router.js index 9a132883a5..318892d625 100644 --- a/lib/router.js +++ b/lib/router.js @@ -3219,6 +3219,9 @@ router.get('/feixuew/:id?', require('./routes/feixuew/index')); // 1X router.get('/1x/:type?/:caty?', require('./routes/1x/index')); +// 剑网3 +router.get('/jx3/:caty?', require('./routes/jx3/news')); + // GQ router.get('/gq/tw/:caty?/:subcaty?', require('./routes/gq/tw/index')); diff --git a/lib/routes/jx3/news.js b/lib/routes/jx3/news.js new file mode 100644 index 0000000000..cbeebe9b56 --- /dev/null +++ b/lib/routes/jx3/news.js @@ -0,0 +1,59 @@ +const url = require('url'); +const got = require('@/utils/got'); +const cheerio = require('cheerio'); + +module.exports = async (ctx) => { + ctx.params.caty = ctx.params.caty || 'allnews'; + + let query; + + if (ctx.params.caty === 'hd') { + query = 'div.article_list article div h3 a'; + } else { + query = 'div.article_list ul li a'; + } + + const rootUrl = 'https://jx3.xoyo.com'; + const currentUrl = `${rootUrl}/${ctx.params.caty}`; + const response = await got({ + method: 'get', + url: currentUrl, + }); + + const $ = cheerio.load(response.data); + + const list = $(query) + .slice(0, 10) + .map((_, item) => { + item = $(item); + return { + title: item.text(), + link: url.resolve(rootUrl, item.attr('href')), + }; + }) + .get(); + + const items = await Promise.all( + list.map( + async (item) => + await ctx.cache.tryGet(item.link, async () => { + const detailResponse = await got({ + method: 'get', + url: item.link, + }); + const content = cheerio.load(detailResponse.data); + + item.description = content('div.detail_con').html(); + item.pubDate = new Date(content('p.detail_time').text() + ' GMT+8').toUTCString(); + + return item; + }) + ) + ); + + ctx.state.data = { + title: `${$('ul.allnews_list_ul li.active').text()} - 剑网3`, + link: currentUrl, + item: items, + }; +}; From 513f942e4527c01fd9d38781438233e4328a5159 Mon Sep 17 00:00:00 2001 From: Ethan Shen Date: Fri, 25 Sep 2020 18:41:38 +0800 Subject: [PATCH 18/44] =?UTF-8?q?fix:=20=E8=99=8E=E5=97=85=E7=BD=91?= =?UTF-8?q?=E8=A1=A5=E5=85=85=E6=96=87=E7=AB=A0=E9=A2=98=E5=9B=BE=20(#5712?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/routes/huxiu/utils.js | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/routes/huxiu/utils.js b/lib/routes/huxiu/utils.js index bc871bc0d4..4d0af4ea4d 100644 --- a/lib/routes/huxiu/utils.js +++ b/lib/routes/huxiu/utils.js @@ -32,9 +32,19 @@ const ProcessFeed = async (list, cache) => { }); $('img.dialog_add_wxxy_qrcode_icon').remove(); + let description = ''; + + const videoMatch = response.data.match(/"hd_link":"(.*)","duration/); + if (videoMatch !== null) { + description += ``; + } + if ($('.top-img').html() !== null) { + description += $('.top-img').html(); + } + return { title: $('.article__title').text().trim(), - description: $('.article__top-img-video') + $('.article__content').html(), + description: description + $('.article__content').html(), pubDate: date($('.article__time').text(), 8), author: $('.author-info__username').text(), link, From 6b7f357e881ea0f17428a0c7a322eec1555fc1ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=93=BC=E5=93=88?= Date: Fri, 25 Sep 2020 18:42:13 +0800 Subject: [PATCH 19/44] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E5=9B=A0?= =?UTF-8?q?=E7=BD=91=E7=AB=99=E6=94=B9=E7=89=88=E9=80=A0=E6=88=90=E7=9A=84?= =?UTF-8?q?=E8=A5=BF=E7=94=B5=E6=95=99=E5=8A=A1=E5=A4=84=E6=8A=93=E5=8F=96?= =?UTF-8?q?=E9=94=99=E8=AF=AF=20(#5715)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: codefactor-io --- docs/university.md | 6 ++-- lib/routes/universities/xidian/jwc.js | 40 ++++++++++++++++++++------- 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/docs/university.md b/docs/university.md index 193aafc34b..f781c751f5 100644 --- a/docs/university.md +++ b/docs/university.md @@ -1547,9 +1547,9 @@ type 列表： 全文内容需使用校园网或 VPN 获取 ::: -| 全部 | 信息发布 | 通知公告 | 教务信息 | 教学研究 | 教学实践 | 招生信息 | 质量监控 | -| :--: | :------: | :------: | :------: | :------: | :------: | :------: | :------: | -| all | xxfb | tzgg | jwxx | jxyj | jxsj | zsxx | zljk | +| 全部 | 教学信息 | 教学研究 | 实践教学 | 质量监控 | 通知公告 | +| :---: | :------: | :------: | :------: | :------: | :------: | +| all | jxxx | jxyj | sjjx | zljk | tzgg | diff --git a/lib/routes/universities/xidian/jwc.js b/lib/routes/universities/xidian/jwc.js index 8041a99383..cacbf51619 100644 --- a/lib/routes/universities/xidian/jwc.js +++ b/lib/routes/universities/xidian/jwc.js @@ -7,38 +7,58 @@ module.exports = async (ctx) => { method: 'get', url: 'https://jwc.xidian.edu.cn', }); - const arrC = ['信息发布', '通知公告', '教务信息', '教学研究', '教学实践', '招生信息', '质量监控']; - const arrE = ['xxfb', 'tzgg', 'jwxx', 'jxyj', 'jxsj', 'zsxx', 'zljk']; + const arrC = ['教学信息', '教学研究', '实践教学', '质量监控', '通知公告']; + const arrE = ['jxxx', 'jxyj', 'sjjx', 'zljk', 'tzgg']; - const data = response.data; + const data = response.body; const $ = cheerio.load(data); - const list = $('#__01 > tbody > tr > td:nth-child(2)'); - const titleTables = list.children().filter((index) => index % 2 !== 1); + const leftItems = $('div .con2 .fl').children(); + const rightItems = $('div .con2 .fr').children(); + const mainItems = $('div .main_right').children(); + const list = { '0': leftItems[1], '1': leftItems[4], '2': rightItems[1], '3': rightItems[4], '4': mainItems[5] }; + const parseTable = (index, item) => { const title = arrC[index]; - const aTags = $(item).next().find('a'); + const aTags = $(item).find('a'); return aTags .map((_, item) => { item = $(item); + let date; + if (index === '0' || index === '1') { + date = new Date(item.find('span').text()).toUTCString(); + } else if (index === '4') { + const a = item.find('div p').text(); + const b = item.find('span').text(); + const c = b + '.' + a; + date = new Date(c).toUTCString(); + } else { + const a = item.find('div p').text(); + const b = item.find('span').text(); + const c = a + '.' + b; + date = new Date(c).toUTCString(); + } return { title: item.attr('title'), description: title + '
' + item.attr('title') + '
全文内容需使用校园网或VPN获取', - pubDate: new Date(item.parent().next().text()).toUTCString(), + pubDate: date, link: `https://jwc.xidian.edu.cn/${item.attr('href')}`, }; }) .get(); }; + const arrIndex = arrE.indexOf(category); - let result; + let result = new Array(); if (arrIndex === -1) { - result = titleTables.map(parseTable).get(); + for (const i in list) { + result = result.concat(parseTable(i, list[i])); + } result.sort((a, b) => new Date(b.pubDate) - new Date(a.pubDate)); } else { - result = parseTable(arrIndex, titleTables[arrIndex]); + result = parseTable(arrIndex, list[arrIndex]); } ctx.state.data = { From 10b975d73da5f5e03cb1ee5208b78ed65785a105 Mon Sep 17 00:00:00 2001 From: Ethan Shen Date: Fri, 25 Sep 2020 18:43:29 +0800 Subject: [PATCH 20/44] feat: add Nobel Prize lists (#5716) Co-authored-by: Henry Wang --- docs/en/other.md | 12 ++++++ docs/other.md | 12 ++++++ lib/router.js | 3 ++ lib/routes/nobelprize/index.js | 79 ++++++++++++++++++++++++++++++++++ 4 files changed, 106 insertions(+) create mode 100644 lib/routes/nobelprize/index.js diff --git a/docs/en/other.md b/docs/en/other.md index 5fd2fa50e5..5bc5e6a04e 100644 --- a/docs/en/other.md +++ b/docs/en/other.md @@ -117,6 +117,18 @@ Official Website: [https://www.ssm.gov.mo/apps1/PreventWuhanInfection/en.aspx](h +## Nobel Prize + +### List + + + +| Physics | Chemistry | Physiology or Medicine | Literature | Peace | Economic Science | +| ------- | --------- | ---------------------- | ---------- | ----- | ----------------- | +| physics | chemistry | physiology-or-medicine | literature | peace | economic-sciences | + + + ## Parcel Tracking ### Hermes UK diff --git a/docs/other.md b/docs/other.md index 6822829170..36f6b48ad4 100644 --- a/docs/other.md +++ b/docs/other.md @@ -521,6 +521,18 @@ type 为 all 时，category 参数不支持 cost 和 free +## 诺贝尔奖 + +### 获奖名单 + + + +| 物理学 | 化学 | 生理学或医学 | 文学 | 和平 | 经济学 | +| ------- | --------- | ---------------------- | ---------- | ----- | ----------------- | +| physics | chemistry | physiology-or-medicine | literature | peace | economic-sciences | + + + ## 且听风吟福利 ### 分类 diff --git a/lib/router.js b/lib/router.js index 318892d625..5a85fd5881 100644 --- a/lib/router.js +++ b/lib/router.js @@ -3240,6 +3240,9 @@ router.get('/appsales/:caty?/:time?', require('./routes/appsales/index')); // CGTN router.get('/cgtn/top', require('./routes/cgtn/top')); +// Nobel Prize +router.get('/nobelprize/:caty?', require('./routes/nobelprize/index')); + // 中華民國國防部 router.get('/gov/taiwan/mnd', require('./routes/gov/taiwan/mnd')); diff --git a/lib/routes/nobelprize/index.js b/lib/routes/nobelprize/index.js new file mode 100644 index 0000000000..c7a59b5066 --- /dev/null +++ b/lib/routes/nobelprize/index.js @@ -0,0 +1,79 @@ +const got = require('@/utils/got'); +const cheerio = require('cheerio'); + +const config = { + physics: { + link: 'all-nobel-prizes-in-physics', + title: 'Physics', + }, + chemistry: { + link: 'all-nobel-prizes-in-chemistry', + title: 'Chemistry', + }, + 'physiology-or-medicine': { + link: 'all-nobel-laureates-in-physiology-or-medicine', + title: 'Physiology or Medicine', + }, + literature: { + link: 'all-nobel-prizes-in-literature', + title: 'Literature', + }, + peace: { + link: 'all-nobel-peace-prizes', + title: 'Peace', + }, + 'economic-sciences': { + link: 'all-prizes-in-economic-sciences', + title: 'Economic Science', + }, +}; + +module.exports = async (ctx) => { + let rootUrl = 'https://www.nobelprize.org/prizes/lists/all-nobel-prizes/'; + + const cfg = config[ctx.params.caty]; + if (cfg) { + rootUrl = `https://www.nobelprize.org/prizes/lists/${cfg.link}/`; + } + + const response = await got({ + method: 'get', + url: rootUrl, + }); + + const $ = cheerio.load(response.data); + + const items = $('div.by_year h3') + .map((_, item) => { + item = $(item); + if (item.text().length === 4) { + return Promise.resolve(''); + } + + const a = item.find('a'); + + let description = ''; + + item.nextAll().each(function () { + if ($(this).get(0).tagName === 'p') { + description += `

${$(this).html()}

`; + return true; + } + return false; + }); + + return { + title: a.text(), + link: a.attr('href'), + pubDate: new Date(`${a.text().match(/([0-9]{4})/)[1]}-01-01 00:00:00 GMT+0`).toUTCString(), + description, + }; + }) + .get(); + + ctx.state.data = { + title: $('title').eq(0).text(), + link: rootUrl, + item: items, + }; +}; From 3f5fc5d7fab83414d4b67d0a0daa8359c267d4a4 Mon Sep 17 00:00:00 2001 From: Ethan Shen Date: Fri, 25 Sep 2020 18:46:27 +0800 Subject: [PATCH 21/44] =?UTF-8?q?feat:=20add=20=E5=8D=8E=E5=B0=94=E8=A1=97?= =?UTF-8?q?=E8=A7=81=E9=97=BB=E5=AE=9E=E6=97=B6=E5=BF=AB=E8=AE=AF=20(#5720?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/traditional-media.md | 10 ++++++++ lib/router.js | 1 + lib/routes/wallstreetcn/live.js | 44 +++++++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+) create mode 100644 lib/routes/wallstreetcn/live.js diff --git a/docs/traditional-media.md b/docs/traditional-media.md index fc91b291fd..528d04f1b2 100644 --- a/docs/traditional-media.md +++ b/docs/traditional-media.md @@ -283,6 +283,16 @@ Type 栏目: +### 实时快讯 + + + +| 要闻 | A 股 | 美股 | 港股 | 外汇 | 商品 | 理财 | +| ------ | ------- | -------- | -------- | ----- | --------- | --------- | +| global | a-stock | us-stock | hk-stock | forex | commodity | financing | + + + ## 极客公园 ### 全球快讯 diff --git a/lib/router.js b/lib/router.js index 5a85fd5881..3a5250e9fc 100644 --- a/lib/router.js +++ b/lib/router.js @@ -1209,6 +1209,7 @@ router.get('/allpoetry/:order?', require('./routes/allpoetry/order')); // 华尔街见闻 router.get('/wallstreetcn/news/global', require('./routes/wallstreetcn/news')); +router.get('/wallstreetcn/live/:channel?', require('./routes/wallstreetcn/live')); // 多抓鱼搜索 router.get('/duozhuayu/search/:wd', require('./routes/duozhuayu/search')); diff --git a/lib/routes/wallstreetcn/live.js b/lib/routes/wallstreetcn/live.js new file mode 100644 index 0000000000..6455fea5d8 --- /dev/null +++ b/lib/routes/wallstreetcn/live.js @@ -0,0 +1,44 @@ +const got = require('@/utils/got'); + +const config = { + global: '要闻', + 'a-stock': 'A股', + 'us-stock': '美股', + 'hk-stock': '港股', + forex: '外汇', + commodity: '商品', + financing: '理财', +}; + +module.exports = async (ctx) => { + ctx.params.channel = ctx.params.channel || 'global'; + + const currentUrl = `https://wallstreetcn.com/live/${ctx.params.channel}`; + const rootUrl = `https://api.wallstcn.com/apiv1/content/lives?channel=${ctx.params.channel}-channel`; + const response = await got({ + method: 'get', + url: rootUrl, + }); + + const items = response.data.data.items.map((item) => { + let title; + const match = item.content_text.match(/【(.*)】/); + if (match !== null) { + title = match[1]; + } else { + title = item.content_text; + } + return { + title, + link: `https://wallstreetcn.com/livenews/${item.id}`, + pubDate: new Date(item.display_time * 1000).toUTCString(), + description: item.content, + }; + }); + + ctx.state.data = { + title: `${config[ctx.params.channel]} - 实时快讯 - 华尔街见闻`, + link: currentUrl, + item: items, + }; +}; From 0481bff2846867d0c7f2fee8f6adff34753e2850 Mon Sep 17 00:00:00 2001 From: Ethan Shen Date: Fri, 25 Sep 2020 18:46:41 +0800 Subject: [PATCH 22/44] feat: add World Economic Forum reports (#5717) Co-authored-by: Henry Wang --- docs/en/finance.md | 16 +++++++++++ docs/finance.md | 16 +++++++++++ lib/router.js | 3 ++ lib/routes/weforum/report.js | 53 ++++++++++++++++++++++++++++++++++++ 4 files changed, 88 insertions(+) create mode 100644 lib/routes/weforum/report.js diff --git a/docs/en/finance.md b/docs/en/finance.md index c8e7a8938b..935b14710b 100644 --- a/docs/en/finance.md +++ b/docs/en/finance.md @@ -15,3 +15,19 @@ pageClass: routes ### US Stock News + +## World Economic Forum + +### Report + + + +Languages + +| English | Español | Français | 中文 | 日本語 | +| ------- | ------- | -------- | ---- | ------ | +| en | es | fr | cn | jp | + +See filters in [Report](https://www.weforum.org/reports) for Year and Platform these two parameters. + + diff --git a/docs/finance.md b/docs/finance.md index c6b8cebb3b..49653029eb 100644 --- a/docs/finance.md +++ b/docs/finance.md @@ -62,6 +62,22 @@ pageClass: routes <Route author="nczitzk" example="/gelonghui/keyword/ 早报" path="/gelonghui/keyword/:keyword" :paramsDesc="[' 搜索关键字']/> +## 世界经济论坛 + +### 报告 + + + +语言 + +| English | Español | Français | 中文 | 日本語 | +| ------- | ------- | -------- | ---- | ------ | +| en | es | fr | cn | jp | + +年份 和 平台 这两个参数请参见 [报告页](https://www.weforum.org/reports) 过滤条件处。 + + + ## 淘股吧股票论坛 ### 论坛总版 diff --git a/lib/router.js b/lib/router.js index 3a5250e9fc..8f0ec72ab9 100644 --- a/lib/router.js +++ b/lib/router.js @@ -3241,6 +3241,9 @@ router.get('/appsales/:caty?/:time?', require('./routes/appsales/index')); // CGTN router.get('/cgtn/top', require('./routes/cgtn/top')); +// World Economic Forum +router.get('/weforum/report/:lang?/:year?/:platform?', require('./routes/weforum/report')); + // Nobel Prize router.get('/nobelprize/:caty?', require('./routes/nobelprize/index')); diff --git a/lib/routes/weforum/report.js b/lib/routes/weforum/report.js new file mode 100644 index 0000000000..69e46f6900 --- /dev/null +++ b/lib/routes/weforum/report.js @@ -0,0 +1,53 @@ +const got = require('@/utils/got'); +const cheerio = require('cheerio'); + +module.exports = async (ctx) => { + ctx.params.lang = ctx.params.lang || 'www'; + ctx.params.platform = ctx.params.platform || ''; + ctx.params.year = ctx.params.year || ''; + + const rootUrl = `https://${ctx.params.lang === 'en' ? 'www' : ctx.params.lang}.weforum.org`; + const currentUrl = `${rootUrl}/reports?platform=${ctx.params.platform}&year=${ctx.params.year}`; + const response = await got({ + method: 'get', + url: currentUrl, + }); + + const $ = cheerio.load(response.data); + + const list = $('.report-listing-tout__content') + .map((_, item) => { + item = $(item); + return { + title: item.find('.report-listing-tout__title').text(), + link: `${rootUrl}${item.find('.report-listing-tout__cta').attr('href')}`, + description: `Download PDF`, + }; + }) + .get(); + + const items = await Promise.all( + list.map( + async (item) => + await ctx.cache.tryGet(item.link, async () => { + const detailResponse = await got({ + method: 'get', + url: item.link, + }); + const content = cheerio.load(detailResponse.data); + + content('.report__meta').remove(); + + item.description += content('.report__display-info').html(); + + return item; + }) + ) + ); + + ctx.state.data = { + title: $('title').text(), + link: currentUrl, + item: items, + }; +}; From b202a0546e73ba6968e3cd014dc0338270651353 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Fri, 25 Sep 2020 10:51:52 +0000 Subject: [PATCH 23/44] style: auto format --- docs/university.md | 6 +++--- lib/routes/universities/xidian/jwc.js | 4 +--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/docs/university.md b/docs/university.md index f781c751f5..3676fe8168 100644 --- a/docs/university.md +++ b/docs/university.md @@ -1547,9 +1547,9 @@ type 列表： 全文内容需使用校园网或 VPN 获取 ::: -| 全部 | 教学信息 | 教学研究 | 实践教学 | 质量监控 | 通知公告 | -| :---: | :------: | :------: | :------: | :------: | :------: | -| all | jxxx | jxyj | sjjx | zljk | tzgg | +| 全部 | 教学信息 | 教学研究 | 实践教学 | 质量监控 | 通知公告 | +| :--: | :------: | :------: | :------: | :------: | :------: | +| all | jxxx | jxyj | sjjx | zljk | tzgg | diff --git a/lib/routes/universities/xidian/jwc.js b/lib/routes/universities/xidian/jwc.js index cacbf51619..cd41edabfc 100644 --- a/lib/routes/universities/xidian/jwc.js +++ b/lib/routes/universities/xidian/jwc.js @@ -16,8 +16,7 @@ module.exports = async (ctx) => { const leftItems = $('div .con2 .fl').children(); const rightItems = $('div .con2 .fr').children(); const mainItems = $('div .main_right').children(); - const list = { '0': leftItems[1], '1': leftItems[4], '2': rightItems[1], '3': rightItems[4], '4': mainItems[5] }; - + const list = { 0: leftItems[1], 1: leftItems[4], 2: rightItems[1], 3: rightItems[4], 4: mainItems[5] }; const parseTable = (index, item) => { const title = arrC[index]; @@ -49,7 +48,6 @@ module.exports = async (ctx) => { .get(); }; - const arrIndex = arrE.indexOf(category); let result = new Array(); if (arrIndex === -1) { From eb4efcb38957d96de093950aa41506cfd6f4fcb8 Mon Sep 17 00:00:00 2001 From: Zuyang Kou Date: Fri, 25 Sep 2020 18:53:52 +0800 Subject: [PATCH 24/44] =?UTF-8?q?revert:=20=E5=9B=9E=E6=BB=9A=E8=B4=A2?= =?UTF-8?q?=E6=96=B0=E6=97=A7=E7=89=88=20API(#5703)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/routes/caixin/article.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/routes/caixin/article.js b/lib/routes/caixin/article.js index 800e3bfa85..29fd6d70a0 100644 --- a/lib/routes/caixin/article.js +++ b/lib/routes/caixin/article.js @@ -3,7 +3,11 @@ const got = require('@/utils/got'); module.exports = async (ctx) => { const response = await got({ method: 'get', - url: 'https://mappsv5.caixin.com/index_page_v5/index_page_1.json', + url: 'http://mapiv5.caixin.com//m/api/getWapIndexListByPage?page=1&callback=&_=1560140003179', + headers: { + Referer: `http://mapiv5.caixin.com/`, + Host: 'mapiv5.caixin.com', + }, }); const data = response.data.data.list; From 7386f88ea88d744b3ef5880287333e36311820a8 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 25 Sep 2020 15:46:19 +0000 Subject: [PATCH 25/44] chore(deps-dev): bump supertest from 4.0.2 to 5.0.0 Bumps [supertest](https://github.com/visionmedia/supertest) from 4.0.2 to 5.0.0. - [Release notes](https://github.com/visionmedia/supertest/releases) - [Commits](https://github.com/visionmedia/supertest/compare/v4.0.2...v5.0.0) Signed-off-by: dependabot-preview[bot] --- package.json | 2 +- yarn.lock | 84 +++++++++++++++++++++++++++++++--------------------- 2 files changed, 51 insertions(+), 35 deletions(-) diff --git a/package.json b/package.json index ef1b1aea13..fa1d521c1a 100644 --- a/package.json +++ b/package.json @@ -59,7 +59,7 @@ "remark-preset-prettier": "0.4.0", "staged-git-files": "1.2.0", "string-width": "4.2.0", - "supertest": "4.0.2", + "supertest": "5.0.0", "vuepress": "1.5.3", "yorkie": "2.0.0" }, diff --git a/yarn.lock b/yarn.lock index 43679d28dd..94c08020a3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3630,7 +3630,7 @@ commondir@^1.0.1: resolved "https://registry.yarnpkg.com/commondir/-/commondir-1.0.1.tgz#ddd800da0c66127393cca5950ea968a3aaf1253b" integrity sha1-3dgA2gxmEnOTzKWVDqloo6rxJTs= -component-emitter@^1.2.0, component-emitter@^1.2.1: +component-emitter@^1.2.1, component-emitter@^1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/component-emitter/-/component-emitter-1.3.0.tgz#16e4070fba8ae29b679f2215853ee181ab2eabc0" integrity sha512-Rd3se6QB+sO1TwqZjscQrurpEPIfO0/yYnSin6Q/rD3mOutHvUrCAhJub3r90uNb+SESBuE0QYoB90YdfatsRg== @@ -3757,7 +3757,7 @@ cookie@^0.4.1: resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.4.1.tgz#afd713fe26ebd21ba95ceb61f9a8116e50a537d1" integrity sha512-ZwrFkGJxUR3EIoXtO+yVE69Eb7KlixbaeAWfBQB9vVsNn/o+Yw69gBWSSDK825hQNdN+wF8zELf3dFNl/kxkUA== -cookiejar@^2.1.0: +cookiejar@^2.1.2: version "2.1.2" resolved "https://registry.yarnpkg.com/cookiejar/-/cookiejar-2.1.2.tgz#dd8a235530752f988f9a0844f3fc589e3111125c" integrity sha512-Mw+adcfzPxcPeI+0WlvRrr/3lGVO0bD75SxX6811cxSh1Wbxx7xZBGK1eVtDf6si8rg2lhnUjsVLMFMfbRIuwA== @@ -4206,7 +4206,7 @@ debug@2.6.9, debug@^2.2.0, debug@^2.3.3, debug@^2.6.9: dependencies: ms "2.0.0" -debug@^3.1.0, debug@^3.2.5, debug@^3.2.6: +debug@^3.2.5, debug@^3.2.6: version "3.2.6" resolved "https://registry.yarnpkg.com/debug/-/debug-3.2.6.tgz#e83d17de16d8a7efb7717edbe5fb10135eee629b" integrity sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ== @@ -5275,6 +5275,11 @@ fast-safe-stringify@^2.0.4: resolved "https://registry.yarnpkg.com/fast-safe-stringify/-/fast-safe-stringify-2.0.6.tgz#04b26106cc56681f51a044cfc0d76cf0008ac2c2" integrity sha512-q8BZ89jjc+mz08rSxROs8VsrBBcn1SIw1kq9NjolL509tkABRk9io01RAjSaEv1Xb2uFLt8VtRiZbGp5H8iDtg== +fast-safe-stringify@^2.0.7: + version "2.0.7" + resolved "https://registry.yarnpkg.com/fast-safe-stringify/-/fast-safe-stringify-2.0.7.tgz#124aa885899261f68aedb42a7c080de9da608743" + integrity sha512-Utm6CdzT+6xsDk2m8S6uL8VHxNwI6Jub+e9NYTcAms28T84pTa25GJQV9j0CY0N1rM8hK4x6grpF2BQf+2qwVA== + fast-text-encoding@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/fast-text-encoding/-/fast-text-encoding-1.0.0.tgz#3e5ce8293409cfaa7177a71b9ca84e1b1e6f25ef" @@ -5448,7 +5453,7 @@ forever-agent@~0.6.1: resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91" integrity sha1-+8cfDEGt6zf5bFd60e1C2P2sypE= -form-data@^2.3.1, form-data@^2.5.0: +form-data@^2.5.0: version "2.5.1" resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.5.1.tgz#f2cbec57b5e59e23716e128fe44d4e5dd23895f4" integrity sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA== @@ -5480,10 +5485,10 @@ format@^0.2.0: resolved "https://registry.yarnpkg.com/format/-/format-0.2.2.tgz#d6170107e9efdc4ed30c9dc39016df942b5cb58b" integrity sha1-1hcBB+nv3E7TDJ3DkBbflCtctYs= -formidable@^1.2.0: - version "1.2.1" - resolved "https://registry.yarnpkg.com/formidable/-/formidable-1.2.1.tgz#70fb7ca0290ee6ff961090415f4b3df3d2082659" - integrity sha512-Fs9VRguL0gqGHkXS5GQiMCr1VhZBxz0JnJs4JmMp/2jL18Fmbzvv7vOFRU+U8TBkHEE/CX1qDXzJplVULgsLeg== +formidable@^1.2.2: + version "1.2.2" + resolved "https://registry.yarnpkg.com/formidable/-/formidable-1.2.2.tgz#bf69aea2972982675f00865342b982986f6b8dd9" + integrity sha512-V8gLm+41I/8kguQ4/o1D3RIHRmhYFG4pnNyonvua+40rqcEmT4+V71yaZ3B457xbbgCsCfjSPi65u/W6vK1U5Q== forwarded@~0.1.2: version "0.1.2" @@ -8272,7 +8277,7 @@ merge2@^1.2.3: resolved "https://registry.yarnpkg.com/merge2/-/merge2-1.2.3.tgz#7ee99dbd69bb6481689253f018488a1b902b0ed5" integrity sha512-gdUU1Fwj5ep4kplwcmftruWofEFt6lfpkkr3h860CXbAB9c3hGb55EOL2ali0Td5oebvW0E1+3Sr+Ur7XfKpRA== -methods@^1.1.1, methods@^1.1.2, methods@~1.1.2: +methods@1.1.2, methods@^1.1.2, methods@~1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/methods/-/methods-1.1.2.tgz#5529a4d67654134edcc5266656835b0f851afcee" integrity sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4= @@ -8329,7 +8334,7 @@ mime-types@^2.1.12, mime-types@^2.1.18, mime-types@~2.1.17, mime-types@~2.1.19, dependencies: mime-db "1.43.0" -mime@1.6.0, mime@^1.3.4, mime@^1.4.1: +mime@1.6.0, mime@^1.3.4: version "1.6.0" resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1" integrity sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg== @@ -8339,6 +8344,11 @@ mime@^2.0.3, mime@^2.2.0, mime@^2.4.2: resolved "https://registry.yarnpkg.com/mime/-/mime-2.4.4.tgz#bd7b91135fc6b01cde3e9bae33d659b63d8857e5" integrity sha512-LRxmNwziLPT828z+4YkNzloCFC2YM4wrB99k+AV5ZbEyfGNWfG8SO1FUXLmLDBSo89NrJZ4DIWeLjy1CHGhMGA== +mime@^2.4.6: + version "2.4.6" + resolved "https://registry.yarnpkg.com/mime/-/mime-2.4.6.tgz#e5b407c90db442f2beb5b162373d07b69affa4d1" + integrity sha512-RZKhC3EmpBchfTGBVb8fb+RL2cWyw/32lshnsETttkBAyAUXSGHxbEJWWRXc751DrIxG1q04b8QwMbAwkRPpUA== + mimic-fn@^2.0.0, mimic-fn@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-2.1.0.tgz#7ed2c2ccccaf84d3ffcb7a69b57711fc2083401b" @@ -10123,7 +10133,7 @@ q@^1.1.2: resolved "https://registry.yarnpkg.com/q/-/q-1.5.1.tgz#7e32f75b41381291d04611f1bf14109ac00651d7" integrity sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc= -qs@6.7.0, qs@^6.5.1: +qs@6.7.0: version "6.7.0" resolved "https://registry.yarnpkg.com/qs/-/qs-6.7.0.tgz#41dc1a015e3d581f1621776be31afb2876a9b1bc" integrity sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ== @@ -10133,6 +10143,11 @@ qs@^6.7.0: resolved "https://registry.yarnpkg.com/qs/-/qs-6.9.1.tgz#20082c65cb78223635ab1a9eaca8875a29bf8ec9" integrity sha512-Cxm7/SS/y/Z3MHWSxXb8lIFqgqBowP5JMlTUFyJN88y0SGQhVmZnqFK/PeuMX9LzUyWsqqhNxIyg0jlzq946yA== +qs@^6.9.4: + version "6.9.4" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.9.4.tgz#9090b290d1f91728d3c22e54843ca44aea5ab687" + integrity sha512-A1kFqHekCTM7cz0udomYUoYNWjBebHm/5wzU/XqrBRBNWectVH0QIiN+NEcZ0Dte5hvzHwbr8+XQmguPhJ6WdQ== + qs@~6.5.2: version "6.5.2" resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.2.tgz#cb3ae806e8740444584ef154ce8ee98d403f3e36" @@ -10255,7 +10270,7 @@ read-pkg@^5.2.0: parse-json "^5.0.0" type-fest "^0.6.0" -"readable-stream@1 || 2", readable-stream@^2.0.0, readable-stream@^2.0.1, readable-stream@^2.0.2, readable-stream@^2.0.6, readable-stream@^2.1.5, readable-stream@^2.2.2, readable-stream@^2.3.3, readable-stream@^2.3.5, readable-stream@^2.3.6, readable-stream@^2.3.7, readable-stream@~2.3.6: +"readable-stream@1 || 2", readable-stream@^2.0.0, readable-stream@^2.0.1, readable-stream@^2.0.2, readable-stream@^2.0.6, readable-stream@^2.1.5, readable-stream@^2.2.2, readable-stream@^2.3.3, readable-stream@^2.3.6, readable-stream@^2.3.7, readable-stream@~2.3.6: version "2.3.7" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57" integrity sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw== @@ -10268,7 +10283,7 @@ read-pkg@^5.2.0: string_decoder "~1.1.1" util-deprecate "~1.0.1" -readable-stream@^3.0.6, readable-stream@^3.1.1, readable-stream@^3.4.0: +readable-stream@^3.0.6, readable-stream@^3.1.1, readable-stream@^3.4.0, readable-stream@^3.6.0: version "3.6.0" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198" integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA== @@ -11589,29 +11604,30 @@ stylus@^0.54.5: sax "0.5.x" source-map "0.1.x" -superagent@^3.8.3: - version "3.8.3" - resolved "https://registry.yarnpkg.com/superagent/-/superagent-3.8.3.tgz#460ea0dbdb7d5b11bc4f78deba565f86a178e128" - integrity sha512-GLQtLMCoEIK4eDv6OGtkOoSMt3D+oq0y3dsxMuYuDvaNUvuT8eFBuLmfR0iYYzHC1e8hpzC6ZsxbuP6DIalMFA== - dependencies: - component-emitter "^1.2.0" - cookiejar "^2.1.0" - debug "^3.1.0" - extend "^3.0.0" - form-data "^2.3.1" - formidable "^1.2.0" - methods "^1.1.1" - mime "^1.4.1" - qs "^6.5.1" - readable-stream "^2.3.5" - -supertest@4.0.2: - version "4.0.2" - resolved "https://registry.yarnpkg.com/supertest/-/supertest-4.0.2.tgz#c2234dbdd6dc79b6f15b99c8d6577b90e4ce3f36" - integrity sha512-1BAbvrOZsGA3YTCWqbmh14L0YEq0EGICX/nBnfkfVJn7SrxQV1I3pMYjSzG9y/7ZU2V9dWqyqk2POwxlb09duQ== +superagent@6.1.0: + version "6.1.0" + resolved "https://registry.yarnpkg.com/superagent/-/superagent-6.1.0.tgz#09f08807bc41108ef164cfb4be293cebd480f4a6" + integrity sha512-OUDHEssirmplo3F+1HWKUrUjvnQuA+nZI6i/JJBdXb5eq9IyEQwPyPpqND+SSsxf6TygpBEkUjISVRN4/VOpeg== dependencies: + component-emitter "^1.3.0" + cookiejar "^2.1.2" + debug "^4.1.1" + fast-safe-stringify "^2.0.7" + form-data "^3.0.0" + formidable "^1.2.2" methods "^1.1.2" - superagent "^3.8.3" + mime "^2.4.6" + qs "^6.9.4" + readable-stream "^3.6.0" + semver "^7.3.2" + +supertest@5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/supertest/-/supertest-5.0.0.tgz#771aedfeb0a95466cc5d100d5d11288736fd25da" + integrity sha512-2JAWpPrUOZF4hHH5ZTCN2xjKXvJS3AEwPNXl0HUseHsfcXFvMy9kcsufIHCNAmQ5hlGCvgeAqaR5PBEouN3hlQ== + dependencies: + methods "1.1.2" + superagent "6.1.0" supports-color@^2.0.0: version "2.0.0" From 292f63218d95c11e78d6f83876328373d02b63a0 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 25 Sep 2020 16:01:19 +0000 Subject: [PATCH 26/44] chore(deps-dev): bump prettier from 2.1.1 to 2.1.2 Bumps [prettier](https://github.com/prettier/prettier) from 2.1.1 to 2.1.2. - [Release notes](https://github.com/prettier/prettier/releases) - [Changelog](https://github.com/prettier/prettier/blob/master/CHANGELOG.md) - [Commits](https://github.com/prettier/prettier/compare/2.1.1...2.1.2) Signed-off-by: dependabot-preview[bot] --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index fa1d521c1a..cd7cf82a06 100644 --- a/package.json +++ b/package.json @@ -50,7 +50,7 @@ "nock": "13.0.4", "nodemon": "2.0.4", "pinyin": "2.9.1", - "prettier": "2.1.1", + "prettier": "2.1.2", "prettier-check": "2.0.0", "pretty-quick": "3.0.2", "remark": "12.0.1", diff --git a/yarn.lock b/yarn.lock index 94c08020a3..7f5da5171a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9926,10 +9926,10 @@ prettier@1.16.3: resolved "https://registry.yarnpkg.com/prettier/-/prettier-1.16.3.tgz#8c62168453badef702f34b45b6ee899574a6a65d" integrity sha512-kn/GU6SMRYPxUakNXhpP0EedT/KmaPzr0H5lIsDogrykbaxOpOfAFfk5XA7DZrJyMAv1wlMV3CPcZruGXVVUZw== -prettier@2.1.1: - version "2.1.1" - resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.1.1.tgz#d9485dd5e499daa6cb547023b87a6cf51bee37d6" - integrity sha512-9bY+5ZWCfqj3ghYBLxApy2zf6m+NJo5GzmLTpr9FsApsfjriNnS2dahWReHMi7qNPhhHl9SYHJs2cHZLgexNIw== +prettier@2.1.2: + version "2.1.2" + resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.1.2.tgz#3050700dae2e4c8b67c4c3f666cdb8af405e1ce5" + integrity sha512-16c7K+x4qVlJg9rEbXl7HEGmQyZlG4R9AgP+oHKRMsMsuk8s+ATStlf1NpDqyBI1HpVyfjLOeMhH2LvuNvV5Vg== pretty-bytes@^5.1.0: version "5.2.0" From aeeaf1c2798548c35efc2f9b119c36f62ccbec93 Mon Sep 17 00:00:00 2001 From: Ethan Shen Date: Sat, 26 Sep 2020 22:26:00 +0800 Subject: [PATCH 27/44] feat: add JavDB (#5739) Co-authored-by: Henry Wang --- docs/multimedia.md | 110 +++++++++++++++++++++++++++++++++++ lib/router.js | 9 +++ lib/routes/javdb/actors.js | 19 ++++++ lib/routes/javdb/home.js | 32 ++++++++++ lib/routes/javdb/makers.js | 20 +++++++ lib/routes/javdb/rankings.js | 12 ++++ lib/routes/javdb/search.js | 26 +++++++++ lib/routes/javdb/series.js | 20 +++++++ lib/routes/javdb/tags.js | 12 ++++ lib/routes/javdb/utils.js | 72 +++++++++++++++++++++++ 10 files changed, 332 insertions(+) create mode 100644 lib/routes/javdb/actors.js create mode 100644 lib/routes/javdb/home.js create mode 100644 lib/routes/javdb/makers.js create mode 100644 lib/routes/javdb/rankings.js create mode 100644 lib/routes/javdb/search.js create mode 100644 lib/routes/javdb/series.js create mode 100644 lib/routes/javdb/tags.js create mode 100644 lib/routes/javdb/utils.js diff --git a/docs/multimedia.md b/docs/multimedia.md index 50cb939d5a..f2d128e227 100644 --- a/docs/multimedia.md +++ b/docs/multimedia.md @@ -131,6 +131,116 @@ pageClass: routes +## JavDB + +### 主页 + + + +分类 + +| 有碼 | 無碼 | 歐美 | FC2 | +| -------- | ---------- | ------- | --- | +| censored | uncensored | western | fc2 | + +排序 + +| 发布日期排序 | 磁鏈更新排序 | +| ------------ | ------------ | +| 1 | 2 | + +过滤 + +| 全部 | 可下载 | 含字幕 | 含短評 | +| ---- | ------ | ------ | ------ | +| 0 | 1 | 2 | 3 | + + + +### 分類 + + + +::: tip 提示 + +在 [分類](https://javdb.com/tags) 中选定分类后，URL 中 `tags?` 后的字段即为筛选参数。 + +如 中 `c2=5&c10=1` 为筛选参数。 + +::: + +分类 + +| 有碼 | 無碼 | 歐美 | FC2 | +| -------- | ---------- | ------- | --- | +| censored | uncensored | western | fc2 | + + + +### 排行榜 + + + +分类 + +| 有碼 | 無碼 | 歐美 | FC2 | +| -------- | ---------- | ------- | --- | +| censored | uncensored | western | fc2 | + +时间 + +| 日榜 | 週榜 | 月榜 | +| ----- | ------ | ------- | +| daily | weekly | monthly | + + + +### 搜索 + + + +| 全部 | 可播放 | 單體作品 | 演員 | 片商 | 導演 | 系列 | 番號 | 可下載 | 字幕 | 預覽圖 | +| ---- | -------- | -------- | ----- | ----- | -------- | ------ | ---- | -------- | ----- | ------- | +| | playable | single | actor | maker | director | series | code | download | cnsub | preview | + + + +### 演員 + + + +| 全部 | 可播放 | 單體作品 | 可下載 | 含字幕 | +| ---- | ------ | -------- | ------ | ------ | +| | p | s | d | c | + +所有演员编号参见 [演員庫](https://javdb.com/actors) + + + +### 系列 + + + +| 全部 | 可播放 | 單體作品 | 可下載 | 字幕 | 預覽圖 | +| ---- | -------- | -------- | -------- | ----- | ------- | +| | playable | single | download | cnsub | preview | + +所有系列编号参见 [系列庫](https://javdb.com/series) + + + +### 片商 + + + +| 全部 | 可播放 | 單體作品 | 可下載 | 字幕 | 預覽圖 | +| ---- | -------- | -------- | -------- | ----- | ------- | +| | playable | single | download | cnsub | preview | + +所有片商编号参见 [片商庫](https://javdb.com/makers) + + + ## JavLibrary ### 影片 diff --git a/lib/router.js b/lib/router.js index 8f0ec72ab9..25c0cee9c7 100644 --- a/lib/router.js +++ b/lib/router.js @@ -3241,6 +3241,15 @@ router.get('/appsales/:caty?/:time?', require('./routes/appsales/index')); // CGTN router.get('/cgtn/top', require('./routes/cgtn/top')); +// JavDB +router.get('/javdb/home/:caty?/:sort?/:filter?', require('./routes/javdb/home')); +router.get('/javdb/search/:keyword?/:filter?', require('./routes/javdb/search')); +router.get('/javdb/tags/:query?/:caty?', require('./routes/javdb/tags')); +router.get('/javdb/actors/:id/:filter?', require('./routes/javdb/actors')); +router.get('/javdb/makers/:id/:filter?', require('./routes/javdb/makers')); +router.get('/javdb/series/:id/:filter?', require('./routes/javdb/series')); +router.get('/javdb/rankings/:caty?/:time?', require('./routes/javdb/rankings')); + // World Economic Forum router.get('/weforum/report/:lang?/:year?/:platform?', require('./routes/weforum/report')); diff --git a/lib/routes/javdb/actors.js b/lib/routes/javdb/actors.js new file mode 100644 index 0000000000..0815709c5d --- /dev/null +++ b/lib/routes/javdb/actors.js @@ -0,0 +1,19 @@ +const utils = require('./utils'); + +module.exports = async (ctx) => { + ctx.params.filter = ctx.params.filter || ''; + + const currentUrl = `${utils.rootUrl}/actors/${ctx.params.id}?t=${ctx.params.filter}`; + + const filter = { + '': '', + p: '可播放', + s: '單體作品', + d: '可下載', + c: '含字幕', + }; + + const title = `JavDB${filter[ctx.params.filter] === '' ? '' : ` - ${filter[ctx.params.filter]}`} `; + + ctx.state.data = await utils.ProcessItems(ctx, currentUrl, title); +}; diff --git a/lib/routes/javdb/home.js b/lib/routes/javdb/home.js new file mode 100644 index 0000000000..8d49953c30 --- /dev/null +++ b/lib/routes/javdb/home.js @@ -0,0 +1,32 @@ +const utils = require('./utils'); + +module.exports = async (ctx) => { + ctx.params.caty = ctx.params.caty || 'censored'; + ctx.params.sort = ctx.params.sort || '2'; + ctx.params.filter = ctx.params.filter || '1'; + + const currentUrl = `${utils.rootUrl}/${ctx.params.caty === 'censored' ? '' : ctx.params.caty}?vft=${ctx.params.filter}&vst=${ctx.params.sort}`; + + const category = { + censored: '有碼', + uncensored: '無碼', + western: '歐美', + fc2: 'FC2', + }; + + const filter = { + 0: '', + 1: '可下载', + 2: '含字幕', + 3: '含短評', + }; + + const sort = { + 1: '发布日期排序', + 2: '磁鏈更新排序', + }; + + const title = `${category[ctx.params.caty]} - JavDB - ${filter[ctx.params.filter] === '' ? '|' : `${filter[ctx.params.filter]} | `}${sort[ctx.params.sort]}`; + + ctx.state.data = await utils.ProcessItems(ctx, currentUrl, title); +}; diff --git a/lib/routes/javdb/makers.js b/lib/routes/javdb/makers.js new file mode 100644 index 0000000000..c6b06f95f0 --- /dev/null +++ b/lib/routes/javdb/makers.js @@ -0,0 +1,20 @@ +const utils = require('./utils'); + +module.exports = async (ctx) => { + ctx.params.filter = ctx.params.filter || ''; + + const currentUrl = `${utils.rootUrl}/makers/${ctx.params.id}?f=${ctx.params.filter}`; + + const filter = { + '': '', + playable: '可播放', + single: '單體作品', + download: '可下載', + cnsub: '字幕', + preview: '預覽圖', + }; + + const title = `JavDB${filter[ctx.params.filter] === '' ? '' : ` - ${filter[ctx.params.filter]}`} `; + + ctx.state.data = await utils.ProcessItems(ctx, currentUrl, title); +}; diff --git a/lib/routes/javdb/rankings.js b/lib/routes/javdb/rankings.js new file mode 100644 index 0000000000..ee429abc26 --- /dev/null +++ b/lib/routes/javdb/rankings.js @@ -0,0 +1,12 @@ +const utils = require('./utils'); + +module.exports = async (ctx) => { + ctx.params.caty = ctx.params.caty || 'censored'; + ctx.params.time = ctx.params.time || 'daily'; + + const currentUrl = `${utils.rootUrl}/rankings/video_${ctx.params.caty}?period=${ctx.params.time}`; + + const title = 'JavDB'; + + ctx.state.data = await utils.ProcessItems(ctx, currentUrl, title); +}; diff --git a/lib/routes/javdb/search.js b/lib/routes/javdb/search.js new file mode 100644 index 0000000000..bdf0bc5ef0 --- /dev/null +++ b/lib/routes/javdb/search.js @@ -0,0 +1,26 @@ +const utils = require('./utils'); + +module.exports = async (ctx) => { + ctx.params.filter = ctx.params.filter || ''; + ctx.params.keyword = ctx.params.keyword || ''; + + const currentUrl = `${utils.rootUrl}/search?q=${ctx.params.keyword}&f=${ctx.params.filter}`; + + const filter = { + '': '', + playable: '可播放', + single: '單體作品', + actor: '演員', + maker: '片商', + director: '導演', + series: '系列', + code: '番號', + download: '可下載', + cnsub: '字幕', + preview: '預覽圖', + }; + + const title = `關鍵字 ${ctx.params.keyword} ${filter[ctx.params.filter] === '' ? '' : `+ ${filter[ctx.params.filter]}`} 搜索結果 - JavDB `; + + ctx.state.data = await utils.ProcessItems(ctx, currentUrl, title); +}; diff --git a/lib/routes/javdb/series.js b/lib/routes/javdb/series.js new file mode 100644 index 0000000000..9189c21bea --- /dev/null +++ b/lib/routes/javdb/series.js @@ -0,0 +1,20 @@ +const utils = require('./utils'); + +module.exports = async (ctx) => { + ctx.params.filter = ctx.params.filter || ''; + + const currentUrl = `${utils.rootUrl}/series/${ctx.params.id}?f=${ctx.params.filter}`; + + const filter = { + '': '', + playable: '可播放', + single: '單體作品', + download: '可下載', + cnsub: '字幕', + preview: '預覽圖', + }; + + const title = `JavDB${filter[ctx.params.filter] === '' ? '' : ` - ${filter[ctx.params.filter]}`} `; + + ctx.state.data = await utils.ProcessItems(ctx, currentUrl, title); +}; diff --git a/lib/routes/javdb/tags.js b/lib/routes/javdb/tags.js new file mode 100644 index 0000000000..efec484cb8 --- /dev/null +++ b/lib/routes/javdb/tags.js @@ -0,0 +1,12 @@ +const utils = require('./utils'); + +module.exports = async (ctx) => { + ctx.params.caty = ctx.params.caty || 'censored'; + ctx.params.query = ctx.params.query || ''; + + const currentUrl = `${utils.rootUrl}/tags${ctx.params.caty === 'censored' ? '' : `/${ctx.params.caty}`}?${ctx.params.query}`; + + const title = `JavDB${ctx.params.query === '' ? '' : ` - ${ctx.params.query}`} `; + + ctx.state.data = await utils.ProcessItems(ctx, currentUrl, title); +}; diff --git a/lib/routes/javdb/utils.js b/lib/routes/javdb/utils.js new file mode 100644 index 0000000000..78e5ad8ee2 --- /dev/null +++ b/lib/routes/javdb/utils.js @@ -0,0 +1,72 @@ +const got = require('@/utils/got'); +const cheerio = require('cheerio'); + +const rootUrl = 'https://javdb.com'; + +module.exports = { + rootUrl: rootUrl, + ProcessItems: async (ctx, currentUrl, title) => { + const response = await got({ + method: 'get', + url: currentUrl, + }); + + const $ = cheerio.load(response.data); + + $('.tags').remove(); + + const list = $('.grid-item a') + .slice(0, 5) + .map((_, item) => { + item = $(item); + return { + title: item.text(), + link: `${rootUrl}${item.attr('href')}`, + pubDate: new Date(item.find('.meta').text()).toUTCString(), + }; + }) + .get(); + + const items = await Promise.all( + list.map( + async (item) => + await ctx.cache.tryGet(item.link, async () => { + const detailResponse = await got({ + method: 'get', + url: item.link, + }); + const content = cheerio.load(detailResponse.data); + + item.enclosure_url = content('#magnets-content button[data-clipboard-text]').eq(0).attr('data-clipboard-text'); + item.enclosure_type = 'application/x-bittorrent'; + + content('icon, .review-buttons, .copy-to-clipboard').remove(); + content('#modal-review-watched, #modal-save-list').remove(); + content('.video-cover').eq(1).remove(); + + content('img.video-cover').attr('src', content(this).parent().attr('href')); + + content('.preview-images img').each(function () { + content(this).removeAttr('data-src'); + content(this).attr('src', content(this).parent().attr('href')); + }); + + content('#preview-video').css('display', 'block'); + + item.description = content('.movie-info-panel .columns').html() + content('#magnets-content').html() + content('.preview-images').html(); + + return item; + }) + ) + ); + + const htmlTitle = $('title').text(); + const subject = htmlTitle.indexOf('|') === -1 ? '' : htmlTitle.split('|')[0]; + + return { + title: subject === '' ? title : `${subject} - ${title}`, + link: currentUrl, + item: items, + }; + }, +}; From b0584950d1a7e55337e53aeb5a898d386beabf58 Mon Sep 17 00:00:00 2001 From: Ethan Shen Date: Sat, 26 Sep 2020 22:26:48 +0800 Subject: [PATCH 28/44] =?UTF-8?q?fix:=20GQ=E5=8F=B0=E7=81=A3=E9=83=A8?= =?UTF-8?q?=E5=88=86=E5=9B=BE=E9=9B=86=20(#5740)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/routes/gq/tw/index.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/routes/gq/tw/index.js b/lib/routes/gq/tw/index.js index 4509bccfd6..1b2a67f248 100644 --- a/lib/routes/gq/tw/index.js +++ b/lib/routes/gq/tw/index.js @@ -40,11 +40,17 @@ module.exports = async (ctx) => { item.title = content('h1').text(); + const article = content('section[data-test-id="ArticleBodyContent"]').html(); + const gallery = content('section[data-test-id="GalleryImage"]').html(); + if (ctx.params.caty === 'video') { item.description = content('div[data-test-id="EmbedIframe"]').html() + content('p[data-test-id="DekText"]').parent().html(); item.pubDate = new Date(detailResponse.data.match(/pubDate":"(.*?)","channel/)[1]).toUTCString(); + } else if (gallery !== null) { + item.description = gallery; + item.pubDate = new Date(detailResponse.data.match(/pubDate":"(.*?)","items/)[1]).toUTCString(); } else { - item.description = content('section[data-test-id="ArticleBodyContent"]').html(); + item.description = article; item.pubDate = new Date(detailResponse.data.match(/pubDate":"(.*?)","publishedRevisions/)[1]).toUTCString(); } From 48a95c138f67be23046168dc86a86c904fbb42b5 Mon Sep 17 00:00:00 2001 From: Jonathan Goh <3853064+hang333@users.noreply.github.com> Date: Sun, 27 Sep 2020 00:15:05 +0800 Subject: [PATCH 29/44] =?UTF-8?q?feat:=20=E4=B8=AD=E5=9B=BD=E9=A9=BB?= =?UTF-8?q?=E5=A4=96=E4=BD=BF=E9=A2=86=E9=A6=86=E9=80=9A=E7=9F=A5=E8=AE=A2?= =?UTF-8?q?=E9=98=85=E6=BA=90=20add=20=E9=A9=BB=E9=A9=AC=E6=9D=A5=E8=A5=BF?= =?UTF-8?q?=E4=BA=9A=E5=A4=A7=E4=BD=BF=E9=A6=86=E6=94=AF=E6=8C=81=20(#5736?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/government.md | 6 ++++++ lib/routes/embassy/supportedList.js | 9 +++++++++ 2 files changed, 15 insertions(+) diff --git a/docs/government.md b/docs/government.md index cd973be0ad..fdf5ca977b 100644 --- a/docs/government.md +++ b/docs/government.md @@ -380,6 +380,12 @@ pageClass: routes * * * +#### 马来西亚 `MY` + +- 大使馆: `/embassy/my` + +* * * + #### 新加坡 `SG` - 大使馆: `/embassy/sg` diff --git a/lib/routes/embassy/supportedList.js b/lib/routes/embassy/supportedList.js index 1a37e2e122..e2327ff6b7 100644 --- a/lib/routes/embassy/supportedList.js +++ b/lib/routes/embassy/supportedList.js @@ -147,6 +147,15 @@ module.exports = { }, }, }, + my: { + country: 'my', + countryCN: '马来西亚', + link: 'http://my.china-embassy.org/chn/zytz/', + list: 'div[class=nbox_ul] > ul a', + title: '#News_Body_Title', + description: '#News_Body_Txt_A', + pubDate: '#News_Body_Time', + }, sg: { country: 'sg', countryCN: '新加坡', From 3d527390aef00ae6ec502c6a72eafa8e81ce73a4 Mon Sep 17 00:00:00 2001 From: Luyu Huang Date: Sun, 27 Sep 2020 00:19:00 +0800 Subject: [PATCH 30/44] fix: ithome failure (#5727) --- lib/routes/ithome/index.js | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/routes/ithome/index.js b/lib/routes/ithome/index.js index 66a41d2263..7dc1a49df3 100644 --- a/lib/routes/ithome/index.js +++ b/lib/routes/ithome/index.js @@ -43,7 +43,7 @@ module.exports = async (ctx) => { }); const $ = cheerio.load(response.data); - const list = $('div.wrap_left_top div.mf div.new-list-1 li[class!="top"] span.title a') + const list = $('#list > div.fl > ul > li > div > h2 > a') .slice(0, 10) .map((_, item) => { item = $(item); @@ -60,16 +60,15 @@ module.exports = async (ctx) => { await ctx.cache.tryGet(item.link, async () => { const res = await got({ method: 'get', url: item.link }); const content = cheerio.load(res.data); - const post = content('div#con div.content'); - const paragraph = post.find('div#paragraph'); - paragraph.find('img[data-original]').each((_, ele) => { + const post = content('#paragraph'); + post.find('img[data-original]').each((_, ele) => { ele = $(ele); ele.attr('src', ele.attr('data-original')); ele.removeAttr('class'); ele.removeAttr('data-original'); }); - item.description = paragraph.html(); - item.pubDate = new Date(post.find('span#pubtime_baidu').text() + ' GMT+8').toUTCString(); + item.description = post.html(); + item.pubDate = new Date(content('#pubtime_baidu').text() + ' GMT+8').toUTCString(); return item; }) ) From c3ff574bf93be6aa4221948055693256729ed843 Mon Sep 17 00:00:00 2001 From: Ethan Shen Date: Sun, 27 Sep 2020 00:21:58 +0800 Subject: [PATCH 31/44] feat: add notefolio (#5730) Co-authored-by: Henry Wang --- docs/design.md | 23 ++++++++++++ docs/en/design.md | 23 ++++++++++++ lib/router.js | 3 ++ lib/routes/notefolio/index.js | 68 +++++++++++++++++++++++++++++++++++ 4 files changed, 117 insertions(+) create mode 100644 lib/routes/notefolio/index.js diff --git a/docs/design.md b/docs/design.md index 7be30d95a5..5b35b64762 100644 --- a/docs/design.md +++ b/docs/design.md @@ -83,6 +83,29 @@ Behance 用户主页 URL 获取用户名，如 +## Notefolio + +### Works + + + +| 分类 | 韩文分类名 | 中文分类名 | +| ---- | --------------- | --------------- | +| all | 전체 | 全部 | +| A7 | 공예 | 工艺品 | +| J7 | 그래픽 디자인 | 平面设计 | +| B7 | 디지털 아트 | 数字艺术 | +| C7 | 영상/모션그래픽 | 视频 / 图形动画 | +| D7 | 브랜딩/편집 | 品牌创建 / 编辑 | +| E7 | 산업 디자인 | 工业设计 | +| F7 | UI/UX | UI/UX | +| G7 | 일러스트레이션 | 插画 | +| K7 | 타이포그래피 | 字体 | +| H7 | 파인아트 | 纯艺术 | +| I7 | 포토그래피 | 摄影 | + + + ## UI 中国 ### 推荐文章 diff --git a/docs/en/design.md b/docs/en/design.md index e945105cb0..7f96898568 100644 --- a/docs/en/design.md +++ b/docs/en/design.md @@ -77,6 +77,29 @@ Default is under 'https://www.methodstudios.com/en/features'. +## Notefolio + +### Works + + + +| Category | Name in Korean | Name in English | +| -------- | -------------- | --------------- | +| all | 전체 | All | +| A7 | 공예 | Crafts | +| J7 | 그래픽 디자인 | Graphic Design | +| B7 | 디지털 아트 | Digital Art | +| C7 | 영상/모션그래픽 | Video / Motion Graphics | +| D7 | 브랜딩/편집 | Branding / Editing | +| E7 | 산업 디자인 | Industrial Design | +| F7 | UI/UX | UI/UX | +| G7 | 일러스트레이션 | Illustration | +| K7 | 타이포그래피 | Typography | +| H7 | 파인아트 | Fine Art | +| I7 | 포토그래피 | Photography | + + + ## Unit Image ### Films diff --git a/lib/router.js b/lib/router.js index 25c0cee9c7..bd3280552a 100644 --- a/lib/router.js +++ b/lib/router.js @@ -3241,6 +3241,9 @@ router.get('/appsales/:caty?/:time?', require('./routes/appsales/index')); // CGTN router.get('/cgtn/top', require('./routes/cgtn/top')); +// notefolio +router.get('/notefolio/:caty?/:order?/:time?/:query?', require('./routes/notefolio/index')); + // JavDB router.get('/javdb/home/:caty?/:sort?/:filter?', require('./routes/javdb/home')); router.get('/javdb/search/:keyword?/:filter?', require('./routes/javdb/search')); diff --git a/lib/routes/notefolio/index.js b/lib/routes/notefolio/index.js new file mode 100644 index 0000000000..c92317858f --- /dev/null +++ b/lib/routes/notefolio/index.js @@ -0,0 +1,68 @@ +const got = require('@/utils/got'); +const cheerio = require('cheerio'); + +module.exports = async (ctx) => { + const caty = ctx.params.caty || 'all'; + const order = ctx.params.order || 'pick'; + const time = ctx.params.time || 'all'; + const query = ctx.params.query || ''; + + const rootUrl = `https://notefolio.net/?work_categories=${caty === 'all' ? '' : caty}&order=${order}&from=${time}&q=${query}`; + const response = await got({ + method: 'get', + url: rootUrl, + }); + + const $ = cheerio.load(response.data); + + $('div.mobile-category-order').remove(); + + const list = $('div.work-list div.work-list-item') + .slice(0, 5) + .map((_, item) => { + item = $(item); + const a = item.find('a.go-to-work-info').eq(0); + return { + title: item.find('div.title').text(), + link: `https://notefolio.net${a.attr('href')}`, + author: item.find('span.go-profile-area').text(), + }; + }) + .get(); + + const items = await Promise.all( + list.map( + async (item) => + await ctx.cache.tryGet(item.link, async () => { + const detailResponse = await got({ + method: 'get', + url: item.link, + }); + + const content = cheerio.load(detailResponse.data); + + item.description = '
    '; + + content('li.block-image a img').each((_, i) => { + i = content(i); + item.description += `
    • `; + }); + + content('li.block-video iframe').each((_, i) => { + i = content(i); + item.description += `