From 7342326a2a91583a6bf048db2f2acae43211ba2b Mon Sep 17 00:00:00 2001 From: Ethan Shen <42264778+nczitzk@users.noreply.github.com> Date: Sat, 27 Aug 2022 20:32:55 +0800 Subject: [PATCH 1/8] =?UTF-8?q?feat(route):=20add=20=E8=89=BE=E7=91=9E?= =?UTF-8?q?=E5=92=A8=E8=AF=A2=E5=91=A8=E5=BA=A6=E5=B8=82=E5=9C=BA=E8=A7=82?= =?UTF-8?q?=E5=AF=9F=20(#10626)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/other.md | 9 +++++++ lib/v2/iresearch/maintainer.js | 1 + lib/v2/iresearch/radar.js | 8 +++++- lib/v2/iresearch/router.js | 1 + lib/v2/iresearch/templates/weekly.art | 11 ++++++++ lib/v2/iresearch/weekly.js | 39 +++++++++++++++++++++++++++ 6 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 lib/v2/iresearch/templates/weekly.art create mode 100644 lib/v2/iresearch/weekly.js diff --git a/docs/other.md b/docs/other.md index 9eb8c156df..2a868d5d97 100644 --- a/docs/other.md +++ b/docs/other.md @@ -334,6 +334,15 @@ type 为 all 时，category 参数不支持 cost 和 free +### 周度市场观察 + + + +| 家电行业 | 服装行业 | 美妆行业 | 食品饮料行业 | +| ---- | ---- | ---- | ------ | + + + ## 爱 Q 生活网 ### 最近更新 diff --git a/lib/v2/iresearch/maintainer.js b/lib/v2/iresearch/maintainer.js index 407334da12..d743147c36 100644 --- a/lib/v2/iresearch/maintainer.js +++ b/lib/v2/iresearch/maintainer.js @@ -1,3 +1,4 @@ module.exports = { '/report': ['brilon', 'Fatpandac'], + '/weekly/:category?': ['nczitzk'], }; diff --git a/lib/v2/iresearch/radar.js b/lib/v2/iresearch/radar.js index e96a6e00d5..aebb77f0e3 100644 --- a/lib/v2/iresearch/radar.js +++ b/lib/v2/iresearch/radar.js @@ -4,10 +4,16 @@ module.exports = { www: [ { title: '研究报告', - docs: 'https://docs.rsshub.app/journal.html#ieee-xplore', + docs: 'https://docs.rsshub.app/other.html#ai-rui-chan-ye-yan-jiu-bao-gao', source: ['/report.shtml'], target: '/iresearch/report', }, + { + title: '周度市场观察', + docs: 'https://docs.rsshub.app/other.html#ai-rui-zhou-du-shi-chang-guan-cha', + source: ['/report.shtml?type=3', ''], + target: '/iresearch/weekly/:category', + }, ], }, }; diff --git a/lib/v2/iresearch/router.js b/lib/v2/iresearch/router.js index a7bfc0bbde..d2fdad0df9 100644 --- a/lib/v2/iresearch/router.js +++ b/lib/v2/iresearch/router.js @@ -1,3 +1,4 @@ module.exports = function (router) { router.get('/report', require('./report')); + router.get('/weekly/:category?', require('./weekly')); }; diff --git a/lib/v2/iresearch/templates/weekly.art b/lib/v2/iresearch/templates/weekly.art new file mode 100644 index 0000000000..ccf67d2b41 --- /dev/null +++ b/lib/v2/iresearch/templates/weekly.art @@ -0,0 +1,11 @@ +{{ if cover }} +{{ content }}

+{{ /if }} +{{ if pages }} +<% for (let i = 1; i <= pages; i++){ %> + +<% } %> +{{ /if }} \ No newline at end of file diff --git a/lib/v2/iresearch/weekly.js b/lib/v2/iresearch/weekly.js new file mode 100644 index 0000000000..f0d516b4a0 --- /dev/null +++ b/lib/v2/iresearch/weekly.js @@ -0,0 +1,39 @@ +const got = require('@/utils/got'); +const { parseDate } = require('@/utils/parse-date'); +const { art } = require('@/utils/render'); +const path = require('path'); + +module.exports = async (ctx) => { + const category = ctx.params.category ?? ''; + + const rootUrl = 'https://www.iresearch.com.cn'; + const currentUrl = `${rootUrl}/report.shtml?type=4`; + const apiUrl = `${rootUrl}/api/json/report/ireport.json`; + + const response = await got({ + method: 'get', + url: apiUrl, + }); + + const items = JSON.parse(response.data.slice(1)) + .slice(0, ctx.query.limit ? parseInt(ctx.query.limit) : 200) + .filter((item) => (category ? item.classname === category : true)) + .map((item) => ({ + title: item.reportname, + pubDate: parseDate(item.addtime), + link: `${rootUrl}/report/detail?id=${item.id}`, + category: [item.classname].concat(item.keywords.split(',')), + description: art(path.join(__dirname, 'templates/weekly.art'), { + id: item.id, + cover: item.reportpic, + content: item.shortcoutent, + pages: item.PagesCount, + }), + })); + + ctx.state.data = { + title: '艾瑞咨询 - 周度市场观察', + link: currentUrl, + item: items, + }; +}; From 043ec5b4770355870a3d2734f6496d1692519290 Mon Sep 17 00:00:00 2001 From: shiningdracon Date: Sat, 27 Aug 2022 05:51:12 -0700 Subject: [PATCH 2/8] fix: trow.cc session cookie (#10627) --- lib/v2/trow/portal.js | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/lib/v2/trow/portal.js b/lib/v2/trow/portal.js index 0e0b6a6254..b2cc303c6b 100644 --- a/lib/v2/trow/portal.js +++ b/lib/v2/trow/portal.js @@ -4,11 +4,21 @@ const { parseDate } = require('@/utils/parse-date'); const timezone = require('@/utils/timezone'); module.exports = async (ctx) => { - const response = await got({ - method: 'get', + let data; + const response = await got.extend({ followRedirect: false }).get({ url: `https://trow.cc`, }); - const data = response.data; + if (response.statusCode === 302) { + const response2 = await got.extend({ followRedirect: false }).get({ + url: `https://trow.cc`, + headers: { + cookie: response.headers['set-cookie'], + }, + }); + data = response2.data; + } else { + data = response.data; + } const $ = cheerio.load(data); const list = $('#portal_content .borderwrap[style="display:show"]'); From f806d0c1453d619695a8ff9355ba8cdea370a410 Mon Sep 17 00:00:00 2001 From: Tony Date: Sat, 27 Aug 2022 15:07:44 +0000 Subject: [PATCH 3/8] docs: replace algolia with meilisearch --- .github/workflows/scrape.yml | 26 +++++++++ docs/.vuepress/config.js | 17 +++--- package.json | 2 + scripts/docs-scraper/docs.rsshub.app.json | 21 ++++++++ yarn.lock | 65 ++++++++++++++++++++++- 5 files changed, 122 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/scrape.yml create mode 100644 scripts/docs-scraper/docs.rsshub.app.json diff --git a/.github/workflows/scrape.yml b/.github/workflows/scrape.yml new file mode 100644 index 0000000000..614c60b77f --- /dev/null +++ b/.github/workflows/scrape.yml @@ -0,0 +1,26 @@ +name: Update meilisearch index + +on: + push: + branches: + - master + paths: + - 'docs/**' + +jobs: + scrape-docs: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Run docs-scraper + env: + HOST_URL: ${{ secrets.MEILISEARCH_HOST_URL }} + API_KEY: ${{ secrets.MEILISEARCH_API_KEY }} + CONFIG_FILE_PATH: ${{ github.workspace }}/scripts/docs-scraper/docs.rsshub.app.json + run: | + docker run -t --rm \ + -e MEILISEARCH_HOST_URL=$HOST_URL \ + -e MEILISEARCH_API_KEY=$API_KEY \ + -v $CONFIG_FILE_PATH:/docs-scraper/config.json \ + getmeili/docs-scraper pipenv run ./docs_scraper config.json diff --git a/docs/.vuepress/config.js b/docs/.vuepress/config.js index 76467d5f7e..a2cce93aae 100644 --- a/docs/.vuepress/config.js +++ b/docs/.vuepress/config.js @@ -20,6 +20,15 @@ module.exports = { }, }, '@vuepress/back-to-top': true, + sitemap: { + hostname: 'https://docs.rsshub.app', + }, + 'vuepress-plugin-meilisearch': { + hostUrl: 'https://meilisearch.rsshub.app', + apiKey: 'd3faa641522b6fb1e1bfaeba838c0627c1a5d235988a8df224cdd6b35abfab22', + indexUid: 'rsshub', + maxSuggestions: 14, + }, }, locales: { '/': { @@ -73,14 +82,6 @@ module.exports = { editLinks: true, docsDir: 'docs', smoothScroll: true, - algolia: { - apiKey: '4c5f9a5a2c10e6bec73c0f07dbe356a6', - indexName: 'rsshub', - appId: 'I63CFUQ7XF', - algoliaOptions: { - hitsPerPage: 14, - }, - }, locales: { '/': { lang: 'zh-CN', diff --git a/package.json b/package.json index 98a93f6938..06fed2bbc1 100644 --- a/package.json +++ b/package.json @@ -74,6 +74,8 @@ "tosource": "2.0.0-alpha.3", "unified": "9.2.2", "vuepress": "1.9.7", + "vuepress-plugin-meilisearch": "0.12.2", + "vuepress-plugin-sitemap": "2.3.1", "yorkie": "2.0.0" }, "dependencies": { diff --git a/scripts/docs-scraper/docs.rsshub.app.json b/scripts/docs-scraper/docs.rsshub.app.json new file mode 100644 index 0000000000..7e67509aa3 --- /dev/null +++ b/scripts/docs-scraper/docs.rsshub.app.json @@ -0,0 +1,21 @@ +{ + "index_uid": "rsshub", + "sitemap_urls": ["https://docs.rsshub.app/sitemap.xml"], + "start_urls": ["https://docs.rsshub.app"], + "selectors": { + "lvl0": { + "selector": ".sidebar-heading.open", + "global": true, + "default_value": "路由" + }, + "lvl1": ".theme-default-content h1", + "lvl2": ".theme-default-content h2", + "lvl3": ".theme-default-content h3", + "lvl4": ".theme-default-content h4", + "lvl5": ".theme-default-content h5", + "text": ".theme-default-content p, .docs-content li" + }, + "strip_chars": " .,;:#", + "scrap_start_urls": true, + "min_indexed_level": 1 +} diff --git a/yarn.lock b/yarn.lock index eafedc3e09..964dccb8af 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3007,6 +3007,13 @@ autocomplete.js@0.36.0: dependencies: immediate "^3.2.3" +autocomplete.js@^0.38.1: + version "0.38.1" + resolved "https://registry.yarnpkg.com/autocomplete.js/-/autocomplete.js-0.38.1.tgz#9b006c985d996165ebbc62af33f5b4c32d209cc2" + integrity sha512-6pSJzuRMY3pqpozt+SXThl2DmJfma8Bi3SVFbZHS0PW/N72bOUv+Db0jAh2cWOhTsA4X+GNmKvIl8wExJTnN9w== + dependencies: + immediate "^3.2.3" + autoprefixer@^9.5.1: version "9.8.8" resolved "https://registry.yarnpkg.com/autoprefixer/-/autoprefixer-9.8.8.tgz#fd4bd4595385fa6f06599de749a4d5f7a474957a" @@ -4391,7 +4398,7 @@ cross-env@7.0.3: dependencies: cross-spawn "^7.0.1" -cross-fetch@3.1.5: +cross-fetch@3.1.5, cross-fetch@^3.1.5: version "3.1.5" resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-3.1.5.tgz#e1389f44d9e7ba767907f7af8454787952ab534f" integrity sha512-lvb1SBsI0Z7GDwmuid+mU3kWVBwTVUbe7S0H52yaaAdQOXq2YktTCZdlAcNKFzE6QtRz0snpw9bNiPeOIkkQvw== @@ -5044,6 +5051,16 @@ dns-txt@^2.0.2: dependencies: buffer-indexof "^1.0.0" +docs-searchbar.js@^2.1.1: + version "2.2.0" + resolved "https://registry.yarnpkg.com/docs-searchbar.js/-/docs-searchbar.js-2.2.0.tgz#e60eb0ee4f67853c41fdf7d3a4ef6ac7ac2c8027" + integrity sha512-rleKMiTkxG14M+6yt3+H8H0w/GKxWsNMinf4+zTVz9aVi04oJ8u37kktTiW3qkpPl9VQ7/i7Yv9wykukMqVxfw== + dependencies: + autocomplete.js "^0.38.1" + meilisearch "^0.27.0" + to-factory "^1.0.0" + zepto "^1.2.0" + docsearch.js@^2.5.2: version "2.6.3" resolved "https://registry.yarnpkg.com/docsearch.js/-/docsearch.js-2.6.3.tgz#57cb4600d3b6553c677e7cbbe6a734593e38625d" @@ -9111,6 +9128,11 @@ lodash.bind@^4.1.4: resolved "https://registry.yarnpkg.com/lodash.bind/-/lodash.bind-4.2.1.tgz#7ae3017e939622ac31b7d7d7dcb1b34db1690d35" integrity sha512-lxdsn7xxlCymgLYo1gGvVrfHmkjDiyqVv62FAeF2i5ta72BipE1SLxw8hPEPLhD4/247Ijw07UQH7Hq/chT5LA== +lodash.chunk@^4.2.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/lodash.chunk/-/lodash.chunk-4.2.0.tgz#66e5ce1f76ed27b4303d8c6512e8d1216e8106bc" + integrity sha512-ZzydJKfUHJwHa+hF5X66zLFCBrWn5GeF28OHEr4WVWtNDXlQ/IjWKPBiikqKo2ne0+v6JgCgJ0GzJp8k8bHC7w== + lodash.clonedeep@^4.5.0: version "4.5.0" resolved "https://registry.yarnpkg.com/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz#e23f3f9c4f8fbdde872529c1071857a086e5ccef" @@ -9166,6 +9188,11 @@ lodash.merge@^4.4.0, lodash.merge@^4.6.2: resolved "https://registry.yarnpkg.com/lodash.merge/-/lodash.merge-4.6.2.tgz#558aa53b43b661e1925a0afdfa36a9a1085fe57a" integrity sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ== +lodash.padstart@^4.6.1: + version "4.6.1" + resolved "https://registry.yarnpkg.com/lodash.padstart/-/lodash.padstart-4.6.1.tgz#d2e3eebff0d9d39ad50f5cbd1b52a7bce6bb611b" + integrity sha512-sW73O6S8+Tg66eY56DBk85aQzzUJDtpoXFBgELMd5P/SotAguo+1kYO6RuYgXxA4HJH3LFTFPASX6ET6bjfriw== + lodash.pick@^4.2.1: version "4.4.0" resolved "https://registry.yarnpkg.com/lodash.pick/-/lodash.pick-4.4.0.tgz#52f05610fff9ded422611441ed1fc123a03001b3" @@ -9558,6 +9585,13 @@ media-typer@0.3.0: resolved "https://registry.yarnpkg.com/media-typer/-/media-typer-0.3.0.tgz#8710d7af0aa626f8fffa1ce00168545263255748" integrity sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ== +meilisearch@^0.27.0: + version "0.27.0" + resolved "https://registry.yarnpkg.com/meilisearch/-/meilisearch-0.27.0.tgz#8bd57ddb77b975f93e054cb977b951c488ece297" + integrity sha512-kZOZFIuSO7c6xRf+Y2/9/h6A9pl0sCl/G44X4KuaSwxGbruOZPhmxbeVEgLHBv4pUFvQ56rNVTA/2d/5GCU1YA== + dependencies: + cross-fetch "^3.1.5" + memory-fs@^0.4.1: version "0.4.1" resolved "https://registry.yarnpkg.com/memory-fs/-/memory-fs-0.4.1.tgz#3a9a20b8462523e447cfbc7e8bb80ed667bfc552" @@ -12555,6 +12589,16 @@ sisteransi@^1.0.5: resolved "https://registry.yarnpkg.com/sisteransi/-/sisteransi-1.0.5.tgz#134d681297756437cc05ca01370d3a7a571075ed" integrity sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg== +sitemap@^3.0.0: + version "3.2.2" + resolved "https://registry.yarnpkg.com/sitemap/-/sitemap-3.2.2.tgz#3f77c358fa97b555c879e457098e39910095c62b" + integrity sha512-TModL/WU4m2q/mQcrDgNANn0P4LwprM9MMvG4hu5zP4c6IIKs2YLTu6nXXnNr8ODW/WFtxKggiJ1EGn2W0GNmg== + dependencies: + lodash.chunk "^4.2.0" + lodash.padstart "^4.6.1" + whatwg-url "^7.0.0" + xmlbuilder "^13.0.0" + slash@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/slash/-/slash-1.0.0.tgz#c41f2f6c39fc16d1cd17ad4b5d896114ae470d55" @@ -14145,6 +14189,20 @@ vuepress-plugin-container@^2.0.2: "@vuepress/shared-utils" "^1.2.0" markdown-it-container "^2.0.0" +vuepress-plugin-meilisearch@0.12.2: + version "0.12.2" + resolved "https://registry.yarnpkg.com/vuepress-plugin-meilisearch/-/vuepress-plugin-meilisearch-0.12.2.tgz#d130fa77a5f1ca5f8ae09af14e504f21779376d1" + integrity sha512-ug9+puCPySr/wJEnymWqFqbo3xsQfqXog2kFU8RPjrOQ20miQjgPYnF3u/8Pb+QT0s5zHA6ykT8lTBx0ovWsVw== + dependencies: + docs-searchbar.js "^2.1.1" + +vuepress-plugin-sitemap@2.3.1: + version "2.3.1" + resolved "https://registry.yarnpkg.com/vuepress-plugin-sitemap/-/vuepress-plugin-sitemap-2.3.1.tgz#51298aca77a5de96396fdbd1103e1637dd61ae6a" + integrity sha512-n+8lbukhrKrsI9H/EX0EBgkE1pn85LAQFvQ5dIvrZP4Kz6JxPOPPNTQmZMhahQV1tXbLZQCEN7A1WZH4x+arJQ== + dependencies: + sitemap "^3.0.0" + vuepress-plugin-smooth-scroll@^0.0.3: version "0.0.3" resolved "https://registry.yarnpkg.com/vuepress-plugin-smooth-scroll/-/vuepress-plugin-smooth-scroll-0.0.3.tgz#6eff2d4c186cca917cc9f7df2b0af7de7c8c6438" @@ -14758,6 +14816,11 @@ xml2js@0.4.23, xml2js@^0.4.19: sax ">=0.6.0" xmlbuilder "~11.0.0" +xmlbuilder@^13.0.0: + version "13.0.2" + resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-13.0.2.tgz#02ae33614b6a047d1c32b5389c1fdacb2bce47a7" + integrity sha512-Eux0i2QdDYKbdbA6AM6xE4m6ZTZr4G4xF9kahI2ukSEMCzwce2eX9WlTI5J3s+NU7hpasFsr8hWIONae7LluAQ== + xmlbuilder@^15.1.1: version "15.1.1" resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-15.1.1.tgz#9dcdce49eea66d8d10b42cae94a79c3c8d0c2ec5" From 3342cb465aa088b662d1e91a96d42ae2d2a4f689 Mon Sep 17 00:00:00 2001 From: Tony Date: Sat, 27 Aug 2022 15:23:27 +0000 Subject: [PATCH 4/8] docs: fix meilisearch apikey --- docs/.vuepress/config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/.vuepress/config.js b/docs/.vuepress/config.js index a2cce93aae..a29df462fe 100644 --- a/docs/.vuepress/config.js +++ b/docs/.vuepress/config.js @@ -25,7 +25,7 @@ module.exports = { }, 'vuepress-plugin-meilisearch': { hostUrl: 'https://meilisearch.rsshub.app', - apiKey: 'd3faa641522b6fb1e1bfaeba838c0627c1a5d235988a8df224cdd6b35abfab22', + apiKey: '375c36cd9573a2c1d1e536214158c37120fdd0ba6cd8829f7a848e940cc22245', indexUid: 'rsshub', maxSuggestions: 14, }, From 7c7d96013c2ad6d574925be2dda0b13f923d5a0e Mon Sep 17 00:00:00 2001 From: Tony Date: Sun, 28 Aug 2022 09:54:35 -0600 Subject: [PATCH 5/8] feat(route): xjtu dyyy (#10640) * feat(route): xjtu dyyy * fix: typo --- docs/university.md | 4 ++++ lib/v2/xjtu/dyyy/index.js | 44 +++++++++++++++++++++++++++++++++++++++ lib/v2/xjtu/maintainer.js | 1 + lib/v2/xjtu/radar.js | 8 +++++++ lib/v2/xjtu/router.js | 1 + 5 files changed, 58 insertions(+) create mode 100644 lib/v2/xjtu/dyyy/index.js diff --git a/docs/university.md b/docs/university.md index 7739212890..a7a72fd6a8 100644 --- a/docs/university.md +++ b/docs/university.md @@ -2710,6 +2710,10 @@ jsjxy.hbut.edu.cn 证书链不全，自建 RSSHub 可设置环境变量 NODE_TLS +### 第一附属医院新闻 + + + ### 第二附属医院新闻 diff --git a/lib/v2/xjtu/dyyy/index.js b/lib/v2/xjtu/dyyy/index.js new file mode 100644 index 0000000000..629720d623 --- /dev/null +++ b/lib/v2/xjtu/dyyy/index.js @@ -0,0 +1,44 @@ +const got = require('@/utils/got'); +const cheerio = require('cheerio'); +const { parseDate } = require('@/utils/parse-date'); + +const baseUrl = 'http://www.dyyy.xjtu.edu.cn'; + +module.exports = async (ctx) => { + const { path } = ctx.params; + const response = await got(`${baseUrl}/${path}.htm`); + + const $ = cheerio.load(response.data); + + const items = $('.list_right_con div li') + .toArray() + .map((item) => { + item = $(item); + return { + title: item.find('a').attr('title'), + link: new URL(item.find('a').attr('href'), response.url).href, + pubDate: parseDate(item.find('.data').text()), + }; + }); + + await Promise.all( + items.map((item) => + ctx.cache.tryGet(item.link, async () => { + const { data: response } = await got(item.link); + const $ = cheerio.load(response); + + item.author = $('.content_source') + .text() + .match(/责任编辑：(.*)\(点击/)[1]; + item.description = $('.content_con').html(); + return item; + }) + ) + ); + + ctx.state.data = { + title: $('head title').text(), + link: `${baseUrl}/${path}.htm`, + item: items, + }; +}; diff --git a/lib/v2/xjtu/maintainer.js b/lib/v2/xjtu/maintainer.js index 9e81888f06..bdd4b9f039 100644 --- a/lib/v2/xjtu/maintainer.js +++ b/lib/v2/xjtu/maintainer.js @@ -1,6 +1,7 @@ module.exports = { '/2yuan/news/:id?': ['nczitzk'], '/dean/:subpath+': ['hoilc'], + '/dyyy/:path+': ['TonyRL'], '/ee/:id?': ['DylanXie123'], '/gs/tzgg': ['nczitzk'], '/international/:subpath+': ['guitaoliu'], diff --git a/lib/v2/xjtu/radar.js b/lib/v2/xjtu/radar.js index 7594dfaac2..bace2feb6c 100644 --- a/lib/v2/xjtu/radar.js +++ b/lib/v2/xjtu/radar.js @@ -49,5 +49,13 @@ module.exports = { target: (params, url) => `/xjtu/std/${new URL(url).toString().match(/\/(\w+)\.htm/)[1]}`, }, ], + 'www.dyyy': [ + { + title: '第一附属医院新闻', + docs: 'https://docs.rsshub.app/university.html#xi-an-jiao-tong-da-xue', + source: ['/*'], + target: (_, url) => `/xjtu/dyyy${new URL(url).pathname.replace('.htm', '')}`, + }, + ], }, }; diff --git a/lib/v2/xjtu/router.js b/lib/v2/xjtu/router.js index 77640e936a..fcf1189718 100644 --- a/lib/v2/xjtu/router.js +++ b/lib/v2/xjtu/router.js @@ -1,6 +1,7 @@ module.exports = function (router) { router.get('/2yuan/news/:id?', require('./2yuan/news')); router.get('/dean/:subpath+', require('./dean')); + router.get('/dyyy/:path+', require('./dyyy/index')); router.get('/ee/:id?', require('./ee')); router.get('/gs/tzgg', require('./gs/tzgg')); router.get('/international/:subpath+', require('./international')); From 42768369ba80ab70af047c6bd3cf7ae8b53ac38d Mon Sep 17 00:00:00 2001 From: Tony Date: Mon, 29 Aug 2022 00:50:48 +0800 Subject: [PATCH 6/8] chore: wait for netlify before scraping --- .github/workflows/scrape.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/scrape.yml b/.github/workflows/scrape.yml index 614c60b77f..f3ef22e85f 100644 --- a/.github/workflows/scrape.yml +++ b/.github/workflows/scrape.yml @@ -5,12 +5,15 @@ on: branches: - master paths: + - '.github/workflows/scrape.yml' - 'docs/**' jobs: scrape-docs: runs-on: ubuntu-latest steps: + - name: Wait for Netlify to finish + run: sleep 3m - name: Checkout uses: actions/checkout@v3 - name: Run docs-scraper From e3ffd5d7c13ee94aec88b22fb6da75379f730f7f Mon Sep 17 00:00:00 2001 From: Tony Date: Sun, 28 Aug 2022 16:33:16 -0600 Subject: [PATCH 7/8] fix(route): mofcom (#10641) * fix(route): mofcom * fix: typo --- docs/government.md | 2 +- lib/router.js | 4 +-- lib/routes/mofcom/article.js | 54 ---------------------------- lib/v2/gov/maintainer.js | 1 + lib/v2/gov/mofcom/article.js | 68 ++++++++++++++++++++++++++++++++++++ lib/v2/gov/radar.js | 11 ++++++ lib/v2/gov/router.js | 1 + 7 files changed, 84 insertions(+), 57 deletions(-) delete mode 100644 lib/routes/mofcom/article.js create mode 100644 lib/v2/gov/mofcom/article.js diff --git a/docs/government.md b/docs/government.md index 5231109df7..6675a51899 100644 --- a/docs/government.md +++ b/docs/government.md @@ -1247,7 +1247,7 @@ pageClass: routes ### 政务公开 - + ## 中华人民共和国生态环境部 diff --git a/lib/router.js b/lib/router.js index ae3b97754c..1f8b037a37 100644 --- a/lib/router.js +++ b/lib/router.js @@ -1945,8 +1945,8 @@ router.get('/dhl/:id', lazyloadRouteHandler('./routes/dhl/shipment-tracking')); // Japanpost router.get('/japanpost/track/:reqCode/:locale?', lazyloadRouteHandler('./routes/japanpost/track')); -// 中华人民共和国商务部 -router.get('/mofcom/article/:suffix', lazyloadRouteHandler('./routes/mofcom/article')); +// 中华人民共和国商务部 migrated to v2 +// router.get('/mofcom/article/:suffix', lazyloadRouteHandler('./routes/mofcom/article')); // 品玩 router.get('/pingwest/status', lazyloadRouteHandler('./routes/pingwest/status')); diff --git a/lib/routes/mofcom/article.js b/lib/routes/mofcom/article.js deleted file mode 100644 index a7cb76d1cc..0000000000 --- a/lib/routes/mofcom/article.js +++ /dev/null @@ -1,54 +0,0 @@ -const cheerio = require('cheerio'); -const got = require('@/utils/got'); -const { resolve } = require('url'); - -const host = 'http://www.mofcom.gov.cn'; - -module.exports = async (ctx) => { - const suffix = ctx.params.suffix; - const url = resolve('http://www.mofcom.gov.cn/article/', suffix); - const res = await got.get(url); - const $ = cheerio.load(res.data); - const title = $('head > title').text(); - const list = $('.txtList_01 li').get(); - - const out = await Promise.all( - list.map(async (item) => { - const $ = cheerio.load(item); - const title = $('a').attr('title'); - const sub_url = $('a').attr('href'); - const itemUrl = resolve(host, sub_url); - const cache = await ctx.cache.get(itemUrl); - if (cache) { - return Promise.resolve(JSON.parse(cache)); - } - - let responses = await got.get(itemUrl); - const redirect = responses.data.match(/window\.location\.href='(.*)'/); - if (redirect) { - responses = await got.get(redirect[1], { - headers: { - Referer: itemUrl, - 'Accept-Encoding': 'gzip, deflate', - 'Accept-Language': 'zh-CN,zh;q=0.9', - Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3', - }, - }); - } - const $d = cheerio.load(responses.data); - - const single = { - title, - link: itemUrl, - description: $d('.art-con').html() || $d('.textlive').html(), - }; - ctx.cache.set(itemUrl, JSON.stringify(single)); - return Promise.resolve(single); - }) - ); - ctx.state.data = { - title, - link: url, - item: out, - }; -}; diff --git a/lib/v2/gov/maintainer.js b/lib/v2/gov/maintainer.js index edf9388ab5..3ec630e6ff 100644 --- a/lib/v2/gov/maintainer.js +++ b/lib/v2/gov/maintainer.js @@ -22,6 +22,7 @@ module.exports = { '/miit/wjgs': ['Yoge-Code'], '/miit/zcjd': ['Yoge-Code'], '/moe/:type': ['Crawler995'], + '/mofcom/article/:suffix+': ['LogicJake'], '/moj/aac/news/:type?': ['TonyRL'], '/nifdc/:path+': ['nczitzk'], '/nmpa/:path+': ['TonyRL'], diff --git a/lib/v2/gov/mofcom/article.js b/lib/v2/gov/mofcom/article.js new file mode 100644 index 0000000000..c5802bc84a --- /dev/null +++ b/lib/v2/gov/mofcom/article.js @@ -0,0 +1,68 @@ +const cheerio = require('cheerio'); +const got = require('@/utils/got'); +const { parseDate } = require('@/utils/parse-date'); +const timezone = require('@/utils/timezone'); + +const host = 'http://www.mofcom.gov.cn'; + +module.exports = async (ctx) => { + const suffix = ctx.params.suffix; + const url = `http://www.mofcom.gov.cn/article/${suffix}/`; + const { data: res } = await got(url); + const $ = cheerio.load(res); + $('.listline').remove(); + const list = $('.txtList_01 li') + .toArray() + .map((item) => { + item = $(item); + const date = + item.find('span').length !== 0 + ? item + .find('span') + .text() + .match(/((\d{4}-\d{2}-\d{2})(\s\d{2}:\d{2}:\d{2})?)/)[1] + : item + .find('a') + .attr('title') + .match(/(\d{4}年\d{1,2}月\d{1,2})/)[1]; + return { + title: item.find('a').attr('title'), + link: host + item.find('a').attr('href'), + pubDate: timezone(parseDate(date, ['YYYY-MM-DD HH:mm:ss', 'YYYY-MM-DD', 'YYYY年M月D']), 8), + }; + }); + + await Promise.all( + list.map((item) => + ctx.cache.tryGet(item.link, async () => { + let responses = await got(item.link); + // xwfb/xwlxfbh || xwfb/xwztfbh + const redirect = responses.data.match(/_cofing1=\{href:"(.*)",type/) || responses.data.match(/window\.location\.href='(.*)'/); + if (redirect) { + responses = await got(redirect[1], { + headers: { + Referer: item.link, + }, + }); + } + const $ = cheerio.load(responses.data); + const iframe = $('.art-con iframe').attr('src'); + if ($('.art-con iframe').attr('src')) { + const { data: res } = await got(iframe); + item.description = cheerio.load(res)('tbody').html(); + } else { + item.description = $('.art-con').html() || /* xwfb/xwztfbh */ $('.textlive').html(); + } + item.pubDate = $('meta[name="PubDate"]').length ? timezone(parseDate($('meta[name="PubDate"]').attr('content'), 'YYYY-MM-DD HH:mm'), 8) : item.pubDate; + + return item; + }) + ) + ); + ctx.state.data = { + title: $('head > title').text(), + description: $('meta[name=description]').attr('content'), + link: url, + item: list, + }; +}; diff --git a/lib/v2/gov/radar.js b/lib/v2/gov/radar.js index 4fde967f95..5a2f9b3437 100644 --- a/lib/v2/gov/radar.js +++ b/lib/v2/gov/radar.js @@ -526,6 +526,17 @@ module.exports = { }, ], }, + 'mofcom.gov.cn': { + _name: '中华人民共和国商务部', + '.': [ + { + title: '新闻发布', + docs: 'https://docs.rsshub.app/government.html#zhong-hua-ren-min-gong-he-guo-shang-wu-bu', + source: ['/article/*'], + target: (_, url) => `/gov/mofcom/article/${new URL(url).pathname.replace('/article/', '')}`, + }, + ], + }, 'moj.gov.tw': { _name: '台灣法務部廉政署', 'www.aac': [ diff --git a/lib/v2/gov/router.js b/lib/v2/gov/router.js index 61483fb1f3..12395f547d 100644 --- a/lib/v2/gov/router.js +++ b/lib/v2/gov/router.js @@ -14,6 +14,7 @@ module.exports = function (router) { router.get('/miit/wjgs', require('./miit/wjgs')); router.get('/miit/zcjd', require('./miit/zcjd')); router.get('/moe/:type', require('./moe/moe')); + router.get('/mofcom/article/:suffix+', require('./mofcom/article')); router.get('/moj/aac/news/:type?', require('./moj/aac/news')); router.get(/nifdc\/([\w/-]+)?/, require('./nifdc')); router.get(/\/nmpa\/([\w][\w/]+)?/, require('./nmpa/generic')); From 19dc2857c68a817ec6c22a3cb8bbf17cf78897b0 Mon Sep 17 00:00:00 2001 From: Tony Date: Sun, 28 Aug 2022 16:44:40 -0600 Subject: [PATCH 8/8] feat(route): stheadline std (#10642) --- docs/traditional-media.md | 6 ++++ lib/v2/stheadline/maintainer.js | 3 ++ lib/v2/stheadline/radar.js | 13 +++++++++ lib/v2/stheadline/router.js | 3 ++ lib/v2/stheadline/std/realtime.js | 46 +++++++++++++++++++++++++++++++ 5 files changed, 71 insertions(+) create mode 100644 lib/v2/stheadline/maintainer.js create mode 100644 lib/v2/stheadline/radar.js create mode 100644 lib/v2/stheadline/router.js create mode 100644 lib/v2/stheadline/std/realtime.js diff --git a/docs/traditional-media.md b/docs/traditional-media.md index 128c4a7757..ab639f15f8 100644 --- a/docs/traditional-media.md +++ b/docs/traditional-media.md @@ -1985,6 +1985,12 @@ category 对应的关键词有 +## 星島日報 + +### 即時 + + + ## 星洲网 ### 首页 diff --git a/lib/v2/stheadline/maintainer.js b/lib/v2/stheadline/maintainer.js new file mode 100644 index 0000000000..46206e7f34 --- /dev/null +++ b/lib/v2/stheadline/maintainer.js @@ -0,0 +1,3 @@ +module.exports = { + '/std/realtime/:category*': ['TonyRL'], +}; diff --git a/lib/v2/stheadline/radar.js b/lib/v2/stheadline/radar.js new file mode 100644 index 0000000000..bfdc2be3b1 --- /dev/null +++ b/lib/v2/stheadline/radar.js @@ -0,0 +1,13 @@ +module.exports = { + 'stheadline.com': { + _name: '星島日報', + std: [ + { + title: '即時', + docs: 'https://docs.rsshub.app/traditional-media.html#xing-dao-ri-bao', + source: ['/realtime/*category'], + target: (params) => `/stdheadline/std/realtime/${params.category}`, + }, + ], + }, +}; diff --git a/lib/v2/stheadline/router.js b/lib/v2/stheadline/router.js new file mode 100644 index 0000000000..ee345be8b5 --- /dev/null +++ b/lib/v2/stheadline/router.js @@ -0,0 +1,3 @@ +module.exports = (router) => { + router.get('/std/realtime/:category*', require('./std/realtime')); +}; diff --git a/lib/v2/stheadline/std/realtime.js b/lib/v2/stheadline/std/realtime.js new file mode 100644 index 0000000000..2fea18068b --- /dev/null +++ b/lib/v2/stheadline/std/realtime.js @@ -0,0 +1,46 @@ +const got = require('@/utils/got'); +const cheerio = require('cheerio'); +const { parseDate } = require('@/utils/parse-date'); +const timezone = require('@/utils/timezone'); + +const baseUrl = 'https://std.stheadline.com'; + +module.exports = async (ctx) => { + const { category = '即時' } = ctx.params; + const url = `${baseUrl}/realtime/${category}`; + const { data: response } = await got(url); + const $ = cheerio.load(response); + + const items = $('.col-md-9 .media-body .h5 a') + .toArray() + .map((item) => { + item = $(item); + return { + title: item.attr('title'), + link: item.attr('href'), + guid: item.attr('href').substring(0, item.attr('href').lastIndexOf('/')), + }; + }); + + await Promise.all( + items.map((item) => + ctx.cache.tryGet(item.link, async () => { + const { data: response } = await got(item.link); + const $ = cheerio.load(response); + + item.description = $('.paragraphs').html(); + item.pubDate = timezone(parseDate($('.content .date').text())); + + return item; + }) + ) + ); + + ctx.state.data = { + title: $('head title').text(), + description: $('meta[name=description]').attr('content'), + image: 'https://std.stheadline.com/dist/images/favicon/icon-512.png', + link: url, + item: items, + }; +};