From a8efd75456ce4fb1d927c76ea072de6b7e3663de Mon Sep 17 00:00:00 2001 From: Tony Date: Mon, 29 Aug 2022 06:44:04 -0600 Subject: [PATCH 01/17] fix(route): tieba forum split (#10644) --- lib/v2/baidu/tieba/forum.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/v2/baidu/tieba/forum.js b/lib/v2/baidu/tieba/forum.js index 1845339b9d..622e8df254 100644 --- a/lib/v2/baidu/tieba/forum.js +++ b/lib/v2/baidu/tieba/forum.js @@ -39,13 +39,12 @@ module.exports = async (ctx) => { .toArray() .map((element) => { const item = $(element); - return ``; + return ``; }) .join(''); return { title, - // description: `

${details}

${medias}

作者：${author_name}

`, description: art(path.join(__dirname, '../templates/forum.art'), { details, medias, From b4f341fef0c3e4d40ac25081858a60c9eb321ea0 Mon Sep 17 00:00:00 2001 From: Cyrus <48074861+Xm798@users.noreply.github.com> Date: Mon, 29 Aug 2022 23:19:13 +0800 Subject: [PATCH 02/17] =?UTF-8?q?fix(route):=20=E5=8D=97=E4=BA=AC=E8=88=AA?= =?UTF-8?q?=E7=A9=BA=E8=88=AA=E5=A4=A9=E5=A4=A7=E5=AD=A6=E7=9B=B8=E5=85=B3?= =?UTF-8?q?=E8=B7=AF=E7=94=B1=20(#10646)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/university.md | 32 +++--- lib/v2/nuaa/college/cae.js | 85 ++++++++++++++++ lib/v2/nuaa/{cs/index.js => college/cs.js} | 18 ++-- lib/v2/nuaa/jwc/jwc.js | 16 +-- lib/v2/nuaa/maintainer.js | 5 +- lib/v2/nuaa/radar.js | 6 ++ lib/v2/nuaa/router.js | 5 +- lib/v2/nuaa/utils/pypasswaf.js | 4 +- lib/v2/nuaa/yjsy/yjsy.js | 107 +++++++++++++-------- 9 files changed, 201 insertions(+), 77 deletions(-) create mode 100644 lib/v2/nuaa/college/cae.js rename lib/v2/nuaa/{cs/index.js => college/cs.js} (73%) diff --git a/docs/university.md b/docs/university.md index a7a72fd6a8..11d5e8cfff 100644 --- a/docs/university.md +++ b/docs/university.md @@ -1811,7 +1811,7 @@ jsjxy.hbut.edu.cn 证书链不全，自建 RSSHub 可设置环境变量 NODE_TLS ### 教务处 - + | 通知公告 | 教学服务 | 教学建设 | 学生培养 | 教学资源 | | ---- | ---- | ---- | ---- | ---- | @@ -1819,9 +1819,27 @@ jsjxy.hbut.edu.cn 证书链不全，自建 RSSHub 可设置环境变量 NODE_TLS +### 研究生院 + + + +| 通知公告 | 新闻动态 | 学术信息 | 师生风采 | +| ---- | ---- | ---- | ---- | +| tzgg | xwdt | xsxx | ssfc | + + + +### 自动化学院 + + + +| 综合新闻 | 党委行政 | 人事 / 合作 | 研究生培养 | 本科生培养 | 学生工作 | 通知公告 | 学术信息 | 答辩公告 | +| ---- | ---- | ------- | ----- | ----- | ---- | ---- | ---- | ---- | +| zhxw | dwxz | rshz | yjs | bks | xsgz | tzgg | xsxx | dbgg | + ### 计算机科学与技术学院 - + | 通知公告 | 热点新闻 | 学科科研 | 教学动态 | 本科生培养 | 研究生培养 | 学生工作 | | ---- | ---- | ---- | ---- | ----- | ----- | ---- | @@ -1829,16 +1847,6 @@ jsjxy.hbut.edu.cn 证书链不全，自建 RSSHub 可设置环境变量 NODE_TLS -### 研究生院 - - - -| 最近动态 | 研院新闻 | 上级文件 | 管理文件 | 信息服务 | -| ------ | ---- | ---- | ---- | ---- | -| latest | yyxw | sjwj | glwj | xxfw | - - - ## 南京理工大学 ### 教务处 diff --git a/lib/v2/nuaa/college/cae.js b/lib/v2/nuaa/college/cae.js new file mode 100644 index 0000000000..c2f2a8822f --- /dev/null +++ b/lib/v2/nuaa/college/cae.js @@ -0,0 +1,85 @@ +const got = require('@/utils/got'); +const cheerio = require('cheerio'); +const { parseDate } = require('@/utils/parse-date'); +const getCookie = require('../utils/pypasswaf'); +const host = 'https://cae.nuaa.edu.cn/'; + +const map = new Map([ + ['zhxw', { title: '综合新闻 | 南京航空航天大学自动化学院', suffix: '5399/list.htm' }], + ['dwxz', { title: '党委行政 | 南京航空航天大学自动化学院', suffix: '13266/list.htm' }], + ['rshz', { title: '人事/合作 | 南京航空航天大学自动化学院', suffix: '13267/list.htm' }], + ['yjs', { title: '研究生培养 | 南京航空航天大学自动化学院', suffix: '13271/list.htm' }], + ['bks', { title: '本科生培养 | 南京航空航天大学自动化学院', suffix: '13270/list.htm' }], + ['xsgz', { title: '学生工作 | 南京航空航天大学自动化学院', suffix: '13268/list.htm' }], + ['tzgg', { title: '通知公告 | 南京航空航天大学自动化学院', suffix: '13264/list.htm' }], + ['xsxx', { title: '学术信息 | 南京航空航天大学自动化学院', suffix: '13265/list.htm' }], + ['dbgg', { title: '答辩公告 | 南京航空航天大学自动化学院', suffix: 'dbgg/list.htm' }], +]); + +module.exports = async (ctx) => { + const type = ctx.params.type; + const suffix = map.get(type).suffix; + const getDescription = Boolean(ctx.params.getDescription) || false; + const link = new URL(suffix, host).href; + const cookie = await getCookie(host); + const gotConfig = { + headers: { + cookie, + }, + }; + const response = await got(link, gotConfig); + const $ = cheerio.load(response.data); + + const list = $('#wp_news_w6 ul li') + .slice(0, 10) + .map(function () { + const info = { + title: $(this).find('a').text(), + link: $(this).find('a').attr('href'), + date: $(this).find('span').text(), + }; + return info; + }) + .get(); + + const out = await Promise.all( + list.map((info) => { + const title = info.title || 'zhxw'; + const date = info.date; + const itemUrl = new URL(info.link, host).href; + + return ctx.cache.tryGet(itemUrl, async () => { + const arr = itemUrl.split('.'); + const pageType = arr[arr.length - 1]; + + // 南航新 WAF 过于敏感 + // 目前 description 需要遍历页面，会被 WAF 拦截导致无法输出 + // 考虑换一种获取 description 的方式或者将标题当作 title。 + let description = title + '
查看原文'; + if (getDescription) { + description = itemUrl; + if (pageType === 'htm' || pageType === 'html') { + const response = await got(itemUrl, gotConfig); + const $ = cheerio.load(response.data); + description = $('.wp_articlecontent').html() + '
查看原文'; + } + } + + const single = { + title, + link: itemUrl, + description, + pubDate: parseDate(date), + }; + return single; + }); + }) + ); + + ctx.state.data = { + title: map.get(type).title, + link, + description: '南京航空航天大学自动化学院RSS', + item: out, + }; +}; diff --git a/lib/v2/nuaa/cs/index.js b/lib/v2/nuaa/college/cs.js similarity index 73% rename from lib/v2/nuaa/cs/index.js rename to lib/v2/nuaa/college/cs.js index bc3cb3571d..36eae3327e 100644 --- a/lib/v2/nuaa/cs/index.js +++ b/lib/v2/nuaa/college/cs.js @@ -2,16 +2,16 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); const { parseDate } = require('@/utils/parse-date'); const getCookie = require('../utils/pypasswaf'); -const host = 'http://cs.nuaa.edu.cn/'; +const host = 'https://cs.nuaa.edu.cn/'; const map = new Map([ - ['tzgg', { title: '南京航空航天大学计算机科学与技术学院 -- 通知公告', suffix: 'tzgg/list.htm' }], - ['rdxw', { title: '南京航空航天大学计算机科学与技术学院 -- 热点新闻', suffix: '10846/list.htm' }], - ['xkky', { title: '南京航空航天大学计算机科学与技术学院 -- 学科科研', suffix: '10849/list.htm' }], - ['be', { title: '南京航空航天大学计算机科学与技术学院 -- 本科生培养', suffix: '10850/list.htm' }], - ['me', { title: '南京航空航天大学计算机科学与技术学院 -- 研究生培养', suffix: '10851/list.htm' }], - ['jxdt', { title: '南京航空航天大学计算机科学与技术学院 -- 教学动态', suffix: '1977/list.htm' }], - ['xsgz', { title: '南京航空航天大学计算机科学与技术学院 -- 学生工作', suffix: '1959/list.htm' }], + ['tzgg', { title: '通知公告 | 南京航空航天大学计算机科学与技术学院', suffix: 'tzgg/list.htm' }], + ['rdxw', { title: '热点新闻 | 南京航空航天大学计算机科学与技术学院', suffix: '10846/list.htm' }], + ['xkky', { title: '学科科研 | 南京航空航天大学计算机科学与技术学院', suffix: '10849/list.htm' }], + ['be', { title: '本科生培养 | 南京航空航天大学计算机科学与技术学院', suffix: '10850/list.htm' }], + ['me', { title: '研究生培养 | 南京航空航天大学计算机科学与技术学院', suffix: '10851/list.htm' }], + ['jxdt', { title: '教学动态 | 南京航空航天大学计算机科学与技术学院', suffix: '1977/list.htm' }], + ['xsgz', { title: '学生工作 | 南京航空航天大学计算机科学与技术学院', suffix: '1959/list.htm' }], ]); module.exports = async (ctx) => { @@ -20,7 +20,7 @@ module.exports = async (ctx) => { const suffix = map.get(type).suffix; const link = new URL(suffix, host).href; - const cookie = await getCookie(); + const cookie = await getCookie(host); const gotConfig = { headers: { cookie, diff --git a/lib/v2/nuaa/jwc/jwc.js b/lib/v2/nuaa/jwc/jwc.js index 8f199edb09..3710b759f3 100644 --- a/lib/v2/nuaa/jwc/jwc.js +++ b/lib/v2/nuaa/jwc/jwc.js @@ -5,11 +5,11 @@ const getCookie = require('../utils/pypasswaf'); const host = 'http://aao.nuaa.edu.cn/'; const map = new Map([ - ['tzgg', { title: '南京航空航天大学教务处 -- 通知公告', suffix: '8222/list.htm' }], - ['jxfw', { title: '南京航空航天大学教务处 -- 教学服务', suffix: '8230/list.htm' }], - ['xspy', { title: '南京航空航天大学教务处 -- 学生培养', suffix: '8231/list.htm' }], - ['jxjs', { title: '南京航空航天大学教务处 -- 教学建设', suffix: '8232/list.htm' }], - ['jxzy', { title: '南京航空航天大学教务处 -- 教学资源', suffix: '8233/list.htm' }], + ['tzgg', { title: '通知公告 | 南京航空航天大学教务处', suffix: '8222/list.htm' }], + ['jxfw', { title: '教学服务 | 南京航空航天大学教务处', suffix: '8230/list.htm' }], + ['xspy', { title: '学生培养 | 南京航空航天大学教务处', suffix: '8231/list.htm' }], + ['jxjs', { title: '教学建设 | 南京航空航天大学教务处', suffix: '8232/list.htm' }], + ['jxzy', { title: '教学资源 | 南京航空航天大学教务处', suffix: '8233/list.htm' }], ]); module.exports = async (ctx) => { @@ -18,7 +18,7 @@ module.exports = async (ctx) => { const getDescription = Boolean(ctx.params.getDescription) || false; const link = new URL(suffix, host).href; - const cookie = await getCookie(); + const cookie = await getCookie(host); const gotConfig = { headers: { cookie, @@ -52,13 +52,13 @@ module.exports = async (ctx) => { // 南航新 WAF 过于敏感 // 目前 description 需要遍历页面，会被 WAF 拦截导致无法输出 // 考虑换一种获取 description 的方式或者将标题当作 title。 - let description = title; + let description = title + '
查看原文'; if (getDescription) { description = itemUrl; if (pageType === 'htm' || pageType === 'html') { const response = await got(itemUrl, gotConfig); const $ = cheerio.load(response.data); - description = $('.wp_articlecontent').html(); + description = $('.wp_articlecontent').html() + '
查看原文'; } } diff --git a/lib/v2/nuaa/maintainer.js b/lib/v2/nuaa/maintainer.js index 1861cbdab9..cf17c207a1 100644 --- a/lib/v2/nuaa/maintainer.js +++ b/lib/v2/nuaa/maintainer.js @@ -1,5 +1,6 @@ module.exports = { '/cs/:type/:getDescription?': ['LogicJake', 'Seiry', 'qrzbing'], - '/jwc/:type/:getDescription?': ['arcosx', 'Seiry', 'qrzbing'], - '/yjsy/:type?': ['junfengP', 'Seiry'], + '/cae/:type/:getDescription?': ['Seiry', 'qrzbing', 'Xm798'], + '/jwc/:type/:getDescription?': ['arcosx', 'Seiry', 'qrzbing', 'Xm798'], + '/yjsy/:type/:getDescription?': ['junfengP', 'Seiry', 'Xm798'], }; diff --git a/lib/v2/nuaa/radar.js b/lib/v2/nuaa/radar.js index df521f4734..b15527d30c 100644 --- a/lib/v2/nuaa/radar.js +++ b/lib/v2/nuaa/radar.js @@ -13,6 +13,12 @@ module.exports = { docs: 'https://docs.rsshub.app/university.html#nan-jing-hang-kong-hang-tian-da-xue', }, ], + cae: [ + { + title: '自动化学院', + docs: 'https://docs.rsshub.app/university.html#nan-jing-hang-kong-hang-tian-da-xue', + }, + ], 'www.graduate': [ { title: '研究生院', diff --git a/lib/v2/nuaa/router.js b/lib/v2/nuaa/router.js index c906a0548a..cf57ad6b5c 100644 --- a/lib/v2/nuaa/router.js +++ b/lib/v2/nuaa/router.js @@ -1,5 +1,6 @@ module.exports = (router) => { - router.get('/cs/:type/:getDescription?', require('./cs/index')); + router.get('/cae/:type/:getDescription?', require('./college/cae')); + router.get('/cs/:type/:getDescription?', require('./college/cs')); router.get('/jwc/:type/:getDescription?', require('./jwc/jwc')); - router.get('/yjsy/:type?', require('./yjsy/yjsy')); + router.get('/yjsy/:type/:getDescription?', require('./yjsy/yjsy')); }; diff --git a/lib/v2/nuaa/utils/pypasswaf.js b/lib/v2/nuaa/utils/pypasswaf.js index 1135d53490..cec40e60a2 100644 --- a/lib/v2/nuaa/utils/pypasswaf.js +++ b/lib/v2/nuaa/utils/pypasswaf.js @@ -1,10 +1,8 @@ -const host = 'http://aao.nuaa.edu.cn/'; - /** * async function 获取cookie * @desc 返回一个可用的cookie，使用 `got` 发起请求的时候，传入到`options.headers.cookie`即可 */ -module.exports = async function getCookie() { +module.exports = async function getCookie(host) { const browser = await require('@/utils/puppeteer')({ stealth: true }); const page = await browser.newPage(); await page.setRequestInterception(true); diff --git a/lib/v2/nuaa/yjsy/yjsy.js b/lib/v2/nuaa/yjsy/yjsy.js index 6f21060fcf..fb8d25aced 100644 --- a/lib/v2/nuaa/yjsy/yjsy.js +++ b/lib/v2/nuaa/yjsy/yjsy.js @@ -1,55 +1,80 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); -const host = 'http://www.graduate.nuaa.edu.cn/'; const { parseDate } = require('@/utils/parse-date'); +const getCookie = require('../utils/pypasswaf'); +const host = 'http://www.graduate.nuaa.edu.cn/'; -const map = { - latest: '2146/list.htm', - yyxw: 'yyxw/list.htm', - sjwj: 'sjwj/list.htm', - glwj: '2017/list.htm', - xxfw: '2147/list.htm', -}; - -const load = (link, ctx) => - ctx.cache.tryGet(link, async () => { - const response = await got(host + link); - const $ = cheerio.load(response.data); - const images = $('img'); - for (let k = 0; k < images.length; k++) { - $(images[k]).replaceWith(``); - } - const description = $('.wp_articlecontent').html(); - return { description }; - }); +const map = new Map([ + ['tzgg', { title: '通知公告 | 南京航空航天大学研究生院', suffix: '2145/list.htm' }], + ['xwdt', { title: '新闻动态 | 南京航空航天大学研究生院', suffix: '13276/list.htm' }], + ['xsxx', { title: '学术信息 | 南京航空航天大学研究生院', suffix: '13277/list.htm' }], + ['ssfc', { title: '师生风采 | 南京航空航天大学研究生院', suffix: '13278/list.htm' }], +]); module.exports = async (ctx) => { - const type = ctx.params.type ?? 'latest'; - const response = await got({ - method: 'get', - url: host + map[type], - }); + const type = ctx.params.type; + const suffix = map.get(type).suffix; + const getDescription = Boolean(ctx.params.getDescription) || false; + const link = new URL(suffix, host).href; + const cookie = await getCookie(host); + const gotConfig = { + headers: { + cookie, + }, + }; + const response = await got(link, gotConfig); const $ = cheerio.load(response.data); - const list = $('#news_list ul.news_ul > li').slice(0, 10).get(); - const process = await Promise.all( - list.map(async (item) => { - const a = $(item).find('a'); - const t = $(item).find('span'); - const itemUrl = a.attr('href'); - const single = { - title: a.text(), - link: new URL(itemUrl, host).href, - pubDate: parseDate(t.text(), 'YYYY-MM-DD'), + + const list = $('#wp_news_w6 ul li') + .slice(0, 10) + .map(function () { + const info = { + title: $(this).find('a').text(), + link: $(this).find('a').attr('href'), + date: $(this).find('span').text(), }; - const other = await load(itemUrl, ctx); - return { ...single, ...other }; + return info; + }) + .get(); + + const out = await Promise.all( + list.map((info) => { + const title = info.title || 'tzgg'; + const date = info.date; + const itemUrl = new URL(info.link, host).href; + + return ctx.cache.tryGet(itemUrl, async () => { + const arr = itemUrl.split('.'); + const pageType = arr[arr.length - 1]; + + // 南航新 WAF 过于敏感 + // 目前 description 需要遍历页面，会被 WAF 拦截导致无法输出 + // 考虑换一种获取 description 的方式或者将标题当作 title。 + let description = title + '
查看原文'; + if (getDescription) { + description = itemUrl; + if (pageType === 'htm' || pageType === 'html') { + const response = await got(itemUrl, gotConfig); + const $ = cheerio.load(response.data); + description = $('.wp_articlecontent').html() + '
查看原文'; + } + } + + const single = { + title, + link: itemUrl, + description, + pubDate: parseDate(date), + }; + return single; + }); }) ); ctx.state.data = { - title: '南航研究生院', - link: host, - description: '南航研究生院RSS', - item: process, + title: map.get(type).title, + link, + description: '南京航空航天大学研究生院RSS', + item: out, }; }; From 256387bd252484f02329d50cf55b3d3c7b424137 Mon Sep 17 00:00:00 2001 From: Tony Date: Mon, 29 Aug 2022 09:40:16 -0600 Subject: [PATCH 03/17] feat(route): pixiv novels (#10647) * feat(route): pixiv novels * fix: radar * fix: cache rename * docs: fix tag closing * fix: sort --- assets/radar-rules.js | 15 ---- docs/en/social-media.md | 4 + docs/social-media.md | 4 + lib/radar-rules.js | 35 -------- lib/router.js | 12 +-- lib/routes/pixiv/token.js | 74 ---------------- lib/v2/nuaa/maintainer.js | 2 +- lib/v2/nuaa/radar.js | 12 +-- lib/{routes => v2}/pixiv/api/getBookmarks.js | 4 +- .../pixiv/api/getIllustFollows.js | 4 +- lib/{routes => v2}/pixiv/api/getIllusts.js | 4 +- lib/{routes => v2}/pixiv/api/getRanking.js | 4 +- lib/{routes => v2}/pixiv/api/getUserDetail.js | 4 +- lib/{routes => v2}/pixiv/api/searchIllust.js | 4 +- .../pixiv/api/searchPopularIllust.js | 4 +- lib/{routes => v2}/pixiv/bookmarks.js | 7 +- lib/{routes => v2}/pixiv/constants.js | 0 lib/{routes => v2}/pixiv/illustfollow.js | 9 +- lib/v2/pixiv/maintainer.js | 8 ++ lib/v2/pixiv/novels.js | 35 ++++++++ lib/{routes => v2}/pixiv/pixiv-got.js | 2 +- lib/v2/pixiv/radar.js | 43 ++++++++++ lib/{routes => v2}/pixiv/ranking.js | 7 +- lib/v2/pixiv/router.js | 8 ++ lib/{routes => v2}/pixiv/search.js | 7 +- lib/v2/pixiv/token.js | 86 +++++++++++++++++++ lib/{routes => v2}/pixiv/user.js | 7 +- lib/{routes => v2}/pixiv/utils.js | 0 28 files changed, 230 insertions(+), 175 deletions(-) delete mode 100644 lib/routes/pixiv/token.js rename lib/{routes => v2}/pixiv/api/getBookmarks.js (84%) rename lib/{routes => v2}/pixiv/api/getIllustFollows.js (84%) rename lib/{routes => v2}/pixiv/api/getIllusts.js (85%) rename lib/{routes => v2}/pixiv/api/getRanking.js (91%) rename lib/{routes => v2}/pixiv/api/getUserDetail.js (88%) rename lib/{routes => v2}/pixiv/api/searchIllust.js (87%) rename lib/{routes => v2}/pixiv/api/searchPopularIllust.js (85%) rename lib/{routes => v2}/pixiv/bookmarks.js (87%) rename lib/{routes => v2}/pixiv/constants.js (100%) rename lib/{routes => v2}/pixiv/illustfollow.js (84%) create mode 100644 lib/v2/pixiv/maintainer.js create mode 100644 lib/v2/pixiv/novels.js rename lib/{routes => v2}/pixiv/pixiv-got.js (97%) create mode 100644 lib/v2/pixiv/radar.js rename lib/{routes => v2}/pixiv/ranking.js (93%) create mode 100644 lib/v2/pixiv/router.js rename lib/{routes => v2}/pixiv/search.js (89%) create mode 100644 lib/v2/pixiv/token.js rename lib/{routes => v2}/pixiv/user.js (85%) rename lib/{routes => v2}/pixiv/utils.js (100%) diff --git a/assets/radar-rules.js b/assets/radar-rules.js index 08f995c041..43ec4ddbc7 100644 --- a/assets/radar-rules.js +++ b/assets/radar-rules.js @@ -34,21 +34,6 @@ s: [{ title: '热搜榜', docs: 'https://docs.rsshub.app/social-media.html#wei-bo', source: '/top/summary', target: '/weibo/search/hot' }], }, 'weibo.cn': { _name: '微博', m: [{ title: '博主', docs: 'https://docs.rsshub.app/social-media.html#wei-bo', source: ['/u/:uid', '/profile/:uid'], target: '/weibo/user/:uid' }] }, - 'pixiv.net': { - _name: 'Pixiv', - www: [ - { title: '用户收藏', docs: 'https://docs.rsshub.app/social-media.html#pixiv', source: '/users/:id/bookmarks/artworks', target: '/pixiv/user/bookmarks/:id' }, - { title: '用户动态', docs: 'https://docs.rsshub.app/social-media.html#pixiv', source: '/users/:id', target: '/pixiv/user/:id' }, - { title: '排行榜', docs: 'https://docs.rsshub.app/social-media.html#pixiv', source: '/ranking.php', target: (params, url) => `/pixiv/ranking/${new URL(url).searchParams.get('mode') || 'daily'}` }, - { - title: '关键词', - docs: 'https://docs.rsshub.app/social-media.html#pixiv', - source: ['/tags/:keyword', '/tags/:keyword/:type?'], - target: (params, url) => `/pixiv/search/:keyword/${new URL(url).searchParams.get('order')}/${new URL(url).searchParams.get('mode')}`, - }, - { title: '关注的新作品', docs: 'https://docs.rsshub.app/social-media.html#pixiv', source: '/bookmark_new_illust.php', target: '/pixiv/user/illustfollows' }, - ], - }, 'github.com': { _name: 'GitHub', '.': [ diff --git a/docs/en/social-media.md b/docs/en/social-media.md index eb80af7181..3cea3b9fc7 100644 --- a/docs/en/social-media.md +++ b/docs/en/social-media.md @@ -214,6 +214,10 @@ Though, every Story expires after 24 hours, so it may be not so serious. +### User Novels + + + ### Rankings diff --git a/docs/social-media.md b/docs/social-media.md index c0ad806d3b..63046888f3 100644 --- a/docs/social-media.md +++ b/docs/social-media.md @@ -562,6 +562,10 @@ Instagram Stories 没有可靠的 guid，你的 RSS 阅读器可能将同一条 +### 用户小说 + + + ### 排行榜 diff --git a/lib/radar-rules.js b/lib/radar-rules.js index 5bc9ce5681..ec011c34d8 100644 --- a/lib/radar-rules.js +++ b/lib/radar-rules.js @@ -45,41 +45,6 @@ module.exports = { }, ], }, - 'pixiv.net': { - _name: 'Pixiv', - www: [ - { - title: '用户收藏', - docs: 'https://docs.rsshub.app/social-media.html#pixiv', - source: '/users/:id/bookmarks/artworks', - target: '/pixiv/user/bookmarks/:id', - }, - { - title: '用户动态', - docs: 'https://docs.rsshub.app/social-media.html#pixiv', - source: '/users/:id', - target: '/pixiv/user/:id', - }, - { - title: '排行榜', - docs: 'https://docs.rsshub.app/social-media.html#pixiv', - source: '/ranking.php', - target: (params, url) => `/pixiv/ranking/${new URL(url).searchParams.get('mode') || 'daily'}`, - }, - { - title: '关键词', - docs: 'https://docs.rsshub.app/social-media.html#pixiv', - source: ['/tags/:keyword', '/tags/:keyword/:type?'], - target: (params, url) => `/pixiv/search/:keyword/${new URL(url).searchParams.get('order')}/${new URL(url).searchParams.get('mode')}`, - }, - { - title: '关注的新作品', - docs: 'https://docs.rsshub.app/social-media.html#pixiv', - source: '/bookmark_new_illust.php', - target: '/pixiv/user/illustfollows', - }, - ], - }, 'smzdm.com': { _name: '什么值得买', www: [ diff --git a/lib/router.js b/lib/router.js index 1f8b037a37..d895e8f616 100644 --- a/lib/router.js +++ b/lib/router.js @@ -116,12 +116,12 @@ router.get('/mzitu/category/:category', lazyloadRouteHandler('./routes/mzitu/cat router.get('/mzitu/post/:id', lazyloadRouteHandler('./routes/mzitu/post')); router.get('/mzitu/tag/:tag', lazyloadRouteHandler('./routes/mzitu/tag')); -// pixiv -router.get('/pixiv/user/bookmarks/:id', lazyloadRouteHandler('./routes/pixiv/bookmarks')); -router.get('/pixiv/user/illustfollows', lazyloadRouteHandler('./routes/pixiv/illustfollow')); -router.get('/pixiv/user/:id', lazyloadRouteHandler('./routes/pixiv/user')); -router.get('/pixiv/ranking/:mode/:date?', lazyloadRouteHandler('./routes/pixiv/ranking')); -router.get('/pixiv/search/:keyword/:order?/:mode?', lazyloadRouteHandler('./routes/pixiv/search')); +// pixiv migrated to v2 +// router.get('/pixiv/user/bookmarks/:id', lazyloadRouteHandler('./routes/pixiv/bookmarks')); +// router.get('/pixiv/user/illustfollows', lazyloadRouteHandler('./routes/pixiv/illustfollow')); +// router.get('/pixiv/user/:id', lazyloadRouteHandler('./routes/pixiv/user')); +// router.get('/pixiv/ranking/:mode/:date?', lazyloadRouteHandler('./routes/pixiv/ranking')); +// router.get('/pixiv/search/:keyword/:order?/:mode?', lazyloadRouteHandler('./routes/pixiv/search')); // pixiv-fanbox router.get('/fanbox/:user?', lazyloadRouteHandler('./routes/fanbox/main')); diff --git a/lib/routes/pixiv/token.js b/lib/routes/pixiv/token.js deleted file mode 100644 index f00067b6ce..0000000000 --- a/lib/routes/pixiv/token.js +++ /dev/null @@ -1,74 +0,0 @@ -const config = require('@/config').value; -const logger = require('@/utils/logger'); -const got = require('./pixiv-got'); -const { maskHeader } = require('./constants'); - -let token = null; - -const authorizationInfo = { - client_id: 'MOBrBDS8blbauoSck0ZfDbtuzpyT', - client_secret: 'lsACyCD94FhDUtGTXi3QzcFE2uU1hqtDaKeqrdwj', - hash_secret: '28c1fdd170a5204386cb1313c7077b34f83e4aaf4aa829ce78c231e05b0bae2c', -}; - -const refreshToken = () => - got({ - method: 'post', - url: 'https://oauth.secure.pixiv.net/auth/token', - form: { - ...authorizationInfo, - get_secure_url: 1, - grant_type: 'refresh_token', - refresh_token: config.pixiv.refreshToken, - }, - headers: { - ...maskHeader, - }, - responseType: 'json', - resolveBodyOnly: true, - }).catch((e) => { - logger.error('Pixiv refresh token failed.'); - logger.debug(e); - }); - -async function tickToken() { - const result = await refreshToken(); - let expireTime; - if (result && result.access_token) { - logger.debug('Pixiv refresh token success.'); - token = result.access_token; - expireTime = result.expires_in; - } else { - expireTime = 30; - logger.error(`Pixiv refresh token failed, retry in ${expireTime} seconds.`); - } - // If is package, the mock server should be closed after request, needn't to refresh - if (!config.isPackage) { - setTimeout(tickToken, expireTime); - } -} - -let tickTokenStarted = false; - -async function startTickToken() { - if (!tickTokenStarted) { - // 如果tickToken没启动 - tickTokenStarted = true; - await tickToken(); // 启动tickToken - } -} - -async function waitForToken() { - while (!token) { - // eslint-disable-next-line no-await-in-loop - await new Promise((resolve) => setTimeout(resolve, 0)); - } - return token; -} - -module.exports = async () => { - await startTickToken(); - return waitForToken(); -}; - -module.exports.tickToken = tickToken; diff --git a/lib/v2/nuaa/maintainer.js b/lib/v2/nuaa/maintainer.js index cf17c207a1..12dfe3d764 100644 --- a/lib/v2/nuaa/maintainer.js +++ b/lib/v2/nuaa/maintainer.js @@ -1,6 +1,6 @@ module.exports = { - '/cs/:type/:getDescription?': ['LogicJake', 'Seiry', 'qrzbing'], '/cae/:type/:getDescription?': ['Seiry', 'qrzbing', 'Xm798'], + '/cs/:type/:getDescription?': ['LogicJake', 'Seiry', 'qrzbing'], '/jwc/:type/:getDescription?': ['arcosx', 'Seiry', 'qrzbing', 'Xm798'], '/yjsy/:type/:getDescription?': ['junfengP', 'Seiry', 'Xm798'], }; diff --git a/lib/v2/nuaa/radar.js b/lib/v2/nuaa/radar.js index b15527d30c..d6045101db 100644 --- a/lib/v2/nuaa/radar.js +++ b/lib/v2/nuaa/radar.js @@ -7,18 +7,18 @@ module.exports = { docs: 'https://docs.rsshub.app/university.html#nan-jing-hang-kong-hang-tian-da-xue', }, ], - cs: [ - { - title: '计算机科学与技术学院', - docs: 'https://docs.rsshub.app/university.html#nan-jing-hang-kong-hang-tian-da-xue', - }, - ], cae: [ { title: '自动化学院', docs: 'https://docs.rsshub.app/university.html#nan-jing-hang-kong-hang-tian-da-xue', }, ], + cs: [ + { + title: '计算机科学与技术学院', + docs: 'https://docs.rsshub.app/university.html#nan-jing-hang-kong-hang-tian-da-xue', + }, + ], 'www.graduate': [ { title: '研究生院', diff --git a/lib/routes/pixiv/api/getBookmarks.js b/lib/v2/pixiv/api/getBookmarks.js similarity index 84% rename from lib/routes/pixiv/api/getBookmarks.js rename to lib/v2/pixiv/api/getBookmarks.js index eafc5e7b32..07b5f5be43 100644 --- a/lib/routes/pixiv/api/getBookmarks.js +++ b/lib/v2/pixiv/api/getBookmarks.js @@ -10,9 +10,7 @@ const queryString = require('query-string'); * @returns {Promise>} */ module.exports = function getBookmarks(user_id, token) { - return got({ - method: 'get', - url: 'https://app-api.pixiv.net/v1/user/bookmarks/illust', + return got('https://app-api.pixiv.net/v1/user/bookmarks/illust', { headers: { ...maskHeader, Authorization: 'Bearer ' + token, diff --git a/lib/routes/pixiv/api/getIllustFollows.js b/lib/v2/pixiv/api/getIllustFollows.js similarity index 84% rename from lib/routes/pixiv/api/getIllustFollows.js rename to lib/v2/pixiv/api/getIllustFollows.js index ea6d436a08..6c492edcdb 100644 --- a/lib/routes/pixiv/api/getIllustFollows.js +++ b/lib/v2/pixiv/api/getIllustFollows.js @@ -8,9 +8,7 @@ const queryString = require('query-string'); * @returns {Promise>} */ module.exports = function getUserIllustFollows(token) { - return got({ - method: 'get', - url: 'https://app-api.pixiv.net/v2/illust/follow', + return got('https://app-api.pixiv.net/v2/illust/follow', { headers: { ...maskHeader, Authorization: 'Bearer ' + token, diff --git a/lib/routes/pixiv/api/getIllusts.js b/lib/v2/pixiv/api/getIllusts.js similarity index 85% rename from lib/routes/pixiv/api/getIllusts.js rename to lib/v2/pixiv/api/getIllusts.js index 33f62e2c10..d27e29c314 100644 --- a/lib/routes/pixiv/api/getIllusts.js +++ b/lib/v2/pixiv/api/getIllusts.js @@ -9,9 +9,7 @@ const queryString = require('query-string'); * @returns {Promise>} */ module.exports = function getIllusts(user_id, token) { - return got({ - method: 'get', - url: 'https://app-api.pixiv.net/v1/user/illusts', + return got('https://app-api.pixiv.net/v1/user/illusts', { headers: { ...maskHeader, Authorization: 'Bearer ' + token, diff --git a/lib/routes/pixiv/api/getRanking.js b/lib/v2/pixiv/api/getRanking.js similarity index 91% rename from lib/routes/pixiv/api/getRanking.js rename to lib/v2/pixiv/api/getRanking.js index fe94389eaa..5bbc0bbef5 100644 --- a/lib/routes/pixiv/api/getRanking.js +++ b/lib/v2/pixiv/api/getRanking.js @@ -14,9 +14,7 @@ const allowMode = ['day', 'week', 'month', 'day_male', 'day_female', 'week_origi */ module.exports = function getRanking(mode, date, token) { assert(allowMode.includes(mode), 'Mode not allow.'); - return got({ - method: 'get', - url: 'https://app-api.pixiv.net/v1/illust/ranking', + return got('https://app-api.pixiv.net/v1/illust/ranking', { headers: { ...maskHeader, Authorization: 'Bearer ' + token, diff --git a/lib/routes/pixiv/api/getUserDetail.js b/lib/v2/pixiv/api/getUserDetail.js similarity index 88% rename from lib/routes/pixiv/api/getUserDetail.js rename to lib/v2/pixiv/api/getUserDetail.js index 5a1d1c25bb..abc51ad47b 100644 --- a/lib/routes/pixiv/api/getUserDetail.js +++ b/lib/v2/pixiv/api/getUserDetail.js @@ -15,9 +15,7 @@ const queryString = require('query-string'); * @returns {Promise>} */ module.exports = function getUserDetail(user_id, token) { - return got({ - method: 'get', - url: 'https://app-api.pixiv.net/v1/user/detail', + return got('https://app-api.pixiv.net/v1/user/detail', { headers: { ...maskHeader, Authorization: 'Bearer ' + token, diff --git a/lib/routes/pixiv/api/searchIllust.js b/lib/v2/pixiv/api/searchIllust.js similarity index 87% rename from lib/routes/pixiv/api/searchIllust.js rename to lib/v2/pixiv/api/searchIllust.js index 3b7277327c..fcc839402b 100644 --- a/lib/routes/pixiv/api/searchIllust.js +++ b/lib/v2/pixiv/api/searchIllust.js @@ -9,9 +9,7 @@ const queryString = require('query-string'); * @returns {Promise>} */ module.exports = function searchIllust(keyword, token) { - return got({ - method: 'get', - url: 'https://app-api.pixiv.net/v1/search/illust', + return got('https://app-api.pixiv.net/v1/search/illust', { headers: { ...maskHeader, Authorization: 'Bearer ' + token, diff --git a/lib/routes/pixiv/api/searchPopularIllust.js b/lib/v2/pixiv/api/searchPopularIllust.js similarity index 85% rename from lib/routes/pixiv/api/searchPopularIllust.js rename to lib/v2/pixiv/api/searchPopularIllust.js index 1756424411..a13dab4661 100644 --- a/lib/routes/pixiv/api/searchPopularIllust.js +++ b/lib/v2/pixiv/api/searchPopularIllust.js @@ -9,9 +9,7 @@ const queryString = require('query-string'); * @returns {Promise>} */ module.exports = function searchPopularIllust(keyword, token) { - return got({ - method: 'get', - url: 'https://app-api.pixiv.net/v1/search/popular-preview/illust', + return got('https://app-api.pixiv.net/v1/search/popular-preview/illust', { headers: { ...maskHeader, Authorization: 'Bearer ' + token, diff --git a/lib/routes/pixiv/bookmarks.js b/lib/v2/pixiv/bookmarks.js similarity index 87% rename from lib/routes/pixiv/bookmarks.js rename to lib/v2/pixiv/bookmarks.js index 89b7e88371..61112a7e75 100644 --- a/lib/routes/pixiv/bookmarks.js +++ b/lib/v2/pixiv/bookmarks.js @@ -1,8 +1,9 @@ -const getToken = require('./token'); +const { getToken } = require('./token'); const getBookmarks = require('./api/getBookmarks'); const getUserDetail = require('./api/getUserDetail'); const config = require('@/config').value; const pixivUtils = require('./utils'); +const { parseDate } = require('@/utils/parse-date'); module.exports = async (ctx) => { if (!config.pixiv || !config.pixiv.refreshToken) { @@ -11,7 +12,7 @@ module.exports = async (ctx) => { const id = ctx.params.id; - const token = await getToken(); + const token = await getToken(ctx.cache.tryGet); if (!token) { throw 'pixiv not login'; } @@ -30,7 +31,7 @@ module.exports = async (ctx) => { return { title: illust.title, author: illust.user.name, - pubDate: new Date(illust.create_date).toUTCString(), + pubDate: parseDate(illust.create_date), description: `

画师：${illust.user.name} - 阅览数：${illust.total_view} - 收藏数：${illust.total_bookmarks}

${images.join('')}`, link: `https://www.pixiv.net/artworks/${illust.id}`, }; diff --git a/lib/routes/pixiv/constants.js b/lib/v2/pixiv/constants.js similarity index 100% rename from lib/routes/pixiv/constants.js rename to lib/v2/pixiv/constants.js diff --git a/lib/routes/pixiv/illustfollow.js b/lib/v2/pixiv/illustfollow.js similarity index 84% rename from lib/routes/pixiv/illustfollow.js rename to lib/v2/pixiv/illustfollow.js index 0c37c0dba4..a40b73586d 100644 --- a/lib/routes/pixiv/illustfollow.js +++ b/lib/v2/pixiv/illustfollow.js @@ -1,15 +1,16 @@ -const getToken = require('./token'); +const { getToken } = require('./token'); const getIllustFollows = require('./api/getIllustFollows'); const config = require('@/config').value; const pixivUtils = require('./utils'); +const { parseDate } = require('@/utils/parse-date'); module.exports = async (ctx) => { if (!config.pixiv || !config.pixiv.refreshToken) { throw 'pixiv RSS is disabled due to the lack of relevant config'; } - const token = await getToken(); - if (!(await getToken())) { + const token = await getToken(ctx.cache.tryGet); + if (!token) { throw 'pixiv not login'; } @@ -24,7 +25,7 @@ module.exports = async (ctx) => { return { title: illust.title, author: illust.user.name, - pubDate: new Date(illust.create_date).toUTCString(), + pubDate: parseDate(illust.create_date), description: `

画师：${illust.user.name} - 阅览数：${illust.total_view} - 收藏数：${illust.total_bookmarks}

${images.join('')}`, link: `https://www.pixiv.net/artworks/${illust.id}`, }; diff --git a/lib/v2/pixiv/maintainer.js b/lib/v2/pixiv/maintainer.js new file mode 100644 index 0000000000..c609eeb9a0 --- /dev/null +++ b/lib/v2/pixiv/maintainer.js @@ -0,0 +1,8 @@ +module.exports = { + '/ranking/:mode/:date?': ['EYHN'], + '/search/:keyword/:order?/:mode?': ['DIYgod'], + '/user/bookmarks/:id': ['EYHN'], + '/user/illustfollows': ['ClarkeCheng'], + '/user/novels/:id': ['TonyRL'], + '/user/:id': ['DIYgod'], +}; diff --git a/lib/v2/pixiv/novels.js b/lib/v2/pixiv/novels.js new file mode 100644 index 0000000000..44567f3b2c --- /dev/null +++ b/lib/v2/pixiv/novels.js @@ -0,0 +1,35 @@ +const got = require('@/utils/got'); +const { parseDate } = require('@/utils/parse-date'); +const baseUrl = 'https://www.pixiv.net'; + +module.exports = async (ctx) => { + const { id } = ctx.params; + const { data } = await got(`${baseUrl}/ajax/user/${id}/profile/all`); + + const items = [ + ...data.body.novelSeries.map((item) => ({ + title: item.title, + description: item.caption, + link: `https://www.pixiv.net/novel/series/${item.id}`, + author: item.userName, + pubDate: parseDate(item.createDate), + updated: parseDate(item.updateDate), + category: item.tags, + })), + ...data.body.pickup.map((item) => ({ + title: item.title, + description: item.description, + link: item.contentUrl, + author: item.userName, + pubDate: parseDate(item.createDate), + updated: parseDate(item.updateDate), + category: item.tags, + })), + ]; + + ctx.state.data = { + title: `${data.body.novelSeries[0].userName || data.body.pickup[0].userName} 的小说`, + link: `${baseUrl}/users/${id}/novels`, + item: items, + }; +}; diff --git a/lib/routes/pixiv/pixiv-got.js b/lib/v2/pixiv/pixiv-got.js similarity index 97% rename from lib/routes/pixiv/pixiv-got.js rename to lib/v2/pixiv/pixiv-got.js index 0fd73f9e9f..f562bc17fb 100644 --- a/lib/routes/pixiv/pixiv-got.js +++ b/lib/v2/pixiv/pixiv-got.js @@ -6,7 +6,7 @@ const pixivConfig = require('@/config').value.pixiv; async function dohResolve(name, jsonDohEndpoint) { try { - const response = await got.get(jsonDohEndpoint, { + const response = await got(jsonDohEndpoint, { searchParams: { name, type: 'A', diff --git a/lib/v2/pixiv/radar.js b/lib/v2/pixiv/radar.js new file mode 100644 index 0000000000..6b2229adbf --- /dev/null +++ b/lib/v2/pixiv/radar.js @@ -0,0 +1,43 @@ +module.exports = { + 'pixiv.net': { + _name: 'Pixiv', + www: [ + { + title: '用户收藏', + docs: 'https://docs.rsshub.app/social-media.html#pixiv', + source: '/users/:id/bookmarks/artworks', + target: '/pixiv/user/bookmarks/:id', + }, + { + title: '用户动态', + docs: 'https://docs.rsshub.app/social-media.html#pixiv', + source: '/users/:id', + target: '/pixiv/user/:id', + }, + { + title: '用户小说', + docs: 'https://docs.rsshub.app/social-media.html#pixiv', + source: '/users/:id/novels', + target: '/pixiv/user/novels/:id', + }, + { + title: '排行榜', + docs: 'https://docs.rsshub.app/social-media.html#pixiv', + source: '/ranking.php', + target: (params, url) => `/pixiv/ranking/${new URL(url).searchParams.get('mode') || 'daily'}`, + }, + { + title: '关键词', + docs: 'https://docs.rsshub.app/social-media.html#pixiv', + source: ['/tags/:keyword', '/tags/:keyword/:type?'], + target: (params, url) => `/pixiv/search/:keyword/${new URL(url).searchParams.get('order')}/${new URL(url).searchParams.get('mode')}`, + }, + { + title: '关注的新作品', + docs: 'https://docs.rsshub.app/social-media.html#pixiv', + source: '/bookmark_new_illust.php', + target: '/pixiv/user/illustfollows', + }, + ], + }, +}; diff --git a/lib/routes/pixiv/ranking.js b/lib/v2/pixiv/ranking.js similarity index 93% rename from lib/routes/pixiv/ranking.js rename to lib/v2/pixiv/ranking.js index b35b2bd592..ab1f139a70 100644 --- a/lib/routes/pixiv/ranking.js +++ b/lib/v2/pixiv/ranking.js @@ -1,7 +1,8 @@ -const getToken = require('./token'); +const { getToken } = require('./token'); const getRanking = require('./api/getRanking'); const config = require('@/config').value; const pixivUtils = require('./utils'); +const { parseDate } = require('@/utils/parse-date'); const titles = { day: 'pixiv 日排行', @@ -56,7 +57,7 @@ module.exports = async (ctx) => { const mode = alias[ctx.params.mode] ? alias[ctx.params.mode] : ctx.params.mode; const date = ctx.params.date ? new Date(ctx.params.date) : new Date(); - const token = await getToken(); + const token = await getToken(ctx.cache.tryGet); if (!token) { throw 'pixiv not login'; } @@ -75,7 +76,7 @@ module.exports = async (ctx) => { const images = pixivUtils.getImgs(illust); return { title: `#${index + 1} ${illust.title}`, - pubDate: new Date(illust.create_date).toUTCString(), + pubDate: parseDate(illust.create_date), description: `

画师：${illust.user.name} - 阅览数：${illust.total_view} - 收藏数：${illust.total_bookmarks}


${images.join('')}`, link: `https://www.pixiv.net/artworks/${illust.id}`, author: illust.user.name, diff --git a/lib/v2/pixiv/router.js b/lib/v2/pixiv/router.js new file mode 100644 index 0000000000..99263ffb9d --- /dev/null +++ b/lib/v2/pixiv/router.js @@ -0,0 +1,8 @@ +module.exports = (router) => { + router.get('/ranking/:mode/:date?', require('./ranking')); + router.get('/search/:keyword/:order?/:mode?', require('./search')); + router.get('/user/bookmarks/:id', require('./bookmarks')); + router.get('/user/illustfollows', require('./illustfollow')); + router.get('/user/novels/:id', require('./novels')); + router.get('/user/:id', require('./user')); +}; diff --git a/lib/routes/pixiv/search.js b/lib/v2/pixiv/search.js similarity index 89% rename from lib/routes/pixiv/search.js rename to lib/v2/pixiv/search.js index 8fcb5d93df..f6fada5f5b 100644 --- a/lib/routes/pixiv/search.js +++ b/lib/v2/pixiv/search.js @@ -1,8 +1,9 @@ -const getToken = require('./token'); +const { getToken } = require('./token'); const searchPopularIllust = require('./api/searchPopularIllust'); const searchIllust = require('./api/searchIllust'); const config = require('@/config').value; const pixivUtils = require('./utils'); +const { parseDate } = require('@/utils/parse-date'); module.exports = async (ctx) => { if (!config.pixiv || !config.pixiv.refreshToken) { @@ -13,7 +14,7 @@ module.exports = async (ctx) => { const order = ctx.params.order || 'date'; const mode = ctx.params.mode; - const token = await getToken(); + const token = await getToken(ctx.cache.tryGet); if (!token) { throw 'pixiv not login'; } @@ -40,7 +41,7 @@ module.exports = async (ctx) => { return { title: illust.title, author: illust.user.name, - pubDate: new Date(illust.create_date).toUTCString(), + pubDate: parseDate(illust.create_date), description: `

画师：${illust.user.name} - 阅览数：${illust.total_view} - 收藏数：${illust.total_bookmarks}

${images.join('')}`, link: `https://www.pixiv.net/artworks/${illust.id}`, }; diff --git a/lib/v2/pixiv/token.js b/lib/v2/pixiv/token.js new file mode 100644 index 0000000000..06911904ba --- /dev/null +++ b/lib/v2/pixiv/token.js @@ -0,0 +1,86 @@ +const config = require('@/config').value; +const logger = require('@/utils/logger'); +const got = require('./pixiv-got'); +const { maskHeader } = require('./constants'); + +let token = null; + +const authorizationInfo = { + client_id: 'MOBrBDS8blbauoSck0ZfDbtuzpyT', + client_secret: 'lsACyCD94FhDUtGTXi3QzcFE2uU1hqtDaKeqrdwj', + hash_secret: '28c1fdd170a5204386cb1313c7077b34f83e4aaf4aa829ce78c231e05b0bae2c', +}; + +const refreshToken = (tryGet) => + tryGet( + 'pixiv:accessToken', + () => + got + .post('https://oauth.secure.pixiv.net/auth/token', { + form: { + ...authorizationInfo, + get_secure_url: 1, + grant_type: 'refresh_token', + refresh_token: config.pixiv.refreshToken, + }, + headers: { + ...maskHeader, + }, + responseType: 'json', + resolveBodyOnly: true, + }) + .catch((e) => { + logger.error('Pixiv refresh token failed.'); + logger.error(e); + }), + 3600, + false + ); + +async function getToken(tryGet) { + const result = await refreshToken(tryGet); + // let expireTime; + if (result && result.access_token) { + logger.debug('Pixiv refresh token success.'); + token = result.access_token; + // expireTime = result.expires_in; + } + // } else { + // expireTime = 30; + // logger.error(`Pixiv refresh token failed, retry in ${expireTime} seconds.`); + // } + // If is package, the mock server should be closed after request, needn't to refresh + // if (!config.isPackage) { + // setTimeout(tickToken, expireTime * 1000); + // } + return token; +} + +module.exports = { + getToken, +}; + +// let tickTokenStarted = false; + +// async function startTickToken(tryGet) { +// if (!tickTokenStarted) { +// // 如果tickToken没启动 +// tickTokenStarted = true; +// await tickToken(tryGet); // 启动tickToken +// } +// } + +// async function waitForToken() { +// while (!token) { +// // eslint-disable-next-line no-await-in-loop +// await new Promise((resolve) => setTimeout(resolve, 0)); +// } +// return token; +// } + +// module.exports = async (tryGet) => { +// await startTickToken(tryGet); +// return waitForToken(); +// }; + +// module.exports.tickToken = tickToken; diff --git a/lib/routes/pixiv/user.js b/lib/v2/pixiv/user.js similarity index 85% rename from lib/routes/pixiv/user.js rename to lib/v2/pixiv/user.js index e9ff62b09f..e3893ec8cf 100644 --- a/lib/routes/pixiv/user.js +++ b/lib/v2/pixiv/user.js @@ -1,7 +1,8 @@ -const getToken = require('./token'); +const { getToken } = require('./token'); const getIllusts = require('./api/getIllusts'); const config = require('@/config').value; const pixivUtils = require('./utils'); +const { parseDate } = require('@/utils/parse-date'); module.exports = async (ctx) => { if (!config.pixiv || !config.pixiv.refreshToken) { @@ -9,7 +10,7 @@ module.exports = async (ctx) => { } const id = ctx.params.id; - const token = await getToken(); + const token = await getToken(ctx.cache.tryGet); if (!token) { throw 'pixiv not login'; } @@ -28,7 +29,7 @@ module.exports = async (ctx) => { return { title: illust.title, author: username, - pubDate: new Date(illust.create_date).toUTCString(), + pubDate: parseDate(illust.create_date), description: `

画师：${username} - 阅览数：${illust.total_view} - 收藏数：${illust.total_bookmarks}

${images.join('')}`, link: `https://www.pixiv.net/artworks/${illust.id}`, }; diff --git a/lib/routes/pixiv/utils.js b/lib/v2/pixiv/utils.js similarity index 100% rename from lib/routes/pixiv/utils.js rename to lib/v2/pixiv/utils.js From d5a952a55ebe72acfdae44e593d9338ff5f01556 Mon Sep 17 00:00:00 2001 From: Yuan68 <69062462+Yuan68@users.noreply.github.com> Date: Mon, 29 Aug 2022 23:52:16 +0800 Subject: [PATCH 04/17] =?UTF-8?q?fix(route):=20=E5=A2=9E=E5=8A=A0BBC?= =?UTF-8?q?=E5=92=8C=E5=BE=AE=E4=BF=A1=EF=BC=88feeddd=E8=8E=B7=E5=8F=96?= =?UTF-8?q?=EF=BC=89=E7=9A=84favicon=20(#10649)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update index.js * Update feeddd.js --- lib/routes/bbc/index.js | 1 + lib/v2/wechat/feeddd.js | 1 + 2 files changed, 2 insertions(+) diff --git a/lib/routes/bbc/index.js b/lib/routes/bbc/index.js index bfd70b9653..71fb375aac 100644 --- a/lib/routes/bbc/index.js +++ b/lib/routes/bbc/index.js @@ -85,6 +85,7 @@ module.exports = async (ctx) => { ctx.state.data = { title, link, + image: 'https://www.bbc.com/favicon.ico', description: title, item: items, }; diff --git a/lib/v2/wechat/feeddd.js b/lib/v2/wechat/feeddd.js index b2e0c1c753..ae687f92ad 100644 --- a/lib/v2/wechat/feeddd.js +++ b/lib/v2/wechat/feeddd.js @@ -31,6 +31,7 @@ module.exports = async (ctx) => { ctx.state.json = { title: response.data.title, link: response.data.feed_url, + image: 'https://mp.weixin.qq.com/favicon.ico', description: response.data.title, item: items, }; From 9757d7e5be6bd5a3f29eef934351e730ce3ab315 Mon Sep 17 00:00:00 2001 From: miles Date: Tue, 30 Aug 2022 01:20:52 +0800 Subject: [PATCH 05/17] =?UTF-8?q?feat(route):=20add=20=E8=B5=B7=E7=82=B9?= =?UTF-8?q?=E4=BD=9C=E8=80=85=E9=A1=B5=E9=9D=A2=E7=9A=84=E4=BD=9C=E5=93=81?= =?UTF-8?q?=20(#10643)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(route): add 起点作者页面的作品 * fix(route): 起点作者缺少 v2 路由必要文件 * fix(route): typo * fix(route): remove redundant code * fix(route): author accepts pure string only * refactor(route): cleanup code * fix(route): remove old date parser * refactor: migrate to v2 --- assets/radar-rules.js | 11 ------- docs/reading.md | 4 +++ lib/radar-rules.js | 31 ------------------- lib/router.js | 10 +++--- lib/v2/qidian/author.js | 41 +++++++++++++++++++++++++ lib/{routes => v2}/qidian/chapter.js | 13 +++----- lib/{routes => v2}/qidian/forum.js | 4 +-- lib/{routes => v2}/qidian/free-next.js | 5 +-- lib/{routes => v2}/qidian/free.js | 8 ++--- lib/v2/qidian/maintainer.js | 7 +++++ lib/v2/qidian/radar.js | 41 +++++++++++++++++++++++++ lib/v2/qidian/router.js | 7 +++++ lib/v2/qidian/templates/description.art | 5 +++ 13 files changed, 119 insertions(+), 68 deletions(-) create mode 100644 lib/v2/qidian/author.js rename lib/{routes => v2}/qidian/chapter.js (75%) rename lib/{routes => v2}/qidian/forum.js (93%) rename lib/{routes => v2}/qidian/free-next.js (93%) rename lib/{routes => v2}/qidian/free.js (88%) create mode 100644 lib/v2/qidian/maintainer.js create mode 100644 lib/v2/qidian/radar.js create mode 100644 lib/v2/qidian/router.js create mode 100644 lib/v2/qidian/templates/description.art diff --git a/assets/radar-rules.js b/assets/radar-rules.js index 43ec4ddbc7..6bd1c0680e 100644 --- a/assets/radar-rules.js +++ b/assets/radar-rules.js @@ -307,17 +307,6 @@ ], sites: [{ title: 'Sites', docs: 'https://docs.rsshub.app/blog.html#google-sites', source: ['/site/:id/*', '/site/:id'], target: '/google/sites/:id' }], }, - 'qidian.com': { - _name: '起点', - book: [ - { title: '章节', docs: 'https://docs.rsshub.app/reading.html#qi-dian', source: '/info/:id', target: '/qidian/chapter/:id' }, - { title: '讨论区', docs: 'https://docs.rsshub.app/reading.html#qi-dian', source: '/info/:id', target: '/qidian/forum/:id' }, - ], - www: [ - { title: '限免', docs: 'https://docs.rsshub.app/reading.html#qi-dian', source: '/free', target: '/qidian/free' }, - { title: '女生限免', docs: 'https://docs.rsshub.app/reading.html#qi-dian', source: '/mm/free', target: '/qidian/free/mm' }, - ], - }, 'hackerone.com': { _name: 'HackerOne', '.': [{ title: 'HackerOne Hacker Activity', docs: 'https://docs.rsshub.app/other.html#hackerone-hacker-activity', source: '/hacktivity', target: '/hackerone/hacktivity' }] }, 'cowlevel.net': { _name: '奶牛关', '.': [{ title: '元素文章', docs: 'https://docs.rsshub.app/game.html#nai-niu-guan', source: ['/element/:id', '/element/:id/article'], target: '/cowlevel/element/:id' }] }, 'beijing.gov.cn': { wjw: [{ title: '北京卫生健康委员会', docs: 'https://docs.rsshub.app/government.html#bei-jing-shi-wei-sheng-jian-kang-wei-yuan-hui', source: '/xwzx_20031/:caty', target: '/gov/beijing/mhc/:caty' }] }, diff --git a/docs/reading.md b/docs/reading.md index 9ff701babd..e6d7ce4920 100644 --- a/docs/reading.md +++ b/docs/reading.md @@ -399,6 +399,10 @@ count 的取值范围为 1-12，为防止请求次数过多，推荐设置为 5 +### 作者 + + + ## 起点图 ### 首订 diff --git a/lib/radar-rules.js b/lib/radar-rules.js index ec011c34d8..0453e4740d 100644 --- a/lib/radar-rules.js +++ b/lib/radar-rules.js @@ -747,37 +747,6 @@ module.exports = { }, ], }, - 'qidian.com': { - _name: '起点', - book: [ - { - title: '章节', - docs: 'https://docs.rsshub.app/reading.html#qi-dian', - source: '/info/:id', - target: '/qidian/chapter/:id', - }, - { - title: '讨论区', - docs: 'https://docs.rsshub.app/reading.html#qi-dian', - source: '/info/:id', - target: '/qidian/forum/:id', - }, - ], - www: [ - { - title: '限免', - docs: 'https://docs.rsshub.app/reading.html#qi-dian', - source: '/free', - target: '/qidian/free', - }, - { - title: '女生限免', - docs: 'https://docs.rsshub.app/reading.html#qi-dian', - source: '/mm/free', - target: '/qidian/free/mm', - }, - ], - }, 'hackerone.com': { _name: 'HackerOne', '.': [ diff --git a/lib/router.js b/lib/router.js index d895e8f616..ed66c96dcc 100644 --- a/lib/router.js +++ b/lib/router.js @@ -338,11 +338,11 @@ router.get('/mi/bbs/board/:boardId', lazyloadRouteHandler('./routes/mi/board')); // Keep // router.get('/keep/user/:id', lazyloadRouteHandler('./routes/keep/user')); -// 起点 -router.get('/qidian/chapter/:id', lazyloadRouteHandler('./routes/qidian/chapter')); -router.get('/qidian/forum/:id', lazyloadRouteHandler('./routes/qidian/forum')); -router.get('/qidian/free/:type?', lazyloadRouteHandler('./routes/qidian/free')); -router.get('/qidian/free-next/:type?', lazyloadRouteHandler('./routes/qidian/free-next')); +// 起点 migrated to v2 +// router.get('/qidian/chapter/:id', lazyloadRouteHandler('./routes/qidian/chapter')); +// router.get('/qidian/forum/:id', lazyloadRouteHandler('./routes/qidian/forum')); +// router.get('/qidian/free/:type?', lazyloadRouteHandler('./routes/qidian/free')); +// router.get('/qidian/free-next/:type?', lazyloadRouteHandler('./routes/qidian/free-next')); // 纵横 router.get('/zongheng/chapter/:id', lazyloadRouteHandler('./routes/zongheng/chapter')); diff --git a/lib/v2/qidian/author.js b/lib/v2/qidian/author.js new file mode 100644 index 0000000000..8b515f0da2 --- /dev/null +++ b/lib/v2/qidian/author.js @@ -0,0 +1,41 @@ +const got = require('@/utils/got'); +const cheerio = require('cheerio'); +const { parseDate, parseRelativeDate } = require('@/utils/parse-date'); +const { art } = require('@/utils/render'); +const path = require('path'); +const timezone = require('@/utils/timezone'); + +module.exports = async (ctx) => { + const id = ctx.params.id; + const rootUrl = 'https://my.qidian.com'; + const currentUrl = `${rootUrl}/author/${id}/`; + const response = await got(currentUrl); + const $ = cheerio.load(response.data); + const authorName = $('.header-msg h1').contents().first().text().trim(); + const items = $('.author-work .author-item') + .toArray() + .map((item) => { + item = $(item); + const messageItem = item.find('.author-item-msg'); + const updatedDate = messageItem.find('.author-item-update span').text().replace('·', '').trim(); + + return { + title: messageItem.find('.author-item-title').text().trim(), + author: authorName, + category: messageItem.find('.author-item-exp a').first().text().trim(), + description: art(path.join(__dirname, 'templates/description.art'), { + description: messageItem.find('.author-item-update a').attr('title'), + image: item.find('a img').attr('src'), + }), + pubDate: timezone(/(今|昨)/.test(updatedDate) ? parseRelativeDate(updatedDate) : parseDate(updatedDate, 'YYYY-MM-DD HH:mm'), +8), + link: messageItem.find('.author-item-update a').attr('href'), + }; + }); + + ctx.state.data = { + title: `${authorName} - 起点中文网`, + description: $('.header-msg-desc').text().trim(), + link: currentUrl, + item: items, + }; +}; diff --git a/lib/routes/qidian/chapter.js b/lib/v2/qidian/chapter.js similarity index 75% rename from lib/routes/qidian/chapter.js rename to lib/v2/qidian/chapter.js index a84cebbce9..f1f806386a 100644 --- a/lib/routes/qidian/chapter.js +++ b/lib/v2/qidian/chapter.js @@ -1,13 +1,11 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); +const { parseDate } = require('@/utils/parse-date'); module.exports = async (ctx) => { const id = ctx.params.id; - const response = await got({ - method: 'get', - url: `https://book.qidian.com/info/${id}#Catalog`, - }); + const response = await got(`https://book.qidian.com/info/${id}/#Catalog`); const $ = cheerio.load(response.data); const name = $('.book-info>h1>em').text(); @@ -15,10 +13,7 @@ module.exports = async (ctx) => { const csrfToken = response.headers['set-cookie'].find((s) => s.startsWith('_csrfToken=')).split(';')[0]; - const chapters_response = await got({ - method: 'get', - url: `https://book.qidian.com/ajax/book/category?${csrfToken}&bookId=${id}`, - }); + const chapters_response = await got(`https://book.qidian.com/ajax/book/category?${csrfToken}&bookId=${id}`); const chapter_item = []; for (let i = 0; i < chapters_response.data.data.vs.length; i++) { @@ -27,7 +22,7 @@ module.exports = async (ctx) => { const chapter = chapters[j]; chapter_item.push({ title: chapter.cN, - pubDate: new Date(chapter.uT).toUTCString(), + pubDate: parseDate(chapter.uT), link: `https://vipreader.qidian.com/chapter/${id}/${chapter.id}`, }); } diff --git a/lib/routes/qidian/forum.js b/lib/v2/qidian/forum.js similarity index 93% rename from lib/routes/qidian/forum.js rename to lib/v2/qidian/forum.js index 177119e46b..8911e247a8 100644 --- a/lib/routes/qidian/forum.js +++ b/lib/v2/qidian/forum.js @@ -7,9 +7,7 @@ module.exports = async (ctx) => { const url = `https://forum.qidian.com/NewForum/List.aspx?BookId=${id}`; - const forum_response = await got({ - method: 'get', - url, + const forum_response = await got(url, { headers: { Referer: `https://book.qidian.com/info/${id}`, }, diff --git a/lib/routes/qidian/free-next.js b/lib/v2/qidian/free-next.js similarity index 93% rename from lib/routes/qidian/free-next.js rename to lib/v2/qidian/free-next.js index cbcb1abaa4..5643035c4c 100644 --- a/lib/routes/qidian/free-next.js +++ b/lib/v2/qidian/free-next.js @@ -13,10 +13,7 @@ module.exports = async (ctx) => { title = '起点中文网'; } - const response = await got({ - method: 'get', - url: link, - }); + const response = await got(link); const $ = cheerio.load(response.data); const list = $('div.other-rec-wrap li'); diff --git a/lib/routes/qidian/free.js b/lib/v2/qidian/free.js similarity index 88% rename from lib/routes/qidian/free.js rename to lib/v2/qidian/free.js index 01fe3d0d47..33253d3dca 100644 --- a/lib/routes/qidian/free.js +++ b/lib/v2/qidian/free.js @@ -1,5 +1,6 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); +const parseDate = require('@/utils/date'); module.exports = async (ctx) => { const type = ctx.params.type; @@ -13,10 +14,7 @@ module.exports = async (ctx) => { title = '起点中文网'; } - const response = await got({ - method: 'get', - url: link, - }); + const response = await got(link); const $ = cheerio.load(response.data); const list = $('#limit-list li'); @@ -31,7 +29,7 @@ module.exports = async (ctx) => { return { title: item.find('.book-mid-info h4 a').text(), description: img + rank + update + '
' + item.find('p.intro').html(), - pubDate: new Date(item.find('p.update span').text()).toUTCString(), + pubDate: parseDate(item.find('p.update span').text()), link: 'https:' + item.find('.book-mid-info h4 a').attr('href'), author: item.find('p.author a.name').text(), }; diff --git a/lib/v2/qidian/maintainer.js b/lib/v2/qidian/maintainer.js new file mode 100644 index 0000000000..a74b4404f0 --- /dev/null +++ b/lib/v2/qidian/maintainer.js @@ -0,0 +1,7 @@ +module.exports = { + '/author/:id': ['miles170'], + '/chapter/:id': ['Chingyat'], + '/forum/:id': ['Chingyat'], + '/free/:type?': ['LogicJake'], + '/free-next/:type?': ['LogicJake'], +}; diff --git a/lib/v2/qidian/radar.js b/lib/v2/qidian/radar.js new file mode 100644 index 0000000000..c823d302fc --- /dev/null +++ b/lib/v2/qidian/radar.js @@ -0,0 +1,41 @@ +module.exports = { + 'qidian.com': { + _name: '起点', + book: [ + { + title: '章节', + docs: 'https://docs.rsshub.app/reading.html#qi-dian', + source: '/info/:id', + target: '/qidian/chapter/:id', + }, + { + title: '讨论区', + docs: 'https://docs.rsshub.app/reading.html#qi-dian', + source: '/info/:id', + target: '/qidian/forum/:id', + }, + ], + my: [ + { + title: '作者', + docs: 'https://docs.rsshub.app/reading.html#qi-dian', + source: '/author/:id', + target: '/qidian/author/:id', + }, + ], + www: [ + { + title: '限免', + docs: 'https://docs.rsshub.app/reading.html#qi-dian', + source: '/free', + target: '/qidian/free', + }, + { + title: '女生限免', + docs: 'https://docs.rsshub.app/reading.html#qi-dian', + source: '/mm/free', + target: '/qidian/free/mm', + }, + ], + }, +}; diff --git a/lib/v2/qidian/router.js b/lib/v2/qidian/router.js new file mode 100644 index 0000000000..c809d5784e --- /dev/null +++ b/lib/v2/qidian/router.js @@ -0,0 +1,7 @@ +module.exports = (router) => { + router.get('/author/:id', require('./author')); + router.get('/chapter/:id', require('./chapter')); + router.get('/forum/:id', require('./forum')); + router.get('/free/:type?', require('./free')); + router.get('/free-next/:type?', require('./free-next')); +}; diff --git a/lib/v2/qidian/templates/description.art b/lib/v2/qidian/templates/description.art new file mode 100644 index 0000000000..33a6ef7e6b --- /dev/null +++ b/lib/v2/qidian/templates/description.art @@ -0,0 +1,5 @@ +

{{ description }}

+{{ if image }} + +{{ /if }} +{{ author }} \ No newline at end of file From c233b1787f62b6e9193183fc7933278f4f2d903c Mon Sep 17 00:00:00 2001 From: TonyRL Date: Mon, 29 Aug 2022 17:42:03 +0000 Subject: [PATCH 06/17] chore: pull image before waiting --- .github/workflows/scrape.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/scrape.yml b/.github/workflows/scrape.yml index f3ef22e85f..4838b7391d 100644 --- a/.github/workflows/scrape.yml +++ b/.github/workflows/scrape.yml @@ -12,10 +12,12 @@ jobs: scrape-docs: runs-on: ubuntu-latest steps: - - name: Wait for Netlify to finish - run: sleep 3m - name: Checkout uses: actions/checkout@v3 + - name: Pull image + run: docker pull getmeili/docs-scraper + - name: Wait for Netlify to finish + run: sleep 3m - name: Run docs-scraper env: HOST_URL: ${{ secrets.MEILISEARCH_HOST_URL }} From 896b450cc88e8e99c4be8724aab05d9dcebbcd71 Mon Sep 17 00:00:00 2001 From: Axojhf <20625305+axojhf@users.noreply.github.com> Date: Tue, 30 Aug 2022 01:52:01 +0800 Subject: [PATCH 07/17] feat(route): Add Love Live Topics Route (#10645) * Add Love Live Topics Route * fix radar issue --- docs/anime.md | 17 +++++- docs/en/anime.md | 17 +++++- lib/v2/lovelive-anime/maintainer.js | 1 + lib/v2/lovelive-anime/radar.js | 15 ++++-- lib/v2/lovelive-anime/router.js | 1 + lib/v2/lovelive-anime/topics.js | 84 +++++++++++++++++++++++++++++ 6 files changed, 130 insertions(+), 5 deletions(-) create mode 100644 lib/v2/lovelive-anime/topics.js diff --git a/docs/anime.md b/docs/anime.md index ce8443c735..74714e06e7 100644 --- a/docs/anime.md +++ b/docs/anime.md @@ -363,7 +363,22 @@ Sources ### Love Live 官网最新 News - + + +### Love Live 官网分类 Topics + + + +| 子企划名（非全称） | Lovelive! | Lovelive! Sunshine!! | Lovelive! Nijigasaki High School Idol Club | Lovelive! Superstar!! | +| ------------------ | ----------- | -------------------- | ------------------------------------------ | --------------------- | +| `abbr`参数 | otonokizaka | uranohoshi | nijigasaki | yuigaoka | + + +| 分类名 | 全てのニュース | 音楽商品 | アニメ映像商品 | キャスト映像商品 | 劇場 | アニメ放送/配信 | キャスト配信/ラジオ | ライブ/イベント | ブック | グッズ | ゲーム | メディア | ご当地情報 | その他 | キャンペーン | +| -------------- | --------------- | -------- | -------------- | ---------------- | ------- | --------------- | ------------------- | --------------- | ------ | ------ | ------ | -------- | ---------- | ------ | ------------ | +| `category`参数 | *无参数* | music | anime_movie | cast_movie | theater | onair | radio | event | books | goods | game | media | local | other | campaign | + + ## Mox.moe diff --git a/docs/en/anime.md b/docs/en/anime.md index 1e5c6d7952..57d57ef406 100644 --- a/docs/en/anime.md +++ b/docs/en/anime.md @@ -72,7 +72,22 @@ When `posts` is selected as the value of the parameter **source**, the parameter ### Love Live! Official Website Latest NEWS - + + +### Love Live Official Website Categories Topics + + + +| Sub-project name (not full name) | Lovelive! | Lovelive! Sunshine!! | Lovelive! Nijigasaki High School Idol Club | Lovelive! Superstar!! | +| -------------------------------- | ----------- | -------------------- | ------------------------------------------ | --------------------- | +| `abbr`parameter | otonokizaka | uranohoshi | nijigasaki | yuigaoka | + + +| category name | 全てのニュース | 音楽商品 | アニメ映像商品 | キャスト映像商品 | 劇場 | アニメ放送/配信 | キャスト配信/ラジオ | ライブ/イベント | ブック | グッズ | ゲーム | メディア | ご当地情報 | その他 | キャンペーン | +| -------------- | --------------- | -------- | -------------- | ---------------- | ------- | --------------- | ------------------- | --------------- | ------ | ------ | ------ | -------- | ---------- | ------ | ------------ | +| `category`parameter | *No parameter* | music | anime_movie | cast_movie | theater | onair | radio | event | books | goods | game | media | local | other | campaign | + + ## Touhougarakuta diff --git a/lib/v2/lovelive-anime/maintainer.js b/lib/v2/lovelive-anime/maintainer.js index 4dc741ff71..1f883a6c90 100644 --- a/lib/v2/lovelive-anime/maintainer.js +++ b/lib/v2/lovelive-anime/maintainer.js @@ -1,3 +1,4 @@ module.exports = { '/news/:option?': ['axojhf'], + '/topics/:abbr/:category?': ['axojhf'], }; diff --git a/lib/v2/lovelive-anime/radar.js b/lib/v2/lovelive-anime/radar.js index fa57df0d35..e64c194838 100644 --- a/lib/v2/lovelive-anime/radar.js +++ b/lib/v2/lovelive-anime/radar.js @@ -3,11 +3,20 @@ module.exports = { _name: 'Love Live 官网', www: [ { - title: '最新NEWS', - docs: 'https://docs.rsshub.app/anime.html#lovelive-anime', - source: ['/'], + title: '最新 NEWS', + docs: 'https://docs.rsshub.app/anime.html#lovelive-anime-love-live-guan-wang-zui-xin-news', + source: ['/', '/news'], target: '/lovelive-anime/news', }, + { + title: '分类 Topics', + docs: 'https://docs.rsshub.app/anime.html#lovelive-anime-love-live-guan-wang-fen-lei-topics', + source: ['/:abbr/topics/', '/:abbr/topics.php'], + target: (params, url) => { + const cat = url.match(/\?cat=(.*)/); + return `/lovelive-anime/topics/${params.abbr}/${null !== cat && cat.length === 2 ? cat[1] : ''}`; + }, + }, ], }, }; diff --git a/lib/v2/lovelive-anime/router.js b/lib/v2/lovelive-anime/router.js index c7bbc2e167..0734739f7b 100644 --- a/lib/v2/lovelive-anime/router.js +++ b/lib/v2/lovelive-anime/router.js @@ -1,3 +1,4 @@ module.exports = (router) => { router.get('/news/:option?', require('./news')); + router.get('/topics/:abbr/:category?/:option?', require('./topics')); }; diff --git a/lib/v2/lovelive-anime/topics.js b/lib/v2/lovelive-anime/topics.js new file mode 100644 index 0000000000..01a65f9401 --- /dev/null +++ b/lib/v2/lovelive-anime/topics.js @@ -0,0 +1,84 @@ +const got = require('@/utils/got'); +const cheerio = require('cheerio'); +const path = require('path'); +const { art } = require('@/utils/render'); +const { parseDate } = require('@/utils/parse-date'); +const renderDescription = (desc) => art(path.join(__dirname, 'templates/description.art'), desc); + +module.exports = async (ctx) => { + const abbr = ctx.params.abbr; + const rootUrl = `https://www.lovelive-anime.jp/${abbr}`; + const topicsUrlPart = 'yuigaoka' === abbr ? 'topics/' : 'topics.php'; + const baseUrl = `${rootUrl}/${'yuigaoka' === abbr ? 'topics/' : ''}`; + const abbrDetail = { + otonokizaka: 'ラブライブ！', + uranohoshi: 'サンシャイン!!', + nijigasaki: '虹ヶ咲学園', + yuigaoka: 'スーパースター!!', + }; + let url; + + if (ctx.params.hasOwnProperty('category') && ctx.params.category !== 'detail') { + url = `${rootUrl}/${topicsUrlPart}?cat=${ctx.params.category}`; + } else { + url = `${rootUrl}/${topicsUrlPart}`; + } + + const response = await got(url); + + const $ = cheerio.load(response.data); + + const categoryName = 'uranohoshi' === abbr ? $('div.llbox > p').text() : $('div.category_title > h2').text(); + + let items = $('ul.listbox > li') + .map((_, item) => { + item = $(item); + + const link = `${baseUrl}${item.find('div > a').attr('href')}`; + const pubDate = parseDate(item.find('a > p.date').text(), 'YYYY/MM/DD'); + const title = item.find('a > p.title').text(); + const category = item.find('a > p.category').text(); + const imglink = `${baseUrl}${ + item + .find('a > img') + .attr('style') + .match(/background-image:url\((.*)\)/)[1] + }`; + + return { + link, + pubDate, + title, + category, + description: renderDescription({ + title, + imglink, + }), + }; + }) + .get(); + + if (ctx.params.option === 'detail' || ctx.params.category === 'detail') { + items = await Promise.all( + items.map((item) => + ctx.cache.tryGet(item.link, async () => { + const detailResp = await got(item.link); + const $ = cheerio.load(detailResp.data); + + const content = $('div.p-page__detail.p-article'); + for (const v of content.find('img')) { + v.attribs.src = `${baseUrl}${v.attribs.src}`; + } + item.description = content.html(); + return item; + }) + ) + ); + } + + ctx.state.data = { + title: `${categoryName} - ${abbrDetail[abbr]} - Love Live Official Website Topics`, + link: url, + item: items, + }; +}; From 57b5195522c13112609238da44052755e3ce1f04 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 29 Aug 2022 17:54:22 +0000 Subject: [PATCH 08/17] style: auto format --- docs/anime.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/anime.md b/docs/anime.md index 74714e06e7..c7b9af176b 100644 --- a/docs/anime.md +++ b/docs/anime.md @@ -370,13 +370,12 @@ Sources | 子企划名（非全称） | Lovelive! | Lovelive! Sunshine!! | Lovelive! Nijigasaki High School Idol Club | Lovelive! Superstar!! | -| ------------------ | ----------- | -------------------- | ------------------------------------------ | --------------------- | -| `abbr`参数 | otonokizaka | uranohoshi | nijigasaki | yuigaoka | +| --------- | ----------- | -------------------- | ------------------------------------------ | --------------------- | +| `abbr`参数 | otonokizaka | uranohoshi | nijigasaki | yuigaoka | - -| 分类名 | 全てのニュース | 音楽商品 | アニメ映像商品 | キャスト映像商品 | 劇場 | アニメ放送/配信 | キャスト配信/ラジオ | ライブ/イベント | ブック | グッズ | ゲーム | メディア | ご当地情報 | その他 | キャンペーン | -| -------------- | --------------- | -------- | -------------- | ---------------- | ------- | --------------- | ------------------- | --------------- | ------ | ------ | ------ | -------- | ---------- | ------ | ------------ | -| `category`参数 | *无参数* | music | anime_movie | cast_movie | theater | onair | radio | event | books | goods | game | media | local | other | campaign | +| 分类名 | 全てのニュース | 音楽商品 | アニメ映像商品 | キャスト映像商品 | 劇場 | アニメ放送 / 配信 | キャスト配信 / ラジオ | ライブ / イベント | ブック | グッズ | ゲーム | メディア | ご当地情報 | その他 | キャンペーン | +| ------------ | ------------ | ----- | ----------- | ---------- | ------- | ---------- | ------------ | ---------- | ----- | ----- | ---- | ----- | ----- | ----- | -------- | +| `category`参数 | *无参数* | music | anime_movie | cast_movie | theater | onair | radio | event | books | goods | game | media | local | other | campaign | From 77db6c2de191cbe23796d3cfbdf2dd7b171395da Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Aug 2022 21:57:43 +0000 Subject: [PATCH 09/17] chore(deps): bump rand-user-agent from 1.0.78 to 1.0.79 (#10650) Bumps [rand-user-agent](https://github.com/WebScrapingAPI/rand-user-agent) from 1.0.78 to 1.0.79. - [Release notes](https://github.com/WebScrapingAPI/rand-user-agent/releases) - [Commits](https://github.com/WebScrapingAPI/rand-user-agent/commits) --- updated-dependencies: - dependency-name: rand-user-agent dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 06fed2bbc1..2fb657a72d 100644 --- a/package.json +++ b/package.json @@ -124,7 +124,7 @@ "puppeteer-extra": "3.3.4", "puppeteer-extra-plugin-stealth": "2.11.1", "query-string": "7.1.1", - "rand-user-agent": "1.0.78", + "rand-user-agent": "1.0.79", "re2": "1.17.7", "require-all": "3.0.0", "rss-parser": "3.12.0", diff --git a/yarn.lock b/yarn.lock index 964dccb8af..d6c25dd7f9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11721,10 +11721,10 @@ ramda@^0.26.1: resolved "https://registry.yarnpkg.com/ramda/-/ramda-0.26.1.tgz#8d41351eb8111c55353617fc3bbffad8e4d35d06" integrity sha512-hLWjpy7EnsDBb0p+Z3B7rPi3GDeRG5ZtiI33kJhTt+ORCd38AbAIjB/9zRIUoeTbE/AVX5ZkU7m6bznsvrf8eQ== -rand-user-agent@1.0.78: - version "1.0.78" - resolved "https://registry.yarnpkg.com/rand-user-agent/-/rand-user-agent-1.0.78.tgz#5472e954b06ab1ef8b7b080ebc0db6fd3a9a142d" - integrity sha512-atAfPjApMtpHK6pz+1zD1NYc8rV4ixqz4UtkYw17LSxKvOaaiLZBeUz6keJuTTNoTUjdYFZXAFBy29WHdvxwAw== +rand-user-agent@1.0.79: + version "1.0.79" + resolved "https://registry.yarnpkg.com/rand-user-agent/-/rand-user-agent-1.0.79.tgz#1472d536fdced3e2e2b64cccafafc20651599d91" + integrity sha512-M/KwE9V2ILytCmKNyg8g/ExrQspUUZcBNFbfryLyJuNcPC/x9IYy+9LEZnSio0a5Rfjv/iHXQYTSEqavL0aZrw== randexp@0.4.6: version "0.4.6" From e1efae05fe8349d88003cb8239f59106944c75a4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Aug 2022 23:22:17 +0000 Subject: [PATCH 10/17] chore(deps-dev): bump eslint from 8.22.0 to 8.23.0 (#10652) Bumps [eslint](https://github.com/eslint/eslint) from 8.22.0 to 8.23.0. - [Release notes](https://github.com/eslint/eslint/releases) - [Changelog](https://github.com/eslint/eslint/blob/main/CHANGELOG.md) - [Commits](https://github.com/eslint/eslint/compare/v8.22.0...v8.23.0) --- updated-dependencies: - dependency-name: eslint dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package.json | 2 +- yarn.lock | 42 +++++++++++++++++++++--------------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/package.json b/package.json index 2fb657a72d..ee0ac679e0 100644 --- a/package.json +++ b/package.json @@ -49,7 +49,7 @@ "@vuepress/plugin-pwa": "1.9.7", "@vuepress/shared-utils": "1.9.7", "cross-env": "7.0.3", - "eslint": "8.22.0", + "eslint": "8.23.0", "eslint-config-prettier": "8.5.0", "eslint-plugin-prettier": "4.2.1", "eslint-plugin-yml": "1.1.0", diff --git a/yarn.lock b/yarn.lock index d6c25dd7f9..c0711de80d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -983,14 +983,14 @@ enabled "2.0.x" kuler "^2.0.0" -"@eslint/eslintrc@^1.3.0": - version "1.3.0" - resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-1.3.0.tgz#29f92c30bb3e771e4a2048c95fa6855392dfac4f" - integrity sha512-UWW0TMTmk2d7hLcWD1/e2g5HDM/HQ3csaLSqXCfqwh4uNDuNqlaKWXmEsL4Cs41Z0KnILNvwbHAah3C2yt06kw== +"@eslint/eslintrc@^1.3.1": + version "1.3.1" + resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-1.3.1.tgz#de0807bfeffc37b964a7d0400e0c348ce5a2543d" + integrity sha512-OhSY22oQQdw3zgPOOwdoj01l/Dzl1Z+xyUP33tkSN+aqyEhymJCcPHyXt+ylW8FSe0TfRC2VG+ROQOapD0aZSQ== dependencies: ajv "^6.12.4" debug "^4.3.2" - espree "^9.3.2" + espree "^9.4.0" globals "^13.15.0" ignore "^5.2.0" import-fresh "^3.2.1" @@ -1049,6 +1049,11 @@ resolved "https://registry.yarnpkg.com/@humanwhocodes/gitignore-to-minimatch/-/gitignore-to-minimatch-1.0.2.tgz#316b0a63b91c10e53f242efb4ace5c3b34e8728d" integrity sha512-rSqmMJDdLFUsyxR6FMtD00nfQKKLFb1kv+qBbOVKqErvloEIJLo5bDTJTQNTYgeyp78JsA7u/NPi5jT1GR/MuA== +"@humanwhocodes/module-importer@^1.0.1": + version "1.0.1" + resolved "https://registry.yarnpkg.com/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz#af5b2691a22b44be847b0ca81641c5fb6ad0172c" + integrity sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA== + "@humanwhocodes/object-schema@^1.2.1": version "1.2.1" resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz#b520529ec21d8e5945a1851dfd1c32e94e39ff45" @@ -5743,14 +5748,15 @@ eslint-visitor-keys@^3.0.0, eslint-visitor-keys@^3.3.0: resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-3.3.0.tgz#f6480fa6b1f30efe2d1968aa8ac745b862469826" integrity sha512-mQ+suqKJVyeuwGYHAdjMFqjCyfl8+Ldnxuyp3ldiMBFKkvytrXUZWaiPCEav8qDHKty44bD+qV1IP4T+w+xXRA== -eslint@8.22.0: - version "8.22.0" - resolved "https://registry.yarnpkg.com/eslint/-/eslint-8.22.0.tgz#78fcb044196dfa7eef30a9d65944f6f980402c48" - integrity sha512-ci4t0sz6vSRKdmkOGmprBo6fmI4PrphDFMy5JEq/fNS0gQkJM3rLmrqcp8ipMcdobH3KtUP40KniAE9W19S4wA== +eslint@8.23.0: + version "8.23.0" + resolved "https://registry.yarnpkg.com/eslint/-/eslint-8.23.0.tgz#a184918d288820179c6041bb3ddcc99ce6eea040" + integrity sha512-pBG/XOn0MsJcKcTRLr27S5HpzQo4kLr+HjLQIyK4EiCsijDl/TB+h5uEuJU6bQ8Edvwz1XWOjpaP2qgnXGpTcA== dependencies: - "@eslint/eslintrc" "^1.3.0" + "@eslint/eslintrc" "^1.3.1" "@humanwhocodes/config-array" "^0.10.4" "@humanwhocodes/gitignore-to-minimatch" "^1.0.2" + "@humanwhocodes/module-importer" "^1.0.1" ajv "^6.10.0" chalk "^4.0.0" cross-spawn "^7.0.2" @@ -5760,7 +5766,7 @@ eslint@8.22.0: eslint-scope "^7.1.1" eslint-utils "^3.0.0" eslint-visitor-keys "^3.3.0" - espree "^9.3.3" + espree "^9.4.0" esquery "^1.4.0" esutils "^2.0.2" fast-deep-equal "^3.1.3" @@ -5786,12 +5792,11 @@ eslint@8.22.0: strip-ansi "^6.0.1" strip-json-comments "^3.1.0" text-table "^0.2.0" - v8-compile-cache "^2.0.3" -espree@^9.3.2, espree@^9.3.3: - version "9.3.3" - resolved "https://registry.yarnpkg.com/espree/-/espree-9.3.3.tgz#2dd37c4162bb05f433ad3c1a52ddf8a49dc08e9d" - integrity sha512-ORs1Rt/uQTqUKjDdGCyrtYxbazf5umATSf/K4qxjmZHORR6HJk+2s/2Pqe+Kk49HHINC/xNIrGfgh8sZcll0ng== +espree@^9.4.0: + version "9.4.0" + resolved "https://registry.yarnpkg.com/espree/-/espree-9.4.0.tgz#cd4bc3d6e9336c433265fc0aa016fc1aaf182f8a" + integrity sha512-DQmnRpLj7f6TgN/NYb0MTzJXL+vJF9h3pHy4JhCIs3zwcgez8xmGg3sXHcEO97BrmO2OSvCwMdfdlyl+E9KjOw== dependencies: acorn "^8.8.0" acorn-jsx "^5.3.2" @@ -14043,11 +14048,6 @@ uuid@^8.0.0, uuid@^8.3.2: resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2" integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg== -v8-compile-cache@^2.0.3: - version "2.3.0" - resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz#2de19618c66dc247dcfb6f99338035d8245a2cee" - integrity sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA== - v8-to-istanbul@^9.0.1: version "9.0.1" resolved "https://registry.yarnpkg.com/v8-to-istanbul/-/v8-to-istanbul-9.0.1.tgz#b6f994b0b5d4ef255e17a0d17dc444a9f5132fa4" From e6341a512cea7c9e19e865ad533cc3f75e6fcf63 Mon Sep 17 00:00:00 2001 From: miles Date: Tue, 30 Aug 2022 19:36:29 +0800 Subject: [PATCH 11/17] refactor(route): migrate greasyfork to v2 (#10655) * feat(route): migrate greasyfork to v2 * fix(route): maintainer path --- docs/en/program-update.md | 2 +- docs/program-update.md | 2 +- lib/router.js | 4 ++-- lib/v2/greasyfork/maintainer.js | 3 +++ lib/v2/greasyfork/radar.js | 13 +++++++++++++ lib/v2/greasyfork/router.js | 3 +++ lib/{routes => v2}/greasyfork/scripts.js | 8 ++++---- 7 files changed, 27 insertions(+), 8 deletions(-) create mode 100644 lib/v2/greasyfork/maintainer.js create mode 100644 lib/v2/greasyfork/radar.js create mode 100644 lib/v2/greasyfork/router.js rename lib/{routes => v2}/greasyfork/scripts.js (86%) diff --git a/docs/en/program-update.md b/docs/en/program-update.md index c3ed1eba37..942c4c8853 100644 --- a/docs/en/program-update.md +++ b/docs/en/program-update.md @@ -236,7 +236,7 @@ Language ### Script Update - + ## Hugo diff --git a/docs/program-update.md b/docs/program-update.md index 542ec10e90..455863da18 100644 --- a/docs/program-update.md +++ b/docs/program-update.md @@ -268,7 +268,7 @@ pageClass: routes ### 脚本更新 - + ## Hugo diff --git a/lib/router.js b/lib/router.js index ed66c96dcc..fa8f760775 100644 --- a/lib/router.js +++ b/lib/router.js @@ -387,8 +387,8 @@ router.get('/xuangubao/subject/:subject_id', lazyloadRouteHandler('./routes/xuan // router.get('/xueqiu/hots', lazyloadRouteHandler('./routes/xueqiu/hots')); // router.get('/xueqiu/stock_comments/:id/:titleLength?', lazyloadRouteHandler('./routes/xueqiu/stock_comments')); -// Greasy Fork -router.get('/greasyfork/:language/:domain?', lazyloadRouteHandler('./routes/greasyfork/scripts')); +// Greasy Fork migrated to v2 +// router.get('/greasyfork/:language/:domain?', lazyloadRouteHandler('./routes/greasyfork/scripts')); // Gwern Bran­wen router.get('/gwern/:category', lazyloadRouteHandler('./routes/gwern/category')); diff --git a/lib/v2/greasyfork/maintainer.js b/lib/v2/greasyfork/maintainer.js new file mode 100644 index 0000000000..0eb4f31b83 --- /dev/null +++ b/lib/v2/greasyfork/maintainer.js @@ -0,0 +1,3 @@ +module.exports = { + '/:language/:domain?': ['imlonghao', 'miles170'], +}; diff --git a/lib/v2/greasyfork/radar.js b/lib/v2/greasyfork/radar.js new file mode 100644 index 0000000000..ee48265de2 --- /dev/null +++ b/lib/v2/greasyfork/radar.js @@ -0,0 +1,13 @@ +module.exports = { + 'greasyfork.org': { + _name: 'Greasy Fork', + '.': [ + { + title: 'User scripts', + docs: 'https://docs.rsshub.app/program-update.html#greasy-fork', + source: ['/:language', '/:language/scripts/by-site/:domain?'], + target: '/greasyfork/:language/:domain?', + }, + ], + }, +}; diff --git a/lib/v2/greasyfork/router.js b/lib/v2/greasyfork/router.js new file mode 100644 index 0000000000..32144b2830 --- /dev/null +++ b/lib/v2/greasyfork/router.js @@ -0,0 +1,3 @@ +module.exports = (router) => { + router.get('/:language/:domain?', require('./scripts')); +}; diff --git a/lib/routes/greasyfork/scripts.js b/lib/v2/greasyfork/scripts.js similarity index 86% rename from lib/routes/greasyfork/scripts.js rename to lib/v2/greasyfork/scripts.js index cee531a42e..b3d0659265 100644 --- a/lib/routes/greasyfork/scripts.js +++ b/lib/v2/greasyfork/scripts.js @@ -7,10 +7,10 @@ module.exports = async (ctx) => { const domain = ctx.params.domain; const filter_locale = ctx.params.language === 'all' ? 0 : 1; const url = domain ? `by-site/${domain}` : ''; - + const currentUrl = `https://greasyfork.org/${language}/scripts/${url}`; const res = await got({ method: 'get', - url: `https://greasyfork.org/${language}/scripts/${url}`, + url: currentUrl, searchParams: queryString.stringify({ filter_locale, sort: 'updated', @@ -21,12 +21,12 @@ module.exports = async (ctx) => { ctx.state.data = { title: $('title').first().text(), - link: `https://greasyfork.org/${language}/scripts/${url}`, + link: currentUrl, description: $('meta[name=description]').attr('content'), item: list && list - .map((index, item) => { + .map((_, item) => { item = $(item); return { title: item.find('a').text(), From 3a65aec136cc8d6819e7997da51d1bb618ffb99e Mon Sep 17 00:00:00 2001 From: Cyrus <48074861+Xm798@users.noreply.github.com> Date: Tue, 30 Aug 2022 19:52:42 +0800 Subject: [PATCH 12/17] =?UTF-8?q?fix(route):=20=E5=8D=97=E4=BA=AC=E8=88=AA?= =?UTF-8?q?=E7=A9=BA=E8=88=AA=E5=A4=A9=E5=A4=A7=E5=AD=A6=E8=B7=AF=E7=94=B1?= =?UTF-8?q?=E7=BC=93=E5=AD=98=20(#10656)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(route): fix 南京航空航天大学路由缓存 * fix(route): sort the routes --- docs/university.md | 8 ++++---- lib/v2/nuaa/college/cae.js | 38 +++++++++++++++-------------------- lib/v2/nuaa/college/cs.js | 41 +++++++++++++++----------------------- lib/v2/nuaa/jwc/jwc.js | 36 ++++++++++++++------------------- lib/v2/nuaa/maintainer.js | 4 ++-- lib/v2/nuaa/yjsy/yjsy.js | 36 ++++++++++++++------------------- 6 files changed, 68 insertions(+), 95 deletions(-) diff --git a/docs/university.md b/docs/university.md index 11d5e8cfff..04b71bf0e7 100644 --- a/docs/university.md +++ b/docs/university.md @@ -1811,7 +1811,7 @@ jsjxy.hbut.edu.cn 证书链不全，自建 RSSHub 可设置环境变量 NODE_TLS ### 教务处 - + | 通知公告 | 教学服务 | 教学建设 | 学生培养 | 教学资源 | | ---- | ---- | ---- | ---- | ---- | @@ -1821,7 +1821,7 @@ jsjxy.hbut.edu.cn 证书链不全，自建 RSSHub 可设置环境变量 NODE_TLS ### 研究生院 - + | 通知公告 | 新闻动态 | 学术信息 | 师生风采 | | ---- | ---- | ---- | ---- | @@ -1831,7 +1831,7 @@ jsjxy.hbut.edu.cn 证书链不全，自建 RSSHub 可设置环境变量 NODE_TLS ### 自动化学院 - + | 综合新闻 | 党委行政 | 人事 / 合作 | 研究生培养 | 本科生培养 | 学生工作 | 通知公告 | 学术信息 | 答辩公告 | | ---- | ---- | ------- | ----- | ----- | ---- | ---- | ---- | ---- | @@ -1839,7 +1839,7 @@ jsjxy.hbut.edu.cn 证书链不全，自建 RSSHub 可设置环境变量 NODE_TLS ### 计算机科学与技术学院 - + | 通知公告 | 热点新闻 | 学科科研 | 教学动态 | 本科生培养 | 研究生培养 | 学生工作 | | ---- | ---- | ---- | ---- | ----- | ----- | ---- | diff --git a/lib/v2/nuaa/college/cae.js b/lib/v2/nuaa/college/cae.js index c2f2a8822f..3f2dfa6c05 100644 --- a/lib/v2/nuaa/college/cae.js +++ b/lib/v2/nuaa/college/cae.js @@ -43,36 +43,30 @@ module.exports = async (ctx) => { .get(); const out = await Promise.all( - list.map((info) => { - const title = info.title || 'zhxw'; + list.map(async (info) => { + const title = info.title || 'tzgg'; const date = info.date; const itemUrl = new URL(info.link, host).href; + let description = title + '
查看原文'; - return ctx.cache.tryGet(itemUrl, async () => { - const arr = itemUrl.split('.'); - const pageType = arr[arr.length - 1]; - - // 南航新 WAF 过于敏感 - // 目前 description 需要遍历页面，会被 WAF 拦截导致无法输出 - // 考虑换一种获取 description 的方式或者将标题当作 title。 - let description = title + '
查看原文'; - if (getDescription) { - description = itemUrl; + if (getDescription) { + description = await ctx.cache.tryGet(itemUrl, async () => { + const arr = itemUrl.split('.'); + const pageType = arr[arr.length - 1]; if (pageType === 'htm' || pageType === 'html') { const response = await got(itemUrl, gotConfig); const $ = cheerio.load(response.data); - description = $('.wp_articlecontent').html() + '
查看原文'; + return $('.wp_articlecontent').html() + '
查看原文'; } - } + }); + } - const single = { - title, - link: itemUrl, - description, - pubDate: parseDate(date), - }; - return single; - }); + return { + title, + link: itemUrl, + description, + pubDate: parseDate(date), + }; }) ); diff --git a/lib/v2/nuaa/college/cs.js b/lib/v2/nuaa/college/cs.js index 36eae3327e..672573432a 100644 --- a/lib/v2/nuaa/college/cs.js +++ b/lib/v2/nuaa/college/cs.js @@ -42,39 +42,30 @@ module.exports = async (ctx) => { .get(); const out = await Promise.all( - list.map((info) => { + list.map(async (info) => { const title = info.title || 'tzgg'; const date = info.date; const itemUrl = new URL(info.link, host).href; + let description = title + '
查看原文'; - return ctx.cache.tryGet(itemUrl, async () => { - const arr = itemUrl.split('.'); - const pageType = arr[arr.length - 1]; - - // 南航新 WAF 过于敏感 - // 目前 description 需要遍历页面，会被 WAF 拦截导致无法输出 - // 考虑换一种获取 description 的方式或者将标题当作 title。 - let description = title; - if (getDescription) { - description = itemUrl; + if (getDescription) { + description = await ctx.cache.tryGet(itemUrl, async () => { + const arr = itemUrl.split('.'); + const pageType = arr[arr.length - 1]; if (pageType === 'htm' || pageType === 'html') { - const response = await got.get(itemUrl, gotConfig); + const response = await got(itemUrl, gotConfig); const $ = cheerio.load(response.data); - description = $('.wp_articlecontent') - .html() - .replace(/src="\//g, `src="${new URL('.', host).href}`) - .trim(); + return $('.wp_articlecontent').html() + '
查看原文'; } - } + }); + } - const single = { - title, - link: itemUrl, - description, - pubDate: parseDate(date), - }; - return single; - }); + return { + title, + link: itemUrl, + description, + pubDate: parseDate(date), + }; }) ); diff --git a/lib/v2/nuaa/jwc/jwc.js b/lib/v2/nuaa/jwc/jwc.js index 3710b759f3..22c8b22294 100644 --- a/lib/v2/nuaa/jwc/jwc.js +++ b/lib/v2/nuaa/jwc/jwc.js @@ -40,36 +40,30 @@ module.exports = async (ctx) => { .get(); const out = await Promise.all( - list.map((info) => { + list.map(async (info) => { const title = info.title || 'tzgg'; const date = info.date; const itemUrl = new URL(info.link, host).href; + let description = title + '
查看原文'; - return ctx.cache.tryGet(itemUrl, async () => { - const arr = itemUrl.split('.'); - const pageType = arr[arr.length - 1]; - - // 南航新 WAF 过于敏感 - // 目前 description 需要遍历页面，会被 WAF 拦截导致无法输出 - // 考虑换一种获取 description 的方式或者将标题当作 title。 - let description = title + '
查看原文'; - if (getDescription) { - description = itemUrl; + if (getDescription) { + description = await ctx.cache.tryGet(itemUrl, async () => { + const arr = itemUrl.split('.'); + const pageType = arr[arr.length - 1]; if (pageType === 'htm' || pageType === 'html') { const response = await got(itemUrl, gotConfig); const $ = cheerio.load(response.data); - description = $('.wp_articlecontent').html() + '
查看原文'; + return $('.wp_articlecontent').html() + '
查看原文'; } - } + }); + } - const single = { - title, - link: itemUrl, - description, - pubDate: parseDate(date), - }; - return single; - }); + return { + title, + link: itemUrl, + description, + pubDate: parseDate(date), + }; }) ); diff --git a/lib/v2/nuaa/maintainer.js b/lib/v2/nuaa/maintainer.js index 12dfe3d764..bd7b7fc5ee 100644 --- a/lib/v2/nuaa/maintainer.js +++ b/lib/v2/nuaa/maintainer.js @@ -1,6 +1,6 @@ module.exports = { - '/cae/:type/:getDescription?': ['Seiry', 'qrzbing', 'Xm798'], - '/cs/:type/:getDescription?': ['LogicJake', 'Seiry', 'qrzbing'], + '/cae/:type/:getDescription?': ['Xm798'], + '/cs/:type/:getDescription?': ['LogicJake', 'Seiry', 'qrzbing', 'Xm798'], '/jwc/:type/:getDescription?': ['arcosx', 'Seiry', 'qrzbing', 'Xm798'], '/yjsy/:type/:getDescription?': ['junfengP', 'Seiry', 'Xm798'], }; diff --git a/lib/v2/nuaa/yjsy/yjsy.js b/lib/v2/nuaa/yjsy/yjsy.js index fb8d25aced..c36bb1c29e 100644 --- a/lib/v2/nuaa/yjsy/yjsy.js +++ b/lib/v2/nuaa/yjsy/yjsy.js @@ -38,36 +38,30 @@ module.exports = async (ctx) => { .get(); const out = await Promise.all( - list.map((info) => { + list.map(async (info) => { const title = info.title || 'tzgg'; const date = info.date; const itemUrl = new URL(info.link, host).href; + let description = title + '
查看原文'; - return ctx.cache.tryGet(itemUrl, async () => { - const arr = itemUrl.split('.'); - const pageType = arr[arr.length - 1]; - - // 南航新 WAF 过于敏感 - // 目前 description 需要遍历页面，会被 WAF 拦截导致无法输出 - // 考虑换一种获取 description 的方式或者将标题当作 title。 - let description = title + '
查看原文'; - if (getDescription) { - description = itemUrl; + if (getDescription) { + description = await ctx.cache.tryGet(itemUrl, async () => { + const arr = itemUrl.split('.'); + const pageType = arr[arr.length - 1]; if (pageType === 'htm' || pageType === 'html') { const response = await got(itemUrl, gotConfig); const $ = cheerio.load(response.data); - description = $('.wp_articlecontent').html() + '
查看原文'; + return $('.wp_articlecontent').html() + '
查看原文'; } - } + }); + } - const single = { - title, - link: itemUrl, - description, - pubDate: parseDate(date), - }; - return single; - }); + return { + title, + link: itemUrl, + description, + pubDate: parseDate(date), + }; }) ); From 3c7cd655e99702cc4940ab5662a7c0426359ccae Mon Sep 17 00:00:00 2001 From: miles Date: Tue, 30 Aug 2022 20:53:52 +0800 Subject: [PATCH 13/17] =?UTF-8?q?feat(route):=20add=20=E8=8F=AF=E8=A6=96?= =?UTF-8?q?=E6=96=B0=E8=81=9E=E7=B6=B2=20(#10658)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(route): add 華視新聞網 * fix(route): rate limit * feat(route): description uses HTML instead of raw text --- docs/traditional-media.md | 12 ++++++++++++ lib/v2/cts/maintainer.js | 3 +++ lib/v2/cts/news.js | 40 +++++++++++++++++++++++++++++++++++++++ lib/v2/cts/radar.js | 13 +++++++++++++ lib/v2/cts/router.js | 3 +++ 5 files changed, 71 insertions(+) create mode 100644 lib/v2/cts/maintainer.js create mode 100644 lib/v2/cts/news.js create mode 100644 lib/v2/cts/radar.js create mode 100644 lib/v2/cts/router.js diff --git a/docs/traditional-media.md b/docs/traditional-media.md index ab639f15f8..10e929b8a7 100644 --- a/docs/traditional-media.md +++ b/docs/traditional-media.md @@ -1152,6 +1152,18 @@ Type 栏目： +## 華視 + +### 新聞 + + + +| 即時 | 氣象 | 政治 | 國際 | 社會 | 運動 | 生活 | 財經 | 台語 | 地方 | 產業 | 綜合 | 藝文 | 娛樂 | +| ---- | ------- | -------- | ------------- | ------- | ------ | ---- | ----- | --------- | ----- | -- | ------- | ---- | --------- | +| real | weather | politics | international | society | sports | life | money | taiwanese | local | pr | general | arts | entertain | + + + ## 环球网 ### 分类 diff --git a/lib/v2/cts/maintainer.js b/lib/v2/cts/maintainer.js new file mode 100644 index 0000000000..5c4aa089da --- /dev/null +++ b/lib/v2/cts/maintainer.js @@ -0,0 +1,3 @@ +module.exports = { + '/:category': ['miles170'], +}; diff --git a/lib/v2/cts/news.js b/lib/v2/cts/news.js new file mode 100644 index 0000000000..e77d0ae898 --- /dev/null +++ b/lib/v2/cts/news.js @@ -0,0 +1,40 @@ +const got = require('@/utils/got'); +const cheerio = require('cheerio'); +const { parseDate } = require('@/utils/parse-date'); +const asyncPool = require('tiny-async-pool'); + +module.exports = async (ctx) => { + const category = ctx.params.category; + const currentUrl = `https://news.cts.com.tw/${category}/index.html`; + const response = await got(currentUrl); + const $ = cheerio.load(response.data); + const items = []; + for await (const data of asyncPool(5, $('#newslist-top a[title]').slice(0, ctx.query.limit ? parseInt(ctx.query.limit) : 20), (item) => { + item = $(item); + const link = item.attr('href'); + return ctx.cache.tryGet(link, async () => { + const response = await got(link); + const $ = cheerio.load(response.data); + const author = $('.artical-content p:eq(0)').text().trim(); + $('.artical-content p:eq(0), .artical-content .flexbox').remove(); + + return { + title: item.attr('title'), + author, + description: $('.artical-content').html(), + category: $('meta[property="article:section"]').attr('content'), + pubDate: parseDate($('meta[property="article:published_time"]').attr('content')), + link, + }; + }); + })) { + items.push(data); + } + + ctx.state.data = { + title: $('title').text(), + link: currentUrl, + description: $('meta[name="description"]').attr('content'), + item: items, + }; +}; diff --git a/lib/v2/cts/radar.js b/lib/v2/cts/radar.js new file mode 100644 index 0000000000..6dc1db9275 --- /dev/null +++ b/lib/v2/cts/radar.js @@ -0,0 +1,13 @@ +module.exports = { + 'cts.com.tw': { + _name: '華視', + news: [ + { + title: '新聞', + docs: 'https://docs.rsshub.app/traditional-media.html#hua-shi-xin-wen', + source: '/:category/index.html', + target: '/cts/:category', + }, + ], + }, +}; diff --git a/lib/v2/cts/router.js b/lib/v2/cts/router.js new file mode 100644 index 0000000000..b9311da838 --- /dev/null +++ b/lib/v2/cts/router.js @@ -0,0 +1,3 @@ +module.exports = function (router) { + router.get('/:category', require('./news')); +}; From fee049146e80077e60657b8f6b3aedd7ffef46a7 Mon Sep 17 00:00:00 2001 From: Tony Date: Tue, 30 Aug 2022 09:29:51 -0900 Subject: [PATCH 14/17] chore: update meilisearch settings (#10659) * chore: update meilisearch settings * fix: docker build --- .github/workflows/scrape.yml | 18 +++++++++++ package.json | 1 + scripts/docs-scraper/settings.js | 55 ++++++++++++++++++++++++++++++++ yarn.lock | 2 +- 4 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 scripts/docs-scraper/settings.js diff --git a/.github/workflows/scrape.yml b/.github/workflows/scrape.yml index 4838b7391d..5c01d6a300 100644 --- a/.github/workflows/scrape.yml +++ b/.github/workflows/scrape.yml @@ -7,6 +7,7 @@ on: paths: - '.github/workflows/scrape.yml' - 'docs/**' + workflow_dispatch: ~ jobs: scrape-docs: @@ -14,10 +15,16 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 + - name: Use Node.js v16 + uses: actions/setup-node@v3 + with: + node-version: 16 + cache: 'yarn' - name: Pull image run: docker pull getmeili/docs-scraper - name: Wait for Netlify to finish run: sleep 3m + if: github.event_name == 'push' - name: Run docs-scraper env: HOST_URL: ${{ secrets.MEILISEARCH_HOST_URL }} @@ -29,3 +36,14 @@ jobs: -e MEILISEARCH_API_KEY=$API_KEY \ -v $CONFIG_FILE_PATH:/docs-scraper/config.json \ getmeili/docs-scraper pipenv run ./docs_scraper config.json + - name: Update meilisearch settings + uses: actions/github-script@v6 + env: + HOST_URL: ${{ secrets.MEILISEARCH_HOST_URL }} + API_KEY: ${{ secrets.MEILISEARCH_API_KEY }} + with: + script: | + const meilisearchHost = process.env.HOST_URL + const meilisearchApiKey = process.env.API_KEY + const script = require(`${process.env.GITHUB_WORKSPACE}/scripts/docs-scraper/settings.js`) + await script({ core }, meilisearchHost, meilisearchApiKey) diff --git a/package.json b/package.json index ee0ac679e0..58a21cbca3 100644 --- a/package.json +++ b/package.json @@ -55,6 +55,7 @@ "eslint-plugin-yml": "1.1.0", "fs-extra": "10.1.0", "jest": "29.0.1", + "meilisearch": "0.27.0", "mockdate": "3.0.5", "nock": "13.2.9", "nodemon": "2.0.19", diff --git a/scripts/docs-scraper/settings.js b/scripts/docs-scraper/settings.js new file mode 100644 index 0000000000..3b96e19521 --- /dev/null +++ b/scripts/docs-scraper/settings.js @@ -0,0 +1,55 @@ +const { MeiliSearch } = require('meilisearch'); + +module.exports = async ({ core }, host, apiKey) => { + const client = new MeiliSearch({ + host, + apiKey, + }); + core.info('Updating displayed attributes'); + core.info( + await client + .index('rsshub') + .updateDisplayedAttributes([ + 'hierarchy_radio_lvl1', + 'hierarchy_radio_lvl2', + 'hierarchy_radio_lvl3', + 'hierarchy_radio_lvl4', + 'hierarchy_radio_lvl5', + 'hierarchy_lvl0', + 'hierarchy_lvl1', + 'hierarchy_lvl2', + 'hierarchy_lvl3', + 'hierarchy_lvl4', + 'hierarchy_lvl5', + 'hierarchy_lvl6', + 'anchor', + 'url', + 'content', + ]) + ); + + core.info('Updating searchable attributes'); + core.info( + await client + .index('rsshub') + .updateSearchableAttributes([ + 'hierarchy_radio_lvl1', + 'hierarchy_radio_lvl2', + 'hierarchy_radio_lvl3', + 'hierarchy_radio_lvl4', + 'hierarchy_radio_lvl5', + 'hierarchy_lvl0', + 'hierarchy_lvl1', + 'hierarchy_lvl2', + 'hierarchy_lvl3', + 'hierarchy_lvl4', + 'hierarchy_lvl5', + 'hierarchy_lvl6', + 'anchor', + 'content', + ]) + ); + + core.info('Updating sortable attributes'); + core.info(await client.index('rsshub').updateSortableAttributes(['anchor'])); +}; diff --git a/yarn.lock b/yarn.lock index c0711de80d..a79a50f651 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9590,7 +9590,7 @@ media-typer@0.3.0: resolved "https://registry.yarnpkg.com/media-typer/-/media-typer-0.3.0.tgz#8710d7af0aa626f8fffa1ce00168545263255748" integrity sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ== -meilisearch@^0.27.0: +meilisearch@0.27.0, meilisearch@^0.27.0: version "0.27.0" resolved "https://registry.yarnpkg.com/meilisearch/-/meilisearch-0.27.0.tgz#8bd57ddb77b975f93e054cb977b951c488ece297" integrity sha512-kZOZFIuSO7c6xRf+Y2/9/h6A9pl0sCl/G44X4KuaSwxGbruOZPhmxbeVEgLHBv4pUFvQ56rNVTA/2d/5GCU1YA== From 29990a0c2a6dde5d2eaf1e87e1d898f73666b814 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 30 Aug 2022 21:10:43 +0000 Subject: [PATCH 15/17] chore(deps): bump dotenv from 16.0.1 to 16.0.2 (#10660) Bumps [dotenv](https://github.com/motdotla/dotenv) from 16.0.1 to 16.0.2. - [Release notes](https://github.com/motdotla/dotenv/releases) - [Changelog](https://github.com/motdotla/dotenv/blob/master/CHANGELOG.md) - [Commits](https://github.com/motdotla/dotenv/compare/v16.0.1...v16.0.2) --- updated-dependencies: - dependency-name: dotenv dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 58a21cbca3..4441410a22 100644 --- a/package.json +++ b/package.json @@ -92,7 +92,7 @@ "crypto-js": "4.1.1", "currency-symbol-map": "5.1.0", "dayjs": "1.11.5", - "dotenv": "16.0.1", + "dotenv": "16.0.2", "emailjs-imap-client": "3.1.0", "entities": "3.0.1", "etag": "1.8.1", diff --git a/yarn.lock b/yarn.lock index a79a50f651..37f450ee04 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5223,10 +5223,10 @@ dot-prop@^5.2.0: dependencies: is-obj "^2.0.0" -dotenv@16.0.1: - version "16.0.1" - resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.0.1.tgz#8f8f9d94876c35dac989876a5d3a82a267fdce1d" - integrity sha512-1K6hR6wtk2FviQ4kEiSjFiH5rpzEVi8WW0x96aztHVMhEspNpc4DVOUTEHtEva5VThQ8IaBX1Pe4gSzpVVUsKQ== +dotenv@16.0.2: + version "16.0.2" + resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.0.2.tgz#0b0f8652c016a3858ef795024508cddc4bffc5bf" + integrity sha512-JvpYKUmzQhYoIFgK2MOnF3bciIZoItIIoryihy0rIA+H4Jy0FmgyKYAHCTN98P5ybGSJcIFbh6QKeJdtZd1qhA== dotenv@^6.2.0: version "6.2.0" From c1176174d4a90105562a1f87af7060233f660670 Mon Sep 17 00:00:00 2001 From: Tony Date: Wed, 31 Aug 2022 01:48:34 +0200 Subject: [PATCH 16/17] chore: install deps --- .github/workflows/scrape.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/scrape.yml b/.github/workflows/scrape.yml index 5c01d6a300..3c626c5144 100644 --- a/.github/workflows/scrape.yml +++ b/.github/workflows/scrape.yml @@ -20,6 +20,8 @@ jobs: with: node-version: 16 cache: 'yarn' + - name: Install dependencies (yarn) + run: yarn - name: Pull image run: docker pull getmeili/docs-scraper - name: Wait for Netlify to finish From 9e1796944fc9f997d2ba422dd10765dd3c69916e Mon Sep 17 00:00:00 2001 From: miles Date: Wed, 31 Aug 2022 16:16:56 +0800 Subject: [PATCH 17/17] =?UTF-8?q?feat(route):=20add=20=E8=81=AF=E5=90=88?= =?UTF-8?q?=E6=96=B0=E8=81=9E=E7=B6=B2=20(#10654)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(route): add 聯合新聞網 * fix(route): concatenate the url correctly * feat(route): fetch link name * feat(route): description uses HTML instead of raw text * feat(route): remove `from` search params from link * feat(route): improve feed description readability * fix(route): article was not properly parsed * refactor(route): improve feed fetching --- docs/traditional-media.md | 12 +++++ lib/v2/udn/breaking-news.js | 85 +++++++++++++++++++++++++++++++++ lib/v2/udn/maintainer.js | 3 ++ lib/v2/udn/radar.js | 13 +++++ lib/v2/udn/router.js | 3 ++ lib/v2/udn/templates/figure.art | 4 ++ 6 files changed, 120 insertions(+) create mode 100644 lib/v2/udn/breaking-news.js create mode 100644 lib/v2/udn/maintainer.js create mode 100644 lib/v2/udn/radar.js create mode 100644 lib/v2/udn/router.js create mode 100644 lib/v2/udn/templates/figure.art diff --git a/docs/traditional-media.md b/docs/traditional-media.md index 10e929b8a7..5dbcb95bf1 100644 --- a/docs/traditional-media.md +++ b/docs/traditional-media.md @@ -1376,6 +1376,18 @@ category 对应的关键词有 +## 聯合新聞網 + +### 即時新聞 + + + +| 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 11 | 12 | 13 | 99 | +| -- | -- | -- | -- | -- | -- | -- | -- | -- | -- | -- | -- | -- | --- | +| 精選 | 要聞 | 社會 | 地方 | 兩岸 | 國際 | 財經 | 運動 | 娛樂 | 生活 | 股市 | 文教 | 數位 | 不分類 | + + + ## 路透社 ::: warning 迁移说明 diff --git a/lib/v2/udn/breaking-news.js b/lib/v2/udn/breaking-news.js new file mode 100644 index 0000000000..9f3ee3c94c --- /dev/null +++ b/lib/v2/udn/breaking-news.js @@ -0,0 +1,85 @@ +const got = require('@/utils/got'); +const { parseDate } = require('@/utils/parse-date'); +const timezone = require('@/utils/timezone'); +const cheerio = require('cheerio'); +const { art } = require('@/utils/render'); +const path = require('path'); + +module.exports = async (ctx) => { + const id = ctx.params.id; + const link = `/news/breaknews/1/${id}#breaknews`; + const name = await getLinkName(ctx, link); + const rootUrl = 'https://udn.com'; + const response = await got(`${rootUrl}/api/more?page=1&channelId=1&cate_id=${id}&type=breaknews`); + const items = await Promise.all( + response.data.lists.map((item) => { + let link = item.titleLink.indexOf('http://') === 0 || item.titleLink.indexOf('https://') === 0 ? item.titleLink : `${rootUrl}${item.titleLink}`; + const linkUrl = new URL(link); + // cleanup query paramter + linkUrl.query = linkUrl.search = ''; + link = linkUrl.toString(); + return ctx.cache.tryGet(link, async () => { + let result = await got(link); + // VIP article requires redirection + // e.g. https://udn.com/news/story/7331/6576320 + const vip = result.data.match(/