From 25bb7dd9746097423095fb4a12f7b11c11fb5548 Mon Sep 17 00:00:00 2001 From: Toby Tso Date: Mon, 28 Dec 2020 16:34:40 +0800 Subject: [PATCH 01/49] WIP: add Media Digest --- lib/router.js | 3 ++ lib/routes/mediadigest/category.js | 69 ++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) create mode 100644 lib/routes/mediadigest/category.js diff --git a/lib/router.js b/lib/router.js index 2da2ee3d3c..dcb959fb8a 100644 --- a/lib/router.js +++ b/lib/router.js @@ -3766,4 +3766,7 @@ router.get('/sciencenet/blog/:type?/:time?/:sort?', require('./routes/sciencenet // DailyArt router.get('/dailyart/:language?', require('./routes/dailyart/index')); +// Media Digest +router.get('/mediadigest/:category?', require('./routes/mediadigest/category')); + module.exports = router; diff --git a/lib/routes/mediadigest/category.js b/lib/routes/mediadigest/category.js new file mode 100644 index 0000000000..2225d6691b --- /dev/null +++ b/lib/routes/mediadigest/category.js @@ -0,0 +1,69 @@ +const got = require('@/utils/got'); +const cheerio = require('cheerio'); + +function getArticle(line) { + const article = ctx.cache.tryGet(line, async () => { + const a_r = await got.get(`https://app3.rthk.hk/mediadigest/${line}`); + const $ = cheerio.load(a_r.data); + // title + const h1 = $('h1.story-title').text(); + // author + const author_list = $('div.story-author'); + const authors = author_list.map((_index, author) => $(author).text()); + const author = authors.map((_index, author) => `${author}
`); + const s_author = author.toArray().join(""); + const author_block = `

${s_author}

`; + // date + const date = $('div.story-calendar').text(); + // desc + const desc = `${$(author_block)}${$('div.story-content').html()}`; + + return { + title: h1, + description: desc, + pubDate: new Date(`${date}T09:00:00+0800`).toUTCString(), + link: line, + } + }); + tasks.push(article); +} + +module.exports = async (ctx) => { + const category = ctx.params.category || 'all'; + const tasks = []; + + if (category !== 'all') { + const response = await got.get(`https://app3.rthk.hk/mediadigest/category.php?cid=${category}`); + const $ = cheerio.load(response.data); + const list = $('div.category-line'); + + list.map((_index, line) => $(line).find('a').first().attr('href')) + .each((_index, line) => { + getArticle(line); + }); + } else { + let cids = [1, 2]; for (let i = 4; i < 28; ++i) {cids.push(i);}; + let urls = cids.map((cid) => `https://app3.rthk.hk/mediadigest/category.php?cid=${cid}`); + const list_allt = await Promise.all( + urls.map(async (url) => { + const response = await got.get(url); + const $ = cheerio.load(response.data); + const list = $('div.category-line').map((_index, line) => $(line).find('a').first().attr('href')); + return Promise.resolve(list.toArray()); + }) + ); + const list_all = [...new Set(list_allt.flat())]; + + list_all.forEach((line) => { + getArticle(line); + }); + }; + + const rss = await Promise.all(tasks); + + ctx.state.data = { + title: '傳媒透視', + link: 'https://app3.rthk.hk/mediadigest/index.php', + item: rss, + }; +}; From 4c2b46207173147c47cbb4dad47881c471b2ebbc Mon Sep 17 00:00:00 2001 From: Toby Tso Date: Mon, 28 Dec 2020 16:36:00 +0800 Subject: [PATCH 02/49] WIP: fix --- lib/routes/mediadigest/category.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/routes/mediadigest/category.js b/lib/routes/mediadigest/category.js index 2225d6691b..d52214530b 100644 --- a/lib/routes/mediadigest/category.js +++ b/lib/routes/mediadigest/category.js @@ -23,7 +23,7 @@ function getArticle(line) { description: desc, pubDate: new Date(`${date}T09:00:00+0800`).toUTCString(), link: line, - } + }; }); tasks.push(article); } From bd6effd2e2e8c0986b331083f443b0d5fe50c9be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=82=AE=E3=83=A3=E3=83=A9?= Date: Mon, 28 Dec 2020 09:08:57 +0000 Subject: [PATCH 03/49] fix --- lib/routes/mediadigest/category.js | 34 +++++++++++++++++++----------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/lib/routes/mediadigest/category.js b/lib/routes/mediadigest/category.js index d52214530b..b42a27f649 100644 --- a/lib/routes/mediadigest/category.js +++ b/lib/routes/mediadigest/category.js @@ -1,7 +1,7 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); -function getArticle(line) { +function getArticle(ctx, line, tasks) { const article = ctx.cache.tryGet(line, async () => { const a_r = await got.get(`https://app3.rthk.hk/mediadigest/${line}`); const $ = cheerio.load(a_r.data); @@ -11,7 +11,7 @@ function getArticle(line) { const author_list = $('div.story-author'); const authors = author_list.map((_index, author) => $(author).text()); const author = authors.map((_index, author) => `${author}
`); - const s_author = author.toArray().join(""); + const s_author = author.toArray().join(''); const author_block = `

${s_author}

`; // date const date = $('div.story-calendar').text(); @@ -37,27 +37,37 @@ module.exports = async (ctx) => { const $ = cheerio.load(response.data); const list = $('div.category-line'); - list.map((_index, line) => $(line).find('a').first().attr('href')) - .each((_index, line) => { - getArticle(line); - }); + list.map((_index, line) => $(line).find('a').first().attr('href')).each((_index, line) => { + getArticle(ctx, line, tasks); + }); } else { - let cids = [1, 2]; for (let i = 4; i < 28; ++i) {cids.push(i);}; - let urls = cids.map((cid) => `https://app3.rthk.hk/mediadigest/category.php?cid=${cid}`); + const cids = [1, 2]; + for (let i = 4; i < 28; ++i) { + cids.push(i); + } + const urls = cids.map((cid) => `https://app3.rthk.hk/mediadigest/category.php?cid=${cid}`); const list_allt = await Promise.all( urls.map(async (url) => { const response = await got.get(url); const $ = cheerio.load(response.data); - const list = $('div.category-line').map((_index, line) => $(line).find('a').first().attr('href')); + const list = $('div.category-line').map((_index, line) => $(line).find('a').first().attr('href')); return Promise.resolve(list.toArray()); }) ); const list_all = [...new Set(list_allt.flat())]; - list_all.forEach((line) => { - getArticle(line); + const list_recent_20 = list_all + .sort((a, b) => { + const aid_a = a.match(/aid=(\d+)/)[1]; + const aid_b = b.match(/aid=(\d+)/)[1]; + // reverse + return aid_b - aid_a; + }) + .slice(0, 20); + list_recent_20.forEach((line) => { + getArticle(ctx, line, tasks); }); - }; + } const rss = await Promise.all(tasks); From e23d6e7c90eafbf6f16975386830c652b76e1117 Mon Sep 17 00:00:00 2001 From: Toby Tso Date: Tue, 29 Dec 2020 15:44:21 +0800 Subject: [PATCH 04/49] WIP: add `:range` to route --- lib/router.js | 2 +- lib/routes/mediadigest/category.js | 285 ++++++++++++++++++++++------- 2 files changed, 223 insertions(+), 64 deletions(-) diff --git a/lib/router.js b/lib/router.js index dcb959fb8a..488de8eb2f 100644 --- a/lib/router.js +++ b/lib/router.js @@ -3767,6 +3767,6 @@ router.get('/sciencenet/blog/:type?/:time?/:sort?', require('./routes/sciencenet router.get('/dailyart/:language?', require('./routes/dailyart/index')); // Media Digest -router.get('/mediadigest/:category?', require('./routes/mediadigest/category')); +router.get('/mediadigest/:range/:category?', require('./routes/mediadigest/category')); module.exports = router; diff --git a/lib/routes/mediadigest/category.js b/lib/routes/mediadigest/category.js index b42a27f649..757541d4b9 100644 --- a/lib/routes/mediadigest/category.js +++ b/lib/routes/mediadigest/category.js @@ -1,79 +1,238 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); -function getArticle(ctx, line, tasks) { - const article = ctx.cache.tryGet(line, async () => { - const a_r = await got.get(`https://app3.rthk.hk/mediadigest/${line}`); - const $ = cheerio.load(a_r.data); - // title - const h1 = $('h1.story-title').text(); - // author - const author_list = $('div.story-author'); - const authors = author_list.map((_index, author) => $(author).text()); - const author = authors.map((_index, author) => `${author}
`); - const s_author = author.toArray().join(''); - const author_block = `

${s_author}

`; - // date - const date = $('div.story-calendar').text(); - // desc - const desc = `${$(author_block)}${$('div.story-content').html()}`; +function getArticle(ctx, list) { + // for 每次可爲 rss 値推入 20 篇文章內容 + for (let start = 0; start < list.length; start += 20) { + // 建立一個切片 (20 個文章 URL) 異步獲取文章內容的任務 + let now = list.slice(start, start+20).forEach((line) => { + ctx.cache.tryGet(line, async() => { + const a_r = await got.get(`https://app3.rthk.hk/mediadigest/${line}`); + const $ = cheerio.load(a_r.data); + // title + const h1 = $('h1.story-title').text(); + // author + const author_list = $('div.story-author'); + const authors = author_list.map((_index, author) => $(author).text()); + const author = authors.map((_index, author) => `${author}
`); + const s_author = author.toArray().join(''); + const author_block = `

${s_author}

`; + // date + const date = $('div.story-calendar').text(); + // desc + const desc = `${$(author_block)}${$('div.story-content').html()}`; - return { - title: h1, - description: desc, - pubDate: new Date(`${date}T09:00:00+0800`).toUTCString(), - link: line, - }; - }); - tasks.push(article); + return { + title: h1, + description: desc, + pubDate: new Date(`${date}T09:00:00+0800`).toUTCString(), + link: line, + }; + }); + }) + // 執行一個切片 (20 個文章 URL) 的任務並將結果資料推至 rss 値 + rss.push(...(await Promise.all(now))); + } } -module.exports = async (ctx) => { +module.exports = async(ctx) => { + const range = ctx.params.range || 'latest'; const category = ctx.params.category || 'all'; - const tasks = []; - if (category !== 'all') { - const response = await got.get(`https://app3.rthk.hk/mediadigest/category.php?cid=${category}`); - const $ = cheerio.load(response.data); - const list = $('div.category-line'); - - list.map((_index, line) => $(line).find('a').first().attr('href')).each((_index, line) => { - getArticle(ctx, line, tasks); - }); - } else { - const cids = [1, 2]; - for (let i = 4; i < 28; ++i) { - cids.push(i); - } - const urls = cids.map((cid) => `https://app3.rthk.hk/mediadigest/category.php?cid=${cid}`); - const list_allt = await Promise.all( - urls.map(async (url) => { - const response = await got.get(url); - const $ = cheerio.load(response.data); - const list = $('div.category-line').map((_index, line) => $(line).find('a').first().attr('href')); - return Promise.resolve(list.toArray()); - }) - ); - const list_all = [...new Set(list_allt.flat())]; - - const list_recent_20 = list_all - .sort((a, b) => { - const aid_a = a.match(/aid=(\d+)/)[1]; - const aid_b = b.match(/aid=(\d+)/)[1]; - // reverse - return aid_b - aid_a; - }) - .slice(0, 20); - list_recent_20.forEach((line) => { - getArticle(ctx, line, tasks); - }); + // for range validation + var num = /^[0-9]+$/; + var current_year = new Date().getFullYear(); + // placeholder for rss + let rss = []; + // cid + const cids = [1, 2]; + for (let i = 4; i < 28; ++i) { + cids.push(i); } - const rss = await Promise.all(tasks); + // latest (50 articles): + // if 'all' (latest 50 articles of the site); + // else if 'cid' (all articles of a specific category); + // else 'error'. + if (range === 'latest') { + if (category === 'all') { + // 獲取全站文章 URL + const urls = cids.map((cid) => `https://app3.rthk.hk/mediadigest/category.php?cid=${cid}`); + const list_allt = await Promise.all( + urls.map(async(url) => { + const response = await got.get(url); + const $ = cheerio.load(response.data); + const list = $('div.category-line').map((_index, line) => $(line).find('a').first().attr('href')); + return Promise.resolve(list.toArray()); + }) + ); + const list_all = [...new Set(list_allt.flat())]; // removed duplicates and flatten + // 時序排列並抽取最新 50 項 + const list = list_all + .sort((a, b) => { + const aid_a = a.match(/aid=(\d+)/)[1]; + const aid_b = b.match(/aid=(\d+)/)[1]; + // reverse + return aid_b - aid_a; + }) + .slice(0, 50); + // getArticle(ctx, list); + // for 每次會可爲 rss 値段推入 20 篇文章內容 + for (let start = 0; start < list.length; start += 20) { + // 建立一個切片 (20 個文章 URL) 異步獲取文章內容的任務 + let now = list.slice(start, start+20).forEach((line) => { + ctx.cache.tryGet(line, async() => { + const a_r = await got.get(`https://app3.rthk.hk/mediadigest/${line}`); + const $ = cheerio.load(a_r.data); + // title + const h1 = $('h1.story-title').text(); + // author + const author_list = $('div.story-author'); + const authors = author_list.map((_index, author) => $(author).text()); + const author = authors.map((_index, author) => `${author}
`); + const s_author = author.toArray().join(''); + const author_block = `

${s_author}

`; + // date + const date = $('div.story-calendar').text(); + // desc + const desc = `${$(author_block)}${$('div.story-content').html()}`; + + return { + title: h1, + description: desc, + pubDate: new Date(`${date}T09:00:00+0800`).toUTCString(), + link: line, + }; + }); + }) + // 執行一個切片 (20 個文章 URL) 的任務並將結果資料推至 rss 値 + rss.push(...(await Promise.all(now))); + } + } else if (cids.includes(category)) { + // 獲取特定 category 文章目錄 + const response = await got.get(`https://app3.rthk.hk/mediadigest/category.php?cid=${category}`); + const $ = cheerio.load(response.data); + + const list = $('div.category-line').map((_index, line) => $(line).find('a').first().attr('href')).toArray(); + // getArticle(ctx, list); + // for 每次會可爲 rss 値推入 20 篇文章內容 + for (let start = 0; start < list.length; start += 20) { + // 建立一個切片 (20 個文章 URL) 異步獲取文章內容的任務 + let now = list.slice(start, start+20).forEach((line) => { + ctx.cache.tryGet(line, async() => { + const a_r = await got.get(`https://app3.rthk.hk/mediadigest/${line}`); + const $ = cheerio.load(a_r.data); + // title + const h1 = $('h1.story-title').text(); + // author + const author_list = $('div.story-author'); + const authors = author_list.map((_index, author) => $(author).text()); + const author = authors.map((_index, author) => `${author}
`); + const s_author = author.toArray().join(''); + const author_block = `

${s_author}

`; + // date + const date = $('div.story-calendar').text(); + // desc + const desc = `${$(author_block)}${$('div.story-content').html()}`; + + return { + title: h1, + description: desc, + pubDate: new Date(`${date}T09:00:00+0800`).toUTCString(), + link: line, + }; + }); + }) + // 執行一個切片 (20 個文章 URL) 的任務並將結果資料推至 rss 値 + rss.push(...(await Promise.all(now))); + } + } else { + rss = ['

Invalid :category? input.

所輸入:category?參數有誤。

所输入:category?参数有误。

']; + } + } + // year (specific year range): + // if 'all' (all articles in a specific year range); + // else 'error'. + else if (range.length === 4 && range !== 0 && num.test(range) && !((range < 1970) || (range > current_year))) { + if (category === 'all') { + // 獲取全站文章 URL + const urls = cids.map((cid) => `https://app3.rthk.hk/mediadigest/category.php?cid=${cid}`); + const list_all = await Promise.all( + // 每個任務是篩選出一個文章目錄裏需要抓取的文章 URL，以供後續「獲取全文」 + urls.map(async(url) => { + const response = await got.get(url); + const $ = cheerio.load(response.data); + // 對應文章 URL 與年份 + // Ref: https://cythilya.github.io/2016/03/13/jquery-map-grep/ + const list_t = $('div.category-line').map((_index, line) => $(line).find('a').first().attr('href')).toArray(); + const year = $('div.category-line div.pull-right').map((_index, date) => new Date($(date).text()).getFullYear()).toArray(); + // 將 list_t 與 year 一一對應，組成 JSON + // Ref: https://stackoverflow.com/questions/30124979/merge-two-arrays-into-one-javascript-object + function article(year, url) { + this.year = year; + this.url = url; + } + var dlist = []; + for (var i = 0; i < list_t.length; i++) { + dlist.push(new article(year[i], list_t[i])); + } + // 篩出 list (需要抓取的文章 URL 並組成數列) + list = dlist.filter((w) => w.year === range).map((article) => (article.url)); + return Promise.resolve(list); + }) + ); + const list = [...new Set(list_all.flat())]; // removed duplicates and flatten + // getArticle(ctx, list); + // for 每次會可爲 rss 値段推入 20 篇文章內容 + for (let start = 0; start < list.length; start += 20) { + // 建立一個切片 (20 個文章 URL) 異步獲取文章內容的任務 + let now = list.slice(start, start+20).forEach((line) => { + ctx.cache.tryGet(line, async() => { + const a_r = await got.get(`https://app3.rthk.hk/mediadigest/${line}`); + const $ = cheerio.load(a_r.data); + // title + const h1 = $('h1.story-title').text(); + // author + const author_list = $('div.story-author'); + const authors = author_list.map((_index, author) => $(author).text()); + const author = authors.map((_index, author) => `${author}
`); + const s_author = author.toArray().join(''); + const author_block = `

${s_author}

`; + // date + const date = $('div.story-calendar').text(); + // desc + const desc = `${$(author_block)}${$('div.story-content').html()}`; + + return { + title: h1, + description: desc, + pubDate: new Date(`${date}T09:00:00+0800`).toUTCString(), + link: line, + }; + }); + }) + // 執行一個切片 (20 個文章 URL) 的任務並將結果資料推至 rss 値 + rss.push(...(await Promise.all(now))); + } + } else { + rss = ['

Please do not filter with both year and category.

請不要同時限定年份及分類。

请不要同时限定年份及分类。

']; + } + } else { + rss = ['

Invalid :range input.

所輸入:range參數有誤。

所输入:range参数有误。

']; + } + +/* // 分批次处理 tasks，每次拉取 20 个 + if (rss.length === 0) { + for (let start = 0; start < tasks.length; start += 20) { + let now = task.slice(start, start+20); + rss.push(...(await Promise.all(now))); + } + } + */ ctx.state.data = { title: '傳媒透視', link: 'https://app3.rthk.hk/mediadigest/index.php', item: rss, }; -}; +}; \ No newline at end of file From 8376d90ccaa37911fbbf7e93d09101a2f42d3afc Mon Sep 17 00:00:00 2001 From: totorowechat <30337116+totorowechat@users.noreply.github.com> Date: Tue, 29 Dec 2020 15:57:07 +0800 Subject: [PATCH 05/49] feat: add scboy --- assets/radar-rules.js | 14 ++++++++++ docs/bbs.md | 11 ++++++++ lib/config.js | 3 +++ lib/router.js | 3 +++ lib/routes/scboy/thread.js | 53 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 84 insertions(+) create mode 100644 lib/routes/scboy/thread.js diff --git a/assets/radar-rules.js b/assets/radar-rules.js index 137e4cc56f..6474ddca1c 100644 --- a/assets/radar-rules.js +++ b/assets/radar-rules.js @@ -2470,4 +2470,18 @@ }, ], }, + 'scboy.com': { + _name: 'scboy 论坛', + www: [ + { + title: '帖子', + docs: 'https://docs.rsshub.app/bbs.html#scboy', + source: '', + target: (params, url) => { + const id = url.includes('thread') ? url.split('-')[1].split('.')[0] : ''; + return id ? `/scboy/thread/${id}` : ''; + }, + }, + ], + }, }); diff --git a/docs/bbs.md b/docs/bbs.md index 392e28a726..cbd036c839 100644 --- a/docs/bbs.md +++ b/docs/bbs.md @@ -180,6 +180,17 @@ pageClass: routes +## SCBOY 论坛 + +### 帖子 + + + +帖子网址如果为 那么帖子 tid 就是 `1789863`。 + +访问水区需要添加环境变量 `SCBOY_BBS_TOKEN`, 详情见部署页面的配置模块。 `SCBOY_BBS_TOKEN`在cookies的`bbs_token`中。 + + ## V2EX ### 最热 / 最新主题 diff --git a/lib/config.js b/lib/config.js index bda7dc67ce..59f0e47e10 100644 --- a/lib/config.js +++ b/lib/config.js @@ -182,6 +182,9 @@ const calculateValue = () => { username: envs.DIDA365_USERNAME, password: envs.DIDA365_PASSWORD, }, + scboy: { + token: envs.SCBOY_BBS_TOKEN, + }, }; }; calculateValue(); diff --git a/lib/router.js b/lib/router.js index 2da2ee3d3c..d1360d84ba 100644 --- a/lib/router.js +++ b/lib/router.js @@ -3766,4 +3766,7 @@ router.get('/sciencenet/blog/:type?/:time?/:sort?', require('./routes/sciencenet // DailyArt router.get('/dailyart/:language?', require('./routes/dailyart/index')); +// SCBOY +router.get('/scboy/thread/:tid', require('./routes/scboy/thread')); + module.exports = router; diff --git a/lib/routes/scboy/thread.js b/lib/routes/scboy/thread.js new file mode 100644 index 0000000000..acf87b25f8 --- /dev/null +++ b/lib/routes/scboy/thread.js @@ -0,0 +1,53 @@ +const got = require('@/utils/got'); +const cheerio = require('cheerio'); +const date = require('@/utils/date'); +const config = require('@/config').value; + +module.exports = async (ctx) => { + const tid = ctx.params.tid; + const link = `https://www.scboy.com/?thread-${tid}.htm`; + let cookieString = 'postlist_orderby=desc'; + if (config.scboy.token) { + cookieString = `postlist_orderby=desc; bbs_token=${config.scboy.token}`; + } + + const res = await got.get({ + method: 'get', + url: link, + responseType: 'buffer', + headers: { + Cookie: cookieString, + }, + }); + + const $ = cheerio.load(res.data); + const title = $('h4 > span:nth-of-type(1)').text(); + + const list = $('li.media.post'); + const count = []; + + for (let i = 0; i < Math.min(list.length, 30); i++) { + count.push(i); + } + + const resultItems = await Promise.all( + count.map(async (i) => { + const each = $(list[i]); + const floor = each.find('span.floor-parent').text(); + const item = { + title: `${title} #${floor ? floor : ' 热门回复'}`, + link: `https://www.scboy.com/?thread-${tid}`, + description: each.find('div.message.mt-1.break-all > div:nth-of-type(1)').html(), + author: each.find('username').text(), + pubDate: date(each.find('.date').text()), + }; + return Promise.resolve(item); + }) + ); + + ctx.state.data = { + title: title, + link: `https://www.scboy.com/?thread-${tid}.htm`, + item: resultItems, + }; +}; From d0130d9dcdff19d94ec5726351d290c40c73090f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=82=AE=E3=83=A3=E3=83=A9?= Date: Tue, 29 Dec 2020 09:31:53 +0000 Subject: [PATCH 06/49] fix --- lib/routes/mediadigest/category.js | 156 ++++------------------------- 1 file changed, 21 insertions(+), 135 deletions(-) diff --git a/lib/routes/mediadigest/category.js b/lib/routes/mediadigest/category.js index 757541d4b9..aab44b5b81 100644 --- a/lib/routes/mediadigest/category.js +++ b/lib/routes/mediadigest/category.js @@ -1,12 +1,12 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); -function getArticle(ctx, list) { +async function getArticle(ctx, list) { // for 每次可爲 rss 値推入 20 篇文章內容 + const res = []; for (let start = 0; start < list.length; start += 20) { // 建立一個切片 (20 個文章 URL) 異步獲取文章內容的任務 - let now = list.slice(start, start+20).forEach((line) => { - ctx.cache.tryGet(line, async() => { + const now = list.slice(start, start + 20).map(async(line) => await ctx.cache.tryGet(line, async() => { const a_r = await got.get(`https://app3.rthk.hk/mediadigest/${line}`); const $ = cheerio.load(a_r.data); // title @@ -28,11 +28,11 @@ function getArticle(ctx, list) { pubDate: new Date(`${date}T09:00:00+0800`).toUTCString(), link: line, }; - }); - }) + })); // 執行一個切片 (20 個文章 URL) 的任務並將結果資料推至 rss 値 - rss.push(...(await Promise.all(now))); + res.push(...await Promise.all(now)); } + return res; } module.exports = async(ctx) => { @@ -40,10 +40,12 @@ module.exports = async(ctx) => { const category = ctx.params.category || 'all'; // for range validation - var num = /^[0-9]+$/; - var current_year = new Date().getFullYear(); + const num = /^[1-9][0-9]{3}$/; + const current_year = new Date().getFullYear(); // placeholder for rss - let rss = []; + let rss = [{ + description: '

Invalid :category? input.

所輸入:category?參數有誤。

所输入:category?参数有误。

' + }]; // cid const cids = [1, 2]; for (let i = 4; i < 28; ++i) { @@ -66,7 +68,7 @@ module.exports = async(ctx) => { return Promise.resolve(list.toArray()); }) ); - const list_all = [...new Set(list_allt.flat())]; // removed duplicates and flatten + const list_all = [...new Set(list_allt.flat())]; // removed duplicates and flatten // 時序排列並抽取最新 50 項 const list = list_all .sort((a, b) => { @@ -77,84 +79,22 @@ module.exports = async(ctx) => { }) .slice(0, 50); // getArticle(ctx, list); - // for 每次會可爲 rss 値段推入 20 篇文章內容 - for (let start = 0; start < list.length; start += 20) { - // 建立一個切片 (20 個文章 URL) 異步獲取文章內容的任務 - let now = list.slice(start, start+20).forEach((line) => { - ctx.cache.tryGet(line, async() => { - const a_r = await got.get(`https://app3.rthk.hk/mediadigest/${line}`); - const $ = cheerio.load(a_r.data); - // title - const h1 = $('h1.story-title').text(); - // author - const author_list = $('div.story-author'); - const authors = author_list.map((_index, author) => $(author).text()); - const author = authors.map((_index, author) => `${author}
`); - const s_author = author.toArray().join(''); - const author_block = `

${s_author}

`; - // date - const date = $('div.story-calendar').text(); - // desc - const desc = `${$(author_block)}${$('div.story-content').html()}`; - - return { - title: h1, - description: desc, - pubDate: new Date(`${date}T09:00:00+0800`).toUTCString(), - link: line, - }; - }); - }) - // 執行一個切片 (20 個文章 URL) 的任務並將結果資料推至 rss 値 - rss.push(...(await Promise.all(now))); - } - } else if (cids.includes(category)) { + rss = await getArticle(ctx, list); + } else if (cids.includes(parseInt(category))) { // 獲取特定 category 文章目錄 const response = await got.get(`https://app3.rthk.hk/mediadigest/category.php?cid=${category}`); const $ = cheerio.load(response.data); const list = $('div.category-line').map((_index, line) => $(line).find('a').first().attr('href')).toArray(); - // getArticle(ctx, list); - // for 每次會可爲 rss 値推入 20 篇文章內容 - for (let start = 0; start < list.length; start += 20) { - // 建立一個切片 (20 個文章 URL) 異步獲取文章內容的任務 - let now = list.slice(start, start+20).forEach((line) => { - ctx.cache.tryGet(line, async() => { - const a_r = await got.get(`https://app3.rthk.hk/mediadigest/${line}`); - const $ = cheerio.load(a_r.data); - // title - const h1 = $('h1.story-title').text(); - // author - const author_list = $('div.story-author'); - const authors = author_list.map((_index, author) => $(author).text()); - const author = authors.map((_index, author) => `${author}
`); - const s_author = author.toArray().join(''); - const author_block = `

${s_author}

`; - // date - const date = $('div.story-calendar').text(); - // desc - const desc = `${$(author_block)}${$('div.story-content').html()}`; - - return { - title: h1, - description: desc, - pubDate: new Date(`${date}T09:00:00+0800`).toUTCString(), - link: line, - }; - }); - }) - // 執行一個切片 (20 個文章 URL) 的任務並將結果資料推至 rss 値 - rss.push(...(await Promise.all(now))); - } - } else { - rss = ['

Invalid :category? input.

所輸入:category?參數有誤。

所输入:category?参数有误。

']; + rss = await getArticle(ctx, list.slice(0, 50)); } } // year (specific year range): // if 'all' (all articles in a specific year range); // else 'error'. - else if (range.length === 4 && range !== 0 && num.test(range) && !((range < 1970) || (range > current_year))) { - if (category === 'all') { + else if (num.test(range)) { + const range_num = parseInt(range); + if (category === 'all' && range_num >= 1970 && range_num <= current_year) { // 獲取全站文章 URL const urls = cids.map((cid) => `https://app3.rthk.hk/mediadigest/category.php?cid=${cid}`); const list_all = await Promise.all( @@ -164,71 +104,17 @@ module.exports = async(ctx) => { const $ = cheerio.load(response.data); // 對應文章 URL 與年份 // Ref: https://cythilya.github.io/2016/03/13/jquery-map-grep/ - const list_t = $('div.category-line').map((_index, line) => $(line).find('a').first().attr('href')).toArray(); + let list = $('div.category-line').map((_index, line) => $(line).find('a').first().attr('href')).toArray(); const year = $('div.category-line div.pull-right').map((_index, date) => new Date($(date).text()).getFullYear()).toArray(); - // 將 list_t 與 year 一一對應，組成 JSON - // Ref: https://stackoverflow.com/questions/30124979/merge-two-arrays-into-one-javascript-object - function article(year, url) { - this.year = year; - this.url = url; - } - var dlist = []; - for (var i = 0; i < list_t.length; i++) { - dlist.push(new article(year[i], list_t[i])); - } + list = list.filter((_url, i) => year[i] === range_num); // 篩出 list (需要抓取的文章 URL 並組成數列) - list = dlist.filter((w) => w.year === range).map((article) => (article.url)); return Promise.resolve(list); }) ); const list = [...new Set(list_all.flat())]; // removed duplicates and flatten - // getArticle(ctx, list); - // for 每次會可爲 rss 値段推入 20 篇文章內容 - for (let start = 0; start < list.length; start += 20) { - // 建立一個切片 (20 個文章 URL) 異步獲取文章內容的任務 - let now = list.slice(start, start+20).forEach((line) => { - ctx.cache.tryGet(line, async() => { - const a_r = await got.get(`https://app3.rthk.hk/mediadigest/${line}`); - const $ = cheerio.load(a_r.data); - // title - const h1 = $('h1.story-title').text(); - // author - const author_list = $('div.story-author'); - const authors = author_list.map((_index, author) => $(author).text()); - const author = authors.map((_index, author) => `${author}
`); - const s_author = author.toArray().join(''); - const author_block = `

${s_author}

`; - // date - const date = $('div.story-calendar').text(); - // desc - const desc = `${$(author_block)}${$('div.story-content').html()}`; - - return { - title: h1, - description: desc, - pubDate: new Date(`${date}T09:00:00+0800`).toUTCString(), - link: line, - }; - }); - }) - // 執行一個切片 (20 個文章 URL) 的任務並將結果資料推至 rss 値 - rss.push(...(await Promise.all(now))); - } - } else { - rss = ['

Please do not filter with both year and category.

請不要同時限定年份及分類。

请不要同时限定年份及分类。

']; - } - } else { - rss = ['

Invalid :range input.

所輸入:range參數有誤。

所输入:range参数有误。

']; - } - -/* // 分批次处理 tasks，每次拉取 20 个 - if (rss.length === 0) { - for (let start = 0; start < tasks.length; start += 20) { - let now = task.slice(start, start+20); - rss.push(...(await Promise.all(now))); + rss = await getArticle(ctx, list); } } - */ ctx.state.data = { title: '傳媒透視', From 473cc49d020b3b1611dd8600351935ba00a1514b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=82=AE=E3=83=A3=E3=83=A9?= Date: Tue, 29 Dec 2020 09:34:39 +0000 Subject: [PATCH 07/49] just async --- lib/routes/mediadigest/category.js | 47 +++++++++++++++++------------- 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/lib/routes/mediadigest/category.js b/lib/routes/mediadigest/category.js index aab44b5b81..3fae548b7f 100644 --- a/lib/routes/mediadigest/category.js +++ b/lib/routes/mediadigest/category.js @@ -2,11 +2,9 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); async function getArticle(ctx, list) { - // for 每次可爲 rss 値推入 20 篇文章內容 - const res = []; - for (let start = 0; start < list.length; start += 20) { - // 建立一個切片 (20 個文章 URL) 異步獲取文章內容的任務 - const now = list.slice(start, start + 20).map(async(line) => await ctx.cache.tryGet(line, async() => { + const now = list.map( + async (line) => + await ctx.cache.tryGet(line, async () => { const a_r = await got.get(`https://app3.rthk.hk/mediadigest/${line}`); const $ = cheerio.load(a_r.data); // title @@ -28,14 +26,13 @@ async function getArticle(ctx, list) { pubDate: new Date(`${date}T09:00:00+0800`).toUTCString(), link: line, }; - })); - // 執行一個切片 (20 個文章 URL) 的任務並將結果資料推至 rss 値 - res.push(...await Promise.all(now)); - } - return res; + }) + ); + // 執行一個切片 (20 個文章 URL) 的任務並將結果資料推至 rss 値 + return await Promise.all(now); } -module.exports = async(ctx) => { +module.exports = async (ctx) => { const range = ctx.params.range || 'latest'; const category = ctx.params.category || 'all'; @@ -43,9 +40,11 @@ module.exports = async(ctx) => { const num = /^[1-9][0-9]{3}$/; const current_year = new Date().getFullYear(); // placeholder for rss - let rss = [{ - description: '

Invalid :category? input.

所輸入:category?參數有誤。

所输入:category?参数有误。

' - }]; + let rss = [ + { + description: '

Invalid :category? input.

所輸入:category?參數有誤。

所输入:category?参数有误。

', + }, + ]; // cid const cids = [1, 2]; for (let i = 4; i < 28; ++i) { @@ -61,7 +60,7 @@ module.exports = async(ctx) => { // 獲取全站文章 URL const urls = cids.map((cid) => `https://app3.rthk.hk/mediadigest/category.php?cid=${cid}`); const list_allt = await Promise.all( - urls.map(async(url) => { + urls.map(async (url) => { const response = await got.get(url); const $ = cheerio.load(response.data); const list = $('div.category-line').map((_index, line) => $(line).find('a').first().attr('href')); @@ -85,7 +84,9 @@ module.exports = async(ctx) => { const response = await got.get(`https://app3.rthk.hk/mediadigest/category.php?cid=${category}`); const $ = cheerio.load(response.data); - const list = $('div.category-line').map((_index, line) => $(line).find('a').first().attr('href')).toArray(); + const list = $('div.category-line') + .map((_index, line) => $(line).find('a').first().attr('href')) + .toArray(); rss = await getArticle(ctx, list.slice(0, 50)); } } @@ -99,20 +100,24 @@ module.exports = async(ctx) => { const urls = cids.map((cid) => `https://app3.rthk.hk/mediadigest/category.php?cid=${cid}`); const list_all = await Promise.all( // 每個任務是篩選出一個文章目錄裏需要抓取的文章 URL，以供後續「獲取全文」 - urls.map(async(url) => { + urls.map(async (url) => { const response = await got.get(url); const $ = cheerio.load(response.data); // 對應文章 URL 與年份 // Ref: https://cythilya.github.io/2016/03/13/jquery-map-grep/ - let list = $('div.category-line').map((_index, line) => $(line).find('a').first().attr('href')).toArray(); - const year = $('div.category-line div.pull-right').map((_index, date) => new Date($(date).text()).getFullYear()).toArray(); + let list = $('div.category-line') + .map((_index, line) => $(line).find('a').first().attr('href')) + .toArray(); + const year = $('div.category-line div.pull-right') + .map((_index, date) => new Date($(date).text()).getFullYear()) + .toArray(); list = list.filter((_url, i) => year[i] === range_num); // 篩出 list (需要抓取的文章 URL 並組成數列) return Promise.resolve(list); }) ); const list = [...new Set(list_all.flat())]; // removed duplicates and flatten - rss = await getArticle(ctx, list); + rss = await getArticle(ctx, list.slice(0, 50)); } } @@ -121,4 +126,4 @@ module.exports = async(ctx) => { link: 'https://app3.rthk.hk/mediadigest/index.php', item: rss, }; -}; \ No newline at end of file +}; From 7deee8ea6fded632116e55c16c30e2dfd99f6c20 Mon Sep 17 00:00:00 2001 From: zph Date: Tue, 29 Dec 2020 17:43:46 +0800 Subject: [PATCH 08/49] Added Radio Free Asia --- docs/en/traditional-media.md | 12 +++++++++ docs/traditional-media.md | 12 +++++++++ lib/router.js | 3 +++ lib/routes/rfa/index.js | 47 ++++++++++++++++++++++++++++++++++++ 4 files changed, 74 insertions(+) create mode 100644 lib/routes/rfa/index.js diff --git a/docs/en/traditional-media.md b/docs/en/traditional-media.md index 983211fe80..4f942f500e 100644 --- a/docs/en/traditional-media.md +++ b/docs/en/traditional-media.md @@ -149,6 +149,18 @@ Generates full-text feeds that the official feed doesn't provide. +## Radio Free Asia (RFA) + + + +Delivers a better experience by supporting parameter specification. + +Parameters can be obtained from the official website, for instance: + +`https://www.rfa.org/cantonese/news` corresponds to `/rfa/cantonese/news` + +`https://www.rfa.org/cantonese/news/htm` corresponds to `/rfa/cantonese/news/htm` + ## Reuters ### Channel diff --git a/docs/traditional-media.md b/docs/traditional-media.md index 94db156aeb..6281d1bee8 100644 --- a/docs/traditional-media.md +++ b/docs/traditional-media.md @@ -965,3 +965,15 @@ category 对应的关键词有 ### 九江新闻 + +## 自由亚洲电台 + + + +通过指定频道参数，提供比官方源更佳的阅读体验。 + +参数均可在官网获取，如： + +`https://www.rfa.org/cantonese/news` 对应 `/rfa/cantonese/news` + +`https://www.rfa.org/cantonese/news/htm` 对应 `/rfa/cantonese/news/htm` diff --git a/lib/router.js b/lib/router.js index 5bb0f656c3..7ca17dea0b 100644 --- a/lib/router.js +++ b/lib/router.js @@ -3780,4 +3780,7 @@ router.get('/nace/blog/:sort?', require('./routes/nace/blog')); // Caixin Latest router.get('/caixin/latest', require('./routes/caixin/latest')); +// Radio Free Asia +router.get('/rfa/:language?/:channel?/:subChannel?', require('./routes/rfa/index')); + module.exports = router; diff --git a/lib/routes/rfa/index.js b/lib/routes/rfa/index.js new file mode 100644 index 0000000000..90791eb2fc --- /dev/null +++ b/lib/routes/rfa/index.js @@ -0,0 +1,47 @@ +const cheerio = require('cheerio'); +const got = require('@/utils/got'); + +module.exports = async (ctx) => { + let url = 'https://www.rfa.org/' + (ctx.params.language || 'english'); + + if (ctx.params.channel) { + url += '/' + ctx.params.channel; + } + if (ctx.params.subChannel) { + url += '/' + ctx.params.subChannel; + } + + const response = await got.get(url); + const $ = cheerio.load(response.data); + + const selectors = ['div[id=topstorywidefulltease]', 'div.two_featured', 'div.three_featured', 'div.single_column_teaser', 'div.sectionteaser', 'div.specialwrap']; + const list = []; + selectors.forEach(function (selector) { + $(selector).each(function (_, e) { + const item = {}; + item.title = $(e).find('h2 a span').first().text(); + item.link = $(e).find('h2 a').first().attr('href'); + list.push(item); + }); + }); + + const result = await Promise.all( + list.map( + async (item) => + await ctx.cache.tryGet(item.link, async () => { + const content = await got.get(item.link); + + const description = cheerio.load(content.data); + item.description = description('div[id=storytext]').html(); + item.pubDate = new Date(description('span[id=story_date]').text()).toUTCString(); + return item; + }) + ) + ); + + ctx.state.data = { + title: 'RFA', + link: 'https://www.rfa.org/', + item: result, + }; +}; From 43476500df54426a887a1f31058204806342f2f0 Mon Sep 17 00:00:00 2001 From: TheresaQWQ Date: Tue, 29 Dec 2020 10:35:10 +0000 Subject: [PATCH 09/49] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=20=E9=B1=BC=E5=A1=98?= =?UTF-8?q?=E7=83=AD=E6=A6=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/new-media.md | 4 ++++ lib/router.js | 3 +++ lib/routes/mofish/index.js | 29 +++++++++++++++++++++++++++++ 3 files changed, 36 insertions(+) create mode 100644 lib/routes/mofish/index.js diff --git a/docs/new-media.md b/docs/new-media.md index dea896d573..bf7fcddde1 100644 --- a/docs/new-media.md +++ b/docs/new-media.md @@ -2109,3 +2109,7 @@ QueryString: ### 全文 + +## 鱼塘热榜 + + \ No newline at end of file diff --git a/lib/router.js b/lib/router.js index 5bb0f656c3..dfc51508a3 100644 --- a/lib/router.js +++ b/lib/router.js @@ -3780,4 +3780,7 @@ router.get('/nace/blog/:sort?', require('./routes/nace/blog')); // Caixin Latest router.get('/caixin/latest', require('./routes/caixin/latest')); +// 鱼塘热榜 +router.get('/mofish/:id', require('./routes/mofish/index')); + module.exports = router; diff --git a/lib/routes/mofish/index.js b/lib/routes/mofish/index.js new file mode 100644 index 0000000000..f99bf3087d --- /dev/null +++ b/lib/routes/mofish/index.js @@ -0,0 +1,29 @@ +const got = require('@/utils/got'); + +module.exports = async (ctx) => { + const id = ctx.params.id; + const page = ctx.query.page || 0; + + const url = `https://api.tophub.fun/v2/GetAllInfoGzip?id=${id}&page=${page}`; + + const response = await got({ + method: 'get', + url: url, + }); + + const data = response.data.Data.data; + + const title = `鱼塘热榜`; + + ctx.state.data = { + title: title, + link: `https://mo.fish/`, + description: title, + item: data.map((item) => ({ + title: item.Title, + pubDate: new Date(item.releaseTime).toUTCString(), + link: item.Url, + guid: item.id, + })), + }; +}; \ No newline at end of file From 961636aebdcfe3965521821f67eff9dad4e9ffe7 Mon Sep 17 00:00:00 2001 From: Toby Tso Date: Tue, 29 Dec 2020 19:59:57 +0800 Subject: [PATCH 10/49] WIP: add Media Digest --- docs/traditional-media.md | 17 +++++++++++++++++ lib/routes/mediadigest/category.js | 14 ++++++++------ 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/docs/traditional-media.md b/docs/traditional-media.md index c9347c4f19..ec3c01e659 100644 --- a/docs/traditional-media.md +++ b/docs/traditional-media.md @@ -957,3 +957,20 @@ category 对应的关键词有 ### 九江新闻 + +### RTHK 傳媒透視 + + + +细则: + +- `:range` 时间范围参数 + (可为 `latest` 或 `四位数字的年份`) + + - `latest`: 最新的 50 篇文章 + - `2020`: 2020 年的所有文章 + +- 全文输出转换为简体字: `?opencc=t2s` + (`opencc` 是 RSSHub 的通用参数，详情请参阅 [「中文简繁体转换」](https://docs.rsshub.app/parameter.html#zhong-wen-jian-fan-ti-zhuan-huan)) + + diff --git a/lib/routes/mediadigest/category.js b/lib/routes/mediadigest/category.js index 3fae548b7f..9c41c61b77 100644 --- a/lib/routes/mediadigest/category.js +++ b/lib/routes/mediadigest/category.js @@ -39,10 +39,11 @@ module.exports = async (ctx) => { // for range validation const num = /^[1-9][0-9]{3}$/; const current_year = new Date().getFullYear(); - // placeholder for rss + // placeholders + let title = '傳媒透視'; let rss = [ { - description: '

Invalid :category? input.

所輸入:category?參數有誤。

所输入:category?参数有误。

', + description: '

Invalid :range input.

所輸入:range參數有誤。

所输入:range参数有误。

', }, ]; // cid @@ -53,7 +54,7 @@ module.exports = async (ctx) => { // latest (50 articles): // if 'all' (latest 50 articles of the site); - // else if 'cid' (all articles of a specific category); + // else if 'cid' (deprecated) (latest 50 articles of a specific category); // else 'error'. if (range === 'latest') { if (category === 'all') { @@ -91,7 +92,7 @@ module.exports = async (ctx) => { } } // year (specific year range): - // if 'all' (all articles in a specific year range); + // if 'all' (latest 200 articles in a specific year range); // else 'error'. else if (num.test(range)) { const range_num = parseInt(range); @@ -117,12 +118,13 @@ module.exports = async (ctx) => { }) ); const list = [...new Set(list_all.flat())]; // removed duplicates and flatten - rss = await getArticle(ctx, list.slice(0, 50)); + rss = await getArticle(ctx, list.slice(0, 200)); + title = `傳媒透視 - ${range}`; } } ctx.state.data = { - title: '傳媒透視', + title: title, link: 'https://app3.rthk.hk/mediadigest/index.php', item: rss, }; From c4e820decc996c4b4219ba04d35d0e04403f5854 Mon Sep 17 00:00:00 2001 From: Toby Tso Date: Tue, 29 Dec 2020 20:02:22 +0800 Subject: [PATCH 11/49] WIP: fix doc --- docs/traditional-media.md | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/docs/traditional-media.md b/docs/traditional-media.md index ec3c01e659..771130a6c7 100644 --- a/docs/traditional-media.md +++ b/docs/traditional-media.md @@ -181,6 +181,23 @@ pageClass: routes +## RTHK 傳媒透視 + + + +细则: + +- `:range` 时间范围参数 + (可为 `latest` 或 `四位数字的年份`) + + - `latest`: 最新的 50 篇文章 + - `2020`: 2020 年的所有文章 + +- 全文输出转换为简体字: `?opencc=t2s` + (`opencc` 是 RSSHub 的通用参数，详情请参阅 [「中文简繁体转换」](https://docs.rsshub.app/parameter.html#zhong-wen-jian-fan-ti-zhuan-huan)) + + + ## Solidot ### 最新消息 @@ -957,20 +974,3 @@ category 对应的关键词有 ### 九江新闻 - -### RTHK 傳媒透視 - - - -细则: - -- `:range` 时间范围参数 - (可为 `latest` 或 `四位数字的年份`) - - - `latest`: 最新的 50 篇文章 - - `2020`: 2020 年的所有文章 - -- 全文输出转换为简体字: `?opencc=t2s` - (`opencc` 是 RSSHub 的通用参数，详情请参阅 [「中文简繁体转换」](https://docs.rsshub.app/parameter.html#zhong-wen-jian-fan-ti-zhuan-huan)) - - From 823b2fc236f7788b96e338960fe11ba51988f0fb Mon Sep 17 00:00:00 2001 From: "St. John Johnson" Date: Tue, 29 Dec 2020 21:00:13 -0800 Subject: [PATCH 12/49] Adding GoComics.com and ComicsKingdom.com --- docs/en/picture.md | 8 ++++ lib/router.js | 6 +++ lib/routes/comicskingdom/index.js | 62 ++++++++++++++++++++++++++++++ lib/routes/gocomics/index.js | 64 +++++++++++++++++++++++++++++++ 4 files changed, 140 insertions(+) create mode 100644 lib/routes/comicskingdom/index.js create mode 100644 lib/routes/gocomics/index.js diff --git a/docs/en/picture.md b/docs/en/picture.md index cd3cb4f024..f41db20042 100644 --- a/docs/en/picture.md +++ b/docs/en/picture.md @@ -42,6 +42,10 @@ pageClass: routes +## ComicsKingdom Comic Strips + + + ## DailyArt @@ -50,6 +54,10 @@ pageClass: routes +## GoComics Comic Strips + + + ## Google Doodles ### Update diff --git a/lib/router.js b/lib/router.js index 5bb0f656c3..4fed5ed9f9 100644 --- a/lib/router.js +++ b/lib/router.js @@ -3780,4 +3780,10 @@ router.get('/nace/blog/:sort?', require('./routes/nace/blog')); // Caixin Latest router.get('/caixin/latest', require('./routes/caixin/latest')); +// GoComics +router.get('/gocomics/:name', require('./routes/gocomics/index')); + +// Comics Kingdom +router.get('/comicskingdom/:name', require('./routes/comicskingdom/index')); + module.exports = router; diff --git a/lib/routes/comicskingdom/index.js b/lib/routes/comicskingdom/index.js new file mode 100644 index 0000000000..3a81c7b47b --- /dev/null +++ b/lib/routes/comicskingdom/index.js @@ -0,0 +1,62 @@ +const got = require('@/utils/got'); +const cheerio = require('cheerio'); + +module.exports = async (ctx) => { + const baseURL = 'https://www.comicskingdom.com'; + const name = ctx.params.name; + const url = `${baseURL}/${name}/archive`; + const response = await got({ + method: 'get', + url: url, + }); + + const data = response.data; + const $ = cheerio.load(data); + + // Determine Comic and Author from main page + const comic = $('title').text().replace('Comics Kingdom - ', '').trim(); + const author = $('div.author p').text(); + + // Find the links for all non-archived items + const links = $('div.archive-tile[data-is-blocked=false]') + .map((i, el) => $(el).find('a[data-prem="Comic Tile"]').first().attr('href')) + .get() + .map((url) => `${baseURL}${url}`); + + if (links.length === 0) { + throw `Comic Not Found - ${name}`; + } + const items = await Promise.all( + links.map( + async (link) => + await ctx.cache.tryGet(link, async () => { + const detailResponse = await got({ + method: 'get', + url: link, + }); + const content = cheerio.load(detailResponse.data); + + const title = content('title').text(); + const image = content('meta[property="og:image"]').attr('content'); + const description = ``; + // Pull the date out of the URL + const pubDate = new Date(link.split('/').slice(-3).join('/')).toUTCString(); + + return { + title: title, + author: author, + category: 'comic', + description: description, + pubDate: pubDate, + link: link, + }; + }) + ) + ); + + ctx.state.data = { + title: `${comic} - Comics Kingdom`, + link: url, + item: items, + }; +}; diff --git a/lib/routes/gocomics/index.js b/lib/routes/gocomics/index.js new file mode 100644 index 0000000000..b743418358 --- /dev/null +++ b/lib/routes/gocomics/index.js @@ -0,0 +1,64 @@ +const got = require('@/utils/got'); +const cheerio = require('cheerio'); + +module.exports = async (ctx) => { + const baseURL = 'https://www.gocomics.com'; + const name = ctx.params.name; + const limit = ctx.query.limit || 5; + const url = `${baseURL}/${name}/`; + const response = await got({ + method: 'get', + url: url, + }); + + const data = response.data; + const $ = cheerio.load(data); + + // Determine Comic and Author from main page + const comic = $('.media-heading').eq(0).text(); + const author = $('.media-subheading').eq(0).text().replace('By ', ''); + + // Load previous comic URL + const items = []; + let previous = $('.gc-deck--cta-0 a').attr('href'); + + if (!previous) { + throw `Comic Not Found - ${name}`; + } + + while (items.length < limit) { + const link = `${baseURL}${previous}`; + /* eslint-disable no-await-in-loop */ + const item = await ctx.cache.tryGet(link, async () => { + const detailResponse = await got({ + method: 'get', + url: link, + }); + const content = cheerio.load(detailResponse.data); + + const title = content('h1.m-0').eq(0).text(); + const image = content('.comic.container').eq(0).attr('data-image'); + const description = ``; + // Pull the date out of the URL + const pubDate = new Date(link.split('/').slice(-3).join('/')).toUTCString(); + const previous = content('.js-previous-comic').eq(0).attr('href'); + + return { + title: title, + author: author, + category: 'comic', + description: description, + pubDate: pubDate, + link: link, + previous: previous, + }; + }); + items.push(item); + previous = item.previous; + } + ctx.state.data = { + title: `${comic} - GoComics`, + link: url, + item: items, + }; +}; From b823e5f9e84b75c7538412d12d12032f5d8e96be Mon Sep 17 00:00:00 2001 From: zph Date: Wed, 30 Dec 2020 16:04:10 +0800 Subject: [PATCH 13/49] Added top stories route for AP --- docs/en/traditional-media.md | 4 +++ docs/traditional-media.md | 4 +++ lib/router.js | 1 + lib/routes/apnews/index.js | 54 ++++++++++++++++++++++++++++++++++++ 4 files changed, 63 insertions(+) create mode 100644 lib/routes/apnews/index.js diff --git a/docs/en/traditional-media.md b/docs/en/traditional-media.md index 983211fe80..d28621dfd2 100644 --- a/docs/en/traditional-media.md +++ b/docs/en/traditional-media.md @@ -20,6 +20,10 @@ Site ## AP News +### Top Stories + + + ### Topics diff --git a/docs/traditional-media.md b/docs/traditional-media.md index 94db156aeb..a574998f57 100644 --- a/docs/traditional-media.md +++ b/docs/traditional-media.md @@ -26,6 +26,10 @@ pageClass: routes ## AP News +### 首页头条 + + + ### 话题 diff --git a/lib/router.js b/lib/router.js index 5bb0f656c3..7e1e6838ad 100644 --- a/lib/router.js +++ b/lib/router.js @@ -2490,6 +2490,7 @@ router.get('/gbcc/trust', require('./routes/gbcc/trust')); // Associated Press router.get('/apnews/topics/:topic', require('./routes/apnews/topics')); +router.get('/apnews', require('./routes/apnews/index')); // CBC router.get('/cbc/topics/:topic?', require('./routes/cbc/topics')); diff --git a/lib/routes/apnews/index.js b/lib/routes/apnews/index.js new file mode 100644 index 0000000000..a59c24f1de --- /dev/null +++ b/lib/routes/apnews/index.js @@ -0,0 +1,54 @@ +const cheerio = require('cheerio'); +const got = require('@/utils/got'); + +module.exports = async (ctx) => { + const response = await got.get('https://apnews.com/'); + const $ = cheerio.load(response.data); + const list = []; + + // main story component + const main = $('div[data-key=main-story]').first(); + + // headline news + const headline = {}; + headline.title = $(main).find('a[data-key=card-headline] h1').first().text(); + headline.link = 'https://apnews.com' + $(main).find('a[data-key=card-headline]').first().attr('href'); + list.push(headline); + + // related stories + $(main) + .find('a[data-key=related-story-link]') + .each(function (_, e) { + const item = {}; + item.title = $(e).find('div[data-key=related-story-headline]').first().text(); + item.link = 'https://apnews.com' + $(e).attr('href'); + list.push(item); + }); + + const result = await Promise.all( + list.map( + async (item) => + await ctx.cache.tryGet(item.link, async () => { + const content = await got.get(item.link); + + const description = cheerio.load(content.data); + const metadata = JSON.parse(description('script[type="application/ld+json"]').html()); + const featureImageURL = metadata.image; + + item.description = ``; + item.description += description('div[class=Article]') + .html() + .replace(/ADVERTISEMENT/g, ''); + item.pubDate = new Date(metadata.datePublished).toISOString(); + item.author = metadata.author[0]; + return item; + }) + ) + ); + + ctx.state.data = { + title: 'Associated Press News', + link: 'https://apnews.com/', + item: result, + }; +}; From 8f7f5918c081e34aece243c0337fa69ccc58f487 Mon Sep 17 00:00:00 2001 From: nczitzk Date: Wed, 30 Dec 2020 19:00:59 +0800 Subject: [PATCH 14/49] =?UTF-8?q?feat:=20add=20=E4=B8=AD=E5=9B=BD=E5=86=9C?= =?UTF-8?q?=E5=B7=A5=E6=B0=91=E4=B8=BB=E5=85=9A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/government.md | 12 +++++++++ lib/router.js | 3 +++ lib/routes/gov/ngd/index.js | 52 +++++++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+) create mode 100644 lib/routes/gov/ngd/index.js diff --git a/docs/government.md b/docs/government.md index 7787edca28..eb76ada010 100644 --- a/docs/government.md +++ b/docs/government.md @@ -185,6 +185,18 @@ pageClass: routes +## 中国农工民主党 + +### 新闻中心 + + + +将目标栏目的网址拆解为 `http://www.ngd.org.cn/` 和后面的字段，去掉 `.htm` 后，把后面的字段中的 `/` 替换为 `-`，即为该路由的 slug + +如：(要闻动态)[http://www.ngd.org.cn/xwzx/ywdt/index.htm] 的网址在 `http://www.ngd.org.cn/` 后的字段是 `xwzx/ywdt/index.htm`，则对应的 slug 为 `xwzx-ywdt-index`，对应的路由即为 `/ngd/xwzx-ywdt-index` + + + ## 中国人大网 diff --git a/lib/router.js b/lib/router.js index 5bb0f656c3..7bb0603bec 100644 --- a/lib/router.js +++ b/lib/router.js @@ -3780,4 +3780,7 @@ router.get('/nace/blog/:sort?', require('./routes/nace/blog')); // Caixin Latest router.get('/caixin/latest', require('./routes/caixin/latest')); +// 中国农工民主党 +router.get('/ngd/:slug?', require('./routes/gov/ngd/index')); + module.exports = router; diff --git a/lib/routes/gov/ngd/index.js b/lib/routes/gov/ngd/index.js new file mode 100644 index 0000000000..5dfd996fe9 --- /dev/null +++ b/lib/routes/gov/ngd/index.js @@ -0,0 +1,52 @@ +const got = require('@/utils/got'); +const cheerio = require('cheerio'); + +module.exports = async (ctx) => { + const slug = ctx.params.slug || 'xwzx-ywdt-index'; + + const rootUrl = 'http://www.ngd.org.cn'; + const currentUrl = `${rootUrl}/${slug.replace(/-/g, '/')}.htm`; + const response = await got({ + method: 'get', + url: currentUrl, + }); + + const $ = cheerio.load(response.data); + + const list = $('.gp-ellipsis a') + .slice(0, 15) + .map((_, item) => { + item = $(item); + return { + title: item.text(), + link: `${currentUrl.replace('/index.htm', '')}/${item.attr('href')}`, + }; + }) + .get(); + + const items = await Promise.all( + list.map( + async (item) => + await ctx.cache.tryGet(item.link, async () => { + const detailResponse = await got({ + method: 'get', + url: item.link, + }); + const content = cheerio.load(detailResponse.data); + const info = content('.articleAuthor').text().split('|'); + + item.author = info[0].replace('来源：', ''); + item.description = content('.gp-article').html(); + item.pubDate = new Date(info[info.length - 1].replace('发布时间：', '')).toUTCString(); + + return item; + }) + ) + ); + + ctx.state.data = { + title: $('title').text(), + link: currentUrl, + item: items, + }; +}; From 27477a37e03cc5e6b976ea0dd2e14894882423b8 Mon Sep 17 00:00:00 2001 From: zph Date: Wed, 30 Dec 2020 21:17:29 +0800 Subject: [PATCH 15/49] Added Pincong --- docs/bbs.md | 18 +++++++++++++++++ lib/router.js | 5 +++++ lib/routes/pincong/hot.js | 33 ++++++++++++++++++++++++++++++ lib/routes/pincong/index.js | 40 +++++++++++++++++++++++++++++++++++++ lib/routes/pincong/topic.js | 32 +++++++++++++++++++++++++++++ 5 files changed, 128 insertions(+) create mode 100644 lib/routes/pincong/hot.js create mode 100644 lib/routes/pincong/index.js create mode 100644 lib/routes/pincong/topic.js diff --git a/docs/bbs.md b/docs/bbs.md index 0a557cbee5..989d4c97b3 100644 --- a/docs/bbs.md +++ b/docs/bbs.md @@ -395,6 +395,24 @@ pageClass: routes +## 品葱 + +### 发现 + + + +| 最新 | 推荐 | 热门 | +| ---- | --------- | ---- | +| new | recommend | hot | + +### 精选 + + + +### 话题 + + + ## 三星盖乐世社区 ### 最新帖子 diff --git a/lib/router.js b/lib/router.js index 5bb0f656c3..150c32d695 100644 --- a/lib/router.js +++ b/lib/router.js @@ -3780,4 +3780,9 @@ router.get('/nace/blog/:sort?', require('./routes/nace/blog')); // Caixin Latest router.get('/caixin/latest', require('./routes/caixin/latest')); +// Pincong 品葱 +router.get('/pincong/category/:category?/:sort?', require('./routes/pincong/index')); +router.get('/pincong/hot/:category?', require('./routes/pincong/hot')); +router.get('/pincong/topic/:topic', require('./routes/pincong/topic')); + module.exports = router; diff --git a/lib/routes/pincong/hot.js b/lib/routes/pincong/hot.js new file mode 100644 index 0000000000..0f65af2ab5 --- /dev/null +++ b/lib/routes/pincong/hot.js @@ -0,0 +1,33 @@ +const cheerio = require('cheerio'); + +module.exports = async (ctx) => { + let url = 'https://pincong.rocks/hot/list/'; + + url += ctx.params.category ? 'category-' + ctx.params.category : 'category-0'; + + // use Puppeteer due to the obstacle by cloudflare challenge + const browser = await require('@/utils/puppeteer')(); + const page = await browser.newPage(); + await page.goto(url); + const html = await page.evaluate( + // eslint-disable-next-line no-undef + () => document.documentElement.innerHTML + ); + + browser.close(); + + const $ = cheerio.load(html); + const list = $('div.aw-item'); + + ctx.state.data = { + title: '品葱 - 精选', + link: 'https://pincong.rocks/hot/', + item: list + .map((_, item) => ({ + title: $(item).find('h2 a').text().trim(), + description: $(item).find('div.markitup-box').html(), + link: 'https://pincong.rocks' + $(item).find('div.mod-head h2 a').attr('href'), + })) + .get(), + }; +}; diff --git a/lib/routes/pincong/index.js b/lib/routes/pincong/index.js new file mode 100644 index 0000000000..f1553de5f2 --- /dev/null +++ b/lib/routes/pincong/index.js @@ -0,0 +1,40 @@ +const cheerio = require('cheerio'); + +module.exports = async (ctx) => { + let url = 'https://pincong.rocks/'; + + const sortMap = { + new: 'sort_type-new', + recommend: 'recommend-1', + hot: 'sort_type-hot__day2', + }; + + url += (ctx.params.sort && sortMap[ctx.params.sort]) || 'recommend-1'; + url += ctx.params.category ? '__category-' + ctx.params.category : ''; + + // use Puppeteer due to the obstacle by cloudflare challenge + const browser = await require('@/utils/puppeteer')(); + const page = await browser.newPage(); + await page.goto(url); + const html = await page.evaluate( + // eslint-disable-next-line no-undef + () => document.querySelector('div.aw-common-list').innerHTML + ); + + browser.close(); + + const $ = cheerio.load(html); + const list = $('div.aw-item'); + + ctx.state.data = { + title: '品葱 - 发现', + link: url, + item: list + .map((_, item) => ({ + title: $(item).find('h4 a').text().trim(), + link: 'https://pincong.rocks' + $(item).find('h4 a').attr('href'), + pubDate: new Date($(item).attr('data-timestamp') * 1000).toISOString(), + })) + .get(), + }; +}; diff --git a/lib/routes/pincong/topic.js b/lib/routes/pincong/topic.js new file mode 100644 index 0000000000..2659515f80 --- /dev/null +++ b/lib/routes/pincong/topic.js @@ -0,0 +1,32 @@ +const cheerio = require('cheerio'); + +module.exports = async (ctx) => { + const url = 'https://pincong.rocks/topic/' + ctx.params.topic; + + // use Puppeteer due to the obstacle by cloudflare challenge + const browser = await require('@/utils/puppeteer')(); + const page = await browser.newPage(); + await page.goto(url); + const html = await page.evaluate( + () => + // eslint-disable-next-line no-undef + (document.querySelector('div.aw-common-list') && document.querySelector('div.aw-common-list').innerHTML) || '' + ); + + browser.close(); + + const $ = cheerio.load(html); + const list = $('div.aw-item'); + + ctx.state.data = { + title: `品葱 - ${ctx.params.topic}`, + link: url, + item: list + .map((_, item) => ({ + title: $(item).find('h4 a').text().trim(), + link: 'https://pincong.rocks' + $(item).find('h4 a').attr('href'), + pubDate: new Date($(item).attr('data-timestamp') * 1000).toISOString(), + })) + .get(), + }; +}; From 3d7eadb502ba5c5eadfabf1170e03ccfdd683e2d Mon Sep 17 00:00:00 2001 From: zph Date: Fri, 1 Jan 2021 02:05:30 +0800 Subject: [PATCH 16/49] Fixed iDaily error --- lib/routes/idaily/index.js | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/lib/routes/idaily/index.js b/lib/routes/idaily/index.js index 3205f932a2..9c3567c3bd 100644 --- a/lib/routes/idaily/index.js +++ b/lib/routes/idaily/index.js @@ -6,19 +6,12 @@ module.exports = async (ctx) => { url: 'http://idaily-cdn.idailycdn.com/api/list/v3/iphone/zh-hans?page=1&ver=iphone', }); - const data = response.data; - - let dataToday = data.filter((item) => item.pubdate_timestamp * 1000 >= new Date().getTime() - 86400000); - - // leverage yesterday's items if today's not published yet - if (dataToday.length === 0) { - dataToday = data.filter((item) => item.pubdate_timestamp * 1000 >= new Date().getTime() - 86400000 * 2); - } + const data = response.data.filter((item) => item.ui_sets && item.ui_sets.caption_subtitle).slice(0, 15); ctx.state.data = { title: `iDaily 每日环球视野`, description: 'iDaily 每日环球视野', - item: dataToday.map((item) => ({ + item: data.map((item) => ({ title: item.ui_sets.caption_subtitle, description: `
${item.content}`, pubDate: new Date(item.pubdate_timestamp * 1000).toUTCString(), From ccf3f042924f290feecbf502a5574f6fc53540ce Mon Sep 17 00:00:00 2001 From: Laam Pui Date: Sat, 2 Jan 2021 11:34:41 +0800 Subject: [PATCH 17/49] fix: /douban/movie/later --- lib/routes/douban/later.js | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/lib/routes/douban/later.js b/lib/routes/douban/later.js index bf6a69ee0e..8d9b784ddd 100644 --- a/lib/routes/douban/later.js +++ b/lib/routes/douban/later.js @@ -1,19 +1,32 @@ const got = require('@/utils/got'); +const cheerio = require('cheerio'); module.exports = async (ctx) => { const response = await got({ method: 'get', - url: 'https://api.douban.com/v2/movie/coming_soon?apikey=0df993c66c0c636e29ecbb5344252a4a', + url: 'https://movie.douban.com/cinema/later/beijing/', }); - const movieList = response.data.subjects; + const $ = cheerio.load(response.data); + + const item = $('#showing-soon .item') + .map((index, ele) => { + const description = $(ele).html(); + const name = $('h3', ele).text().trim(); + const date = $('ul li', ele).eq(0).text().trim(); + const type = $('ul li', ele).eq(1).text().trim(); + const link = $('a.thumb', ele).attr('href'); + + return { + title: `${date} - 《${name}》 - ${type}`, + link, + description, + }; + }) + .get(); ctx.state.data = { title: '即将上映的电影', link: 'https://movie.douban.com/cinema/later/', - item: movieList.map((item) => ({ - title: item.title, - description: `标题：${item.title}
影片类型：${item.genres.join(' | ')}
评分：${item.rating.stars === '00' ? '无' : item.rating.average}
`, - link: item.alt, - })), + item, }; }; From c4dd02cae7d458486d140b175da9cc0336205e01 Mon Sep 17 00:00:00 2001 From: duhd1993 Date: Sat, 2 Jan 2021 19:25:25 -0500 Subject: [PATCH 18/49] caixin: add fulltext to the article route --- lib/routes/caixin/article.js | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/lib/routes/caixin/article.js b/lib/routes/caixin/article.js index 29fd6d70a0..fe8d617c01 100644 --- a/lib/routes/caixin/article.js +++ b/lib/routes/caixin/article.js @@ -1,4 +1,5 @@ const got = require('@/utils/got'); +const cheerio = require('cheerio'); module.exports = async (ctx) => { const response = await got({ @@ -12,15 +13,33 @@ module.exports = async (ctx) => { const data = response.data.data.list; + const items = await Promise.all( + data.map(async (item) => { + const link = item.web_url; + const summary = `

摘要:${item.summary}

`; + + const fullText = await ctx.cache.tryGet(link, async () => { + const result = await got.get(link); + + const $ = cheerio.load(result.data); + + return $('#Main_Content_Val').html(); + }); + + return { + title: item.title, + description: summary + fullText, + link: link, + pubDate: new Date(item.time * 1000), + author: item.author_name, + }; + }) + ); + ctx.state.data = { title: `财新网 - 首页`, link: `http://www.caixin.com/`, description: '财新网 - 首页', - item: data.map((item) => ({ - title: item.title, - description: `

${item.summary}

`, - link: item.web_url, - pubDate: new Date(item.time * 1000), - })), + item: items, }; }; From df5e015dd79663fece927c25c28515f1c6727a47 Mon Sep 17 00:00:00 2001 From: duhd1993 Date: Sat, 2 Jan 2021 19:52:43 -0500 Subject: [PATCH 19/49] caixin: do not display fullText when it's not article --- lib/routes/caixin/article.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/routes/caixin/article.js b/lib/routes/caixin/article.js index fe8d617c01..bef43439cd 100644 --- a/lib/routes/caixin/article.js +++ b/lib/routes/caixin/article.js @@ -28,7 +28,7 @@ module.exports = async (ctx) => { return { title: item.title, - description: summary + fullText, + description: fullText ? summary + fullText : summary, link: link, pubDate: new Date(item.time * 1000), author: item.author_name, From e1a3f8c8698e6df2cf16db82a12d5f662f30382d Mon Sep 17 00:00:00 2001 From: duhd1993 Date: Sat, 2 Jan 2021 20:36:18 -0500 Subject: [PATCH 20/49] caixin: style of article route --- lib/routes/caixin/article.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/routes/caixin/article.js b/lib/routes/caixin/article.js index bef43439cd..6874fe8aad 100644 --- a/lib/routes/caixin/article.js +++ b/lib/routes/caixin/article.js @@ -16,7 +16,7 @@ module.exports = async (ctx) => { const items = await Promise.all( data.map(async (item) => { const link = item.web_url; - const summary = `

摘要:${item.summary}

`; + const summary = `

${item.summary}

`; const fullText = await ctx.cache.tryGet(link, async () => { const result = await got.get(link); From 3694d4f5253f82ffc67c65947a97b1fe239730ed Mon Sep 17 00:00:00 2001 From: Laam Pui Date: Sun, 3 Jan 2021 13:25:14 +0800 Subject: [PATCH 21/49] =?UTF-8?q?fix:=20=E5=8C=97=E4=BA=AC=E7=90=86?= =?UTF-8?q?=E5=B7=A5=E5=A4=A7=E5=AD=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/routes/universities/bit/cs/cs.js | 3 +++ lib/routes/universities/bit/cs/utils.js | 6 +++++- lib/routes/universities/bit/jwc/jwc.js | 3 +++ lib/routes/universities/bit/jwc/utils.js | 6 +++++- 4 files changed, 16 insertions(+), 2 deletions(-) diff --git a/lib/routes/universities/bit/cs/cs.js b/lib/routes/universities/bit/cs/cs.js index 605491fa6c..ab923eb31e 100644 --- a/lib/routes/universities/bit/cs/cs.js +++ b/lib/routes/universities/bit/cs/cs.js @@ -6,6 +6,9 @@ module.exports = async (ctx) => { const response = await got({ method: 'get', url: 'http://cs.bit.edu.cn/tzgg', + https: { + rejectUnauthorized: false, + }, }); const $ = cheerio.load(response.data); diff --git a/lib/routes/universities/bit/cs/utils.js b/lib/routes/universities/bit/cs/utils.js index d3d07dd1b6..2efa8cc240 100644 --- a/lib/routes/universities/bit/cs/utils.js +++ b/lib/routes/universities/bit/cs/utils.js @@ -5,7 +5,11 @@ const url = require('url'); // 专门定义一个function用于加载文章内容 async function load(link) { // 异步请求文章 - const response = await got.get(link); + const response = await got.get(link, { + https: { + rejectUnauthorized: false, + }, + }); // 加载文章内容 const $ = cheerio.load(response.data); diff --git a/lib/routes/universities/bit/jwc/jwc.js b/lib/routes/universities/bit/jwc/jwc.js index 175d6cdde3..52859cfe6f 100644 --- a/lib/routes/universities/bit/jwc/jwc.js +++ b/lib/routes/universities/bit/jwc/jwc.js @@ -6,6 +6,9 @@ module.exports = async (ctx) => { const response = await got({ method: 'get', url: 'http://jwc.bit.edu.cn/tzgg', + https: { + rejectUnauthorized: false, + }, }); const $ = cheerio.load(response.data); diff --git a/lib/routes/universities/bit/jwc/utils.js b/lib/routes/universities/bit/jwc/utils.js index 1c32dcfb67..d2891db294 100644 --- a/lib/routes/universities/bit/jwc/utils.js +++ b/lib/routes/universities/bit/jwc/utils.js @@ -5,7 +5,11 @@ const url = require('url'); // 专门定义一个function用于加载文章内容 async function load(link) { // 异步请求文章 - const response = await got.get(link); + const response = await got.get(link, { + https: { + rejectUnauthorized: false, + }, + }); // 加载文章内容 const $ = cheerio.load(response.data); From b7bfe1cc00c0613be2dd7d300ff5c3975a82a78a Mon Sep 17 00:00:00 2001 From: Laam Pui Date: Sun, 3 Jan 2021 13:46:33 +0800 Subject: [PATCH 22/49] =?UTF-8?q?fix:=20=E4=B8=AD=E5=9B=BD=E8=8D=AF?= =?UTF-8?q?=E7=A7=91=E5=A4=A7=E5=AD=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/routes/universities/cpu/home.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/routes/universities/cpu/home.js b/lib/routes/universities/cpu/home.js index e7f1306d89..f87507ffc5 100644 --- a/lib/routes/universities/cpu/home.js +++ b/lib/routes/universities/cpu/home.js @@ -15,13 +15,13 @@ module.exports = async (ctx) => { const data = response.data; const $ = cheerio.load(data); - const $list = $('div#wp_news_w3 a').slice(0, 10).get(); + const $list = $('div#wp_news_w3 a').get(); const resultItem = await Promise.all( $list.map(async (item) => { const title = $(item).attr('title'); const href = $(item).attr('href'); - const detail_url = 'http://www.cpu.edu.cn' + href; + const detail_url = href.startsWith('/') ? `http://www.cpu.edu.cn${href}` : href; const single = { title: title, link: detail_url, @@ -37,7 +37,7 @@ module.exports = async (ctx) => { { const detail_data = detail.data; const $ = cheerio.load(detail_data); - single.description = $('table[bgcolor="#FFFFFF"]').html(); + single.description = $('table[bgcolor="#FFFFFF"]').html() || $('table.nyxx').html() || $('div.inner div.article').html(); } return Promise.resolve(single); }) From 94214a9d5e1c1cf93dc52643c5e49a021485c8bc Mon Sep 17 00:00:00 2001 From: jinkin Date: Tue, 5 Jan 2021 11:22:31 +0800 Subject: [PATCH 23/49] add mcdonalds news --- docs/shopping.md | 11 ++++++++ lib/router.js | 3 +++ lib/routes/mcdonalds/news.js | 51 ++++++++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+) create mode 100644 lib/routes/mcdonalds/news.js diff --git a/docs/shopping.md b/docs/shopping.md index eae0a16575..caba5c7a34 100644 --- a/docs/shopping.md +++ b/docs/shopping.md @@ -347,3 +347,14 @@ For instance, in + +## 麦当劳 + +### 麦当劳活动资讯 + + +| 全部分类 | 社会责任 | 人员品牌 | 产品故事 | 优惠 | 品牌文化 | 活动速报 | +| -------- | ------- | --- | ------- | ------ | ------- | -------- | +| news_list | responsibility | brand | product | sales | culture | event | + + diff --git a/lib/router.js b/lib/router.js index 5bb0f656c3..2f05b7b278 100644 --- a/lib/router.js +++ b/lib/router.js @@ -3780,4 +3780,7 @@ router.get('/nace/blog/:sort?', require('./routes/nace/blog')); // Caixin Latest router.get('/caixin/latest', require('./routes/caixin/latest')); +// Mcdonalds +router.get('/mcdonalds/:category', require('./routes/mcdonalds/news')); + module.exports = router; diff --git a/lib/routes/mcdonalds/news.js b/lib/routes/mcdonalds/news.js new file mode 100644 index 0000000000..93949dcc7b --- /dev/null +++ b/lib/routes/mcdonalds/news.js @@ -0,0 +1,51 @@ +const got = require('@/utils/got'); +const cheerio = require('cheerio'); + +module.exports = async (ctx) => { + const category_param = ctx.params.category || 'news_list'; + const categories = category_param.split('+'); + const baseUrl = "https://www.mcdonalds.com.cn/news/"; + + const get_news_list = async (cates) => await Promise.all( + cates.map(async (cate) => { + const response = await got.get(baseUrl + cate); + const $ = cheerio.load(response.data); + // console.log($('title').text()); + const news = $('.news_list .box-container > div').slice(0, 10).map((idx, item) => { + item = $(item); + return item.find('a[target]').attr('href'); + }) + .get(); + // console.log(news); + return Promise.resolve(news); + }) + ); + const all_news = [].concat.apply([], await get_news_list(categories)); + + const out = await Promise.all( + all_news.map(async (news_url) => { + const news_detail = await ctx.cache.tryGet(news_url, async () => { + const result = await got.get(news_url); + const $ = cheerio.load(result.data); + + console.log($('h3 ~ time').text(), news_url); + + return { + title: $('h3').text(), + link: news_url, + // author: author, + pubDate: new Date($('h3 ~ time').text()).toUTCString(), + description: $('.cmsPage').html(), + }; + }); + return Promise.resolve(news_detail); + }) + ); + + ctx.state.data = { + title: '麦当劳资讯', + link: baseUrl, + item: out, + }; + +}; \ No newline at end of file From ae1ceb6c6057de36150a571d752d7ffd3300c26d Mon Sep 17 00:00:00 2001 From: Jinkin Date: Tue, 5 Jan 2021 18:21:35 +0800 Subject: [PATCH 24/49] Update news.js --- lib/routes/mcdonalds/news.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/routes/mcdonalds/news.js b/lib/routes/mcdonalds/news.js index 93949dcc7b..f62e83a77b 100644 --- a/lib/routes/mcdonalds/news.js +++ b/lib/routes/mcdonalds/news.js @@ -28,7 +28,7 @@ module.exports = async (ctx) => { const result = await got.get(news_url); const $ = cheerio.load(result.data); - console.log($('h3 ~ time').text(), news_url); +// console.log($('h3 ~ time').text(), news_url); return { title: $('h3').text(), @@ -48,4 +48,4 @@ module.exports = async (ctx) => { item: out, }; -}; \ No newline at end of file +}; From 8cc0dfa231fc4ec02c3d9930d3bfaa6ed27d806b Mon Sep 17 00:00:00 2001 From: zytomorrow Date: Wed, 6 Jan 2021 15:49:58 +0800 Subject: [PATCH 25/49] =?UTF-8?q?feat:=20=E7=AE=80=E9=98=85(simpread)=20no?= =?UTF-8?q?tice?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/program-update.md | 6 ++++++ lib/router.js | 3 +++ lib/routes/simpread/notice.js | 20 ++++++++++++++++++++ 3 files changed, 29 insertions(+) create mode 100644 lib/routes/simpread/notice.js diff --git a/docs/program-update.md b/docs/program-update.md index 5cddf1086a..a2dafca86a 100644 --- a/docs/program-update.md +++ b/docs/program-update.md @@ -433,3 +433,9 @@ pageClass: routes | | -commentCount | -createdAt | createdAt | + +## simpread + +### 消息通知 + + diff --git a/lib/router.js b/lib/router.js index 5bb0f656c3..e9f57e3871 100644 --- a/lib/router.js +++ b/lib/router.js @@ -3780,4 +3780,7 @@ router.get('/nace/blog/:sort?', require('./routes/nace/blog')); // Caixin Latest router.get('/caixin/latest', require('./routes/caixin/latest')); +// SimpRead +router.get('/simpread/notice', require('./routes/simpread/notice')); + module.exports = router; diff --git a/lib/routes/simpread/notice.js b/lib/routes/simpread/notice.js new file mode 100644 index 0000000000..a6ef1a11dd --- /dev/null +++ b/lib/routes/simpread/notice.js @@ -0,0 +1,20 @@ +const got = require('@/utils/got'); +const md = require('markdown-it')(); + +module.exports = async (ctx) => { + const url = 'https://static.simp.red/notice'; + const response = await got.get(url); + const data = response.data.notice; + ctx.state.data = { + title: 'SimpRead 消息通知', + link: url, + description: 'SimpRead 消息通知', + item: data.map((item) => ({ + description: md.render(item.content), + link: url, + pubDate: item.date, + title: `${item.category.name}-${item.title}`, + author: 'SimpRead', + })), + }; +}; From 5b1c4cf80269f4bdd7bf71db82dd2a1674301fa5 Mon Sep 17 00:00:00 2001 From: hellodword <46193371+hellodword@users.noreply.github.com> Date: Thu, 7 Jan 2021 11:22:29 +0800 Subject: [PATCH 26/49] feat: add microsoft store updates --- docs/en/program-update.md | 6 +++++ docs/program-update.md | 6 +++++ lib/router.js | 3 +++ lib/routes/microsoft-store/updates.js | 32 +++++++++++++++++++++++++++ 4 files changed, 47 insertions(+) create mode 100644 lib/routes/microsoft-store/updates.js diff --git a/docs/en/program-update.md b/docs/en/program-update.md index 9325fd5e2b..13b7028215 100644 --- a/docs/en/program-update.md +++ b/docs/en/program-update.md @@ -164,6 +164,12 @@ The owner of the official image fills in the library, for example: https://rsshu +## Microsoft Store + +### Updates + + + ## Minecraft Refer to [#minecraft](/en/game.html#minecraft) diff --git a/docs/program-update.md b/docs/program-update.md index 5cddf1086a..ed7354551d 100644 --- a/docs/program-update.md +++ b/docs/program-update.md @@ -212,6 +212,12 @@ pageClass: routes +## Microsoft Store + +### Updates + + + ## Minecraft 见 [#minecraft](/game.html#minecraft) diff --git a/lib/router.js b/lib/router.js index 71be05cda9..e50f2c3a04 100644 --- a/lib/router.js +++ b/lib/router.js @@ -2853,6 +2853,9 @@ router.get('/xposed/module/:mod', require('./routes/xposed/module')); // Microsoft Edge router.get('/edge/addon/:crxid', require('./routes/edge/addon')); +// Microsoft Store +router.get('/microsoft-store/updates/:productid/:market?', require('./routes/microsoft-store/updates')); + // 上海立信会计金融学院 router.get('/slu/tzgg/:id', require('./routes/universities/slu/tzgg')); router.get('/slu/jwc/:id', require('./routes/universities/slu/jwc')); diff --git a/lib/routes/microsoft-store/updates.js b/lib/routes/microsoft-store/updates.js new file mode 100644 index 0000000000..4c5f63f625 --- /dev/null +++ b/lib/routes/microsoft-store/updates.js @@ -0,0 +1,32 @@ +const got = require('@/utils/got'); + +module.exports = async (ctx) => { + const { market = 'CN', productid } = ctx.params; + + const { data } = await got({ + method: 'get', + url: `https://displaycatalog.mp.microsoft.com/v7.0/products/${productid}/?fieldsTemplate=&market=${market}&languages=en`, + headers: { + 'Content-Type': 'application/json', + 'MS-CV': `${Array(16) + .join() + .split(',') + .map(function () { + return 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'.charAt(Math.floor(Math.random() * 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'.length)); + }) + .join('')}.1`, + }, + }); + + ctx.state.data = { + title: `${data.Product.LocalizedProperties[0].ProductTitle} - Microsoft Store Updates`, + link: `https://www.microsoft.com/store/productId/${productid}`, + item: [ + { + title: data.Product.DisplaySkuAvailabilities[0].Sku.Properties.Packages[0].PackageFullName, + pubDate: new Date(data.Product.DisplaySkuAvailabilities[0].Sku.LastModifiedDate), + link: `https://www.microsoft.com/store/productId/${productid}`, + }, + ], + }; +}; From 333f99215aafb356e5d6773fa50ab38fea4b9d3b Mon Sep 17 00:00:00 2001 From: KTachibanaM Date: Wed, 6 Jan 2021 21:55:49 -0800 Subject: [PATCH 27/49] add instruction for ansible deployment --- docs/en/install/README.md | 30 ++++++++++++++++++++++++++++++ docs/install/README.md | 30 ++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/docs/en/install/README.md b/docs/en/install/README.md index 460be758ea..3623834bbe 100644 --- a/docs/en/install/README.md +++ b/docs/en/install/README.md @@ -104,6 +104,36 @@ $ docker run -d --name rsshub -p 1200:1200 -e CACHE_EXPIRE=3600 -e GITHUB_ACCESS To configure more options please refer to [Configuration](#configuration). +# Ansible Deployment + +[This Ansible playbook](https://github.com/k-t-corp/RSSHub-ansible) includes RSSHub, Redis, browserless and Caddy + +This Ansible playbook currently only support Ubuntu 20.04 + +### Install + +```bash +sudo apt update +sudo apt install ansible +cd ~ +git clone https://github.com/k-t-corp/RSSHub-ansible.git +cd RSSHub-ansible +ansible-playbook rsshub.yaml +# When prompt to enter a domain name, enter the domain name that this machine/VM will use +# For example, if your users use https://rsshub.exmaple.com to access your RSSHub instance, enter rsshub.exmaple.com (remove the https://) +``` + +### Update + +```bash +cd ~ +git clone https://github.com/k-t-corp/RSSHub-ansible.git +cd RSSHub-ansible +ansible-playbook rsshub.yaml +# When prompt to enter a domain name, enter the domain name that this machine/VM will use +# For example, if your users use https://rsshub.exmaple.com to access your RSSHub instance, enter rsshub.exmaple.com (remove the https://) +``` + ## Manual Deployment The most direct way to deploy `RSSHub`, you can follow the steps below to deploy`RSSHub` on your computer, server or anywhere. diff --git a/docs/install/README.md b/docs/install/README.md index 82c18370ac..dfa6a09489 100644 --- a/docs/install/README.md +++ b/docs/install/README.md @@ -106,6 +106,36 @@ $ docker run -d --name rsshub -p 1200:1200 -e CACHE_EXPIRE=3600 -e GITHUB_ACCESS 更多配置项请看 [#配置](#pei-zhi) +## Ansible 部署 + +[该 Ansible playbook](https://github.com/k-t-corp/RSSHub-ansible) 包括了 RSSHub, Redis, browserless 以及 Caddy 的安装 + +该 Ansible playbook 目前只支持 Ubuntu 20.04 + +### 安装 + +```bash +sudo apt update +sudo apt install ansible +cd ~ +git clone https://github.com/k-t-corp/RSSHub-ansible.git +cd RSSHub-ansible +ansible-playbook rsshub.yaml +# 当提示输入 domain name 的时候，输入该主机所使用的域名 +# 举例：如果您的 RSSHub 用户使用 https://rsshub.exmaple.com 访问您的 RSSHub 实例，输入 rsshub.exmaple.com（去掉 https://） +``` + +### 更新 + +```bash +cd ~ +git clone https://github.com/k-t-corp/RSSHub-ansible.git +cd RSSHub-ansible +ansible-playbook rsshub.yaml +# 当提示输入 domain name 的时候，输入该主机所使用的域名 +# 举例：如果您的 RSSHub 用户使用 https://rsshub.exmaple.com 访问您的 RSSHub 实例，输入 rsshub.exmaple.com（去掉 https://） +``` + ## 手动部署 部署 `RSSHub` 最直接的方式，您可以按照以下步骤将 `RSSHub` 部署在您的电脑、服务器或者其他任何地方 From 866baa903a0b6c87bfa3fb31d6da96a3d1f42253 Mon Sep 17 00:00:00 2001 From: KTachibanaM Date: Wed, 6 Jan 2021 21:58:36 -0800 Subject: [PATCH 28/49] specify caddy 2 for ansible deployment --- docs/en/install/README.md | 2 +- docs/install/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/install/README.md b/docs/en/install/README.md index 3623834bbe..6217303343 100644 --- a/docs/en/install/README.md +++ b/docs/en/install/README.md @@ -106,7 +106,7 @@ To configure more options please refer to [Configuration](#configuration). # Ansible Deployment -[This Ansible playbook](https://github.com/k-t-corp/RSSHub-ansible) includes RSSHub, Redis, browserless and Caddy +[This Ansible playbook](https://github.com/k-t-corp/RSSHub-ansible) includes RSSHub, Redis, browserless and Caddy 2 This Ansible playbook currently only support Ubuntu 20.04 diff --git a/docs/install/README.md b/docs/install/README.md index dfa6a09489..793fd15b85 100644 --- a/docs/install/README.md +++ b/docs/install/README.md @@ -108,7 +108,7 @@ $ docker run -d --name rsshub -p 1200:1200 -e CACHE_EXPIRE=3600 -e GITHUB_ACCESS ## Ansible 部署 -[该 Ansible playbook](https://github.com/k-t-corp/RSSHub-ansible) 包括了 RSSHub, Redis, browserless 以及 Caddy 的安装 +[该 Ansible playbook](https://github.com/k-t-corp/RSSHub-ansible) 包括了 RSSHub, Redis, browserless 以及 Caddy 2 该 Ansible playbook 目前只支持 Ubuntu 20.04 From da9be824ee48c647499cc8d12bdddacc3e948e85 Mon Sep 17 00:00:00 2001 From: zytomorrow Date: Thu, 7 Jan 2021 17:57:33 +0800 Subject: [PATCH 29/49] =?UTF-8?q?feat:=20=E7=AE=80=E9=98=85(simpread)=20ch?= =?UTF-8?q?angelog?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/program-update.md | 3 ++ lib/router.js | 5 ++- lib/routes/simpread/changelog.js | 77 ++++++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 lib/routes/simpread/changelog.js diff --git a/docs/program-update.md b/docs/program-update.md index a2dafca86a..706791de63 100644 --- a/docs/program-update.md +++ b/docs/program-update.md @@ -439,3 +439,6 @@ pageClass: routes ### 消息通知 + +### 更新日志 + diff --git a/lib/router.js b/lib/router.js index e9f57e3871..2da4052374 100644 --- a/lib/router.js +++ b/lib/router.js @@ -3780,7 +3780,10 @@ router.get('/nace/blog/:sort?', require('./routes/nace/blog')); // Caixin Latest router.get('/caixin/latest', require('./routes/caixin/latest')); -// SimpRead +// SimpRead-消息通知 router.get('/simpread/notice', require('./routes/simpread/notice')); +// SimpRead-更新日志 +router.get('/simpread/changelog', require('./routes/simpread/changelog')); + module.exports = router; diff --git a/lib/routes/simpread/changelog.js b/lib/routes/simpread/changelog.js new file mode 100644 index 0000000000..d7ad05d766 --- /dev/null +++ b/lib/routes/simpread/changelog.js @@ -0,0 +1,77 @@ +const got = require('@/utils/got'); +const cheerio = require('cheerio'); + +module.exports = async (ctx) => { + const url = 'https://simpread.pro/changelog.html'; + const response = await got.get(url); + const data = response.data; + const $ = cheerio.load(data); + ctx.state.data = { + title: 'SimpRead 更新日志', + link: url, + description: $('body > div.container.changelog > div.desc').html(), + item: $('.version') + .map((index, item) => { + const year = $(item).find('.year').html(); + const month_day = $(item).find('.day').html(); + let version = ''; + let detail = ''; + + // 结构异化-收个version与后续version不同级 + if (index === 0) { + // 版本名称处理 + version = $($(item).find('.num > a')[0]).clone().children().remove().end().text(); + // detail处理 + detail = $($(item).find('.details')[0]); + } else { + // 版本名称处理 + version = $(item).find('.num > a').clone().children().remove().end().text(); + // detail处理 + detail = $(item).find('.details'); + } + if (version === '') { + version = $(item).find('.num').clone().children().remove().end().text(); + } + let version_type = $(item).find('.num > a > i').attr('class'); + // 部分结构不一致处理 + if (version_type === undefined) { + version_type = $(item).find('.num > i').attr('class'); + } + if (version_type.indexOf('chrome') !== -1) { + version_type = 'Chrome'; + } + if (version_type.indexOf('apple') !== -1) { + version_type = 'Safari'; + } + if (version_type.indexOf('code') !== -1) { + version_type = 'UserScript'; + } + if (version_type.indexOf('firefox') !== -1) { + version_type = 'Firefox'; + } + + $(detail) + .find('li') + .map((index, item) => { + const level = $(item).find('span').attr('class'); + if (level !== undefined) { + if (level.indexOf('empty') !== -1) { + $(item).wrap('
    '); + } else { + $(item).find('span').append('
    '); + } + } + return {}; + }); + + return { + description: $(detail).html(), + link: url, + pubDate: `${year}-${month_day.replace('.', '-')} 00:00:00 GMT`, + title: `${version_type}${version}`, + author: 'SimpRead', + }; + }) + .get(), + }; +}; From 002f41f55918a884ff588945868239ec22334bff Mon Sep 17 00:00:00 2001 From: Ethan Shen Date: Thu, 7 Jan 2021 21:31:14 +0800 Subject: [PATCH 30/49] =?UTF-8?q?fix:=20=E5=B0=8F=E9=BB=91=E7=9B=92?= =?UTF-8?q?=E6=B8=B8=E6=88=8F=E6=96=B0=E9=97=BB=E5=8E=9F=E6=96=87=E9=93=BE?= =?UTF-8?q?=E6=8E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/routes/xiaoheihe/news.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/routes/xiaoheihe/news.js b/lib/routes/xiaoheihe/news.js index 1eb2d2f827..a00a6730bf 100644 --- a/lib/routes/xiaoheihe/news.js +++ b/lib/routes/xiaoheihe/news.js @@ -38,6 +38,7 @@ module.exports = async (ctx) => { // 存放到缓存区 ctx.cache.set(cacheKey, content); news.description = content; + news.link = `https://api.xiaoheihe.cn/maxnews/app/share/detail/${newsId}`; } return Promise.resolve(news); From 2aca91a25e71b508cf74a9fb48bf57acc0bbd54c Mon Sep 17 00:00:00 2001 From: zytomorrow Date: Fri, 8 Jan 2021 08:57:25 +0800 Subject: [PATCH 31/49] =?UTF-8?q?fix:=20=E7=AE=80=E9=98=85(simpread)=20doc?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/program-update.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/program-update.md b/docs/program-update.md index 706791de63..b120035886 100644 --- a/docs/program-update.md +++ b/docs/program-update.md @@ -438,7 +438,7 @@ pageClass: routes ### 消息通知 - + ### 更新日志 - + From 97b89d68ef50e1cd300b73d77956a8963cab3b47 Mon Sep 17 00:00:00 2001 From: KTachibanaM Date: Thu, 7 Jan 2021 19:59:44 -0800 Subject: [PATCH 32/49] move ansible scripts inside --- docs/en/install/README.md | 19 +++-- docs/install/README.md | 19 +++-- scripts/ansible/.gitignore | 90 +++++++++++++++++++++ scripts/ansible/README.md | 20 +++++ scripts/ansible/Vagrantfile | 5 ++ scripts/ansible/rsshub.Caddyfile | 3 + scripts/ansible/rsshub.env | 3 + scripts/ansible/rsshub.service | 11 +++ scripts/ansible/rsshub.yaml | 130 +++++++++++++++++++++++++++++++ scripts/ansible/try.sh | 5 ++ 10 files changed, 285 insertions(+), 20 deletions(-) create mode 100644 scripts/ansible/.gitignore create mode 100644 scripts/ansible/README.md create mode 100644 scripts/ansible/Vagrantfile create mode 100644 scripts/ansible/rsshub.Caddyfile create mode 100644 scripts/ansible/rsshub.env create mode 100644 scripts/ansible/rsshub.service create mode 100644 scripts/ansible/rsshub.yaml create mode 100755 scripts/ansible/try.sh diff --git a/docs/en/install/README.md b/docs/en/install/README.md index 6217303343..7dc0b7d954 100644 --- a/docs/en/install/README.md +++ b/docs/en/install/README.md @@ -106,19 +106,20 @@ To configure more options please refer to [Configuration](#configuration). # Ansible Deployment -[This Ansible playbook](https://github.com/k-t-corp/RSSHub-ansible) includes RSSHub, Redis, browserless and Caddy 2 +This Ansible playbook includes RSSHub, Redis, browserless and Caddy 2 -This Ansible playbook currently only support Ubuntu 20.04 +Currently only support Ubuntu 20.04 + +Requires sudo privilege ### Install ```bash sudo apt update sudo apt install ansible -cd ~ -git clone https://github.com/k-t-corp/RSSHub-ansible.git -cd RSSHub-ansible -ansible-playbook rsshub.yaml +git clone https://github.com/DIYgod/RSSHub.git ~/RSSHub +cd ~/RSSHub/scripts/ansible +sudo ansible-playbook rsshub.yaml # When prompt to enter a domain name, enter the domain name that this machine/VM will use # For example, if your users use https://rsshub.exmaple.com to access your RSSHub instance, enter rsshub.exmaple.com (remove the https://) ``` @@ -126,10 +127,8 @@ ansible-playbook rsshub.yaml ### Update ```bash -cd ~ -git clone https://github.com/k-t-corp/RSSHub-ansible.git -cd RSSHub-ansible -ansible-playbook rsshub.yaml +cd ~/RSSHub/scripts/ansible +sudo ansible-playbook rsshub.yaml # When prompt to enter a domain name, enter the domain name that this machine/VM will use # For example, if your users use https://rsshub.exmaple.com to access your RSSHub instance, enter rsshub.exmaple.com (remove the https://) ``` diff --git a/docs/install/README.md b/docs/install/README.md index 793fd15b85..ed84552889 100644 --- a/docs/install/README.md +++ b/docs/install/README.md @@ -108,19 +108,20 @@ $ docker run -d --name rsshub -p 1200:1200 -e CACHE_EXPIRE=3600 -e GITHUB_ACCESS ## Ansible 部署 -[该 Ansible playbook](https://github.com/k-t-corp/RSSHub-ansible) 包括了 RSSHub, Redis, browserless 以及 Caddy 2 +这个 Ansible playbook 包括了 RSSHub, Redis, browserless 以及 Caddy 2 -该 Ansible playbook 目前只支持 Ubuntu 20.04 +目前只支持 Ubuntu 20.04 + +需要 sudo 权限 ### 安装 ```bash sudo apt update sudo apt install ansible -cd ~ -git clone https://github.com/k-t-corp/RSSHub-ansible.git -cd RSSHub-ansible -ansible-playbook rsshub.yaml +git clone https://github.com/DIYgod/RSSHub.git ~/RSSHub +cd ~/RSSHub/scripts/ansible +sudo ansible-playbook rsshub.yaml # 当提示输入 domain name 的时候，输入该主机所使用的域名 # 举例：如果您的 RSSHub 用户使用 https://rsshub.exmaple.com 访问您的 RSSHub 实例，输入 rsshub.exmaple.com（去掉 https://） ``` @@ -128,10 +129,8 @@ ansible-playbook rsshub.yaml ### 更新 ```bash -cd ~ -git clone https://github.com/k-t-corp/RSSHub-ansible.git -cd RSSHub-ansible -ansible-playbook rsshub.yaml +cd ~/RSSHub/scripts/ansible +sudo ansible-playbook rsshub.yaml # 当提示输入 domain name 的时候，输入该主机所使用的域名 # 举例：如果您的 RSSHub 用户使用 https://rsshub.exmaple.com 访问您的 RSSHub 实例，输入 rsshub.exmaple.com（去掉 https://） ``` diff --git a/scripts/ansible/.gitignore b/scripts/ansible/.gitignore new file mode 100644 index 0000000000..6117c52186 --- /dev/null +++ b/scripts/ansible/.gitignore @@ -0,0 +1,90 @@ +# Created by https://www.toptal.com/developers/gitignore/api/windows,linux,macos,ansible,vagrant +# Edit at https://www.toptal.com/developers/gitignore?templates=windows,linux,macos,ansible,vagrant + +### Ansible ### +*.retry + +### Linux ### +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + +### macOS ### +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +### Vagrant ### +# General +.vagrant/ + +# Log files (if you are creating logs in debug mode, uncomment this) +# *.log + +### Vagrant Patch ### +*.box + +### Windows ### +# Windows thumbnail cache files +Thumbs.db +Thumbs.db:encryptable +ehthumbs.db +ehthumbs_vista.db + +# Dump file +*.stackdump + +# Folder config file +[Dd]esktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msix +*.msm +*.msp + +# Windows shortcuts +*.lnk + +# End of https://www.toptal.com/developers/gitignore/api/windows,linux,macos,ansible,vagrant + +# vagrant logs +*.log \ No newline at end of file diff --git a/scripts/ansible/README.md b/scripts/ansible/README.md new file mode 100644 index 0000000000..50cae9b717 --- /dev/null +++ b/scripts/ansible/README.md @@ -0,0 +1,20 @@ +# Readme + +Ansible playbook to deploy [RSSHub](https://github.com/DIYgod/RSSHub) on bare-metal with Redis, browserless and Caddy 2 + +Requires sudo permission + +## Usage +On `Ubuntu 20.04`, [install ansible](https://www.digitalocean.com/community/tutorials/how-to-install-and-configure-ansible-on-ubuntu-20-04), then + +```bash +sudo ansible-playbook rsshub.yaml +``` + +## Development +Install `vagrant`, then + +```bash +./try.sh +ansible-playbook rsshub.yaml +``` diff --git a/scripts/ansible/Vagrantfile b/scripts/ansible/Vagrantfile new file mode 100644 index 0000000000..604af7f5ad --- /dev/null +++ b/scripts/ansible/Vagrantfile @@ -0,0 +1,5 @@ +Vagrant.configure("2") do |config| + config.vm.box = "generic/ubuntu2004" + config.vm.synced_folder ".", "/vagrant", type: "rsync", rsync__exclude: ".git/" + config.ssh.extra_args = ["-t", "cd /vagrant; bash --login"] +end diff --git a/scripts/ansible/rsshub.Caddyfile b/scripts/ansible/rsshub.Caddyfile new file mode 100644 index 0000000000..7781eb24fd --- /dev/null +++ b/scripts/ansible/rsshub.Caddyfile @@ -0,0 +1,3 @@ +{{ domain_name }} + +reverse_proxy localhost:1200 diff --git a/scripts/ansible/rsshub.env b/scripts/ansible/rsshub.env new file mode 100644 index 0000000000..d4a283725c --- /dev/null +++ b/scripts/ansible/rsshub.env @@ -0,0 +1,3 @@ +NODE_ENV=production +CACHE_TYPE=redis +PUPPETEER_WS_ENDPOINT=ws://localhost:3000 diff --git a/scripts/ansible/rsshub.service b/scripts/ansible/rsshub.service new file mode 100644 index 0000000000..0c8352a44e --- /dev/null +++ b/scripts/ansible/rsshub.service @@ -0,0 +1,11 @@ +[Unit] +Description=RSSHub is an open source, easy to use, and extensible RSS feed aggregator + +[Service] +User=rsshub +WorkingDirectory=/home/rsshub/app +ExecStart=yarn start +EnvironmentFile=/home/rsshub/app/.env + +[Install] +WantedBy=multi-user.target diff --git a/scripts/ansible/rsshub.yaml b/scripts/ansible/rsshub.yaml new file mode 100644 index 0000000000..33671a51ce --- /dev/null +++ b/scripts/ansible/rsshub.yaml @@ -0,0 +1,130 @@ +- + name: Install RSSHub + hosts: localhost + become: true + vars_prompt: + - + name: domain_name + prompt: What is the domain name (without www, e.g. rsshub.example.com)? Use "http://localhost" for development in Vagrant VM. + private: no + tasks: + - + name: Check OS + fail: + msg: This playbook can only be run on Ubuntu 20.04 at this moment + when: ansible_distribution != 'Ubuntu' or ansible_distribution_version !='20.04' + - + name: Install GPG keys for repos + apt_key: + url: '{{ item }}' + state: present + with_items: + - https://deb.nodesource.com/gpgkey/nodesource.gpg.key + - https://dl.yarnpkg.com/debian/pubkey.gpg + - https://download.docker.com/linux/ubuntu/gpg + - https://dl.cloudsmith.io/public/caddy/stable/cfg/gpg/gpg.155B6D79CA56EA34.key + - + name: Install repos + apt_repository: + repo: '{{ item }}' + state: present + update_cache: yes + with_items: + - deb https://deb.nodesource.com/node_12.x focal main + - deb https://dl.yarnpkg.com/debian/ stable main + - deb https://download.docker.com/linux/ubuntu focal stable + - deb https://dl.cloudsmith.io/public/caddy/stable/deb/debian any-version main + - + name: Install prerequisites + apt: + name: + - nodejs + - yarn + - build-essential + - python-is-python2 + - redis-server + - docker-ce + - python3-pip + - virtualenv + - python3-setuptools + - caddy + state: present + update_cache: yes + - + name: Install python module for docker + pip: + name: docker + - + name: Pull docker image for browserless + docker_image: + name: browserless/chrome + source: pull + - + name: Start redis + systemd: + state: restarted + enabled: yes + name: redis + daemon_reload: yes + - + name: Copy caddy configuration + template: + src: rsshub.Caddyfile + dest: /etc/caddy/Caddyfile + - + name: Start caddy + systemd: + state: restarted + enabled: yes + name: caddy + daemon_reload: yes + - + name: Create and start browserless container + docker_container: + name: browserless + image: browserless/chrome + state: started + restart_policy: always + published_ports: + - "3000:3000" + - + name: Create the user + user: + name: rsshub + create_home: true + shell: /bin/bash + - + name: Clone the repo + git: + repo: https://github.com/DIYgod/RSSHub.git + dest: /home/rsshub/app + update: yes + - + name: Install repo dependencies + command: yarn install --production + args: + chdir: /home/rsshub/app + - + name: Copy configuration + copy: + src: rsshub.env + dest: /home/rsshub/app/.env + - + name: Own repo to the user + file: + path: /home/rsshub/app + owner: rsshub + group: rsshub + recurse: yes + - + name: Install the systemd unit + copy: + src: rsshub.service + dest: /etc/systemd/system/rsshub.service + - + name: Start the systemd service + systemd: + state: restarted + enabled: yes + name: rsshub + daemon_reload: yes diff --git a/scripts/ansible/try.sh b/scripts/ansible/try.sh new file mode 100755 index 0000000000..cd8f1df76d --- /dev/null +++ b/scripts/ansible/try.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set -e + +vagrant rsync +vagrant ssh From 7aec48e8d0f83c1fd640fef3583a235d07a5b08a Mon Sep 17 00:00:00 2001 From: zytomorrow Date: Fri, 8 Jan 2021 12:41:47 +0800 Subject: [PATCH 33/49] =?UTF-8?q?fix:=20=E7=AE=80=E9=98=85(simpread)=20cha?= =?UTF-8?q?ngelog=E7=BB=93=E6=9E=84bug=E4=BF=AE=E5=A4=8D=E9=80=82=E9=85=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/routes/simpread/changelog.js | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/lib/routes/simpread/changelog.js b/lib/routes/simpread/changelog.js index d7ad05d766..5be450af73 100644 --- a/lib/routes/simpread/changelog.js +++ b/lib/routes/simpread/changelog.js @@ -16,19 +16,10 @@ module.exports = async (ctx) => { const month_day = $(item).find('.day').html(); let version = ''; let detail = ''; - - // 结构异化-收个version与后续version不同级 - if (index === 0) { - // 版本名称处理 - version = $($(item).find('.num > a')[0]).clone().children().remove().end().text(); - // detail处理 - detail = $($(item).find('.details')[0]); - } else { - // 版本名称处理 - version = $(item).find('.num > a').clone().children().remove().end().text(); - // detail处理 - detail = $(item).find('.details'); - } + // 版本名称处理 + version = $(item).find('.num > a').clone().children().remove().end().text(); + // detail处理 + detail = $(item).find('.details'); if (version === '') { version = $(item).find('.num').clone().children().remove().end().text(); } From 92ddaf6178c193ce799f82df00b4862332168cab Mon Sep 17 00:00:00 2001 From: zytomorrow Date: Fri, 8 Jan 2021 13:20:07 +0800 Subject: [PATCH 34/49] =?UTF-8?q?fix:=20=E7=AE=80=E9=98=85(simpread)=20?= =?UTF-8?q?=E5=8D=A0=E4=BD=8D=E5=AD=97=E7=AC=A6=E5=8E=BB=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/routes/simpread/changelog.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/routes/simpread/changelog.js b/lib/routes/simpread/changelog.js index 5be450af73..dfa3f8a140 100644 --- a/lib/routes/simpread/changelog.js +++ b/lib/routes/simpread/changelog.js @@ -45,11 +45,13 @@ module.exports = async (ctx) => { .find('li') .map((index, item) => { const level = $(item).find('span').attr('class'); + const text = $(item).find('span').html(); + $(item).find('span').remove(); if (level !== undefined) { if (level.indexOf('empty') !== -1) { $(item).wrap('
      '); } else { - $(item).find('span').append('
      '); + $(item).prepend(`${text}: `); } } return {}; From f17c4e92b8d199599b91379493b8970a3cbd5e21 Mon Sep 17 00:00:00 2001 From: zytomorrow Date: Fri, 8 Jan 2021 14:31:28 +0800 Subject: [PATCH 35/49] =?UTF-8?q?fix:=20=E7=AE=80=E9=98=85(simpread)=20?= =?UTF-8?q?=E6=A0=BC=E5=BC=8F=E7=BE=8E=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/routes/simpread/changelog.js | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/lib/routes/simpread/changelog.js b/lib/routes/simpread/changelog.js index dfa3f8a140..9665b4cb5d 100644 --- a/lib/routes/simpread/changelog.js +++ b/lib/routes/simpread/changelog.js @@ -2,7 +2,7 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); module.exports = async (ctx) => { - const url = 'https://simpread.pro/changelog.html'; + const url = 'http://ksria.com/simpread/changelog.html'; const response = await got.get(url); const data = response.data; const $ = cheerio.load(data); @@ -41,17 +41,26 @@ module.exports = async (ctx) => { version_type = 'Firefox'; } + // detail + const text_color = { + important: '#9c27b0', + add: '#4caf50', + change: '#ffc107', + fix: '#f44336', + complete: '#03a9f4', + }; $(detail) .find('li') .map((index, item) => { - const level = $(item).find('span').attr('class'); + let span_class = $(item).find('span').attr('class'); const text = $(item).find('span').html(); $(item).find('span').remove(); - if (level !== undefined) { - if (level.indexOf('empty') !== -1) { + if (span_class !== undefined) { + span_class = span_class.split(' '); + if (span_class[1] === 'empty') { $(item).wrap('
        '); } else { - $(item).prepend(`${text}: `); + $(item).prepend(`${text}: `); } } return {}; From ce132be12e11a3b8565fa74503f8bbc34bdcf61e Mon Sep 17 00:00:00 2001 From: zytomorrow Date: Fri, 8 Jan 2021 15:06:05 +0800 Subject: [PATCH 36/49] =?UTF-8?q?fix:=20=E7=AE=80=E9=98=85(simpread)=20?= =?UTF-8?q?=E6=A0=BC=E5=BC=8F=E7=BE=8E=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/routes/simpread/notice.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/routes/simpread/notice.js b/lib/routes/simpread/notice.js index a6ef1a11dd..8da1c99bc0 100644 --- a/lib/routes/simpread/notice.js +++ b/lib/routes/simpread/notice.js @@ -7,11 +7,11 @@ module.exports = async (ctx) => { const data = response.data.notice; ctx.state.data = { title: 'SimpRead 消息通知', - link: url, + link: 'http://ksria.com/simpread/changelog.html', description: 'SimpRead 消息通知', item: data.map((item) => ({ description: md.render(item.content), - link: url, + link: 'http://ksria.com/simpread/changelog.html', pubDate: item.date, title: `${item.category.name}-${item.title}`, author: 'SimpRead', From 0747bfb9927475669b7e7f6602714d208509e6f2 Mon Sep 17 00:00:00 2001 From: zytomorrow Date: Fri, 8 Jan 2021 15:13:42 +0800 Subject: [PATCH 37/49] =?UTF-8?q?fix:=20=E7=AE=80=E9=98=85(simpread)=20?= =?UTF-8?q?=E6=A0=BC=E5=BC=8F=E7=BE=8E=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/routes/simpread/changelog.js | 4 ++-- lib/routes/simpread/notice.js | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/routes/simpread/changelog.js b/lib/routes/simpread/changelog.js index 9665b4cb5d..59ecaea8f7 100644 --- a/lib/routes/simpread/changelog.js +++ b/lib/routes/simpread/changelog.js @@ -8,7 +8,7 @@ module.exports = async (ctx) => { const $ = cheerio.load(data); ctx.state.data = { title: 'SimpRead 更新日志', - link: url, + link: 'https://simpread.pro/changelog.html', description: $('body > div.container.changelog > div.desc').html(), item: $('.version') .map((index, item) => { @@ -68,7 +68,7 @@ module.exports = async (ctx) => { return { description: $(detail).html(), - link: url, + link: 'https://simpread.pro/changelog.html', pubDate: `${year}-${month_day.replace('.', '-')} 00:00:00 GMT`, title: `${version_type}${version}`, author: 'SimpRead', diff --git a/lib/routes/simpread/notice.js b/lib/routes/simpread/notice.js index 8da1c99bc0..79dd7950ef 100644 --- a/lib/routes/simpread/notice.js +++ b/lib/routes/simpread/notice.js @@ -7,11 +7,11 @@ module.exports = async (ctx) => { const data = response.data.notice; ctx.state.data = { title: 'SimpRead 消息通知', - link: 'http://ksria.com/simpread/changelog.html', + link: 'https://simpread.pro/changelog.html', description: 'SimpRead 消息通知', item: data.map((item) => ({ description: md.render(item.content), - link: 'http://ksria.com/simpread/changelog.html', + link: 'https://simpread.pro/changelog.html', pubDate: item.date, title: `${item.category.name}-${item.title}`, author: 'SimpRead', From 01b8757729ec085d47a96bdeac18f524094c5ed9 Mon Sep 17 00:00:00 2001 From: KTachibanaM Date: Fri, 8 Jan 2021 23:19:11 -0800 Subject: [PATCH 38/49] mention docker for ansible --- docs/en/install/README.md | 2 +- docs/install/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/install/README.md b/docs/en/install/README.md index 7dc0b7d954..d271aa069a 100644 --- a/docs/en/install/README.md +++ b/docs/en/install/README.md @@ -106,7 +106,7 @@ To configure more options please refer to [Configuration](#configuration). # Ansible Deployment -This Ansible playbook includes RSSHub, Redis, browserless and Caddy 2 +This Ansible playbook includes RSSHub, Redis, browserless (uses Docker) and Caddy 2 Currently only support Ubuntu 20.04 diff --git a/docs/install/README.md b/docs/install/README.md index ed84552889..3d69e43013 100644 --- a/docs/install/README.md +++ b/docs/install/README.md @@ -108,7 +108,7 @@ $ docker run -d --name rsshub -p 1200:1200 -e CACHE_EXPIRE=3600 -e GITHUB_ACCESS ## Ansible 部署 -这个 Ansible playbook 包括了 RSSHub, Redis, browserless 以及 Caddy 2 +这个 Ansible playbook 包括了 RSSHub, Redis, browserless (依赖 Docker) 以及 Caddy 2 目前只支持 Ubuntu 20.04 From 2a6e1176aace62dce1a8ba05aabe5900b992c890 Mon Sep 17 00:00:00 2001 From: KTachibanaM Date: Fri, 8 Jan 2021 23:24:37 -0800 Subject: [PATCH 39/49] mentions docker will be automatically installed for ansible deployment --- docs/en/install/README.md | 2 +- docs/install/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/install/README.md b/docs/en/install/README.md index d271aa069a..b612600623 100644 --- a/docs/en/install/README.md +++ b/docs/en/install/README.md @@ -110,7 +110,7 @@ This Ansible playbook includes RSSHub, Redis, browserless (uses Docker) and Cadd Currently only support Ubuntu 20.04 -Requires sudo privilege +Requires sudo privilege and virtualization capability (Docker will be automatically installed) ### Install diff --git a/docs/install/README.md b/docs/install/README.md index 3d69e43013..a8c59fa918 100644 --- a/docs/install/README.md +++ b/docs/install/README.md @@ -112,7 +112,7 @@ $ docker run -d --name rsshub -p 1200:1200 -e CACHE_EXPIRE=3600 -e GITHUB_ACCESS 目前只支持 Ubuntu 20.04 -需要 sudo 权限 +需要 sudo 权限和虚拟化能力（Docker 将会被自动安装） ### 安装 From f8da7ce83547e4e25c02efe522ba433d12e7e825 Mon Sep 17 00:00:00 2001 From: nczitzk Date: Sat, 9 Jan 2021 22:33:00 +0800 Subject: [PATCH 40/49] feat: add cgtn opinions --- docs/en/new-media.md | 4 +++ docs/new-media.md | 4 +++ lib/router.js | 1 + lib/routes/cgtn/opinions.js | 51 +++++++++++++++++++++++++++++++++++++ 4 files changed, 60 insertions(+) create mode 100644 lib/routes/cgtn/opinions.js diff --git a/docs/en/new-media.md b/docs/en/new-media.md index 070815cbe9..d0b372cc8b 100644 --- a/docs/en/new-media.md +++ b/docs/en/new-media.md @@ -67,6 +67,10 @@ Compared to the official one, the RSS feed generated by RSSHub not only has more ## CGTN +### Opinions + + + ### Most Read & Most Share diff --git a/docs/new-media.md b/docs/new-media.md index dea896d573..ebc5e352c5 100644 --- a/docs/new-media.md +++ b/docs/new-media.md @@ -122,6 +122,10 @@ pageClass: routes ## CGTN +### Opinions + + + ### Most Read & Most Share diff --git a/lib/router.js b/lib/router.js index 5bb0f656c3..8daef9d3dd 100644 --- a/lib/router.js +++ b/lib/router.js @@ -3338,6 +3338,7 @@ router.get('/fulinian/:caty?', require('./routes/fulinian/index')); // CGTN router.get('/cgtn/most/:type?/:time?', require('./routes/cgtn/most')); +router.get('/cgtn/opinions', require('./routes/cgtn/opinions')); // AppSales router.get('/appsales/:caty?/:time?', require('./routes/appsales/index')); diff --git a/lib/routes/cgtn/opinions.js b/lib/routes/cgtn/opinions.js new file mode 100644 index 0000000000..c10c3216c5 --- /dev/null +++ b/lib/routes/cgtn/opinions.js @@ -0,0 +1,51 @@ +const got = require('@/utils/got'); +const cheerio = require('cheerio'); + +module.exports = async (ctx) => { + const rootUrl = `https://www.cgtn.com/opinions`; + const response = await got({ + method: 'get', + url: rootUrl, + }); + + const $ = cheerio.load(response.data); + + $('.cg-pic').parent().remove(); + + const list = $(`.cg-title h4`) + .slice(0, 15) + .map((_, item) => { + item = $(item); + const a = item.find('a'); + return { + title: a.text(), + link: a.attr('href'), + pubDate: new Date(parseInt(a.attr('data-time'))).toUTCString(), + }; + }) + .get(); + + const items = await Promise.all( + list.map( + async (item) => + await ctx.cache.tryGet(item.link, async () => { + const detailResponse = await got({ + method: 'get', + url: item.link, + }); + const content = cheerio.load(detailResponse.data); + + item.author = content('.news-author-name').text(); + item.description = content('#cmsMainContent').html(); + + return item; + }) + ) + ); + + ctx.state.data = { + title: 'CGTN - Opinions', + link: rootUrl, + item: items, + }; +}; From 56655c77e2cdfeeb0b4418d9caa676223a329f42 Mon Sep 17 00:00:00 2001 From: qixingchen <4182240+Qixingchen@users.noreply.github.com> Date: Sat, 9 Jan 2021 23:59:02 +0800 Subject: [PATCH 41/49] Update live.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修正 哔哩哔哩直播分区不存在导致的文档示例api不可用的问题 --- docs/live.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/live.md b/docs/live.md index f41ff9d02d..84f153c57d 100644 --- a/docs/live.md +++ b/docs/live.md @@ -22,7 +22,7 @@ pageClass: routes ### 直播分区 - + ::: warning 注意 From f954c2515903f02fd935e731c2455dee43c3d018 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Sun, 10 Jan 2021 05:34:24 +0000 Subject: [PATCH 42/49] style: auto format --- docs/program-update.md | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/docs/program-update.md b/docs/program-update.md index 5eb9cf3bd5..a78f659c95 100644 --- a/docs/program-update.md +++ b/docs/program-update.md @@ -316,6 +316,16 @@ pageClass: routes +## simpread + +### 消息通知 + + + +### 更新日志 + + + ## sketch.com ### beta 更新 @@ -439,12 +449,3 @@ pageClass: routes | | -commentCount | -createdAt | createdAt | - -## simpread - -### 消息通知 - - - -### 更新日志 - From e8011374ffdf24b9b2b37342c931d7ed9190c485 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Sun, 10 Jan 2021 05:48:46 +0000 Subject: [PATCH 43/49] style: auto format --- lib/router.js | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/router.js b/lib/router.js index 9d824c12e4..7ea0250b69 100644 --- a/lib/router.js +++ b/lib/router.js @@ -3785,7 +3785,6 @@ router.get('/nace/blog/:sort?', require('./routes/nace/blog')); // Caixin Latest router.get('/caixin/latest', require('./routes/caixin/latest')); - // Media Digest router.get('/mediadigest/:range/:category?', require('./routes/mediadigest/category')); From 12d99d804eb179ad8cae0addbe5603276d163885 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Sun, 10 Jan 2021 06:47:40 +0000 Subject: [PATCH 44/49] style: auto format --- lib/router.js | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/router.js b/lib/router.js index 37bb9f50fd..0056df6949 100644 --- a/lib/router.js +++ b/lib/router.js @@ -3806,5 +3806,4 @@ router.get('/simpread/changelog', require('./routes/simpread/changelog')); // Radio Free Asia router.get('/rfa/:language?/:channel?/:subChannel?', require('./routes/rfa/index')); - module.exports = router; From fa3ac74fbb0cd378a44f3ffb48baa87636d42ff0 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Sun, 10 Jan 2021 06:55:05 +0000 Subject: [PATCH 45/49] style: auto format --- lib/router.js | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/router.js b/lib/router.js index 9efa7ca9ab..0db1c19aae 100644 --- a/lib/router.js +++ b/lib/router.js @@ -3786,7 +3786,6 @@ router.get('/nace/blog/:sort?', require('./routes/nace/blog')); // Caixin Latest router.get('/caixin/latest', require('./routes/caixin/latest')); - // Pincong 品葱 router.get('/pincong/category/:category?/:sort?', require('./routes/pincong/index')); router.get('/pincong/hot/:category?', require('./routes/pincong/hot')); From 7974dd54d6e25d3edb1bf41a96290ab6a2fe859f Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Sun, 10 Jan 2021 06:58:39 +0000 Subject: [PATCH 46/49] style: auto format --- docs/shopping.md | 23 ++++++++++++----------- lib/router.js | 2 -- lib/routes/mcdonalds/news.js | 20 +++++++++++--------- 3 files changed, 23 insertions(+), 22 deletions(-) diff --git a/docs/shopping.md b/docs/shopping.md index caba5c7a34..a058a5241b 100644 --- a/docs/shopping.md +++ b/docs/shopping.md @@ -166,6 +166,18 @@ For instance, in +## 麦当劳 + +### 麦当劳活动资讯 + + + +| 全部分类 | 社会责任 | 人员品牌 | 产品故事 | 优惠 | 品牌文化 | 活动速报 | +| --------- | -------------- | -------- | -------- | ----- | -------- | -------- | +| news_list | responsibility | brand | product | sales | culture | event | + + + ## 缺书网 ### 促销 @@ -347,14 +359,3 @@ For instance, in - -## 麦当劳 - -### 麦当劳活动资讯 - - -| 全部分类 | 社会责任 | 人员品牌 | 产品故事 | 优惠 | 品牌文化 | 活动速报 | -| -------- | ------- | --- | ------- | ------ | ------- | -------- | -| news_list | responsibility | brand | product | sales | culture | event | - - diff --git a/lib/router.js b/lib/router.js index f99dcd25dd..3f293e8317 100644 --- a/lib/router.js +++ b/lib/router.js @@ -3786,7 +3786,6 @@ router.get('/nace/blog/:sort?', require('./routes/nace/blog')); // Caixin Latest router.get('/caixin/latest', require('./routes/caixin/latest')); - // Mcdonalds router.get('/mcdonalds/:category', require('./routes/mcdonalds/news')); @@ -3815,5 +3814,4 @@ router.get('/simpread/changelog', require('./routes/simpread/changelog')); // Radio Free Asia router.get('/rfa/:language?/:channel?/:subChannel?', require('./routes/rfa/index')); - module.exports = router; diff --git a/lib/routes/mcdonalds/news.js b/lib/routes/mcdonalds/news.js index f62e83a77b..f8fc8e3717 100644 --- a/lib/routes/mcdonalds/news.js +++ b/lib/routes/mcdonalds/news.js @@ -4,18 +4,21 @@ const cheerio = require('cheerio'); module.exports = async (ctx) => { const category_param = ctx.params.category || 'news_list'; const categories = category_param.split('+'); - const baseUrl = "https://www.mcdonalds.com.cn/news/"; + const baseUrl = 'https://www.mcdonalds.com.cn/news/'; - const get_news_list = async (cates) => await Promise.all( + const get_news_list = async (cates) => + await Promise.all( cates.map(async (cate) => { const response = await got.get(baseUrl + cate); const $ = cheerio.load(response.data); // console.log($('title').text()); - const news = $('.news_list .box-container > div').slice(0, 10).map((idx, item) => { - item = $(item); - return item.find('a[target]').attr('href'); - }) - .get(); + const news = $('.news_list .box-container > div') + .slice(0, 10) + .map((idx, item) => { + item = $(item); + return item.find('a[target]').attr('href'); + }) + .get(); // console.log(news); return Promise.resolve(news); }) @@ -28,7 +31,7 @@ module.exports = async (ctx) => { const result = await got.get(news_url); const $ = cheerio.load(result.data); -// console.log($('h3 ~ time').text(), news_url); + // console.log($('h3 ~ time').text(), news_url); return { title: $('h3').text(), @@ -47,5 +50,4 @@ module.exports = async (ctx) => { link: baseUrl, item: out, }; - }; From de41c4aff2f146d8385f33d5ed9bbb303c45fcba Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Sun, 10 Jan 2021 07:41:20 +0000 Subject: [PATCH 47/49] style: auto format --- docs/new-media.md | 7 +++---- lib/router.js | 1 - lib/routes/mofish/index.js | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/docs/new-media.md b/docs/new-media.md index eec38865ff..a9b2f164d2 100644 --- a/docs/new-media.md +++ b/docs/new-media.md @@ -2032,6 +2032,9 @@ column 为 third 时可选的 category: +## 鱼塘热榜 + + ## 遠見 @@ -2113,7 +2116,3 @@ QueryString: ### 全文 - -## 鱼塘热榜 - - \ No newline at end of file diff --git a/lib/router.js b/lib/router.js index 9f95ea46b6..75ec51820e 100644 --- a/lib/router.js +++ b/lib/router.js @@ -3817,5 +3817,4 @@ router.get('/simpread/changelog', require('./routes/simpread/changelog')); // Radio Free Asia router.get('/rfa/:language?/:channel?/:subChannel?', require('./routes/rfa/index')); - module.exports = router; diff --git a/lib/routes/mofish/index.js b/lib/routes/mofish/index.js index f99bf3087d..e8890bf1ca 100644 --- a/lib/routes/mofish/index.js +++ b/lib/routes/mofish/index.js @@ -26,4 +26,4 @@ module.exports = async (ctx) => { guid: item.id, })), }; -}; \ No newline at end of file +}; From 26ad38a0da22bff03dcf68b435eeded440307032 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Sun, 10 Jan 2021 07:53:24 +0000 Subject: [PATCH 48/49] style: auto format --- docs/bbs.md | 2 +- lib/router.js | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/bbs.md b/docs/bbs.md index a3eac63265..81779d9bf7 100644 --- a/docs/bbs.md +++ b/docs/bbs.md @@ -188,7 +188,7 @@ pageClass: routes 帖子网址如果为 那么帖子 tid 就是 `1789863`。 -访问水区需要添加环境变量 `SCBOY_BBS_TOKEN`, 详情见部署页面的配置模块。 `SCBOY_BBS_TOKEN`在cookies的`bbs_token`中。 +访问水区需要添加环境变量 `SCBOY_BBS_TOKEN`, 详情见部署页面的配置模块。 `SCBOY_BBS_TOKEN`在 cookies 的`bbs_token`中。 ## V2EX diff --git a/lib/router.js b/lib/router.js index a14f5c4e2c..46d98594be 100644 --- a/lib/router.js +++ b/lib/router.js @@ -3774,7 +3774,6 @@ router.get('/sciencenet/blog/:type?/:time?/:sort?', require('./routes/sciencenet // DailyArt router.get('/dailyart/:language?', require('./routes/dailyart/index')); - // SCBOY router.get('/scboy/thread/:tid', require('./routes/scboy/thread')); @@ -3821,5 +3820,4 @@ router.get('/simpread/changelog', require('./routes/simpread/changelog')); // Radio Free Asia router.get('/rfa/:language?/:channel?/:subChannel?', require('./routes/rfa/index')); - module.exports = router; From 3e83f6927614480a8109c7bdaf72da43e15297fc Mon Sep 17 00:00:00 2001 From: NeverBehave Date: Sun, 10 Jan 2021 03:16:50 -0500 Subject: [PATCH 49/49] docs: add checklist for PR --- .github/PULL_REQUEST_TEMPLATE.md | 39 ++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index d1ffaeb445..f390335ae9 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -11,3 +11,42 @@ Close # To simplify the testing workflow, please include the complete route, with all required and optional parameters, otherwise your pull request will be closed. --> + +## 新RSS检查列表 / New RSS Script Checklist + + + +- [ ] 这是在提交一个新的RSS吗? Is this a new RSS Script? + - **如果不是, 请留空本列表**. **LEAVE BLANK** if it is not a submitting new RSS Script +- [ ] 是否提供了文档? Documentation provided? + - [ ] 是否提供了英文文档? EN Documentation provided? +- [ ] 是否支持全文获取? Is this RSS Script support fulltext? + - [ ] 如果全文获取中需要访问文章链接, 是否使用了缓存? If fulltext requires to fetch detail pages, is cache used in the process? + - [缓存说明](https://docs.rsshub.app/joinus/#ti-jiao-xin-de-rsshub-gui-ze-bian-xie-jiao-ben-shi-yong-huan-cun) | [How to use cache](https://docs.rsshub.app/joinus/#ti-jiao-xin-de-rsshub-gui-ze-bian-xie-jiao-ben-shi-yong-huan-cun) +- [ ] 目标是否有明显的反爬/频率限制? Is there any sign of anti-bot or rate limit? + - [ ] 如果有, 是否有对应的措施? (延长缓存时间, 写文档说明, etc.) If yes, do your code reflect this sign? (e.g. write documentations, use long cache time) +- [ ] 是否引入的新的包? Any new package introduced? + - 如果有, 请说明原因. If yes, please state your reason +- [ ] 是否使用了`Puppeteer`? Make use of `Puppeteer`? + - 如果有, 请说明原因. If yes, please state your reason + + +## 说明 / Note + +