opensource/RSSHub
3
0
Fork 0
mirror of https://github.com/DIYgod/RSSHub.git synced 2026-08-31 01:01:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: add addNoReferrer in common-utils (#2571)

Browse Source 
* fix: no-referrer tag

* chore: merge qdaily routes

* feat: add addNoReferrer in common-utils

This adds no-referrer attribute to images.
Parameters:

- add no-referrer attribute to images
- $: cheerio selector
- source: source selector, string
- target: target attribute name, typically for lazyload imgs, string
- srcPrefix: prefix for src, string
- removeAttr: attributes to remove, array: ['attrA','attrB']

* test: add test case

* doc: update
This commit is contained in:
Henry Wang
2019-07-08 04:45:02 +01:00
committed by DIYgod
parent ffcca63d36
commit d09affa703
30 changed files with 127 additions and 308 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/social-media.md
+6 -8
View File
@@ -532,17 +532,15 @@ pageClass: routes
## 好奇心日报
### 分类
### 标签，栏目，分类
<Route author="WenhuWee" example="/qdaily/category/5" path="/qdaily/category/:id" :paramsDesc="['分类 id，可在分类 URL 找到']"/>
<Route author="WenhuWee emdoe SivaGao HenryQW" example="/qdaily/column/59" path="/qdaily/:type/:id" :paramsDesc="['类型，见下表', '对应 id，可在 URL 找到']">
### 栏目
| 标签 | 栏目 | 分类 |
| ---- | ------ | ---------- |
| tag | column | researcach |
<Route author="WenhuWee emdoe" example="/qdaily/column/59" path="/qdaily/column/:id" :paramsDesc="['栏目 id，可在栏目 URL 找到']"/>
### 标签
<Route author="SivaGao" example="/qdaily/tag/29" path="/qdaily/tag/:id" :paramsDesc="['标签 id，可在 tag URL 找到']"/>
</Route>
## 虎扑
lib/router.js
+1 -3
View File
@@ -207,11 +207,9 @@ router.get('/geektime/news', require('./routes/geektime/news'));
router.get('/jiemian/list/:cid', require('./routes/jiemian/list.js'));
// 好奇心日报
router.get('/qdaily/column/:id', require('./routes/qdaily/column'));
router.get('/qdaily/category/:id', require('./routes/qdaily/category'));
router.get('/qdaily/tag/:id', require('./routes/qdaily/tag'));
router.get('/qdaily/notch/posts', require('./routes/qdaily/notch/index'));
router.get('/qdaily/notch/explore/:id', require('./routes/qdaily/notch/explore'));
router.get('/qdaily/:type/:id', require('./routes/qdaily/index'));
// 爱奇艺
router.get('/iqiyi/dongman/:id', require('./routes/iqiyi/dongman'));
lib/routes/21caijing/channel.js
+4 -9
View File
@@ -1,6 +1,7 @@
const got = require('@/utils/got');
const date = require('@/utils/date');
const cheerio = require('cheerio');
const { addNoReferrer } = require('@/utils/common-utils');
module.exports = async (ctx) => {
const channel = ctx.params.name;
@@ -35,15 +36,6 @@ module.exports = async (ctx) => {
const $ = cheerio.load(response.data);
$('img').each((index, elem) => {
const $elem = $(elem);
const src = $elem.attr('data-original');
if (src) {
$elem.attr('src', src);
$elem.removeAttr('data-original');
}
});
const div = $('div.editor');
if (div) {
div.remove();
@@ -51,6 +43,9 @@ module.exports = async (ctx) => {
const title = $('h1').text();
const dateStr = $('div.newsDate').text();
addNoReferrer($, 'div.txtContent img', 'data-original');
const content = $('div.txtContent').html();
const single = {
lib/routes/8btc/author.js
+2 -5
View File
@@ -1,5 +1,6 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const { addNoReferrer } = require('@/utils/common-utils');
module.exports = async (ctx) => {
const authorid = ctx.params.authorid;
@@ -8,11 +9,7 @@ module.exports = async (ctx) => {
const ProcessFeed = (data) => {
const $ = cheerio.load(data);
$('img').each((index, elem) => {
const $elem = $(elem);
$elem.attr('referrerpolicy', 'no-referrer');
});
addNoReferrer($, '.bbt-html');
// 提取内容
return $('.bbt-html').html();
};
lib/routes/archdaily/home.js
+2 -5
View File
@@ -1,6 +1,7 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const parser = require('@/utils/rss-parser');
const { addNoReferrer } = require('@/utils/common-utils');
module.exports = async (ctx) => {
const feed = await parser.parseURL('https://feeds.feedburner.com/ArchdailyCN');
@@ -13,11 +14,7 @@ module.exports = async (ctx) => {
const $ = cheerio.load(response.data);
// 还原图片地址
$('img').each((index, elem) => {
const $elem = $(elem);
$elem.attr('referrerpolicy', 'no-referrer');
});
addNoReferrer($, '#single-content');
// 提取内容
return $('#single-content').html();
lib/routes/bilibili/followings_article.js
+3 -3
View File
@@ -2,6 +2,7 @@ const got = require('@/utils/got');
const cache = require('./cache');
const config = require('@/config');
const cheerio = require('cheerio');
const { addNoReferrer } = require('@/utils/common-utils');
module.exports = async (ctx) => {
const uid = String(ctx.params.uid);
@@ -31,9 +32,8 @@ module.exports = async (ctx) => {
const result = await got.get(link);
const $ = cheerio.load(result.data);
$('img').each(function(i, e) {
$(e).attr('src', $(e).attr('data-src'));
});
addNoReferrer($, '.article-holder', 'data-src');
return $('.article-holder').html();
});
lib/routes/cnbeta/home.js
+2 -10
View File
@@ -1,6 +1,7 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const parser = require('@/utils/rss-parser');
const { addNoReferrer } = require('@/utils/common-utils');
module.exports = async (ctx) => {
const feed = await parser.parseURL('https://rss.cnbeta.com/');
@@ -8,16 +9,7 @@ module.exports = async (ctx) => {
const ProcessFeed = (data) => {
const $ = cheerio.load(data);
// 还原图片地址
$('img').each((index, elem) => {
const $elem = $(elem);
const src = $elem.attr('data-original');
if (src && src !== '') {
$elem.attr('src', `${src}`);
}
$elem.attr('referrerpolicy', 'no-referrer');
});
addNoReferrer($, '.article-content', 'data-original');
// 提取内容
return $('.article-summary p').html() + '<br>' + $('.article-content').html();
};
lib/routes/dockone/weekly.js
+2 -4
View File
@@ -1,5 +1,6 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const { addNoReferrer } = require('@/utils/common-utils');
module.exports = async (ctx) => {
const baseUrl = 'http://weekly.dockone.io';
@@ -18,10 +19,7 @@ module.exports = async (ctx) => {
const ProcessFeed = (data) => {
const $ = cheerio.load(data);
$('img').each((index, elem) => {
const $elem = $(elem);
$elem.attr('referrerpolicy', 'no-referrer');
});
addNoReferrer($, '.aw-question-detail .content');
// 提取内容
return $('.aw-question-detail .content').html();
lib/routes/gcores/category.js
+2 -8
View File
@@ -1,5 +1,6 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const { addNoReferrer } = require('@/utils/common-utils');
const sourceTimezoneOffset = -8;
module.exports = async (ctx) => {
@@ -45,14 +46,7 @@ module.exports = async (ctx) => {
const itemPage = itemRes.data;
$ = cheerio.load(itemPage);
$('img').each((index, elem) => {
const $elem = $(elem);
const src = $elem.attr('data-original');
if (src) {
$elem.attr('src', src);
$elem.removeAttr('data-original');
}
});
addNoReferrer($, '.story', 'data-original');
const content = $('.story').html();
const pageInfo = $('.story_info').text();
lib/routes/indienova/article.js
+2 -4
View File
@@ -1,6 +1,7 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const url = require('url');
const { addNoReferrer } = require('@/utils/common-utils');
module.exports = async (ctx) => {
const type = ctx.params.type;
@@ -30,10 +31,7 @@ module.exports = async (ctx) => {
const $$$ = cheerio.load(resp.data);
$('img').each((index, elem) => {
const $elem = $(elem);
$elem.attr('referrerpolicy', 'no-referrer');
});
addNoReferrer($$$, '.indienova-single-post');
const description = $$$('.indienova-single-post').html();
const single = {
lib/routes/iresearch/report.js
+1 -1
View File
@@ -21,7 +21,7 @@ async function processReport(item, ctx) {
const reportId = data.id;
for (let i = 1; i <= data.PagesCount; i++) {
description += `<img src="http://report.iresearch.cn/rimgs/${reportId}/${i}.jpg" rel="noreferrer"><br>`;
description += `<img src="http://report.iresearch.cn/rimgs/${reportId}/${i}.jpg" rel="no-referrer"><br>`;
}
await ctx.cache.set(apiUrl, description);
lib/routes/jianshu/utils.js
+3 -9
View File
@@ -1,6 +1,7 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const url = require('url');
const { addNoReferrer } = require('@/utils/common-utils');
// 加载文章页
async function load(link) {
@@ -17,15 +18,8 @@ async function load(link) {
const serverOffset = date.getTimezoneOffset() / 60;
const pubDate = new Date(date.getTime() - 60 * 60 * 1000 * (timeZone + serverOffset)).toUTCString();
// 还原图片地址
$('img').each((index, elem) => {
const $elem = $(elem);
const src = $elem.attr('data-original-src');
if (src && src !== '') {
$elem.attr('src', `https:${src}`);
}
$elem.attr('referrerpolicy', 'no-referrer');
});
addNoReferrer($, '.show-content-free', 'data-original-src', 'https:');
// 去除样式
$('.image-container, .image-container-fill').removeAttr('style');
// 处理视频
lib/routes/kkj/news.js
+5 -11
View File
@@ -1,6 +1,7 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const parser = require('@/utils/rss-parser');
const { addNoReferrer } = require('@/utils/common-utils');
module.exports = async (ctx) => {
const feed = await parser.parseURL('http://rss.mydrivers.com/rss.aspx?Tid=1');
@@ -8,19 +9,12 @@ module.exports = async (ctx) => {
const ProcessFeed = (data) => {
const $ = cheerio.load(data);
$('img').each((index, elem) => {
const $elem = $(elem);
const src = $elem.attr('src');
if (src && src !== '') {
$elem.attr('src', `https:${src}`);
}
$elem.attr('referrerpolicy', 'no-referrer');
});
addNoReferrer($, '.news_info', '', 'https:');
// 移除广告等内容
$('.weixin ~ div').remove();
$('.weixin').remove();
$('script').remove();
$('.news_info .weixin ~ div').remove();
$('.news_info .weixin').remove();
$('.news_info script').remove();
// 提取内容
return $('.news_info').html();
lib/routes/oschina/u.js
+1 -1
View File
@@ -41,7 +41,7 @@ module.exports = async (ctx) => {
},
});
const content = cheerio.load(detail.data);
content('.ad-wrap').remove();
content('#articleContent .ad-wrap').remove();
item.description = content('#articleContent').html();
ctx.cache.set(key, item.description);
}
lib/routes/qdaily/category.js
-59
View File
@@ -1,59 +0,0 @@
const cheerio = require('cheerio');
const got = require('@/utils/got');
module.exports = async (ctx) => {
const url = `https://www.qdaily.com/categories/${ctx.params.id}.html`;
const res = await got.get(url);
const $ = cheerio.load(res.data);
const list = $('.article').get();
const proList = [];
const indexList = [];
const out = await Promise.all(
list.map(async (item, i) => {
const $ = cheerio.load(item);
const time = $('.smart-date').attr('data-origindate');
let title = $('.title').text();
if (!title) {
title = $('.smart-dotdotdot').text();
}
const itemUrl = $('a').attr('href');
const allUrl = `https://www.qdaily.com${itemUrl}`;
const cache = await ctx.cache.get(allUrl);
if (cache) {
return Promise.resolve(JSON.parse(cache));
}
const single = {
title,
pubDate: new Date(time).toUTCString(),
link: allUrl,
guid: allUrl,
};
proList.push(got.get(allUrl));
indexList.push(i);
return Promise.resolve(single);
})
);
const responses = await got.all(proList);
for (let i = 0; i < responses.length; i++) {
const res = responses[i];
const $ = cheerio.load(res.data);
$('img').each((index, item) => {
item = $(item);
item.attr('src', item.attr('data-src'));
item.attr('referrerpolicy', 'no-referrer');
});
$('.article-detail-bd .author-share').remove();
$('.article-detail-ft').remove();
out[indexList[i]].description = $('.main .com-article-detail').html();
ctx.cache.set(out[indexList[i]].link, JSON.stringify(out[i]));
}
ctx.state.data = {
title: $('title').text(),
link: url,
item: out,
};
};
lib/routes/qdaily/column.js → lib/routes/qdaily/index.js
+16 -6
View File
@@ -1,8 +1,22 @@
const cheerio = require('cheerio');
const got = require('@/utils/got');
const { addNoReferrer } = require('@/utils/common-utils');
module.exports = async (ctx) => {
const url = `http://www.qdaily.com/special_columns/${ctx.params.id}.html`;
let type;
switch (ctx.params.type) {
case 'tag':
type = 'tags';
break;
case 'category':
type = 'categories';
break;
case 'column':
type = 'special_columns';
break;
}
const url = `http://www.qdaily.com/${type}/${ctx.params.id}.html`;
const res = await got.get(url);
const $ = cheerio.load(res.data);
@@ -48,14 +62,10 @@ module.exports = async (ctx) => {
const res = responses[i];
const $ = cheerio.load(res.data);
$('img').each((index, item) => {
item = $(item);
item.attr('src', item.attr('data-src'));
item.attr('referrerpolicy', 'no-referrer');
});
$('.article-detail-bd .author-share').remove();
$('.article-detail-ft').remove();
addNoReferrer($, '.main .com-article-detail', 'data-src');
if (out[indexList[i]].link.match('/weeklies/')) {
out[indexList[i]].description = $('.packery-container').html();
} else {
lib/routes/qdaily/tag.js
-59
View File
@@ -1,59 +0,0 @@
const cheerio = require('cheerio');
const got = require('@/utils/got');
module.exports = async (ctx) => {
const url = `https://www.qdaily.com/tags/${ctx.params.id}.html`;
const res = await got.get(url);
const $ = cheerio.load(res.data);
const list = $('.article').get();
const proList = [];
const indexList = [];
const out = await Promise.all(
list.map(async (item, i) => {
const $ = cheerio.load(item);
const time = $('.smart-date').attr('data-origindate');
let title = $('.title').text();
if (!title) {
title = $('.smart-dotdotdot').text();
}
const itemUrl = $('a').attr('href');
const allUrl = `https://www.qdaily.com${itemUrl}`;
const cache = await ctx.cache.get(allUrl);
if (cache) {
return Promise.resolve(JSON.parse(cache));
}
const single = {
title,
pubDate: new Date(time).toUTCString(),
link: allUrl,
guid: allUrl,
};
proList.push(got.get(allUrl));
indexList.push(i);
return Promise.resolve(single);
})
);
const responses = await got.all(proList);
for (let i = 0; i < responses.length; i++) {
const res = responses[i];
const $ = cheerio.load(res.data);
$('img').each((index, item) => {
item = $(item);
item.attr('src', item.attr('data-src'));
item.attr('referrerpolicy', 'no-referrer');
});
$('.article-detail-bd .author-share').remove();
$('.article-detail-ft').remove();
out[indexList[i]].description = $('.main .com-article-detail').html();
ctx.cache.set(out[indexList[i]].link, JSON.stringify(out[i]));
}
ctx.state.data = {
title: $('title').text(),
link: url,
item: out,
};
};
lib/routes/sixthtone/news.js
+3 -5
View File
@@ -1,6 +1,7 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const parser = require('@/utils/rss-parser');
const { addNoReferrer } = require('@/utils/common-utils');
module.exports = async (ctx) => {
const feed = await parser.parseURL('https://www.sixthtone.com/rss');
@@ -8,14 +9,11 @@ module.exports = async (ctx) => {
const ProcessFeed = (data) => {
const $ = cheerio.load(data);
$('img').each((index, elem) => {
const $elem = $(elem);
$elem.attr('referrerpolicy', 'no-referrer');
});
// 移除广告等内容
$('[class*="advertise-"]').remove();
addNoReferrer($, '.content');
// 提取内容
return $('.content').html();
};
lib/routes/tencent/wechat/miniprogram/plugins.js
+3 -5
View File
@@ -1,5 +1,6 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const { addNoReferrer } = require('@/utils/common-utils');
module.exports = async (ctx) => {
const link = 'https://developers.weixin.qq.com/community/plugins';
@@ -37,11 +38,8 @@ module.exports = async (ctx) => {
});
const id = 'x-rsshub';
const $ = cheerio.load(`<div id="${id}">${pluginDetail.data.data.Content}</div>`);
$('img').each((index, item) => {
item = $(item);
item.attr('src', item.attr('data-src'));
item.attr('referrerpolicy', 'no-referrer');
});
addNoReferrer($, `#${id}`, 'data-src');
item.description = $(`#${id}`).html();
ctx.cache.set(key, item.description);
lib/routes/universities/bit/cs/utils.js
+2 -8
View File
@@ -1,6 +1,7 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const url = require('url');
const { addNoReferrer } = require('@/utils/common-utils');
// 专门定义一个function用于加载文章内容
async function load(link) {
@@ -30,14 +31,7 @@ async function load(link) {
const pubDate = new Date(date.getTime() - 60 * 60 * 1000 * (timeZone + serverOffset)).toUTCString();
// 还原图片地址, 去除图片onClick事件
$('img').each((index, elem) => {
const $elem = $(elem);
const src = $elem.attr('src');
if (src && src !== '') {
$elem.attr('src', `http://cs.bit.edu.cn/tzgg/${src}`);
}
$elem.removeAttr('onclick');
});
addNoReferrer($, '.wz_art', '', 'http://cs.bit.edu.cn/tzgg/', ['onclick']);
// 提取内容
const description = $('.wz_art').html();
lib/routes/universities/bit/jwc/utils.js
+2 -8
View File
@@ -1,6 +1,7 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const url = require('url');
const { addNoReferrer } = require('@/utils/common-utils');
// 专门定义一个function用于加载文章内容
async function load(link) {
@@ -20,14 +21,7 @@ async function load(link) {
const pubDate = new Date(date.getTime() - 60 * 60 * 1000 * (timeZone + serverOffset)).toUTCString();
// 还原图片地址, 去除图片onClick事件
$('img').each((index, elem) => {
const $elem = $(elem);
const src = $elem.attr('src');
if (src && src !== '') {
$elem.attr('src', `http://jwc.bit.edu.cn/tzgg/${src}`);
}
$elem.removeAttr('onclick');
});
addNoReferrer($, '.aca_article_con', '', 'http://jwc.bit.edu.cn/tzgg/', ['onclick']);
// 提取内容
const description = $('.aca_article_con').html();
lib/routes/universities/buaa/utils.js
+2 -9
View File
@@ -1,6 +1,7 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const url = require('url');
const { addNoReferrer } = require('@/utils/common-utils');
// 完整文章页
async function load(link) {
@@ -17,15 +18,7 @@ async function load(link) {
const serverOffset = date.getTimezoneOffset() / 60;
const pubDate = new Date(date.getTime() - 60 * 60 * 1000 * (timeZone + serverOffset)).toUTCString();
// 还原图片地址
$('img').each((index, elem) => {
const $elem = $(elem);
const src = $elem.attr('src');
if (src && src !== '') {
$elem.attr('src', `https://news.buaa.edu.cn${src}`);
}
});
addNoReferrer($, '.newsleftconbox > #vsb_content', '', 'https://news.buaa.edu.cn');
// 提取内容
const description = $('.newsleftconbox > #vsb_content').html();
lib/routes/universities/bupt/utils.js
+4 -11
View File
@@ -31,7 +31,7 @@ const ProcessFeed = async (list, cache, current, type) =>
$ = cheerio.load(data); // 使用 cheerio 加载返回的 HTML
// 还原图片地址
$('img').each((index, elem) => {
$(`${current.selector.content} img`).each((index, elem) => {
const $elem = $(elem);
const src = $elem.attr('src');
if (src && src !== '') {
@@ -39,7 +39,7 @@ const ProcessFeed = async (list, cache, current, type) =>
}
});
// 还原链接地址
$('a').each((index, elem) => {
$(`${current.selector.content} a`).each((index, elem) => {
const $elem = $(elem);
const src = $elem.attr('href');
if (src && src !== '') {
@@ -47,15 +47,8 @@ const ProcessFeed = async (list, cache, current, type) =>
}
});
// 去除样式
$('img').removeAttr('style');
$('div').removeAttr('style');
$('span').removeAttr('style');
$('p').removeAttr('style');
$('table').removeAttr('style');
$('td').removeAttr('style');
$('tr').removeAttr('style');
$('style').remove();
$('script').remove();
$('img, div, span, p, table, td, tr').removeAttr('style');
$('style, script').remove();
let title = '';
if (type === 'sci') {
lib/routes/universities/shmtu/jwc.js
+1 -29
View File
@@ -7,7 +7,7 @@ async function load(link) {
const response = await got.get(link);
const $ = cheerio.load(response.data);
$('img').each((index, elem) => {
$('.field-item img').each((index, elem) => {
const $elem = $(elem);
const src = $elem.attr('src');
if (src && src !== '') {
@@ -17,34 +17,6 @@ async function load(link) {
$elem.removeAttr('alt');
});
$('div').each((index, elem) => {
const $elem = $(elem);
$elem.replaceWith($elem.html().trim());
});
$('p').each((index, elem) => {
const $elem = $(elem);
$elem.replaceWith($elem.html().trim());
});
$('b').each((index, elem) => {
const $elem = $(elem);
$elem.replaceWith($elem.html().trim());
});
$('u').each((index, elem) => {
const $elem = $(elem);
$elem.replaceWith($elem.html().trim());
});
$('strong').each((index, elem) => {
const $elem = $(elem);
$elem.replaceWith($elem.html().trim());
});
while ($('span').text() !== '') {
$('span').each((index, elem) => {
const $elem = $(elem);
$elem.replaceWith($elem.html().trim());
});
}
const description = $('.field-item').html();
return { description };
lib/routes/universities/sjtu/gs/utils.js
+2 -2
View File
@@ -32,12 +32,12 @@ const ProcessFeed = async (list, caches, host) => {
for (let i = 0; i < responses.length; i++) {
const $ = cheerio.load(responses[i].data);
$('a').each(function() {
$('.ssdd a').each(function() {
if ($(this).attr('href')) {
$(this).attr('href', url.resolve(host, $(this).attr('href')));
}
});
$('img').each(function() {
$('.ssdd img').each(function() {
if ($(this).attr('src')) {
$(this).attr('src', url.resolve(host, $(this).attr('src')));
}
lib/routes/universities/szu/yz/utils.js
+4 -11
View File
@@ -34,7 +34,7 @@ const ProcessFeed = async (list, cache, current) =>
$ = cheerio.load(data); // 使用 cheerio 加载返回的 HTML
// 还原图片地址
$('img').each((index, elem) => {
$(`${current.selector.content} img`).each((index, elem) => {
const $elem = $(elem);
const src = $elem.attr('src');
if (src && src !== '') {
@@ -43,7 +43,7 @@ const ProcessFeed = async (list, cache, current) =>
});
// 还原链接地址
$('a').each((index, elem) => {
$(`${current.selector.content} a`).each((index, elem) => {
const $elem = $(elem);
const src = $elem.attr('href');
if (src && src !== '') {
@@ -52,15 +52,8 @@ const ProcessFeed = async (list, cache, current) =>
});
// 去除样式
$('img').removeAttr('style');
$('div').removeAttr('style');
$('span').removeAttr('style');
$('p').removeAttr('style');
$('table').removeAttr('style');
$('td').removeAttr('style');
$('tr').removeAttr('style');
$('style').remove();
$('script').remove();
$('img, div, span, p, table, td, tr').removeAttr('style');
$('style, script').remove();
const title = $('h2').text();
lib/routes/zaker/source.js
+2 -8
View File
@@ -1,6 +1,7 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const date_util = require('@/utils/date');
const { addNoReferrer } = require('@/utils/common-utils');
module.exports = async (ctx) => {
const type = ctx.params.type || 'source';
@@ -46,14 +47,7 @@ module.exports = async (ctx) => {
const $ = cheerio.load(response.data);
$('img').each((index, elem) => {
const $elem = $(elem);
const src = $elem.attr('data-original');
if (src && src !== '') {
$elem.attr('src', `${src}`);
}
$elem.attr('referrerpolicy', 'no-referrer');
});
addNoReferrer($, 'div.article_content div', 'data-original');
const description = $('div.article_content div').html() || '原文已被删除';
lib/routes/zreading/home.js
+4 -7
View File
@@ -1,6 +1,7 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const parser = require('@/utils/rss-parser');
const { addNoReferrer } = require('@/utils/common-utils');
module.exports = async (ctx) => {
const feed = await parser.parseURL('http://www.zreading.cn/feed');
@@ -8,14 +9,10 @@ module.exports = async (ctx) => {
const ProcessFeed = (data) => {
const $ = cheerio.load(data);
$('script').remove();
$('.adsbygoogle').remove();
$('.grap script').remove();
$('.grap .adsbygoogle').remove();
// 还原图片地址
$('img').each((index, elem) => {
const $elem = $(elem);
$elem.attr('referrerpolicy', 'no-referrer');
});
addNoReferrer($, '.grap');
// 提取内容
return $('.grap').html();
lib/utils/common-utils.js
+34
View File
@@ -6,6 +6,40 @@ const toTitleCase = (str) =>
.map((word) => word.replace(word[0], word[0].toUpperCase()))
.join(' ');
// add no-referrer attribute to images
// $: cheerio selector
// source: source selector, string
// target: target attribute name, typically for lazyload imgs, string
// srcPrefix: prefix for src, string
// removeAttr: attributes to remove, array: ['attrA','attrA']
const addNoReferrer = ($, source, target, srcPrefix, removeAttr) => {
$(`${source} img`).each((index, e) => {
const $e = $(e);
let src = target ? $e.attr(target) : $e.attr('src');
if (src) {
if (srcPrefix) {
src = srcPrefix + src;
}
if (target) {
$e.removeAttr(target);
}
if (removeAttr) {
removeAttr.forEach((e) => {
$e.removeAttr(e);
});
}
$e.attr('src', src);
$e.attr('referrerpolicy', 'no-referrer');
}
});
};
module.exports = {
toTitleCase,
addNoReferrer,
};
test/utils/common-utils.js
+12
View File
@@ -1,7 +1,19 @@
const utils = require('../../lib/utils/common-utils');
const cheerio = require('cheerio');
describe('common-utils', () => {
it('toTitleCase', async () => {
expect(utils.toTitleCase('RSSHub IS AS aweSOme aS henry')).toBe('Rsshub Is As Awesome As Henry');
});
it('addNoReferrer', async () => {
const data = "<div><img data-src='user-images.githubusercontent.com/5896343/60762836-02746d80-a060-11e9-8947-76b3b3d3b731.png' remove='remove' remove2='remove2'></div>";
const $ = cheerio.load(data);
utils.addNoReferrer($, 'div', 'data-src', 'https://', ['remove', 'remove2']);
const result = $('div').html();
expect(result).toBe('<img src="https://user-images.githubusercontent.com/5896343/60762836-02746d80-a060-11e9-8947-76b3b3d3b731.png" referrerpolicy="no-referrer">');
});
});