diff --git a/lib/v2/wsj/news.js b/lib/v2/wsj/news.js index 2f06c536fc..8518ed9429 100644 --- a/lib/v2/wsj/news.js +++ b/lib/v2/wsj/news.js @@ -40,7 +40,7 @@ module.exports = async (ctx) => { item.test = key; return item; }); - const items = await asyncPoolAll(1, list, (item) => parseArticle(item, ctx)); + const items = await asyncPoolAll(10, list, (item) => parseArticle(item, ctx)); ctx.state.data = { title: `WSJ${subTitle}`, diff --git a/lib/v2/wsj/templates/article-description.art b/lib/v2/wsj/templates/article-description.art new file mode 100644 index 0000000000..53569ee56d --- /dev/null +++ b/lib/v2/wsj/templates/article-description.art @@ -0,0 +1,6 @@ +
+ {{@ item.subTitle}} + {{@ item.article}} + +
+ diff --git a/lib/v2/wsj/utils.js b/lib/v2/wsj/utils.js index f1a31aef21..266d61d686 100644 --- a/lib/v2/wsj/utils.js +++ b/lib/v2/wsj/utils.js @@ -2,6 +2,8 @@ const asyncPool = require('tiny-async-pool'); const cheerio = require('cheerio'); const got = require('@/utils/got'); const { parseDate } = require('@/utils/parse-date'); +const { art } = require('@/utils/render'); +const path = require('path'); const UA = require('@/utils/rand-user-agent')({ browser: 'chrome', os: 'android', device: 'mobile' }); // const chromeMobileUserAgent = 'Mozilla/5.0 (Linux; Android 7.0; SM-G892A Build/NRD90M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/67.0.3396.87 Mobile Safari/537.36'; @@ -18,15 +20,6 @@ const parseArticle = (item, ctx) => }); const html = response.data; const $ = cheerio.load(html); - const content = $('.articleBody > section'); - - // Cover - const cover = $('.articleLead > div.is-lead-inset > div.header > .img-header > div.image-container > amp-img > img'); - - if (cover.length > 0) { - $(``).insertBefore(content[0].childNodes[0]); - $(cover).remove(); - } // Summary const summary = $('head > meta[name="description"]').attr('content'); @@ -41,8 +34,18 @@ const parseArticle = (item, ctx) => .split(',') .map((c) => c.trim()); + const article = $('article'); + item.subTitle = $('h2.sub-head').html(); + + // Remove h1 and h2 + article.find('h1').remove(); + article.find('h2').remove(); + + // Remove podcast + article.find('.media-object-podcast').remove(); + // Images - content.find('amp-img').each((i, e) => { + article.find('amp-img').each((i, e) => { const img = $(`${e.attribs.alt}`); // Caption follows, no need to handle caption @@ -51,7 +54,7 @@ const parseArticle = (item, ctx) => }); // iframes (youtube videos and interactive elements) - content.find('amp-iframe').each((i, e) => { + article.find('amp-iframe').each((i, e) => { const iframe = $(`