feat(route/WSJ): enrich content and speed up crawling (#12118)

* fix(route): empty content and html style title for the-atlantic

* feat(wsj): enrich content

* chore: speed up crawling

lib/v2/wsj/news.js

@@ -40,7 +40,7 @@ module.exports = async (ctx) => {
         item.test = key;
         return item;
     });
-    const items = await asyncPoolAll(1, list, (item) => parseArticle(item, ctx));
+    const items = await asyncPoolAll(10, list, (item) => parseArticle(item, ctx));
 
     ctx.state.data = {
         title: `WSJ${subTitle}`,

lib/v2/wsj/templates/article-description.art

@@ -0,0 +1,6 @@
+<div>
+    {{@ item.subTitle}}
+    {{@ item.article}}
+
+</div>
+

lib/v2/wsj/utils.js

@@ -2,6 +2,8 @@ const asyncPool = require('tiny-async-pool');
 const cheerio = require('cheerio');
 const got = require('@/utils/got');
 const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
 const UA = require('@/utils/rand-user-agent')({ browser: 'chrome', os: 'android', device: 'mobile' });
 
 // const chromeMobileUserAgent = 'Mozilla/5.0 (Linux; Android 7.0; SM-G892A Build/NRD90M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/67.0.3396.87 Mobile Safari/537.36';
@@ -18,15 +20,6 @@ const parseArticle = (item, ctx) =>
         });
         const html = response.data;
         const $ = cheerio.load(html);
-        const content = $('.articleBody > section');
-
-        // Cover
-        const cover = $('.articleLead > div.is-lead-inset > div.header > .img-header > div.image-container > amp-img > img');
-
-        if (cover.length > 0) {
-            $(`<img src=${cover[0].attribs.content}>`).insertBefore(content[0].childNodes[0]);
-            $(cover).remove();
-        }
 
         // Summary
         const summary = $('head > meta[name="description"]').attr('content');
@@ -41,8 +34,18 @@ const parseArticle = (item, ctx) =>
             .split(',')
             .map((c) => c.trim());
 
+        const article = $('article');
+        item.subTitle = $('h2.sub-head').html();
+
+        // Remove h1 and h2
+        article.find('h1').remove();
+        article.find('h2').remove();
+
+        // Remove podcast
+        article.find('.media-object-podcast').remove();
+
         // Images
-        content.find('amp-img').each((i, e) => {
+        article.find('amp-img').each((i, e) => {
             const img = $(`<img width="${e.attribs.width}" height="${e.attribs.height}" src="${e.attribs.src}" alt="${e.attribs.alt}">`);
 
             // Caption follows, no need to handle caption
@@ -51,7 +54,7 @@ const parseArticle = (item, ctx) =>
         });
 
         // iframes (youtube videos and interactive elements)
-        content.find('amp-iframe').each((i, e) => {
+        article.find('amp-iframe').each((i, e) => {
             const iframe = $(`<iframe width="${e.attribs.width}" height="${e.attribs.height}" src="${e.attribs.src}">`);
             $(iframe).insertBefore(e);
             $(e).remove();
@@ -60,17 +63,21 @@ const parseArticle = (item, ctx) =>
         // Remove unwanted DOMs
         const unwanted_element_selectors = ['amp-ad', '.wsj-ad'];
         unwanted_element_selectors.forEach((selector) => {
-            content.find(selector).each((i, e) => {
+            article.find(selector).each((i, e) => {
                 $(e).remove();
             });
         });
 
         // Paywall
-        content.find('.paywall').each((i, e) => {
+        article.find('.paywall').each((i, e) => {
             // Caption follows, no need to handle caption
             $(e.childNodes).insertBefore(e);
             $(e).remove();
         });
+        item.article = article.html();
+        item.description = art(path.join(__dirname, 'templates/article-description.art'), {
+            item,
+        });
 
         return {
             title: item.title,
@@ -79,7 +86,7 @@ const parseArticle = (item, ctx) =>
             author,
             link: item.link,
             summary,
-            description: content.html(),
+            description: item.description,
             category: categories,
             icon: 'https://s.wsj.net/media/wsj_launcher-icon-4x.png',
             logo: 'https://vir.wsj.net/fp/assets/webpack4/img/wsj-logo-big-black.165e51cc.svg',