feat(route): add LinkedIn company posts (#18799)

* feat(route): add linkedin company posts - basic setup path: /linkedin/company/google/posts * feat(route): linkedIn company post returns realworld data * fix: InvalidDate issue for pubDate * refactor: to using arrow function for handler * feat: add feed description * chore: update parameter company_id description remove trailing white space * perf: add await to puppeteer page.close() * perf: close puppeteer browser missed this step in previous commits * perf: load response in cheerio only once
2026-08-30 16:55:08 +08:00 · 2025-04-09 14:23:54 +01:00
parent eb5e4c609b
commit facc6d543c
2 changed files with 139 additions and 1 deletions
@@ -0,0 +1,64 @@
+import { Route } from '@/types';
+import puppeteer from '@/utils/puppeteer';
+import { load } from 'cheerio';
+import { parseCompanyName, parseCompanyPosts, BASE_URL } from './utils';
+import logger from '@/utils/logger';
+
+export const route: Route = {
+    path: '/company/:company_id/posts',
+    categories: ['social-media'],
+    example: '/linkedin/company/google/posts',
+    parameters: { company_id: "Company's LinkedIn profile ID" },
+    description: "Get company's LinkedIn posts by company ID",
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportRadar: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    name: 'Company Posts',
+    maintainers: ['saifazmi'],
+    handler: async (ctx) => {
+        const company_id = ctx.req.param('company_id');
+
+        // Puppeteer setup
+        const browser = await puppeteer();
+        const page = await browser.newPage();
+        await page.setRequestInterception(true);
+
+        page.on('request', (request) => {
+            request.resourceType() === 'document' ? request.continue() : request.abort();
+        });
+
+        const url = new URL(`${BASE_URL}/company/${company_id}`);
+
+        logger.http(`Requesting ${url.href}`);
+        await page.goto(url.href, {
+            waitUntil: 'domcontentloaded',
+        });
+
+        const response = await page.content();
+        await page.close();
+
+        const $ = load(response);
+        const companyName = parseCompanyName($);
+        const posts = parseCompanyPosts($);
+
+        await browser.close();
+
+        return {
+            title: `LinkedIn - ${companyName}'s Posts`,
+            link: url.href,
+            description: `This feed gets ${companyName}'s posts from LinkedIn`,
+            item: posts.map((post) => ({
+                title: post.text,
+                description: post.text,
+                link: post.link,
+                pubDate: post.date,
+            })),
+        };
+    },
+};
@@ -1,9 +1,12 @@
 import { load } from 'cheerio';
 import { Job } from './models';
+import dayjs from 'dayjs';

 /**
 * Constants
 */
+const BASE_URL = 'https://www.linkedin.com';
+
 const KEYWORDS_QUERY_KEY = 'keywords';

 const JOB_TYPES_QUERY_KEY = 'f_JT';
@@ -123,4 +126,75 @@ const parseRouteParam = (searchParam: string | null): string => {
    return encodeURIComponent(searchParam.split(',').join('-'));
 };

-export { parseParamsToSearchParams, parseParamsToString, parseJobDetail, parseJobSearch, parseRouteParam, JOB_TYPES, JOB_TYPES_QUERY_KEY, EXP_LEVELS, EXP_LEVELS_QUERY_KEY, KEYWORDS_QUERY_KEY };
+/**
+ * Parse company profile page for posts
+ * Example page: https://www.linkedin.com/company/google/
+ *
+ * @param {Cheerio} $ HTML string of company profile page
+ * @returns {Array<JSON>} Array of company posts
+ */
+function parseCompanyPosts($) {
+    const posts = $('ul.updates__list > li')
+        .toArray() // Convert the Cheerio object to a plain array
+        .map((elem) => {
+            const elemHtml = $(elem);
+            const link = elemHtml.find('a.main-feed-card__overlay-link').attr('href');
+            const text = elemHtml.find('p.attributed-text-segment-list__content').text().trim();
+            const date = parseRelativeShorthandDate(elemHtml.find('time').text().trim());
+
+            return { link, text, date };
+        });
+
+    return posts;
+}
+
+/**
+ * Parse company profile page for its name
+ * Example page: https://www.linkedin.com/company/google/
+ *
+ * @param {Cheerio} $ HTML string of company profile page
+ * @returns {String} Company name
+ */
+function parseCompanyName($) {
+    return $('h1.top-card-layout__title').text().trim();
+}
+
+/**
+ * Parse relative date shorthand string into a Date object
+ *
+ * @param {String} shorthand The shorthand string representing the date
+ * @returns {Date|null} The parsed date or null if the format is invalid
+ */
+function parseRelativeShorthandDate(shorthand) {
+    const match = shorthand.match(/^(\d+)([wdmyh])$/);
+    if (!match) {
+        return null;
+    }
+
+    const [, amount, unit] = match;
+    const unitMap = {
+        w: 'week',
+        d: 'day',
+        m: 'month',
+        y: 'year',
+        h: 'hour',
+    };
+
+    return dayjs().subtract(Number.parseInt(amount), unitMap[unit]);
+}
+
+export {
+    parseCompanyPosts,
+    parseCompanyName,
+    parseParamsToSearchParams,
+    parseParamsToString,
+    parseJobDetail,
+    parseJobSearch,
+    parseRouteParam,
+    BASE_URL,
+    JOB_TYPES,
+    JOB_TYPES_QUERY_KEY,
+    EXP_LEVELS,
+    EXP_LEVELS_QUERY_KEY,
+    KEYWORDS_QUERY_KEY,
+};