mirror of
https://github.com/aiclientproxy/proxycast.git
synced 2026-09-24 23:10:56 +08:00
feat: ship lime 1.6.1 rollout
This commit is contained in:
@@ -218,6 +218,12 @@
|
||||
sleep(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
},
|
||||
resolveRoot(value) {
|
||||
if (value && typeof value === "object" && "nodeType" in value) {
|
||||
return value;
|
||||
}
|
||||
return document.body;
|
||||
},
|
||||
text(value) {
|
||||
return String(value?.textContent || "")
|
||||
.replace(/\s+/g, " ")
|
||||
@@ -252,6 +258,212 @@
|
||||
}
|
||||
return null;
|
||||
},
|
||||
async waitForDomStable(options = {}) {
|
||||
const root = siteAdapterHelpers.resolveRoot(options.root);
|
||||
const stableMs = Math.max(100, Number(options.stableMs) || 500);
|
||||
const timeoutMs = Math.max(stableMs, Number(options.timeoutMs) || 2500);
|
||||
|
||||
if (!root || typeof MutationObserver === "undefined") {
|
||||
await siteAdapterHelpers.sleep(Math.min(timeoutMs, stableMs));
|
||||
return true;
|
||||
}
|
||||
|
||||
return await new Promise((resolve) => {
|
||||
let finished = false;
|
||||
let settleTimer = null;
|
||||
let timeoutTimer = null;
|
||||
|
||||
const finish = () => {
|
||||
if (finished) {
|
||||
return;
|
||||
}
|
||||
finished = true;
|
||||
if (settleTimer) clearTimeout(settleTimer);
|
||||
if (timeoutTimer) clearTimeout(timeoutTimer);
|
||||
observer.disconnect();
|
||||
resolve(true);
|
||||
};
|
||||
|
||||
const schedule = () => {
|
||||
if (settleTimer) clearTimeout(settleTimer);
|
||||
settleTimer = setTimeout(finish, stableMs);
|
||||
};
|
||||
|
||||
const observer = new MutationObserver(() => {
|
||||
schedule();
|
||||
});
|
||||
|
||||
observer.observe(root, {
|
||||
childList: true,
|
||||
subtree: true,
|
||||
attributes: true,
|
||||
characterData: true,
|
||||
});
|
||||
|
||||
timeoutTimer = setTimeout(finish, timeoutMs);
|
||||
schedule();
|
||||
});
|
||||
},
|
||||
async scrollUntilSettled(options = {}) {
|
||||
const root = siteAdapterHelpers.resolveRoot(options.root);
|
||||
const maxScrolls = Math.max(1, Number(options.maxScrolls) || 8);
|
||||
const delayMs = Math.max(80, Number(options.delayMs) || 350);
|
||||
const settleRounds = Math.max(1, Number(options.settleRounds) || 2);
|
||||
const viewportFactor = Math.max(
|
||||
0.2,
|
||||
Number(options.viewportFactor) || 0.9,
|
||||
);
|
||||
const getSnapshot =
|
||||
typeof options.getSnapshot === "function" ? options.getSnapshot : () => ({});
|
||||
const scrollingElement =
|
||||
document.scrollingElement || document.documentElement || document.body;
|
||||
let lastSignature = "";
|
||||
let lastScrollY = Math.round(window.scrollY);
|
||||
let stableCount = 0;
|
||||
let scrolls = 0;
|
||||
|
||||
while (scrolls < maxScrolls) {
|
||||
const stepPx = Math.max(
|
||||
320,
|
||||
Number(options.stepPx) || Math.round(window.innerHeight * viewportFactor),
|
||||
);
|
||||
window.scrollBy(0, stepPx);
|
||||
await siteAdapterHelpers.sleep(delayMs);
|
||||
await siteAdapterHelpers.waitForDomStable({
|
||||
root,
|
||||
stableMs: Math.min(600, delayMs),
|
||||
timeoutMs: Math.max(1200, delayMs * 4),
|
||||
});
|
||||
|
||||
let snapshot = {};
|
||||
try {
|
||||
snapshot = getSnapshot() || {};
|
||||
} catch {
|
||||
snapshot = {};
|
||||
}
|
||||
|
||||
const signature = JSON.stringify({
|
||||
scrollY: Math.round(window.scrollY),
|
||||
scrollHeight: Math.round(scrollingElement?.scrollHeight || 0),
|
||||
...snapshot,
|
||||
});
|
||||
|
||||
if (
|
||||
signature === lastSignature ||
|
||||
Math.round(window.scrollY) === lastScrollY
|
||||
) {
|
||||
stableCount += 1;
|
||||
} else {
|
||||
stableCount = 0;
|
||||
}
|
||||
|
||||
lastSignature = signature;
|
||||
lastScrollY = Math.round(window.scrollY);
|
||||
scrolls += 1;
|
||||
|
||||
if (stableCount >= settleRounds) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
scrolls,
|
||||
scrollY: Math.round(window.scrollY),
|
||||
scrollHeight: Math.round(scrollingElement?.scrollHeight || 0),
|
||||
stableCount,
|
||||
};
|
||||
},
|
||||
async waitForImagesReady(rootOrOptions, maybeOptions = {}) {
|
||||
const root =
|
||||
rootOrOptions &&
|
||||
typeof rootOrOptions === "object" &&
|
||||
"nodeType" in rootOrOptions
|
||||
? rootOrOptions
|
||||
: siteAdapterHelpers.resolveRoot(maybeOptions.root);
|
||||
const options =
|
||||
rootOrOptions &&
|
||||
typeof rootOrOptions === "object" &&
|
||||
"nodeType" in rootOrOptions
|
||||
? maybeOptions
|
||||
: rootOrOptions || {};
|
||||
const timeoutMs = Math.max(200, Number(options.timeoutMs) || 3000);
|
||||
const intervalMs = Math.max(80, Number(options.intervalMs) || 200);
|
||||
const stableRounds = Math.max(1, Number(options.stableRounds) || 2);
|
||||
|
||||
const collectCandidates = () => {
|
||||
const target = siteAdapterHelpers.resolveRoot(root);
|
||||
const entries = [];
|
||||
const seen = new Set();
|
||||
const push = (element) => {
|
||||
if (!element || seen.has(element)) {
|
||||
return;
|
||||
}
|
||||
seen.add(element);
|
||||
entries.push(element);
|
||||
};
|
||||
|
||||
if (target?.matches?.("img, [role='img'], [style*='background-image']")) {
|
||||
push(target);
|
||||
}
|
||||
target
|
||||
?.querySelectorAll?.("img, [role='img'], [style*='background-image']")
|
||||
?.forEach?.((element) => push(element));
|
||||
|
||||
const details = entries.map((element) => {
|
||||
const inlineStyle = element.getAttribute?.("style") || "";
|
||||
const computedStyle =
|
||||
typeof getComputedStyle === "function"
|
||||
? getComputedStyle(element).backgroundImage || ""
|
||||
: "";
|
||||
const bgMatch = String(`${inlineStyle};${computedStyle}`).match(
|
||||
/url\((['"]?)(.*?)\1\)/,
|
||||
);
|
||||
const source =
|
||||
element.currentSrc ||
|
||||
element.getAttribute?.("src") ||
|
||||
element.getAttribute?.("data-src") ||
|
||||
element.getAttribute?.("data-image-url") ||
|
||||
element.getAttribute?.("srcset") ||
|
||||
bgMatch?.[2] ||
|
||||
"";
|
||||
return String(source || "").trim();
|
||||
});
|
||||
|
||||
return {
|
||||
total: details.length,
|
||||
ready: details.filter(Boolean).length,
|
||||
};
|
||||
};
|
||||
|
||||
const startedAt = Date.now();
|
||||
let previousSignature = "";
|
||||
let stableCount = 0;
|
||||
let lastSnapshot = collectCandidates();
|
||||
|
||||
while (Date.now() - startedAt < timeoutMs) {
|
||||
lastSnapshot = collectCandidates();
|
||||
const signature = JSON.stringify(lastSnapshot);
|
||||
|
||||
if (lastSnapshot.total > 0 && lastSnapshot.ready >= lastSnapshot.total) {
|
||||
return lastSnapshot;
|
||||
}
|
||||
|
||||
if (signature === previousSignature) {
|
||||
stableCount += 1;
|
||||
} else {
|
||||
stableCount = 0;
|
||||
}
|
||||
previousSignature = signature;
|
||||
|
||||
if (stableCount >= stableRounds) {
|
||||
return lastSnapshot;
|
||||
}
|
||||
|
||||
await siteAdapterHelpers.sleep(intervalMs);
|
||||
}
|
||||
|
||||
return lastSnapshot;
|
||||
},
|
||||
take(items, limit) {
|
||||
return items.slice(0, Math.max(1, limit));
|
||||
},
|
||||
|
||||
@@ -512,11 +512,40 @@
|
||||
"x/article-export": (async (args, helpers) => {
|
||||
const ARTICLE_ROOT_SELECTOR = '[data-testid="twitterArticleReadView"]';
|
||||
const ARTICLE_TITLE_SELECTOR = '[data-testid="twitter-article-title"]';
|
||||
const ARTICLE_CONTENT_SELECTOR =
|
||||
'[data-testid="longformRichTextComponent"] [data-contents="true"], [data-testid="longformRichTextComponent"]';
|
||||
const ARTICLE_CONTENT_SELECTORS = [
|
||||
'[data-testid="twitterArticleRichTextView"]',
|
||||
'[data-testid="longformRichTextComponent"]',
|
||||
];
|
||||
const ARTICLE_CONTENT_SELECTOR = ARTICLE_CONTENT_SELECTORS.join(", ");
|
||||
const IMAGE_SELECTOR = '[data-testid="tweetPhoto"] img';
|
||||
const CODE_BLOCK_SELECTOR =
|
||||
'[data-testid="markdown-code-block"], [data-testid="prism-code-block"], pre';
|
||||
const IMAGE_BLOCK_SELECTOR =
|
||||
'[data-testid="tweetPhoto"], figure, [role="img"], [style*="background-image"]';
|
||||
const CODE_BLOCK_SELECTOR = [
|
||||
'[data-testid="markdown-code-block"]',
|
||||
'[data-testid="prism-code-block"]',
|
||||
'[data-testid*="code"]',
|
||||
'[data-testid*="Code"]',
|
||||
'.prism-code',
|
||||
'.react-syntax-highlighter-line-number',
|
||||
'[class*="syntax-highlighter"]',
|
||||
'[class*="code-block"]',
|
||||
'[class*="CodeBlock"]',
|
||||
'[class*="language-"]',
|
||||
'pre',
|
||||
].join(", ");
|
||||
const IGNORED_BLOCK_SELECTOR = [
|
||||
ARTICLE_TITLE_SELECTOR,
|
||||
"script",
|
||||
"style",
|
||||
"noscript",
|
||||
"svg",
|
||||
"button",
|
||||
"[role='button']",
|
||||
"time",
|
||||
"[data-testid='UserAvatar-Container']",
|
||||
"[data-testid='User-Name']",
|
||||
"[data-testid='socialContext']",
|
||||
].join(", ");
|
||||
|
||||
function normalizeText(value) {
|
||||
return String(value || "")
|
||||
@@ -534,8 +563,17 @@
|
||||
.trim();
|
||||
}
|
||||
|
||||
function normalizeCodeText(value) {
|
||||
return String(value || "")
|
||||
.replace(/\r\n/g, "\n")
|
||||
.replace(/\u00a0/g, " ")
|
||||
.replace(/\n{3,}/g, "\n\n")
|
||||
.replace(/^\n+/, "")
|
||||
.replace(/\n+$/, "");
|
||||
}
|
||||
|
||||
function escapeMarkdownText(value) {
|
||||
return String(value || "").replace(/([\\`*_{}\[\]()#+\-.!|>])/g, "\\$1");
|
||||
return String(value || "").replace(/([\\`*_{}\[\]()#+\-!|>])/g, "\\$1");
|
||||
}
|
||||
|
||||
function normalizeArticleUrl(rawUrl) {
|
||||
@@ -569,6 +607,9 @@
|
||||
}
|
||||
|
||||
function normalizeImageUrl(rawUrl) {
|
||||
if (!normalizeText(rawUrl)) {
|
||||
return "";
|
||||
}
|
||||
try {
|
||||
const url = new URL(rawUrl, location.href);
|
||||
if (url.hostname.includes("pbs.twimg.com")) {
|
||||
@@ -601,11 +642,19 @@
|
||||
const pictureSources = picture
|
||||
? Array.from(picture.querySelectorAll("source"))
|
||||
: [];
|
||||
const candidate = [
|
||||
imageElement.currentSrc,
|
||||
const explicitSourceCandidates = [
|
||||
imageElement.getAttribute("src"),
|
||||
imageElement.getAttribute("data-src"),
|
||||
imageElement.getAttribute("data-image-url"),
|
||||
imageElement.getAttribute("srcset"),
|
||||
...pictureSources.map((source) => source.getAttribute("srcset")),
|
||||
].filter((value) => normalizeText(value));
|
||||
const candidate = [
|
||||
explicitSourceCandidates.length > 0 ||
|
||||
normalizeText(imageElement.currentSrc) !== location.href
|
||||
? imageElement.currentSrc
|
||||
: "",
|
||||
...explicitSourceCandidates,
|
||||
resolveSrcsetUrl(imageElement.getAttribute("srcset")),
|
||||
...pictureSources.map((source) =>
|
||||
resolveSrcsetUrl(source.getAttribute("srcset")),
|
||||
@@ -659,8 +708,7 @@
|
||||
};
|
||||
}
|
||||
|
||||
function registerImage(state, imageElement) {
|
||||
const rawUrl = resolveImageSource(imageElement);
|
||||
function registerImageUrl(state, rawUrl, altText, suggestedFileName) {
|
||||
const normalizedUrl = normalizeImageUrl(rawUrl);
|
||||
if (!normalizedUrl) {
|
||||
return "";
|
||||
@@ -670,16 +718,319 @@
|
||||
state.imageUrls.add(normalizedUrl);
|
||||
state.images.push({
|
||||
url: normalizedUrl,
|
||||
alt: normalizeText(imageElement.getAttribute("alt")) || undefined,
|
||||
suggested_file_name: resolveSuggestedImageName(
|
||||
normalizedUrl,
|
||||
state.images.length,
|
||||
),
|
||||
alt: normalizeText(altText) || undefined,
|
||||
suggested_file_name:
|
||||
normalizeText(suggestedFileName) ||
|
||||
resolveSuggestedImageName(normalizedUrl, state.images.length),
|
||||
});
|
||||
}
|
||||
|
||||
const altText = normalizeText(imageElement.getAttribute("alt")) || "插图";
|
||||
return ``;
|
||||
const markdownAltText = normalizeText(altText) || "插图";
|
||||
return ``;
|
||||
}
|
||||
|
||||
function registerImage(state, imageElement) {
|
||||
const rawUrl = resolveImageSource(imageElement);
|
||||
return registerImageUrl(
|
||||
state,
|
||||
rawUrl,
|
||||
imageElement.getAttribute("alt"),
|
||||
undefined,
|
||||
);
|
||||
}
|
||||
|
||||
function extractBackgroundImageUrl(value) {
|
||||
const matches = Array.from(
|
||||
String(value || "").matchAll(/url\((['"]?)(.*?)\1\)/g),
|
||||
);
|
||||
for (const match of matches) {
|
||||
const candidate = normalizeImageUrl(match[2]);
|
||||
if (candidate) {
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
function resolveBackgroundImageSource(element) {
|
||||
const inlineStyle = element.getAttribute("style") || "";
|
||||
const inlineUrl = extractBackgroundImageUrl(inlineStyle);
|
||||
if (inlineUrl) {
|
||||
return inlineUrl;
|
||||
}
|
||||
|
||||
if (typeof getComputedStyle === "function") {
|
||||
const computedUrl = extractBackgroundImageUrl(
|
||||
getComputedStyle(element).backgroundImage,
|
||||
);
|
||||
if (computedUrl) {
|
||||
return computedUrl;
|
||||
}
|
||||
}
|
||||
|
||||
return "";
|
||||
}
|
||||
|
||||
function resolveElementAltText(element) {
|
||||
return (
|
||||
normalizeText(element.getAttribute("aria-label")) ||
|
||||
normalizeText(element.getAttribute("title")) ||
|
||||
normalizeText(element.getAttribute("data-alt")) ||
|
||||
normalizeText(element.getAttribute("alt")) ||
|
||||
""
|
||||
);
|
||||
}
|
||||
|
||||
function resolveElementDimension(element, property) {
|
||||
const rect =
|
||||
typeof element.getBoundingClientRect === "function"
|
||||
? element.getBoundingClientRect()
|
||||
: null;
|
||||
const rectValue = property === "width" ? rect?.width : rect?.height;
|
||||
if (Number.isFinite(rectValue) && rectValue > 0) {
|
||||
return rectValue;
|
||||
}
|
||||
|
||||
const attributeValue = Number(element.getAttribute(property));
|
||||
if (Number.isFinite(attributeValue) && attributeValue > 0) {
|
||||
return attributeValue;
|
||||
}
|
||||
|
||||
const styleValue = Number.parseFloat(
|
||||
property === "width" ? element.style?.width : element.style?.height,
|
||||
);
|
||||
if (Number.isFinite(styleValue) && styleValue > 0) {
|
||||
return styleValue;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
function isLikelyContentImageElement(element) {
|
||||
const width = resolveElementDimension(element, "width");
|
||||
const height = resolveElementDimension(element, "height");
|
||||
if (width === 0 && height === 0) {
|
||||
return true;
|
||||
}
|
||||
return width >= 48 && height >= 48;
|
||||
}
|
||||
|
||||
function registerBackgroundImage(state, element) {
|
||||
const rawUrl = resolveBackgroundImageSource(element);
|
||||
if (!rawUrl || !isLikelyContentImageElement(element)) {
|
||||
return "";
|
||||
}
|
||||
|
||||
return registerImageUrl(
|
||||
state,
|
||||
rawUrl,
|
||||
resolveElementAltText(element),
|
||||
undefined,
|
||||
);
|
||||
}
|
||||
|
||||
function collectImageMarkdown(element, state) {
|
||||
const markdown = [];
|
||||
const seenUrls = new Set();
|
||||
|
||||
const pushMarkdown = (value) => {
|
||||
if (!value) {
|
||||
return;
|
||||
}
|
||||
const urlMatch = value.match(/\]\((.+)\)$/);
|
||||
const dedupeKey = urlMatch?.[1] || value;
|
||||
if (seenUrls.has(dedupeKey)) {
|
||||
return;
|
||||
}
|
||||
seenUrls.add(dedupeKey);
|
||||
markdown.push(value);
|
||||
};
|
||||
|
||||
if (element.tagName === "IMG") {
|
||||
pushMarkdown(registerImage(state, element));
|
||||
}
|
||||
|
||||
Array.from(element.querySelectorAll("img")).forEach((image) => {
|
||||
pushMarkdown(registerImage(state, image));
|
||||
});
|
||||
|
||||
const backgroundCandidates = [
|
||||
element,
|
||||
...Array.from(
|
||||
element.querySelectorAll("[role='img'], [style*='background-image']"),
|
||||
),
|
||||
];
|
||||
backgroundCandidates.forEach((candidate) => {
|
||||
pushMarkdown(registerBackgroundImage(state, candidate));
|
||||
});
|
||||
|
||||
return markdown;
|
||||
}
|
||||
|
||||
function hasNestedSerializableContent(element) {
|
||||
if (!(element instanceof Element)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return Boolean(
|
||||
element.querySelector(
|
||||
[
|
||||
"img",
|
||||
"[role='img']",
|
||||
"[style*='background-image']",
|
||||
"pre",
|
||||
"[data-language]",
|
||||
"[data-testid*='code']",
|
||||
"[data-testid*='Code']",
|
||||
".prism-code",
|
||||
"[class*='syntax-highlighter']",
|
||||
"[class*='code-block']",
|
||||
"[class*='CodeBlock']",
|
||||
"[class*='language-']",
|
||||
].join(", "),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
function shouldIgnoreElement(element) {
|
||||
if (!(element instanceof Element)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (
|
||||
element.matches?.(ARTICLE_CONTENT_SELECTOR) ||
|
||||
element.matches?.(IMAGE_SELECTOR) ||
|
||||
element.matches?.(IMAGE_BLOCK_SELECTOR) ||
|
||||
element.matches?.(CODE_BLOCK_SELECTOR) ||
|
||||
element.tagName === "IMG" ||
|
||||
hasNestedSerializableContent(element)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return element.matches(IGNORED_BLOCK_SELECTOR);
|
||||
}
|
||||
|
||||
function countCodeLikeLines(lines) {
|
||||
return lines.filter((line) =>
|
||||
/^\s*(#\s*[\w./-]+|---$|name:|description:|metadata:|Step \d+:|class |def |function |const |let |var |async |await |if |for |while |return |import |from |export |\{|\}|\[|\]|<\/?[a-z])/i.test(
|
||||
line,
|
||||
),
|
||||
).length;
|
||||
}
|
||||
|
||||
function looksLikeCodeBlock(element) {
|
||||
if (!(element instanceof Element)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (element.matches(CODE_BLOCK_SELECTOR)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const dataTestId = normalizeText(element.getAttribute("data-testid"));
|
||||
if (/code|syntax|prism/i.test(dataTestId)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const className =
|
||||
typeof element.className === "string"
|
||||
? element.className
|
||||
: element.getAttribute("class") || "";
|
||||
if (/prism|syntax-highlighter|code-block|CodeBlock|highlight/i.test(className)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (element.querySelector("pre, code, [data-language], [class*='language-']")) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const rawText = normalizeCodeText(element.innerText || element.textContent || "");
|
||||
if (!rawText) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const lines = rawText.split("\n").filter((line) => line.trim().length > 0);
|
||||
if (lines.length < 3) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const codeLikeLines = countCodeLikeLines(lines);
|
||||
const style = typeof getComputedStyle === "function" ? getComputedStyle(element) : null;
|
||||
const fontFamily = normalizeText(style?.fontFamily);
|
||||
const whiteSpace = normalizeText(style?.whiteSpace);
|
||||
const looksMonospace = /mono|courier|menlo|monaco|consolas/i.test(fontFamily);
|
||||
const preservesWhitespace = /pre|break-spaces/i.test(whiteSpace);
|
||||
|
||||
return preservesWhitespace || looksMonospace || codeLikeLines >= 2;
|
||||
}
|
||||
|
||||
function serializeCodeBlock(element) {
|
||||
const codeText = normalizeCodeText(element.innerText || element.textContent || "");
|
||||
if (!codeText) {
|
||||
return [];
|
||||
}
|
||||
const language = resolveCodeLanguage(element);
|
||||
return [`\`\`\`${language}\n${codeText}\n\`\`\``];
|
||||
}
|
||||
|
||||
function isSerializableArticleSegment(element) {
|
||||
if (!(element instanceof Element)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (shouldIgnoreElement(element) || element.matches?.(ARTICLE_TITLE_SELECTOR)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return (
|
||||
element.matches?.(ARTICLE_CONTENT_SELECTOR) ||
|
||||
element.matches?.(IMAGE_SELECTOR) ||
|
||||
element.matches?.(IMAGE_BLOCK_SELECTOR) ||
|
||||
looksLikeCodeBlock(element)
|
||||
);
|
||||
}
|
||||
|
||||
function collectArticleSegments(articleRoot, preferredRoot) {
|
||||
const preferredRoots = [
|
||||
...(preferredRoot ? [preferredRoot] : []),
|
||||
...Array.from(articleRoot.querySelectorAll(ARTICLE_CONTENT_SELECTOR)).filter(
|
||||
(element) => element !== preferredRoot,
|
||||
),
|
||||
];
|
||||
const rawCandidates = Array.from(articleRoot.querySelectorAll("*")).filter(
|
||||
(element) =>
|
||||
isSerializableArticleSegment(element) &&
|
||||
!preferredRoots.includes(element),
|
||||
);
|
||||
const orderedCandidates = [...preferredRoots, ...rawCandidates];
|
||||
const segments = [];
|
||||
|
||||
for (const candidate of orderedCandidates) {
|
||||
if (
|
||||
segments.some(
|
||||
(existing) =>
|
||||
existing === candidate ||
|
||||
existing.contains(candidate) ||
|
||||
candidate.contains(existing),
|
||||
)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
segments.push(candidate);
|
||||
}
|
||||
|
||||
return segments.length > 0 ? segments : [preferredRoot || articleRoot];
|
||||
}
|
||||
|
||||
function resolveArticleContentRoot(articleRoot) {
|
||||
for (const selector of ARTICLE_CONTENT_SELECTORS) {
|
||||
const matched = articleRoot.querySelector(selector);
|
||||
if (matched) {
|
||||
return matched;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function collectInlineMarkdown(node, state) {
|
||||
@@ -700,6 +1051,17 @@
|
||||
return registerImage(state, element);
|
||||
}
|
||||
|
||||
if (element.matches?.(IMAGE_BLOCK_SELECTOR)) {
|
||||
return collectImageMarkdown(element, state).join("\n\n");
|
||||
}
|
||||
|
||||
if (
|
||||
!hasMeaningfulText(element) &&
|
||||
element.querySelector?.("img, [role='img'], [style*='background-image']")
|
||||
) {
|
||||
return collectImageMarkdown(element, state).join("\n\n");
|
||||
}
|
||||
|
||||
if (element.matches?.(CODE_BLOCK_SELECTOR)) {
|
||||
return "";
|
||||
}
|
||||
@@ -744,6 +1106,21 @@
|
||||
return normalizeText(element.textContent || "").length > 0;
|
||||
}
|
||||
|
||||
function hasRenderableMedia(element) {
|
||||
if (!(element instanceof Element)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return (
|
||||
element.matches?.(IMAGE_SELECTOR) ||
|
||||
element.matches?.(IMAGE_BLOCK_SELECTOR) ||
|
||||
element.tagName === "IMG" ||
|
||||
Boolean(
|
||||
element.querySelector("img, [role='img'], [style*='background-image']"),
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
function serializeList(element, state, depth = 0) {
|
||||
const items = Array.from(element.children)
|
||||
.filter((child) => child.tagName === "LI")
|
||||
@@ -783,26 +1160,16 @@
|
||||
return [];
|
||||
}
|
||||
|
||||
if (element.matches(ARTICLE_TITLE_SELECTOR)) {
|
||||
if (shouldIgnoreElement(element)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
if (element.matches(CODE_BLOCK_SELECTOR)) {
|
||||
const codeText = normalizeMultilineText(element.innerText || element.textContent || "");
|
||||
if (!codeText) {
|
||||
return [];
|
||||
}
|
||||
const language = resolveCodeLanguage(element);
|
||||
return [`\`\`\`${language}\n${codeText}\n\`\`\``];
|
||||
if (looksLikeCodeBlock(element)) {
|
||||
return serializeCodeBlock(element);
|
||||
}
|
||||
|
||||
if (element.matches(IMAGE_SELECTOR) || element.matches('[data-testid="tweetPhoto"], figure')) {
|
||||
const images = element.matches(IMAGE_SELECTOR)
|
||||
? [element]
|
||||
: Array.from(element.querySelectorAll("img"));
|
||||
return images
|
||||
.map((image) => registerImage(state, image))
|
||||
.filter(Boolean);
|
||||
if (element.matches(IMAGE_SELECTOR) || element.matches(IMAGE_BLOCK_SELECTOR)) {
|
||||
return collectImageMarkdown(element, state);
|
||||
}
|
||||
|
||||
if (element.matches("ul,ol")) {
|
||||
@@ -832,12 +1199,15 @@
|
||||
if (child.matches(CODE_BLOCK_SELECTOR)) {
|
||||
return true;
|
||||
}
|
||||
if (child.matches(IMAGE_SELECTOR) || child.matches('[data-testid="tweetPhoto"], figure')) {
|
||||
if (child.matches(IMAGE_SELECTOR) || child.matches(IMAGE_BLOCK_SELECTOR)) {
|
||||
return true;
|
||||
}
|
||||
if (child.matches("ul,ol,blockquote,h1,h2,h3,h4,h5,h6,hr")) {
|
||||
return true;
|
||||
}
|
||||
if (looksLikeCodeBlock(child)) {
|
||||
return true;
|
||||
}
|
||||
return child.children.length > 0 && !["A", "SPAN", "EM", "STRONG", "I", "B", "CODE"].includes(child.tagName);
|
||||
});
|
||||
|
||||
@@ -845,7 +1215,7 @@
|
||||
return directBlockChildren.flatMap((child) => serializeBlock(child, state));
|
||||
}
|
||||
|
||||
if (!hasMeaningfulText(element) && !element.querySelector("img")) {
|
||||
if (!hasMeaningfulText(element) && !hasRenderableMedia(element)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
@@ -867,6 +1237,170 @@
|
||||
return fallback;
|
||||
}
|
||||
|
||||
function serializeArticleSegments(segments, state) {
|
||||
const blocks = segments
|
||||
.flatMap((segment) => {
|
||||
if (segment.matches?.(ARTICLE_CONTENT_SELECTOR)) {
|
||||
const serialized = serializeArticle(segment, state);
|
||||
return serialized ? [serialized] : [];
|
||||
}
|
||||
return serializeBlock(segment, state);
|
||||
})
|
||||
.map((block) => normalizeMultilineText(block))
|
||||
.filter(Boolean);
|
||||
|
||||
return blocks.join("\n\n");
|
||||
}
|
||||
|
||||
function collectPotentialCodeBlockElements(root) {
|
||||
const resolveNodeDepth = (element) => {
|
||||
let depth = 0;
|
||||
let current = element.parentElement;
|
||||
while (current) {
|
||||
depth += 1;
|
||||
current = current.parentElement;
|
||||
}
|
||||
return depth;
|
||||
};
|
||||
const candidates = Array.from(root.querySelectorAll("*"))
|
||||
.filter((element) => !shouldIgnoreElement(element) && looksLikeCodeBlock(element))
|
||||
.sort((left, right) => {
|
||||
const leftDepth = resolveNodeDepth(left);
|
||||
const rightDepth = resolveNodeDepth(right);
|
||||
return rightDepth - leftDepth;
|
||||
});
|
||||
const uniqueCandidates = [];
|
||||
|
||||
for (const candidate of candidates) {
|
||||
if (
|
||||
uniqueCandidates.some(
|
||||
(existing) =>
|
||||
existing === candidate ||
|
||||
existing.contains(candidate) ||
|
||||
candidate.contains(existing),
|
||||
)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
uniqueCandidates.push(candidate);
|
||||
}
|
||||
|
||||
return uniqueCandidates;
|
||||
}
|
||||
|
||||
function collectFallbackCodeBlocks(root) {
|
||||
const uniqueCandidates = collectPotentialCodeBlockElements(root);
|
||||
return uniqueCandidates.flatMap((candidate) => serializeCodeBlock(candidate));
|
||||
}
|
||||
|
||||
function collectPotentialMediaCarriers(root) {
|
||||
if (!(root instanceof Element)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const carriers = [];
|
||||
const seen = new Set();
|
||||
const push = (element) => {
|
||||
if (!(element instanceof Element) || seen.has(element)) {
|
||||
return;
|
||||
}
|
||||
if (shouldIgnoreElement(element)) {
|
||||
return;
|
||||
}
|
||||
seen.add(element);
|
||||
carriers.push(element);
|
||||
};
|
||||
|
||||
if (
|
||||
root.matches?.(IMAGE_SELECTOR) ||
|
||||
root.matches?.(IMAGE_BLOCK_SELECTOR) ||
|
||||
root.tagName === "IMG"
|
||||
) {
|
||||
push(root);
|
||||
}
|
||||
|
||||
Array.from(
|
||||
root.querySelectorAll("img, [data-testid='tweetPhoto'], figure, [role='img'], [style*='background-image']"),
|
||||
).forEach((element) => push(element));
|
||||
|
||||
return carriers;
|
||||
}
|
||||
|
||||
function appendMissingBlocks(markdown, blocks) {
|
||||
const normalizedMarkdown = normalizeMultilineText(markdown);
|
||||
const appendedBlocks = [];
|
||||
const appendedSet = new Set();
|
||||
|
||||
blocks.forEach((block) => {
|
||||
const normalizedBlock = normalizeMultilineText(block);
|
||||
if (!normalizedBlock) {
|
||||
return;
|
||||
}
|
||||
if (
|
||||
normalizedMarkdown.includes(normalizedBlock) ||
|
||||
appendedSet.has(normalizedBlock)
|
||||
) {
|
||||
return;
|
||||
}
|
||||
appendedSet.add(normalizedBlock);
|
||||
appendedBlocks.push(normalizedBlock);
|
||||
});
|
||||
|
||||
if (appendedBlocks.length === 0) {
|
||||
return normalizedMarkdown;
|
||||
}
|
||||
|
||||
return [normalizedMarkdown, ...appendedBlocks].filter(Boolean).join("\n\n");
|
||||
}
|
||||
|
||||
function countExtractedCodeBlocks(markdown) {
|
||||
return Array.from(String(markdown || "").matchAll(/^```/gm)).length;
|
||||
}
|
||||
|
||||
function buildWarmupSnapshot(articleRoot) {
|
||||
return {
|
||||
articleBlocks: articleRoot.querySelectorAll(ARTICLE_CONTENT_SELECTOR).length,
|
||||
codeBlocks: collectPotentialCodeBlockElements(articleRoot).length,
|
||||
mediaCarriers: collectPotentialMediaCarriers(articleRoot).length,
|
||||
textLength: normalizeText(articleRoot.textContent || "").length,
|
||||
};
|
||||
}
|
||||
|
||||
async function warmupArticleContent(articleRoot) {
|
||||
if (!(articleRoot instanceof Element)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const initialScrollY = Math.round(window.scrollY);
|
||||
await helpers.waitForDomStable?.({
|
||||
root: articleRoot,
|
||||
stableMs: 400,
|
||||
timeoutMs: 2500,
|
||||
});
|
||||
await helpers.scrollUntilSettled?.({
|
||||
root: articleRoot,
|
||||
maxScrolls: 12,
|
||||
delayMs: 350,
|
||||
settleRounds: 2,
|
||||
getSnapshot: () => buildWarmupSnapshot(articleRoot),
|
||||
});
|
||||
await helpers.waitForDomStable?.({
|
||||
root: articleRoot,
|
||||
stableMs: 500,
|
||||
timeoutMs: 3000,
|
||||
});
|
||||
await helpers.waitForImagesReady?.(articleRoot, {
|
||||
timeoutMs: 3200,
|
||||
intervalMs: 200,
|
||||
stableRounds: 2,
|
||||
});
|
||||
|
||||
if (Math.round(window.scrollY) !== initialScrollY) {
|
||||
window.scrollTo(0, initialScrollY);
|
||||
await helpers.sleep?.(80);
|
||||
}
|
||||
}
|
||||
|
||||
const requestedUrl = String(args.url || "").trim();
|
||||
if (requestedUrl && !normalizeArticleUrl(requestedUrl)) {
|
||||
return {
|
||||
@@ -896,11 +1430,17 @@
|
||||
};
|
||||
}
|
||||
|
||||
await warmupArticleContent(articleRoot);
|
||||
|
||||
const title =
|
||||
normalizeText(articleRoot.querySelector(ARTICLE_TITLE_SELECTOR)?.textContent) ||
|
||||
normalizeText(document.title.replace(/\s+on X.*$/i, ""));
|
||||
const contentRoot = articleRoot.querySelector(ARTICLE_CONTENT_SELECTOR);
|
||||
if (!contentRoot) {
|
||||
const richTextRoot = await helpers.waitFor(
|
||||
() => resolveArticleContentRoot(articleRoot),
|
||||
5000,
|
||||
250,
|
||||
);
|
||||
if (!richTextRoot) {
|
||||
return {
|
||||
ok: false,
|
||||
error_code: helpers.looksLikeLoginWall() ? "auth_required" : "adapter_runtime_error",
|
||||
@@ -909,7 +1449,12 @@
|
||||
}
|
||||
|
||||
const state = createState();
|
||||
const markdown = serializeArticle(contentRoot, state);
|
||||
const articleSegments = collectArticleSegments(articleRoot, richTextRoot);
|
||||
const fallbackCodeBlocks = collectFallbackCodeBlocks(articleRoot);
|
||||
const potentialMediaCarriers = collectPotentialMediaCarriers(articleRoot);
|
||||
let markdown = serializeArticleSegments(articleSegments, state);
|
||||
markdown = appendMissingBlocks(markdown, fallbackCodeBlocks);
|
||||
markdown = appendMissingBlocks(markdown, collectImageMarkdown(articleRoot, state));
|
||||
if (!markdown) {
|
||||
return {
|
||||
ok: false,
|
||||
@@ -918,6 +1463,22 @@
|
||||
};
|
||||
}
|
||||
|
||||
if (potentialMediaCarriers.length > 0 && state.images.length === 0) {
|
||||
return {
|
||||
ok: false,
|
||||
error_code: "adapter_runtime_error",
|
||||
error_message: "X 长文中的图片资源尚未完全加载,请稍后重试导出。",
|
||||
};
|
||||
}
|
||||
|
||||
if (fallbackCodeBlocks.length > 0 && countExtractedCodeBlocks(markdown) === 0) {
|
||||
return {
|
||||
ok: false,
|
||||
error_code: "adapter_runtime_error",
|
||||
error_message: "X 长文中的代码示例尚未完全加载,请稍后重试导出。",
|
||||
};
|
||||
}
|
||||
|
||||
const publishedAt =
|
||||
articleRoot.querySelector("time[datetime]")?.getAttribute("datetime") ||
|
||||
document.querySelector("time[datetime]")?.getAttribute("datetime") ||
|
||||
|
||||
Reference in New Issue
Block a user