From f4664006b2a5ad17e121cca9140ea4b8b08b10f0 Mon Sep 17 00:00:00 2001 From: tom <62763456+amplitudesxd@users.noreply.github.com> Date: Mon, 1 Sep 2025 20:31:23 +0100 Subject: [PATCH] Fix go-html-to-md on Windows (ENG-3398) (#2082) --- .github/workflows/test-server-self-host.yml | 4 +- .github/workflows/test-server.yml | 4 +- apps/api/Dockerfile | 6 +- apps/api/sharedLibs/go-html-to-md/.gitignore | 4 +- apps/api/sharedLibs/go-html-to-md/README.md | 15 +- apps/api/src/harness.ts | 229 ++++++++++--------- apps/api/src/lib/crawler.ts | 113 ++++----- apps/api/src/lib/html-to-markdown.ts | 20 +- apps/api/src/lib/html-transformer.ts | 180 +++++++++------ apps/api/src/lib/pdf-parser.ts | 112 +++++---- apps/api/src/natives.ts | 39 ++++ 11 files changed, 412 insertions(+), 314 deletions(-) create mode 100644 apps/api/src/natives.ts diff --git a/.github/workflows/test-server-self-host.yml b/.github/workflows/test-server-self-host.yml index d7d36244f..18730fa4b 100644 --- a/.github/workflows/test-server-self-host.yml +++ b/.github/workflows/test-server-self-host.yml @@ -70,8 +70,8 @@ jobs: - name: Build go-html-to-md run: | go mod tidy - go build -o html-to-markdown.so -buildmode=c-shared html-to-markdown.go - chmod +x html-to-markdown.so + go build -o libhtml-to-markdown.so -buildmode=c-shared html-to-markdown.go + chmod +x libhtml-to-markdown.so working-directory: ./apps/api/sharedLibs/go-html-to-md - name: Build html-transformer run: | diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.yml index 2202fcf27..8c76cd08b 100644 --- a/.github/workflows/test-server.yml +++ b/.github/workflows/test-server.yml @@ -87,8 +87,8 @@ jobs: - name: Build go-html-to-md run: | go mod tidy - go build -o html-to-markdown.so -buildmode=c-shared html-to-markdown.go - chmod +x html-to-markdown.so + go build -o libhtml-to-markdown.so -buildmode=c-shared html-to-markdown.go + chmod +x libhtml-to-markdown.so working-directory: ./apps/api/sharedLibs/go-html-to-md - name: Build html-transformer run: | diff --git a/apps/api/Dockerfile b/apps/api/Dockerfile index 5a8c62464..1297dc47d 100644 --- a/apps/api/Dockerfile +++ b/apps/api/Dockerfile @@ -20,8 +20,8 @@ COPY sharedLibs/go-html-to-md /app/sharedLibs/go-html-to-md # Install Go dependencies and build parser lib RUN cd /app/sharedLibs/go-html-to-md && \ go mod download && \ - go build -o html-to-markdown.so -buildmode=c-shared html-to-markdown.go && \ - chmod +x html-to-markdown.so + go build -o libhtml-to-markdown.so -buildmode=c-shared html-to-markdown.go && \ + chmod +x libhtml-to-markdown.so # Install Rust FROM rust:1-slim AS rust-base @@ -47,7 +47,7 @@ RUN cd /app/sharedLibs/crawler && \ FROM base COPY --from=build /app/dist /app/dist COPY --from=prod-deps /app/node_modules /app/node_modules -COPY --from=go-base /app/sharedLibs/go-html-to-md/html-to-markdown.so /app/sharedLibs/go-html-to-md/html-to-markdown.so +COPY --from=go-base /app/sharedLibs/go-html-to-md/libhtml-to-markdown.so /app/sharedLibs/go-html-to-md/libhtml-to-markdown.so COPY --from=rust-base /app/sharedLibs/html-transformer/target/release/libhtml_transformer.so /app/sharedLibs/html-transformer/target/release/libhtml_transformer.so COPY --from=rust-base /app/sharedLibs/pdf-parser/target/release/libpdf_parser.so /app/sharedLibs/pdf-parser/target/release/libpdf_parser.so COPY --from=rust-base /app/sharedLibs/crawler/target/release/libcrawler.so /app/sharedLibs/crawler/target/release/libcrawler.so diff --git a/apps/api/sharedLibs/go-html-to-md/.gitignore b/apps/api/sharedLibs/go-html-to-md/.gitignore index bdab47c67..9e40cf424 100644 --- a/apps/api/sharedLibs/go-html-to-md/.gitignore +++ b/apps/api/sharedLibs/go-html-to-md/.gitignore @@ -1,2 +1,2 @@ -html-to-markdown.so -html-to-markdown.h \ No newline at end of file +html-to-markdown.* +!html-to-markdown.go \ No newline at end of file diff --git a/apps/api/sharedLibs/go-html-to-md/README.md b/apps/api/sharedLibs/go-html-to-md/README.md index 4ad510c30..2994ccf35 100644 --- a/apps/api/sharedLibs/go-html-to-md/README.md +++ b/apps/api/sharedLibs/go-html-to-md/README.md @@ -1,7 +1,12 @@ -To build the go-html-to-md library, run the following command: +To build the `go-html-to-md` library, run the following command: ```bash -cd apps/api/src/lib/go-html-to-md -go build -o html-to-markdown.so -buildmode=c-shared html-to-markdown.go -chmod +x html-to-markdown.so -``` \ No newline at end of file +cd apps/api/sharedLibs/go-html-to-md +go build -o -buildmode=c-shared html-to-markdown.go +``` + +Replace `` with the correct filename for your OS: + +- Windows → `html-to-markdown.dll` +- Linux → `libhtml-to-markdown.so` +- macOS → `libhtml-to-markdown.dylib` diff --git a/apps/api/src/harness.ts b/apps/api/src/harness.ts index 9f1fe14db..ebda6594b 100644 --- a/apps/api/src/harness.ts +++ b/apps/api/src/harness.ts @@ -1,126 +1,147 @@ import { exec } from "child_process"; import * as net from "net"; +import { basename } from "path"; +import { HTML_TO_MARKDOWN_PATH } from "./natives"; function waitForPort(port: number, host: string): Promise { - return new Promise((resolve) => { - const checkPort = () => { - const socket = new net.Socket(); - - const onError = () => { - socket.destroy(); - setTimeout(checkPort, 1000); // Try again in 1 second - }; + return new Promise((resolve) => { + const checkPort = () => { + const socket = new net.Socket(); - socket.once('error', onError); - - socket.connect(port, host, () => { - socket.destroy(); - resolve(); - }); - }; - - checkPort(); - }); + const onError = () => { + socket.destroy(); + setTimeout(checkPort, 1000); // Try again in 1 second + }; + + socket.once("error", onError); + + socket.connect(port, host, () => { + socket.destroy(); + resolve(); + }); + }; + + checkPort(); + }); } if (process.argv.length < 3) { - console.error("Usage: pnpm harness "); - console.error() - console.error("The harness ensures that the dependencies are up to date, everything is built, and the API and Worker are running, before running a command that would require the above."); - console.error("It also tears down the API and Worker after the command is run."); - process.exit(1); + console.error("Usage: pnpm harness "); + console.error(); + console.error( + "The harness ensures that the dependencies are up to date, everything is built, and the API and Worker are running, before running a command that would require the above.", + ); + console.error( + "It also tears down the API and Worker after the command is run.", + ); + process.exit(1); } const command = process.argv.slice(2); function execForward(fancyName: string, command: string): Promise { - return new Promise((resolve, reject) => { - const child = exec(command); - let stdoutBuffer = ""; - let stderrBuffer = ""; - child.stdout?.on("data", (data) => { - stdoutBuffer += data; - while(stdoutBuffer.includes("\n")) { - const split = stdoutBuffer.split("\n"); - const line = split[0]; - stdoutBuffer = split.slice(1).join("\n"); - process.stdout.write(`[${fancyName}] ${line}\n`); - } - }); - child.stderr?.on("data", (data) => { - stderrBuffer += data; - while(stderrBuffer.includes("\n")) { - const split = stderrBuffer.split("\n"); - const line = split[0]; - stderrBuffer = split.slice(1).join("\n"); - process.stderr.write(`[${fancyName}] ${line}\n`); - } - }); - child.on("close", (code) => { - if (code !== 0) { - reject(new Error(`Command ${JSON.stringify(command)} failed with code ${code}`)); - } else { - resolve(); - } - }); + return new Promise((resolve, reject) => { + const child = exec(command); + let stdoutBuffer = ""; + let stderrBuffer = ""; + child.stdout?.on("data", (data) => { + stdoutBuffer += data; + while (stdoutBuffer.includes("\n")) { + const split = stdoutBuffer.split("\n"); + const line = split[0]; + stdoutBuffer = split.slice(1).join("\n"); + process.stdout.write(`[${fancyName}] ${line}\n`); + } }); + child.stderr?.on("data", (data) => { + stderrBuffer += data; + while (stderrBuffer.includes("\n")) { + const split = stderrBuffer.split("\n"); + const line = split[0]; + stderrBuffer = split.slice(1).join("\n"); + process.stderr.write(`[${fancyName}] ${line}\n`); + } + }); + child.on("close", (code) => { + if (code !== 0) { + reject( + new Error( + `Command ${JSON.stringify(command)} failed with code ${code}`, + ), + ); + } else { + resolve(); + } + }); + }); } (async () => { - console.log("=== Installing dependencies and building all components..."); - await Promise.all([ - (async () => { - const install = execForward("api@install", "pnpm install"); - await install; - - const build = execForward("api@build", "pnpm build"); - await build; - })(), - execForward("sharedLibs/crawler@build", "cd sharedLibs/crawler && cargo build --release"), - execForward("sharedLibs/html-transformer@build", "cd sharedLibs/html-transformer && cargo build --release"), - execForward("sharedLibs/pdf-parser@build", "cd sharedLibs/pdf-parser && cargo build --release"), - (async () => { - const install = execForward("sharedLibs/go-html-to-md@install", "cd sharedLibs/go-html-to-md && go mod tidy"); - await install; + console.log("=== Installing dependencies and building all components..."); + await Promise.all([ + (async () => { + const install = execForward("api@install", "pnpm install"); + await install; - const build = execForward("sharedLibs/go-html-to-md@build", "cd sharedLibs/go-html-to-md && go build -o html-to-markdown.so -buildmode=c-shared html-to-markdown.go"); - await build; - })(), + const build = execForward("api@build", "pnpm build"); + await build; + })(), + execForward( + "sharedLibs/crawler@build", + "cd sharedLibs/crawler && cargo build --release", + ), + execForward( + "sharedLibs/html-transformer@build", + "cd sharedLibs/html-transformer && cargo build --release", + ), + execForward( + "sharedLibs/pdf-parser@build", + "cd sharedLibs/pdf-parser && cargo build --release", + ), + (async () => { + const install = execForward( + "sharedLibs/go-html-to-md@install", + "cd sharedLibs/go-html-to-md && go mod tidy", + ); + await install; + + const build = execForward( + "sharedLibs/go-html-to-md@build", + `cd sharedLibs/go-html-to-md && go build -o ${basename(HTML_TO_MARKDOWN_PATH)} -buildmode=c-shared html-to-markdown.go`, + ); + await build; + })(), + ]); + + console.log("=== Starting API, Worker, and Index Worker..."); + + const api = execForward("api", "pnpm start:production:nobuild"); + const worker = execForward("worker", "pnpm worker:production"); + const indexWorker = execForward( + "index-worker", + "pnpm index-worker:production", + ); + + try { + await Promise.race([ + waitForPort(3002, "localhost"), + new Promise((reject) => + setTimeout(() => reject(new Error("API did not start in time")), 10000), + ), ]); - console.log("=== Starting API, Worker, and Index Worker..."); + console.log("=== Running command..."); - const api = execForward("api", "pnpm start:production:nobuild"); - const worker = execForward("worker", "pnpm worker:production"); - const indexWorker = execForward("index-worker", "pnpm index-worker:production"); + const cmd = execForward("command", command.join(" ")); - try { - await Promise.race([ - waitForPort(3002, "localhost"), - new Promise((reject) => setTimeout(() => reject(new Error("API did not start in time")), 10000)), - ]); - - console.log("=== Running command..."); - - const cmd = execForward("command", command.join(" ")); - - await Promise.race([ - cmd, - api, - worker, - indexWorker, - ]); - } finally { - console.log("=== Tearing down API, Worker, and Index Worker..."); - exec("pkill -f 'queue-worker.js'"); - exec("pkill -f 'index.js'"); - exec("pkill -f 'index-worker.js'"); - await Promise.all([ - api, - worker, - indexWorker, - ]); - } - - console.log("=== Goodbye!"); -})(); \ No newline at end of file + await Promise.race([cmd, api, worker, indexWorker]); + } finally { + console.log("=== Tearing down API, Worker, and Index Worker..."); + exec("pkill -f 'queue-worker.js'"); + exec("pkill -f 'index.js'"); + exec("pkill -f 'index-worker.js'"); + await Promise.all([api, worker, indexWorker]); + } + + console.log("=== Goodbye!"); +})(); diff --git a/apps/api/src/lib/crawler.ts b/apps/api/src/lib/crawler.ts index 862988dce..cf4c311d5 100644 --- a/apps/api/src/lib/crawler.ts +++ b/apps/api/src/lib/crawler.ts @@ -1,34 +1,28 @@ import koffi, { KoffiFunction } from "koffi"; -import { join } from "path"; import { stat } from "fs/promises"; -import { platform } from "os"; import type { DenialReason } from "../scraper/WebScraper/crawler"; +import { CRAWLER_PATH } from "../natives"; // TODO: add a timeout to the Rust transformer -const rustExecutablePath = join( - process.cwd(), - "sharedLibs/crawler/target/release/", - platform() === "darwin" ? "libcrawler.dylib" : "libcrawler.so" -); export type FilterLinksCall = { - links: string[], - limit: number | undefined, - max_depth: number, - base_url: string, - initial_url: string, - regex_on_full_url: boolean, - excludes: string[], - includes: string[], - allow_backward_crawling: boolean, - ignore_robots_txt: boolean, - robots_txt: string, -} + links: string[]; + limit: number | undefined; + max_depth: number; + base_url: string; + initial_url: string; + regex_on_full_url: boolean; + excludes: string[]; + includes: string[]; + allow_backward_crawling: boolean; + ignore_robots_txt: boolean; + robots_txt: string; +}; export type FilterLinksResult = { - links: string[], - denial_reasons: Map, -} + links: string[]; + denial_reasons: Map; +}; class RustCrawler { private static instance: RustCrawler; @@ -38,19 +32,27 @@ class RustCrawler { private _processSitemap: KoffiFunction; private constructor() { - const lib = koffi.load(rustExecutablePath); + const lib = koffi.load(CRAWLER_PATH); this._freeString = lib.func("free_string", "void", ["string"]); const cstn = "CString:" + crypto.randomUUID(); - const freedResultString = koffi.disposable(cstn, "string", this._freeString); + const freedResultString = koffi.disposable( + cstn, + "string", + this._freeString, + ); this._filterLinks = lib.func("filter_links", freedResultString, ["string"]); - this._parseSitemapXml = lib.func("parse_sitemap_xml", freedResultString, ["string"]); - this._processSitemap = lib.func("process_sitemap", freedResultString, ["string"]); + this._parseSitemapXml = lib.func("parse_sitemap_xml", freedResultString, [ + "string", + ]); + this._processSitemap = lib.func("process_sitemap", freedResultString, [ + "string", + ]); } public static async getInstance(): Promise { if (!RustCrawler.instance) { try { - await stat(rustExecutablePath); + await stat(CRAWLER_PATH); } catch (_) { throw Error("Rust crawler shared library not found"); } @@ -61,30 +63,35 @@ class RustCrawler { public async filterLinks(call: FilterLinksCall): Promise { return new Promise((resolve, reject) => { - this._filterLinks.async(JSON.stringify(call), (err: Error, res: string) => { - if (err) { - reject(err); - } else { - if (res.startsWith("RUSTFC:ERROR:")) { - return reject(new Error(res.split("RUSTFC:ERROR:")[1])); - } + this._filterLinks.async( + JSON.stringify(call), + (err: Error, res: string) => { + if (err) { + reject(err); + } else { + if (res.startsWith("RUSTFC:ERROR:")) { + return reject(new Error(res.split("RUSTFC:ERROR:")[1])); + } - if (res === "RUSTFC:ERROR") { - return reject(new Error("Something went wrong on the Rust side.")); - } + if (res === "RUSTFC:ERROR") { + return reject( + new Error("Something went wrong on the Rust side."), + ); + } - try { - const raw = JSON.parse(res); - const result: FilterLinksResult = { - links: raw.links, - denial_reasons: new Map(Object.entries(raw.denial_reasons)), + try { + const raw = JSON.parse(res); + const result: FilterLinksResult = { + links: raw.links, + denial_reasons: new Map(Object.entries(raw.denial_reasons)), + }; + resolve(result); + } catch (e) { + reject(e); } - resolve(result); - } catch (e) { - reject(e); } - } - }); + }, + ); }); } @@ -142,16 +149,16 @@ class RustCrawler { export async function filterLinks( call: FilterLinksCall, ): Promise { - const converter = await RustCrawler.getInstance(); - return await converter.filterLinks(call); + const converter = await RustCrawler.getInstance(); + return await converter.filterLinks(call); } export async function parseSitemapXml(xmlContent: string): Promise { - const converter = await RustCrawler.getInstance(); - return await converter.parseSitemapXml(xmlContent); + const converter = await RustCrawler.getInstance(); + return await converter.parseSitemapXml(xmlContent); } export async function processSitemap(xmlContent: string): Promise { - const converter = await RustCrawler.getInstance(); - return await converter.processSitemap(xmlContent); + const converter = await RustCrawler.getInstance(); + return await converter.processSitemap(xmlContent); } diff --git a/apps/api/src/lib/html-to-markdown.ts b/apps/api/src/lib/html-to-markdown.ts index a474e34e2..5a21b9b1f 100644 --- a/apps/api/src/lib/html-to-markdown.ts +++ b/apps/api/src/lib/html-to-markdown.ts @@ -1,38 +1,34 @@ import koffi from "koffi"; -import { join } from "path"; import "../services/sentry"; import * as Sentry from "@sentry/node"; import dotenv from "dotenv"; import { logger } from "./logger"; import { stat } from "fs/promises"; +import { HTML_TO_MARKDOWN_PATH } from "../natives"; dotenv.config(); // TODO: add a timeout to the Go parser -const goExecutablePath = join( - process.cwd(), - "sharedLibs", - "go-html-to-md", - "html-to-markdown.so", -); class GoMarkdownConverter { private static instance: GoMarkdownConverter; private convert: any; private free: any; - + private constructor() { - const lib = koffi.load(goExecutablePath); + const lib = koffi.load(HTML_TO_MARKDOWN_PATH); this.free = lib.func("FreeCString", "void", ["string"]); const cstn = "CString:" + crypto.randomUUID(); const freedResultString = koffi.disposable(cstn, "string", this.free); - this.convert = lib.func("ConvertHTMLToMarkdown", freedResultString, ["string"]); + this.convert = lib.func("ConvertHTMLToMarkdown", freedResultString, [ + "string", + ]); } public static async getInstance(): Promise { if (!GoMarkdownConverter.instance) { try { - await stat(goExecutablePath); + await stat(HTML_TO_MARKDOWN_PATH); } catch (_) { throw Error("Go shared library not found"); } @@ -83,7 +79,7 @@ export async function parseMarkdown( } else { logger.warn( "Tried to use Go parser, but it doesn't exist in the file system.", - { goExecutablePath }, + { HTML_TO_MARKDOWN_PATH }, ); } } diff --git a/apps/api/src/lib/html-transformer.ts b/apps/api/src/lib/html-transformer.ts index 4f05fe78c..247837e25 100644 --- a/apps/api/src/lib/html-transformer.ts +++ b/apps/api/src/lib/html-transformer.ts @@ -1,22 +1,14 @@ import koffi, { KoffiFunction } from "koffi"; -import { join } from "path"; import { stat } from "fs/promises"; -import { platform } from "os"; - -// TODO: add a timeout to the Rust transformer -const rustExecutablePath = join( - process.cwd(), - "sharedLibs/html-transformer/target/release/", - platform() === "darwin" ? "libhtml_transformer.dylib" : "libhtml_transformer.so" -); +import { HTML_TRANSFORMER_PATH } from "../natives"; type TransformHtmlOptions = { - html: string, - url: string, - include_tags: string[], - exclude_tags: string[], - only_main_content: boolean, - omce_signatures?: string[], + html: string; + url: string; + include_tags: string[]; + exclude_tags: string[]; + only_main_content: boolean; + omce_signatures?: string[]; }; class RustHTMLTransformer { @@ -31,23 +23,45 @@ class RustHTMLTransformer { private _extractAttributes: KoffiFunction; private constructor() { - const lib = koffi.load(rustExecutablePath); + const lib = koffi.load(HTML_TRANSFORMER_PATH); this._freeString = lib.func("free_string", "void", ["string"]); const cstn = "CString:" + crypto.randomUUID(); - const freedResultString = koffi.disposable(cstn, "string", this._freeString); - this._extractLinks = lib.func("extract_links", freedResultString, ["string"]); - this._extractImages = lib.func("extract_images", freedResultString, ["string", "string"]); - this._extractBaseHref = lib.func("extract_base_href", freedResultString, ["string", "string"]); - this._extractMetadata = lib.func("extract_metadata", freedResultString, ["string"]); - this._transformHtml = lib.func("transform_html", freedResultString, ["string"]); - this._getInnerJSON = lib.func("get_inner_json", freedResultString, ["string"]); - this._extractAttributes = lib.func("extract_attributes", freedResultString, ["string", "string"]); + const freedResultString = koffi.disposable( + cstn, + "string", + this._freeString, + ); + this._extractLinks = lib.func("extract_links", freedResultString, [ + "string", + ]); + this._extractImages = lib.func("extract_images", freedResultString, [ + "string", + "string", + ]); + this._extractBaseHref = lib.func("extract_base_href", freedResultString, [ + "string", + "string", + ]); + this._extractMetadata = lib.func("extract_metadata", freedResultString, [ + "string", + ]); + this._transformHtml = lib.func("transform_html", freedResultString, [ + "string", + ]); + this._getInnerJSON = lib.func("get_inner_json", freedResultString, [ + "string", + ]); + this._extractAttributes = lib.func( + "extract_attributes", + freedResultString, + ["string", "string"], + ); } public static async getInstance(): Promise { if (!RustHTMLTransformer.instance) { try { - await stat(rustExecutablePath); + await stat(HTML_TRANSFORMER_PATH); } catch (_) { throw Error("Rust html-transformer shared library not found"); } @@ -110,17 +124,27 @@ class RustHTMLTransformer { public async transformHtml(opts: TransformHtmlOptions): Promise { return new Promise((resolve, reject) => { - this._transformHtml.async(JSON.stringify(opts), (err: Error, res: string) => { - if (err) { - reject(err); - } else { - if (res === "RUSTFC:ERROR" || res.startsWith("RUSTFC:ERROR:")) { - reject(new Error(res.startsWith("RUSTFC:ERROR:") ? ("Something went wrong on the Rust side. " + res.split("RUSTFC:ERROR:")[1]) : "Something went wrong on the Rust side.")); + this._transformHtml.async( + JSON.stringify(opts), + (err: Error, res: string) => { + if (err) { + reject(err); } else { - resolve(res); + if (res === "RUSTFC:ERROR" || res.startsWith("RUSTFC:ERROR:")) { + reject( + new Error( + res.startsWith("RUSTFC:ERROR:") + ? "Something went wrong on the Rust side. " + + res.split("RUSTFC:ERROR:")[1] + : "Something went wrong on the Rust side.", + ), + ); + } else { + resolve(res); + } } - } - }); + }, + ); }); } @@ -140,19 +164,31 @@ class RustHTMLTransformer { }); } - public async extractAttributes(html: string, options: string): Promise { + public async extractAttributes( + html: string, + options: string, + ): Promise { return new Promise((resolve, reject) => { - this._extractAttributes.async(html, options, (err: Error, res: string) => { - if (err) { - reject(err); - } else { - if (res.startsWith("RUSTFC:ERROR:")) { - reject(new Error("Rust attribute extraction failed: " + res.split("RUSTFC:ERROR:")[1])); + this._extractAttributes.async( + html, + options, + (err: Error, res: string) => { + if (err) { + reject(err); } else { - resolve(res); + if (res.startsWith("RUSTFC:ERROR:")) { + reject( + new Error( + "Rust attribute extraction failed: " + + res.split("RUSTFC:ERROR:")[1], + ), + ); + } else { + resolve(res); + } } - } - }); + }, + ); }); } } @@ -160,47 +196,47 @@ class RustHTMLTransformer { export async function extractLinks( html: string | null | undefined, ): Promise { - if (!html) { - return []; - } + if (!html) { + return []; + } - const converter = await RustHTMLTransformer.getInstance(); - return await converter.extractLinks(html); + const converter = await RustHTMLTransformer.getInstance(); + return await converter.extractLinks(html); } export async function extractImages( html: string | null | undefined, - baseUrl: string = '' + baseUrl: string = "", ): Promise { - if (!html) { - return []; - } + if (!html) { + return []; + } - const converter = await RustHTMLTransformer.getInstance(); - return await converter.extractImages(html, baseUrl); + const converter = await RustHTMLTransformer.getInstance(); + return await converter.extractImages(html, baseUrl); } export async function extractBaseHref( html: string | null | undefined, - url: string + url: string, ): Promise { - if (!html) { - return url; - } + if (!html) { + return url; + } - const converter = await RustHTMLTransformer.getInstance(); - return await converter.extractBaseHref(html, url); + const converter = await RustHTMLTransformer.getInstance(); + return await converter.extractBaseHref(html, url); } export async function extractMetadata( - html: string | null | undefined, + html: string | null | undefined, ): Promise { - if (!html) { - return []; - } + if (!html) { + return []; + } - const converter = await RustHTMLTransformer.getInstance(); - return await converter.extractMetadata(html); + const converter = await RustHTMLTransformer.getInstance(); + return await converter.extractMetadata(html); } export async function transformHtml( @@ -210,9 +246,7 @@ export async function transformHtml( return await converter.transformHtml(opts); } -export async function getInnerJSON( - html: string, -): Promise { +export async function getInnerJSON(html: string): Promise { const converter = await RustHTMLTransformer.getInstance(); return await converter.getInnerJSON(html); } @@ -230,7 +264,7 @@ export type AttributeResult = { export async function extractAttributesRust( html: string, - selectors: AttributeSelector[] + selectors: AttributeSelector[], ): Promise { if (!html || selectors.length === 0) { return []; @@ -243,6 +277,8 @@ export async function extractAttributesRust( try { return JSON.parse(resultJson); } catch (error) { - throw new Error(`Failed to parse Rust attribute extraction result: ${error}`); + throw new Error( + `Failed to parse Rust attribute extraction result: ${error}`, + ); } } diff --git a/apps/api/src/lib/pdf-parser.ts b/apps/api/src/lib/pdf-parser.ts index 0c24637d6..b930b78a7 100644 --- a/apps/api/src/lib/pdf-parser.ts +++ b/apps/api/src/lib/pdf-parser.ts @@ -2,79 +2,73 @@ import koffi, { KoffiFunction } from "koffi"; import { join } from "path"; import { stat } from "fs/promises"; import { platform } from "os"; +import { PDF_PARSER_PATH } from "../natives"; // TODO: add a timeout to the Rust parser -const rustExecutablePath = join( - process.cwd(), - "sharedLibs/pdf-parser/target/release/", - platform() === "darwin" ? "libpdf_parser.dylib" : "libpdf_parser.so" -); export type PDFMetadata = { - numPages: number; - title?: string; -} + numPages: number; + title?: string; +}; class RustPDFParser { - private static instance: RustPDFParser; - private _getPDFMetadata: KoffiFunction; + private static instance: RustPDFParser; + private _getPDFMetadata: KoffiFunction; - private constructor() { - const lib = koffi.load(rustExecutablePath); - this._getPDFMetadata = lib.func("get_pdf_metadata", "string", ["string"]); + private constructor() { + const lib = koffi.load(PDF_PARSER_PATH); + this._getPDFMetadata = lib.func("get_pdf_metadata", "string", ["string"]); + } + + public static async isParserAvailable(): Promise { + if (RustPDFParser.instance) { + return true; } - public static async isParserAvailable(): Promise { - if (RustPDFParser.instance) { - return true; - } - - try { - await stat(rustExecutablePath); - RustPDFParser.instance = new RustPDFParser(); - return true; - } catch (_) { - return false; - } + try { + await stat(PDF_PARSER_PATH); + RustPDFParser.instance = new RustPDFParser(); + return true; + } catch (_) { + return false; } + } - public static async getInstance(): Promise { - if (!RustPDFParser.instance) { + public static async getInstance(): Promise { + if (!RustPDFParser.instance) { + try { + await stat(PDF_PARSER_PATH); + } catch (_) { + throw Error("Rust pdf-parser shared library not found"); + } + RustPDFParser.instance = new RustPDFParser(); + } + return RustPDFParser.instance; + } + + public async getPDFMetadata(path: string): Promise { + return new Promise((resolve, reject) => { + this._getPDFMetadata.async(path, (err: Error, res: string) => { + if (err) { + reject(err); + } else { + if (res.startsWith("RUSTFC:ERROR:")) { + reject(new Error(res.replace("RUSTFC:ERROR:", ""))); + } else { try { - await stat(rustExecutablePath); - } catch (_) { - throw Error("Rust pdf-parser shared library not found"); + const metadata = JSON.parse(res) as PDFMetadata; + resolve(metadata); + } catch (e) { + reject(new Error("Failed to parse PDF metadata.")); } - RustPDFParser.instance = new RustPDFParser(); + } } - return RustPDFParser.instance; - } - - public async getPDFMetadata(path: string): Promise { - return new Promise((resolve, reject) => { - this._getPDFMetadata.async(path, (err: Error, res: string) => { - if (err) { - reject(err); - } else { - if (res.startsWith("RUSTFC:ERROR:")) { - reject(new Error(res.replace("RUSTFC:ERROR:", ""))); - } else { - try { - const metadata = JSON.parse(res) as PDFMetadata; - resolve(metadata); - } catch (e) { - reject(new Error("Failed to parse PDF metadata.")); - } - } - } - }); - }); - } + }); + }); + } } -export async function getPDFMetadata( - path: string, -): Promise { - const converter = await RustPDFParser.getInstance(); - return await converter.getPDFMetadata(path); +export async function getPDFMetadata(path: string): Promise { + const converter = await RustPDFParser.getInstance(); + return await converter.getPDFMetadata(path); } diff --git a/apps/api/src/natives.ts b/apps/api/src/natives.ts new file mode 100644 index 000000000..560a14cb7 --- /dev/null +++ b/apps/api/src/natives.ts @@ -0,0 +1,39 @@ +import { platform } from "os"; +import { join } from "path"; + +const currentPlatform = platform(); +const isWindows = currentPlatform === "win32"; + +const EXTENSIONS = { + win32: ".dll", + darwin: ".dylib", + default: ".so", +} as const; + +function createNativePath(subPath: string, filename: string): string { + const extension = + EXTENSIONS[currentPlatform as keyof typeof EXTENSIONS] ?? + EXTENSIONS.default; + const fullFilename = `${isWindows ? "" : "lib"}${filename}${extension}`; + return join(process.cwd(), "sharedLibs", subPath, fullFilename); +} + +export const HTML_TRANSFORMER_PATH = createNativePath( + "html-transformer/target/release", + "html_transformer", +); + +export const PDF_PARSER_PATH = createNativePath( + "pdf-parser/target/release", + "pdf_parser", +); + +export const CRAWLER_PATH = createNativePath( + "crawler/target/release", + "crawler", +); + +export const HTML_TO_MARKDOWN_PATH = createNativePath( + "go-html-to-md", + "html-to-markdown", +);