diff --git a/apps/api/sharedLibs/go-html-to-md/go.mod b/apps/api/sharedLibs/go-html-to-md/go.mod index 99da57619..5db5ba7b3 100644 --- a/apps/api/sharedLibs/go-html-to-md/go.mod +++ b/apps/api/sharedLibs/go-html-to-md/go.mod @@ -4,13 +4,15 @@ go 1.23.0 toolchain go1.24.0 -require github.com/firecrawl/html-to-markdown v0.0.0-20250922154302-32a7ad4a22c3 +require ( + github.com/PuerkitoBio/goquery v1.10.3 + github.com/firecrawl/html-to-markdown v0.0.0-20250922154302-32a7ad4a22c3 + golang.org/x/net v0.41.0 +) require ( - github.com/PuerkitoBio/goquery v1.10.3 // indirect github.com/andybalholm/cascadia v1.3.3 // indirect github.com/kr/pretty v0.3.0 // indirect - golang.org/x/net v0.41.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/apps/api/sharedLibs/go-html-to-md/html-to-markdown.go b/apps/api/sharedLibs/go-html-to-md/html-to-markdown.go index efc137a8c..88dccfe39 100644 --- a/apps/api/sharedLibs/go-html-to-md/html-to-markdown.go +++ b/apps/api/sharedLibs/go-html-to-md/html-to-markdown.go @@ -5,11 +5,17 @@ package main */ import "C" import ( + "strings" "unsafe" + // "log" + "unicode/utf8" + + "github.com/PuerkitoBio/goquery" md "github.com/firecrawl/html-to-markdown" "github.com/firecrawl/html-to-markdown/plugin" + "golang.org/x/net/html" ) //export ConvertHTMLToMarkdown @@ -17,6 +23,8 @@ func ConvertHTMLToMarkdown(html *C.char) *C.char { converter := md.NewConverter("", true, nil) converter.Use(plugin.GitHubFlavored()) + addGenericPreRule(converter) + markdown, err := converter.ConvertString(C.GoString(html)) if err != nil { // log.Fatal(err) @@ -32,3 +40,115 @@ func FreeCString(s *C.char) { func main() { // This function is required for the main package } + +// addGenericPreRule adds a robust PRE handler that extracts nested code text +// (e.g., tables/rows/gutters) and outputs fenced blocks with detected language. +func addGenericPreRule(conv *md.Converter) { + isGutter := func(class string) bool { + c := strings.ToLower(class) + return strings.Contains(c, "gutter") || strings.Contains(c, "line-numbers") + } + + detectLang := func(sel *goquery.Selection) string { + classes := sel.AttrOr("class", "") + lower := strings.ToLower(classes) + for _, part := range strings.Fields(lower) { + if strings.HasPrefix(part, "language-") { + return strings.TrimPrefix(part, "language-") + } + if strings.HasPrefix(part, "lang-") { + return strings.TrimPrefix(part, "lang-") + } + } + return "" + } + + // Collect text recursively; insert newlines after block elements and br + var collect func(n *html.Node, b *strings.Builder) + collect = func(n *html.Node, b *strings.Builder) { + if n == nil { + return + } + switch n.Type { + case html.TextNode: + b.WriteString(n.Data) + case html.ElementNode: + name := strings.ToLower(n.Data) + // Skip gutters + if name != "" { + // check class attr for gutters + for _, a := range n.Attr { + if a.Key == "class" && isGutter(a.Val) { + return + } + } + } + + if name == "br" { + b.WriteString("\n") + } + + for c := n.FirstChild; c != nil; c = c.NextSibling { + collect(c, b) + } + + // Newline after block-ish wrappers to preserve lines + switch name { + case "p", "div", "li", "tr", "table", "thead", "tbody", "tfoot", "section", "article", "blockquote", "pre", "h1", "h2", "h3", "h4", "h5", "h6": + b.WriteString("\n") + } + } + } + + conv.AddRules(md.Rule{ + Filter: []string{"pre"}, + Replacement: func(_ string, selec *goquery.Selection, opt *md.Options) *string { + // find inner if present for language + codeSel := selec.Find("code").First() + lang := detectLang(codeSel) + if lang == "" { + lang = detectLang(selec) + } + + var b strings.Builder + for _, n := range selec.Nodes { + collect(n, &b) + } + content := strings.TrimRight(b.String(), "\n") + + fenceChar, _ := utf8.DecodeRuneInString(opt.Fence) + fence := md.CalculateCodeFence(fenceChar, content) + text := "\n\n" + fence + lang + "\n" + content + "\n" + fence + "\n\n" + return md.String(text) + }, + }) + + // Inline code: robustly extract text and fence with backticks + conv.AddRules(md.Rule{ + Filter: []string{"code"}, + Replacement: func(_ string, selec *goquery.Selection, opt *md.Options) *string { + // If inside pre, let the PRE rule handle it + if selec.ParentsFiltered("pre").Length() > 0 { + return nil + } + var b strings.Builder + for _, n := range selec.Nodes { + collect(n, &b) + } + code := b.String() + // collapse multiple newlines for inline code + code = md.TrimTrailingSpaces(strings.ReplaceAll(code, "\r\n", "\n")) + + // Choose fence length safely + fence := "`" + if strings.Contains(code, "`") { + fence = "``" + if strings.Contains(code, "``") { + fence = "```" + } + } + out := fence + code + fence + return md.String(out) + }, + }) +} diff --git a/apps/api/src/__tests__/snips/v2/scrape.test.ts b/apps/api/src/__tests__/snips/v2/scrape.test.ts index bc7418c35..32cdd8f5b 100644 --- a/apps/api/src/__tests__/snips/v2/scrape.test.ts +++ b/apps/api/src/__tests__/snips/v2/scrape.test.ts @@ -1582,6 +1582,23 @@ describe("Scrape tests", () => { scrapeTimeout, ); + concurrentIf(ALLOW_TEST_SUITE_WEBSITE)( + "nested code blocks are converted to markdown correctly", + async () => { + const response = await scrape( + { + url: `${base}/code-block`, + formats: ["markdown"], + }, + identity, + ); + + expect(response.markdown).toBeDefined(); + expect(response.markdown).toContain("MyCustomClient"); + }, + scrapeTimeout, + ); + // TODO: check if these are required describeIf(ALLOW_TEST_SUITE_WEBSITE)( "__experimental_omceDomain functionality", diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts index 784103074..356b0b69d 100644 --- a/apps/api/src/controllers/v1/types.ts +++ b/apps/api/src/controllers/v1/types.ts @@ -517,7 +517,12 @@ const baseScrapeOptions = z useMock: z.string().optional(), blockAds: z.boolean().default(true), proxy: z.enum(["basic", "stealth", "auto"]).default("basic"), - maxAge: z.number().int().gte(0).safe().default(1 * 24 * 60 * 60 * 1000), + maxAge: z + .number() + .int() + .gte(0) + .safe() + .default(1 * 24 * 60 * 60 * 1000), storeInCache: z.boolean().default(true), // @deprecated __experimental_cache: z.boolean().default(false).optional(), diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.ts index 0f393edbc..6bc0875f2 100644 --- a/apps/api/src/lib/crawl-redis.ts +++ b/apps/api/src/lib/crawl-redis.ts @@ -49,14 +49,8 @@ export async function saveCrawl(id: string, crawl: StoredCrawl) { }); } -export async function recordRobotsBlocked( - crawlId: string, - url: string, -) { - await redisEvictConnection.sadd( - "crawl:" + crawlId + ":robots_blocked", - url, - ); +export async function recordRobotsBlocked(crawlId: string, url: string) { + await redisEvictConnection.sadd("crawl:" + crawlId + ":robots_blocked", url); await redisEvictConnection.expire( "crawl:" + crawlId + ":robots_blocked", 24 * 60 * 60, diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts index c3f6448e6..185e1c64c 100644 --- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts +++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts @@ -67,8 +67,6 @@ async function scrapePDFWithRunPodMU( maxPages, }); - - if ( process.env.PDF_MU_V2_EXPERIMENT === "true" && process.env.PDF_MU_V2_BASE_URL && @@ -115,7 +113,6 @@ async function scrapePDFWithRunPodMU( })(); } - const muV1StartedAt = Date.now(); const podStart = await robustFetch({ url: diff --git a/apps/test-site/src/pages/code-block.astro b/apps/test-site/src/pages/code-block.astro new file mode 100644 index 000000000..9c96fdec9 --- /dev/null +++ b/apps/test-site/src/pages/code-block.astro @@ -0,0 +1,116 @@ +--- +import BaseHead from '../components/BaseHead.astro'; +import Footer from '../components/Footer.astro'; +import Header from '../components/Header.astro'; +--- + + + + + + + +
+
+

Inline Code Backticks Test

+

+ This page tests the conversion of inline code elements that contain backticks. + The HTML <code> tag should be converted to markdown without + escaping backticks, relying on longer fences instead. +

+

+ Here are some examples: +

+
    +
  • Command: ls - should render as ls in markdown
    • +
  • Template literal: `template` - should render without escaped backticks
    • +
  • Nested backticks: ``double`` - should use longer fence
    • +
  • HTML entity: <code> - should handle entity decoding
    • +
+

+ The key test case is that <code>ls</code> should + render as markdown without literal backslashes before backticks. Specifically, + the HTML entity sequence <code>ls</code> should render + as ls in markdown (using longer fences if needed), not as + \`ls\` with escaped backticks. +

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ 1 + + client<llm> MyCustomClient { +
+ 2 + + provider "openai" +
+ 3 + + options { +
+ 4 + + model "gpt-5-mini" +
+ 5 + + // Set the API key using an environment variable +
+ 6 + + api_key env.MY_SUPER_SECRET_API_KEY +
+ 7 + + } +
+ 8 + + } +
+
+ +
+