mirror of
https://github.com/simstudioai/sim.git
synced 2026-09-24 15:45:35 +08:00
feat(gitlab): sync repository files (code/docs) (#4864)
* feat(gitlab): sync repository files (code/docs) alongside wiki and issues * fix(gitlab): follow full keyset next-link for repo tree + skip disabled wiki gracefully in all/both * fix(gitlab): error on bad user branch (tree 404), warn on resolveRef fallback, normalize pathPrefix to directory boundary * fix(gitlab): preserve slashes in branch ref for file source URLs (GitFlow branches) * fix(gitlab): never abort sync on repo-tree 404 (empty repo); validate user branch exists at setup instead * fix(gitlab): validate ref via commits endpoint so tags and commit SHAs are accepted, not just branches * fix(gitlab): skip repo phase on tree 403 (missing read_repository) so wiki/issues still sync under all * fix(byok): add Fal icon and repair corrupted Ollama icon path The Ollama BYOK icon rendered blank because its SVG path had spaces stripped between arc-command flags (e.g. `a5.05 5.05 0 12.05-.636`), producing invalid tokens. Replaced with the canonical Ollama path. Also added a dedicated FalIcon (was falling back to the generic ImageIcon) and wired it into the BYOK provider list. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> * fix(icons): repair corrupted Fireworks icon arc command The leftmost spark of the Fireworks icon never rendered because its third subpath used a corrupted arc command (`a34.59 34.59 0 17.15 37.65`) with collapsed flags, yielding an invalid sweep-flag of 7 that aborts the path parse. Replaced with the canonical lobehub Fireworks source. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Waleed
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
b329c36b1a
commit
cd66774960
@@ -20,13 +20,13 @@ import {
|
||||
BasetenIcon,
|
||||
BrandfetchIcon,
|
||||
ExaAIIcon,
|
||||
FalIcon,
|
||||
FindymailIcon,
|
||||
FirecrawlIcon,
|
||||
FireworksIcon,
|
||||
GeminiIcon,
|
||||
GoogleIcon,
|
||||
HunterIOIcon,
|
||||
ImageIcon,
|
||||
JinaAIIcon,
|
||||
LinkupIcon,
|
||||
MistralIcon,
|
||||
@@ -118,7 +118,7 @@ const PROVIDERS: {
|
||||
{
|
||||
id: 'falai',
|
||||
name: 'Fal.ai',
|
||||
icon: ImageIcon,
|
||||
icon: FalIcon,
|
||||
description: 'Image and video generation',
|
||||
placeholder: 'Enter your Fal.ai API key',
|
||||
},
|
||||
|
||||
@@ -3365,7 +3365,18 @@ export const OllamaIcon = (props: SVGProps<SVGSVGElement>) => (
|
||||
xmlns='http://www.w3.org/2000/svg'
|
||||
>
|
||||
<title>Ollama</title>
|
||||
<path d='M7.91 1.09c.216.09.411.23.588.41.3.306.54.744.73 1.26.191.52.315 1.1.36 1.68a5.05 5.05 0 12.05-.636l.051-.004c.87-.07 1.73.09 2.48.47.101.05.2.11.3.17.05-.569.17-1.13.36-1.64.19-.52.44-.957.73-1.26a1.67 1.67 0 1.59-.41c.257-.1.53-.118.8-.42.4.114.75.368 1.02.737.25.337.43.769.56 1.29.23.93.27 2.16.115 3.65l.053.4.03.019c.757.58 1.28 1.4 1.56 2.35.44 1.49.216 3.16-.534 4.09l-.18.02.2c.417.76.67 1.57.724 2.4l.002.03c.064 1.07-.2 2.14-.814 3.19l-.007.01.1.02c.472 1.16.62 2.32.438 3.49l-.6.04a.651.65 0 01-.747.54.648.65 0 01-.54-.742c.167-1.03.01-2.07-.48-3.12a.643.64 0 01.04-.617l.004-.006c.604-.924.85-1.83.8-2.72-.046-.779-.325-1.54-.8-2.27a.644.64 0 01.18-.886l.009-.006c.243-.159.47-.565.58-1.12a4.23 4.23 0 00-.095-1.97c-.205-.7-.58-1.28-1.1-1.68-.595-.454-1.38-.673-2.38-.61a.653.65 0 01-.632-.371c-.314-.665-.772-1.14-1.34-1.44a3.29 3.29 0 00-1.77-.332c-1.25.099-2.34.801-2.67 1.69a.652.65 0 01-.61.43c-1.07.002-1.89.252-2.5.703-.522.39-.878.94-1.07 1.59a4.07 4.07 0 00-.068 1.89c.112.56.331 1.2.58 1.27l.8.01c.212.21.257.53.11.785-.36.62-.629 1.55-.673 2.44-.05 1.02.186 1.9.719 2.54l.16.02a.643.64 0 1.1.69c-.576 1.24-.753 2.25-.562 3.05a.652.65 0 01-1.27.298c-.243-1.02-.078-2.18.473-3.5l.014-.035-.008-.012a4.34 4.34 0 01-.598-1.31l-.005-.019a5.76 5.76 0 01-.177-1.78c.044-.91.28-1.84.622-2.59l.012-.026-.002-.002c-.293-.418-.51-.953-.63-1.54l-.005-.024a5.35 5.35 0 1.09-2.49c.262-.915.78-1.7 1.54-2.27.06-.45.12-.9.19-.132-.159-1.49-.119-2.73.11-3.67.13-.518.31-.95.56-1.29.27-.368.61-.622 1.01-.737.27-.076.54-.59.8.042zm4.12 9.09c.936 0 1.8.31 2.45.855.63.53 1 1.24 1 1.94 0 .888-.406 1.58-1.13 2.02-.62.38-1.45.557-2.4.557-1.01 0-1.87-.259-2.49-.734-.617-.47-.963-1.13-.963-1.84 0-.707.4-1.42 1.06-1.95.668-.537 1.55-.849 2.49-.849zm0 .896a3.07 3.07 0 00-1.92.65c-.461.37-.722.84-.722 1.25 0 .428.21.83.61 1.13.455.35 1.12.548 1.94.548.8 0 1.47-.147 1.93-.426.46-.28.7-.686.7-1.26 0-.423-.246-.89-.683-1.26-.484-.405-1.14-.643-1.86-.643zm.662 1.21l.4c.12.15.095.37-.056.49l-.292.23v.446a.375.38 0 01-.376.37.375.38 0 01-.376-.373v-.46l-.271-.218a.347.35 0 01-.052-.49.35.353 0 1.49-.051l.215.17.22-.174a.353.35 0 01.49.05zm-5.04-1.92c.478 0 .867.39.87.871a.87.87 0 01-.868.87.87.87 0 01-.867-.87.87.87 0 1.87-.872zm8.71 0c.48 0 .868.39.87.871a.87.87 0 01-.868.87.87.87 0 01-.867-.87.87.87 0 1.87-.872zM7.44 2.3l-.3a.659.66 0 00-.285.24l-.5.01c-.138.19-.258.47-.348.83-.17.69-.216 1.63-.124 2.78.43-.128.9-.208 1.4-.237l.01-.1.02-.034c.046-.82.1-.161.15-.239.12-.771.02-1.69-.253-2.44-.134-.364-.297-.65-.453-.813a.628.63 0 00-.107-.09L7.44 2.3zm9.17.04l-.2a.628.63 0 00-.107.09c-.156.16-.32.45-.453.81-.29.79-.387 1.78-.23 2.57l.58.1.8.01h.03a5.18 5.18 0 11.47.212c.086-1.12.038-2.04-.128-2.72-.09-.365-.21-.643-.349-.832l-.004-.006a.659.66 0 00-.285-.239h-.004z' />
|
||||
<path d='M7.905 1.09c.216.085.411.225.588.41.295.306.544.744.734 1.263.191.522.315 1.1.362 1.68a5.054 5.054 0 012.049-.636l.051-.004c.87-.07 1.73.087 2.48.474.101.053.2.11.297.17.05-.569.172-1.134.36-1.644.19-.52.439-.957.733-1.264a1.67 1.67 0 01.589-.41c.257-.1.53-.118.796-.042.401.114.745.368 1.016.737.248.337.434.769.561 1.287.23.934.27 2.163.115 3.645l.053.04.026.019c.757.576 1.284 1.397 1.563 2.35.435 1.487.216 3.155-.534 4.088l-.018.021.002.003c.417.762.67 1.567.724 2.4l.002.03c.064 1.065-.2 2.137-.814 3.19l-.007.01.01.024c.472 1.157.62 2.322.438 3.486l-.006.039a.651.651 0 01-.747.536.648.648 0 01-.54-.742c.167-1.033.01-2.069-.48-3.123a.643.643 0 01.04-.617l.004-.006c.604-.924.854-1.83.8-2.72-.046-.779-.325-1.544-.8-2.273a.644.644 0 01.18-.886l.009-.006c.243-.159.467-.565.58-1.12a4.229 4.229 0 00-.095-1.974c-.205-.7-.58-1.284-1.105-1.683-.595-.454-1.383-.673-2.38-.61a.653.653 0 01-.632-.371c-.314-.665-.772-1.141-1.343-1.436a3.288 3.288 0 00-1.772-.332c-1.245.099-2.343.801-2.67 1.686a.652.652 0 01-.61.425c-1.067.002-1.893.252-2.497.703-.522.39-.878.935-1.066 1.588a4.07 4.07 0 00-.068 1.886c.112.558.331 1.02.582 1.269l.008.007c.212.207.257.53.109.785-.36.622-.629 1.549-.673 2.44-.05 1.018.186 1.902.719 2.536l.016.019a.643.643 0 01.095.69c-.576 1.236-.753 2.252-.562 3.052a.652.652 0 01-1.269.298c-.243-1.018-.078-2.184.473-3.498l.014-.035-.008-.012a4.339 4.339 0 01-.598-1.309l-.005-.019a5.764 5.764 0 01-.177-1.785c.044-.91.278-1.842.622-2.59l.012-.026-.002-.002c-.293-.418-.51-.953-.63-1.545l-.005-.024a5.352 5.352 0 01.093-2.49c.262-.915.777-1.701 1.536-2.269.06-.045.123-.09.186-.132-.159-1.493-.119-2.73.112-3.67.127-.518.314-.95.562-1.287.27-.368.614-.622 1.015-.737.266-.076.54-.059.797.042zm4.116 9.09c.936 0 1.8.313 2.446.855.63.527 1.005 1.235 1.005 1.94 0 .888-.406 1.58-1.133 2.022-.62.375-1.451.557-2.403.557-1.009 0-1.871-.259-2.493-.734-.617-.47-.963-1.13-.963-1.845 0-.707.398-1.417 1.056-1.946.668-.537 1.55-.849 2.485-.849zm0 .896a3.07 3.07 0 00-1.916.65c-.461.37-.722.835-.722 1.25 0 .428.21.829.61 1.134.455.347 1.124.548 1.943.548.799 0 1.473-.147 1.932-.426.463-.28.7-.686.7-1.257 0-.423-.246-.89-.683-1.256-.484-.405-1.14-.643-1.864-.643zm.662 1.21l.004.004c.12.151.095.37-.056.49l-.292.23v.446a.375.375 0 01-.376.373.375.375 0 01-.376-.373v-.46l-.271-.218a.347.347 0 01-.052-.49.353.353 0 01.494-.051l.215.172.22-.174a.353.353 0 01.49.051zm-5.04-1.919c.478 0 .867.39.867.871a.87.87 0 01-.868.871.87.87 0 01-.867-.87.87.87 0 01.867-.872zm8.706 0c.48 0 .868.39.868.871a.87.87 0 01-.868.871.87.87 0 01-.867-.87.87.87 0 01.867-.872zM7.44 2.3l-.003.002a.659.659 0 00-.285.238l-.005.006c-.138.189-.258.467-.348.832-.17.692-.216 1.631-.124 2.782.43-.128.899-.208 1.404-.237l.01-.001.019-.034c.046-.082.095-.161.148-.239.123-.771.022-1.692-.253-2.444-.134-.364-.297-.65-.453-.813a.628.628 0 00-.107-.09L7.44 2.3zm9.174.04l-.002.001a.628.628 0 00-.107.09c-.156.163-.32.45-.453.814-.29.794-.387 1.776-.23 2.572l.058.097.008.014h.03a5.184 5.184 0 011.466.212c.086-1.124.038-2.043-.128-2.722-.09-.365-.21-.643-.349-.832l-.004-.006a.659.659 0 00-.285-.239h-.004z' />
|
||||
</svg>
|
||||
)
|
||||
export const FalIcon = (props: SVGProps<SVGSVGElement>) => (
|
||||
<svg {...props} height='1em' viewBox='0 0 24 24' width='1em' xmlns='http://www.w3.org/2000/svg'>
|
||||
<title>Fal</title>
|
||||
<path
|
||||
clipRule='evenodd'
|
||||
d='M15.477 0c.415 0 .749.338.788.752a7.775 7.775 0 006.985 6.984c.413.04.752.373.752.788v6.952c0 .415-.338.748-.752.788a7.775 7.775 0 00-6.985 6.984c-.04.414-.373.752-.788.752H8.525c-.416 0-.749-.338-.789-.752a7.775 7.775 0 00-6.984-6.984c-.414-.04-.752-.373-.752-.788V8.524c0-.415.338-.748.752-.788A7.775 7.775 0 007.736.752C7.776.338 8.11 0 8.526 0h6.95zM4.819 11.98a7.226 7.226 0 007.223 7.23 7.226 7.226 0 007.223-7.23c0-3.994-3.234-7.23-7.223-7.23a7.227 7.227 0 00-7.223 7.23z'
|
||||
fill='#EC0648'
|
||||
fillRule='evenodd'
|
||||
/>
|
||||
</svg>
|
||||
)
|
||||
export function ShieldCheckIcon(props: SVGProps<SVGSVGElement>) {
|
||||
@@ -3982,16 +3993,16 @@ export function FireworksIcon(props: SVGProps<SVGSVGElement>) {
|
||||
return (
|
||||
<svg
|
||||
{...props}
|
||||
viewBox='0 0 512 512'
|
||||
height='1em'
|
||||
width='1em'
|
||||
viewBox='0 0 24 24'
|
||||
xmlns='http://www.w3.org/2000/svg'
|
||||
fillRule='evenodd'
|
||||
clipRule='evenodd'
|
||||
strokeLinejoin='round'
|
||||
strokeMiterlimit={2}
|
||||
>
|
||||
<path
|
||||
d='M314.33 110.17L255.98 251.73l-58.42-141.56h-37.46l64 154.75c5.23 12.85 17.77 21.31 31.65 21.31s26.42-8.44 31.65-21.27l64.4-154.79h-37.46zm24.92 215.67L446 216.58l-14.56-34.77-116.58 119.56c-9.71 9.96-12.54 24.83-7.15 37.65 5.29 12.73 17.79 21.08 31.58 21.08l.42.06L506 359.75l-14.56-34.77-152.15.853h-.042zM66 216.5l14.56-34.77 116.58 119.56a34.59 34.59 0 17.15 37.65C199 351.67 186.5 360.02 172.71 360.02l-166.67-.375-.42.04 14.56-34.77 152.15.875L66 216.5z'
|
||||
fill='#5019c5'
|
||||
d='M14.8 5l-2.801 6.795L9.195 5H7.397l3.072 7.428a1.64 1.64 0 003.038.002L16.598 5H14.8zm1.196 10.352l5.124-5.244-.699-1.669-5.596 5.739a1.664 1.664 0 00-.343 1.807 1.642 1.642 0 001.516 1.012L16 17l8-.02-.699-1.669-7.303.041h-.002zM2.88 10.104l.699-1.669 5.596 5.739c.468.479.603 1.189.343 1.807a1.643 1.643 0 01-1.516 1.012l-8-.018-.002.002.699-1.669 7.303.042-5.122-5.246z'
|
||||
fill='#5019C5'
|
||||
/>
|
||||
</svg>
|
||||
)
|
||||
|
||||
@@ -9,18 +9,120 @@ const logger = createLogger('GitLabConnector')
|
||||
|
||||
const DEFAULT_HOST = 'gitlab.com'
|
||||
const PAGE_SIZE = 100
|
||||
/** Max repository file size to index. Larger blobs are skipped. */
|
||||
const MAX_FILE_SIZE = 10 * 1024 * 1024
|
||||
/** Bytes sniffed for NUL when detecting binary files (matches git's heuristic). */
|
||||
const BINARY_SNIFF_BYTES = 8000
|
||||
|
||||
/**
|
||||
* Prefix encoded into each document's externalId so getDocument can route to the
|
||||
* correct GitLab resource. Wiki pages are addressed by slug, issues by iid.
|
||||
* correct GitLab resource. Wiki pages are addressed by slug, issues by iid, and
|
||||
* repository files by their repo-relative path.
|
||||
*/
|
||||
const WIKI_PREFIX = 'wiki:'
|
||||
const ISSUE_PREFIX = 'issue:'
|
||||
const FILE_PREFIX = 'file:'
|
||||
|
||||
/**
|
||||
* Selects which GitLab resources to sync.
|
||||
* Selects which GitLab resources to sync. `repo` = repository files (code/docs),
|
||||
* `all` = repo + wiki + issues. `both` is retained for backward compatibility and
|
||||
* means wiki + issues (no repository files).
|
||||
*/
|
||||
type ContentTypeChoice = 'wiki' | 'issues' | 'both'
|
||||
type ContentTypeChoice = 'repo' | 'wiki' | 'issues' | 'both' | 'all'
|
||||
|
||||
/** Listing phases, walked in order: repository files ➜ wiki ➜ issues. */
|
||||
type SyncPhase = 'repo' | 'wiki' | 'issues'
|
||||
|
||||
interface GitLabTreeEntry {
|
||||
id: string
|
||||
name: string
|
||||
type: 'blob' | 'tree'
|
||||
path: string
|
||||
mode?: string
|
||||
}
|
||||
|
||||
interface GitLabFile {
|
||||
file_path?: string
|
||||
blob_id?: string
|
||||
content?: string
|
||||
encoding?: string
|
||||
size?: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Heuristic binary detection: a NUL byte in the first 8 KB marks the file as
|
||||
* binary, matching `git diff` / `git grep` semantics.
|
||||
*/
|
||||
function isBinaryBuffer(buf: Buffer): boolean {
|
||||
const len = Math.min(buf.length, BINARY_SNIFF_BYTES)
|
||||
for (let i = 0; i < len; i++) {
|
||||
if (buf[i] === 0) return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses a comma-separated extension filter into a normalized set (leading dot,
|
||||
* lowercased). Returns null when no filter is configured (accept all files).
|
||||
*/
|
||||
function parseExtensions(raw: unknown): Set<string> | null {
|
||||
const trimmed = typeof raw === 'string' ? raw.trim() : ''
|
||||
if (!trimmed) return null
|
||||
const exts = trimmed
|
||||
.split(',')
|
||||
.map((e) => e.trim().toLowerCase())
|
||||
.filter(Boolean)
|
||||
.map((e) => (e.startsWith('.') ? e : `.${e}`))
|
||||
return exts.length > 0 ? new Set(exts) : null
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true when the file path matches the extension filter (or no filter set).
|
||||
*/
|
||||
function matchesExtension(filePath: string, extSet: Set<string> | null): boolean {
|
||||
if (!extSet) return true
|
||||
const lastDot = filePath.lastIndexOf('.')
|
||||
if (lastDot === -1) return false
|
||||
return extSet.has(filePath.slice(lastDot).toLowerCase())
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts the full `rel="next"` URL from a keyset-pagination `Link` response
|
||||
* header. GitLab's guidance is to follow this link verbatim rather than rebuild
|
||||
* the URL, so the connector stores and re-fetches it as-is — this is robust to
|
||||
* whichever continuation parameter the endpoint uses (`page_token`, `cursor`,
|
||||
* `id_after`, …). Returns undefined when there is no next page.
|
||||
*/
|
||||
function parseNextLink(linkHeader: string | null): string | undefined {
|
||||
if (!linkHeader) return undefined
|
||||
for (const part of linkHeader.split(',')) {
|
||||
if (!/rel="?next"?/i.test(part)) continue
|
||||
const urlMatch = part.match(/<([^>]+)>/)
|
||||
if (urlMatch) return urlMatch[1]
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the ordered list of active sync phases for a content-type choice.
|
||||
*/
|
||||
function activePhases(choice: ContentTypeChoice): SyncPhase[] {
|
||||
const phases: SyncPhase[] = []
|
||||
if (choice === 'repo' || choice === 'all') phases.push('repo')
|
||||
if (choice === 'wiki' || choice === 'both' || choice === 'all') phases.push('wiki')
|
||||
if (choice === 'issues' || choice === 'both' || choice === 'all') phases.push('issues')
|
||||
return phases
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the phase following `current` for a choice, or undefined when `current`
|
||||
* is the last active phase.
|
||||
*/
|
||||
function nextPhase(current: SyncPhase, choice: ContentTypeChoice): SyncPhase | undefined {
|
||||
const phases = activePhases(choice)
|
||||
const idx = phases.indexOf(current)
|
||||
return idx >= 0 && idx + 1 < phases.length ? phases[idx + 1] : undefined
|
||||
}
|
||||
|
||||
interface GitLabWikiPage {
|
||||
slug: string
|
||||
@@ -57,6 +159,7 @@ interface GitLabProject {
|
||||
id: number
|
||||
path_with_namespace?: string
|
||||
web_url?: string
|
||||
default_branch?: string
|
||||
wiki_access_level?: string
|
||||
wiki_enabled?: boolean
|
||||
}
|
||||
@@ -94,7 +197,15 @@ function encodeProjectId(project: unknown): string {
|
||||
*/
|
||||
function getContentTypeChoice(sourceConfig: Record<string, unknown>): ContentTypeChoice {
|
||||
const value = typeof sourceConfig.contentTypes === 'string' ? sourceConfig.contentTypes : 'both'
|
||||
if (value === 'wiki' || value === 'issues') return value
|
||||
if (
|
||||
value === 'repo' ||
|
||||
value === 'wiki' ||
|
||||
value === 'issues' ||
|
||||
value === 'both' ||
|
||||
value === 'all'
|
||||
) {
|
||||
return value
|
||||
}
|
||||
return 'both'
|
||||
}
|
||||
|
||||
@@ -136,6 +247,118 @@ function buildIssueContentHash(projectId: string, iid: number, updatedAt: string
|
||||
return `gitlab:issue:${projectId}:${iid}:${updatedAt}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the change-detection hash for a repository file. The git blob SHA is
|
||||
* content-addressable, so it changes exactly when the file content changes — and
|
||||
* it is available both on the tree listing (`tree entry.id`) and the file fetch
|
||||
* (`blob_id`), so the stub and hydrated document hash identically without a
|
||||
* content fetch during listing.
|
||||
*/
|
||||
function buildFileContentHash(projectId: string, path: string, blobSha: string): string {
|
||||
return `gitlab:file:${projectId}:${path}:${blobSha}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the web UI URL for a repository file at a given ref.
|
||||
*/
|
||||
function buildFileSourceUrl(
|
||||
apiBase: string,
|
||||
encodedProject: string,
|
||||
host: string,
|
||||
projectPath: string,
|
||||
ref: string,
|
||||
path: string
|
||||
): string {
|
||||
const encodedPath = path.split('/').map(encodeURIComponent).join('/')
|
||||
if (projectPath) {
|
||||
const encodedRef = ref.split('/').map(encodeURIComponent).join('/')
|
||||
return `https://${host}/${projectPath}/-/blob/${encodedRef}/${encodedPath}`
|
||||
}
|
||||
return `${apiBase}/projects/${encodedProject}/repository/files/${encodeURIComponent(path)}/raw?ref=${encodeURIComponent(ref)}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds a deferred stub for a repository file from a tree entry. Content is empty
|
||||
* and fetched lazily via getDocument for new/changed files only.
|
||||
*/
|
||||
function treeEntryToStub(
|
||||
apiBase: string,
|
||||
encodedProject: string,
|
||||
host: string,
|
||||
projectPath: string,
|
||||
ref: string,
|
||||
entry: GitLabTreeEntry
|
||||
): ExternalDocument {
|
||||
return {
|
||||
externalId: `${FILE_PREFIX}${entry.path}`,
|
||||
title: entry.name || entry.path,
|
||||
content: '',
|
||||
contentDeferred: true,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl: buildFileSourceUrl(apiBase, encodedProject, host, projectPath, ref, entry.path),
|
||||
contentHash: buildFileContentHash(encodedProject, entry.path, entry.id),
|
||||
metadata: {
|
||||
contentType: 'file',
|
||||
title: entry.name || entry.path,
|
||||
path: entry.path,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds a repository-file document from a fetched (non-raw) file response. Returns
|
||||
* null for binary, oversized, or empty files so they are not indexed.
|
||||
*/
|
||||
function fileToDocument(
|
||||
apiBase: string,
|
||||
encodedProject: string,
|
||||
host: string,
|
||||
projectPath: string,
|
||||
ref: string,
|
||||
path: string,
|
||||
file: GitLabFile
|
||||
): ExternalDocument | null {
|
||||
const blobSha = file.blob_id?.trim()
|
||||
if (!blobSha) return null
|
||||
|
||||
if (typeof file.size === 'number' && file.size > MAX_FILE_SIZE) {
|
||||
logger.info('Skipping oversized GitLab file', { path, size: file.size })
|
||||
return null
|
||||
}
|
||||
|
||||
const raw = typeof file.content === 'string' ? file.content : ''
|
||||
const buffer = file.encoding === 'base64' ? Buffer.from(raw, 'base64') : Buffer.from(raw, 'utf8')
|
||||
if (isBinaryBuffer(buffer)) {
|
||||
logger.info('Skipping binary GitLab file', { path })
|
||||
return null
|
||||
}
|
||||
if (buffer.byteLength > MAX_FILE_SIZE) {
|
||||
logger.info('Skipping oversized GitLab file', { path, size: buffer.byteLength })
|
||||
return null
|
||||
}
|
||||
|
||||
const content = buffer.toString('utf8')
|
||||
const title = path.split('/').pop() || path
|
||||
const body = composeBody(title, content)
|
||||
if (!body.trim()) return null
|
||||
|
||||
return {
|
||||
externalId: `${FILE_PREFIX}${path}`,
|
||||
title,
|
||||
content: body,
|
||||
contentDeferred: false,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl: buildFileSourceUrl(apiBase, encodedProject, host, projectPath, ref, path),
|
||||
contentHash: buildFileContentHash(encodedProject, path, blobSha),
|
||||
metadata: {
|
||||
contentType: 'file',
|
||||
title,
|
||||
path,
|
||||
size: buffer.byteLength,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Composes the document body as "Title\n\n<content>".
|
||||
*/
|
||||
@@ -251,30 +474,73 @@ async function fetchProject(
|
||||
* issues via the X-Next-Page header.
|
||||
*/
|
||||
interface CursorState {
|
||||
phase: 'wiki' | 'issues'
|
||||
phase: SyncPhase
|
||||
issuePage: number
|
||||
/** Full `rel="next"` URL for the repository-tree keyset page to fetch next. */
|
||||
fileNextUrl?: string
|
||||
}
|
||||
|
||||
function encodeCursor(state: CursorState): string {
|
||||
return Buffer.from(JSON.stringify(state), 'utf8').toString('base64url')
|
||||
}
|
||||
|
||||
function decodeCursor(cursor: string | undefined, initialPhase: 'wiki' | 'issues'): CursorState {
|
||||
function decodeCursor(cursor: string | undefined, initialPhase: SyncPhase): CursorState {
|
||||
if (!cursor) return { phase: initialPhase, issuePage: 1 }
|
||||
try {
|
||||
const parsed = JSON.parse(Buffer.from(cursor, 'base64url').toString('utf8')) as Partial<{
|
||||
phase: 'wiki' | 'issues'
|
||||
phase: SyncPhase
|
||||
issuePage: number
|
||||
fileNextUrl: string
|
||||
}>
|
||||
const phase: SyncPhase =
|
||||
parsed.phase === 'repo' || parsed.phase === 'issues' || parsed.phase === 'wiki'
|
||||
? parsed.phase
|
||||
: initialPhase
|
||||
return {
|
||||
phase: parsed.phase === 'issues' ? 'issues' : 'wiki',
|
||||
phase,
|
||||
issuePage: Number(parsed.issuePage) > 0 ? Number(parsed.issuePage) : 1,
|
||||
fileNextUrl: typeof parsed.fileNextUrl === 'string' ? parsed.fileNextUrl : undefined,
|
||||
}
|
||||
} catch {
|
||||
return { phase: initialPhase, issuePage: 1 }
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves the git ref (branch/tag) to sync repository files from. Uses the
|
||||
* user-configured `ref` when set, otherwise the project's default branch, which
|
||||
* is cached on syncContext to avoid repeat lookups across pages and getDocument.
|
||||
*/
|
||||
async function resolveRef(
|
||||
sourceConfig: Record<string, unknown>,
|
||||
syncContext: Record<string, unknown> | undefined,
|
||||
apiBase: string,
|
||||
encodedProject: string,
|
||||
accessToken: string
|
||||
): Promise<string> {
|
||||
const configured = typeof sourceConfig.ref === 'string' ? sourceConfig.ref.trim() : ''
|
||||
if (configured) return configured
|
||||
|
||||
const cached = syncContext?.defaultBranch as string | undefined
|
||||
if (cached) return cached
|
||||
|
||||
const response = await fetchProject(apiBase, encodedProject, accessToken)
|
||||
if (response.ok) {
|
||||
const project = (await response.json()) as GitLabProject
|
||||
const branch = project.default_branch?.trim() || 'main'
|
||||
if (syncContext) {
|
||||
syncContext.defaultBranch = branch
|
||||
if (project.path_with_namespace) syncContext.projectPath = project.path_with_namespace
|
||||
}
|
||||
return branch
|
||||
}
|
||||
logger.warn('Failed to fetch GitLab project for default branch; falling back to "main"', {
|
||||
project: encodedProject,
|
||||
status: response.status,
|
||||
})
|
||||
return 'main'
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies the optional maxItems cap to a batch, tracking the running total in
|
||||
* syncContext and flagging `listingCapped` when the cap is hit.
|
||||
@@ -298,7 +564,8 @@ function applyMaxItemsCap(
|
||||
export const gitlabConnector: ConnectorConfig = {
|
||||
id: 'gitlab',
|
||||
name: 'GitLab',
|
||||
description: 'Sync wiki pages and issues from a GitLab project into your knowledge base',
|
||||
description:
|
||||
'Sync repository files, wiki pages, and issues from a GitLab project into your knowledge base',
|
||||
version: '1.0.0',
|
||||
icon: GitLabIcon,
|
||||
|
||||
@@ -310,8 +577,9 @@ export const gitlabConnector: ConnectorConfig = {
|
||||
|
||||
/**
|
||||
* Incremental sync applies to issues only (via the `updated_after` filter
|
||||
* derived from lastSyncAt). Wikis lack a change timestamp, so they are always
|
||||
* re-listed in full and reconciled by content hash.
|
||||
* derived from lastSyncAt). Wikis and repository files lack a change timestamp
|
||||
* on listing, so they are always re-listed in full and reconciled by content
|
||||
* hash (wiki: content digest, file: git blob SHA) — unchanged docs are skipped.
|
||||
*/
|
||||
supportsIncrementalSync: true,
|
||||
|
||||
@@ -338,10 +606,42 @@ export const gitlabConnector: ConnectorConfig = {
|
||||
type: 'dropdown',
|
||||
required: false,
|
||||
options: [
|
||||
{ label: 'Code, Wiki & Issues', id: 'all' },
|
||||
{ label: 'Code (repository files) only', id: 'repo' },
|
||||
{ label: 'Wiki only', id: 'wiki' },
|
||||
{ label: 'Issues only', id: 'issues' },
|
||||
{ label: 'Both', id: 'both' },
|
||||
{ label: 'Wiki & Issues', id: 'both' },
|
||||
],
|
||||
description: 'Which content to index. "Code" syncs repository files (READMEs, docs, source).',
|
||||
},
|
||||
{
|
||||
id: 'ref',
|
||||
title: 'Branch',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
mode: 'advanced',
|
||||
placeholder: 'Default branch',
|
||||
description: 'Branch or tag to sync repository files from. Applies only when syncing Code.',
|
||||
},
|
||||
{
|
||||
id: 'pathPrefix',
|
||||
title: 'Path Filter',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
mode: 'advanced',
|
||||
placeholder: 'e.g. docs/',
|
||||
description:
|
||||
'Only sync repository files under this path prefix. Applies only when syncing Code.',
|
||||
},
|
||||
{
|
||||
id: 'fileExtensions',
|
||||
title: 'File Extensions',
|
||||
type: 'short-input',
|
||||
required: false,
|
||||
mode: 'advanced',
|
||||
placeholder: 'e.g. .md, .txt, .mdx',
|
||||
description:
|
||||
'Only sync repository files with these extensions (comma-separated). Leave blank for all text files. Applies only when syncing Code.',
|
||||
},
|
||||
{
|
||||
id: 'issueState',
|
||||
@@ -398,13 +698,13 @@ export const gitlabConnector: ConnectorConfig = {
|
||||
const choice = getContentTypeChoice(sourceConfig)
|
||||
const maxItems = sourceConfig.maxItems ? Number(sourceConfig.maxItems) : 0
|
||||
|
||||
const wantsWiki = choice === 'wiki' || choice === 'both'
|
||||
const wantsIssues = choice === 'issues' || choice === 'both'
|
||||
|
||||
if (!encodedProject) {
|
||||
throw new Error('Project is required')
|
||||
}
|
||||
|
||||
const phases = activePhases(choice)
|
||||
if (phases.length === 0) return { documents: [], hasMore: false }
|
||||
|
||||
let projectPath = (syncContext?.projectPath as string) ?? ''
|
||||
if (!projectPath && syncContext) {
|
||||
const projectResponse = await fetchProject(apiBase, encodedProject, accessToken)
|
||||
@@ -412,13 +712,98 @@ export const gitlabConnector: ConnectorConfig = {
|
||||
const project = (await projectResponse.json()) as GitLabProject
|
||||
projectPath = project.path_with_namespace ?? ''
|
||||
syncContext.projectPath = projectPath
|
||||
if (project.default_branch && !syncContext.defaultBranch) {
|
||||
syncContext.defaultBranch = project.default_branch
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const initialPhase: 'wiki' | 'issues' = wantsWiki ? 'wiki' : 'issues'
|
||||
const state = decodeCursor(cursor, initialPhase)
|
||||
let state = decodeCursor(cursor, phases[0])
|
||||
if (!phases.includes(state.phase)) state = { phase: phases[0], issuePage: 1 }
|
||||
|
||||
if (state.phase === 'wiki' && wantsWiki) {
|
||||
/** Cursor that advances to the first page of the phase after `current`, if any. */
|
||||
const advance = (current: SyncPhase): { nextCursor?: string; hasMore: boolean } => {
|
||||
const next = nextPhase(current, choice)
|
||||
if (!next) return { hasMore: false }
|
||||
return { nextCursor: encodeCursor({ phase: next, issuePage: 1 }), hasMore: true }
|
||||
}
|
||||
|
||||
if (state.phase === 'repo') {
|
||||
const ref = await resolveRef(sourceConfig, syncContext, apiBase, encodedProject, accessToken)
|
||||
const extSet = parseExtensions(sourceConfig.fileExtensions)
|
||||
const rawPrefix =
|
||||
typeof sourceConfig.pathPrefix === 'string' ? sourceConfig.pathPrefix.trim() : ''
|
||||
const pathPrefix = rawPrefix && !rawPrefix.endsWith('/') ? `${rawPrefix}/` : rawPrefix
|
||||
|
||||
const treeParams = new URLSearchParams({
|
||||
ref,
|
||||
recursive: 'true',
|
||||
per_page: String(PAGE_SIZE),
|
||||
pagination: 'keyset',
|
||||
})
|
||||
const url =
|
||||
state.fileNextUrl ??
|
||||
`${apiBase}/projects/${encodedProject}/repository/tree?${treeParams.toString()}`
|
||||
logger.info('Listing GitLab repository files', {
|
||||
host,
|
||||
project: encodedProject,
|
||||
ref,
|
||||
continued: Boolean(state.fileNextUrl),
|
||||
})
|
||||
|
||||
const response = await fetchWithRetry(url, {
|
||||
method: 'GET',
|
||||
headers: authHeaders(accessToken),
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status === 404 || response.status === 403) {
|
||||
logger.warn('GitLab repository tree unavailable; skipping files', {
|
||||
host,
|
||||
project: encodedProject,
|
||||
ref,
|
||||
status: response.status,
|
||||
})
|
||||
const adv = advance('repo')
|
||||
return { documents: [], nextCursor: adv.nextCursor, hasMore: adv.hasMore }
|
||||
}
|
||||
const errorText = await response.text().catch(() => '')
|
||||
logger.error('Failed to list GitLab repository tree', {
|
||||
status: response.status,
|
||||
error: errorText.slice(0, 500),
|
||||
})
|
||||
throw new Error(`Failed to list GitLab repository tree: ${response.status}`)
|
||||
}
|
||||
|
||||
const entries = (await response.json()) as GitLabTreeEntry[]
|
||||
const documents: ExternalDocument[] = []
|
||||
for (const entry of entries) {
|
||||
if (entry.type !== 'blob' || !entry.path) continue
|
||||
if (pathPrefix && !entry.path.startsWith(pathPrefix)) continue
|
||||
if (!matchesExtension(entry.path, extSet)) continue
|
||||
documents.push(treeEntryToStub(apiBase, encodedProject, host, projectPath, ref, entry))
|
||||
}
|
||||
|
||||
const { documents: capped, capped: hitLimit } = applyMaxItemsCap(
|
||||
documents,
|
||||
maxItems,
|
||||
syncContext
|
||||
)
|
||||
if (hitLimit) return { documents: capped, hasMore: false }
|
||||
|
||||
const nextLink = parseNextLink(response.headers.get('link'))
|
||||
if (nextLink) {
|
||||
return {
|
||||
documents: capped,
|
||||
nextCursor: encodeCursor({ phase: 'repo', issuePage: 1, fileNextUrl: nextLink }),
|
||||
hasMore: true,
|
||||
}
|
||||
}
|
||||
const adv = advance('repo')
|
||||
return { documents: capped, nextCursor: adv.nextCursor, hasMore: adv.hasMore }
|
||||
}
|
||||
|
||||
if (state.phase === 'wiki') {
|
||||
const url = `${apiBase}/projects/${encodedProject}/wikis?with_content=1`
|
||||
logger.info('Listing GitLab wiki pages', { host, project: encodedProject })
|
||||
|
||||
@@ -428,6 +813,15 @@ export const gitlabConnector: ConnectorConfig = {
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status === 403 || response.status === 404) {
|
||||
logger.warn('GitLab wiki unavailable; skipping wiki phase', {
|
||||
host,
|
||||
project: encodedProject,
|
||||
status: response.status,
|
||||
})
|
||||
const adv = advance('wiki')
|
||||
return { documents: [], nextCursor: adv.nextCursor, hasMore: adv.hasMore }
|
||||
}
|
||||
const errorText = await response.text().catch(() => '')
|
||||
logger.error('Failed to list GitLab wiki pages', {
|
||||
status: response.status,
|
||||
@@ -450,18 +844,15 @@ export const gitlabConnector: ConnectorConfig = {
|
||||
syncContext
|
||||
)
|
||||
|
||||
if (hitLimit || !wantsIssues) {
|
||||
if (hitLimit) {
|
||||
return { documents: capped, hasMore: false }
|
||||
}
|
||||
|
||||
return {
|
||||
documents: capped,
|
||||
nextCursor: encodeCursor({ phase: 'issues', issuePage: 1 }),
|
||||
hasMore: true,
|
||||
}
|
||||
const adv = advance('wiki')
|
||||
return { documents: capped, nextCursor: adv.nextCursor, hasMore: adv.hasMore }
|
||||
}
|
||||
|
||||
if (wantsIssues) {
|
||||
if (state.phase === 'issues') {
|
||||
const params = new URLSearchParams({
|
||||
per_page: String(PAGE_SIZE),
|
||||
page: String(state.issuePage),
|
||||
@@ -586,6 +977,32 @@ export const gitlabConnector: ConnectorConfig = {
|
||||
return issueToDocument(encodedProject, host, projectPath, issue)
|
||||
}
|
||||
|
||||
if (externalId.startsWith(FILE_PREFIX)) {
|
||||
const path = externalId.slice(FILE_PREFIX.length)
|
||||
if (!path) return null
|
||||
|
||||
const ref = await resolveRef(
|
||||
sourceConfig,
|
||||
syncContext,
|
||||
apiBase,
|
||||
encodedProject,
|
||||
accessToken
|
||||
)
|
||||
const url = `${apiBase}/projects/${encodedProject}/repository/files/${encodeURIComponent(path)}?ref=${encodeURIComponent(ref)}`
|
||||
const response = await fetchWithRetry(url, {
|
||||
method: 'GET',
|
||||
headers: authHeaders(accessToken),
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status === 404) return null
|
||||
throw new Error(`Failed to fetch GitLab file: ${response.status}`)
|
||||
}
|
||||
|
||||
const file = (await response.json()) as GitLabFile
|
||||
return fileToDocument(apiBase, encodedProject, host, projectPath, ref, path, file)
|
||||
}
|
||||
|
||||
return null
|
||||
} catch (error) {
|
||||
logger.warn(`Failed to fetch GitLab document ${externalId}`, {
|
||||
@@ -634,7 +1051,7 @@ export const gitlabConnector: ConnectorConfig = {
|
||||
|
||||
const projectRecord = (await response.json()) as GitLabProject
|
||||
|
||||
if (choice === 'wiki' || choice === 'both') {
|
||||
if (activePhases(choice).includes('wiki')) {
|
||||
const accessLevel = projectRecord.wiki_access_level
|
||||
const enabled =
|
||||
accessLevel != null ? accessLevel !== 'disabled' : projectRecord.wiki_enabled !== false
|
||||
@@ -642,7 +1059,28 @@ export const gitlabConnector: ConnectorConfig = {
|
||||
if (choice === 'wiki') {
|
||||
return { valid: false, error: 'The wiki feature is disabled for this project' }
|
||||
}
|
||||
logger.warn('Wiki feature disabled; only issues will sync', { project })
|
||||
logger.warn('Wiki feature disabled; it will be skipped', { project })
|
||||
}
|
||||
}
|
||||
|
||||
const userRef = typeof sourceConfig.ref === 'string' ? sourceConfig.ref.trim() : ''
|
||||
if (userRef && activePhases(choice).includes('repo')) {
|
||||
const refResponse = await fetchWithRetry(
|
||||
`${apiBase}/projects/${encodedProject}/repository/commits/${encodeURIComponent(userRef)}`,
|
||||
{ method: 'GET', headers: authHeaders(accessToken) },
|
||||
VALIDATE_RETRY_OPTIONS
|
||||
)
|
||||
if (refResponse.status === 404) {
|
||||
return {
|
||||
valid: false,
|
||||
error: `Branch, tag, or commit "${userRef}" not found in project "${project}"`,
|
||||
}
|
||||
}
|
||||
if (!refResponse.ok) {
|
||||
return {
|
||||
valid: false,
|
||||
error: `Cannot verify ref "${userRef}": ${refResponse.status}`,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -659,16 +1097,17 @@ export const gitlabConnector: ConnectorConfig = {
|
||||
{ id: 'author', displayName: 'Author', fieldType: 'text' },
|
||||
{ id: 'labels', displayName: 'Labels', fieldType: 'text' },
|
||||
{ id: 'milestone', displayName: 'Milestone', fieldType: 'text' },
|
||||
{ id: 'path', displayName: 'File Path', fieldType: 'text' },
|
||||
{ id: 'size', displayName: 'File Size (bytes)', fieldType: 'number' },
|
||||
{ id: 'createdAt', displayName: 'Created At', fieldType: 'date' },
|
||||
{ id: 'updatedAt', displayName: 'Updated At', fieldType: 'date' },
|
||||
],
|
||||
|
||||
/**
|
||||
* Maps document metadata to tag slots. The `contentType` and `title` tags
|
||||
* apply to both wikis and issues. The remaining tags (state, author, labels,
|
||||
* milestone, createdAt, updatedAt) are issue-only — wiki pages expose none of
|
||||
* them in the REST API, so wiki documents leave those metadata fields empty
|
||||
* and the type/empty guards below skip them.
|
||||
* Maps document metadata to tag slots. `contentType` and `title` apply to every
|
||||
* document type. `state`/`author`/`labels`/`milestone`/`createdAt`/`updatedAt`
|
||||
* are issue-only and `path`/`size` are repository-file-only; each document type
|
||||
* leaves the others' fields empty and the type/empty guards below skip them.
|
||||
*/
|
||||
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
|
||||
const result: Record<string, unknown> = {}
|
||||
@@ -693,6 +1132,15 @@ export const gitlabConnector: ConnectorConfig = {
|
||||
result.milestone = metadata.milestone
|
||||
}
|
||||
|
||||
if (typeof metadata.path === 'string' && metadata.path.trim()) {
|
||||
result.path = metadata.path
|
||||
}
|
||||
|
||||
if (metadata.size != null) {
|
||||
const num = Number(metadata.size)
|
||||
if (!Number.isNaN(num)) result.size = num
|
||||
}
|
||||
|
||||
const createdAt = parseTagDate(metadata.createdAt)
|
||||
if (createdAt) result.createdAt = createdAt
|
||||
|
||||
|
||||
Reference in New Issue
Block a user