mirror of
https://github.com/simstudioai/sim.git
synced 2026-10-24 01:23:02 +08:00
fix(mcp): tighten OAuth probe signal and clear stale popup interval
- probe: only classify as OAuth on resource_metadata or scope params. Bare `Bearer error="invalid_token"` is generic and used by API-key servers, so it must not auto-flip the auth type to OAuth. - popup hook: clear any existing close-watcher interval before overwriting when startOauthForServer is invoked twice for the same serverId. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Waleed Latif
co-authored by
Claude Opus 4.7
Claude Opus 4.7
parent
5be7f799f1
commit
05c4bc19e8
@@ -0,0 +1,534 @@
|
||||
---
|
||||
name: "source-command-add-connector"
|
||||
description: "Add a knowledge base connector for syncing documents from an external source"
|
||||
---
|
||||
|
||||
# source-command-add-connector
|
||||
|
||||
Use this skill when the user asks to run the migrated source command `add-connector`.
|
||||
|
||||
## Command Template
|
||||
|
||||
# Add Connector Skill
|
||||
|
||||
You are an expert at adding knowledge base connectors to Sim. A connector syncs documents from an external source (Confluence, Google Drive, Notion, etc.) into a knowledge base.
|
||||
|
||||
## Your Task
|
||||
|
||||
When the user asks you to create a connector:
|
||||
1. Use Context7 or WebFetch to read the service's API documentation
|
||||
2. Determine the auth mode: **OAuth** (if Sim already has an OAuth provider for the service) or **API key** (if the service uses API key / Bearer token auth)
|
||||
3. Create the connector directory and config
|
||||
4. Register it in the connector registry
|
||||
|
||||
## Directory Structure
|
||||
|
||||
Create files in `apps/sim/connectors/{service}/`:
|
||||
```
|
||||
connectors/{service}/
|
||||
├── index.ts # Barrel export
|
||||
└── {service}.ts # ConnectorConfig definition
|
||||
```
|
||||
|
||||
## Authentication
|
||||
|
||||
Connectors use a discriminated union for auth config (`ConnectorAuthConfig` in `connectors/types.ts`):
|
||||
|
||||
```typescript
|
||||
type ConnectorAuthConfig =
|
||||
| { mode: 'oauth'; provider: OAuthService; requiredScopes?: string[] }
|
||||
| { mode: 'apiKey'; label?: string; placeholder?: string }
|
||||
```
|
||||
|
||||
### OAuth mode
|
||||
For services with existing OAuth providers in `apps/sim/lib/oauth/types.ts`. The `provider` must match an `OAuthService`. The modal shows a credential picker and handles token refresh automatically.
|
||||
|
||||
### API key mode
|
||||
For services that use API key / Bearer token auth. The modal shows a password input with the configured `label` and `placeholder`. The API key is encrypted at rest using AES-256-GCM and stored in a dedicated `encryptedApiKey` column on the connector record. The sync engine decrypts it automatically — connectors receive the raw access token in `listDocuments`, `getDocument`, and `validateConfig`.
|
||||
|
||||
## ConnectorConfig Structure
|
||||
|
||||
### OAuth connector example
|
||||
|
||||
```typescript
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { {Service}Icon } from '@/components/icons'
|
||||
import { fetchWithRetry } from '@/lib/knowledge/documents/utils'
|
||||
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
|
||||
|
||||
const logger = createLogger('{Service}Connector')
|
||||
|
||||
export const {service}Connector: ConnectorConfig = {
|
||||
id: '{service}',
|
||||
name: '{Service}',
|
||||
description: 'Sync documents from {Service} into your knowledge base',
|
||||
version: '1.0.0',
|
||||
icon: {Service}Icon,
|
||||
|
||||
auth: {
|
||||
mode: 'oauth',
|
||||
provider: '{service}', // Must match OAuthService in lib/oauth/types.ts
|
||||
requiredScopes: ['read:...'],
|
||||
},
|
||||
|
||||
configFields: [
|
||||
// Rendered dynamically by the add-connector modal UI
|
||||
// Supports 'short-input' and 'dropdown' types
|
||||
],
|
||||
|
||||
listDocuments: async (accessToken, sourceConfig, cursor) => {
|
||||
// Return metadata stubs with contentDeferred: true (if per-doc content fetch needed)
|
||||
// Or full documents with content (if list API returns content inline)
|
||||
// Return { documents: ExternalDocument[], nextCursor?, hasMore }
|
||||
},
|
||||
|
||||
getDocument: async (accessToken, sourceConfig, externalId) => {
|
||||
// Fetch full content for a single document
|
||||
// Return ExternalDocument with contentDeferred: false, or null
|
||||
},
|
||||
|
||||
validateConfig: async (accessToken, sourceConfig) => {
|
||||
// Return { valid: true } or { valid: false, error: 'message' }
|
||||
},
|
||||
|
||||
// Optional: map source metadata to semantic tag keys (translated to slots by sync engine)
|
||||
mapTags: (metadata) => {
|
||||
// Return Record<string, unknown> with keys matching tagDefinitions[].id
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
### API key connector example
|
||||
|
||||
```typescript
|
||||
export const {service}Connector: ConnectorConfig = {
|
||||
id: '{service}',
|
||||
name: '{Service}',
|
||||
description: 'Sync documents from {Service} into your knowledge base',
|
||||
version: '1.0.0',
|
||||
icon: {Service}Icon,
|
||||
|
||||
auth: {
|
||||
mode: 'apiKey',
|
||||
label: 'API Key', // Shown above the input field
|
||||
placeholder: 'Enter your {Service} API key', // Input placeholder
|
||||
},
|
||||
|
||||
configFields: [ /* ... */ ],
|
||||
listDocuments: async (accessToken, sourceConfig, cursor) => { /* ... */ },
|
||||
getDocument: async (accessToken, sourceConfig, externalId) => { /* ... */ },
|
||||
validateConfig: async (accessToken, sourceConfig) => { /* ... */ },
|
||||
}
|
||||
```
|
||||
|
||||
## ConfigField Types
|
||||
|
||||
The add-connector modal renders these automatically — no custom UI needed.
|
||||
|
||||
Three field types are supported: `short-input`, `dropdown`, and `selector`.
|
||||
|
||||
```typescript
|
||||
// Text input
|
||||
{
|
||||
id: 'domain',
|
||||
title: 'Domain',
|
||||
type: 'short-input',
|
||||
placeholder: 'yoursite.example.com',
|
||||
required: true,
|
||||
}
|
||||
|
||||
// Dropdown (static options)
|
||||
{
|
||||
id: 'contentType',
|
||||
title: 'Content Type',
|
||||
type: 'dropdown',
|
||||
required: false,
|
||||
options: [
|
||||
{ label: 'Pages only', id: 'page' },
|
||||
{ label: 'Blog posts only', id: 'blogpost' },
|
||||
{ label: 'All content', id: 'all' },
|
||||
],
|
||||
}
|
||||
```
|
||||
|
||||
## Dynamic Selectors (Canonical Pairs)
|
||||
|
||||
Use `type: 'selector'` to fetch options dynamically from the existing selector registry (`hooks/selectors/registry.ts`). Selectors are always paired with a manual fallback input using the **canonical pair** pattern — a `selector` field (basic mode) and a `short-input` field (advanced mode) linked by `canonicalParamId`.
|
||||
|
||||
The user sees a toggle button (ArrowLeftRight) to switch between the selector dropdown and manual text input. On submit, the modal resolves each canonical pair to the active mode's value, keyed by `canonicalParamId`.
|
||||
|
||||
### Rules
|
||||
|
||||
1. **Every selector field MUST have a canonical pair** — a corresponding `short-input` (or `dropdown`) field with the same `canonicalParamId` and `mode: 'advanced'`.
|
||||
2. **`required` must be set identically on both fields** in a pair. If the selector is required, the manual input must also be required.
|
||||
3. **`canonicalParamId` must match the key the connector expects in `sourceConfig`** (e.g. `baseId`, `channel`, `teamId`). The advanced field's `id` should typically match `canonicalParamId`.
|
||||
4. **`dependsOn` references the selector field's `id`**, not the `canonicalParamId`. The modal propagates dependency clearing across canonical siblings automatically — changing either field in a parent pair clears dependent children.
|
||||
|
||||
### Selector canonical pair example (Airtable base → table cascade)
|
||||
|
||||
```typescript
|
||||
configFields: [
|
||||
// Base: selector (basic) + manual (advanced)
|
||||
{
|
||||
id: 'baseSelector',
|
||||
title: 'Base',
|
||||
type: 'selector',
|
||||
selectorKey: 'airtable.bases', // Must exist in hooks/selectors/registry.ts
|
||||
canonicalParamId: 'baseId',
|
||||
mode: 'basic',
|
||||
placeholder: 'Select a base',
|
||||
required: true,
|
||||
},
|
||||
{
|
||||
id: 'baseId',
|
||||
title: 'Base ID',
|
||||
type: 'short-input',
|
||||
canonicalParamId: 'baseId',
|
||||
mode: 'advanced',
|
||||
placeholder: 'e.g. appXXXXXXXXXXXXXX',
|
||||
required: true,
|
||||
},
|
||||
// Table: selector depends on base (basic) + manual (advanced)
|
||||
{
|
||||
id: 'tableSelector',
|
||||
title: 'Table',
|
||||
type: 'selector',
|
||||
selectorKey: 'airtable.tables',
|
||||
canonicalParamId: 'tableIdOrName',
|
||||
mode: 'basic',
|
||||
dependsOn: ['baseSelector'], // References the selector field ID
|
||||
placeholder: 'Select a table',
|
||||
required: true,
|
||||
},
|
||||
{
|
||||
id: 'tableIdOrName',
|
||||
title: 'Table Name or ID',
|
||||
type: 'short-input',
|
||||
canonicalParamId: 'tableIdOrName',
|
||||
mode: 'advanced',
|
||||
placeholder: 'e.g. Tasks',
|
||||
required: true,
|
||||
},
|
||||
// Non-selector fields stay as-is
|
||||
{ id: 'maxRecords', title: 'Max Records', type: 'short-input', ... },
|
||||
]
|
||||
```
|
||||
|
||||
### Selector with domain dependency (Jira/Confluence pattern)
|
||||
|
||||
When a selector depends on a plain `short-input` field (no canonical pair), `dependsOn` references that field's `id` directly. The `domain` field's value maps to `SelectorContext.domain` automatically via `SELECTOR_CONTEXT_FIELDS`.
|
||||
|
||||
```typescript
|
||||
configFields: [
|
||||
{
|
||||
id: 'domain',
|
||||
title: 'Jira Domain',
|
||||
type: 'short-input',
|
||||
placeholder: 'yoursite.atlassian.net',
|
||||
required: true,
|
||||
},
|
||||
{
|
||||
id: 'projectSelector',
|
||||
title: 'Project',
|
||||
type: 'selector',
|
||||
selectorKey: 'jira.projects',
|
||||
canonicalParamId: 'projectKey',
|
||||
mode: 'basic',
|
||||
dependsOn: ['domain'],
|
||||
placeholder: 'Select a project',
|
||||
required: true,
|
||||
},
|
||||
{
|
||||
id: 'projectKey',
|
||||
title: 'Project Key',
|
||||
type: 'short-input',
|
||||
canonicalParamId: 'projectKey',
|
||||
mode: 'advanced',
|
||||
placeholder: 'e.g. ENG, PROJ',
|
||||
required: true,
|
||||
},
|
||||
]
|
||||
```
|
||||
|
||||
### How `dependsOn` maps to `SelectorContext`
|
||||
|
||||
The connector selector field builds a `SelectorContext` from dependency values. For the mapping to work, each dependency's `canonicalParamId` (or field `id` for non-canonical fields) must exist in `SELECTOR_CONTEXT_FIELDS` (`lib/workflows/subblocks/context.ts`):
|
||||
|
||||
```
|
||||
oauthCredential, domain, teamId, projectId, knowledgeBaseId, planId,
|
||||
siteId, collectionId, spreadsheetId, fileId, baseId, datasetId, serviceDeskId
|
||||
```
|
||||
|
||||
### Available selector keys
|
||||
|
||||
Check `hooks/selectors/types.ts` for the full `SelectorKey` union. Common ones for connectors:
|
||||
|
||||
| SelectorKey | Context Deps | Returns |
|
||||
|-------------|-------------|---------|
|
||||
| `airtable.bases` | credential | Base ID + name |
|
||||
| `airtable.tables` | credential, `baseId` | Table ID + name |
|
||||
| `slack.channels` | credential | Channel ID + name |
|
||||
| `gmail.labels` | credential | Label ID + name |
|
||||
| `google.calendar` | credential | Calendar ID + name |
|
||||
| `linear.teams` | credential | Team ID + name |
|
||||
| `linear.projects` | credential, `teamId` | Project ID + name |
|
||||
| `jira.projects` | credential, `domain` | Project key + name |
|
||||
| `confluence.spaces` | credential, `domain` | Space key + name |
|
||||
| `notion.databases` | credential | Database ID + name |
|
||||
| `asana.workspaces` | credential | Workspace GID + name |
|
||||
| `microsoft.teams` | credential | Team ID + name |
|
||||
| `microsoft.channels` | credential, `teamId` | Channel ID + name |
|
||||
| `webflow.sites` | credential | Site ID + name |
|
||||
| `outlook.folders` | credential | Folder ID + name |
|
||||
|
||||
## ExternalDocument Shape
|
||||
|
||||
Every document returned from `listDocuments`/`getDocument` must include:
|
||||
|
||||
```typescript
|
||||
{
|
||||
externalId: string // Source-specific unique ID
|
||||
title: string // Document title
|
||||
content: string // Extracted plain text (or '' if contentDeferred)
|
||||
contentDeferred?: boolean // true = content will be fetched via getDocument
|
||||
mimeType: 'text/plain' // Always text/plain (content is extracted)
|
||||
contentHash: string // Metadata-based hash for change detection
|
||||
sourceUrl?: string // Link back to original (stored on document record)
|
||||
metadata?: Record<string, unknown> // Source-specific data (fed to mapTags)
|
||||
}
|
||||
```
|
||||
|
||||
## Content Deferral (Required for file/content-download connectors)
|
||||
|
||||
**All connectors that require per-document API calls to fetch content MUST use `contentDeferred: true`.** This is the standard pattern — `listDocuments` returns lightweight metadata stubs, and content is fetched lazily by the sync engine via `getDocument` only for new/changed documents.
|
||||
|
||||
This pattern is critical for reliability: the sync engine processes documents in batches and enqueues each batch for processing immediately. If a sync times out, all previously-batched documents are already queued. Without deferral, content downloads during listing can exhaust the sync task's time budget before any documents are saved.
|
||||
|
||||
### When to use `contentDeferred: true`
|
||||
|
||||
- The service's list API does NOT return document content (only metadata)
|
||||
- Content requires a separate download/export API call per document
|
||||
- Examples: Google Drive, OneDrive, SharePoint, Dropbox, Notion, Confluence, Gmail, Obsidian, Evernote, GitHub
|
||||
|
||||
### When NOT to use `contentDeferred`
|
||||
|
||||
- The list API already returns the full content inline (e.g., Slack messages, Reddit posts, HubSpot notes)
|
||||
- No per-document API call is needed to get content
|
||||
|
||||
### Content Hash Strategy
|
||||
|
||||
Use a **metadata-based** `contentHash` — never a content-based hash. The hash must be derivable from the list response metadata alone, so the sync engine can detect changes without downloading content.
|
||||
|
||||
Good metadata hash sources:
|
||||
- `modifiedTime` / `lastModifiedDateTime` — changes when file is edited
|
||||
- Git blob SHA — unique per content version
|
||||
- API-provided content hash (e.g., Dropbox `content_hash`)
|
||||
- Version number (e.g., Confluence page version)
|
||||
|
||||
Format: `{service}:{id}:{changeIndicator}`
|
||||
|
||||
```typescript
|
||||
// Google Drive: modifiedTime changes on edit
|
||||
contentHash: `gdrive:${file.id}:${file.modifiedTime ?? ''}`
|
||||
|
||||
// GitHub: blob SHA is a content-addressable hash
|
||||
contentHash: `gitsha:${item.sha}`
|
||||
|
||||
// Dropbox: API provides content_hash
|
||||
contentHash: `dropbox:${entry.id}:${entry.content_hash ?? entry.server_modified}`
|
||||
|
||||
// Confluence: version number increments on edit
|
||||
contentHash: `confluence:${page.id}:${page.version.number}`
|
||||
```
|
||||
|
||||
**Critical invariant:** The `contentHash` MUST be identical whether produced by `listDocuments` (stub) or `getDocument` (full doc). Both should use the same stub function to guarantee this.
|
||||
|
||||
### Implementation Pattern
|
||||
|
||||
```typescript
|
||||
// 1. Create a stub function (sync, no API calls)
|
||||
function fileToStub(file: ServiceFile): ExternalDocument {
|
||||
return {
|
||||
externalId: file.id,
|
||||
title: file.name || 'Untitled',
|
||||
content: '',
|
||||
contentDeferred: true,
|
||||
mimeType: 'text/plain',
|
||||
sourceUrl: `https://service.com/file/${file.id}`,
|
||||
contentHash: `service:${file.id}:${file.modifiedTime ?? ''}`,
|
||||
metadata: { /* fields needed by mapTags */ },
|
||||
}
|
||||
}
|
||||
|
||||
// 2. listDocuments returns stubs (fast, metadata only)
|
||||
listDocuments: async (accessToken, sourceConfig, cursor) => {
|
||||
const response = await fetchWithRetry(listUrl, { ... })
|
||||
const files = (await response.json()).files
|
||||
const documents = files.map(fileToStub)
|
||||
return { documents, nextCursor, hasMore }
|
||||
}
|
||||
|
||||
// 3. getDocument fetches content and returns full doc with SAME contentHash
|
||||
getDocument: async (accessToken, sourceConfig, externalId) => {
|
||||
const metadata = await fetchWithRetry(metadataUrl, { ... })
|
||||
const file = await metadata.json()
|
||||
if (file.trashed) return null
|
||||
|
||||
try {
|
||||
const content = await fetchContent(accessToken, file)
|
||||
if (!content.trim()) return null
|
||||
const stub = fileToStub(file)
|
||||
return { ...stub, content, contentDeferred: false }
|
||||
} catch (error) {
|
||||
logger.warn(`Failed to fetch content for: ${file.name}`, { error })
|
||||
return null
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Reference Implementations
|
||||
|
||||
- **Google Drive**: `connectors/google-drive/google-drive.ts` — file download/export with `modifiedTime` hash
|
||||
- **GitHub**: `connectors/github/github.ts` — git blob SHA hash
|
||||
- **Notion**: `connectors/notion/notion.ts` — blocks API with `last_edited_time` hash
|
||||
- **Confluence**: `connectors/confluence/confluence.ts` — version number hash
|
||||
|
||||
## tagDefinitions — Declared Tag Definitions
|
||||
|
||||
Declare which tags the connector populates using semantic IDs. Shown in the add-connector modal as opt-out checkboxes.
|
||||
On connector creation, slots are **dynamically assigned** via `getNextAvailableSlot` — connectors never hardcode slot names.
|
||||
|
||||
```typescript
|
||||
tagDefinitions: [
|
||||
{ id: 'labels', displayName: 'Labels', fieldType: 'text' },
|
||||
{ id: 'version', displayName: 'Version', fieldType: 'number' },
|
||||
{ id: 'lastModified', displayName: 'Last Modified', fieldType: 'date' },
|
||||
],
|
||||
```
|
||||
|
||||
Each entry has:
|
||||
- `id`: Semantic key matching a key returned by `mapTags` (e.g. `'labels'`, `'version'`)
|
||||
- `displayName`: Human-readable name shown in the UI (e.g. "Labels", "Last Modified")
|
||||
- `fieldType`: `'text'` | `'number'` | `'date'` | `'boolean'` — determines which slot pool to draw from
|
||||
|
||||
Users can opt out of specific tags in the modal. Disabled IDs are stored in `sourceConfig.disabledTagIds`.
|
||||
The assigned mapping (`semantic id → slot`) is stored in `sourceConfig.tagSlotMapping`.
|
||||
|
||||
## mapTags — Metadata to Semantic Keys
|
||||
|
||||
Maps source metadata to semantic tag keys. Required if `tagDefinitions` is set.
|
||||
The sync engine calls this automatically and translates semantic keys to actual DB slots
|
||||
using the `tagSlotMapping` stored on the connector.
|
||||
|
||||
Return keys must match the `id` values declared in `tagDefinitions`.
|
||||
|
||||
```typescript
|
||||
mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
|
||||
const result: Record<string, unknown> = {}
|
||||
|
||||
// Validate arrays before casting — metadata may be malformed
|
||||
const labels = Array.isArray(metadata.labels) ? (metadata.labels as string[]) : []
|
||||
if (labels.length > 0) result.labels = labels.join(', ')
|
||||
|
||||
// Validate numbers — guard against NaN
|
||||
if (metadata.version != null) {
|
||||
const num = Number(metadata.version)
|
||||
if (!Number.isNaN(num)) result.version = num
|
||||
}
|
||||
|
||||
// Validate dates — guard against Invalid Date
|
||||
if (typeof metadata.lastModified === 'string') {
|
||||
const date = new Date(metadata.lastModified)
|
||||
if (!Number.isNaN(date.getTime())) result.lastModified = date
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
```
|
||||
|
||||
## External API Calls — Use `fetchWithRetry`
|
||||
|
||||
All external API calls must use `fetchWithRetry` from `@/lib/knowledge/documents/utils` instead of raw `fetch()`. This provides exponential backoff with retries on 429/502/503/504 errors. It returns a standard `Response` — all `.ok`, `.json()`, `.text()` checks work unchanged.
|
||||
|
||||
For `validateConfig` (user-facing, called on save), pass `VALIDATE_RETRY_OPTIONS` to cap wait time at ~7s. Background operations (`listDocuments`, `getDocument`) use the built-in defaults (5 retries, ~31s max).
|
||||
|
||||
```typescript
|
||||
import { VALIDATE_RETRY_OPTIONS, fetchWithRetry } from '@/lib/knowledge/documents/utils'
|
||||
|
||||
// Background sync — use defaults
|
||||
const response = await fetchWithRetry(url, {
|
||||
method: 'GET',
|
||||
headers: { Authorization: `Bearer ${accessToken}` },
|
||||
})
|
||||
|
||||
// validateConfig — tighter retry budget
|
||||
const response = await fetchWithRetry(url, { ... }, VALIDATE_RETRY_OPTIONS)
|
||||
```
|
||||
|
||||
## sourceUrl
|
||||
|
||||
If `ExternalDocument.sourceUrl` is set, the sync engine stores it on the document record. Always construct the full URL (not a relative path).
|
||||
|
||||
## Sync Engine Behavior (Do Not Modify)
|
||||
|
||||
The sync engine (`lib/knowledge/connectors/sync-engine.ts`) is connector-agnostic. It:
|
||||
1. Calls `listDocuments` with pagination until `hasMore` is false
|
||||
2. Compares `contentHash` to detect new/changed/unchanged documents
|
||||
3. Stores `sourceUrl` and calls `mapTags` on insert/update automatically
|
||||
4. Handles soft-delete of removed documents
|
||||
5. Resolves access tokens automatically — OAuth tokens are refreshed, API keys are decrypted from the `encryptedApiKey` column
|
||||
|
||||
You never need to modify the sync engine when adding a connector.
|
||||
|
||||
## Icon
|
||||
|
||||
The `icon` field on `ConnectorConfig` is used throughout the UI — in the connector list, the add-connector modal, and as the document icon in the knowledge base table (replacing the generic file type icon for connector-sourced documents). The icon is read from `CONNECTOR_REGISTRY[connectorType].icon` at runtime — no separate icon map to maintain.
|
||||
|
||||
If the service already has an icon in `apps/sim/components/icons.tsx` (from a tool integration), reuse it. Otherwise, ask the user to provide the SVG.
|
||||
|
||||
## Registering
|
||||
|
||||
Add one line to `apps/sim/connectors/registry.ts`:
|
||||
|
||||
```typescript
|
||||
import { {service}Connector } from '@/connectors/{service}'
|
||||
|
||||
export const CONNECTOR_REGISTRY: ConnectorRegistry = {
|
||||
// ... existing connectors ...
|
||||
{service}: {service}Connector,
|
||||
}
|
||||
```
|
||||
|
||||
## Reference Implementations
|
||||
|
||||
- **OAuth + contentDeferred**: `apps/sim/connectors/google-drive/google-drive.ts` — file download with metadata-based hash, `orderBy` for deterministic pagination
|
||||
- **OAuth + contentDeferred (blocks API)**: `apps/sim/connectors/notion/notion.ts` — complex block content extraction deferred to `getDocument`
|
||||
- **OAuth + contentDeferred (git)**: `apps/sim/connectors/github/github.ts` — blob SHA hash, tree listing
|
||||
- **OAuth + inline content**: `apps/sim/connectors/confluence/confluence.ts` — multiple config field types, `mapTags`, label fetching
|
||||
- **API key**: `apps/sim/connectors/fireflies/fireflies.ts` — GraphQL API with Bearer token auth
|
||||
|
||||
## Checklist
|
||||
|
||||
- [ ] Created `connectors/{service}/{service}.ts` with full ConnectorConfig
|
||||
- [ ] Created `connectors/{service}/index.ts` barrel export
|
||||
- [ ] **Auth configured correctly:**
|
||||
- OAuth: `auth.provider` matches an existing `OAuthService` in `lib/oauth/types.ts`
|
||||
- API key: `auth.label` and `auth.placeholder` set appropriately
|
||||
- [ ] **Selector fields configured correctly (if applicable):**
|
||||
- Every `type: 'selector'` field has a canonical pair (`short-input` or `dropdown` with same `canonicalParamId` and `mode: 'advanced'`)
|
||||
- `required` is identical on both fields in each canonical pair
|
||||
- `selectorKey` exists in `hooks/selectors/registry.ts`
|
||||
- `dependsOn` references selector field IDs (not `canonicalParamId`)
|
||||
- Dependency `canonicalParamId` values exist in `SELECTOR_CONTEXT_FIELDS`
|
||||
- [ ] `listDocuments` handles pagination with metadata-based content hashes
|
||||
- [ ] `contentDeferred: true` used if content requires per-doc API calls (file download, export, blocks fetch)
|
||||
- [ ] `contentHash` is metadata-based (not content-based) and identical between stub and `getDocument`
|
||||
- [ ] `sourceUrl` set on each ExternalDocument (full URL, not relative)
|
||||
- [ ] `metadata` includes source-specific data for tag mapping
|
||||
- [ ] `tagDefinitions` declared for each semantic key returned by `mapTags`
|
||||
- [ ] `mapTags` implemented if source has useful metadata (labels, dates, versions)
|
||||
- [ ] `validateConfig` verifies the source is accessible
|
||||
- [ ] All external API calls use `fetchWithRetry` (not raw `fetch`)
|
||||
- [ ] All optional config fields validated in `validateConfig`
|
||||
- [ ] Icon exists in `components/icons.tsx` (or asked user to provide SVG)
|
||||
- [ ] Registered in `connectors/registry.ts`
|
||||
@@ -0,0 +1,327 @@
|
||||
---
|
||||
name: "source-command-add-tools"
|
||||
description: "Create tool configurations for a Sim integration by reading API docs"
|
||||
---
|
||||
|
||||
# source-command-add-tools
|
||||
|
||||
Use this skill when the user asks to run the migrated source command `add-tools`.
|
||||
|
||||
## Command Template
|
||||
|
||||
# Add Tools Skill
|
||||
|
||||
You are an expert at creating tool configurations for Sim integrations. Your job is to read API documentation and create properly structured tool files.
|
||||
|
||||
## Your Task
|
||||
|
||||
When the user asks you to create tools for a service:
|
||||
1. Use Context7 or WebFetch to read the service's API documentation
|
||||
2. Create the tools directory structure
|
||||
3. Generate properly typed tool configurations
|
||||
|
||||
## Directory Structure
|
||||
|
||||
Create files in `apps/sim/tools/{service}/`:
|
||||
```
|
||||
tools/{service}/
|
||||
├── index.ts # Barrel export
|
||||
├── types.ts # Parameter & response types
|
||||
└── {action}.ts # Individual tool files (one per operation)
|
||||
```
|
||||
|
||||
## Tool Configuration Structure
|
||||
|
||||
Every tool MUST follow this exact structure:
|
||||
|
||||
```typescript
|
||||
import type { {ServiceName}{Action}Params } from '@/tools/{service}/types'
|
||||
import type { ToolConfig } from '@/tools/types'
|
||||
|
||||
interface {ServiceName}{Action}Response {
|
||||
success: boolean
|
||||
output: {
|
||||
// Define output structure here
|
||||
}
|
||||
}
|
||||
|
||||
export const {serviceName}{Action}Tool: ToolConfig<
|
||||
{ServiceName}{Action}Params,
|
||||
{ServiceName}{Action}Response
|
||||
> = {
|
||||
id: '{service}_{action}', // snake_case, matches tool name
|
||||
name: '{Service} {Action}', // Human readable
|
||||
description: 'Brief description', // One sentence
|
||||
version: '1.0.0',
|
||||
|
||||
// OAuth config (if service uses OAuth)
|
||||
oauth: {
|
||||
required: true,
|
||||
provider: '{service}', // Must match OAuth provider ID
|
||||
},
|
||||
|
||||
params: {
|
||||
// Hidden params (system-injected, only use hidden for oauth accessToken)
|
||||
accessToken: {
|
||||
type: 'string',
|
||||
required: true,
|
||||
visibility: 'hidden',
|
||||
description: 'OAuth access token',
|
||||
},
|
||||
// User-only params (credentials, api key, IDs user must provide)
|
||||
someId: {
|
||||
type: 'string',
|
||||
required: true,
|
||||
visibility: 'user-only',
|
||||
description: 'The ID of the resource',
|
||||
},
|
||||
// User-or-LLM params (everything else, can be provided by user OR computed by LLM)
|
||||
query: {
|
||||
type: 'string',
|
||||
required: false, // Use false for optional
|
||||
visibility: 'user-or-llm',
|
||||
description: 'Search query',
|
||||
},
|
||||
},
|
||||
|
||||
request: {
|
||||
url: (params) => `https://api.service.com/v1/resource/${params.id}`,
|
||||
method: 'POST',
|
||||
headers: (params) => ({
|
||||
Authorization: `Bearer ${params.accessToken}`,
|
||||
'Content-Type': 'application/json',
|
||||
}),
|
||||
body: (params) => ({
|
||||
// Request body - only for POST/PUT/PATCH
|
||||
// Trim ID fields to prevent copy-paste whitespace errors:
|
||||
// userId: params.userId?.trim(),
|
||||
}),
|
||||
},
|
||||
|
||||
transformResponse: async (response: Response) => {
|
||||
const data = await response.json()
|
||||
return {
|
||||
success: true,
|
||||
output: {
|
||||
// Map API response to output
|
||||
// Use ?? null for nullable fields
|
||||
// Use ?? [] for optional arrays
|
||||
},
|
||||
}
|
||||
},
|
||||
|
||||
outputs: {
|
||||
// Define each output field
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
## Critical Rules for Parameters
|
||||
|
||||
### Visibility Options
|
||||
- `'hidden'` - System-injected (OAuth tokens, internal params). User never sees.
|
||||
- `'user-only'` - User must provide (credentials, api keys, account-specific IDs)
|
||||
- `'user-or-llm'` - User provides OR LLM can compute (search queries, content, filters, most fall into this category)
|
||||
|
||||
### Parameter Types
|
||||
- `'string'` - Text values
|
||||
- `'number'` - Numeric values
|
||||
- `'boolean'` - True/false
|
||||
- `'json'` - Complex objects (NOT 'object', use 'json')
|
||||
- `'file'` - Single file
|
||||
- `'file[]'` - Multiple files
|
||||
|
||||
### Required vs Optional
|
||||
- Always explicitly set `required: true` or `required: false`
|
||||
- Optional params should have `required: false`
|
||||
|
||||
## Critical Rules for Outputs
|
||||
|
||||
### Output Types
|
||||
- `'string'`, `'number'`, `'boolean'` - Primitives
|
||||
- `'json'` - Complex objects (use this, NOT 'object')
|
||||
- `'array'` - Arrays with `items` property
|
||||
- `'object'` - Objects with `properties` property
|
||||
|
||||
### Optional Outputs
|
||||
Add `optional: true` for fields that may not exist in the response:
|
||||
```typescript
|
||||
closedAt: {
|
||||
type: 'string',
|
||||
description: 'When the issue was closed',
|
||||
optional: true,
|
||||
},
|
||||
```
|
||||
|
||||
### Typed JSON Outputs
|
||||
|
||||
When using `type: 'json'` and you know the object shape in advance, **always define the inner structure** using `properties` so downstream consumers know what fields are available:
|
||||
|
||||
```typescript
|
||||
// BAD: Opaque json with no info about what's inside
|
||||
metadata: {
|
||||
type: 'json',
|
||||
description: 'Response metadata',
|
||||
},
|
||||
|
||||
// GOOD: Define the known properties
|
||||
metadata: {
|
||||
type: 'json',
|
||||
description: 'Response metadata',
|
||||
properties: {
|
||||
id: { type: 'string', description: 'Unique ID' },
|
||||
status: { type: 'string', description: 'Current status' },
|
||||
count: { type: 'number', description: 'Total count' },
|
||||
},
|
||||
},
|
||||
```
|
||||
|
||||
For arrays of objects, define the item structure:
|
||||
```typescript
|
||||
items: {
|
||||
type: 'array',
|
||||
description: 'List of items',
|
||||
items: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
id: { type: 'string', description: 'Item ID' },
|
||||
name: { type: 'string', description: 'Item name' },
|
||||
},
|
||||
},
|
||||
},
|
||||
```
|
||||
|
||||
Only use bare `type: 'json'` without `properties` when the shape is truly dynamic or unknown.
|
||||
|
||||
## Critical Rules for transformResponse
|
||||
|
||||
### Handle Nullable Fields
|
||||
ALWAYS use `?? null` for fields that may be undefined:
|
||||
```typescript
|
||||
transformResponse: async (response: Response) => {
|
||||
const data = await response.json()
|
||||
return {
|
||||
success: true,
|
||||
output: {
|
||||
id: data.id,
|
||||
title: data.title,
|
||||
body: data.body ?? null, // May be undefined
|
||||
assignee: data.assignee ?? null, // May be undefined
|
||||
labels: data.labels ?? [], // Default to empty array
|
||||
closedAt: data.closed_at ?? null, // May be undefined
|
||||
},
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Never Output Raw JSON Dumps
|
||||
DON'T do this:
|
||||
```typescript
|
||||
output: {
|
||||
data: data, // BAD - raw JSON dump
|
||||
}
|
||||
```
|
||||
|
||||
DO this instead - extract meaningful fields:
|
||||
```typescript
|
||||
output: {
|
||||
id: data.id,
|
||||
name: data.name,
|
||||
status: data.status,
|
||||
metadata: {
|
||||
createdAt: data.created_at,
|
||||
updatedAt: data.updated_at,
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
## Types File Pattern
|
||||
|
||||
Create `types.ts` with interfaces for all params and responses:
|
||||
|
||||
```typescript
|
||||
import type { ToolResponse } from '@/tools/types'
|
||||
|
||||
// Parameter interfaces
|
||||
export interface {Service}{Action}Params {
|
||||
accessToken: string
|
||||
requiredField: string
|
||||
optionalField?: string
|
||||
}
|
||||
|
||||
// Response interfaces (extend ToolResponse)
|
||||
export interface {Service}{Action}Response extends ToolResponse {
|
||||
output: {
|
||||
field1: string
|
||||
field2: number
|
||||
optionalField?: string | null
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Index.ts Barrel Export Pattern
|
||||
|
||||
```typescript
|
||||
// Export all tools
|
||||
export { serviceTool1 } from './{action1}'
|
||||
export { serviceTool2 } from './{action2}'
|
||||
|
||||
// Export types
|
||||
export * from './types'
|
||||
```
|
||||
|
||||
## Registering Tools
|
||||
|
||||
After creating tools, remind the user to:
|
||||
1. Import tools in `apps/sim/tools/registry.ts`
|
||||
2. Add to the `tools` object with snake_case keys:
|
||||
```typescript
|
||||
import { serviceActionTool } from '@/tools/{service}'
|
||||
|
||||
export const tools = {
|
||||
// ... existing tools ...
|
||||
{service}_{action}: serviceActionTool,
|
||||
}
|
||||
```
|
||||
|
||||
## V2 Tool Pattern
|
||||
|
||||
If creating V2 tools (API-aligned outputs), use `_v2` suffix:
|
||||
- Tool ID: `{service}_{action}_v2`
|
||||
- Variable name: `{action}V2Tool`
|
||||
- Version: `'2.0.0'`
|
||||
- Outputs: Flat, API-aligned (no content/metadata wrapper)
|
||||
|
||||
## Naming Convention
|
||||
|
||||
All tool IDs MUST use `snake_case`: `{service}_{action}` (e.g., `x_create_tweet`, `slack_send_message`). Never use camelCase or PascalCase for tool IDs.
|
||||
|
||||
## Checklist Before Finishing
|
||||
|
||||
- [ ] All tool IDs use snake_case
|
||||
- [ ] All params have explicit `required: true` or `required: false`
|
||||
- [ ] All params have appropriate `visibility`
|
||||
- [ ] All nullable response fields use `?? null`
|
||||
- [ ] All optional outputs have `optional: true`
|
||||
- [ ] No raw JSON dumps in outputs
|
||||
- [ ] Types file has all interfaces
|
||||
- [ ] Index.ts exports all tools
|
||||
|
||||
## Final Validation (Required)
|
||||
|
||||
After creating all tools, you MUST validate every tool before finishing:
|
||||
|
||||
1. **Read every tool file** you created — do not skip any
|
||||
2. **Cross-reference with the API docs** to verify:
|
||||
- All required params are marked `required: true`
|
||||
- All optional params are marked `required: false`
|
||||
- Param types match the API (string, number, boolean, json)
|
||||
- Request URL, method, headers, and body match the API spec
|
||||
- `transformResponse` extracts the correct fields from the API response
|
||||
- All output fields match what the API actually returns
|
||||
- No fields are missing from outputs that the API provides
|
||||
- No extra fields are defined in outputs that the API doesn't return
|
||||
3. **Verify consistency** across tools:
|
||||
- Shared types in `types.ts` match all tools that use them
|
||||
- Tool IDs in the barrel export match the tool file definitions
|
||||
- Error handling is consistent (error checks, meaningful messages)
|
||||
@@ -0,0 +1,503 @@
|
||||
---
|
||||
name: "source-command-add-trigger"
|
||||
description: "Create webhook or polling triggers for a Sim integration"
|
||||
---
|
||||
|
||||
# source-command-add-trigger
|
||||
|
||||
Use this skill when the user asks to run the migrated source command `add-trigger`.
|
||||
|
||||
## Command Template
|
||||
|
||||
# Add Trigger
|
||||
|
||||
You are an expert at creating webhook and polling triggers for Sim. You understand the trigger system, the generic `buildTriggerSubBlocks` helper, polling infrastructure, and how triggers connect to blocks.
|
||||
|
||||
## Your Task
|
||||
|
||||
1. Research what webhook events the service supports — if the service lacks reliable webhooks, use polling
|
||||
2. Create the trigger files using the generic builder (webhook) or manual config (polling)
|
||||
3. Create a provider handler (webhook) or polling handler (polling)
|
||||
4. Register triggers and connect them to the block
|
||||
|
||||
## Directory Structure
|
||||
|
||||
```
|
||||
apps/sim/triggers/{service}/
|
||||
├── index.ts # Barrel exports
|
||||
├── utils.ts # Service-specific helpers (options, instructions, extra fields, outputs)
|
||||
├── {event_a}.ts # Primary trigger (includes dropdown)
|
||||
├── {event_b}.ts # Secondary trigger (no dropdown)
|
||||
└── webhook.ts # Generic webhook trigger (optional, for "all events")
|
||||
|
||||
apps/sim/lib/webhooks/
|
||||
├── provider-subscription-utils.ts # Shared subscription helpers (getProviderConfig, getNotificationUrl)
|
||||
├── providers/
|
||||
│ ├── {service}.ts # Provider handler (auth, formatInput, matchEvent, subscriptions)
|
||||
│ ├── types.ts # WebhookProviderHandler interface
|
||||
│ ├── utils.ts # Shared helpers (createHmacVerifier, verifyTokenAuth, skipByEventTypes)
|
||||
│ └── registry.ts # Handler map + default handler
|
||||
```
|
||||
|
||||
## Step 1: Create `utils.ts`
|
||||
|
||||
This file contains all service-specific helpers used by triggers.
|
||||
|
||||
```typescript
|
||||
import type { SubBlockConfig } from '@/blocks/types'
|
||||
import type { TriggerOutput } from '@/triggers/types'
|
||||
|
||||
export const {service}TriggerOptions = [
|
||||
{ label: 'Event A', id: '{service}_event_a' },
|
||||
{ label: 'Event B', id: '{service}_event_b' },
|
||||
]
|
||||
|
||||
export function {service}SetupInstructions(eventType: string): string {
|
||||
const instructions = [
|
||||
'Copy the <strong>Webhook URL</strong> above',
|
||||
'Go to <strong>{Service} Settings > Webhooks</strong>',
|
||||
`Select the <strong>${eventType}</strong> event type`,
|
||||
'Paste the webhook URL and save',
|
||||
'Click "Save" above to activate your trigger',
|
||||
]
|
||||
return instructions
|
||||
.map((instruction, index) =>
|
||||
`<div class="mb-3"><strong>${index + 1}.</strong> ${instruction}</div>`
|
||||
)
|
||||
.join('')
|
||||
}
|
||||
|
||||
export function build{Service}ExtraFields(triggerId: string): SubBlockConfig[] {
|
||||
return [
|
||||
{
|
||||
id: 'projectId',
|
||||
title: 'Project ID (Optional)',
|
||||
type: 'short-input',
|
||||
placeholder: 'Leave empty for all projects',
|
||||
mode: 'trigger',
|
||||
condition: { field: 'selectedTriggerId', value: triggerId },
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
export function build{Service}Outputs(): Record<string, TriggerOutput> {
|
||||
return {
|
||||
eventType: { type: 'string', description: 'The type of event' },
|
||||
resourceId: { type: 'string', description: 'ID of the affected resource' },
|
||||
resource: {
|
||||
id: { type: 'string', description: 'Resource ID' },
|
||||
name: { type: 'string', description: 'Resource name' },
|
||||
},
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Step 2: Create Trigger Files
|
||||
|
||||
**Primary trigger** — MUST include `includeDropdown: true`:
|
||||
|
||||
```typescript
|
||||
import { {Service}Icon } from '@/components/icons'
|
||||
import { buildTriggerSubBlocks } from '@/triggers'
|
||||
import { build{Service}ExtraFields, build{Service}Outputs, {service}SetupInstructions, {service}TriggerOptions } from '@/triggers/{service}/utils'
|
||||
import type { TriggerConfig } from '@/triggers/types'
|
||||
|
||||
export const {service}EventATrigger: TriggerConfig = {
|
||||
id: '{service}_event_a',
|
||||
name: '{Service} Event A',
|
||||
provider: '{service}',
|
||||
description: 'Trigger workflow when Event A occurs',
|
||||
version: '1.0.0',
|
||||
icon: {Service}Icon,
|
||||
subBlocks: buildTriggerSubBlocks({
|
||||
triggerId: '{service}_event_a',
|
||||
triggerOptions: {service}TriggerOptions,
|
||||
includeDropdown: true,
|
||||
setupInstructions: {service}SetupInstructions('Event A'),
|
||||
extraFields: build{Service}ExtraFields('{service}_event_a'),
|
||||
}),
|
||||
outputs: build{Service}Outputs(),
|
||||
webhook: { method: 'POST', headers: { 'Content-Type': 'application/json' } },
|
||||
}
|
||||
```
|
||||
|
||||
**Secondary triggers** — NO `includeDropdown` (it's already in the primary):
|
||||
|
||||
```typescript
|
||||
export const {service}EventBTrigger: TriggerConfig = {
|
||||
// Same as above but: id: '{service}_event_b', no includeDropdown
|
||||
}
|
||||
```
|
||||
|
||||
## Step 3: Register and Wire
|
||||
|
||||
### `apps/sim/triggers/{service}/index.ts`
|
||||
|
||||
```typescript
|
||||
export { {service}EventATrigger } from './event_a'
|
||||
export { {service}EventBTrigger } from './event_b'
|
||||
```
|
||||
|
||||
### `apps/sim/triggers/registry.ts`
|
||||
|
||||
```typescript
|
||||
import { {service}EventATrigger, {service}EventBTrigger } from '@/triggers/{service}'
|
||||
|
||||
export const TRIGGER_REGISTRY: TriggerRegistry = {
|
||||
// ... existing ...
|
||||
{service}_event_a: {service}EventATrigger,
|
||||
{service}_event_b: {service}EventBTrigger,
|
||||
}
|
||||
```
|
||||
|
||||
### Block file (`apps/sim/blocks/blocks/{service}.ts`)
|
||||
|
||||
Wire triggers into the block so the trigger UI appears and `generate-docs.ts` discovers them. Two changes are needed:
|
||||
|
||||
1. **Spread trigger subBlocks** at the end of the block's `subBlocks` array
|
||||
2. **Add `triggers` property** after `outputs` with `enabled: true` and `available: [...]`
|
||||
|
||||
```typescript
|
||||
import { getTrigger } from '@/triggers'
|
||||
|
||||
export const {Service}Block: BlockConfig = {
|
||||
// ...
|
||||
subBlocks: [
|
||||
// Regular tool subBlocks first...
|
||||
...getTrigger('{service}_event_a').subBlocks,
|
||||
...getTrigger('{service}_event_b').subBlocks,
|
||||
],
|
||||
// ... tools, inputs, outputs ...
|
||||
triggers: {
|
||||
enabled: true,
|
||||
available: ['{service}_event_a', '{service}_event_b'],
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
**Versioned blocks (V1 + V2):** Many integrations have a hidden V1 block and a visible V2 block. Where you add the trigger wiring depends on how V2 inherits from V1:
|
||||
|
||||
- **V2 uses `...V1Block` spread** (e.g., Google Calendar): Add trigger to V1 — V2 inherits both `subBlocks` and `triggers` automatically.
|
||||
- **V2 defines its own `subBlocks`** (e.g., Google Sheets): Add trigger to V2 (the visible block). V1 is hidden and doesn't need it.
|
||||
- **Single block, no V2** (e.g., Google Drive): Add trigger directly.
|
||||
|
||||
`generate-docs.ts` deduplicates by base type (first match wins). If V1 is processed first without triggers, the V2 triggers won't appear in `integrations.json`. Always verify by checking the output after running the script.
|
||||
|
||||
## Provider Handler
|
||||
|
||||
All provider-specific webhook logic lives in a single handler file: `apps/sim/lib/webhooks/providers/{service}.ts`.
|
||||
|
||||
### When to Create a Handler
|
||||
|
||||
| Behavior | Method | Examples |
|
||||
|---|---|---|
|
||||
| HMAC signature auth | `verifyAuth` via `createHmacVerifier` | Ashby, Jira, Linear, Typeform |
|
||||
| Custom token auth | `verifyAuth` via `verifyTokenAuth` | Generic, Google Forms |
|
||||
| Event filtering | `matchEvent` | GitHub, Jira, Attio, HubSpot |
|
||||
| Idempotency dedup | `extractIdempotencyId` | Slack, Stripe, Linear, Jira |
|
||||
| Custom input formatting | `formatInput` | Slack, Teams, Attio, Ashby |
|
||||
| Auto webhook creation | `createSubscription` | Ashby, Grain, Calendly, Airtable |
|
||||
| Auto webhook deletion | `deleteSubscription` | Ashby, Grain, Calendly, Airtable |
|
||||
| Challenge/verification | `handleChallenge` | Slack, WhatsApp, Teams |
|
||||
| Custom success response | `formatSuccessResponse` | Slack, Twilio Voice, Teams |
|
||||
|
||||
If none apply, you don't need a handler. The default handler provides bearer token auth.
|
||||
|
||||
### Example Handler
|
||||
|
||||
```typescript
|
||||
import crypto from 'crypto'
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { safeCompare } from '@/lib/core/security/encryption'
|
||||
import type { EventMatchContext, FormatInputContext, FormatInputResult, WebhookProviderHandler } from '@/lib/webhooks/providers/types'
|
||||
import { createHmacVerifier } from '@/lib/webhooks/providers/utils'
|
||||
|
||||
const logger = createLogger('WebhookProvider:{Service}')
|
||||
|
||||
function validate{Service}Signature(secret: string, signature: string, body: string): boolean {
|
||||
if (!secret || !signature || !body) return false
|
||||
const computed = crypto.createHmac('sha256', secret).update(body, 'utf8').digest('hex')
|
||||
return safeCompare(computed, signature)
|
||||
}
|
||||
|
||||
export const {service}Handler: WebhookProviderHandler = {
|
||||
verifyAuth: createHmacVerifier({
|
||||
configKey: 'webhookSecret',
|
||||
headerName: 'X-{Service}-Signature',
|
||||
validateFn: validate{Service}Signature,
|
||||
providerLabel: '{Service}',
|
||||
}),
|
||||
|
||||
async matchEvent({ body, requestId, providerConfig }: EventMatchContext) {
|
||||
const triggerId = providerConfig.triggerId as string | undefined
|
||||
if (triggerId && triggerId !== '{service}_webhook') {
|
||||
const { is{Service}EventMatch } = await import('@/triggers/{service}/utils')
|
||||
if (!is{Service}EventMatch(triggerId, body as Record<string, unknown>)) return false
|
||||
}
|
||||
return true
|
||||
},
|
||||
|
||||
async formatInput({ body }: FormatInputContext): Promise<FormatInputResult> {
|
||||
const b = body as Record<string, unknown>
|
||||
return {
|
||||
input: {
|
||||
eventType: b.type,
|
||||
resourceId: (b.data as Record<string, unknown>)?.id || '',
|
||||
resource: b.data,
|
||||
},
|
||||
}
|
||||
},
|
||||
|
||||
extractIdempotencyId(body: unknown) {
|
||||
const obj = body as Record<string, unknown>
|
||||
return obj.id && obj.type ? `${obj.type}:${obj.id}` : null
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
### Register the Handler
|
||||
|
||||
In `apps/sim/lib/webhooks/providers/registry.ts`:
|
||||
|
||||
```typescript
|
||||
import { {service}Handler } from '@/lib/webhooks/providers/{service}'
|
||||
|
||||
const PROVIDER_HANDLERS: Record<string, WebhookProviderHandler> = {
|
||||
// ... existing (alphabetical) ...
|
||||
{service}: {service}Handler,
|
||||
}
|
||||
```
|
||||
|
||||
## Output Alignment (Critical)
|
||||
|
||||
There are two sources of truth that **MUST be aligned**:
|
||||
|
||||
1. **Trigger `outputs`** — schema defining what fields SHOULD be available (UI tag dropdown)
|
||||
2. **`formatInput` on the handler** — implementation that transforms raw payload into actual data
|
||||
|
||||
If they differ: the tag dropdown shows fields that don't exist, or actual data has fields users can't discover.
|
||||
|
||||
**Rules for `formatInput`:**
|
||||
- Return `{ input: { ... } }` where inner keys match trigger `outputs` exactly
|
||||
- Return `{ input: ..., skip: { message: '...' } }` to skip execution
|
||||
- No wrapper objects or duplication
|
||||
- Use `null` for missing optional data
|
||||
|
||||
## Automatic Webhook Registration
|
||||
|
||||
If the service API supports programmatic webhook creation, implement `createSubscription` and `deleteSubscription` on the handler. The orchestration layer calls these automatically — **no code touches `route.ts`, `provider-subscriptions.ts`, or `deploy.ts`**.
|
||||
|
||||
```typescript
|
||||
import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils'
|
||||
import type { DeleteSubscriptionContext, SubscriptionContext, SubscriptionResult } from '@/lib/webhooks/providers/types'
|
||||
|
||||
export const {service}Handler: WebhookProviderHandler = {
|
||||
async createSubscription(ctx: SubscriptionContext): Promise<SubscriptionResult | undefined> {
|
||||
const config = getProviderConfig(ctx.webhook)
|
||||
const apiKey = config.apiKey as string
|
||||
if (!apiKey) throw new Error('{Service} API Key is required.')
|
||||
|
||||
const res = await fetch('https://api.{service}.com/webhooks', {
|
||||
method: 'POST',
|
||||
headers: { Authorization: `Bearer ${apiKey}`, 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ url: getNotificationUrl(ctx.webhook) }),
|
||||
})
|
||||
|
||||
if (!res.ok) throw new Error(`{Service} error: ${res.status}`)
|
||||
const { id } = (await res.json()) as { id: string }
|
||||
return { providerConfigUpdates: { externalId: id } }
|
||||
},
|
||||
|
||||
async deleteSubscription(ctx: DeleteSubscriptionContext): Promise<void> {
|
||||
const config = getProviderConfig(ctx.webhook)
|
||||
const { apiKey, externalId } = config as { apiKey?: string; externalId?: string }
|
||||
if (!apiKey || !externalId) return
|
||||
await fetch(`https://api.{service}.com/webhooks/${externalId}`, {
|
||||
method: 'DELETE',
|
||||
headers: { Authorization: `Bearer ${apiKey}` },
|
||||
}).catch(() => {})
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
**Key points:**
|
||||
- Throw from `createSubscription` — orchestration rolls back the DB webhook
|
||||
- Never throw from `deleteSubscription` — log non-fatally
|
||||
- Return `{ providerConfigUpdates: { externalId } }` — orchestration merges into `providerConfig`
|
||||
- Add `apiKey` field to `build{Service}ExtraFields` with `password: true`
|
||||
|
||||
## Trigger Outputs Schema
|
||||
|
||||
Trigger outputs use the same schema as block outputs (NOT tool outputs).
|
||||
|
||||
**Supported:** `type` + `description` for leaf fields, nested objects for complex data.
|
||||
**NOT supported:** `optional: true`, `items` (those are tool-output-only features).
|
||||
|
||||
```typescript
|
||||
export function buildOutputs(): Record<string, TriggerOutput> {
|
||||
return {
|
||||
eventType: { type: 'string', description: 'Event type' },
|
||||
timestamp: { type: 'string', description: 'When it occurred' },
|
||||
payload: { type: 'json', description: 'Full event payload' },
|
||||
resource: {
|
||||
id: { type: 'string', description: 'Resource ID' },
|
||||
name: { type: 'string', description: 'Resource name' },
|
||||
},
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Polling Triggers
|
||||
|
||||
Use polling when the service lacks reliable webhooks (e.g., Google Sheets, Google Drive, Google Calendar, Gmail, RSS, IMAP). Polling triggers do NOT use `buildTriggerSubBlocks` — they define subBlocks manually.
|
||||
|
||||
### Directory Structure
|
||||
|
||||
```
|
||||
apps/sim/triggers/{service}/
|
||||
├── index.ts # Barrel export
|
||||
└── poller.ts # TriggerConfig with polling: true
|
||||
|
||||
apps/sim/lib/webhooks/polling/
|
||||
└── {service}.ts # PollingProviderHandler implementation
|
||||
```
|
||||
|
||||
### Polling Handler (`apps/sim/lib/webhooks/polling/{service}.ts`)
|
||||
|
||||
```typescript
|
||||
import { pollingIdempotency } from '@/lib/core/idempotency/service'
|
||||
import type { PollingProviderHandler, PollWebhookContext } from '@/lib/webhooks/polling/types'
|
||||
import { markWebhookFailed, markWebhookSuccess, resolveOAuthCredential, updateWebhookProviderConfig } from '@/lib/webhooks/polling/utils'
|
||||
import { processPolledWebhookEvent } from '@/lib/webhooks/processor'
|
||||
|
||||
export const {service}PollingHandler: PollingProviderHandler = {
|
||||
provider: '{service}',
|
||||
label: '{Service}',
|
||||
|
||||
async pollWebhook(ctx: PollWebhookContext): Promise<'success' | 'failure'> {
|
||||
const { webhookData, workflowData, requestId, logger } = ctx
|
||||
const webhookId = webhookData.id
|
||||
|
||||
try {
|
||||
// For OAuth services:
|
||||
const accessToken = await resolveOAuthCredential(webhookData, '{service}', requestId, logger)
|
||||
const config = webhookData.providerConfig as unknown as {Service}WebhookConfig
|
||||
|
||||
// First poll: seed state, emit nothing
|
||||
if (!config.lastCheckedTimestamp) {
|
||||
await updateWebhookProviderConfig(webhookId, { lastCheckedTimestamp: new Date().toISOString() }, logger)
|
||||
await markWebhookSuccess(webhookId, logger)
|
||||
return 'success'
|
||||
}
|
||||
|
||||
// Fetch changes since last poll, process with idempotency
|
||||
// ...
|
||||
|
||||
await markWebhookSuccess(webhookId, logger)
|
||||
return 'success'
|
||||
} catch (error) {
|
||||
logger.error(`[${requestId}] Error processing {service} webhook ${webhookId}:`, error)
|
||||
await markWebhookFailed(webhookId, logger)
|
||||
return 'failure'
|
||||
}
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
**Key patterns:**
|
||||
- First poll seeds state and emits nothing (avoids flooding with existing data)
|
||||
- Use `pollingIdempotency.executeWithIdempotency(provider, key, callback)` for dedup
|
||||
- Use `processPolledWebhookEvent(webhookData, workflowData, payload, requestId)` to fire the workflow
|
||||
- Use `updateWebhookProviderConfig(webhookId, partialConfig, logger)` for read-merge-write on state
|
||||
- Use the latest server-side timestamp from API responses (not wall clock) to avoid clock skew
|
||||
|
||||
### Trigger Config (`apps/sim/triggers/{service}/poller.ts`)
|
||||
|
||||
```typescript
|
||||
import { {Service}Icon } from '@/components/icons'
|
||||
import type { TriggerConfig } from '@/triggers/types'
|
||||
|
||||
export const {service}PollingTrigger: TriggerConfig = {
|
||||
id: '{service}_poller',
|
||||
name: '{Service} Trigger',
|
||||
provider: '{service}',
|
||||
description: 'Triggers when ...',
|
||||
version: '1.0.0',
|
||||
icon: {Service}Icon,
|
||||
polling: true, // REQUIRED — routes to polling infrastructure
|
||||
|
||||
subBlocks: [
|
||||
{ id: 'triggerCredentials', type: 'oauth-input', title: 'Credentials', serviceId: '{service}', requiredScopes: [], required: true, mode: 'trigger', supportsCredentialSets: true },
|
||||
// ... service-specific config fields (dropdowns, inputs, switches) ...
|
||||
{ id: 'triggerInstructions', type: 'text', title: 'Setup Instructions', hideFromPreview: true, mode: 'trigger', defaultValue: '...' },
|
||||
],
|
||||
|
||||
outputs: {
|
||||
// Must match the payload shape from processPolledWebhookEvent
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
### Registration (3 places)
|
||||
|
||||
1. **`apps/sim/triggers/constants.ts`** — add provider to `POLLING_PROVIDERS` Set
|
||||
2. **`apps/sim/lib/webhooks/polling/registry.ts`** — import handler, add to `POLLING_HANDLERS`
|
||||
3. **`apps/sim/triggers/registry.ts`** — import trigger config, add to `TRIGGER_REGISTRY`
|
||||
|
||||
### Helm Cron Job
|
||||
|
||||
Add to `helm/sim/values.yaml` under the existing polling cron jobs:
|
||||
|
||||
```yaml
|
||||
{service}WebhookPoll:
|
||||
schedule: "*/1 * * * *"
|
||||
concurrencyPolicy: Forbid
|
||||
url: "http://sim:3000/api/webhooks/poll/{service}"
|
||||
```
|
||||
|
||||
### Reference Implementations
|
||||
|
||||
- Simple: `apps/sim/lib/webhooks/polling/rss.ts` + `apps/sim/triggers/rss/poller.ts`
|
||||
- Complex (OAuth, attachments): `apps/sim/lib/webhooks/polling/gmail.ts` + `apps/sim/triggers/gmail/poller.ts`
|
||||
- Cursor-based (changes API): `apps/sim/lib/webhooks/polling/google-drive.ts`
|
||||
- Timestamp-based: `apps/sim/lib/webhooks/polling/google-calendar.ts`
|
||||
|
||||
## Checklist
|
||||
|
||||
### Trigger Definition
|
||||
- [ ] Created `utils.ts` with options, instructions, extra fields, and output builders
|
||||
- [ ] Primary trigger has `includeDropdown: true`; secondary triggers do NOT
|
||||
- [ ] All triggers use `buildTriggerSubBlocks` helper
|
||||
- [ ] Created `index.ts` barrel export
|
||||
|
||||
### Registration
|
||||
- [ ] All triggers in `triggers/registry.ts` → `TRIGGER_REGISTRY`
|
||||
- [ ] Block has `triggers.enabled: true` and lists all trigger IDs in `triggers.available`
|
||||
- [ ] Block spreads all trigger subBlocks: `...getTrigger('id').subBlocks`
|
||||
|
||||
### Provider Handler (if needed)
|
||||
- [ ] Handler file at `apps/sim/lib/webhooks/providers/{service}.ts`
|
||||
- [ ] Registered in `providers/registry.ts` (alphabetical)
|
||||
- [ ] Signature validator is a private function inside the handler file
|
||||
- [ ] `formatInput` output keys match trigger `outputs` exactly
|
||||
- [ ] Event matching uses dynamic `await import()` for trigger utils
|
||||
|
||||
### Auto Registration (if supported)
|
||||
- [ ] `createSubscription` and `deleteSubscription` on the handler
|
||||
- [ ] NO changes to `route.ts`, `provider-subscriptions.ts`, or `deploy.ts`
|
||||
- [ ] API key field uses `password: true`
|
||||
|
||||
### Polling Trigger (if applicable)
|
||||
- [ ] Handler implements `PollingProviderHandler` at `lib/webhooks/polling/{service}.ts`
|
||||
- [ ] Trigger config has `polling: true` and defines subBlocks manually (no `buildTriggerSubBlocks`)
|
||||
- [ ] Provider string matches across: trigger config, handler, `POLLING_PROVIDERS`, polling registry
|
||||
- [ ] First poll seeds state and emits nothing
|
||||
- [ ] Added provider to `POLLING_PROVIDERS` in `triggers/constants.ts`
|
||||
- [ ] Added handler to `POLLING_HANDLERS` in `lib/webhooks/polling/registry.ts`
|
||||
- [ ] Added cron job to `helm/sim/values.yaml`
|
||||
- [ ] Payload shape matches trigger `outputs` schema
|
||||
|
||||
### Testing
|
||||
- [ ] `bun run type-check` passes
|
||||
- [ ] Manually verify output keys match trigger `outputs` keys
|
||||
- [ ] Trigger UI shows correctly in the block
|
||||
@@ -0,0 +1,18 @@
|
||||
---
|
||||
name: "source-command-council"
|
||||
description: "Spawn task agents to explore a given area of interest in the codebase"
|
||||
---
|
||||
|
||||
# source-command-council
|
||||
|
||||
Use this skill when the user asks to run the migrated source command `council`.
|
||||
|
||||
## Command Template
|
||||
|
||||
Based on the given area of interest, please:
|
||||
|
||||
1. Dig around the codebase in terms of that given area of interest, gather general information such as keywords and architecture overview.
|
||||
2. Spawn off n=10 (unless specified otherwise) task agents to dig deeper into the codebase in terms of that given area of interest, some of them should be out of the box for variance.
|
||||
3. Once the task agents are done, use the information to do what the user wants.
|
||||
|
||||
If user is in plan mode, use the information to create the plan.
|
||||
@@ -0,0 +1,90 @@
|
||||
---
|
||||
name: "source-command-ship"
|
||||
description: "Commit, push, and open a PR to staging in one shot"
|
||||
---
|
||||
|
||||
# source-command-ship
|
||||
|
||||
Use this skill when the user asks to run the migrated source command `ship`.
|
||||
|
||||
## Command Template
|
||||
|
||||
# Ship Command
|
||||
|
||||
You help ship code by creating commits, pushing to the remote branch, and creating PRs in the user's voice.
|
||||
|
||||
## Your Task
|
||||
|
||||
When the user runs `/ship`:
|
||||
|
||||
1. **Check git status** - See what files have changed
|
||||
2. **Generate a commit message** following this format: `type(scope): description`
|
||||
- Types: `fix`, `feat`, `improvement`, `chore`
|
||||
- Scope: short identifier (e.g., `undo-redo`, `api`, `ui`)
|
||||
- Keep it concise
|
||||
|
||||
3. **Run pre-ship checks** from the repo root before staging:
|
||||
- `bun run lint` to fix formatting issues
|
||||
- `bun run check:api-validation:strict` to catch boundary contract failures before CI
|
||||
|
||||
4. **Stage and commit** the changes with the generated message
|
||||
|
||||
5. **Push to origin** using the current branch name
|
||||
|
||||
6. **Create a PR** to staging with a description in the user's voice
|
||||
|
||||
## Commit Message Format
|
||||
|
||||
Based on the repo's commit history:
|
||||
```
|
||||
fix(scope): description for bug fixes
|
||||
feat(scope): description for new features
|
||||
improvement(scope): description for enhancements
|
||||
chore(scope): description for maintenance
|
||||
```
|
||||
|
||||
## PR Description Format
|
||||
|
||||
Use this exact template in the user's voice (concise, bullet points):
|
||||
|
||||
```markdown
|
||||
## Summary
|
||||
- bullet point describing what changed
|
||||
- another bullet point if needed
|
||||
|
||||
## Type of Change
|
||||
- [x] Bug fix (or appropriate type)
|
||||
|
||||
## Testing
|
||||
Tested manually (or describe testing)
|
||||
|
||||
## Checklist
|
||||
- [x] Code follows project style guidelines
|
||||
- [x] Self-reviewed my changes
|
||||
- [ ] Tests added/updated and passing
|
||||
- [x] No new warnings introduced
|
||||
- [x] I confirm that I have read and agree to the terms outlined in the [Contributor License Agreement (CLA)](./CONTRIBUTING.md#contributor-license-agreement-cla)
|
||||
```
|
||||
|
||||
## PR Creation Command
|
||||
|
||||
Use this command structure:
|
||||
```bash
|
||||
gh pr create --base staging --title "COMMIT_MESSAGE" --body "PR_BODY"
|
||||
```
|
||||
|
||||
## Important Notes
|
||||
|
||||
- Always confirm the commit message and PR description with the user before executing
|
||||
- The PR should be created against `staging` branch
|
||||
- Keep descriptions concise and in active voice
|
||||
- Match the user's previous PR style: direct, no fluff, bullet points
|
||||
- **DO NOT add "Co-Authored-By" lines to commits** - keep commit messages clean
|
||||
|
||||
## User's Voice Characteristics (based on previous PRs)
|
||||
|
||||
- Short, direct bullet points
|
||||
- No unnecessary explanation
|
||||
- "Tested manually" is acceptable for testing section; include lint and boundary validation results when run
|
||||
- Checkboxes filled in appropriately
|
||||
- No screenshots section unless UI changes
|
||||
@@ -0,0 +1,322 @@
|
||||
---
|
||||
name: "source-command-validate-connector"
|
||||
description: "Validate an existing knowledge base connector against its service's API docs"
|
||||
---
|
||||
|
||||
# source-command-validate-connector
|
||||
|
||||
Use this skill when the user asks to run the migrated source command `validate-connector`.
|
||||
|
||||
## Command Template
|
||||
|
||||
# Validate Connector Skill
|
||||
|
||||
You are an expert auditor for Sim knowledge base connectors. Your job is to thoroughly validate that an existing connector is correct, complete, and follows all conventions.
|
||||
|
||||
## Your Task
|
||||
|
||||
When the user asks you to validate a connector:
|
||||
1. Read the service's API documentation (via Context7 or WebFetch)
|
||||
2. Read the connector implementation, OAuth config, and registry entries
|
||||
3. Cross-reference everything against the API docs and Sim conventions
|
||||
4. Report all issues found, grouped by severity (critical, warning, suggestion)
|
||||
5. Fix all issues after reporting them
|
||||
|
||||
## Step 1: Gather All Files
|
||||
|
||||
Read **every** file for the connector — do not skip any:
|
||||
|
||||
```
|
||||
apps/sim/connectors/{service}/{service}.ts # Connector implementation
|
||||
apps/sim/connectors/{service}/index.ts # Barrel export
|
||||
apps/sim/connectors/registry.ts # Connector registry entry
|
||||
apps/sim/connectors/types.ts # ConnectorConfig interface, ExternalDocument, etc.
|
||||
apps/sim/connectors/utils.ts # Shared utilities (computeContentHash, htmlToPlainText, etc.)
|
||||
apps/sim/lib/oauth/oauth.ts # OAUTH_PROVIDERS — single source of truth for scopes
|
||||
apps/sim/lib/oauth/utils.ts # getCanonicalScopesForProvider, getScopesForService, SCOPE_DESCRIPTIONS
|
||||
apps/sim/lib/oauth/types.ts # OAuthService union type
|
||||
apps/sim/components/icons.tsx # Icon definition for the service
|
||||
```
|
||||
|
||||
If the connector uses selectors, also read:
|
||||
```
|
||||
apps/sim/hooks/selectors/registry.ts # Selector key definitions
|
||||
apps/sim/hooks/selectors/types.ts # SelectorKey union type
|
||||
apps/sim/lib/workflows/subblocks/context.ts # SELECTOR_CONTEXT_FIELDS
|
||||
```
|
||||
|
||||
## Step 2: Pull API Documentation
|
||||
|
||||
Fetch the official API docs for the service. This is the **source of truth** for:
|
||||
- Endpoint URLs, HTTP methods, and auth headers
|
||||
- Required vs optional parameters
|
||||
- Parameter types and allowed values
|
||||
- Response shapes and field names
|
||||
- Pagination patterns (cursor, offset, next token)
|
||||
- Rate limits and error formats
|
||||
- OAuth scopes and their meanings
|
||||
|
||||
Use Context7 (resolve-library-id → query-docs) or WebFetch to retrieve documentation. If both fail, note which claims are based on training knowledge vs verified docs.
|
||||
|
||||
## Step 3: Validate API Endpoints
|
||||
|
||||
For **every** API call in the connector (`listDocuments`, `getDocument`, `validateConfig`, and any helper functions), verify against the API docs:
|
||||
|
||||
### URLs and Methods
|
||||
- [ ] Base URL is correct for the service's API version
|
||||
- [ ] Endpoint paths match the API docs exactly
|
||||
- [ ] HTTP method is correct (GET, POST, PUT, PATCH, DELETE)
|
||||
- [ ] Path parameters are correctly interpolated and URI-encoded where needed
|
||||
- [ ] Query parameters use correct names and formats per the API docs
|
||||
|
||||
### Headers
|
||||
- [ ] Authorization header uses the correct format:
|
||||
- OAuth: `Authorization: Bearer ${accessToken}`
|
||||
- API Key: correct header name per the service's docs
|
||||
- [ ] `Content-Type` is set for POST/PUT/PATCH requests
|
||||
- [ ] Any service-specific headers are present (e.g., `Notion-Version`, `Dropbox-API-Arg`)
|
||||
- [ ] No headers are sent that the API doesn't support or silently ignores
|
||||
|
||||
### Request Bodies
|
||||
- [ ] POST/PUT body fields match API parameter names exactly
|
||||
- [ ] Required fields are always sent
|
||||
- [ ] Optional fields are conditionally included (not sent as `null` or empty unless the API expects that)
|
||||
- [ ] Field value types match API expectations (string vs number vs boolean)
|
||||
|
||||
### Input Sanitization
|
||||
- [ ] User-controlled values interpolated into query strings are properly escaped:
|
||||
- OData `$filter`: single quotes escaped with `''` (e.g., `externalId.replace(/'/g, "''")`)
|
||||
- SOQL: single quotes escaped with `\'`
|
||||
- GraphQL variables: passed as variables, not interpolated into query strings
|
||||
- URL path segments: `encodeURIComponent()` applied
|
||||
- [ ] URL-type config fields (e.g., `siteUrl`, `instanceUrl`) are normalized:
|
||||
- Strip `https://` / `http://` prefix if the API expects bare domains
|
||||
- Strip trailing `/`
|
||||
- Apply `.trim()` before validation
|
||||
|
||||
### Response Parsing
|
||||
- [ ] Response structure is correctly traversed (e.g., `data.results` vs `data.items` vs `data`)
|
||||
- [ ] Field names extracted match what the API actually returns
|
||||
- [ ] Nullable fields are handled with `?? null` or `|| undefined`
|
||||
- [ ] Error responses are checked before accessing data fields
|
||||
|
||||
## Step 4: Validate OAuth Scopes (if OAuth connector)
|
||||
|
||||
Scopes must be correctly declared and sufficient for all API calls the connector makes.
|
||||
|
||||
### Connector requiredScopes
|
||||
- [ ] `requiredScopes` in the connector's `auth` config lists all scopes needed by the connector
|
||||
- [ ] Each scope in `requiredScopes` is a real, valid scope recognized by the service's API
|
||||
- [ ] No invalid, deprecated, or made-up scopes are listed
|
||||
- [ ] No unnecessary excess scopes beyond what the connector actually needs
|
||||
|
||||
### Scope Subset Validation (CRITICAL)
|
||||
- [ ] Every scope in `requiredScopes` exists in the OAuth provider's `scopes` array in `lib/oauth/oauth.ts`
|
||||
- [ ] Find the provider in `OAUTH_PROVIDERS[providerGroup].services[serviceId].scopes`
|
||||
- [ ] Verify: `requiredScopes` ⊆ `OAUTH_PROVIDERS scopes` (every required scope is present in the provider config)
|
||||
- [ ] If a required scope is NOT in the provider config, flag as **critical** — the connector will fail at runtime
|
||||
|
||||
### Scope Sufficiency
|
||||
For each API endpoint the connector calls:
|
||||
- [ ] Identify which scopes are required per the API docs
|
||||
- [ ] Verify those scopes are included in the connector's `requiredScopes`
|
||||
- [ ] If the connector calls endpoints requiring scopes not in `requiredScopes`, flag as **warning**
|
||||
|
||||
### Token Refresh Config
|
||||
- [ ] Check the `getOAuthTokenRefreshConfig` function in `lib/oauth/oauth.ts` for this provider
|
||||
- [ ] `useBasicAuth` matches the service's token exchange requirements
|
||||
- [ ] `supportsRefreshTokenRotation` matches whether the service issues rotating refresh tokens
|
||||
- [ ] Token endpoint URL is correct
|
||||
|
||||
## Step 5: Validate Pagination
|
||||
|
||||
### listDocuments Pagination
|
||||
- [ ] Cursor/pagination parameter name matches the API docs
|
||||
- [ ] Response pagination field is correctly extracted (e.g., `next_cursor`, `nextPageToken`, `@odata.nextLink`, `offset`)
|
||||
- [ ] `hasMore` is correctly determined from the response
|
||||
- [ ] `nextCursor` is correctly passed back for the next page
|
||||
- [ ] `maxItems` / `maxRecords` cap is correctly applied across pages using `syncContext.totalDocsFetched`
|
||||
- [ ] Page size is within the API's allowed range (not exceeding max page size)
|
||||
- [ ] Last page precision: when a `maxItems` cap exists, the final page request uses `Math.min(PAGE_SIZE, remaining)` to avoid fetching more records than needed
|
||||
- [ ] No off-by-one errors in pagination tracking
|
||||
- [ ] The connector does NOT hit known API pagination limits silently (e.g., HubSpot search 10k cap)
|
||||
|
||||
### Pagination State Across Pages
|
||||
- [ ] `syncContext` is used to cache state across pages (user names, field maps, instance URLs, portal IDs, etc.)
|
||||
- [ ] Cached state in `syncContext` is correctly initialized on first page and reused on subsequent pages
|
||||
|
||||
## Step 6: Validate Data Transformation
|
||||
|
||||
### ExternalDocument Construction
|
||||
- [ ] `externalId` is a stable, unique identifier from the source API
|
||||
- [ ] `title` is extracted from the correct field and has a sensible fallback (e.g., `'Untitled'`)
|
||||
- [ ] `content` is plain text — HTML content is stripped using `htmlToPlainText` from `@/connectors/utils`
|
||||
- [ ] `mimeType` is `'text/plain'`
|
||||
- [ ] `contentHash` is computed using `computeContentHash` from `@/connectors/utils`
|
||||
- [ ] `sourceUrl` is a valid, complete URL back to the original resource (not relative)
|
||||
- [ ] `metadata` contains all fields referenced by `mapTags` and `tagDefinitions`
|
||||
|
||||
### Content Extraction
|
||||
- [ ] Rich text / HTML fields are converted to plain text before indexing
|
||||
- [ ] Important content is not silently dropped (e.g., nested blocks, table cells, code blocks)
|
||||
- [ ] Content is not silently truncated without logging a warning
|
||||
- [ ] Empty/blank documents are properly filtered out
|
||||
- [ ] Size checks use `Buffer.byteLength(text, 'utf8')` not `text.length` when comparing against byte-based limits (e.g., `MAX_FILE_SIZE` in bytes)
|
||||
|
||||
## Step 7: Validate Tag Definitions and mapTags
|
||||
|
||||
### tagDefinitions
|
||||
- [ ] Each `tagDefinition` has an `id`, `displayName`, and `fieldType`
|
||||
- [ ] `fieldType` matches the actual data type: `'text'` for strings, `'number'` for numbers, `'date'` for dates, `'boolean'` for booleans
|
||||
- [ ] Every `id` in `tagDefinitions` is returned by `mapTags`
|
||||
- [ ] No `tagDefinition` references a field that `mapTags` never produces
|
||||
|
||||
### mapTags
|
||||
- [ ] Return keys match `tagDefinition` `id` values exactly
|
||||
- [ ] Date values are properly parsed using `parseTagDate` from `@/connectors/utils`
|
||||
- [ ] Array values are properly joined using `joinTagArray` from `@/connectors/utils`
|
||||
- [ ] Number values are validated (not `NaN`)
|
||||
- [ ] Metadata field names accessed in `mapTags` match what `listDocuments`/`getDocument` store in `metadata`
|
||||
|
||||
## Step 8: Validate Config Fields and Validation
|
||||
|
||||
### configFields
|
||||
- [ ] Every field has `id`, `title`, `type`
|
||||
- [ ] `required` is set explicitly (not omitted)
|
||||
- [ ] Dropdown fields have `options` with `label` and `id` for each option
|
||||
- [ ] Selector fields follow the canonical pair pattern:
|
||||
- A `type: 'selector'` field with `selectorKey`, `canonicalParamId`, `mode: 'basic'`
|
||||
- A `type: 'short-input'` field with the same `canonicalParamId`, `mode: 'advanced'`
|
||||
- `required` is identical on both fields in the pair
|
||||
- [ ] `selectorKey` values exist in the selector registry
|
||||
- [ ] `dependsOn` references selector field `id` values, not `canonicalParamId`
|
||||
|
||||
### validateConfig
|
||||
- [ ] Validates all required fields are present before making API calls
|
||||
- [ ] Validates optional numeric fields (checks `Number.isNaN`, positive values)
|
||||
- [ ] Makes a lightweight API call to verify access (e.g., fetch 1 record, get profile)
|
||||
- [ ] Uses `VALIDATE_RETRY_OPTIONS` for retry budget
|
||||
- [ ] Returns `{ valid: true }` on success
|
||||
- [ ] Returns `{ valid: false, error: 'descriptive message' }` on failure
|
||||
- [ ] Catches exceptions and returns user-friendly error messages
|
||||
- [ ] Does NOT make expensive calls (full data listing, large queries)
|
||||
|
||||
## Step 9: Validate getDocument
|
||||
|
||||
- [ ] Fetches a single document by `externalId`
|
||||
- [ ] Returns `null` for 404 / not found (does not throw)
|
||||
- [ ] Returns the same `ExternalDocument` shape as `listDocuments`
|
||||
- [ ] Handles all content types that `listDocuments` can produce (e.g., if `listDocuments` returns both pages and blogposts, `getDocument` must handle both — not hardcode one endpoint)
|
||||
- [ ] Forwards `syncContext` if it needs cached state (user names, field maps, etc.)
|
||||
- [ ] Error handling is graceful (catches, logs, returns null or throws with context)
|
||||
- [ ] Does not redundantly re-fetch data already included in the initial API response (e.g., if comments come back with the post, don't fetch them again separately)
|
||||
|
||||
## Step 10: Validate General Quality
|
||||
|
||||
### fetchWithRetry Usage
|
||||
- [ ] All external API calls use `fetchWithRetry` from `@/lib/knowledge/documents/utils`
|
||||
- [ ] No raw `fetch()` calls to external APIs
|
||||
- [ ] `VALIDATE_RETRY_OPTIONS` used in `validateConfig`
|
||||
- [ ] If `validateConfig` calls a shared helper (e.g., `linearGraphQL`, `resolveId`), that helper must accept and forward `retryOptions` to `fetchWithRetry`
|
||||
- [ ] Default retry options used in `listDocuments`/`getDocument`
|
||||
|
||||
### API Efficiency
|
||||
- [ ] APIs that support field selection (e.g., `$select`, `sysparm_fields`, `fields`) should request only the fields the connector needs — in both `listDocuments` AND `getDocument`
|
||||
- [ ] No redundant API calls: if a helper already fetches data (e.g., site metadata), callers should reuse the result instead of making a second call for the same information
|
||||
- [ ] Sequential per-item API calls (fetching details for each document in a loop) should be batched with `Promise.all` and a concurrency limit of 3-5
|
||||
|
||||
### Error Handling
|
||||
- [ ] Individual document failures are caught and logged without aborting the sync
|
||||
- [ ] API error responses include status codes in error messages
|
||||
- [ ] No unhandled promise rejections in concurrent operations
|
||||
|
||||
### Concurrency
|
||||
- [ ] Concurrent API calls use reasonable batch sizes (3-5 is typical)
|
||||
- [ ] No unbounded `Promise.all` over large arrays
|
||||
|
||||
### Logging
|
||||
- [ ] Uses `createLogger` from `@sim/logger` (not `console.log`)
|
||||
- [ ] Logs sync progress at `info` level
|
||||
- [ ] Logs errors at `warn` or `error` level with context
|
||||
|
||||
### Registry
|
||||
- [ ] Connector is exported from `connectors/{service}/index.ts`
|
||||
- [ ] Connector is registered in `connectors/registry.ts`
|
||||
- [ ] Registry key matches the connector's `id` field
|
||||
|
||||
## Step 11: Report and Fix
|
||||
|
||||
### Report Format
|
||||
|
||||
Group findings by severity:
|
||||
|
||||
**Critical** (will cause runtime errors, data loss, or auth failures):
|
||||
- Wrong API endpoint URL or HTTP method
|
||||
- Invalid or missing OAuth scopes (not in provider config)
|
||||
- Incorrect response field mapping (accessing wrong path)
|
||||
- SOQL/query fields that don't exist on the target object
|
||||
- Pagination that silently hits undocumented API limits
|
||||
- Missing error handling that would crash the sync
|
||||
- `requiredScopes` not a subset of OAuth provider scopes
|
||||
- Query/filter injection: user-controlled values interpolated into OData `$filter`, SOQL, or query strings without escaping
|
||||
|
||||
**Warning** (incorrect behavior, data quality issues, or convention violations):
|
||||
- HTML content not stripped via `htmlToPlainText`
|
||||
- `getDocument` not forwarding `syncContext`
|
||||
- `getDocument` hardcoded to one content type when `listDocuments` returns multiple (e.g., only pages but not blogposts)
|
||||
- Missing `tagDefinition` for metadata fields returned by `mapTags`
|
||||
- Incorrect `useBasicAuth` or `supportsRefreshTokenRotation` in token refresh config
|
||||
- Invalid scope names that the API doesn't recognize (even if silently ignored)
|
||||
- Private resources excluded from name-based lookup despite scopes being available
|
||||
- Silent data truncation without logging
|
||||
- Size checks using `text.length` (character count) instead of `Buffer.byteLength` (byte count) for byte-based limits
|
||||
- URL-type config fields not normalized (protocol prefix, trailing slashes cause API failures)
|
||||
- `VALIDATE_RETRY_OPTIONS` not threaded through helper functions called by `validateConfig`
|
||||
|
||||
**Suggestion** (minor improvements):
|
||||
- Missing incremental sync support despite API supporting it
|
||||
- Overly broad scopes that could be narrowed (not wrong, but could be tighter)
|
||||
- Source URL format could be more specific
|
||||
- Missing `orderBy` for deterministic pagination
|
||||
- Redundant API calls that could be cached in `syncContext`
|
||||
- Sequential per-item API calls that could be batched with `Promise.all` (concurrency 3-5)
|
||||
- API supports field selection but connector fetches all fields (e.g., missing `$select`, `sysparm_fields`, `fields`)
|
||||
- `getDocument` re-fetches data already included in the initial API response (e.g., comments returned with post)
|
||||
- Last page of pagination requests full `PAGE_SIZE` when fewer records remain (`Math.min(PAGE_SIZE, remaining)`)
|
||||
|
||||
### Fix All Issues
|
||||
|
||||
After reporting, fix every **critical** and **warning** issue. Apply **suggestions** where they don't add unnecessary complexity.
|
||||
|
||||
### Validation Output
|
||||
|
||||
After fixing, confirm:
|
||||
1. `bun run lint` passes
|
||||
2. TypeScript compiles clean
|
||||
3. Re-read all modified files to verify fixes are correct
|
||||
|
||||
## Checklist Summary
|
||||
|
||||
- [ ] Read connector implementation, types, utils, registry, and OAuth config
|
||||
- [ ] Pulled and read official API documentation for the service
|
||||
- [ ] Validated every API endpoint URL, method, headers, and body against API docs
|
||||
- [ ] Validated input sanitization: no query/filter injection, URL fields normalized
|
||||
- [ ] Validated OAuth scopes: `requiredScopes` ⊆ OAuth provider `scopes` in `oauth.ts`
|
||||
- [ ] Validated each scope is real and recognized by the service's API
|
||||
- [ ] Validated scopes are sufficient for all API endpoints the connector calls
|
||||
- [ ] Validated token refresh config (`useBasicAuth`, `supportsRefreshTokenRotation`)
|
||||
- [ ] Validated pagination: cursor names, page sizes, hasMore logic, no silent caps
|
||||
- [ ] Validated data transformation: plain text extraction, HTML stripping, content hashing
|
||||
- [ ] Validated tag definitions match mapTags output, correct fieldTypes
|
||||
- [ ] Validated config fields: canonical pairs, selector keys, required flags
|
||||
- [ ] Validated validateConfig: lightweight check, error messages, retry options
|
||||
- [ ] Validated getDocument: null on 404, all content types handled, no redundant re-fetches, syncContext forwarding
|
||||
- [ ] Validated fetchWithRetry used for all external calls (no raw fetch), VALIDATE_RETRY_OPTIONS threaded through helpers
|
||||
- [ ] Validated API efficiency: field selection used, no redundant calls, sequential fetches batched
|
||||
- [ ] Validated error handling: graceful failures, no unhandled rejections
|
||||
- [ ] Validated logging: createLogger, no console.log
|
||||
- [ ] Validated registry: correct export, correct key
|
||||
- [ ] Reported all issues grouped by severity
|
||||
- [ ] Fixed all critical and warning issues
|
||||
- [ ] Ran `bun run lint` after fixes
|
||||
- [ ] Verified TypeScript compiles clean
|
||||
@@ -0,0 +1,295 @@
|
||||
---
|
||||
name: "source-command-validate-integration"
|
||||
description: "Validate an existing Sim integration (tools, block, registry) against the service's API docs"
|
||||
---
|
||||
|
||||
# source-command-validate-integration
|
||||
|
||||
Use this skill when the user asks to run the migrated source command `validate-integration`.
|
||||
|
||||
## Command Template
|
||||
|
||||
# Validate Integration Skill
|
||||
|
||||
You are an expert auditor for Sim integrations. Your job is to thoroughly validate that an existing integration is correct, complete, and follows all conventions.
|
||||
|
||||
## Your Task
|
||||
|
||||
When the user asks you to validate an integration:
|
||||
1. Read the service's API documentation (via WebFetch or Context7)
|
||||
2. Read every tool, the block, and registry entries
|
||||
3. Cross-reference everything against the API docs and Sim conventions
|
||||
4. Report all issues found, grouped by severity (critical, warning, suggestion)
|
||||
5. Fix all issues after reporting them
|
||||
|
||||
## Step 1: Gather All Files
|
||||
|
||||
Read **every** file for the integration — do not skip any:
|
||||
|
||||
```
|
||||
apps/sim/tools/{service}/ # All tool files, types.ts, index.ts
|
||||
apps/sim/blocks/blocks/{service}.ts # Block definition
|
||||
apps/sim/tools/registry.ts # Tool registry entries for this service
|
||||
apps/sim/blocks/registry.ts # Block registry entry for this service
|
||||
apps/sim/components/icons.tsx # Icon definition
|
||||
apps/sim/lib/auth/auth.ts # OAuth config — should use getCanonicalScopesForProvider()
|
||||
apps/sim/lib/oauth/oauth.ts # OAuth provider config — single source of truth for scopes
|
||||
apps/sim/lib/oauth/utils.ts # Scope utilities, SCOPE_DESCRIPTIONS for modal UI
|
||||
```
|
||||
|
||||
## Step 2: Pull API Documentation
|
||||
|
||||
Fetch the official API docs for the service. This is the **source of truth** for:
|
||||
- Endpoint URLs, HTTP methods, and auth headers
|
||||
- Required vs optional parameters
|
||||
- Parameter types and allowed values
|
||||
- Response shapes and field names
|
||||
- Pagination patterns (which param name, which response field)
|
||||
- Rate limits and error formats
|
||||
|
||||
## Step 3: Validate Tools
|
||||
|
||||
For **every** tool file, check:
|
||||
|
||||
### Tool ID and Naming
|
||||
- [ ] Tool ID uses `snake_case`: `{service}_{action}` (e.g., `x_create_tweet`, `slack_send_message`)
|
||||
- [ ] Tool `name` is human-readable (e.g., `'X Create Tweet'`)
|
||||
- [ ] Tool `description` is a concise one-liner describing what it does
|
||||
- [ ] Tool `version` is set (`'1.0.0'` or `'2.0.0'` for V2)
|
||||
|
||||
### Params
|
||||
- [ ] All required API params are marked `required: true`
|
||||
- [ ] All optional API params are marked `required: false`
|
||||
- [ ] Every param has explicit `required: true` or `required: false` — never omitted
|
||||
- [ ] Param types match the API (`'string'`, `'number'`, `'boolean'`, `'json'`)
|
||||
- [ ] Visibility is correct:
|
||||
- `'hidden'` — ONLY for OAuth access tokens and system-injected params
|
||||
- `'user-only'` — for API keys, credentials, and account-specific IDs the user must provide
|
||||
- `'user-or-llm'` — for everything else (search queries, content, filters, IDs that could come from other blocks)
|
||||
- [ ] Every param has a `description` that explains what it does
|
||||
|
||||
### Request
|
||||
- [ ] URL matches the API endpoint exactly (correct base URL, path segments, path params)
|
||||
- [ ] HTTP method matches the API spec (GET, POST, PUT, PATCH, DELETE)
|
||||
- [ ] Headers include correct auth pattern:
|
||||
- OAuth: `Authorization: Bearer ${params.accessToken}`
|
||||
- API Key: correct header name and format per the service's docs
|
||||
- [ ] `Content-Type` header is set for POST/PUT/PATCH requests
|
||||
- [ ] Body sends all required fields and only includes optional fields when provided
|
||||
- [ ] For GET requests with query params: URL is constructed correctly with query string
|
||||
- [ ] ID fields in URL paths are `.trim()`-ed to prevent copy-paste whitespace errors
|
||||
- [ ] Path params use template literals correctly: `` `https://api.service.com/v1/${params.id.trim()}` ``
|
||||
|
||||
### Response / transformResponse
|
||||
- [ ] Correctly parses the API response (`await response.json()`)
|
||||
- [ ] Extracts the right fields from the response structure (e.g., `data.data` vs `data` vs `data.results`)
|
||||
- [ ] All nullable fields use `?? null`
|
||||
- [ ] All optional arrays use `?? []`
|
||||
- [ ] Error cases are handled: checks for missing/empty data and returns meaningful error
|
||||
- [ ] Does NOT do raw JSON dumps — extracts meaningful, individual fields
|
||||
|
||||
### Outputs
|
||||
- [ ] All output fields match what the API actually returns
|
||||
- [ ] No fields are missing that the API provides and users would commonly need
|
||||
- [ ] No phantom fields defined that the API doesn't return
|
||||
- [ ] `optional: true` is set on fields that may not exist in all responses
|
||||
- [ ] When using `type: 'json'` and the shape is known, `properties` defines the inner fields
|
||||
- [ ] When using `type: 'array'`, `items` defines the item structure with `properties`
|
||||
- [ ] Field descriptions are accurate and helpful
|
||||
|
||||
### Types (types.ts)
|
||||
- [ ] Has param interfaces for every tool (e.g., `XCreateTweetParams`)
|
||||
- [ ] Has response interfaces for every tool (extending `ToolResponse`)
|
||||
- [ ] Optional params use `?` in the interface (e.g., `replyTo?: string`)
|
||||
- [ ] Field names in types match actual API field names
|
||||
- [ ] Shared response types are properly reused (e.g., `XTweetResponse` shared across tweet tools)
|
||||
|
||||
### Barrel Export (index.ts)
|
||||
- [ ] Every tool is exported
|
||||
- [ ] All types are re-exported (`export * from './types'`)
|
||||
- [ ] No orphaned exports (tools that don't exist)
|
||||
|
||||
### Tool Registry (tools/registry.ts)
|
||||
- [ ] Every tool is imported and registered
|
||||
- [ ] Registry keys use snake_case and match tool IDs exactly
|
||||
- [ ] Entries are in alphabetical order within the file
|
||||
|
||||
## Step 4: Validate Block
|
||||
|
||||
### Block ↔ Tool Alignment (CRITICAL)
|
||||
|
||||
This is the most important validation — the block must be perfectly aligned with every tool it references.
|
||||
|
||||
For **each tool** in `tools.access`:
|
||||
- [ ] The operation dropdown has an option whose ID matches the tool ID (or the `tools.config.tool` function correctly maps to it)
|
||||
- [ ] Every **required** tool param (except `accessToken`) has a corresponding subBlock input that is:
|
||||
- Shown when that operation is selected (correct `condition`)
|
||||
- Marked as `required: true` (or conditionally required)
|
||||
- [ ] Every **optional** tool param has a corresponding subBlock input (or is intentionally omitted if truly never needed)
|
||||
- [ ] SubBlock `id` values are unique across the entire block — no duplicates even across different conditions
|
||||
- [ ] The `tools.config.tool` function returns the correct tool ID for every possible operation value
|
||||
- [ ] The `tools.config.params` function correctly maps subBlock IDs to tool param names when they differ
|
||||
|
||||
### SubBlocks
|
||||
- [ ] Operation dropdown lists ALL tool operations available in `tools.access`
|
||||
- [ ] Dropdown option labels are human-readable and descriptive
|
||||
- [ ] Conditions use correct syntax:
|
||||
- Single value: `{ field: 'operation', value: 'x_create_tweet' }`
|
||||
- Multiple values (OR): `{ field: 'operation', value: ['x_create_tweet', 'x_delete_tweet'] }`
|
||||
- Negation: `{ field: 'operation', value: 'delete', not: true }`
|
||||
- Compound: `{ field: 'op', value: 'send', and: { field: 'type', value: 'dm' } }`
|
||||
- [ ] Condition arrays include ALL operations that use that field — none missing
|
||||
- [ ] `dependsOn` is set for fields that need other values (selectors depending on credential, cascading dropdowns)
|
||||
- [ ] SubBlock types match tool param types:
|
||||
- Enum/fixed options → `dropdown`
|
||||
- Free text → `short-input`
|
||||
- Long text/content → `long-input`
|
||||
- True/false → `dropdown` with Yes/No options (not `switch` unless purely UI toggle)
|
||||
- Credentials → `oauth-input` with correct `serviceId`
|
||||
- [ ] Dropdown `value: () => 'default'` is set for dropdowns with a sensible default
|
||||
|
||||
### Advanced Mode
|
||||
- [ ] Optional, rarely-used fields are set to `mode: 'advanced'`:
|
||||
- Pagination tokens / next tokens
|
||||
- Time range filters (start/end time)
|
||||
- Sort order / direction options
|
||||
- Max results / per page limits
|
||||
- Reply settings / threading options
|
||||
- Rarely used IDs (reply-to, quote-tweet, etc.)
|
||||
- Exclude filters
|
||||
- [ ] **Required** fields are NEVER set to `mode: 'advanced'`
|
||||
- [ ] Fields that users fill in most of the time are NOT set to `mode: 'advanced'`
|
||||
|
||||
### WandConfig
|
||||
- [ ] Timestamp fields have `wandConfig` with `generationType: 'timestamp'`
|
||||
- [ ] Comma-separated list fields have `wandConfig` with a descriptive prompt
|
||||
- [ ] Complex filter/query fields have `wandConfig` with format examples in the prompt
|
||||
- [ ] All `wandConfig` prompts end with "Return ONLY the [format] - no explanations, no extra text."
|
||||
- [ ] `wandConfig.placeholder` describes what to type in natural language
|
||||
|
||||
### Tools Config
|
||||
- [ ] `tools.access` lists **every** tool ID the block can use — none missing
|
||||
- [ ] `tools.config.tool` returns the correct tool ID for each operation
|
||||
- [ ] Type coercions are in `tools.config.params` (runs at execution time), NOT in `tools.config.tool` (runs at serialization time before variable resolution)
|
||||
- [ ] `tools.config.params` handles:
|
||||
- `Number()` conversion for numeric params that come as strings from inputs
|
||||
- `Boolean` / string-to-boolean conversion for toggle params
|
||||
- Empty string → `undefined` conversion for optional dropdown values
|
||||
- Any subBlock ID → tool param name remapping
|
||||
- [ ] No `Number()`, `JSON.parse()`, or other coercions in `tools.config.tool` — these would destroy dynamic references like `<Block.output>`
|
||||
|
||||
### Block Outputs
|
||||
- [ ] Outputs cover the key fields returned by ALL tools (not just one operation)
|
||||
- [ ] Output types are correct (`'string'`, `'number'`, `'boolean'`, `'json'`)
|
||||
- [ ] `type: 'json'` outputs either:
|
||||
- Describe inner fields in the description string (GOOD): `'User profile (id, name, username, bio)'`
|
||||
- Use nested output definitions (BEST): `{ id: { type: 'string' }, name: { type: 'string' } }`
|
||||
- [ ] No opaque `type: 'json'` with vague descriptions like `'Response data'`
|
||||
- [ ] Outputs that only appear for certain operations use `condition` if supported, or document which operations return them
|
||||
|
||||
### Block Metadata
|
||||
- [ ] `type` is snake_case (e.g., `'x'`, `'cloudflare'`)
|
||||
- [ ] `name` is human-readable (e.g., `'X'`, `'Cloudflare'`)
|
||||
- [ ] `description` is a concise one-liner
|
||||
- [ ] `longDescription` provides detail for docs
|
||||
- [ ] `docsLink` points to `'https://docs.sim.ai/tools/{service}'`
|
||||
- [ ] `category` is `'tools'`
|
||||
- [ ] `bgColor` uses the service's brand color hex
|
||||
- [ ] `icon` references the correct icon component from `@/components/icons`
|
||||
- [ ] `authMode` is set correctly (`AuthMode.OAuth` or `AuthMode.ApiKey`)
|
||||
- [ ] Block is registered in `blocks/registry.ts` alphabetically
|
||||
|
||||
### Block Inputs
|
||||
- [ ] `inputs` section lists all subBlock params that the block accepts
|
||||
- [ ] Input types match the subBlock types
|
||||
- [ ] When using `canonicalParamId`, inputs list the canonical ID (not the raw subBlock IDs)
|
||||
|
||||
## Step 5: Validate OAuth Scopes (if OAuth service)
|
||||
|
||||
Scopes are centralized — the single source of truth is `OAUTH_PROVIDERS` in `lib/oauth/oauth.ts`.
|
||||
|
||||
- [ ] Scopes defined in `lib/oauth/oauth.ts` under `OAUTH_PROVIDERS[provider].services[service].scopes`
|
||||
- [ ] `auth.ts` uses `getCanonicalScopesForProvider(providerId)` — NOT a hardcoded array
|
||||
- [ ] Block `requiredScopes` uses `getScopesForService(serviceId)` — NOT a hardcoded array
|
||||
- [ ] No hardcoded scope arrays in `auth.ts` or block files (should all use utility functions)
|
||||
- [ ] Each scope has a human-readable description in `SCOPE_DESCRIPTIONS` within `lib/oauth/utils.ts`
|
||||
- [ ] No excess scopes that aren't needed by any tool
|
||||
|
||||
## Step 6: Validate Pagination Consistency
|
||||
|
||||
If any tools support pagination:
|
||||
- [ ] Pagination param names match the API docs (e.g., `pagination_token` vs `next_token` vs `cursor`)
|
||||
- [ ] Different API endpoints that use different pagination param names have separate subBlocks in the block
|
||||
- [ ] Pagination response fields (`nextToken`, `cursor`, etc.) are included in tool outputs
|
||||
- [ ] Pagination subBlocks are set to `mode: 'advanced'`
|
||||
|
||||
## Step 7: Validate Error Handling
|
||||
|
||||
- [ ] `transformResponse` checks for error conditions before accessing data
|
||||
- [ ] Error responses include meaningful messages (not just generic "failed")
|
||||
- [ ] HTTP error status codes are handled (check `response.ok` or status codes)
|
||||
|
||||
## Step 8: Report and Fix
|
||||
|
||||
### Report Format
|
||||
|
||||
Group findings by severity:
|
||||
|
||||
**Critical** (will cause runtime errors or incorrect behavior):
|
||||
- Wrong endpoint URL or HTTP method
|
||||
- Missing required params or wrong `required` flag
|
||||
- Incorrect response field mapping (accessing wrong path in response)
|
||||
- Missing error handling that would cause crashes
|
||||
- Tool ID mismatch between tool file, registry, and block `tools.access`
|
||||
- OAuth scopes missing in `auth.ts` that tools need
|
||||
- `tools.config.tool` returning wrong tool ID for an operation
|
||||
- Type coercions in `tools.config.tool` instead of `tools.config.params`
|
||||
|
||||
**Warning** (follows conventions incorrectly or has usability issues):
|
||||
- Optional field not set to `mode: 'advanced'`
|
||||
- Missing `wandConfig` on timestamp/complex fields
|
||||
- Wrong `visibility` on params (e.g., `'hidden'` instead of `'user-or-llm'`)
|
||||
- Missing `optional: true` on nullable outputs
|
||||
- Opaque `type: 'json'` without property descriptions
|
||||
- Missing `.trim()` on ID fields in request URLs
|
||||
- Missing `?? null` on nullable response fields
|
||||
- Block condition array missing an operation that uses that field
|
||||
- Hardcoded scope arrays instead of using `getScopesForService()` / `getCanonicalScopesForProvider()`
|
||||
- Missing scope description in `SCOPE_DESCRIPTIONS` within `lib/oauth/utils.ts`
|
||||
|
||||
**Suggestion** (minor improvements):
|
||||
- Better description text
|
||||
- Inconsistent naming across tools
|
||||
- Missing `longDescription` or `docsLink`
|
||||
- Pagination fields that could benefit from `wandConfig`
|
||||
|
||||
### Fix All Issues
|
||||
|
||||
After reporting, fix every **critical** and **warning** issue. Apply **suggestions** where they don't add unnecessary complexity.
|
||||
|
||||
### Validation Output
|
||||
|
||||
After fixing, confirm:
|
||||
1. `bun run lint` passes with no fixes needed
|
||||
2. TypeScript compiles clean (no type errors)
|
||||
3. Re-read all modified files to verify fixes are correct
|
||||
|
||||
## Checklist Summary
|
||||
|
||||
- [ ] Read ALL tool files, block, types, index, and registries
|
||||
- [ ] Pulled and read official API documentation
|
||||
- [ ] Validated every tool's ID, params, request, response, outputs, and types against API docs
|
||||
- [ ] Validated block ↔ tool alignment (every tool param has a subBlock, every condition is correct)
|
||||
- [ ] Validated advanced mode on optional/rarely-used fields
|
||||
- [ ] Validated wandConfig on timestamps and complex inputs
|
||||
- [ ] Validated tools.config mapping, tool selector, and type coercions
|
||||
- [ ] Validated block outputs match what tools return, with typed JSON where possible
|
||||
- [ ] Validated OAuth scopes use centralized utilities (getScopesForService, getCanonicalScopesForProvider) — no hardcoded arrays
|
||||
- [ ] Validated scope descriptions exist in `SCOPE_DESCRIPTIONS` within `lib/oauth/utils.ts` for all scopes
|
||||
- [ ] Validated pagination consistency across tools and block
|
||||
- [ ] Validated error handling (error checks, meaningful messages)
|
||||
- [ ] Validated registry entries (tools and block, alphabetical, correct imports)
|
||||
- [ ] Reported all issues grouped by severity
|
||||
- [ ] Fixed all critical and warning issues
|
||||
- [ ] Ran `bun run lint` after fixes
|
||||
- [ ] Verified TypeScript compiles clean
|
||||
@@ -0,0 +1,218 @@
|
||||
---
|
||||
name: "source-command-validate-trigger"
|
||||
description: "Validate an existing Sim webhook trigger against provider API docs and repository conventions"
|
||||
---
|
||||
|
||||
# source-command-validate-trigger
|
||||
|
||||
Use this skill when the user asks to run the migrated source command `validate-trigger`.
|
||||
|
||||
## Command Template
|
||||
|
||||
# Validate Trigger
|
||||
|
||||
You are an expert auditor for Sim webhook triggers. Your job is to validate that an existing trigger implementation is correct, complete, secure, and aligned across all layers.
|
||||
|
||||
## Your Task
|
||||
|
||||
1. Read the service's webhook/API documentation (via WebFetch)
|
||||
2. Read every trigger file, provider handler, and registry entry
|
||||
3. Cross-reference against the API docs and Sim conventions
|
||||
4. Report all issues grouped by severity (critical, warning, suggestion)
|
||||
5. Fix all issues after reporting them
|
||||
|
||||
## Step 1: Gather All Files
|
||||
|
||||
Read **every** file for the trigger — do not skip any:
|
||||
|
||||
```
|
||||
apps/sim/triggers/{service}/ # All trigger files, utils.ts, index.ts
|
||||
apps/sim/lib/webhooks/providers/{service}.ts # Provider handler (if exists)
|
||||
apps/sim/lib/webhooks/providers/registry.ts # Handler registry
|
||||
apps/sim/triggers/registry.ts # Trigger registry
|
||||
apps/sim/blocks/blocks/{service}.ts # Block definition (trigger wiring)
|
||||
```
|
||||
|
||||
Also read for reference:
|
||||
```
|
||||
apps/sim/lib/webhooks/providers/types.ts # WebhookProviderHandler interface
|
||||
apps/sim/lib/webhooks/providers/utils.ts # Shared helpers (createHmacVerifier, etc.)
|
||||
apps/sim/lib/webhooks/provider-subscription-utils.ts # Subscription helpers
|
||||
apps/sim/lib/webhooks/processor.ts # Central webhook processor
|
||||
```
|
||||
|
||||
## Step 2: Pull API Documentation
|
||||
|
||||
Fetch the service's official webhook documentation. This is the **source of truth** for:
|
||||
- Webhook event types and payload shapes
|
||||
- Signature/auth verification method (HMAC algorithm, header names, secret format)
|
||||
- Challenge/verification handshake requirements
|
||||
- Webhook subscription API (create/delete endpoints, if applicable)
|
||||
- Retry behavior and delivery guarantees
|
||||
|
||||
## Step 3: Validate Trigger Definitions
|
||||
|
||||
### utils.ts
|
||||
- [ ] `{service}TriggerOptions` lists all trigger IDs accurately
|
||||
- [ ] `{service}SetupInstructions` provides clear, correct steps for the service
|
||||
- [ ] `build{Service}ExtraFields` includes relevant filter/config fields with correct `condition`
|
||||
- [ ] Output builders expose all meaningful fields from the webhook payload
|
||||
- [ ] Output builders do NOT use `optional: true` or `items` (tool-output-only features)
|
||||
- [ ] Nested output objects correctly model the payload structure
|
||||
|
||||
### Trigger Files
|
||||
- [ ] Exactly one primary trigger has `includeDropdown: true`
|
||||
- [ ] All secondary triggers do NOT have `includeDropdown`
|
||||
- [ ] All triggers use `buildTriggerSubBlocks` helper (not hand-rolled subBlocks)
|
||||
- [ ] Every trigger's `id` matches the convention `{service}_{event_name}`
|
||||
- [ ] Every trigger's `provider` matches the service name used in the handler registry
|
||||
- [ ] `index.ts` barrel exports all triggers
|
||||
|
||||
### Trigger ↔ Provider Alignment (CRITICAL)
|
||||
- [ ] Every trigger ID referenced in `matchEvent` logic exists in `{service}TriggerOptions`
|
||||
- [ ] Event matching logic in the provider correctly maps trigger IDs to service event types
|
||||
- [ ] Event matching logic in `is{Service}EventMatch` (if exists) correctly identifies events per the API docs
|
||||
|
||||
## Step 4: Validate Provider Handler
|
||||
|
||||
### Auth Verification
|
||||
- [ ] `verifyAuth` correctly validates webhook signatures per the service's documentation
|
||||
- [ ] HMAC algorithm matches (SHA-1, SHA-256, SHA-512)
|
||||
- [ ] Signature header name matches the API docs exactly
|
||||
- [ ] Signature format is handled (raw hex, `sha256=` prefix, base64, etc.)
|
||||
- [ ] Uses `safeCompare` for timing-safe comparison (no `===`)
|
||||
- [ ] If `webhookSecret` is required, handler rejects when it's missing (fail-closed)
|
||||
- [ ] Signature is computed over raw body (not parsed JSON)
|
||||
|
||||
### Event Matching
|
||||
- [ ] `matchEvent` returns `boolean` (not `NextResponse` or other values)
|
||||
- [ ] Challenge/verification events are excluded from matching (e.g., `endpoint.url_validation`)
|
||||
- [ ] When `triggerId` is a generic webhook ID, all events pass through
|
||||
- [ ] When `triggerId` is specific, only matching events pass
|
||||
- [ ] Event matching logic uses dynamic `await import()` for trigger utils (avoids circular deps)
|
||||
|
||||
### formatInput (CRITICAL)
|
||||
- [ ] Every key in the `formatInput` return matches a key in the trigger `outputs` schema
|
||||
- [ ] Every key in the trigger `outputs` schema is populated by `formatInput`
|
||||
- [ ] No extra undeclared keys that users can't discover in the UI
|
||||
- [ ] No wrapper objects (`webhook: { ... }`, `{service}: { ... }`)
|
||||
- [ ] Nested output paths exist at the correct depth (e.g., `resource.id` actually has `resource: { id: ... }`)
|
||||
- [ ] `null` is used for missing optional fields (not empty strings or empty objects)
|
||||
- [ ] Returns `{ input: { ... } }` — not a bare object
|
||||
|
||||
### Idempotency
|
||||
- [ ] `extractIdempotencyId` returns a stable, unique key per delivery
|
||||
- [ ] Uses provider-specific delivery IDs when available (e.g., `X-Request-Id`, `Linear-Delivery`, `svix-id`)
|
||||
- [ ] Falls back to content-based ID (e.g., `${type}:${id}`) when no delivery header exists
|
||||
- [ ] Does NOT include timestamps in the idempotency key (would break dedup on retries)
|
||||
|
||||
### Challenge Handling (if applicable)
|
||||
- [ ] `handleChallenge` correctly implements the service's URL verification handshake
|
||||
- [ ] Returns the expected response format per the API docs
|
||||
- [ ] Env-backed secrets are resolved via `resolveEnvVarsInObject` if needed
|
||||
|
||||
## Step 5: Validate Automatic Subscription Lifecycle
|
||||
|
||||
If the service supports programmatic webhook creation:
|
||||
|
||||
### createSubscription
|
||||
- [ ] Calls the correct API endpoint to create a webhook
|
||||
- [ ] Sends the correct event types/filters
|
||||
- [ ] Passes the notification URL from `getNotificationUrl(ctx.webhook)`
|
||||
- [ ] Returns `{ providerConfigUpdates: { externalId } }` with the external webhook ID
|
||||
- [ ] Throws on failure (orchestration handles rollback)
|
||||
- [ ] Provides user-friendly error messages (401 → "Invalid API Key", etc.)
|
||||
|
||||
### deleteSubscription
|
||||
- [ ] Calls the correct API endpoint to delete the webhook
|
||||
- [ ] Handles 404 gracefully (webhook already deleted)
|
||||
- [ ] Never throws — catches errors and logs non-fatally
|
||||
- [ ] Skips gracefully when `apiKey` or `externalId` is missing
|
||||
|
||||
### Orchestration Isolation
|
||||
- [ ] NO provider-specific logic in `route.ts`, `provider-subscriptions.ts`, or `deploy.ts`
|
||||
- [ ] All subscription logic lives on the handler (`createSubscription`/`deleteSubscription`)
|
||||
|
||||
## Step 6: Validate Registration and Block Wiring
|
||||
|
||||
### Trigger Registry (`triggers/registry.ts`)
|
||||
- [ ] All triggers are imported and registered
|
||||
- [ ] Registry keys match trigger IDs exactly
|
||||
- [ ] No orphaned entries (triggers that don't exist)
|
||||
|
||||
### Provider Handler Registry (`providers/registry.ts`)
|
||||
- [ ] Handler is imported and registered (if handler exists)
|
||||
- [ ] Registry key matches the `provider` field on the trigger configs
|
||||
- [ ] Entries are in alphabetical order
|
||||
|
||||
### Block Wiring (`blocks/blocks/{service}.ts`)
|
||||
- [ ] Block has `triggers.enabled: true`
|
||||
- [ ] `triggers.available` lists all trigger IDs
|
||||
- [ ] All trigger subBlocks are spread into `subBlocks`: `...getTrigger('id').subBlocks`
|
||||
- [ ] No trigger IDs in `triggers.available` that aren't in the registry
|
||||
- [ ] No trigger subBlocks spread that aren't in `triggers.available`
|
||||
|
||||
## Step 7: Validate Security
|
||||
|
||||
- [ ] Webhook secrets are never logged (not even at debug level)
|
||||
- [ ] Auth verification runs before any event processing
|
||||
- [ ] No secret comparison uses `===` (must use `safeCompare` or `crypto.timingSafeEqual`)
|
||||
- [ ] Timestamp/replay protection is reasonable (not too tight for retries, not too loose for security)
|
||||
- [ ] Raw body is used for signature verification (not re-serialized JSON)
|
||||
|
||||
## Step 8: Report and Fix
|
||||
|
||||
### Report Format
|
||||
|
||||
Group findings by severity:
|
||||
|
||||
**Critical** (runtime errors, security issues, or data loss):
|
||||
- Wrong HMAC algorithm or header name
|
||||
- `formatInput` keys don't match trigger `outputs`
|
||||
- Missing `verifyAuth` when the service sends signed webhooks
|
||||
- `matchEvent` returns non-boolean values
|
||||
- Provider-specific logic leaking into shared orchestration files
|
||||
- Trigger IDs mismatch between trigger files, registry, and block
|
||||
- `createSubscription` calling wrong API endpoint
|
||||
- Auth comparison using `===` instead of `safeCompare`
|
||||
|
||||
**Warning** (convention violations or usability issues):
|
||||
- Missing `extractIdempotencyId` when the service provides delivery IDs
|
||||
- Timestamps in idempotency keys (breaks dedup on retries)
|
||||
- Missing challenge handling when the service requires URL verification
|
||||
- Output schema missing fields that `formatInput` returns (undiscoverable data)
|
||||
- Overly tight timestamp skew window that rejects legitimate retries
|
||||
- `matchEvent` not filtering challenge/verification events
|
||||
- Setup instructions missing important steps
|
||||
|
||||
**Suggestion** (minor improvements):
|
||||
- More specific output field descriptions
|
||||
- Additional output fields that could be exposed
|
||||
- Better error messages in `createSubscription`
|
||||
- Logging improvements
|
||||
|
||||
### Fix All Issues
|
||||
|
||||
After reporting, fix every **critical** and **warning** issue. Apply **suggestions** where they don't add unnecessary complexity.
|
||||
|
||||
### Validation Output
|
||||
|
||||
After fixing, confirm:
|
||||
1. `bun run type-check` passes
|
||||
2. Re-read all modified files to verify fixes are correct
|
||||
3. Provider handler tests pass (if they exist): `bun test {service}`
|
||||
|
||||
## Checklist Summary
|
||||
|
||||
- [ ] Read all trigger files, provider handler, types, registries, and block
|
||||
- [ ] Pulled and read official webhook/API documentation
|
||||
- [ ] Validated trigger definitions: options, instructions, extra fields, outputs
|
||||
- [ ] Validated primary/secondary trigger distinction (`includeDropdown`)
|
||||
- [ ] Validated provider handler: auth, matchEvent, formatInput, idempotency
|
||||
- [ ] Validated output alignment: every `outputs` key ↔ every `formatInput` key
|
||||
- [ ] Validated subscription lifecycle: createSubscription, deleteSubscription, no shared-file edits
|
||||
- [ ] Validated registration: trigger registry, handler registry, block wiring
|
||||
- [ ] Validated security: safe comparison, no secret logging, replay protection
|
||||
- [ ] Reported all issues grouped by severity
|
||||
- [ ] Fixed all critical and warning issues
|
||||
- [ ] `bun run type-check` passes after fixes
|
||||
@@ -113,6 +113,8 @@ export function useMcpOauthPopup({ workspaceId }: UseMcpOauthPopupProps) {
|
||||
return
|
||||
}
|
||||
const { popup } = result
|
||||
const existing = popupIntervalsRef.current.get(serverId)
|
||||
if (existing !== undefined) window.clearInterval(existing)
|
||||
const interval = window.setInterval(() => {
|
||||
if (popup.closed) clear()
|
||||
}, 500)
|
||||
|
||||
@@ -49,7 +49,11 @@ export async function detectMcpAuthType(url: string): Promise<McpAuthType> {
|
||||
|
||||
if (res.status === 401) {
|
||||
const params = extractWWWAuthenticateParams(res)
|
||||
if (params.resourceMetadataUrl || params.scope || params.error) {
|
||||
// Per RFC 9728, an OAuth-protected resource signals OAuth via
|
||||
// `resource_metadata=...` in WWW-Authenticate. `scope=...` is also an
|
||||
// OAuth-specific hint. A bare `error="invalid_token"` is generic Bearer
|
||||
// and used by plain API-key servers too, so it must not classify as OAuth.
|
||||
if (params.resourceMetadataUrl || params.scope) {
|
||||
return 'oauth'
|
||||
}
|
||||
return 'headers'
|
||||
|
||||
Reference in New Issue
Block a user