mirror of
https://github.com/simstudioai/sim.git
synced 2026-09-24 15:45:35 +08:00
chore(snowflake): drop the local write caps in favor of the shared limit (#6461)
The 1000-row and 1 MB bound-value caps duplicated the shared 10MB tool request
body gate on the same axis, at a lower threshold. Nothing fails between the two
thresholds: Snowflake's 1 MB guidance is a recommendation about metadata
retention rather than a hard limit, and statements above it still execute. The
row array is already resident before the statement is built, so the caps also
bounded no allocation the request body did not already bound.
These were removed once before in 624b7373f5 and should have stayed removed.
This commit is contained in:
@@ -164,7 +164,7 @@ Insert structured JSON rows using bound values.
|
||||
| `database` | string | Yes | Database name |
|
||||
| `schema` | string | Yes | Schema name |
|
||||
| `table` | string | Yes | Target Snowflake table name within the selected database and schema context |
|
||||
| `rows` | json | Yes | Non-empty JSON array of row objects with matching keys. Max 1000 rows and 1 MB of bound data per call - stage the files and use Load Data for bulk ingest. |
|
||||
| `rows` | json | Yes | Non-empty JSON array of row objects with matching keys. For bulk loads, stage the files and use Load Data instead. |
|
||||
|
||||
#### Output
|
||||
|
||||
@@ -210,7 +210,7 @@ Update matching rows with a bound MERGE statement without inserting new rows.
|
||||
| `database` | string | Yes | Database name |
|
||||
| `schema` | string | Yes | Schema name |
|
||||
| `table` | string | Yes | Target Snowflake table name within the selected database and schema context |
|
||||
| `rows` | json | Yes | Non-empty JSON array of row objects with matching keys. Max 1000 rows and 1 MB of bound data per call - stage the files and use Load Data for bulk ingest. |
|
||||
| `rows` | json | Yes | Non-empty JSON array of row objects with matching keys. For bulk loads, stage the files and use Load Data instead. |
|
||||
| `matchColumns` | array | Yes | Columns used to match target rows. Match values must be non-null and unique across the submitted rows. |
|
||||
|
||||
#### Output
|
||||
@@ -257,7 +257,7 @@ Update matching rows and insert unmatched rows with a bound MERGE statement.
|
||||
| `database` | string | Yes | Database name |
|
||||
| `schema` | string | Yes | Schema name |
|
||||
| `table` | string | Yes | Target Snowflake table name within the selected database and schema context |
|
||||
| `rows` | json | Yes | Non-empty JSON array of row objects with matching keys. Max 1000 rows and 1 MB of bound data per call - stage the files and use Load Data for bulk ingest. |
|
||||
| `rows` | json | Yes | Non-empty JSON array of row objects with matching keys. For bulk loads, stage the files and use Load Data instead. |
|
||||
| `matchColumns` | array | Yes | Columns used to match target rows. Match values must be non-null and unique across the submitted rows. |
|
||||
|
||||
#### Output
|
||||
|
||||
@@ -649,7 +649,7 @@ export const SnowflakeBlock: BlockConfig<SnowflakeStatementResponse> = {
|
||||
rows: {
|
||||
type: 'string',
|
||||
description:
|
||||
'Structured rows as a JSON array. Max 1000 rows and 1 MB of bound data per call - stage the files and use Load Data for bulk ingest.',
|
||||
'Structured rows as a JSON array. For bulk loads, stage the files and use Load Data instead.',
|
||||
},
|
||||
matchColumns: {
|
||||
type: 'string',
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -81,7 +81,7 @@ export const insertRowsTool: ToolConfig<SnowflakeInsertRowsParams, SnowflakeStat
|
||||
required: true,
|
||||
visibility: 'user-or-llm',
|
||||
description:
|
||||
'Non-empty JSON array of row objects with matching keys. Max 1000 rows and 1 MB of bound data per call - stage the files and use Load Data for bulk ingest.',
|
||||
'Non-empty JSON array of row objects with matching keys. For bulk loads, stage the files and use Load Data instead.',
|
||||
},
|
||||
},
|
||||
request: snowflakeStatementRequest((params) =>
|
||||
|
||||
@@ -172,45 +172,6 @@ describe('Snowflake SQL builders', () => {
|
||||
)
|
||||
})
|
||||
|
||||
it('caps the number of rows per structured write', () => {
|
||||
const rows = Array.from({ length: 1001 }, (_, index) => ({ id: index }))
|
||||
expect(() => buildInsertRows({ ...table, rows })).toThrow('cannot exceed 1000 per call')
|
||||
expect(() => buildInsertRows({ ...table, rows })).toThrow('snowflake_load_data')
|
||||
expect(() => buildInsertRows({ ...table, rows: [{ blob: 'x'.repeat(1_000_001) }] })).toThrow(
|
||||
'statement budget'
|
||||
)
|
||||
})
|
||||
|
||||
it('counts the bound value budget in UTF-8 bytes, not UTF-16 code units', () => {
|
||||
expect(() => buildInsertRows({ ...table, rows: [{ blob: '中'.repeat(999_999) }] })).toThrow(
|
||||
'statement budget'
|
||||
)
|
||||
expect(() =>
|
||||
buildInsertRows({ ...table, rows: [{ blob: '中'.repeat(333_333) }] })
|
||||
).not.toThrow()
|
||||
})
|
||||
|
||||
it('applies the same statement budget to explicit bindings', () => {
|
||||
expect(() =>
|
||||
normalizeBindings({ '1': { type: 'TEXT', value: 'x'.repeat(1_000_001) } })
|
||||
).toThrow('statement budget')
|
||||
expect(() =>
|
||||
normalizeBindings({
|
||||
'1': { type: 'TEXT', value: 'x'.repeat(600_000) },
|
||||
'2': { type: 'TEXT', value: 'x'.repeat(600_000) },
|
||||
})
|
||||
).toThrow('statement budget')
|
||||
expect(() =>
|
||||
buildCallProcedure({
|
||||
...context,
|
||||
database: 'ANALYTICS',
|
||||
schema: 'PUBLIC',
|
||||
procedureName: 'REFRESH_MODEL',
|
||||
procedureArguments: [{ type: 'TEXT', value: '中'.repeat(999_999) }],
|
||||
})
|
||||
).toThrow('statement budget')
|
||||
})
|
||||
|
||||
it('routes a mixed semi-structured column through a single whole-column PARSE_JSON', () => {
|
||||
const result = buildInsertRows({
|
||||
...table,
|
||||
|
||||
@@ -90,26 +90,6 @@ function requireQueryId(queryId: string): string {
|
||||
return trimmed
|
||||
}
|
||||
|
||||
/**
|
||||
* Snowflake recommends limiting query text to 1 MB per statement, and that limit explicitly covers
|
||||
* values supplied through bindings. Statements above it still execute but are truncated before the
|
||||
* metadata store persists them, so they can no longer be retried or inspected.
|
||||
*
|
||||
* The budget is a byte budget, so it is measured against the UTF-8 encoding rather than the
|
||||
* JavaScript string length — a multi-byte string is up to 3x longer on the wire than in code units.
|
||||
*/
|
||||
const MAX_BOUND_VALUE_BYTES = 1_000_000
|
||||
|
||||
const BULK_INGEST_HINT = 'stage the data and use snowflake_load_data for bulk ingest'
|
||||
|
||||
function assertBoundBytesWithinBudget(boundBytes: number): void {
|
||||
if (boundBytes > MAX_BOUND_VALUE_BYTES) {
|
||||
throw new Error(
|
||||
`Snowflake bound values exceed the ${MAX_BOUND_VALUE_BYTES} byte statement budget; send fewer rows per call or ${BULK_INGEST_HINT}`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
export function normalizeBindings(
|
||||
input?: Record<string, SnowflakeBinding>
|
||||
): Record<string, SnowflakeBinding> | undefined {
|
||||
@@ -119,7 +99,6 @@ export function normalizeBindings(
|
||||
}
|
||||
const normalized: Record<string, SnowflakeBinding> = {}
|
||||
let hasBindings = false
|
||||
let boundBytes = 0
|
||||
for (const position in input) {
|
||||
if (!Object.hasOwn(input, position)) continue
|
||||
hasBindings = true
|
||||
@@ -136,27 +115,16 @@ export function normalizeBindings(
|
||||
if (typeof binding.value !== 'string') {
|
||||
throw new Error(`binding ${position} value must be a string`)
|
||||
}
|
||||
boundBytes += Buffer.byteLength(binding.value, 'utf8')
|
||||
assertBoundBytesWithinBudget(boundBytes)
|
||||
normalized[position] = { type: binding.type, value: binding.value }
|
||||
}
|
||||
return hasBindings ? normalized : undefined
|
||||
}
|
||||
|
||||
/**
|
||||
* Structured writes build a single statement holding every row, so the row count is capped well
|
||||
* below the byte budget to keep a typical write far away from the 1 MB statement recommendation.
|
||||
*/
|
||||
const MAX_WRITE_ROWS = 1_000
|
||||
|
||||
class BindingsBuilder {
|
||||
readonly bindings: Record<string, SnowflakeBinding> = {}
|
||||
private position = 0
|
||||
private boundBytes = 0
|
||||
|
||||
private addBinding(type: SnowflakeBinding['type'], value: string): string {
|
||||
this.boundBytes += Buffer.byteLength(value, 'utf8')
|
||||
assertBoundBytesWithinBudget(this.boundBytes)
|
||||
this.position += 1
|
||||
const key = String(this.position)
|
||||
this.bindings[key] = { type, value }
|
||||
@@ -195,11 +163,6 @@ class BindingsBuilder {
|
||||
|
||||
function validateRows(rows: Array<Record<string, unknown>>): string[] {
|
||||
if (!Array.isArray(rows) || rows.length === 0) throw new Error('rows must be a non-empty array')
|
||||
if (rows.length > MAX_WRITE_ROWS) {
|
||||
throw new Error(
|
||||
`rows cannot exceed ${MAX_WRITE_ROWS} per call; send the rows in smaller batches or ${BULK_INGEST_HINT}`
|
||||
)
|
||||
}
|
||||
const columns = Object.keys(rows[0] ?? {})
|
||||
if (columns.length === 0) throw new Error('rows must contain at least one column')
|
||||
const identifierKeys = new Set<string>()
|
||||
|
||||
@@ -91,7 +91,7 @@ export const updateRowsTool: ToolConfig<SnowflakeUpdateRowsParams, SnowflakeStat
|
||||
required: true,
|
||||
visibility: 'user-or-llm',
|
||||
description:
|
||||
'Non-empty JSON array of row objects with matching keys. Max 1000 rows and 1 MB of bound data per call - stage the files and use Load Data for bulk ingest.',
|
||||
'Non-empty JSON array of row objects with matching keys. For bulk loads, stage the files and use Load Data instead.',
|
||||
},
|
||||
matchColumns: {
|
||||
type: 'array',
|
||||
|
||||
@@ -91,7 +91,7 @@ export const upsertRowsTool: ToolConfig<SnowflakeUpsertRowsParams, SnowflakeStat
|
||||
required: true,
|
||||
visibility: 'user-or-llm',
|
||||
description:
|
||||
'Non-empty JSON array of row objects with matching keys. Max 1000 rows and 1 MB of bound data per call - stage the files and use Load Data for bulk ingest.',
|
||||
'Non-empty JSON array of row objects with matching keys. For bulk loads, stage the files and use Load Data instead.',
|
||||
},
|
||||
matchColumns: {
|
||||
type: 'array',
|
||||
|
||||
Reference in New Issue
Block a user