mirror of
https://github.com/simstudioai/sim.git
synced 2026-10-28 02:41:36 +08:00
feat(db): auto-apply tracked script data migrations in db:migrate (#5497)
* feat(db): auto-apply tracked script data migrations in db:migrate * fix(db): reset session lock_timeout before script migrations, guard journal insert * improvement(db): re-verify advisory-lock session before script migrations
This commit is contained in:
Theodore Li
@@ -1,71 +0,0 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*
|
||||
* Locks in the BYTEWISE ordering the rest of the stack depends on: `order_key` is
|
||||
* stored `COLLATE "C"` (migration 0228) so Postgres compares keys the same way this
|
||||
* library does. If the library's order ever diverged from ASCII byte order — e.g.
|
||||
* across the `Z` (0x5A) < `a` (0x61) integer-head boundary, exactly where the
|
||||
* `en_US.UTF-8` locale disagrees — inserts would mint keys that fail the `a >= b`
|
||||
* assertion and rows would display out of order. These tests would catch that.
|
||||
*/
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
BASE_62_DIGITS,
|
||||
generateKeyBetween,
|
||||
generateNKeysBetween,
|
||||
} from '@/lib/fractional-indexing/fractional-indexing'
|
||||
|
||||
/** Bytewise (ASCII / UTF-16 code-unit) comparator — what `COLLATE "C"` and the library use. */
|
||||
const byteCompare = (a: string, b: string): number => (a < b ? -1 : a > b ? 1 : 0)
|
||||
|
||||
describe('fractional-indexing bytewise ordering', () => {
|
||||
it('uses an alphabet in ascending char-code order with uppercase before lowercase', () => {
|
||||
for (let i = 1; i < BASE_62_DIGITS.length; i++) {
|
||||
expect(BASE_62_DIGITS.charCodeAt(i)).toBeGreaterThan(BASE_62_DIGITS.charCodeAt(i - 1))
|
||||
}
|
||||
// The boundary en_US.UTF-8 inverts: bytewise 'Z' < 'a', locale 'a' < 'Z'.
|
||||
expect('Z' < 'a').toBe(true)
|
||||
expect(BASE_62_DIGITS.indexOf('Z')).toBeLessThan(BASE_62_DIGITS.indexOf('a'))
|
||||
})
|
||||
|
||||
it('produces strictly bytewise-increasing keys on repeated append', () => {
|
||||
let prev: string | null = null
|
||||
let last = ''
|
||||
for (let i = 0; i < 200; i++) {
|
||||
const key: string = generateKeyBetween(prev, null)
|
||||
if (last) expect(byteCompare(last, key)).toBe(-1)
|
||||
last = key
|
||||
prev = key
|
||||
}
|
||||
})
|
||||
|
||||
it('stays bytewise-ordered when prepends cross the Z/a integer-head boundary', () => {
|
||||
// Repeated prepend walks the integer head down out of 'a' into 'Z','Y',… —
|
||||
// the exact uppercase region en_US.UTF-8 sorts wrong. The first prepend before
|
||||
// "a0" already yields a 'Z'-headed key ("Zz").
|
||||
const keys: string[] = []
|
||||
let next: string | null = null
|
||||
for (let i = 0; i < 60; i++) {
|
||||
const key: string = generateKeyBetween(null, next)
|
||||
keys.push(key)
|
||||
next = key
|
||||
}
|
||||
const ascending = [...keys].reverse() // prepends are emitted largest → smallest
|
||||
expect([...ascending].sort(byteCompare)).toEqual(ascending)
|
||||
expect(new Set(keys).size).toBe(keys.length) // all distinct
|
||||
// An uppercase-headed key sorts before the lowercase-headed "a0" at the tail.
|
||||
expect(ascending.some((k) => k[0] >= 'A' && k[0] <= 'Z')).toBe(true)
|
||||
expect(ascending[ascending.length - 1][0]).toBe('a')
|
||||
})
|
||||
|
||||
it('mints a contiguous run that is bytewise-sorted and distinct', () => {
|
||||
const keys = generateNKeysBetween(null, null, 500)
|
||||
expect([...keys].sort(byteCompare)).toEqual(keys)
|
||||
expect(new Set(keys).size).toBe(keys.length)
|
||||
})
|
||||
|
||||
it('throws when bounds are out of order — the contract COLLATE "C" must satisfy', () => {
|
||||
expect(() => generateKeyBetween('a1', 'a0')).toThrow()
|
||||
expect(() => generateKeyBetween('a0', 'a0')).toThrow()
|
||||
})
|
||||
})
|
||||
@@ -1,281 +0,0 @@
|
||||
/**
|
||||
* Fractional indexing — generate ordering strings that sort lexicographically.
|
||||
*
|
||||
* In-house port of David Greenspan's algorithm
|
||||
* (https://observablehq.com/@dgreensp/implementing-fractional-indexing),
|
||||
* behavior-identical to the `fractional-indexing` npm package (CC0). A key is a
|
||||
* variable-length base-62 string: between any two keys there is always room for
|
||||
* another, so inserts never renumber existing rows. The only cost is gradual
|
||||
* length growth under repeated same-spot inserts.
|
||||
*
|
||||
* A key is `<integer part><fraction>`. The integer part's first character
|
||||
* encodes its own length (`a..z` → 2..27, `A..Z` → 27..2), letting integers
|
||||
* grow without bound in both directions. The fraction is plain base-62 digits
|
||||
* with no trailing zero.
|
||||
*/
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Digits
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** Default digit alphabet. Must be in ascending character-code order. */
|
||||
export const BASE_62_DIGITS = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Integer-part helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** Length the integer part must have, derived from its first character. */
|
||||
function getIntegerLength(head: string): number {
|
||||
if (head >= 'a' && head <= 'z') {
|
||||
return head.charCodeAt(0) - 'a'.charCodeAt(0) + 2
|
||||
}
|
||||
if (head >= 'A' && head <= 'Z') {
|
||||
return 'Z'.charCodeAt(0) - head.charCodeAt(0) + 2
|
||||
}
|
||||
throw new Error(`invalid order key head: ${head}`)
|
||||
}
|
||||
|
||||
function validateInteger(int: string): void {
|
||||
if (int.length !== getIntegerLength(int[0])) {
|
||||
throw new Error(`invalid integer part of order key: ${int}`)
|
||||
}
|
||||
}
|
||||
|
||||
function getIntegerPart(key: string): string {
|
||||
const integerPartLength = getIntegerLength(key[0])
|
||||
if (integerPartLength > key.length) {
|
||||
throw new Error(`invalid order key: ${key}`)
|
||||
}
|
||||
return key.slice(0, integerPartLength)
|
||||
}
|
||||
|
||||
function validateOrderKey(key: string, digits: string): void {
|
||||
if (key === `A${digits[0].repeat(26)}`) {
|
||||
throw new Error(`invalid order key: ${key}`)
|
||||
}
|
||||
// getIntegerPart throws if the head is bad or the key is too short.
|
||||
const i = getIntegerPart(key)
|
||||
const f = key.slice(i.length)
|
||||
if (f.slice(-1) === digits[0]) {
|
||||
throw new Error(`invalid order key: ${key}`)
|
||||
}
|
||||
}
|
||||
|
||||
/** Increment the integer part; returns null past the largest integer. */
|
||||
function incrementInteger(x: string, digits: string): string | null {
|
||||
validateInteger(x)
|
||||
const [head, ...digs] = x.split('')
|
||||
let carry = true
|
||||
for (let i = digs.length - 1; carry && i >= 0; i--) {
|
||||
const d = digits.indexOf(digs[i]) + 1
|
||||
if (d === digits.length) {
|
||||
digs[i] = digits[0]
|
||||
} else {
|
||||
digs[i] = digits[d]
|
||||
carry = false
|
||||
}
|
||||
}
|
||||
if (carry) {
|
||||
if (head === 'Z') {
|
||||
return `a${digits[0]}`
|
||||
}
|
||||
if (head === 'z') {
|
||||
return null
|
||||
}
|
||||
const h = String.fromCharCode(head.charCodeAt(0) + 1)
|
||||
if (h > 'a') {
|
||||
digs.push(digits[0])
|
||||
} else {
|
||||
digs.pop()
|
||||
}
|
||||
return h + digs.join('')
|
||||
}
|
||||
return head + digs.join('')
|
||||
}
|
||||
|
||||
/** Decrement the integer part; returns null past the smallest integer. */
|
||||
function decrementInteger(x: string, digits: string): string | null {
|
||||
validateInteger(x)
|
||||
const [head, ...digs] = x.split('')
|
||||
let borrow = true
|
||||
for (let i = digs.length - 1; borrow && i >= 0; i--) {
|
||||
const d = digits.indexOf(digs[i]) - 1
|
||||
if (d === -1) {
|
||||
digs[i] = digits.slice(-1)
|
||||
} else {
|
||||
digs[i] = digits[d]
|
||||
borrow = false
|
||||
}
|
||||
}
|
||||
if (borrow) {
|
||||
if (head === 'a') {
|
||||
return `Z${digits.slice(-1)}`
|
||||
}
|
||||
if (head === 'A') {
|
||||
return null
|
||||
}
|
||||
const h = String.fromCharCode(head.charCodeAt(0) - 1)
|
||||
if (h < 'Z') {
|
||||
digs.push(digits.slice(-1))
|
||||
} else {
|
||||
digs.pop()
|
||||
}
|
||||
return h + digs.join('')
|
||||
}
|
||||
return head + digs.join('')
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Midpoint
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Fraction strictly between `a` and `b` (both without integer parts). `a` may be
|
||||
* empty; `b` is null (open end) or non-empty and `> a`. No trailing zeros.
|
||||
*/
|
||||
function midpoint(a: string, b: string | null | undefined, digits: string): string {
|
||||
const zero = digits[0]
|
||||
if (b != null && a >= b) {
|
||||
throw new Error(`${a} >= ${b}`)
|
||||
}
|
||||
if (a.slice(-1) === zero || (b && b.slice(-1) === zero)) {
|
||||
throw new Error('trailing zero')
|
||||
}
|
||||
if (b) {
|
||||
// Strip the longest common prefix, padding `a` with zeros as we go. `b`
|
||||
// needs no padding — it can't end before `a` within the common prefix.
|
||||
let n = 0
|
||||
while ((a[n] || zero) === b[n]) {
|
||||
n++
|
||||
}
|
||||
if (n > 0) {
|
||||
return b.slice(0, n) + midpoint(a.slice(n), b.slice(n), digits)
|
||||
}
|
||||
}
|
||||
// First digits (or lack thereof) differ.
|
||||
const digitA = a ? digits.indexOf(a[0]) : 0
|
||||
const digitB = b != null ? digits.indexOf(b[0]) : digits.length
|
||||
if (digitB - digitA > 1) {
|
||||
const midDigit = Math.round(0.5 * (digitA + digitB))
|
||||
return digits[midDigit]
|
||||
}
|
||||
// First digits are consecutive.
|
||||
if (b && b.length > 1) {
|
||||
return b.slice(0, 1)
|
||||
}
|
||||
// `b` is null or a single digit; recurse into `a`'s tail.
|
||||
return digits[digitA] + midpoint(a.slice(1), null, digits)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Public API
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Returns a key that sorts strictly between `a` and `b`. Either may be null for
|
||||
* an open end. `a < b` lexicographically when both are non-null.
|
||||
*
|
||||
* @throws if `a`/`b` are invalid keys or `a >= b`.
|
||||
*/
|
||||
export function generateKeyBetween(
|
||||
a: string | null | undefined,
|
||||
b: string | null | undefined,
|
||||
digits: string = BASE_62_DIGITS
|
||||
): string {
|
||||
if (a != null) {
|
||||
validateOrderKey(a, digits)
|
||||
}
|
||||
if (b != null) {
|
||||
validateOrderKey(b, digits)
|
||||
}
|
||||
if (a != null && b != null && a >= b) {
|
||||
throw new Error(`${a} >= ${b}`)
|
||||
}
|
||||
if (a == null) {
|
||||
if (b == null) {
|
||||
return `a${digits[0]}`
|
||||
}
|
||||
const ib = getIntegerPart(b)
|
||||
const fb = b.slice(ib.length)
|
||||
if (ib === `A${digits[0].repeat(26)}`) {
|
||||
return ib + midpoint('', fb, digits)
|
||||
}
|
||||
if (ib < b) {
|
||||
return ib
|
||||
}
|
||||
const res = decrementInteger(ib, digits)
|
||||
if (res == null) {
|
||||
throw new Error('cannot decrement any more')
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
if (b == null) {
|
||||
const ia = getIntegerPart(a)
|
||||
const fa = a.slice(ia.length)
|
||||
const i = incrementInteger(ia, digits)
|
||||
return i == null ? ia + midpoint(fa, null, digits) : i
|
||||
}
|
||||
|
||||
const ia = getIntegerPart(a)
|
||||
const fa = a.slice(ia.length)
|
||||
const ib = getIntegerPart(b)
|
||||
const fb = b.slice(ib.length)
|
||||
if (ia === ib) {
|
||||
return ia + midpoint(fa, fb, digits)
|
||||
}
|
||||
const i = incrementInteger(ia, digits)
|
||||
if (i == null) {
|
||||
throw new Error('cannot increment any more')
|
||||
}
|
||||
if (i < b) {
|
||||
return i
|
||||
}
|
||||
return ia + midpoint(fa, null, digits)
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns `n` distinct keys in sorted order, strictly between `a` and `b` (same
|
||||
* open-end semantics as {@link generateKeyBetween}). When both ends are null,
|
||||
* returns a contiguous run of "integer" keys.
|
||||
*/
|
||||
export function generateNKeysBetween(
|
||||
a: string | null | undefined,
|
||||
b: string | null | undefined,
|
||||
n: number,
|
||||
digits: string = BASE_62_DIGITS
|
||||
): string[] {
|
||||
if (n === 0) {
|
||||
return []
|
||||
}
|
||||
if (n === 1) {
|
||||
return [generateKeyBetween(a, b, digits)]
|
||||
}
|
||||
if (b == null) {
|
||||
let c = generateKeyBetween(a, b, digits)
|
||||
const result = [c]
|
||||
for (let i = 0; i < n - 1; i++) {
|
||||
c = generateKeyBetween(c, b, digits)
|
||||
result.push(c)
|
||||
}
|
||||
return result
|
||||
}
|
||||
if (a == null) {
|
||||
let c = generateKeyBetween(a, b, digits)
|
||||
const result = [c]
|
||||
for (let i = 0; i < n - 1; i++) {
|
||||
c = generateKeyBetween(a, c, digits)
|
||||
result.push(c)
|
||||
}
|
||||
result.reverse()
|
||||
return result
|
||||
}
|
||||
const mid = Math.floor(n / 2)
|
||||
const c = generateKeyBetween(a, b, digits)
|
||||
return [
|
||||
...generateNKeysBetween(a, c, mid, digits),
|
||||
c,
|
||||
...generateNKeysBetween(c, b, n - mid - 1, digits),
|
||||
]
|
||||
}
|
||||
@@ -11,10 +11,7 @@
|
||||
* repeated same-spot inserts.
|
||||
*/
|
||||
|
||||
import {
|
||||
generateKeyBetween,
|
||||
generateNKeysBetween,
|
||||
} from '@/lib/fractional-indexing/fractional-indexing'
|
||||
import { generateKeyBetween, generateNKeysBetween } from '@sim/utils/fractional-indexing'
|
||||
|
||||
/**
|
||||
* Returns a key that sorts strictly between `a` and `b`. Pass `null` for an open
|
||||
|
||||
@@ -1,130 +0,0 @@
|
||||
#!/usr/bin/env bun
|
||||
|
||||
/**
|
||||
* Backfills the `order_key` column on `user_table_rows`.
|
||||
*
|
||||
* Row ordering is moving from the contiguous integer `position` to a fractional
|
||||
* string `order_key` (O(1) insert/delete — no reshift/recompact). This script
|
||||
* assigns each existing row a key derived from its current `position` order, so
|
||||
* the new ordering matches today's once the `TABLES_FRACTIONAL_ORDERING` flag is
|
||||
* flipped on.
|
||||
*
|
||||
* Per-table-atomic: each table is keyed inside one transaction holding the same
|
||||
* per-table advisory lock the app uses for inserts, so a concurrent insert can't
|
||||
* interleave. Idempotent: tables already fully keyed are skipped; a table with
|
||||
* any NULL key is fully re-keyed from `position` order (deterministic, so a
|
||||
* re-run after a partial failure is safe).
|
||||
*
|
||||
* Usage:
|
||||
* DATABASE_URL=... bun run apps/sim/scripts/backfill-table-order-keys.ts
|
||||
* DATABASE_URL=... bun run apps/sim/scripts/backfill-table-order-keys.ts --dry-run
|
||||
*/
|
||||
|
||||
import { userTableRows } from '@sim/db/schema'
|
||||
import { getErrorMessage } from '@sim/utils/errors'
|
||||
import { asc, eq, isNull, sql } from 'drizzle-orm'
|
||||
import { drizzle } from 'drizzle-orm/postgres-js'
|
||||
import postgres from 'postgres'
|
||||
import { nKeysBetween } from '@/lib/table/order-key'
|
||||
|
||||
/**
|
||||
* Rows written per `UPDATE … FROM (VALUES …)`. One statement for a whole large table builds an
|
||||
* enormous VALUES list that overflows the JS call stack while drizzle assembles it (and would
|
||||
* exceed Postgres's 65535-bound-param limit at ~32k rows, 2 params/row). 5000 keeps ~10k params
|
||||
* — well under both ceilings — while minimizing round-trips. Chunks share the one per-table
|
||||
* transaction, so the table is still keyed atomically.
|
||||
*/
|
||||
const WRITE_CHUNK_SIZE = 5000
|
||||
|
||||
export async function runBackfill(): Promise<void> {
|
||||
const dryRun = process.argv.includes('--dry-run')
|
||||
const connectionString = process.env.DATABASE_URL ?? process.env.POSTGRES_URL
|
||||
if (!connectionString) {
|
||||
console.error('Missing DATABASE_URL or POSTGRES_URL')
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const client = postgres(connectionString, {
|
||||
prepare: false,
|
||||
idle_timeout: 20,
|
||||
connect_timeout: 30,
|
||||
max: 5,
|
||||
onnotice: () => {},
|
||||
})
|
||||
const db = drizzle(client)
|
||||
|
||||
const stats = { tables: 0, tablesKeyed: 0, rowsKeyed: 0, failed: 0 }
|
||||
|
||||
try {
|
||||
// Tables that still have at least one un-keyed row.
|
||||
const pending = await db
|
||||
.selectDistinct({ tableId: userTableRows.tableId })
|
||||
.from(userTableRows)
|
||||
.where(isNull(userTableRows.orderKey))
|
||||
|
||||
console.log(
|
||||
`Backfill starting — ${pending.length} table(s) with NULL order_key${dryRun ? ' [DRY RUN]' : ''}`
|
||||
)
|
||||
|
||||
for (const { tableId } of pending) {
|
||||
stats.tables += 1
|
||||
try {
|
||||
const keyed = await db.transaction(async (trx) => {
|
||||
// Serialize with concurrent inserts on this table (same lock the app uses).
|
||||
await trx.execute(
|
||||
sql`SELECT pg_advisory_xact_lock(hashtextextended(${`user_table_rows_pos:${tableId}`}, 0))`
|
||||
)
|
||||
const rows = await trx
|
||||
.select({ id: userTableRows.id })
|
||||
.from(userTableRows)
|
||||
.where(eq(userTableRows.tableId, tableId))
|
||||
.orderBy(asc(userTableRows.position), asc(userTableRows.id))
|
||||
|
||||
if (rows.length === 0) return 0
|
||||
const keys = nKeysBetween(null, null, rows.length)
|
||||
if (dryRun) return rows.length
|
||||
|
||||
// Chunked UPDATE … FROM (VALUES …) mapping id → key (see WRITE_CHUNK_SIZE).
|
||||
for (let start = 0; start < rows.length; start += WRITE_CHUNK_SIZE) {
|
||||
const chunk = rows.slice(start, start + WRITE_CHUNK_SIZE)
|
||||
const values = sql.join(
|
||||
chunk.map((r, i) => sql`(${r.id}, ${keys[start + i]})`),
|
||||
sql`, `
|
||||
)
|
||||
await trx.execute(sql`
|
||||
UPDATE user_table_rows AS t
|
||||
SET order_key = v.order_key
|
||||
FROM (VALUES ${values}) AS v(id, order_key)
|
||||
WHERE t.id = v.id AND t.table_id = ${tableId}
|
||||
`)
|
||||
}
|
||||
return rows.length
|
||||
})
|
||||
stats.tablesKeyed += 1
|
||||
stats.rowsKeyed += keyed
|
||||
console.log(` ${tableId}: keyed ${keyed} rows`)
|
||||
} catch (error) {
|
||||
stats.failed += 1
|
||||
console.error(` ${tableId}: FAILED — ${getErrorMessage(error)}`)
|
||||
}
|
||||
}
|
||||
|
||||
console.log('Backfill complete.')
|
||||
console.log(` tables scanned: ${stats.tables}`)
|
||||
console.log(` tables keyed: ${stats.tablesKeyed}`)
|
||||
console.log(` rows keyed: ${stats.rowsKeyed}`)
|
||||
console.log(` failed: ${stats.failed}`)
|
||||
if (stats.failed > 0) process.exitCode = 1
|
||||
} finally {
|
||||
await client.end({ timeout: 5 }).catch(() => {})
|
||||
}
|
||||
}
|
||||
|
||||
if ((import.meta as { main?: boolean }).main) {
|
||||
try {
|
||||
await runBackfill()
|
||||
} catch (error) {
|
||||
console.error('Backfill aborted:', getErrorMessage(error))
|
||||
process.exitCode = 1
|
||||
}
|
||||
}
|
||||
@@ -20,7 +20,8 @@
|
||||
* the original backfill also used — minting a fresh, evenly-spaced, distinct run
|
||||
* with `nKeysBetween`.
|
||||
*
|
||||
* Distinct from `backfill-table-order-keys.ts`, which keys tables with NULL keys;
|
||||
* Distinct from the `0001_backfill_table_order_keys` script migration
|
||||
* (`packages/db/script-migrations/`), which keys tables with NULL keys;
|
||||
* this one repairs tables that are fully keyed but bytewise-disordered. Run it
|
||||
* AFTER migration 0228 so the re-key writes and sorts under `COLLATE "C"`.
|
||||
*
|
||||
@@ -41,7 +42,7 @@ import { drizzle } from 'drizzle-orm/postgres-js'
|
||||
import postgres from 'postgres'
|
||||
import { nKeysBetween } from '@/lib/table/order-key'
|
||||
|
||||
/** See backfill-table-order-keys.ts — keeps each VALUES list well under the param/stack ceilings. */
|
||||
/** Keeps each VALUES list well under Postgres's 65535-bound-param and JS call-stack ceilings. */
|
||||
const WRITE_CHUNK_SIZE = 5000
|
||||
|
||||
export async function runRepair(): Promise<void> {
|
||||
|
||||
Reference in New Issue
Block a user