mirror of
https://gitee.com/samwaf/SamWaf.git
synced 2026-10-31 10:11:42 +08:00
* fix: make archived log shard connections concurrency-safe and recoverable * fix: build the WebSocket online table before its consumers start * perf: cap stored payloads, index log lookups by IP and rule, batch IP tag writes * fix: keep archived log shards readable when the live table gains a column * feat: merge IP tag history when their database location changes * refactor: move request payloads out of the access log into their own table * feat: scope log database export to a time range and selected tiers * feat: add an IP watchlist that full-captures watched IPs * feat: split log storage into security events, access rows, and payload tiers * feat: roll up requests per actor and path into daily analysis tables * fix:actor key by ip * feat: add a source and path analysis page over the daily rollups * feat: exclude chosen source IPs from web log recording * docs: add upgrade notes for the analysis page and log IP exclusion * feat: derive the log partition from the time range and look up identifiers across all of them * feat: add time partition verbs and the period naming they share * feat: cut log archives per month and expire them by dropping partitions * feat: manage log partitions, and answer why an IP has no logs * fix: read archived SQLite log shards read-only and choose tiers by data * fix: count the live log shard from access_log instead of a stale snapshot * fix:export db bug * fix: default backend keep-alive pool (512/128/90s) to stop connect-close churn under load * chore: gen_swagger.bat outputs json only, syncs doc site, runs route test * feat: pin web frontend version and embed it into the binary
194 lines
8.6 KiB
Go
194 lines
8.6 KiB
Go
package dialect
|
|
|
|
import (
|
|
"time"
|
|
|
|
"gorm.io/gorm"
|
|
)
|
|
|
|
// ColumnMeta holds metadata for a single table column.
|
|
type ColumnMeta struct {
|
|
Cid int
|
|
Name string
|
|
Type string
|
|
NotNull bool
|
|
DefaultVal string
|
|
PrimaryKey bool
|
|
}
|
|
|
|
// IndexColumnMeta describes a column participating in an index.
|
|
type IndexColumnMeta struct {
|
|
SeqNo int
|
|
Cid int
|
|
Name string
|
|
}
|
|
|
|
// IndexMeta holds metadata for a table index.
|
|
type IndexMeta struct {
|
|
Name string
|
|
Unique bool
|
|
Origin string // SQLite: "c"=created, "u"=unique, "pk"=primary; empty for others
|
|
Columns []IndexColumnMeta
|
|
}
|
|
|
|
// DBMetrics holds runtime performance metrics. Fields are driver-specific;
|
|
// irrelevant fields are zero-valued (omitempty in JSON output).
|
|
type DBMetrics struct {
|
|
DatabaseName string `json:"database_name"`
|
|
DatabasePath string `json:"database_path,omitempty"` // file-based only
|
|
FileSize int64 `json:"file_size,omitempty"`
|
|
FileSizeMB float64 `json:"file_size_mb,omitempty"`
|
|
ConnectionCount int `json:"connection_count"`
|
|
MaxConnections int `json:"max_connections"`
|
|
IdleConnections int `json:"idle_connections"`
|
|
InUseConnections int `json:"in_use_connections"`
|
|
Timestamp time.Time `json:"timestamp"`
|
|
|
|
// SQLite-specific (PRAGMA)
|
|
PageCount int64 `json:"page_count,omitempty"`
|
|
PageSize int64 `json:"page_size,omitempty"`
|
|
FreelistCount int64 `json:"freelist_count,omitempty"`
|
|
CacheSize int64 `json:"cache_size,omitempty"`
|
|
JournalMode string `json:"journal_mode,omitempty"`
|
|
SynchronousMode string `json:"synchronous_mode,omitempty"`
|
|
TempStore string `json:"temp_store,omitempty"`
|
|
CacheHitRatio float64 `json:"cache_hit_ratio,omitempty"`
|
|
|
|
// MySQL / SQL Server specific
|
|
EngineVersion string `json:"engine_version,omitempty"`
|
|
DataSizeMB float64 `json:"data_size_mb,omitempty"`
|
|
IndexSizeMB float64 `json:"index_size_mb,omitempty"`
|
|
CurrentConns int `json:"current_connections,omitempty"`
|
|
}
|
|
|
|
// DBDialect abstracts SQL dialect differences so business logic stays
|
|
// database-agnostic. Each database engine provides one implementation.
|
|
// Register the active implementation once at startup via Register().
|
|
type DBDialect interface {
|
|
// Name returns the driver identifier: "sqlite" | "mysql" | "sqlserver".
|
|
Name() string
|
|
|
|
// IsFileBased returns true for file-based databases (SQLite).
|
|
// Used to guard backup/repair operations that operate on files.
|
|
IsFileBased() bool
|
|
|
|
// SupportsBackup reports whether the built-in backup tool is available.
|
|
SupportsBackup() bool
|
|
|
|
// SupportsRepair reports whether the built-in repair tool is available.
|
|
SupportsRepair() bool
|
|
|
|
// ForceIndexClause returns the table reference with an index hint.
|
|
// SQLite: "tbl INDEXED BY idx"
|
|
// MySQL: "tbl FORCE INDEX (idx)"
|
|
// SQL Server:"tbl WITH (INDEX(idx))"
|
|
// PostgreSQL has no index hints: it returns the bare table name.
|
|
ForceIndexClause(table, idx string) string
|
|
|
|
// Quote wraps an identifier in the engine's quoting characters.
|
|
// SQLite / PostgreSQL: "x"
|
|
// MySQL: `x`
|
|
Quote(ident string) string
|
|
|
|
// BatchDeleteSQL builds a "delete at most limit rows matching where" statement.
|
|
// where is a caller-built fragment that may contain ? placeholders; it appears
|
|
// exactly once in the result on every engine, so the caller's argument list is
|
|
// identical across dialects.
|
|
//
|
|
// Not every table has a primary key (innerbean.WebLog / web_logs has none), so
|
|
// this cannot key off an id column and must use each engine's physical row handle:
|
|
// SQLite: DELETE FROM t WHERE rowid IN (SELECT rowid FROM t WHERE <where> LIMIT n)
|
|
// PostgreSQL: DELETE FROM t WHERE ctid IN (SELECT ctid FROM t WHERE <where> LIMIT n)
|
|
// MySQL: DELETE FROM t WHERE <where> LIMIT n
|
|
// MySQL has no row-handle pseudo-column and rejects a LIMIT-bearing subquery on the
|
|
// table being deleted from (error 1093), hence the flat DELETE ... LIMIT form.
|
|
BatchDeleteSQL(table, where string, limit int) string
|
|
|
|
// GroupConcatDistinct aggregates expr into one comma-separated string, de-duplicated.
|
|
// SQLite / MySQL: GROUP_CONCAT(DISTINCT expr)
|
|
// PostgreSQL: string_agg(DISTINCT expr, ',')
|
|
GroupConcatDistinct(expr string) string
|
|
|
|
// InsertIgnoreSQL builds a multi-row INSERT that silently skips rows conflicting
|
|
// with an existing primary/unique key, so a re-run is idempotent.
|
|
// SQLite: INSERT OR IGNORE INTO t (cols) VALUES (...),(...)
|
|
// MySQL: INSERT IGNORE INTO t (cols) VALUES (...),(...)
|
|
// PostgreSQL: INSERT INTO t (cols) VALUES (...),(...) ON CONFLICT DO NOTHING
|
|
// quotedCols and rowPlaceholders are pre-built, already-quoted SQL fragments.
|
|
// Note: with no unique key there is nothing to conflict on, so this degrades to a
|
|
// plain INSERT (web_logs has no primary key on any engine).
|
|
InsertIgnoreSQL(table, quotedCols, rowPlaceholders string) string
|
|
|
|
// UpsertExcludedRef returns how the conflict branch of an upsert refers to the
|
|
// value the statement tried to insert, for accumulating updates such as
|
|
// "cnt = table.cnt + <incoming cnt>".
|
|
// SQLite / PostgreSQL: excluded.<col>
|
|
// MySQL: VALUES(<col>)
|
|
UpsertExcludedRef(col string) string
|
|
|
|
// FormatLocalTime returns a SQL expression that renders a DATETIME column as
|
|
// 'YYYY-MM-DD HH:MM:SS' in local time, matching customtype.JsonTime.MarshalJSON.
|
|
//
|
|
// Storage contract: SamWaf writes time.Time in LOCAL time, never UTC.
|
|
// MySQL: DSN carries loc=Local, so the DATETIME column holds the local
|
|
// wall clock with no zone info — format it as-is, no CONVERT_TZ.
|
|
// SQLite: go-wxsqlite3 binds time.Time as '2006-01-02 15:04:05.999999999-07:00',
|
|
// so the text carries a zone suffix that SQLite normalizes to UTC;
|
|
// the local offset has to be added back.
|
|
FormatLocalTime(colExpr string) string
|
|
|
|
// RenameTable renames src to dst within the same schema.
|
|
// For SQLite the caller handles file-level rename; this handles in-DB rename
|
|
// for server databases (MySQL: RENAME TABLE; MSSQL: sp_rename).
|
|
RenameTable(db *gorm.DB, src, dst string) error
|
|
|
|
// ShardSwapTable atomically archives liveTable to archiveTable and leaves an
|
|
// empty liveTable in place (same structure + indexes), used for log table
|
|
// sharding on server databases.
|
|
// MySQL: CREATE TABLE <live>_tmp LIKE <live>; RENAME TABLE <live> TO <archive>, <live>_tmp TO <live>
|
|
// File-based databases (SQLite) shard via OS-level file rename instead and
|
|
// return an error here ("not supported").
|
|
ShardSwapTable(db *gorm.DB, liveTable, archiveTable string) error
|
|
|
|
// CreatePartition creates an empty time partition of baseTable, cloning its
|
|
// structure and indexes. partTable must be named "<baseTable>_<period>".
|
|
// MySQL: CREATE TABLE <part> LIKE <base>
|
|
// Postgres: CREATE TABLE <part> (LIKE <base> INCLUDING ALL)
|
|
// A partition here is a period-named table, not a native RANGE partition -
|
|
// see wafdb/dialect/partition.go for why. File-based databases (SQLite)
|
|
// partition by file and return an error; check IsFileBased() first.
|
|
CreatePartition(db *gorm.DB, baseTable, partTable string) error
|
|
|
|
// ListPartitions returns the partition tables of baseTable ("<base>_<suffix>"),
|
|
// sorted by name. File-based databases (SQLite) partition by file, so this
|
|
// legitimately returns nothing there.
|
|
ListPartitions(db *gorm.DB, baseTable string) ([]string, error)
|
|
|
|
// DropPartition drops one whole partition table, replacing per-row DELETE +
|
|
// VACUUM for expiry. partTable must be named "<baseTable>_<period>" - the pair
|
|
// is validated so an unrelated table cannot be dropped by mistake.
|
|
// File-based databases (SQLite) delete the partition file instead and return an error.
|
|
DropPartition(db *gorm.DB, baseTable, partTable string) error
|
|
|
|
// TableSizeMB returns the on-disk size (data + index) of a table in MB,
|
|
// used by the sharding task to detect the size threshold on server databases.
|
|
// File-based databases (SQLite) return 0 (the caller uses the file size instead).
|
|
TableSizeMB(db *gorm.DB, table string) (int64, error)
|
|
|
|
// ListTables returns all user-defined table names in the current schema.
|
|
ListTables(db *gorm.DB) ([]string, error)
|
|
|
|
// TableExists reports whether the named table exists.
|
|
TableExists(db *gorm.DB, name string) bool
|
|
|
|
// ColumnInfo returns column metadata for the given table.
|
|
ColumnInfo(db *gorm.DB, table string) ([]ColumnMeta, error)
|
|
|
|
// IndexInfo returns index metadata for the given table.
|
|
IndexInfo(db *gorm.DB, table string) ([]IndexMeta, error)
|
|
|
|
// CollectMetrics gathers runtime performance metrics.
|
|
// name is a human-readable label; path is the file path (SQLite) or empty.
|
|
CollectMetrics(db *gorm.DB, name, path string) (*DBMetrics, error)
|
|
}
|