Merge pull request #4216 from bestony/agent/devbox-coding/3ff3c99d

feat(ops): add Host filtering to system logs
This commit is contained in:
Wesley Liddick
2026-07-14 10:13:40 +08:00
committed by GitHub
17 changed files with 286 additions and 4 deletions
@@ -15,6 +15,7 @@ import (
type opsSystemLogCleanupRequest struct {
StartTime string `json:"start_time"`
EndTime string `json:"end_time"`
Host string `json:"host"`
Level string `json:"level"`
Component string `json:"component"`
@@ -56,6 +57,7 @@ func (h *OpsHandler) ListSystemLogs(c *gin.Context) {
PageSize: pageSize,
StartTime: &start,
EndTime: &end,
Host: strings.TrimSpace(c.Query("host")),
Level: strings.TrimSpace(c.Query("level")),
Component: strings.TrimSpace(c.Query("component")),
RequestID: strings.TrimSpace(c.Query("request_id")),
@@ -153,6 +155,7 @@ func (h *OpsHandler) CleanupSystemLogs(c *gin.Context) {
filter := &service.OpsSystemLogCleanupFilter{
StartTime: start,
EndTime: end,
Host: strings.TrimSpace(req.Host),
Level: strings.TrimSpace(req.Level),
Component: strings.TrimSpace(req.Component),
RequestID: strings.TrimSpace(req.RequestID),
@@ -2,6 +2,7 @@ package admin
import (
"bytes"
"context"
"encoding/json"
"net/http"
"net/http/httptest"
@@ -19,6 +20,26 @@ type responseEnvelope struct {
Data json.RawMessage `json:"data"`
}
type opsSystemLogCaptureRepo struct {
service.OpsRepository
listFilter *service.OpsSystemLogFilter
cleanupFilter *service.OpsSystemLogCleanupFilter
}
func (r *opsSystemLogCaptureRepo) ListSystemLogs(_ context.Context, filter *service.OpsSystemLogFilter) (*service.OpsSystemLogList, error) {
r.listFilter = filter
return &service.OpsSystemLogList{Logs: []*service.OpsSystemLog{}, Page: filter.Page, PageSize: filter.PageSize}, nil
}
func (r *opsSystemLogCaptureRepo) DeleteSystemLogs(_ context.Context, filter *service.OpsSystemLogCleanupFilter) (int64, error) {
r.cleanupFilter = filter
return 1, nil
}
func (r *opsSystemLogCaptureRepo) InsertSystemLogCleanupAudit(_ context.Context, _ *service.OpsSystemLogCleanupAudit) error {
return nil
}
func newOpsSystemLogTestRouter(handler *OpsHandler, withUser bool) *gin.Engine {
gin.SetMode(gin.TestMode)
r := gin.New()
@@ -121,6 +142,23 @@ func TestOpsSystemLogHandler_ListSuccess(t *testing.T) {
}
}
func TestOpsSystemLogHandler_ListAcceptsHost(t *testing.T) {
repo := &opsSystemLogCaptureRepo{}
svc := service.NewOpsService(repo, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
h := NewOpsHandler(svc)
r := newOpsSystemLogTestRouter(h, false)
w := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/logs?host=api-node-1", nil)
r.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("status=%d, want 200", w.Code)
}
if repo.listFilter == nil || repo.listFilter.Host != "api-node-1" {
t.Fatalf("host filter = %+v, want api-node-1", repo.listFilter)
}
}
func TestOpsSystemLogHandler_CleanupUnauthorized(t *testing.T) {
svc := service.NewOpsService(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
h := NewOpsHandler(svc)
@@ -205,6 +243,24 @@ func TestOpsSystemLogHandler_CleanupAcceptsAPIKeyID(t *testing.T) {
}
}
func TestOpsSystemLogHandler_CleanupAcceptsHost(t *testing.T) {
repo := &opsSystemLogCaptureRepo{}
svc := service.NewOpsService(repo, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
h := NewOpsHandler(svc)
r := newOpsSystemLogTestRouter(h, true)
w := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodPost, "/logs/cleanup", bytes.NewBufferString(`{"host":"api-node-1"}`))
req.Header.Set("Content-Type", "application/json")
r.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("status=%d, want 200", w.Code)
}
if repo.cleanupFilter == nil || repo.cleanupFilter.Host != "api-node-1" {
t.Fatalf("host filter = %+v, want api-node-1", repo.cleanupFilter)
}
}
func TestOpsSystemLogHandler_CleanupInvalidAPIKeyID(t *testing.T) {
svc := service.NewOpsService(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)
h := NewOpsHandler(svc)
+10
View File
@@ -718,6 +718,7 @@ func (r *opsRepository) BatchInsertSystemLogs(ctx context.Context, inputs []*ser
stmt, err := tx.PrepareContext(ctx, pq.CopyIn(
"ops_system_logs",
"created_at",
"host",
"level",
"component",
"message",
@@ -760,6 +761,7 @@ func (r *opsRepository) BatchInsertSystemLogs(ctx context.Context, inputs []*ser
if _, err := stmt.ExecContext(
ctx,
createdAt.UTC(),
opsNullString(input.Host),
level,
component,
message,
@@ -827,6 +829,7 @@ func (r *opsRepository) ListSystemLogs(ctx context.Context, filter *service.OpsS
SELECT
l.id,
l.created_at,
COALESCE(l.host, ''),
l.level,
COALESCE(l.component, ''),
COALESCE(l.message, ''),
@@ -859,6 +862,7 @@ LIMIT $` + itoa(len(args)+1) + ` OFFSET $` + itoa(len(args)+2)
if err := rows.Scan(
&item.ID,
&item.CreatedAt,
&item.Host,
&item.Level,
&item.Component,
&item.Message,
@@ -1130,6 +1134,11 @@ func buildOpsSystemLogsWhere(filter *service.OpsSystemLogFilter) (string, []any,
hasConstraint = true
}
if filter != nil {
if v := strings.TrimSpace(filter.Host); v != "" {
args = append(args, v)
clauses = append(clauses, "l.host = $"+itoa(len(args)))
hasConstraint = true
}
if v := strings.ToLower(strings.TrimSpace(filter.Level)); v != "" {
args = append(args, v)
clauses = append(clauses, "LOWER(COALESCE(l.level,'')) = $"+itoa(len(args)))
@@ -1194,6 +1203,7 @@ func buildOpsSystemLogsCleanupWhere(filter *service.OpsSystemLogCleanupFilter) (
listFilter := &service.OpsSystemLogFilter{
StartTime: filter.StartTime,
EndTime: filter.EndTime,
Host: filter.Host,
Level: filter.Level,
Component: filter.Component,
RequestID: filter.RequestID,
@@ -18,6 +18,7 @@ func TestBuildOpsSystemLogsWhere_WithClientRequestIDAndUserID(t *testing.T) {
filter := &service.OpsSystemLogFilter{
StartTime: &start,
EndTime: &end,
Host: "api-node-1",
Level: "warn",
Component: "http.access",
RequestID: "req-1",
@@ -37,8 +38,11 @@ func TestBuildOpsSystemLogsWhere_WithClientRequestIDAndUserID(t *testing.T) {
if where == "" {
t.Fatalf("where should not be empty")
}
if len(args) != 12 {
t.Fatalf("args len = %d, want 12", len(args))
if len(args) != 13 {
t.Fatalf("args len = %d, want 13", len(args))
}
if !contains(where, "l.host = $") {
t.Fatalf("where should include host condition: %s", where)
}
if !contains(where, "COALESCE(l.client_request_id,'') = $") {
t.Fatalf("where should include client_request_id condition: %s", where)
@@ -68,6 +72,7 @@ func TestBuildOpsSystemLogsCleanupWhere_WithClientRequestIDAndUserID(t *testing.
userID := int64(9)
apiKeyID := int64(10)
filter := &service.OpsSystemLogCleanupFilter{
Host: "api-node-2",
ClientRequestID: "creq-9",
UserID: &userID,
APIKeyID: &apiKeyID,
@@ -77,8 +82,11 @@ func TestBuildOpsSystemLogsCleanupWhere_WithClientRequestIDAndUserID(t *testing.
if !hasConstraint {
t.Fatalf("expected hasConstraint=true")
}
if len(args) != 3 {
t.Fatalf("args len = %d, want 3", len(args))
if len(args) != 4 {
t.Fatalf("args len = %d, want 4", len(args))
}
if !contains(where, "l.host = $") {
t.Fatalf("where should include host condition: %s", where)
}
if !contains(where, "COALESCE(l.client_request_id,'') = $") {
t.Fatalf("where should include client_request_id condition: %s", where)
+1
View File
@@ -8,6 +8,7 @@ import (
type OpsSystemLog struct {
ID int64 `json:"id"`
CreatedAt time.Time `json:"created_at"`
Host string `json:"host"`
Level string `json:"level"`
Component string `json:"component"`
Message string `json:"message"`
+3
View File
@@ -194,6 +194,7 @@ type OpsInsertSystemMetricsInput struct {
type OpsInsertSystemLogInput struct {
CreatedAt time.Time
Host string
Level string
Component string
Message string
@@ -210,6 +211,7 @@ type OpsInsertSystemLogInput struct {
type OpsSystemLogFilter struct {
StartTime *time.Time
EndTime *time.Time
Host string
Level string
Component string
@@ -230,6 +232,7 @@ type OpsSystemLogFilter struct {
type OpsSystemLogCleanupFilter struct {
StartTime *time.Time
EndTime *time.Time
Host string
Level string
Component string
@@ -89,6 +89,7 @@ func marshalSystemLogCleanupConditions(filter *OpsSystemLogCleanupFilter) string
return "{}"
}
payload := map[string]any{
"host": strings.TrimSpace(filter.Host),
"level": strings.TrimSpace(filter.Level),
"component": strings.TrimSpace(filter.Component),
"request_id": strings.TrimSpace(filter.RequestID),
@@ -101,6 +101,7 @@ func TestOpsServiceCleanupSystemLogs_SuccessAndAudit(t *testing.T) {
now := time.Now().UTC()
filter := &OpsSystemLogCleanupFilter{
StartTime: &now,
Host: "api-node-1",
Level: "warn",
RequestID: "req-1",
ClientRequestID: "creq-1",
@@ -119,6 +120,9 @@ func TestOpsServiceCleanupSystemLogs_SuccessAndAudit(t *testing.T) {
if audit == nil {
t.Fatalf("expected cleanup audit")
}
if !strings.Contains(audit.Conditions, `"host":"api-node-1"`) {
t.Fatalf("audit conditions should include host: %s", audit.Conditions)
}
if !strings.Contains(audit.Conditions, `"client_request_id":"creq-1"`) {
t.Fatalf("audit conditions should include client_request_id: %s", audit.Conditions)
}
@@ -27,6 +27,7 @@ type OpsSystemLogSinkHealth struct {
type OpsSystemLogSink struct {
opsRepo OpsRepository
host string
queue chan *logger.LogEvent
@@ -45,10 +46,14 @@ type OpsSystemLogSink struct {
lastError atomic.Value
}
const maxSystemLogHostLength = 255
func NewOpsSystemLogSink(opsRepo OpsRepository) *OpsSystemLogSink {
ctx, cancel := context.WithCancel(context.Background())
rawHost, err := os.Hostname()
s := &OpsSystemLogSink{
opsRepo: opsRepo,
host: normalizeSystemLogHost(rawHost, err),
queue: make(chan *logger.LogEvent, 5000),
batchSize: 200,
flushInterval: time.Second,
@@ -59,6 +64,18 @@ func NewOpsSystemLogSink(opsRepo OpsRepository) *OpsSystemLogSink {
return s
}
func normalizeSystemLogHost(host string, err error) string {
host = strings.TrimSpace(host)
if err != nil || host == "" {
return "unknown"
}
runes := []rune(host)
if len(runes) > maxSystemLogHostLength {
return string(runes[:maxSystemLogHostLength])
}
return host
}
func (s *OpsSystemLogSink) Start() {
if s == nil || s.opsRepo == nil {
return
@@ -220,6 +237,7 @@ func (s *OpsSystemLogSink) flushBatch(baseCtx context.Context, batch []*logger.L
inputs = append(inputs, &OpsInsertSystemLogInput{
CreatedAt: createdAt,
Host: s.host,
Level: strings.ToLower(strings.TrimSpace(event.Level)),
Component: component,
Message: message,
@@ -140,6 +140,7 @@ func TestOpsSystemLogSink_StartStopAndFlushSuccess(t *testing.T) {
}
sink := NewOpsSystemLogSink(repo)
sink.host = "api-node-1"
sink.batchSize = 1
sink.flushInterval = 10 * time.Millisecond
sink.Start()
@@ -172,6 +173,9 @@ func TestOpsSystemLogSink_StartStopAndFlushSuccess(t *testing.T) {
t.Fatalf("captured len = %d, want 1", len(captured))
}
item := captured[0]
if item.Host != "api-node-1" {
t.Fatalf("host = %q, want api-node-1", item.Host)
}
if item.RequestID != "req-1" || item.ClientRequestID != "creq-1" {
t.Fatalf("unexpected request ids: %+v", item)
}
@@ -324,3 +328,20 @@ func TestOpsSystemLogSink_HelperFunctions(t *testing.T) {
}
}
}
func TestNormalizeSystemLogHost(t *testing.T) {
if got := normalizeSystemLogHost(" api-node-1 ", nil); got != "api-node-1" {
t.Fatalf("trimmed host = %q, want api-node-1", got)
}
if got := normalizeSystemLogHost("", nil); got != "unknown" {
t.Fatalf("empty host = %q, want unknown", got)
}
if got := normalizeSystemLogHost("api-node-1", errors.New("hostname unavailable")); got != "unknown" {
t.Fatalf("errored host = %q, want unknown", got)
}
longHost := strings.Repeat("节", maxSystemLogHostLength+1)
got := normalizeSystemLogHost(longHost, nil)
if runeCount := len([]rune(got)); runeCount != maxSystemLogHostLength {
t.Fatalf("truncated host rune count = %d, want %d", runeCount, maxSystemLogHostLength)
}
}
@@ -0,0 +1,3 @@
-- Track the application host that emitted each indexed system log.
ALTER TABLE ops_system_logs
ADD COLUMN IF NOT EXISTS host VARCHAR(255);
@@ -0,0 +1,2 @@
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_ops_system_logs_host_created_at
ON ops_system_logs (host, created_at DESC);
+3
View File
@@ -828,6 +828,7 @@ export interface OpsRuntimeLogConfig {
export interface OpsSystemLog {
id: number
created_at: string
host: string
level: string
component: string
message: string
@@ -849,6 +850,7 @@ export interface OpsSystemLogQuery {
time_range?: '5m' | '30m' | '1h' | '6h' | '24h' | '7d' | '30d'
start_time?: string
end_time?: string
host?: string
level?: string
component?: string
request_id?: string
@@ -864,6 +866,7 @@ export interface OpsSystemLogQuery {
export interface OpsSystemLogCleanupRequest {
start_time?: string
end_time?: string
host?: string
level?: string
component?: string
request_id?: string
@@ -50,6 +50,7 @@ export default {
timeRange: 'Time range',
startTime: 'Start time (optional)',
endTime: 'End time (optional)',
host: 'Host',
component: 'Component',
componentPlaceholder: 'e.g. http.access',
keyId: 'KEY ID',
@@ -50,6 +50,7 @@ export default {
timeRange: '时间范围',
startTime: '开始时间(可选)',
endTime: '结束时间(可选)',
host: 'Host',
component: '组件',
componentPlaceholder: '例如 http.access',
keyId: 'KEY ID',
@@ -48,6 +48,7 @@ const filters = reactive({
time_range: '1h' as '5m' | '30m' | '1h' | '6h' | '24h' | '7d' | '30d',
start_time: '',
end_time: '',
host: '',
level: '',
component: '',
request_id: '',
@@ -175,6 +176,7 @@ const buildQuery = () => {
}
if (filters.start_time) query.start_time = toRFC3339(filters.start_time)
if (filters.end_time) query.end_time = toRFC3339(filters.end_time)
if (filters.host.trim()) query.host = filters.host.trim()
if (filters.level.trim()) query.level = filters.level.trim()
if (filters.component.trim()) query.component = filters.component.trim()
if (filters.request_id.trim()) query.request_id = filters.request_id.trim()
@@ -288,6 +290,7 @@ const cleanupCurrentFilter = async () => {
const payload = {
start_time: toRFC3339(filters.start_time),
end_time: toRFC3339(filters.end_time),
host: filters.host.trim() || undefined,
level: filters.level.trim() || undefined,
component: filters.component.trim() || undefined,
request_id: filters.request_id.trim() || undefined,
@@ -313,6 +316,7 @@ const resetFilters = () => {
filters.time_range = '1h'
filters.start_time = ''
filters.end_time = ''
filters.host = ''
filters.level = ''
filters.component = ''
filters.request_id = ''
@@ -454,6 +458,10 @@ onMounted(async () => {
{{ t('admin.ops.systemLogs.component') }}
<input v-model="filters.component" type="text" class="input mt-1" :placeholder="t('admin.ops.systemLogs.componentPlaceholder')" />
</label>
<label class="text-xs text-gray-600 dark:text-gray-300">
{{ t('admin.ops.systemLogs.host') }}
<input v-model="filters.host" type="text" class="input mt-1" />
</label>
<label class="text-xs text-gray-600 dark:text-gray-300">
request_id
<input v-model="filters.request_id" type="text" class="input mt-1" />
@@ -503,6 +511,7 @@ onMounted(async () => {
<thead class="bg-gray-50 dark:bg-dark-900">
<tr>
<th class="w-[170px] px-3 py-2 text-left text-[11px] font-semibold text-gray-500">{{ t('admin.ops.systemLogs.time') }}</th>
<th class="w-[160px] px-3 py-2 text-left text-[11px] font-semibold text-gray-500">{{ t('admin.ops.systemLogs.host') }}</th>
<th class="w-[80px] px-3 py-2 text-left text-[11px] font-semibold text-gray-500">{{ t('admin.ops.systemLogs.level') }}</th>
<th class="px-3 py-2 text-left text-[11px] font-semibold text-gray-500">{{ t('admin.ops.systemLogs.logDetails') }}</th>
</tr>
@@ -510,6 +519,9 @@ onMounted(async () => {
<tbody class="divide-y divide-gray-100 dark:divide-dark-800">
<tr v-for="row in logs" :key="row.id" class="align-top">
<td class="px-3 py-2 text-xs text-gray-700 dark:text-gray-300">{{ formatTime(row.created_at) }}</td>
<td class="px-3 py-2 text-xs text-gray-700 dark:text-gray-300">
<span class="block truncate" :title="row.host || '-'">{{ row.host || '-' }}</span>
</td>
<td class="px-3 py-2 text-xs">
<span class="inline-flex rounded-full px-2 py-0.5 font-semibold" :class="levelBadgeClass(row.level)">
{{ row.level }}
@@ -0,0 +1,135 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
import { defineComponent } from 'vue'
import { flushPromises, mount } from '@vue/test-utils'
import OpsSystemLogTable from '../OpsSystemLogTable.vue'
import enLocale from '@/i18n/locales/en'
import zhLocale from '@/i18n/locales/zh'
const mockListSystemLogs = vi.fn()
const mockCleanupSystemLogs = vi.fn()
const mockGetSystemLogSinkHealth = vi.fn()
const mockGetRuntimeLogConfig = vi.fn()
vi.mock('@/api/admin/ops', () => ({
opsAPI: {
listSystemLogs: (...args: any[]) => mockListSystemLogs(...args),
cleanupSystemLogs: (...args: any[]) => mockCleanupSystemLogs(...args),
getSystemLogSinkHealth: (...args: any[]) => mockGetSystemLogSinkHealth(...args),
getRuntimeLogConfig: (...args: any[]) => mockGetRuntimeLogConfig(...args),
},
}))
vi.mock('@/stores', () => ({
useAppStore: () => ({
showError: vi.fn(),
showSuccess: vi.fn(),
}),
}))
vi.mock('vue-i18n', async (importOriginal) => {
const actual = await importOriginal<typeof import('vue-i18n')>()
return {
...actual,
useI18n: () => ({ t: (key: string) => key }),
}
})
const SelectStub = defineComponent({
name: 'SelectControlStub',
props: {
modelValue: {
type: [String, Number],
default: '',
},
},
emits: ['update:modelValue'],
template: '<div class="select-stub" />',
})
const PaginationStub = defineComponent({
name: 'PaginationStub',
template: '<div class="pagination-stub" />',
})
const runtimeConfig = {
level: 'info',
enable_sampling: false,
sampling_initial: 100,
sampling_thereafter: 100,
caller: true,
stacktrace_level: 'error',
retention_days: 30,
}
const sinkHealth = {
queue_depth: 0,
queue_capacity: 5000,
dropped_count: 0,
write_failed_count: 0,
written_count: 1,
avg_write_delay_ms: 0,
}
describe('OpsSystemLogTable host support', () => {
beforeEach(() => {
vi.clearAllMocks()
vi.spyOn(window, 'confirm').mockReturnValue(true)
mockListSystemLogs.mockResolvedValue({
items: [
{
id: 1,
created_at: '2026-07-14T00:10:01Z',
host: 'api-node-1',
level: 'warn',
component: 'app',
message: 'request failed',
},
],
total: 1,
page: 1,
page_size: 20,
})
mockCleanupSystemLogs.mockResolvedValue({ deleted: 1 })
mockGetSystemLogSinkHealth.mockResolvedValue(sinkHealth)
mockGetRuntimeLogConfig.mockResolvedValue(runtimeConfig)
})
it('renders the host and sends it with list and cleanup filters', async () => {
const wrapper = mount(OpsSystemLogTable, {
global: {
stubs: {
Select: SelectStub,
Pagination: PaginationStub,
},
},
})
await flushPromises()
expect(wrapper.text()).toContain('api-node-1')
const hostLabel = wrapper.findAll('label').find((label) => label.text().includes('admin.ops.systemLogs.host'))
expect(hostLabel).toBeDefined()
await hostLabel!.find('input').setValue(' api-node-2 ')
const searchButton = wrapper.findAll('button').find((button) => button.text() === 'admin.ops.systemLogs.search')
expect(searchButton).toBeDefined()
await searchButton!.trigger('click')
await flushPromises()
expect(mockListSystemLogs).toHaveBeenLastCalledWith(expect.objectContaining({ host: 'api-node-2' }))
const cleanupButton = wrapper.findAll('button').find((button) => button.text() === 'admin.ops.systemLogs.cleanCurrentFilters')
expect(cleanupButton).toBeDefined()
await cleanupButton!.trigger('click')
await flushPromises()
expect(mockCleanupSystemLogs).toHaveBeenCalledWith(expect.objectContaining({ host: 'api-node-2' }))
})
it.each([
['zh', zhLocale],
['en', enLocale],
])('defines the Host translation for %s', (_name, locale) => {
expect(locale.admin.ops.systemLogs.host).toBe('Host')
})
})