diff --git a/pkg/hostman/guestman/guestman.go b/pkg/hostman/guestman/guestman.go index f4fb096431..52a6be7048 100644 --- a/pkg/hostman/guestman/guestman.go +++ b/pkg/hostman/guestman/guestman.go @@ -206,6 +206,7 @@ func (m *SGuestManager) startContainerSyncLoop() { m.reconcileContainerLoop(m.podCache) }() } + StartContainerLogRotateLoop(m) } } diff --git a/pkg/hostman/guestman/pod.go b/pkg/hostman/guestman/pod.go index a1f02db827..3663302c8b 100644 --- a/pkg/hostman/guestman/pod.go +++ b/pkg/hostman/guestman/pod.go @@ -147,6 +147,11 @@ type PodInstance interface { IsInternalRemoved(ctrCriId string) bool GetPodContainerCriIds() []string + + // For container log rotation: log dir, relative log path per container, and ctrId->criId map + GetPodLogDir() string + GetContainerLogPath(ctrId string) string + ListContainerCriIds() map[string]string } type sContainer struct { @@ -643,6 +648,26 @@ func (s *sPodGuestInstance) getPodLogDir() string { return filepath.Join(s.HomeDir(), "logs") } +func (s *sPodGuestInstance) GetPodLogDir() string { + return s.getPodLogDir() +} + +func (s *sPodGuestInstance) getContainerLogPath(ctrId string) string { + return filepath.Join(fmt.Sprintf("%s.log", ctrId)) +} + +func (s *sPodGuestInstance) GetContainerLogPath(ctrId string) string { + return s.getContainerLogPath(ctrId) +} + +func (s *sPodGuestInstance) ListContainerCriIds() map[string]string { + out := make(map[string]string, len(s.containers)) + for ctrId, c := range s.containers { + out[ctrId] = c.CRIId + } + return out +} + func (s *sPodGuestInstance) getShmDir() string { return filepath.Join(s.HomeDir(), "shm") } @@ -1726,10 +1751,6 @@ func (s *sPodGuestInstance) CreateContainer(ctx context.Context, userCred mcclie return nil, nil } -func (s *sPodGuestInstance) getContainerLogPath(ctrId string) string { - return filepath.Join(fmt.Sprintf("%s.log", ctrId)) -} - func (s *sPodGuestInstance) getLxcfsMounts() []*runtimeapi.Mount { // lxcfsPath := "/var/lib/lxc/lxcfs" lxcfsPath := options.HostOptions.LxcfsPath diff --git a/pkg/hostman/guestman/pod_logrotate.go b/pkg/hostman/guestman/pod_logrotate.go new file mode 100644 index 0000000000..afd02cb05c --- /dev/null +++ b/pkg/hostman/guestman/pod_logrotate.go @@ -0,0 +1,200 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package guestman + +import ( + "context" + "os" + "path/filepath" + "strconv" + "sync" + "time" + + "github.com/docker/go-units" + runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" + + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/hostman/options" +) + +const ( + containerLogRotateInterval = 10 * time.Minute +) + +var ( + containerLogRotateMu sync.Mutex +) + +// RunContainerLogRotate runs log rotation for all running pod containers once. +// It is safe to call concurrently; only one run executes at a time. +func RunContainerLogRotate(ctx context.Context, manager *SGuestManager, maxSizeBytes int64, maxFiles int) { + if maxSizeBytes <= 0 || maxFiles <= 0 { + return + } + if !containerLogRotateMu.TryLock() { + return + } + defer containerLogRotateMu.Unlock() + + cri := manager.host.GetCRI() + if cri == nil { + return + } + runtimeClient := cri.GetRuntimeClient() + if runtimeClient == nil { + return + } + + manager.Servers.Range(func(_id, value interface{}) bool { + select { + case <-ctx.Done(): + return false + default: + } + pod, ok := value.(PodInstance) + if !ok { + return true + } + if !pod.IsRunning() { + return true + } + logDir := pod.GetPodLogDir() + for ctrId, criId := range pod.ListContainerCriIds() { + if criId == "" { + continue + } + logPath := filepath.Join(logDir, pod.GetContainerLogPath(ctrId)) + if err := rotateContainerLog(ctx, logPath, criId, maxSizeBytes, maxFiles, runtimeClient); err != nil { + log.Warningf("rotate container log %s (cri %s): %v", logPath, criId, err) + } + } + return true + }) +} + +// rotateContainerLog rotates the container log file at logPath if it exceeds maxSizeBytes, +// keeps up to maxFiles (current + rotated), then calls ReopenContainerLog for the container. +func rotateContainerLog(ctx context.Context, logPath, criId string, maxSizeBytes int64, maxFiles int, runtimeClient runtimeapi.RuntimeServiceClient) error { + dir := filepath.Dir(logPath) + base := filepath.Base(logPath) + // Always try to cleanup stale rotated logs, even if we don't rotate this time. + cleanupRotatedLogs(dir, base, maxFiles) + + info, err := os.Stat(logPath) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + if !info.Mode().IsRegular() { + return nil + } + if info.Size() < maxSizeBytes { + return nil + } + + // Rename from high to low so we don't overwrite: .(n-1)->.n, ..., .1->.2, then main->.1 + for i := maxFiles - 1; i >= 2; i-- { + src := filepath.Join(dir, base+"."+strconv.Itoa(i-1)) + dst := filepath.Join(dir, base+"."+strconv.Itoa(i)) + if _, err := os.Stat(src); err != nil { + if os.IsNotExist(err) { + continue + } + return err + } + if err := os.Rename(src, dst); err != nil { + log.Warningf("rename %s -> %s: %v", src, dst, err) + } + } + // Then rotate current log to .1 + dst1 := filepath.Join(dir, base+".1") + if err := os.Rename(logPath, dst1); err != nil { + return errors.Wrapf(err, "rename %s -> %s", logPath, dst1) + } + // Cleanup again after shift. + cleanupRotatedLogs(dir, base, maxFiles) + + _, err = runtimeClient.ReopenContainerLog(ctx, &runtimeapi.ReopenContainerLogRequest{ + ContainerId: criId, + }) + if err != nil { + // If runtime failed to reopen the log, try best to rename back so containerd keeps writing to logPath. + if _, statErr := os.Stat(logPath); os.IsNotExist(statErr) { + if rbErr := os.Rename(dst1, logPath); rbErr != nil && !os.IsNotExist(rbErr) { + log.Warningf("reopen log failed, rename back %s -> %s: %v", dst1, logPath, rbErr) + } + } + return errors.Wrap(err, "ReopenContainerLog") + } + return nil +} + +func cleanupRotatedLogs(dir, base string, maxFiles int) { + // Keep only .1 .. .(maxFiles-1). Remove .maxFiles and above. + if maxFiles <= 0 { + return + } + // Stop after some consecutive not-exist to avoid infinite loop. + miss := 0 + for i := maxFiles; i < maxFiles+100; i++ { + p := filepath.Join(dir, base+"."+strconv.Itoa(i)) + if err := os.Remove(p); err != nil { + if os.IsNotExist(err) { + miss++ + if miss >= 20 { + return + } + continue + } + log.Errorf("remove old container log %s: %v", p, err) + continue + } + log.Infof("remove old container log %s", p) + miss = 0 + } +} + +// StartContainerLogRotateLoop starts a goroutine that periodically runs container log rotation +// when options are enabled. Call from guestman after manager and host are ready. +func StartContainerLogRotateLoop(manager *SGuestManager) { + maxSizeStr := options.HostOptions.ContainerLogMaxSize + maxFiles := options.HostOptions.ContainerLogMaxFiles + if maxSizeStr == "" || maxFiles <= 0 { + return + } + maxSizeBytes, err := units.FromHumanSize(maxSizeStr) + if err != nil { + log.Warningf("parse ContainerLogMaxSize %q: %v, disable container log rotate", maxSizeStr, err) + return + } + if maxSizeBytes <= 0 { + return + } + + go func() { + ticker := time.NewTicker(containerLogRotateInterval) + defer ticker.Stop() + for range ticker.C { + ctx, cancel := context.WithTimeout(context.Background(), 2*containerLogRotateInterval) + RunContainerLogRotate(ctx, manager, maxSizeBytes, maxFiles) + cancel() + } + }() + log.Infof("container log rotate started: maxSize=%s, maxFiles=%d", maxSizeStr, maxFiles) +} diff --git a/pkg/hostman/options/options.go b/pkg/hostman/options/options.go index 26d1b72cdb..17577b8291 100644 --- a/pkg/hostman/options/options.go +++ b/pkg/hostman/options/options.go @@ -268,6 +268,10 @@ type SHostOptions struct { EnableDirtyRecoverySeconds int `help:"Seconds to delay enable dirty guests recovery feature, default 15 minutes" default:"900"` EnableContainerCniPortmap bool `help:"Use container cni portmap plugin" default:"false"` DisableReconcileContainer bool `help:"disable reconcile container" default:"false"` + + // Container log rotation (Docker-style max-size and max-file) + ContainerLogMaxSize string `help:"Max size of container log file before rotation (e.g. 10m, 100k). Disabled if empty or <= 0" default:"256m"` + ContainerLogMaxFiles int `help:"Max number of container log files to keep (current + rotated). Disabled if <= 0" default:"1"` } func (o SHostOptions) HostLocalNetconfPath(br string) string {