fix(scheduler): refactor the pending usage logic (#24109)

pkg/scheduler/cache/cache.go

@@ -18,6 +18,7 @@ import (
 	"fmt"
 	"reflect"
 	"sync"
+	"time"
 
 	"yunion.io/x/log"
 	expirationcache "yunion.io/x/pkg/util/cache"
@@ -145,16 +146,17 @@ func (c *schedulerCache) updateAllObjects() {
 		// if ids is nil and err is normalError then return.
 		return
 	} else if len(ids) > 0 {
-		log.V(10).Debugf("Update host/baremetal status list: %v", ids)
 		c.loadObjects(ids)
 	}
 }
 
 func (c *schedulerCache) loadObjects(ids []string) ([]interface{}, error) {
+	startTime := time.Now()
 	log.Infof("Start load %s, period: %v, ttl: %v", c.Name(), c.item.Period(), c.item.TTL())
 
 	defer func() {
-		log.Infof("End load %s", c.Name())
+		duration := time.Since(startTime)
+		log.Infof("End load %s, duration: %v", c.Name(), duration)
 	}()
 
 	var (

pkg/scheduler/data_manager/candidate_manager.go

@@ -18,12 +18,14 @@ import (
 	"fmt"
 	"time"
 
+	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/utils"
 
 	"yunion.io/x/onecloud/pkg/scheduler/cache"
 	candidatecache "yunion.io/x/onecloud/pkg/scheduler/cache/candidate"
 	"yunion.io/x/onecloud/pkg/scheduler/core"
+	schedmodels "yunion.io/x/onecloud/pkg/scheduler/models"
 )
 
 type CandidateGetArgs struct {
@@ -363,12 +365,24 @@ func (cm *CandidateManager) Reload(resType string, candidateIds []string) (
 }
 
 func (cm *CandidateManager) ReloadAll(resType string) ([]interface{}, error) {
+	// Mark the start of ReloadAll to protect pending usage added during reload
+	schedmodels.HostPendingUsageManager.SetReloadAllStartTime()
+
 	impl, err := cm.getImpl(resType)
 	if err != nil {
 		return nil, err
 	}
 
-	return impl.ReloadAll()
+	result, err := impl.ReloadAll()
+	if err == nil {
+		// Clear pending usage created before ReloadAll started
+		// This ensures pending usage doesn't leak when cache is fully rebuilt
+		// but protects pending usage added during reload
+		schedmodels.HostPendingUsageManager.ClearAllPendingUsage()
+	} else {
+		log.Errorf("[CandidateManager] Failed to reload all %q candidates: %v", resType, err)
+	}
+	return result, err
 }
 
 //type IDirtyPoolItem interface {

pkg/scheduler/manager/expire_queue.go

@@ -30,14 +30,16 @@ type ExpireManager struct {
 	expireChannel chan *api.ExpireArgs
 	stopCh        <-chan struct{}
 
-	mergeLock *sync.Mutex
+	mergeLock         *sync.Mutex
+	reloadCancelQueue *ReloadCancelQueue
 }
 
 func NewExpireManager(stopCh <-chan struct{}) *ExpireManager {
 	return &ExpireManager{
-		expireChannel: make(chan *api.ExpireArgs, o.Options.ExpireQueueMaxLength),
-		stopCh:        stopCh,
-		mergeLock:     new(sync.Mutex),
+		expireChannel:     make(chan *api.ExpireArgs, o.Options.ExpireQueueMaxLength),
+		stopCh:            stopCh,
+		mergeLock:         new(sync.Mutex),
+		reloadCancelQueue: NewReloadCancelQueue(stopCh),
 	}
 }
 
@@ -112,46 +114,37 @@ func (e *ExpireManager) batchMergeExpire() {
 		}
 	}
 	log.V(4).Infof("batchMergeExpire dirtyHosts: %v, dirtyBaremetals: %v", dirtyHosts, dirtyBaremetals)
-	wg := &sync.WaitGroup{}
-	wg.Add(2)
-	go func() {
-		defer wg.Done()
-		//dirtyHosts = notInSession(dirtyHosts, "host")
-		if len(dirtyHosts) > 0 {
-			log.V(10).Debugf("CleanDirty Hosts: %v\n", dirtyHosts)
-			if _, err := schedManager.CandidateManager.Reload("host", dirtyHostSets.List()); err != nil {
-				log.Errorf("Clean dirty hosts %v: %v", dirtyHosts, err)
-			}
-			schedManager.HistoryManager.CancelCandidatesPendingUsage(dirtyHosts)
-		}
-	}()
 
-	go func() {
-		defer wg.Done()
-		//dirtyBaremetals = notInSession(dirtyBaremetals, "baremetal")
-		if len(dirtyBaremetals) > 0 {
-			log.V(10).Debugf("CleanDirty Baremetals: %v\n", dirtyBaremetals)
-			if _, err := schedManager.CandidateManager.Reload("baremetal", dirtyBaremetalSets.List()); err != nil {
-				log.Errorf("Clean dirty baremetals %v: %v", dirtyBaremetals, err)
-			}
-			schedManager.HistoryManager.CancelCandidatesPendingUsage(dirtyBaremetals)
+	// Use queue to ensure reload completes before cancel
+	var hostTask, baremetalTask *ReloadCancelTask
+
+	if len(dirtyHosts) > 0 {
+		hostTask = &ReloadCancelTask{
+			ResType:     "host",
+			HostIds:     dirtyHostSets.List(),
+			ExpireHosts: dirtyHosts,
 		}
-	}()
-	if ok := e.waitTimeOut(wg, u.ToDuration(o.Options.ExpireQueueConsumptionTimeout)); !ok {
-		log.Errorln("time out reload data.")
-	}
-}
-
-func (e *ExpireManager) waitTimeOut(wg *sync.WaitGroup, timeout time.Duration) bool {
-	ch := make(chan struct{})
-	go func() {
-		wg.Wait()
-		close(ch)
-	}()
-	select {
-	case <-ch:
-		return true
-	case <-time.After(timeout):
-		return false
+	}
+
+	if len(dirtyBaremetals) > 0 {
+		baremetalTask = &ReloadCancelTask{
+			ResType:     "baremetal",
+			HostIds:     dirtyBaremetalSets.List(),
+			ExpireHosts: dirtyBaremetals,
+		}
+	}
+
+	// Add tasks to queue (will be processed asynchronously)
+	if hostTask != nil || baremetalTask != nil {
+		tasks := make([]*ReloadCancelTask, 0, 2)
+		if hostTask != nil {
+			tasks = append(tasks, hostTask)
+		}
+		if baremetalTask != nil {
+			tasks = append(tasks, baremetalTask)
+		}
+		e.reloadCancelQueue.AddBatch(tasks, nil)
+		log.Infof("Added reload+cancel tasks to queue: hosts=%d, baremetals=%d",
+			len(dirtyHosts), len(dirtyBaremetals))
 	}
 }

pkg/scheduler/manager/manager.go

@@ -46,6 +46,8 @@ type SchedulerManager struct {
 	DataManager        *data_manager.DataManager
 	CandidateManager   *data_manager.CandidateManager
 	KubeClusterManager *k8s.SKubeClusterManager
+
+	stopCh <-chan struct{}
 }
 
 func NewSchedulerManager(stopCh <-chan struct{}) *SchedulerManager {
@@ -57,6 +59,7 @@ func NewSchedulerManager(stopCh <-chan struct{}) *SchedulerManager {
 	sm.HistoryManager = NewHistoryManager(stopCh)
 	sm.TaskManager = NewTaskManager(stopCh)
 	sm.KubeClusterManager = k8s.NewKubeClusterManager(o.Options.Region, 30*time.Second)
+	sm.stopCh = stopCh
 
 	return sm
 }
@@ -80,8 +83,26 @@ func InitAndStart(stopCh <-chan struct{}) {
 }
 
 func (sm *SchedulerManager) start() {
+	// Safety net: periodically GC session pending usages older than 30 minutes.
+	go func() {
+		t := time.NewTicker(1 * time.Minute)
+		defer t.Stop()
+		for {
+			select {
+			case <-t.C:
+				n := schedmodels.HostPendingUsageManager.GCExpiredSessionUsages(30 * time.Minute)
+				if n > 0 {
+					log.Warningf("[PendingUsage] GC expired session usages: cleared=%d", n)
+				}
+			case <-sm.stopCh:
+				return
+			}
+		}
+	}()
+
 	startFuncs := []func(){
 		sm.ExpireManager.Run,
+		sm.ExpireManager.reloadCancelQueue.Run,
 		sm.CompletedManager.Run,
 		sm.HistoryManager.Run,
 		sm.TaskManager.Run,

pkg/scheduler/manager/reload_cancel_queue.go

@@ -0,0 +1,153 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package manager
+
+import (
+	"sync"
+	"time"
+
+	"yunion.io/x/log"
+
+	schedmodels "yunion.io/x/onecloud/pkg/scheduler/models"
+	o "yunion.io/x/onecloud/pkg/scheduler/options"
+)
+
+// ReloadCancelTask represents a task to reload candidates and then cancel pending usage
+type ReloadCancelTask struct {
+	ResType     string        // "host" or "baremetal"
+	HostIds     []string      // host/baremetal IDs to reload
+	ExpireHosts []*expireHost // hosts to cancel pending usage
+}
+
+// ReloadCancelQueue manages a queue of reload+cancel tasks
+// It ensures that cancel happens only after reload completes
+type ReloadCancelQueue struct {
+	queue  chan *ReloadCancelTask
+	stopCh <-chan struct{}
+	wg     sync.WaitGroup
+}
+
+// NewReloadCancelQueue creates a new ReloadCancelQueue
+func NewReloadCancelQueue(stopCh <-chan struct{}) *ReloadCancelQueue {
+	queueSize := o.Options.ExpireQueueMaxLength
+	return &ReloadCancelQueue{
+		queue:  make(chan *ReloadCancelTask, queueSize),
+		stopCh: stopCh,
+	}
+}
+
+// Add adds a reload+cancel task to the queue
+func (q *ReloadCancelQueue) Add(task *ReloadCancelTask) {
+	select {
+	case q.queue <- task:
+		log.Debugf("Added reload+cancel task: resType=%s, hostIds=%v", task.ResType, task.HostIds)
+	default:
+		log.Warningf("ReloadCancelQueue is full, dropping task: resType=%s", task.ResType)
+	}
+}
+
+// Run starts the queue worker
+func (q *ReloadCancelQueue) Run() {
+	defer close(q.queue)
+
+	// Start multiple workers for better throughput
+	workerCount := 2
+	for i := 0; i < workerCount; i++ {
+		q.wg.Add(1)
+		go q.worker(i)
+	}
+
+	<-q.stopCh
+	log.Infof("ReloadCancelQueue stopping...")
+
+	// Wait for all workers to finish current tasks
+	close(q.queue)
+	q.wg.Wait()
+	log.Infof("ReloadCancelQueue stopped")
+}
+
+// worker processes tasks from the queue
+func (q *ReloadCancelQueue) worker(id int) {
+	defer q.wg.Done()
+
+	log.Infof("ReloadCancelQueue worker %d started", id)
+	defer log.Infof("ReloadCancelQueue worker %d stopped", id)
+
+	for task := range q.queue {
+		if task == nil {
+			continue
+		}
+
+		q.processTask(task)
+	}
+}
+
+// processTask executes reload first, then cancel pending usage
+func (q *ReloadCancelQueue) processTask(task *ReloadCancelTask) {
+	startTime := time.Now()
+	log.Infof("[ReloadCancelQueue] Processing task: resType=%s, hostIds=%v, expireHosts=%d",
+		task.ResType, task.HostIds, len(task.ExpireHosts))
+
+	// Step 1: Reload candidates
+	if len(task.HostIds) > 0 {
+		log.Infof("[ReloadCancelQueue] Step 1: Starting reload for %s, hostIds=%v", task.ResType, task.HostIds)
+		// Mark the start of Reload to protect pending usage added during reload
+		schedmodels.HostPendingUsageManager.SetReloadStartTime()
+
+		reloadStart := time.Now()
+		if _, err := schedManager.CandidateManager.Reload(task.ResType, task.HostIds); err != nil {
+			log.Errorf("[ReloadCancelQueue] Failed to reload %s candidates %v: %v", task.ResType, task.HostIds, err)
+			// Continue to cancel even if reload fails, as cancel is independent
+		} else {
+			reloadDuration := time.Since(reloadStart)
+			log.Infof("[ReloadCancelQueue] Successfully reloaded %s candidates: %v (duration=%v)",
+				task.ResType, task.HostIds, reloadDuration)
+			/*
+				log.Infof("[ReloadCancelQueue] Step 2: Clearing pending usage for reloaded hosts")
+				// Clear pending usage created before Reload started (partial reload)
+				// This protects pending usage added during reload
+				cutoffTime := schedmodels.HostPendingUsageManager.GetReloadStartTime()
+				if cutoffTime.IsZero() {
+					log.Warningf("[ReloadCancelQueue] No cutoff time set, skipping pending usage cleanup for hosts %v", task.HostIds)
+				} else {
+					log.Infof("[ReloadCancelQueue] Clearing pending usage for hosts %v created before %v", task.HostIds, cutoffTime)
+					schedmodels.HostPendingUsageManager.GetStore().ClearHostPendingUsageBefore(task.HostIds, cutoffTime)
+					log.Infof("[ReloadCancelQueue] Cleared pending usage for hosts %v created before %v", task.HostIds, cutoffTime)
+				}
+			*/
+		}
+	}
+
+	// Step 2: Cancel pending usage (always execute, even if reload failed)
+	if len(task.ExpireHosts) > 0 {
+		log.Infof("[ReloadCancelQueue] Step 3: Canceling pending usage for %d expire hosts", len(task.ExpireHosts))
+		schedManager.HistoryManager.CancelCandidatesPendingUsage(task.ExpireHosts)
+		log.Infof("[ReloadCancelQueue] Canceled pending usage for %s: %v", task.ResType, task.ExpireHosts)
+	}
+
+	duration := time.Since(startTime)
+	log.Infof("[ReloadCancelQueue] Completed task: resType=%s, duration=%v", task.ResType, duration)
+}
+
+// AddBatch adds multiple tasks in batch
+func (q *ReloadCancelQueue) AddBatch(tasks []*ReloadCancelTask, _ []*ReloadCancelTask) {
+	// Simply add all tasks to queue
+	// Deduplication is not needed here as ExpireManager already merges by host ID
+	for _, task := range tasks {
+		if task != nil {
+			q.Add(task)
+		}
+	}
+}

pkg/scheduler/manager/task_history.go

@@ -217,10 +217,6 @@ func (m *HistoryManager) GetHistory(sessionId string) *HistoryItem {
 }
 
 func (m *HistoryManager) GetCancelUsage(sessionId string, hostId string) *models.SessionPendingUsage {
-	item := m.GetHistory(sessionId)
-	if item == nil {
-		return nil
-	}
 	usage, _ := models.HostPendingUsageManager.GetSessionUsage(sessionId, hostId)
 	return usage
 }
@@ -239,8 +235,11 @@ func (m *HistoryManager) CancelCandidatesPendingUsage(hosts []*expireHost) {
 		}
 		if err := models.HostPendingUsageManager.CancelPendingUsage(hostId, cancelUsage); err != nil {
 			log.Errorf("Cancel host %s usage %#v: %v", hostId, cancelUsage, err)
-		} else {
-			cancelUsage.StopTimer()
+		}
+		// Delete session usage after cancel
+		if cancelUsage.Usage.IsEmpty() {
+			models.HostPendingUsageManager.DeleteSessionUsage(cancelUsage)
+			log.Infof("Deleted empty session usage for session: %s, host: %s", sid, hostId)
 		}
 	}
 }

pkg/scheduler/models/pending_usage.go

@@ -17,12 +17,12 @@ package models
 import (
 	"context"
 	"fmt"
-	"runtime/debug"
 	"sync"
 	"time"
 
 	"github.com/pkg/errors"
 
+	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 
 	schedapi "yunion.io/x/onecloud/pkg/apis/scheduler"
@@ -30,20 +30,25 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/quotas"
 	computemodels "yunion.io/x/onecloud/pkg/compute/models"
 	"yunion.io/x/onecloud/pkg/scheduler/api"
-	"yunion.io/x/onecloud/pkg/scheduler/options"
 )
 
 var HostPendingUsageManager *SHostPendingUsageManager
 
 type SHostPendingUsageManager struct {
-	store *SHostMemoryPendingUsageStore
+	store              *SHostMemoryPendingUsageStore
+	reloadAllStartTime time.Time    // 记录 ReloadAll 开始时间
+	reloadStartTime    time.Time    // 记录部分 Reload 开始时间
+	reloadAllLock      sync.RWMutex // 保护 reloadAllStartTime
+	reloadLock         sync.RWMutex // 保护 reloadStartTime
 }
 
 func init() {
 	pendingStore := NewHostMemoryPendingUsageStore()
 
 	HostPendingUsageManager = &SHostPendingUsageManager{
-		store: pendingStore,
+		store:              pendingStore,
+		reloadAllStartTime: time.Time{}, // 初始化为零值
+		reloadStartTime:    time.Time{}, // 初始化为零值
 	}
 }
 
@@ -54,6 +59,7 @@ func (m *SHostPendingUsageManager) Keyword() string {
 func (m *SHostPendingUsageManager) newSessionUsage(req *api.SchedInfo, hostId string) *SessionPendingUsage {
 	su := NewSessionUsage(req.SessionId, hostId)
 	su.Usage = NewPendingUsageBySchedInfo(hostId, req, nil)
+	// CreatedAt is already set in NewSessionUsage
 	return su
 }
 
@@ -83,11 +89,13 @@ func (m *SHostPendingUsageManager) GetSessionUsage(sessionId, hostId string) (*S
 
 func (m *SHostPendingUsageManager) AddPendingUsage(req *api.SchedInfo, candidate *schedapi.CandidateResource) {
 	hostId := candidate.HostId
+	log.Infof("[PendingUsage] AddPendingUsage: sessionId=%s, hostId=%s, memory=%dMB, cpu=%d",
+		req.SessionId, hostId, req.Memory, req.Ncpu)
 
 	sessionUsage, _ := m.GetSessionUsage(req.SessionId, hostId)
 	if sessionUsage == nil {
 		sessionUsage = m.newSessionUsage(req, hostId)
-		sessionUsage.StartTimer()
+		log.Infof("[PendingUsage] Created new SessionPendingUsage: %s", sessionUsage)
 	}
 	m.addSessionUsage(candidate.HostId, candidate, sessionUsage)
 	if candidate.BackupCandidate != nil {
@@ -123,9 +131,15 @@ func (m *SHostPendingUsageManager) CancelPendingUsage(hostId string, su *Session
 	if su == nil {
 		return nil
 	}
+
+	oldMemory := pendingUsage.Memory
+	oldCpu := pendingUsage.Cpu
 	pendingUsage.Sub(su.Usage)
 	m.store.SetPendingUsage(hostId, pendingUsage)
 	su.SubCount()
+
+	log.Infof("[PendingUsage] CancelPendingUsage: %s, host %s pending usage memory: %d->%dMB, cpu: %d->%d",
+		su, hostId, oldMemory, pendingUsage.Memory, oldCpu, pendingUsage.Cpu)
 	return nil
 }
 
@@ -133,6 +147,85 @@ func (m *SHostPendingUsageManager) DeleteSessionUsage(usage *SessionPendingUsage
 	m.store.DeleteSessionUsage(usage)
 }
 
+// GCExpiredSessionUsages releases session pending usage that has lived longer than ttl.
+// This is a safety net to avoid leaked pending usages.
+func (m *SHostPendingUsageManager) GCExpiredSessionUsages(ttl time.Duration) int {
+	if ttl <= 0 {
+		return 0
+	}
+	now := time.Now()
+	expired := make([]*SessionPendingUsage, 0)
+
+	m.store.RangeSessionUsages(func(su *SessionPendingUsage) bool {
+		if su == nil {
+			return true
+		}
+		if now.Sub(su.CreatedAt) > ttl {
+			expired = append(expired, su)
+		}
+		return true
+	})
+
+	cleared := 0
+	for _, su := range expired {
+		hostId := su.Usage.HostId
+		// best-effort cancel + delete
+		_ = m.CancelPendingUsage(hostId, su)
+		m.DeleteSessionUsage(su)
+		cleared++
+		log.Warningf("[PendingUsage] GCExpiredSessionUsage cleared: ttl=%v, now=%v, %s", ttl, now, su)
+	}
+	return cleared
+}
+
+// SetReloadStartTime marks the start of a partial reload operation
+// This should be called before Reload to protect pending usage added during reload
+func (m *SHostPendingUsageManager) SetReloadStartTime() {
+	m.reloadLock.Lock()
+	defer m.reloadLock.Unlock()
+	m.reloadStartTime = time.Now()
+	log.Infof("[PendingUsage] SetReloadStartTime: cutoff time set to %v", m.reloadStartTime)
+}
+
+// GetReloadStartTime returns the cutoff time for partial reload
+func (m *SHostPendingUsageManager) GetReloadStartTime() time.Time {
+	m.reloadLock.RLock()
+	defer m.reloadLock.RUnlock()
+	return m.reloadStartTime
+}
+
+// GetStore returns the underlying store for direct access
+func (m *SHostPendingUsageManager) GetStore() *SHostMemoryPendingUsageStore {
+	return m.store
+}
+
+// SetReloadAllStartTime marks the start of a full reload operation
+// This should be called before ReloadAll to protect pending usage added during reload
+func (m *SHostPendingUsageManager) SetReloadAllStartTime() {
+	m.reloadAllLock.Lock()
+	defer m.reloadAllLock.Unlock()
+	m.reloadAllStartTime = time.Now()
+	log.Infof("[PendingUsage] SetReloadAllStartTime: cutoff time set to %v", m.reloadAllStartTime)
+}
+
+// ClearAllPendingUsage clears all pending usage created before the last ReloadAll start
+// This is called when all hosts are fully reloaded
+func (m *SHostPendingUsageManager) ClearAllPendingUsage() {
+	m.reloadAllLock.RLock()
+	cutoffTime := m.reloadAllStartTime
+	m.reloadAllLock.RUnlock()
+
+	if cutoffTime.IsZero() {
+		// No ReloadAll has been started, clear all
+		log.Warningf("[PendingUsage] ClearAllPendingUsage: skipping clear all (no cutoff time)")
+	} else {
+		// Only clear pending usage created before ReloadAll started
+		log.Infof("[PendingUsage] ClearAllPendingUsage: clearing created before %v", cutoffTime)
+		m.store.clearAllPendingUsageBefore(cutoffTime)
+		log.Infof("[PendingUsage] Cleared pending usage created before %v", cutoffTime)
+	}
+}
+
 type SHostMemoryPendingUsageStore struct {
 	store *sync.Map
 }
@@ -143,6 +236,16 @@ func NewHostMemoryPendingUsageStore() *SHostMemoryPendingUsageStore {
 	}
 }
 
+func (s *SHostMemoryPendingUsageStore) RangeSessionUsages(f func(*SessionPendingUsage) bool) {
+	s.store.Range(func(_, v interface{}) bool {
+		su, ok := v.(*SessionPendingUsage)
+		if !ok || su == nil {
+			return true
+		}
+		return f(su)
+	})
+}
+
 func (self *SHostMemoryPendingUsageStore) sessionUsageKey(sid, hostId string) string {
 	return fmt.Sprintf("%s-%s", sid, hostId)
 }
@@ -194,13 +297,98 @@ func (self *SHostMemoryPendingUsageStore) GetNetPendingUsage(id string) int {
 	return total
 }
 
+// clearPendingUsageBefore is the common implementation for clearing pending usage based on cutoffTime
+func (self *SHostMemoryPendingUsageStore) clearPendingUsageBefore(
+	shouldDelete func(*SessionPendingUsage) bool,
+	logPrefix string,
+	cutoffTime time.Time,
+) {
+	sessionKeysToDelete := make([]string, 0)
+	hostIdsToDelete := make(map[string]bool)
+	sessionUsagesToDelete := make(map[string]*SessionPendingUsage) // key -> sessionUsage
+
+	// First pass: collect session usages to delete
+	self.store.Range(func(key, value interface{}) bool {
+		keyStr, ok := key.(string)
+		if !ok {
+			return true
+		}
+
+		// Check if it's a session usage
+		if su, ok := value.(*SessionPendingUsage); ok {
+			if shouldDelete(su) {
+				sessionKeysToDelete = append(sessionKeysToDelete, keyStr)
+				hostIdsToDelete[su.Usage.HostId] = true
+				sessionUsagesToDelete[keyStr] = su
+			}
+		}
+		return true
+	})
+
+	log.Infof("[PendingUsage] %s: found %d session usages to delete (cutoff=%v)",
+		logPrefix, len(sessionKeysToDelete), cutoffTime)
+
+	// Delete session usages and update pending usage
+	deletedCount := 0
+	for _, key := range sessionKeysToDelete {
+		su := sessionUsagesToDelete[key]
+		if su != nil {
+			hostId := su.Usage.HostId
+			// Update pending usage by subtracting this session usage
+			if pendingUsage, err := self.GetPendingUsage(hostId); err == nil {
+				oldMemory := pendingUsage.Memory
+				oldCpu := pendingUsage.Cpu
+				pendingUsage.Sub(su.Usage)
+				if pendingUsage.IsEmpty() {
+					self.store.Delete(hostId)
+					log.Infof("[PendingUsage] Deleted empty pending usage for host %s", hostId)
+				} else {
+					self.store.Store(hostId, pendingUsage)
+					log.Debugf("[PendingUsage] Updated pending usage for host %s: memory %d->%dMB, cpu %d->%d",
+						hostId, oldMemory, pendingUsage.Memory, oldCpu, pendingUsage.Cpu)
+				}
+			}
+		}
+		// Delete session usage
+		self.store.Delete(key)
+		deletedCount++
+	}
+}
+
+// ClearHostPendingUsageBefore clears pending usage for specified hosts created before cutoffTime
+func (self *SHostMemoryPendingUsageStore) ClearHostPendingUsageBefore(hostIds []string, cutoffTime time.Time) {
+	hostIdSet := make(map[string]bool)
+	for _, hostId := range hostIds {
+		hostIdSet[hostId] = true
+	}
+
+	self.clearPendingUsageBefore(
+		func(su *SessionPendingUsage) bool {
+			return hostIdSet[su.Usage.HostId] && su.CreatedAt.Before(cutoffTime)
+		},
+		fmt.Sprintf("ClearHostPendingUsageBefore: hosts %v", hostIds),
+		cutoffTime,
+	)
+}
+
+// clearAllPendingUsageBefore clears pending usage and session usages created before cutoffTime
+func (self *SHostMemoryPendingUsageStore) clearAllPendingUsageBefore(cutoffTime time.Time) {
+	self.clearPendingUsageBefore(
+		func(su *SessionPendingUsage) bool {
+			return su.CreatedAt.Before(cutoffTime)
+		},
+		"clearAllPendingUsageBefore",
+		cutoffTime,
+	)
+}
+
 type SessionPendingUsage struct {
 	HostId    string
 	SessionId string
 	Usage     *SPendingUsage
 	countLock *sync.Mutex
 	count     int
-	cancelCh  chan string
+	CreatedAt time.Time // 记录创建时间，用于 ReloadAll 时判断是否应该清空
 }
 
 func NewSessionUsage(sid, hostId string) *SessionPendingUsage {
@@ -210,7 +398,7 @@ func NewSessionUsage(sid, hostId string) *SessionPendingUsage {
 		Usage:     NewPendingUsageBySchedInfo(hostId, nil, nil),
 		count:     0,
 		countLock: new(sync.Mutex),
-		cancelCh:  make(chan string),
+		CreatedAt: time.Now(), // 记录创建时间
 	}
 	return su
 }
@@ -231,6 +419,14 @@ func (su *SessionPendingUsage) SubCount() {
 	su.count--
 }
 
+func (su *SessionPendingUsage) String() string {
+	if su == nil {
+		return "<nil>"
+	}
+	return fmt.Sprintf("SessionPendingUsage{sessionId=%s, hostId=%s, count=%d, createdAt=%v}: %s",
+		su.SessionId, su.HostId, su.count, su.CreatedAt, jsonutils.Marshal(su.Usage.ToMap()))
+}
+
 type SResourcePendingUsage struct {
 	store *sync.Map
 }
@@ -445,46 +641,3 @@ func (self *SPendingUsage) IsEmpty() bool {
 	}
 	return true
 }
-
-func (self *SessionPendingUsage) cancelSelf() {
-	hostId := self.Usage.HostId
-	count := self.count
-
-	for i := 0; i <= count; i++ {
-		HostPendingUsageManager.CancelPendingUsage(hostId, self)
-	}
-}
-
-func (self *SessionPendingUsage) StartTimer() {
-	timeout := time.Duration(options.Options.ExpireSessionUsageTimeout) * time.Second
-	go func() {
-		for {
-			select {
-			case <-time.After(timeout):
-				log.Infof("timeout cancel session usage %#v", self)
-				self.cancelSelf()
-				goto ForEnd
-			case sid := <-self.cancelCh:
-				log.Infof("Cancel session %s usage, count: %d", sid, self.count)
-				if self.count <= 0 {
-					goto ForEnd
-				} else {
-					log.Infof("continue waiting next cancel...")
-				}
-			}
-		}
-	ForEnd:
-		log.Infof("delete session usage %#v", self)
-		HostPendingUsageManager.DeleteSessionUsage(self)
-	}()
-}
-
-func (self *SessionPendingUsage) StopTimer() {
-	defer func() {
-		if r := recover(); r != nil {
-			log.Errorf("SessionPendingUsage %#v stop timer: %v", self, r)
-			debug.PrintStack()
-		}
-	}()
-	self.cancelCh <- self.SessionId
-}

pkg/scheduler/options/options.go

@@ -52,8 +52,6 @@ type SchedOptions struct {
 	ExpireQueueMaxLength          int    `help:"Expire queue max length" default:"1000"`
 	ExpireQueueDealLength         int    `help:"Expire queue deal length" default:"100"`
 
-	ExpireSessionUsageTimeout int `help:"Expire Session usage timeout second" default:"60"`
-
 	// completed queue options
 	CompletedQueueConsumptionPeriod  string `help:"Completed queue consumption period" default:"30s"`
 	CompletedQueueConsumptionTimeout string `help:"Completed queue consumption timeout" default:"30s"`