- local disk snapshot create and delete

- auto snapshot everyday
- rewrite cornman
- make dep and some fix
- fix conflict

cmd/climc/shell/disks.go

@@ -120,10 +120,11 @@ func init() {
 	})
 
 	type DiskUpdateOptions struct {
-		ID         string `help:"ID or name of disk"`
-		Name       string `help:"New name of disk"`
-		Desc       string `help:"Description" metavar:"DESCRIPTION"`
-		AutoDelete string `help:"enable/disable auto delete of disk" choices:"enable|disable"`
+		ID           string `help:"ID or name of disk"`
+		Name         string `help:"New name of disk"`
+		Desc         string `help:"Description" metavar:"DESCRIPTION"`
+		AutoDelete   string `help:"enable/disable auto delete of disk" choices:"enable|disable"`
+		AutoSnapshot string `help:"enable/disable auto snapshot of disk" choices:"enable|disable"`
 	}
 	R(&DiskUpdateOptions{}, "disk-update", "Update property of a virtual disk", func(s *mcclient.ClientSession, args *DiskUpdateOptions) error {
 		params := jsonutils.NewDict()
@@ -140,6 +141,13 @@ func init() {
 				params.Add(jsonutils.JSONFalse, "auto_delete")
 			}
 		}
+		if len(args.AutoSnapshot) > 0 {
+			if args.AutoSnapshot == "enable" {
+				params.Add(jsonutils.JSONTrue, "auto_snapshot")
+			} else {
+				params.Add(jsonutils.JSONFalse, "auto_snapshot")
+			}
+		}
 		if params.Size() == 0 {
 			return InvalidUpdateError()
 		}

cmd/climc/shell/servers.go

@@ -352,6 +352,23 @@ func init() {
 		return nil
 	})
 
+	type ServerDiskSnapshotOptions struct {
+		SERVER       string `help:"server ID or Name"`
+		DISK         string `help:"create snapshot disk id"`
+		SNAPSHOTNAME string `help:"Snapshot name"`
+	}
+	R(&ServerDiskSnapshotOptions{}, "server-disk-create-snapshot", "Task server disk snapshot", func(s *mcclient.ClientSession, args *ServerDiskSnapshotOptions) error {
+		params := jsonutils.NewDict()
+		params.Set("disk_id", jsonutils.NewString(args.DISK))
+		params.Set("name", jsonutils.NewString(args.SNAPSHOTNAME))
+		srv, err := modules.Servers.PerformAction(s, args.SERVER, "disk-snapshot", params)
+		if err != nil {
+			return err
+		}
+		printObject(srv)
+		return nil
+	})
+
 	type ServerInsertISOOptions struct {
 		ID  string `help:"server ID or Name"`
 		ISO string `help:"Glance image ID of the ISO"`

cmd/climc/shell/snapshots.go

@@ -0,0 +1,40 @@
+package shell
+
+import (
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/mcclient/options"
+)
+
+func init() {
+	type SnapshotsListOptions struct {
+		options.BaseListOptions
+		Disk string `help:"Disk snapshots"`
+	}
+	R(&SnapshotsListOptions{}, "snapshot-list", "Show snapshots", func(s *mcclient.ClientSession, args *SnapshotsListOptions) error {
+		params, err := args.BaseListOptions.Params()
+		if err != nil {
+			return err
+		}
+		params.Add(jsonutils.NewString(args.Disk), "disk_id")
+		result, err := modules.Snapshots.List(s, params)
+		if err != nil {
+			return err
+		}
+		printList(result, modules.Snapshots.GetColumns(s))
+		return nil
+	})
+
+	type SnapshotDeleteOptions struct {
+		ID string `help:"Delete snapshot id"`
+	}
+	R(&SnapshotDeleteOptions{}, "snapshot-delete", "Delete snapshots", func(s *mcclient.ClientSession, args *SnapshotDeleteOptions) error {
+		result, err := modules.Snapshots.Delete(s, args.ID, nil)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+}

pkg/appsrv/dispatcher/dispatcher.go

@@ -205,7 +205,6 @@ func performClassActionHandler(ctx context.Context, w http.ResponseWriter, r *ht
 	var data jsonutils.JSONObject
 	if body != nil {
 		data, _ = body.Get(manager.KeywordPlural())
-		// about string ??
 		if data == nil {
 			data = body.(*jsonutils.JSONDict)
 		}
@@ -217,13 +216,21 @@ func performClassActionHandler(ctx context.Context, w http.ResponseWriter, r *ht
 		httperrors.GeneralServerError(w, err)
 		return
 	}
+	if results == nil {
+		results = jsonutils.NewDict()
+	}
 	appsrv.SendJSON(w, wrapBody(results, manager.KeywordPlural()))
 }
 
 func performActionHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
 	manager, params, query, body := fetchEnv(ctx, w, r)
-	data, _ := body.Get(manager.Keyword())
-	if data == nil {
+	var data jsonutils.JSONObject
+	if body != nil {
+		data, _ = body.Get(manager.Keyword())
+		if data == nil {
+			data = body.(*jsonutils.JSONDict)
+		}
+	} else {
 		data = jsonutils.NewDict()
 	}
 	result, err := manager.PerformAction(ctx, params["<resid>"], params["<action>"], query, data)

pkg/cloudcommon/cronman/cronman.go

@@ -1,96 +1,192 @@
 package cronman
 
 import (
+	"container/heap"
 	"context"
-	"reflect"
-	"runtime"
 	"runtime/debug"
 	"time"
 
 	"yunion.io/x/log"
-
 	"yunion.io/x/onecloud/pkg/appctx"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
 )
 
-const (
-	DEFAULT_CRON_INTERVAL = 60 * time.Second // default resolution is 1 monutes
-)
+var manager *SCronJobManager
 
-type SCronJobManager struct {
-	checkInterval time.Duration
-	timer         *time.Timer
-	jobs          []SCronJob
+func init() {
+	manager = &SCronJobManager{
+		jobs: make([]*SCronJob, 0),
+	}
+}
+
+type ICronTimer interface {
+	Next(time.Time) time.Time
+}
+
+type Timer1 struct {
+	dur time.Duration
+}
+
+func (t *Timer1) Next(now time.Time) time.Time {
+	return now.Add(t.dur)
+}
+
+type Timer2 struct {
+	day, hour, min, sec int
+}
+
+func (t *Timer2) Next(now time.Time) time.Time {
+	next := now.Add(time.Hour * time.Duration(t.day) * 24)
+	return time.Date(next.Year(), next.Month(), next.Day(), t.hour, t.min, t.sec, 0, next.Location())
 }
 
 type SCronJob struct {
-	name        string
-	runInterval time.Duration
-	job         func(ctx context.Context, userCred mcclient.TokenCredential)
-	lastRun     time.Time
+	Name  string
+	job   func(ctx context.Context, userCred mcclient.TokenCredential)
+	Timer ICronTimer
+	Next  time.Time
 }
 
-func NewCronJobManager(interval time.Duration) *SCronJobManager {
-	if interval == 0 {
-		interval = DEFAULT_CRON_INTERVAL
+type CronJobTimerHeap []*SCronJob
+
+func (cjth CronJobTimerHeap) Len() int {
+	return len(cjth)
+}
+
+func (cjth CronJobTimerHeap) Swap(i, j int) {
+	cjth[i], cjth[j] = cjth[j], cjth[i]
+}
+
+func (cjth CronJobTimerHeap) Less(i, j int) bool {
+	if cjth[i].Next.IsZero() {
+		return false
+	}
+	if cjth[j].Next.IsZero() {
+		return true
+	}
+	return cjth[i].Next.Before(cjth[j].Next)
+}
+
+func (cjth *CronJobTimerHeap) Push(x interface{}) {
+	*cjth = append(*cjth, x.(*SCronJob))
+}
+
+func (cjth *CronJobTimerHeap) Pop() interface{} {
+	old := *cjth
+	n := old.Len()
+	x := old[n-1]
+	*cjth = old[0 : n-1]
+	return x
+}
+
+type SCronJobManager struct {
+	jobs    CronJobTimerHeap
+	add     chan *SCronJob
+	stop    chan struct{}
+	running bool
+}
+
+func GetCronJobManager() *SCronJobManager {
+	return manager
+}
+
+func (self *SCronJobManager) AddJob1(name string, interval time.Duration, jobFunc func(ctx context.Context, userCred mcclient.TokenCredential)) {
+	t := Timer1{
+		dur: interval,
+	}
+	job := SCronJob{
+		Name:  name,
+		job:   jobFunc,
+		Timer: &t,
+	}
+	if !self.running {
+		self.jobs = append(self.jobs, &job)
+	} else {
+		self.add <- &job
+	}
+}
+
+func (self *SCronJobManager) AddJob2(name string, day, hour, min, sec int, jobFunc func(ctx context.Context, userCred mcclient.TokenCredential)) {
+	t := Timer2{
+		day:  day,
+		hour: hour,
+		min:  min,
+		sec:  sec,
+	}
+	job := SCronJob{
+		Name:  name,
+		job:   jobFunc,
+		Timer: &t,
+	}
+	if !self.running {
+		self.jobs = append(self.jobs, &job)
+	} else {
+		self.add <- &job
+	}
+}
+
+func (self *SCronJobManager) Next(now time.Time) {
+	for _, job := range self.jobs {
+		job.Next = job.Timer.Next(now)
 	}
-	cron := SCronJobManager{checkInterval: interval, jobs: make([]SCronJob, 0)}
-	return &cron
 }
 
 func (self *SCronJobManager) Start() {
-	if self.timer != nil {
+	if self.running {
 		return
 	}
-	self.timer = time.AfterFunc(self.checkInterval, self.runCronJobs)
+	self.running = true
+	go self.run()
 }
 
 func (self *SCronJobManager) Stop() {
-	if self.timer != nil {
-		self.timer.Stop()
-		self.timer = nil
-	}
+	close(self.stop)
 }
 
-func getFunctionName(i interface{}) string {
-	return runtime.FuncForPC(reflect.ValueOf(i).Pointer()).Name()
-}
-
-func (self *SCronJobManager) AddJob(name string, interval time.Duration, jobFunc func(ctx context.Context, userCred mcclient.TokenCredential)) {
-	// name := getFunctionName(jobFunc)
-	log.Debugf("Add cronjob %s", name)
-	job := SCronJob{name: name, job: jobFunc, runInterval: interval}
-	self.jobs = append(self.jobs, job)
-}
-
-func (self *SCronJobManager) runCronJobs() {
+func (self *SCronJobManager) run() {
 	now := time.Now()
-	for i := 0; i < len(self.jobs); i += 1 {
-		self.jobs[i].run(now)
-	}
-	self.timer = nil
-	self.Start() // schedule next run
-}
-
-func (self *SCronJob) run(now time.Time) {
-	if self.lastRun.IsZero() || now.Sub(self.lastRun) >= self.runInterval {
-		log.Debugf("Run cronjob %s", self.name)
-		go runJob(self.name, self.job)
-		self.lastRun = now
+	self.Next(now)
+	heap.Init(&self.jobs)
+	for {
+		var timer *time.Timer
+		if len(self.jobs) == 0 || self.jobs[0].Next.IsZero() {
+			timer = time.NewTimer(100000 * time.Hour)
+		} else {
+			timer = time.NewTimer(self.jobs[0].Next.Sub(now))
+		}
+		select {
+		case now = <-timer.C:
+			for i, job := range self.jobs {
+				if job.Next.After(now) || job.Next.IsZero() {
+					break
+				}
+				go job.runJob()
+				job.Next = job.Timer.Next(now)
+				heap.Fix(&self.jobs, i)
+			}
+		case newJob := <-self.add:
+			now = time.Now()
+			newJob.Next = newJob.Timer.Next(now)
+			heap.Push(&self.jobs, newJob)
+		case <-self.stop:
+			timer.Stop()
+			return
+		}
 	}
 }
 
-func runJob(name string, job func(ctx context.Context, userCred mcclient.TokenCredential)) {
+func (job *SCronJob) runJob() {
 	defer func() {
 		if r := recover(); r != nil {
-			log.Errorf("CronJob task %s run error: %s", name, r)
+			log.Errorf("CronJob task %s run error: %s", job.Name, r)
 			debug.PrintStack()
 		}
 	}()
 
+	log.Debugf("Cron job: %s started", job.Name)
 	ctx := context.Background()
 	ctx = context.WithValue(ctx, appctx.APP_CONTEXT_KEY_APPNAME, "Region-Corn-Service")
 	userCred := auth.AdminCredential()
-	job(ctx, userCred)
+	job.job(ctx, userCred)
 }

pkg/compute/guestdrivers/base.go

@@ -129,3 +129,19 @@ func (self *SBaseGuestDriver) RequestGuestHotAddIso(ctx context.Context, guest *
 func (self *SBaseGuestDriver) RequestRebuildRootDisk(ctx context.Context, guest *models.SGuest, task taskman.ITask) error {
 	return fmt.Errorf("Not Implement")
 }
+
+func (self *SBaseGuestDriver) StartGuestDiskSnapshotTask(ctx context.Context, userCred mcclient.TokenCredential, guest *models.SGuest, params *jsonutils.JSONDict) error {
+	return fmt.Errorf("Not Implement")
+}
+
+func (self *SBaseGuestDriver) RequestDiskSnapshot(ctx context.Context, guest *models.SGuest, task taskman.ITask, snapshotId, diskId string) error {
+	return fmt.Errorf("Not Implement")
+}
+
+func (self *SBaseGuestDriver) RequestDeleteSnapshot(ctx context.Context, guest *models.SGuest, task taskman.ITask, params *jsonutils.JSONDict) error {
+	return fmt.Errorf("Not Implement")
+}
+
+func (self *SBaseGuestDriver) RequestReloadDiskSnapshot(ctx context.Context, guest *models.SGuest, task taskman.ITask, params *jsonutils.JSONDict) error {
+	return fmt.Errorf("Not Implement")
+}

pkg/compute/guestdrivers/kvm.go

@@ -47,6 +47,48 @@ func (self *SKVMGuestDriver) DoGuestCreateDisksTask(ctx context.Context, guest *
 	return nil
 }
 
+func (self *SKVMGuestDriver) StartGuestDiskSnapshotTask(ctx context.Context, userCred mcclient.TokenCredential, guest *models.SGuest, params *jsonutils.JSONDict) error {
+	task, err := taskman.TaskManager.NewTask(ctx, "GuestDiskSnapshotTask", guest, userCred, params, "", "", nil)
+	if err != nil {
+		return err
+	}
+	task.ScheduleRun(nil)
+	return nil
+}
+
+func (self *SKVMGuestDriver) RequestDiskSnapshot(ctx context.Context, guest *models.SGuest, task taskman.ITask, snapshotId, diskId string) error {
+	url := fmt.Sprintf("/servers/%s/snapshot", guest.Id)
+	body := jsonutils.NewDict()
+	body.Set("disk_id", jsonutils.NewString(diskId))
+	body.Set("snapshot_id", jsonutils.NewString(snapshotId))
+	header := http.Header{}
+	header.Set("X-Task-Id", task.GetTaskId())
+	header.Set("X-Region-Version", "v2")
+	host := guest.GetHost()
+	_, err := host.Request(task.GetUserCred(), "POST", url, header, body)
+	return err
+}
+
+func (self *SKVMGuestDriver) RequestDeleteSnapshot(ctx context.Context, guest *models.SGuest, task taskman.ITask, params *jsonutils.JSONDict) error {
+	url := fmt.Sprintf("/servers/%s/delete-snapshot", guest.Id)
+	header := http.Header{}
+	header.Set("X-Task-Id", task.GetTaskId())
+	header.Set("X-Region-Version", "v2")
+	host := guest.GetHost()
+	_, err := host.Request(task.GetUserCred(), "POST", url, header, params)
+	return err
+}
+
+func (self *SKVMGuestDriver) RequestReloadDiskSnapshot(ctx context.Context, guest *models.SGuest, task taskman.ITask, params *jsonutils.JSONDict) error {
+	url := fmt.Sprintf("/servers/%s/reload-disk-snapshot", guest.Id)
+	header := http.Header{}
+	header.Set("X-Task-Id", task.GetTaskId())
+	header.Set("X-Region-Version", "v2")
+	host := guest.GetHost()
+	_, err := host.Request(task.GetUserCred(), "POST", url, header, params)
+	return err
+}
+
 func findVNCPort(results string) int {
 	reg := regexp.MustCompile(`(\d+\.\d+\.\d+\.\d+):([\d]+)`)
 	finds := reg.FindStringSubmatch(results)

pkg/compute/handlers.go

@@ -57,6 +57,7 @@ func InitHandlers(app *appsrv.Application) {
 		models.VCenterManager,
 		models.DnsRecordManager,
 		models.ElasticipManager,
+		models.SnapshotManager,
 	} {
 		db.RegisterModelManager(manager)
 		handler := db.NewModelHandler(manager)

pkg/compute/hostdrivers/aliyun.go

@@ -13,6 +13,7 @@ import (
 )
 
 type SAliyunHostDriver struct {
+	SBaseHostDriver
 }
 
 func init() {

pkg/compute/hostdrivers/base.go

@@ -0,0 +1,16 @@
+package hostdrivers
+
+import (
+	"context"
+	"fmt"
+
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	"yunion.io/x/onecloud/pkg/compute/models"
+)
+
+type SBaseHostDriver struct {
+}
+
+func (self *SBaseHostDriver) RequestDeleteSnapshotWithStorage(ctx context.Context, host *models.SHost, snapshot *models.SSnapshot, task taskman.ITask) error {
+	return fmt.Errorf("Not Implement")
+}

pkg/compute/hostdrivers/kvm.go

@@ -13,6 +13,7 @@ import (
 )
 
 type SKVMHostDriver struct {
+	SBaseHostDriver
 }
 
 func init() {
@@ -147,3 +148,14 @@ func (self *SKVMHostDriver) RequestSaveUploadImageOnHost(ctx context.Context, ho
 	_, err := host.Request(task.GetUserCred(), "POST", url, header, body)
 	return err
 }
+
+func (self *SKVMHostDriver) RequestDeleteSnapshotWithStorage(ctx context.Context, host *models.SHost, snapshot *models.SSnapshot, task taskman.ITask) error {
+	url := fmt.Sprintf("/storages/%s/delete-snapshots", snapshot.StorageId)
+	body := jsonutils.NewDict()
+	body.Set("disk_id", jsonutils.NewString(snapshot.DiskId))
+	header := http.Header{}
+	header.Add("X-Task-Id", task.GetTaskId())
+	header.Add("X-Region-Version", "v2")
+	_, err := host.Request(task.GetUserCred(), "POST", url, header, body)
+	return err
+}

pkg/compute/models/disks.go

@@ -53,6 +53,9 @@ const (
 	DISK_POST_MIGRATE  = "post_migrate"
 	DISK_MIGRATING     = "migrating"
 
+	DISK_START_SNAPSHOT = "start_snapshot"
+	DISK_SNAPSHOTING    = "snapshoting"
+
 	DISK_TYPE_SYS  = "sys"
 	DISK_TYPE_SWAP = "swap"
 	DISK_TYPE_DATA = "data"
@@ -89,6 +92,7 @@ type SDisk struct {
 	DiskType string `width:"32" charset:"ascii" nullable:"true" list:"user"` // Column(VARCHAR(32, charset='ascii'), nullable=True)
 	// # is persistent
 	Nonpersistent bool `default:"false" list:"user"` // Column(Boolean, default=False)
+	AutoSnapshot  bool `default:"false" nullable:"true" get:"user" update:"user"`
 }
 
 func (manager *SDiskManager) GetContextManager() []db.IModelManager {
@@ -299,6 +303,13 @@ func (self *SDisk) StartDiskCreateTask(ctx context.Context, userCred mcclient.To
 	return nil
 }
 
+func (self *SDisk) GetSnapshotCount() int {
+	q := SnapshotManager.Query()
+	count := q.Filter(sqlchemy.AND(sqlchemy.Equals(q.Field("disk_id"), self.Id),
+		sqlchemy.Equals(q.Field("out_of_chain"), false))).Count()
+	return count
+}
+
 func (self *SDisk) StartAllocate(ctx context.Context, host *SHost, storage *SStorage, taskId string, userCred mcclient.TokenCredential, rebuild bool, snapshot string, task taskman.ITask) error {
 	log.Infof("Allocating disk on host %s ...", host.GetName())
 
@@ -331,6 +342,30 @@ func (self *SDisk) StartAllocate(ctx context.Context, host *SHost, storage *SSto
 	return host.GetHostDriver().RequestAllocateDiskOnStorage(ctx, host, storage, self, task, content)
 }
 
+func (self *SDisk) AllowGetDetailsConvertSnapshot(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool {
+	return self.IsOwner(userCred)
+}
+
+func (self *SDisk) GetDetailsConvertSnapshot(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	deleteSnapshot := SnapshotManager.GetDiskFirstSnapshot(self.Id)
+	if deleteSnapshot == nil {
+		return nil, httperrors.NewNotFoundError("Can not get disk snapshot")
+	}
+	convertSnapshot, err := SnapshotManager.GetConvertSnapshot(deleteSnapshot)
+	if err != nil {
+		return nil, httperrors.NewBadRequestError("Get convert snapshot failed: %s", err.Error())
+	}
+	var pendingDelete bool
+	if deleteSnapshot.CreatedBy == MANUAL && !deleteSnapshot.PendingDeleted {
+		pendingDelete = true
+	}
+	ret := jsonutils.NewDict()
+	ret.Set("delete_snapshot", jsonutils.NewString(deleteSnapshot.Id))
+	ret.Set("convert_snapshot", jsonutils.NewString(convertSnapshot.Id))
+	ret.Set("pending_delete", jsonutils.NewBool(pendingDelete))
+	return ret, nil
+}
+
 func (self *SDisk) AllowPerformResize(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
 	return self.IsOwner(userCred)
 }
@@ -1047,7 +1082,7 @@ func (self *SDisk) isInit() bool {
 	return self.Status == DISK_INIT
 }
 
-func (model *SDisk) AllowPerformCancelDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
+func (self *SDisk) AllowPerformCancelDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
 	return userCred.IsSystemAdmin()
 }
 
@@ -1060,7 +1095,7 @@ func (self *SDisk) PerformCancelDelete(ctx context.Context, userCred mcclient.To
 }
 
 func (manager *SDiskManager) getExpiredPendingDeleteDisks() []SDisk {
-	deadline := time.Now().Add(time.Duration(options.Options.PendingDeleteExpireSeconds) * time.Second)
+	deadline := time.Now().Add(time.Duration(options.Options.PendingDeleteExpireSeconds*-1) * time.Second)
 
 	q := manager.Query()
 	q = q.IsTrue("pending_deleted").LT("pending_deleted_at", deadline).Limit(options.Options.PendingDeleteMaxCleanBatchSize)
@@ -1084,3 +1119,43 @@ func (manager *SDiskManager) CleanPendingDeleteDisks(ctx context.Context, userCr
 		disks[i].StartDiskDeleteTask(ctx, userCred, "", false)
 	}
 }
+
+func (manager *SDiskManager) getAutoSnapshotDisks() []SDisk {
+	q := manager.Query().SubQuery()
+	dest := make([]SDisk, 0)
+	err := q.Query().Filter(sqlchemy.Equals(q.Field("auto_snapshot"), true)).All(&dest)
+	if err != nil {
+		return nil
+	}
+	return dest
+}
+
+func (manager *SDiskManager) AutoDiskSnapshot(ctx context.Context, userCred mcclient.TokenCredential) {
+	disks := manager.getAutoSnapshotDisks()
+	if disks == nil {
+		return
+	}
+	for _, disk := range disks {
+		snapCount := disk.GetSnapshotCount()
+		if snapCount >= DISK_MAX_SNAPSHOT {
+			continue
+		}
+		guests := disk.GetGuests()
+		if guests == nil || len(guests) > 1 {
+			log.Errorln("Disk %s not attach or attached more than one guest", disk.Id)
+			continue
+		}
+		// if !utils.IsInStringArray(guests[0].Status, []string{VM_RUNNING, VM_READY}) {
+		// 	log.Errorln("Guest(%s) in status(%s) cannot do snapshot action", guests[0].Id, guests[0].Status)
+		// 	continue
+		// }
+		// name
+		name := disk.Name + time.Now().Format("2006-01-02#15:04:05")
+		snap, err := SnapshotManager.CreateSnapshot(ctx, userCred, AUTO, disk.Id, guests[0].Id, "", name)
+		if err != nil {
+			log.Errorln(err)
+			continue
+		}
+		guests[0].StartDiskSnapshot(ctx, userCred, disk.Id, snap.Id)
+	}
+}

pkg/compute/models/guestdrivers.go

@@ -93,6 +93,11 @@ type IGuestDriver interface {
 
 	RequestGuestHotAddIso(ctx context.Context, guest *SGuest, path string, task taskman.ITask) error
 	RequestRebuildRootDisk(ctx context.Context, guest *SGuest, task taskman.ITask) error
+
+	StartGuestDiskSnapshotTask(ctx context.Context, userCred mcclient.TokenCredential, guest *SGuest, params *jsonutils.JSONDict) error
+	RequestDiskSnapshot(ctx context.Context, guest *SGuest, task taskman.ITask, snapshotId, diskId string) error
+	RequestDeleteSnapshot(ctx context.Context, guest *SGuest, task taskman.ITask, params *jsonutils.JSONDict) error
+	RequestReloadDiskSnapshot(ctx context.Context, guest *SGuest, task taskman.ITask, params *jsonutils.JSONDict) error
 }
 
 var guestDrivers map[string]IGuestDriver

pkg/compute/models/guests.go

@@ -75,7 +75,7 @@ const (
 
 	VM_CHANGE_FLAVOR     = "change_flavor"
 	VM_REBUILD_ROOT      = "rebuild_root"
-	VM_REBUILD_ROOT_FAIL = "rebld_root_fail"
+	VM_REBUILD_ROOT_FAIL = "rebuild_root_fail"
 
 	VM_START_SNAPSHOT  = "snapshot_start"
 	VM_SNAPSHOT        = "snapshot"
@@ -3406,6 +3406,52 @@ func (self *SGuest) PerformReset(ctx context.Context, userCred mcclient.TokenCre
 	return nil, httperrors.NewInvalidStatusError("Cannot reset VM in status %s", self.Status)
 }
 
+func (self *SGuest) AllowPerformDiskSnapshot(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
+	return self.IsOwner(userCred)
+}
+
+func (self *SGuest) PerformDiskSnapshot(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	if !utils.IsInStringArray(self.Status, []string{VM_RUNNING, VM_READY}) {
+		return nil, httperrors.NewInvalidStatusError("Cannot do snapshot when VM in status %s", self.Status)
+	}
+	diskId, err := data.GetString("disk_id")
+	if err != nil {
+		return nil, err
+	}
+	name, err := data.GetString("name")
+	if err != nil {
+		return nil, err
+	}
+	if self.GetGuestDisk(diskId) == nil {
+		return nil, httperrors.NewNotFoundError("Guest disk %s not found", diskId)
+	}
+	snapshots := SnapshotManager.GetDiskSnapshotsByCreate(diskId, MANUAL)
+	if snapshots != nil {
+		if len(snapshots) >= DISK_MAX_MANUAL_SNAPSHOT {
+			return nil, httperrors.NewBadRequestError("Disk %s snapshot full, cannot take any more", diskId)
+		}
+		for _, snapshot := range snapshots {
+			if snapshot.Name == name {
+				return nil, httperrors.NewBadRequestError("Name Conflict")
+			}
+		}
+	}
+	snapshot, err := SnapshotManager.CreateSnapshot(ctx, userCred, MANUAL, diskId, self.Id, "", name)
+	if err != nil {
+		return nil, err
+	}
+	err = self.StartDiskSnapshot(ctx, userCred, diskId, snapshot.Id)
+	return nil, err
+}
+
+func (self *SGuest) StartDiskSnapshot(ctx context.Context, userCred mcclient.TokenCredential, diskId, snapshotId string) error {
+	self.SetStatus(userCred, VM_START_SNAPSHOT, "StartDiskSnapshot")
+	params := jsonutils.NewDict()
+	params.Set("disk_id", jsonutils.NewString(diskId))
+	params.Set("snapshot_id", jsonutils.NewString(snapshotId))
+	return self.GetDriver().StartGuestDiskSnapshotTask(ctx, userCred, self, params)
+}
+
 func (self *SGuest) AllowPerformStop(ctx context.Context,
 	userCred mcclient.TokenCredential,
 	query jsonutils.JSONObject,
@@ -3556,7 +3602,7 @@ func (manager *SGuestManager) getIpsByExit(ips []string, isExitOnly bool) []stri
 }
 
 func (manager *SGuestManager) getExpiredPendingDeleteGuests() []SGuest {
-	deadline := time.Now().Add(time.Duration(options.Options.PendingDeleteExpireSeconds) * time.Second)
+	deadline := time.Now().Add(time.Duration(options.Options.PendingDeleteExpireSeconds*-1) * time.Second)
 
 	q := manager.Query()
 	q = q.IsTrue("pending_deleted").LT("pending_deleted_at", deadline).In("hypervisor", []string{"aliyun"}).Limit(options.Options.PendingDeleteMaxCleanBatchSize)

pkg/compute/models/hostdrivers.go

@@ -18,6 +18,7 @@ type IHostDriver interface {
 	RequestDeallocateDiskOnHost(host *SHost, storage *SStorage, disk *SDisk, task taskman.ITask) error
 	RequestResizeDiskOnHostOnline(host *SHost, storage *SStorage, disk *SDisk, size int64, task taskman.ITask) error
 	RequestResizeDiskOnHost(host *SHost, storage *SStorage, disk *SDisk, size int64, task taskman.ITask) error
+	RequestDeleteSnapshotWithStorage(ctx context.Context, host *SHost, snapshot *SSnapshot, task taskman.ITask) error
 }
 
 var hostDrivers map[string]IHostDriver

pkg/compute/models/snapshots.go

@@ -0,0 +1,278 @@
+package models
+
+import (
+	"context"
+	"fmt"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/sqlchemy"
+
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient"
+)
+
+const (
+	DISK_MAX_SNAPSHOT        = 9
+	DISK_MAX_MANUAL_SNAPSHOT = 2
+
+	// create by
+	MANUAL = "manual"
+	AUTO   = "auto"
+
+	SNAPSHOT_FAILED = "create_failed"
+	SNAPSHOT_READY  = "ready"
+)
+
+type SSnapshotManager struct {
+	db.SVirtualResourceBaseManager
+}
+
+type SSnapshot struct {
+	db.SVirtualResourceBase
+	DiskId     string `width:"36" charset:"ascii" nullable:"false" create:"required" key_index:"true" list:"user"`
+	StorageId  string `width:"36" charset:"ascii" nullable:"true" list:"admin"`
+	CreatedBy  string `width:"36" charset:"ascii" nullable:"false" default:"manual" list:"admin"`
+	Location   string `charset:"ascii" nullable:"false" list:"admin"`
+	Size       int    `nullable:"false" list:"user"` // MB
+	OutOfChain bool   `nullable:"false" default:"false" index:"true" get:"admin"`
+}
+
+var SnapshotManager *SSnapshotManager
+
+func init() {
+	SnapshotManager = &SSnapshotManager{SVirtualResourceBaseManager: db.NewVirtualResourceBaseManager(SSnapshot{}, "snapshots_tbl", "snapshot", "snapshots")}
+}
+
+func (self *SSnapshot) AllowListItems(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool {
+	return self.IsOwner(userCred)
+}
+
+func (self *SSnapshot) AllowCreateItem(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
+	return false
+}
+
+func (self *SSnapshot) AllowGetDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool {
+	return false
+}
+
+func (self *SSnapshot) AllowUpdateItem(ctx context.Context, userCred mcclient.TokenCredential) bool {
+	return false
+}
+
+func (self *SSnapshot) CustomizeCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data jsonutils.JSONObject) error {
+	return self.SVirtualResourceBase.CustomizeCreate(ctx, userCred, ownerProjId, query, data)
+}
+
+func (self *SSnapshot) GetGuest() (*SGuest, error) {
+	iDisk, err := DiskManager.FetchById(self.DiskId)
+	if err != nil {
+		return nil, err
+	}
+	disk := iDisk.(*SDisk)
+	guests := disk.GetGuests()
+	if len(guests) > 1 {
+		return nil, fmt.Errorf("Snapshot disk attach mutil guest")
+	} else if len(guests) == 1 {
+		return &guests[0], nil
+	} else {
+		return nil, nil
+	}
+}
+
+func (self *SSnapshot) GetHost() *SHost {
+	iStorage, err := StorageManager.FetchById(self.StorageId)
+	if err != nil {
+		log.Errorln(err)
+		return nil
+	}
+	storage := iStorage.(*SStorage)
+	return storage.GetMasterHost()
+}
+
+func (self *SSnapshotManager) GetDiskSnapshotsByCreate(diskId, createdBy string) []SSnapshot {
+	dest := make([]SSnapshot, 0)
+	q := self.Query().SubQuery()
+	err := q.Query().Filter(sqlchemy.AND(sqlchemy.Equals(q.Field("disk_id"), diskId),
+		sqlchemy.Equals(q.Field("created_by"), createdBy),
+		sqlchemy.Equals(q.Field("pending_deleted"), false))).All(&dest)
+	if err != nil {
+		log.Errorf("GetDiskSnapshots error: %s", err)
+		return nil
+	}
+	for i := 0; i < len(dest); i++ {
+		dest[i].SetModelManager(self)
+	}
+	return dest
+}
+
+func (self *SSnapshotManager) GetDiskSnapshots(diskId string) []SSnapshot {
+	dest := make([]SSnapshot, 0)
+	q := self.Query().SubQuery()
+	err := q.Query().Filter(sqlchemy.AND(sqlchemy.Equals(q.Field("disk_id"), diskId))).All(&dest)
+	if err != nil {
+		log.Errorf("GetDiskSnapshots error: %s", err)
+		return nil
+	}
+	for i := 0; i < len(dest); i++ {
+		dest[i].SetModelManager(self)
+	}
+	return dest
+}
+
+func (self *SSnapshotManager) GetDiskFirstSnapshot(diskId string) *SSnapshot {
+	dest := &SSnapshot{}
+	q := self.Query().SubQuery()
+	err := q.Query().Filter(sqlchemy.AND(sqlchemy.Equals(q.Field("disk_id"), diskId),
+		sqlchemy.Equals(q.Field("status"), SNAPSHOT_READY),
+		sqlchemy.Equals(q.Field("out_of_chain"), false))).Asc("created_at").First(dest)
+	if err != nil {
+		log.Errorf("Get Disk First snapshot error: %s", err.Error())
+		return nil
+	}
+	dest.SetModelManager(self)
+	return dest
+}
+
+func (self *SSnapshotManager) GetDiskSnapshotCount(diskId string) int {
+	q := self.Query().SubQuery()
+	return q.Query().Filter(sqlchemy.AND(sqlchemy.Equals(q.Field("disk_id"), diskId),
+		sqlchemy.Equals(q.Field("pending_deleted"), false))).Count()
+}
+
+func (self *SSnapshotManager) CreateSnapshot(ctx context.Context, userCred mcclient.TokenCredential, createdBy, diskId, guestId, location, name string) (*SSnapshot, error) {
+	iDisk, err := DiskManager.FetchById(diskId)
+	if err != nil {
+		return nil, err
+	}
+	disk := iDisk.(*SDisk)
+	snapshot := &SSnapshot{}
+	snapshot.SetModelManager(self)
+	snapshot.ProjectId = userCred.GetProjectId()
+	snapshot.DiskId = disk.Id
+	snapshot.StorageId = disk.StorageId
+	snapshot.Size = disk.DiskSize
+	snapshot.Location = location
+	snapshot.CreatedBy = createdBy
+	snapshot.Name = name
+	err = SnapshotManager.TableSpec().Insert(snapshot)
+	if err != nil {
+		return nil, err
+	}
+	return snapshot, nil
+}
+
+func (self *SSnapshot) AllowDeleteItem(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
+	return self.IsOwner(userCred)
+}
+
+func (self *SSnapshot) StartSnapshotDeleteTask(ctx context.Context, userCred mcclient.TokenCredential, reloadDisk bool, parentTaskId string) error {
+	params := jsonutils.NewDict()
+	params.Set("reload_disk", jsonutils.NewBool(reloadDisk))
+	task, err := taskman.TaskManager.NewTask(ctx, "SnapshotDeleteTask", self, userCred, params, parentTaskId, "", nil)
+	if err != nil {
+		log.Errorf(err.Error())
+		return err
+	} else {
+		task.ScheduleRun(nil)
+	}
+	return nil
+}
+
+func (self *SSnapshot) ValidateDeleteCondition(ctx context.Context) error {
+	return nil
+}
+
+func (self *SSnapshot) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error {
+	if self.CreatedBy == MANUAL {
+		if !self.PendingDeleted {
+			return self.SVirtualResourceBase.PendingDelete()
+		} else {
+			return self.StartSnapshotDeleteTask(ctx, userCred, false, "")
+		}
+	} else {
+		return httperrors.NewBadRequestError("Cannot delete snapshot created by %s", self.CreatedBy)
+	}
+}
+
+func (self *SSnapshot) AllowPerformDeleted(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
+	return self.IsOwner(userCred)
+}
+
+func (self *SSnapshot) PerformDeleted(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	self.GetModelManager().TableSpec().Update(self, func() error {
+		self.OutOfChain = true
+		return nil
+	})
+	err := self.StartSnapshotDeleteTask(ctx, userCred, true, "")
+	return nil, err
+}
+
+func (self *SSnapshotManager) AllowGetPropertyMaxCount(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool {
+	return true
+}
+
+func (self *SSnapshotManager) GetPropertyMaxCount(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	ret := jsonutils.NewDict()
+	ret.Set("max_count", jsonutils.NewInt(int64(DISK_MAX_SNAPSHOT)))
+	return ret, nil
+}
+
+func (self *SSnapshotManager) GetConvertSnapshot(deleteSnapshot *SSnapshot) (*SSnapshot, error) {
+	dest := make([]SSnapshot, 0)
+	q := self.Query().SubQuery()
+	err := q.Query().Filter(sqlchemy.AND(sqlchemy.Equals(q.Field("disk_id"), deleteSnapshot.DiskId),
+		sqlchemy.Equals(q.Field("status"), SNAPSHOT_READY),
+		sqlchemy.Equals(q.Field("out_of_chain"), false))).
+		Asc("created_at").Limit(2).All(&dest)
+	if err != nil {
+		return nil, err
+	}
+	if len(dest) == 2 && dest[0].Id == deleteSnapshot.Id {
+		dest[1].SetModelManager(self)
+		return &dest[1], nil
+	}
+	return nil, fmt.Errorf("Snapshot %s cannot convert", deleteSnapshot.Id)
+}
+
+func (self *SSnapshotManager) AllowPerformDeleteDiskSnapshots(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
+	return userCred.IsAdmin()
+}
+
+func (self *SSnapshotManager) PerformDeleteDiskSnapshots(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	diskId, err := data.GetString("disk_id")
+	if err != nil {
+		return nil, err
+	}
+	disk, err := DiskManager.FetchById(diskId)
+	if disk != nil {
+		return nil, httperrors.NewBadRequestError("Cannot Delete disk %s snapshots, disk exist", diskId)
+	}
+	snapshots := self.GetDiskSnapshots(diskId)
+	if snapshots == nil || len(snapshots) == 0 {
+		return nil, httperrors.NewNotFoundError("Disk %s dose not have snapshot", diskId)
+	}
+	err = snapshots[0].StartSnapshotsDeleteTask(ctx, userCred, "")
+	return nil, err
+}
+
+func (self *SSnapshot) StartSnapshotsDeleteTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string) error {
+	task, err := taskman.TaskManager.NewTask(ctx, "BatchSnapshostDeleteTask", self, userCred, nil, parentTaskId, "", nil)
+	if err != nil {
+		log.Errorf(err.Error())
+		return err
+	} else {
+		task.ScheduleRun(nil)
+	}
+	return nil
+}
+
+func (self *SSnapshot) RealDelete(ctx context.Context, userCred mcclient.TokenCredential) error {
+	return db.DeleteModel(ctx, userCred, self)
+}
+
+func (self *SSnapshot) Delete(ctx context.Context, userCred mcclient.TokenCredential) error {
+	return nil
+}

pkg/compute/options/options.go

@@ -49,6 +49,9 @@ type ComputeOptions struct {
 
 	BaremetalPreparePackageUrl string `help:"Baremetal online register package"`
 
+	AutoSnapshotDay  int `default:"1" help:"Days auto snapshot disks, default 1 day"`
+	AutoSnapshotHour int `default:"2" help:"What hour take sanpshot, default 02:00"`
+
 	cloudcommon.DBOptions
 }
 

pkg/compute/service/service.go

@@ -52,10 +52,11 @@ func StartService() {
 	if db.CheckSync(options.Options.AutoSyncTable) {
 		err := models.InitDB()
 		if err == nil {
+			cron := cronman.GetCronJobManager()
+			cron.AddJob1("CleanPendingDeleteServers", time.Duration(options.Options.PendingDeleteCheckSeconds)*time.Second, models.GuestManager.CleanPendingDeleteServers)
+			cron.AddJob1("CleanPendingDeleteDisks", time.Duration(options.Options.PendingDeleteCheckSeconds)*time.Second, models.DiskManager.CleanPendingDeleteDisks)
+			cron.AddJob2("AutoDiskSnapshot", options.Options.AutoSnapshotDay, options.Options.AutoSnapshotHour, 0, 0, models.DiskManager.AutoDiskSnapshot)
 
-			cron := cronman.NewCronJobManager(0)
-			cron.AddJob("CleanPendingDeleteServers", time.Duration(options.Options.PendingDeleteCheckSeconds)*time.Second, models.GuestManager.CleanPendingDeleteServers)
-			cron.AddJob("CleanPendingDeleteDisks", time.Duration(options.Options.PendingDeleteCheckSeconds)*time.Second, models.DiskManager.CleanPendingDeleteDisks)
 			cron.Start()
 			defer cron.Stop()
 

pkg/compute/tasks/guest_disk_snapshot_task.go

@@ -0,0 +1,267 @@
+package tasks
+
+import (
+	"context"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	"yunion.io/x/onecloud/pkg/compute/models"
+)
+
+type GuestDiskSnapshotTask struct {
+	SGuestBaseTask
+}
+
+func init() {
+	taskman.RegisterTask(GuestDiskSnapshotTask{})
+	taskman.RegisterTask(SnapshotDeleteTask{})
+	taskman.RegisterTask(BatchSnapshostDeleteTask{})
+}
+
+func (self *GuestDiskSnapshotTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
+	guest := obj.(*models.SGuest)
+	self.DoDiskSnapshot(ctx, guest)
+}
+
+func (self *GuestDiskSnapshotTask) DoDiskSnapshot(ctx context.Context, guest *models.SGuest) {
+	diskId, err := self.Params.GetString("disk_id")
+	if err != nil {
+		self.TaskFailed(ctx, guest, err.Error())
+		return
+	}
+	snapshotId, err := self.Params.GetString("snapshot_id")
+	if err != nil {
+		self.TaskFailed(ctx, guest, err.Error())
+		return
+	}
+	self.SetStage("OnDiskSnapshotComplete", nil)
+	guest.SetStatus(self.UserCred, models.VM_SNAPSHOT, "")
+	err = guest.GetDriver().RequestDiskSnapshot(ctx, guest, self, snapshotId, diskId)
+	if err != nil {
+		self.TaskFailed(ctx, guest, err.Error())
+		return
+	}
+}
+
+func (self *GuestDiskSnapshotTask) OnDiskSnapshotComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
+	res := data.(*jsonutils.JSONDict)
+	location, err := res.GetString("location")
+	if err != nil {
+		log.Infof("OnDiskSnapshotComplete called with data no location")
+		return
+	}
+	snapshotId, _ := self.Params.GetString("snapshot_id")
+	iSnapshot, _ := models.SnapshotManager.FetchById(snapshotId)
+	snapshot := iSnapshot.(*models.SSnapshot)
+	models.SnapshotManager.TableSpec().Update(snapshot, func() error {
+		snapshot.Location = location
+		snapshot.Status = models.SNAPSHOT_READY
+		return nil
+	})
+	guest.SetStatus(self.UserCred, models.VM_SNAPSHOT_SUCC, "")
+	self.TaskComplete(ctx, guest, nil)
+}
+
+func (self *GuestDiskSnapshotTask) OnDiskSnapshotCompleteFailed(ctx context.Context, guest *models.SGuest, err jsonutils.JSONObject) {
+	self.TaskFailed(ctx, guest, err.String())
+}
+
+func (self *GuestDiskSnapshotTask) OnAutoDeleteSnapshot(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
+	self.TaskComplete(ctx, guest, data)
+}
+
+func (self *GuestDiskSnapshotTask) OnAutoDeleteSnapshotFailed(ctx context.Context, guest *models.SGuest, err jsonutils.JSONObject) {
+	log.Errorf("Auto Delete Snapshot Failed %s", err.String())
+	self.TaskComplete(ctx, guest, err)
+}
+
+func (self *GuestDiskSnapshotTask) TaskComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
+	guest.StartSyncstatus(ctx, self.UserCred, self.GetTaskId())
+	self.SetStage("OnSyncStatus", nil)
+}
+
+func (self *GuestDiskSnapshotTask) OnSyncStatus(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
+	self.SetStageComplete(ctx, nil)
+}
+
+func (self *GuestDiskSnapshotTask) TaskFailed(ctx context.Context, guest *models.SGuest, reason string) {
+	snapshotId, _ := self.Params.GetString("snapshot_id")
+	iSnapshot, _ := models.SnapshotManager.FetchById(snapshotId)
+	snapshot := iSnapshot.(*models.SSnapshot)
+	models.SnapshotManager.TableSpec().Update(snapshot, func() error {
+		snapshot.Status = models.SNAPSHOT_FAILED
+		return nil
+	})
+	self.SetStageFailed(ctx, reason)
+	guest.SetStatus(self.UserCred, models.VM_SNAPSHOT_FAILED, reason)
+}
+
+/***************************** Snapshot Delete Task *****************************/
+
+type SnapshotDeleteTask struct {
+	taskman.STask
+}
+
+func (self *SnapshotDeleteTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
+	snapshot := obj.(*models.SSnapshot)
+	guest, err := snapshot.GetGuest()
+	if err != nil {
+		self.SetStageFailed(ctx, err.Error())
+		return
+	}
+	if guest == nil {
+		self.SetStageFailed(ctx, "Cannot delete without guest")
+		return
+	}
+	if jsonutils.QueryBoolean(self.Params, "reload_disk", false) && snapshot.OutOfChain {
+		self.StartReloadDisk(ctx, snapshot, guest)
+	} else {
+		self.StartDeleteSnapshot(ctx, snapshot, guest)
+	}
+}
+
+func (self *SnapshotDeleteTask) StartReloadDisk(ctx context.Context, snapshot *models.SSnapshot, guest *models.SGuest) {
+	self.SetStage("OnReloadDiskSnapshot", nil)
+	guest.SetStatus(self.UserCred, models.VM_SNAPSHOT, "Start Reload Snapshot")
+	params := jsonutils.NewDict()
+	params.Set("disk_id", jsonutils.NewString(snapshot.DiskId))
+	err := guest.GetDriver().RequestReloadDiskSnapshot(ctx, guest, self, params)
+	if err != nil {
+		self.TaskFailed(ctx, snapshot, err.Error())
+	}
+}
+
+func (self *SnapshotDeleteTask) StartDeleteSnapshot(ctx context.Context, snapshot *models.SSnapshot, guest *models.SGuest) {
+	convertSnapshot, err := models.SnapshotManager.GetConvertSnapshot(snapshot)
+	if err != nil {
+		self.TaskFailed(ctx, snapshot, err.Error())
+		return
+	}
+	params := jsonutils.NewDict()
+	params.Set("delete_snapshot", jsonutils.NewString(snapshot.Id))
+	params.Set("disk_id", jsonutils.NewString(snapshot.DiskId))
+	if !snapshot.OutOfChain {
+		params.Set("convert_snapshot", jsonutils.NewString(convertSnapshot.Id))
+		var pendingDelete = jsonutils.JSONFalse
+		if snapshot.CreatedBy == models.MANUAL && snapshot.PendingDeleted == false {
+			pendingDelete = jsonutils.JSONTrue
+		}
+		params.Set("pending_delete", pendingDelete)
+	} else {
+		params.Set("auto_deleted", jsonutils.JSONTrue)
+	}
+
+	self.SetStage("OnDeleteSnapshot", nil)
+	guest.SetStatus(self.UserCred, models.VM_SNAPSHOT, "Start Delete Snapshot")
+	err = guest.GetDriver().RequestDeleteSnapshot(ctx, guest, self, params)
+	if err != nil {
+		self.TaskFailed(ctx, snapshot, err.Error())
+	}
+}
+
+func (self *SnapshotDeleteTask) OnDeleteSnapshot(ctx context.Context, snapshot *models.SSnapshot, data jsonutils.JSONObject) {
+	if !jsonutils.QueryBoolean(data, "deleted", false) {
+		log.Infof("OnDeleteSnapshot with no deleted")
+		return
+	}
+	if snapshot.OutOfChain {
+		snapshot.RealDelete(ctx, self.UserCred)
+		self.TaskComplete(ctx, snapshot, nil)
+	} else {
+		guest, err := snapshot.GetGuest()
+		if err != nil {
+			self.SetStageFailed(ctx, err.Error())
+			return
+		}
+		var pendingDelete = false
+		if snapshot.CreatedBy == models.MANUAL && snapshot.PendingDeleted == false &&
+			!jsonutils.QueryBoolean(self.Params, "force_delete", false) {
+			pendingDelete = true
+		}
+		if pendingDelete {
+			snapshot.PendingDelete()
+		} else {
+			snapshot.RealDelete(ctx, self.UserCred)
+		}
+		self.SetStage("TaskComplete", nil)
+		guest.StartSyncstatus(ctx, self.UserCred, "")
+	}
+}
+
+func (self *SnapshotDeleteTask) OnDeleteSnapshotFailed(ctx context.Context, snapshot *models.SSnapshot, data jsonutils.JSONObject) {
+	self.SetStageFailed(ctx, data.String())
+}
+
+func (self *SnapshotDeleteTask) OnReloadDiskSnapshot(ctx context.Context, snapshot *models.SSnapshot, data jsonutils.JSONObject) {
+	if !jsonutils.QueryBoolean(data, "reopen", false) {
+		log.Infof("OnDeleteSnapshot with no reopen")
+		return
+	}
+	if snapshot.CreatedBy == models.AUTO {
+		err := snapshot.RealDelete(ctx, self.UserCred)
+		if err != nil {
+			self.TaskFailed(ctx, snapshot, err.Error())
+			return
+		}
+	} else {
+		err := snapshot.SVirtualResourceBase.PendingDelete()
+		if err != nil {
+			self.TaskFailed(ctx, snapshot, err.Error())
+			return
+		}
+	}
+	self.SetStage("TaskComplete", nil)
+	guest, err := snapshot.GetGuest()
+	if err != nil {
+		self.SetStageFailed(ctx, err.Error())
+		return
+	}
+	guest.StartSyncstatus(ctx, self.UserCred, self.GetTaskId())
+}
+
+func (self *SnapshotDeleteTask) TaskComplete(ctx context.Context, snapshot *models.SSnapshot, data jsonutils.JSONObject) {
+	self.SetStageComplete(ctx, nil)
+}
+
+func (self *SnapshotDeleteTask) TaskFailed(ctx context.Context, snapshot *models.SSnapshot, reason string) {
+	self.SetStageFailed(ctx, reason)
+	guest, err := snapshot.GetGuest()
+	if err != nil {
+		log.Errorln(err.Error())
+		return
+	}
+	guest.StartSyncstatus(ctx, self.UserCred, "")
+}
+
+type BatchSnapshostDeleteTask struct {
+	taskman.STask
+}
+
+func (self *BatchSnapshostDeleteTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
+	snapshot := obj.(*models.SSnapshot)
+	self.StartStorageDeleteSnapshot(ctx, snapshot)
+}
+
+func (self *BatchSnapshostDeleteTask) StartStorageDeleteSnapshot(ctx context.Context, snapshot *models.SSnapshot) {
+	host := snapshot.GetHost()
+	if host == nil {
+		self.SetStageFailed(ctx, "Cannot found snapshot host")
+		return
+	}
+	self.SetStage("OnStorageDeleteSnapshot", nil)
+	err := host.GetHostDriver().RequestDeleteSnapshotWithStorage(ctx, host, snapshot, self)
+	if err != nil {
+		self.SetStageFailed(ctx, err.Error())
+	}
+}
+
+func (self *BatchSnapshostDeleteTask) OnStorageDeleteSnapshot(ctx context.Context, snapshot *models.SSnapshot, data jsonutils.JSONObject) {
+	snapshots := models.SnapshotManager.GetDiskSnapshots(snapshot.DiskId)
+	for i := 0; i < len(snapshots); i++ {
+		snapshots[i].RealDelete(ctx, self.UserCred)
+	}
+	self.SetStageComplete(ctx, nil)
+}

pkg/mcclient/modules/mod_snapshots.go

@@ -0,0 +1,14 @@
+package modules
+
+var (
+	Snapshots ResourceManager
+)
+
+func init() {
+	Snapshots = NewComputeManager("snapshot", "snapshots",
+		[]string{"ID", "Name", "Size", "Status",
+			"Disk_id", "Guest_id", "Created_at"},
+		[]string{"Storage_id", "Create_by", "Location"})
+
+	registerCompute(&Snapshots)
+}

pkg/util/httputils/httputils.go

@@ -16,6 +16,7 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/gotypes"
 	"yunion.io/x/pkg/trace"
 
 	"yunion.io/x/onecloud/pkg/appctx"
@@ -154,7 +155,7 @@ func Request(client *http.Client, ctx context.Context, method string, urlStr str
 
 func JSONRequest(client *http.Client, ctx context.Context, method string, urlStr string, header http.Header, body jsonutils.JSONObject, debug bool) (http.Header, jsonutils.JSONObject, error) {
 	bodystr := ""
-	if body != nil {
+	if !gotypes.IsNil(body) {
 		bodystr = body.String()
 	}
 	jbody := strings.NewReader(bodystr)