Merge branch 'master' into hotfix/qx-secgroup-rules

.github/workflows/docker.yml

@@ -31,7 +31,7 @@ jobs:
         make cmd/apigateway cmd/baremetal-agent cmd/climc cmd/keystone
         make cmd/logger cmd/region cmd/scheduler cmd/webconsole
         make cmd/yunionconf cmd/glance cmd/torrent cmd/s3gateway
-        make cmd/ansibleserver
+        make cmd/ansibleserver cmd/notify
 
     - name: Image baremetal-agent
       uses: elgohr/Publish-Docker-Github-Action@master
@@ -122,3 +122,13 @@ jobs:
         registry: registry.cn-beijing.aliyuncs.com
         snapshot: true
         dockerfile: build/docker/Dockerfile.glance
+
+    - name: Image notify
+      uses: elgohr/Publish-Docker-Github-Action@master
+      with:
+        name: registry.cn-beijing.aliyuncs.com/yunionio/notify
+        username: ${{ secrets.DOCKER_USERNAME }}
+        password: ${{ secrets.DOCKER_PASSWORD }}
+        registry: registry.cn-beijing.aliyuncs.com
+        snapshot: true
+        dockerfile: build/docker/Dockerfile.notify

build/docker/Dockerfile.notify

@@ -0,0 +1,15 @@
+FROM frolvlad/alpine-glibc:glibc-2.28
+
+MAINTAINER "Zexi Li <lizexi@yunionyun.com>"
+
+ENV TZ Asia/Shanghai
+
+RUN mkdir -p /opt/yunion/bin
+
+RUN apk update && \
+    apk add --no-cache tzdata && \
+    rm -rf /var/cache/apk/*
+
+RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
+
+ADD ./_output/bin/notify /opt/yunion/bin/notify

build/docker/Dockerfile.region

@@ -12,4 +12,5 @@ ENV TZ Asia/Shanghai
 
 RUN mkdir -p /opt/yunion/bin
 
+COPY ./build/region/root/opt/ /opt/
 ADD ./_output/bin/region /opt/yunion/bin/region

pkg/apigateway/handler/auth.go

@@ -21,9 +21,9 @@ import (
 	"strings"
 	"time"
 
-	"github.com/pkg/errors"
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 
 	"yunion.io/x/onecloud/pkg/apigateway/clientman"
 	"yunion.io/x/onecloud/pkg/apigateway/constants"
@@ -104,11 +104,11 @@ func (h *AuthHandlers) Bind(app *appsrv.Application) {
 func (h *AuthHandlers) GetRegionsResponse(ctx context.Context, w http.ResponseWriter, req *http.Request) (*jsonutils.JSONDict, error) {
 	adminToken := auth.AdminCredential()
 	if adminToken == nil {
-		return nil, errors.New("failed to get admin credential")
+		return nil, errors.Error("failed to get admin credential")
 	}
 	regions := adminToken.GetRegions()
 	if len(regions) == 0 {
-		return nil, errors.New("region is empty")
+		return nil, errors.Error("region is empty")
 	}
 	regionsJson := jsonutils.NewStringArray(regions)
 	s := auth.GetAdminSession(ctx, regions[0], "")
@@ -360,7 +360,7 @@ func isUserAllowWebconsole(ctx context.Context, w http.ResponseWriter, req *http
 		return false
 	}
 	if !jsonutils.QueryBoolean(usr, "allow_web_console", true) {
-		httperrors.ForbiddenError(w, "forbidden user %q login from web")
+		httperrors.ForbiddenError(w, "forbidden user %q login from web", usr.String())
 		return false
 	}
 	return true
@@ -507,10 +507,29 @@ func (h *AuthHandlers) postLogoutHandler(ctx context.Context, w http.ResponseWri
 
 func FetchRegion(req *http.Request) string {
 	r, e := req.Cookie("region")
-	if e != nil {
+	if e == nil && len(r.Value) > 0 {
+		return r.Value
+	}
+	if len(options.Options.DefaultRegion) > 0 {
 		return options.Options.DefaultRegion
 	}
-	return r.Value
+	adminToken := auth.AdminCredential()
+	if adminToken == nil {
+		log.Errorf("FetchRegion: nil adminTken")
+		return ""
+	}
+	regions := adminToken.GetRegions()
+	if len(regions) == 0 {
+		log.Errorf("FetchRegion: empty region list")
+		return ""
+	}
+	for _, r := range regions {
+		if len(r) > 0 {
+			return r
+		}
+	}
+	log.Errorf("FetchRegion: no valid region")
+	return ""
 }
 
 func fetchDomain(req *http.Request) string {

pkg/appsrv/workers.go

@@ -144,9 +144,15 @@ type SWorkerManager struct {
 	workerLock     *sync.Mutex
 	workerId       uint64
 	dbWorker       bool
+
+	ignoreOverflow bool
 }
 
 func NewWorkerManager(name string, workerCount int, backlog int, dbWorker bool) *SWorkerManager {
+	return NewWorkerManagerIgnoreOverflow(name, workerCount, backlog, dbWorker, false)
+}
+
+func NewWorkerManagerIgnoreOverflow(name string, workerCount int, backlog int, dbWorker bool, ignoreOverflow bool) *SWorkerManager {
 	manager := SWorkerManager{name: name,
 		queue:          NewRing(workerCount * backlog),
 		workerCount:    workerCount,
@@ -156,6 +162,8 @@ func NewWorkerManager(name string, workerCount int, backlog int, dbWorker bool)
 		workerLock:     &sync.Mutex{},
 		workerId:       0,
 		dbWorker:       dbWorker,
+
+		ignoreOverflow: ignoreOverflow,
 	}
 
 	workerManagers = append(workerManagers, &manager)
@@ -176,8 +184,8 @@ func (wm *SWorkerManager) Run(task func(), worker chan *SWorker, onErr func(erro
 	ret := wm.queue.Push(&sWorkerTask{task: task, worker: worker, onError: onErr})
 	if ret {
 		wm.schedule()
-	} else {
-		log.Warningf("queue full, task dropped")
+	} else if !wm.ignoreOverflow {
+		log.Warningf("[%s] queue full, task dropped", wm)
 	}
 	return ret
 }

pkg/cloudcommon/db/db_dispatcher.go

@@ -1019,9 +1019,10 @@ func _doCreateItem(
 	}
 
 	// run name validation after validate create data
+	parentId := manager.FetchParentId(ctx, dataDict)
 	name, _ := dataDict.GetString("name")
 	if len(name) > 0 {
-		err = NewNameValidator(manager, ownerId, name)
+		err = NewNameValidator(manager, ownerId, name, parentId)
 		if err != nil {
 			return nil, err
 		}

pkg/cloudcommon/db/interface.go

@@ -68,6 +68,7 @@ type IModelManager interface {
 	FilterByOwner(q *sqlchemy.SQuery, userCred mcclient.IIdentityProvider, scope rbacutils.TRbacScope) *sqlchemy.SQuery
 	FilterBySystemAttributes(q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject, scope rbacutils.TRbacScope) *sqlchemy.SQuery
 	FilterByHiddenSystemAttributes(q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject, scope rbacutils.TRbacScope) *sqlchemy.SQuery
+	FilterByParentId(q *sqlchemy.SQuery, parentId string) *sqlchemy.SQuery
 
 	// GetOwnerId(userCred mcclient.IIdentityProvider) mcclient.IIdentityProvider
 
@@ -107,6 +108,7 @@ type IModelManager interface {
 
 	// fetch owner Id from query when create resource
 	FetchOwnerId(ctx context.Context, data jsonutils.JSONObject) (mcclient.IIdentityProvider, error)
+	FetchParentId(ctx context.Context, data jsonutils.JSONObject) string
 
 	/* name uniqueness scope, system/domain/project, default is system */
 	NamespaceScope() rbacutils.TRbacScope
@@ -170,6 +172,7 @@ type IModel interface {
 	DeleteInContext(ctx context.Context, userCred mcclient.TokenCredential, ctxObjs []IModel, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error)
 
 	GetOwnerId() mcclient.IIdentityProvider
+	GetParentId() string
 
 	IsSharable(reqCred mcclient.IIdentityProvider) bool
 

pkg/cloudcommon/db/modelbase.go

@@ -159,6 +159,10 @@ func (manager *SModelBaseManager) FilterByHiddenSystemAttributes(q *sqlchemy.SQu
 	return q
 }
 
+func (manager *SModelBaseManager) FilterByParentId(q *sqlchemy.SQuery, parentId string) *sqlchemy.SQuery {
+	return q
+}
+
 func (manager *SModelBaseManager) FetchById(idStr string) (IModel, error) {
 	return nil, sql.ErrNoRows
 }
@@ -256,6 +260,10 @@ func (manager *SModelBaseManager) FetchOwnerId(ctx context.Context, data jsonuti
 	return nil, nil
 }
 
+func (manager *SModelBaseManager) FetchParentId(ctx context.Context, data jsonutils.JSONObject) string {
+	return ""
+}
+
 func (manager *SModelBaseManager) NamespaceScope() rbacutils.TRbacScope {
 	return rbacutils.ScopeSystem
 }
@@ -492,6 +500,10 @@ func (model *SModelBase) GetOwnerId() mcclient.IIdentityProvider {
 	return nil
 }
 
+func (model *SModelBase) GetParentId() string {
+	return ""
+}
+
 func (model *SModelBase) IsSharable(ownerId mcclient.IIdentityProvider) bool {
 	return false
 }

pkg/cloudcommon/db/namevalidator.go

@@ -23,11 +23,12 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient"
 )
 
-func isNameUnique(manager IModelManager, ownerId mcclient.IIdentityProvider, name string) (bool, error) {
+func isNameUnique(manager IModelManager, ownerId mcclient.IIdentityProvider, name string, parentId string) (bool, error) {
 	q := manager.Query()
 	q = manager.FilterByName(q, name)
 	q = manager.FilterByOwner(q, ownerId, manager.NamespaceScope())
 	q = manager.FilterBySystemAttributes(q, nil, nil, manager.ResourceScope())
+	q = manager.FilterByParentId(q, parentId)
 	cnt, err := q.CountWithError()
 	if err != nil {
 		return false, err
@@ -35,12 +36,12 @@ func isNameUnique(manager IModelManager, ownerId mcclient.IIdentityProvider, nam
 	return cnt == 0, nil
 }
 
-func NewNameValidator(manager IModelManager, ownerId mcclient.IIdentityProvider, name string) error {
+func NewNameValidator(manager IModelManager, ownerId mcclient.IIdentityProvider, name string, parentId string) error {
 	err := manager.ValidateName(name)
 	if err != nil {
 		return err
 	}
-	uniq, err := isNameUnique(manager, ownerId, name)
+	uniq, err := isNameUnique(manager, ownerId, name, parentId)
 	if err != nil {
 		return err
 	}
@@ -57,6 +58,7 @@ func isAlterNameUnique(model IModel, name string) (bool, error) {
 	q = manager.FilterByOwner(q, model.GetOwnerId(), manager.NamespaceScope())
 	q = manager.FilterBySystemAttributes(q, nil, nil, manager.ResourceScope())
 	q = manager.FilterByNotId(q, model.GetId())
+	q = manager.FilterByParentId(q, model.GetParentId())
 	cnt, err := q.CountWithError()
 	if err != nil {
 		return false, err
@@ -96,7 +98,7 @@ func GenerateName2(manager IModelManager, ownerId mcclient.IIdentityProvider, hi
 		var uniq bool
 		var err error
 		if model == nil {
-			uniq, err = isNameUnique(manager, ownerId, name)
+			uniq, err = isNameUnique(manager, ownerId, name, "")
 		} else {
 			uniq, err = isAlterNameUnique(model, name)
 		}

pkg/cloudcommon/policy/defaults.go

@@ -258,6 +258,12 @@ var (
 					Action:   PolicyActionUpdate,
 					Result:   rbacutils.Allow,
 				},
+				{
+					Service:  "notify",
+					Resource: "contacts",
+					Action:   PolicyActionList,
+					Result:   rbacutils.Allow,
+				},
 			},
 		},
 		{

pkg/cloudcommon/policy/policy.go

@@ -62,7 +62,8 @@ func init() {
 	}
 	DefaultPolicyFetcher = remotePolicyFetcher
 
-	syncWorkerManager = appsrv.NewWorkerManager("sync_policy_worker", 1, 1000, false)
+	// no need to queue many sync tasks
+	syncWorkerManager = appsrv.NewWorkerManagerIgnoreOverflow("sync_policy_worker", 1, 2, true, true)
 }
 
 type SPolicyManager struct {

pkg/cloudcommon/policy/resources.go

@@ -36,7 +36,7 @@ var (
 		"metadatas",
 		"loadbalancerclusters",
 		"loadbalanceragents",
-		"netowkinterfaces",
+		"networkinterfaces",
 		"natgateways",
 		"natsentries",
 		"natdentries",

pkg/compute/models/guest_actions.go

@@ -386,7 +386,7 @@ func (self *SGuest) PerformClone(ctx context.Context, userCred mcclient.TokenCre
 	if len(cloneInput.Name) == 0 {
 		return nil, httperrors.NewMissingParameterError("name")
 	}
-	err = db.NewNameValidator(GuestManager, userCred, cloneInput.Name)
+	err = db.NewNameValidator(GuestManager, userCred, cloneInput.Name, "")
 	if err != nil {
 		return nil, err
 	}
@@ -3085,7 +3085,7 @@ func (man *SGuestManager) PerformImport(ctx context.Context, userCred mcclient.T
 	if obj, _ := man.FetchByIdOrName(userCred, desc.Id); obj != nil {
 		return nil, httperrors.NewInputParameterError("Server %s already exists", desc.Id)
 	}
-	if err := db.NewNameValidator(man, userCred, desc.Name); err != nil {
+	if err := db.NewNameValidator(man, userCred, desc.Name, ""); err != nil {
 		return nil, err
 	}
 	if hostObj, _ := HostManager.FetchByIdOrName(userCred, desc.HostId); hostObj == nil {

pkg/compute/models/guest_queries.go

@@ -55,6 +55,16 @@ func (manager *SGuestManager) FetchCustomizeColumns(ctx context.Context, userCre
 			}
 		}
 	}
+	if len(fields) == 0 || fields.Contains("nics") {
+		nicsMap := fetchGuestNICs(ctx, guestIds, tristate.False)
+		if nicsMap != nil {
+			for i := range rows {
+				if nics, ok := nicsMap[objs[i].GetId()]; ok {
+					rows[i].Add(nics, "nics")
+				}
+			}
+		}
+	}
 	if len(fields) == 0 || fields.Contains("vpc") || fields.Contains("vpc_id") {
 		gvpcs := fetchGuestVpcs(guestIds)
 		if gvpcs != nil {
@@ -218,6 +228,26 @@ func fetchGuestIPs(guestIds []string, virtual tristate.TriState) map[string][]st
 	return ret
 }
 
+func fetchGuestNICs(ctx context.Context, guestIds []string, virtual tristate.TriState) map[string]*jsonutils.JSONArray {
+	q := GuestnetworkManager.Query().In("guest_id", guestIds)
+	nics := make([]SGuestnetwork, 0)
+	if err := q.All(&nics); err != nil {
+		return nil
+	}
+	ret := make(map[string]*jsonutils.JSONArray)
+	for i := range nics {
+		desc := nics[i].GetShortDesc(ctx)
+		li := ret[nics[i].GuestId]
+		if li == nil {
+			li = jsonutils.NewArray(desc)
+			ret[nics[i].GuestId] = li
+		} else {
+			li.Add(desc)
+		}
+	}
+	return ret
+}
+
 func (self *SGuest) GetRealIPs() []string {
 	result := fetchGuestIPs([]string{self.Id}, tristate.False)
 	if result == nil {

pkg/compute/models/guestnetworks.go

@@ -546,6 +546,12 @@ func calculateNics(q *sqlchemy.SQuery) GuestnicsCount {
 }
 
 func (self *SGuestnetwork) IsExit() bool {
+	if self.IpAddr != "" {
+		addr, err := netutils.NewIPV4Addr(self.IpAddr)
+		if err == nil {
+			return netutils.IsExitAddress(addr)
+		}
+	}
 	net := self.GetNetwork()
 	if net != nil {
 		return net.IsExitNetwork()
@@ -690,6 +696,7 @@ func (self *SGuestnetwork) GetShortDesc(ctx context.Context) *jsonutils.JSONDict
 	desc := jsonutils.NewDict()
 	if len(self.IpAddr) > 0 {
 		desc.Add(jsonutils.NewString(self.IpAddr), "ip_addr")
+		desc.Add(jsonutils.NewBool(self.IsExit()), "is_exit")
 	}
 	if len(self.Ip6Addr) > 0 {
 		desc.Add(jsonutils.NewString(self.Ip6Addr), "ip6_addr")

pkg/compute/models/guests.go

@@ -893,10 +893,6 @@ func (manager *SGuestManager) validateCreateData(
 			return nil, httperrors.NewBadRequestError("Snapshot error: disk index 0 but disk type is %s", diskConfig.DiskType)
 		}
 
-		if len(diskConfig.ImageId) == 0 && len(diskConfig.SnapshotId) == 0 && !data.Contains("cdrom") {
-			return nil, httperrors.NewBadRequestError("Miss operating system???")
-		}
-
 		// if len(diskConfig.Backend) == 0 {
 		// 	diskConfig.Backend = STORAGE_LOCAL
 		// }
@@ -953,6 +949,10 @@ func (manager *SGuestManager) validateCreateData(
 		return nil, err
 	}
 
+	if input.Hypervisor != api.HYPERVISOR_KVM && len(input.Disks[0].ImageId) == 0 && len(input.Disks[0].SnapshotId) == 0 && input.Cdrom == "" {
+		return nil, httperrors.NewBadRequestError("Miss operating system???")
+	}
+
 	hypervisor = input.Hypervisor
 	if hypervisor != api.HYPERVISOR_CONTAINER {
 		// support sku here

pkg/compute/models/hosts.go

@@ -3007,7 +3007,7 @@ func (self *SHost) PerformInitialize(
 	if err != nil || self.GetBaremetalServer() != nil {
 		return nil, nil
 	}
-	err = db.NewNameValidator(GuestManager, userCred, name)
+	err = db.NewNameValidator(GuestManager, userCred, name, "")
 	if err != nil {
 		return nil, err
 	}

pkg/compute/models/networks.go

@@ -1939,7 +1939,7 @@ func (self *SNetwork) PerformSplit(ctx context.Context, userCred mcclient.TokenC
 	defer lockman.ReleaseClass(ctx, NetworkManager, db.GetLockClassKey(NetworkManager, userCred))
 
 	if len(name) > 0 {
-		if err := db.NewNameValidator(NetworkManager, userCred, name); err != nil {
+		if err := db.NewNameValidator(NetworkManager, userCred, name, ""); err != nil {
 			return nil, httperrors.NewInputParameterError("Duplicate name %s", name)
 		}
 	} else {

pkg/compute/models/snapshotpolicy.go

@@ -17,6 +17,7 @@ package models
 import (
 	"context"
 	"fmt"
+
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
@@ -151,7 +152,7 @@ func (manager *SSnapshotPolicyManager) ValidateCreateData(ctx context.Context, u
 	input.ProjectId = ownerId.GetProjectId()
 	input.DomainId = ownerId.GetProjectDomainId()
 
-	err = db.NewNameValidator(manager, ownerId, input.Name)
+	err = db.NewNameValidator(manager, ownerId, input.Name, "")
 	if err != nil {
 		return nil, err
 	}

pkg/compute/tasks/guest_disk_snapshot_task.go

@@ -88,6 +88,7 @@ func (self *GuestDiskSnapshotTask) OnDiskSnapshotCompleteFailed(ctx context.Cont
 func (self *GuestDiskSnapshotTask) TaskComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
 	logclient.AddActionLogWithStartable(self, guest, logclient.ACT_DISK_CREATE_SNAPSHOT, nil, self.UserCred, true)
 	guest.StartSyncstatus(ctx, self.UserCred, "")
+	self.SetStageComplete(ctx, nil)
 }
 
 func (self *GuestDiskSnapshotTask) TaskFailed(ctx context.Context, guest *models.SGuest, reason string) {

pkg/hostman/guestfs/fsdriver/winscripts.go

@@ -339,7 +339,7 @@ function mtw_assign_volume_letter(volume_no_set, letter_offset) {
                 'select volume=' + volume_no,
                 'assign letter=' + String.fromCharCode(charcode),
                 'select partition 1',
-                'format fs ntfs'
+                'format fs ntfs quick'
             );
         }
     }
@@ -347,7 +347,7 @@ function mtw_assign_volume_letter(volume_no_set, letter_offset) {
         'select volume=' + volume_no_set,
         'assign letter=' + letter_set,
         'select partition 1',
-        'format fs ntfs'
+        'format fs ntfs quick'
     );
     mtw_execute_diskpart(cmd_lines);
 

pkg/hostman/guestman/guesttasks.go

@@ -811,7 +811,7 @@ func (s *SGuestDiskSnapshotTask) onReloadBlkdevSucc(res string) {
 func (s *SGuestDiskSnapshotTask) onSnapshotBlkdevFail(string) {
 	snapshotDir := s.disk.GetSnapshotDir()
 	snapshotPath := path.Join(snapshotDir, s.snapshotId)
-	_, err := procutils.NewCommand("rm", "-rf", snapshotPath).Run()
+	_, err := procutils.NewCommand("mv", "-f", snapshotPath, s.disk.GetPath()).Run()
 	if err != nil {
 		log.Errorln(err)
 	}
@@ -908,7 +908,7 @@ func (s *SGuestSnapshotDeleteTask) onReloadBlkdevSucc(err string) {
 
 func (s *SGuestSnapshotDeleteTask) onSnapshotBlkdevFail(res string) {
 	snapshotPath := path.Join(s.disk.GetSnapshotDir(), s.convertSnapshot)
-	if _, err := procutils.NewCommand("rm", "-f", s.tmpPath, snapshotPath).Run(); err != nil {
+	if _, err := procutils.NewCommand("mv", "-f", s.tmpPath, snapshotPath).Run(); err != nil {
 		log.Errorln(err)
 	}
 	s.taskFailed("Reload blkdev failed")
@@ -923,8 +923,7 @@ func (s *SGuestSnapshotDeleteTask) onResumeSucc(res string) {
 		}
 	}
 	if !s.pendingDelete {
-		snapshotDir := s.disk.GetSnapshotDir()
-		procutils.NewCommand("rm", "-f", path.Join(snapshotDir, s.deleteSnapshot))
+		s.disk.DoDeleteSnapshot(s.deleteSnapshot)
 	}
 	body := jsonutils.NewDict()
 	body.Set("deleted", jsonutils.JSONTrue)

pkg/hostman/storageman/disk_base.go

@@ -37,6 +37,7 @@ type IDisk interface {
 	GetDiskSetupScripts(idx int) string
 	GetSnapshotLocation() string
 	OnRebuildRoot(ctx context.Context, params jsonutils.JSONObject) error
+	DoDeleteSnapshot(snapshotId string) error
 
 	DeleteAllSnapshot() error
 	DiskSnapshot(ctx context.Context, params interface{}) (jsonutils.JSONObject, error)
@@ -173,3 +174,7 @@ func (d *SBaseDisk) DiskDeleteSnapshot(ctx context.Context, params interface{})
 func (d *SBaseDisk) CreateFromRbdSnapshot(ctx context.Context, napshotUrl, srcDiskId, srcPool string) error {
 	return fmt.Errorf("Not implement disk.CreateFromRbdSnapshot")
 }
+
+func (d *SBaseDisk) DoDeleteSnapshot(snapshotId string) error {
+	return fmt.Errorf("Not implement disk.DoDeleteSnapshot")
+}

pkg/hostman/storageman/disk_local.go

@@ -541,3 +541,8 @@ func (d *SLocalDisk) PrepareMigrate(liveMigrate bool) (string, error) {
 	}
 	return "", nil
 }
+
+func (d *SLocalDisk) DoDeleteSnapshot(snapshotId string) error {
+	snapshotPath := path.Join(d.GetSnapshotDir(), snapshotId)
+	return d.Storage.DeleteDiskfile(snapshotPath)
+}

pkg/hostman/storageman/storage_base.go

@@ -387,11 +387,11 @@ func checkSnapshots(storage IStorage, snapshotDir string, maxSnapshotCount int)
 
 	// if snapshot count greater than maxsnapshot count, do convert
 	if len(snapshots) >= maxSnapshotCount {
-		requestConvertSnapshot(snapshotPath, diskId)
+		requestConvertSnapshot(storage, snapshotPath, diskId)
 	}
 }
 
-func requestConvertSnapshot(snapshotPath, diskId string) {
+func requestConvertSnapshot(storage IStorage, snapshotPath, diskId string) {
 	log.Infof("SNPASHOT path %s", snapshotPath)
 	res, err := modules.Disks.GetSpecific(
 		hostutils.GetComputeSession(context.Background()), diskId, "convert-snapshot", nil)
@@ -421,11 +421,13 @@ func requestConvertSnapshot(snapshotPath, diskId string) {
 		return
 	}
 	requestDeleteSnapshot(
-		diskId, snapshotPath, deleteSnapshot, convertSnapshotPath, outfile, pendingDelete)
+		storage, diskId, snapshotPath, deleteSnapshot,
+		convertSnapshotPath, outfile, pendingDelete,
+	)
 }
 
 func requestDeleteSnapshot(
-	diskId, snapshotPath, deleteSnapshot, convertSnapshotPath,
+	storage IStorage, diskId, snapshotPath, deleteSnapshot, convertSnapshotPath,
 	outfile string, pendingDelete bool,
 ) {
 	deleteSnapshotPath := path.Join(snapshotPath, deleteSnapshot)
@@ -446,8 +448,8 @@ func requestDeleteSnapshot(
 		return
 	}
 	if !pendingDelete {
-		if out, err := procutils.NewCommand("rm", "-f", deleteSnapshotPath).Run(); err != nil {
-			log.Errorf("%s", out)
+		if err := storage.DeleteDiskfile(deleteSnapshotPath); err != nil {
+			log.Errorln(err)
 			return
 		}
 	}

pkg/keystone/tokens/handlers.go

@@ -179,7 +179,7 @@ func verifyTokensV3(ctx context.Context, w http.ResponseWriter, r *http.Request)
 
 func verifyCommon(ctx context.Context, w http.ResponseWriter, tokenStr string) (*SAuthToken, error) {
 	adminToken := policy.FetchUserCredential(ctx)
-	if !adminToken.IsAllow(rbacutils.ScopeSystem, api.SERVICE_TYPE, "tokens", "perform", "auth") {
+	if adminToken == nil || !adminToken.IsAllow(rbacutils.ScopeSystem, api.SERVICE_TYPE, "tokens", "perform", "auth") {
 		return nil, httperrors.NewForbiddenError("not allow to auth")
 	}
 	token := SAuthToken{}

pkg/mcclient/options/servers.go

@@ -260,6 +260,8 @@ type ServerCreateOptions struct {
 	DryRun           *bool    `help:"Dry run to test scheduler" json:"-"`
 	UserDataFile     string   `help:"user_data file path" json:"-"`
 
+	OsType string `help:"os type, e.g. Linux, Windows, etc."`
+
 	Duration string `help:"valid duration of the server, e.g. 1H, 1D, 1W, 1M, 1Y, ADMIN ONLY option"`
 
 	AutoPrepaidRecycle bool `help:"automatically enable prepaid recycling after server is created successfully" json:"auto_prepaid_recycle,omitfalse"`
@@ -342,6 +344,7 @@ func (opts *ServerCreateOptions) Params() (*computeapi.ServerCreateInput, error)
 		EipChargeType:      opts.EipChargeType,
 		Eip:                opts.Eip,
 		EnableCloudInit:    opts.EnableCloudInit,
+		OsType:             opts.OsType,
 	}
 
 	if opts.GenerateName {

pkg/mcclient/token3.go

@@ -274,7 +274,6 @@ func (catalog KeystoneServiceCatalogV3) getRegions() []string {
 			}
 		}
 	}
-	fmt.Println("getRegions", regions)
 	return regions
 }
 

pkg/notify/options/options.go

@@ -24,10 +24,10 @@ type NotifyOption struct {
 
 	DingtalkEnabled       bool   `help:"Enable dingtalk"`
 	SocketFileDir         string `help:"Socket file directory" default:"/etc/yunion/notify"`
-	UpdateInterval        int    `help:"Update send services interval(unit:s)" default:30`
+	UpdateInterval        int    `help:"Update send services interval(unit:s)" default:"30"`
 	VerifyEmailUrl        string `help:"url of verify email"`
 	ReSendScope           int    `help:"Resend all messages that have not been sent successfully within ReSendScope minutes"`
-	InitNotificationScope int    `help:"initialize data of notification with in InitNotificationScope hours" default:100`
+	InitNotificationScope int    `help:"initialize data of notification with in InitNotificationScope hours" default:"100"`
 }
 
 var Options NotifyOption

pkg/notify/service.go

@@ -63,7 +63,7 @@ func StartService() {
 	models.NotifyService.InitAll()
 	defer models.NotifyService.StopAll()
 
-	cron := cronman.GetCronJobManager()
+	cron := cronman.InitCronJobManager(true, 2)
 	// update service
 	cron.AddJobAtIntervals("UpdateServices", time.Duration(opts.UpdateInterval)*time.Second, models.NotifyService.UpdateServices)
 

pkg/scheduler/algorithm/predicates/baremetal/network_predicate.go

@@ -1,156 +0,0 @@
-// Copyright 2019 Yunion
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package baremetal
-
-import (
-	"fmt"
-	"strings"
-	"sync"
-
-	"yunion.io/x/pkg/utils"
-
-	computeapi "yunion.io/x/onecloud/pkg/apis/compute"
-	"yunion.io/x/onecloud/pkg/scheduler/algorithm/plugin"
-	"yunion.io/x/onecloud/pkg/scheduler/algorithm/predicates"
-	"yunion.io/x/onecloud/pkg/scheduler/core"
-)
-
-type NetworkPredicate struct {
-	BasePredicate
-	plugin.BasePlugin
-	SelectedNetworks sync.Map
-}
-
-func (p *NetworkPredicate) Name() string {
-	return "baremetal_network"
-}
-
-func (p *NetworkPredicate) Clone() core.FitPredicate {
-	return &NetworkPredicate{}
-}
-
-func (p *NetworkPredicate) PreExecute(u *core.Unit, cs []core.Candidater) (bool, error) {
-	notIgnore, _ := p.BasePredicate.PreExecute(u, cs)
-	if !notIgnore {
-		return false, nil
-	}
-
-	u.AppendSelectPlugin(p)
-	d := u.SchedData()
-	if len(d.HostId) > 0 && len(d.Networks) == 0 {
-		return false, nil
-	}
-
-	return true, nil
-}
-
-func (p *NetworkPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core.PredicateFailureReason, error) {
-	h := predicates.NewPredicateHelper(p, u, c)
-	schedData := u.SchedData()
-
-	networks := c.Getter().Networks()
-
-	counters := core.NewCounters()
-
-	isMigrate := func() bool {
-		return len(schedData.HostId) > 0
-	}
-
-	isRandomNetworkAvailable := func(private bool, exit bool, wire string) string {
-		var errMsgs []string
-		for _, network := range networks {
-			appendError := func(errMsg string) {
-				errMsgs = append(errMsgs, fmt.Sprintf("%s: %s", network.Id, errMsg))
-			}
-			if !((network.GetPorts() > 0 || isMigrate()) && network.IsExitNetwork() == exit) {
-				appendError(predicates.ErrNoPorts)
-			}
-			if wire != "" && !utils.HasPrefix(wire, network.WireId) && !utils.HasPrefix(wire, network.GetWire().GetName()) { // re
-				appendError(predicates.ErrWireIsNotMatch)
-			}
-			if (!private && network.IsPublic) || (private && !network.IsPublic && network.ProjectId == schedData.Project) {
-				// TODO: support reservedNetworks
-				reservedNetworks := 0
-				restPort := int64(network.GetPorts() - reservedNetworks)
-				if restPort == 0 {
-					appendError("not enough network port")
-					continue
-				}
-				counter := u.CounterManager.GetOrCreate("net:"+network.Id, func() core.Counter {
-					return core.NewNormalCounter(restPort)
-				})
-
-				u.SharedResourceManager.Add(network.Id, counter)
-				counters.Add(counter)
-				p.SelectedNetworks.Store(network.Id, counter.GetCount())
-				return ""
-			} else {
-				appendError(predicates.ErrNotOwner)
-			}
-		}
-
-		return strings.Join(errMsgs, ";")
-	}
-
-	filterByRandomNetwork := func() {
-		if err_msg := isRandomNetworkAvailable(false, false, ""); err_msg != "" {
-			h.Exclude(err_msg)
-		}
-		h.SetCapacityCounter(counters)
-	}
-
-	isNetworkAvaliable := func(network *computeapi.NetworkConfig) string {
-		if network.Network == "" {
-			return isRandomNetworkAvailable(network.Private, network.Exit, network.Wire)
-		}
-		for _, net := range networks {
-			if (network.Network == net.Id || network.Network == net.Name) && (net.IsPublic || net.ProjectId == schedData.Project) && (net.GetPorts() > 0 || isMigrate()) {
-				h.SetCapacity(1)
-				return ""
-			}
-		}
-
-		return predicates.ErrUnknown
-	}
-
-	filterBySpecifiedNetworks := func() {
-		var errMsgs []string
-
-		for _, network := range schedData.Networks {
-			if err_msg := isNetworkAvaliable(network); err_msg != "" {
-				errMsgs = append(errMsgs, err_msg)
-			}
-		}
-
-		if len(errMsgs) > 0 {
-			h.Exclude(strings.Join(errMsgs, ", "))
-		}
-	}
-
-	loadUnknownNetworks := func() bool {
-		return true // TODO: ???
-	}
-
-	loadUnknownNetworks()
-
-	// Randomly assign networks if no network is specified.
-	if len(schedData.Networks) == 0 {
-		filterByRandomNetwork()
-	} else {
-		filterBySpecifiedNetworks()
-	}
-
-	return h.GetResult()
-}

pkg/scheduler/algorithm/predicates/error.go

@@ -118,6 +118,7 @@ const (
 	NetworkDomain    = "network_domain"
 	NetworkRange     = "network_range"
 	NetworkFreeCount = "network_free_count"
+	NetworkPort      = "network_port"
 
 	StorageEnable = "storage_status"
 	StorageMatch  = "storage_match"

pkg/scheduler/algorithm/predicates/guest/network_predicate.go

@@ -1,217 +0,0 @@
-// Copyright 2019 Yunion
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package guest
-
-import (
-	"fmt"
-	"strings"
-	"sync"
-
-	"yunion.io/x/pkg/util/sets"
-	"yunion.io/x/pkg/utils"
-
-	"yunion.io/x/onecloud/pkg/compute/models"
-
-	computeapi "yunion.io/x/onecloud/pkg/apis/compute"
-	"yunion.io/x/onecloud/pkg/scheduler/algorithm/plugin"
-	"yunion.io/x/onecloud/pkg/scheduler/algorithm/predicates"
-	"yunion.io/x/onecloud/pkg/scheduler/api"
-	"yunion.io/x/onecloud/pkg/scheduler/core"
-)
-
-// NetworkPredicate will filter the current network information with
-// the specified scheduling information to match, if not specified will
-// randomly match the available network resources.
-type NetworkPredicate struct {
-	predicates.BasePredicate
-	plugin.BasePlugin
-	SelectedNetworks sync.Map
-}
-
-func (p *NetworkPredicate) Name() string {
-	return "host_network"
-}
-
-func (p *NetworkPredicate) Clone() core.FitPredicate {
-	return &NetworkPredicate{}
-}
-
-func (p *NetworkPredicate) PreExecute(u *core.Unit, cs []core.Candidater) (bool, error) {
-	data := u.SchedData()
-	if len(data.HostId) > 0 && len(data.Networks) == 0 {
-		return false, nil
-	}
-
-	return true, nil
-}
-
-func (p *NetworkPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core.PredicateFailureReason, error) {
-	h := predicates.NewPredicateHelper(p, u, c)
-
-	getter := c.Getter()
-	networks := getter.Networks()
-
-	d := u.SchedData()
-
-	isMigrate := func() bool {
-		return len(d.HostId) > 0
-	}
-
-	// ServerType's value is 'guest', 'container' or ''(support all type) will return true.
-	isMatchServerType := func(network *models.SNetwork) bool {
-		return sets.NewString("guest", "", "container").Has(network.ServerType)
-	}
-
-	counterOfNetwork := func(u *core.Unit, n *models.SNetwork, r int) core.Counter {
-		counter := u.CounterManager.GetOrCreate("net:"+n.Id, func() core.Counter {
-			return core.NewNormalCounter(int64(getter.GetFreePort(n.Id) - r))
-		})
-
-		u.SharedResourceManager.Add(n.GetId(), counter)
-		return counter
-	}
-
-	isRandomNetworkAvailable := func(private bool, exit bool, wire string,
-		counters core.MultiCounter) string {
-
-		var fullErrMsgs []string
-		found := false
-
-		for _, n := range networks {
-			errMsgs := []string{}
-			appendError := func(errMsg string) {
-				errMsgs = append(errMsgs, errMsg)
-			}
-
-			if !isMatchServerType(n.SNetwork) {
-				appendError(predicates.ErrServerTypeIsNotMatch)
-			}
-
-			if n.IsExitNetwork() != exit {
-				appendError(predicates.ErrExitIsNotMatch)
-			}
-
-			if !(n.GetPorts() > 0 || isMigrate()) {
-				appendError(predicates.ErrNoPorts)
-			}
-
-			if wire != "" && !utils.HasPrefix(wire, n.WireId) && !utils.HasPrefix(wire, n.GetWire().GetName()) { // re
-				appendError(predicates.ErrWireIsNotMatch)
-			}
-
-			if !((!private && n.IsPublic) || (private && !n.IsPublic && n.ProjectId == d.Project)) {
-				appendError(predicates.ErrNotOwner)
-			}
-
-			if len(errMsgs) == 0 {
-				// add resource
-				reservedNetworks := 0
-				counter := counterOfNetwork(u, n.SNetwork, reservedNetworks)
-				p.SelectedNetworks.Store(n.GetId(), counter.GetCount())
-				counters.Add(counter)
-				found = true
-			} else {
-				fullErrMsgs = append(fullErrMsgs,
-					fmt.Sprintf("%s: %s", n.Id, strings.Join(errMsgs, ",")),
-				)
-			}
-		}
-
-		if !found {
-			return strings.Join(fullErrMsgs, "; ")
-		}
-
-		return ""
-	}
-
-	filterByRandomNetwork := func() {
-		counters := core.NewCounters()
-		if err_msg := isRandomNetworkAvailable(false, false, "", counters); err_msg != "" {
-			h.Exclude(err_msg)
-		}
-		h.SetCapacityCounter(counters)
-	}
-
-	isNetworkAvaliable := func(n *computeapi.NetworkConfig, counters *core.MinCounters,
-		networks []*api.CandidateNetwork) string {
-		if n.Network == "" {
-			counters0 := core.NewCounters()
-			ret_msg := isRandomNetworkAvailable(n.Private, n.Exit, n.Wire, counters0)
-			counters.Add(counters0)
-			return ret_msg
-		}
-		if len(networks) == 0 {
-			return predicates.ErrNoAvailableNetwork
-		}
-
-		errMsgs := make([]string, 0)
-
-		for _, net := range networks {
-			/*if !isMatchServerType(net) {
-				errMsgs = append(errMsgs, fmt.Sprintf("%v(%v): server type not matched", net.Name, net.ID))
-				continue
-			}*/
-			if !(n.Network == net.GetId() || n.Network == net.GetName()) {
-				errMsgs = append(errMsgs, fmt.Sprintf("%v(%v): id/name not matched", net.Name, net.Id))
-			} else if !(net.IsPublic || net.ProjectId == d.Project) {
-				errMsgs = append(errMsgs, fmt.Sprintf("%v(%v): not owner (%v != %v)", net.Name, net.Id, net.ProjectId, d.Project))
-			} else if !(net.GetPorts() > 0 || isMigrate()) {
-				errMsgs = append(errMsgs, fmt.Sprintf("%v(%v): ports use up", net.Name, net.Id))
-			} else {
-				// add resource
-				reservedNetworks := 0
-				counter := counterOfNetwork(u, net.SNetwork, reservedNetworks)
-				if counter.GetCount() < int64(d.Count) {
-					errMsgs = append(errMsgs, fmt.Sprintf("%s: ports not enough, free: %d, required: %d", net.Name, counter.GetCount(), d.Count))
-					continue
-				}
-				p.SelectedNetworks.Store(net.Id, counter.GetCount())
-				counters.Add(counter)
-				return ""
-			}
-		}
-
-		if len(errMsgs) == 0 {
-			return predicates.ErrUnknown
-		}
-
-		return strings.Join(errMsgs, "; ")
-	}
-
-	filterBySpecifiedNetworks := func() {
-		counters := core.NewMinCounters()
-		var errMsgs []string
-
-		for _, n := range d.Networks {
-			if err_msg := isNetworkAvaliable(n, counters, networks); err_msg != "" {
-				errMsgs = append(errMsgs, err_msg)
-			}
-		}
-
-		if len(errMsgs) > 0 {
-			h.Exclude(strings.Join(errMsgs, ", "))
-		} else {
-			h.SetCapacityCounter(counters)
-		}
-	}
-
-	if len(d.Networks) == 0 {
-		filterByRandomNetwork()
-	} else {
-		filterBySpecifiedNetworks()
-	}
-
-	return h.GetResult()
-}

pkg/scheduler/algorithm/predicates/network_predicate.go

@@ -0,0 +1,280 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package predicates
+
+import (
+	"fmt"
+	"sync"
+
+	"yunion.io/x/pkg/util/netutils"
+	"yunion.io/x/pkg/util/sets"
+	"yunion.io/x/pkg/utils"
+
+	computeapi "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/compute/models"
+	"yunion.io/x/onecloud/pkg/scheduler/algorithm/plugin"
+	"yunion.io/x/onecloud/pkg/scheduler/api"
+	"yunion.io/x/onecloud/pkg/scheduler/core"
+	"yunion.io/x/onecloud/pkg/util/rbacutils"
+)
+
+// NetworkPredicate will filter the current network information with
+// the specified scheduling information to match, if not specified will
+// randomly match the available network resources.
+type NetworkPredicate struct {
+	BasePredicate
+	plugin.BasePlugin
+	SelectedNetworks sync.Map
+}
+
+func (p *NetworkPredicate) Name() string {
+	return "host_network"
+}
+
+func (p *NetworkPredicate) Clone() core.FitPredicate {
+	return &NetworkPredicate{}
+}
+
+func (p *NetworkPredicate) PreExecute(u *core.Unit, cs []core.Candidater) (bool, error) {
+	data := u.SchedData()
+	if len(data.HostId) > 0 && len(data.Networks) == 0 {
+		return false, nil
+	}
+
+	return true, nil
+}
+
+func (p *NetworkPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core.PredicateFailureReason, error) {
+	h := NewPredicateHelper(p, u, c)
+
+	getter := c.Getter()
+	networks := getter.Networks()
+
+	d := u.SchedData()
+
+	isMigrate := func() bool {
+		return len(d.HostId) > 0
+	}
+
+	counterOfNetwork := func(u *core.Unit, n *models.SNetwork, r int) core.Counter {
+		counter := u.CounterManager.GetOrCreate("net:"+n.Id, func() core.Counter {
+			return core.NewNormalCounter(int64(getter.GetFreePort(n.Id) - r))
+		})
+
+		u.SharedResourceManager.Add(n.GetId(), counter)
+		return counter
+	}
+
+	isMatchServerType := func(network *models.SNetwork) bool {
+		if d.Hypervisor == computeapi.HYPERVISOR_BAREMETAL {
+			return network.ServerType == computeapi.NETWORK_TYPE_BAREMETAL
+		}
+		return sets.NewString(
+			"", computeapi.NETWORK_TYPE_GUEST,
+			computeapi.NETWORK_TYPE_CONTAINER).Has(network.ServerType)
+	}
+
+	checkAddress := func(addr string, net *models.SNetwork) error {
+		if len(addr) == 0 {
+			return nil
+		}
+		ipAddr, err := netutils.NewIPV4Addr(addr)
+		if err != nil {
+			return fmt.Errorf("Invalid ip address %s: %v", addr, err)
+		}
+		if !net.GetIPRange().Contains(ipAddr) {
+			return fmt.Errorf("Address %s not in network %s range", addr, net.Name)
+		}
+		return nil
+	}
+
+	checkNetCount := func(net *models.SNetwork, reqCount int) (core.Counter, core.PredicateFailureReason) {
+		counter := counterOfNetwork(u, net, 0)
+		if counter.GetCount() < int64(reqCount) {
+			return nil, FailReason{
+				Reason: fmt.Sprintf("%s: ports not enough, free: %d, required: %d", net.Name, counter.GetCount(), d.Count),
+				Type:   NetworkFreeCount,
+			}
+		}
+		return counter, nil
+	}
+
+	isRandomNetworkAvailable := func(address, domain string, private bool, exit bool, wire string, counters core.MultiCounter) []core.PredicateFailureReason {
+		var fullErrMsgs []core.PredicateFailureReason
+		found := false
+
+		for _, n := range networks {
+			errMsgs := []core.PredicateFailureReason{}
+			appendError := func(msg core.PredicateFailureReason) {
+				errMsgs = append(errMsgs, msg)
+			}
+
+			if !isMatchServerType(n.SNetwork) {
+				appendError(FailReason{
+					Reason: fmt.Sprintf("Network %s type %s match", n.Name, n.ServerType),
+					Type:   NetworkTypeMatch,
+				})
+			}
+
+			if n.IsExitNetwork() != exit {
+				appendError(FailReason{Reason: ErrExitIsNotMatch})
+			}
+
+			if !(n.GetPorts() > 0 || isMigrate()) {
+				appendError(FailReason{
+					Reason: fmt.Sprintf("%v(%v): ports use up", n.Name, n.Id),
+					Type:   NetworkPort,
+				})
+			}
+
+			if wire != "" && !utils.HasPrefix(wire, n.WireId) && !utils.HasPrefix(wire, n.GetWire().GetName()) {
+				appendError(FailReason{
+					Reason: fmt.Sprintf("Wire %s != %s", wire, n.WireId),
+					Type:   NetworkWire,
+				})
+			}
+
+			schedData := u.SchedData()
+			if private {
+				if n.IsPublic {
+					appendError(FailReason{
+						Reason: fmt.Sprintf("Network %s is public", n.Name),
+						Type:   NetworkPublic,
+					})
+				} else if n.ProjectId != schedData.Project && utils.IsInStringArray(schedData.Project, n.GetSharedProjects()) {
+					appendError(FailReason{
+						Reason: fmt.Sprintf("Network project %s + %v not owner by %s", n.ProjectId, n.GetSharedProjects(), schedData.Project),
+						Type:   NetworkOwner,
+					})
+				}
+			} else {
+				if !n.IsPublic {
+					appendError(FailReason{Reason: fmt.Sprintf("Network %s is private", n.Name), Type: NetworkPrivate})
+				} else if rbacutils.TRbacScope(n.PublicScope) == rbacutils.ScopeDomain {
+					netDomain := n.DomainId
+					reqDomain := domain
+					if netDomain != reqDomain {
+						appendError(FailReason{Reason: fmt.Sprintf("Network domain scope %s not owner by %s", netDomain, reqDomain), Type: NetworkDomain})
+					}
+				}
+			}
+
+			if err := checkAddress(address, n.SNetwork); err != nil {
+				appendError(FailReason{Reason: err.Error(), Type: NetworkRange})
+			}
+
+			if len(errMsgs) == 0 {
+				// add resource
+				counter := counterOfNetwork(u, n.SNetwork, 0)
+				p.SelectedNetworks.Store(n.GetId(), counter.GetCount())
+				counters.Add(counter)
+				found = true
+			} else {
+				fullErrMsgs = append(fullErrMsgs, errMsgs...)
+			}
+		}
+
+		if counters.GetCount() < int64(d.Count) {
+			found = false
+			fullErrMsgs = append(fullErrMsgs, FailReason{
+				Reason: fmt.Sprintf("total random ports not enough, free: %d, required: %d", counters.GetCount(), d.Count),
+				Type:   NetworkFreeCount,
+			})
+		}
+
+		if !found {
+			return fullErrMsgs
+		}
+
+		return nil
+	}
+
+	filterByRandomNetwork := func() {
+		counters := core.NewCounters()
+		if errMsg := isRandomNetworkAvailable("", "", false, false, "", counters); len(errMsg) != 0 {
+			h.ExcludeByErrors(errMsg)
+		}
+		h.SetCapacityCounter(counters)
+	}
+
+	isNetworkAvaliable := func(n *computeapi.NetworkConfig, counters *core.MinCounters, networks []*api.CandidateNetwork) []core.PredicateFailureReason {
+		if len(networks) == 0 {
+			return []core.PredicateFailureReason{
+				FailReason{Reason: ErrNoAvailableNetwork},
+			}
+		}
+
+		if n.Network == "" {
+			counters0 := core.NewCounters()
+			retMsg := isRandomNetworkAvailable(n.Address, n.Domain, n.Private, n.Exit, n.Wire, counters0)
+			counters.Add(counters0)
+			return retMsg
+		}
+
+		errMsgs := make([]core.PredicateFailureReason, 0)
+
+		for _, net := range networks {
+			if !(n.Network == net.GetId() || n.Network == net.GetName()) {
+				errMsgs = append(errMsgs, &FailReason{
+					Reason: fmt.Sprintf("%v(%v): id/name not matched", net.Name, net.Id),
+					Type:   NetworkMatch,
+				})
+				continue
+			}
+			if !(net.GetPorts() > 0 || isMigrate()) {
+				errMsgs = append(errMsgs, &FailReason{
+					Reason: fmt.Sprintf("%v(%v): ports use up", net.Name, net.Id),
+					Type:   NetworkPort,
+				})
+				continue
+			}
+			counter, err := checkNetCount(net.SNetwork, d.Count)
+			if err != nil {
+				errMsgs = append(errMsgs, err)
+				continue
+			}
+			p.SelectedNetworks.Store(net.Id, counter.GetCount())
+			counters.Add(counter)
+			return nil
+		}
+
+		return nil
+	}
+
+	filterBySpecifiedNetworks := func() {
+		counters := core.NewMinCounters()
+		var errMsgs []core.PredicateFailureReason
+
+		for _, n := range d.Networks {
+			if errMsg := isNetworkAvaliable(n, counters, networks); len(errMsg) != 0 {
+				errMsgs = append(errMsgs, errMsg...)
+			}
+		}
+
+		if len(errMsgs) > 0 {
+			h.ExcludeByErrors(errMsgs)
+		} else {
+			h.SetCapacityCounter(counters)
+		}
+	}
+
+	if len(d.Networks) == 0 {
+		filterByRandomNetwork()
+	} else {
+		filterBySpecifiedNetworks()
+	}
+
+	return h.GetResult()
+}

pkg/scheduler/algorithm/predicates/network_schedtag_predicate.go

@@ -171,22 +171,6 @@ func (p *NetworkSchedtagPredicate) IsResourceFitInput(u *core.Unit, c core.Candi
 			}
 		}
 	}
-	free, err := network.GetFreeAddressCount()
-	if err != nil {
-		return &FailReason{
-			Reason: fmt.Sprintf("get free address count: %v", err),
-			Type:   NetworkFreeCount,
-		}
-	}
-	req := u.SchedData().Count
-	if free < req {
-		return &FailReason{
-			Reason: fmt.Sprintf("Network %s no free IPs, free %d, require %d", network.Name, free, req),
-			Type:   NetworkFreeCount,
-		}
-	}
-	h := NewPredicateHelper(p, u, c)
-	h.SetCapacity(int64(free))
 	return nil
 }
 

pkg/scheduler/algorithmprovider/baremetal.go

@@ -34,7 +34,7 @@ func baremetalPredicates() sets.String {
 		factory.RegisterFitPredicate("c-BaremetalCPUFilter", &predicatebm.CPUPredicate{}),
 		factory.RegisterFitPredicate("d-BaremetalMemoryFilter", &predicatebm.MemoryPredicate{}),
 		factory.RegisterFitPredicate("e-BaremetalStorageFilter", &predicatebm.StoragePredicate{}),
-		//factory.RegisterFitPredicate("f-BaremetalNetFilter", &predicatebm.NetworkPredicate{}),
+		factory.RegisterFitPredicate("f-BaremetalNetFilter", &predicates.NetworkPredicate{}),
 		factory.RegisterFitPredicate("g-BaremetalResourceTypeFilter", &predicates.ResourceTypePredicate{}),
 		factory.RegisterFitPredicate("h-DiskschedtagFilter", &predicates.DiskSchedtagPredicate{}),
 		factory.RegisterFitPredicate("i-NetschedtagFilter", &predicates.NetworkSchedtagPredicate{}),

pkg/scheduler/algorithmprovider/defaults.go

@@ -39,7 +39,7 @@ func defaultPredicates() sets.String {
 		factory.RegisterFitPredicate("g-GuestCPUFilter", &predicateguest.CPUPredicate{}),
 		factory.RegisterFitPredicate("h-GuestMemoryFilter", &predicateguest.MemoryPredicate{}),
 		factory.RegisterFitPredicate("i-GuestStorageFilter", &predicateguest.StoragePredicate{}),
-		//factory.RegisterFitPredicate("j-GuestNetworkFilter", &predicateguest.NetworkPredicate{}),
+		factory.RegisterFitPredicate("j-GuestNetworkFilter", &predicates.NetworkPredicate{}),
 		factory.RegisterFitPredicate("k-GuestIsolatedDeviceFilter", &predicateguest.IsolatedDevicePredicate{}),
 		factory.RegisterFitPredicate("l-GuestResourceTypeFilter", &predicates.ResourceTypePredicate{}),
 		factory.RegisterFitPredicate("m-GuestDiskschedtagFilter", &predicates.DiskSchedtagPredicate{}),

scripts/docker_push.sh

@@ -14,7 +14,7 @@ REGISTRY=${REGISTRY:-docker.io/yunion}
 TAG=${TAG:-latest}
 
 build_bin() {
-    CGO_ENABLED=0 make cmd/$1
+    make cmd/$1
 }
 
 build_image() {