Merge pull request #808 from swordqiu/feature/qj-keystone-support-phase1

Feature/qj keystone support phase1

Gopkg.lock

@@ -508,6 +508,14 @@
   revision = "5b77d2a35fb0ede96d138fc9a99f5c9b6aef11b4"
   version = "v1.7.0"
 
+[[projects]]
+  branch = "master"
+  digest = "1:660e6c59164572d31b7983efcddc8584a112fdf7cdd23b02417a1db16be2990d"
+  name = "github.com/fernet/fernet-go"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "9eac43b88a5efb8651d24de9b68e87567e029736"
+
 [[projects]]
   branch = "master"
   digest = "1:50a46ab1d5edbbdd55125b4d37f1bf503d0807c26461f9ad7b358d6006641d09"
@@ -1287,6 +1295,17 @@
   pruneopts = "UT"
   revision = "9c7f9b7a2bc3a520f7c7b30b34b7f85f47fe27b6"
 
+[[projects]]
+  digest = "1:01d5dc3bfb14cf8fb2c924dcf026231218604b8ed15daaa028875d49e7f09071"
+  name = "github.com/vmihailenco/msgpack"
+  packages = [
+    ".",
+    "codes",
+  ]
+  pruneopts = "UT"
+  revision = "c2fc210f30a2aca9db880cc017a92c169c999253"
+  version = "v4.0.4"
+
 [[projects]]
   digest = "1:506f2f0fee9b0e75b987286233a3b0d3fbf7cb145d754b0719b672822246cbc6"
   name = "github.com/vmware/govmomi"
@@ -1337,9 +1356,11 @@
 
 [[projects]]
   branch = "master"
-  digest = "1:bb52d5f63c8e08efb5bca4cf600f07690f75f3aa738e0bdfc25112385a2e674c"
+  digest = "1:cc72e3461310e6287e526d8611d91dabd5d69971cb24c995b95c3820d5242ff1"
   name = "golang.org/x/crypto"
   packages = [
+    "bcrypt",
+    "blowfish",
     "curve25519",
     "ed25519",
     "ed25519/internal/edwards25519",
@@ -1432,14 +1453,18 @@
   revision = "85acf8d2951cb2a3bde7632f9ff273ef0379bcbd"
 
 [[projects]]
-  digest = "1:a623ed4965cad144a32551cb009b9bad9ad247c22ca1afdd934fc85d0c891f93"
+  digest = "1:aee4de7ff868145d797de04621e3f7ae0d363244acd3df52bf5c49ad808f4ca8"
   name = "google.golang.org/appengine"
   packages = [
+    ".",
     "cloudsql",
+    "datastore",
     "internal",
+    "internal/app_identity",
     "internal/base",
     "internal/datastore",
     "internal/log",
+    "internal/modules",
     "internal/remote_api",
     "internal/urlfetch",
     "urlfetch",
@@ -1497,6 +1522,14 @@
   revision = "df014850f6dee74ba2fc94874043a9f3f75fbfd8"
   version = "v1.17.0"
 
+[[projects]]
+  digest = "1:af07c44dc04418be522bfd4e21ca9130d58169ea084e3a883e23772003a381c4"
+  name = "gopkg.in/asn1-ber.v1"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "f715ec2f112d1e4195b827ad68cf44017a3ef2b1"
+  version = "v1.3"
+
 [[projects]]
   digest = "1:2655728699af00d823b0a68bd7169fd241a30c04252f456b34421908701b9c3f"
   name = "gopkg.in/gin-gonic/gin.v1"
@@ -1521,6 +1554,14 @@
   revision = "d2d2541c53f18d2a059457998ce2876cc8e67cbf"
   version = "v0.9.1"
 
+[[projects]]
+  digest = "1:e9a0fa7c2dfc90e0fae16be5825ad98074d8704f5fcebfdc289a8e8fb0f8e4b5"
+  name = "gopkg.in/ldap.v3"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "9f0d712775a0973b7824a1585a86a4ea1d5263d9"
+  version = "v3.0.3"
+
 [[projects]]
   digest = "1:4d2e5a73dc1500038e504a8d78b986630e3626dc027bc030ba5c75da257cdb96"
   name = "gopkg.in/yaml.v2"
@@ -1705,26 +1746,26 @@
 
 [[projects]]
   branch = "master"
-  digest = "1:53c214ff8c2fd57db0962140b9a5e57b01b63066e7de7fe6897a633d15b78248"
+  digest = "1:dd1650e297358d848ef6dcee19dfc771c3797a4d4de4a35308776a6cb7ef4f55"
   name = "yunion.io/x/jsonutils"
   packages = ["."]
   pruneopts = "UT"
-  revision = "028b9007e455103eabacf3e6d71b5ce5060c3984"
+  revision = "c321606ff64b549fbcfc530f26f8b1858695918d"
 
 [[projects]]
   branch = "master"
-  digest = "1:20590bdd812710ad3257e92c23c7d19d9595a48ed69b200ac1d23ac76eee848e"
+  digest = "1:b41b7202c73bb184085e87e1373d2729f0a4fb5776c0b41376c14809159508af"
   name = "yunion.io/x/log"
   packages = [
     ".",
     "hooks",
   ]
   pruneopts = "UT"
-  revision = "3c4f24289a1b2fd39b94a9ccbd40931b3a20aab7"
+  revision = "04ce53b17c6b0db6eaff6bd810db2c6106232176"
 
 [[projects]]
   branch = "master"
-  digest = "1:777d274ba9317e4901e544a75138520b2fb0fd88d6c66fb446262824571835a8"
+  digest = "1:0fbe21d28e25757df5cfc5266c9c8d8183da6b1749dd20dfe06a94ba801db4fe"
   name = "yunion.io/x/pkg"
   packages = [
     "gotypes",
@@ -1758,23 +1799,23 @@
     "utils",
   ]
   pruneopts = "UT"
-  revision = "df1afe907f81bfa2adddae023d8577074784a964"
+  revision = "740ce1e70d2417c195f669bbddf8ade8ec967419"
 
 [[projects]]
   branch = "master"
-  digest = "1:ff4e73f338d767968f1a664f407877a9f4e1adc21ea938387a6e93130d23be4d"
+  digest = "1:66d19ef38985070b76f8ca9f9fc468e69e923bf331cb9bf4488d7997f2f4a7b0"
   name = "yunion.io/x/sqlchemy"
   packages = ["."]
   pruneopts = "UT"
-  revision = "b34ca088f7b65fa0e466e39daa9d99e01fcb073d"
+  revision = "b100553e017505920d398be5bb4bafe7914d6ef0"
 
 [[projects]]
   branch = "master"
-  digest = "1:cfa79b54c4ec8d1fe6b1599846b3975c50212f58114d807a6db69bbee5a16b7a"
+  digest = "1:505f3ac78ea58d528c24f804b58e03638515fb817bd0e07f4809bc0200316d33"
   name = "yunion.io/x/structarg"
   packages = ["."]
   pruneopts = "UT"
-  revision = "5b725f46d67547e7fb603b2118eddc5358ac2a39"
+  revision = "4a479a4597e8856097209f3e3b38b8e39fbbd9e1"
 
 [solve-meta]
   analyzer-name = "dep"
@@ -1834,6 +1875,7 @@
     "github.com/coredns/coredns/plugin/trace",
     "github.com/coredns/coredns/request",
     "github.com/fatih/color",
+    "github.com/fernet/fernet-go",
     "github.com/fsnotify/fsnotify",
     "github.com/go-sql-driver/mysql",
     "github.com/golang-plus/errors",
@@ -1855,6 +1897,7 @@
     "github.com/moul/http2curl",
     "github.com/nelsonken/cos-go-sdk-v5/cos",
     "github.com/pierrec/lz4",
+    "github.com/pkg/errors",
     "github.com/serialx/hashring",
     "github.com/shirou/gopsutil/cpu",
     "github.com/shirou/gopsutil/host",
@@ -1869,6 +1912,7 @@
     "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/http",
     "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/profile",
     "github.com/tredoe/osutil/user/crypt/sha512_crypt",
+    "github.com/vmihailenco/msgpack",
     "github.com/vmware/govmomi",
     "github.com/vmware/govmomi/object",
     "github.com/vmware/govmomi/property",
@@ -1878,11 +1922,13 @@
     "github.com/vmware/govmomi/vim25/mo",
     "github.com/vmware/govmomi/vim25/types",
     "go.etcd.io/etcd/clientv3",
+    "golang.org/x/crypto/bcrypt",
     "golang.org/x/crypto/ssh",
     "golang.org/x/net/bpf",
     "golang.org/x/net/ipv4",
     "golang.org/x/sys/unix",
     "gopkg.in/gin-gonic/gin.v1",
+    "gopkg.in/ldap.v3",
     "gopkg.in/yaml.v2",
     "k8s.io/api/core/v1",
     "k8s.io/apimachinery/pkg/api/errors",

Gopkg.toml

@@ -207,6 +207,15 @@
 [[constraint]]
   name="github.com/ceph/go-ceph"
   branch = "master"
+
 [[constraint]]
   branch = "master"
   name = "github.com/libvirt/libvirt-go-xml"
+
+[[constraint]]
+  version = "v3.0.3"
+  name = "gopkg.in/ldap.v3"
+
+[[constraint]]
+  branch = "master"
+  name = "github.com/fernet/fernet-go"

cmd/climc/shell/baremetalagents.go

@@ -64,4 +64,13 @@ func init() {
 		printObject(result)
 		return nil
 	})
+
+	R(&BaremetalAgentOpsOperations{}, "baremetal-agent-delete", "Delete baremetal agent", func(s *mcclient.ClientSession, args *BaremetalAgentOpsOperations) error {
+		result, err := modules.Baremetalagents.Delete(s, args.ID, nil)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
 }

cmd/climc/shell/disks.go

@@ -91,12 +91,12 @@ func init() {
 		return nil
 	})
 
-	R(&DiskDetailOptions{}, "disk-purge", "Delete a disk record in database, not actually do deletion", func(s *mcclient.ClientSession, args *DiskDetailOptions) error {
-		disk, e := modules.Disks.PerformAction(s, args.ID, "purge", nil)
-		if e != nil {
-			return e
-		}
-		printObject(disk)
+	type DiskBatchOpsOptions struct {
+		ID []string `help:"id list of disks to operate"`
+	}
+	R(&DiskBatchOpsOptions{}, "disk-purge", "Delete a disk record in database, not actually do deletion", func(s *mcclient.ClientSession, args *DiskBatchOpsOptions) error {
+		ret := modules.Disks.BatchPerformAction(s, args.ID, "purge", nil)
+		printBatchResults(ret, modules.Disks.GetColumns(s))
 		return nil
 	})
 

cmd/climc/shell/domains.go

@@ -19,6 +19,7 @@ import (
 
 	"yunion.io/x/jsonutils"
 
+	api "yunion.io/x/onecloud/pkg/apis/identity"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 )
@@ -111,103 +112,13 @@ func init() {
 	}) */
 
 	type DomainConfigLDAPOptions struct {
-		ID                      string   `help:"ID of domain to config"`
-		URL                     string   `help:"LDAP server URL"`
-		SUFFIX                  string   `help:"Suffix"`
-		QueryScope              string   `help:"Query scope, either one or sub" choices:"one|sub" default:"sub"`
-		PageSize                int      `help:"Page size, default 20" default:"20"`
-		User                    string   `help:"User"`
-		Password                string   `help:"Password"`
-		UserTreeDN              string   `help:"User tree distinguished name"`
-		UserFilter              string   `help:"user_filter"`
-		UserObjectclass         string   `help:"user_objectclass"`
-		UserIdAttribute         string   `help:"user_id_attribute"`
-		UserNameAttribute       string   `help:"user_name_attribute"`
-		UserEnabledAttribute    string   `help:"user_enabled_attribute"`
-		UserEnabledMask         int64    `help:"user_enabled_mask" default:"-1"`
-		UserEnabledDefault      string   `help:"user_enabled_default"`
-		UserEnabledInvert       string   `help:"user_enabled_invert" choices:"true|false"`
-		UserAdditionalAttribute []string `help:"user_additional_attribute_mapping"`
-		GroupTreeDN             string   `help:"User tree distinguished name"`
-		GroupFilter             string   `help:"group_filter"`
-		GroupObjectclass        string   `help:"group_objectclass"`
-		GroupIdAttribute        string   `help:"group_id_attribute"`
-		GroupNameAttribute      string   `help:"group_name_attribute"`
-		GroupMemberAttribute    string   `help:"group_member_attribute"`
-		GroupMembersAreIds      bool     `help:"group_members_are_ids"`
+		ID string `help:"ID of domain to config" json:"-"`
+		api.SDomainLDAPConfigOptions
 	}
 	R(&DomainConfigLDAPOptions{}, "domain-config-ldap", "Config a domain with LDAP driver", func(s *mcclient.ClientSession, args *DomainConfigLDAPOptions) error {
 		config := jsonutils.NewDict()
 		config.Add(jsonutils.NewString("ldap"), "config", "identity", "driver")
-		config.Add(jsonutils.NewString(args.URL), "config", "ldap", "url")
-		config.Add(jsonutils.NewString(args.SUFFIX), "config", "ldap", "suffix")
-		if len(args.QueryScope) > 0 {
-			config.Add(jsonutils.NewString(args.QueryScope), "config", "ldap", "query_scope")
-		}
-		if args.PageSize > 0 {
-			config.Add(jsonutils.NewInt(int64(args.PageSize)), "config", "ldap", "page_size")
-		}
-		if len(args.User) > 0 {
-			config.Add(jsonutils.NewString(args.User), "config", "ldap", "user")
-		}
-		if len(args.Password) > 0 {
-			config.Add(jsonutils.NewString(args.Password), "config", "ldap", "password")
-		}
-		if len(args.UserTreeDN) > 0 {
-			config.Add(jsonutils.NewString(args.UserTreeDN), "config", "ldap", "user_tree_dn")
-		}
-		if len(args.UserFilter) > 0 {
-			config.Add(jsonutils.NewString(args.UserFilter), "config", "ldap", "user_filter")
-		}
-		if len(args.UserObjectclass) > 0 {
-			config.Add(jsonutils.NewString(args.UserObjectclass), "config", "ldap", "user_objectclass")
-		}
-		if len(args.UserIdAttribute) > 0 {
-			config.Add(jsonutils.NewString(args.UserIdAttribute), "config", "ldap", "user_id_attribute")
-		}
-		if len(args.UserNameAttribute) > 0 {
-			config.Add(jsonutils.NewString(args.UserNameAttribute), "config", "ldap", "user_name_attribute")
-		}
-		if len(args.UserEnabledAttribute) > 0 {
-			config.Add(jsonutils.NewString(args.UserEnabledAttribute), "config", "ldap", "user_enabled_attribute")
-		}
-		if args.UserEnabledMask >= 0 {
-			config.Add(jsonutils.NewInt(int64(args.UserEnabledMask)), "config", "ldap", "user_enabled_mask")
-		}
-		if len(args.UserEnabledDefault) > 0 {
-			config.Add(jsonutils.NewString(args.UserEnabledDefault), "config", "ldap", "user_enabled_default")
-		}
-		if len(args.UserEnabledInvert) > 0 {
-			if args.UserEnabledInvert == "true" {
-				config.Add(jsonutils.JSONTrue, "config", "ldap", "user_enabled_invert")
-			} else {
-				config.Add(jsonutils.JSONFalse, "config", "ldap", "user_enabled_invert")
-			}
-		}
-		if len(args.UserAdditionalAttribute) > 0 {
-			config.Add(jsonutils.NewStringArray(args.UserAdditionalAttribute), "config", "ldap", "user_additional_attribute_mapping")
-		}
-		if len(args.GroupTreeDN) > 0 {
-			config.Add(jsonutils.NewString(args.GroupTreeDN), "config", "ldap", "group_tree_dn")
-		}
-		if len(args.GroupFilter) > 0 {
-			config.Add(jsonutils.NewString(args.GroupFilter), "config", "ldap", "group_filter")
-		}
-		if len(args.GroupObjectclass) > 0 {
-			config.Add(jsonutils.NewString(args.GroupObjectclass), "config", "ldap", "group_objectclass")
-		}
-		if len(args.GroupIdAttribute) > 0 {
-			config.Add(jsonutils.NewString(args.GroupIdAttribute), "config", "ldap", "group_id_attribute")
-		}
-		if len(args.GroupNameAttribute) > 0 {
-			config.Add(jsonutils.NewString(args.GroupNameAttribute), "config", "ldap", "group_name_attribute")
-		}
-		if len(args.GroupMemberAttribute) > 0 {
-			config.Add(jsonutils.NewString(args.GroupMemberAttribute), "config", "ldap", "group_member_attribute")
-		}
-		if args.GroupMembersAreIds {
-			config.Add(jsonutils.JSONTrue, "config", "ldap", "group_members_are_ids")
-		}
+		config.Add(jsonutils.Marshal(args), "config", "ldap")
 		objId, err := modules.Domains.GetId(s, args.ID, nil)
 		if err != nil {
 			return err

cmd/climc/shell/endpoints.go

@@ -30,10 +30,6 @@ func init() {
 		Interface string `help:"Search by interface"`
 	}
 	R(&EndpointListOptions{}, "endpoint-list", "List service endpoints", func(s *mcclient.ClientSession, args *EndpointListOptions) error {
-		mod, err := modules.GetModule(s, "endpoints")
-		if err != nil {
-			return err
-		}
 		query := jsonutils.NewDict()
 		if args.Limit > 0 {
 			query.Add(jsonutils.NewInt(args.Limit), "limit")
@@ -45,11 +41,7 @@ func init() {
 			query.Add(jsonutils.NewString(args.Region), "region_id")
 		}
 		if len(args.Service) > 0 {
-			srvMod, err := modules.GetModule(s, "services")
-			if err != nil {
-				return err
-			}
-			srvId, err := srvMod.GetId(s, args.Service, nil)
+			srvId, err := modules.ServicesV3.GetId(s, args.Service, nil)
 			if err != nil {
 				return err
 			}
@@ -58,11 +50,11 @@ func init() {
 		if len(args.Interface) > 0 {
 			query.Add(jsonutils.NewString(args.Interface), "interface")
 		}
-		result, err := mod.List(s, query)
+		result, err := modules.EndpointsV3.List(s, query)
 		if err != nil {
 			return err
 		}
-		printList(result, mod.GetColumns(s))
+		printList(result, modules.EndpointsV3.GetColumns(s))
 		return nil
 	})
 
@@ -70,11 +62,7 @@ func init() {
 		ID string `help:"ID or name of endpoints"`
 	}
 	R(&EndpointDetailOptions{}, "endpoint-show", "Show details of an endpoint", func(s *mcclient.ClientSession, args *EndpointDetailOptions) error {
-		mod, err := modules.GetModule(s, "endpoints")
-		if err != nil {
-			return err
-		}
-		ep, err := mod.Get(s, args.ID, nil)
+		ep, err := modules.EndpointsV3.Get(s, args.ID, nil)
 		if err != nil {
 			return err
 		}
@@ -83,11 +71,7 @@ func init() {
 	})
 
 	R(&EndpointDetailOptions{}, "endpoint-delete", "Delete an endpoint", func(s *mcclient.ClientSession, args *EndpointDetailOptions) error {
-		mod, err := modules.GetModule(s, "endpoints")
-		if err != nil {
-			return err
-		}
-		ep, err := mod.Delete(s, args.ID, nil)
+		ep, err := modules.EndpointsV3.Delete(s, args.ID, nil)
 		if err != nil {
 			return err
 		}

cmd/climc/shell/events.go

@@ -56,6 +56,10 @@ func doImageEventList(s *mcclient.ClientSession, args *EventListOptions) error {
 	return doEventList(modules.ImageLogs, s, args)
 }
 
+func doIdentityEventList(s *mcclient.ClientSession, args *EventListOptions) error {
+	return doEventList(modules.IdentityLogs, s, args)
+}
+
 func doEventList(man modules.ResourceManager, s *mcclient.ClientSession, args *EventListOptions) error {
 	params := jsonutils.NewDict()
 	if len(args.Type) > 0 {
@@ -167,4 +171,49 @@ func init() {
 		nargs := EventListOptions{BaseEventListOptions: args.BaseEventListOptions, Id: args.ID, Type: []string{"image"}}
 		return doImageEventList(s, &nargs)
 	})
+
+	R(&TypeEventListOptions{}, "user-event", "Show operation event logs of keystone users", func(s *mcclient.ClientSession, args *TypeEventListOptions) error {
+		nargs := EventListOptions{BaseEventListOptions: args.BaseEventListOptions, Id: args.ID, Type: []string{"user"}}
+		return doIdentityEventList(s, &nargs)
+	})
+
+	R(&TypeEventListOptions{}, "group-event", "Show operation event logs of keystone groups", func(s *mcclient.ClientSession, args *TypeEventListOptions) error {
+		nargs := EventListOptions{BaseEventListOptions: args.BaseEventListOptions, Id: args.ID, Type: []string{"group"}}
+		return doIdentityEventList(s, &nargs)
+	})
+
+	R(&TypeEventListOptions{}, "domain-event", "Show operation event logs of keystone domains", func(s *mcclient.ClientSession, args *TypeEventListOptions) error {
+		nargs := EventListOptions{BaseEventListOptions: args.BaseEventListOptions, Id: args.ID, Type: []string{"domain"}}
+		return doIdentityEventList(s, &nargs)
+	})
+
+	R(&TypeEventListOptions{}, "project-event", "Show operation event logs of keystone projects", func(s *mcclient.ClientSession, args *TypeEventListOptions) error {
+		nargs := EventListOptions{BaseEventListOptions: args.BaseEventListOptions, Id: args.ID, Type: []string{"project"}}
+		return doIdentityEventList(s, &nargs)
+	})
+
+	R(&TypeEventListOptions{}, "role-event", "Show operation event logs of keystone roles", func(s *mcclient.ClientSession, args *TypeEventListOptions) error {
+		nargs := EventListOptions{BaseEventListOptions: args.BaseEventListOptions, Id: args.ID, Type: []string{"role"}}
+		return doIdentityEventList(s, &nargs)
+	})
+
+	R(&TypeEventListOptions{}, "policy-event", "Show operation event logs of keystone policies", func(s *mcclient.ClientSession, args *TypeEventListOptions) error {
+		nargs := EventListOptions{BaseEventListOptions: args.BaseEventListOptions, Id: args.ID, Type: []string{"policy"}}
+		return doIdentityEventList(s, &nargs)
+	})
+
+	R(&TypeEventListOptions{}, "endpoint-event", "Show operation event logs of keystone endpoints", func(s *mcclient.ClientSession, args *TypeEventListOptions) error {
+		nargs := EventListOptions{BaseEventListOptions: args.BaseEventListOptions, Id: args.ID, Type: []string{"endpoint"}}
+		return doIdentityEventList(s, &nargs)
+	})
+
+	R(&TypeEventListOptions{}, "service-event", "Show operation event logs of keystone services", func(s *mcclient.ClientSession, args *TypeEventListOptions) error {
+		nargs := EventListOptions{BaseEventListOptions: args.BaseEventListOptions, Id: args.ID, Type: []string{"service"}}
+		return doIdentityEventList(s, &nargs)
+	})
+
+	R(&TypeEventListOptions{}, "credential-event", "Show operation event logs of keystone credentials", func(s *mcclient.ClientSession, args *TypeEventListOptions) error {
+		nargs := EventListOptions{BaseEventListOptions: args.BaseEventListOptions, Id: args.ID, Type: []string{"credential"}}
+		return doIdentityEventList(s, &nargs)
+	})
 }

cmd/climc/shell/groups.go

@@ -23,6 +23,7 @@ import (
 
 func init() {
 	type GroupListOptions struct {
+		Admin  bool   `help:"admin mode"`
 		Name   string `help:"Name of the groups to filter"`
 		Domain string `help:"Domain to filter"`
 		Limit  int64  `help:"Items per page" default:"20"`
@@ -40,6 +41,10 @@ func init() {
 				return e
 			}
 			params.Add(jsonutils.NewString(domainId), "domain_id")
+			params.Add(jsonutils.JSONTrue, "admin")
+		}
+		if args.Admin {
+			params.Add(jsonutils.JSONTrue, "admin")
 		}
 		if args.Limit > 0 {
 			params.Add(jsonutils.NewInt(args.Limit), "limit")

cmd/climc/shell/nodealert.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package shell
 
 import (

cmd/climc/shell/policies.go

@@ -62,8 +62,10 @@ func init() {
 	})
 
 	type PolicyCreateOptions struct {
-		TYPE string `help:"type of the policy"`
-		FILE string `help:"path to policy file"`
+		TYPE     string `help:"type of the policy"`
+		FILE     string `help:"path to policy file"`
+		Enabled  bool   `help:"create policy enabled"`
+		Disabled bool   `help:"create policy disabled"`
 	}
 	R(&PolicyCreateOptions{}, "policy-create", "Create a new policy", func(s *mcclient.ClientSession, args *PolicyCreateOptions) error {
 		policyBytes, err := ioutil.ReadFile(args.FILE)
@@ -74,6 +76,11 @@ func init() {
 		params := jsonutils.NewDict()
 		params.Add(jsonutils.NewString(args.TYPE), "type")
 		params.Add(jsonutils.NewString(string(policyBytes)), "policy")
+		if args.Enabled {
+			params.Add(jsonutils.JSONTrue, "enabled")
+		} else if args.Disabled {
+			params.Add(jsonutils.JSONFalse, "enabled")
+		}
 
 		result, err := modules.Policies.Create(s, params)
 		if err != nil {
@@ -86,9 +93,11 @@ func init() {
 	})
 
 	type PolicyPatchOptions struct {
-		ID   string `help:"ID of policy"`
-		File string `help:"path to policy file"`
-		Type string `help:"policy type"`
+		ID       string `help:"ID of policy"`
+		File     string `help:"path to policy file"`
+		Type     string `help:"policy type"`
+		Enabled  bool   `help:"update policy enabled"`
+		Disabled bool   `help:"update policy disabled"`
 	}
 	R(&PolicyPatchOptions{}, "policy-patch", "Patch policy", func(s *mcclient.ClientSession, args *PolicyPatchOptions) error {
 		policyId, err := modules.Policies.GetId(s, args.ID, nil)
@@ -106,6 +115,11 @@ func init() {
 			}
 			params.Add(jsonutils.NewString(string(policyBytes)), "policy")
 		}
+		if args.Enabled {
+			params.Add(jsonutils.JSONTrue, "enabled")
+		} else if args.Disabled {
+			params.Add(jsonutils.JSONFalse, "enabled")
+		}
 		result, err := modules.Policies.Patch(s, policyId, params)
 		if err != nil {
 			return err

cmd/climc/shell/projects.go

@@ -154,7 +154,7 @@ func init() {
 		if args.Enabled && !args.Disabled {
 			params.Add(jsonutils.JSONTrue, "enabled")
 		} else if !args.Enabled && args.Disabled {
-			params.Add(jsonutils.JSONTrue, "disabled")
+			params.Add(jsonutils.JSONFalse, "enabled")
 		}
 		if len(args.Desc) > 0 {
 			params.Add(jsonutils.NewString(args.Desc), "description")

cmd/climc/shell/roles.go

@@ -30,10 +30,6 @@ func init() {
 		Search string `help:"search text"`
 	}
 	R(&RoleListOptions{}, "role-list", "List keystone Roles", func(s *mcclient.ClientSession, args *RoleListOptions) error {
-		mod, err := modules.GetModule(s, "roles")
-		if err != nil {
-			return err
-		}
 		query := jsonutils.NewDict()
 		if len(args.Name) > 0 {
 			query.Add(jsonutils.NewString(args.Name), "name")
@@ -54,11 +50,11 @@ func init() {
 		if len(args.Search) > 0 {
 			query.Add(jsonutils.NewString(args.Search), "name__icontains")
 		}
-		result, err := mod.List(s, query)
+		result, err := modules.RolesV3.List(s, query)
 		if err != nil {
 			return err
 		}
-		printList(result, mod.GetColumns(s))
+		printList(result, modules.RolesV3.GetColumns(s))
 		return nil
 	})
 
@@ -67,10 +63,6 @@ func init() {
 		Domain string `help:"Domain"`
 	}
 	R(&RoleDetailOptions{}, "role-show", "Show details of a role", func(s *mcclient.ClientSession, args *RoleDetailOptions) error {
-		mod, err := modules.GetModule(s, "roles")
-		if err != nil {
-			return err
-		}
 		query := jsonutils.NewDict()
 		if len(args.Domain) > 0 {
 			domainId, err := modules.Domains.GetId(s, args.Domain, nil)
@@ -79,7 +71,7 @@ func init() {
 			}
 			query.Add(jsonutils.NewString(domainId), "domain_id")
 		}
-		role, err := mod.Get(s, args.ID, query)
+		role, err := modules.RolesV3.Get(s, args.ID, query)
 		if err != nil {
 			return err
 		}
@@ -95,15 +87,11 @@ func init() {
 			}
 			query.Add(jsonutils.NewString(domainId), "domain_id")
 		}
-		mod, err := modules.GetModule(s, "roles")
+		rid, err := modules.RolesV3.GetId(s, args.ID, query)
 		if err != nil {
 			return err
 		}
-		rid, err := mod.GetId(s, args.ID, query)
-		if err != nil {
-			return err
-		}
-		role, err := mod.Delete(s, rid, nil)
+		role, err := modules.RolesV3.Delete(s, rid, nil)
 		if err != nil {
 			return err
 		}
@@ -129,11 +117,7 @@ func init() {
 		if len(args.Desc) > 0 {
 			params.Add(jsonutils.NewString(args.Desc), "description")
 		}
-		mod, err := modules.GetModule(s, "roles")
-		if err != nil {
-			return err
-		}
-		role, err := mod.Create(s, params)
+		role, err := modules.RolesV3.Create(s, params)
 		if err != nil {
 			return err
 		}
@@ -156,11 +140,7 @@ func init() {
 			}
 			query.Add(jsonutils.NewString(domainId), "domain_id")
 		}
-		mod, err := modules.GetModule(s, "roles")
-		if err != nil {
-			return err
-		}
-		rid, err := mod.GetId(s, args.ID, query)
+		rid, err := modules.RolesV3.GetId(s, args.ID, query)
 		if err != nil {
 			return err
 		}
@@ -171,7 +151,7 @@ func init() {
 		if len(args.Desc) > 0 {
 			params.Add(jsonutils.NewString(args.Desc), "description")
 		}
-		role, err := mod.Patch(s, rid, params)
+		role, err := modules.RolesV3.Patch(s, rid, params)
 		if err != nil {
 			return err
 		}

cmd/climc/shell/services.go

@@ -27,12 +27,9 @@ func init() {
 		Offset int64  `help:"Offset, default 0, i.e. no offset"`
 		Name   string `help:"Search by name"`
 		Type   string `help:"Search by type"`
+		Search string `help:"search any fields"`
 	}
 	R(&ServiceListOptions{}, "service-list", "List services", func(s *mcclient.ClientSession, args *ServiceListOptions) error {
-		mod, err := modules.GetModule(s, "services")
-		if err != nil {
-			return err
-		}
 		query := jsonutils.NewDict()
 		if args.Limit > 0 {
 			query.Add(jsonutils.NewInt(args.Limit), "limit")
@@ -46,11 +43,14 @@ func init() {
 		if len(args.Type) > 0 {
 			query.Add(jsonutils.NewString(args.Type), "type__icontains")
 		}
-		result, err := mod.List(s, query)
+		if len(args.Search) > 0 {
+			query.Add(jsonutils.NewString(args.Search), "search")
+		}
+		result, err := modules.ServicesV3.List(s, query)
 		if err != nil {
 			return err
 		}
-		printList(result, mod.GetColumns(s))
+		printList(result, modules.ServicesV3.GetColumns(s))
 		return nil
 	})
 
@@ -58,15 +58,11 @@ func init() {
 		ID string `help:"ID of service"`
 	}
 	R(&ServiceShowOptions{}, "service-show", "Show details of a service", func(s *mcclient.ClientSession, args *ServiceShowOptions) error {
-		mod, err := modules.GetModule(s, "services")
+		srvId, err := modules.ServicesV3.GetId(s, args.ID, nil)
 		if err != nil {
 			return err
 		}
-		srvId, err := mod.GetId(s, args.ID, nil)
-		if err != nil {
-			return err
-		}
-		result, err := mod.Get(s, srvId, nil)
+		result, err := modules.ServicesV3.Get(s, srvId, nil)
 		if err != nil {
 			return err
 		}
@@ -74,15 +70,11 @@ func init() {
 		return nil
 	})
 	R(&ServiceShowOptions{}, "service-delete", "Delete a service", func(s *mcclient.ClientSession, args *ServiceShowOptions) error {
-		mod, err := modules.GetModule(s, "services")
+		srvId, err := modules.ServicesV3.GetId(s, args.ID, nil)
 		if err != nil {
 			return err
 		}
-		srvId, err := mod.GetId(s, args.ID, nil)
-		if err != nil {
-			return err
-		}
-		result, err := mod.Delete(s, srvId, nil)
+		result, err := modules.ServicesV3.Delete(s, srvId, nil)
 		if err != nil {
 			return err
 		}
@@ -109,11 +101,7 @@ func init() {
 		} else if !args.Enabled && args.Disabled {
 			params.Add(jsonutils.JSONFalse, "enabled")
 		}
-		mod, err := modules.GetModule(s, "services")
-		if err != nil {
-			return err
-		}
-		srv, err := mod.Create(s, params)
+		srv, err := modules.ServicesV3.Create(s, params)
 		if err != nil {
 			return err
 		}
@@ -130,11 +118,7 @@ func init() {
 		Disabled bool   `help:"Disabled"`
 	}
 	R(&ServiceUpdateOptions{}, "service-update", "Update a service", func(s *mcclient.ClientSession, args *ServiceUpdateOptions) error {
-		mod, err := modules.GetModule(s, "services")
-		if err != nil {
-			return err
-		}
-		srvId, err := mod.GetId(s, args.ID, nil)
+		srvId, err := modules.ServicesV3.GetId(s, args.ID, nil)
 		if err != nil {
 			return err
 		}
@@ -153,7 +137,7 @@ func init() {
 		} else if !args.Enabled && args.Disabled {
 			params.Add(jsonutils.JSONFalse, "enabled")
 		}
-		srv, err := mod.Patch(s, srvId, params)
+		srv, err := modules.ServicesV3.Patch(s, srvId, params)
 		if err != nil {
 			return err
 		}

cmd/climc/shell/usages.go

@@ -143,4 +143,15 @@ func init() {
 		printObject(result)
 		return nil
 	})
+
+	type IdentityUsageOptions struct {
+	}
+	R(&IdentityUsageOptions{}, "identity-usage", "Show general usage of identity", func(s *mcclient.ClientSession, args *IdentityUsageOptions) error {
+		result, err := modules.IdentityUsages.GetUsage(s, nil)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
 }

cmd/climc/shell/users.go

@@ -23,6 +23,7 @@ import (
 
 func init() {
 	type UserListOptions struct {
+		Admin            bool   `help:"admin mode"`
 		Domain           string `help:"Filter by domain"`
 		Name             string `help:"Filter by name"`
 		Limit            int64  `help:"Limit, default 0, i.e. no limit"`
@@ -32,10 +33,6 @@ func init() {
 		NoDefaultProject bool   `help:"Filter users without valid default_project_id"`
 	}
 	R(&UserListOptions{}, "user-list", "List users", func(s *mcclient.ClientSession, args *UserListOptions) error {
-		mod, err := modules.GetModule(s, "users")
-		if err != nil {
-			return err
-		}
 		params := jsonutils.NewDict()
 		if len(args.Domain) > 0 {
 			domainId, err := modules.Domains.GetId(s, args.Domain, nil)
@@ -43,6 +40,10 @@ func init() {
 				return err
 			}
 			params.Add(jsonutils.NewString(domainId), "domain_id")
+			params.Add(jsonutils.JSONTrue, "admin")
+		}
+		if args.Admin {
+			params.Add(jsonutils.JSONTrue, "admin")
 		}
 		if len(args.Name) > 0 {
 			params.Add(jsonutils.NewString(args.Name), "name")
@@ -65,11 +66,11 @@ func init() {
 		} else if args.NoDefaultProject {
 			params.Add(jsonutils.NewString(""), "default_project_id__iempty")
 		}
-		result, err := mod.List(s, params)
+		result, err := modules.UsersV3.List(s, params)
 		if err != nil {
 			return err
 		}
-		printList(result, mod.GetColumns(s))
+		printList(result, modules.UsersV3.GetColumns(s))
 		return nil
 	})
 
@@ -78,10 +79,6 @@ func init() {
 		Domain string `help:"Domain"`
 	}
 	R(&UserDetailOptions{}, "user-show", "Show details of user", func(s *mcclient.ClientSession, args *UserDetailOptions) error {
-		mod, e := modules.GetModule(s, "users")
-		if e != nil {
-			return e
-		}
 		query := jsonutils.NewDict()
 		if len(args.Domain) > 0 {
 			domainId, err := modules.Domains.GetId(s, args.Domain, nil)
@@ -90,7 +87,7 @@ func init() {
 			}
 			query.Add(jsonutils.NewString(domainId), "domain_id")
 		}
-		user, e := mod.Get(s, args.ID, query)
+		user, e := modules.UsersV3.Get(s, args.ID, query)
 		if e != nil {
 			return e
 		}
@@ -98,10 +95,6 @@ func init() {
 		return nil
 	})
 	R(&UserDetailOptions{}, "user-delete", "Delete user", func(s *mcclient.ClientSession, args *UserDetailOptions) error {
-		mod, e := modules.GetModule(s, "users")
-		if e != nil {
-			return e
-		}
 		query := jsonutils.NewDict()
 		if len(args.Domain) > 0 {
 			domainId, err := modules.Domains.GetId(s, args.Domain, nil)
@@ -110,11 +103,11 @@ func init() {
 			}
 			query.Add(jsonutils.NewString(domainId), "domain_id")
 		}
-		uid, e := mod.GetId(s, args.ID, query)
+		uid, e := modules.UsersV3.GetId(s, args.ID, query)
 		if e != nil {
 			return e
 		}
-		_, e = mod.Delete(s, uid, nil)
+		_, e = modules.UsersV3.Delete(s, uid, nil)
 		if e != nil {
 			return e
 		}
@@ -155,11 +148,11 @@ func init() {
 		if err != nil {
 			return err
 		}
-		projects, e := modules.UsersV3.GetGroups(s, uid)
+		groups, e := modules.UsersV3.GetGroups(s, uid)
 		if e != nil {
 			return e
 		}
-		printList(projects, modules.Groups.GetColumns(s))
+		printList(groups, modules.Groups.GetColumns(s))
 		return nil
 	})
 
@@ -195,10 +188,6 @@ func init() {
 		DefaultProject string `help:"Default project"`
 	}
 	R(&UserCreateOptions{}, "user-create", "Create a user", func(s *mcclient.ClientSession, args *UserCreateOptions) error {
-		mod, err := modules.GetModule(s, "users")
-		if err != nil {
-			return err
-		}
 		params := jsonutils.NewDict()
 		params.Add(jsonutils.NewString(args.NAME), "name")
 		if len(args.Domain) > 0 {
@@ -237,7 +226,7 @@ func init() {
 			params.Add(jsonutils.NewString(projId), "default_project_id")
 		}
 
-		user, err := mod.Create(s, params)
+		user, err := modules.UsersV3.Create(s, params)
 		if err != nil {
 			return err
 		}

cmd/cryptool/shell/fernet.go

@@ -0,0 +1,129 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package shell
+
+import (
+	"fmt"
+	"time"
+
+	"yunion.io/x/pkg/util/timeutils"
+
+	"yunion.io/x/onecloud/pkg/keystone/tokens"
+	"yunion.io/x/onecloud/pkg/util/fernetool"
+	"yunion.io/x/onecloud/pkg/util/shellutils"
+)
+
+func init() {
+	type FernetInitKeysOptions struct {
+		PATH  string `help:"path that stores fernet keys"`
+		COUNT int    `help:"number of keys to init"`
+	}
+	shellutils.R(&FernetInitKeysOptions{}, "fernet-initkeys", "Initialze fernet keys", func(args *FernetInitKeysOptions) error {
+		fm := fernetool.SFernetKeyManager{}
+		err := fm.InitKeys(args.PATH, args.COUNT)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+
+	type FernetEncryptOptions struct {
+		PATH string `help:"path that stores fernet keys"`
+		MSG  string `help:"message to encrypt"`
+	}
+	shellutils.R(&FernetEncryptOptions{}, "fernet-encrypt", "Encrypt message with fernet keys", func(args *FernetEncryptOptions) error {
+		fm := fernetool.SFernetKeyManager{}
+		err := fm.LoadKeys(args.PATH)
+		if err != nil {
+			return err
+		}
+		ret, err := fm.Encrypt([]byte(args.MSG))
+		if err != nil {
+			return err
+		}
+		fmt.Println(string(ret))
+		return nil
+	})
+
+	type FernetEncryptTokenOptions struct {
+		PATH      string `help:"path that stores fernet keys"`
+		USERID    string `help:"UserId"`
+		METHOD    string `help:"auth method" choices:"password|token"`
+		EXPIREAT  string `help:"expired time"`
+		ProjectId string `help:"project Id"`
+		DomainId  string `help:"domainId"`
+		AUDITID   string `help:"audit ID"`
+	}
+	shellutils.R(&FernetEncryptTokenOptions{}, "fernet-encrypt-token", "Encrypt auth token with fernet keys", func(args *FernetEncryptTokenOptions) error {
+		token := tokens.SAuthToken{}
+		token.UserId = args.USERID
+		token.Method = args.METHOD
+		token.ProjectId = args.ProjectId
+		token.DomainId = args.DomainId
+		token.ExpiresAt, _ = timeutils.ParseFullIsoTime(args.EXPIREAT)
+		token.AuditIds = []string{args.AUDITID}
+
+		tk, err := token.Encode()
+		if err != nil {
+			return err
+		}
+
+		fmt.Println(len(tk))
+		fmt.Println(string(tk))
+		fmt.Println(tk)
+		fmt.Printf("%x\n", tk)
+
+		fm := fernetool.SFernetKeyManager{}
+		err = fm.LoadKeys(args.PATH)
+		if err != nil {
+			return err
+		}
+		ret, err := fm.Encrypt(tk)
+		if err != nil {
+			return err
+		}
+		fmt.Println(string(ret))
+		return nil
+	})
+
+	shellutils.R(&FernetEncryptOptions{}, "fernet-decrypt", "Decrypt message with fernet keys", func(args *FernetEncryptOptions) error {
+		fm := fernetool.SFernetKeyManager{}
+		if args.PATH == "empty" {
+			fm.InitEmpty()
+		} else {
+			err := fm.LoadKeys(args.PATH)
+			if err != nil {
+				return err
+			}
+		}
+		fmt.Println("primary key hash:", fm.PrimaryKeyHash())
+
+		ret := fm.Decrypt([]byte(args.MSG), time.Hour*-1)
+		if len(ret) == 0 {
+			return fmt.Errorf("invalid message")
+		}
+		fmt.Println(len(ret))
+		fmt.Println(string(ret))
+		fmt.Println(ret)
+		fmt.Printf("%x\n", ret)
+		token := tokens.SAuthToken{}
+		err := token.Decode(ret)
+		if err != nil {
+			return err
+		}
+		fmt.Println(token)
+		return nil
+	})
+}

cmd/keystone/main.go

@@ -0,0 +1,23 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"yunion.io/x/onecloud/pkg/keystone/service"
+)
+
+func main() {
+	service.StartService()
+}

cmd/ldapcli/main.go

@@ -0,0 +1,135 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"yunion.io/x/log"
+	"yunion.io/x/structarg"
+
+	_ "yunion.io/x/onecloud/cmd/ldapcli/shell"
+	"yunion.io/x/onecloud/pkg/util/ldaputils"
+	"yunion.io/x/onecloud/pkg/util/shellutils"
+)
+
+type BaseOptions struct {
+	Debug    bool   `help:"debug mode"`
+	Help     bool   `help:"Show help"`
+	Url      string `help:"ldap url, like ldap://10.168.222.23:389 or ldaps://10.168.222.23:389" default:"$LDAP_URL"`
+	Account  string `help:"LDAP Account" default:"$LDAP_ACCOUNT"`
+	Password string `help:"LDAP password" default:"$LDAP_PASSWORD"`
+	BaseDN   string `help:"LDAP base DN" default:"$LDAP_BASEDN"`
+
+	SUBCOMMAND string `help:"aliyuncli subcommand" subcommand:"true"`
+}
+
+func getSubcommandParser() (*structarg.ArgumentParser, error) {
+	parse, e := structarg.NewArgumentParser(&BaseOptions{},
+		"ldapcli",
+		"Command-line tool for LDAP API.",
+		`See "ldapcli help COMMAND" for help on a specific command.`)
+
+	if e != nil {
+		return nil, e
+	}
+
+	subcmd := parse.GetSubcommand()
+	if subcmd == nil {
+		return nil, fmt.Errorf("No subcommand argument.")
+	}
+	type HelpOptions struct {
+		SUBCOMMAND string `help:"sub-command name"`
+	}
+	shellutils.R(&HelpOptions{}, "help", "Show help of a subcommand", func(args *HelpOptions) error {
+		helpstr, e := subcmd.SubHelpString(args.SUBCOMMAND)
+		if e != nil {
+			return e
+		} else {
+			fmt.Print(helpstr)
+			return nil
+		}
+	})
+	for _, v := range shellutils.CommandTable {
+		_, e := subcmd.AddSubParser(v.Options, v.Command, v.Desc, v.Callback)
+		if e != nil {
+			return nil, e
+		}
+	}
+	return parse, nil
+}
+
+func showErrorAndExit(e error) {
+	log.Errorf("%s", e)
+	os.Exit(1)
+}
+
+func newClient(options *BaseOptions) (*ldaputils.SLDAPClient, error) {
+	if len(options.Url) == 0 {
+		return nil, fmt.Errorf("Missing ldap URL")
+	}
+	if len(options.BaseDN) == 0 {
+		return nil, fmt.Errorf("Missing BaseDN, e.g. DC=example,DC=com")
+	}
+
+	cli := ldaputils.NewLDAPClient(options.Url, options.Account, options.Password, options.BaseDN, options.Debug)
+
+	err := cli.Connect()
+	if err != nil {
+		log.Errorf("connect fail %s", err)
+		return nil, err
+	}
+	return cli, nil
+}
+
+func main() {
+	parser, e := getSubcommandParser()
+	if e != nil {
+		showErrorAndExit(e)
+	}
+	e = parser.ParseArgs(os.Args[1:], false)
+	options := parser.Options().(*BaseOptions)
+
+	if options.Help {
+		fmt.Print(parser.HelpString())
+	} else {
+		subcmd := parser.GetSubcommand()
+		subparser := subcmd.GetSubParser()
+		if e != nil {
+			if subparser != nil {
+				fmt.Print(subparser.Usage())
+			} else {
+				fmt.Print(parser.Usage())
+			}
+			showErrorAndExit(e)
+		} else {
+			var cli *ldaputils.SLDAPClient
+			suboptions := subparser.Options()
+			if options.SUBCOMMAND == "help" {
+				e = subcmd.Invoke(suboptions)
+			} else {
+				cli, e = newClient(options)
+				if e != nil {
+					showErrorAndExit(e)
+				}
+				e = subcmd.Invoke(cli, suboptions)
+			}
+			if e != nil {
+				showErrorAndExit(e)
+			}
+		}
+	}
+}

cmd/ldapcli/shell/search.go

@@ -0,0 +1,66 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package shell
+
+import (
+	"fmt"
+	"strings"
+	"yunion.io/x/onecloud/pkg/util/ldaputils"
+	"yunion.io/x/onecloud/pkg/util/shellutils"
+)
+
+func init() {
+	type LdapSearchOptions struct {
+		Base        string   `help:"base DN, e.g. OU=tech,DC=example,DC=com"`
+		Objectclass string   `help:"objectclass, e.g. organizationalPerson"`
+		Search      []string `help:"search conditions, in format of field:value"`
+		Field       []string `help:"retrieve field info"`
+	}
+	shellutils.R(&LdapSearchOptions{}, "search", "search ldap", func(cli *ldaputils.SLDAPClient, args *LdapSearchOptions) error {
+		search := make(map[string]string)
+		for _, s := range args.Search {
+			colonPos := strings.IndexByte(s, ':')
+			if colonPos <= 0 {
+				return fmt.Errorf("invalid search condition %s", s)
+			}
+			search[s[:colonPos]] = s[(colonPos + 1):]
+		}
+		entries, err := cli.Search(args.Base, args.Objectclass, search, args.Field)
+		if err != nil {
+			return err
+		}
+		for _, entry := range entries {
+			entry.PrettyPrint(2)
+		}
+		return nil
+	})
+
+	type LdapAuthOptions struct {
+		Base        string   `help:"base DN, e.g. OU=tech,DC=example,DC=com"`
+		Objectclass string   `help:"objectclass, e.g. organizationalPerson"`
+		ATTR        string   `help:"account attribute name"`
+		ACCOUNT     string   `help:"account name to auth"`
+		PASSWORD    string   `help:"Password to auth"`
+		Field       []string `help:"retrieve field info"`
+	}
+	shellutils.R(&LdapAuthOptions{}, "auth", "authenticate against ldap", func(cli *ldaputils.SLDAPClient, args *LdapAuthOptions) error {
+		entry, err := cli.Authenticate(args.Base, args.Objectclass, args.ATTR, args.ACCOUNT, args.PASSWORD, args.Field)
+		if err != nil {
+			return err
+		}
+		entry.PrettyPrint(2)
+		return nil
+	})
+}

cmd/ucloudcli/main.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package main
 
 import (

pkg/apis/billing/billing_const.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package billing
 
 const (

pkg/apis/billing/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package billing // import "yunion.io/x/onecloud/pkg/apis/billing"

pkg/apis/compute/agent_const.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package compute
 
 const (

pkg/apis/compute/cachedimages_const.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package compute
 
 const (

pkg/apis/compute/cloudaccount_const.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package compute
 
 const (

pkg/apis/compute/cloudregions_const.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package compute
 
 const (

pkg/apis/compute/elasticips_const.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package compute
 
 const (

pkg/apis/compute/esxi_const.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package compute
 
 const (

pkg/apis/compute/guest_import_api.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package compute
 
 type SImportNic struct {

pkg/apis/compute/sku_const.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package compute
 
 const (

pkg/apis/compute/snapshot.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package compute
 
 import "yunion.io/x/onecloud/pkg/apis"

pkg/apis/compute/snapshot_const.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package compute
 
 const (

pkg/apis/compute/vpcs_const.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package compute
 
 const (

pkg/apis/compute/zones_const.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package compute
 
 const (

pkg/apis/identity/config.go

@@ -0,0 +1,48 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package identity
+
+const (
+	QueryScopeOne = "one"
+	QUeryScopeSub = "sub"
+)
+
+type SDomainLDAPConfigOptions struct {
+	Url        string `json:"url,omitempty" help:"LDAP server URL" required:"true"`
+	Suffix     string `json:"suffix,omitempty" required:"true"`
+	QueryScope string `json:"query_scope,omitempty" help:"Query scope, either one or sub" choices:"one|sub" default:"sub"`
+	PageSize   int    `json:"page_size,omitzero" help:"Page size, default 20" default:"20"`
+	User       string `json:"user,omitempty"`
+	Password   string `json:"password,omitempty"`
+
+	UserTreeDN              string   `json:"user_tree_dn,omitempty" help:"User tree distinguished name"`
+	UserFilter              string   `json:"user_filter,omitempty"`
+	UserObjectclass         string   `json:"user_objectclass,omitempty"`
+	UserIdAttribute         string   `json:"user_id_attribute,omitempty"`
+	UserNameAttribute       string   `json:"user_name_attribute,omitempty"`
+	UserEnabledAttribute    string   `json:"user_enabled_attribute,omitempty"`
+	UserEnabledMask         int64    `json:"user_enabled_mask,allowzero" default:"-1"`
+	UserEnabledDefault      string   `json:"user_enabled_default,omitempty"`
+	UserEnabledInvert       bool     `json:"user_enabled_invert,allowfalse"`
+	UserAdditionalAttribute []string `json:"user_additional_attribute_mapping,omitempty" token:"user_additional_attribute"`
+
+	GroupTreeDN          string `json:"group_tree_dn,omitempty" help:"Group tree distinguished name"`
+	GroupFilter          string `json:"group_filter,omitempty"`
+	GroupObjectclass     string `json:"group_objectclass,omitempty"`
+	GroupIdAttribute     string `json:"group_id_attribute,omitempty"`
+	GroupNameAttribute   string `json:"group_name_attribute,omitempty"`
+	GroupMemberAttribute string `json:"group_member_attribute,omitempty"`
+	GroupMembersAreIds   bool   `json:"group_members_are_ids,allowfalse"`
+}

pkg/apis/identity/consts.go

@@ -0,0 +1,56 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package identity
+
+const (
+	SERVICE_TYPE = "identity"
+
+	DEFAULT_DOMAIN_ID   = "default"
+	DEFAULT_DOMAIN_NAME = "Default"
+
+	AUTH_METHOD_PASSWORD = "password"
+	AUTH_METHOD_TOKEN    = "token"
+
+	AUTH_METHOD_ID_PASSWORD = 1
+	AUTH_METHOD_ID_TOKEN    = 2
+
+	AUTH_TOKEN_HEADER         = "X-Auth-Token"
+	AUTH_SUBJECT_TOKEN_HEADER = "X-Subject-Token"
+
+	AssignmentUserProject  = "UserProject"
+	AssignmentGroupProject = "GroupProject"
+	AssignmentUserDomain   = "UserDomain"
+	AssignmentGroupDomain  = "GroupDomain"
+
+	EndpointInterfacePublic   = "public"
+	EndpointInterfaceInternal = "internal"
+	EndpointInterfaceAdmin    = "admin"
+
+	KeystoneDomainRoot = "<<keystone.domain.root>>"
+
+	IdMappingEntityUser  = "user"
+	IdMappingEntityGroup = "group"
+
+	IdentityDriverSQL  = "sql"
+	IdentityDriverLDAP = "ldap"
+)
+
+var (
+	AUTH_METHODS = []string{AUTH_METHOD_PASSWORD, AUTH_METHOD_TOKEN}
+
+	SensitiveDomainConfigMap = map[string]string{
+		"ldap": "password",
+	}
+)

pkg/apis/identity/doc.go

@@ -0,0 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package identity // import "yunion.io/x/onecloud/pkg/apis/identity"

pkg/apis/image/consts.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package image
 
 type TImageType string

pkg/apis/image/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package image // import "yunion.io/x/onecloud/pkg/apis/image"

pkg/appsrv/appsrv.go

@@ -350,7 +350,7 @@ func (app *Application) initServer(addr string) *http.Server {
 		db.SetMaxOpenConns(app.connMax + 1)
 	}
 	*/
-	app.addDefaultHandlers()
+
 	s := &http.Server{
 		Addr:              addr,
 		Handler:           app,
@@ -424,18 +424,28 @@ func (app *Application) ListenAndServeWithCleanup(addr string, onStop func()) {
 }
 
 func (app *Application) ListenAndServeTLSWithCleanup(addr string, certFile, keyFile string, onStop func()) {
-	s := app.initServer(addr)
-	app.registerCleanShutdown(s, onStop)
-	app.listenAndServe(s, certFile, keyFile)
-	app.waitCleanShutdown()
+	app.ListenAndServeTLSWithCleanup2(addr, certFile, keyFile, onStop, true)
 }
 
 func (app *Application) ListenAndServeWithoutCleanup(addr, certFile, keyFile string) {
-	s := app.initServer(addr)
-	app.listenAndServe(s, certFile, keyFile)
+	app.ListenAndServeTLSWithCleanup2(addr, certFile, keyFile, nil, false)
 }
 
-func (app *Application) listenAndServe(s *http.Server, certFile, keyFile string) {
+func (app *Application) ListenAndServeTLSWithCleanup2(addr string, certFile, keyFile string, onStop func(), isMaster bool) {
+	if isMaster {
+		app.addDefaultHandlers()
+	}
+	s := app.initServer(addr)
+	if isMaster {
+		app.registerCleanShutdown(s, onStop)
+	}
+	app.listenAndServeInternal(s, certFile, keyFile)
+	if isMaster {
+		app.waitCleanShutdown()
+	}
+}
+
+func (app *Application) listenAndServeInternal(s *http.Server, certFile, keyFile string) {
 	var err error
 	if len(certFile) == 0 && len(keyFile) == 0 {
 		err = s.ListenAndServe()
@@ -443,7 +453,7 @@ func (app *Application) listenAndServe(s *http.Server, certFile, keyFile string)
 		err = s.ListenAndServeTLS(certFile, keyFile)
 	}
 	if err != nil && err != http.ErrServerClosed {
-		log.Fatalf("ListAndServer fail: %s", err)
+		log.Fatalf("ListAndServer fail: %s (cert=%s key=%s)", err, certFile, keyFile)
 	}
 }
 

pkg/appsrv/dispatcher/dispatcher.go

@@ -23,6 +23,7 @@ import (
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/utils"
 
+	"strings"
 	"yunion.io/x/onecloud/pkg/appctx"
 	"yunion.io/x/onecloud/pkg/appsrv"
 	"yunion.io/x/onecloud/pkg/httperrors"
@@ -38,16 +39,20 @@ func AddModelDispatcher(prefix string, app *appsrv.Application, manager IModelDi
 		manager.Filter(listHandler), metadata, "list", tags)
 	manager.CustomizeHandlerInfo(h)
 
-	ctxs := manager.ContextKeywordPlural()
+	ctxss := manager.ContextKeywordPlurals()
 	// list in context
-	if ctxs != nil && len(ctxs) > 0 {
-		for _, ctx := range ctxs {
-			h = app.AddHandler2("GET",
-				fmt.Sprintf("%s/%s/<resid>/%s", prefix, ctx, manager.KeywordPlural()),
-				manager.Filter(listInContextHandler), metadata, fmt.Sprintf("list_in_%s", ctx), tags)
-			manager.CustomizeHandlerInfo(h)
+	for _, ctxs := range ctxss {
+		segs := make([]string, 0)
+		segs = append(segs, prefix)
+		for i, ctx := range ctxs {
+			segs = append(segs, ctx, fmt.Sprintf("<resid_%d>", i))
 		}
+		segs = append(segs, manager.KeywordPlural())
+		h = app.AddHandler2("GET", strings.Join(segs, "/"),
+			manager.Filter(listInContextHandler), metadata, fmt.Sprintf("list_in_%s", strings.Join(ctxs, "_")), tags)
+		manager.CustomizeHandlerInfo(h)
 	}
+
 	// Head
 	h = app.AddHandler2("HEAD",
 		fmt.Sprintf("%s/%s/<resid>", prefix, manager.KeywordPlural()),
@@ -69,22 +74,26 @@ func AddModelDispatcher(prefix string, app *appsrv.Application, manager IModelDi
 		fmt.Sprintf("%s/%s", prefix, manager.KeywordPlural()),
 		manager.Filter(createHandler), metadata, "create", tags)
 	manager.CustomizeHandlerInfo(h)
+
 	// create in context
-	if ctxs != nil && len(ctxs) > 0 {
-		for _, ctx := range ctxs {
-			h = app.AddHandler2("POST",
-				fmt.Sprintf("%s/%s/<resid>/%s", prefix, ctx, manager.KeywordPlural()),
-				manager.Filter(createInContextHandler), metadata, fmt.Sprintf("create_in_%s", ctx), tags)
-			manager.CustomizeHandlerInfo(h)
+	for _, ctxs := range ctxss {
+		segs := make([]string, 0)
+		segs = append(segs, prefix)
+		for i, ctx := range ctxs {
+			segs = append(segs, ctx, fmt.Sprintf("<resid_%d>", i))
 		}
+		segs = append(segs, manager.KeywordPlural())
+		h = app.AddHandler2("POST", strings.Join(segs, "/"),
+			manager.Filter(createInContextHandler), metadata, fmt.Sprintf("create_in_%s", strings.Join(ctxs, "_")), tags)
+		manager.CustomizeHandlerInfo(h)
 	}
+
 	// batchPerformAction
 	h = app.AddHandler2("POST",
 		fmt.Sprintf("%s/%s/<action>", prefix, manager.KeywordPlural()),
 		manager.Filter(performClassActionHandler), metadata, "perform_class_action", tags)
 	manager.CustomizeHandlerInfo(h)
 	// performAction
-	// create in context
 	h = app.AddHandler2("POST",
 		fmt.Sprintf("%s/%s/<resid>/<action>", prefix, manager.KeywordPlural()),
 		manager.Filter(performActionHandler), metadata, "perform_action", tags)
@@ -99,6 +108,39 @@ func AddModelDispatcher(prefix string, app *appsrv.Application, manager IModelDi
 		fmt.Sprintf("%s/%s/<resid>", prefix, manager.KeywordPlural()),
 		manager.Filter(updateHandler), metadata, "update", tags)
 	manager.CustomizeHandlerInfo(h)
+	// patch
+	h = app.AddHandler2("PATCH",
+		fmt.Sprintf("%s/%s/<resid>", prefix, manager.KeywordPlural()),
+		manager.Filter(updateHandler), metadata, "patch", tags)
+	manager.CustomizeHandlerInfo(h)
+
+	// update/patch in context
+	for _, ctxs := range ctxss {
+		segs := make([]string, 0)
+		segs = append(segs, prefix)
+		for i, ctx := range ctxs {
+			segs = append(segs, ctx, fmt.Sprintf("<resid_%d>", i))
+		}
+		segs = append(segs, manager.KeywordPlural(), "<resid>")
+		h = app.AddHandler2("PUT", strings.Join(segs, "/"),
+			manager.Filter(updateInContextHandler), metadata, fmt.Sprintf("update_in_%s", strings.Join(ctxs, "_")), tags)
+		manager.CustomizeHandlerInfo(h)
+		h = app.AddHandler2("PATCH", strings.Join(segs, "/"),
+			manager.Filter(updateInContextHandler), metadata, fmt.Sprintf("patch_in_%s", strings.Join(ctxs, "_")), tags)
+		manager.CustomizeHandlerInfo(h)
+	}
+
+	// update spec
+	h = app.AddHandler2("PUT",
+		fmt.Sprintf("%s/%s/<resid>/<spec>", prefix, manager.KeywordPlural()),
+		manager.Filter(updateSpecHandler), metadata, "update_spec", tags)
+	manager.CustomizeHandlerInfo(h)
+	// patch spec
+	h = app.AddHandler2("PATCH",
+		fmt.Sprintf("%s/%s/<resid>/<spec>", prefix, manager.KeywordPlural()),
+		manager.Filter(updateSpecHandler), metadata, "patch_spec", tags)
+	manager.CustomizeHandlerInfo(h)
+
 	// batch Delete
 	/* app.AddHandler2("DELTE",
 	fmt.Sprintf("%s/%s", prefix, manager.KeywordPlural()),
@@ -109,6 +151,25 @@ func AddModelDispatcher(prefix string, app *appsrv.Application, manager IModelDi
 		fmt.Sprintf("%s/%s/<resid>", prefix, manager.KeywordPlural()),
 		manager.Filter(deleteHandler), metadata, "delete", tags)
 	manager.CustomizeHandlerInfo(h)
+
+	// delete in context
+	for _, ctxs := range ctxss {
+		segs := make([]string, 0)
+		segs = append(segs, prefix)
+		for i, ctx := range ctxs {
+			segs = append(segs, ctx, fmt.Sprintf("<resid_%d>", i))
+		}
+		segs = append(segs, manager.KeywordPlural(), "<resid>")
+		h = app.AddHandler2("DELETE", strings.Join(segs, "/"),
+			manager.Filter(deleteInContextHandler), metadata, fmt.Sprintf("delete_in_%s", strings.Join(ctxs, "_")), tags)
+		manager.CustomizeHandlerInfo(h)
+	}
+
+	// Delete Spec
+	h = app.AddHandler2("DELETE",
+		fmt.Sprintf("%s/%s/<resid>/<spec>", prefix, manager.KeywordPlural()),
+		manager.Filter(deleteSpecHandler), metadata, "delete_spec", tags)
+	manager.CustomizeHandlerInfo(h)
 }
 
 func fetchEnv(ctx context.Context, w http.ResponseWriter, r *http.Request) (IModelDispatchHandler, map[string]string, jsonutils.JSONObject, jsonutils.JSONObject) {
@@ -136,11 +197,11 @@ func mergeQueryParams(params map[string]string, query jsonutils.JSONObject, excl
 
 func listHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
 	manager, params, query, _ := fetchEnv(ctx, w, r)
-	handleList(ctx, w, manager, "", mergeQueryParams(params, query))
+	handleList(ctx, w, manager, nil, mergeQueryParams(params, query))
 }
 
-func handleList(ctx context.Context, w http.ResponseWriter, manager IModelDispatchHandler, ctxId string, query jsonutils.JSONObject) {
-	listResult, err := manager.List(ctx, query, ctxId)
+func handleList(ctx context.Context, w http.ResponseWriter, manager IModelDispatchHandler, ctxIds []SResourceContext, query jsonutils.JSONObject) {
+	listResult, err := manager.List(ctx, query, ctxIds)
 	if err != nil {
 		httperrors.GeneralServerError(w, err)
 		return
@@ -148,10 +209,26 @@ func handleList(ctx context.Context, w http.ResponseWriter, manager IModelDispat
 	appsrv.SendJSON(w, modules.ListResult2JSONWithKey(listResult, manager.KeywordPlural()))
 }
 
+func fetchContextIds(segs []string, params map[string]string) ([]SResourceContext, []string) {
+	ctxIds := make([]SResourceContext, 0)
+	keys := make([]string, 0)
+	idx := 0
+	key := fmt.Sprintf("<resid_%d>", idx)
+	for i := 0; i < len(segs); i += 1 {
+		if segs[i] == key {
+			ctxIds = append(ctxIds, SResourceContext{Type: segs[i-1], Id: params[key]})
+			keys = append(keys, key)
+			idx += 1
+			key = fmt.Sprintf("<resid_%d>", idx)
+		}
+	}
+	return ctxIds, keys
+}
+
 func listInContextHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
 	manager, params, query, _ := fetchEnv(ctx, w, r)
-	ctxId := params["<resid>"]
-	handleList(ctx, w, manager, ctxId, mergeQueryParams(params, query, "<resid>"))
+	ctxIds, ctxKeys := fetchContextIds(appctx.AppContextCurrentRoot(ctx), params)
+	handleList(ctx, w, manager, ctxIds, mergeQueryParams(params, query, ctxKeys...))
 }
 
 func wrapBody(body jsonutils.JSONObject, key string) jsonutils.JSONObject {
@@ -179,6 +256,17 @@ func headHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+func sendJSON(ctx context.Context, w http.ResponseWriter, result jsonutils.JSONObject, keyword string) {
+	appParams := appsrv.AppContextGetParams(ctx)
+	var body jsonutils.JSONObject
+	if appParams != nil && appParams.OverrideResponseBodyWrapper {
+		body = result
+	} else {
+		body = wrapBody(result, keyword)
+	}
+	appsrv.SendJSON(w, body)
+}
+
 func getHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
 	manager, params, query, _ := fetchEnv(ctx, w, r)
 	result, err := manager.Get(ctx, params["<resid>"], mergeQueryParams(params, query, "<resid>"), false)
@@ -187,14 +275,7 @@ func getHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	if result != nil {
-		appParams := appsrv.AppContextGetParams(ctx)
-		var body jsonutils.JSONObject
-		if appParams != nil && appParams.OverrideResponseBodyWrapper {
-			body = result
-		} else {
-			body = wrapBody(result, manager.Keyword())
-		}
-		appsrv.SendJSON(w, body)
+		sendJSON(ctx, w, result, manager.Keyword())
 	}
 }
 
@@ -206,16 +287,16 @@ func getSpecHandler(ctx context.Context, w http.ResponseWriter, r *http.Request)
 		return
 	}
 	if result != nil {
-		appsrv.SendJSON(w, wrapBody(result, manager.Keyword()))
+		sendJSON(ctx, w, result, manager.Keyword())
 	}
 }
 
 func createHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
 	manager, params, query, body := fetchEnv(ctx, w, r)
-	handleCreate(ctx, w, manager, "", mergeQueryParams(params, query), body, r)
+	handleCreate(ctx, w, manager, nil, mergeQueryParams(params, query), body, r)
 }
 
-func handleCreate(ctx context.Context, w http.ResponseWriter, manager IModelDispatchHandler, ctxId string, query jsonutils.JSONObject, body jsonutils.JSONObject, r *http.Request) {
+func handleCreate(ctx context.Context, w http.ResponseWriter, manager IModelDispatchHandler, ctxIds []SResourceContext, query jsonutils.JSONObject, body jsonutils.JSONObject, r *http.Request) {
 	count := int64(1)
 	var data jsonutils.JSONObject
 	var err error
@@ -236,14 +317,14 @@ func handleCreate(ctx context.Context, w http.ResponseWriter, manager IModelDisp
 		}
 	}
 	if count <= 1 {
-		result, err := manager.Create(ctx, query, data, ctxId)
+		result, err := manager.Create(ctx, query, data, ctxIds)
 		if err != nil {
 			httperrors.GeneralServerError(w, err)
 			return
 		}
 		appsrv.SendJSON(w, wrapBody(result, manager.Keyword()))
 	} else {
-		results, err := manager.BatchCreate(ctx, query, data, int(count), ctxId)
+		results, err := manager.BatchCreate(ctx, query, data, int(count), ctxIds)
 		if err != nil {
 			httperrors.GeneralServerError(w, err)
 			return
@@ -261,17 +342,21 @@ func handleCreate(ctx context.Context, w http.ResponseWriter, manager IModelDisp
 
 func createInContextHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
 	manager, params, query, body := fetchEnv(ctx, w, r)
-	ctxId := params["<resid>"]
-	handleCreate(ctx, w, manager, ctxId, mergeQueryParams(params, query, "<resid>"), body, r)
+	ctxIds, ctxKeys := fetchContextIds(appctx.AppContextCurrentRoot(ctx), params)
+	handleCreate(ctx, w, manager, ctxIds, mergeQueryParams(params, query, ctxKeys...), body, r)
 }
 
 func performClassActionHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
 	manager, params, query, body := fetchEnv(ctx, w, r)
 	var data jsonutils.JSONObject
 	if body != nil {
-		data, _ = body.Get(manager.KeywordPlural())
-		if data == nil {
-			data = body.(*jsonutils.JSONDict)
+		if body.Contains(manager.KeywordPlural()) {
+			data, _ = body.Get(manager.KeywordPlural())
+			if data == nil {
+				data = body.(*jsonutils.JSONDict)
+			}
+		} else {
+			data = body
 		}
 	} else {
 		data = jsonutils.NewDict()
@@ -284,16 +369,21 @@ func performClassActionHandler(ctx context.Context, w http.ResponseWriter, r *ht
 	if results == nil {
 		results = jsonutils.NewDict()
 	}
-	appsrv.SendJSON(w, wrapBody(results, manager.KeywordPlural()))
+	sendJSON(ctx, w, results, manager.KeywordPlural())
+	// appsrv.SendJSON(w, wrapBody(results, manager.KeywordPlural()))
 }
 
 func performActionHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
 	manager, params, query, body := fetchEnv(ctx, w, r)
 	var data jsonutils.JSONObject
 	if body != nil {
-		data, _ = body.Get(manager.Keyword())
-		if data == nil {
-			data = body.(*jsonutils.JSONDict)
+		if body.Contains(manager.Keyword()) {
+			data, _ = body.Get(manager.Keyword())
+			if data == nil {
+				data = body.(*jsonutils.JSONDict)
+			}
+		} else {
+			data = body
 		}
 	} else {
 		data = jsonutils.NewDict()
@@ -303,7 +393,8 @@ func performActionHandler(ctx context.Context, w http.ResponseWriter, r *http.Re
 		httperrors.GeneralServerError(w, err)
 		return
 	}
-	appsrv.SendJSON(w, wrapBody(result, manager.Keyword()))
+	sendJSON(ctx, w, result, manager.Keyword())
+	// appsrv.SendJSON(w, wrapBody(result, manager.Keyword()))
 }
 
 /*
@@ -326,14 +417,22 @@ func updateClassHandler(ctx context.Context, w http.ResponseWriter, r *http.Requ
 
 func updateHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
 	manager, params, query, body := fetchEnv(ctx, w, r)
+	handleUpdate(ctx, w, manager, params["<resid>"], nil, mergeQueryParams(params, query, "<resid>"), body, r)
+}
+
+func handleUpdate(ctx context.Context, w http.ResponseWriter, manager IModelDispatchHandler, resId string, ctxIds []SResourceContext, query jsonutils.JSONObject, body jsonutils.JSONObject, r *http.Request) {
 	var data jsonutils.JSONObject
 	var err error
 	if body != nil {
-		data, err = body.Get(manager.Keyword())
-		if err != nil {
-			httperrors.InvalidInputError(w,
-				fmt.Sprintf("No request key: %s", manager.Keyword()))
-			return
+		if body.Contains(manager.Keyword()) {
+			data, err = body.Get(manager.Keyword())
+			if err != nil {
+				httperrors.InvalidInputError(w,
+					fmt.Sprintf("No request key: %s", manager.Keyword()))
+				return
+			}
+		} else {
+			data = body
 		}
 	} else {
 		data, err = manager.FetchUpdateHeaderData(ctx, r.Header)
@@ -343,12 +442,52 @@ func updateHandler(ctx context.Context, w http.ResponseWriter, r *http.Request)
 			return
 		}
 	}
-	result, err := manager.Update(ctx, params["<resid>"], mergeQueryParams(params, query, "<resid>"), data)
+	result, err := manager.Update(ctx, resId, query, data, ctxIds)
 	if err != nil {
 		httperrors.GeneralServerError(w, err)
 		return
 	}
-	appsrv.SendJSON(w, wrapBody(result, manager.Keyword()))
+	sendJSON(ctx, w, result, manager.Keyword())
+	// appsrv.SendJSON(w, wrapBody(result, manager.Keyword()))
+}
+
+func updateInContextHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	manager, params, query, body := fetchEnv(ctx, w, r)
+	ctxIds, ctxKeys := fetchContextIds(appctx.AppContextCurrentRoot(ctx), params)
+	ctxKeys = append(ctxKeys, "<resid>")
+	handleUpdate(ctx, w, manager, params["<resid>"], ctxIds, mergeQueryParams(params, query, ctxKeys...), body, r)
+}
+
+func updateSpecHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	manager, params, query, body := fetchEnv(ctx, w, r)
+	var data jsonutils.JSONObject
+	var err error
+	if body != nil {
+		if body.Contains(manager.Keyword()) {
+			data, err = body.Get(manager.Keyword())
+			if err != nil {
+				httperrors.InvalidInputError(w,
+					fmt.Sprintf("No request key: %s", manager.Keyword()))
+				return
+			}
+		} else {
+			data = body
+		}
+	} else {
+		data, err = manager.FetchUpdateHeaderData(ctx, r.Header)
+		if err != nil {
+			httperrors.InvalidInputError(w,
+				fmt.Sprintf("In valid request header: %s", err))
+			return
+		}
+	}
+	result, err := manager.UpdateSpec(ctx, params["<resid>"], params["<spec>"], mergeQueryParams(params, query, "<resid>", "<spec>"), data)
+	if err != nil {
+		httperrors.GeneralServerError(w, err)
+		return
+	}
+	sendJSON(ctx, w, result, manager.Keyword())
+	// appsrv.SendJSON(w, wrapBody(result, manager.Keyword()))
 }
 
 /*
@@ -374,22 +513,65 @@ func deleteClassHandler(ctx context.Context, w http.ResponseWriter, r *http.Requ
 
 func deleteHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
 	manager, params, query, body := fetchEnv(ctx, w, r)
+	handleDelete(ctx, w, manager, params["<resid>"], nil, mergeQueryParams(params, query, "<resid>"), body, r)
+}
+
+func handleDelete(ctx context.Context, w http.ResponseWriter, manager IModelDispatchHandler, resId string, ctxIds []SResourceContext, query jsonutils.JSONObject, body jsonutils.JSONObject, r *http.Request) {
 	var data jsonutils.JSONObject
 	var err error
 	if body != nil {
-		data, err = body.Get(manager.Keyword())
-		if err != nil {
-			httperrors.InvalidInputError(w,
-				fmt.Sprintf("No request key: %s", manager.Keyword()))
-			return
+		if body.Contains(manager.Keyword()) {
+			data, err = body.Get(manager.Keyword())
+			if err != nil {
+				httperrors.InvalidInputError(w,
+					fmt.Sprintf("No request key: %s", manager.Keyword()))
+				return
+			}
+		} else {
+			data = body
 		}
 	} else {
 		data = jsonutils.NewDict()
 	}
-	result, err := manager.Delete(ctx, params["<resid>"], mergeQueryParams(params, query, "<resid>"), data)
+	result, err := manager.Delete(ctx, resId, query, data, ctxIds)
 	if err != nil {
 		httperrors.GeneralServerError(w, err)
 		return
 	}
-	appsrv.SendJSON(w, wrapBody(result, manager.Keyword()))
+	sendJSON(ctx, w, result, manager.Keyword())
+	// appsrv.SendJSON(w, wrapBody(result, manager.Keyword()))
+}
+
+func deleteInContextHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	manager, params, query, body := fetchEnv(ctx, w, r)
+	ctxIds, ctxKeys := fetchContextIds(appctx.AppContextCurrentRoot(ctx), params)
+	ctxKeys = append(ctxKeys, "<resid>")
+	handleDelete(ctx, w, manager, params["<resid>"], ctxIds, mergeQueryParams(params, query, ctxKeys...), body, r)
+}
+
+func deleteSpecHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	manager, params, query, body := fetchEnv(ctx, w, r)
+	var data jsonutils.JSONObject
+	var err error
+	if body != nil {
+		if body.Contains(manager.Keyword()) {
+			data, err = body.Get(manager.Keyword())
+			if err != nil {
+				httperrors.InvalidInputError(w,
+					fmt.Sprintf("No request key: %s", manager.Keyword()))
+				return
+			}
+		} else {
+			data = body
+		}
+	} else {
+		data = jsonutils.NewDict()
+	}
+	result, err := manager.DeleteSpec(ctx, params["<resid>"], params["<spec>"], mergeQueryParams(params, query, "<resid>", "<spec>"), data)
+	if err != nil {
+		httperrors.GeneralServerError(w, err)
+		return
+	}
+	sendJSON(ctx, w, result, manager.Keyword())
+	// appsrv.SendJSON(w, wrapBody(result, manager.Keyword()))
 }

pkg/appsrv/dispatcher/dispatcher_test.go

@@ -0,0 +1,49 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dispatcher
+
+import "testing"
+
+func TestFetchContextIds(t *testing.T) {
+	cases := []struct {
+		segs   []string
+		params map[string]string
+	}{
+		{
+			[]string{"servers", "<resid_0>", "disks", "<resid_1>", "test"},
+			map[string]string{
+				"<resid_0>": "12345",
+				"<resid_1>": "23",
+			},
+		},
+		{
+			[]string{"servers", "<resid_0>", "disks", "<resid_1>", "test"},
+			map[string]string{
+				"<resid_0>": "12345",
+			},
+		},
+		{
+			[]string{"servers", "<resid_0>"},
+			map[string]string{
+				"<resid_0>": "12345",
+				"<resid_1>": "23",
+			},
+		},
+	}
+	for _, c := range cases {
+		ctxIdx, keys := fetchContextIds(c.segs, c.params)
+		t.Logf("%#v %#v", ctxIdx, keys)
+	}
+}

pkg/appsrv/dispatcher/handlers.go

@@ -28,24 +28,35 @@ type IMiddlewareFilter interface {
 	Filter(appsrv.FilterHandler) appsrv.FilterHandler
 }
 
+type SResourceContext struct {
+	Type string
+	Id   string
+}
+
 type IModelDispatchHandler interface {
 	IMiddlewareFilter
 
 	Keyword() string
 	KeywordPlural() string
-	ContextKeywordPlural() []string
+	ContextKeywordPlurals() [][]string
 
-	List(ctx context.Context, query jsonutils.JSONObject, ctxId string) (*modules.ListResult, error)
+	List(ctx context.Context, query jsonutils.JSONObject, ctxIds []SResourceContext) (*modules.ListResult, error)
 	Get(ctx context.Context, idstr string, query jsonutils.JSONObject, isHead bool) (jsonutils.JSONObject, error)
 	GetSpecific(ctx context.Context, idstr string, spec string, query jsonutils.JSONObject) (jsonutils.JSONObject, error)
-	Create(ctx context.Context, query jsonutils.JSONObject, data jsonutils.JSONObject, ctxId string) (jsonutils.JSONObject, error)
-	BatchCreate(ctx context.Context, query jsonutils.JSONObject, data jsonutils.JSONObject, count int, ctxId string) ([]modules.SubmitResult, error)
+
+	Create(ctx context.Context, query jsonutils.JSONObject, data jsonutils.JSONObject, ctxIds []SResourceContext) (jsonutils.JSONObject, error)
+	BatchCreate(ctx context.Context, query jsonutils.JSONObject, data jsonutils.JSONObject, count int, ctxIds []SResourceContext) ([]modules.SubmitResult, error)
+
 	PerformClassAction(ctx context.Context, action string, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error)
 	PerformAction(ctx context.Context, idstr string, action string, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error)
+
 	// UpdateClass(ctx context.Context, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error)
-	Update(ctx context.Context, idstr string, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error)
+	Update(ctx context.Context, idstr string, query jsonutils.JSONObject, data jsonutils.JSONObject, ctxIds []SResourceContext) (jsonutils.JSONObject, error)
+	UpdateSpec(ctx context.Context, idstr string, spec string, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error)
+
 	// DeleteClass(ctx context.Context, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error)
-	Delete(ctx context.Context, idstr string, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error)
+	Delete(ctx context.Context, idstr string, query jsonutils.JSONObject, data jsonutils.JSONObject, ctxIds []SResourceContext) (jsonutils.JSONObject, error)
+	DeleteSpec(ctx context.Context, idstr string, spec string, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error)
 
 	CustomizeHandlerInfo(info *appsrv.SHandlerInfo)
 	FetchCreateHeaderData(ctx context.Context, header http.Header) (jsonutils.JSONObject, error)
@@ -60,7 +71,7 @@ type IJointModelDispatchHandler interface {
 	MasterKeywordPlural() string
 	SlaveKeywordPlural() string
 
-	List(ctx context.Context, query jsonutils.JSONObject, ctxId string) (*modules.ListResult, error)
+	List(ctx context.Context, query jsonutils.JSONObject, ctxIds []SResourceContext) (*modules.ListResult, error)
 	ListMasterDescendent(ctx context.Context, idStr string, query jsonutils.JSONObject) (*modules.ListResult, error)
 	ListSlaveDescendent(ctx context.Context, idStr string, query jsonutils.JSONObject) (*modules.ListResult, error)
 	Get(ctx context.Context, id1 string, id2 string, query jsonutils.JSONObject) (jsonutils.JSONObject, error)

pkg/appsrv/dispatcher/jointdispatcher.go

@@ -116,7 +116,7 @@ func fetchJointEnv(ctx context.Context, w http.ResponseWriter, r *http.Request)
 
 func jointListHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
 	manager, params, query, _ := fetchJointEnv(ctx, w, r)
-	listResult, err := manager.List(ctx, mergeQueryParams(params, query), "")
+	listResult, err := manager.List(ctx, mergeQueryParams(params, query), nil)
 	if err != nil {
 		httperrors.GeneralServerError(w, err)
 		return

pkg/appsrv/send.go

@@ -29,14 +29,22 @@ func Send(w http.ResponseWriter, text string) {
 
 func SendStruct(w http.ResponseWriter, obj interface{}) {
 	w.Header().Set("Content-Type", "application/json")
-	b, e := json.Marshal(obj)
-	if e != nil {
-		log.Errorln("SendStruct json Marshal error: ", e)
+	if obj != nil {
+		b, e := json.Marshal(obj)
+		if e != nil {
+			log.Errorln("SendStruct json Marshal error: ", e)
+		}
+		w.Write(b)
+	} else {
+		w.Write([]byte{})
 	}
-	w.Write(b)
 }
 
 func SendJSON(w http.ResponseWriter, obj jsonutils.JSONObject) {
 	w.Header().Set("Content-Type", "application/json")
-	w.Write([]byte(obj.String()))
+	if obj != nil {
+		w.Write([]byte(obj.String()))
+	} else {
+		w.Write([]byte{})
+	}
 }

pkg/baremetal/service/service.go

@@ -42,12 +42,12 @@ func (s *BaremetalService) StartService() {
 	common_options.ParseOptions(&o.Options, os.Args, "baremetal.conf", "baremetal")
 	app_common.InitAuth(&o.Options.CommonOptions, s.startAgent)
 
-	app := app_common.InitApp(&o.Options.CommonOptions, false)
+	app := app_common.InitApp(&o.Options.BaseOptions, false)
 	handler.InitHandlers(app)
 
 	s.startFileServer()
 
-	app_common.ServeForeverWithCleanup(app, &o.Options.CommonOptions, func() {
+	app_common.ServeForeverWithCleanup(app, &o.Options.BaseOptions, func() {
 		tasks.OnStop()
 		baremetal.Stop()
 	})

pkg/cloudcommon/app/app.go

@@ -26,7 +26,7 @@ import (
 	"yunion.io/x/onecloud/pkg/util/seclib2"
 )
 
-func InitApp(options *common_options.CommonOptions, dbAccess bool) *appsrv.Application {
+func InitApp(options *common_options.BaseOptions, dbAccess bool) *appsrv.Application {
 	// cache := appsrv.NewCache(options.AuthTokenCacheSize)
 	app := appsrv.NewApplication(options.ApplicationID, options.RequestWorkerCount, dbAccess)
 	app.CORSAllowHosts(options.CorsHosts)
@@ -38,19 +38,25 @@ func InitApp(options *common_options.CommonOptions, dbAccess bool) *appsrv.Appli
 	return app
 }
 
-func ServeForever(app *appsrv.Application, options *common_options.CommonOptions) {
+func ServeForever(app *appsrv.Application, options *common_options.BaseOptions) {
 	ServeForeverWithCleanup(app, options, nil)
 }
 
-func ServeForeverWithCleanup(app *appsrv.Application, options *common_options.CommonOptions, onStop func()) {
-	addr := net.JoinHostPort(options.Address, strconv.Itoa(options.Port))
+func ServeForeverWithCleanup(app *appsrv.Application, options *common_options.BaseOptions, onStop func()) {
+	ServeForeverExtended(app, options, options.Port, onStop, true)
+}
+
+func ServeForeverExtended(app *appsrv.Application, options *common_options.BaseOptions, port int, onStop func(), isMaster bool) {
+	addr := net.JoinHostPort(options.Address, strconv.Itoa(port))
 	proto := "http"
 	if options.EnableSsl {
 		proto = "https"
 	}
 	log.Infof("Start listen on %s://%s", proto, addr)
+	var certfile string
+	var sslfile string
 	if options.EnableSsl {
-		certfile := options.SslCertfile
+		certfile = options.SslCertfile
 		if len(options.SslCaCerts) > 0 {
 			var err error
 			certfile, err = seclib2.MergeCaCertFiles(options.SslCaCerts, options.SslCertfile)
@@ -65,8 +71,7 @@ func ServeForeverWithCleanup(app *appsrv.Application, options *common_options.Co
 		if len(options.SslKeyfile) == 0 {
 			log.Fatalf("Missing ssl-keyfile")
 		}
-		app.ListenAndServeTLSWithCleanup(addr, certfile, options.SslKeyfile, onStop)
-	} else {
-		app.ListenAndServeWithCleanup(addr, onStop)
+		sslfile = options.SslKeyfile
 	}
+	app.ListenAndServeTLSWithCleanup2(addr, certfile, sslfile, onStop, isMaster)
 }

pkg/cloudcommon/app/auth.go

@@ -67,6 +67,10 @@ func InitAuth(options *common_options.CommonOptions, authComplete auth.AuthCompl
 
 	authComplete()
 
+	InitBaseAuth(&options.BaseOptions)
+}
+
+func InitBaseAuth(options *common_options.BaseOptions) {
 	if options.EnableRbac {
 		policy.EnableGlobalRbac(time.Duration(options.RbacPolicySyncPeriodSeconds)*time.Second,
 			time.Duration(options.RbacPolicySyncFailedRetrySeconds)*time.Second,

pkg/cloudcommon/db/count.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package db
 
 import (

pkg/cloudcommon/db/db_dispatcher.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"net/http"
 	"reflect"
+	"sort"
 	"strings"
 
 	"yunion.io/x/jsonutils"
@@ -30,6 +31,7 @@ import (
 	"yunion.io/x/sqlchemy"
 
 	"yunion.io/x/onecloud/pkg/appsrv"
+	"yunion.io/x/onecloud/pkg/appsrv/dispatcher"
 	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
 	"yunion.io/x/onecloud/pkg/cloudcommon/policy"
@@ -59,12 +61,15 @@ func (dispatcher *DBModelDispatcher) KeywordPlural() string {
 	return dispatcher.modelManager.KeywordPlural()
 }
 
-func (dispatcher *DBModelDispatcher) ContextKeywordPlural() []string {
-	ctxMans := dispatcher.modelManager.GetContextManager()
+func (dispatcher *DBModelDispatcher) ContextKeywordPlurals() [][]string {
+	ctxMans := dispatcher.modelManager.GetContextManagers()
 	if ctxMans != nil {
-		keys := make([]string, len(ctxMans))
+		keys := make([][]string, len(ctxMans))
 		for i := 0; i < len(ctxMans); i += 1 {
-			keys[i] = ctxMans[i].KeywordPlural()
+			keys[i] = make([]string, len(ctxMans[i]))
+			for j := 0; j < len(ctxMans[i]); j += 1 {
+				keys[i][j] = ctxMans[i][j].KeywordPlural()
+			}
 		}
 		return keys
 	}
@@ -84,11 +89,7 @@ func (dispatcher *DBModelDispatcher) CustomizeHandlerInfo(handler *appsrv.SHandl
 }
 
 func fetchUserCredential(ctx context.Context) mcclient.TokenCredential {
-	token := auth.FetchUserCredential(ctx, policy.FilterPolicyCredential)
-	if token == nil && !consts.IsRbacEnabled() {
-		log.Fatalf("user token credential not found?")
-	}
-	return token
+	return policy.FetchUserCredential(ctx)
 }
 
 /**
@@ -112,7 +113,7 @@ func listFields(manager IModelManager, userCred mcclient.TokenCredential) []stri
 	return ret
 }
 
-func searchFields(manager IModelManager, userCred mcclient.TokenCredential) []string {
+func searchFields(manager IModelManager, userCred mcclient.TokenCredential) stringutils2.SSortedStrings {
 	ret := make([]string, 0)
 	for _, col := range manager.TableSpec().Columns() {
 		tags := col.Tags()
@@ -122,7 +123,8 @@ func searchFields(manager IModelManager, userCred mcclient.TokenCredential) []st
 			ret = append(ret, col.Name())
 		}
 	}
-	return ret
+	sort.Strings(ret)
+	return stringutils2.SSortedStrings(ret)
 }
 
 func GetDetailFields(manager IModelManager, userCred mcclient.TokenCredential) []string {
@@ -147,7 +149,6 @@ func createRequireFields(manager IModelManager, userCred mcclient.TokenCredentia
 			ret = append(ret, col.Name())
 		}
 	}
-	log.Debugf("CreateRequiredFields for %s: %s", manager.Keyword(), ret)
 	return ret
 }
 
@@ -161,7 +162,6 @@ func createFields(manager IModelManager, userCred mcclient.TokenCredential) []st
 			ret = append(ret, col.Name())
 		}
 	}
-	log.Debugf("CreateFields for %s: %s", manager.Keyword(), ret)
 	return ret
 }
 
@@ -177,6 +177,28 @@ func updateFields(manager IModelManager, userCred mcclient.TokenCredential) []st
 	return ret
 }
 
+var (
+	searchOps = map[string]string{
+		"contains":   "contains",
+		"startswith": "startswith",
+		"endswith":   "endswith",
+		"empty":      "isnullorempty",
+	}
+)
+
+func parseSearchFieldkey(key string) (string, string) {
+	for op, fn := range searchOps {
+		if strings.HasSuffix(key, "__"+op) {
+			key = key[:len(key)-(2+len(op))]
+			return key, fn
+		} else if strings.HasSuffix(key, "__i"+op) {
+			key = key[:len(key)-(3+len(op))]
+			return key, fn
+		}
+	}
+	return key, ""
+}
+
 func listItemsQueryByColumn(manager IModelManager, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) {
 	if query == nil {
 		return q, nil
@@ -185,16 +207,26 @@ func listItemsQueryByColumn(manager IModelManager, q *sqlchemy.SQuery, userCred
 	if err != nil {
 		return nil, err
 	}
+
 	listF := searchFields(manager, userCred)
-	for k, v := range qdata {
-		searchable, _ := utils.InStringArray(k, listF)
-		if searchable {
-			colSpec := manager.TableSpec().ColumnSpec(k)
+	for key, val := range qdata {
+		fn, op := parseSearchFieldkey(key)
+		if listF.Contains(fn) {
+			colSpec := manager.TableSpec().ColumnSpec(fn)
 			if colSpec != nil {
-				strV, _ := v.GetString()
-				if len(strV) > 0 {
+				strV, _ := val.GetString()
+				if len(op) > 0 {
+					filter := fmt.Sprintf("%s.%s(%s)", fn, op, strV)
+					fc := filterclause.ParseFilterClause(filter)
+					if fc != nil {
+						cond := fc.QueryCondition(q)
+						if cond != nil {
+							q = q.Filter(cond)
+						}
+					}
+				} else if len(strV) > 0 {
 					strV := colSpec.ConvertFromString(strV)
-					q = q.Equals(k, strV)
+					q = q.Equals(fn, strV)
 				}
 			}
 		}
@@ -234,8 +266,7 @@ func applyListItemsGeneralFilters(manager IModelManager, q *sqlchemy.SQuery,
 	for _, f := range filters {
 		fc := filterclause.ParseFilterClause(f)
 		if fc != nil {
-			ok, _ := utils.InStringArray(fc.GetField(), schFields)
-			if ok {
+			if schFields.Contains(fc.GetField()) {
 				cond := fc.QueryCondition(q)
 				if cond != nil {
 					conds = append(conds, cond)
@@ -260,7 +291,7 @@ func applyListItemsGeneralJointFilters(manager IModelManager, q *sqlchemy.SQuery
 		if jfc != nil {
 			jointModelManager := GetModelManager(jfc.GetJointModelName())
 			schFields := searchFields(jointModelManager, userCred)
-			if ok, _ := utils.InStringArray(jfc.GetField(), schFields); ok {
+			if schFields.Contains(jfc.GetField()) {
 				sq := jointModelManager.Query(jfc.RelatedKey)
 				cond := jfc.GetJointFilter(sq)
 				if cond != nil {
@@ -411,38 +442,65 @@ func Query2List(manager IModelManager, ctx context.Context, userCred mcclient.To
 	return results, nil
 }
 
-func fetchContextObjectId(manager IModelManager, ctx context.Context, userCred mcclient.TokenCredential, ctxId string, queryDict *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
-	ctxMans := manager.GetContextManager()
-	if ctxMans == nil {
-		return nil, fmt.Errorf("No context manager")
-	}
-	find := false
-	keys := make([]string, 0)
-	for i := 0; i < len(ctxMans); i += 1 {
-		ctxObj, err := fetchItem(ctxMans[i], ctx, userCred, ctxId, nil)
+func fetchContextObjectsIds(manager IModelManager, ctx context.Context, userCred mcclient.TokenCredential, ctxIds []dispatcher.SResourceContext, queryDict *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+	var err error
+	for i := 0; i < len(ctxIds); i += 1 {
+		queryDict, err = fetchContextObjectIds(manager, ctx, userCred, ctxIds[i], queryDict)
 		if err != nil {
-			if err == sql.ErrNoRows {
-				keys = append(keys, ctxMans[i].KeywordPlural())
-				continue
-			} else {
-				return nil, err
-			}
-		} else {
-			find = true
-			queryDict.Add(jsonutils.NewString(ctxObj.GetId()), fmt.Sprintf("%s_id", ctxObj.GetModelManager().Keyword()))
-			if len(ctxObj.GetModelManager().Alias()) > 0 {
-				queryDict.Add(jsonutils.NewString(ctxObj.GetId()), fmt.Sprintf("%s_id", ctxObj.GetModelManager().Alias()))
+			return nil, err
+		}
+	}
+	return queryDict, nil
+}
+
+func fetchContextObjectIds(manager IModelManager, ctx context.Context, userCred mcclient.TokenCredential, ctxId dispatcher.SResourceContext, queryDict *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+	ctxObj, err := fetchContextObject(manager, ctx, userCred, ctxId)
+	if err != nil {
+		return nil, err
+	}
+	queryDict.Add(jsonutils.NewString(ctxObj.GetId()), fmt.Sprintf("%s_id", ctxObj.GetModelManager().Keyword()))
+	if len(ctxObj.GetModelManager().Alias()) > 0 {
+		queryDict.Add(jsonutils.NewString(ctxObj.GetId()), fmt.Sprintf("%s_id", ctxObj.GetModelManager().Alias()))
+	}
+	return queryDict, nil
+}
+
+func fetchContextObjects(manager IModelManager, ctx context.Context, userCred mcclient.TokenCredential, ctxIds []dispatcher.SResourceContext) ([]IModel, error) {
+	ctxObjs := make([]IModel, len(ctxIds))
+	for i := 0; i < len(ctxIds); i += 1 {
+		ctxObj, err := fetchContextObject(manager, ctx, userCred, ctxIds[i])
+		if err != nil {
+			return nil, err
+		}
+		ctxObjs[i] = ctxObj
+	}
+	return ctxObjs, nil
+}
+
+func fetchContextObject(manager IModelManager, ctx context.Context, userCred mcclient.TokenCredential, ctxId dispatcher.SResourceContext) (IModel, error) {
+	ctxMans := manager.GetContextManagers()
+	if ctxMans == nil {
+		return nil, httperrors.NewInternalServerError("No context manager")
+	}
+	for i := 0; i < len(ctxMans); i += 1 {
+		for j := 0; j < len(ctxMans); j += 1 {
+			if ctxMans[i][j].KeywordPlural() == ctxId.Type {
+				ctxObj, err := fetchItem(ctxMans[i][j], ctx, userCred, ctxId.Id, nil)
+				if err != nil {
+					if err == sql.ErrNoRows {
+						return nil, httperrors.NewResourceNotFoundError2(ctxMans[i][j].Keyword(), ctxId.Id)
+					} else {
+						return nil, err
+					}
+				}
+				return ctxObj, nil
 			}
 		}
 	}
-	if !find {
-		return nil, httperrors.NewResourceNotFoundError("Resource %s not found in %s", ctxId, strings.Join(keys, ", "))
-	} else {
-		return queryDict, nil
-	}
+	return nil, httperrors.NewInternalServerError("No such context %s(%s)", ctxId.Type, ctxId.Id)
 }
 
-func ListItems(manager IModelManager, ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, ctxId string) (*modules.ListResult, error) {
+func ListItems(manager IModelManager, ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, ctxIds []dispatcher.SResourceContext) (*modules.ListResult, error) {
 	var maxLimit int64 = 2048
 	limit, _ := query.Int("limit")
 	offset, _ := query.Int("offset")
@@ -452,8 +510,8 @@ func ListItems(manager IModelManager, ctx context.Context, userCred mcclient.Tok
 	if !ok {
 		return nil, fmt.Errorf("invalid query format")
 	}
-	if len(ctxId) > 0 {
-		queryDict, err = fetchContextObjectId(manager, ctx, userCred, ctxId, queryDict)
+	if len(ctxIds) > 0 {
+		queryDict, err = fetchContextObjectsIds(manager, ctx, userCred, ctxIds, queryDict)
 		if err != nil {
 			return nil, err
 		}
@@ -558,7 +616,7 @@ func calculateListResult(data []jsonutils.JSONObject, total, limit, offset int64
 	return &retResult
 }
 
-func (dispatcher *DBModelDispatcher) List(ctx context.Context, query jsonutils.JSONObject, ctxId string) (*modules.ListResult, error) {
+func (dispatcher *DBModelDispatcher) List(ctx context.Context, query jsonutils.JSONObject, ctxIds []dispatcher.SResourceContext) (*modules.ListResult, error) {
 	userCred := fetchUserCredential(ctx)
 
 	var isAllow bool
@@ -578,7 +636,7 @@ func (dispatcher *DBModelDispatcher) List(ctx context.Context, query jsonutils.J
 		return nil, httperrors.NewForbiddenError("Not allow to list")
 	}
 
-	items, err := ListItems(dispatcher.modelManager, ctx, userCred, query, ctxId)
+	items, err := ListItems(dispatcher.modelManager, ctx, userCred, query, ctxIds)
 	if err != nil {
 		log.Errorf("Fail to list items: %s", err)
 		return nil, httperrors.NewGeneralError(err)
@@ -922,7 +980,7 @@ func doCreateItem(manager IModelManager, ctx context.Context, userCred mcclient.
 	if err != nil {
 		return nil, httperrors.NewGeneralError(err)
 	}
-	err = manager.TableSpec().Insert(model)
+	err = manager.TableSpec().InsertOrUpdate(model)
 	if err != nil {
 		return nil, httperrors.NewGeneralError(err)
 	}
@@ -935,23 +993,28 @@ func (dispatcher *DBModelDispatcher) FetchCreateHeaderData(ctx context.Context,
 	return dispatcher.modelManager.FetchCreateHeaderData(ctx, header)
 }
 
-func (dispatcher *DBModelDispatcher) Create(ctx context.Context, query jsonutils.JSONObject, data jsonutils.JSONObject, ctxId string) (jsonutils.JSONObject, error) {
+func (dispatcher *DBModelDispatcher) Create(ctx context.Context, query jsonutils.JSONObject, data jsonutils.JSONObject, ctxIds []dispatcher.SResourceContext) (jsonutils.JSONObject, error) {
 	userCred := fetchUserCredential(ctx)
 
-	ownerProjId, err := fetchOwnerProjectId(ctx, dispatcher.modelManager, userCred, data)
-	if err != nil {
-		return nil, httperrors.NewGeneralError(err)
+	var ownerProjId string
+	var err error
+
+	if len(dispatcher.modelManager.GetOwnerId(userCred)) > 0 {
+		ownerProjId, err = fetchOwnerProjectId(ctx, dispatcher.modelManager, userCred, data)
+		if err != nil {
+			return nil, httperrors.NewGeneralError(err)
+		}
 	}
 
-	if len(ctxId) > 0 {
+	if len(ctxIds) > 0 {
 		dataDict, ok := data.(*jsonutils.JSONDict)
 		if !ok {
 			log.Errorf("fail to convert body into jsondict")
 			return nil, fmt.Errorf("fail to parse body")
 		}
-		data, err = fetchContextObjectId(dispatcher.modelManager, ctx, userCred, ctxId, dataDict)
+		data, err = fetchContextObjectsIds(dispatcher.modelManager, ctx, userCred, ctxIds, dataDict)
 		if err != nil {
-			log.Errorf("fail to find context object %s", ctxId)
+			log.Errorf("fail to find context object %s", ctxIds)
 			return nil, err
 		}
 	}
@@ -1005,7 +1068,7 @@ func expandMultiCreateParams(data jsonutils.JSONObject, count int) ([]jsonutils.
 	return ret, nil
 }
 
-func (dispatcher *DBModelDispatcher) BatchCreate(ctx context.Context, query jsonutils.JSONObject, data jsonutils.JSONObject, count int, ctxId string) ([]modules.SubmitResult, error) {
+func (dispatcher *DBModelDispatcher) BatchCreate(ctx context.Context, query jsonutils.JSONObject, data jsonutils.JSONObject, count int, ctxIds []dispatcher.SResourceContext) ([]modules.SubmitResult, error) {
 	userCred := fetchUserCredential(ctx)
 
 	ownerProjId, err := fetchOwnerProjectId(ctx, dispatcher.modelManager, userCred, data)
@@ -1013,12 +1076,12 @@ func (dispatcher *DBModelDispatcher) BatchCreate(ctx context.Context, query json
 		return nil, httperrors.NewGeneralError(err)
 	}
 
-	if len(ctxId) > 0 {
+	if len(ctxIds) > 0 {
 		dataDict, ok := data.(*jsonutils.JSONDict)
 		if !ok {
 			return nil, fmt.Errorf("fail to parse body")
 		}
-		data, err = fetchContextObjectId(dispatcher.modelManager, ctx, userCred, ctxId, dataDict)
+		data, err = fetchContextObjectsIds(dispatcher.modelManager, ctx, userCred, ctxIds, dataDict)
 		if err != nil {
 			return nil, err
 		}
@@ -1160,27 +1223,55 @@ func (dispatcher *DBModelDispatcher) PerformAction(ctx context.Context, idStr st
 	lockman.LockObject(ctx, model)
 	defer lockman.ReleaseObject(ctx, model)
 
-	modelValue := reflect.ValueOf(model)
-	result, err := objectPerformAction(dispatcher, model, modelValue, ctx, userCred, action, query, data)
+	return objectPerformAction(dispatcher, model, reflect.ValueOf(model), ctx, userCred, action, query, data)
+}
+
+func objectPerformAction(dispatcher *DBModelDispatcher, model IModel, modelValue reflect.Value, ctx context.Context, userCred mcclient.TokenCredential, action string, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	return reflectDispatcher(dispatcher, model, modelValue, ctx, userCred, policy.PolicyActionPerform, "PerformAction", "Perform", action, query, data)
+}
+
+func reflectDispatcher(
+	dispatcher *DBModelDispatcher,
+	model IModel,
+	modelValue reflect.Value,
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	operator string,
+	generalFuncName string,
+	funcPrefix string,
+	spec string,
+	query jsonutils.JSONObject,
+	data jsonutils.JSONObject,
+) (jsonutils.JSONObject, error) {
+	result, err := reflectDispatcherInternal(
+		dispatcher, model, modelValue, ctx, userCred, operator, generalFuncName, funcPrefix, spec, query, data)
 	if err == nil && result == nil {
 		return getItemDetails(dispatcher.modelManager, model, ctx, userCred, query)
 	} else {
 		return result, err
 	}
 }
-
-func objectPerformAction(dispatcher *DBModelDispatcher, model IModel, modelValue reflect.Value, ctx context.Context, userCred mcclient.TokenCredential, action string, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
-	const generalFuncName = "PerformAction"
-	// const generalAllowFuncName = "AllowPerformAction"
-
+func reflectDispatcherInternal(
+	dispatcher *DBModelDispatcher,
+	model IModel,
+	modelValue reflect.Value,
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	operator string,
+	generalFuncName string,
+	funcPrefix string,
+	spec string,
+	query jsonutils.JSONObject,
+	data jsonutils.JSONObject,
+) (jsonutils.JSONObject, error) {
 	isGeneral := false
-	funcName := fmt.Sprintf("Perform%s", utils.Kebab2Camel(action, "-"))
+	funcName := fmt.Sprintf("%s%s", funcPrefix, utils.Kebab2Camel(spec, "-"))
 	funcValue := modelValue.MethodByName(funcName)
 
 	if !funcValue.IsValid() || funcValue.IsNil() {
 		funcValue = modelValue.MethodByName(generalFuncName)
 		if !funcValue.IsValid() || funcValue.IsNil() {
-			msg := fmt.Sprintf("%s perform action %s not found", dispatcher.Keyword(), action)
+			msg := fmt.Sprintf("%s %s %s not found", dispatcher.Keyword(), operator, spec)
 			log.Errorf(msg)
 			return nil, httperrors.NewActionNotFoundError(msg)
 		} else {
@@ -1195,7 +1286,7 @@ func objectPerformAction(dispatcher *DBModelDispatcher, model IModel, modelValue
 		params = []reflect.Value{
 			reflect.ValueOf(ctx),
 			reflect.ValueOf(userCred),
-			reflect.ValueOf(action),
+			reflect.ValueOf(spec),
 			reflect.ValueOf(query),
 			reflect.ValueOf(data),
 		}
@@ -1212,15 +1303,15 @@ func objectPerformAction(dispatcher *DBModelDispatcher, model IModel, modelValue
 	if consts.IsRbacEnabled() {
 		if model == nil {
 			ownerProjId, _ := fetchOwnerProjectId(ctx, dispatcher.modelManager, userCred, data)
-			isAllow = isClassActionRbacAllowed(dispatcher.modelManager, userCred, ownerProjId, policy.PolicyActionPerform, action)
+			isAllow = isClassActionRbacAllowed(dispatcher.modelManager, userCred, ownerProjId, operator, spec)
 		} else {
-			isAllow = isObjectRbacAllowed(dispatcher.modelManager, model, userCred, policy.PolicyActionPerform, action)
+			isAllow = isObjectRbacAllowed(dispatcher.modelManager, model, userCred, operator, spec)
 		}
 	} else {
 		allowFuncName := "Allow" + funcName
 		allowFuncValue := modelValue.MethodByName(allowFuncName)
 		if !allowFuncValue.IsValid() || allowFuncValue.IsNil() {
-			msg := fmt.Sprintf("%s allow perform action %s not found", dispatcher.Keyword(), action)
+			msg := fmt.Sprintf("%s allow %s %s not found", dispatcher.Keyword(), operator, spec)
 			log.Errorf(msg)
 			return nil, httperrors.NewActionNotFoundError(msg)
 		}
@@ -1233,7 +1324,7 @@ func objectPerformAction(dispatcher *DBModelDispatcher, model IModel, modelValue
 		isAllow = outs[0].Bool()
 	}
 	if !isAllow {
-		return nil, httperrors.NewForbiddenError("%s not allow to perform action %s", dispatcher.Keyword(), action)
+		return nil, httperrors.NewForbiddenError("%s not allow to %s %s", dispatcher.Keyword(), operator, spec)
 	}
 
 	outs := funcValue.Call(params)
@@ -1311,7 +1402,7 @@ func (dispatcher *DBModelDispatcher) FetchUpdateHeaderData(ctx context.Context,
 	return dispatcher.modelManager.FetchUpdateHeaderData(ctx, header)
 }
 
-func (dispatcher *DBModelDispatcher) Update(ctx context.Context, idStr string, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+func (dispatcher *DBModelDispatcher) Update(ctx context.Context, idStr string, query jsonutils.JSONObject, data jsonutils.JSONObject, ctxIds []dispatcher.SResourceContext) (jsonutils.JSONObject, error) {
 	userCred := fetchUserCredential(ctx)
 	model, err := fetchItem(dispatcher.modelManager, ctx, userCred, idStr, nil)
 	if err == sql.ErrNoRows {
@@ -1330,10 +1421,38 @@ func (dispatcher *DBModelDispatcher) Update(ctx context.Context, idStr string, q
 		return nil, httperrors.NewForbiddenError("Not allow to update item")
 	}
 
+	if len(ctxIds) > 0 {
+		ctxObjs, err := fetchContextObjects(dispatcher.modelManager, ctx, userCred, ctxIds)
+		if err != nil {
+			return nil, httperrors.NewGeneralError(err)
+		}
+
+		return model.UpdateInContext(ctx, userCred, ctxObjs, query, data)
+	} else {
+		lockman.LockObject(ctx, model)
+		defer lockman.ReleaseObject(ctx, model)
+
+		return updateItem(dispatcher.modelManager, model, ctx, userCred, query, data)
+	}
+}
+
+func (dispatcher *DBModelDispatcher) UpdateSpec(ctx context.Context, idStr string, spec string, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	userCred := fetchUserCredential(ctx)
+	model, err := fetchItem(dispatcher.modelManager, ctx, userCred, idStr, nil)
+	if err == sql.ErrNoRows {
+		return nil, httperrors.NewResourceNotFoundError2(dispatcher.modelManager.Keyword(), idStr)
+	} else if err != nil {
+		return nil, httperrors.NewGeneralError(err)
+	}
+
 	lockman.LockObject(ctx, model)
 	defer lockman.ReleaseObject(ctx, model)
 
-	return updateItem(dispatcher.modelManager, model, ctx, userCred, query, data)
+	return objectUpdateSpec(dispatcher, model, reflect.ValueOf(model), ctx, userCred, spec, query, data)
+}
+
+func objectUpdateSpec(dispatcher *DBModelDispatcher, model IModel, modelValue reflect.Value, ctx context.Context, userCred mcclient.TokenCredential, spec string, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	return reflectDispatcher(dispatcher, model, modelValue, ctx, userCred, policy.PolicyActionUpdate, "UpdateSpec", "Update", spec, query, data)
 }
 
 func DeleteModel(ctx context.Context, userCred mcclient.TokenCredential, item IModel) error {
@@ -1352,18 +1471,6 @@ func DeleteModel(ctx context.Context, userCred mcclient.TokenCredential, item IM
 
 func deleteItem(manager IModelManager, model IModel, ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
 	// log.Debugf("deleteItem %s", jsonutils.Marshal(model))
-
-	var isAllow bool
-	if consts.IsRbacEnabled() {
-		isAllow = isObjectRbacAllowed(manager, model, userCred, policy.PolicyActionDelete)
-	} else {
-		isAllow = model.AllowDeleteItem(ctx, userCred, query, data)
-	}
-	if !isAllow {
-		log.Errorf("not allow to delete")
-		return nil, httperrors.NewForbiddenError("%s(%s) not allow to delete", manager.KeywordPlural(), model.GetId())
-	}
-
 	err := model.ValidateDeleteCondition(ctx)
 	if err != nil {
 		log.Errorf("validate delete condition error: %s", err)
@@ -1398,7 +1505,7 @@ func deleteItem(manager IModelManager, model IModel, ctx context.Context, userCr
 	return details, nil
 }
 
-func (dispatcher *DBModelDispatcher) Delete(ctx context.Context, idstr string, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+func (dispatcher *DBModelDispatcher) Delete(ctx context.Context, idstr string, query jsonutils.JSONObject, data jsonutils.JSONObject, ctxIds []dispatcher.SResourceContext) (jsonutils.JSONObject, error) {
 	userCred := fetchUserCredential(ctx)
 	model, err := fetchItem(dispatcher.modelManager, ctx, userCred, idstr, nil)
 	if err == sql.ErrNoRows {
@@ -1408,8 +1515,47 @@ func (dispatcher *DBModelDispatcher) Delete(ctx context.Context, idstr string, q
 	}
 	// log.Debugf("Delete %s", model.GetShortDesc(ctx))
 
+	var isAllow bool
+	if consts.IsRbacEnabled() {
+		isAllow = isObjectRbacAllowed(dispatcher.modelManager, model, userCred, policy.PolicyActionDelete)
+	} else {
+		isAllow = model.AllowDeleteItem(ctx, userCred, query, data)
+	}
+	if !isAllow {
+		log.Errorf("not allow to delete")
+		return nil, httperrors.NewForbiddenError("%s(%s) not allow to delete", dispatcher.modelManager.KeywordPlural(), model.GetId())
+	}
+
+	if len(ctxIds) > 0 {
+		ctxObjs, err := fetchContextObjects(dispatcher.modelManager, ctx, userCred, ctxIds)
+		if err != nil {
+			return nil, httperrors.NewGeneralError(err)
+		}
+
+		return model.DeleteInContext(ctx, userCred, ctxObjs, query, data)
+	} else {
+		lockman.LockObject(ctx, model)
+		defer lockman.ReleaseObject(ctx, model)
+
+		return deleteItem(dispatcher.modelManager, model, ctx, userCred, query, data)
+	}
+}
+
+func (dispatcher *DBModelDispatcher) DeleteSpec(ctx context.Context, idstr string, spec string, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	userCred := fetchUserCredential(ctx)
+	model, err := fetchItem(dispatcher.modelManager, ctx, userCred, idstr, nil)
+	if err == sql.ErrNoRows {
+		return nil, httperrors.NewResourceNotFoundError2(dispatcher.modelManager.Keyword(), idstr)
+	} else if err != nil {
+		return nil, httperrors.NewGeneralError(err)
+	}
+
 	lockman.LockObject(ctx, model)
 	defer lockman.ReleaseObject(ctx, model)
 
-	return deleteItem(dispatcher.modelManager, model, ctx, userCred, query, data)
+	return objectDeleteSpec(dispatcher, model, reflect.ValueOf(model), ctx, userCred, spec, query, data)
+}
+
+func objectDeleteSpec(dispatcher *DBModelDispatcher, model IModel, modelValue reflect.Value, ctx context.Context, userCred mcclient.TokenCredential, spec string, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	return reflectDispatcher(dispatcher, model, modelValue, ctx, userCred, policy.PolicyActionDelete, "DeleteSpec", "Delete", spec, query, data)
 }

pkg/cloudcommon/db/db_joint_dispatcher.go

@@ -123,7 +123,7 @@ func (dispatcher *DBJointModelDispatcher) _listJoint(ctx context.Context, userCr
 		return nil, httperrors.NewForbiddenError("Not allow to list")
 	}
 
-	items, err := ListItems(dispatcher.JointModelManager(), ctx, userCred, queryDict, "")
+	items, err := ListItems(dispatcher.JointModelManager(), ctx, userCred, queryDict, nil)
 	if err != nil {
 		log.Errorf("Fail to list items: %s", err)
 		return nil, httperrors.NewGeneralError(err)

pkg/cloudcommon/db/interface.go

@@ -30,7 +30,7 @@ import (
 type IModelManager interface {
 	lockman.ILockedClass
 
-	GetContextManager() []IModelManager
+	GetContextManagers() [][]IModelManager
 
 	// Table() *sqlchemy.STable
 	TableSpec() *sqlchemy.STableSpec
@@ -128,6 +128,8 @@ type IModel interface {
 	PreUpdate(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject)
 	PostUpdate(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject)
 
+	UpdateInContext(ctx context.Context, userCred mcclient.TokenCredential, ctxObjs []IModel, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error)
+
 	// delete hooks
 	AllowDeleteItem(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool
 	ValidateDeleteCondition(ctx context.Context) error
@@ -137,6 +139,8 @@ type IModel interface {
 	Delete(ctx context.Context, userCred mcclient.TokenCredential) error
 	PostDelete(ctx context.Context, userCred mcclient.TokenCredential)
 
+	DeleteInContext(ctx context.Context, userCred mcclient.TokenCredential, ctxObjs []IModel, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error)
+
 	GetOwnerProjectId() string
 	IsSharable() bool
 

pkg/cloudcommon/db/modelbase.go

@@ -64,7 +64,7 @@ func (manager *SModelBaseManager) KeywordPlural() string {
 	return manager.keywordPlural
 }
 
-func (manager *SModelBaseManager) GetContextManager() []IModelManager {
+func (manager *SModelBaseManager) GetContextManagers() [][]IModelManager {
 	return nil
 }
 
@@ -344,3 +344,11 @@ func (model *SModelBase) IsSharable() bool {
 func (model *SModelBase) CustomizedGetDetailsBody(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) {
 	return nil, nil
 }
+
+func (model *SModelBase) UpdateInContext(ctx context.Context, userCred mcclient.TokenCredential, ctxObjs []IModel, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	return nil, nil
+}
+
+func (model *SModelBase) DeleteInContext(ctx context.Context, userCred mcclient.TokenCredential, ctxObjs []IModel, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	return nil, nil
+}

pkg/cloudcommon/db/models.go

@@ -59,15 +59,26 @@ func mustCheckModelManager(modelMan IModelManager) {
 
 func CheckSync(autoSync bool) bool {
 	log.Infof("Start check database ...")
+	examinedTables := make(map[string]bool)
+	allDropFKSqls := make([]string, 0)
 	allSqls := make([]string, 0)
 	for modelName, modelMan := range globalTables {
 		log.Infof("# check table of model %s", modelName)
 		tableSpec := modelMan.TableSpec()
+		if _, ok := examinedTables[tableSpec.Name()]; ok {
+			continue
+		}
+		examinedTables[tableSpec.Name()] = true
+		dropFKSqls := tableSpec.DropForeignKeySQL()
+		if len(dropFKSqls) > 0 {
+			allDropFKSqls = append(allDropFKSqls, dropFKSqls...)
+		}
 		sqls := tableSpec.SyncSQL()
-		for _, sql := range sqls {
-			allSqls = append(allSqls, sql)
+		if len(sqls) > 0 {
+			allSqls = append(allSqls, sqls...)
 		}
 	}
+	allSqls = append(allDropFKSqls, allSqls...)
 	if len(allSqls) > 0 {
 		if autoSync {
 			err := commitSqlDIffs(allSqls)

pkg/cloudcommon/db/opslog.go

@@ -354,6 +354,8 @@ func (manager *SOpsLogManager) ListItemFilter(ctx context.Context, q *sqlchemy.S
 	if objIds != nil && len(objIds) > 0 {
 		q = q.Filter(sqlchemy.OR(sqlchemy.In(q.Field("obj_id"), objIds), sqlchemy.In(q.Field("obj_name"), objIds)))
 	}
+	queryDict := query.(*jsonutils.JSONDict)
+	queryDict.Remove("obj_id")
 	action := jsonutils.GetQueryStringArray(query, "action")
 	if action != nil && len(action) > 0 {
 		q = q.Filter(sqlchemy.In(q.Field("action"), action))

pkg/cloudcommon/db/rbac.go

@@ -186,9 +186,23 @@ func IsAdminAllowUpdate(userCred mcclient.TokenCredential, obj IModel) bool {
 	return userCred.IsAdminAllow(consts.GetServiceType(), obj.KeywordPlural(), policy.PolicyActionUpdate)
 }
 
+func IsAdminAllowUpdateSpec(userCred mcclient.TokenCredential, obj IModel, spec string) bool {
+	if userCred == nil {
+		return false
+	}
+	return userCred.IsAdminAllow(consts.GetServiceType(), obj.KeywordPlural(), policy.PolicyActionUpdate, spec)
+}
+
 func IsAdminAllowDelete(userCred mcclient.TokenCredential, obj IModel) bool {
 	if userCred == nil {
 		return false
 	}
 	return userCred.IsAdminAllow(consts.GetServiceType(), obj.KeywordPlural(), policy.PolicyActionDelete)
 }
+
+func IsAdminAllowDeleteSpec(userCred mcclient.TokenCredential, obj IModel, spec string) bool {
+	if userCred == nil {
+		return false
+	}
+	return userCred.IsAdminAllow(consts.GetServiceType(), obj.KeywordPlural(), policy.PolicyActionDelete, spec)
+}

pkg/cloudcommon/db/resourcebase.go

@@ -31,7 +31,7 @@ type SResourceBase struct {
 	UpdatedAt     time.Time `nullable:"false" updated_at:"true" list:"user"`
 	UpdateVersion int       `default:"0" nullable:"false" auto_version:"true" list:"user"`
 	DeletedAt     time.Time ``
-	Deleted       bool      `nullable:"false" default:"false" index:"true"`
+	Deleted       bool      `nullable:"false" default:"false"`
 }
 
 type SResourceBaseManager struct {

pkg/cloudcommon/db/standalone.go

@@ -31,6 +31,12 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient"
 )
 
+type UUIDGenerator func() string
+
+var (
+	DefaultUUIDGenerator = stringutils.UUID4
+)
+
 type SStandaloneResourceBase struct {
 	SResourceBase
 
@@ -45,7 +51,7 @@ type SStandaloneResourceBase struct {
 
 func (model *SStandaloneResourceBase) BeforeInsert() {
 	if len(model.Id) == 0 {
-		model.Id = stringutils.UUID4()
+		model.Id = DefaultUUIDGenerator()
 	}
 }
 

pkg/cloudcommon/db/taskman/tasks.go

@@ -750,13 +750,9 @@ func (manager *STaskManager) QueryTasksOfObject(obj db.IStandaloneModel, since t
 		}
 	}
 
-	// subq1 and subq2 do not intersect for the fact that they have
-	// different condition on tasks_tbl.obj_id field
-	uq := sqlchemy.Union(subq1, subq2)
-	uq = uq.Desc("created_at")
-
-	q := uq.SubQuery().Query()
-	return q
+	// subq1 and subq2 do not overlap for the fact that they have
+	// different conditions on tasks_tbl.obj_id field
+	return sqlchemy.Union(subq1, subq2).Query().Desc("created_at")
 }
 
 func (manager *STaskManager) IsInTask(obj db.IStandaloneModel) bool {

pkg/cloudcommon/etcd/handler/dispatcher.go

@@ -26,6 +26,7 @@ import (
 	"yunion.io/x/pkg/utils"
 
 	"yunion.io/x/onecloud/pkg/appsrv"
+	"yunion.io/x/onecloud/pkg/appsrv/dispatcher"
 	"yunion.io/x/onecloud/pkg/cloudcommon/etcd"
 	"yunion.io/x/onecloud/pkg/cloudcommon/etcd/models/base"
 	"yunion.io/x/onecloud/pkg/cloudcommon/policy"
@@ -56,7 +57,7 @@ func (disp *SEtcdModelHandler) KeywordPlural() string {
 	return disp.manager.KeywordPlural()
 }
 
-func (disp *SEtcdModelHandler) ContextKeywordPlural() []string {
+func (disp *SEtcdModelHandler) ContextKeywordPlurals() [][]string {
 	return nil
 }
 
@@ -72,7 +73,7 @@ func (disp *SEtcdModelHandler) FetchUpdateHeaderData(ctx context.Context, header
 	return disp.manager.FetchUpdateHeaderData(ctx, header)
 }
 
-func (disp *SEtcdModelHandler) List(ctx context.Context, query jsonutils.JSONObject, ctxId string) (*modules.ListResult, error) {
+func (disp *SEtcdModelHandler) List(ctx context.Context, query jsonutils.JSONObject, ctxIds []dispatcher.SResourceContext) (*modules.ListResult, error) {
 	objs, err := disp.manager.AllJson(ctx)
 	if err != nil {
 		return nil, httperrors.NewGeneralError(err)
@@ -166,11 +167,11 @@ func (disp *SEtcdModelHandler) GetSpecific(ctx context.Context, idstr string, sp
 	}
 }
 
-func (disp *SEtcdModelHandler) Create(ctx context.Context, query jsonutils.JSONObject, data jsonutils.JSONObject, ctxId string) (jsonutils.JSONObject, error) {
+func (disp *SEtcdModelHandler) Create(ctx context.Context, query jsonutils.JSONObject, data jsonutils.JSONObject, ctxIds []dispatcher.SResourceContext) (jsonutils.JSONObject, error) {
 	return nil, httperrors.NewNotImplementedError("not implemented")
 }
 
-func (disp *SEtcdModelHandler) BatchCreate(ctx context.Context, query jsonutils.JSONObject, data jsonutils.JSONObject, count int, ctxId string) ([]modules.SubmitResult, error) {
+func (disp *SEtcdModelHandler) BatchCreate(ctx context.Context, query jsonutils.JSONObject, data jsonutils.JSONObject, count int, ctxIds []dispatcher.SResourceContext) ([]modules.SubmitResult, error) {
 	return nil, httperrors.NewNotImplementedError("not implemented")
 }
 
@@ -182,10 +183,18 @@ func (disp *SEtcdModelHandler) PerformAction(ctx context.Context, idstr string,
 	return nil, httperrors.NewNotImplementedError("not implemented")
 }
 
-func (disp *SEtcdModelHandler) Update(ctx context.Context, idstr string, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+func (disp *SEtcdModelHandler) Update(ctx context.Context, idstr string, query jsonutils.JSONObject, data jsonutils.JSONObject, ctxIds []dispatcher.SResourceContext) (jsonutils.JSONObject, error) {
 	return nil, httperrors.NewNotImplementedError("not implemented")
 }
 
-func (disp *SEtcdModelHandler) Delete(ctx context.Context, idstr string, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+func (disp *SEtcdModelHandler) Delete(ctx context.Context, idstr string, query jsonutils.JSONObject, data jsonutils.JSONObject, ctxIds []dispatcher.SResourceContext) (jsonutils.JSONObject, error) {
+	return nil, httperrors.NewNotImplementedError("not implemented")
+}
+
+func (disp *SEtcdModelHandler) UpdateSpec(ctx context.Context, idstr string, spec string, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	return nil, httperrors.NewNotImplementedError("not implemented")
+}
+
+func (disp *SEtcdModelHandler) DeleteSpec(ctx context.Context, idstr string, spec string, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
 	return nil, httperrors.NewNotImplementedError("not implemented")
 }

pkg/cloudcommon/options/options.go

@@ -32,7 +32,9 @@ import (
 	"yunion.io/x/onecloud/pkg/util/atexit"
 )
 
-type CommonOptions struct {
+type BaseOptions struct {
+	Region string `help:"Region name or ID" alias:"auth-region"`
+
 	Port    int    `help:"The port that the service runs on" alias:"bind-port"`
 	Address string `help:"The IP address to serve on (set to 0.0.0.0 for all interfaces)" default:"0.0.0.0" alias:"bind-host"`
 
@@ -40,29 +42,20 @@ type CommonOptions struct {
 	LogVerboseLevel int    `help:"log verbosity level" default:"0"`
 	LogFilePrefix   string `help:"prefix of log files"`
 
-	Region             string   `help:"Region name or ID" alias:"auth-region"`
-	AuthURL            string   `help:"Keystone auth URL" alias:"auth-uri"`
-	AdminUser          string   `help:"Admin username"`
-	AdminDomain        string   `help:"Admin user domain"`
-	AdminPassword      string   `help:"Admin password" alias:"admin-passwd"`
-	AdminProject       string   `help:"Admin project" default:"system" alias:"admin-tenant-name"`
-	CorsHosts          []string `help:"List of hostname that allow CORS"`
-	AuthTokenCacheSize uint32   `help:"Auth token Cache Size" default:"2048"`
-	TempPath           string   `help:"Path for store temp file, at least 40G space" default:"/opt/yunion/tmp"`
-
-	DebugClient bool `help:"Switch on/off mcclient debugs" default:"false"`
+	CorsHosts []string `help:"List of hostname that allow CORS"`
+	TempPath  string   `help:"Path for store temp file, at least 40G space" default:"/opt/yunion/tmp"`
 
 	ApplicationID      string `help:"Application ID"`
 	RequestWorkerCount int    `default:"4" help:"Request worker thread count, default is 4"`
 
-	NotifyAdminUsers  []string `default:"sysadmin" help:"System administrator user ID or name to notify system events, if domain is not default, specify domain as prefix ending with double backslash, e.g. domain\\\\user"`
-	NotifyAdminGroups []string `help:"System administrator group ID or name to notify system events, if domain is not default, specify domain as prefix ending with double backslash, e.g. domain\\\\group"`
-
 	EnableSsl   bool   `help:"Enable https"`
 	SslCaCerts  string `help:"ssl certificate ca root file, separating ca and cert file is not encouraged" alias:"ca-file"`
 	SslCertfile string `help:"ssl certification file, normally combines all the certificates in the chain" alias:"cert-file"`
 	SslKeyfile  string `help:"ssl certification private key file" alias:"key-file"`
 
+	NotifyAdminUsers  []string `default:"sysadmin" help:"System administrator user ID or name to notify system events, if domain is not default, specify domain as prefix ending with double backslash, e.g. domain\\\\user"`
+	NotifyAdminGroups []string `help:"System administrator group ID or name to notify system events, if domain is not default, specify domain as prefix ending with double backslash, e.g. domain\\\\group"`
+
 	EnableRbac                       bool `help:"Switch on Role-based Access Control" default:"true"`
 	RbacDebug                        bool `help:"turn on rbac debug log" default:"false"`
 	RbacPolicySyncPeriodSeconds      int  `help:"policy sync interval in seconds, default 15 minutes" default:"900"`
@@ -71,8 +64,21 @@ type CommonOptions struct {
 	structarg.BaseOptions
 }
 
+type CommonOptions struct {
+	AuthURL            string `help:"Keystone auth URL" alias:"auth-uri"`
+	AdminUser          string `help:"Admin username"`
+	AdminDomain        string `help:"Admin user domain"`
+	AdminPassword      string `help:"Admin password" alias:"admin-passwd"`
+	AdminProject       string `help:"Admin project" default:"system" alias:"admin-tenant-name"`
+	AuthTokenCacheSize uint32 `help:"Auth token Cache Size" default:"2048"`
+
+	DebugClient bool `help:"Switch on/off mcclient debugs" default:"false"`
+
+	BaseOptions
+}
+
 type DBOptions struct {
-	SqlConnection string `help:"SQL connection string"`
+	SqlConnection string `help:"SQL connection string" alias:"connection"`
 	AutoSyncTable bool   `help:"Automatically synchronize table changes if differences are detected"`
 
 	GlobalVirtualResourceNamespace bool `help:"Per project namespace or global namespace for virtual resources"`
@@ -105,7 +111,7 @@ func ParseOptions(optStruct interface{}, args []string, configFileName string, s
 		log.Fatalf("Parse arguments error: %v", err)
 	}
 
-	var optionsRef *CommonOptions
+	var optionsRef *BaseOptions
 
 	err = reflectutils.FindAnonymouStructPointer(optStruct, &optionsRef)
 	if err != nil {
@@ -178,5 +184,7 @@ func ParseOptions(optStruct interface{}, args []string, configFileName string, s
 
 	log.V(10).Debugf("Parsed options: %#v", optStruct)
 
-	consts.SetRegion(optionsRef.Region)
+	if len(optionsRef.Region) > 0 {
+		consts.SetRegion(optionsRef.Region)
+	}
 }

pkg/cloudcommon/policy/defaults.go

@@ -14,7 +14,9 @@
 
 package policy
 
-import "yunion.io/x/onecloud/pkg/util/rbacutils"
+import (
+	"yunion.io/x/onecloud/pkg/util/rbacutils"
+)
 
 var (
 	defaultRules = []rbacutils.SRbacRule{

pkg/cloudcommon/policy/policy.go

@@ -25,6 +25,7 @@ import (
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 
+	"yunion.io/x/onecloud/pkg/appsrv"
 	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
@@ -46,14 +47,22 @@ const (
 	PolicyActionPerform = "perform"
 )
 
+type PolicyFetchFunc func() (map[string]rbacutils.SRbacPolicy, map[string]rbacutils.SRbacPolicy, error)
+
 var (
-	PolicyManager *SPolicyManager
+	PolicyManager        *SPolicyManager
+	DefaultPolicyFetcher PolicyFetchFunc
+
+	syncWorkerManager *appsrv.SWorkerManager
 )
 
 func init() {
 	PolicyManager = &SPolicyManager{
 		lock: &sync.Mutex{},
 	}
+	DefaultPolicyFetcher = remotePolicyFetcher
+
+	syncWorkerManager = appsrv.NewWorkerManager("sync_policy_worker", 1, 1000, false)
 }
 
 type SPolicyManager struct {
@@ -62,6 +71,8 @@ type SPolicyManager struct {
 	defaultPolicy *rbacutils.SRbacPolicy
 	lastSync      time.Time
 
+	defaultAdminPolicy *rbacutils.SRbacPolicy
+
 	failedRetryInterval time.Duration
 	refreshInterval     time.Duration
 
@@ -92,7 +103,7 @@ func parseJsonPolicy(obj jsonutils.JSONObject) (string, rbacutils.SRbacPolicy, e
 	return typeStr, policy, nil
 }
 
-func fetchPolicies() (map[string]rbacutils.SRbacPolicy, map[string]rbacutils.SRbacPolicy, error) {
+func remotePolicyFetcher() (map[string]rbacutils.SRbacPolicy, map[string]rbacutils.SRbacPolicy, error) {
 	s := auth.GetAdminSession(context.Background(), consts.GetRegion(), "v1")
 
 	policies := make(map[string]rbacutils.SRbacPolicy)
@@ -144,11 +155,15 @@ func (manager *SPolicyManager) start(refreshInterval time.Duration, retryInterva
 	}
 
 	manager.cache = hashcache.NewCache(2048, manager.refreshInterval/2)
-	manager.sync()
+	manager.SyncOnce()
 }
 
-func (manager *SPolicyManager) SyncOnce() error {
-	policies, adminPolicies, err := fetchPolicies()
+func (manager *SPolicyManager) SyncOnce() {
+	syncWorkerManager.Run(manager.sync, nil, nil)
+}
+
+func (manager *SPolicyManager) doSync() error {
+	policies, adminPolicies, err := DefaultPolicyFetcher()
 	if err != nil {
 		log.Errorf("sync rbac policy failed: %s", err)
 		return err
@@ -166,15 +181,19 @@ func (manager *SPolicyManager) SyncOnce() error {
 	return nil
 }
 
+func (manager *SPolicyManager) RegisterDefaultAdminPolicy(policy *rbacutils.SRbacPolicy) {
+	manager.defaultAdminPolicy = policy
+}
+
 func (manager *SPolicyManager) sync() {
-	err := manager.SyncOnce()
+	err := manager.doSync()
 	var interval time.Duration
 	if err != nil {
 		interval = manager.failedRetryInterval
 	} else {
 		interval = manager.refreshInterval
 	}
-	time.AfterFunc(interval, manager.sync)
+	time.AfterFunc(interval, manager.SyncOnce)
 }
 
 func queryKey(isAdmin bool, userCred mcclient.TokenCredential, service string, resource string, action string, extra ...string) string {
@@ -278,6 +297,14 @@ func (manager *SPolicyManager) allowWithoutCache(isAdmin bool, userCred mcclient
 			}
 		}
 	}
+	if isAdmin && manager.defaultAdminPolicy != nil && manager.defaultAdminPolicy.Match(userCred) {
+		rule := manager.defaultAdminPolicy.GetMatchRule(service, resource, action, extra...)
+		if rule != nil {
+			if currentPriv.StricterThan(rule.Result) {
+				currentPriv = rule.Result
+			}
+		}
+	}
 	if consts.IsRbacDebug() {
 		log.Debugf("[RBAC: %v] %s %s %s %#v permission %s userCred: %s", isAdmin, service, resource, action, extra, currentPriv, userCred)
 	}

pkg/cloudcommon/policy/token.go

@@ -15,10 +15,14 @@
 package policy
 
 import (
+	"context"
+
+	"yunion.io/x/log"
 	"yunion.io/x/pkg/gotypes"
 
 	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	"yunion.io/x/onecloud/pkg/util/rbacutils"
 )
 
@@ -65,3 +69,11 @@ func FilterPolicyCredential(token mcclient.TokenCredential) mcclient.TokenCreden
 		return &SPolicyTokenCredential{TokenCredential: token}
 	}
 }
+
+func FetchUserCredential(ctx context.Context) mcclient.TokenCredential {
+	token := auth.FetchUserCredential(ctx, FilterPolicyCredential)
+	if token == nil && !consts.IsRbacEnabled() {
+		log.Fatalf("user token credential not found?")
+	}
+	return token
+}

pkg/cloudcommon/validators/validators.go

@@ -34,6 +34,7 @@ import (
 	"yunion.io/x/pkg/util/regutils"
 	"yunion.io/x/sqlchemy"
 
+	identity "yunion.io/x/onecloud/pkg/apis/identity"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/util/choices"
 )
@@ -395,6 +396,10 @@ func (v *ValidatorModelIdOrName) GetTenantId() string {
 	return v.ProjectId
 }
 
+func (v *ValidatorModelIdOrName) GetProjectDomainId() string {
+	return identity.DEFAULT_DOMAIN_ID
+}
+
 func (v *ValidatorModelIdOrName) getValue() interface{} {
 	return v.Model
 }

pkg/cloudir/service/service.go

@@ -28,6 +28,7 @@ import (
 
 func StartService() {
 	opts := &options.Options
+	baseOpts := &opts.BaseOptions
 	commonOpts := &opts.CommonOptions
 	common_options.ParseOptions(opts, os.Args, "cloudir.conf", "cloudir")
 
@@ -41,11 +42,11 @@ func StartService() {
 		return
 	}
 
-	app := app_common.InitApp(commonOpts, false)
+	app := app_common.InitApp(baseOpts, false)
 	cloudcommon.AppDBInit(app)
 	initHandlers(app)
 
-	app_common.ServeForeverWithCleanup(app, commonOpts, func() {
+	app_common.ServeForeverWithCleanup(app, baseOpts, func() {
 		etcd.CloseDefaultEtcdClient()
 	})
 }

pkg/cloutpost/service/service.go

@@ -33,6 +33,7 @@ const (
 
 func StartService() {
 	opts := &options.Options
+	baseOpts := &opts.BaseOptions
 	commonOpts := &opts.CommonOptions
 	common_options.ParseOptions(opts, os.Args, "cloutpost.conf", SERVICE_TYPE)
 
@@ -46,7 +47,7 @@ func StartService() {
 	}
 	defer etcd.CloseDefaultEtcdClient()
 
-	app := app_common.InitApp(commonOpts, false)
+	app := app_common.InitApp(baseOpts, false)
 	cloudcommon.AppDBInit(app)
 	initHandlers(app)
 
@@ -65,5 +66,5 @@ func StartService() {
 		log.Fatalf("fail to register service %s", err)
 	}
 
-	app_common.ServeForever(app, commonOpts)
+	app_common.ServeForever(app, baseOpts)
 }

pkg/compute/models/cachedimages.go

@@ -390,7 +390,6 @@ func (self *SCachedimage) ChooseSourceStoragecacheInRange(hostType string, exclu
 		return nil, nil
 	}
 
-	rand.Seed(time.Now().Unix())
 	return &scimgs[rand.Intn(len(scimgs))], nil
 }
 

pkg/compute/models/cloudregionresource.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package models
 
 import (

pkg/compute/models/disks.go

@@ -92,8 +92,10 @@ type SDisk struct {
 	AutoSnapshot  bool `default:"false" nullable:"true" get:"user" update:"user"`
 }
 
-func (manager *SDiskManager) GetContextManager() []db.IModelManager {
-	return []db.IModelManager{StorageManager}
+func (manager *SDiskManager) GetContextManagers() [][]db.IModelManager {
+	return [][]db.IModelManager{
+		{StorageManager},
+	}
 }
 
 func (manager *SDiskManager) FetchDiskById(diskId string) *SDisk {

pkg/compute/models/guest_queries.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package models
 
 import (
@@ -278,7 +292,7 @@ func fetchSecgroups(guestIds []string) map[string][]sSecgroupInfo {
 	q1 = q1.Filter(sqlchemy.In(guests.Field("id"), guestIds))
 	q2 := guestsecgroups.Query(guestsecgroups.Field("guest_id"), guestsecgroups.Field("secgroup_id"))
 	q2 = q2.Filter(sqlchemy.In(guestsecgroups.Field("guest_id"), guestIds))
-	uq := sqlchemy.Union(q1, q2).SubQuery()
+	uq := sqlchemy.Union(q1, q2)
 	q := uq.Query(uq.Field("guest_id"), uq.Field("secgroup_id"), secgroups.Field("name").Label("secgroup_name"))
 	q = q.Join(secgroups, sqlchemy.Equals(uq.Field("secgroup_id"), secgroups.Field("id")))
 

pkg/compute/models/hosts.go

@@ -193,8 +193,10 @@ type SHost struct {
 	IsImport bool `nullable:"true" default:"false" list:"admin" create:"admin_optional"`
 }
 
-func (manager *SHostManager) GetContextManager() []db.IModelManager {
-	return []db.IModelManager{ZoneManager}
+func (manager *SHostManager) GetContextManagers() [][]db.IModelManager {
+	return [][]db.IModelManager{
+		{ZoneManager},
+	}
 }
 
 func (self *SHostManager) AllowListItems(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool {

pkg/compute/models/networks.go

@@ -107,8 +107,10 @@ type SNetwork struct {
 	AllocTimoutSeconds int `default:"0" nullable:"true" get:"admin"`
 }
 
-func (manager *SNetworkManager) GetContextManager() []db.IModelManager {
-	return []db.IModelManager{WireManager}
+func (manager *SNetworkManager) GetContextManagers() [][]db.IModelManager {
+	return [][]db.IModelManager{
+		{WireManager},
+	}
 }
 
 func (self *SNetwork) GetWire() *SWire {

pkg/compute/models/networkschedtags.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package models
 
 import (

pkg/compute/models/purge.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package models
 
 import (

pkg/compute/models/storages.go

@@ -148,8 +148,11 @@ type SStorage struct {
 	IsSysDiskStore bool `nullable:"false" default:"true" list:"user" create:"optional" update:"admin"`
 }
 
-func (manager *SStorageManager) GetContextManager() []db.IModelManager {
-	return []db.IModelManager{ZoneManager, StoragecacheManager}
+func (manager *SStorageManager) GetContextManagers() [][]db.IModelManager {
+	return [][]db.IModelManager{
+		{ZoneManager},
+		{StoragecacheManager},
+	}
 }
 
 func (manager *SStorageManager) AllowListItems(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool {

pkg/compute/models/vpcs.go

@@ -63,8 +63,10 @@ type SVpc struct {
 	CloudregionId string `width:"36" charset:"ascii" nullable:"false" list:"admin" create:"admin_required"`
 }
 
-func (manager *SVpcManager) GetContextManager() []db.IModelManager {
-	return []db.IModelManager{CloudregionManager}
+func (manager *SVpcManager) GetContextManagers() [][]db.IModelManager {
+	return [][]db.IModelManager{
+		{CloudregionManager},
+	}
 }
 
 func (self *SVpcManager) AllowListItems(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool {

pkg/compute/models/wires.go

@@ -63,8 +63,11 @@ type SWire struct {
 	VpcId        string `wdith:"36" charset:"ascii" nullable:"false" list:"admin" create:"admin_required"`
 }
 
-func (manager *SWireManager) GetContextManager() []db.IModelManager {
-	return []db.IModelManager{ZoneManager, VpcManager}
+func (manager *SWireManager) GetContextManagers() [][]db.IModelManager {
+	return [][]db.IModelManager{
+		{ZoneManager},
+		{VpcManager},
+	}
 }
 
 func (self *SWireManager) AllowListItems(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool {

pkg/compute/models/zoneresource.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package models
 
 import (

pkg/compute/models/zones.go

@@ -63,8 +63,10 @@ type SZone struct {
 	CloudregionId string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"admin_required"`
 }
 
-func (manager *SZoneManager) GetContextManager() []db.IModelManager {
-	return []db.IModelManager{CloudregionManager}
+func (manager *SZoneManager) GetContextManagers() [][]db.IModelManager {
+	return [][]db.IModelManager{
+		{CloudregionManager},
+	}
 }
 
 func (self *SZoneManager) AllowCreateItem(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {

pkg/compute/options/namesync.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package options
 
 import (

pkg/compute/regiondrivers/ucloud.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package regiondrivers
 
 import (

pkg/compute/service/service.go

@@ -48,6 +48,7 @@ func StartService() {
 
 	opts := &options.Options
 	commonOpts := &options.Options.CommonOptions
+	baseOpts := &options.Options.BaseOptions
 	dbOpts := &options.Options.DBOptions
 	common_options.ParseOptions(opts, os.Args, "region.conf", "compute")
 
@@ -65,7 +66,7 @@ func StartService() {
 	cloudcommon.InitDB(dbOpts)
 	defer cloudcommon.CloseDB()
 
-	app := app_common.InitApp(commonOpts, true)
+	app := app_common.InitApp(baseOpts, true)
 	cloudcommon.AppDBInit(app)
 	InitHandlers(app)
 
@@ -104,5 +105,5 @@ func StartService() {
 		defer cron.Stop()
 	}
 
-	app_common.ServeForever(app, commonOpts)
+	app_common.ServeForever(app, baseOpts)
 }

pkg/compute/tasks/cloud_account_delete_task.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package tasks
 
 import (

pkg/compute/tasks/host_import_libvirt_servers_task.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package tasks
 
 import (

pkg/hostman/guestman/libvirt.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package guestman
 
 import (

pkg/hostman/host_services.go

@@ -50,7 +50,7 @@ func (host *SHostService) StartService() {
 	}
 
 	options.HostOptions.EnableRbac = false // disable rbac
-	app := app_common.InitApp(&options.HostOptions.CommonOptions, false)
+	app := app_common.InitApp(&options.HostOptions.BaseOptions, false)
 	hostInstance := hostinfo.Instance()
 	if err := hostInstance.Init(); err != nil {
 		log.Fatalf(err.Error())
@@ -77,14 +77,14 @@ func (host *SHostService) StartService() {
 
 	// Init Metadata handler
 	go metadata.StartService(
-		app_common.InitApp(&options.HostOptions.CommonOptions, false),
+		app_common.InitApp(&options.HostOptions.BaseOptions, false),
 		options.HostOptions.Address, options.HostOptions.Port+1000)
 
 	cronManager := cronman.GetCronJobManager(false)
 	cronManager.AddJob2(
 		"CleanRecycleDiskFiles", 1, 3, 0, 0, storageman.CleanRecycleDiskfiles, false)
 
-	app_common.ServeForeverWithCleanup(app, &options.HostOptions.CommonOptions, func() {
+	app_common.ServeForeverWithCleanup(app, &options.HostOptions.BaseOptions, func() {
 		hostinfo.Stop()
 		storageman.Stop()
 		hostmetrics.Stop()

pkg/hostman/hostinfo/hostbridge/linux_bridge.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package hostbridge
 
 import (

pkg/hostman/hostinfo/hostbridge/ovs.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package hostbridge
 
 import (

pkg/hostman/system_service/servicemanager.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package system_service
 
 type SServiceStatus struct {

pkg/hostman/system_service/systemd.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package system_service
 
 import (

pkg/hostman/system_service/systemd_test.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package system_service
 
 import (

pkg/hostman/system_service/sysv.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package system_service
 
 import (

pkg/hostman/system_service/sysv_test.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package system_service
 
 import (

pkg/httperrors/httperrors.go

@@ -18,6 +18,7 @@ import (
 	"net/http"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
 
 	"yunion.io/x/onecloud/pkg/util/httputils"
 )
@@ -38,6 +39,7 @@ func HTTPError(w http.ResponseWriter, msg string, statusCode int, class string,
 	err.Add(jsonutils.NewStringArray(error.Fields), "fields")
 	body.Add(err, "data")
 	w.Write([]byte(body.String()))
+	log.Errorf("Send error %s", err)
 }
 
 func JsonClientError(w http.ResponseWriter, e *httputils.JSONClientError) {

pkg/image/models/images.go

@@ -159,7 +159,7 @@ func (manager *SImageManager) GetPropertyDetail(ctx context.Context, userCred mc
 	queryDict := query.(*jsonutils.JSONDict)
 	queryDict.Add(jsonutils.JSONTrue, "details")
 
-	items, err := db.ListItems(manager, ctx, userCred, queryDict, "")
+	items, err := db.ListItems(manager, ctx, userCred, queryDict, nil)
 	if err != nil {
 		log.Errorf("Fail to list items: %s", err)
 		return nil, httperrors.NewGeneralError(err)

pkg/image/service/service.go

@@ -43,6 +43,7 @@ const (
 func StartService() {
 	opts := &options.Options
 	commonOpts := &opts.CommonOptions
+	baseOpts := &opts.BaseOptions
 	dbOpts := &opts.DBOptions
 	common_options.ParseOptions(opts, os.Args, "glance-api.conf", SERVICE_TYPE)
 
@@ -91,7 +92,7 @@ func StartService() {
 
 	cloudcommon.InitDB(dbOpts)
 
-	app := app_common.InitApp(commonOpts, true)
+	app := app_common.InitApp(baseOpts, true)
 	initHandlers(app)
 
 	if !db.CheckSync(opts.AutoSyncTable) {
@@ -108,7 +109,7 @@ func StartService() {
 	cron.Start()
 
 	cloudcommon.AppDBInit(app)
-	app_common.ServeForeverWithCleanup(app, commonOpts, func() {
+	app_common.ServeForeverWithCleanup(app, baseOpts, func() {
 		cloudcommon.CloseDB()
 
 		cron.Stop()

pkg/keystone/driver/base.go

@@ -0,0 +1,50 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package driver
+
+import (
+	"yunion.io/x/onecloud/pkg/keystone/models"
+)
+
+type SBaseDomainDriver struct {
+	virtual interface{}
+
+	config   models.TDomainConfigs
+	domainId string
+}
+
+func (base *SBaseDomainDriver) IIdentityBackend() IIdentityBackend {
+	return base.virtual.(IIdentityBackend)
+}
+
+func NewBaseDomainDriver(domainId string, conf models.TDomainConfigs) SBaseDomainDriver {
+	return SBaseDomainDriver{
+		domainId: domainId,
+		config:   conf,
+	}
+}
+
+func GetDriver(domainId string, conf models.TDomainConfigs) (IIdentityBackend, error) {
+	if ident, ok := conf["identity"]; ok {
+		if driverJson, ok := ident["driver"]; ok {
+			driver, _ := driverJson.GetString()
+			switch driver {
+			case "ldap":
+				return NewLDAPDriver(domainId, conf)
+			}
+		}
+	}
+	return NewSQLDriver(domainId, conf)
+}

pkg/keystone/driver/doc.go

@@ -0,0 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package driver // import "yunion.io/x/onecloud/pkg/keystone/driver"

pkg/keystone/driver/driver.go

@@ -0,0 +1,40 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package driver
+
+import (
+	"context"
+
+	"yunion.io/x/onecloud/pkg/keystone/models"
+	"yunion.io/x/onecloud/pkg/mcclient"
+)
+
+type IIdentityBackend interface {
+	Authenticate(ctx context.Context, identity mcclient.SAuthenticationIdentity) (*models.SUserExtended, error)
+}
+
+type SUserInfo struct {
+	DN      string
+	Id      string
+	Name    string
+	Enabled bool
+	Extra   map[string]string
+}
+
+type SGroupInfo struct {
+	Id      string
+	Name    string
+	Members []string
+}