Merge branch 'release/2.2.0' of ssh://git.yunion.io/~quxuan/onecloud into feature/qx-azure

cmd/climc/shell/hosts.go

@@ -474,4 +474,38 @@ func init() {
 		return nil
 	})
 
+	type HostCreateOptions struct {
+		ZONE       string `help:"Zone where the host is located"`
+		NAME       string `help:"Name of baremetal"`
+		MAC        string `help:"Default MAC address of baremetal"`
+		Rack       string `help:"Rack number of baremetal"`
+		Slots      string `help:"Slots number of baremetal"`
+		IpmiPasswd string `help:"IPMI user password"`
+		IpmiAddr   string `help:"IPMI IP address"`
+	}
+	R(&HostCreateOptions{}, "host-create", "Create a baremetal host", func(s *mcclient.ClientSession, args *HostCreateOptions) error {
+		params := jsonutils.NewDict()
+		params.Add(jsonutils.NewString(args.NAME), "name")
+		params.Add(jsonutils.NewString(args.MAC), "access_mac")
+		params.Add(jsonutils.NewString("baremetal"), "host_type")
+		if len(args.Rack) > 0 {
+			params.Add(jsonutils.NewString(args.Rack), "rack")
+		}
+		if len(args.Slots) > 0 {
+			params.Add(jsonutils.NewString(args.Slots), "slots")
+		}
+		if len(args.IpmiPasswd) > 0 {
+			params.Add(jsonutils.NewString(args.IpmiPasswd), "ipmi_password")
+		}
+		if len(args.IpmiAddr) > 0 {
+			params.Add(jsonutils.NewString(args.IpmiAddr), "ipmi_ip_addr")
+		}
+		result, err := modules.Hosts.CreateInContext(s, params, &modules.Zones, args.ZONE)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
 }

cmd/climc/shell/k8s/configmap.go

@@ -17,7 +17,8 @@ func initConfigMap() {
 	R(&listOpt{}, cmdN("list"), "List k8s configmap", func(s *mcclient.ClientSession, args *listOpt) error {
 		params := fetchNamespaceParams(args.namespaceListOptions)
 		params.Update(fetchPagingParams(args.baseListOptions))
-		ret, err := k8s.ConfigMaps.ListInContexts(s, params, args.ClusterContext())
+		params.Update(args.ClusterParams())
+		ret, err := k8s.ConfigMaps.List(s, params)
 		if err != nil {
 			return err
 		}

cmd/climc/shell/k8s/deployment.go

@@ -25,7 +25,8 @@ func initDeployment() {
 	R(&listOpt{}, cmdN("list"), "List k8s deployment", func(s *mcclient.ClientSession, args *listOpt) error {
 		params := fetchNamespaceParams(args.namespaceListOptions)
 		params.Update(fetchPagingParams(args.baseListOptions))
-		ret, err := k8s.Deployments.ListInContexts(s, params, args.ClusterContext())
+		params.Update(args.ClusterParams())
+		ret, err := k8s.Deployments.List(s, params)
 		if err != nil {
 			return err
 		}
@@ -45,7 +46,7 @@ func initDeployment() {
 		Net             string   `help:"Network config, e.g. net1, net1:10.168.222.171"`
 	}
 	R(&createOpt{}, cmdN("create"), "Create deployment resource", func(s *mcclient.ClientSession, args *createOpt) error {
-		params := jsonutils.NewDict()
+		params := args.ClusterParams()
 		if len(args.Image) == 0 {
 			return fmt.Errorf("Image must provided")
 		}
@@ -74,7 +75,7 @@ func initDeployment() {
 			}
 			params.Add(net, "networkConfig")
 		}
-		ret, err := k8s.Deployments.CreateInContexts(s, params, args.ClusterContext())
+		ret, err := k8s.Deployments.Create(s, params)
 		if err != nil {
 			return err
 		}
@@ -87,11 +88,11 @@ func initDeployment() {
 	}
 	R(&getOpt{}, cmdN("show"), "Get deployment details", func(s *mcclient.ClientSession, args *getOpt) error {
 		id := args.NAME
-		params := jsonutils.NewDict()
+		params := args.ClusterParams()
 		if args.Namespace != "" {
 			params.Add(jsonutils.NewString(args.Namespace), "namespace")
 		}
-		ret, err := k8s.Deployments.GetInContexts(s, id, params, args.ClusterContext())
+		ret, err := k8s.Deployments.Get(s, id, params)
 		if err != nil {
 			return err
 		}

cmd/climc/shell/k8s/k8s.go

@@ -33,8 +33,10 @@ type clusterBaseOptions struct {
 	Cluster string `default:"$K8S_CLUSTER|default" help:"Kubernetes cluster name"`
 }
 
-func (o clusterBaseOptions) ClusterContext() []modules.ManagerContext {
-	return []modules.ManagerContext{clusterContext(o.Cluster)}
+func (o clusterBaseOptions) ClusterParams() *jsonutils.JSONDict {
+	ret := jsonutils.NewDict()
+	ret.Add(jsonutils.NewString(o.Cluster), "cluster")
+	return ret
 }
 
 type baseListOptions struct {
@@ -70,7 +72,7 @@ type resourceGetOptions struct {
 }
 
 func (o resourceGetOptions) ToJSON() *jsonutils.JSONDict {
-	params := jsonutils.NewDict()
+	params := o.ClusterParams()
 	if o.Namespace != "" {
 		params.Add(jsonutils.NewString(o.Namespace), "namespace")
 	}

cmd/climc/shell/k8s/pods.go

@@ -19,7 +19,8 @@ func initPod() {
 	R(&listOpt{}, cmdN("list"), "List k8s pod", func(s *mcclient.ClientSession, args *listOpt) error {
 		params := fetchNamespaceParams(args.namespaceListOptions)
 		params.Update(fetchPagingParams(args.baseListOptions))
-		ret, err := k8s.Pods.ListInContexts(s, params, args.ClusterContext())
+		params.Update(args.ClusterParams())
+		ret, err := k8s.Pods.List(s, params)
 		if err != nil {
 			return err
 		}
@@ -32,7 +33,7 @@ func initPod() {
 	}
 	R(&deleteOpt{}, cmdN("delete"), "Delete pod", func(s *mcclient.ClientSession, args *deleteOpt) error {
 		id := args.NAME
-		ret, err := k8s.Pods.DeleteInContexts(s, id, args.ToJSON(), args.ClusterContext())
+		ret, err := k8s.Pods.Delete(s, id, args.ToJSON())
 		if err != nil {
 			return err
 		}

cmd/climc/shell/k8s/raw.go

@@ -17,7 +17,7 @@ type rawDeleteOpt struct {
 
 func initRaw() {
 	R(&rawGetOpt{}, "k8s-get", "Get k8s resource instance raw info", func(s *mcclient.ClientSession, args *rawGetOpt) error {
-		obj, err := k8s.RawResource.Get(s, args.KIND, args.Namespace, args.NAME, nil, args.ClusterContext())
+		obj, err := k8s.RawResource.Get(s, args.KIND, args.Namespace, args.NAME, args.ClusterParams())
 		if err != nil {
 			return err
 		}
@@ -26,7 +26,7 @@ func initRaw() {
 	})
 
 	R(&rawDeleteOpt{}, "k8s-delete", "Delete k8s resource instance", func(s *mcclient.ClientSession, args *rawDeleteOpt) error {
-		err := k8s.RawResource.Delete(s, args.KIND, args.Namespace, args.NAME, nil, args.ClusterContext())
+		err := k8s.RawResource.Delete(s, args.KIND, args.Namespace, args.NAME, args.ClusterParams())
 		if err != nil {
 			return err
 		}

cmd/climc/shell/k8s/releases.go

@@ -29,6 +29,7 @@ func initRelease() {
 	R(&listOpt{}, cmdN("list"), "List k8s cluster helm releases", func(s *mcclient.ClientSession, args *listOpt) error {
 		params := fetchNamespaceParams(args.namespaceListOptions)
 		params.Update(fetchPagingParams(args.baseListOptions))
+		params.Update(args.ClusterParams())
 		if args.Filter != "" {
 			params.Add(json.NewString(args.Filter), "filter")
 		}
@@ -57,7 +58,7 @@ func initRelease() {
 		if args.Pending {
 			params.Add(json.JSONTrue, "pending")
 		}
-		ret, err := k8s.Releases.ListInContexts(s, params, args.ClusterContext())
+		ret, err := k8s.Releases.List(s, params)
 		if err != nil {
 			return err
 		}
@@ -70,7 +71,7 @@ func initRelease() {
 		NAME string `help:"Release instance name"`
 	}
 	R(&showOpt{}, cmdN("show"), "Get helm release details", func(s *mcclient.ClientSession, args *showOpt) error {
-		ret, err := k8s.Releases.GetInContexts(s, args.NAME, nil, args.ClusterContext())
+		ret, err := k8s.Releases.Get(s, args.NAME, args.ClusterParams())
 		if err != nil {
 			return err
 		}
@@ -121,6 +122,7 @@ func initRelease() {
 		if err != nil {
 			return err
 		}
+		params.Update(args.ClusterParams())
 		params.Add(json.NewString(args.CHARTNAME), "chart_name")
 		if args.Namespace != "" {
 			params.Add(json.NewString(args.Namespace), "namespace")
@@ -128,7 +130,7 @@ func initRelease() {
 		if args.Name != "" {
 			params.Add(json.NewString(args.Name), "release_name")
 		}
-		ret, err := k8s.Releases.CreateInContexts(s, params, args.ClusterContext())
+		ret, err := k8s.Releases.Create(s, params)
 		if err != nil {
 			return err
 		}
@@ -149,6 +151,7 @@ func initRelease() {
 		if err != nil {
 			return err
 		}
+		params.Update(args.ClusterParams())
 		params.Add(json.NewString(args.CHARTNAME), "chart_name")
 		params.Add(json.NewString(args.NAME), "release_name")
 		if args.ReuseValues {
@@ -158,7 +161,7 @@ func initRelease() {
 			params.Add(json.JSONTrue, "reset_values")
 		}
 
-		res, err := k8s.Releases.PutInContexts(s, args.NAME, params, args.ClusterContext())
+		res, err := k8s.Releases.Put(s, args.NAME, params)
 		if err != nil {
 			return err
 		}
@@ -171,7 +174,7 @@ func initRelease() {
 		NAME string `help:"Release instance name"`
 	}
 	R(&deleteOpt{}, cmdN("delete"), "Delete release", func(s *mcclient.ClientSession, args *deleteOpt) error {
-		_, err := k8s.Releases.DeleteInContexts(s, args.NAME, nil, args.ClusterContext())
+		_, err := k8s.Releases.Delete(s, args.NAME, args.ClusterParams())
 		return err
 	})
 }

cmd/climc/shell/k8s/service.go

@@ -17,7 +17,8 @@ func initService() {
 	R(&listOpt{}, cmdN("list"), "List k8s service", func(s *mcclient.ClientSession, args *listOpt) error {
 		params := fetchNamespaceParams(args.namespaceListOptions)
 		params.Update(fetchPagingParams(args.baseListOptions))
-		ret, err := k8s.Services.ListInContexts(s, params, args.ClusterContext())
+		params.Update(args.ClusterParams())
+		ret, err := k8s.Services.List(s, params)
 		if err != nil {
 			return err
 		}

cmd/climc/shell/k8s/tiller.go

@@ -28,7 +28,7 @@ func initTiller() {
 		MaxHistory int `json:"history_max"`
 	}
 	R(&createOpt{}, cmdN("create"), "Install helm tiller server to Kubernetes cluster", func(s *mcclient.ClientSession, args *createOpt) error {
-		params := json.NewDict()
+		params := args.ClusterParams()
 		if len(args.KubeContext) > 0 {
 			params.Add(json.NewString(args.KubeContext), "kube_context")
 		}
@@ -46,7 +46,7 @@ func initTiller() {
 		if args.MaxHistory > 0 {
 			params.Add(json.NewInt(int64(args.MaxHistory)), "history_max")
 		}
-		ret, err := k8s.Tiller.CreateInContexts(s, params, args.ClusterContext())
+		ret, err := k8s.Tiller.Create(s, params)
 		if err != nil {
 			return err
 		}

cmd/climc/shell/networks.go

@@ -142,6 +142,15 @@ func init() {
 		return nil
 	})
 
+	R(&NetworkShowOptions{}, "network-metadata", "Show metadata of a network", func(s *mcclient.ClientSession, args *NetworkShowOptions) error {
+		result, err := modules.Networks.GetMetadata(s, args.ID, nil)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
 	R(&NetworkShowOptions{}, "network-private", "Make a network private", func(s *mcclient.ClientSession, args *NetworkShowOptions) error {
 		result, err := modules.Networks.PerformAction(s, args.ID, "private", nil)
 		if err != nil {
@@ -288,4 +297,102 @@ func init() {
 		return nil
 	})
 
+	type NetworkStaticRoutesOptions struct {
+		NETWORK string   `help:"ID or name of the network"`
+		Net     []string `help:"destination network of static route"`
+		Gw      []string `help:"gateway address for the static route"`
+	}
+	R(&NetworkStaticRoutesOptions{}, "network-set-static-routes", "Set static routes for a network", func(s *mcclient.ClientSession, args *NetworkStaticRoutesOptions) error {
+		params := jsonutils.NewDict()
+		if len(args.Net) > 0 && len(args.Gw) > 0 {
+			if len(args.Net) != len(args.Gw) {
+				return fmt.Errorf("Inconsistent network and gateway pairs")
+			}
+			routes := jsonutils.NewDict()
+			for i := 0; i < len(args.Net); i += 1 {
+				routes.Add(jsonutils.NewString(args.Gw[i]), args.Net[i])
+			}
+			params.Add(routes, "static_routes")
+		} else {
+			params.Add(jsonutils.JSONNull, "static_routes")
+		}
+		result, err := modules.Networks.PerformAction(s, args.NETWORK, "metadata", params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
+	type NetworkAddDnsUpdateTargetOptions struct {
+		ID     string `help:"Network ID or name"`
+		DNS    string `help:"DNS server address" metavar:"DNS_SERVER"`
+		KEY    string `help:"DNS update key name" metavar:"DNS_UPDATE_KEY"`
+		SECRET string `help:"DNS update key secret" metavar:"DNS_UPDATE_SECRET"`
+	}
+	R(&NetworkAddDnsUpdateTargetOptions{}, "network-add-dns-update-target", "Add a dns update target to a network", func(s *mcclient.ClientSession, args *NetworkAddDnsUpdateTargetOptions) error {
+		params := jsonutils.NewDict()
+		params.Add(jsonutils.NewString(args.DNS), "server")
+		params.Add(jsonutils.NewString(args.KEY), "key")
+		params.Add(jsonutils.NewString(args.SECRET), "secret")
+		result, err := modules.Networks.PerformAction(s, args.ID, "add-dns-update-target", params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
+	type NetworkRemoveDnsUpdateTargetOptions struct {
+		ID  string `help:"Network ID or name"`
+		DNS string `help:"DNS server address" metavar:"DNS_SERVER"`
+		KEY string `help:"DNS update key name" metavar:"DNS_UPDATE_KEY"`
+	}
+	R(&NetworkRemoveDnsUpdateTargetOptions{}, "network-remove-dns-update-target", "Remove a dns update target from a network", func(s *mcclient.ClientSession, args *NetworkRemoveDnsUpdateTargetOptions) error {
+		params := jsonutils.NewDict()
+		params.Add(jsonutils.NewString(args.DNS), "server")
+		params.Add(jsonutils.NewString(args.KEY), "key")
+		result, err := modules.Networks.PerformAction(s, args.ID, "remove-dns-update-target", params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
+	type NetworkSetDnsUpdateKeyOptions struct {
+		ID     string `help:"ID of virtual network to update"`
+		KEY    string `help:"Key name of secret" metavar:"KEYNAME"`
+		SECRET string `help:"Key secret"`
+		SERVER string `help:"Alternate DNS update server"`
+	}
+	R(&NetworkSetDnsUpdateKeyOptions{}, "network-set-dns-update-key", "Set DNS update key info for a virtual network", func(s *mcclient.ClientSession, args *NetworkSetDnsUpdateKeyOptions) error {
+		params := jsonutils.NewDict()
+		params.Add(jsonutils.NewString(args.KEY), "dns_update_key_name")
+		params.Add(jsonutils.NewString(args.SECRET), "dns_update_key_secret")
+		params.Add(jsonutils.NewString(args.SERVER), "dns_update_server")
+		result, err := modules.Networks.PerformAction(s, args.ID, "metadata", params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
+	type NetworkRemoveDnsUpdateKeyOptions struct {
+		ID string `help:"ID of virtual network"`
+	}
+	R(&NetworkRemoveDnsUpdateKeyOptions{}, "network-remove-dns-update-key", "Set DNS update key info for a virtual network", func(s *mcclient.ClientSession, args *NetworkRemoveDnsUpdateKeyOptions) error {
+		params := jsonutils.NewDict()
+		params.Add(jsonutils.JSONNull, "dns_update_key_name")
+		params.Add(jsonutils.JSONNull, "dns_update_key_secret")
+		params.Add(jsonutils.JSONNull, "dns_update_server")
+		result, err := modules.Networks.PerformAction(s, args.ID, "metadata", params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
 }

pkg/cloudcommon/db/db_dispatcher.go

@@ -21,6 +21,7 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	"yunion.io/x/onecloud/pkg/util/httputils"
+	"yunion.io/x/onecloud/pkg/util/logclient"
 )
 
 type DBModelDispatcher struct {
@@ -768,6 +769,7 @@ func (dispatcher *DBModelDispatcher) Create(ctx context.Context, query jsonutils
 		return nil, httperrors.NewGeneralError(err)
 	}
 	OpsLog.LogEvent(model, ACT_CREATE, model.GetShortDesc(), userCred)
+	logclient.AddActionLog(model, logclient.ACT_CREATE, "", userCred, true)
 	dispatcher.modelManager.OnCreateComplete(ctx, []IModel{model}, userCred, query, data)
 	return getItemDetails(dispatcher.modelManager, model, ctx, userCred, query)
 }
@@ -986,11 +988,13 @@ func updateItem(manager IModelManager, item IModel, ctx context.Context, userCre
 
 	if err != nil {
 		log.Errorf("validate update condition error: %s", err)
+		logclient.AddActionLog(item, logclient.ACT_UPDATE, err.Error(), userCred, false)
 		return nil, httperrors.NewGeneralError(err)
 	}
 
 	dataDict, ok := data.(*jsonutils.JSONDict)
 	if !ok {
+		logclient.AddActionLog(item, logclient.ACT_UPDATE, "Invalid data JSONObject", userCred, false)
 		return nil, httperrors.NewInternalServerError("Invalid data JSONObject")
 	}
 
@@ -998,13 +1002,16 @@ func updateItem(manager IModelManager, item IModel, ctx context.Context, userCre
 	if len(name) > 0 {
 		err = alterNameValidator(item, name)
 		if err != nil {
+			logclient.AddActionLog(item, logclient.ACT_UPDATE, err.Error(), userCred, false)
 			return nil, err
 		}
 	}
 
 	dataDict, err = item.ValidateUpdateData(ctx, userCred, query, dataDict)
 	if err != nil {
-		log.Errorf("validate update data error: %s", err)
+		errMsg := fmt.Sprintf("validate update data error: %s", err)
+		log.Errorf(errMsg)
+		logclient.AddActionLog(item, logclient.ACT_UPDATE, errMsg, userCred, false)
 		return nil, httperrors.NewGeneralError(err)
 	}
 	item.PreUpdate(ctx, userCred, query, dataDict)
@@ -1012,7 +1019,9 @@ func updateItem(manager IModelManager, item IModel, ctx context.Context, userCre
 		filterData := dataDict.CopyIncludes(updateFields(manager, userCred)...)
 		err = filterData.Unmarshal(item)
 		if err != nil {
-			log.Errorf("unmarshal fail: %s", err)
+			errMsg := fmt.Sprintf("unmarshal fail: %s", err)
+			logclient.AddActionLog(item, logclient.ACT_UPDATE, errMsg, userCred, false)
+			log.Errorf(errMsg)
 			return httperrors.NewGeneralError(err)
 		}
 		return nil
@@ -1026,7 +1035,10 @@ func updateItem(manager IModelManager, item IModel, ctx context.Context, userCre
 		diffStr := sqlchemy.UpdateDiffString(diff)
 		if len(diffStr) > 0 {
 			OpsLog.LogEvent(item, ACT_UPDATE, diffStr, userCred)
+			logclient.AddActionLog(item, logclient.ACT_UPDATE, diffStr, userCred, true)
 		}
+	} else {
+		logclient.AddActionLog(item, logclient.ACT_UPDATE, "", userCred, true)
 	}
 	return getItemDetails(manager, item, ctx, userCred, query)
 }
@@ -1054,10 +1066,13 @@ func DeleteModel(ctx context.Context, userCred mcclient.TokenCredential, item IM
 		return item.MarkDelete()
 	})
 	if err != nil {
-		log.Errorf("save update error %s", err)
+		msg := fmt.Sprintf("save update error %s", err)
+		log.Errorf(msg)
+		logclient.AddActionLog(item, logclient.ACT_DELETE, msg, userCred, false)
 		return httperrors.NewGeneralError(err)
 	}
 	OpsLog.LogEvent(item, ACT_DELETE, item.GetShortDesc(), userCred)
+	logclient.AddActionLog(item, logclient.ACT_DELETE, item.GetShortDesc(), userCred, true)
 	return nil
 }
 

pkg/cloudcommon/db/opslog.go

@@ -10,6 +10,7 @@ import (
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/logclient"
 	"yunion.io/x/pkg/util/stringutils"
 	"yunion.io/x/sqlchemy"
 )
@@ -113,6 +114,9 @@ const (
 	ACT_UNCACHE_IMAGE_FAIL = "uncache_image_fail"
 	ACT_UNCACHED_IMAGE     = "uncached_image"
 
+	ACT_SYNC_CLOUD_DISK   = "sync_cloud_disk"
+	ACT_SYNC_CLOUD_SERVER = "sync_cloud_server"
+
 	ACT_SPLIT = "net_split"
 
 	ACT_PENDING_DELETE = "pending_delete"
@@ -302,6 +306,7 @@ func (manager *SOpsLogManager) SyncOwner(m IModel, former *STenant, userCred mcc
 	notes.Add(jsonutils.NewString(former.GetId()), "former_project_id")
 	notes.Add(jsonutils.NewString(former.GetName()), "form_project")
 	manager.LogEvent(m, ACT_CHANGE_OWNER, notes, userCred)
+	logclient.AddActionLog(m, logclient.ACT_CHANGE_OWNER, nil, userCred, true)
 }
 
 func (manager *SOpsLogManager) AllowListItems(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool {

pkg/cloudcommon/db/resourcebase.go

@@ -13,7 +13,7 @@ type SResourceBase struct {
 	SModelBase
 
 	CreatedAt     time.Time `nullable:"false" created_at:"true" get:"user"`
-	UpdatedAt     time.Time `nullable:"false" updated_at:"true"`
+	UpdatedAt     time.Time `nullable:"false" updated_at:"true" list:"user"`
 	UpdateVersion int       `default:"0" nullable:"false" auto_version:"true"`
 	DeletedAt     time.Time ``
 	Deleted       bool      `nullable:"false" default:"false"`

pkg/cloudcommon/db/statusstandalone.go

@@ -3,9 +3,11 @@ package db
 import (
 	"context"
 	"fmt"
+	"strings"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/logclient"
 )
 
 type SStatusStandaloneResourceBase struct {
@@ -40,6 +42,9 @@ func (model *SStatusStandaloneResourceBase) SetStatus(userCred mcclient.TokenCre
 			notes = fmt.Sprintf("%s: %s", notes, reason)
 		}
 		OpsLog.LogEvent(model, ACT_UPDATE_STATUS, notes, userCred)
+		if strings.Contains(notes, "fail") {
+			logclient.AddActionLog(model, logclient.ACT_VM_SYNC_STATUS, notes, userCred, false)
+		}
 	}
 	return nil
 }

pkg/cloudprovider/resources.go

@@ -56,6 +56,7 @@ type ICloudZone interface {
 type ICloudImage interface {
 	ICloudResource
 
+	Delete() error
 	GetIStoragecache() ICloudStoragecache
 }
 
@@ -66,6 +67,9 @@ type ICloudStoragecache interface {
 
 	GetManagerId() string
 
+	CreateIImage(snapshotId, imageName, imageDesc string) (ICloudImage, error)
+
+	DownloadImage(userCred mcclient.TokenCredential, imageId string, extId string) (jsonutils.JSONObject, error)
 	UploadImage(userCred mcclient.TokenCredential, imageId string, extId string, isForce bool) (string, error)
 }
 
@@ -201,9 +205,18 @@ type ICloudDisk interface {
 	GetMountpoint() string
 	Delete() error
 
+	CreateISnapshot(name string, desc string) (ICloudSnapshot, error)
+	GetISnapshot(idStr string) (ICloudSnapshot, error)
+	GetISnapshots() ([]ICloudSnapshot, error)
+
 	Resize(newSize int64) error
 }
 
+type ICloudSnapshot interface {
+	ICloudResource
+	Delete() error
+}
+
 type ICloudVpc interface {
 	ICloudResource
 

pkg/compute/guestdrivers/aliyun.go

@@ -121,7 +121,6 @@ func (self *SAliyunGuestDriver) GetJsonDescAtHost(ctx context.Context, guest *mo
 			config.DataDisks[i-1] = disk.DiskSize / 1024 // MB => GB
 		}
 	}
-
 	return jsonutils.Marshal(&config)
 }
 
@@ -243,14 +242,10 @@ func (self *SAliyunGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gu
 
 		params := task.GetParams()
 		log.Debugf("Deploy VM params %s", params.String())
-		var name string
-		if v, e := params.GetString("name"); e != nil {
-			name = v
-		}
-		var description string
-		if v, e := params.GetString("description"); e != nil {
-			description = v
-		}
+
+		name, _ := params.GetString("name")
+		description, _ := params.GetString("description")
+		publicKey, _ := config.GetString("public_key")
 		resetPassword := jsonutils.QueryBoolean(params, "reset_password", false)
 		deleteKeypair := jsonutils.QueryBoolean(params, "__delete_keypair__", false)
 		password, _ := params.GetString("password")
@@ -258,11 +253,6 @@ func (self *SAliyunGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gu
 			password = seclib2.RandomPassword2(12)
 		}
 
-		publicKey := ""
-		if k, e := config.GetString("public_key"); e != nil {
-			publicKey = k
-		}
-
 		taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) {
 			encpasswd, err := utils.EncryptAESBase64(guest.Id, password)
 			if err != nil {

pkg/compute/guestdrivers/baremetals.go

@@ -9,6 +9,7 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/quotas"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
 	"yunion.io/x/onecloud/pkg/compute/models"
+	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
 )
 
@@ -172,3 +173,7 @@ func (self *SBaremetalGuestDriver) StartGuestDetachdiskTask(ctx context.Context,
 func (self *SBaremetalGuestDriver) StartSuspendTask(ctx context.Context, userCred mcclient.TokenCredential, guest *models.SGuest, params *jsonutils.JSONDict, parentTaskId string) error {
 	return fmt.Errorf("Cannot suspend a baremetal serer")
 }
+
+func (self *SBaremetalGuestDriver) StartGuestSaveImage(ctx context.Context, userCred mcclient.TokenCredential, guest *models.SGuest, params *jsonutils.JSONDict, parentTaskId string) error {
+	return httperrors.NewUnsupportOperationError("Cannot save image for baremtal")
+}

pkg/compute/guestdrivers/virtualization.go

@@ -193,3 +193,12 @@ func (self *SVirtualizedGuestDriver) StartSuspendTask(ctx context.Context, userC
 	task.ScheduleRun(nil)
 	return nil
 }
+
+func (self *SVirtualizedGuestDriver) StartGuestSaveImage(ctx context.Context, userCred mcclient.TokenCredential, guest *models.SGuest, params *jsonutils.JSONDict, parentTaskId string) error {
+	if task, err := taskman.TaskManager.NewTask(ctx, "GuestSaveImageTask", guest, userCred, params, parentTaskId, "", nil); err != nil {
+		return err
+	} else {
+		task.ScheduleRun(nil)
+	}
+	return nil
+}

pkg/compute/hostdrivers/aliyun.go

@@ -2,11 +2,13 @@ package hostdrivers
 
 import (
 	"context"
+	"fmt"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
 	"yunion.io/x/onecloud/pkg/compute/models"
+	"yunion.io/x/onecloud/pkg/httperrors"
 )
 
 type SAliyunHostDriver struct {
@@ -48,6 +50,53 @@ func (self *SAliyunHostDriver) CheckAndSetCacheImage(ctx context.Context, host *
 	return nil
 }
 
+func (self *SAliyunHostDriver) RequestPrepareSaveDiskOnHost(ctx context.Context, host *models.SHost, disk *models.SDisk, imageId string, task taskman.ITask) error {
+	task.ScheduleRun(nil)
+	return nil
+}
+
+func (self *SAliyunHostDriver) RequestSaveUploadImageOnHost(ctx context.Context, host *models.SHost, disk *models.SDisk, imageId string, task taskman.ITask, data jsonutils.JSONObject) error {
+	if iDisk, err := disk.GetIDisk(); err != nil {
+		return err
+	} else if iStorage, err := disk.GetIStorage(); err != nil {
+		return err
+	} else if iStoragecache := iStorage.GetIStoragecache(); iStoragecache == nil {
+		return httperrors.NewResourceNotFoundError("fail to find iStoragecache for storage: %s", iStorage.GetName())
+	} else {
+		taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) {
+			if snapshot, err := iDisk.CreateISnapshot(fmt.Sprintf("Snapshot-%s", imageId), "PrepareSaveImage"); err != nil {
+				return nil, err
+			} else {
+				scimg := models.StoragecachedimageManager.Register(ctx, task.GetUserCred(), iStoragecache.GetId(), imageId)
+				if scimg.Status != models.CACHED_IMAGE_STATUS_READY {
+					scimg.SetStatus(task.GetUserCred(), models.CACHED_IMAGE_STATUS_CACHING, "request_prepare_save_disk_on_host")
+				}
+				if iImage, err := iStoragecache.CreateIImage(snapshot.GetId(), fmt.Sprintf("Image-%s", imageId), ""); err != nil {
+					log.Errorf("fail to create iImage: %v", err)
+					scimg.SetStatus(task.GetUserCred(), models.CACHED_IMAGE_STATUS_CACHE_FAILED, err.Error())
+					return nil, err
+				} else {
+					scimg.SetExternalId(iImage.GetId())
+					if result, err := iStoragecache.DownloadImage(task.GetUserCred(), imageId, iImage.GetId()); err != nil {
+						scimg.SetStatus(task.GetUserCred(), models.CACHED_IMAGE_STATUS_CACHE_FAILED, err.Error())
+						return nil, err
+					} else {
+						if err := iImage.Delete(); err != nil {
+							log.Errorf("Delete iImage %s failed: %v", iImage.GetId(), err)
+						}
+						if err := snapshot.Delete(); err != nil {
+							log.Errorf("Delete snapshot %s failed: %v", snapshot.GetId(), err)
+						}
+						scimg.SetStatus(task.GetUserCred(), models.CACHED_IMAGE_STATUS_READY, "")
+						return result, nil
+					}
+				}
+			}
+		})
+	}
+	return nil
+}
+
 func (self *SAliyunHostDriver) RequestAllocateDiskOnStorage(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask, content *jsonutils.JSONDict) error {
 	if iCloudStorage, err := storage.GetIStorage(); err != nil {
 		return err

pkg/compute/hostdrivers/azure.go

@@ -7,6 +7,7 @@ import (
 	"yunion.io/x/log"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
 	"yunion.io/x/onecloud/pkg/compute/models"
+	"yunion.io/x/onecloud/pkg/httperrors"
 )
 
 type SAzureHostDriver struct {
@@ -120,3 +121,12 @@ func (self *SAzureHostDriver) RequestResizeDiskOnHost(host *models.SHost, storag
 	}
 	return nil
 }
+
+func (self *SAzureHostDriver) RequestPrepareSaveDiskOnHost(ctx context.Context, host *models.SHost, disk *models.SDisk, imageId string, task taskman.ITask) error {
+	task.ScheduleRun(nil)
+	return nil
+}
+
+func (self *SAzureHostDriver) RequestSaveUploadImageOnHost(ctx context.Context, host *models.SHost, disk *models.SDisk, imageId string, task taskman.ITask, data jsonutils.JSONObject) error {
+	return httperrors.NewNotImplementedError("not implement")
+}

pkg/compute/hostdrivers/kvm.go

@@ -124,3 +124,26 @@ func (self *SKVMHostDriver) RequestResizeDiskOnHostOnline(host *models.SHost, st
 	}
 	return nil
 }
+
+func (self *SKVMHostDriver) RequestPrepareSaveDiskOnHost(ctx context.Context, host *models.SHost, disk *models.SDisk, imageId string, task taskman.ITask) error {
+	body := jsonutils.NewDict()
+	body.Add(jsonutils.Marshal(map[string]string{"image_id": imageId}), "disk")
+	url := fmt.Sprintf("/disks/%s/save-prepare/%s", disk.StorageId, disk.Id)
+	header := http.Header{"X-Task-Id": []string{task.GetTaskId()}, "X-Region-Version": []string{"v2"}}
+	_, err := host.Request(task.GetUserCred(), "POST", url, header, body)
+	return err
+}
+
+func (self *SKVMHostDriver) RequestSaveUploadImageOnHost(ctx context.Context, host *models.SHost, disk *models.SDisk, imageId string, task taskman.ITask, data jsonutils.JSONObject) error {
+	body := jsonutils.NewDict()
+	backup, _ := data.GetString("backup")
+	content := map[string]string{"image_path": backup, "image_id": imageId, "storagecached_id": disk.GetStorage().StoragecacheId}
+	if data.Contains("format") {
+		content["format"], _ = data.GetString("format")
+	}
+	body.Add(jsonutils.Marshal(content), "disk")
+	url := fmt.Sprintf("/disks/%s/upload", disk.StorageId)
+	header := http.Header{"X-Task-Id": []string{task.GetTaskId()}, "X-Region-Version": []string{"v2"}}
+	_, err := host.Request(task.GetUserCred(), "POST", url, header, body)
+	return err
+}

pkg/compute/models/disks.go

@@ -27,6 +27,8 @@ import (
 	"yunion.io/x/onecloud/pkg/compute/options"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
 )
 
 const (
@@ -358,6 +360,94 @@ func (self *SDisk) PerformResize(ctx context.Context, userCred mcclient.TokenCre
 	}
 }
 
+func (self *SDisk) GetIStorage() (cloudprovider.ICloudStorage, error) {
+	if storage := self.GetStorage(); storage == nil {
+		return nil, httperrors.NewResourceNotFoundError("fail to find storage for disk %s", self.GetName())
+	} else if provider, err := storage.GetDriver(); err != nil {
+		return nil, err
+	} else {
+		return provider.GetIStorageById(storage.GetExternalId())
+	}
+}
+
+func (self *SDisk) GetIDisk() (cloudprovider.ICloudDisk, error) {
+	if iStorage, err := self.GetIStorage(); err != nil {
+		log.Errorf("fail to find iStorage: %v", err)
+		return nil, err
+	} else {
+		return iStorage.GetIDisk(self.GetExternalId())
+	}
+}
+
+func (self *SDisk) GetZone() *SZone {
+	if storage := self.GetStorage(); storage != nil {
+		return storage.getZone()
+	}
+	return nil
+}
+
+func (self *SDisk) PrepareSaveImage(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict) (string, error) {
+	if zone := self.GetZone(); zone == nil {
+		return "", httperrors.NewResourceNotFoundError("No zone for this disk")
+	}
+	data.Add(jsonutils.NewString(self.DiskFormat), "disk_format")
+	name, _ := data.GetString("name")
+	s := auth.GetAdminSession(options.Options.Region, "")
+	if imageList, err := modules.Images.List(s, jsonutils.Marshal(map[string]string{"name": name, "admin": "true"})); err != nil {
+		return "", err
+	} else if imageList.Total > 0 {
+		return "", httperrors.NewConflictError("Duplicate image name %s", name)
+	}
+	quota := SQuota{Image: 1}
+	if _, err := QuotaManager.CheckQuota(ctx, userCred, userCred.GetProjectId(), &quota); err != nil {
+		return "", err
+	}
+	data.Add(jsonutils.NewInt(int64(self.DiskSize)), "virtual_size")
+	if result, err := modules.Images.Create(s, data); err != nil {
+		return "", err
+	} else if imageId, err := result.GetString("id"); err != nil {
+		return "", err
+	} else {
+		return imageId, nil
+	}
+}
+
+func (self *SDisk) AllowPerformSave(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
+	return self.IsOwner(userCred)
+}
+
+func (self *SDisk) PerformSave(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	if self.Status != DISK_READY {
+		return nil, httperrors.NewResourceNotReadyError("Save disk when disk is READY")
+
+	}
+	if self.GetRuningGuestCount() > 0 {
+		return nil, httperrors.NewResourceNotReadyError("Save disk when not being USED")
+	}
+
+	if name, err := data.GetString("name"); err != nil || len(name) == 0 {
+		return nil, httperrors.NewInputParameterError("Image name is required")
+	}
+	kwargs := data.(*jsonutils.JSONDict)
+	if imageId, err := self.PrepareSaveImage(ctx, userCred, kwargs); err != nil {
+		return nil, err
+	} else {
+		kwargs.Add(jsonutils.NewString(imageId), "image_id")
+		return nil, self.StartDiskSaveTask(ctx, userCred, kwargs, "")
+	}
+}
+
+func (self *SDisk) StartDiskSaveTask(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict, parentTaskId string) error {
+	self.SetStatus(userCred, DISK_START_SAVE, "")
+	if task, err := taskman.TaskManager.NewTask(ctx, "DiskSaveTask", self, userCred, data, parentTaskId, "", nil); err != nil {
+		log.Errorf("Start DiskSaveTask failed:%v", err)
+		return err
+	} else {
+		task.ScheduleRun(nil)
+	}
+	return nil
+}
+
 func (self *SDisk) ValidateDeleteCondition(ctx context.Context) error {
 	if self.GetGuestDiskCount() > 0 {
 		return httperrors.NewNotEmptyError("Virtual disk used by virtual servers")
@@ -591,6 +681,7 @@ func (manager *SDiskManager) newFromCloudDisk(ctx context.Context, userCred mccl
 		}
 	}
 
+	db.OpsLog.LogEvent(&disk, db.ACT_SYNC_CLOUD_DISK, disk.GetShortDesc(), userCred)
 	return &disk, nil
 }
 
@@ -925,6 +1016,14 @@ func (self *SDisk) GetShortDesc() *jsonutils.JSONDict {
 		desc.Add(jsonutils.NewString(priceKey), "price_key")
 	}
 
+	if hypervisor := self.GetMetadata("hypervisor", nil); len(hypervisor) > 0 {
+		desc.Add(jsonutils.NewString(hypervisor), "hypervisor")
+	}
+
+	if len(self.ExternalId) > 0 {
+		desc.Add(jsonutils.NewString(self.ExternalId), "externalId")
+	}
+
 	fs := self.GetFsFormat()
 	if len(fs) > 0 {
 		desc.Add(jsonutils.NewString(fs), "fs_format")

pkg/compute/models/guestdrivers.go

@@ -61,6 +61,8 @@ type IGuestDriver interface {
 
 	StartDeleteGuestTask(ctx context.Context, userCred mcclient.TokenCredential, guest *SGuest, params *jsonutils.JSONDict, parentTaskId string) error
 
+	StartGuestSaveImage(ctx context.Context, userCred mcclient.TokenCredential, guest *SGuest, params *jsonutils.JSONDict, parentTaskId string) error
+
 	RequestStopGuestForDelete(ctx context.Context, guest *SGuest, task taskman.ITask) error
 
 	RequestDetachDisksFromGuestForDelete(ctx context.Context, guest *SGuest, task taskman.ITask) error

pkg/compute/models/guests.go

@@ -1311,6 +1311,7 @@ func (manager *SGuestManager) newCloudVM(ctx context.Context, userCred mcclient.
 		}
 	}
 
+	db.OpsLog.LogEvent(&guest, db.ACT_SYNC_CLOUD_SERVER, guest.GetShortDesc(), userCred)
 	return &guest, nil
 }
 
@@ -1566,6 +1567,43 @@ func (self *SGuest) attach2Disk(disk *SDisk, userCred mcclient.TokenCredential,
 	return err
 }
 
+func (self *SGuest) AllowPerformSaveImage(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
+	log.Infof("permission: %s", self.IsOwner(userCred))
+	return self.IsOwner(userCred)
+}
+
+func (self *SGuest) PerformSaveImage(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	if !utils.IsInStringArray(self.Status, []string{VM_READY}) {
+		return nil, httperrors.NewInputParameterError("Cannot save image in status %s", self.Status)
+	} else if !data.Contains("name") {
+		return nil, httperrors.NewInputParameterError("Image name is required")
+	} else if disks := self.CategorizeDisks(); disks.Root == nil {
+		return nil, httperrors.NewInputParameterError("No root image")
+	} else {
+		kwargs := data.(*jsonutils.JSONDict)
+		restart := self.Status == VM_RUNNING
+		properties := jsonutils.NewDict()
+		if notes, err := data.GetString("notes"); err != nil && len(notes) > 0 {
+			properties.Add(jsonutils.NewString(notes), "notes")
+		}
+		properties.Add(jsonutils.NewString(self.OsType), "os_type")
+		kwargs.Add(properties, "properties")
+		kwargs.Add(jsonutils.NewBool(restart), "restart")
+		lockman.LockObject(ctx, disks.Root)
+		defer lockman.ReleaseObject(ctx, disks.Root)
+		if imageId, err := disks.Root.PrepareSaveImage(ctx, userCred, kwargs); err != nil {
+			return nil, err
+		} else {
+			kwargs.Add(jsonutils.NewString(imageId), "image_id")
+		}
+		return nil, self.StartGuestSaveImage(ctx, userCred, kwargs, "")
+	}
+}
+
+func (self *SGuest) StartGuestSaveImage(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict, parentTaskId string) error {
+	return self.GetDriver().StartGuestSaveImage(ctx, userCred, self, data, parentTaskId)
+}
+
 func (self *SGuest) AllowPerformSync(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
 	return self.IsOwner(userCred)
 }
@@ -3191,6 +3229,10 @@ func (self *SGuest) GetShortDesc() *jsonutils.JSONDict {
 		desc.Add(jsonutils.NewString(priceKey), "price_key")
 	}
 
+	if len(self.ExternalId) > 0 {
+		desc.Add(jsonutils.NewString(self.ExternalId), "externalId")
+	}
+
 	desc.Set("hypervisor", jsonutils.NewString(self.GetHypervisor()))
 	spec := self.GetSpec(false)
 	if self.GetHypervisor() == HYPERVISOR_BAREMETAL {
@@ -3506,3 +3548,15 @@ func (manager *SGuestManager) CleanPendingDeleteServers(ctx context.Context, use
 		guests[i].StartDeleteGuestTask(ctx, userCred, "", false, true)
 	}
 }
+
+func (self *SGuest) SetDisableDelete(val bool) error {
+	_, err := self.GetModelManager().TableSpec().Update(self, func() error {
+		if val {
+			self.DisableDelete = tristate.True
+		} else {
+			self.DisableDelete = tristate.False
+		}
+		return nil
+	})
+	return err
+}

pkg/compute/models/hostdrivers.go

@@ -12,6 +12,8 @@ import (
 type IHostDriver interface {
 	GetHostType() string
 	CheckAndSetCacheImage(ctx context.Context, host *SHost, storagecache *SStoragecache, scimg *SStoragecachedimage, task taskman.ITask) error
+	RequestPrepareSaveDiskOnHost(ctx context.Context, host *SHost, disk *SDisk, imageId string, task taskman.ITask) error
+	RequestSaveUploadImageOnHost(ctx context.Context, host *SHost, disk *SDisk, imageId string, task taskman.ITask, data jsonutils.JSONObject) error
 	RequestAllocateDiskOnStorage(ctx context.Context, host *SHost, storage *SStorage, disk *SDisk, task taskman.ITask, content *jsonutils.JSONDict) error
 	RequestDeallocateDiskOnHost(host *SHost, storage *SStorage, disk *SDisk, task taskman.ITask) error
 	RequestResizeDiskOnHostOnline(host *SHost, storage *SStorage, disk *SDisk, size int64, task taskman.ITask) error

pkg/compute/models/networks.go

@@ -13,6 +13,7 @@ import (
 	"yunion.io/x/pkg/util/fileutils"
 	"yunion.io/x/pkg/util/netutils"
 	"yunion.io/x/pkg/util/regutils"
+	"yunion.io/x/pkg/util/sets"
 	"yunion.io/x/pkg/utils"
 	"yunion.io/x/sqlchemy"
 
@@ -30,6 +31,7 @@ const (
 
 	SERVER_TYPE_GUEST     = "guest"
 	SERVER_TYPE_BAREMETAL = "baremetal"
+	SERVER_TYPE_CONTAINER = "container"
 
 	STATIC_ALLOC = "static"
 
@@ -966,7 +968,7 @@ func (manager *SNetworkManager) ValidateCreateData(ctx context.Context, userCred
 	serverTypeStr, _ := data.GetString("server_type")
 	if len(serverTypeStr) == 0 {
 		serverTypeStr = SERVER_TYPE_GUEST
-	} else if serverTypeStr != SERVER_TYPE_GUEST && serverTypeStr != SERVER_TYPE_BAREMETAL {
+	} else if !sets.NewString(SERVER_TYPE_GUEST, SERVER_TYPE_BAREMETAL, SERVER_TYPE_CONTAINER).Has(serverTypeStr) {
 		return nil, httperrors.NewInputParameterError("Invalid server_type: %s", serverTypeStr)
 	}
 	data.Add(jsonutils.NewString(serverTypeStr), "server_type")

pkg/compute/models/storages.go

@@ -68,7 +68,7 @@ type SStorage struct {
 
 	Capacity    int                  `nullable:"false" list:"admin" update:"admin" create:"admin_required"`                            // Column(Integer, nullable=False) # capacity of disk in MB
 	Reserved    int                  `nullable:"true" default:"0" list:"admin" update:"admin"`                                         // Column(Integer, nullable=True, default=0)
-	StorageType string               `width:"32" charset:"ascii" nullable:"false" list:"admin" update:"admin" create:"admin_required"` // Column(VARCHAR(32, charset='ascii'), nullable=False)
+	StorageType string               `width:"32" charset:"ascii" nullable:"false" list:"user" update:"admin" create:"admin_required"`  // Column(VARCHAR(32, charset='ascii'), nullable=False)
 	MediumType  string               `width:"32" charset:"ascii" nullable:"false" list:"admin" update:"admin" create:"admin_required"` // Column(VARCHAR(32, charset='ascii'), nullable=False)
 	Cmtbound    float32              `nullable:"true" list:"admin" update:"admin"`                                                     // Column(Float, nullable=True)
 	StorageConf jsonutils.JSONObject `nullable:"true" get:"admin" update:"admin"`                                                      // = Column(JSONEncodedDict, nullable=True)
@@ -105,6 +105,30 @@ func (manager *SStorageManager) AllowListItems(ctx context.Context, userCred mcc
 	return true
 }
 
+func (self *SStorage) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSONDict {
+	used := self.GetUsedCapacity(tristate.True)
+	waste := self.GetUsedCapacity(tristate.False)
+	vcapa := float32(self.GetCapacity()) * self.GetOvercommitBound()
+	extra.Add(jsonutils.NewInt(int64(used)), "used_capacity")
+	extra.Add(jsonutils.NewInt(int64(waste)), "waste_capacity")
+	extra.Add(jsonutils.NewFloat(float64(vcapa)), "virtual_capacity")
+	extra.Add(jsonutils.NewFloat(float64(vcapa-float32(used)-float32(waste))), "free_capacity")
+	if self.GetCapacity() > 0 {
+		value := float64(used * 1.0 / self.GetCapacity())
+		value = float64(int(value*100+0.5) / 100.0)
+		extra.Add(jsonutils.NewFloat(value), "commit_rate")
+	} else {
+		extra.Add(jsonutils.NewFloat(0.0), "commit_rate")
+	}
+	extra.Add(jsonutils.NewFloat(float64(self.GetOvercommitBound())), "commit_bound")
+	return extra
+}
+
+func (self *SStorage) GetCustomizeColumns(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict {
+	extra := self.SEnabledStatusStandaloneResourceBase.GetCustomizeColumns(ctx, userCred, query)
+	return self.getMoreDetails(extra)
+}
+
 func (self *SStorage) GetUsedCapacity(isReady tristate.TriState) int {
 	disks := DiskManager.Query().SubQuery()
 	q := disks.Query(sqlchemy.SUM("sum", disks.Field("disk_size"))).Equals("storage_id", self.Id)

pkg/compute/tasks/cloud_provider_sync_info_task.go

@@ -10,6 +10,7 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/compute/models"
+	"yunion.io/x/onecloud/pkg/util/logclient"
 	"yunion.io/x/pkg/utils"
 )
 
@@ -21,6 +22,25 @@ func init() {
 	taskman.RegisterTask(CloudProviderSyncInfoTask{})
 }
 
+func getAction(params *jsonutils.JSONDict) string {
+	fullSync := jsonutils.QueryBoolean(params, "full_sync", false)
+	if !fullSync {
+		syncRangeJson, _ := params.Get("sync_range")
+		if syncRangeJson != nil {
+			fullSync = jsonutils.QueryBoolean(syncRangeJson, "full_sync", false)
+		}
+	}
+
+	action := ""
+
+	if fullSync {
+		action = logclient.ACT_CLOUD_FULLSYNC
+	} else {
+		action = logclient.ACT_CLOUD_SYNC
+	}
+	return action
+}
+
 func taskFail(ctx context.Context, task *CloudProviderSyncInfoTask, provider *models.SCloudprovider, reason string) {
 	provider.SetStatus(task.UserCred, models.CLOUD_PROVIDER_DISCONNECTED, reason)
 	task.SetStageFailed(ctx, reason)
@@ -31,7 +51,8 @@ func (self *CloudProviderSyncInfoTask) OnInit(ctx context.Context, obj db.IStand
 	provider.MarkStartSync(self.UserCred)
 	// do sync
 
-	log.Infof("Start sync cloud provider status ...")
+	notes := fmt.Sprintf("Start sync cloud provider status ...")
+	log.Infof(notes)
 	driver, err := provider.GetDriver()
 	if err != nil {
 		reason := fmt.Sprintf("Invalid cloud provider %s", err)
@@ -60,16 +81,18 @@ func (self *CloudProviderSyncInfoTask) OnInit(ctx context.Context, obj db.IStand
 
 	provider.SetStatus(self.UserCred, models.CLOUD_PROVIDER_CONNECTED, "")
 	self.SetStageComplete(ctx, nil)
+	logclient.AddActionLog(provider, getAction(self.Params), body, self.UserCred, true)
 }
 
 func logSyncFailed(provider *models.SCloudprovider, task taskman.ITask, reason string) {
 	db.OpsLog.LogEvent(provider, db.ACT_SYNC_HOST_COMPLETE, reason, task.GetUserCred())
+	logclient.AddActionLog(provider, getAction(task.GetParams()), reason, task.GetUserCred(), false)
 }
 
 func syncCloudProviderInfo(ctx context.Context, provider *models.SCloudprovider, task *CloudProviderSyncInfoTask, driver cloudprovider.ICloudProvider, syncRange *models.SSyncRange) {
-	log.Infof("Start sync host info ...")
+	notes := fmt.Sprintf("Start sync host info ...")
+	log.Infof(notes)
 	db.OpsLog.LogEvent(provider, db.ACT_SYNC_HOST_START, "", task.UserCred)
-
 	regions := driver.GetIRegions()
 	localRegions, remoteRegions, result := models.CloudregionManager.SyncRegions(ctx, task.UserCred, provider.Provider, regions)
 	msg := result.Result()
@@ -80,6 +103,7 @@ func syncCloudProviderInfo(ctx context.Context, provider *models.SCloudprovider,
 	}
 
 	db.OpsLog.LogEvent(provider, db.ACT_SYNC_HOST_COMPLETE, msg, task.UserCred)
+	logclient.AddActionLog(provider, getAction(task.Params), "", task.UserCred, true)
 	for i := 0; i < len(localRegions); i += 1 {
 		if len(syncRange.Region) > 0 && !utils.IsInStringArray(localRegions[i].Id, syncRange.Region) {
 			continue
@@ -112,12 +136,14 @@ func syncRegionZones(ctx context.Context, provider *models.SCloudprovider, task
 	}
 	localZones, remoteZones, result := models.ZoneManager.SyncZones(ctx, task.UserCred, localRegion, zones)
 	msg := result.Result()
-	log.Infof("SyncZones for region %s result: %s", localRegion.Name, msg)
+	notes := fmt.Sprintf("SyncZones for region %s result: %s", localRegion.Name, msg)
+	log.Infof(notes)
 	if result.IsError() {
 		logSyncFailed(provider, task, msg)
 		return nil, nil
 	}
 	db.OpsLog.LogEvent(provider, db.ACT_SYNC_HOST_COMPLETE, msg, task.UserCred)
+	logclient.AddActionLog(provider, getAction(task.Params), notes, task.UserCred, true)
 	return localZones, remoteZones
 }
 
@@ -132,13 +158,14 @@ func syncRegionVPCs(ctx context.Context, provider *models.SCloudprovider, task *
 
 	localVpcs, remoteVpcs, result := models.VpcManager.SyncVPCs(ctx, task.UserCred, provider, localRegion, vpcs)
 	msg := result.Result()
-	log.Infof("SyncVPCs for region %s result: %s", localRegion.Name, msg)
+	notes := fmt.Sprintf("SyncVPCs for region %s result: %s", localRegion.Name, msg)
+	log.Infof(notes)
 	if result.IsError() {
 		logSyncFailed(provider, task, msg)
 		return
 	}
 	db.OpsLog.LogEvent(provider, db.ACT_SYNC_HOST_COMPLETE, msg, task.UserCred)
-
+	logclient.AddActionLog(provider, getAction(task.Params), notes, task.UserCred, true)
 	for j := 0; j < len(localVpcs); j += 1 {
 		syncVpcWires(ctx, provider, task, &localVpcs[j], remoteVpcs[j])
 		syncVpcSecGroup(ctx, provider, task, &localVpcs[j], remoteVpcs[j])
@@ -154,7 +181,8 @@ func syncVpcSecGroup(ctx context.Context, provider *models.SCloudprovider, task
 	} else {
 		_, _, result := models.SecurityGroupManager.SyncSecgroups(ctx, task.UserCred, secgroups)
 		msg := result.Result()
-		log.Infof("SyncSecurityGroup for VPC %s result: %s", localVpc.Name, msg)
+		notes := fmt.Sprintf("SyncSecurityGroup for VPC %s result: %s", localVpc.Name, msg)
+		log.Infof(notes)
 		if result.IsError() {
 			logSyncFailed(provider, task, msg)
 			return
@@ -172,12 +200,14 @@ func syncVpcWires(ctx context.Context, provider *models.SCloudprovider, task tas
 	}
 	localWires, remoteWires, result := models.WireManager.SyncWires(ctx, task.GetUserCred(), localVpc, wires)
 	msg := result.Result()
-	log.Infof("SyncWires for VPC %s result: %s", localVpc.Name, msg)
+	notes := fmt.Sprintf("SyncWires for VPC %s result: %s", localVpc.Name, msg)
+	log.Infof(notes)
 	if result.IsError() {
 		logSyncFailed(provider, task, msg)
 		return
 	}
 	db.OpsLog.LogEvent(provider, db.ACT_SYNC_HOST_COMPLETE, msg, task.GetUserCred())
+	logclient.AddActionLog(provider, getAction(task.GetParams()), notes, task.GetUserCred(), true)
 	for i := 0; i < len(localWires); i += 1 {
 		syncWireNetworks(ctx, provider, task, &localWires[i], remoteWires[i])
 	}
@@ -193,12 +223,14 @@ func syncWireNetworks(ctx context.Context, provider *models.SCloudprovider, task
 	}
 	_, _, result := models.NetworkManager.SyncNetworks(ctx, task.GetUserCred(), localWire, nets)
 	msg := result.Result()
-	log.Infof("SyncNetworks for wire %s result: %s", localWire.Name, msg)
+	notes := fmt.Sprintf("SyncNetworks for wire %s result: %s", localWire.Name, msg)
+	log.Infof(notes)
 	if result.IsError() {
 		logSyncFailed(provider, task, msg)
 		return
 	}
 	db.OpsLog.LogEvent(provider, db.ACT_SYNC_HOST_COMPLETE, msg, task.GetUserCred())
+	logclient.AddActionLog(provider, getAction(task.GetParams()), notes, task.GetUserCred(), true)
 }
 
 func syncZoneStorages(ctx context.Context, provider *models.SCloudprovider, task *CloudProviderSyncInfoTask, localZone *models.SZone, remoteZone cloudprovider.ICloudZone) {
@@ -211,12 +243,14 @@ func syncZoneStorages(ctx context.Context, provider *models.SCloudprovider, task
 	}
 	localStorages, remoteStorages, result := models.StorageManager.SyncStorages(ctx, task.UserCred, provider, localZone, storages)
 	msg := result.Result()
-	log.Infof("SyncZones for region %s result: %s", localZone.Name, msg)
+	notes := fmt.Sprintf("SyncZones for region %s result: %s", localZone.Name, msg)
+	log.Infof(notes)
 	if result.IsError() {
 		logSyncFailed(provider, task, msg)
 		return
 	}
 	db.OpsLog.LogEvent(provider, db.ACT_SYNC_HOST_COMPLETE, msg, task.UserCred)
+	logclient.AddActionLog(provider, getAction(task.GetParams()), notes, task.GetUserCred(), true)
 
 	for i := 0; i < len(localStorages); i += 1 {
 		syncStorageCaches(ctx, provider, task, &localStorages[i], remoteStorages[i])
@@ -251,12 +285,14 @@ func syncStorageDisks(ctx context.Context, provider *models.SCloudprovider, task
 	}
 	_, _, result := models.DiskManager.SyncDisks(ctx, task.UserCred, localStorage, disks)
 	msg := result.Result()
-	log.Infof("SyncDisks for storage %s result: %s", localStorage.Name, msg)
+	notes := fmt.Sprintf("SyncDisks for storage %s result: %s", localStorage.Name, msg)
+	log.Infof(notes)
 	if result.IsError() {
 		logSyncFailed(provider, task, msg)
 		return
 	}
 	db.OpsLog.LogEvent(provider, db.ACT_SYNC_HOST_COMPLETE, msg, task.UserCred)
+	logclient.AddActionLog(provider, getAction(task.Params), notes, task.UserCred, true)
 }
 
 func syncZoneHosts(ctx context.Context, provider *models.SCloudprovider, task *CloudProviderSyncInfoTask, localZone *models.SZone, remoteZone cloudprovider.ICloudZone, syncRange *models.SSyncRange) {
@@ -269,13 +305,14 @@ func syncZoneHosts(ctx context.Context, provider *models.SCloudprovider, task *C
 	}
 	localHosts, remoteHosts, result := models.HostManager.SyncHosts(ctx, task.UserCred, provider, localZone, hosts)
 	msg := result.Result()
-	log.Infof("SyncHosts for zone %s result: %s", localZone.Name, msg)
+	notes := fmt.Sprintf("SyncHosts for zone %s result: %s", localZone.Name, msg)
+	log.Infof(notes)
 	if result.IsError() {
 		logSyncFailed(provider, task, msg)
 		return
 	}
 	db.OpsLog.LogEvent(provider, db.ACT_SYNC_HOST_COMPLETE, msg, task.UserCred)
-
+	logclient.AddActionLog(provider, getAction(task.Params), notes, task.UserCred, true)
 	for i := 0; i < len(localHosts); i += 1 {
 		if len(syncRange.Host) > 0 && !utils.IsInStringArray(localHosts[i].Id, syncRange.Host) {
 			continue
@@ -296,12 +333,14 @@ func syncHostStorages(ctx context.Context, provider *models.SCloudprovider, task
 	}
 	result := localHost.SyncHostStorages(ctx, task.UserCred, storages)
 	msg := result.Result()
-	log.Infof("SyncHostStorages for host %s result: %s", localHost.Name, msg)
+	notes := fmt.Sprintf("SyncHostStorages for host %s result: %s", localHost.Name, msg)
+	log.Infof(notes)
 	if result.IsError() {
 		logSyncFailed(provider, task, msg)
 		return
 	}
 	db.OpsLog.LogEvent(provider, db.ACT_SYNC_HOST_COMPLETE, msg, task.UserCred)
+	logclient.AddActionLog(provider, getAction(task.Params), notes, task.UserCred, true)
 }
 
 func syncHostWires(ctx context.Context, provider *models.SCloudprovider, task taskman.ITask, localHost *models.SHost, remoteHost cloudprovider.ICloudHost) {
@@ -314,12 +353,14 @@ func syncHostWires(ctx context.Context, provider *models.SCloudprovider, task ta
 	}
 	result := localHost.SyncHostWires(ctx, task.GetUserCred(), wires)
 	msg := result.Result()
-	log.Infof("SyncHostWires for host %s result: %s", localHost.Name, msg)
+	notes := fmt.Sprintf("SyncHostWires for host %s result: %s", localHost.Name, msg)
+	log.Infof(notes)
 	if result.IsError() {
 		logSyncFailed(provider, task, msg)
 		return
 	}
 	db.OpsLog.LogEvent(provider, db.ACT_SYNC_HOST_COMPLETE, msg, task.GetUserCred())
+	logclient.AddActionLog(provider, getAction(task.GetParams()), notes, task.GetUserCred(), true)
 }
 
 func syncHostVMs(ctx context.Context, provider *models.SCloudprovider, task *CloudProviderSyncInfoTask, localHost *models.SHost, remoteHost cloudprovider.ICloudHost) {
@@ -332,13 +373,14 @@ func syncHostVMs(ctx context.Context, provider *models.SCloudprovider, task *Clo
 	}
 	localVMs, remoteVMs, result := localHost.SyncHostVMs(ctx, task.UserCred, vms)
 	msg := result.Result()
-	log.Infof("SyncHostVMs for host %s result: %s", localHost.Name, msg)
+	notes := fmt.Sprintf("SyncHostVMs for host %s result: %s", localHost.Name, msg)
+	log.Infof(notes)
 	if result.IsError() {
 		logSyncFailed(provider, task, msg)
 		return
 	}
 	db.OpsLog.LogEvent(provider, db.ACT_SYNC_HOST_COMPLETE, msg, task.UserCred)
-
+	logclient.AddActionLog(provider, getAction(task.Params), notes, task.UserCred, true)
 	for i := 0; i < len(localVMs); i += 1 {
 		syncVMNics(ctx, provider, task, localHost, &localVMs[i], remoteVMs[i])
 		syncVMDisks(ctx, provider, task, localHost, &localVMs[i], remoteVMs[i])
@@ -355,12 +397,14 @@ func syncVMNics(ctx context.Context, provider *models.SCloudprovider, task *Clou
 	}
 	result := localVM.SyncVMNics(ctx, task.UserCred, host, nics)
 	msg := result.Result()
-	log.Infof("syncVMNics for VM %s result: %s", localVM.Name, msg)
+	notes := fmt.Sprintf("syncVMNics for VM %s result: %s", localVM.Name, msg)
+	log.Infof(notes)
 	if result.IsError() {
 		logSyncFailed(provider, task, msg)
 		return
 	}
 	db.OpsLog.LogEvent(provider, db.ACT_SYNC_HOST_COMPLETE, msg, task.UserCred)
+	logclient.AddActionLog(provider, getAction(task.Params), notes, task.UserCred, true)
 }
 
 func syncVMDisks(ctx context.Context, provider *models.SCloudprovider, task *CloudProviderSyncInfoTask, host *models.SHost, localVM *models.SGuest, remoteVM cloudprovider.ICloudVM) {
@@ -373,10 +417,12 @@ func syncVMDisks(ctx context.Context, provider *models.SCloudprovider, task *Clo
 	}
 	result := localVM.SyncVMDisks(ctx, task.UserCred, host, disks)
 	msg := result.Result()
-	log.Infof("syncVMNics for VM %s result: %s", localVM.Name, msg)
+	notes := fmt.Sprintf("syncVMNics for VM %s result: %s", localVM.Name, msg)
+	log.Infof(notes)
 	if result.IsError() {
 		logSyncFailed(provider, task, msg)
 		return
 	}
 	db.OpsLog.LogEvent(provider, db.ACT_SYNC_HOST_COMPLETE, msg, task.UserCred)
+	logclient.AddActionLog(provider, getAction(task.Params), notes, task.UserCred, true)
 }

pkg/compute/tasks/disk_save_task.go

@@ -0,0 +1,107 @@
+package tasks
+
+import (
+	"context"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	"yunion.io/x/onecloud/pkg/compute/models"
+	"yunion.io/x/onecloud/pkg/compute/options"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
+	mc "yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+type DiskSaveTask struct {
+	SDiskBaseTask
+}
+
+func init() {
+	taskman.RegisterTask(DiskSaveTask{})
+}
+
+func (self *DiskSaveTask) GetMasterHost(disk *models.SDisk) *models.SHost {
+	if guests := disk.GetGuests(); len(guests) == 1 {
+		if host := guests[0].GetHost(); host == nil {
+			if storage := disk.GetStorage(); storage != nil {
+				return storage.GetMasterHost()
+			}
+		} else {
+			return host
+		}
+	}
+	return nil
+}
+
+func (self *DiskSaveTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
+	disk := obj.(*models.SDisk)
+	if host := self.GetMasterHost(disk); host == nil {
+		resion := "Cannot find host for disk"
+		disk.SetDiskReady(ctx, self.GetUserCred(), resion)
+		self.TaskFailed(ctx, resion)
+		db.OpsLog.LogEvent(disk, db.ACT_SAVE_FAIL, resion, self.GetUserCred())
+	} else {
+		disk.SetStatus(self.GetUserCred(), models.DISK_START_SAVE, "")
+		for _, guest := range disk.GetGuests() {
+			guest.SetStatus(self.GetUserCred(), models.VM_SAVE_DISK, "")
+		}
+		self.StartBackupDisk(ctx, disk, host)
+	}
+}
+
+func (self *DiskSaveTask) StartBackupDisk(ctx context.Context, disk *models.SDisk, host *models.SHost) {
+	self.SetStage("on_disk_backup_complete", nil)
+	disk.SetStatus(self.GetUserCred(), models.DISK_SAVING, "")
+	imageId, _ := self.GetParams().GetString("image_id")
+	if err := host.GetHostDriver().RequestPrepareSaveDiskOnHost(ctx, host, disk, imageId, self); err != nil {
+		log.Errorf("Backup failed: %v", err)
+		disk.SetDiskReady(ctx, self.GetUserCred(), err.Error())
+		self.TaskFailed(ctx, err.Error())
+		db.OpsLog.LogEvent(disk, db.ACT_SAVE_FAIL, err.Error(), self.GetUserCred())
+	}
+}
+
+func (self *DiskSaveTask) OnDiskBackupCompleteFailed(ctx context.Context, disk *models.SDisk, data jsonutils.JSONObject) {
+	disk.SetDiskReady(ctx, self.GetUserCred(), data.String())
+	db.OpsLog.LogEvent(disk, db.ACT_SAVE_FAIL, data.String(), self.GetUserCred())
+}
+
+func (self *DiskSaveTask) OnDiskBackupComplete(ctx context.Context, disk *models.SDisk, data *jsonutils.JSONDict) {
+	disk.SetDiskReady(ctx, self.GetUserCred(), "")
+	db.OpsLog.LogEvent(disk, db.ACT_SAVE, disk.GetShortDesc(), self.GetUserCred())
+	self.SetStageComplete(ctx, nil)
+	imageId, _ := self.GetParams().GetString("image_id")
+	if host := self.GetMasterHost(disk); host == nil {
+		log.Errorf("Saved disk Host mast not be nil")
+		self.TaskFailed(ctx, "Saved disk Host mast not be nil")
+	} else {
+		if self.Params.Contains("format") {
+			format, _ := self.Params.Get("format")
+			data.Add(format, "format")
+		}
+		if err := self.UploadDisk(ctx, host, disk, imageId, data); err != nil {
+			log.Errorf("UploadDisk failed: %v", err)
+			self.TaskFailed(ctx, err.Error())
+		}
+		self.RefreshImageCache(ctx, imageId)
+	}
+}
+
+func (self *DiskSaveTask) RefreshImageCache(ctx context.Context, imageId string) {
+	models.CachedimageManager.GetImageById(ctx, self.GetUserCred(), imageId, true)
+}
+
+func (self *DiskSaveTask) UploadDisk(ctx context.Context, host *models.SHost, disk *models.SDisk, imageId string, data *jsonutils.JSONDict) error {
+	return host.GetHostDriver().RequestSaveUploadImageOnHost(ctx, host, disk, imageId, self, jsonutils.Marshal(data))
+}
+
+func (self *DiskSaveTask) TaskFailed(ctx context.Context, resion string) {
+	self.SetStageFailed(ctx, resion)
+	if imageId, err := self.GetParams().GetString("image_id"); err != nil && len(imageId) > 0 {
+		log.Errorf("save disk task failed, set image %s killed", imageId)
+		s := auth.GetAdminSession(options.Options.Region, "")
+		mc.Images.Update(s, imageId, jsonutils.Marshal(map[string]string{"status": "killed"}))
+	}
+}

pkg/compute/tasks/guest_batch_create_task.go

@@ -77,6 +77,9 @@ func (self *GuestBatchCreateTask) onScheduleFail(ctx context.Context, guest *mod
 	if len(msg) > 0 {
 		reason = fmt.Sprintf("%s: %s", reason, msg)
 	}
+	if guest.DisableDelete.IsTrue() {
+		guest.SetDisableDelete(false)
+	}
 	guest.SetStatus(self.UserCred, models.VM_SCHEDULE_FAILED, reason)
 	db.OpsLog.LogEvent(guest, db.ACT_ALLOCATE_FAIL, reason, self.UserCred)
 	notifyclient.NotifySystemError(guest.Id, guest.Name, models.VM_SCHEDULE_FAILED, reason)

pkg/compute/tasks/guest_change_config_task.go

@@ -9,6 +9,7 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
 	"yunion.io/x/onecloud/pkg/compute/models"
+	"yunion.io/x/onecloud/pkg/util/logclient"
 )
 
 type GuestChangeConfigTask struct {
@@ -41,37 +42,44 @@ func (self *GuestChangeConfigTask) OnDisksResizeComplete(ctx context.Context, ob
 		iResizeSet, err := resizeDisks.GetAt(i)
 		if err != nil {
 			self.SetStageFailed(ctx, err.Error())
+			logclient.AddActionLog(obj, logclient.ACT_VM_CHANGE_FLAVOR, err, self.UserCred, false)
 			return
 		}
 		resizeSet := iResizeSet.(*jsonutils.JSONArray)
 		diskId, err := resizeSet.GetAt(0)
 		if err != nil {
 			self.SetStageFailed(ctx, err.Error())
+			logclient.AddActionLog(obj, logclient.ACT_VM_CHANGE_FLAVOR, err, self.UserCred, false)
 			return
 		}
 		idStr, err := diskId.GetString()
 		if err != nil {
 			self.SetStageFailed(ctx, err.Error())
+			logclient.AddActionLog(obj, logclient.ACT_VM_CHANGE_FLAVOR, err, self.UserCred, false)
 			return
 		}
 		jSize, err := resizeSet.GetAt(1)
 		if err != nil {
 			self.SetStageFailed(ctx, err.Error())
+			logclient.AddActionLog(obj, logclient.ACT_VM_CHANGE_FLAVOR, err, self.UserCred, false)
 			return
 		}
 		size, err := jSize.Int()
 		if err != nil {
 			self.SetStageFailed(ctx, err.Error())
+			logclient.AddActionLog(obj, logclient.ACT_VM_CHANGE_FLAVOR, err, self.UserCred, false)
 			return
 		}
 		iDisk, err := models.DiskManager.FetchById(idStr)
 		if err != nil {
 			self.SetStageFailed(ctx, err.Error())
+			logclient.AddActionLog(obj, logclient.ACT_VM_CHANGE_FLAVOR, err, self.UserCred, false)
 			return
 		}
 		disk := iDisk.(*models.SDisk)
 		if err != nil {
 			self.SetStageFailed(ctx, err.Error())
+			logclient.AddActionLog(disk, logclient.ACT_VM_CHANGE_FLAVOR, err, self.UserCred, false)
 			return
 		}
 		if disk.DiskSize < int(size) {
@@ -79,6 +87,7 @@ func (self *GuestChangeConfigTask) OnDisksResizeComplete(ctx context.Context, ob
 			err = self.GetPendingUsage(&pendingUsage)
 			if err != nil {
 				self.SetStageFailed(ctx, err.Error())
+				logclient.AddActionLog(disk, logclient.ACT_VM_CHANGE_FLAVOR, err, self.UserCred, false)
 				return
 			}
 			disk.StartDiskResizeTask(ctx, self.UserCred, size, self.GetTaskId(), &pendingUsage)
@@ -112,6 +121,7 @@ func (self *GuestChangeConfigTask) OnCreateDisksComplete(ctx context.Context, ob
 			vcpuCount, err = iVcpuCount.Int()
 			if err != nil {
 				self.SetStageFailed(ctx, err.Error())
+				logclient.AddActionLog(guest, logclient.ACT_VM_CHANGE_FLAVOR, err, self.UserCred, false)
 				return
 			}
 		}
@@ -119,12 +129,14 @@ func (self *GuestChangeConfigTask) OnCreateDisksComplete(ctx context.Context, ob
 			vmemSize, err = iVmemSize.Int()
 			if err != nil {
 				self.SetStageFailed(ctx, err.Error())
+				logclient.AddActionLog(guest, logclient.ACT_VM_CHANGE_FLAVOR, err, self.UserCred, false)
 				return
 			}
 		}
 		err = guest.GetDriver().RequestChangeVmConfig(ctx, guest, self, vcpuCount, vmemSize)
 		if err != nil {
 			self.SetStageFailed(ctx, err.Error())
+			logclient.AddActionLog(guest, logclient.ACT_VM_CHANGE_FLAVOR, err, self.UserCred, false)
 			return
 		}
 		var addCpu, addMem = 0, 0
@@ -151,12 +163,14 @@ func (self *GuestChangeConfigTask) OnCreateDisksComplete(ctx context.Context, ob
 		})
 		if err != nil {
 			self.SetStageFailed(ctx, err.Error())
+			logclient.AddActionLog(guest, logclient.ACT_VM_CHANGE_FLAVOR, err, self.UserCred, false)
 			return
 		}
 		var pendingUsage models.SQuota
 		err = self.GetPendingUsage(&pendingUsage)
 		if err != nil {
 			self.SetStageFailed(ctx, err.Error())
+			logclient.AddActionLog(guest, logclient.ACT_VM_CHANGE_FLAVOR, err, self.UserCred, false)
 			return
 		}
 		// ownerCred := guest.GetOwnerUserCred()
@@ -172,11 +186,13 @@ func (self *GuestChangeConfigTask) OnCreateDisksComplete(ctx context.Context, ob
 		err = models.QuotaManager.CancelPendingUsage(ctx, self.UserCred, guest.ProjectId, &pendingUsage, &cancelUsage)
 		if err != nil {
 			self.SetStageFailed(ctx, err.Error())
+			logclient.AddActionLog(guest, logclient.ACT_VM_CHANGE_FLAVOR, err, self.UserCred, false)
 			return
 		}
 		err = self.SetPendingUsage(&pendingUsage)
 		if err != nil {
 			self.SetStageFailed(ctx, err.Error())
+			logclient.AddActionLog(guest, logclient.ACT_VM_CHANGE_FLAVOR, err, self.UserCred, false)
 			return
 		}
 	}
@@ -184,6 +200,7 @@ func (self *GuestChangeConfigTask) OnCreateDisksComplete(ctx context.Context, ob
 	err = guest.StartSyncstatus(ctx, self.UserCred, self.GetTaskId())
 	if err != nil {
 		self.SetStageFailed(ctx, err.Error())
+		logclient.AddActionLog(guest, logclient.ACT_VM_CHANGE_FLAVOR, err, self.UserCred, false)
 		return
 	}
 }
@@ -193,10 +210,12 @@ func (self *GuestChangeConfigTask) OnSyncStatusComplete(ctx context.Context, obj
 	if guest.Status == models.VM_READY && jsonutils.QueryBoolean(self.Params, "auto_start", false) {
 		self.SetStage("on_guest_start_complete", nil)
 		guest.StartGueststartTask(ctx, self.UserCred, nil, self.GetTaskId())
+		logclient.AddActionLog(guest, logclient.ACT_VM_CHANGE_FLAVOR, "", self.UserCred, true)
 	} else {
 		dt := jsonutils.NewDict()
 		dt.Add(jsonutils.NewString(guest.Id), "id")
 		self.SetStageComplete(ctx, dt)
+		logclient.AddActionLog(guest, logclient.ACT_VM_CHANGE_FLAVOR, "", self.UserCred, true)
 	}
 }
 

pkg/compute/tasks/guest_create_task.go

@@ -11,6 +11,7 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
 	"yunion.io/x/onecloud/pkg/cloudcommon/notifyclient"
 	"yunion.io/x/onecloud/pkg/compute/models"
+	"yunion.io/x/onecloud/pkg/util/logclient"
 )
 
 type GuestCreateTask struct {
@@ -49,6 +50,7 @@ func (self *GuestCreateTask) OnDiskPreparedFailed(ctx context.Context, obj db.IS
 	guest := obj.(*models.SGuest)
 	guest.SetStatus(self.UserCred, models.VM_DISK_FAILED, "allocation failed")
 	db.OpsLog.LogEvent(guest, db.ACT_ALLOCATE_FAIL, data, self.UserCred)
+	logclient.AddActionLog(guest, logclient.ACT_ALLOCATE, data, self.UserCred, false)
 	notifyclient.NotifySystemError(guest.Id, guest.Name, models.VM_DISK_FAILED, data.String())
 }
 
@@ -76,6 +78,7 @@ func (self *GuestCreateTask) OnCdromPreparedFailed(ctx context.Context, obj db.I
 	guest := obj.(*models.SGuest)
 	guest.SetStatus(self.UserCred, models.VM_DISK_FAILED, "")
 	db.OpsLog.LogEvent(guest, db.ACT_ALLOCATE_FAIL, data, self.UserCred)
+	logclient.AddActionLog(guest, logclient.ACT_ALLOCATE, data, self.UserCred, false)
 	notifyclient.NotifySystemError(guest.Id, guest.Name, models.VM_DISK_FAILED, fmt.Sprintf("cdrom_failed %s", data))
 }
 
@@ -102,6 +105,7 @@ func (self *GuestCreateTask) OnDeployGuestDescCompleteFailed(ctx context.Context
 	guest := obj.(*models.SGuest)
 	guest.SetStatus(self.UserCred, models.VM_DEPLOY_FAILED, "deploy_failed")
 	db.OpsLog.LogEvent(guest, db.ACT_ALLOCATE_FAIL, data, self.UserCred)
+	logclient.AddActionLog(guest, logclient.ACT_ALLOCATE, data, self.UserCred, false)
 	notifyclient.NotifySystemError(guest.Id, guest.Name, models.VM_DEPLOY_FAILED, data.String())
 }
 

pkg/compute/tasks/guest_delete_task.go

@@ -5,13 +5,14 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
-	"yunion.io/x/pkg/utils"
 
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
 	"yunion.io/x/onecloud/pkg/cloudcommon/notifyclient"
 	"yunion.io/x/onecloud/pkg/compute/models"
 	"yunion.io/x/onecloud/pkg/compute/options"
+	"yunion.io/x/onecloud/pkg/util/logclient"
+	"yunion.io/x/pkg/utils"
 )
 
 type GuestDeleteTask struct {
@@ -93,6 +94,7 @@ func (self *GuestDeleteTask) OnGuestDeleteCompleteFailed(ctx context.Context, ob
 	guest := obj.(*models.SGuest)
 	guest.SetStatus(self.UserCred, models.VM_DELETE_FAIL, err.String())
 	db.OpsLog.LogEvent(guest, db.ACT_DELOCATE_FAIL, err, self.UserCred)
+	logclient.AddActionLog(guest, logclient.ACT_DELETE, err, self.UserCred, false)
 }
 
 func (self *GuestDeleteTask) OnGuestDeleteComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
@@ -108,6 +110,7 @@ func (self *GuestDeleteTask) DeleteGuest(ctx context.Context, guest *models.SGue
 	guest.RealDelete(ctx, self.UserCred)
 	guest.RemoveAllMetadata(ctx, self.UserCred)
 	db.OpsLog.LogEvent(guest, db.ACT_DELOCATE, nil, self.UserCred)
+	logclient.AddActionLog(guest, logclient.ACT_DELETE, nil, self.UserCred, true)
 	if !guest.IsSystem && !guest.PendingDeleted {
 		self.NotifyServerDeleted(ctx, guest)
 	}

pkg/compute/tasks/guest_deploy_task.go

@@ -9,6 +9,7 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
 	"yunion.io/x/onecloud/pkg/compute/models"
+	"yunion.io/x/onecloud/pkg/util/logclient"
 )
 
 type GuestDeployTask struct {
@@ -61,6 +62,7 @@ func (self *GuestDeployTask) StartDeployGuestOnHost(ctx context.Context, guest *
 func (self *GuestDeployTask) OnDeployGuestFail(ctx context.Context, guest *models.SGuest, err error) {
 	guest.SetStatus(self.UserCred, models.VM_DEPLOY_FAILED, err.Error())
 	self.SetStageFailed(ctx, err.Error())
+	logclient.AddActionLog(guest, logclient.ACT_VM_DEPLOY, err, self.UserCred, false)
 }
 
 func (self *GuestDeployTask) OnDeployGuestComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
@@ -68,6 +70,30 @@ func (self *GuestDeployTask) OnDeployGuestComplete(ctx context.Context, obj db.I
 	guest := obj.(*models.SGuest)
 	guest.GetDriver().OnGuestDeployTaskDataReceived(ctx, guest, self, data)
 	guest.GetDriver().OnGuestDeployTaskComplete(ctx, guest, self)
+	action, _ := self.Params.GetString("deploy_action")
+	keypair, _ := self.Params.GetString("keypair")
+	reset_password := jsonutils.QueryBoolean(self.Params, "reset_password", false)
+	unbind_kp := jsonutils.QueryBoolean(self.Params, "__delete_keypair__", false)
+	_log := false
+	if action == "deploy" {
+		if len(keypair) >= 32 {
+			if unbind_kp {
+				logclient.AddActionLog(guest, logclient.ACT_VM_UNBIND_KEYPAIR, nil, self.UserCred, true)
+				_log = true
+			} else {
+				logclient.AddActionLog(guest, logclient.ACT_VM_BIND_KEYPAIR, nil, self.UserCred, true)
+				_log = true
+			}
+
+		} else if reset_password {
+			logclient.AddActionLog(guest, logclient.ACT_VM_RESET_PSWD, "", self.UserCred, true)
+			_log = true
+		}
+	}
+	if !_log {
+		// 如果 deploy 有其他事件，统一记在这里。
+		logclient.AddActionLog(guest, "misc部署", self.Params, self.UserCred, true)
+	}
 }
 
 func (self *GuestDeployTask) OnDeployGuestCompleteFailed(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {

pkg/compute/tasks/guest_rebuild_root_task.go

@@ -11,6 +11,7 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
 	"yunion.io/x/onecloud/pkg/cloudcommon/notifyclient"
 	"yunion.io/x/onecloud/pkg/compute/models"
+	"yunion.io/x/onecloud/pkg/util/logclient"
 )
 
 func init() {
@@ -48,6 +49,7 @@ func (self *GuestRebuildRootTask) StartRebuildRootDisk(ctx context.Context, gues
 	})
 	if err != nil {
 		self.SetStageFailed(ctx, err.Error())
+		logclient.AddActionLog(guest, logclient.ACT_VM_REBUILD, err, self.UserCred, false)
 		return
 	} else {
 		db.OpsLog.LogEvent(gds.Root, db.ACT_UPDATE_STATUS,
@@ -64,16 +66,19 @@ func (self *GuestRebuildRootTask) OnRebuildRootDiskComplete(ctx context.Context,
 	imginfo, err := models.CachedimageManager.GetImageById(ctx, self.UserCred, imgId, false)
 	if err != nil {
 		self.SetStageFailed(ctx, err.Error())
+		logclient.AddActionLog(guest, logclient.ACT_VM_REBUILD, err, self.UserCred, false)
 		return
 	}
 	osprof, err := osprofile.GetOSProfileFromImageProperties(imginfo.Properties, guest.Hypervisor)
 	if err != nil {
 		self.SetStageFailed(ctx, err.Error())
+		logclient.AddActionLog(guest, logclient.ACT_VM_REBUILD, err, self.UserCred, false)
 		return
 	}
 	err = guest.SetMetadata(ctx, "__os_profile__", osprof, self.UserCred)
 	if err != nil {
 		self.SetStageFailed(ctx, err.Error())
+		logclient.AddActionLog(guest, logclient.ACT_VM_REBUILD, err, self.UserCred, false)
 		return
 	}
 	if guest.OsType != osprof.OSType {
@@ -83,20 +88,20 @@ func (self *GuestRebuildRootTask) OnRebuildRootDiskComplete(ctx context.Context,
 		})
 		if err != nil {
 			self.SetStageFailed(ctx, err.Error())
+			logclient.AddActionLog(guest, logclient.ACT_VM_REBUILD, err, self.UserCred, false)
 			return
 		}
 	}
 	db.OpsLog.LogEvent(guest, db.ACT_REBUILD_ROOT, "", self.UserCred)
-	// TODO: logclient
 	guest.NotifyServerEvent(notifyclient.SERVER_REBUILD_ROOT, notifyclient.PRIORITY_IMPORTANT, true)
 	self.SetStage("OnSyncStatusComplete", nil)
 	guest.StartSyncstatus(ctx, self.UserCred, self.GetTaskId())
 }
 
 func (self *GuestRebuildRootTask) OnRebuildRootDiskCompleteFailed(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
-	db.OpsLog.LogEvent(guest, db.ACT_REBUILD_ROOT_FAIL, data.String(), self.UserCred)
+	db.OpsLog.LogEvent(guest, db.ACT_REBUILD_ROOT_FAIL, data, self.UserCred)
 	guest.SetStatus(self.UserCred, models.VM_REBUILD_ROOT_FAIL, "")
-	// TODO: logclient
+	logclient.AddActionLog(guest, logclient.ACT_VM_REBUILD, data, self.UserCred, false)
 }
 
 func (self *GuestRebuildRootTask) OnSyncStatusComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
@@ -106,6 +111,7 @@ func (self *GuestRebuildRootTask) OnSyncStatusComplete(ctx context.Context, gues
 	} else {
 		self.SetStageComplete(ctx, nil)
 	}
+	logclient.AddActionLog(guest, logclient.ACT_VM_REBUILD, "", self.UserCred, true)
 }
 
 func (self *GuestRebuildRootTask) OnGuestStartComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
@@ -137,8 +143,10 @@ func (self *KVMGuestRebuildRootTask) OnRebuildRootDiskComplete(ctx context.Conte
 
 func (self *KVMGuestRebuildRootTask) OnRebuildRootDiskCompleteFailed(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
 	self.SetStageFailed(ctx, data.String())
+	logclient.AddActionLog(guest, logclient.ACT_VM_REBUILD, data, self.UserCred, false)
 }
 
 func (self *KVMGuestRebuildRootTask) OnGuestDeployComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
 	self.SetStageComplete(ctx, nil)
-}
+	logclient.AddActionLog(guest, logclient.ACT_VM_REBUILD, nil, self.UserCred, true)
+}

pkg/compute/tasks/guest_save_image_task.go

@@ -0,0 +1,65 @@
+package tasks
+
+import (
+	"context"
+	"fmt"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	"yunion.io/x/onecloud/pkg/compute/models"
+)
+
+type GuestSaveImageTask struct {
+	SGuestBaseTask
+}
+
+func init() {
+	taskman.RegisterTask(GuestSaveImageTask{})
+}
+
+func (self *GuestSaveImageTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) {
+	guest := obj.(*models.SGuest)
+	log.Infof("Saving server image: %s", guest.Name)
+	if restart, _ := self.GetParams().Bool("restart"); restart {
+		self.SetStage("on_stop_server_complete", nil)
+		guest.StartGuestStopTask(ctx, self.GetUserCred(), false, self.GetTaskId())
+	} else {
+		self.OnStopServerComplete(ctx, guest, nil)
+	}
+}
+
+func (self *GuestSaveImageTask) OnStopServerComplete(ctx context.Context, guest *models.SGuest, body jsonutils.JSONObject) {
+	if guest.Status != models.VM_READY {
+		resion := fmt.Sprintf("Server %s not in ready status", guest.Name)
+		log.Errorf(resion)
+		self.SetStageFailed(ctx, resion)
+	} else {
+		self.SetStage("on_save_root_image_complete", nil)
+		guest.SetStatus(self.GetUserCred(), models.VM_START_SAVE_DISK, "")
+		disks := guest.CategorizeDisks()
+		if err := disks.Root.StartDiskSaveTask(ctx, self.GetUserCred(), self.GetParams(), self.GetTaskId()); err != nil {
+			self.SetStageFailed(ctx, err.Error())
+		}
+	}
+}
+
+func (self *GuestSaveImageTask) OnSaveRootImageComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
+	if restart, _ := self.GetParams().Bool("restart"); restart {
+		self.SetStage("on_start_server_complete", nil)
+		guest.StartGueststartTask(ctx, self.GetUserCred(), nil, self.GetTaskId())
+	} else {
+		self.SetStageComplete(ctx, nil)
+	}
+}
+
+func (self *GuestSaveImageTask) OnSaveRootImageCompleteFailed(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
+	log.Errorf("Guest save root image failed: %s", data.PrettyString())
+	guest.SetStatus(self.GetUserCred(), models.VM_SAVE_DISK_FAILED, data.PrettyString())
+	self.SetStageFailed(ctx, data.PrettyString())
+}
+
+func (self *GuestSaveImageTask) OnStartServerCompleteFailed(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
+	self.SetStageComplete(ctx, nil)
+}

pkg/compute/tasks/guest_start_task.go

@@ -4,10 +4,10 @@ import (
 	"context"
 
 	"yunion.io/x/jsonutils"
-
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
 	"yunion.io/x/onecloud/pkg/compute/models"
+	"yunion.io/x/onecloud/pkg/util/logclient"
 )
 
 type GuestStartTask struct {
@@ -59,6 +59,7 @@ func (self *GuestStartTask) OnStartComplete(ctx context.Context, obj db.IStandal
 	db.OpsLog.LogEvent(guest, db.ACT_START, guest.GetShortDesc(), self.UserCred)
 	self.SetStage("on_guest_syncstatus_after_start", nil)
 	guest.StartSyncstatus(ctx, self.UserCred, self.GetTaskId())
+	logclient.AddActionLog(guest, logclient.ACT_VM_START, "", self.UserCred, true)
 	// self.taskComplete(ctx, guest)
 }
 
@@ -76,6 +77,7 @@ func (self *GuestStartTask) onStartGuestFailed(ctx context.Context, guest *model
 	guest.SetStatus(self.UserCred, models.VM_START_FAILED, err.Error())
 	self.SetStageFailed(ctx, err.Error())
 	self.OnStartCompleteFailed(ctx, guest, jsonutils.NewString(err.Error()))
+	logclient.AddActionLog(guest, logclient.ACT_VM_START, err, self.UserCred, false)
 }
 
 func (self *GuestStartTask) taskComplete(ctx context.Context, guest *models.SGuest) {

pkg/compute/tasks/guest_stop_task.go

@@ -6,10 +6,10 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
-
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
 	"yunion.io/x/onecloud/pkg/compute/models"
+	"yunion.io/x/onecloud/pkg/util/logclient"
 )
 
 type GuestStopTask struct {
@@ -58,10 +58,12 @@ func (self *GuestStopTask) OnGuestStopTaskComplete(ctx context.Context, obj db.I
 	if guest.Status == models.VM_READY && guest.DisableDelete.IsFalse() && guest.ShutdownBehavior == models.SHUTDOWN_TERMINATE {
 		guest.StartAutoDeleteGuestTask(ctx, self.UserCred, "")
 	}
+	logclient.AddActionLog(guest, logclient.ACT_VM_STOP, "", self.UserCred, true)
 }
 
 func (self *GuestStopTask) OnStopGuestFail(ctx context.Context, guest *models.SGuest, err error) {
 	guest.SetStatus(self.UserCred, models.VM_STOP_FAILED, err.Error())
 	db.OpsLog.LogEvent(guest, db.ACT_STOP_FAIL, err.Error(), self.UserCred)
 	self.SetStageFailed(ctx, err.Error())
+	logclient.AddActionLog(guest, logclient.ACT_VM_STOP, err, self.UserCred, false)
 }

pkg/compute/tasks/guest_syncstatus_task.go

@@ -9,6 +9,7 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/compute/models"
+	"yunion.io/x/onecloud/pkg/util/logclient"
 )
 
 type GuestSyncstatusTask struct {
@@ -53,9 +54,11 @@ func (self *GuestSyncstatusTask) OnGetStatusSucc(ctx context.Context, guest *mod
 	statusData.Add(jsonutils.NewString(statusStr), "status")
 	guest.PerformStatus(ctx, self.UserCred, nil, statusData)
 	self.SetStageComplete(ctx, nil)
+	logclient.AddActionLog(guest, logclient.ACT_VM_SYNC_STATUS, "", self.UserCred, true)
 }
 
 func (self *GuestSyncstatusTask) OnGetStatusFail(ctx context.Context, guest *models.SGuest, err error) {
 	guest.SetStatus(self.UserCred, models.VM_UNKNOWN, err.Error())
 	self.SetStageComplete(ctx, nil)
+	logclient.AddActionLog(guest, logclient.ACT_VM_SYNC_STATUS, err, self.UserCred, false)
 }

pkg/mcclient/auth/auth.go

@@ -123,6 +123,9 @@ func newAuthManager(cli *mcclient.Client, info *AuthInfo) *authManager {
 }
 
 func (a *authManager) verify(token string) (mcclient.TokenCredential, error) {
+	if a.adminCredential == nil {
+		return nil, fmt.Errorf("No valid admin token credential")
+	}
 	cred, err := a.tokenCacheVerify.Verify(a.client, a.adminCredential.GetTokenString(), token)
 	if err != nil {
 		return nil, err

pkg/mcclient/modules/k8s/raw.go

@@ -2,13 +2,11 @@ package k8s
 
 import (
 	"fmt"
-	"net/url"
 	"strings"
 
 	"yunion.io/x/jsonutils"
 
 	"yunion.io/x/onecloud/pkg/mcclient"
-	"yunion.io/x/onecloud/pkg/mcclient/modules"
 )
 
 var RawResource *RawResourceManager
@@ -19,33 +17,23 @@ func init() {
 	}
 }
 
-type rawResourceContext struct {
+type rawResource struct {
 	kind  string
 	name  string
 	query jsonutils.JSONObject
-	ctxs  []modules.ManagerContext
 }
 
-func newRawResourceContext(kind, namespace, name string, query jsonutils.JSONObject, ctxs []modules.ManagerContext) *rawResourceContext {
+func newRawResource(kind, namespace, name string, query jsonutils.JSONObject) *rawResource {
 	nsQuery := getNamespaceQuery(namespace)
 	if query != nil {
 		nsQuery.Update(query)
 	}
-	ctx := &rawResourceContext{kind: kind, name: name, query: nsQuery, ctxs: ctxs}
+	ctx := &rawResource{kind: kind, name: name, query: nsQuery}
 	return ctx
 }
 
-func (ctx rawResourceContext) contextPath() string {
+func (ctx rawResource) path() string {
 	segs := make([]string, 0)
-	ctxs := ctx.ctxs
-	if ctxs != nil && len(ctxs) > 0 {
-		for _, c := range ctxs {
-			segs = append(segs, c.InstanceManager.KeyString())
-			if len(c.InstanceId) > 0 {
-				segs = append(segs, url.PathEscape(c.InstanceId))
-			}
-		}
-	}
 	segs = append(segs, "_raw", ctx.kind, ctx.name)
 	path := fmt.Sprintf("/%s", strings.Join(segs, "/"))
 	if ctx.query != nil {
@@ -74,22 +62,22 @@ func getNamespaceQuery(namespace string) *jsonutils.JSONDict {
 	return query
 }
 
-func (m *RawResourceManager) Get(s *mcclient.ClientSession, kind string, namespace string, name string, query jsonutils.JSONObject, ctxs []modules.ManagerContext) (jsonutils.JSONObject, error) {
-	ctx := newRawResourceContext(kind, namespace, name, query, ctxs)
-	return m.request(s, "GET", ctx.contextPath(), nil)
+func (m *RawResourceManager) Get(s *mcclient.ClientSession, kind string, namespace string, name string, query jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	ctx := newRawResource(kind, namespace, name, query)
+	return m.request(s, "GET", ctx.path(), nil)
 }
 
-func (m *RawResourceManager) Put(s *mcclient.ClientSession, kind string, namespace string, name string, body jsonutils.JSONObject, ctxs []modules.ManagerContext) error {
+func (m *RawResourceManager) Put(s *mcclient.ClientSession, kind string, namespace string, name string, body jsonutils.JSONObject) error {
 	rawBytes := body.String()
 	newBody := jsonutils.NewDict()
 	newBody.Add(jsonutils.NewString(rawBytes), "raw")
-	ctx := newRawResourceContext(kind, namespace, name, nil, ctxs)
-	_, err := m.request(s, "PUT", ctx.contextPath(), newBody)
+	ctx := newRawResource(kind, namespace, name, nil)
+	_, err := m.request(s, "PUT", ctx.path(), newBody)
 	return err
 }
 
-func (m *RawResourceManager) Delete(s *mcclient.ClientSession, kind string, namespace string, name string, query jsonutils.JSONObject, ctxs []modules.ManagerContext) error {
-	ctx := newRawResourceContext(kind, namespace, name, query, ctxs)
-	_, err := m.request(s, "DELETE", ctx.contextPath(), nil)
+func (m *RawResourceManager) Delete(s *mcclient.ClientSession, kind string, namespace string, name string, query jsonutils.JSONObject) error {
+	ctx := newRawResource(kind, namespace, name, query)
+	_, err := m.request(s, "DELETE", ctx.path(), nil)
 	return err
 }

pkg/mcclient/modules/mod_images.go

@@ -428,24 +428,32 @@ func (this *ImageManager) _create(s *mcclient.ClientSession, params jsonutils.JS
 	if !exists {
 		return nil, fmt.Errorf("Unsupported image format %s", format)
 	}
-	osType, err := params.GetString("properties", "os_type")
-	if err != nil {
-		return nil, fmt.Errorf("Can't get os_type from params: %s", params.String())
-	}
-	exists, _ = utils.InStringArray(osType, []string{"Windows", "Linux", "Freebsd", "Android", "macOS", "VMWare"})
-	if !exists {
-		return nil, fmt.Errorf("OS type must be specified")
-	}
-	name, _ := params.GetString("name")
-	if len(name) == 0 {
-		return nil, fmt.Errorf("Missing name")
-	}
-	dupName, e := this.IsNameDuplicate(s, name)
-	if dupName {
-		return nil, fmt.Errorf("Duplicate name %s", name)
-	}
-	if e != nil {
-		return nil, fmt.Errorf("Check name duplicate error %s", e)
+	imageId, _ := params.GetString("image_id")
+	path := fmt.Sprintf("/%s", this.URLPath())
+	method := "POST"
+	if len(imageId) == 0 {
+		osType, err := params.GetString("properties", "os_type")
+		if err != nil {
+			return nil, fmt.Errorf("Can't get os_type from params: %s", params.String())
+		}
+		exists, _ = utils.InStringArray(osType, []string{"Windows", "Linux", "Freebsd", "Android", "macOS", "VMWare"})
+		if !exists {
+			return nil, fmt.Errorf("OS type must be specified")
+		}
+		name, _ := params.GetString("name")
+		if len(name) == 0 {
+			return nil, fmt.Errorf("Missing name")
+		}
+		dupName, e := this.IsNameDuplicate(s, name)
+		if dupName {
+			return nil, fmt.Errorf("Duplicate name %s", name)
+		}
+		if e != nil {
+			return nil, fmt.Errorf("Check name duplicate error %s", e)
+		}
+	} else {
+		path = fmt.Sprintf("/%s/%s", this.URLPath(), imageId)
+		method = "PUT"
 	}
 	headers, e := setImageMeta(params)
 	if e != nil {
@@ -467,8 +475,7 @@ func (this *ImageManager) _create(s *mcclient.ClientSession, params jsonutils.JS
 			headers.Add("Content-Length", fmt.Sprintf("%d", size))
 		}
 	}
-	path := fmt.Sprintf("/%s", this.URLPath())
-	resp, err := this.rawRequest(s, "POST", path, headers, body)
+	resp, err := this.rawRequest(s, method, path, headers, body)
 	_, json, err := s.ParseJSONResponse(resp, err)
 	if err != nil {
 		return nil, err

pkg/mcclient/options/base.go

@@ -144,7 +144,7 @@ func ListStructToParams(v interface{}) (*jsonutils.JSONDict, error) {
 type BaseListOptions struct {
 	Limit         *int     `default:"20" help:"Page limit"`
 	Offset        *int     `default:"0" help:"Page offset"`
-	OrderBy       string   `help:"Name of the field to be ordered by"`
+	OrderBy       []string `help:"Name of the field to be ordered by"`
 	Order         string   `help:"List order" choices:"desc|asc"`
 	Details       *bool    `help:"Show more details"`
 	Search        string   `help:"Filter results by a simple keyword search"`

pkg/util/aliyun/disk.go

@@ -61,6 +61,8 @@ func (self *SDisk) GetMetadata() *jsonutils.JSONDict {
 	priceKey := fmt.Sprintf("%s::%s::%s", self.RegionId, self.Category, self.Type)
 	data.Add(jsonutils.NewString(priceKey), "price_key")
 
+	data.Add(jsonutils.NewString(models.HYPERVISOR_ALIYUN), "hypervisor")
+
 	return data
 }
 
@@ -107,6 +109,11 @@ func (self *SDisk) GetId() string {
 }
 
 func (self *SDisk) Delete() error {
+	if _, err := self.storage.zone.region.getDisk(self.DiskId); err == cloudprovider.ErrNotFound {
+		// 未找到disk, 说明disk已经被删除了. 避免回收站中disk-delete循环删除失败
+		log.Errorf("Failed to find disk %s when delete", self.DiskId)
+		return nil
+	}
 	return self.storage.zone.region.deleteDisk(self.DiskId)
 }
 
@@ -260,3 +267,110 @@ func (self *SRegion) resizeDisk(diskId string, size int64) error {
 
 	return nil
 }
+
+func (self *SDisk) CreateISnapshot(name, desc string) (cloudprovider.ICloudSnapshot, error) {
+	if snapshotId, err := self.storage.zone.region.CreateSnapshot(self.DiskId, name, desc); err != nil {
+		log.Errorf("createSnapshot fail %s", err)
+		return nil, err
+	} else if snapshot, err := self.getSnapshot(snapshotId); err != nil {
+		return nil, err
+	} else {
+		snapshot.disk = self
+		if err := cloudprovider.WaitStatus(snapshot, string(SnapshotStatusAccoplished), 15*time.Second, 3600*time.Second); err != nil {
+			return nil, err
+		}
+		return snapshot, nil
+	}
+}
+
+func (self *SRegion) CreateSnapshot(diskId, name, desc string) (string, error) {
+	params := make(map[string]string)
+	params["RegionId"] = self.RegionId
+	params["DiskId"] = diskId
+	params["SnapshotName"] = name
+	params["Description"] = desc
+
+	if body, err := self.ecsRequest("CreateSnapshot", params); err != nil {
+		log.Errorf("CreateSnapshot fail %s", err)
+		return "", err
+	} else {
+		return body.GetString("SnapshotId")
+	}
+}
+
+func (self *SDisk) GetISnapshot(snapshotId string) (cloudprovider.ICloudSnapshot, error) {
+	if snapshot, err := self.getSnapshot(snapshotId); err != nil {
+		return nil, err
+	} else {
+		snapshot.disk = self
+		return snapshot, nil
+	}
+}
+
+func (self *SDisk) getSnapshot(snapshotId string) (*SSnapshot, error) {
+	if snapshots, total, err := self.storage.zone.region.GetSnapshots("", "", "", []string{snapshotId}, 0, 1); err != nil {
+		return nil, err
+	} else if total != 1 {
+		return nil, cloudprovider.ErrNotFound
+	} else {
+		return &snapshots[0], nil
+	}
+}
+
+func (self *SDisk) GetISnapshots() ([]cloudprovider.ICloudSnapshot, error) {
+	snapshots := make([]SSnapshot, 0)
+	for {
+		if parts, total, err := self.storage.zone.region.GetSnapshots("", self.DiskId, "", []string{}, 0, 20); err != nil {
+			log.Errorf("GetDisks fail %s", err)
+			return nil, err
+		} else {
+			snapshots = append(snapshots, parts...)
+			if len(snapshots) >= total {
+				break
+			}
+		}
+	}
+	isnapshots := make([]cloudprovider.ICloudSnapshot, len(snapshots))
+	for i := 0; i < len(snapshots); i++ {
+		snapshots[i].disk = self
+		isnapshots[i] = &snapshots[i]
+	}
+	return isnapshots, nil
+}
+
+func (self *SRegion) GetSnapshots(instanceId string, diskId string, snapshotName string, snapshotIds []string, offset int, limit int) ([]SSnapshot, int, error) {
+	if limit > 50 || limit <= 0 {
+		limit = 50
+	}
+	params := make(map[string]string)
+	params["RegionId"] = self.RegionId
+	params["PageSize"] = fmt.Sprintf("%d", limit)
+	params["PageNumber"] = fmt.Sprintf("%d", (offset/limit)+1)
+
+	if len(instanceId) > 0 {
+		params["InstanceId"] = instanceId
+	}
+	if len(diskId) > 0 {
+		params["diskId"] = diskId
+	}
+	if len(snapshotName) > 0 {
+		params["SnapshotName"] = snapshotName
+	}
+	if snapshotIds != nil && len(snapshotIds) > 0 {
+		params["SnapshotIds"] = jsonutils.Marshal(snapshotIds).String()
+	}
+
+	if body, err := self.ecsRequest("DescribeSnapshots", params); err != nil {
+		log.Errorf("GetSnapshots fail %s", err)
+		return nil, 0, err
+	} else {
+		snapshots := make([]SSnapshot, 0)
+		if err := body.Unmarshal(&snapshots, "Snapshots", "Snapshot"); err != nil {
+			log.Errorf("Unmarshal snapshot details fail %s", err)
+			return nil, 0, err
+		}
+		total, _ := body.Int("TotalCount")
+		return snapshots, int(total), nil
+	}
+
+}

pkg/util/aliyun/image.go

@@ -5,6 +5,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/aliyun/aliyun-oss-go-sdk/oss"
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
@@ -71,6 +72,10 @@ func (self *SImage) IsEmulated() bool {
 	return false
 }
 
+func (self *SImage) Delete() error {
+	return self.storageCache.region.DeleteImage(self.ImageId)
+}
+
 func (self *SImage) GetGlobalId() string {
 	return fmt.Sprintf("%s-%s")
 }
@@ -102,6 +107,32 @@ func (self *SImage) Refresh() error {
 	return jsonutils.Update(self, new)
 }
 
+type ImageExportTask struct {
+	ImageId  string
+	RegionId string
+	// RequestId string
+	TaskId string
+}
+
+func (self *SRegion) ExportImage(imageId string, bucket *oss.Bucket) (*ImageExportTask, error) {
+	params := make(map[string]string)
+	params["RegionId"] = self.RegionId
+	params["ImageId"] = imageId
+	params["OssBucket"] = bucket.BucketName
+	params["OssPrefix"] = fmt.Sprintf("%sexport", strings.Replace(imageId, "-", "", -1))
+
+	if body, err := self.ecsRequest("ExportImage", params); err != nil {
+		return nil, err
+	} else {
+		result := ImageExportTask{}
+		if err := body.Unmarshal(&result); err != nil {
+			log.Errorf("unmarshal result error %s", err)
+			return nil, err
+		}
+		return &result, nil
+	}
+}
+
 // {"ImageId":"m-j6c1qlpa7oebbg1n2k60","RegionId":"cn-hongkong","RequestId":"F8B2F6A1-F6AA-4C92-A54C-C4A309CF811F","TaskId":"t-j6c1qlpa7oebbg1rcl9t"}
 
 type ImageImportTask struct {

pkg/util/aliyun/shell/snapshot.go

@@ -0,0 +1,45 @@
+package shell
+
+import (
+	"yunion.io/x/onecloud/pkg/util/aliyun"
+	"yunion.io/x/onecloud/pkg/util/shellutils"
+)
+
+func init() {
+	type SnapshotListOptions struct {
+		DiskId      string   `help:"Disk ID"`
+		InstanceId  string   `help:"Instance ID"`
+		SnapshotIds []string `helo:"Snapshot ids"`
+		Name        string   `help:"Snapshot Name"`
+		Limit       int      `help:"page size"`
+		Offset      int      `help:"page offset"`
+	}
+	shellutils.R(&SnapshotListOptions{}, "snapshot-list", "List snapshot", func(cli *aliyun.SRegion, args *SnapshotListOptions) error {
+		if snapshots, total, err := cli.GetSnapshots(args.InstanceId, args.DiskId, args.Name, args.SnapshotIds, args.Offset, args.Limit); err != nil {
+			return err
+		} else {
+			printList(snapshots, total, args.Offset, args.Limit, []string{})
+			return nil
+		}
+	})
+
+	type SnapshotDeleteOptions struct {
+		ID string `help:"Snapshot ID"`
+	}
+
+	shellutils.R(&SnapshotDeleteOptions{}, "snapshot-delete", "Delete snapshot", func(cli *aliyun.SRegion, args *SnapshotDeleteOptions) error {
+		return cli.DeleteSnapshot(args.ID)
+	})
+
+	type SnapshotCreateOptions struct {
+		DiskId string `help:"Disk ID"`
+		Name   string `help:"Snapeshot Name"`
+		Desc   string `help:"Snapshot Desc"`
+	}
+
+	shellutils.R(&SnapshotCreateOptions{}, "snapshot-create", "Create snapshot", func(cli *aliyun.SRegion, args *SnapshotCreateOptions) error {
+		_, err := cli.CreateSnapshot(args.DiskId, args.Name, args.Desc)
+		return err
+	})
+
+}

pkg/util/aliyun/snapshot.go

@@ -0,0 +1,73 @@
+package aliyun
+
+import (
+	"fmt"
+
+	"yunion.io/x/jsonutils"
+)
+
+type SnapshotStatusType string
+
+const (
+	SnapshotStatusAccoplished SnapshotStatusType = "accomplished"
+	SnapshotStatusProgress    SnapshotStatusType = "progressing"
+)
+
+type SSnapshot struct {
+	disk           *SDisk
+	Progress       string
+	SnapshotId     string
+	SnapshotName   string
+	SourceDiskId   string
+	SourceDiskSize int32
+	SourceDiskType string
+	Status         SnapshotStatusType
+	Usage          string
+}
+
+func (self *SSnapshot) GetId() string {
+	return self.SnapshotId
+}
+
+func (self *SSnapshot) GetName() string {
+	return self.SnapshotName
+}
+
+func (self *SSnapshot) GetStatus() string {
+	return string(self.Status)
+}
+
+func (self *SSnapshot) Refresh() error {
+	if snapshot, err := self.disk.getSnapshot(self.SnapshotId); err != nil {
+		return err
+	} else if err := jsonutils.Update(self, snapshot); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (self *SSnapshot) GetGlobalId() string {
+	return fmt.Sprintf("%s", self.SnapshotId)
+}
+
+func (self *SSnapshot) IsEmulated() bool {
+	return false
+}
+
+func (self *SRegion) DeleteSnapshot(snapshotId string) error {
+	params := make(map[string]string)
+	params["SnapshotId"] = snapshotId
+	_, err := self.ecsRequest("DeleteSnapshot", params)
+	return err
+}
+
+func (self *SSnapshot) Delete() error {
+	if self.disk == nil {
+		return fmt.Errorf("not init disk for snapshot %s", self.SnapshotId)
+	}
+	return self.disk.storage.zone.region.DeleteSnapshot(self.SnapshotId)
+}
+
+func (self *SSnapshot) GetMetadata() *jsonutils.JSONDict {
+	return nil
+}

pkg/util/aliyun/storagecache.go

@@ -2,13 +2,17 @@ package aliyun
 
 import (
 	"fmt"
+	"os"
 	"strings"
 	"time"
 
+	"github.com/aliyun/aliyun-oss-go-sdk/oss"
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
+	compute "yunion.io/x/onecloud/pkg/compute/models"
 	"yunion.io/x/onecloud/pkg/compute/options"
+	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
@@ -106,7 +110,7 @@ func (self *SStoragecache) uploadImage(userCred mcclient.TokenCredential, imageI
 		log.Errorf("GetOssClient err %s", err)
 		return "", err
 	}
-	bucketName := strings.ToLower(fmt.Sprintf("imgcache-%s", self.region.GetId()))
+	bucketName := strings.ToLower(fmt.Sprintf("imgcache-%s-%s", self.region.GetId(), self.region.client.providerId))
 	exist, err := oss.IsBucketExist(bucketName)
 	if err != nil {
 		log.Errorf("IsBucketExist err %s", err)
@@ -173,3 +177,127 @@ func (self *SStoragecache) uploadImage(userCred mcclient.TokenCredential, imageI
 
 	return task.ImageId, nil
 }
+
+func (self *SStoragecache) CreateIImage(snapshoutId, imageName, imageDesc string) (cloudprovider.ICloudImage, error) {
+	if imageId, err := self.region.createIImage(snapshoutId, imageName, imageDesc); err != nil {
+		return nil, err
+	} else if image, err := self.region.GetImage(imageId); err != nil {
+		return nil, err
+	} else {
+		image.storageCache = self
+		iimage := make([]cloudprovider.ICloudImage, 1)
+		iimage[0] = image
+		if err := cloudprovider.WaitStatus(iimage[0], compute.IMAGE_STATUS_ACTIVE, 15*time.Second, 3600*time.Second); err != nil {
+			return nil, err
+		}
+		return iimage[0], nil
+	}
+}
+
+func (self *SRegion) CheckBucket(bucketName string) (*oss.Bucket, error) {
+	return self.checkBucket(bucketName)
+}
+
+func (self *SRegion) checkBucket(bucketName string) (*oss.Bucket, error) {
+	oss, err := self.GetOssClient()
+	if err != nil {
+		log.Errorf("GetOssClient err %s", err)
+		return nil, err
+	}
+	if exist, err := oss.IsBucketExist(bucketName); err != nil {
+		log.Errorf("IsBucketExist err %s", err)
+		return nil, err
+	} else if !exist {
+		log.Debugf("Bucket %s not exists, to create ...", bucketName)
+		if err := oss.CreateBucket(bucketName); err != nil {
+			log.Errorf("Create bucket error %s", err)
+			return nil, err
+		}
+	}
+	log.Debugf("Bucket %s exists", bucketName)
+	if bucket, err := oss.Bucket(bucketName); err != nil {
+		log.Errorf("Bucket error %s %s", bucketName, err)
+		return nil, err
+	} else {
+		return bucket, nil
+	}
+}
+
+func (self *SRegion) createIImage(snapshoutId, imageName, imageDesc string) (string, error) {
+	params := make(map[string]string)
+	params["RegionId"] = self.RegionId
+	params["OssBucket"] = strings.ToLower(fmt.Sprintf("imgcache-%s", self.GetId()))
+	params["SnapshotId"] = snapshoutId
+	params["ImageName"] = imageName
+	params["Description"] = imageDesc
+
+	if _, err := self.checkBucket(params["OssBucket"]); err != nil {
+		return "", err
+	}
+
+	if body, err := self.ecsRequest("CreateImage", params); err != nil {
+		log.Errorf("CreateImage fail %s", err)
+		return "", err
+	} else {
+		log.Infof("%s", body)
+		return body.GetString("ImageId")
+	}
+}
+
+func (self *SStoragecache) DownloadImage(userCred mcclient.TokenCredential, imageId string, extId string) (jsonutils.JSONObject, error) {
+	return self.downloadImage(userCred, imageId, extId)
+}
+
+// 定义进度条监听器。
+type OssProgressListener struct {
+}
+
+// 定义进度变更事件处理函数。
+func (listener *OssProgressListener) ProgressChanged(event *oss.ProgressEvent) {
+	switch event.EventType {
+	case oss.TransferStartedEvent:
+		log.Debugf("Transfer Started, ConsumedBytes: %d, TotalBytes %d.\n",
+			event.ConsumedBytes, event.TotalBytes)
+	case oss.TransferDataEvent:
+		log.Debugf("\rTransfer Data, ConsumedBytes: %d, TotalBytes %d, %d%%.",
+			event.ConsumedBytes, event.TotalBytes, event.ConsumedBytes*100/event.TotalBytes)
+	case oss.TransferCompletedEvent:
+		log.Debugf("\nTransfer Completed, ConsumedBytes: %d, TotalBytes %d.\n",
+			event.ConsumedBytes, event.TotalBytes)
+	case oss.TransferFailedEvent:
+		log.Debugf("\nTransfer Failed, ConsumedBytes: %d, TotalBytes %d.\n",
+			event.ConsumedBytes, event.TotalBytes)
+	default:
+	}
+}
+
+func (self *SStoragecache) downloadImage(userCred mcclient.TokenCredential, imageId string, extId string) (jsonutils.JSONObject, error) {
+	tmpImageFile := fmt.Sprintf("/tmp/%s", extId)
+	bucketName := strings.ToLower(fmt.Sprintf("imgcache-%s", self.region.GetId()))
+	if bucket, err := self.region.checkBucket(bucketName); err != nil {
+		return nil, err
+	} else if _, err := self.region.GetImage(extId); err != nil {
+		return nil, err
+	} else if task, err := self.region.ExportImage(extId, bucket); err != nil {
+		return nil, err
+	} else if err := self.region.waitTaskStatus(ExportImageTask, task.TaskId, "Finished", 15*time.Second, 3600*time.Second); err != nil {
+		return nil, err
+	} else if imageList, err := bucket.ListObjects(oss.Prefix(fmt.Sprintf("%sexport", strings.Replace(extId, "-", "", -1)))); err != nil {
+		return nil, err
+	} else if len(imageList.Objects) != 1 {
+		return nil, httperrors.NewResourceNotFoundError("exported image not find")
+	} else if err := bucket.DownloadFile(imageList.Objects[0].Key, tmpImageFile, 12*1024*1024, oss.Routines(3), oss.Progress(&OssProgressListener{})); err != nil {
+		return nil, err
+	} else {
+		s := auth.GetAdminSession(options.Options.Region, "")
+		params := jsonutils.Marshal(map[string]string{"image_id": imageId, "disk-format": "raw"})
+		if file, err := os.Open(tmpImageFile); err != nil {
+			return nil, err
+		} else if result, err := modules.Images.Upload(s, params, file, imageList.Objects[0].Size); err != nil {
+			return nil, err
+		} else {
+			os.Remove(tmpImageFile)
+			return result, nil
+		}
+	}
+}

pkg/util/azure/azure.go

@@ -2,13 +2,13 @@ package azure
 
 import (
 	"context"
-	"reflect"
 	"strings"
 
 	"github.com/Azure/azure-sdk-for-go/services/preview/subscription/mgmt/2018-03-01-preview/subscription"
 	"github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2018-05-01/resources"
 
 	"github.com/Azure/go-autorest/autorest"
+	azureenv "github.com/Azure/go-autorest/autorest/azure"
 	"github.com/Azure/go-autorest/autorest/azure/auth"
 
 	"yunion.io/x/jsonutils"
@@ -25,13 +25,6 @@ const (
 	AZURE_API_VERSION = "2018-04-01"
 )
 
-var authAddr = map[string]string{
-	"https://management.azure.com":         "https://login.microsoftonline.com",
-	"https://management.usgovcloudapi.net": "https://login.microsoftonline.us",
-	"https://management.chinacloudapi.cn":  "https://login.chinacloudapi.cn",
-	"https://management.microsoftazure.de": "https://login.microsoftonline.de",
-}
-
 var DefaultResourceGroup = map[string]string{
 	"disk":     "YunionDiskResource",
 	"instance": "YunionInstanceResource",
@@ -50,18 +43,23 @@ type SAzureClient struct {
 	clientScret    string
 	baseUrl        string
 	secret         string
+	envName        string
+	env            azureenv.Environment
 	authorizer     autorest.Authorizer
 	iregions       []cloudprovider.ICloudRegion
 }
 
-func NewAzureClient(providerId string, providerName string, accessKey string, secret string, url string) (*SAzureClient, error) {
-	if _, ok := authAddr[url]; !ok {
-		return nil, httperrors.NewUnauthorizedError("Access url choices: %v", reflect.ValueOf(authAddr).MapKeys())
-	}
+func NewAzureClient(providerId string, providerName string, accessKey string, secret string, envName string) (*SAzureClient, error) {
 	if clientInfo, accountInfo := strings.Split(secret, "/"), strings.Split(accessKey, "/"); len(clientInfo) == 2 && len(accountInfo) == 2 {
-		client := SAzureClient{providerId: providerId, providerName: providerName, secret: secret, baseUrl: url}
+		client := SAzureClient{providerId: providerId, providerName: providerName, secret: secret, envName: envName}
 		client.clientId, client.clientScret = clientInfo[0], clientInfo[1]
 		client.tenantId, client.subscriptionId = accountInfo[0], accountInfo[1]
+		if env, err := azureenv.EnvironmentFromName(envName); err != nil {
+			return nil, err
+		} else {
+			client.env = env
+			client.baseUrl = env.ResourceManagerEndpoint
+		}
 		if err := client.fetchAzureInof(); err != nil {
 			return nil, err
 		} else if err := client.fetchRegions(); err != nil {
@@ -114,8 +112,8 @@ func (self *SAzureClient) fetchAzueResourceGroup() error {
 
 func (self *SAzureClient) fetchAzureInof() error {
 	conf := auth.NewClientCredentialsConfig(self.clientId, self.clientScret, self.tenantId)
-	conf.Resource = self.baseUrl
-	conf.AADEndpoint = authAddr[self.baseUrl]
+	conf.Resource = self.env.ResourceManagerEndpoint
+	conf.AADEndpoint = self.env.ActiveDirectoryEndpoint
 	if authorizer, err := conf.Authorizer(); err != nil {
 		return err
 	} else {
@@ -124,14 +122,20 @@ func (self *SAzureClient) fetchAzureInof() error {
 	return nil
 }
 
-func (self *SAzureClient) UpdateAccount(tenantId, secret string) error {
-	if self.tenantId != tenantId || self.secret != secret {
+func (self *SAzureClient) UpdateAccount(tenantId, secret, envName string) error {
+	if self.tenantId != tenantId || self.secret != secret || self.envName != envName {
+		if env, err := azureenv.EnvironmentFromName(envName); err != nil {
+			return err
+		} else {
+			self.env = env
+			self.baseUrl = env.ResourceManagerEndpoint
+		}
 		if clientInfo, accountInfo := strings.Split(secret, "/"), strings.Split(tenantId, "/"); len(clientInfo) == 2 && len(accountInfo) == 2 {
 			self.clientId, self.clientScret = clientInfo[0], clientInfo[1]
 			self.tenantId, self.subscriptionId = accountInfo[0], accountInfo[1]
 			conf := auth.NewClientCredentialsConfig(self.clientId, self.clientScret, self.tenantId)
-			conf.Resource = self.baseUrl
-			conf.AADEndpoint = strings.Replace(self.baseUrl, "management", "login", -1)
+			conf.Resource = self.env.ResourceManagerEndpoint
+			conf.AADEndpoint = self.env.ActiveDirectoryEndpoint
 			if authorizer, err := conf.Authorizer(); err != nil {
 				return err
 			} else {

pkg/util/azure/disk.go

@@ -239,3 +239,15 @@ func (self *SDisk) GetDiskType() string {
 	}
 	return models.DISK_TYPE_DATA
 }
+
+func (self *SDisk) CreateISnapshot(name, desc string) (cloudprovider.ICloudSnapshot, error) {
+	return nil, cloudprovider.ErrNotImplemented
+}
+
+func (self *SDisk) GetISnapshot(snapshotId string) (cloudprovider.ICloudSnapshot, error) {
+	return nil, cloudprovider.ErrNotImplemented
+}
+
+func (self *SDisk) GetISnapshots() ([]cloudprovider.ICloudSnapshot, error) {
+	return nil, cloudprovider.ErrNotImplemented
+}

pkg/util/azure/image.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-06-01/compute"
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/compute/models"
 )
@@ -130,6 +131,27 @@ func (self *SRegion) GetImage(imageId string) (*SImage, error) {
 	return &image, nil
 }
 
+func (self *SRegion) GetImageByName(imageId string) (*SImage, error) {
+	return self.GetImage(imageId)
+}
+
+func (self *SRegion) CreateImageByBlob(imageName, osType, blobURI string) (*SImage, error) {
+	imageClient := compute.NewImagesClientWithBaseURI(self.client.baseUrl, self.SubscriptionID)
+	imageClient.Authorizer = self.client.authorizer
+	storageProfile := compute.ImageStorageProfile{OsDisk: &compute.ImageOSDisk{OsType: compute.OperatingSystemTypes(osType), OsState: compute.OperatingSystemStateTypes("Generalized"), BlobURI: &blobURI}}
+	params := compute.Image{Name: &imageName, Location: &self.Name, ImageProperties: &compute.ImageProperties{StorageProfile: &storageProfile}}
+	if result, err := imageClient.CreateOrUpdate(context.Background(), DefaultResourceGroup["image"], imageName, params); err != nil {
+		log.Errorf("Create image from blob error: %v", err)
+		return nil, err
+	} else if err := result.WaitForCompletion(context.Background(), imageClient.Client); err != nil {
+		return nil, err
+	} else if image, err := self.GetImageByName(imageName); err != nil {
+		return nil, err
+	} else {
+		return image, nil
+	}
+}
+
 func (self *SRegion) GetImages() ([]SImage, error) {
 	images := make([]SImage, 0)
 	imageClient := compute.NewImagesClientWithBaseURI(self.client.baseUrl, self.SubscriptionID)
@@ -141,3 +163,7 @@ func (self *SRegion) GetImages() ([]SImage, error) {
 	}
 	return images, nil
 }
+
+func (self *SImage) Delete() error {
+	return cloudprovider.ErrNotImplemented
+}

pkg/util/azure/provider/provider.go

@@ -18,7 +18,7 @@ func (self *SAzureProviderFactory) GetId() string {
 func (self *SAzureProviderFactory) GetProvider(providerId, providerName, url, account, secret string) (cloudprovider.ICloudProvider, error) {
 	provider, ok := self.providerTable[providerId]
 	if ok {
-		err := provider.client.UpdateAccount(account, secret)
+		err := provider.client.UpdateAccount(account, secret, url)
 		if err != nil {
 			return nil, err
 		} else {

pkg/util/azure/shell/blob.go

@@ -14,15 +14,33 @@ func init() {
 		Offset         int    `help:"page offset"`
 	}
 	shellutils.R(&BlobListOptions{}, "blob-file-list", "List intances", func(cli *azure.SRegion, args *BlobListOptions) error {
-		if err := cli.CheckBlob(args.ResourceGroup, args.StorageAccount, args.BlobName); err != nil {
+		if err := cli.CheckBlobContainer(args.ResourceGroup, args.StorageAccount, args.BlobName); err != nil {
 			return err
+		} else if result, err := cli.ListContainerFiles(args.ResourceGroup, args.StorageAccount, args.BlobName); err != nil {
+			return err
+		} else {
+			printList(result, len(result), args.Offset, args.Limit, []string{})
+			return nil
+		}
+	})
+
+	type BlobUploadOptions struct {
+		ResourceGroup  string `help:"Resource Group Name`
+		StorageAccount string `helo:"Storage Account"`
+		BlobName       string `helo:"Blob name`
+		FilePath       string `helo:"Filet path to upload`
+		Limit          int    `help:"page size"`
+		Offset         int    `help:"page offset"`
+	}
+
+	shellutils.R(&BlobUploadOptions{}, "blob-file-upload", "Upload file to blob", func(cli *azure.SRegion, args *BlobUploadOptions) error {
+		if err := cli.CheckBlobContainer(args.ResourceGroup, args.StorageAccount, args.BlobName); err != nil {
+			return err
+		} else if url, err := cli.UploadContainerFiles(args.ResourceGroup, args.StorageAccount, args.BlobName, args.FilePath); err != nil {
+			return err
+		} else {
+			printObject(url)
+			return nil
 		}
-		return nil
-		// if images, err := cli.GetImages(); err != nil {
-		// 	return err
-		// } else {
-		// 	printList(images, len(images), args.Offset, args.Limit, []string{})
-		// 	return nil
-		// }
 	})
 }

pkg/util/azure/shell/image.go

@@ -10,7 +10,7 @@ func init() {
 		Limit  int `help:"page size"`
 		Offset int `help:"page offset"`
 	}
-	shellutils.R(&ImageListOptions{}, "image-list", "List intances", func(cli *azure.SRegion, args *ImageListOptions) error {
+	shellutils.R(&ImageListOptions{}, "image-list", "List images", func(cli *azure.SRegion, args *ImageListOptions) error {
 		if images, err := cli.GetImages(); err != nil {
 			return err
 		} else {
@@ -18,4 +18,19 @@ func init() {
 			return nil
 		}
 	})
+
+	type ImageCreateOptions struct {
+		Name    string `helo:"Image name"`
+		OsType  string `helo:"Operation system" choices:"Linux|Windows"`
+		BlobURI string `helo:"page blob uri"`
+	}
+
+	shellutils.R(&ImageCreateOptions{}, "image-create", "Create image", func(cli *azure.SRegion, args *ImageCreateOptions) error {
+		if image, err := cli.CreateImageByBlob(args.Name, args.OsType, args.BlobURI); err != nil {
+			return err
+		} else {
+			printObject(image)
+			return nil
+		}
+	})
 }

pkg/util/azure/storagecache.go

@@ -3,17 +3,28 @@ package azure
 import (
 	"context"
 	"fmt"
+	"io"
+	"os"
+	"strings"
+	"time"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/compute/options"
 	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
 
-	"github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2018-02-01/storage"
+	storageaccount "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2018-02-01/storage"
+	"github.com/Azure/azure-sdk-for-go/storage"
 )
 
 const (
-	DefaultStorageAccount string = "storageimagecache"
+	DefaultStorageAccount string = "storage"
 	DefaultBlobContainer  string = "image-cache"
+
+	DefaultReadBlockSize uint64 = 4 * 1024 * 1024
 )
 
 type SStoragecache struct {
@@ -87,77 +98,72 @@ func (self *SStoragecache) UploadImage(userCred mcclient.TokenCredential, imageI
 }
 
 func (self *SRegion) CreateStorageAccount(resourceGroup, storageAccount string) error {
-	storageClinet := storage.NewAccountsClientWithBaseURI(self.client.baseUrl, self.SubscriptionID)
+	storageClinet := storageaccount.NewAccountsClientWithBaseURI(self.client.baseUrl, self.SubscriptionID)
 	storageClinet.Authorizer = self.client.authorizer
-	sku := storage.Sku{Name: storage.SkuName("Standard_GRS")}
-	params := storage.AccountCreateParameters{Sku: &sku, Location: &self.Name, Kind: storage.Kind("Storage")}
+	sku := storageaccount.Sku{Name: storageaccount.SkuName("Standard_GRS")}
+	params := storageaccount.AccountCreateParameters{Sku: &sku, Location: &self.Name, Kind: storageaccount.Kind("Storage")}
 	if len(resourceGroup) == 0 {
 		resourceGroup = DefaultResourceGroup["storage"]
 	}
 	if len(storageAccount) == 0 {
-		storageAccount = DefaultStorageAccount
+		storageAccount = fmt.Sprintf("%s%s", self.Name, DefaultStorageAccount)
 	}
-	if _, err := storageClinet.Create(context.Background(), resourceGroup, storageAccount, params); err != nil {
+	if result, err := storageClinet.Create(context.Background(), resourceGroup, storageAccount, params); err != nil {
+		return err
+	} else if err := result.WaitForCompletion(context.Background(), storageClinet.Client); err != nil {
 		return err
 	}
 	return nil
 }
 
-func (self *SRegion) CreateStorageBlob(resourceGroup, storageAccount, containerName string) error {
-	blobClient := storage.NewBlobContainersClientWithBaseURI(self.client.baseUrl, self.SubscriptionID)
-	blobClient.Authorizer = self.client.authorizer
-	if len(resourceGroup) == 0 {
-		resourceGroup = DefaultResourceGroup["storage"]
-	}
-	if len(storageAccount) == 0 {
-		storageAccount = DefaultStorageAccount
-	}
-	if len(containerName) == 0 {
-		containerName = DefaultBlobContainer
-	}
-	properties := storage.ContainerProperties{PublicAccess: storage.PublicAccess("PublicAccessNone"), LeaseDuration: storage.LeaseDuration("Infinite")}
-	_type := "Microsoft.Storage/storageAccounts"
-	params := storage.BlobContainer{Name: &containerName, ContainerProperties: &properties, Type: &_type}
-	if _, err := blobClient.Create(context.Background(), resourceGroup, storageAccount, containerName, params); err != nil {
+func (self *SRegion) checkStorageContainer(storageAccount, accessKey, containerName string) error {
+	if client, err := storage.NewBasicClientOnSovereignCloud(storageAccount, accessKey, self.client.env); err != nil {
+		return err
+	} else {
+		blob := client.GetBlobService()
+		container := blob.GetContainerReference(containerName)
+		option := storage.CreateContainerOptions{Timeout: 10, Access: storage.ContainerAccessType("")}
+		_, err := container.CreateIfNotExists(&option)
 		return err
 	}
 	return nil
 }
 
-func (self *SRegion) isStorageBlobExist(resourceGroup, storageAccount, containerName string) bool {
-	blobClient := storage.NewBlobContainersClientWithBaseURI(self.client.baseUrl, self.SubscriptionID)
-	blobClient.Authorizer = self.client.authorizer
-	if len(resourceGroup) == 0 {
-		resourceGroup = DefaultResourceGroup["storage"]
-	}
-	if len(storageAccount) == 0 {
-		storageAccount = DefaultStorageAccount
-	}
-	if len(containerName) == 0 {
-		containerName = DefaultBlobContainer
-	}
-	if _, err := blobClient.Get(context.Background(), resourceGroup, storageAccount, containerName); err != nil {
-		return false
-	}
-	return true
-}
-
 func (self *SRegion) isStorageAccountExist(resourceGroup, storageAccount string) bool {
-	storageClinet := storage.NewAccountsClientWithBaseURI(self.client.baseUrl, self.SubscriptionID)
+	storageClinet := storageaccount.NewAccountsClientWithBaseURI(self.client.baseUrl, self.SubscriptionID)
 	storageClinet.Authorizer = self.client.authorizer
-	if len(resourceGroup) == 0 {
-		resourceGroup = DefaultResourceGroup["storage"]
-	}
-	if len(storageAccount) == 0 {
-		storageAccount = DefaultStorageAccount
-	}
 	if _, err := storageClinet.GetProperties(context.Background(), resourceGroup, storageAccount); err != nil {
 		return false
 	}
 	return true
 }
 
-func (self *SRegion) CheckBlob(resourceGroup, storageAccount, blobName string) error {
+func (self *SRegion) getStorageAccountKey(resourceGroup, storageAccount string) (string, error) {
+	storageClinet := storageaccount.NewAccountsClientWithBaseURI(self.client.baseUrl, self.SubscriptionID)
+	storageClinet.Authorizer = self.client.authorizer
+	if result, err := storageClinet.ListKeys(context.Background(), resourceGroup, storageAccount); err != nil {
+		return "", err
+	} else {
+		for _, key := range *result.Keys {
+			permission := strings.ToLower(string(key.Permissions))
+			if permission == "full" {
+				return *key.Value, nil
+			}
+		}
+	}
+	return "", fmt.Errorf("not find storage account accessKey")
+}
+
+func (self *SRegion) CheckBlobContainer(resourceGroup, storageAccount, blobName string) error {
+	if len(resourceGroup) == 0 {
+		resourceGroup = DefaultResourceGroup["storage"]
+	}
+	if len(storageAccount) == 0 {
+		storageAccount = fmt.Sprintf("%s%s", self.Name, DefaultStorageAccount)
+	}
+	if len(blobName) == 0 {
+		blobName = DefaultBlobContainer
+	}
 	if err := self.client.fetchAzueResourceGroup(); err != nil {
 		return err
 	} else if !self.isStorageAccountExist(resourceGroup, storageAccount) {
@@ -165,14 +171,225 @@ func (self *SRegion) CheckBlob(resourceGroup, storageAccount, blobName string) e
 			return err
 		}
 	}
-	if !self.isStorageBlobExist(resourceGroup, storageAccount, blobName) {
-		if err := self.CreateStorageBlob(resourceGroup, storageAccount, blobName); err != nil {
-			return err
+	if accessKey, err := self.getStorageAccountKey(resourceGroup, storageAccount); err != nil {
+		return err
+	} else {
+		return self.checkStorageContainer(storageAccount, accessKey, blobName)
+	}
+}
+
+type BlobProperties struct {
+	LastModified          time.Time
+	ContentMD5            string
+	ContentLength         int64
+	ContentType           string
+	ContentEncoding       string
+	CacheControl          string
+	ContentLanguage       string
+	ContentDisposition    string
+	BlobType              string
+	SequenceNumber        int64
+	CopyID                string
+	CopyStatus            string
+	CopySource            string
+	CopyProgress          string
+	CopyCompletionTime    time.Time
+	CopyStatusDescription string
+	LeaseStatus           string
+	LeaseState            string
+	LeaseDuration         string
+	ServerEncrypted       bool
+	IncrementalCopy       bool
+}
+
+type BlobMetadata map[string]string
+
+type ContainerProperties struct {
+	LastModified  string
+	Etag          string
+	LeaseStatus   string
+	LeaseState    string
+	LeaseDuration string
+	PublicAccess  string
+}
+
+type Container struct {
+	Name       string
+	Metadata   map[string]string
+	Properties ContainerProperties
+}
+
+type Blob struct {
+	Container  Container
+	Name       string
+	Snapshot   time.Time
+	Properties BlobProperties
+	Metadata   BlobMetadata
+}
+
+func (self *SRegion) getContainerFiles(storageAccount, accessKey, containerName string) ([]Blob, error) {
+	if client, err := storage.NewBasicClientOnSovereignCloud(storageAccount, accessKey, self.client.env); err != nil {
+		return nil, err
+	} else {
+		blob := client.GetBlobService()
+		container := blob.GetContainerReference(containerName)
+		params := storage.ListBlobsParameters{}
+		blobs := make([]Blob, 0)
+		if result, err := container.ListBlobs(params); err != nil {
+			return nil, err
+		} else if err := jsonutils.Update(&blobs, result.Blobs); err != nil {
+			return nil, err
+		} else {
+			return blobs, nil
 		}
 	}
-	return nil
+}
+
+func (self *SRegion) ListContainerFiles(resourceGroup, storageAccount, blobName string) ([]Blob, error) {
+	if len(resourceGroup) == 0 {
+		resourceGroup = DefaultResourceGroup["storage"]
+	}
+	if len(storageAccount) == 0 {
+		storageAccount = fmt.Sprintf("%s%s", self.Name, DefaultStorageAccount)
+	}
+	if len(blobName) == 0 {
+		blobName = DefaultBlobContainer
+	}
+	if accessKey, err := self.getStorageAccountKey(resourceGroup, storageAccount); err != nil {
+		return nil, err
+	} else {
+		return self.getContainerFiles(storageAccount, accessKey, blobName)
+	}
+}
+
+func (self *SRegion) uploadContainerFileByReader(storageAccount, accessKey, containerName, fileName string, content io.Reader, size int64) (string, error) {
+	if client, err := storage.NewBasicClientOnSovereignCloud(storageAccount, accessKey, self.client.env); err != nil {
+		return "", err
+	} else {
+		blob := client.GetBlobService()
+		container := blob.GetContainerReference(containerName)
+		blobClient := container.GetBlobReference(fileName)
+		if _, err := blobClient.DeleteIfExists(&storage.DeleteBlobOptions{}); err != nil {
+			return "", err
+		}
+		blobClient.Properties.ContentLength = size
+		if err := blobClient.PutPageBlob(&storage.PutBlobOptions{}); err != nil {
+			return "", err
+		}
+		var readed uint64 = 0
+		for i := 0; i < int(uint64(size)/DefaultReadBlockSize); i++ {
+			if err := blobClient.WriteRange(storage.BlobRange{Start: readed, End: readed + DefaultReadBlockSize - 1}, content, &storage.PutPageOptions{}); err != nil {
+				return "", err
+			}
+			readed += DefaultReadBlockSize
+			log.Debugf("Upload %s %f%% to %s", fileName, float64(readed)/float64(size)*100, storageAccount)
+		}
+		if extraSize := uint64(size) % DefaultReadBlockSize; extraSize > 0 {
+			log.Debugf("Upload %s extra size: %d to %s", fileName, extraSize, storageAccount)
+			if err := blobClient.WriteRange(storage.BlobRange{Start: readed, End: readed + extraSize - 1}, content, &storage.PutPageOptions{}); err != nil {
+				return "", err
+			}
+		}
+		log.Debugf("Upload %s complate", fileName)
+		return blobClient.GetURL(), nil
+	}
+}
+
+func (self *SRegion) uploadContainerFileByPath(storageAccount, accessKey, containerName, filePath string) (string, error) {
+	if f, err := os.Open(filePath); err != nil {
+		return "", err
+	} else {
+		defer f.Close()
+		if finfo, err := f.Stat(); err != nil {
+			return "", err
+		} else {
+			return self.uploadContainerFileByReader(storageAccount, accessKey, containerName, finfo.Name(), f, finfo.Size())
+		}
+	}
+}
+
+func (self *SRegion) UploadContainerFiles(resourceGroup, storageAccount, containerName, filePath string) (string, error) {
+	if len(resourceGroup) == 0 {
+		resourceGroup = DefaultResourceGroup["storage"]
+	}
+	if len(storageAccount) == 0 {
+		storageAccount = fmt.Sprintf("%s%s", self.Name, DefaultStorageAccount)
+	}
+	if len(containerName) == 0 {
+		containerName = DefaultBlobContainer
+	}
+	if accessKey, err := self.getStorageAccountKey(resourceGroup, storageAccount); err != nil {
+		return "", err
+	} else {
+		return self.uploadContainerFileByPath(storageAccount, accessKey, containerName, filePath)
+	}
 }
 
 func (self *SStoragecache) uploadImage(userCred mcclient.TokenCredential, imageId string, isForce bool) (string, error) {
-	return "", nil
+	s := auth.GetAdminSession(options.Options.Region, "")
+
+	if meta, reader, err := modules.Images.Download(s, imageId); err != nil {
+		return "", err
+	} else {
+		// {"checksum":"d0ab0450979977c6ada8d85066a6e484","container_format":"bare","created_at":"2018-08-10T04:18:07","deleted":"False","disk_format":"vhd","id":"64189033-3ad4-413c-b074-6bf0b6be8508","is_public":"False","min_disk":"0","min_ram":"0","name":"centos-7.3.1611-20180104.vhd","owner":"5124d80475434da8b41fee48d5be94df","properties":{"os_arch":"x86_64","os_distribution":"CentOS","os_type":"Linux","os_version":"7.3.1611-VHD"},"protected":"False","size":"2028505088","status":"active","updated_at":"2018-08-10T04:20:59"}
+		log.Infof("meta data %s", meta)
+
+		osType, _ := meta.GetString("properties", "os_type")
+		imageNameOnBlob, _ := meta.GetString("name")
+		if !strings.HasSuffix(imageNameOnBlob, ".vhd") {
+			imageNameOnBlob = fmt.Sprintf("%s.vhd", imageNameOnBlob)
+		}
+
+		storageAccount := fmt.Sprintf("%s%s", self.region.Name, DefaultStorageAccount)
+
+		if err := self.region.CheckBlobContainer(DefaultResourceGroup["storage"], storageAccount, DefaultBlobContainer); err != nil {
+			return "", err
+		}
+
+		size, _ := meta.Int("size")
+		accessKey, err := self.region.getStorageAccountKey(DefaultResourceGroup["storage"], storageAccount)
+		if err != nil {
+			return "", err
+		}
+
+		blobURI, err := self.region.uploadContainerFileByReader(storageAccount, accessKey, DefaultBlobContainer, imageNameOnBlob, reader, size)
+		if err != nil {
+			log.Errorf("uploadContainerFileByReader error: %v", err)
+			return "", err
+		}
+
+		imageBaseName := imageId
+		if imageBaseName[0] >= '0' && imageBaseName[0] <= '9' {
+			imageBaseName = fmt.Sprintf("img%s", imageId)
+		}
+		imageName := imageBaseName
+		nameIdx := 1
+
+		// check image name, avoid name conflict
+		for {
+			if _, err = self.region.GetImageByName(imageName); err != nil {
+				if err == cloudprovider.ErrNotFound {
+					break
+				} else {
+					return "", err
+				}
+			}
+			imageName = fmt.Sprintf("%s-%d", imageBaseName, nameIdx)
+			nameIdx += 1
+		}
+
+		if image, err := self.region.CreateImageByBlob(imageName, osType, blobURI); err != nil {
+			return "", err
+		} else {
+			return image.GetId(), nil
+		}
+	}
+}
+
+func (self *SStoragecache) CreateIImage(snapshoutId, imageName, imageDesc string) (cloudprovider.ICloudImage, error) {
+	return nil, cloudprovider.ErrNotImplemented
+}
+
+func (self *SStoragecache) DownloadImage(userCred mcclient.TokenCredential, imageId string, extId string) (jsonutils.JSONObject, error) {
+	return nil, cloudprovider.ErrNotImplemented
 }

pkg/util/logclient/doc.go

@@ -0,0 +1 @@
+package logclient // import "yunion.io/x/onecloud/pkg/util/logclient"

pkg/util/logclient/logclient.go

@@ -0,0 +1,97 @@
+package logclient
+
+import (
+	"fmt"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/pkg/util/stringutils"
+)
+
+const (
+	ACT_ADDTAG                       = "添加标签"
+	ACT_ALLOCATE                     = "分配"
+	ACT_BM_CONVERT_HYPER             = "转换为宿主机"
+	ACT_BM_MAINTENANCE               = "进入离线状态"
+	ACT_BM_UNCONVERT_HYPER           = "转换为受管物理机"
+	ACT_BM_UNMAINTENANCE             = "退出离线状态"
+	ACT_CANCEL_DELETE                = "恢复"
+	ACT_CHANGE_OWNER                 = "更改项目"
+	ACT_CLOUD_FULLSYNC               = "全量同步"
+	ACT_CLOUD_SYNC                   = "同步"
+	ACT_CREATE                       = "创建"
+	ACT_DELETE                       = "删除"
+	ACT_DISABLE                      = "禁用"
+	ACT_ENABLE                       = "启用"
+	ACT_GUEST_ATTACH_ISOLATED_DEVICE = "挂载透传设备"
+	ACT_GUEST_DETACH_ISOLATED_DEVICE = "卸载透传设备"
+	ACT_MERGE                        = "合并"
+	ACT_OFFLINE                      = "下线"
+	ACT_ONLINE                       = "上线"
+	ACT_PRIVATE                      = "设为私有"
+	ACT_PUBLIC                       = "设为共享"
+	ACT_RELEASE_IP                   = "释放IP"
+	ACT_RESERVE_IP                   = "预留IP"
+	ACT_RESIZE                       = "扩容"
+	ACT_RMTAG                        = "删除标签"
+	ACT_SPLIT                        = "分割"
+	ACT_UNCACHED_IMAGE               = "清除缓存"
+	ACT_UPDATE                       = "更新"
+	ACT_VM_ATTACH_DISK               = "挂载磁盘"
+	ACT_VM_BIND_KEYPAIR              = "绑定密钥"
+	ACT_VM_CHANGE_FLAVOR             = "调整配置"
+	ACT_VM_DEPLOY                    = "部署"
+	ACT_VM_DETACH_DISK               = "卸载磁盘"
+	ACT_VM_PURGE                     = "清除"
+	ACT_VM_REBUILD                   = "重装系统"
+	ACT_VM_RESET_PSWD                = "重置密码"
+	ACT_VM_START                     = "开机"
+	ACT_VM_STOP                      = "关机"
+	ACT_VM_SYNC_CONF                 = "同步配置"
+	ACT_VM_SYNC_STATUS               = "同步状态"
+	ACT_VM_UNBIND_KEYPAIR            = "解绑密钥"
+)
+
+type IObject interface {
+	GetId() string
+	GetName() string
+	Keyword() string
+}
+
+func AddActionLog(model IObject, action string, iNotes interface{}, userCred mcclient.TokenCredential, success bool) {
+
+	token := userCred
+	notes := stringutils.Interface2String(iNotes)
+
+	s := auth.GetSession(userCred, "", "")
+
+	logentry := jsonutils.NewDict()
+	logentry.Add(jsonutils.NewString(model.GetName()), "obj_name")
+	logentry.Add(jsonutils.NewString(model.Keyword()), "obj_type")
+	logentry.Add(jsonutils.NewString(model.GetId()), "obj_id")
+	logentry.Add(jsonutils.NewString(action), "action")
+	logentry.Add(jsonutils.NewString(token.GetUserId()), "user_id")
+	logentry.Add(jsonutils.NewString(token.GetUserName()), "user")
+	logentry.Add(jsonutils.NewString(token.GetTenantId()), "tenant_id")
+	logentry.Add(jsonutils.NewString(token.GetTenantName()), "tenant")
+
+	if !success {
+		// 失败日志
+		logentry.Add(jsonutils.JSONFalse, "success")
+	} else {
+		// 成功日志
+		logentry.Add(jsonutils.JSONTrue, "success")
+	}
+	// TODO delete tag when done.
+	notes = fmt.Sprintf("[a2]%s", notes)
+	logentry.Add(jsonutils.NewString(notes), "notes")
+
+	_, err := modules.Actions.Create(s, logentry)
+	if err != nil {
+		fmt.Printf("create action log failed %s", err)
+	} else {
+		fmt.Println("create action log success")
+	}
+}

vendor/github.com/Azure/azure-sdk-for-go/storage/README.md

@@ -0,0 +1,22 @@
+# Azure Storage SDK for Go (Preview)
+
+:exclamation: IMPORTANT: This package is in maintenance only and will be deprecated in the
+future. Please use one of the following packages instead.
+
+| Service | Import Path/Repo |
+|---------|------------------|
+| Storage - Blobs | [github.com/Azure/azure-storage-blob-go](https://github.com/Azure/azure-storage-blob-go) |
+| Storage - Files | [github.com/Azure/azure-storage-file-go](https://github.com/Azure/azure-storage-file-go) |
+| Storage - Queues | [github.com/Azure/azure-storage-queue-go](https://github.com/Azure/azure-storage-queue-go) |
+
+The `github.com/Azure/azure-sdk-for-go/storage` package is used to manage
+[Azure Storage](https://docs.microsoft.com/en-us/azure/storage/) data plane
+resources: containers, blobs, tables, and queues.
+
+To manage storage *accounts* use Azure Resource Manager (ARM) via the packages
+at [github.com/Azure/azure-sdk-for-go/services/storage](https://github.com/Azure/azure-sdk-for-go/tree/master/services/storage).
+
+This package also supports the [Azure Storage
+Emulator](https://azure.microsoft.com/documentation/articles/storage-use-emulator/)
+(Windows only).
+

vendor/github.com/Azure/azure-sdk-for-go/storage/appendblob.go

@@ -0,0 +1,91 @@
+package storage
+
+// Copyright 2017 Microsoft Corporation
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+import (
+	"bytes"
+	"crypto/md5"
+	"encoding/base64"
+	"fmt"
+	"net/http"
+	"net/url"
+	"time"
+)
+
+// PutAppendBlob initializes an empty append blob with specified name. An
+// append blob must be created using this method before appending blocks.
+//
+// See CreateBlockBlobFromReader for more info on creating blobs.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Put-Blob
+func (b *Blob) PutAppendBlob(options *PutBlobOptions) error {
+	params := url.Values{}
+	headers := b.Container.bsc.client.getStandardHeaders()
+	headers["x-ms-blob-type"] = string(BlobTypeAppend)
+	headers = mergeHeaders(headers, headersFromStruct(b.Properties))
+	headers = b.Container.bsc.client.addMetadataToHeaders(headers, b.Metadata)
+
+	if options != nil {
+		params = addTimeout(params, options.Timeout)
+		headers = mergeHeaders(headers, headersFromStruct(*options))
+	}
+	uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), params)
+
+	resp, err := b.Container.bsc.client.exec(http.MethodPut, uri, headers, nil, b.Container.bsc.auth)
+	if err != nil {
+		return err
+	}
+	return b.respondCreation(resp, BlobTypeAppend)
+}
+
+// AppendBlockOptions includes the options for an append block operation
+type AppendBlockOptions struct {
+	Timeout           uint
+	LeaseID           string     `header:"x-ms-lease-id"`
+	MaxSize           *uint      `header:"x-ms-blob-condition-maxsize"`
+	AppendPosition    *uint      `header:"x-ms-blob-condition-appendpos"`
+	IfModifiedSince   *time.Time `header:"If-Modified-Since"`
+	IfUnmodifiedSince *time.Time `header:"If-Unmodified-Since"`
+	IfMatch           string     `header:"If-Match"`
+	IfNoneMatch       string     `header:"If-None-Match"`
+	RequestID         string     `header:"x-ms-client-request-id"`
+	ContentMD5        bool
+}
+
+// AppendBlock appends a block to an append blob.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Append-Block
+func (b *Blob) AppendBlock(chunk []byte, options *AppendBlockOptions) error {
+	params := url.Values{"comp": {"appendblock"}}
+	headers := b.Container.bsc.client.getStandardHeaders()
+	headers["x-ms-blob-type"] = string(BlobTypeAppend)
+	headers["Content-Length"] = fmt.Sprintf("%v", len(chunk))
+
+	if options != nil {
+		params = addTimeout(params, options.Timeout)
+		headers = mergeHeaders(headers, headersFromStruct(*options))
+		if options.ContentMD5 {
+			md5sum := md5.Sum(chunk)
+			headers[headerContentMD5] = base64.StdEncoding.EncodeToString(md5sum[:])
+		}
+	}
+	uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), params)
+
+	resp, err := b.Container.bsc.client.exec(http.MethodPut, uri, headers, bytes.NewReader(chunk), b.Container.bsc.auth)
+	if err != nil {
+		return err
+	}
+	return b.respondCreation(resp, BlobTypeAppend)
+}

vendor/github.com/Azure/azure-sdk-for-go/storage/authorization.go

@@ -0,0 +1,246 @@
+// Package storage provides clients for Microsoft Azure Storage Services.
+package storage
+
+// Copyright 2017 Microsoft Corporation
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+import (
+	"bytes"
+	"fmt"
+	"net/url"
+	"sort"
+	"strings"
+)
+
+// See: https://docs.microsoft.com/rest/api/storageservices/fileservices/authentication-for-the-azure-storage-services
+
+type authentication string
+
+const (
+	sharedKey             authentication = "sharedKey"
+	sharedKeyForTable     authentication = "sharedKeyTable"
+	sharedKeyLite         authentication = "sharedKeyLite"
+	sharedKeyLiteForTable authentication = "sharedKeyLiteTable"
+
+	// headers
+	headerAcceptCharset           = "Accept-Charset"
+	headerAuthorization           = "Authorization"
+	headerContentLength           = "Content-Length"
+	headerDate                    = "Date"
+	headerXmsDate                 = "x-ms-date"
+	headerXmsVersion              = "x-ms-version"
+	headerContentEncoding         = "Content-Encoding"
+	headerContentLanguage         = "Content-Language"
+	headerContentType             = "Content-Type"
+	headerContentMD5              = "Content-MD5"
+	headerIfModifiedSince         = "If-Modified-Since"
+	headerIfMatch                 = "If-Match"
+	headerIfNoneMatch             = "If-None-Match"
+	headerIfUnmodifiedSince       = "If-Unmodified-Since"
+	headerRange                   = "Range"
+	headerDataServiceVersion      = "DataServiceVersion"
+	headerMaxDataServiceVersion   = "MaxDataServiceVersion"
+	headerContentTransferEncoding = "Content-Transfer-Encoding"
+)
+
+func (c *Client) addAuthorizationHeader(verb, url string, headers map[string]string, auth authentication) (map[string]string, error) {
+	if !c.sasClient {
+		authHeader, err := c.getSharedKey(verb, url, headers, auth)
+		if err != nil {
+			return nil, err
+		}
+		headers[headerAuthorization] = authHeader
+	}
+	return headers, nil
+}
+
+func (c *Client) getSharedKey(verb, url string, headers map[string]string, auth authentication) (string, error) {
+	canRes, err := c.buildCanonicalizedResource(url, auth, false)
+	if err != nil {
+		return "", err
+	}
+
+	canString, err := buildCanonicalizedString(verb, headers, canRes, auth)
+	if err != nil {
+		return "", err
+	}
+	return c.createAuthorizationHeader(canString, auth), nil
+}
+
+func (c *Client) buildCanonicalizedResource(uri string, auth authentication, sas bool) (string, error) {
+	errMsg := "buildCanonicalizedResource error: %s"
+	u, err := url.Parse(uri)
+	if err != nil {
+		return "", fmt.Errorf(errMsg, err.Error())
+	}
+
+	cr := bytes.NewBufferString("")
+	if c.accountName != StorageEmulatorAccountName || !sas {
+		cr.WriteString("/")
+		cr.WriteString(c.getCanonicalizedAccountName())
+	}
+
+	if len(u.Path) > 0 {
+		// Any portion of the CanonicalizedResource string that is derived from
+		// the resource's URI should be encoded exactly as it is in the URI.
+		// -- https://msdn.microsoft.com/en-gb/library/azure/dd179428.aspx
+		cr.WriteString(u.EscapedPath())
+	}
+
+	params, err := url.ParseQuery(u.RawQuery)
+	if err != nil {
+		return "", fmt.Errorf(errMsg, err.Error())
+	}
+
+	// See https://github.com/Azure/azure-storage-net/blob/master/Lib/Common/Core/Util/AuthenticationUtility.cs#L277
+	if auth == sharedKey {
+		if len(params) > 0 {
+			cr.WriteString("\n")
+
+			keys := []string{}
+			for key := range params {
+				keys = append(keys, key)
+			}
+			sort.Strings(keys)
+
+			completeParams := []string{}
+			for _, key := range keys {
+				if len(params[key]) > 1 {
+					sort.Strings(params[key])
+				}
+
+				completeParams = append(completeParams, fmt.Sprintf("%s:%s", key, strings.Join(params[key], ",")))
+			}
+			cr.WriteString(strings.Join(completeParams, "\n"))
+		}
+	} else {
+		// search for "comp" parameter, if exists then add it to canonicalizedresource
+		if v, ok := params["comp"]; ok {
+			cr.WriteString("?comp=" + v[0])
+		}
+	}
+
+	return string(cr.Bytes()), nil
+}
+
+func (c *Client) getCanonicalizedAccountName() string {
+	// since we may be trying to access a secondary storage account, we need to
+	// remove the -secondary part of the storage name
+	return strings.TrimSuffix(c.accountName, "-secondary")
+}
+
+func buildCanonicalizedString(verb string, headers map[string]string, canonicalizedResource string, auth authentication) (string, error) {
+	contentLength := headers[headerContentLength]
+	if contentLength == "0" {
+		contentLength = ""
+	}
+	date := headers[headerDate]
+	if v, ok := headers[headerXmsDate]; ok {
+		if auth == sharedKey || auth == sharedKeyLite {
+			date = ""
+		} else {
+			date = v
+		}
+	}
+	var canString string
+	switch auth {
+	case sharedKey:
+		canString = strings.Join([]string{
+			verb,
+			headers[headerContentEncoding],
+			headers[headerContentLanguage],
+			contentLength,
+			headers[headerContentMD5],
+			headers[headerContentType],
+			date,
+			headers[headerIfModifiedSince],
+			headers[headerIfMatch],
+			headers[headerIfNoneMatch],
+			headers[headerIfUnmodifiedSince],
+			headers[headerRange],
+			buildCanonicalizedHeader(headers),
+			canonicalizedResource,
+		}, "\n")
+	case sharedKeyForTable:
+		canString = strings.Join([]string{
+			verb,
+			headers[headerContentMD5],
+			headers[headerContentType],
+			date,
+			canonicalizedResource,
+		}, "\n")
+	case sharedKeyLite:
+		canString = strings.Join([]string{
+			verb,
+			headers[headerContentMD5],
+			headers[headerContentType],
+			date,
+			buildCanonicalizedHeader(headers),
+			canonicalizedResource,
+		}, "\n")
+	case sharedKeyLiteForTable:
+		canString = strings.Join([]string{
+			date,
+			canonicalizedResource,
+		}, "\n")
+	default:
+		return "", fmt.Errorf("%s authentication is not supported yet", auth)
+	}
+	return canString, nil
+}
+
+func buildCanonicalizedHeader(headers map[string]string) string {
+	cm := make(map[string]string)
+
+	for k, v := range headers {
+		headerName := strings.TrimSpace(strings.ToLower(k))
+		if strings.HasPrefix(headerName, "x-ms-") {
+			cm[headerName] = v
+		}
+	}
+
+	if len(cm) == 0 {
+		return ""
+	}
+
+	keys := []string{}
+	for key := range cm {
+		keys = append(keys, key)
+	}
+
+	sort.Strings(keys)
+
+	ch := bytes.NewBufferString("")
+
+	for _, key := range keys {
+		ch.WriteString(key)
+		ch.WriteRune(':')
+		ch.WriteString(cm[key])
+		ch.WriteRune('\n')
+	}
+
+	return strings.TrimSuffix(string(ch.Bytes()), "\n")
+}
+
+func (c *Client) createAuthorizationHeader(canonicalizedString string, auth authentication) string {
+	signature := c.computeHmac256(canonicalizedString)
+	var key string
+	switch auth {
+	case sharedKey, sharedKeyForTable:
+		key = "SharedKey"
+	case sharedKeyLite, sharedKeyLiteForTable:
+		key = "SharedKeyLite"
+	}
+	return fmt.Sprintf("%s %s:%s", key, c.getCanonicalizedAccountName(), signature)
+}

vendor/github.com/Azure/azure-sdk-for-go/storage/blob.go

@@ -0,0 +1,632 @@
+package storage
+
+// Copyright 2017 Microsoft Corporation
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+import (
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// A Blob is an entry in BlobListResponse.
+type Blob struct {
+	Container  *Container
+	Name       string         `xml:"Name"`
+	Snapshot   time.Time      `xml:"Snapshot"`
+	Properties BlobProperties `xml:"Properties"`
+	Metadata   BlobMetadata   `xml:"Metadata"`
+}
+
+// PutBlobOptions includes the options any put blob operation
+// (page, block, append)
+type PutBlobOptions struct {
+	Timeout           uint
+	LeaseID           string     `header:"x-ms-lease-id"`
+	Origin            string     `header:"Origin"`
+	IfModifiedSince   *time.Time `header:"If-Modified-Since"`
+	IfUnmodifiedSince *time.Time `header:"If-Unmodified-Since"`
+	IfMatch           string     `header:"If-Match"`
+	IfNoneMatch       string     `header:"If-None-Match"`
+	RequestID         string     `header:"x-ms-client-request-id"`
+}
+
+// BlobMetadata is a set of custom name/value pairs.
+//
+// See https://msdn.microsoft.com/en-us/library/azure/dd179404.aspx
+type BlobMetadata map[string]string
+
+type blobMetadataEntries struct {
+	Entries []blobMetadataEntry `xml:",any"`
+}
+type blobMetadataEntry struct {
+	XMLName xml.Name
+	Value   string `xml:",chardata"`
+}
+
+// UnmarshalXML converts the xml:Metadata into Metadata map
+func (bm *BlobMetadata) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
+	var entries blobMetadataEntries
+	if err := d.DecodeElement(&entries, &start); err != nil {
+		return err
+	}
+	for _, entry := range entries.Entries {
+		if *bm == nil {
+			*bm = make(BlobMetadata)
+		}
+		(*bm)[strings.ToLower(entry.XMLName.Local)] = entry.Value
+	}
+	return nil
+}
+
+// MarshalXML implements the xml.Marshaler interface. It encodes
+// metadata name/value pairs as they would appear in an Azure
+// ListBlobs response.
+func (bm BlobMetadata) MarshalXML(enc *xml.Encoder, start xml.StartElement) error {
+	entries := make([]blobMetadataEntry, 0, len(bm))
+	for k, v := range bm {
+		entries = append(entries, blobMetadataEntry{
+			XMLName: xml.Name{Local: http.CanonicalHeaderKey(k)},
+			Value:   v,
+		})
+	}
+	return enc.EncodeElement(blobMetadataEntries{
+		Entries: entries,
+	}, start)
+}
+
+// BlobProperties contains various properties of a blob
+// returned in various endpoints like ListBlobs or GetBlobProperties.
+type BlobProperties struct {
+	LastModified          TimeRFC1123 `xml:"Last-Modified"`
+	Etag                  string      `xml:"Etag"`
+	ContentMD5            string      `xml:"Content-MD5" header:"x-ms-blob-content-md5"`
+	ContentLength         int64       `xml:"Content-Length"`
+	ContentType           string      `xml:"Content-Type" header:"x-ms-blob-content-type"`
+	ContentEncoding       string      `xml:"Content-Encoding" header:"x-ms-blob-content-encoding"`
+	CacheControl          string      `xml:"Cache-Control" header:"x-ms-blob-cache-control"`
+	ContentLanguage       string      `xml:"Cache-Language" header:"x-ms-blob-content-language"`
+	ContentDisposition    string      `xml:"Content-Disposition" header:"x-ms-blob-content-disposition"`
+	BlobType              BlobType    `xml:"BlobType"`
+	SequenceNumber        int64       `xml:"x-ms-blob-sequence-number"`
+	CopyID                string      `xml:"CopyId"`
+	CopyStatus            string      `xml:"CopyStatus"`
+	CopySource            string      `xml:"CopySource"`
+	CopyProgress          string      `xml:"CopyProgress"`
+	CopyCompletionTime    TimeRFC1123 `xml:"CopyCompletionTime"`
+	CopyStatusDescription string      `xml:"CopyStatusDescription"`
+	LeaseStatus           string      `xml:"LeaseStatus"`
+	LeaseState            string      `xml:"LeaseState"`
+	LeaseDuration         string      `xml:"LeaseDuration"`
+	ServerEncrypted       bool        `xml:"ServerEncrypted"`
+	IncrementalCopy       bool        `xml:"IncrementalCopy"`
+}
+
+// BlobType defines the type of the Azure Blob.
+type BlobType string
+
+// Types of page blobs
+const (
+	BlobTypeBlock  BlobType = "BlockBlob"
+	BlobTypePage   BlobType = "PageBlob"
+	BlobTypeAppend BlobType = "AppendBlob"
+)
+
+func (b *Blob) buildPath() string {
+	return b.Container.buildPath() + "/" + b.Name
+}
+
+// Exists returns true if a blob with given name exists on the specified
+// container of the storage account.
+func (b *Blob) Exists() (bool, error) {
+	uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), nil)
+	headers := b.Container.bsc.client.getStandardHeaders()
+	resp, err := b.Container.bsc.client.exec(http.MethodHead, uri, headers, nil, b.Container.bsc.auth)
+	if resp != nil {
+		defer drainRespBody(resp)
+		if resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusNotFound {
+			return resp.StatusCode == http.StatusOK, nil
+		}
+	}
+	return false, err
+}
+
+// GetURL gets the canonical URL to the blob with the specified name in the
+// specified container.
+// This method does not create a publicly accessible URL if the blob or container
+// is private and this method does not check if the blob exists.
+func (b *Blob) GetURL() string {
+	container := b.Container.Name
+	if container == "" {
+		container = "$root"
+	}
+	return b.Container.bsc.client.getEndpoint(blobServiceName, pathForResource(container, b.Name), nil)
+}
+
+// GetBlobRangeOptions includes the options for a get blob range operation
+type GetBlobRangeOptions struct {
+	Range              *BlobRange
+	GetRangeContentMD5 bool
+	*GetBlobOptions
+}
+
+// GetBlobOptions includes the options for a get blob operation
+type GetBlobOptions struct {
+	Timeout           uint
+	Snapshot          *time.Time
+	LeaseID           string     `header:"x-ms-lease-id"`
+	Origin            string     `header:"Origin"`
+	IfModifiedSince   *time.Time `header:"If-Modified-Since"`
+	IfUnmodifiedSince *time.Time `header:"If-Unmodified-Since"`
+	IfMatch           string     `header:"If-Match"`
+	IfNoneMatch       string     `header:"If-None-Match"`
+	RequestID         string     `header:"x-ms-client-request-id"`
+}
+
+// BlobRange represents the bytes range to be get
+type BlobRange struct {
+	Start uint64
+	End   uint64
+}
+
+func (br BlobRange) String() string {
+	if br.End == 0 {
+		return fmt.Sprintf("bytes=%d-", br.Start)
+	}
+	return fmt.Sprintf("bytes=%d-%d", br.Start, br.End)
+}
+
+// Get returns a stream to read the blob. Caller must call both Read and Close()
+// to correctly close the underlying connection.
+//
+// See the GetRange method for use with a Range header.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Get-Blob
+func (b *Blob) Get(options *GetBlobOptions) (io.ReadCloser, error) {
+	rangeOptions := GetBlobRangeOptions{
+		GetBlobOptions: options,
+	}
+	resp, err := b.getRange(&rangeOptions)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := checkRespCode(resp, []int{http.StatusOK}); err != nil {
+		return nil, err
+	}
+	if err := b.writeProperties(resp.Header, true); err != nil {
+		return resp.Body, err
+	}
+	return resp.Body, nil
+}
+
+// GetRange reads the specified range of a blob to a stream. The bytesRange
+// string must be in a format like "0-", "10-100" as defined in HTTP 1.1 spec.
+// Caller must call both Read and Close()// to correctly close the underlying
+// connection.
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Get-Blob
+func (b *Blob) GetRange(options *GetBlobRangeOptions) (io.ReadCloser, error) {
+	resp, err := b.getRange(options)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := checkRespCode(resp, []int{http.StatusPartialContent}); err != nil {
+		return nil, err
+	}
+	// Content-Length header should not be updated, as the service returns the range length
+	// (which is not alwys the full blob length)
+	if err := b.writeProperties(resp.Header, false); err != nil {
+		return resp.Body, err
+	}
+	return resp.Body, nil
+}
+
+func (b *Blob) getRange(options *GetBlobRangeOptions) (*http.Response, error) {
+	params := url.Values{}
+	headers := b.Container.bsc.client.getStandardHeaders()
+
+	if options != nil {
+		if options.Range != nil {
+			headers["Range"] = options.Range.String()
+			if options.GetRangeContentMD5 {
+				headers["x-ms-range-get-content-md5"] = "true"
+			}
+		}
+		if options.GetBlobOptions != nil {
+			headers = mergeHeaders(headers, headersFromStruct(*options.GetBlobOptions))
+			params = addTimeout(params, options.Timeout)
+			params = addSnapshot(params, options.Snapshot)
+		}
+	}
+	uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), params)
+
+	resp, err := b.Container.bsc.client.exec(http.MethodGet, uri, headers, nil, b.Container.bsc.auth)
+	if err != nil {
+		return nil, err
+	}
+	return resp, err
+}
+
+// SnapshotOptions includes the options for a snapshot blob operation
+type SnapshotOptions struct {
+	Timeout           uint
+	LeaseID           string     `header:"x-ms-lease-id"`
+	IfModifiedSince   *time.Time `header:"If-Modified-Since"`
+	IfUnmodifiedSince *time.Time `header:"If-Unmodified-Since"`
+	IfMatch           string     `header:"If-Match"`
+	IfNoneMatch       string     `header:"If-None-Match"`
+	RequestID         string     `header:"x-ms-client-request-id"`
+}
+
+// CreateSnapshot creates a snapshot for a blob
+// See https://msdn.microsoft.com/en-us/library/azure/ee691971.aspx
+func (b *Blob) CreateSnapshot(options *SnapshotOptions) (snapshotTimestamp *time.Time, err error) {
+	params := url.Values{"comp": {"snapshot"}}
+	headers := b.Container.bsc.client.getStandardHeaders()
+	headers = b.Container.bsc.client.addMetadataToHeaders(headers, b.Metadata)
+
+	if options != nil {
+		params = addTimeout(params, options.Timeout)
+		headers = mergeHeaders(headers, headersFromStruct(*options))
+	}
+	uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), params)
+
+	resp, err := b.Container.bsc.client.exec(http.MethodPut, uri, headers, nil, b.Container.bsc.auth)
+	if err != nil || resp == nil {
+		return nil, err
+	}
+	defer drainRespBody(resp)
+
+	if err := checkRespCode(resp, []int{http.StatusCreated}); err != nil {
+		return nil, err
+	}
+
+	snapshotResponse := resp.Header.Get(http.CanonicalHeaderKey("x-ms-snapshot"))
+	if snapshotResponse != "" {
+		snapshotTimestamp, err := time.Parse(time.RFC3339, snapshotResponse)
+		if err != nil {
+			return nil, err
+		}
+		return &snapshotTimestamp, nil
+	}
+
+	return nil, errors.New("Snapshot not created")
+}
+
+// GetBlobPropertiesOptions includes the options for a get blob properties operation
+type GetBlobPropertiesOptions struct {
+	Timeout           uint
+	Snapshot          *time.Time
+	LeaseID           string     `header:"x-ms-lease-id"`
+	IfModifiedSince   *time.Time `header:"If-Modified-Since"`
+	IfUnmodifiedSince *time.Time `header:"If-Unmodified-Since"`
+	IfMatch           string     `header:"If-Match"`
+	IfNoneMatch       string     `header:"If-None-Match"`
+	RequestID         string     `header:"x-ms-client-request-id"`
+}
+
+// GetProperties provides various information about the specified blob.
+// See https://msdn.microsoft.com/en-us/library/azure/dd179394.aspx
+func (b *Blob) GetProperties(options *GetBlobPropertiesOptions) error {
+	params := url.Values{}
+	headers := b.Container.bsc.client.getStandardHeaders()
+
+	if options != nil {
+		params = addTimeout(params, options.Timeout)
+		params = addSnapshot(params, options.Snapshot)
+		headers = mergeHeaders(headers, headersFromStruct(*options))
+	}
+	uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), params)
+
+	resp, err := b.Container.bsc.client.exec(http.MethodHead, uri, headers, nil, b.Container.bsc.auth)
+	if err != nil {
+		return err
+	}
+	defer drainRespBody(resp)
+
+	if err = checkRespCode(resp, []int{http.StatusOK}); err != nil {
+		return err
+	}
+	return b.writeProperties(resp.Header, true)
+}
+
+func (b *Blob) writeProperties(h http.Header, includeContentLen bool) error {
+	var err error
+
+	contentLength := b.Properties.ContentLength
+	if includeContentLen {
+		contentLengthStr := h.Get("Content-Length")
+		if contentLengthStr != "" {
+			contentLength, err = strconv.ParseInt(contentLengthStr, 0, 64)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	var sequenceNum int64
+	sequenceNumStr := h.Get("x-ms-blob-sequence-number")
+	if sequenceNumStr != "" {
+		sequenceNum, err = strconv.ParseInt(sequenceNumStr, 0, 64)
+		if err != nil {
+			return err
+		}
+	}
+
+	lastModified, err := getTimeFromHeaders(h, "Last-Modified")
+	if err != nil {
+		return err
+	}
+
+	copyCompletionTime, err := getTimeFromHeaders(h, "x-ms-copy-completion-time")
+	if err != nil {
+		return err
+	}
+
+	b.Properties = BlobProperties{
+		LastModified:          TimeRFC1123(*lastModified),
+		Etag:                  h.Get("Etag"),
+		ContentMD5:            h.Get("Content-MD5"),
+		ContentLength:         contentLength,
+		ContentEncoding:       h.Get("Content-Encoding"),
+		ContentType:           h.Get("Content-Type"),
+		ContentDisposition:    h.Get("Content-Disposition"),
+		CacheControl:          h.Get("Cache-Control"),
+		ContentLanguage:       h.Get("Content-Language"),
+		SequenceNumber:        sequenceNum,
+		CopyCompletionTime:    TimeRFC1123(*copyCompletionTime),
+		CopyStatusDescription: h.Get("x-ms-copy-status-description"),
+		CopyID:                h.Get("x-ms-copy-id"),
+		CopyProgress:          h.Get("x-ms-copy-progress"),
+		CopySource:            h.Get("x-ms-copy-source"),
+		CopyStatus:            h.Get("x-ms-copy-status"),
+		BlobType:              BlobType(h.Get("x-ms-blob-type")),
+		LeaseStatus:           h.Get("x-ms-lease-status"),
+		LeaseState:            h.Get("x-ms-lease-state"),
+	}
+	b.writeMetadata(h)
+	return nil
+}
+
+// SetBlobPropertiesOptions contains various properties of a blob and is an entry
+// in SetProperties
+type SetBlobPropertiesOptions struct {
+	Timeout              uint
+	LeaseID              string     `header:"x-ms-lease-id"`
+	Origin               string     `header:"Origin"`
+	IfModifiedSince      *time.Time `header:"If-Modified-Since"`
+	IfUnmodifiedSince    *time.Time `header:"If-Unmodified-Since"`
+	IfMatch              string     `header:"If-Match"`
+	IfNoneMatch          string     `header:"If-None-Match"`
+	SequenceNumberAction *SequenceNumberAction
+	RequestID            string `header:"x-ms-client-request-id"`
+}
+
+// SequenceNumberAction defines how the blob's sequence number should be modified
+type SequenceNumberAction string
+
+// Options for sequence number action
+const (
+	SequenceNumberActionMax       SequenceNumberAction = "max"
+	SequenceNumberActionUpdate    SequenceNumberAction = "update"
+	SequenceNumberActionIncrement SequenceNumberAction = "increment"
+)
+
+// SetProperties replaces the BlobHeaders for the specified blob.
+//
+// Some keys may be converted to Camel-Case before sending. All keys
+// are returned in lower case by GetBlobProperties. HTTP header names
+// are case-insensitive so case munging should not matter to other
+// applications either.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Set-Blob-Properties
+func (b *Blob) SetProperties(options *SetBlobPropertiesOptions) error {
+	params := url.Values{"comp": {"properties"}}
+	headers := b.Container.bsc.client.getStandardHeaders()
+	headers = mergeHeaders(headers, headersFromStruct(b.Properties))
+
+	if options != nil {
+		params = addTimeout(params, options.Timeout)
+		headers = mergeHeaders(headers, headersFromStruct(*options))
+	}
+	uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), params)
+
+	if b.Properties.BlobType == BlobTypePage {
+		headers = addToHeaders(headers, "x-ms-blob-content-length", fmt.Sprintf("%v", b.Properties.ContentLength))
+		if options != nil && options.SequenceNumberAction != nil {
+			headers = addToHeaders(headers, "x-ms-sequence-number-action", string(*options.SequenceNumberAction))
+			if *options.SequenceNumberAction != SequenceNumberActionIncrement {
+				headers = addToHeaders(headers, "x-ms-blob-sequence-number", fmt.Sprintf("%v", b.Properties.SequenceNumber))
+			}
+		}
+	}
+
+	resp, err := b.Container.bsc.client.exec(http.MethodPut, uri, headers, nil, b.Container.bsc.auth)
+	if err != nil {
+		return err
+	}
+	defer drainRespBody(resp)
+	return checkRespCode(resp, []int{http.StatusOK})
+}
+
+// SetBlobMetadataOptions includes the options for a set blob metadata operation
+type SetBlobMetadataOptions struct {
+	Timeout           uint
+	LeaseID           string     `header:"x-ms-lease-id"`
+	IfModifiedSince   *time.Time `header:"If-Modified-Since"`
+	IfUnmodifiedSince *time.Time `header:"If-Unmodified-Since"`
+	IfMatch           string     `header:"If-Match"`
+	IfNoneMatch       string     `header:"If-None-Match"`
+	RequestID         string     `header:"x-ms-client-request-id"`
+}
+
+// SetMetadata replaces the metadata for the specified blob.
+//
+// Some keys may be converted to Camel-Case before sending. All keys
+// are returned in lower case by GetBlobMetadata. HTTP header names
+// are case-insensitive so case munging should not matter to other
+// applications either.
+//
+// See https://msdn.microsoft.com/en-us/library/azure/dd179414.aspx
+func (b *Blob) SetMetadata(options *SetBlobMetadataOptions) error {
+	params := url.Values{"comp": {"metadata"}}
+	headers := b.Container.bsc.client.getStandardHeaders()
+	headers = b.Container.bsc.client.addMetadataToHeaders(headers, b.Metadata)
+
+	if options != nil {
+		params = addTimeout(params, options.Timeout)
+		headers = mergeHeaders(headers, headersFromStruct(*options))
+	}
+	uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), params)
+
+	resp, err := b.Container.bsc.client.exec(http.MethodPut, uri, headers, nil, b.Container.bsc.auth)
+	if err != nil {
+		return err
+	}
+	defer drainRespBody(resp)
+	return checkRespCode(resp, []int{http.StatusOK})
+}
+
+// GetBlobMetadataOptions includes the options for a get blob metadata operation
+type GetBlobMetadataOptions struct {
+	Timeout           uint
+	Snapshot          *time.Time
+	LeaseID           string     `header:"x-ms-lease-id"`
+	IfModifiedSince   *time.Time `header:"If-Modified-Since"`
+	IfUnmodifiedSince *time.Time `header:"If-Unmodified-Since"`
+	IfMatch           string     `header:"If-Match"`
+	IfNoneMatch       string     `header:"If-None-Match"`
+	RequestID         string     `header:"x-ms-client-request-id"`
+}
+
+// GetMetadata returns all user-defined metadata for the specified blob.
+//
+// All metadata keys will be returned in lower case. (HTTP header
+// names are case-insensitive.)
+//
+// See https://msdn.microsoft.com/en-us/library/azure/dd179414.aspx
+func (b *Blob) GetMetadata(options *GetBlobMetadataOptions) error {
+	params := url.Values{"comp": {"metadata"}}
+	headers := b.Container.bsc.client.getStandardHeaders()
+
+	if options != nil {
+		params = addTimeout(params, options.Timeout)
+		params = addSnapshot(params, options.Snapshot)
+		headers = mergeHeaders(headers, headersFromStruct(*options))
+	}
+	uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), params)
+
+	resp, err := b.Container.bsc.client.exec(http.MethodGet, uri, headers, nil, b.Container.bsc.auth)
+	if err != nil {
+		return err
+	}
+	defer drainRespBody(resp)
+
+	if err := checkRespCode(resp, []int{http.StatusOK}); err != nil {
+		return err
+	}
+
+	b.writeMetadata(resp.Header)
+	return nil
+}
+
+func (b *Blob) writeMetadata(h http.Header) {
+	b.Metadata = BlobMetadata(writeMetadata(h))
+}
+
+// DeleteBlobOptions includes the options for a delete blob operation
+type DeleteBlobOptions struct {
+	Timeout           uint
+	Snapshot          *time.Time
+	LeaseID           string `header:"x-ms-lease-id"`
+	DeleteSnapshots   *bool
+	IfModifiedSince   *time.Time `header:"If-Modified-Since"`
+	IfUnmodifiedSince *time.Time `header:"If-Unmodified-Since"`
+	IfMatch           string     `header:"If-Match"`
+	IfNoneMatch       string     `header:"If-None-Match"`
+	RequestID         string     `header:"x-ms-client-request-id"`
+}
+
+// Delete deletes the given blob from the specified container.
+// If the blob does not exists at the time of the Delete Blob operation, it
+// returns error.
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Delete-Blob
+func (b *Blob) Delete(options *DeleteBlobOptions) error {
+	resp, err := b.delete(options)
+	if err != nil {
+		return err
+	}
+	defer drainRespBody(resp)
+	return checkRespCode(resp, []int{http.StatusAccepted})
+}
+
+// DeleteIfExists deletes the given blob from the specified container If the
+// blob is deleted with this call, returns true. Otherwise returns false.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Delete-Blob
+func (b *Blob) DeleteIfExists(options *DeleteBlobOptions) (bool, error) {
+	resp, err := b.delete(options)
+	if resp != nil {
+		defer drainRespBody(resp)
+		if resp.StatusCode == http.StatusAccepted || resp.StatusCode == http.StatusNotFound {
+			return resp.StatusCode == http.StatusAccepted, nil
+		}
+	}
+	return false, err
+}
+
+func (b *Blob) delete(options *DeleteBlobOptions) (*http.Response, error) {
+	params := url.Values{}
+	headers := b.Container.bsc.client.getStandardHeaders()
+
+	if options != nil {
+		params = addTimeout(params, options.Timeout)
+		params = addSnapshot(params, options.Snapshot)
+		headers = mergeHeaders(headers, headersFromStruct(*options))
+		if options.DeleteSnapshots != nil {
+			if *options.DeleteSnapshots {
+				headers["x-ms-delete-snapshots"] = "include"
+			} else {
+				headers["x-ms-delete-snapshots"] = "only"
+			}
+		}
+	}
+	uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), params)
+	return b.Container.bsc.client.exec(http.MethodDelete, uri, headers, nil, b.Container.bsc.auth)
+}
+
+// helper method to construct the path to either a blob or container
+func pathForResource(container, name string) string {
+	if name != "" {
+		return fmt.Sprintf("/%s/%s", container, name)
+	}
+	return fmt.Sprintf("/%s", container)
+}
+
+func (b *Blob) respondCreation(resp *http.Response, bt BlobType) error {
+	defer drainRespBody(resp)
+	err := checkRespCode(resp, []int{http.StatusCreated})
+	if err != nil {
+		return err
+	}
+	b.Properties.BlobType = bt
+	return nil
+}

vendor/github.com/Azure/azure-sdk-for-go/storage/blobsasuri.go

@@ -0,0 +1,174 @@
+package storage
+
+// Copyright 2017 Microsoft Corporation
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+import (
+	"errors"
+	"fmt"
+	"net/url"
+	"strings"
+	"time"
+)
+
+// OverrideHeaders defines overridable response heaedrs in
+// a request using a SAS URI.
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/constructing-a-service-sas
+type OverrideHeaders struct {
+	CacheControl       string
+	ContentDisposition string
+	ContentEncoding    string
+	ContentLanguage    string
+	ContentType        string
+}
+
+// BlobSASOptions are options to construct a blob SAS
+// URI.
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/constructing-a-service-sas
+type BlobSASOptions struct {
+	BlobServiceSASPermissions
+	OverrideHeaders
+	SASOptions
+}
+
+// BlobServiceSASPermissions includes the available permissions for
+// blob service SAS URI.
+type BlobServiceSASPermissions struct {
+	Read   bool
+	Add    bool
+	Create bool
+	Write  bool
+	Delete bool
+}
+
+func (p BlobServiceSASPermissions) buildString() string {
+	permissions := ""
+	if p.Read {
+		permissions += "r"
+	}
+	if p.Add {
+		permissions += "a"
+	}
+	if p.Create {
+		permissions += "c"
+	}
+	if p.Write {
+		permissions += "w"
+	}
+	if p.Delete {
+		permissions += "d"
+	}
+	return permissions
+}
+
+// GetSASURI creates an URL to the blob which contains the Shared
+// Access Signature with the specified options.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/constructing-a-service-sas
+func (b *Blob) GetSASURI(options BlobSASOptions) (string, error) {
+	uri := b.GetURL()
+	signedResource := "b"
+	canonicalizedResource, err := b.Container.bsc.client.buildCanonicalizedResource(uri, b.Container.bsc.auth, true)
+	if err != nil {
+		return "", err
+	}
+
+	permissions := options.BlobServiceSASPermissions.buildString()
+	return b.Container.bsc.client.blobAndFileSASURI(options.SASOptions, uri, permissions, canonicalizedResource, signedResource, options.OverrideHeaders)
+}
+
+func (c *Client) blobAndFileSASURI(options SASOptions, uri, permissions, canonicalizedResource, signedResource string, headers OverrideHeaders) (string, error) {
+	start := ""
+	if options.Start != (time.Time{}) {
+		start = options.Start.UTC().Format(time.RFC3339)
+	}
+
+	expiry := options.Expiry.UTC().Format(time.RFC3339)
+
+	// We need to replace + with %2b first to avoid being treated as a space (which is correct for query strings, but not the path component).
+	canonicalizedResource = strings.Replace(canonicalizedResource, "+", "%2b", -1)
+	canonicalizedResource, err := url.QueryUnescape(canonicalizedResource)
+	if err != nil {
+		return "", err
+	}
+
+	protocols := ""
+	if options.UseHTTPS {
+		protocols = "https"
+	}
+	stringToSign, err := blobSASStringToSign(permissions, start, expiry, canonicalizedResource, options.Identifier, options.IP, protocols, c.apiVersion, headers)
+	if err != nil {
+		return "", err
+	}
+
+	sig := c.computeHmac256(stringToSign)
+	sasParams := url.Values{
+		"sv":  {c.apiVersion},
+		"se":  {expiry},
+		"sr":  {signedResource},
+		"sp":  {permissions},
+		"sig": {sig},
+	}
+
+	if start != "" {
+		sasParams.Add("st", start)
+	}
+
+	if c.apiVersion >= "2015-04-05" {
+		if protocols != "" {
+			sasParams.Add("spr", protocols)
+		}
+		if options.IP != "" {
+			sasParams.Add("sip", options.IP)
+		}
+	}
+
+	// Add override response hedaers
+	addQueryParameter(sasParams, "rscc", headers.CacheControl)
+	addQueryParameter(sasParams, "rscd", headers.ContentDisposition)
+	addQueryParameter(sasParams, "rsce", headers.ContentEncoding)
+	addQueryParameter(sasParams, "rscl", headers.ContentLanguage)
+	addQueryParameter(sasParams, "rsct", headers.ContentType)
+
+	sasURL, err := url.Parse(uri)
+	if err != nil {
+		return "", err
+	}
+	sasURL.RawQuery = sasParams.Encode()
+	return sasURL.String(), nil
+}
+
+func blobSASStringToSign(signedPermissions, signedStart, signedExpiry, canonicalizedResource, signedIdentifier, signedIP, protocols, signedVersion string, headers OverrideHeaders) (string, error) {
+	rscc := headers.CacheControl
+	rscd := headers.ContentDisposition
+	rsce := headers.ContentEncoding
+	rscl := headers.ContentLanguage
+	rsct := headers.ContentType
+
+	if signedVersion >= "2015-02-21" {
+		canonicalizedResource = "/blob" + canonicalizedResource
+	}
+
+	// https://msdn.microsoft.com/en-us/library/azure/dn140255.aspx#Anchor_12
+	if signedVersion >= "2015-04-05" {
+		return fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s", signedPermissions, signedStart, signedExpiry, canonicalizedResource, signedIdentifier, signedIP, protocols, signedVersion, rscc, rscd, rsce, rscl, rsct), nil
+	}
+
+	// reference: http://msdn.microsoft.com/en-us/library/azure/dn140255.aspx
+	if signedVersion >= "2013-08-15" {
+		return fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s", signedPermissions, signedStart, signedExpiry, canonicalizedResource, signedIdentifier, signedVersion, rscc, rscd, rsce, rscl, rsct), nil
+	}
+
+	return "", errors.New("storage: not implemented SAS for versions earlier than 2013-08-15")
+}

vendor/github.com/Azure/azure-sdk-for-go/storage/blobserviceclient.go

@@ -0,0 +1,186 @@
+package storage
+
+// Copyright 2017 Microsoft Corporation
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+import (
+	"encoding/xml"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+)
+
+// BlobStorageClient contains operations for Microsoft Azure Blob Storage
+// Service.
+type BlobStorageClient struct {
+	client Client
+	auth   authentication
+}
+
+// GetServiceProperties gets the properties of your storage account's blob service.
+// See: https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/get-blob-service-properties
+func (b *BlobStorageClient) GetServiceProperties() (*ServiceProperties, error) {
+	return b.client.getServiceProperties(blobServiceName, b.auth)
+}
+
+// SetServiceProperties sets the properties of your storage account's blob service.
+// See: https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/set-blob-service-properties
+func (b *BlobStorageClient) SetServiceProperties(props ServiceProperties) error {
+	return b.client.setServiceProperties(props, blobServiceName, b.auth)
+}
+
+// ListContainersParameters defines the set of customizable parameters to make a
+// List Containers call.
+//
+// See https://msdn.microsoft.com/en-us/library/azure/dd179352.aspx
+type ListContainersParameters struct {
+	Prefix     string
+	Marker     string
+	Include    string
+	MaxResults uint
+	Timeout    uint
+}
+
+// GetContainerReference returns a Container object for the specified container name.
+func (b *BlobStorageClient) GetContainerReference(name string) *Container {
+	return &Container{
+		bsc:  b,
+		Name: name,
+	}
+}
+
+// GetContainerReferenceFromSASURI returns a Container object for the specified
+// container SASURI
+func GetContainerReferenceFromSASURI(sasuri url.URL) (*Container, error) {
+	path := strings.Split(sasuri.Path, "/")
+	if len(path) <= 1 {
+		return nil, fmt.Errorf("could not find a container in URI: %s", sasuri.String())
+	}
+	c, err := newSASClientFromURL(&sasuri)
+	if err != nil {
+		return nil, err
+	}
+	cli := c.GetBlobService()
+	return &Container{
+		bsc:    &cli,
+		Name:   path[1],
+		sasuri: sasuri,
+	}, nil
+}
+
+// ListContainers returns the list of containers in a storage account along with
+// pagination token and other response details.
+//
+// See https://msdn.microsoft.com/en-us/library/azure/dd179352.aspx
+func (b BlobStorageClient) ListContainers(params ListContainersParameters) (*ContainerListResponse, error) {
+	q := mergeParams(params.getParameters(), url.Values{"comp": {"list"}})
+	uri := b.client.getEndpoint(blobServiceName, "", q)
+	headers := b.client.getStandardHeaders()
+
+	type ContainerAlias struct {
+		bsc        *BlobStorageClient
+		Name       string              `xml:"Name"`
+		Properties ContainerProperties `xml:"Properties"`
+		Metadata   BlobMetadata
+		sasuri     url.URL
+	}
+	type ContainerListResponseAlias struct {
+		XMLName    xml.Name         `xml:"EnumerationResults"`
+		Xmlns      string           `xml:"xmlns,attr"`
+		Prefix     string           `xml:"Prefix"`
+		Marker     string           `xml:"Marker"`
+		NextMarker string           `xml:"NextMarker"`
+		MaxResults int64            `xml:"MaxResults"`
+		Containers []ContainerAlias `xml:"Containers>Container"`
+	}
+
+	var outAlias ContainerListResponseAlias
+	resp, err := b.client.exec(http.MethodGet, uri, headers, nil, b.auth)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	err = xmlUnmarshal(resp.Body, &outAlias)
+	if err != nil {
+		return nil, err
+	}
+
+	out := ContainerListResponse{
+		XMLName:    outAlias.XMLName,
+		Xmlns:      outAlias.Xmlns,
+		Prefix:     outAlias.Prefix,
+		Marker:     outAlias.Marker,
+		NextMarker: outAlias.NextMarker,
+		MaxResults: outAlias.MaxResults,
+		Containers: make([]Container, len(outAlias.Containers)),
+	}
+	for i, cnt := range outAlias.Containers {
+		out.Containers[i] = Container{
+			bsc:        &b,
+			Name:       cnt.Name,
+			Properties: cnt.Properties,
+			Metadata:   map[string]string(cnt.Metadata),
+			sasuri:     cnt.sasuri,
+		}
+	}
+
+	return &out, err
+}
+
+func (p ListContainersParameters) getParameters() url.Values {
+	out := url.Values{}
+
+	if p.Prefix != "" {
+		out.Set("prefix", p.Prefix)
+	}
+	if p.Marker != "" {
+		out.Set("marker", p.Marker)
+	}
+	if p.Include != "" {
+		out.Set("include", p.Include)
+	}
+	if p.MaxResults != 0 {
+		out.Set("maxresults", strconv.FormatUint(uint64(p.MaxResults), 10))
+	}
+	if p.Timeout != 0 {
+		out.Set("timeout", strconv.FormatUint(uint64(p.Timeout), 10))
+	}
+
+	return out
+}
+
+func writeMetadata(h http.Header) map[string]string {
+	metadata := make(map[string]string)
+	for k, v := range h {
+		// Can't trust CanonicalHeaderKey() to munge case
+		// reliably. "_" is allowed in identifiers:
+		// https://msdn.microsoft.com/en-us/library/azure/dd179414.aspx
+		// https://msdn.microsoft.com/library/aa664670(VS.71).aspx
+		// http://tools.ietf.org/html/rfc7230#section-3.2
+		// ...but "_" is considered invalid by
+		// CanonicalMIMEHeaderKey in
+		// https://golang.org/src/net/textproto/reader.go?s=14615:14659#L542
+		// so k can be "X-Ms-Meta-Lol" or "x-ms-meta-lol_rofl".
+		k = strings.ToLower(k)
+		if len(v) == 0 || !strings.HasPrefix(k, strings.ToLower(userDefinedMetadataHeaderPrefix)) {
+			continue
+		}
+		// metadata["lol"] = content of the last X-Ms-Meta-Lol header
+		k = k[len(userDefinedMetadataHeaderPrefix):]
+		metadata[k] = v[len(v)-1]
+	}
+	return metadata
+}

vendor/github.com/Azure/azure-sdk-for-go/storage/blockblob.go

@@ -0,0 +1,270 @@
+package storage
+
+// Copyright 2017 Microsoft Corporation
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+import (
+	"bytes"
+	"encoding/xml"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// BlockListType is used to filter out types of blocks in a Get Blocks List call
+// for a block blob.
+//
+// See https://msdn.microsoft.com/en-us/library/azure/dd179400.aspx for all
+// block types.
+type BlockListType string
+
+// Filters for listing blocks in block blobs
+const (
+	BlockListTypeAll         BlockListType = "all"
+	BlockListTypeCommitted   BlockListType = "committed"
+	BlockListTypeUncommitted BlockListType = "uncommitted"
+)
+
+// Maximum sizes (per REST API) for various concepts
+const (
+	MaxBlobBlockSize = 100 * 1024 * 1024
+	MaxBlobPageSize  = 4 * 1024 * 1024
+)
+
+// BlockStatus defines states a block for a block blob can
+// be in.
+type BlockStatus string
+
+// List of statuses that can be used to refer to a block in a block list
+const (
+	BlockStatusUncommitted BlockStatus = "Uncommitted"
+	BlockStatusCommitted   BlockStatus = "Committed"
+	BlockStatusLatest      BlockStatus = "Latest"
+)
+
+// Block is used to create Block entities for Put Block List
+// call.
+type Block struct {
+	ID     string
+	Status BlockStatus
+}
+
+// BlockListResponse contains the response fields from Get Block List call.
+//
+// See https://msdn.microsoft.com/en-us/library/azure/dd179400.aspx
+type BlockListResponse struct {
+	XMLName           xml.Name        `xml:"BlockList"`
+	CommittedBlocks   []BlockResponse `xml:"CommittedBlocks>Block"`
+	UncommittedBlocks []BlockResponse `xml:"UncommittedBlocks>Block"`
+}
+
+// BlockResponse contains the block information returned
+// in the GetBlockListCall.
+type BlockResponse struct {
+	Name string `xml:"Name"`
+	Size int64  `xml:"Size"`
+}
+
+// CreateBlockBlob initializes an empty block blob with no blocks.
+//
+// See CreateBlockBlobFromReader for more info on creating blobs.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Put-Blob
+func (b *Blob) CreateBlockBlob(options *PutBlobOptions) error {
+	return b.CreateBlockBlobFromReader(nil, options)
+}
+
+// CreateBlockBlobFromReader initializes a block blob using data from
+// reader. Size must be the number of bytes read from reader. To
+// create an empty blob, use size==0 and reader==nil.
+//
+// Any headers set in blob.Properties or metadata in blob.Metadata
+// will be set on the blob.
+//
+// The API rejects requests with size > 256 MiB (but this limit is not
+// checked by the SDK). To write a larger blob, use CreateBlockBlob,
+// PutBlock, and PutBlockList.
+//
+// To create a blob from scratch, call container.GetBlobReference() to
+// get an empty blob, fill in blob.Properties and blob.Metadata as
+// appropriate then call this method.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Put-Blob
+func (b *Blob) CreateBlockBlobFromReader(blob io.Reader, options *PutBlobOptions) error {
+	params := url.Values{}
+	headers := b.Container.bsc.client.getStandardHeaders()
+	headers["x-ms-blob-type"] = string(BlobTypeBlock)
+
+	headers["Content-Length"] = "0"
+	var n int64
+	var err error
+	if blob != nil {
+		type lener interface {
+			Len() int
+		}
+		// TODO(rjeczalik): handle io.ReadSeeker, in case blob is *os.File etc.
+		if l, ok := blob.(lener); ok {
+			n = int64(l.Len())
+		} else {
+			var buf bytes.Buffer
+			n, err = io.Copy(&buf, blob)
+			if err != nil {
+				return err
+			}
+			blob = &buf
+		}
+
+		headers["Content-Length"] = strconv.FormatInt(n, 10)
+	}
+	b.Properties.ContentLength = n
+
+	headers = mergeHeaders(headers, headersFromStruct(b.Properties))
+	headers = b.Container.bsc.client.addMetadataToHeaders(headers, b.Metadata)
+
+	if options != nil {
+		params = addTimeout(params, options.Timeout)
+		headers = mergeHeaders(headers, headersFromStruct(*options))
+	}
+	uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), params)
+
+	resp, err := b.Container.bsc.client.exec(http.MethodPut, uri, headers, blob, b.Container.bsc.auth)
+	if err != nil {
+		return err
+	}
+	return b.respondCreation(resp, BlobTypeBlock)
+}
+
+// PutBlockOptions includes the options for a put block operation
+type PutBlockOptions struct {
+	Timeout    uint
+	LeaseID    string `header:"x-ms-lease-id"`
+	ContentMD5 string `header:"Content-MD5"`
+	RequestID  string `header:"x-ms-client-request-id"`
+}
+
+// PutBlock saves the given data chunk to the specified block blob with
+// given ID.
+//
+// The API rejects chunks larger than 100 MiB (but this limit is not
+// checked by the SDK).
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Put-Block
+func (b *Blob) PutBlock(blockID string, chunk []byte, options *PutBlockOptions) error {
+	return b.PutBlockWithLength(blockID, uint64(len(chunk)), bytes.NewReader(chunk), options)
+}
+
+// PutBlockWithLength saves the given data stream of exactly specified size to
+// the block blob with given ID. It is an alternative to PutBlocks where data
+// comes as stream but the length is known in advance.
+//
+// The API rejects requests with size > 100 MiB (but this limit is not
+// checked by the SDK).
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Put-Block
+func (b *Blob) PutBlockWithLength(blockID string, size uint64, blob io.Reader, options *PutBlockOptions) error {
+	query := url.Values{
+		"comp":    {"block"},
+		"blockid": {blockID},
+	}
+	headers := b.Container.bsc.client.getStandardHeaders()
+	headers["Content-Length"] = fmt.Sprintf("%v", size)
+
+	if options != nil {
+		query = addTimeout(query, options.Timeout)
+		headers = mergeHeaders(headers, headersFromStruct(*options))
+	}
+	uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), query)
+
+	resp, err := b.Container.bsc.client.exec(http.MethodPut, uri, headers, blob, b.Container.bsc.auth)
+	if err != nil {
+		return err
+	}
+	return b.respondCreation(resp, BlobTypeBlock)
+}
+
+// PutBlockListOptions includes the options for a put block list operation
+type PutBlockListOptions struct {
+	Timeout           uint
+	LeaseID           string     `header:"x-ms-lease-id"`
+	IfModifiedSince   *time.Time `header:"If-Modified-Since"`
+	IfUnmodifiedSince *time.Time `header:"If-Unmodified-Since"`
+	IfMatch           string     `header:"If-Match"`
+	IfNoneMatch       string     `header:"If-None-Match"`
+	RequestID         string     `header:"x-ms-client-request-id"`
+}
+
+// PutBlockList saves list of blocks to the specified block blob.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Put-Block-List
+func (b *Blob) PutBlockList(blocks []Block, options *PutBlockListOptions) error {
+	params := url.Values{"comp": {"blocklist"}}
+	blockListXML := prepareBlockListRequest(blocks)
+	headers := b.Container.bsc.client.getStandardHeaders()
+	headers["Content-Length"] = fmt.Sprintf("%v", len(blockListXML))
+	headers = mergeHeaders(headers, headersFromStruct(b.Properties))
+	headers = b.Container.bsc.client.addMetadataToHeaders(headers, b.Metadata)
+
+	if options != nil {
+		params = addTimeout(params, options.Timeout)
+		headers = mergeHeaders(headers, headersFromStruct(*options))
+	}
+	uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), params)
+
+	resp, err := b.Container.bsc.client.exec(http.MethodPut, uri, headers, strings.NewReader(blockListXML), b.Container.bsc.auth)
+	if err != nil {
+		return err
+	}
+	defer drainRespBody(resp)
+	return checkRespCode(resp, []int{http.StatusCreated})
+}
+
+// GetBlockListOptions includes the options for a get block list operation
+type GetBlockListOptions struct {
+	Timeout   uint
+	Snapshot  *time.Time
+	LeaseID   string `header:"x-ms-lease-id"`
+	RequestID string `header:"x-ms-client-request-id"`
+}
+
+// GetBlockList retrieves list of blocks in the specified block blob.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Get-Block-List
+func (b *Blob) GetBlockList(blockType BlockListType, options *GetBlockListOptions) (BlockListResponse, error) {
+	params := url.Values{
+		"comp":          {"blocklist"},
+		"blocklisttype": {string(blockType)},
+	}
+	headers := b.Container.bsc.client.getStandardHeaders()
+
+	if options != nil {
+		params = addTimeout(params, options.Timeout)
+		params = addSnapshot(params, options.Snapshot)
+		headers = mergeHeaders(headers, headersFromStruct(*options))
+	}
+	uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), params)
+
+	var out BlockListResponse
+	resp, err := b.Container.bsc.client.exec(http.MethodGet, uri, headers, nil, b.Container.bsc.auth)
+	if err != nil {
+		return out, err
+	}
+	defer resp.Body.Close()
+
+	err = xmlUnmarshal(resp.Body, &out)
+	return out, err
+}

vendor/github.com/Azure/azure-sdk-for-go/storage/client.go

@@ -0,0 +1,988 @@
+// Package storage provides clients for Microsoft Azure Storage Services.
+package storage
+
+// Copyright 2017 Microsoft Corporation
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+import (
+	"bufio"
+	"encoding/base64"
+	"encoding/json"
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"mime"
+	"mime/multipart"
+	"net/http"
+	"net/url"
+	"regexp"
+	"runtime"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/Azure/azure-sdk-for-go/version"
+	"github.com/Azure/go-autorest/autorest"
+	"github.com/Azure/go-autorest/autorest/azure"
+)
+
+const (
+	// DefaultBaseURL is the domain name used for storage requests in the
+	// public cloud when a default client is created.
+	DefaultBaseURL = "core.windows.net"
+
+	// DefaultAPIVersion is the Azure Storage API version string used when a
+	// basic client is created.
+	DefaultAPIVersion = "2016-05-31"
+
+	defaultUseHTTPS      = true
+	defaultRetryAttempts = 5
+	defaultRetryDuration = time.Second * 5
+
+	// StorageEmulatorAccountName is the fixed storage account used by Azure Storage Emulator
+	StorageEmulatorAccountName = "devstoreaccount1"
+
+	// StorageEmulatorAccountKey is the the fixed storage account used by Azure Storage Emulator
+	StorageEmulatorAccountKey = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
+
+	blobServiceName  = "blob"
+	tableServiceName = "table"
+	queueServiceName = "queue"
+	fileServiceName  = "file"
+
+	storageEmulatorBlob  = "127.0.0.1:10000"
+	storageEmulatorTable = "127.0.0.1:10002"
+	storageEmulatorQueue = "127.0.0.1:10001"
+
+	userAgentHeader = "User-Agent"
+
+	userDefinedMetadataHeaderPrefix = "x-ms-meta-"
+
+	connectionStringAccountName      = "accountname"
+	connectionStringAccountKey       = "accountkey"
+	connectionStringEndpointSuffix   = "endpointsuffix"
+	connectionStringEndpointProtocol = "defaultendpointsprotocol"
+
+	connectionStringBlobEndpoint  = "blobendpoint"
+	connectionStringFileEndpoint  = "fileendpoint"
+	connectionStringQueueEndpoint = "queueendpoint"
+	connectionStringTableEndpoint = "tableendpoint"
+	connectionStringSAS           = "sharedaccesssignature"
+)
+
+var (
+	validStorageAccount     = regexp.MustCompile("^[0-9a-z]{3,24}$")
+	defaultValidStatusCodes = []int{
+		http.StatusRequestTimeout,      // 408
+		http.StatusInternalServerError, // 500
+		http.StatusBadGateway,          // 502
+		http.StatusServiceUnavailable,  // 503
+		http.StatusGatewayTimeout,      // 504
+	}
+)
+
+// Sender sends a request
+type Sender interface {
+	Send(*Client, *http.Request) (*http.Response, error)
+}
+
+// DefaultSender is the default sender for the client. It implements
+// an automatic retry strategy.
+type DefaultSender struct {
+	RetryAttempts    int
+	RetryDuration    time.Duration
+	ValidStatusCodes []int
+	attempts         int // used for testing
+}
+
+// Send is the default retry strategy in the client
+func (ds *DefaultSender) Send(c *Client, req *http.Request) (resp *http.Response, err error) {
+	rr := autorest.NewRetriableRequest(req)
+	for attempts := 0; attempts < ds.RetryAttempts; attempts++ {
+		err = rr.Prepare()
+		if err != nil {
+			return resp, err
+		}
+		resp, err = c.HTTPClient.Do(rr.Request())
+		if err != nil || !autorest.ResponseHasStatusCode(resp, ds.ValidStatusCodes...) {
+			return resp, err
+		}
+		drainRespBody(resp)
+		autorest.DelayForBackoff(ds.RetryDuration, attempts, req.Cancel)
+		ds.attempts = attempts
+	}
+	ds.attempts++
+	return resp, err
+}
+
+// Client is the object that needs to be constructed to perform
+// operations on the storage account.
+type Client struct {
+	// HTTPClient is the http.Client used to initiate API
+	// requests. http.DefaultClient is used when creating a
+	// client.
+	HTTPClient *http.Client
+
+	// Sender is an interface that sends the request. Clients are
+	// created with a DefaultSender. The DefaultSender has an
+	// automatic retry strategy built in. The Sender can be customized.
+	Sender Sender
+
+	accountName      string
+	accountKey       []byte
+	useHTTPS         bool
+	UseSharedKeyLite bool
+	baseURL          string
+	apiVersion       string
+	userAgent        string
+	sasClient        bool
+	accountSASToken  url.Values
+}
+
+type odataResponse struct {
+	resp  *http.Response
+	odata odataErrorWrapper
+}
+
+// AzureStorageServiceError contains fields of the error response from
+// Azure Storage Service REST API. See https://msdn.microsoft.com/en-us/library/azure/dd179382.aspx
+// Some fields might be specific to certain calls.
+type AzureStorageServiceError struct {
+	Code                      string `xml:"Code"`
+	Message                   string `xml:"Message"`
+	AuthenticationErrorDetail string `xml:"AuthenticationErrorDetail"`
+	QueryParameterName        string `xml:"QueryParameterName"`
+	QueryParameterValue       string `xml:"QueryParameterValue"`
+	Reason                    string `xml:"Reason"`
+	Lang                      string
+	StatusCode                int
+	RequestID                 string
+	Date                      string
+	APIVersion                string
+}
+
+type odataErrorMessage struct {
+	Lang  string `json:"lang"`
+	Value string `json:"value"`
+}
+
+type odataError struct {
+	Code    string            `json:"code"`
+	Message odataErrorMessage `json:"message"`
+}
+
+type odataErrorWrapper struct {
+	Err odataError `json:"odata.error"`
+}
+
+// UnexpectedStatusCodeError is returned when a storage service responds with neither an error
+// nor with an HTTP status code indicating success.
+type UnexpectedStatusCodeError struct {
+	allowed []int
+	got     int
+	inner   error
+}
+
+func (e UnexpectedStatusCodeError) Error() string {
+	s := func(i int) string { return fmt.Sprintf("%d %s", i, http.StatusText(i)) }
+
+	got := s(e.got)
+	expected := []string{}
+	for _, v := range e.allowed {
+		expected = append(expected, s(v))
+	}
+	return fmt.Sprintf("storage: status code from service response is %s; was expecting %s.  Inner error: %+v", got, strings.Join(expected, " or "), e.inner)
+}
+
+// Got is the actual status code returned by Azure.
+func (e UnexpectedStatusCodeError) Got() int {
+	return e.got
+}
+
+// Inner returns any inner error info.
+func (e UnexpectedStatusCodeError) Inner() error {
+	return e.inner
+}
+
+// NewClientFromConnectionString creates a Client from the connection string.
+func NewClientFromConnectionString(input string) (Client, error) {
+	// build a map of connection string key/value pairs
+	parts := map[string]string{}
+	for _, pair := range strings.Split(input, ";") {
+		if pair == "" {
+			continue
+		}
+
+		equalDex := strings.IndexByte(pair, '=')
+		if equalDex <= 0 {
+			return Client{}, fmt.Errorf("Invalid connection segment %q", pair)
+		}
+
+		value := strings.TrimSpace(pair[equalDex+1:])
+		key := strings.TrimSpace(strings.ToLower(pair[:equalDex]))
+		parts[key] = value
+	}
+
+	// TODO: validate parameter sets?
+
+	if parts[connectionStringAccountName] == StorageEmulatorAccountName {
+		return NewEmulatorClient()
+	}
+
+	if parts[connectionStringSAS] != "" {
+		endpoint := ""
+		if parts[connectionStringBlobEndpoint] != "" {
+			endpoint = parts[connectionStringBlobEndpoint]
+		} else if parts[connectionStringFileEndpoint] != "" {
+			endpoint = parts[connectionStringFileEndpoint]
+		} else if parts[connectionStringQueueEndpoint] != "" {
+			endpoint = parts[connectionStringQueueEndpoint]
+		} else {
+			endpoint = parts[connectionStringTableEndpoint]
+		}
+
+		return NewAccountSASClientFromEndpointToken(endpoint, parts[connectionStringSAS])
+	}
+
+	useHTTPS := defaultUseHTTPS
+	if parts[connectionStringEndpointProtocol] != "" {
+		useHTTPS = parts[connectionStringEndpointProtocol] == "https"
+	}
+
+	return NewClient(parts[connectionStringAccountName], parts[connectionStringAccountKey],
+		parts[connectionStringEndpointSuffix], DefaultAPIVersion, useHTTPS)
+}
+
+// NewBasicClient constructs a Client with given storage service name and
+// key.
+func NewBasicClient(accountName, accountKey string) (Client, error) {
+	if accountName == StorageEmulatorAccountName {
+		return NewEmulatorClient()
+	}
+	return NewClient(accountName, accountKey, DefaultBaseURL, DefaultAPIVersion, defaultUseHTTPS)
+}
+
+// NewBasicClientOnSovereignCloud constructs a Client with given storage service name and
+// key in the referenced cloud.
+func NewBasicClientOnSovereignCloud(accountName, accountKey string, env azure.Environment) (Client, error) {
+	if accountName == StorageEmulatorAccountName {
+		return NewEmulatorClient()
+	}
+	return NewClient(accountName, accountKey, env.StorageEndpointSuffix, DefaultAPIVersion, defaultUseHTTPS)
+}
+
+//NewEmulatorClient contructs a Client intended to only work with Azure
+//Storage Emulator
+func NewEmulatorClient() (Client, error) {
+	return NewClient(StorageEmulatorAccountName, StorageEmulatorAccountKey, DefaultBaseURL, DefaultAPIVersion, false)
+}
+
+// NewClient constructs a Client. This should be used if the caller wants
+// to specify whether to use HTTPS, a specific REST API version or a custom
+// storage endpoint than Azure Public Cloud.
+func NewClient(accountName, accountKey, serviceBaseURL, apiVersion string, useHTTPS bool) (Client, error) {
+	var c Client
+	if !IsValidStorageAccount(accountName) {
+		return c, fmt.Errorf("azure: account name is not valid: it must be between 3 and 24 characters, and only may contain numbers and lowercase letters: %v", accountName)
+	} else if accountKey == "" {
+		return c, fmt.Errorf("azure: account key required")
+	} else if serviceBaseURL == "" {
+		return c, fmt.Errorf("azure: base storage service url required")
+	}
+
+	key, err := base64.StdEncoding.DecodeString(accountKey)
+	if err != nil {
+		return c, fmt.Errorf("azure: malformed storage account key: %v", err)
+	}
+
+	c = Client{
+		HTTPClient:       http.DefaultClient,
+		accountName:      accountName,
+		accountKey:       key,
+		useHTTPS:         useHTTPS,
+		baseURL:          serviceBaseURL,
+		apiVersion:       apiVersion,
+		sasClient:        false,
+		UseSharedKeyLite: false,
+		Sender: &DefaultSender{
+			RetryAttempts:    defaultRetryAttempts,
+			ValidStatusCodes: defaultValidStatusCodes,
+			RetryDuration:    defaultRetryDuration,
+		},
+	}
+	c.userAgent = c.getDefaultUserAgent()
+	return c, nil
+}
+
+// IsValidStorageAccount checks if the storage account name is valid.
+// See https://docs.microsoft.com/en-us/azure/storage/storage-create-storage-account
+func IsValidStorageAccount(account string) bool {
+	return validStorageAccount.MatchString(account)
+}
+
+// NewAccountSASClient contructs a client that uses accountSAS authorization
+// for its operations.
+func NewAccountSASClient(account string, token url.Values, env azure.Environment) Client {
+	return newSASClient(account, env.StorageEndpointSuffix, token)
+}
+
+// NewAccountSASClientFromEndpointToken constructs a client that uses accountSAS authorization
+// for its operations using the specified endpoint and SAS token.
+func NewAccountSASClientFromEndpointToken(endpoint string, sasToken string) (Client, error) {
+	u, err := url.Parse(endpoint)
+	if err != nil {
+		return Client{}, err
+	}
+	_, err = url.ParseQuery(sasToken)
+	if err != nil {
+		return Client{}, err
+	}
+	u.RawQuery = sasToken
+	return newSASClientFromURL(u)
+}
+
+func newSASClient(accountName, baseURL string, sasToken url.Values) Client {
+	c := Client{
+		HTTPClient: http.DefaultClient,
+		apiVersion: DefaultAPIVersion,
+		sasClient:  true,
+		Sender: &DefaultSender{
+			RetryAttempts:    defaultRetryAttempts,
+			ValidStatusCodes: defaultValidStatusCodes,
+			RetryDuration:    defaultRetryDuration,
+		},
+		accountName:     accountName,
+		baseURL:         baseURL,
+		accountSASToken: sasToken,
+	}
+	c.userAgent = c.getDefaultUserAgent()
+	// Get API version and protocol from token
+	c.apiVersion = sasToken.Get("sv")
+	c.useHTTPS = sasToken.Get("spr") == "https"
+	return c
+}
+
+func newSASClientFromURL(u *url.URL) (Client, error) {
+	// the host name will look something like this
+	// - foo.blob.core.windows.net
+	// "foo" is the account name
+	// "core.windows.net" is the baseURL
+
+	// find the first dot to get account name
+	i1 := strings.IndexByte(u.Host, '.')
+	if i1 < 0 {
+		return Client{}, fmt.Errorf("failed to find '.' in %s", u.Host)
+	}
+
+	// now find the second dot to get the base URL
+	i2 := strings.IndexByte(u.Host[i1+1:], '.')
+	if i2 < 0 {
+		return Client{}, fmt.Errorf("failed to find '.' in %s", u.Host[i1+1:])
+	}
+
+	sasToken := u.Query()
+	c := newSASClient(u.Host[:i1], u.Host[i1+i2+2:], sasToken)
+	if spr := sasToken.Get("spr"); spr == "" {
+		// infer from URL if not in the query params set
+		c.useHTTPS = u.Scheme == "https"
+	}
+	return c, nil
+}
+
+func (c Client) isServiceSASClient() bool {
+	return c.sasClient && c.accountSASToken == nil
+}
+
+func (c Client) isAccountSASClient() bool {
+	return c.sasClient && c.accountSASToken != nil
+}
+
+func (c Client) getDefaultUserAgent() string {
+	return fmt.Sprintf("Go/%s (%s-%s) azure-storage-go/%s api-version/%s",
+		runtime.Version(),
+		runtime.GOARCH,
+		runtime.GOOS,
+		version.Number,
+		c.apiVersion,
+	)
+}
+
+// AddToUserAgent adds an extension to the current user agent
+func (c *Client) AddToUserAgent(extension string) error {
+	if extension != "" {
+		c.userAgent = fmt.Sprintf("%s %s", c.userAgent, extension)
+		return nil
+	}
+	return fmt.Errorf("Extension was empty, User Agent stayed as %s", c.userAgent)
+}
+
+// protectUserAgent is used in funcs that include extraheaders as a parameter.
+// It prevents the User-Agent header to be overwritten, instead if it happens to
+// be present, it gets added to the current User-Agent. Use it before getStandardHeaders
+func (c *Client) protectUserAgent(extraheaders map[string]string) map[string]string {
+	if v, ok := extraheaders[userAgentHeader]; ok {
+		c.AddToUserAgent(v)
+		delete(extraheaders, userAgentHeader)
+	}
+	return extraheaders
+}
+
+func (c Client) getBaseURL(service string) *url.URL {
+	scheme := "http"
+	if c.useHTTPS {
+		scheme = "https"
+	}
+	host := ""
+	if c.accountName == StorageEmulatorAccountName {
+		switch service {
+		case blobServiceName:
+			host = storageEmulatorBlob
+		case tableServiceName:
+			host = storageEmulatorTable
+		case queueServiceName:
+			host = storageEmulatorQueue
+		}
+	} else {
+		host = fmt.Sprintf("%s.%s.%s", c.accountName, service, c.baseURL)
+	}
+
+	return &url.URL{
+		Scheme: scheme,
+		Host:   host,
+	}
+}
+
+func (c Client) getEndpoint(service, path string, params url.Values) string {
+	u := c.getBaseURL(service)
+
+	// API doesn't accept path segments not starting with '/'
+	if !strings.HasPrefix(path, "/") {
+		path = fmt.Sprintf("/%v", path)
+	}
+
+	if c.accountName == StorageEmulatorAccountName {
+		path = fmt.Sprintf("/%v%v", StorageEmulatorAccountName, path)
+	}
+
+	u.Path = path
+	u.RawQuery = params.Encode()
+	return u.String()
+}
+
+// AccountSASTokenOptions includes options for constructing
+// an account SAS token.
+// https://docs.microsoft.com/en-us/rest/api/storageservices/constructing-an-account-sas
+type AccountSASTokenOptions struct {
+	APIVersion    string
+	Services      Services
+	ResourceTypes ResourceTypes
+	Permissions   Permissions
+	Start         time.Time
+	Expiry        time.Time
+	IP            string
+	UseHTTPS      bool
+}
+
+// Services specify services accessible with an account SAS.
+type Services struct {
+	Blob  bool
+	Queue bool
+	Table bool
+	File  bool
+}
+
+// ResourceTypes specify the resources accesible with an
+// account SAS.
+type ResourceTypes struct {
+	Service   bool
+	Container bool
+	Object    bool
+}
+
+// Permissions specifies permissions for an accountSAS.
+type Permissions struct {
+	Read    bool
+	Write   bool
+	Delete  bool
+	List    bool
+	Add     bool
+	Create  bool
+	Update  bool
+	Process bool
+}
+
+// GetAccountSASToken creates an account SAS token
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/constructing-an-account-sas
+func (c Client) GetAccountSASToken(options AccountSASTokenOptions) (url.Values, error) {
+	if options.APIVersion == "" {
+		options.APIVersion = c.apiVersion
+	}
+
+	if options.APIVersion < "2015-04-05" {
+		return url.Values{}, fmt.Errorf("account SAS does not support API versions prior to 2015-04-05. API version : %s", options.APIVersion)
+	}
+
+	// build services string
+	services := ""
+	if options.Services.Blob {
+		services += "b"
+	}
+	if options.Services.Queue {
+		services += "q"
+	}
+	if options.Services.Table {
+		services += "t"
+	}
+	if options.Services.File {
+		services += "f"
+	}
+
+	// build resources string
+	resources := ""
+	if options.ResourceTypes.Service {
+		resources += "s"
+	}
+	if options.ResourceTypes.Container {
+		resources += "c"
+	}
+	if options.ResourceTypes.Object {
+		resources += "o"
+	}
+
+	// build permissions string
+	permissions := ""
+	if options.Permissions.Read {
+		permissions += "r"
+	}
+	if options.Permissions.Write {
+		permissions += "w"
+	}
+	if options.Permissions.Delete {
+		permissions += "d"
+	}
+	if options.Permissions.List {
+		permissions += "l"
+	}
+	if options.Permissions.Add {
+		permissions += "a"
+	}
+	if options.Permissions.Create {
+		permissions += "c"
+	}
+	if options.Permissions.Update {
+		permissions += "u"
+	}
+	if options.Permissions.Process {
+		permissions += "p"
+	}
+
+	// build start time, if exists
+	start := ""
+	if options.Start != (time.Time{}) {
+		start = options.Start.UTC().Format(time.RFC3339)
+	}
+
+	// build expiry time
+	expiry := options.Expiry.UTC().Format(time.RFC3339)
+
+	protocol := "https,http"
+	if options.UseHTTPS {
+		protocol = "https"
+	}
+
+	stringToSign := strings.Join([]string{
+		c.accountName,
+		permissions,
+		services,
+		resources,
+		start,
+		expiry,
+		options.IP,
+		protocol,
+		options.APIVersion,
+		"",
+	}, "\n")
+	signature := c.computeHmac256(stringToSign)
+
+	sasParams := url.Values{
+		"sv":  {options.APIVersion},
+		"ss":  {services},
+		"srt": {resources},
+		"sp":  {permissions},
+		"se":  {expiry},
+		"spr": {protocol},
+		"sig": {signature},
+	}
+	if start != "" {
+		sasParams.Add("st", start)
+	}
+	if options.IP != "" {
+		sasParams.Add("sip", options.IP)
+	}
+
+	return sasParams, nil
+}
+
+// GetBlobService returns a BlobStorageClient which can operate on the blob
+// service of the storage account.
+func (c Client) GetBlobService() BlobStorageClient {
+	b := BlobStorageClient{
+		client: c,
+	}
+	b.client.AddToUserAgent(blobServiceName)
+	b.auth = sharedKey
+	if c.UseSharedKeyLite {
+		b.auth = sharedKeyLite
+	}
+	return b
+}
+
+// GetQueueService returns a QueueServiceClient which can operate on the queue
+// service of the storage account.
+func (c Client) GetQueueService() QueueServiceClient {
+	q := QueueServiceClient{
+		client: c,
+	}
+	q.client.AddToUserAgent(queueServiceName)
+	q.auth = sharedKey
+	if c.UseSharedKeyLite {
+		q.auth = sharedKeyLite
+	}
+	return q
+}
+
+// GetTableService returns a TableServiceClient which can operate on the table
+// service of the storage account.
+func (c Client) GetTableService() TableServiceClient {
+	t := TableServiceClient{
+		client: c,
+	}
+	t.client.AddToUserAgent(tableServiceName)
+	t.auth = sharedKeyForTable
+	if c.UseSharedKeyLite {
+		t.auth = sharedKeyLiteForTable
+	}
+	return t
+}
+
+// GetFileService returns a FileServiceClient which can operate on the file
+// service of the storage account.
+func (c Client) GetFileService() FileServiceClient {
+	f := FileServiceClient{
+		client: c,
+	}
+	f.client.AddToUserAgent(fileServiceName)
+	f.auth = sharedKey
+	if c.UseSharedKeyLite {
+		f.auth = sharedKeyLite
+	}
+	return f
+}
+
+func (c Client) getStandardHeaders() map[string]string {
+	return map[string]string{
+		userAgentHeader: c.userAgent,
+		"x-ms-version":  c.apiVersion,
+		"x-ms-date":     currentTimeRfc1123Formatted(),
+	}
+}
+
+func (c Client) exec(verb, url string, headers map[string]string, body io.Reader, auth authentication) (*http.Response, error) {
+	headers, err := c.addAuthorizationHeader(verb, url, headers, auth)
+	if err != nil {
+		return nil, err
+	}
+
+	req, err := http.NewRequest(verb, url, body)
+	if err != nil {
+		return nil, errors.New("azure/storage: error creating request: " + err.Error())
+	}
+
+	// http.NewRequest() will automatically set req.ContentLength for a handful of types
+	// otherwise we will handle here.
+	if req.ContentLength < 1 {
+		if clstr, ok := headers["Content-Length"]; ok {
+			if cl, err := strconv.ParseInt(clstr, 10, 64); err == nil {
+				req.ContentLength = cl
+			}
+		}
+	}
+
+	for k, v := range headers {
+		req.Header[k] = append(req.Header[k], v) // Must bypass case munging present in `Add` by using map functions directly. See https://github.com/Azure/azure-sdk-for-go/issues/645
+	}
+
+	if c.isAccountSASClient() {
+		// append the SAS token to the query params
+		v := req.URL.Query()
+		v = mergeParams(v, c.accountSASToken)
+		req.URL.RawQuery = v.Encode()
+	}
+
+	resp, err := c.Sender.Send(&c, req)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode >= 400 && resp.StatusCode <= 505 {
+		return resp, getErrorFromResponse(resp)
+	}
+
+	return resp, nil
+}
+
+func (c Client) execInternalJSONCommon(verb, url string, headers map[string]string, body io.Reader, auth authentication) (*odataResponse, *http.Request, *http.Response, error) {
+	headers, err := c.addAuthorizationHeader(verb, url, headers, auth)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	req, err := http.NewRequest(verb, url, body)
+	for k, v := range headers {
+		req.Header.Add(k, v)
+	}
+
+	resp, err := c.Sender.Send(&c, req)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	respToRet := &odataResponse{resp: resp}
+
+	statusCode := resp.StatusCode
+	if statusCode >= 400 && statusCode <= 505 {
+		var respBody []byte
+		respBody, err = readAndCloseBody(resp.Body)
+		if err != nil {
+			return nil, nil, nil, err
+		}
+
+		requestID, date, version := getDebugHeaders(resp.Header)
+		if len(respBody) == 0 {
+			// no error in response body, might happen in HEAD requests
+			err = serviceErrFromStatusCode(resp.StatusCode, resp.Status, requestID, date, version)
+			return respToRet, req, resp, err
+		}
+		// try unmarshal as odata.error json
+		err = json.Unmarshal(respBody, &respToRet.odata)
+	}
+
+	return respToRet, req, resp, err
+}
+
+func (c Client) execInternalJSON(verb, url string, headers map[string]string, body io.Reader, auth authentication) (*odataResponse, error) {
+	respToRet, _, _, err := c.execInternalJSONCommon(verb, url, headers, body, auth)
+	return respToRet, err
+}
+
+func (c Client) execBatchOperationJSON(verb, url string, headers map[string]string, body io.Reader, auth authentication) (*odataResponse, error) {
+	// execute common query, get back generated request, response etc... for more processing.
+	respToRet, req, resp, err := c.execInternalJSONCommon(verb, url, headers, body, auth)
+	if err != nil {
+		return nil, err
+	}
+
+	// return the OData in the case of executing batch commands.
+	// In this case we need to read the outer batch boundary and contents.
+	// Then we read the changeset information within the batch
+	var respBody []byte
+	respBody, err = readAndCloseBody(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	// outer multipart body
+	_, batchHeader, err := mime.ParseMediaType(resp.Header["Content-Type"][0])
+	if err != nil {
+		return nil, err
+	}
+
+	// batch details.
+	batchBoundary := batchHeader["boundary"]
+	batchPartBuf, changesetBoundary, err := genBatchReader(batchBoundary, respBody)
+	if err != nil {
+		return nil, err
+	}
+
+	// changeset details.
+	err = genChangesetReader(req, respToRet, batchPartBuf, changesetBoundary)
+	if err != nil {
+		return nil, err
+	}
+
+	return respToRet, nil
+}
+
+func genChangesetReader(req *http.Request, respToRet *odataResponse, batchPartBuf io.Reader, changesetBoundary string) error {
+	changesetMultiReader := multipart.NewReader(batchPartBuf, changesetBoundary)
+	changesetPart, err := changesetMultiReader.NextPart()
+	if err != nil {
+		return err
+	}
+
+	changesetPartBufioReader := bufio.NewReader(changesetPart)
+	changesetResp, err := http.ReadResponse(changesetPartBufioReader, req)
+	if err != nil {
+		return err
+	}
+
+	if changesetResp.StatusCode != http.StatusNoContent {
+		changesetBody, err := readAndCloseBody(changesetResp.Body)
+		err = json.Unmarshal(changesetBody, &respToRet.odata)
+		if err != nil {
+			return err
+		}
+		respToRet.resp = changesetResp
+	}
+
+	return nil
+}
+
+func genBatchReader(batchBoundary string, respBody []byte) (io.Reader, string, error) {
+	respBodyString := string(respBody)
+	respBodyReader := strings.NewReader(respBodyString)
+
+	// reading batchresponse
+	batchMultiReader := multipart.NewReader(respBodyReader, batchBoundary)
+	batchPart, err := batchMultiReader.NextPart()
+	if err != nil {
+		return nil, "", err
+	}
+	batchPartBufioReader := bufio.NewReader(batchPart)
+
+	_, changesetHeader, err := mime.ParseMediaType(batchPart.Header.Get("Content-Type"))
+	if err != nil {
+		return nil, "", err
+	}
+	changesetBoundary := changesetHeader["boundary"]
+	return batchPartBufioReader, changesetBoundary, nil
+}
+
+func readAndCloseBody(body io.ReadCloser) ([]byte, error) {
+	defer body.Close()
+	out, err := ioutil.ReadAll(body)
+	if err == io.EOF {
+		err = nil
+	}
+	return out, err
+}
+
+// reads the response body then closes it
+func drainRespBody(resp *http.Response) {
+	io.Copy(ioutil.Discard, resp.Body)
+	resp.Body.Close()
+}
+
+func serviceErrFromXML(body []byte, storageErr *AzureStorageServiceError) error {
+	if err := xml.Unmarshal(body, storageErr); err != nil {
+		storageErr.Message = fmt.Sprintf("Response body could no be unmarshaled: %v. Body: %v.", err, string(body))
+		return err
+	}
+	return nil
+}
+
+func serviceErrFromJSON(body []byte, storageErr *AzureStorageServiceError) error {
+	odataError := odataErrorWrapper{}
+	if err := json.Unmarshal(body, &odataError); err != nil {
+		storageErr.Message = fmt.Sprintf("Response body could no be unmarshaled: %v. Body: %v.", err, string(body))
+		return err
+	}
+	storageErr.Code = odataError.Err.Code
+	storageErr.Message = odataError.Err.Message.Value
+	storageErr.Lang = odataError.Err.Message.Lang
+	return nil
+}
+
+func serviceErrFromStatusCode(code int, status string, requestID, date, version string) AzureStorageServiceError {
+	return AzureStorageServiceError{
+		StatusCode: code,
+		Code:       status,
+		RequestID:  requestID,
+		Date:       date,
+		APIVersion: version,
+		Message:    "no response body was available for error status code",
+	}
+}
+
+func (e AzureStorageServiceError) Error() string {
+	return fmt.Sprintf("storage: service returned error: StatusCode=%d, ErrorCode=%s, ErrorMessage=%s, RequestInitiated=%s, RequestId=%s, API Version=%s, QueryParameterName=%s, QueryParameterValue=%s",
+		e.StatusCode, e.Code, e.Message, e.Date, e.RequestID, e.APIVersion, e.QueryParameterName, e.QueryParameterValue)
+}
+
+// checkRespCode returns UnexpectedStatusError if the given response code is not
+// one of the allowed status codes; otherwise nil.
+func checkRespCode(resp *http.Response, allowed []int) error {
+	for _, v := range allowed {
+		if resp.StatusCode == v {
+			return nil
+		}
+	}
+	err := getErrorFromResponse(resp)
+	return UnexpectedStatusCodeError{
+		allowed: allowed,
+		got:     resp.StatusCode,
+		inner:   err,
+	}
+}
+
+func (c Client) addMetadataToHeaders(h map[string]string, metadata map[string]string) map[string]string {
+	metadata = c.protectUserAgent(metadata)
+	for k, v := range metadata {
+		h[userDefinedMetadataHeaderPrefix+k] = v
+	}
+	return h
+}
+
+func getDebugHeaders(h http.Header) (requestID, date, version string) {
+	requestID = h.Get("x-ms-request-id")
+	version = h.Get("x-ms-version")
+	date = h.Get("Date")
+	return
+}
+
+func getErrorFromResponse(resp *http.Response) error {
+	respBody, err := readAndCloseBody(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	requestID, date, version := getDebugHeaders(resp.Header)
+	if len(respBody) == 0 {
+		// no error in response body, might happen in HEAD requests
+		err = serviceErrFromStatusCode(resp.StatusCode, resp.Status, requestID, date, version)
+	} else {
+		storageErr := AzureStorageServiceError{
+			StatusCode: resp.StatusCode,
+			RequestID:  requestID,
+			Date:       date,
+			APIVersion: version,
+		}
+		// response contains storage service error object, unmarshal
+		if resp.Header.Get("Content-Type") == "application/xml" {
+			errIn := serviceErrFromXML(respBody, &storageErr)
+			if err != nil { // error unmarshaling the error response
+				err = errIn
+			}
+		} else {
+			errIn := serviceErrFromJSON(respBody, &storageErr)
+			if err != nil { // error unmarshaling the error response
+				err = errIn
+			}
+		}
+		err = storageErr
+	}
+	return err
+}

vendor/github.com/Azure/azure-sdk-for-go/storage/commonsasuri.go

@@ -0,0 +1,38 @@
+package storage
+
+// Copyright 2017 Microsoft Corporation
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+import (
+	"net/url"
+	"time"
+)
+
+// SASOptions includes options used by SAS URIs for different
+// services and resources.
+type SASOptions struct {
+	APIVersion string
+	Start      time.Time
+	Expiry     time.Time
+	IP         string
+	UseHTTPS   bool
+	Identifier string
+}
+
+func addQueryParameter(query url.Values, key, value string) url.Values {
+	if value != "" {
+		query.Add(key, value)
+	}
+	return query
+}

vendor/github.com/Azure/azure-sdk-for-go/storage/container.go

@@ -0,0 +1,640 @@
+package storage
+
+// Copyright 2017 Microsoft Corporation
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+import (
+	"encoding/xml"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// Container represents an Azure container.
+type Container struct {
+	bsc        *BlobStorageClient
+	Name       string              `xml:"Name"`
+	Properties ContainerProperties `xml:"Properties"`
+	Metadata   map[string]string
+	sasuri     url.URL
+}
+
+// Client returns the HTTP client used by the Container reference.
+func (c *Container) Client() *Client {
+	return &c.bsc.client
+}
+
+func (c *Container) buildPath() string {
+	return fmt.Sprintf("/%s", c.Name)
+}
+
+// GetURL gets the canonical URL to the container.
+// This method does not create a publicly accessible URL if the container
+// is private and this method does not check if the blob exists.
+func (c *Container) GetURL() string {
+	container := c.Name
+	if container == "" {
+		container = "$root"
+	}
+	return c.bsc.client.getEndpoint(blobServiceName, pathForResource(container, ""), nil)
+}
+
+// ContainerSASOptions are options to construct a container SAS
+// URI.
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/constructing-a-service-sas
+type ContainerSASOptions struct {
+	ContainerSASPermissions
+	OverrideHeaders
+	SASOptions
+}
+
+// ContainerSASPermissions includes the available permissions for
+// a container SAS URI.
+type ContainerSASPermissions struct {
+	BlobServiceSASPermissions
+	List bool
+}
+
+// GetSASURI creates an URL to the container which contains the Shared
+// Access Signature with the specified options.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/constructing-a-service-sas
+func (c *Container) GetSASURI(options ContainerSASOptions) (string, error) {
+	uri := c.GetURL()
+	signedResource := "c"
+	canonicalizedResource, err := c.bsc.client.buildCanonicalizedResource(uri, c.bsc.auth, true)
+	if err != nil {
+		return "", err
+	}
+
+	// build permissions string
+	permissions := options.BlobServiceSASPermissions.buildString()
+	if options.List {
+		permissions += "l"
+	}
+
+	return c.bsc.client.blobAndFileSASURI(options.SASOptions, uri, permissions, canonicalizedResource, signedResource, options.OverrideHeaders)
+}
+
+// ContainerProperties contains various properties of a container returned from
+// various endpoints like ListContainers.
+type ContainerProperties struct {
+	LastModified  string              `xml:"Last-Modified"`
+	Etag          string              `xml:"Etag"`
+	LeaseStatus   string              `xml:"LeaseStatus"`
+	LeaseState    string              `xml:"LeaseState"`
+	LeaseDuration string              `xml:"LeaseDuration"`
+	PublicAccess  ContainerAccessType `xml:"PublicAccess"`
+}
+
+// ContainerListResponse contains the response fields from
+// ListContainers call.
+//
+// See https://msdn.microsoft.com/en-us/library/azure/dd179352.aspx
+type ContainerListResponse struct {
+	XMLName    xml.Name    `xml:"EnumerationResults"`
+	Xmlns      string      `xml:"xmlns,attr"`
+	Prefix     string      `xml:"Prefix"`
+	Marker     string      `xml:"Marker"`
+	NextMarker string      `xml:"NextMarker"`
+	MaxResults int64       `xml:"MaxResults"`
+	Containers []Container `xml:"Containers>Container"`
+}
+
+// BlobListResponse contains the response fields from ListBlobs call.
+//
+// See https://msdn.microsoft.com/en-us/library/azure/dd135734.aspx
+type BlobListResponse struct {
+	XMLName    xml.Name `xml:"EnumerationResults"`
+	Xmlns      string   `xml:"xmlns,attr"`
+	Prefix     string   `xml:"Prefix"`
+	Marker     string   `xml:"Marker"`
+	NextMarker string   `xml:"NextMarker"`
+	MaxResults int64    `xml:"MaxResults"`
+	Blobs      []Blob   `xml:"Blobs>Blob"`
+
+	// BlobPrefix is used to traverse blobs as if it were a file system.
+	// It is returned if ListBlobsParameters.Delimiter is specified.
+	// The list here can be thought of as "folders" that may contain
+	// other folders or blobs.
+	BlobPrefixes []string `xml:"Blobs>BlobPrefix>Name"`
+
+	// Delimiter is used to traverse blobs as if it were a file system.
+	// It is returned if ListBlobsParameters.Delimiter is specified.
+	Delimiter string `xml:"Delimiter"`
+}
+
+// IncludeBlobDataset has options to include in a list blobs operation
+type IncludeBlobDataset struct {
+	Snapshots        bool
+	Metadata         bool
+	UncommittedBlobs bool
+	Copy             bool
+}
+
+// ListBlobsParameters defines the set of customizable
+// parameters to make a List Blobs call.
+//
+// See https://msdn.microsoft.com/en-us/library/azure/dd135734.aspx
+type ListBlobsParameters struct {
+	Prefix     string
+	Delimiter  string
+	Marker     string
+	Include    *IncludeBlobDataset
+	MaxResults uint
+	Timeout    uint
+	RequestID  string
+}
+
+func (p ListBlobsParameters) getParameters() url.Values {
+	out := url.Values{}
+
+	if p.Prefix != "" {
+		out.Set("prefix", p.Prefix)
+	}
+	if p.Delimiter != "" {
+		out.Set("delimiter", p.Delimiter)
+	}
+	if p.Marker != "" {
+		out.Set("marker", p.Marker)
+	}
+	if p.Include != nil {
+		include := []string{}
+		include = addString(include, p.Include.Snapshots, "snapshots")
+		include = addString(include, p.Include.Metadata, "metadata")
+		include = addString(include, p.Include.UncommittedBlobs, "uncommittedblobs")
+		include = addString(include, p.Include.Copy, "copy")
+		fullInclude := strings.Join(include, ",")
+		out.Set("include", fullInclude)
+	}
+	if p.MaxResults != 0 {
+		out.Set("maxresults", strconv.FormatUint(uint64(p.MaxResults), 10))
+	}
+	if p.Timeout != 0 {
+		out.Set("timeout", strconv.FormatUint(uint64(p.Timeout), 10))
+	}
+
+	return out
+}
+
+func addString(datasets []string, include bool, text string) []string {
+	if include {
+		datasets = append(datasets, text)
+	}
+	return datasets
+}
+
+// ContainerAccessType defines the access level to the container from a public
+// request.
+//
+// See https://msdn.microsoft.com/en-us/library/azure/dd179468.aspx and "x-ms-
+// blob-public-access" header.
+type ContainerAccessType string
+
+// Access options for containers
+const (
+	ContainerAccessTypePrivate   ContainerAccessType = ""
+	ContainerAccessTypeBlob      ContainerAccessType = "blob"
+	ContainerAccessTypeContainer ContainerAccessType = "container"
+)
+
+// ContainerAccessPolicy represents each access policy in the container ACL.
+type ContainerAccessPolicy struct {
+	ID         string
+	StartTime  time.Time
+	ExpiryTime time.Time
+	CanRead    bool
+	CanWrite   bool
+	CanDelete  bool
+}
+
+// ContainerPermissions represents the container ACLs.
+type ContainerPermissions struct {
+	AccessType     ContainerAccessType
+	AccessPolicies []ContainerAccessPolicy
+}
+
+// ContainerAccessHeader references header used when setting/getting container ACL
+const (
+	ContainerAccessHeader string = "x-ms-blob-public-access"
+)
+
+// GetBlobReference returns a Blob object for the specified blob name.
+func (c *Container) GetBlobReference(name string) *Blob {
+	return &Blob{
+		Container: c,
+		Name:      name,
+	}
+}
+
+// CreateContainerOptions includes the options for a create container operation
+type CreateContainerOptions struct {
+	Timeout   uint
+	Access    ContainerAccessType `header:"x-ms-blob-public-access"`
+	RequestID string              `header:"x-ms-client-request-id"`
+}
+
+// Create creates a blob container within the storage account
+// with given name and access level. Returns error if container already exists.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Create-Container
+func (c *Container) Create(options *CreateContainerOptions) error {
+	resp, err := c.create(options)
+	if err != nil {
+		return err
+	}
+	defer drainRespBody(resp)
+	return checkRespCode(resp, []int{http.StatusCreated})
+}
+
+// CreateIfNotExists creates a blob container if it does not exist. Returns
+// true if container is newly created or false if container already exists.
+func (c *Container) CreateIfNotExists(options *CreateContainerOptions) (bool, error) {
+	resp, err := c.create(options)
+	if resp != nil {
+		defer drainRespBody(resp)
+		if resp.StatusCode == http.StatusCreated || resp.StatusCode == http.StatusConflict {
+			return resp.StatusCode == http.StatusCreated, nil
+		}
+	}
+	return false, err
+}
+
+func (c *Container) create(options *CreateContainerOptions) (*http.Response, error) {
+	query := url.Values{"restype": {"container"}}
+	headers := c.bsc.client.getStandardHeaders()
+	headers = c.bsc.client.addMetadataToHeaders(headers, c.Metadata)
+
+	if options != nil {
+		query = addTimeout(query, options.Timeout)
+		headers = mergeHeaders(headers, headersFromStruct(*options))
+	}
+	uri := c.bsc.client.getEndpoint(blobServiceName, c.buildPath(), query)
+
+	return c.bsc.client.exec(http.MethodPut, uri, headers, nil, c.bsc.auth)
+}
+
+// Exists returns true if a container with given name exists
+// on the storage account, otherwise returns false.
+func (c *Container) Exists() (bool, error) {
+	q := url.Values{"restype": {"container"}}
+	var uri string
+	if c.bsc.client.isServiceSASClient() {
+		q = mergeParams(q, c.sasuri.Query())
+		newURI := c.sasuri
+		newURI.RawQuery = q.Encode()
+		uri = newURI.String()
+
+	} else {
+		uri = c.bsc.client.getEndpoint(blobServiceName, c.buildPath(), q)
+	}
+	headers := c.bsc.client.getStandardHeaders()
+
+	resp, err := c.bsc.client.exec(http.MethodHead, uri, headers, nil, c.bsc.auth)
+	if resp != nil {
+		defer drainRespBody(resp)
+		if resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusNotFound {
+			return resp.StatusCode == http.StatusOK, nil
+		}
+	}
+	return false, err
+}
+
+// SetContainerPermissionOptions includes options for a set container permissions operation
+type SetContainerPermissionOptions struct {
+	Timeout           uint
+	LeaseID           string     `header:"x-ms-lease-id"`
+	IfModifiedSince   *time.Time `header:"If-Modified-Since"`
+	IfUnmodifiedSince *time.Time `header:"If-Unmodified-Since"`
+	RequestID         string     `header:"x-ms-client-request-id"`
+}
+
+// SetPermissions sets up container permissions
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Set-Container-ACL
+func (c *Container) SetPermissions(permissions ContainerPermissions, options *SetContainerPermissionOptions) error {
+	body, length, err := generateContainerACLpayload(permissions.AccessPolicies)
+	if err != nil {
+		return err
+	}
+	params := url.Values{
+		"restype": {"container"},
+		"comp":    {"acl"},
+	}
+	headers := c.bsc.client.getStandardHeaders()
+	headers = addToHeaders(headers, ContainerAccessHeader, string(permissions.AccessType))
+	headers["Content-Length"] = strconv.Itoa(length)
+
+	if options != nil {
+		params = addTimeout(params, options.Timeout)
+		headers = mergeHeaders(headers, headersFromStruct(*options))
+	}
+	uri := c.bsc.client.getEndpoint(blobServiceName, c.buildPath(), params)
+
+	resp, err := c.bsc.client.exec(http.MethodPut, uri, headers, body, c.bsc.auth)
+	if err != nil {
+		return err
+	}
+	defer drainRespBody(resp)
+	return checkRespCode(resp, []int{http.StatusOK})
+}
+
+// GetContainerPermissionOptions includes options for a get container permissions operation
+type GetContainerPermissionOptions struct {
+	Timeout   uint
+	LeaseID   string `header:"x-ms-lease-id"`
+	RequestID string `header:"x-ms-client-request-id"`
+}
+
+// GetPermissions gets the container permissions as per https://msdn.microsoft.com/en-us/library/azure/dd179469.aspx
+// If timeout is 0 then it will not be passed to Azure
+// leaseID will only be passed to Azure if populated
+func (c *Container) GetPermissions(options *GetContainerPermissionOptions) (*ContainerPermissions, error) {
+	params := url.Values{
+		"restype": {"container"},
+		"comp":    {"acl"},
+	}
+	headers := c.bsc.client.getStandardHeaders()
+
+	if options != nil {
+		params = addTimeout(params, options.Timeout)
+		headers = mergeHeaders(headers, headersFromStruct(*options))
+	}
+	uri := c.bsc.client.getEndpoint(blobServiceName, c.buildPath(), params)
+
+	resp, err := c.bsc.client.exec(http.MethodGet, uri, headers, nil, c.bsc.auth)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	var ap AccessPolicy
+	err = xmlUnmarshal(resp.Body, &ap.SignedIdentifiersList)
+	if err != nil {
+		return nil, err
+	}
+	return buildAccessPolicy(ap, &resp.Header), nil
+}
+
+func buildAccessPolicy(ap AccessPolicy, headers *http.Header) *ContainerPermissions {
+	// containerAccess. Blob, Container, empty
+	containerAccess := headers.Get(http.CanonicalHeaderKey(ContainerAccessHeader))
+	permissions := ContainerPermissions{
+		AccessType:     ContainerAccessType(containerAccess),
+		AccessPolicies: []ContainerAccessPolicy{},
+	}
+
+	for _, policy := range ap.SignedIdentifiersList.SignedIdentifiers {
+		capd := ContainerAccessPolicy{
+			ID:         policy.ID,
+			StartTime:  policy.AccessPolicy.StartTime,
+			ExpiryTime: policy.AccessPolicy.ExpiryTime,
+		}
+		capd.CanRead = updatePermissions(policy.AccessPolicy.Permission, "r")
+		capd.CanWrite = updatePermissions(policy.AccessPolicy.Permission, "w")
+		capd.CanDelete = updatePermissions(policy.AccessPolicy.Permission, "d")
+
+		permissions.AccessPolicies = append(permissions.AccessPolicies, capd)
+	}
+	return &permissions
+}
+
+// DeleteContainerOptions includes options for a delete container operation
+type DeleteContainerOptions struct {
+	Timeout           uint
+	LeaseID           string     `header:"x-ms-lease-id"`
+	IfModifiedSince   *time.Time `header:"If-Modified-Since"`
+	IfUnmodifiedSince *time.Time `header:"If-Unmodified-Since"`
+	RequestID         string     `header:"x-ms-client-request-id"`
+}
+
+// Delete deletes the container with given name on the storage
+// account. If the container does not exist returns error.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/delete-container
+func (c *Container) Delete(options *DeleteContainerOptions) error {
+	resp, err := c.delete(options)
+	if err != nil {
+		return err
+	}
+	defer drainRespBody(resp)
+	return checkRespCode(resp, []int{http.StatusAccepted})
+}
+
+// DeleteIfExists deletes the container with given name on the storage
+// account if it exists. Returns true if container is deleted with this call, or
+// false if the container did not exist at the time of the Delete Container
+// operation.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/delete-container
+func (c *Container) DeleteIfExists(options *DeleteContainerOptions) (bool, error) {
+	resp, err := c.delete(options)
+	if resp != nil {
+		defer drainRespBody(resp)
+		if resp.StatusCode == http.StatusAccepted || resp.StatusCode == http.StatusNotFound {
+			return resp.StatusCode == http.StatusAccepted, nil
+		}
+	}
+	return false, err
+}
+
+func (c *Container) delete(options *DeleteContainerOptions) (*http.Response, error) {
+	query := url.Values{"restype": {"container"}}
+	headers := c.bsc.client.getStandardHeaders()
+
+	if options != nil {
+		query = addTimeout(query, options.Timeout)
+		headers = mergeHeaders(headers, headersFromStruct(*options))
+	}
+	uri := c.bsc.client.getEndpoint(blobServiceName, c.buildPath(), query)
+
+	return c.bsc.client.exec(http.MethodDelete, uri, headers, nil, c.bsc.auth)
+}
+
+// ListBlobs returns an object that contains list of blobs in the container,
+// pagination token and other information in the response of List Blobs call.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/List-Blobs
+func (c *Container) ListBlobs(params ListBlobsParameters) (BlobListResponse, error) {
+	q := mergeParams(params.getParameters(), url.Values{
+		"restype": {"container"},
+		"comp":    {"list"},
+	})
+	var uri string
+	if c.bsc.client.isServiceSASClient() {
+		q = mergeParams(q, c.sasuri.Query())
+		newURI := c.sasuri
+		newURI.RawQuery = q.Encode()
+		uri = newURI.String()
+	} else {
+		uri = c.bsc.client.getEndpoint(blobServiceName, c.buildPath(), q)
+	}
+
+	headers := c.bsc.client.getStandardHeaders()
+	headers = addToHeaders(headers, "x-ms-client-request-id", params.RequestID)
+
+	var out BlobListResponse
+	resp, err := c.bsc.client.exec(http.MethodGet, uri, headers, nil, c.bsc.auth)
+	if err != nil {
+		return out, err
+	}
+	defer resp.Body.Close()
+
+	err = xmlUnmarshal(resp.Body, &out)
+	for i := range out.Blobs {
+		out.Blobs[i].Container = c
+	}
+	return out, err
+}
+
+// ContainerMetadataOptions includes options for container metadata operations
+type ContainerMetadataOptions struct {
+	Timeout   uint
+	LeaseID   string `header:"x-ms-lease-id"`
+	RequestID string `header:"x-ms-client-request-id"`
+}
+
+// SetMetadata replaces the metadata for the specified container.
+//
+// Some keys may be converted to Camel-Case before sending. All keys
+// are returned in lower case by GetBlobMetadata. HTTP header names
+// are case-insensitive so case munging should not matter to other
+// applications either.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/set-container-metadata
+func (c *Container) SetMetadata(options *ContainerMetadataOptions) error {
+	params := url.Values{
+		"comp":    {"metadata"},
+		"restype": {"container"},
+	}
+	headers := c.bsc.client.getStandardHeaders()
+	headers = c.bsc.client.addMetadataToHeaders(headers, c.Metadata)
+
+	if options != nil {
+		params = addTimeout(params, options.Timeout)
+		headers = mergeHeaders(headers, headersFromStruct(*options))
+	}
+
+	uri := c.bsc.client.getEndpoint(blobServiceName, c.buildPath(), params)
+
+	resp, err := c.bsc.client.exec(http.MethodPut, uri, headers, nil, c.bsc.auth)
+	if err != nil {
+		return err
+	}
+	defer drainRespBody(resp)
+	return checkRespCode(resp, []int{http.StatusOK})
+}
+
+// GetMetadata returns all user-defined metadata for the specified container.
+//
+// All metadata keys will be returned in lower case. (HTTP header
+// names are case-insensitive.)
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/get-container-metadata
+func (c *Container) GetMetadata(options *ContainerMetadataOptions) error {
+	params := url.Values{
+		"comp":    {"metadata"},
+		"restype": {"container"},
+	}
+	headers := c.bsc.client.getStandardHeaders()
+
+	if options != nil {
+		params = addTimeout(params, options.Timeout)
+		headers = mergeHeaders(headers, headersFromStruct(*options))
+	}
+
+	uri := c.bsc.client.getEndpoint(blobServiceName, c.buildPath(), params)
+
+	resp, err := c.bsc.client.exec(http.MethodGet, uri, headers, nil, c.bsc.auth)
+	if err != nil {
+		return err
+	}
+	defer drainRespBody(resp)
+	if err := checkRespCode(resp, []int{http.StatusOK}); err != nil {
+		return err
+	}
+
+	c.writeMetadata(resp.Header)
+	return nil
+}
+
+func (c *Container) writeMetadata(h http.Header) {
+	c.Metadata = writeMetadata(h)
+}
+
+func generateContainerACLpayload(policies []ContainerAccessPolicy) (io.Reader, int, error) {
+	sil := SignedIdentifiers{
+		SignedIdentifiers: []SignedIdentifier{},
+	}
+	for _, capd := range policies {
+		permission := capd.generateContainerPermissions()
+		signedIdentifier := convertAccessPolicyToXMLStructs(capd.ID, capd.StartTime, capd.ExpiryTime, permission)
+		sil.SignedIdentifiers = append(sil.SignedIdentifiers, signedIdentifier)
+	}
+	return xmlMarshal(sil)
+}
+
+func (capd *ContainerAccessPolicy) generateContainerPermissions() (permissions string) {
+	// generate the permissions string (rwd).
+	// still want the end user API to have bool flags.
+	permissions = ""
+
+	if capd.CanRead {
+		permissions += "r"
+	}
+
+	if capd.CanWrite {
+		permissions += "w"
+	}
+
+	if capd.CanDelete {
+		permissions += "d"
+	}
+
+	return permissions
+}
+
+// GetProperties updated the properties of the container.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/get-container-properties
+func (c *Container) GetProperties() error {
+	params := url.Values{
+		"restype": {"container"},
+	}
+	headers := c.bsc.client.getStandardHeaders()
+
+	uri := c.bsc.client.getEndpoint(blobServiceName, c.buildPath(), params)
+
+	resp, err := c.bsc.client.exec(http.MethodGet, uri, headers, nil, c.bsc.auth)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if err := checkRespCode(resp, []int{http.StatusOK}); err != nil {
+		return err
+	}
+
+	// update properties
+	c.Properties.Etag = resp.Header.Get(headerEtag)
+	c.Properties.LeaseStatus = resp.Header.Get("x-ms-lease-status")
+	c.Properties.LeaseState = resp.Header.Get("x-ms-lease-state")
+	c.Properties.LeaseDuration = resp.Header.Get("x-ms-lease-duration")
+	c.Properties.LastModified = resp.Header.Get("Last-Modified")
+	c.Properties.PublicAccess = ContainerAccessType(resp.Header.Get(ContainerAccessHeader))
+
+	return nil
+}

vendor/github.com/Azure/azure-sdk-for-go/storage/copyblob.go

@@ -0,0 +1,237 @@
+package storage
+
+// Copyright 2017 Microsoft Corporation
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+import (
+	"errors"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+)
+
+const (
+	blobCopyStatusPending = "pending"
+	blobCopyStatusSuccess = "success"
+	blobCopyStatusAborted = "aborted"
+	blobCopyStatusFailed  = "failed"
+)
+
+// CopyOptions includes the options for a copy blob operation
+type CopyOptions struct {
+	Timeout   uint
+	Source    CopyOptionsConditions
+	Destiny   CopyOptionsConditions
+	RequestID string
+}
+
+// IncrementalCopyOptions includes the options for an incremental copy blob operation
+type IncrementalCopyOptions struct {
+	Timeout     uint
+	Destination IncrementalCopyOptionsConditions
+	RequestID   string
+}
+
+// CopyOptionsConditions includes some conditional options in a copy blob operation
+type CopyOptionsConditions struct {
+	LeaseID           string
+	IfModifiedSince   *time.Time
+	IfUnmodifiedSince *time.Time
+	IfMatch           string
+	IfNoneMatch       string
+}
+
+// IncrementalCopyOptionsConditions includes some conditional options in a copy blob operation
+type IncrementalCopyOptionsConditions struct {
+	IfModifiedSince   *time.Time
+	IfUnmodifiedSince *time.Time
+	IfMatch           string
+	IfNoneMatch       string
+}
+
+// Copy starts a blob copy operation and waits for the operation to
+// complete. sourceBlob parameter must be a canonical URL to the blob (can be
+// obtained using the GetURL method.) There is no SLA on blob copy and therefore
+// this helper method works faster on smaller files.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Copy-Blob
+func (b *Blob) Copy(sourceBlob string, options *CopyOptions) error {
+	copyID, err := b.StartCopy(sourceBlob, options)
+	if err != nil {
+		return err
+	}
+
+	return b.WaitForCopy(copyID)
+}
+
+// StartCopy starts a blob copy operation.
+// sourceBlob parameter must be a canonical URL to the blob (can be
+// obtained using the GetURL method.)
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Copy-Blob
+func (b *Blob) StartCopy(sourceBlob string, options *CopyOptions) (string, error) {
+	params := url.Values{}
+	headers := b.Container.bsc.client.getStandardHeaders()
+	headers["x-ms-copy-source"] = sourceBlob
+	headers = b.Container.bsc.client.addMetadataToHeaders(headers, b.Metadata)
+
+	if options != nil {
+		params = addTimeout(params, options.Timeout)
+		headers = addToHeaders(headers, "x-ms-client-request-id", options.RequestID)
+		// source
+		headers = addToHeaders(headers, "x-ms-source-lease-id", options.Source.LeaseID)
+		headers = addTimeToHeaders(headers, "x-ms-source-if-modified-since", options.Source.IfModifiedSince)
+		headers = addTimeToHeaders(headers, "x-ms-source-if-unmodified-since", options.Source.IfUnmodifiedSince)
+		headers = addToHeaders(headers, "x-ms-source-if-match", options.Source.IfMatch)
+		headers = addToHeaders(headers, "x-ms-source-if-none-match", options.Source.IfNoneMatch)
+		//destiny
+		headers = addToHeaders(headers, "x-ms-lease-id", options.Destiny.LeaseID)
+		headers = addTimeToHeaders(headers, "x-ms-if-modified-since", options.Destiny.IfModifiedSince)
+		headers = addTimeToHeaders(headers, "x-ms-if-unmodified-since", options.Destiny.IfUnmodifiedSince)
+		headers = addToHeaders(headers, "x-ms-if-match", options.Destiny.IfMatch)
+		headers = addToHeaders(headers, "x-ms-if-none-match", options.Destiny.IfNoneMatch)
+	}
+	uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), params)
+
+	resp, err := b.Container.bsc.client.exec(http.MethodPut, uri, headers, nil, b.Container.bsc.auth)
+	if err != nil {
+		return "", err
+	}
+	defer drainRespBody(resp)
+
+	if err := checkRespCode(resp, []int{http.StatusAccepted, http.StatusCreated}); err != nil {
+		return "", err
+	}
+
+	copyID := resp.Header.Get("x-ms-copy-id")
+	if copyID == "" {
+		return "", errors.New("Got empty copy id header")
+	}
+	return copyID, nil
+}
+
+// AbortCopyOptions includes the options for an abort blob operation
+type AbortCopyOptions struct {
+	Timeout   uint
+	LeaseID   string `header:"x-ms-lease-id"`
+	RequestID string `header:"x-ms-client-request-id"`
+}
+
+// AbortCopy aborts a BlobCopy which has already been triggered by the StartBlobCopy function.
+// copyID is generated from StartBlobCopy function.
+// currentLeaseID is required IF the destination blob has an active lease on it.
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Abort-Copy-Blob
+func (b *Blob) AbortCopy(copyID string, options *AbortCopyOptions) error {
+	params := url.Values{
+		"comp":   {"copy"},
+		"copyid": {copyID},
+	}
+	headers := b.Container.bsc.client.getStandardHeaders()
+	headers["x-ms-copy-action"] = "abort"
+
+	if options != nil {
+		params = addTimeout(params, options.Timeout)
+		headers = mergeHeaders(headers, headersFromStruct(*options))
+	}
+	uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), params)
+
+	resp, err := b.Container.bsc.client.exec(http.MethodPut, uri, headers, nil, b.Container.bsc.auth)
+	if err != nil {
+		return err
+	}
+	defer drainRespBody(resp)
+	return checkRespCode(resp, []int{http.StatusNoContent})
+}
+
+// WaitForCopy loops until a BlobCopy operation is completed (or fails with error)
+func (b *Blob) WaitForCopy(copyID string) error {
+	for {
+		err := b.GetProperties(nil)
+		if err != nil {
+			return err
+		}
+
+		if b.Properties.CopyID != copyID {
+			return errBlobCopyIDMismatch
+		}
+
+		switch b.Properties.CopyStatus {
+		case blobCopyStatusSuccess:
+			return nil
+		case blobCopyStatusPending:
+			continue
+		case blobCopyStatusAborted:
+			return errBlobCopyAborted
+		case blobCopyStatusFailed:
+			return fmt.Errorf("storage: blob copy failed. Id=%s Description=%s", b.Properties.CopyID, b.Properties.CopyStatusDescription)
+		default:
+			return fmt.Errorf("storage: unhandled blob copy status: '%s'", b.Properties.CopyStatus)
+		}
+	}
+}
+
+// IncrementalCopyBlob copies a snapshot of a source blob and copies to referring blob
+// sourceBlob parameter must be a valid snapshot URL of the original blob.
+// THe original blob mut be public, or use a Shared Access Signature.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/incremental-copy-blob .
+func (b *Blob) IncrementalCopyBlob(sourceBlobURL string, snapshotTime time.Time, options *IncrementalCopyOptions) (string, error) {
+	params := url.Values{"comp": {"incrementalcopy"}}
+
+	// need formatting to 7 decimal places so it's friendly to Windows and *nix
+	snapshotTimeFormatted := snapshotTime.Format("2006-01-02T15:04:05.0000000Z")
+	u, err := url.Parse(sourceBlobURL)
+	if err != nil {
+		return "", err
+	}
+	query := u.Query()
+	query.Add("snapshot", snapshotTimeFormatted)
+	encodedQuery := query.Encode()
+	encodedQuery = strings.Replace(encodedQuery, "%3A", ":", -1)
+	u.RawQuery = encodedQuery
+	snapshotURL := u.String()
+
+	headers := b.Container.bsc.client.getStandardHeaders()
+	headers["x-ms-copy-source"] = snapshotURL
+
+	if options != nil {
+		addTimeout(params, options.Timeout)
+		headers = addToHeaders(headers, "x-ms-client-request-id", options.RequestID)
+		headers = addTimeToHeaders(headers, "x-ms-if-modified-since", options.Destination.IfModifiedSince)
+		headers = addTimeToHeaders(headers, "x-ms-if-unmodified-since", options.Destination.IfUnmodifiedSince)
+		headers = addToHeaders(headers, "x-ms-if-match", options.Destination.IfMatch)
+		headers = addToHeaders(headers, "x-ms-if-none-match", options.Destination.IfNoneMatch)
+	}
+
+	// get URI of destination blob
+	uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), params)
+
+	resp, err := b.Container.bsc.client.exec(http.MethodPut, uri, headers, nil, b.Container.bsc.auth)
+	if err != nil {
+		return "", err
+	}
+	defer drainRespBody(resp)
+
+	if err := checkRespCode(resp, []int{http.StatusAccepted}); err != nil {
+		return "", err
+	}
+
+	copyID := resp.Header.Get("x-ms-copy-id")
+	if copyID == "" {
+		return "", errors.New("Got empty copy id header")
+	}
+	return copyID, nil
+}

vendor/github.com/Azure/azure-sdk-for-go/storage/directory.go

@@ -0,0 +1,238 @@
+package storage
+
+// Copyright 2017 Microsoft Corporation
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+import (
+	"encoding/xml"
+	"net/http"
+	"net/url"
+	"sync"
+)
+
+// Directory represents a directory on a share.
+type Directory struct {
+	fsc        *FileServiceClient
+	Metadata   map[string]string
+	Name       string `xml:"Name"`
+	parent     *Directory
+	Properties DirectoryProperties
+	share      *Share
+}
+
+// DirectoryProperties contains various properties of a directory.
+type DirectoryProperties struct {
+	LastModified string `xml:"Last-Modified"`
+	Etag         string `xml:"Etag"`
+}
+
+// ListDirsAndFilesParameters defines the set of customizable parameters to
+// make a List Files and Directories call.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/List-Directories-and-Files
+type ListDirsAndFilesParameters struct {
+	Prefix     string
+	Marker     string
+	MaxResults uint
+	Timeout    uint
+}
+
+// DirsAndFilesListResponse contains the response fields from
+// a List Files and Directories call.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/List-Directories-and-Files
+type DirsAndFilesListResponse struct {
+	XMLName     xml.Name    `xml:"EnumerationResults"`
+	Xmlns       string      `xml:"xmlns,attr"`
+	Marker      string      `xml:"Marker"`
+	MaxResults  int64       `xml:"MaxResults"`
+	Directories []Directory `xml:"Entries>Directory"`
+	Files       []File      `xml:"Entries>File"`
+	NextMarker  string      `xml:"NextMarker"`
+}
+
+// builds the complete directory path for this directory object.
+func (d *Directory) buildPath() string {
+	path := ""
+	current := d
+	for current.Name != "" {
+		path = "/" + current.Name + path
+		current = current.parent
+	}
+	return d.share.buildPath() + path
+}
+
+// Create this directory in the associated share.
+// If a directory with the same name already exists, the operation fails.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Create-Directory
+func (d *Directory) Create(options *FileRequestOptions) error {
+	// if this is the root directory exit early
+	if d.parent == nil {
+		return nil
+	}
+
+	params := prepareOptions(options)
+	headers, err := d.fsc.createResource(d.buildPath(), resourceDirectory, params, mergeMDIntoExtraHeaders(d.Metadata, nil), []int{http.StatusCreated})
+	if err != nil {
+		return err
+	}
+
+	d.updateEtagAndLastModified(headers)
+	return nil
+}
+
+// CreateIfNotExists creates this directory under the associated share if the
+// directory does not exists. Returns true if the directory is newly created or
+// false if the directory already exists.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Create-Directory
+func (d *Directory) CreateIfNotExists(options *FileRequestOptions) (bool, error) {
+	// if this is the root directory exit early
+	if d.parent == nil {
+		return false, nil
+	}
+
+	params := prepareOptions(options)
+	resp, err := d.fsc.createResourceNoClose(d.buildPath(), resourceDirectory, params, nil)
+	if resp != nil {
+		defer drainRespBody(resp)
+		if resp.StatusCode == http.StatusCreated || resp.StatusCode == http.StatusConflict {
+			if resp.StatusCode == http.StatusCreated {
+				d.updateEtagAndLastModified(resp.Header)
+				return true, nil
+			}
+
+			return false, d.FetchAttributes(nil)
+		}
+	}
+
+	return false, err
+}
+
+// Delete removes this directory.  It must be empty in order to be deleted.
+// If the directory does not exist the operation fails.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Delete-Directory
+func (d *Directory) Delete(options *FileRequestOptions) error {
+	return d.fsc.deleteResource(d.buildPath(), resourceDirectory, options)
+}
+
+// DeleteIfExists removes this directory if it exists.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Delete-Directory
+func (d *Directory) DeleteIfExists(options *FileRequestOptions) (bool, error) {
+	resp, err := d.fsc.deleteResourceNoClose(d.buildPath(), resourceDirectory, options)
+	if resp != nil {
+		defer drainRespBody(resp)
+		if resp.StatusCode == http.StatusAccepted || resp.StatusCode == http.StatusNotFound {
+			return resp.StatusCode == http.StatusAccepted, nil
+		}
+	}
+	return false, err
+}
+
+// Exists returns true if this directory exists.
+func (d *Directory) Exists() (bool, error) {
+	exists, headers, err := d.fsc.resourceExists(d.buildPath(), resourceDirectory)
+	if exists {
+		d.updateEtagAndLastModified(headers)
+	}
+	return exists, err
+}
+
+// FetchAttributes retrieves metadata for this directory.
+//  See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/get-directory-properties
+func (d *Directory) FetchAttributes(options *FileRequestOptions) error {
+	params := prepareOptions(options)
+	headers, err := d.fsc.getResourceHeaders(d.buildPath(), compNone, resourceDirectory, params, http.MethodHead)
+	if err != nil {
+		return err
+	}
+
+	d.updateEtagAndLastModified(headers)
+	d.Metadata = getMetadataFromHeaders(headers)
+
+	return nil
+}
+
+// GetDirectoryReference returns a child Directory object for this directory.
+func (d *Directory) GetDirectoryReference(name string) *Directory {
+	return &Directory{
+		fsc:    d.fsc,
+		Name:   name,
+		parent: d,
+		share:  d.share,
+	}
+}
+
+// GetFileReference returns a child File object for this directory.
+func (d *Directory) GetFileReference(name string) *File {
+	return &File{
+		fsc:    d.fsc,
+		Name:   name,
+		parent: d,
+		share:  d.share,
+		mutex:  &sync.Mutex{},
+	}
+}
+
+// ListDirsAndFiles returns a list of files and directories under this directory.
+// It also contains a pagination token and other response details.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/List-Directories-and-Files
+func (d *Directory) ListDirsAndFiles(params ListDirsAndFilesParameters) (*DirsAndFilesListResponse, error) {
+	q := mergeParams(params.getParameters(), getURLInitValues(compList, resourceDirectory))
+
+	resp, err := d.fsc.listContent(d.buildPath(), q, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	defer resp.Body.Close()
+	var out DirsAndFilesListResponse
+	err = xmlUnmarshal(resp.Body, &out)
+	return &out, err
+}
+
+// SetMetadata replaces the metadata for this directory.
+//
+// Some keys may be converted to Camel-Case before sending. All keys
+// are returned in lower case by GetDirectoryMetadata. HTTP header names
+// are case-insensitive so case munging should not matter to other
+// applications either.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Set-Directory-Metadata
+func (d *Directory) SetMetadata(options *FileRequestOptions) error {
+	headers, err := d.fsc.setResourceHeaders(d.buildPath(), compMetadata, resourceDirectory, mergeMDIntoExtraHeaders(d.Metadata, nil), options)
+	if err != nil {
+		return err
+	}
+
+	d.updateEtagAndLastModified(headers)
+	return nil
+}
+
+// updates Etag and last modified date
+func (d *Directory) updateEtagAndLastModified(headers http.Header) {
+	d.Properties.Etag = headers.Get("Etag")
+	d.Properties.LastModified = headers.Get("Last-Modified")
+}
+
+// URL gets the canonical URL to this directory.
+// This method does not create a publicly accessible URL if the directory
+// is private and this method does not check if the directory exists.
+func (d *Directory) URL() string {
+	return d.fsc.client.getEndpoint(fileServiceName, d.buildPath(), url.Values{})
+}

vendor/github.com/Azure/azure-sdk-for-go/storage/entity.go

@@ -0,0 +1,456 @@
+package storage
+
+// Copyright 2017 Microsoft Corporation
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+import (
+	"bytes"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/satori/go.uuid"
+)
+
+// Annotating as secure for gas scanning
+/* #nosec */
+const (
+	partitionKeyNode  = "PartitionKey"
+	rowKeyNode        = "RowKey"
+	etagErrorTemplate = "Etag didn't match: %v"
+)
+
+var (
+	errEmptyPayload      = errors.New("Empty payload is not a valid metadata level for this operation")
+	errNilPreviousResult = errors.New("The previous results page is nil")
+	errNilNextLink       = errors.New("There are no more pages in this query results")
+)
+
+// Entity represents an entity inside an Azure table.
+type Entity struct {
+	Table         *Table
+	PartitionKey  string
+	RowKey        string
+	TimeStamp     time.Time
+	OdataMetadata string
+	OdataType     string
+	OdataID       string
+	OdataEtag     string
+	OdataEditLink string
+	Properties    map[string]interface{}
+}
+
+// GetEntityReference returns an Entity object with the specified
+// partition key and row key.
+func (t *Table) GetEntityReference(partitionKey, rowKey string) *Entity {
+	return &Entity{
+		PartitionKey: partitionKey,
+		RowKey:       rowKey,
+		Table:        t,
+	}
+}
+
+// EntityOptions includes options for entity operations.
+type EntityOptions struct {
+	Timeout   uint
+	RequestID string `header:"x-ms-client-request-id"`
+}
+
+// GetEntityOptions includes options for a get entity operation
+type GetEntityOptions struct {
+	Select    []string
+	RequestID string `header:"x-ms-client-request-id"`
+}
+
+// Get gets the referenced entity. Which properties to get can be
+// specified using the select option.
+// See:
+// https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/query-entities
+// https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/querying-tables-and-entities
+func (e *Entity) Get(timeout uint, ml MetadataLevel, options *GetEntityOptions) error {
+	if ml == EmptyPayload {
+		return errEmptyPayload
+	}
+	// RowKey and PartitionKey could be lost if not included in the query
+	// As those are the entity identifiers, it is best if they are not lost
+	rk := e.RowKey
+	pk := e.PartitionKey
+
+	query := url.Values{
+		"timeout": {strconv.FormatUint(uint64(timeout), 10)},
+	}
+	headers := e.Table.tsc.client.getStandardHeaders()
+	headers[headerAccept] = string(ml)
+
+	if options != nil {
+		if len(options.Select) > 0 {
+			query.Add("$select", strings.Join(options.Select, ","))
+		}
+		headers = mergeHeaders(headers, headersFromStruct(*options))
+	}
+
+	uri := e.Table.tsc.client.getEndpoint(tableServiceName, e.buildPath(), query)
+	resp, err := e.Table.tsc.client.exec(http.MethodGet, uri, headers, nil, e.Table.tsc.auth)
+	if err != nil {
+		return err
+	}
+	defer drainRespBody(resp)
+
+	if err = checkRespCode(resp, []int{http.StatusOK}); err != nil {
+		return err
+	}
+
+	respBody, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	err = json.Unmarshal(respBody, e)
+	if err != nil {
+		return err
+	}
+	e.PartitionKey = pk
+	e.RowKey = rk
+
+	return nil
+}
+
+// Insert inserts the referenced entity in its table.
+// The function fails if there is an entity with the same
+// PartitionKey and RowKey in the table.
+// ml determines the level of detail of metadata in the operation response,
+// or no data at all.
+// See: https://docs.microsoft.com/rest/api/storageservices/fileservices/insert-entity
+func (e *Entity) Insert(ml MetadataLevel, options *EntityOptions) error {
+	query, headers := options.getParameters()
+	headers = mergeHeaders(headers, e.Table.tsc.client.getStandardHeaders())
+
+	body, err := json.Marshal(e)
+	if err != nil {
+		return err
+	}
+	headers = addBodyRelatedHeaders(headers, len(body))
+	headers = addReturnContentHeaders(headers, ml)
+
+	uri := e.Table.tsc.client.getEndpoint(tableServiceName, e.Table.buildPath(), query)
+	resp, err := e.Table.tsc.client.exec(http.MethodPost, uri, headers, bytes.NewReader(body), e.Table.tsc.auth)
+	if err != nil {
+		return err
+	}
+	defer drainRespBody(resp)
+
+	if ml != EmptyPayload {
+		if err = checkRespCode(resp, []int{http.StatusCreated}); err != nil {
+			return err
+		}
+		data, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return err
+		}
+		if err = e.UnmarshalJSON(data); err != nil {
+			return err
+		}
+	} else {
+		if err = checkRespCode(resp, []int{http.StatusNoContent}); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// Update updates the contents of an entity. The function fails if there is no entity
+// with the same PartitionKey and RowKey in the table or if the ETag is different
+// than the one in Azure.
+// See: https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/update-entity2
+func (e *Entity) Update(force bool, options *EntityOptions) error {
+	return e.updateMerge(force, http.MethodPut, options)
+}
+
+// Merge merges the contents of entity specified with PartitionKey and RowKey
+// with the content specified in Properties.
+// The function fails if there is no entity with the same PartitionKey and
+// RowKey in the table or if the ETag is different than the one in Azure.
+// Read more: https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/merge-entity
+func (e *Entity) Merge(force bool, options *EntityOptions) error {
+	return e.updateMerge(force, "MERGE", options)
+}
+
+// Delete deletes the entity.
+// The function fails if there is no entity with the same PartitionKey and
+// RowKey in the table or if the ETag is different than the one in Azure.
+// See: https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/delete-entity1
+func (e *Entity) Delete(force bool, options *EntityOptions) error {
+	query, headers := options.getParameters()
+	headers = mergeHeaders(headers, e.Table.tsc.client.getStandardHeaders())
+
+	headers = addIfMatchHeader(headers, force, e.OdataEtag)
+	headers = addReturnContentHeaders(headers, EmptyPayload)
+
+	uri := e.Table.tsc.client.getEndpoint(tableServiceName, e.buildPath(), query)
+	resp, err := e.Table.tsc.client.exec(http.MethodDelete, uri, headers, nil, e.Table.tsc.auth)
+	if err != nil {
+		if resp.StatusCode == http.StatusPreconditionFailed {
+			return fmt.Errorf(etagErrorTemplate, err)
+		}
+		return err
+	}
+	defer drainRespBody(resp)
+
+	if err = checkRespCode(resp, []int{http.StatusNoContent}); err != nil {
+		return err
+	}
+
+	return e.updateTimestamp(resp.Header)
+}
+
+// InsertOrReplace inserts an entity or replaces the existing one.
+// Read more: https://docs.microsoft.com/rest/api/storageservices/fileservices/insert-or-replace-entity
+func (e *Entity) InsertOrReplace(options *EntityOptions) error {
+	return e.insertOr(http.MethodPut, options)
+}
+
+// InsertOrMerge inserts an entity or merges the existing one.
+// Read more: https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/insert-or-merge-entity
+func (e *Entity) InsertOrMerge(options *EntityOptions) error {
+	return e.insertOr("MERGE", options)
+}
+
+func (e *Entity) buildPath() string {
+	return fmt.Sprintf("%s(PartitionKey='%s', RowKey='%s')", e.Table.buildPath(), e.PartitionKey, e.RowKey)
+}
+
+// MarshalJSON is a custom marshaller for entity
+func (e *Entity) MarshalJSON() ([]byte, error) {
+	completeMap := map[string]interface{}{}
+	completeMap[partitionKeyNode] = e.PartitionKey
+	completeMap[rowKeyNode] = e.RowKey
+	for k, v := range e.Properties {
+		typeKey := strings.Join([]string{k, OdataTypeSuffix}, "")
+		switch t := v.(type) {
+		case []byte:
+			completeMap[typeKey] = OdataBinary
+			completeMap[k] = t
+		case time.Time:
+			completeMap[typeKey] = OdataDateTime
+			completeMap[k] = t.Format(time.RFC3339Nano)
+		case uuid.UUID:
+			completeMap[typeKey] = OdataGUID
+			completeMap[k] = t.String()
+		case int64:
+			completeMap[typeKey] = OdataInt64
+			completeMap[k] = fmt.Sprintf("%v", v)
+		default:
+			completeMap[k] = v
+		}
+		if strings.HasSuffix(k, OdataTypeSuffix) {
+			if !(completeMap[k] == OdataBinary ||
+				completeMap[k] == OdataDateTime ||
+				completeMap[k] == OdataGUID ||
+				completeMap[k] == OdataInt64) {
+				return nil, fmt.Errorf("Odata.type annotation %v value is not valid", k)
+			}
+			valueKey := strings.TrimSuffix(k, OdataTypeSuffix)
+			if _, ok := completeMap[valueKey]; !ok {
+				return nil, fmt.Errorf("Odata.type annotation %v defined without value defined", k)
+			}
+		}
+	}
+	return json.Marshal(completeMap)
+}
+
+// UnmarshalJSON is a custom unmarshaller for entities
+func (e *Entity) UnmarshalJSON(data []byte) error {
+	errorTemplate := "Deserializing error: %v"
+
+	props := map[string]interface{}{}
+	err := json.Unmarshal(data, &props)
+	if err != nil {
+		return err
+	}
+
+	// deselialize metadata
+	e.OdataMetadata = stringFromMap(props, "odata.metadata")
+	e.OdataType = stringFromMap(props, "odata.type")
+	e.OdataID = stringFromMap(props, "odata.id")
+	e.OdataEtag = stringFromMap(props, "odata.etag")
+	e.OdataEditLink = stringFromMap(props, "odata.editLink")
+	e.PartitionKey = stringFromMap(props, partitionKeyNode)
+	e.RowKey = stringFromMap(props, rowKeyNode)
+
+	// deserialize timestamp
+	timeStamp, ok := props["Timestamp"]
+	if ok {
+		str, ok := timeStamp.(string)
+		if !ok {
+			return fmt.Errorf(errorTemplate, "Timestamp casting error")
+		}
+		t, err := time.Parse(time.RFC3339Nano, str)
+		if err != nil {
+			return fmt.Errorf(errorTemplate, err)
+		}
+		e.TimeStamp = t
+	}
+	delete(props, "Timestamp")
+	delete(props, "Timestamp@odata.type")
+
+	// deserialize entity (user defined fields)
+	for k, v := range props {
+		if strings.HasSuffix(k, OdataTypeSuffix) {
+			valueKey := strings.TrimSuffix(k, OdataTypeSuffix)
+			str, ok := props[valueKey].(string)
+			if !ok {
+				return fmt.Errorf(errorTemplate, fmt.Sprintf("%v casting error", v))
+			}
+			switch v {
+			case OdataBinary:
+				props[valueKey], err = base64.StdEncoding.DecodeString(str)
+				if err != nil {
+					return fmt.Errorf(errorTemplate, err)
+				}
+			case OdataDateTime:
+				t, err := time.Parse("2006-01-02T15:04:05Z", str)
+				if err != nil {
+					return fmt.Errorf(errorTemplate, err)
+				}
+				props[valueKey] = t
+			case OdataGUID:
+				props[valueKey] = uuid.FromStringOrNil(str)
+			case OdataInt64:
+				i, err := strconv.ParseInt(str, 10, 64)
+				if err != nil {
+					return fmt.Errorf(errorTemplate, err)
+				}
+				props[valueKey] = i
+			default:
+				return fmt.Errorf(errorTemplate, fmt.Sprintf("%v is not supported", v))
+			}
+			delete(props, k)
+		}
+	}
+
+	e.Properties = props
+	return nil
+}
+
+func getAndDelete(props map[string]interface{}, key string) interface{} {
+	if value, ok := props[key]; ok {
+		delete(props, key)
+		return value
+	}
+	return nil
+}
+
+func addIfMatchHeader(h map[string]string, force bool, etag string) map[string]string {
+	if force {
+		h[headerIfMatch] = "*"
+	} else {
+		h[headerIfMatch] = etag
+	}
+	return h
+}
+
+// updates Etag and timestamp
+func (e *Entity) updateEtagAndTimestamp(headers http.Header) error {
+	e.OdataEtag = headers.Get(headerEtag)
+	return e.updateTimestamp(headers)
+}
+
+func (e *Entity) updateTimestamp(headers http.Header) error {
+	str := headers.Get(headerDate)
+	t, err := time.Parse(time.RFC1123, str)
+	if err != nil {
+		return fmt.Errorf("Update timestamp error: %v", err)
+	}
+	e.TimeStamp = t
+	return nil
+}
+
+func (e *Entity) insertOr(verb string, options *EntityOptions) error {
+	query, headers := options.getParameters()
+	headers = mergeHeaders(headers, e.Table.tsc.client.getStandardHeaders())
+
+	body, err := json.Marshal(e)
+	if err != nil {
+		return err
+	}
+	headers = addBodyRelatedHeaders(headers, len(body))
+	headers = addReturnContentHeaders(headers, EmptyPayload)
+
+	uri := e.Table.tsc.client.getEndpoint(tableServiceName, e.buildPath(), query)
+	resp, err := e.Table.tsc.client.exec(verb, uri, headers, bytes.NewReader(body), e.Table.tsc.auth)
+	if err != nil {
+		return err
+	}
+	defer drainRespBody(resp)
+
+	if err = checkRespCode(resp, []int{http.StatusNoContent}); err != nil {
+		return err
+	}
+
+	return e.updateEtagAndTimestamp(resp.Header)
+}
+
+func (e *Entity) updateMerge(force bool, verb string, options *EntityOptions) error {
+	query, headers := options.getParameters()
+	headers = mergeHeaders(headers, e.Table.tsc.client.getStandardHeaders())
+
+	body, err := json.Marshal(e)
+	if err != nil {
+		return err
+	}
+	headers = addBodyRelatedHeaders(headers, len(body))
+	headers = addIfMatchHeader(headers, force, e.OdataEtag)
+	headers = addReturnContentHeaders(headers, EmptyPayload)
+
+	uri := e.Table.tsc.client.getEndpoint(tableServiceName, e.buildPath(), query)
+	resp, err := e.Table.tsc.client.exec(verb, uri, headers, bytes.NewReader(body), e.Table.tsc.auth)
+	if err != nil {
+		if resp.StatusCode == http.StatusPreconditionFailed {
+			return fmt.Errorf(etagErrorTemplate, err)
+		}
+		return err
+	}
+	defer drainRespBody(resp)
+
+	if err = checkRespCode(resp, []int{http.StatusNoContent}); err != nil {
+		return err
+	}
+
+	return e.updateEtagAndTimestamp(resp.Header)
+}
+
+func stringFromMap(props map[string]interface{}, key string) string {
+	value := getAndDelete(props, key)
+	if value != nil {
+		return value.(string)
+	}
+	return ""
+}
+
+func (options *EntityOptions) getParameters() (url.Values, map[string]string) {
+	query := url.Values{}
+	headers := map[string]string{}
+	if options != nil {
+		query = addTimeout(query, options.Timeout)
+		headers = headersFromStruct(*options)
+	}
+	return query, headers
+}

vendor/github.com/Azure/azure-sdk-for-go/storage/file.go

@@ -0,0 +1,480 @@
+package storage
+
+// Copyright 2017 Microsoft Corporation
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strconv"
+	"sync"
+)
+
+const fourMB = uint64(4194304)
+const oneTB = uint64(1099511627776)
+
+// Export maximum range and file sizes
+const MaxRangeSize = fourMB
+const MaxFileSize = oneTB
+
+// File represents a file on a share.
+type File struct {
+	fsc                *FileServiceClient
+	Metadata           map[string]string
+	Name               string `xml:"Name"`
+	parent             *Directory
+	Properties         FileProperties `xml:"Properties"`
+	share              *Share
+	FileCopyProperties FileCopyState
+	mutex              *sync.Mutex
+}
+
+// FileProperties contains various properties of a file.
+type FileProperties struct {
+	CacheControl string `header:"x-ms-cache-control"`
+	Disposition  string `header:"x-ms-content-disposition"`
+	Encoding     string `header:"x-ms-content-encoding"`
+	Etag         string
+	Language     string `header:"x-ms-content-language"`
+	LastModified string
+	Length       uint64 `xml:"Content-Length" header:"x-ms-content-length"`
+	MD5          string `header:"x-ms-content-md5"`
+	Type         string `header:"x-ms-content-type"`
+}
+
+// FileCopyState contains various properties of a file copy operation.
+type FileCopyState struct {
+	CompletionTime string
+	ID             string `header:"x-ms-copy-id"`
+	Progress       string
+	Source         string
+	Status         string `header:"x-ms-copy-status"`
+	StatusDesc     string
+}
+
+// FileStream contains file data returned from a call to GetFile.
+type FileStream struct {
+	Body       io.ReadCloser
+	ContentMD5 string
+}
+
+// FileRequestOptions will be passed to misc file operations.
+// Currently just Timeout (in seconds) but could expand.
+type FileRequestOptions struct {
+	Timeout uint // timeout duration in seconds.
+}
+
+func prepareOptions(options *FileRequestOptions) url.Values {
+	params := url.Values{}
+	if options != nil {
+		params = addTimeout(params, options.Timeout)
+	}
+	return params
+}
+
+// FileRanges contains a list of file range information for a file.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/List-Ranges
+type FileRanges struct {
+	ContentLength uint64
+	LastModified  string
+	ETag          string
+	FileRanges    []FileRange `xml:"Range"`
+}
+
+// FileRange contains range information for a file.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/List-Ranges
+type FileRange struct {
+	Start uint64 `xml:"Start"`
+	End   uint64 `xml:"End"`
+}
+
+func (fr FileRange) String() string {
+	return fmt.Sprintf("bytes=%d-%d", fr.Start, fr.End)
+}
+
+// builds the complete file path for this file object
+func (f *File) buildPath() string {
+	return f.parent.buildPath() + "/" + f.Name
+}
+
+// ClearRange releases the specified range of space in a file.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Put-Range
+func (f *File) ClearRange(fileRange FileRange, options *FileRequestOptions) error {
+	var timeout *uint
+	if options != nil {
+		timeout = &options.Timeout
+	}
+	headers, err := f.modifyRange(nil, fileRange, timeout, nil)
+	if err != nil {
+		return err
+	}
+
+	f.updateEtagAndLastModified(headers)
+	return nil
+}
+
+// Create creates a new file or replaces an existing one.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Create-File
+func (f *File) Create(maxSize uint64, options *FileRequestOptions) error {
+	if maxSize > oneTB {
+		return fmt.Errorf("max file size is 1TB")
+	}
+	params := prepareOptions(options)
+	headers := headersFromStruct(f.Properties)
+	headers["x-ms-content-length"] = strconv.FormatUint(maxSize, 10)
+	headers["x-ms-type"] = "file"
+
+	outputHeaders, err := f.fsc.createResource(f.buildPath(), resourceFile, params, mergeMDIntoExtraHeaders(f.Metadata, headers), []int{http.StatusCreated})
+	if err != nil {
+		return err
+	}
+
+	f.Properties.Length = maxSize
+	f.updateEtagAndLastModified(outputHeaders)
+	return nil
+}
+
+// CopyFile operation copied a file/blob from the sourceURL to the path provided.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/copy-file
+func (f *File) CopyFile(sourceURL string, options *FileRequestOptions) error {
+	extraHeaders := map[string]string{
+		"x-ms-type":        "file",
+		"x-ms-copy-source": sourceURL,
+	}
+	params := prepareOptions(options)
+
+	headers, err := f.fsc.createResource(f.buildPath(), resourceFile, params, mergeMDIntoExtraHeaders(f.Metadata, extraHeaders), []int{http.StatusAccepted})
+	if err != nil {
+		return err
+	}
+
+	f.updateEtagAndLastModified(headers)
+	f.FileCopyProperties.ID = headers.Get("X-Ms-Copy-Id")
+	f.FileCopyProperties.Status = headers.Get("X-Ms-Copy-Status")
+	return nil
+}
+
+// Delete immediately removes this file from the storage account.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Delete-File2
+func (f *File) Delete(options *FileRequestOptions) error {
+	return f.fsc.deleteResource(f.buildPath(), resourceFile, options)
+}
+
+// DeleteIfExists removes this file if it exists.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Delete-File2
+func (f *File) DeleteIfExists(options *FileRequestOptions) (bool, error) {
+	resp, err := f.fsc.deleteResourceNoClose(f.buildPath(), resourceFile, options)
+	if resp != nil {
+		defer drainRespBody(resp)
+		if resp.StatusCode == http.StatusAccepted || resp.StatusCode == http.StatusNotFound {
+			return resp.StatusCode == http.StatusAccepted, nil
+		}
+	}
+	return false, err
+}
+
+// GetFileOptions includes options for a get file operation
+type GetFileOptions struct {
+	Timeout       uint
+	GetContentMD5 bool
+}
+
+// DownloadToStream operation downloads the file.
+//
+// See: https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/get-file
+func (f *File) DownloadToStream(options *FileRequestOptions) (io.ReadCloser, error) {
+	params := prepareOptions(options)
+	resp, err := f.fsc.getResourceNoClose(f.buildPath(), compNone, resourceFile, params, http.MethodGet, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	if err = checkRespCode(resp, []int{http.StatusOK}); err != nil {
+		drainRespBody(resp)
+		return nil, err
+	}
+	return resp.Body, nil
+}
+
+// DownloadRangeToStream operation downloads the specified range of this file with optional MD5 hash.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/get-file
+func (f *File) DownloadRangeToStream(fileRange FileRange, options *GetFileOptions) (fs FileStream, err error) {
+	extraHeaders := map[string]string{
+		"Range": fileRange.String(),
+	}
+	params := url.Values{}
+	if options != nil {
+		if options.GetContentMD5 {
+			if isRangeTooBig(fileRange) {
+				return fs, fmt.Errorf("must specify a range less than or equal to 4MB when getContentMD5 is true")
+			}
+			extraHeaders["x-ms-range-get-content-md5"] = "true"
+		}
+		params = addTimeout(params, options.Timeout)
+	}
+
+	resp, err := f.fsc.getResourceNoClose(f.buildPath(), compNone, resourceFile, params, http.MethodGet, extraHeaders)
+	if err != nil {
+		return fs, err
+	}
+
+	if err = checkRespCode(resp, []int{http.StatusOK, http.StatusPartialContent}); err != nil {
+		drainRespBody(resp)
+		return fs, err
+	}
+
+	fs.Body = resp.Body
+	if options != nil && options.GetContentMD5 {
+		fs.ContentMD5 = resp.Header.Get("Content-MD5")
+	}
+	return fs, nil
+}
+
+// Exists returns true if this file exists.
+func (f *File) Exists() (bool, error) {
+	exists, headers, err := f.fsc.resourceExists(f.buildPath(), resourceFile)
+	if exists {
+		f.updateEtagAndLastModified(headers)
+		f.updateProperties(headers)
+	}
+	return exists, err
+}
+
+// FetchAttributes updates metadata and properties for this file.
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/get-file-properties
+func (f *File) FetchAttributes(options *FileRequestOptions) error {
+	params := prepareOptions(options)
+	headers, err := f.fsc.getResourceHeaders(f.buildPath(), compNone, resourceFile, params, http.MethodHead)
+	if err != nil {
+		return err
+	}
+
+	f.updateEtagAndLastModified(headers)
+	f.updateProperties(headers)
+	f.Metadata = getMetadataFromHeaders(headers)
+	return nil
+}
+
+// returns true if the range is larger than 4MB
+func isRangeTooBig(fileRange FileRange) bool {
+	if fileRange.End-fileRange.Start > fourMB {
+		return true
+	}
+
+	return false
+}
+
+// ListRangesOptions includes options for a list file ranges operation
+type ListRangesOptions struct {
+	Timeout   uint
+	ListRange *FileRange
+}
+
+// ListRanges returns the list of valid ranges for this file.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/List-Ranges
+func (f *File) ListRanges(options *ListRangesOptions) (*FileRanges, error) {
+	params := url.Values{"comp": {"rangelist"}}
+
+	// add optional range to list
+	var headers map[string]string
+	if options != nil {
+		params = addTimeout(params, options.Timeout)
+		if options.ListRange != nil {
+			headers = make(map[string]string)
+			headers["Range"] = options.ListRange.String()
+		}
+	}
+
+	resp, err := f.fsc.listContent(f.buildPath(), params, headers)
+	if err != nil {
+		return nil, err
+	}
+
+	defer resp.Body.Close()
+	var cl uint64
+	cl, err = strconv.ParseUint(resp.Header.Get("x-ms-content-length"), 10, 64)
+	if err != nil {
+		ioutil.ReadAll(resp.Body)
+		return nil, err
+	}
+
+	var out FileRanges
+	out.ContentLength = cl
+	out.ETag = resp.Header.Get("ETag")
+	out.LastModified = resp.Header.Get("Last-Modified")
+
+	err = xmlUnmarshal(resp.Body, &out)
+	return &out, err
+}
+
+// modifies a range of bytes in this file
+func (f *File) modifyRange(bytes io.Reader, fileRange FileRange, timeout *uint, contentMD5 *string) (http.Header, error) {
+	if err := f.fsc.checkForStorageEmulator(); err != nil {
+		return nil, err
+	}
+	if fileRange.End < fileRange.Start {
+		return nil, errors.New("the value for rangeEnd must be greater than or equal to rangeStart")
+	}
+	if bytes != nil && isRangeTooBig(fileRange) {
+		return nil, errors.New("range cannot exceed 4MB in size")
+	}
+
+	params := url.Values{"comp": {"range"}}
+	if timeout != nil {
+		params = addTimeout(params, *timeout)
+	}
+
+	uri := f.fsc.client.getEndpoint(fileServiceName, f.buildPath(), params)
+
+	// default to clear
+	write := "clear"
+	cl := uint64(0)
+
+	// if bytes is not nil then this is an update operation
+	if bytes != nil {
+		write = "update"
+		cl = (fileRange.End - fileRange.Start) + 1
+	}
+
+	extraHeaders := map[string]string{
+		"Content-Length": strconv.FormatUint(cl, 10),
+		"Range":          fileRange.String(),
+		"x-ms-write":     write,
+	}
+
+	if contentMD5 != nil {
+		extraHeaders["Content-MD5"] = *contentMD5
+	}
+
+	headers := mergeHeaders(f.fsc.client.getStandardHeaders(), extraHeaders)
+	resp, err := f.fsc.client.exec(http.MethodPut, uri, headers, bytes, f.fsc.auth)
+	if err != nil {
+		return nil, err
+	}
+	defer drainRespBody(resp)
+	return resp.Header, checkRespCode(resp, []int{http.StatusCreated})
+}
+
+// SetMetadata replaces the metadata for this file.
+//
+// Some keys may be converted to Camel-Case before sending. All keys
+// are returned in lower case by GetFileMetadata. HTTP header names
+// are case-insensitive so case munging should not matter to other
+// applications either.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Set-File-Metadata
+func (f *File) SetMetadata(options *FileRequestOptions) error {
+	headers, err := f.fsc.setResourceHeaders(f.buildPath(), compMetadata, resourceFile, mergeMDIntoExtraHeaders(f.Metadata, nil), options)
+	if err != nil {
+		return err
+	}
+
+	f.updateEtagAndLastModified(headers)
+	return nil
+}
+
+// SetProperties sets system properties on this file.
+//
+// Some keys may be converted to Camel-Case before sending. All keys
+// are returned in lower case by SetFileProperties. HTTP header names
+// are case-insensitive so case munging should not matter to other
+// applications either.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Set-File-Properties
+func (f *File) SetProperties(options *FileRequestOptions) error {
+	headers, err := f.fsc.setResourceHeaders(f.buildPath(), compProperties, resourceFile, headersFromStruct(f.Properties), options)
+	if err != nil {
+		return err
+	}
+
+	f.updateEtagAndLastModified(headers)
+	return nil
+}
+
+// updates Etag and last modified date
+func (f *File) updateEtagAndLastModified(headers http.Header) {
+	f.Properties.Etag = headers.Get("Etag")
+	f.Properties.LastModified = headers.Get("Last-Modified")
+}
+
+// updates file properties from the specified HTTP header
+func (f *File) updateProperties(header http.Header) {
+	size, err := strconv.ParseUint(header.Get("Content-Length"), 10, 64)
+	if err == nil {
+		f.Properties.Length = size
+	}
+
+	f.updateEtagAndLastModified(header)
+	f.Properties.CacheControl = header.Get("Cache-Control")
+	f.Properties.Disposition = header.Get("Content-Disposition")
+	f.Properties.Encoding = header.Get("Content-Encoding")
+	f.Properties.Language = header.Get("Content-Language")
+	f.Properties.MD5 = header.Get("Content-MD5")
+	f.Properties.Type = header.Get("Content-Type")
+}
+
+// URL gets the canonical URL to this file.
+// This method does not create a publicly accessible URL if the file
+// is private and this method does not check if the file exists.
+func (f *File) URL() string {
+	return f.fsc.client.getEndpoint(fileServiceName, f.buildPath(), nil)
+}
+
+// WriteRangeOptions includes options for a write file range operation
+type WriteRangeOptions struct {
+	Timeout    uint
+	ContentMD5 string
+}
+
+// WriteRange writes a range of bytes to this file with an optional MD5 hash of the content (inside
+// options parameter). Note that the length of bytes must match (rangeEnd - rangeStart) + 1 with
+// a maximum size of 4MB.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Put-Range
+func (f *File) WriteRange(bytes io.Reader, fileRange FileRange, options *WriteRangeOptions) error {
+	if bytes == nil {
+		return errors.New("bytes cannot be nil")
+	}
+	var timeout *uint
+	var md5 *string
+	if options != nil {
+		timeout = &options.Timeout
+		md5 = &options.ContentMD5
+	}
+
+	headers, err := f.modifyRange(bytes, fileRange, timeout, md5)
+	if err != nil {
+		return err
+	}
+	// it's perfectly legal for multiple go routines to call WriteRange
+	// on the same *File (e.g. concurrently writing non-overlapping ranges)
+	// so we must take the file mutex before updating our properties.
+	f.mutex.Lock()
+	f.updateEtagAndLastModified(headers)
+	f.mutex.Unlock()
+	return nil
+}

vendor/github.com/Azure/azure-sdk-for-go/storage/fileserviceclient.go

@@ -0,0 +1,338 @@
+package storage
+
+// Copyright 2017 Microsoft Corporation
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+import (
+	"encoding/xml"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strconv"
+)
+
+// FileServiceClient contains operations for Microsoft Azure File Service.
+type FileServiceClient struct {
+	client Client
+	auth   authentication
+}
+
+// ListSharesParameters defines the set of customizable parameters to make a
+// List Shares call.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/List-Shares
+type ListSharesParameters struct {
+	Prefix     string
+	Marker     string
+	Include    string
+	MaxResults uint
+	Timeout    uint
+}
+
+// ShareListResponse contains the response fields from
+// ListShares call.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/List-Shares
+type ShareListResponse struct {
+	XMLName    xml.Name `xml:"EnumerationResults"`
+	Xmlns      string   `xml:"xmlns,attr"`
+	Prefix     string   `xml:"Prefix"`
+	Marker     string   `xml:"Marker"`
+	NextMarker string   `xml:"NextMarker"`
+	MaxResults int64    `xml:"MaxResults"`
+	Shares     []Share  `xml:"Shares>Share"`
+}
+
+type compType string
+
+const (
+	compNone       compType = ""
+	compList       compType = "list"
+	compMetadata   compType = "metadata"
+	compProperties compType = "properties"
+	compRangeList  compType = "rangelist"
+)
+
+func (ct compType) String() string {
+	return string(ct)
+}
+
+type resourceType string
+
+const (
+	resourceDirectory resourceType = "directory"
+	resourceFile      resourceType = ""
+	resourceShare     resourceType = "share"
+)
+
+func (rt resourceType) String() string {
+	return string(rt)
+}
+
+func (p ListSharesParameters) getParameters() url.Values {
+	out := url.Values{}
+
+	if p.Prefix != "" {
+		out.Set("prefix", p.Prefix)
+	}
+	if p.Marker != "" {
+		out.Set("marker", p.Marker)
+	}
+	if p.Include != "" {
+		out.Set("include", p.Include)
+	}
+	if p.MaxResults != 0 {
+		out.Set("maxresults", strconv.FormatUint(uint64(p.MaxResults), 10))
+	}
+	if p.Timeout != 0 {
+		out.Set("timeout", strconv.FormatUint(uint64(p.Timeout), 10))
+	}
+
+	return out
+}
+
+func (p ListDirsAndFilesParameters) getParameters() url.Values {
+	out := url.Values{}
+
+	if p.Prefix != "" {
+		out.Set("prefix", p.Prefix)
+	}
+	if p.Marker != "" {
+		out.Set("marker", p.Marker)
+	}
+	if p.MaxResults != 0 {
+		out.Set("maxresults", strconv.FormatUint(uint64(p.MaxResults), 10))
+	}
+	out = addTimeout(out, p.Timeout)
+
+	return out
+}
+
+// returns url.Values for the specified types
+func getURLInitValues(comp compType, res resourceType) url.Values {
+	values := url.Values{}
+	if comp != compNone {
+		values.Set("comp", comp.String())
+	}
+	if res != resourceFile {
+		values.Set("restype", res.String())
+	}
+	return values
+}
+
+// GetShareReference returns a Share object for the specified share name.
+func (f *FileServiceClient) GetShareReference(name string) *Share {
+	return &Share{
+		fsc:  f,
+		Name: name,
+		Properties: ShareProperties{
+			Quota: -1,
+		},
+	}
+}
+
+// ListShares returns the list of shares in a storage account along with
+// pagination token and other response details.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/list-shares
+func (f FileServiceClient) ListShares(params ListSharesParameters) (*ShareListResponse, error) {
+	q := mergeParams(params.getParameters(), url.Values{"comp": {"list"}})
+
+	var out ShareListResponse
+	resp, err := f.listContent("", q, nil)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	err = xmlUnmarshal(resp.Body, &out)
+
+	// assign our client to the newly created Share objects
+	for i := range out.Shares {
+		out.Shares[i].fsc = &f
+	}
+	return &out, err
+}
+
+// GetServiceProperties gets the properties of your storage account's file service.
+// File service does not support logging
+// See: https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/get-file-service-properties
+func (f *FileServiceClient) GetServiceProperties() (*ServiceProperties, error) {
+	return f.client.getServiceProperties(fileServiceName, f.auth)
+}
+
+// SetServiceProperties sets the properties of your storage account's file service.
+// File service does not support logging
+// See: https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/set-file-service-properties
+func (f *FileServiceClient) SetServiceProperties(props ServiceProperties) error {
+	return f.client.setServiceProperties(props, fileServiceName, f.auth)
+}
+
+// retrieves directory or share content
+func (f FileServiceClient) listContent(path string, params url.Values, extraHeaders map[string]string) (*http.Response, error) {
+	if err := f.checkForStorageEmulator(); err != nil {
+		return nil, err
+	}
+
+	uri := f.client.getEndpoint(fileServiceName, path, params)
+	extraHeaders = f.client.protectUserAgent(extraHeaders)
+	headers := mergeHeaders(f.client.getStandardHeaders(), extraHeaders)
+
+	resp, err := f.client.exec(http.MethodGet, uri, headers, nil, f.auth)
+	if err != nil {
+		return nil, err
+	}
+
+	if err = checkRespCode(resp, []int{http.StatusOK}); err != nil {
+		drainRespBody(resp)
+		return nil, err
+	}
+
+	return resp, nil
+}
+
+// returns true if the specified resource exists
+func (f FileServiceClient) resourceExists(path string, res resourceType) (bool, http.Header, error) {
+	if err := f.checkForStorageEmulator(); err != nil {
+		return false, nil, err
+	}
+
+	uri := f.client.getEndpoint(fileServiceName, path, getURLInitValues(compNone, res))
+	headers := f.client.getStandardHeaders()
+
+	resp, err := f.client.exec(http.MethodHead, uri, headers, nil, f.auth)
+	if resp != nil {
+		defer drainRespBody(resp)
+		if resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusNotFound {
+			return resp.StatusCode == http.StatusOK, resp.Header, nil
+		}
+	}
+	return false, nil, err
+}
+
+// creates a resource depending on the specified resource type
+func (f FileServiceClient) createResource(path string, res resourceType, urlParams url.Values, extraHeaders map[string]string, expectedResponseCodes []int) (http.Header, error) {
+	resp, err := f.createResourceNoClose(path, res, urlParams, extraHeaders)
+	if err != nil {
+		return nil, err
+	}
+	defer drainRespBody(resp)
+	return resp.Header, checkRespCode(resp, expectedResponseCodes)
+}
+
+// creates a resource depending on the specified resource type, doesn't close the response body
+func (f FileServiceClient) createResourceNoClose(path string, res resourceType, urlParams url.Values, extraHeaders map[string]string) (*http.Response, error) {
+	if err := f.checkForStorageEmulator(); err != nil {
+		return nil, err
+	}
+
+	values := getURLInitValues(compNone, res)
+	combinedParams := mergeParams(values, urlParams)
+	uri := f.client.getEndpoint(fileServiceName, path, combinedParams)
+	extraHeaders = f.client.protectUserAgent(extraHeaders)
+	headers := mergeHeaders(f.client.getStandardHeaders(), extraHeaders)
+
+	return f.client.exec(http.MethodPut, uri, headers, nil, f.auth)
+}
+
+// returns HTTP header data for the specified directory or share
+func (f FileServiceClient) getResourceHeaders(path string, comp compType, res resourceType, params url.Values, verb string) (http.Header, error) {
+	resp, err := f.getResourceNoClose(path, comp, res, params, verb, nil)
+	if err != nil {
+		return nil, err
+	}
+	defer drainRespBody(resp)
+
+	if err = checkRespCode(resp, []int{http.StatusOK}); err != nil {
+		return nil, err
+	}
+
+	return resp.Header, nil
+}
+
+// gets the specified resource, doesn't close the response body
+func (f FileServiceClient) getResourceNoClose(path string, comp compType, res resourceType, params url.Values, verb string, extraHeaders map[string]string) (*http.Response, error) {
+	if err := f.checkForStorageEmulator(); err != nil {
+		return nil, err
+	}
+
+	params = mergeParams(params, getURLInitValues(comp, res))
+	uri := f.client.getEndpoint(fileServiceName, path, params)
+	headers := mergeHeaders(f.client.getStandardHeaders(), extraHeaders)
+
+	return f.client.exec(verb, uri, headers, nil, f.auth)
+}
+
+// deletes the resource and returns the response
+func (f FileServiceClient) deleteResource(path string, res resourceType, options *FileRequestOptions) error {
+	resp, err := f.deleteResourceNoClose(path, res, options)
+	if err != nil {
+		return err
+	}
+	defer drainRespBody(resp)
+	return checkRespCode(resp, []int{http.StatusAccepted})
+}
+
+// deletes the resource and returns the response, doesn't close the response body
+func (f FileServiceClient) deleteResourceNoClose(path string, res resourceType, options *FileRequestOptions) (*http.Response, error) {
+	if err := f.checkForStorageEmulator(); err != nil {
+		return nil, err
+	}
+
+	values := mergeParams(getURLInitValues(compNone, res), prepareOptions(options))
+	uri := f.client.getEndpoint(fileServiceName, path, values)
+	return f.client.exec(http.MethodDelete, uri, f.client.getStandardHeaders(), nil, f.auth)
+}
+
+// merges metadata into extraHeaders and returns extraHeaders
+func mergeMDIntoExtraHeaders(metadata, extraHeaders map[string]string) map[string]string {
+	if metadata == nil && extraHeaders == nil {
+		return nil
+	}
+	if extraHeaders == nil {
+		extraHeaders = make(map[string]string)
+	}
+	for k, v := range metadata {
+		extraHeaders[userDefinedMetadataHeaderPrefix+k] = v
+	}
+	return extraHeaders
+}
+
+// sets extra header data for the specified resource
+func (f FileServiceClient) setResourceHeaders(path string, comp compType, res resourceType, extraHeaders map[string]string, options *FileRequestOptions) (http.Header, error) {
+	if err := f.checkForStorageEmulator(); err != nil {
+		return nil, err
+	}
+
+	params := mergeParams(getURLInitValues(comp, res), prepareOptions(options))
+	uri := f.client.getEndpoint(fileServiceName, path, params)
+	extraHeaders = f.client.protectUserAgent(extraHeaders)
+	headers := mergeHeaders(f.client.getStandardHeaders(), extraHeaders)
+
+	resp, err := f.client.exec(http.MethodPut, uri, headers, nil, f.auth)
+	if err != nil {
+		return nil, err
+	}
+	defer drainRespBody(resp)
+
+	return resp.Header, checkRespCode(resp, []int{http.StatusOK})
+}
+
+//checkForStorageEmulator determines if the client is setup for use with
+//Azure Storage Emulator, and returns a relevant error
+func (f FileServiceClient) checkForStorageEmulator() error {
+	if f.client.accountName == StorageEmulatorAccountName {
+		return fmt.Errorf("Error: File service is not currently supported by Azure Storage Emulator")
+	}
+	return nil
+}

vendor/github.com/Azure/azure-sdk-for-go/storage/leaseblob.go

@@ -0,0 +1,201 @@
+package storage
+
+// Copyright 2017 Microsoft Corporation
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+import (
+	"errors"
+	"net/http"
+	"net/url"
+	"strconv"
+	"time"
+)
+
+// lease constants.
+const (
+	leaseHeaderPrefix = "x-ms-lease-"
+	headerLeaseID     = "x-ms-lease-id"
+	leaseAction       = "x-ms-lease-action"
+	leaseBreakPeriod  = "x-ms-lease-break-period"
+	leaseDuration     = "x-ms-lease-duration"
+	leaseProposedID   = "x-ms-proposed-lease-id"
+	leaseTime         = "x-ms-lease-time"
+
+	acquireLease = "acquire"
+	renewLease   = "renew"
+	changeLease  = "change"
+	releaseLease = "release"
+	breakLease   = "break"
+)
+
+// leasePut is common PUT code for the various acquire/release/break etc functions.
+func (b *Blob) leaseCommonPut(headers map[string]string, expectedStatus int, options *LeaseOptions) (http.Header, error) {
+	params := url.Values{"comp": {"lease"}}
+
+	if options != nil {
+		params = addTimeout(params, options.Timeout)
+		headers = mergeHeaders(headers, headersFromStruct(*options))
+	}
+	uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), params)
+
+	resp, err := b.Container.bsc.client.exec(http.MethodPut, uri, headers, nil, b.Container.bsc.auth)
+	if err != nil {
+		return nil, err
+	}
+	defer drainRespBody(resp)
+
+	if err := checkRespCode(resp, []int{expectedStatus}); err != nil {
+		return nil, err
+	}
+
+	return resp.Header, nil
+}
+
+// LeaseOptions includes options for all operations regarding leasing blobs
+type LeaseOptions struct {
+	Timeout           uint
+	Origin            string     `header:"Origin"`
+	IfMatch           string     `header:"If-Match"`
+	IfNoneMatch       string     `header:"If-None-Match"`
+	IfModifiedSince   *time.Time `header:"If-Modified-Since"`
+	IfUnmodifiedSince *time.Time `header:"If-Unmodified-Since"`
+	RequestID         string     `header:"x-ms-client-request-id"`
+}
+
+// AcquireLease creates a lease for a blob
+// returns leaseID acquired
+// In API Versions starting on 2012-02-12, the minimum leaseTimeInSeconds is 15, the maximum
+// non-infinite leaseTimeInSeconds is 60. To specify an infinite lease, provide the value -1.
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Lease-Blob
+func (b *Blob) AcquireLease(leaseTimeInSeconds int, proposedLeaseID string, options *LeaseOptions) (returnedLeaseID string, err error) {
+	headers := b.Container.bsc.client.getStandardHeaders()
+	headers[leaseAction] = acquireLease
+
+	if leaseTimeInSeconds == -1 {
+		// Do nothing, but don't trigger the following clauses.
+	} else if leaseTimeInSeconds > 60 || b.Container.bsc.client.apiVersion < "2012-02-12" {
+		leaseTimeInSeconds = 60
+	} else if leaseTimeInSeconds < 15 {
+		leaseTimeInSeconds = 15
+	}
+
+	headers[leaseDuration] = strconv.Itoa(leaseTimeInSeconds)
+
+	if proposedLeaseID != "" {
+		headers[leaseProposedID] = proposedLeaseID
+	}
+
+	respHeaders, err := b.leaseCommonPut(headers, http.StatusCreated, options)
+	if err != nil {
+		return "", err
+	}
+
+	returnedLeaseID = respHeaders.Get(http.CanonicalHeaderKey(headerLeaseID))
+
+	if returnedLeaseID != "" {
+		return returnedLeaseID, nil
+	}
+
+	return "", errors.New("LeaseID not returned")
+}
+
+// BreakLease breaks the lease for a blob
+// Returns the timeout remaining in the lease in seconds
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Lease-Blob
+func (b *Blob) BreakLease(options *LeaseOptions) (breakTimeout int, err error) {
+	headers := b.Container.bsc.client.getStandardHeaders()
+	headers[leaseAction] = breakLease
+	return b.breakLeaseCommon(headers, options)
+}
+
+// BreakLeaseWithBreakPeriod breaks the lease for a blob
+// breakPeriodInSeconds is used to determine how long until new lease can be created.
+// Returns the timeout remaining in the lease in seconds
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Lease-Blob
+func (b *Blob) BreakLeaseWithBreakPeriod(breakPeriodInSeconds int, options *LeaseOptions) (breakTimeout int, err error) {
+	headers := b.Container.bsc.client.getStandardHeaders()
+	headers[leaseAction] = breakLease
+	headers[leaseBreakPeriod] = strconv.Itoa(breakPeriodInSeconds)
+	return b.breakLeaseCommon(headers, options)
+}
+
+// breakLeaseCommon is common code for both version of BreakLease (with and without break period)
+func (b *Blob) breakLeaseCommon(headers map[string]string, options *LeaseOptions) (breakTimeout int, err error) {
+
+	respHeaders, err := b.leaseCommonPut(headers, http.StatusAccepted, options)
+	if err != nil {
+		return 0, err
+	}
+
+	breakTimeoutStr := respHeaders.Get(http.CanonicalHeaderKey(leaseTime))
+	if breakTimeoutStr != "" {
+		breakTimeout, err = strconv.Atoi(breakTimeoutStr)
+		if err != nil {
+			return 0, err
+		}
+	}
+
+	return breakTimeout, nil
+}
+
+// ChangeLease changes a lease ID for a blob
+// Returns the new LeaseID acquired
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Lease-Blob
+func (b *Blob) ChangeLease(currentLeaseID string, proposedLeaseID string, options *LeaseOptions) (newLeaseID string, err error) {
+	headers := b.Container.bsc.client.getStandardHeaders()
+	headers[leaseAction] = changeLease
+	headers[headerLeaseID] = currentLeaseID
+	headers[leaseProposedID] = proposedLeaseID
+
+	respHeaders, err := b.leaseCommonPut(headers, http.StatusOK, options)
+	if err != nil {
+		return "", err
+	}
+
+	newLeaseID = respHeaders.Get(http.CanonicalHeaderKey(headerLeaseID))
+	if newLeaseID != "" {
+		return newLeaseID, nil
+	}
+
+	return "", errors.New("LeaseID not returned")
+}
+
+// ReleaseLease releases the lease for a blob
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Lease-Blob
+func (b *Blob) ReleaseLease(currentLeaseID string, options *LeaseOptions) error {
+	headers := b.Container.bsc.client.getStandardHeaders()
+	headers[leaseAction] = releaseLease
+	headers[headerLeaseID] = currentLeaseID
+
+	_, err := b.leaseCommonPut(headers, http.StatusOK, options)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// RenewLease renews the lease for a blob as per https://msdn.microsoft.com/en-us/library/azure/ee691972.aspx
+func (b *Blob) RenewLease(currentLeaseID string, options *LeaseOptions) error {
+	headers := b.Container.bsc.client.getStandardHeaders()
+	headers[leaseAction] = renewLease
+	headers[headerLeaseID] = currentLeaseID
+
+	_, err := b.leaseCommonPut(headers, http.StatusOK, options)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

vendor/github.com/Azure/azure-sdk-for-go/storage/message.go

@@ -0,0 +1,171 @@
+package storage
+
+// Copyright 2017 Microsoft Corporation
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+import (
+	"encoding/xml"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strconv"
+	"time"
+)
+
+// Message represents an Azure message.
+type Message struct {
+	Queue        *Queue
+	Text         string      `xml:"MessageText"`
+	ID           string      `xml:"MessageId"`
+	Insertion    TimeRFC1123 `xml:"InsertionTime"`
+	Expiration   TimeRFC1123 `xml:"ExpirationTime"`
+	PopReceipt   string      `xml:"PopReceipt"`
+	NextVisible  TimeRFC1123 `xml:"TimeNextVisible"`
+	DequeueCount int         `xml:"DequeueCount"`
+}
+
+func (m *Message) buildPath() string {
+	return fmt.Sprintf("%s/%s", m.Queue.buildPathMessages(), m.ID)
+}
+
+// PutMessageOptions is the set of options can be specified for Put Messsage
+// operation. A zero struct does not use any preferences for the request.
+type PutMessageOptions struct {
+	Timeout           uint
+	VisibilityTimeout int
+	MessageTTL        int
+	RequestID         string `header:"x-ms-client-request-id"`
+}
+
+// Put operation adds a new message to the back of the message queue.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Put-Message
+func (m *Message) Put(options *PutMessageOptions) error {
+	query := url.Values{}
+	headers := m.Queue.qsc.client.getStandardHeaders()
+
+	req := putMessageRequest{MessageText: m.Text}
+	body, nn, err := xmlMarshal(req)
+	if err != nil {
+		return err
+	}
+	headers["Content-Length"] = strconv.Itoa(nn)
+
+	if options != nil {
+		if options.VisibilityTimeout != 0 {
+			query.Set("visibilitytimeout", strconv.Itoa(options.VisibilityTimeout))
+		}
+		if options.MessageTTL != 0 {
+			query.Set("messagettl", strconv.Itoa(options.MessageTTL))
+		}
+		query = addTimeout(query, options.Timeout)
+		headers = mergeHeaders(headers, headersFromStruct(*options))
+	}
+
+	uri := m.Queue.qsc.client.getEndpoint(queueServiceName, m.Queue.buildPathMessages(), query)
+	resp, err := m.Queue.qsc.client.exec(http.MethodPost, uri, headers, body, m.Queue.qsc.auth)
+	if err != nil {
+		return err
+	}
+	defer drainRespBody(resp)
+	err = checkRespCode(resp, []int{http.StatusCreated})
+	if err != nil {
+		return err
+	}
+	err = xmlUnmarshal(resp.Body, m)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// UpdateMessageOptions is the set of options can be specified for Update Messsage
+// operation. A zero struct does not use any preferences for the request.
+type UpdateMessageOptions struct {
+	Timeout           uint
+	VisibilityTimeout int
+	RequestID         string `header:"x-ms-client-request-id"`
+}
+
+// Update operation updates the specified message.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Update-Message
+func (m *Message) Update(options *UpdateMessageOptions) error {
+	query := url.Values{}
+	if m.PopReceipt != "" {
+		query.Set("popreceipt", m.PopReceipt)
+	}
+
+	headers := m.Queue.qsc.client.getStandardHeaders()
+	req := putMessageRequest{MessageText: m.Text}
+	body, nn, err := xmlMarshal(req)
+	if err != nil {
+		return err
+	}
+	headers["Content-Length"] = strconv.Itoa(nn)
+	// visibilitytimeout is required for Update (zero or greater) so set the default here
+	query.Set("visibilitytimeout", "0")
+	if options != nil {
+		if options.VisibilityTimeout != 0 {
+			query.Set("visibilitytimeout", strconv.Itoa(options.VisibilityTimeout))
+		}
+		query = addTimeout(query, options.Timeout)
+		headers = mergeHeaders(headers, headersFromStruct(*options))
+	}
+	uri := m.Queue.qsc.client.getEndpoint(queueServiceName, m.buildPath(), query)
+
+	resp, err := m.Queue.qsc.client.exec(http.MethodPut, uri, headers, body, m.Queue.qsc.auth)
+	if err != nil {
+		return err
+	}
+	defer drainRespBody(resp)
+
+	m.PopReceipt = resp.Header.Get("x-ms-popreceipt")
+	nextTimeStr := resp.Header.Get("x-ms-time-next-visible")
+	if nextTimeStr != "" {
+		nextTime, err := time.Parse(time.RFC1123, nextTimeStr)
+		if err != nil {
+			return err
+		}
+		m.NextVisible = TimeRFC1123(nextTime)
+	}
+
+	return checkRespCode(resp, []int{http.StatusNoContent})
+}
+
+// Delete operation deletes the specified message.
+//
+// See https://msdn.microsoft.com/en-us/library/azure/dd179347.aspx
+func (m *Message) Delete(options *QueueServiceOptions) error {
+	params := url.Values{"popreceipt": {m.PopReceipt}}
+	headers := m.Queue.qsc.client.getStandardHeaders()
+
+	if options != nil {
+		params = addTimeout(params, options.Timeout)
+		headers = mergeHeaders(headers, headersFromStruct(*options))
+	}
+	uri := m.Queue.qsc.client.getEndpoint(queueServiceName, m.buildPath(), params)
+
+	resp, err := m.Queue.qsc.client.exec(http.MethodDelete, uri, headers, nil, m.Queue.qsc.auth)
+	if err != nil {
+		return err
+	}
+	defer drainRespBody(resp)
+	return checkRespCode(resp, []int{http.StatusNoContent})
+}
+
+type putMessageRequest struct {
+	XMLName     xml.Name `xml:"QueueMessage"`
+	MessageText string   `xml:"MessageText"`
+}

vendor/github.com/Azure/azure-sdk-for-go/storage/odata.go

@@ -0,0 +1,47 @@
+package storage
+
+// Copyright 2017 Microsoft Corporation
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+// MetadataLevel determines if operations should return a paylod,
+// and it level of detail.
+type MetadataLevel string
+
+// This consts are meant to help with Odata supported operations
+const (
+	OdataTypeSuffix = "@odata.type"
+
+	// Types
+
+	OdataBinary   = "Edm.Binary"
+	OdataDateTime = "Edm.DateTime"
+	OdataGUID     = "Edm.Guid"
+	OdataInt64    = "Edm.Int64"
+
+	// Query options
+
+	OdataFilter  = "$filter"
+	OdataOrderBy = "$orderby"
+	OdataTop     = "$top"
+	OdataSkip    = "$skip"
+	OdataCount   = "$count"
+	OdataExpand  = "$expand"
+	OdataSelect  = "$select"
+	OdataSearch  = "$search"
+
+	EmptyPayload    MetadataLevel = ""
+	NoMetadata      MetadataLevel = "application/json;odata=nometadata"
+	MinimalMetadata MetadataLevel = "application/json;odata=minimalmetadata"
+	FullMetadata    MetadataLevel = "application/json;odata=fullmetadata"
+)

vendor/github.com/Azure/azure-sdk-for-go/storage/pageblob.go

@@ -0,0 +1,203 @@
+package storage
+
+// Copyright 2017 Microsoft Corporation
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+import (
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"time"
+)
+
+// GetPageRangesResponse contains the response fields from
+// Get Page Ranges call.
+//
+// See https://msdn.microsoft.com/en-us/library/azure/ee691973.aspx
+type GetPageRangesResponse struct {
+	XMLName  xml.Name    `xml:"PageList"`
+	PageList []PageRange `xml:"PageRange"`
+}
+
+// PageRange contains information about a page of a page blob from
+// Get Pages Range call.
+//
+// See https://msdn.microsoft.com/en-us/library/azure/ee691973.aspx
+type PageRange struct {
+	Start int64 `xml:"Start"`
+	End   int64 `xml:"End"`
+}
+
+var (
+	errBlobCopyAborted    = errors.New("storage: blob copy is aborted")
+	errBlobCopyIDMismatch = errors.New("storage: blob copy id is a mismatch")
+)
+
+// PutPageOptions includes the options for a put page operation
+type PutPageOptions struct {
+	Timeout                           uint
+	LeaseID                           string     `header:"x-ms-lease-id"`
+	IfSequenceNumberLessThanOrEqualTo *int       `header:"x-ms-if-sequence-number-le"`
+	IfSequenceNumberLessThan          *int       `header:"x-ms-if-sequence-number-lt"`
+	IfSequenceNumberEqualTo           *int       `header:"x-ms-if-sequence-number-eq"`
+	IfModifiedSince                   *time.Time `header:"If-Modified-Since"`
+	IfUnmodifiedSince                 *time.Time `header:"If-Unmodified-Since"`
+	IfMatch                           string     `header:"If-Match"`
+	IfNoneMatch                       string     `header:"If-None-Match"`
+	RequestID                         string     `header:"x-ms-client-request-id"`
+}
+
+// WriteRange writes a range of pages to a page blob.
+// Ranges must be aligned with 512-byte boundaries and chunk must be of size
+// multiplies by 512.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Put-Page
+func (b *Blob) WriteRange(blobRange BlobRange, bytes io.Reader, options *PutPageOptions) error {
+	if bytes == nil {
+		return errors.New("bytes cannot be nil")
+	}
+	return b.modifyRange(blobRange, bytes, options)
+}
+
+// ClearRange clears the given range in a page blob.
+// Ranges must be aligned with 512-byte boundaries and chunk must be of size
+// multiplies by 512.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Put-Page
+func (b *Blob) ClearRange(blobRange BlobRange, options *PutPageOptions) error {
+	return b.modifyRange(blobRange, nil, options)
+}
+
+func (b *Blob) modifyRange(blobRange BlobRange, bytes io.Reader, options *PutPageOptions) error {
+	if blobRange.End < blobRange.Start {
+		return errors.New("the value for rangeEnd must be greater than or equal to rangeStart")
+	}
+	if blobRange.Start%512 != 0 {
+		return errors.New("the value for rangeStart must be a multiple of 512")
+	}
+	if blobRange.End%512 != 511 {
+		return errors.New("the value for rangeEnd must be a multiple of 512 - 1")
+	}
+
+	params := url.Values{"comp": {"page"}}
+
+	// default to clear
+	write := "clear"
+	var cl uint64
+
+	// if bytes is not nil then this is an update operation
+	if bytes != nil {
+		write = "update"
+		cl = (blobRange.End - blobRange.Start) + 1
+	}
+
+	headers := b.Container.bsc.client.getStandardHeaders()
+	headers["x-ms-blob-type"] = string(BlobTypePage)
+	headers["x-ms-page-write"] = write
+	headers["x-ms-range"] = blobRange.String()
+	headers["Content-Length"] = fmt.Sprintf("%v", cl)
+
+	if options != nil {
+		params = addTimeout(params, options.Timeout)
+		headers = mergeHeaders(headers, headersFromStruct(*options))
+	}
+	uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), params)
+
+	resp, err := b.Container.bsc.client.exec(http.MethodPut, uri, headers, bytes, b.Container.bsc.auth)
+	if err != nil {
+		return err
+	}
+	defer drainRespBody(resp)
+	return checkRespCode(resp, []int{http.StatusCreated})
+}
+
+// GetPageRangesOptions includes the options for a get page ranges operation
+type GetPageRangesOptions struct {
+	Timeout          uint
+	Snapshot         *time.Time
+	PreviousSnapshot *time.Time
+	Range            *BlobRange
+	LeaseID          string `header:"x-ms-lease-id"`
+	RequestID        string `header:"x-ms-client-request-id"`
+}
+
+// GetPageRanges returns the list of valid page ranges for a page blob.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Get-Page-Ranges
+func (b *Blob) GetPageRanges(options *GetPageRangesOptions) (GetPageRangesResponse, error) {
+	params := url.Values{"comp": {"pagelist"}}
+	headers := b.Container.bsc.client.getStandardHeaders()
+
+	if options != nil {
+		params = addTimeout(params, options.Timeout)
+		params = addSnapshot(params, options.Snapshot)
+		if options.PreviousSnapshot != nil {
+			params.Add("prevsnapshot", timeRFC3339Formatted(*options.PreviousSnapshot))
+		}
+		if options.Range != nil {
+			headers["Range"] = options.Range.String()
+		}
+		headers = mergeHeaders(headers, headersFromStruct(*options))
+	}
+	uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), params)
+
+	var out GetPageRangesResponse
+	resp, err := b.Container.bsc.client.exec(http.MethodGet, uri, headers, nil, b.Container.bsc.auth)
+	if err != nil {
+		return out, err
+	}
+	defer drainRespBody(resp)
+
+	if err = checkRespCode(resp, []int{http.StatusOK}); err != nil {
+		return out, err
+	}
+	err = xmlUnmarshal(resp.Body, &out)
+	return out, err
+}
+
+// PutPageBlob initializes an empty page blob with specified name and maximum
+// size in bytes (size must be aligned to a 512-byte boundary). A page blob must
+// be created using this method before writing pages.
+//
+// See CreateBlockBlobFromReader for more info on creating blobs.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Put-Blob
+func (b *Blob) PutPageBlob(options *PutBlobOptions) error {
+	if b.Properties.ContentLength%512 != 0 {
+		return errors.New("Content length must be aligned to a 512-byte boundary")
+	}
+
+	params := url.Values{}
+	headers := b.Container.bsc.client.getStandardHeaders()
+	headers["x-ms-blob-type"] = string(BlobTypePage)
+	headers["x-ms-blob-content-length"] = fmt.Sprintf("%v", b.Properties.ContentLength)
+	headers["x-ms-blob-sequence-number"] = fmt.Sprintf("%v", b.Properties.SequenceNumber)
+	headers = mergeHeaders(headers, headersFromStruct(b.Properties))
+	headers = b.Container.bsc.client.addMetadataToHeaders(headers, b.Metadata)
+
+	if options != nil {
+		params = addTimeout(params, options.Timeout)
+		headers = mergeHeaders(headers, headersFromStruct(*options))
+	}
+	uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), params)
+
+	resp, err := b.Container.bsc.client.exec(http.MethodPut, uri, headers, nil, b.Container.bsc.auth)
+	if err != nil {
+		return err
+	}
+	return b.respondCreation(resp, BlobTypePage)
+}

vendor/github.com/Azure/azure-sdk-for-go/storage/queue.go

@@ -0,0 +1,436 @@
+package storage
+
+// Copyright 2017 Microsoft Corporation
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+import (
+	"encoding/xml"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strconv"
+	"time"
+)
+
+const (
+	// casing is per Golang's http.Header canonicalizing the header names.
+	approximateMessagesCountHeader = "X-Ms-Approximate-Messages-Count"
+)
+
+// QueueAccessPolicy represents each access policy in the queue ACL.
+type QueueAccessPolicy struct {
+	ID         string
+	StartTime  time.Time
+	ExpiryTime time.Time
+	CanRead    bool
+	CanAdd     bool
+	CanUpdate  bool
+	CanProcess bool
+}
+
+// QueuePermissions represents the queue ACLs.
+type QueuePermissions struct {
+	AccessPolicies []QueueAccessPolicy
+}
+
+// SetQueuePermissionOptions includes options for a set queue permissions operation
+type SetQueuePermissionOptions struct {
+	Timeout   uint
+	RequestID string `header:"x-ms-client-request-id"`
+}
+
+// Queue represents an Azure queue.
+type Queue struct {
+	qsc               *QueueServiceClient
+	Name              string
+	Metadata          map[string]string
+	AproxMessageCount uint64
+}
+
+func (q *Queue) buildPath() string {
+	return fmt.Sprintf("/%s", q.Name)
+}
+
+func (q *Queue) buildPathMessages() string {
+	return fmt.Sprintf("%s/messages", q.buildPath())
+}
+
+// QueueServiceOptions includes options for some queue service operations
+type QueueServiceOptions struct {
+	Timeout   uint
+	RequestID string `header:"x-ms-client-request-id"`
+}
+
+// Create operation creates a queue under the given account.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Create-Queue4
+func (q *Queue) Create(options *QueueServiceOptions) error {
+	params := url.Values{}
+	headers := q.qsc.client.getStandardHeaders()
+	headers = q.qsc.client.addMetadataToHeaders(headers, q.Metadata)
+
+	if options != nil {
+		params = addTimeout(params, options.Timeout)
+		headers = mergeHeaders(headers, headersFromStruct(*options))
+	}
+	uri := q.qsc.client.getEndpoint(queueServiceName, q.buildPath(), params)
+
+	resp, err := q.qsc.client.exec(http.MethodPut, uri, headers, nil, q.qsc.auth)
+	if err != nil {
+		return err
+	}
+	defer drainRespBody(resp)
+	return checkRespCode(resp, []int{http.StatusCreated})
+}
+
+// Delete operation permanently deletes the specified queue.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Delete-Queue3
+func (q *Queue) Delete(options *QueueServiceOptions) error {
+	params := url.Values{}
+	headers := q.qsc.client.getStandardHeaders()
+
+	if options != nil {
+		params = addTimeout(params, options.Timeout)
+		headers = mergeHeaders(headers, headersFromStruct(*options))
+	}
+	uri := q.qsc.client.getEndpoint(queueServiceName, q.buildPath(), params)
+	resp, err := q.qsc.client.exec(http.MethodDelete, uri, headers, nil, q.qsc.auth)
+	if err != nil {
+		return err
+	}
+	defer drainRespBody(resp)
+	return checkRespCode(resp, []int{http.StatusNoContent})
+}
+
+// Exists returns true if a queue with given name exists.
+func (q *Queue) Exists() (bool, error) {
+	uri := q.qsc.client.getEndpoint(queueServiceName, q.buildPath(), url.Values{"comp": {"metadata"}})
+	resp, err := q.qsc.client.exec(http.MethodGet, uri, q.qsc.client.getStandardHeaders(), nil, q.qsc.auth)
+	if resp != nil {
+		defer drainRespBody(resp)
+		if resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusNotFound {
+			return resp.StatusCode == http.StatusOK, nil
+		}
+		err = getErrorFromResponse(resp)
+	}
+	return false, err
+}
+
+// SetMetadata operation sets user-defined metadata on the specified queue.
+// Metadata is associated with the queue as name-value pairs.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Set-Queue-Metadata
+func (q *Queue) SetMetadata(options *QueueServiceOptions) error {
+	params := url.Values{"comp": {"metadata"}}
+	headers := q.qsc.client.getStandardHeaders()
+	headers = q.qsc.client.addMetadataToHeaders(headers, q.Metadata)
+
+	if options != nil {
+		params = addTimeout(params, options.Timeout)
+		headers = mergeHeaders(headers, headersFromStruct(*options))
+	}
+	uri := q.qsc.client.getEndpoint(queueServiceName, q.buildPath(), params)
+
+	resp, err := q.qsc.client.exec(http.MethodPut, uri, headers, nil, q.qsc.auth)
+	if err != nil {
+		return err
+	}
+	defer drainRespBody(resp)
+	return checkRespCode(resp, []int{http.StatusNoContent})
+}
+
+// GetMetadata operation retrieves user-defined metadata and queue
+// properties on the specified queue. Metadata is associated with
+// the queue as name-values pairs.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Set-Queue-Metadata
+//
+// Because the way Golang's http client (and http.Header in particular)
+// canonicalize header names, the returned metadata names would always
+// be all lower case.
+func (q *Queue) GetMetadata(options *QueueServiceOptions) error {
+	params := url.Values{"comp": {"metadata"}}
+	headers := q.qsc.client.getStandardHeaders()
+
+	if options != nil {
+		params = addTimeout(params, options.Timeout)
+		headers = mergeHeaders(headers, headersFromStruct(*options))
+	}
+	uri := q.qsc.client.getEndpoint(queueServiceName, q.buildPath(), url.Values{"comp": {"metadata"}})
+
+	resp, err := q.qsc.client.exec(http.MethodGet, uri, headers, nil, q.qsc.auth)
+	if err != nil {
+		return err
+	}
+	defer drainRespBody(resp)
+
+	if err := checkRespCode(resp, []int{http.StatusOK}); err != nil {
+		return err
+	}
+
+	aproxMessagesStr := resp.Header.Get(http.CanonicalHeaderKey(approximateMessagesCountHeader))
+	if aproxMessagesStr != "" {
+		aproxMessages, err := strconv.ParseUint(aproxMessagesStr, 10, 64)
+		if err != nil {
+			return err
+		}
+		q.AproxMessageCount = aproxMessages
+	}
+
+	q.Metadata = getMetadataFromHeaders(resp.Header)
+	return nil
+}
+
+// GetMessageReference returns a message object with the specified text.
+func (q *Queue) GetMessageReference(text string) *Message {
+	return &Message{
+		Queue: q,
+		Text:  text,
+	}
+}
+
+// GetMessagesOptions is the set of options can be specified for Get
+// Messsages operation. A zero struct does not use any preferences for the
+// request.
+type GetMessagesOptions struct {
+	Timeout           uint
+	NumOfMessages     int
+	VisibilityTimeout int
+	RequestID         string `header:"x-ms-client-request-id"`
+}
+
+type messages struct {
+	XMLName  xml.Name  `xml:"QueueMessagesList"`
+	Messages []Message `xml:"QueueMessage"`
+}
+
+// GetMessages operation retrieves one or more messages from the front of the
+// queue.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Get-Messages
+func (q *Queue) GetMessages(options *GetMessagesOptions) ([]Message, error) {
+	query := url.Values{}
+	headers := q.qsc.client.getStandardHeaders()
+
+	if options != nil {
+		if options.NumOfMessages != 0 {
+			query.Set("numofmessages", strconv.Itoa(options.NumOfMessages))
+		}
+		if options.VisibilityTimeout != 0 {
+			query.Set("visibilitytimeout", strconv.Itoa(options.VisibilityTimeout))
+		}
+		query = addTimeout(query, options.Timeout)
+		headers = mergeHeaders(headers, headersFromStruct(*options))
+	}
+	uri := q.qsc.client.getEndpoint(queueServiceName, q.buildPathMessages(), query)
+
+	resp, err := q.qsc.client.exec(http.MethodGet, uri, headers, nil, q.qsc.auth)
+	if err != nil {
+		return []Message{}, err
+	}
+	defer resp.Body.Close()
+
+	var out messages
+	err = xmlUnmarshal(resp.Body, &out)
+	if err != nil {
+		return []Message{}, err
+	}
+	for i := range out.Messages {
+		out.Messages[i].Queue = q
+	}
+	return out.Messages, err
+}
+
+// PeekMessagesOptions is the set of options can be specified for Peek
+// Messsage operation. A zero struct does not use any preferences for the
+// request.
+type PeekMessagesOptions struct {
+	Timeout       uint
+	NumOfMessages int
+	RequestID     string `header:"x-ms-client-request-id"`
+}
+
+// PeekMessages retrieves one or more messages from the front of the queue, but
+// does not alter the visibility of the message.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Peek-Messages
+func (q *Queue) PeekMessages(options *PeekMessagesOptions) ([]Message, error) {
+	query := url.Values{"peekonly": {"true"}} // Required for peek operation
+	headers := q.qsc.client.getStandardHeaders()
+
+	if options != nil {
+		if options.NumOfMessages != 0 {
+			query.Set("numofmessages", strconv.Itoa(options.NumOfMessages))
+		}
+		query = addTimeout(query, options.Timeout)
+		headers = mergeHeaders(headers, headersFromStruct(*options))
+	}
+	uri := q.qsc.client.getEndpoint(queueServiceName, q.buildPathMessages(), query)
+
+	resp, err := q.qsc.client.exec(http.MethodGet, uri, headers, nil, q.qsc.auth)
+	if err != nil {
+		return []Message{}, err
+	}
+	defer resp.Body.Close()
+
+	var out messages
+	err = xmlUnmarshal(resp.Body, &out)
+	if err != nil {
+		return []Message{}, err
+	}
+	for i := range out.Messages {
+		out.Messages[i].Queue = q
+	}
+	return out.Messages, err
+}
+
+// ClearMessages operation deletes all messages from the specified queue.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Clear-Messages
+func (q *Queue) ClearMessages(options *QueueServiceOptions) error {
+	params := url.Values{}
+	headers := q.qsc.client.getStandardHeaders()
+
+	if options != nil {
+		params = addTimeout(params, options.Timeout)
+		headers = mergeHeaders(headers, headersFromStruct(*options))
+	}
+	uri := q.qsc.client.getEndpoint(queueServiceName, q.buildPathMessages(), params)
+
+	resp, err := q.qsc.client.exec(http.MethodDelete, uri, headers, nil, q.qsc.auth)
+	if err != nil {
+		return err
+	}
+	defer drainRespBody(resp)
+	return checkRespCode(resp, []int{http.StatusNoContent})
+}
+
+// SetPermissions sets up queue permissions
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/set-queue-acl
+func (q *Queue) SetPermissions(permissions QueuePermissions, options *SetQueuePermissionOptions) error {
+	body, length, err := generateQueueACLpayload(permissions.AccessPolicies)
+	if err != nil {
+		return err
+	}
+
+	params := url.Values{
+		"comp": {"acl"},
+	}
+	headers := q.qsc.client.getStandardHeaders()
+	headers["Content-Length"] = strconv.Itoa(length)
+
+	if options != nil {
+		params = addTimeout(params, options.Timeout)
+		headers = mergeHeaders(headers, headersFromStruct(*options))
+	}
+	uri := q.qsc.client.getEndpoint(queueServiceName, q.buildPath(), params)
+	resp, err := q.qsc.client.exec(http.MethodPut, uri, headers, body, q.qsc.auth)
+	if err != nil {
+		return err
+	}
+	defer drainRespBody(resp)
+	return checkRespCode(resp, []int{http.StatusNoContent})
+}
+
+func generateQueueACLpayload(policies []QueueAccessPolicy) (io.Reader, int, error) {
+	sil := SignedIdentifiers{
+		SignedIdentifiers: []SignedIdentifier{},
+	}
+	for _, qapd := range policies {
+		permission := qapd.generateQueuePermissions()
+		signedIdentifier := convertAccessPolicyToXMLStructs(qapd.ID, qapd.StartTime, qapd.ExpiryTime, permission)
+		sil.SignedIdentifiers = append(sil.SignedIdentifiers, signedIdentifier)
+	}
+	return xmlMarshal(sil)
+}
+
+func (qapd *QueueAccessPolicy) generateQueuePermissions() (permissions string) {
+	// generate the permissions string (raup).
+	// still want the end user API to have bool flags.
+	permissions = ""
+
+	if qapd.CanRead {
+		permissions += "r"
+	}
+
+	if qapd.CanAdd {
+		permissions += "a"
+	}
+
+	if qapd.CanUpdate {
+		permissions += "u"
+	}
+
+	if qapd.CanProcess {
+		permissions += "p"
+	}
+
+	return permissions
+}
+
+// GetQueuePermissionOptions includes options for a get queue permissions operation
+type GetQueuePermissionOptions struct {
+	Timeout   uint
+	RequestID string `header:"x-ms-client-request-id"`
+}
+
+// GetPermissions gets the queue permissions as per https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/get-queue-acl
+// If timeout is 0 then it will not be passed to Azure
+func (q *Queue) GetPermissions(options *GetQueuePermissionOptions) (*QueuePermissions, error) {
+	params := url.Values{
+		"comp": {"acl"},
+	}
+	headers := q.qsc.client.getStandardHeaders()
+
+	if options != nil {
+		params = addTimeout(params, options.Timeout)
+		headers = mergeHeaders(headers, headersFromStruct(*options))
+	}
+	uri := q.qsc.client.getEndpoint(queueServiceName, q.buildPath(), params)
+	resp, err := q.qsc.client.exec(http.MethodGet, uri, headers, nil, q.qsc.auth)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	var ap AccessPolicy
+	err = xmlUnmarshal(resp.Body, &ap.SignedIdentifiersList)
+	if err != nil {
+		return nil, err
+	}
+	return buildQueueAccessPolicy(ap, &resp.Header), nil
+}
+
+func buildQueueAccessPolicy(ap AccessPolicy, headers *http.Header) *QueuePermissions {
+	permissions := QueuePermissions{
+		AccessPolicies: []QueueAccessPolicy{},
+	}
+
+	for _, policy := range ap.SignedIdentifiersList.SignedIdentifiers {
+		qapd := QueueAccessPolicy{
+			ID:         policy.ID,
+			StartTime:  policy.AccessPolicy.StartTime,
+			ExpiryTime: policy.AccessPolicy.ExpiryTime,
+		}
+		qapd.CanRead = updatePermissions(policy.AccessPolicy.Permission, "r")
+		qapd.CanAdd = updatePermissions(policy.AccessPolicy.Permission, "a")
+		qapd.CanUpdate = updatePermissions(policy.AccessPolicy.Permission, "u")
+		qapd.CanProcess = updatePermissions(policy.AccessPolicy.Permission, "p")
+
+		permissions.AccessPolicies = append(permissions.AccessPolicies, qapd)
+	}
+	return &permissions
+}

vendor/github.com/Azure/azure-sdk-for-go/storage/queuesasuri.go

@@ -0,0 +1,146 @@
+package storage
+
+// Copyright 2017 Microsoft Corporation
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+import (
+	"errors"
+	"fmt"
+	"net/url"
+	"strings"
+	"time"
+)
+
+// QueueSASOptions are options to construct a blob SAS
+// URI.
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/constructing-a-service-sas
+type QueueSASOptions struct {
+	QueueSASPermissions
+	SASOptions
+}
+
+// QueueSASPermissions includes the available permissions for
+// a queue SAS URI.
+type QueueSASPermissions struct {
+	Read    bool
+	Add     bool
+	Update  bool
+	Process bool
+}
+
+func (q QueueSASPermissions) buildString() string {
+	permissions := ""
+
+	if q.Read {
+		permissions += "r"
+	}
+	if q.Add {
+		permissions += "a"
+	}
+	if q.Update {
+		permissions += "u"
+	}
+	if q.Process {
+		permissions += "p"
+	}
+	return permissions
+}
+
+// GetSASURI creates an URL to the specified queue which contains the Shared
+// Access Signature with specified permissions and expiration time.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/constructing-a-service-sas
+func (q *Queue) GetSASURI(options QueueSASOptions) (string, error) {
+	canonicalizedResource, err := q.qsc.client.buildCanonicalizedResource(q.buildPath(), q.qsc.auth, true)
+	if err != nil {
+		return "", err
+	}
+
+	// "The canonicalizedresouce portion of the string is a canonical path to the signed resource.
+	// It must include the service name (blob, table, queue or file) for version 2015-02-21 or
+	// later, the storage account name, and the resource name, and must be URL-decoded.
+	// -- https://msdn.microsoft.com/en-us/library/azure/dn140255.aspx
+	// We need to replace + with %2b first to avoid being treated as a space (which is correct for query strings, but not the path component).
+	canonicalizedResource = strings.Replace(canonicalizedResource, "+", "%2b", -1)
+	canonicalizedResource, err = url.QueryUnescape(canonicalizedResource)
+	if err != nil {
+		return "", err
+	}
+
+	signedStart := ""
+	if options.Start != (time.Time{}) {
+		signedStart = options.Start.UTC().Format(time.RFC3339)
+	}
+	signedExpiry := options.Expiry.UTC().Format(time.RFC3339)
+
+	protocols := "https,http"
+	if options.UseHTTPS {
+		protocols = "https"
+	}
+
+	permissions := options.QueueSASPermissions.buildString()
+	stringToSign, err := queueSASStringToSign(q.qsc.client.apiVersion, canonicalizedResource, signedStart, signedExpiry, options.IP, permissions, protocols, options.Identifier)
+	if err != nil {
+		return "", err
+	}
+
+	sig := q.qsc.client.computeHmac256(stringToSign)
+	sasParams := url.Values{
+		"sv":  {q.qsc.client.apiVersion},
+		"se":  {signedExpiry},
+		"sp":  {permissions},
+		"sig": {sig},
+	}
+
+	if q.qsc.client.apiVersion >= "2015-04-05" {
+		sasParams.Add("spr", protocols)
+		addQueryParameter(sasParams, "sip", options.IP)
+	}
+
+	uri := q.qsc.client.getEndpoint(queueServiceName, q.buildPath(), nil)
+	sasURL, err := url.Parse(uri)
+	if err != nil {
+		return "", err
+	}
+	sasURL.RawQuery = sasParams.Encode()
+	return sasURL.String(), nil
+}
+
+func queueSASStringToSign(signedVersion, canonicalizedResource, signedStart, signedExpiry, signedIP, signedPermissions, protocols, signedIdentifier string) (string, error) {
+
+	if signedVersion >= "2015-02-21" {
+		canonicalizedResource = "/queue" + canonicalizedResource
+	}
+
+	// https://msdn.microsoft.com/en-us/library/azure/dn140255.aspx#Anchor_12
+	if signedVersion >= "2015-04-05" {
+		return fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s",
+			signedPermissions,
+			signedStart,
+			signedExpiry,
+			canonicalizedResource,
+			signedIdentifier,
+			signedIP,
+			protocols,
+			signedVersion), nil
+
+	}
+
+	// reference: http://msdn.microsoft.com/en-us/library/azure/dn140255.aspx
+	if signedVersion >= "2013-08-15" {
+		return fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s", signedPermissions, signedStart, signedExpiry, canonicalizedResource, signedIdentifier, signedVersion), nil
+	}
+
+	return "", errors.New("storage: not implemented SAS for versions earlier than 2013-08-15")
+}

vendor/github.com/Azure/azure-sdk-for-go/storage/queueserviceclient.go

@@ -0,0 +1,42 @@
+package storage
+
+// Copyright 2017 Microsoft Corporation
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+// QueueServiceClient contains operations for Microsoft Azure Queue Storage
+// Service.
+type QueueServiceClient struct {
+	client Client
+	auth   authentication
+}
+
+// GetServiceProperties gets the properties of your storage account's queue service.
+// See: https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/get-queue-service-properties
+func (q *QueueServiceClient) GetServiceProperties() (*ServiceProperties, error) {
+	return q.client.getServiceProperties(queueServiceName, q.auth)
+}
+
+// SetServiceProperties sets the properties of your storage account's queue service.
+// See: https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/set-queue-service-properties
+func (q *QueueServiceClient) SetServiceProperties(props ServiceProperties) error {
+	return q.client.setServiceProperties(props, queueServiceName, q.auth)
+}
+
+// GetQueueReference returns a Container object for the specified queue name.
+func (q *QueueServiceClient) GetQueueReference(name string) *Queue {
+	return &Queue{
+		qsc:  q,
+		Name: name,
+	}
+}

vendor/github.com/Azure/azure-sdk-for-go/storage/share.go

@@ -0,0 +1,216 @@
+package storage
+
+// Copyright 2017 Microsoft Corporation
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+	"strconv"
+)
+
+// Share represents an Azure file share.
+type Share struct {
+	fsc        *FileServiceClient
+	Name       string          `xml:"Name"`
+	Properties ShareProperties `xml:"Properties"`
+	Metadata   map[string]string
+}
+
+// ShareProperties contains various properties of a share.
+type ShareProperties struct {
+	LastModified string `xml:"Last-Modified"`
+	Etag         string `xml:"Etag"`
+	Quota        int    `xml:"Quota"`
+}
+
+// builds the complete path for this share object.
+func (s *Share) buildPath() string {
+	return fmt.Sprintf("/%s", s.Name)
+}
+
+// Create this share under the associated account.
+// If a share with the same name already exists, the operation fails.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Create-Share
+func (s *Share) Create(options *FileRequestOptions) error {
+	extraheaders := map[string]string{}
+	if s.Properties.Quota > 0 {
+		extraheaders["x-ms-share-quota"] = strconv.Itoa(s.Properties.Quota)
+	}
+
+	params := prepareOptions(options)
+	headers, err := s.fsc.createResource(s.buildPath(), resourceShare, params, mergeMDIntoExtraHeaders(s.Metadata, extraheaders), []int{http.StatusCreated})
+	if err != nil {
+		return err
+	}
+
+	s.updateEtagAndLastModified(headers)
+	return nil
+}
+
+// CreateIfNotExists creates this share under the associated account if
+// it does not exist. Returns true if the share is newly created or false if
+// the share already exists.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Create-Share
+func (s *Share) CreateIfNotExists(options *FileRequestOptions) (bool, error) {
+	extraheaders := map[string]string{}
+	if s.Properties.Quota > 0 {
+		extraheaders["x-ms-share-quota"] = strconv.Itoa(s.Properties.Quota)
+	}
+
+	params := prepareOptions(options)
+	resp, err := s.fsc.createResourceNoClose(s.buildPath(), resourceShare, params, extraheaders)
+	if resp != nil {
+		defer drainRespBody(resp)
+		if resp.StatusCode == http.StatusCreated || resp.StatusCode == http.StatusConflict {
+			if resp.StatusCode == http.StatusCreated {
+				s.updateEtagAndLastModified(resp.Header)
+				return true, nil
+			}
+			return false, s.FetchAttributes(nil)
+		}
+	}
+
+	return false, err
+}
+
+// Delete marks this share for deletion. The share along with any files
+// and directories contained within it are later deleted during garbage
+// collection.  If the share does not exist the operation fails
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Delete-Share
+func (s *Share) Delete(options *FileRequestOptions) error {
+	return s.fsc.deleteResource(s.buildPath(), resourceShare, options)
+}
+
+// DeleteIfExists operation marks this share for deletion if it exists.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Delete-Share
+func (s *Share) DeleteIfExists(options *FileRequestOptions) (bool, error) {
+	resp, err := s.fsc.deleteResourceNoClose(s.buildPath(), resourceShare, options)
+	if resp != nil {
+		defer drainRespBody(resp)
+		if resp.StatusCode == http.StatusAccepted || resp.StatusCode == http.StatusNotFound {
+			return resp.StatusCode == http.StatusAccepted, nil
+		}
+	}
+	return false, err
+}
+
+// Exists returns true if this share already exists
+// on the storage account, otherwise returns false.
+func (s *Share) Exists() (bool, error) {
+	exists, headers, err := s.fsc.resourceExists(s.buildPath(), resourceShare)
+	if exists {
+		s.updateEtagAndLastModified(headers)
+		s.updateQuota(headers)
+	}
+	return exists, err
+}
+
+// FetchAttributes retrieves metadata and properties for this share.
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/get-share-properties
+func (s *Share) FetchAttributes(options *FileRequestOptions) error {
+	params := prepareOptions(options)
+	headers, err := s.fsc.getResourceHeaders(s.buildPath(), compNone, resourceShare, params, http.MethodHead)
+	if err != nil {
+		return err
+	}
+
+	s.updateEtagAndLastModified(headers)
+	s.updateQuota(headers)
+	s.Metadata = getMetadataFromHeaders(headers)
+
+	return nil
+}
+
+// GetRootDirectoryReference returns a Directory object at the root of this share.
+func (s *Share) GetRootDirectoryReference() *Directory {
+	return &Directory{
+		fsc:   s.fsc,
+		share: s,
+	}
+}
+
+// ServiceClient returns the FileServiceClient associated with this share.
+func (s *Share) ServiceClient() *FileServiceClient {
+	return s.fsc
+}
+
+// SetMetadata replaces the metadata for this share.
+//
+// Some keys may be converted to Camel-Case before sending. All keys
+// are returned in lower case by GetShareMetadata. HTTP header names
+// are case-insensitive so case munging should not matter to other
+// applications either.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/set-share-metadata
+func (s *Share) SetMetadata(options *FileRequestOptions) error {
+	headers, err := s.fsc.setResourceHeaders(s.buildPath(), compMetadata, resourceShare, mergeMDIntoExtraHeaders(s.Metadata, nil), options)
+	if err != nil {
+		return err
+	}
+
+	s.updateEtagAndLastModified(headers)
+	return nil
+}
+
+// SetProperties sets system properties for this share.
+//
+// Some keys may be converted to Camel-Case before sending. All keys
+// are returned in lower case by SetShareProperties. HTTP header names
+// are case-insensitive so case munging should not matter to other
+// applications either.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Set-Share-Properties
+func (s *Share) SetProperties(options *FileRequestOptions) error {
+	extraheaders := map[string]string{}
+	if s.Properties.Quota > 0 {
+		if s.Properties.Quota > 5120 {
+			return fmt.Errorf("invalid value %v for quota, valid values are [1, 5120]", s.Properties.Quota)
+		}
+		extraheaders["x-ms-share-quota"] = strconv.Itoa(s.Properties.Quota)
+	}
+
+	headers, err := s.fsc.setResourceHeaders(s.buildPath(), compProperties, resourceShare, extraheaders, options)
+	if err != nil {
+		return err
+	}
+
+	s.updateEtagAndLastModified(headers)
+	return nil
+}
+
+// updates Etag and last modified date
+func (s *Share) updateEtagAndLastModified(headers http.Header) {
+	s.Properties.Etag = headers.Get("Etag")
+	s.Properties.LastModified = headers.Get("Last-Modified")
+}
+
+// updates quota value
+func (s *Share) updateQuota(headers http.Header) {
+	quota, err := strconv.Atoi(headers.Get("x-ms-share-quota"))
+	if err == nil {
+		s.Properties.Quota = quota
+	}
+}
+
+// URL gets the canonical URL to this share. This method does not create a publicly accessible
+// URL if the share is private and this method does not check if the share exists.
+func (s *Share) URL() string {
+	return s.fsc.client.getEndpoint(fileServiceName, s.buildPath(), url.Values{})
+}

vendor/github.com/Azure/azure-sdk-for-go/storage/storagepolicy.go

@@ -0,0 +1,61 @@
+package storage
+
+// Copyright 2017 Microsoft Corporation
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+import (
+	"strings"
+	"time"
+)
+
+// AccessPolicyDetailsXML has specifics about an access policy
+// annotated with XML details.
+type AccessPolicyDetailsXML struct {
+	StartTime  time.Time `xml:"Start"`
+	ExpiryTime time.Time `xml:"Expiry"`
+	Permission string    `xml:"Permission"`
+}
+
+// SignedIdentifier is a wrapper for a specific policy
+type SignedIdentifier struct {
+	ID           string                 `xml:"Id"`
+	AccessPolicy AccessPolicyDetailsXML `xml:"AccessPolicy"`
+}
+
+// SignedIdentifiers part of the response from GetPermissions call.
+type SignedIdentifiers struct {
+	SignedIdentifiers []SignedIdentifier `xml:"SignedIdentifier"`
+}
+
+// AccessPolicy is the response type from the GetPermissions call.
+type AccessPolicy struct {
+	SignedIdentifiersList SignedIdentifiers `xml:"SignedIdentifiers"`
+}
+
+// convertAccessPolicyToXMLStructs converts between AccessPolicyDetails which is a struct better for API usage to the
+// AccessPolicy struct which will get converted to XML.
+func convertAccessPolicyToXMLStructs(id string, startTime time.Time, expiryTime time.Time, permissions string) SignedIdentifier {
+	return SignedIdentifier{
+		ID: id,
+		AccessPolicy: AccessPolicyDetailsXML{
+			StartTime:  startTime.UTC().Round(time.Second),
+			ExpiryTime: expiryTime.UTC().Round(time.Second),
+			Permission: permissions,
+		},
+	}
+}
+
+func updatePermissions(permissions, permission string) bool {
+	return strings.Contains(permissions, permission)
+}

vendor/github.com/Azure/azure-sdk-for-go/storage/storageservice.go

@@ -0,0 +1,131 @@
+package storage
+
+// Copyright 2017 Microsoft Corporation
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+import (
+	"net/http"
+	"net/url"
+	"strconv"
+)
+
+// ServiceProperties represents the storage account service properties
+type ServiceProperties struct {
+	Logging       *Logging
+	HourMetrics   *Metrics
+	MinuteMetrics *Metrics
+	Cors          *Cors
+}
+
+// Logging represents the Azure Analytics Logging settings
+type Logging struct {
+	Version         string
+	Delete          bool
+	Read            bool
+	Write           bool
+	RetentionPolicy *RetentionPolicy
+}
+
+// RetentionPolicy indicates if retention is enabled and for how many days
+type RetentionPolicy struct {
+	Enabled bool
+	Days    *int
+}
+
+// Metrics provide request statistics.
+type Metrics struct {
+	Version         string
+	Enabled         bool
+	IncludeAPIs     *bool
+	RetentionPolicy *RetentionPolicy
+}
+
+// Cors includes all the CORS rules
+type Cors struct {
+	CorsRule []CorsRule
+}
+
+// CorsRule includes all settings for a Cors rule
+type CorsRule struct {
+	AllowedOrigins  string
+	AllowedMethods  string
+	MaxAgeInSeconds int
+	ExposedHeaders  string
+	AllowedHeaders  string
+}
+
+func (c Client) getServiceProperties(service string, auth authentication) (*ServiceProperties, error) {
+	query := url.Values{
+		"restype": {"service"},
+		"comp":    {"properties"},
+	}
+	uri := c.getEndpoint(service, "", query)
+	headers := c.getStandardHeaders()
+
+	resp, err := c.exec(http.MethodGet, uri, headers, nil, auth)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if err := checkRespCode(resp, []int{http.StatusOK}); err != nil {
+		return nil, err
+	}
+
+	var out ServiceProperties
+	err = xmlUnmarshal(resp.Body, &out)
+	if err != nil {
+		return nil, err
+	}
+
+	return &out, nil
+}
+
+func (c Client) setServiceProperties(props ServiceProperties, service string, auth authentication) error {
+	query := url.Values{
+		"restype": {"service"},
+		"comp":    {"properties"},
+	}
+	uri := c.getEndpoint(service, "", query)
+
+	// Ideally, StorageServiceProperties would be the output struct
+	// This is to avoid golint stuttering, while generating the correct XML
+	type StorageServiceProperties struct {
+		Logging       *Logging
+		HourMetrics   *Metrics
+		MinuteMetrics *Metrics
+		Cors          *Cors
+	}
+	input := StorageServiceProperties{
+		Logging:       props.Logging,
+		HourMetrics:   props.HourMetrics,
+		MinuteMetrics: props.MinuteMetrics,
+		Cors:          props.Cors,
+	}
+
+	body, length, err := xmlMarshal(input)
+	if err != nil {
+		return err
+	}
+
+	headers := c.getStandardHeaders()
+	headers["Content-Length"] = strconv.Itoa(length)
+
+	resp, err := c.exec(http.MethodPut, uri, headers, body, auth)
+	if err != nil {
+		return err
+	}
+	defer drainRespBody(resp)
+	return checkRespCode(resp, []int{http.StatusAccepted})
+}

vendor/github.com/Azure/azure-sdk-for-go/storage/table.go

@@ -0,0 +1,419 @@
+package storage
+
+// Copyright 2017 Microsoft Corporation
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+)
+
+const (
+	tablesURIPath                  = "/Tables"
+	nextTableQueryParameter        = "NextTableName"
+	headerNextPartitionKey         = "x-ms-continuation-NextPartitionKey"
+	headerNextRowKey               = "x-ms-continuation-NextRowKey"
+	nextPartitionKeyQueryParameter = "NextPartitionKey"
+	nextRowKeyQueryParameter       = "NextRowKey"
+)
+
+// TableAccessPolicy are used for SETTING table policies
+type TableAccessPolicy struct {
+	ID         string
+	StartTime  time.Time
+	ExpiryTime time.Time
+	CanRead    bool
+	CanAppend  bool
+	CanUpdate  bool
+	CanDelete  bool
+}
+
+// Table represents an Azure table.
+type Table struct {
+	tsc           *TableServiceClient
+	Name          string `json:"TableName"`
+	OdataEditLink string `json:"odata.editLink"`
+	OdataID       string `json:"odata.id"`
+	OdataMetadata string `json:"odata.metadata"`
+	OdataType     string `json:"odata.type"`
+}
+
+// EntityQueryResult contains the response from
+// ExecuteQuery and ExecuteQueryNextResults functions.
+type EntityQueryResult struct {
+	OdataMetadata string    `json:"odata.metadata"`
+	Entities      []*Entity `json:"value"`
+	QueryNextLink
+	table *Table
+}
+
+type continuationToken struct {
+	NextPartitionKey string
+	NextRowKey       string
+}
+
+func (t *Table) buildPath() string {
+	return fmt.Sprintf("/%s", t.Name)
+}
+
+func (t *Table) buildSpecificPath() string {
+	return fmt.Sprintf("%s('%s')", tablesURIPath, t.Name)
+}
+
+// Get gets the referenced table.
+// See: https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/querying-tables-and-entities
+func (t *Table) Get(timeout uint, ml MetadataLevel) error {
+	if ml == EmptyPayload {
+		return errEmptyPayload
+	}
+
+	query := url.Values{
+		"timeout": {strconv.FormatUint(uint64(timeout), 10)},
+	}
+	headers := t.tsc.client.getStandardHeaders()
+	headers[headerAccept] = string(ml)
+
+	uri := t.tsc.client.getEndpoint(tableServiceName, t.buildSpecificPath(), query)
+	resp, err := t.tsc.client.exec(http.MethodGet, uri, headers, nil, t.tsc.auth)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if err = checkRespCode(resp, []int{http.StatusOK}); err != nil {
+		return err
+	}
+
+	respBody, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	err = json.Unmarshal(respBody, t)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// Create creates the referenced table.
+// This function fails if the name is not compliant
+// with the specification or the tables already exists.
+// ml determines the level of detail of metadata in the operation response,
+// or no data at all.
+// See https://docs.microsoft.com/rest/api/storageservices/fileservices/create-table
+func (t *Table) Create(timeout uint, ml MetadataLevel, options *TableOptions) error {
+	uri := t.tsc.client.getEndpoint(tableServiceName, tablesURIPath, url.Values{
+		"timeout": {strconv.FormatUint(uint64(timeout), 10)},
+	})
+
+	type createTableRequest struct {
+		TableName string `json:"TableName"`
+	}
+	req := createTableRequest{TableName: t.Name}
+	buf := new(bytes.Buffer)
+	if err := json.NewEncoder(buf).Encode(req); err != nil {
+		return err
+	}
+
+	headers := t.tsc.client.getStandardHeaders()
+	headers = addReturnContentHeaders(headers, ml)
+	headers = addBodyRelatedHeaders(headers, buf.Len())
+	headers = options.addToHeaders(headers)
+
+	resp, err := t.tsc.client.exec(http.MethodPost, uri, headers, buf, t.tsc.auth)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if ml == EmptyPayload {
+		if err := checkRespCode(resp, []int{http.StatusNoContent}); err != nil {
+			return err
+		}
+	} else {
+		if err := checkRespCode(resp, []int{http.StatusCreated}); err != nil {
+			return err
+		}
+	}
+
+	if ml != EmptyPayload {
+		data, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return err
+		}
+		err = json.Unmarshal(data, t)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// Delete deletes the referenced table.
+// This function fails if the table is not present.
+// Be advised: Delete deletes all the entries that may be present.
+// See https://docs.microsoft.com/rest/api/storageservices/fileservices/delete-table
+func (t *Table) Delete(timeout uint, options *TableOptions) error {
+	uri := t.tsc.client.getEndpoint(tableServiceName, t.buildSpecificPath(), url.Values{
+		"timeout": {strconv.Itoa(int(timeout))},
+	})
+
+	headers := t.tsc.client.getStandardHeaders()
+	headers = addReturnContentHeaders(headers, EmptyPayload)
+	headers = options.addToHeaders(headers)
+
+	resp, err := t.tsc.client.exec(http.MethodDelete, uri, headers, nil, t.tsc.auth)
+	if err != nil {
+		return err
+	}
+	defer drainRespBody(resp)
+
+	return checkRespCode(resp, []int{http.StatusNoContent})
+}
+
+// QueryOptions includes options for a query entities operation.
+// Top, filter and select are OData query options.
+type QueryOptions struct {
+	Top       uint
+	Filter    string
+	Select    []string
+	RequestID string
+}
+
+func (options *QueryOptions) getParameters() (url.Values, map[string]string) {
+	query := url.Values{}
+	headers := map[string]string{}
+	if options != nil {
+		if options.Top > 0 {
+			query.Add(OdataTop, strconv.FormatUint(uint64(options.Top), 10))
+		}
+		if options.Filter != "" {
+			query.Add(OdataFilter, options.Filter)
+		}
+		if len(options.Select) > 0 {
+			query.Add(OdataSelect, strings.Join(options.Select, ","))
+		}
+		headers = addToHeaders(headers, "x-ms-client-request-id", options.RequestID)
+	}
+	return query, headers
+}
+
+// QueryEntities returns the entities in the table.
+// You can use query options defined by the OData Protocol specification.
+//
+// See: https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/query-entities
+func (t *Table) QueryEntities(timeout uint, ml MetadataLevel, options *QueryOptions) (*EntityQueryResult, error) {
+	if ml == EmptyPayload {
+		return nil, errEmptyPayload
+	}
+	query, headers := options.getParameters()
+	query = addTimeout(query, timeout)
+	uri := t.tsc.client.getEndpoint(tableServiceName, t.buildPath(), query)
+	return t.queryEntities(uri, headers, ml)
+}
+
+// NextResults returns the next page of results
+// from a QueryEntities or NextResults operation.
+//
+// See: https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/query-entities
+// See https://docs.microsoft.com/rest/api/storageservices/fileservices/query-timeout-and-pagination
+func (eqr *EntityQueryResult) NextResults(options *TableOptions) (*EntityQueryResult, error) {
+	if eqr == nil {
+		return nil, errNilPreviousResult
+	}
+	if eqr.NextLink == nil {
+		return nil, errNilNextLink
+	}
+	headers := options.addToHeaders(map[string]string{})
+	return eqr.table.queryEntities(*eqr.NextLink, headers, eqr.ml)
+}
+
+// SetPermissions sets up table ACL permissions
+// See https://docs.microsoft.com/rest/api/storageservices/fileservices/Set-Table-ACL
+func (t *Table) SetPermissions(tap []TableAccessPolicy, timeout uint, options *TableOptions) error {
+	params := url.Values{"comp": {"acl"},
+		"timeout": {strconv.Itoa(int(timeout))},
+	}
+
+	uri := t.tsc.client.getEndpoint(tableServiceName, t.Name, params)
+	headers := t.tsc.client.getStandardHeaders()
+	headers = options.addToHeaders(headers)
+
+	body, length, err := generateTableACLPayload(tap)
+	if err != nil {
+		return err
+	}
+	headers["Content-Length"] = strconv.Itoa(length)
+
+	resp, err := t.tsc.client.exec(http.MethodPut, uri, headers, body, t.tsc.auth)
+	if err != nil {
+		return err
+	}
+	defer drainRespBody(resp)
+
+	return checkRespCode(resp, []int{http.StatusNoContent})
+}
+
+func generateTableACLPayload(policies []TableAccessPolicy) (io.Reader, int, error) {
+	sil := SignedIdentifiers{
+		SignedIdentifiers: []SignedIdentifier{},
+	}
+	for _, tap := range policies {
+		permission := generateTablePermissions(&tap)
+		signedIdentifier := convertAccessPolicyToXMLStructs(tap.ID, tap.StartTime, tap.ExpiryTime, permission)
+		sil.SignedIdentifiers = append(sil.SignedIdentifiers, signedIdentifier)
+	}
+	return xmlMarshal(sil)
+}
+
+// GetPermissions gets the table ACL permissions
+// See https://docs.microsoft.com/rest/api/storageservices/fileservices/get-table-acl
+func (t *Table) GetPermissions(timeout int, options *TableOptions) ([]TableAccessPolicy, error) {
+	params := url.Values{"comp": {"acl"},
+		"timeout": {strconv.Itoa(int(timeout))},
+	}
+
+	uri := t.tsc.client.getEndpoint(tableServiceName, t.Name, params)
+	headers := t.tsc.client.getStandardHeaders()
+	headers = options.addToHeaders(headers)
+
+	resp, err := t.tsc.client.exec(http.MethodGet, uri, headers, nil, t.tsc.auth)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if err = checkRespCode(resp, []int{http.StatusOK}); err != nil {
+		return nil, err
+	}
+
+	var ap AccessPolicy
+	err = xmlUnmarshal(resp.Body, &ap.SignedIdentifiersList)
+	if err != nil {
+		return nil, err
+	}
+	return updateTableAccessPolicy(ap), nil
+}
+
+func (t *Table) queryEntities(uri string, headers map[string]string, ml MetadataLevel) (*EntityQueryResult, error) {
+	headers = mergeHeaders(headers, t.tsc.client.getStandardHeaders())
+	if ml != EmptyPayload {
+		headers[headerAccept] = string(ml)
+	}
+
+	resp, err := t.tsc.client.exec(http.MethodGet, uri, headers, nil, t.tsc.auth)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if err = checkRespCode(resp, []int{http.StatusOK}); err != nil {
+		return nil, err
+	}
+
+	data, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+	var entities EntityQueryResult
+	err = json.Unmarshal(data, &entities)
+	if err != nil {
+		return nil, err
+	}
+
+	for i := range entities.Entities {
+		entities.Entities[i].Table = t
+	}
+	entities.table = t
+
+	contToken := extractContinuationTokenFromHeaders(resp.Header)
+	if contToken == nil {
+		entities.NextLink = nil
+	} else {
+		originalURI, err := url.Parse(uri)
+		if err != nil {
+			return nil, err
+		}
+		v := originalURI.Query()
+		v.Set(nextPartitionKeyQueryParameter, contToken.NextPartitionKey)
+		v.Set(nextRowKeyQueryParameter, contToken.NextRowKey)
+		newURI := t.tsc.client.getEndpoint(tableServiceName, t.buildPath(), v)
+		entities.NextLink = &newURI
+		entities.ml = ml
+	}
+
+	return &entities, nil
+}
+
+func extractContinuationTokenFromHeaders(h http.Header) *continuationToken {
+	ct := continuationToken{
+		NextPartitionKey: h.Get(headerNextPartitionKey),
+		NextRowKey:       h.Get(headerNextRowKey),
+	}
+
+	if ct.NextPartitionKey != "" && ct.NextRowKey != "" {
+		return &ct
+	}
+	return nil
+}
+
+func updateTableAccessPolicy(ap AccessPolicy) []TableAccessPolicy {
+	taps := []TableAccessPolicy{}
+	for _, policy := range ap.SignedIdentifiersList.SignedIdentifiers {
+		tap := TableAccessPolicy{
+			ID:         policy.ID,
+			StartTime:  policy.AccessPolicy.StartTime,
+			ExpiryTime: policy.AccessPolicy.ExpiryTime,
+		}
+		tap.CanRead = updatePermissions(policy.AccessPolicy.Permission, "r")
+		tap.CanAppend = updatePermissions(policy.AccessPolicy.Permission, "a")
+		tap.CanUpdate = updatePermissions(policy.AccessPolicy.Permission, "u")
+		tap.CanDelete = updatePermissions(policy.AccessPolicy.Permission, "d")
+
+		taps = append(taps, tap)
+	}
+	return taps
+}
+
+func generateTablePermissions(tap *TableAccessPolicy) (permissions string) {
+	// generate the permissions string (raud).
+	// still want the end user API to have bool flags.
+	permissions = ""
+
+	if tap.CanRead {
+		permissions += "r"
+	}
+
+	if tap.CanAppend {
+		permissions += "a"
+	}
+
+	if tap.CanUpdate {
+		permissions += "u"
+	}
+
+	if tap.CanDelete {
+		permissions += "d"
+	}
+	return permissions
+}

vendor/github.com/Azure/azure-sdk-for-go/storage/table_batch.go

@@ -0,0 +1,328 @@
+package storage
+
+// Copyright 2017 Microsoft Corporation
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"net/textproto"
+	"sort"
+	"strings"
+
+	"github.com/marstr/guid"
+)
+
+// Operation type. Insert, Delete, Replace etc.
+type Operation int
+
+// consts for batch operations.
+const (
+	InsertOp          = Operation(1)
+	DeleteOp          = Operation(2)
+	ReplaceOp         = Operation(3)
+	MergeOp           = Operation(4)
+	InsertOrReplaceOp = Operation(5)
+	InsertOrMergeOp   = Operation(6)
+)
+
+// BatchEntity used for tracking Entities to operate on and
+// whether operations (replace/merge etc) should be forced.
+// Wrapper for regular Entity with additional data specific for the entity.
+type BatchEntity struct {
+	*Entity
+	Force bool
+	Op    Operation
+}
+
+// TableBatch stores all the entities that will be operated on during a batch process.
+// Entities can be inserted, replaced or deleted.
+type TableBatch struct {
+	BatchEntitySlice []BatchEntity
+
+	// reference to table we're operating on.
+	Table *Table
+}
+
+// defaultChangesetHeaders for changeSets
+var defaultChangesetHeaders = map[string]string{
+	"Accept":       "application/json;odata=minimalmetadata",
+	"Content-Type": "application/json",
+	"Prefer":       "return-no-content",
+}
+
+// NewBatch return new TableBatch for populating.
+func (t *Table) NewBatch() *TableBatch {
+	return &TableBatch{
+		Table: t,
+	}
+}
+
+// InsertEntity adds an entity in preparation for a batch insert.
+func (t *TableBatch) InsertEntity(entity *Entity) {
+	be := BatchEntity{Entity: entity, Force: false, Op: InsertOp}
+	t.BatchEntitySlice = append(t.BatchEntitySlice, be)
+}
+
+// InsertOrReplaceEntity adds an entity in preparation for a batch insert or replace.
+func (t *TableBatch) InsertOrReplaceEntity(entity *Entity, force bool) {
+	be := BatchEntity{Entity: entity, Force: false, Op: InsertOrReplaceOp}
+	t.BatchEntitySlice = append(t.BatchEntitySlice, be)
+}
+
+// InsertOrReplaceEntityByForce adds an entity in preparation for a batch insert or replace. Forces regardless of ETag
+func (t *TableBatch) InsertOrReplaceEntityByForce(entity *Entity) {
+	t.InsertOrReplaceEntity(entity, true)
+}
+
+// InsertOrMergeEntity adds an entity in preparation for a batch insert or merge.
+func (t *TableBatch) InsertOrMergeEntity(entity *Entity, force bool) {
+	be := BatchEntity{Entity: entity, Force: false, Op: InsertOrMergeOp}
+	t.BatchEntitySlice = append(t.BatchEntitySlice, be)
+}
+
+// InsertOrMergeEntityByForce adds an entity in preparation for a batch insert or merge. Forces regardless of ETag
+func (t *TableBatch) InsertOrMergeEntityByForce(entity *Entity) {
+	t.InsertOrMergeEntity(entity, true)
+}
+
+// ReplaceEntity adds an entity in preparation for a batch replace.
+func (t *TableBatch) ReplaceEntity(entity *Entity) {
+	be := BatchEntity{Entity: entity, Force: false, Op: ReplaceOp}
+	t.BatchEntitySlice = append(t.BatchEntitySlice, be)
+}
+
+// DeleteEntity adds an entity in preparation for a batch delete
+func (t *TableBatch) DeleteEntity(entity *Entity, force bool) {
+	be := BatchEntity{Entity: entity, Force: false, Op: DeleteOp}
+	t.BatchEntitySlice = append(t.BatchEntitySlice, be)
+}
+
+// DeleteEntityByForce adds an entity in preparation for a batch delete. Forces regardless of ETag
+func (t *TableBatch) DeleteEntityByForce(entity *Entity, force bool) {
+	t.DeleteEntity(entity, true)
+}
+
+// MergeEntity adds an entity in preparation for a batch merge
+func (t *TableBatch) MergeEntity(entity *Entity) {
+	be := BatchEntity{Entity: entity, Force: false, Op: MergeOp}
+	t.BatchEntitySlice = append(t.BatchEntitySlice, be)
+}
+
+// ExecuteBatch executes many table operations in one request to Azure.
+// The operations can be combinations of Insert, Delete, Replace and Merge
+// Creates the inner changeset body (various operations, Insert, Delete etc) then creates the outer request packet that encompasses
+// the changesets.
+// As per document https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/performing-entity-group-transactions
+func (t *TableBatch) ExecuteBatch() error {
+
+	// Using `github.com/marstr/guid` is in response to issue #947 (https://github.com/Azure/azure-sdk-for-go/issues/947).
+	id, err := guid.NewGUIDs(guid.CreationStrategyVersion1)
+	if err != nil {
+		return err
+	}
+
+	changesetBoundary := fmt.Sprintf("changeset_%s", id.String())
+	uri := t.Table.tsc.client.getEndpoint(tableServiceName, "$batch", nil)
+	changesetBody, err := t.generateChangesetBody(changesetBoundary)
+	if err != nil {
+		return err
+	}
+
+	id, err = guid.NewGUIDs(guid.CreationStrategyVersion1)
+	if err != nil {
+		return err
+	}
+
+	boundary := fmt.Sprintf("batch_%s", id.String())
+	body, err := generateBody(changesetBody, changesetBoundary, boundary)
+	if err != nil {
+		return err
+	}
+
+	headers := t.Table.tsc.client.getStandardHeaders()
+	headers[headerContentType] = fmt.Sprintf("multipart/mixed; boundary=%s", boundary)
+
+	resp, err := t.Table.tsc.client.execBatchOperationJSON(http.MethodPost, uri, headers, bytes.NewReader(body.Bytes()), t.Table.tsc.auth)
+	if err != nil {
+		return err
+	}
+	defer drainRespBody(resp.resp)
+
+	if err = checkRespCode(resp.resp, []int{http.StatusAccepted}); err != nil {
+
+		// check which batch failed.
+		operationFailedMessage := t.getFailedOperation(resp.odata.Err.Message.Value)
+		requestID, date, version := getDebugHeaders(resp.resp.Header)
+		return AzureStorageServiceError{
+			StatusCode: resp.resp.StatusCode,
+			Code:       resp.odata.Err.Code,
+			RequestID:  requestID,
+			Date:       date,
+			APIVersion: version,
+			Message:    operationFailedMessage,
+		}
+	}
+
+	return nil
+}
+
+// getFailedOperation parses the original Azure error string and determines which operation failed
+// and generates appropriate message.
+func (t *TableBatch) getFailedOperation(errorMessage string) string {
+	// errorMessage consists of "number:string" we just need the number.
+	sp := strings.Split(errorMessage, ":")
+	if len(sp) > 1 {
+		msg := fmt.Sprintf("Element %s in the batch returned an unexpected response code.\n%s", sp[0], errorMessage)
+		return msg
+	}
+
+	// cant parse the message, just return the original message to client
+	return errorMessage
+}
+
+// generateBody generates the complete body for the batch request.
+func generateBody(changeSetBody *bytes.Buffer, changesetBoundary string, boundary string) (*bytes.Buffer, error) {
+
+	body := new(bytes.Buffer)
+	writer := multipart.NewWriter(body)
+	writer.SetBoundary(boundary)
+	h := make(textproto.MIMEHeader)
+	h.Set(headerContentType, fmt.Sprintf("multipart/mixed; boundary=%s\r\n", changesetBoundary))
+	batchWriter, err := writer.CreatePart(h)
+	if err != nil {
+		return nil, err
+	}
+	batchWriter.Write(changeSetBody.Bytes())
+	writer.Close()
+	return body, nil
+}
+
+// generateChangesetBody generates the individual changesets for the various operations within the batch request.
+// There is a changeset for Insert, Delete, Merge etc.
+func (t *TableBatch) generateChangesetBody(changesetBoundary string) (*bytes.Buffer, error) {
+
+	body := new(bytes.Buffer)
+	writer := multipart.NewWriter(body)
+	writer.SetBoundary(changesetBoundary)
+
+	for _, be := range t.BatchEntitySlice {
+		t.generateEntitySubset(&be, writer)
+	}
+
+	writer.Close()
+	return body, nil
+}
+
+// generateVerb generates the HTTP request VERB required for each changeset.
+func generateVerb(op Operation) (string, error) {
+	switch op {
+	case InsertOp:
+		return http.MethodPost, nil
+	case DeleteOp:
+		return http.MethodDelete, nil
+	case ReplaceOp, InsertOrReplaceOp:
+		return http.MethodPut, nil
+	case MergeOp, InsertOrMergeOp:
+		return "MERGE", nil
+	default:
+		return "", errors.New("Unable to detect operation")
+	}
+}
+
+// generateQueryPath generates the query path for within the changesets
+// For inserts it will just be a table query path (table name)
+// but for other operations (modifying an existing entity) then
+// the partition/row keys need to be generated.
+func (t *TableBatch) generateQueryPath(op Operation, entity *Entity) string {
+	if op == InsertOp {
+		return entity.Table.buildPath()
+	}
+	return entity.buildPath()
+}
+
+// generateGenericOperationHeaders generates common headers for a given operation.
+func generateGenericOperationHeaders(be *BatchEntity) map[string]string {
+	retval := map[string]string{}
+
+	for k, v := range defaultChangesetHeaders {
+		retval[k] = v
+	}
+
+	if be.Op == DeleteOp || be.Op == ReplaceOp || be.Op == MergeOp {
+		if be.Force || be.Entity.OdataEtag == "" {
+			retval["If-Match"] = "*"
+		} else {
+			retval["If-Match"] = be.Entity.OdataEtag
+		}
+	}
+
+	return retval
+}
+
+// generateEntitySubset generates body payload for particular batch entity
+func (t *TableBatch) generateEntitySubset(batchEntity *BatchEntity, writer *multipart.Writer) error {
+
+	h := make(textproto.MIMEHeader)
+	h.Set(headerContentType, "application/http")
+	h.Set(headerContentTransferEncoding, "binary")
+
+	verb, err := generateVerb(batchEntity.Op)
+	if err != nil {
+		return err
+	}
+
+	genericOpHeadersMap := generateGenericOperationHeaders(batchEntity)
+	queryPath := t.generateQueryPath(batchEntity.Op, batchEntity.Entity)
+	uri := t.Table.tsc.client.getEndpoint(tableServiceName, queryPath, nil)
+
+	operationWriter, err := writer.CreatePart(h)
+	if err != nil {
+		return err
+	}
+
+	urlAndVerb := fmt.Sprintf("%s %s HTTP/1.1\r\n", verb, uri)
+	operationWriter.Write([]byte(urlAndVerb))
+	writeHeaders(genericOpHeadersMap, &operationWriter)
+	operationWriter.Write([]byte("\r\n")) // additional \r\n is needed per changeset separating the "headers" and the body.
+
+	// delete operation doesn't need a body.
+	if batchEntity.Op != DeleteOp {
+		//var e Entity = batchEntity.Entity
+		body, err := json.Marshal(batchEntity.Entity)
+		if err != nil {
+			return err
+		}
+		operationWriter.Write(body)
+	}
+
+	return nil
+}
+
+func writeHeaders(h map[string]string, writer *io.Writer) {
+	// This way it is guaranteed the headers will be written in a sorted order
+	var keys []string
+	for k := range h {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, k := range keys {
+		(*writer).Write([]byte(fmt.Sprintf("%s: %s\r\n", k, h[k])))
+	}
+}

vendor/github.com/Azure/azure-sdk-for-go/storage/tableserviceclient.go

@@ -0,0 +1,204 @@
+package storage
+
+// Copyright 2017 Microsoft Corporation
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strconv"
+)
+
+const (
+	headerAccept          = "Accept"
+	headerEtag            = "Etag"
+	headerPrefer          = "Prefer"
+	headerXmsContinuation = "x-ms-Continuation-NextTableName"
+)
+
+// TableServiceClient contains operations for Microsoft Azure Table Storage
+// Service.
+type TableServiceClient struct {
+	client Client
+	auth   authentication
+}
+
+// TableOptions includes options for some table operations
+type TableOptions struct {
+	RequestID string
+}
+
+func (options *TableOptions) addToHeaders(h map[string]string) map[string]string {
+	if options != nil {
+		h = addToHeaders(h, "x-ms-client-request-id", options.RequestID)
+	}
+	return h
+}
+
+// QueryNextLink includes information for getting the next page of
+// results in query operations
+type QueryNextLink struct {
+	NextLink *string
+	ml       MetadataLevel
+}
+
+// GetServiceProperties gets the properties of your storage account's table service.
+// See: https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/get-table-service-properties
+func (t *TableServiceClient) GetServiceProperties() (*ServiceProperties, error) {
+	return t.client.getServiceProperties(tableServiceName, t.auth)
+}
+
+// SetServiceProperties sets the properties of your storage account's table service.
+// See: https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/set-table-service-properties
+func (t *TableServiceClient) SetServiceProperties(props ServiceProperties) error {
+	return t.client.setServiceProperties(props, tableServiceName, t.auth)
+}
+
+// GetTableReference returns a Table object for the specified table name.
+func (t *TableServiceClient) GetTableReference(name string) *Table {
+	return &Table{
+		tsc:  t,
+		Name: name,
+	}
+}
+
+// QueryTablesOptions includes options for some table operations
+type QueryTablesOptions struct {
+	Top       uint
+	Filter    string
+	RequestID string
+}
+
+func (options *QueryTablesOptions) getParameters() (url.Values, map[string]string) {
+	query := url.Values{}
+	headers := map[string]string{}
+	if options != nil {
+		if options.Top > 0 {
+			query.Add(OdataTop, strconv.FormatUint(uint64(options.Top), 10))
+		}
+		if options.Filter != "" {
+			query.Add(OdataFilter, options.Filter)
+		}
+		headers = addToHeaders(headers, "x-ms-client-request-id", options.RequestID)
+	}
+	return query, headers
+}
+
+// QueryTables returns the tables in the storage account.
+// You can use query options defined by the OData Protocol specification.
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/query-tables
+func (t *TableServiceClient) QueryTables(ml MetadataLevel, options *QueryTablesOptions) (*TableQueryResult, error) {
+	query, headers := options.getParameters()
+	uri := t.client.getEndpoint(tableServiceName, tablesURIPath, query)
+	return t.queryTables(uri, headers, ml)
+}
+
+// NextResults returns the next page of results
+// from a QueryTables or a NextResults operation.
+//
+// See https://docs.microsoft.com/rest/api/storageservices/fileservices/query-tables
+// See https://docs.microsoft.com/rest/api/storageservices/fileservices/query-timeout-and-pagination
+func (tqr *TableQueryResult) NextResults(options *TableOptions) (*TableQueryResult, error) {
+	if tqr == nil {
+		return nil, errNilPreviousResult
+	}
+	if tqr.NextLink == nil {
+		return nil, errNilNextLink
+	}
+	headers := options.addToHeaders(map[string]string{})
+
+	return tqr.tsc.queryTables(*tqr.NextLink, headers, tqr.ml)
+}
+
+// TableQueryResult contains the response from
+// QueryTables and QueryTablesNextResults functions.
+type TableQueryResult struct {
+	OdataMetadata string  `json:"odata.metadata"`
+	Tables        []Table `json:"value"`
+	QueryNextLink
+	tsc *TableServiceClient
+}
+
+func (t *TableServiceClient) queryTables(uri string, headers map[string]string, ml MetadataLevel) (*TableQueryResult, error) {
+	if ml == EmptyPayload {
+		return nil, errEmptyPayload
+	}
+	headers = mergeHeaders(headers, t.client.getStandardHeaders())
+	headers[headerAccept] = string(ml)
+
+	resp, err := t.client.exec(http.MethodGet, uri, headers, nil, t.auth)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if err := checkRespCode(resp, []int{http.StatusOK}); err != nil {
+		return nil, err
+	}
+
+	respBody, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+	var out TableQueryResult
+	err = json.Unmarshal(respBody, &out)
+	if err != nil {
+		return nil, err
+	}
+
+	for i := range out.Tables {
+		out.Tables[i].tsc = t
+	}
+	out.tsc = t
+
+	nextLink := resp.Header.Get(http.CanonicalHeaderKey(headerXmsContinuation))
+	if nextLink == "" {
+		out.NextLink = nil
+	} else {
+		originalURI, err := url.Parse(uri)
+		if err != nil {
+			return nil, err
+		}
+		v := originalURI.Query()
+		v.Set(nextTableQueryParameter, nextLink)
+		newURI := t.client.getEndpoint(tableServiceName, tablesURIPath, v)
+		out.NextLink = &newURI
+		out.ml = ml
+	}
+
+	return &out, nil
+}
+
+func addBodyRelatedHeaders(h map[string]string, length int) map[string]string {
+	h[headerContentType] = "application/json"
+	h[headerContentLength] = fmt.Sprintf("%v", length)
+	h[headerAcceptCharset] = "UTF-8"
+	return h
+}
+
+func addReturnContentHeaders(h map[string]string, ml MetadataLevel) map[string]string {
+	if ml != EmptyPayload {
+		h[headerPrefer] = "return-content"
+		h[headerAccept] = string(ml)
+	} else {
+		h[headerPrefer] = "return-no-content"
+		// From API version 2015-12-11 onwards, Accept header is required
+		h[headerAccept] = string(NoMetadata)
+	}
+	return h
+}

vendor/github.com/Azure/azure-sdk-for-go/storage/util.go

@@ -0,0 +1,244 @@
+package storage
+
+// Copyright 2017 Microsoft Corporation
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+import (
+	"bytes"
+	"crypto/hmac"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/xml"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"reflect"
+	"strconv"
+	"strings"
+	"time"
+)
+
+var (
+	fixedTime         = time.Date(2050, time.December, 20, 21, 55, 0, 0, time.FixedZone("GMT", -6))
+	accountSASOptions = AccountSASTokenOptions{
+		Services: Services{
+			Blob: true,
+		},
+		ResourceTypes: ResourceTypes{
+			Service:   true,
+			Container: true,
+			Object:    true,
+		},
+		Permissions: Permissions{
+			Read:    true,
+			Write:   true,
+			Delete:  true,
+			List:    true,
+			Add:     true,
+			Create:  true,
+			Update:  true,
+			Process: true,
+		},
+		Expiry:   fixedTime,
+		UseHTTPS: true,
+	}
+)
+
+func (c Client) computeHmac256(message string) string {
+	h := hmac.New(sha256.New, c.accountKey)
+	h.Write([]byte(message))
+	return base64.StdEncoding.EncodeToString(h.Sum(nil))
+}
+
+func currentTimeRfc1123Formatted() string {
+	return timeRfc1123Formatted(time.Now().UTC())
+}
+
+func timeRfc1123Formatted(t time.Time) string {
+	return t.Format(http.TimeFormat)
+}
+
+func timeRFC3339Formatted(t time.Time) string {
+	return t.Format("2006-01-02T15:04:05.0000000Z")
+}
+
+func mergeParams(v1, v2 url.Values) url.Values {
+	out := url.Values{}
+	for k, v := range v1 {
+		out[k] = v
+	}
+	for k, v := range v2 {
+		vals, ok := out[k]
+		if ok {
+			vals = append(vals, v...)
+			out[k] = vals
+		} else {
+			out[k] = v
+		}
+	}
+	return out
+}
+
+func prepareBlockListRequest(blocks []Block) string {
+	s := `<?xml version="1.0" encoding="utf-8"?><BlockList>`
+	for _, v := range blocks {
+		s += fmt.Sprintf("<%s>%s</%s>", v.Status, v.ID, v.Status)
+	}
+	s += `</BlockList>`
+	return s
+}
+
+func xmlUnmarshal(body io.Reader, v interface{}) error {
+	data, err := ioutil.ReadAll(body)
+	if err != nil {
+		return err
+	}
+	return xml.Unmarshal(data, v)
+}
+
+func xmlMarshal(v interface{}) (io.Reader, int, error) {
+	b, err := xml.Marshal(v)
+	if err != nil {
+		return nil, 0, err
+	}
+	return bytes.NewReader(b), len(b), nil
+}
+
+func headersFromStruct(v interface{}) map[string]string {
+	headers := make(map[string]string)
+	value := reflect.ValueOf(v)
+	for i := 0; i < value.NumField(); i++ {
+		key := value.Type().Field(i).Tag.Get("header")
+		if key != "" {
+			reflectedValue := reflect.Indirect(value.Field(i))
+			var val string
+			if reflectedValue.IsValid() {
+				switch reflectedValue.Type() {
+				case reflect.TypeOf(fixedTime):
+					val = timeRfc1123Formatted(reflectedValue.Interface().(time.Time))
+				case reflect.TypeOf(uint64(0)), reflect.TypeOf(uint(0)):
+					val = strconv.FormatUint(reflectedValue.Uint(), 10)
+				case reflect.TypeOf(int(0)):
+					val = strconv.FormatInt(reflectedValue.Int(), 10)
+				default:
+					val = reflectedValue.String()
+				}
+			}
+			if val != "" {
+				headers[key] = val
+			}
+		}
+	}
+	return headers
+}
+
+// merges extraHeaders into headers and returns headers
+func mergeHeaders(headers, extraHeaders map[string]string) map[string]string {
+	for k, v := range extraHeaders {
+		headers[k] = v
+	}
+	return headers
+}
+
+func addToHeaders(h map[string]string, key, value string) map[string]string {
+	if value != "" {
+		h[key] = value
+	}
+	return h
+}
+
+func addTimeToHeaders(h map[string]string, key string, value *time.Time) map[string]string {
+	if value != nil {
+		h = addToHeaders(h, key, timeRfc1123Formatted(*value))
+	}
+	return h
+}
+
+func addTimeout(params url.Values, timeout uint) url.Values {
+	if timeout > 0 {
+		params.Add("timeout", fmt.Sprintf("%v", timeout))
+	}
+	return params
+}
+
+func addSnapshot(params url.Values, snapshot *time.Time) url.Values {
+	if snapshot != nil {
+		params.Add("snapshot", timeRFC3339Formatted(*snapshot))
+	}
+	return params
+}
+
+func getTimeFromHeaders(h http.Header, key string) (*time.Time, error) {
+	var out time.Time
+	var err error
+	outStr := h.Get(key)
+	if outStr != "" {
+		out, err = time.Parse(time.RFC1123, outStr)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return &out, nil
+}
+
+// TimeRFC1123 is an alias for time.Time needed for custom Unmarshalling
+type TimeRFC1123 time.Time
+
+// UnmarshalXML is a custom unmarshaller that overrides the default time unmarshal which uses a different time layout.
+func (t *TimeRFC1123) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
+	var value string
+	d.DecodeElement(&value, &start)
+	parse, err := time.Parse(time.RFC1123, value)
+	if err != nil {
+		return err
+	}
+	*t = TimeRFC1123(parse)
+	return nil
+}
+
+// MarshalXML marshals using time.RFC1123.
+func (t *TimeRFC1123) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
+	return e.EncodeElement(time.Time(*t).Format(time.RFC1123), start)
+}
+
+// returns a map of custom metadata values from the specified HTTP header
+func getMetadataFromHeaders(header http.Header) map[string]string {
+	metadata := make(map[string]string)
+	for k, v := range header {
+		// Can't trust CanonicalHeaderKey() to munge case
+		// reliably. "_" is allowed in identifiers:
+		// https://msdn.microsoft.com/en-us/library/azure/dd179414.aspx
+		// https://msdn.microsoft.com/library/aa664670(VS.71).aspx
+		// http://tools.ietf.org/html/rfc7230#section-3.2
+		// ...but "_" is considered invalid by
+		// CanonicalMIMEHeaderKey in
+		// https://golang.org/src/net/textproto/reader.go?s=14615:14659#L542
+		// so k can be "X-Ms-Meta-Lol" or "x-ms-meta-lol_rofl".
+		k = strings.ToLower(k)
+		if len(v) == 0 || !strings.HasPrefix(k, strings.ToLower(userDefinedMetadataHeaderPrefix)) {
+			continue
+		}
+		// metadata["lol"] = content of the last X-Ms-Meta-Lol header
+		k = k[len(userDefinedMetadataHeaderPrefix):]
+		metadata[k] = v[len(v)-1]
+	}
+
+	if len(metadata) == 0 {
+		return nil
+	}
+
+	return metadata
+}

vendor/github.com/aliyun/alibaba-cloud-sdk-go/sdk/auth/roa_signature_composer.go

@@ -42,9 +42,9 @@ func completeROASignParams(request requests.AcsRequest, signer Signer, regionId
 
 	// complete query params
 	queryParams := request.GetQueryParams()
-	if _, ok := queryParams["RegionId"]; !ok {
-		queryParams["RegionId"] = regionId
-	}
+	//if _, ok := queryParams["RegionId"]; !ok {
+	//	queryParams["RegionId"] = regionId
+	//}
 	if extraParam := signer.GetExtraParam(); extraParam != nil {
 		for key, value := range extraParam {
 			if key == "SecurityToken" {

vendor/github.com/aliyun/alibaba-cloud-sdk-go/sdk/client.go

@@ -250,7 +250,9 @@ func (client *Client) DoActionWithSigner(request requests.AcsRequest, response r
 		var timeout bool
 		// receive error
 		if err != nil {
-			if timeout = isTimeout(err); !timeout {
+			if !client.config.AutoRetry {
+				return
+			} else if timeout = isTimeout(err); !timeout {
 				// if not timeout error, return
 				return
 			} else if retryTimes >= client.config.MaxRetryTime {

vendor/github.com/aliyun/alibaba-cloud-sdk-go/sdk/errors/client_error.go

@@ -24,7 +24,7 @@ const (
 	UnsupportedCredentialErrorMessage = "Specified credential (type = %s) is not supported, please check"
 
 	CanNotResolveEndpointErrorCode    = "SDK.CanNotResolveEndpoint"
-	CanNotResolveEndpointErrorMessage = "Can not resolve endpoint(param = %s), please check the user guide\n %s"
+	CanNotResolveEndpointErrorMessage = "Can not resolve endpoint(param = %s), please check your accessKey with secret, and read the user guide\n %s"
 
 	UnsupportedParamPositionErrorCode    = "SDK.UnsupportedParamPosition"
 	UnsupportedParamPositionErrorMessage = "Specified param position (%s) is not supported, please upgrade sdk and retry"

vendor/github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests/roa_request.go

@@ -77,7 +77,9 @@ func (request *RoaRequest) buildQueries(needParamEncode bool) string {
 	// append urlBuilder
 	urlBuilder := bytes.Buffer{}
 	urlBuilder.WriteString(path)
-	urlBuilder.WriteString("?")
+	if len(queryKeys) > 0 {
+		urlBuilder.WriteString("?")
+	}
 	for i := 0; i < len(queryKeys); i++ {
 		queryKey := queryKeys[i]
 		urlBuilder.WriteString(queryKey)

vendor/github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests/rpc_request.go

@@ -52,7 +52,7 @@ func (request *RpcRequest) GetQueries() string {
 }
 
 func (request *RpcRequest) BuildUrl() string {
-	return strings.ToLower(request.Scheme) + "://" + request.Domain + request.BuildQueries()
+	return strings.ToLower(request.Scheme) + "://" + request.Domain + ":" + request.Port + request.BuildQueries()
 }
 
 func (request *RpcRequest) GetUrl() string {

vendor/github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests/types.go

@@ -8,11 +8,11 @@ func NewInteger(integer int) Integer {
 	return Integer(strconv.Itoa(integer))
 }
 
-func (integer Integer) hasValue() bool {
+func (integer Integer) HasValue() bool {
 	return integer != ""
 }
 
-func (integer Integer) getValue() (int, error) {
+func (integer Integer) GetValue() (int, error) {
 	return strconv.Atoi(string(integer))
 }
 
@@ -20,7 +20,7 @@ func NewInteger64(integer int64) Integer {
 	return Integer(strconv.FormatInt(integer, 10))
 }
 
-func (integer Integer) getValue64() (int64, error) {
+func (integer Integer) GetValue64() (int64, error) {
 	return strconv.ParseInt(string(integer), 10, 0)
 }
 
@@ -30,11 +30,11 @@ func NewBoolean(bool bool) Boolean {
 	return Boolean(strconv.FormatBool(bool))
 }
 
-func (boolean Boolean) hasValue() bool {
+func (boolean Boolean) HasValue() bool {
 	return boolean != ""
 }
 
-func (boolean Boolean) getValue() (bool, error) {
+func (boolean Boolean) GetValue() (bool, error) {
 	return strconv.ParseBool(string(boolean))
 }
 
@@ -44,10 +44,10 @@ func NewFloat(f float64) Float {
 	return Float(strconv.FormatFloat(f, 'f', 6, 64))
 }
 
-func (float Float) hasValue() bool {
+func (float Float) HasValue() bool {
 	return float != ""
 }
 
-func (float Float) getValue() (float64, error) {
+func (float Float) GetValue() (float64, error) {
 	return strconv.ParseFloat(string(float), 64)
 }

vendor/github.com/aliyun/alibaba-cloud-sdk-go/sdk/responses/response.go

@@ -47,6 +47,11 @@ func Unmarshal(response AcsResponse, httpResponse *http.Response, format string)
 		// common response need not unmarshal
 		return
 	}
+
+	if len(response.GetHttpContentBytes()) == 0 {
+		return
+	}
+
 	if strings.ToUpper(format) == "JSON" {
 		initJsonParserOnce()
 		err = jsonParser.Unmarshal(response.GetHttpContentBytes(), response)

vendor/github.com/coredns/coredns/core/dnsserver/zdirectives.go

@@ -39,6 +39,7 @@ var Directives = []string{
 	"auto",
 	"secondary",
 	"etcd",
+	"loop",
 	"forward",
 	"proxy",
 	"erratic",

vendor/github.com/coredns/coredns/coremain/run.go

@@ -42,7 +42,7 @@ func Run() {
 	caddy.TrapSignals()
 
 	// Reset flag.CommandLine to get rid of unwanted flags for instance from glog (used in kubernetes).
-	// And readd the once we want to keep.
+	// And read the ones we want to keep.
 	flag.VisitAll(func(f *flag.Flag) {
 		if _, ok := flagsBlacklist[f.Name]; ok {
 			return

vendor/github.com/coredns/coredns/coremain/version.go

@@ -2,7 +2,7 @@ package coremain
 
 // Various CoreDNS constants.
 const (
-	CoreVersion = "1.2.0"
+	CoreVersion = "1.2.2"
 	coreName    = "CoreDNS"
 	serverType  = "dns"
 )

vendor/github.com/coredns/coredns/plugin/backend_lookup.go

@@ -40,24 +40,22 @@ func A(b ServiceBackend, zone string, state request.Request, previousRecords []d
 			if dnsutil.DuplicateCNAME(newRecord, previousRecords) {
 				continue
 			}
+			if dns.IsSubDomain(zone, dns.Fqdn(serv.Host)) {
+				state1 := state.NewWithQuestion(serv.Host, state.QType())
+				state1.Zone = zone
+				nextRecords, err := A(b, zone, state1, append(previousRecords, newRecord), opt)
 
-			state1 := state.NewWithQuestion(serv.Host, state.QType())
-			nextRecords, err := A(b, zone, state1, append(previousRecords, newRecord), opt)
-
-			if err == nil {
-				// Not only have we found something we should add the CNAME and the IP addresses.
-				if len(nextRecords) > 0 {
-					records = append(records, newRecord)
-					records = append(records, nextRecords...)
+				if err == nil {
+					// Not only have we found something we should add the CNAME and the IP addresses.
+					if len(nextRecords) > 0 {
+						records = append(records, newRecord)
+						records = append(records, nextRecords...)
+					}
 				}
 				continue
 			}
 			// This means we can not complete the CNAME, try to look else where.
 			target := newRecord.Target
-			if dns.IsSubDomain(zone, target) {
-				// We should already have found it
-				continue
-			}
 			// Lookup
 			m1, e1 := b.Lookup(state, target, state.QType())
 			if e1 != nil {
@@ -110,19 +108,20 @@ func AAAA(b ServiceBackend, zone string, state request.Request, previousRecords
 			if dnsutil.DuplicateCNAME(newRecord, previousRecords) {
 				continue
 			}
+			if dns.IsSubDomain(zone, dns.Fqdn(serv.Host)) {
+				state1 := state.NewWithQuestion(serv.Host, state.QType())
+				state1.Zone = zone
+				nextRecords, err := AAAA(b, zone, state1, append(previousRecords, newRecord), opt)
 
-			state1 := state.NewWithQuestion(serv.Host, state.QType())
-			nextRecords, err := AAAA(b, zone, state1, append(previousRecords, newRecord), opt)
-
-			if err == nil {
-				// Not only have we found something we should add the CNAME and the IP addresses.
-				if len(nextRecords) > 0 {
-					records = append(records, newRecord)
-					records = append(records, nextRecords...)
+				if err == nil {
+					// Not only have we found something we should add the CNAME and the IP addresses.
+					if len(nextRecords) > 0 {
+						records = append(records, newRecord)
+						records = append(records, nextRecords...)
+					}
 				}
 				continue
 			}
-
 			// This means we can not complete the CNAME, try to look else where.
 			target := newRecord.Target
 			m1, e1 := b.Lookup(state, target, state.QType())
@@ -173,6 +172,11 @@ func SRV(b ServiceBackend, zone string, state request.Request, opt Options) (rec
 		w[serv.Priority] += weight
 	}
 	for _, serv := range services {
+		// Don't add the entry if the port is -1 (invalid). The kubernetes plugin uses port -1 when a service/endpoint
+		// does not have any declared ports.
+		if serv.Port == -1 {
+			continue
+		}
 		w1 := 100.0 / float64(w[serv.Priority])
 		if serv.Weight == 0 {
 			w1 *= 100

vendor/github.com/coredns/coredns/plugin/cache/README.md

@@ -52,11 +52,12 @@ cache [TTL] [ZONES...] {
 
 ## Capacity and Eviction
 
-When specifying **CAPACITY**, the minimum cache capacity is 131,072.  Specifying a lower value will be
-ignored. Specifying a **CAPACITY** of zero does not disable the cache.
-
-Eviction is done per shard - i.e. when a shard reaches capacity, items are evicted from that shard.  Since shards don't fill up perfectly evenly, evictions will occur before the entire cache reaches full capacity. Each shard capacity is equal to the total cache size / number of shards (256).
+If **CAPACITY** is not specified, the default cache size is 10,000 per cache. The minimum allowed cache size is 1024.
 
+Eviction is done per shard. In effect, when a shard reaches capacity, items are evicted from that shard.
+Since shards don't fill up perfectly evenly, evictions will occur before the entire cache reaches full capacity.
+Each shard capacity is equal to the total cache size / number of shards (256). Eviction is random, not TTL based.
+Entries with 0 TTL will remain in the cache until randomly evicted when the shard reaches capacity.
 
 ## Metrics
 
@@ -89,3 +90,13 @@ Proxy to Google Public DNS and only cache responses for example.org (or below).
     cache example.org
 }
 ~~~
+
+Enable caching for all zones, keep a positive cache size of 5000 and a negative cache size of 2500:
+ ~~~ corefile
+ . {
+     cache {
+         success 5000
+         denial 2500
+    }
+ }
+ ~~~

vendor/github.com/coredns/coredns/plugin/dnstap/handler.go

@@ -51,7 +51,7 @@ func TapperFromContext(ctx context.Context) (t Tapper) {
 }
 
 // TapMessage implements Tapper.
-func (h *Dnstap) TapMessage(m *tap.Message) {
+func (h Dnstap) TapMessage(m *tap.Message) {
 	t := tap.Dnstap_MESSAGE
 	h.IO.Dnstap(tap.Dnstap{
 		Type:    &t,

vendor/github.com/coredns/coredns/plugin/etcd/msg/service.go

@@ -164,7 +164,7 @@ func targetStrip(name string, targetStrip int) string {
 		offset, end = dns.NextLabel(name, offset)
 	}
 	if end {
-		// We overshot the name, use the orignal one.
+		// We overshot the name, use the original one.
 		offset = 0
 	}
 	name = name[offset:]