diff --git a/cmd/climc/shell/buckets.go b/cmd/climc/shell/buckets.go index adb5e9febe..7b3a8c0271 100644 --- a/cmd/climc/shell/buckets.go +++ b/cmd/climc/shell/buckets.go @@ -157,8 +157,8 @@ func init() { KEY string `help:"Key of object to upload"` Path string `help:"Path to file to upload" required:"true"` - ContentLength int64 `help:"Content lenght (bytes)"` - ContentType string `help:"Content type" required:"true"` + ContentLength int64 `help:"Content lenght (bytes)" default:"-1"` + ContentType string `help:"Content type"` StorageClass string `help:"storage CLass"` Acl string `help:"object acl." choices:"private|public-read|public-read-write"` } @@ -182,7 +182,7 @@ func init() { body = os.Stdin } - if args.ContentLength <= 0 { + if args.ContentLength < 0 { return fmt.Errorf("required content-length") } diff --git a/cmd/climc/shell/cloudproviders.go b/cmd/climc/shell/cloudproviders.go index ff28c3c0eb..0626f8dddf 100644 --- a/cmd/climc/shell/cloudproviders.go +++ b/cmd/climc/shell/cloudproviders.go @@ -28,7 +28,11 @@ func init() { type CloudproviderListOptions struct { options.BaseListOptions + Usable bool `help:"Vpc & Network usable"` + + HasObjectStorage bool `help:"filter cloudproviders that has object storage"` + NoObjectStorage bool `help:"filter cloudproviders that has no object storage"` } R(&CloudproviderListOptions{}, "cloud-provider-list", "List cloud providers", func(s *mcclient.ClientSession, args *CloudproviderListOptions) error { var params *jsonutils.JSONDict @@ -42,6 +46,12 @@ func init() { if args.Usable { params.Add(jsonutils.NewBool(true), "usable") } + + if args.HasObjectStorage { + params.Add(jsonutils.JSONTrue, "has_object_storage") + } else if args.NoObjectStorage { + params.Add(jsonutils.JSONFalse, "has_object_storage") + } } result, err := modules.Cloudproviders.List(s, params) if err != nil { diff --git a/pkg/cloudprovider/objectstore.go b/pkg/cloudprovider/objectstore.go index 221b44b08b..9e449fd06f 100644 --- a/pkg/cloudprovider/objectstore.go +++ b/pkg/cloudprovider/objectstore.go @@ -327,3 +327,17 @@ func UploadObject(ctx context.Context, bucket ICloudBucket, key string, blocksz } return nil } + +func DeletePrefix(ctx context.Context, bucket ICloudBucket, prefix string) error { + objs, err := bucket.GetIObjects(prefix, true) + if err != nil { + return errors.Wrap(err, "bucket.GetIObjects") + } + for i := range objs { + err := bucket.DeleteObject(ctx, objs[i].GetKey()) + if err != nil { + return errors.Wrap(err, "bucket.DeleteObject") + } + } + return nil +} diff --git a/pkg/compute/models/buckets.go b/pkg/compute/models/buckets.go index 073cdc577f..9cea5a22e0 100644 --- a/pkg/compute/models/buckets.go +++ b/pkg/compute/models/buckets.go @@ -624,12 +624,17 @@ func (bucket *SBucket) PerformMakedir( data jsonutils.JSONObject, ) (jsonutils.JSONObject, error) { key, _ := data.GetString("key") - if key[len(key)-1] != '/' { - return nil, httperrors.NewInputParameterError("directory must ends with /") + for len(key) > 0 && key[len(key)-1] == '/' { + key = key[:len(key)-1] } + + if len(key) == 0 { + return nil, httperrors.NewInputParameterError("empty directory name") + } + err := s3utils.CheckValidObjectName(key) if err != nil { - return nil, httperrors.NewInputParameterError("invalid key: %s", err) + return nil, httperrors.NewInputParameterError("invalid key %s: %s", key, err) } iBucket, err := bucket.GetIBucket() @@ -637,7 +642,7 @@ func (bucket *SBucket) PerformMakedir( return nil, httperrors.NewInternalServerError("fail to find external bucket: %s", err) } - err = cloudprovider.Makedir(ctx, iBucket, key) + err = cloudprovider.Makedir(ctx, iBucket, key+"/") if err != nil { return nil, httperrors.NewInternalServerError("fail to mkdir: %s", err) } @@ -674,9 +679,13 @@ func (bucket *SBucket) PerformDelete( } ok := jsonutils.NewDict() results := modules.BatchDo(keyStrs, func(key string) (jsonutils.JSONObject, error) { - err := iBucket.DeleteObject(ctx, key) + if strings.HasSuffix(key, "/") { + err = cloudprovider.DeletePrefix(ctx, iBucket, key) + } else { + err = iBucket.DeleteObject(ctx, key) + } if err != nil { - return nil, err + return nil, errors.Wrap(err, "DeletePrefix") } else { return ok, nil } @@ -704,6 +713,11 @@ func (bucket *SBucket) PerformUpload( appParams := appsrv.AppContextGetParams(ctx) key := appParams.Request.Header.Get(api.BUCKET_UPLOAD_OBJECT_KEY_HEADER) + + if strings.HasSuffix(key, "/") { + return nil, httperrors.NewInputParameterError("object key should not ends with /") + } + err := s3utils.CheckValidObjectName(key) if err != nil { return nil, httperrors.NewInputParameterError("invalid object key: %s", err) @@ -723,19 +737,21 @@ func (bucket *SBucket) PerformUpload( if err != nil { return nil, httperrors.NewInputParameterError("Illegal Content-Length %s", sizeStr) } - if sizeBytes <= 0 { - return nil, httperrors.NewInputParameterError("Content-Length not positive %d", sizeBytes) + if sizeBytes < 0 { + return nil, httperrors.NewInputParameterError("Content-Length negative %d", sizeBytes) } storageClass := appParams.Request.Header.Get(api.BUCKET_UPLOAD_OBJECT_STORAGECLASS_HEADER) - aclStr := cloudprovider.TBucketACLType(appParams.Request.Header.Get(api.BUCKET_UPLOAD_OBJECT_ACL_HEADER)) - switch aclStr { - case cloudprovider.ACLPrivate, cloudprovider.ACLAuthRead, cloudprovider.ACLPublicRead, cloudprovider.ACLPublicReadWrite: - // do nothing - default: - return nil, httperrors.NewInputParameterError("invalid acl: %s", aclStr) + aclStr := appParams.Request.Header.Get(api.BUCKET_UPLOAD_OBJECT_ACL_HEADER) + if len(aclStr) > 0 { + switch cloudprovider.TBucketACLType(aclStr) { + case cloudprovider.ACLPrivate, cloudprovider.ACLAuthRead, cloudprovider.ACLPublicRead, cloudprovider.ACLPublicReadWrite: + // do nothing + default: + return nil, httperrors.NewInputParameterError("invalid acl: %s", aclStr) + } } - err = cloudprovider.UploadObject(ctx, iBucket, key, 0, appParams.Request.Body, sizeBytes, contType, aclStr, storageClass, false) + err = cloudprovider.UploadObject(ctx, iBucket, key, 0, appParams.Request.Body, sizeBytes, contType, cloudprovider.TBucketACLType(aclStr), storageClass, false) if err != nil { return nil, httperrors.NewInternalServerError("put object error %s", err) } diff --git a/pkg/compute/models/cloudproviders.go b/pkg/compute/models/cloudproviders.go index 338250e58a..6a91bae2b5 100644 --- a/pkg/compute/models/cloudproviders.go +++ b/pkg/compute/models/cloudproviders.go @@ -1010,6 +1010,17 @@ func (manager *SCloudproviderManager) ListItemFilter(ctx context.Context, q *sql q = q.Filter(sqlchemy.IsTrue(cloudaccounts.Field("is_on_premise"))) } + if query.Contains("has_object_storage") { + hasObjectStorage, _ := query.Bool("has_object_storage") + cloudaccounts := CloudaccountManager.Query().SubQuery() + q = q.Join(cloudaccounts, sqlchemy.Equals(cloudaccounts.Field("id"), q.Field("cloudaccount_id"))) + if hasObjectStorage { + q = q.Filter(sqlchemy.IsTrue(cloudaccounts.Field("has_object_storage"))) + } else { + q = q.Filter(sqlchemy.IsFalse(cloudaccounts.Field("has_object_storage"))) + } + } + return q, nil } diff --git a/pkg/multicloud/aws/bucket.go b/pkg/multicloud/aws/bucket.go index a3486bb63f..1634a38584 100644 --- a/pkg/multicloud/aws/bucket.go +++ b/pkg/multicloud/aws/bucket.go @@ -210,7 +210,7 @@ func (b *SBucket) GetIObjects(prefix string, isRecursive bool) ([]cloudprovider. } func (b *SBucket) PutObject(ctx context.Context, key string, body io.Reader, sizeBytes int64, contType string, cannedAcl cloudprovider.TBucketACLType, storageClassStr string) error { - if sizeBytes <= 0 { + if sizeBytes < 0 { return errors.Error("content length expected") } s3cli, err := b.region.GetS3Client() diff --git a/pkg/util/fileutils2/seeker_test.go b/pkg/util/fileutils2/seeker_test.go index 4c6e211e70..cb69a73711 100644 --- a/pkg/util/fileutils2/seeker_test.go +++ b/pkg/util/fileutils2/seeker_test.go @@ -21,25 +21,29 @@ import ( ) func TestNewReadSeeker(t *testing.T) { - testStr := "This is a test reader string" - seeker, err := NewReadSeeker(strings.NewReader(testStr), int64(len(testStr))) - if err != nil { - t.Fatalf("NewReadSeeker error %s", err) - } - defer seeker.Close() - buf1 := make([]byte, 1024) - n, err := seeker.Read(buf1) - if n != len(testStr) { - t.Fatalf("read buf1 error %s", err) - } - buf2 := make([]byte, 1024) - n, err = seeker.Read(buf2) - if n != 0 { - t.Fatalf("read buf2 should fail") - } - seeker.Seek(0, io.SeekStart) - n, err = seeker.Read(buf2) - if n != len(testStr) { - t.Fatalf("read buf2 error %s", err) + for _, testStr := range []string{ + "This is a test reader string", + "", + } { + seeker, err := NewReadSeeker(strings.NewReader(testStr), int64(len(testStr))) + if err != nil { + t.Fatalf("NewReadSeeker error %s", err) + } + defer seeker.Close() + buf1 := make([]byte, 1024) + n, err := seeker.Read(buf1) + if n != len(testStr) { + t.Fatalf("read buf1 error %s", err) + } + buf2 := make([]byte, 1024) + n, err = seeker.Read(buf2) + if n != 0 { + t.Fatalf("read buf2 should fail") + } + seeker.Seek(0, io.SeekStart) + n, err = seeker.Read(buf2) + if n != len(testStr) { + t.Fatalf("read buf2 error %s", err) + } } }