opensource/cloudpods
3
0
Fork 0
mirror of https://github.com/yunionio/cloudpods.git synced 2026-09-21 14:19:49 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2260 from swordqiu/hotfix/qj-bucket-misc-bugfix-20190813

Browse Source 
fix: 1. aws s3 fail to mkdir 2. put object requires acl
This commit is contained in:
Yousong Zhou
2019-08-14 09:50:16 +08:00
committed by GitHub
parent 2532b7e77b 021c8682db
commit a4424561a1
7 changed files with 94 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/climc/shell/buckets.go
+3 -3
View File
@@ -157,8 +157,8 @@ func init() {
KEY string `help:"Key of object to upload"`
Path string `help:"Path to file to upload" required:"true"`
ContentLength int64 `help:"Content lenght (bytes)"`
ContentType string `help:"Content type" required:"true"`
ContentLength int64 `help:"Content lenght (bytes)" default:"-1"`
ContentType string `help:"Content type"`
StorageClass string `help:"storage CLass"`
Acl string `help:"object acl." choices:"private|public-read|public-read-write"`
}
@@ -182,7 +182,7 @@ func init() {
body = os.Stdin
}
if args.ContentLength <= 0 {
if args.ContentLength < 0 {
return fmt.Errorf("required content-length")
}
cmd/climc/shell/cloudproviders.go
+10
View File
@@ -28,7 +28,11 @@ func init() {
type CloudproviderListOptions struct {
options.BaseListOptions
Usable bool `help:"Vpc & Network usable"`
HasObjectStorage bool `help:"filter cloudproviders that has object storage"`
NoObjectStorage bool `help:"filter cloudproviders that has no object storage"`
}
R(&CloudproviderListOptions{}, "cloud-provider-list", "List cloud providers", func(s *mcclient.ClientSession, args *CloudproviderListOptions) error {
var params *jsonutils.JSONDict
@@ -42,6 +46,12 @@ func init() {
if args.Usable {
params.Add(jsonutils.NewBool(true), "usable")
}
if args.HasObjectStorage {
params.Add(jsonutils.JSONTrue, "has_object_storage")
} else if args.NoObjectStorage {
params.Add(jsonutils.JSONFalse, "has_object_storage")
}
}
result, err := modules.Cloudproviders.List(s, params)
if err != nil {
pkg/cloudprovider/objectstore.go
+14
View File
@@ -327,3 +327,17 @@ func UploadObject(ctx context.Context, bucket ICloudBucket, key string, blocksz
}
return nil
}
func DeletePrefix(ctx context.Context, bucket ICloudBucket, prefix string) error {
objs, err := bucket.GetIObjects(prefix, true)
if err != nil {
return errors.Wrap(err, "bucket.GetIObjects")
}
for i := range objs {
err := bucket.DeleteObject(ctx, objs[i].GetKey())
if err != nil {
return errors.Wrap(err, "bucket.DeleteObject")
}
}
return nil
}
pkg/compute/models/buckets.go
+31 -15
View File
@@ -624,12 +624,17 @@ func (bucket *SBucket) PerformMakedir(
data jsonutils.JSONObject,
) (jsonutils.JSONObject, error) {
key, _ := data.GetString("key")
if key[len(key)-1] != '/' {
return nil, httperrors.NewInputParameterError("directory must ends with /")
for len(key) > 0 && key[len(key)-1] == '/' {
key = key[:len(key)-1]
}
if len(key) == 0 {
return nil, httperrors.NewInputParameterError("empty directory name")
}
err := s3utils.CheckValidObjectName(key)
if err != nil {
return nil, httperrors.NewInputParameterError("invalid key: %s", err)
return nil, httperrors.NewInputParameterError("invalid key %s: %s", key, err)
}
iBucket, err := bucket.GetIBucket()
@@ -637,7 +642,7 @@ func (bucket *SBucket) PerformMakedir(
return nil, httperrors.NewInternalServerError("fail to find external bucket: %s", err)
}
err = cloudprovider.Makedir(ctx, iBucket, key)
err = cloudprovider.Makedir(ctx, iBucket, key+"/")
if err != nil {
return nil, httperrors.NewInternalServerError("fail to mkdir: %s", err)
}
@@ -674,9 +679,13 @@ func (bucket *SBucket) PerformDelete(
}
ok := jsonutils.NewDict()
results := modules.BatchDo(keyStrs, func(key string) (jsonutils.JSONObject, error) {
err := iBucket.DeleteObject(ctx, key)
if strings.HasSuffix(key, "/") {
err = cloudprovider.DeletePrefix(ctx, iBucket, key)
} else {
err = iBucket.DeleteObject(ctx, key)
}
if err != nil {
return nil, err
return nil, errors.Wrap(err, "DeletePrefix")
} else {
return ok, nil
}
@@ -704,6 +713,11 @@ func (bucket *SBucket) PerformUpload(
appParams := appsrv.AppContextGetParams(ctx)
key := appParams.Request.Header.Get(api.BUCKET_UPLOAD_OBJECT_KEY_HEADER)
if strings.HasSuffix(key, "/") {
return nil, httperrors.NewInputParameterError("object key should not ends with /")
}
err := s3utils.CheckValidObjectName(key)
if err != nil {
return nil, httperrors.NewInputParameterError("invalid object key: %s", err)
@@ -723,19 +737,21 @@ func (bucket *SBucket) PerformUpload(
if err != nil {
return nil, httperrors.NewInputParameterError("Illegal Content-Length %s", sizeStr)
}
if sizeBytes <= 0 {
return nil, httperrors.NewInputParameterError("Content-Length not positive %d", sizeBytes)
if sizeBytes < 0 {
return nil, httperrors.NewInputParameterError("Content-Length negative %d", sizeBytes)
}
storageClass := appParams.Request.Header.Get(api.BUCKET_UPLOAD_OBJECT_STORAGECLASS_HEADER)
aclStr := cloudprovider.TBucketACLType(appParams.Request.Header.Get(api.BUCKET_UPLOAD_OBJECT_ACL_HEADER))
switch aclStr {
case cloudprovider.ACLPrivate, cloudprovider.ACLAuthRead, cloudprovider.ACLPublicRead, cloudprovider.ACLPublicReadWrite:
// do nothing
default:
return nil, httperrors.NewInputParameterError("invalid acl: %s", aclStr)
aclStr := appParams.Request.Header.Get(api.BUCKET_UPLOAD_OBJECT_ACL_HEADER)
if len(aclStr) > 0 {
switch cloudprovider.TBucketACLType(aclStr) {
case cloudprovider.ACLPrivate, cloudprovider.ACLAuthRead, cloudprovider.ACLPublicRead, cloudprovider.ACLPublicReadWrite:
// do nothing
default:
return nil, httperrors.NewInputParameterError("invalid acl: %s", aclStr)
}
}
err = cloudprovider.UploadObject(ctx, iBucket, key, 0, appParams.Request.Body, sizeBytes, contType, aclStr, storageClass, false)
err = cloudprovider.UploadObject(ctx, iBucket, key, 0, appParams.Request.Body, sizeBytes, contType, cloudprovider.TBucketACLType(aclStr), storageClass, false)
if err != nil {
return nil, httperrors.NewInternalServerError("put object error %s", err)
}
pkg/compute/models/cloudproviders.go
+11
View File
@@ -1010,6 +1010,17 @@ func (manager *SCloudproviderManager) ListItemFilter(ctx context.Context, q *sql
q = q.Filter(sqlchemy.IsTrue(cloudaccounts.Field("is_on_premise")))
}
if query.Contains("has_object_storage") {
hasObjectStorage, _ := query.Bool("has_object_storage")
cloudaccounts := CloudaccountManager.Query().SubQuery()
q = q.Join(cloudaccounts, sqlchemy.Equals(cloudaccounts.Field("id"), q.Field("cloudaccount_id")))
if hasObjectStorage {
q = q.Filter(sqlchemy.IsTrue(cloudaccounts.Field("has_object_storage")))
} else {
q = q.Filter(sqlchemy.IsFalse(cloudaccounts.Field("has_object_storage")))
}
}
return q, nil
}
pkg/multicloud/aws/bucket.go
+1 -1
View File
@@ -210,7 +210,7 @@ func (b *SBucket) GetIObjects(prefix string, isRecursive bool) ([]cloudprovider.
}
func (b *SBucket) PutObject(ctx context.Context, key string, body io.Reader, sizeBytes int64, contType string, cannedAcl cloudprovider.TBucketACLType, storageClassStr string) error {
if sizeBytes <= 0 {
if sizeBytes < 0 {
return errors.Error("content length expected")
}
s3cli, err := b.region.GetS3Client()
pkg/util/fileutils2/seeker_test.go
+24 -20
View File
@@ -21,25 +21,29 @@ import (
)
func TestNewReadSeeker(t *testing.T) {
testStr := "This is a test reader string"
seeker, err := NewReadSeeker(strings.NewReader(testStr), int64(len(testStr)))
if err != nil {
t.Fatalf("NewReadSeeker error %s", err)
}
defer seeker.Close()
buf1 := make([]byte, 1024)
n, err := seeker.Read(buf1)
if n != len(testStr) {
t.Fatalf("read buf1 error %s", err)
}
buf2 := make([]byte, 1024)
n, err = seeker.Read(buf2)
if n != 0 {
t.Fatalf("read buf2 should fail")
}
seeker.Seek(0, io.SeekStart)
n, err = seeker.Read(buf2)
if n != len(testStr) {
t.Fatalf("read buf2 error %s", err)
for _, testStr := range []string{
"This is a test reader string",
"",
} {
seeker, err := NewReadSeeker(strings.NewReader(testStr), int64(len(testStr)))
if err != nil {
t.Fatalf("NewReadSeeker error %s", err)
}
defer seeker.Close()
buf1 := make([]byte, 1024)
n, err := seeker.Read(buf1)
if n != len(testStr) {
t.Fatalf("read buf1 error %s", err)
}
buf2 := make([]byte, 1024)
n, err = seeker.Read(buf2)
if n != 0 {
t.Fatalf("read buf2 should fail")
}
seeker.Seek(0, io.SeekStart)
n, err = seeker.Read(buf2)
if n != len(testStr) {
t.Fatalf("read buf2 error %s", err)
}
}
}