diff --git a/alibaba.go b/alibaba.go index 6c4427a..f45b71f 100644 --- a/alibaba.go +++ b/alibaba.go @@ -53,7 +53,6 @@ func NewAlibabaCloudOSSBackend(bucket string, prefix string, endpoint string, ss } client, err := oss.New(endpoint, accessKeyId, accessKeySecret) - if err != nil { panic("Failed to create OSS client: " + err.Error()) } @@ -76,7 +75,7 @@ func NewAlibabaCloudOSSBackend(bucket string, prefix string, endpoint string, ss func (b AlibabaCloudOSSBackend) ListObjects(prefix string) ([]Object, error) { var objects []Object - prefix = pathutil.Join(b.Prefix, prefix) + prefix = joinAndNormalizePrefix(b.Prefix, prefix) ossPrefix := oss.Prefix(prefix) marker := oss.Marker("") for { @@ -113,7 +112,6 @@ func (b AlibabaCloudOSSBackend) GetObject(path string) (Object, error) { var content []byte key := pathutil.Join(b.Prefix, path) body, err := b.Bucket.GetObject(key) - if err != nil { return object, err } diff --git a/amazon.go b/amazon.go index 731cc08..bc2c3bc 100644 --- a/amazon.go +++ b/amazon.go @@ -18,12 +18,13 @@ package storage import ( "bytes" + "crypto/tls" "io/ioutil" - pathutil "path" - "strings" "net/http" - "crypto/tls" "os" + pathutil "path" + "strings" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" @@ -133,7 +134,7 @@ func NewAmazonS3BackendWithCredentials(bucket string, prefix string, region stri // ListObjects lists all objects in Amazon S3 bucket, at prefix func (b AmazonS3Backend) ListObjects(prefix string) ([]Object, error) { var objects []Object - prefix = pathutil.Join(b.Prefix, prefix) + prefix = joinAndNormalizePrefix(b.Prefix, prefix) s3Input := &s3.ListObjectsInput{ Bucket: aws.String(b.Bucket), Prefix: aws.String(prefix), diff --git a/baidu.go b/baidu.go index 4ca1097..0174e6a 100644 --- a/baidu.go +++ b/baidu.go @@ -52,7 +52,6 @@ func NewBaiDuBOSBackend(bucket string, prefix string, endpoint string) *BaiduBOS } client, err := bos.NewClient(accessKeyId, accessKeySecret, endpoint) - if err != nil { panic("Failed to create BOS client: " + err.Error()) } @@ -67,44 +66,44 @@ func NewBaiDuBOSBackend(bucket string, prefix string, endpoint string) *BaiduBOS // ListObjects lists all objects in Baidu Cloud BOS bucket, at prefix func (b BaiduBOSBackend) ListObjects(prefix string) ([]Object, error) { - var objects []Object - - prefix = pathutil.Join(b.Prefix, prefix) - listObjectsArgs := &api.ListObjectsArgs{ - Prefix: prefix, - Marker: "", - MaxKeys: 1000, - } - for { - lor, err := b.Client.ListObjects(b.Bucket, listObjectsArgs) - if err != nil { - return objects, err - } - - for _, obj := range lor.Contents { - path := removePrefixFromObjectPath(prefix, obj.Key) - if objectPathIsInvalid(path) { - continue - } - lastModified, err := time.Parse(time.RFC3339, obj.LastModified) - if err != nil { - continue - } - object := Object{ - Path: path, - Content: []byte{}, - LastModified: lastModified, - } - objects = append(objects, object) - } - if !lor.IsTruncated { - break - } - listObjectsArgs.Prefix = lor.Prefix - listObjectsArgs.Marker = lor.NextMarker - } - - return objects, nil + var objects []Object + + prefix = joinAndNormalizePrefix(b.Prefix, prefix) + listObjectsArgs := &api.ListObjectsArgs{ + Prefix: prefix, + Marker: "", + MaxKeys: 1000, + } + for { + lor, err := b.Client.ListObjects(b.Bucket, listObjectsArgs) + if err != nil { + return objects, err + } + + for _, obj := range lor.Contents { + path := removePrefixFromObjectPath(prefix, obj.Key) + if objectPathIsInvalid(path) { + continue + } + lastModified, err := time.Parse(time.RFC3339, obj.LastModified) + if err != nil { + continue + } + object := Object{ + Path: path, + Content: []byte{}, + LastModified: lastModified, + } + objects = append(objects, object) + } + if !lor.IsTruncated { + break + } + listObjectsArgs.Prefix = lor.Prefix + listObjectsArgs.Marker = lor.NextMarker + } + + return objects, nil } // GetObject retrieves an object from Baidu Cloud BOS bucket, at prefix diff --git a/etcd.go b/etcd.go index a553996..eb823a2 100644 --- a/etcd.go +++ b/etcd.go @@ -2,6 +2,7 @@ package storage import ( "context" + "errors" "fmt" pathutil "path" "strconv" @@ -9,8 +10,8 @@ import ( "sync" "time" - clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/pkg/v3/transport" + clientv3 "go.etcd.io/etcd/client/v3" ) const DefaultPrefix = "/chart_backend_bucket" @@ -18,8 +19,8 @@ const DefaultPrefix = "/chart_backend_bucket" var ( DefileDialTimeOut = "5s" TimeStampKey = "timestamp" - ErrNotExistEndpoints = fmt.Errorf("endpoints cannot connect !") - ErrNotExist = fmt.Errorf("not exist!") + ErrNotExistEndpoints = errors.New("endpoints cannot connect !") + ErrNotExist = errors.New("not exist!") ) type etcdOpts struct { @@ -38,7 +39,7 @@ type etcdStorage struct { mu sync.RWMutex } -//connection cut off +// connection cut off func isServerErr(err error) bool { if err != nil { if err == context.Canceled { @@ -52,11 +53,9 @@ func isServerErr(err error) bool { return false } -//prapare {basepath} dir +// prapare {basepath} dir func (e *etcdStorage) probe() error { - var ( - err error - ) + var err error ctx, cancel := context.WithCancel(e.ctx) _, err = e.c.Put(ctx, e.base, "") cancel() @@ -100,13 +99,10 @@ func (e *etcdStorage) delTimeStamp(path string) error { return err } -// func (e *etcdStorage) ListObjects(prefix string) ([]Object, error) { - var ( - objs []Object - ) + var objs []Object ctx, cancel := context.WithTimeout(e.ctx, e.opts.dialtimeout) - newpath := pathutil.Join(e.base, prefix) + newpath := joinAndNormalizePrefix(e.base, prefix) resps, err := e.c.Get(ctx, newpath, clientv3.WithPrefix()) cancel() if err != nil { @@ -118,7 +114,7 @@ func (e *etcdStorage) ListObjects(prefix string) ([]Object, error) { if objectPathIsInvalid(path) { continue } - //TODO need optimizate + // TODO need optimizate if strings.HasSuffix(path, TimeStampKey) { continue } @@ -134,13 +130,10 @@ func (e *etcdStorage) ListObjects(prefix string) ([]Object, error) { } } return objs, nil - } func (e *etcdStorage) GetObject(path string) (Object, error) { - var ( - modifytime time.Time - ) + var modifytime time.Time ctx, cancel := context.WithTimeout(e.ctx, e.opts.dialtimeout) newpath := pathutil.Join(e.base, path) resps, err := e.c.Get(ctx, newpath) @@ -164,9 +157,7 @@ func (e *etcdStorage) GetObject(path string) (Object, error) { } func (e *etcdStorage) PutObject(path string, content []byte) error { - var ( - updatetime = time.Now() - ) + updatetime := time.Now() ctx, cancel := context.WithTimeout(e.ctx, e.opts.dialtimeout) newpath := pathutil.Join(e.base, path) _, err := e.c.Put(ctx, newpath, string(content)) @@ -191,9 +182,7 @@ func (e *etcdStorage) DeleteObject(path string) error { } func parseConf(endpoints string, cafile, certfile, keyfile string, dialtime time.Duration) clientv3.Config { - var ( - es []string - ) + var es []string if endpoints == "" { panic(ErrNotExistEndpoints) } @@ -215,9 +204,7 @@ func parseConf(endpoints string, cafile, certfile, keyfile string, dialtime time } func NewEtcdCSBackend(endpoints string, cafile, certfile, keyfile string, prefix string) Backend { - var ( - basepath string - ) + var basepath string DialTimeOut, _ := time.ParseDuration(DefileDialTimeOut) cli, err := clientv3.New(parseConf(endpoints, cafile, certfile, keyfile, DialTimeOut)) if err != nil { @@ -249,5 +236,4 @@ func NewEtcdCSBackend(endpoints string, cafile, certfile, keyfile string, prefix panic(err) } return e - } diff --git a/google.go b/google.go index 9a82aaf..03759ea 100644 --- a/google.go +++ b/google.go @@ -52,7 +52,7 @@ func NewGoogleCSBackend(bucket string, prefix string) *GoogleCSBackend { // ListObjects lists all objects in Google Cloud Storage bucket, at prefix func (b GoogleCSBackend) ListObjects(prefix string) ([]Object, error) { var objects []Object - prefix = pathutil.Join(b.Prefix, prefix) + prefix = joinAndNormalizePrefix(b.Prefix, prefix) listQuery := &storage.Query{ Prefix: prefix, } diff --git a/microsoft.go b/microsoft.go index bbe52bb..d7ebdbe 100644 --- a/microsoft.go +++ b/microsoft.go @@ -19,11 +19,10 @@ package storage import ( "errors" "io/ioutil" + "os" pathutil "path" "time" - "os" - microsoft_storage "github.com/Azure/azure-sdk-for-go/storage" ) @@ -39,7 +38,6 @@ type MicrosoftBlobBackend struct { // NewMicrosoftBlobBackend creates a new instance of MicrosoftBlobBackend func NewMicrosoftBlobBackend(container string, prefix string) *MicrosoftBlobBackend { - // From the Azure portal, get your storage account name and key and set environment variables. accountName, accountKey := os.Getenv("AZURE_STORAGE_ACCOUNT"), os.Getenv("AZURE_STORAGE_ACCESS_KEY") var serviceBaseURL, apiVersion string @@ -78,7 +76,7 @@ func (b MicrosoftBlobBackend) ListObjects(prefix string) ([]Object, error) { } var params microsoft_storage.ListBlobsParameters - prefix = pathutil.Join(b.Prefix, prefix) + prefix = joinAndNormalizePrefix(b.Prefix, prefix) params.Prefix = prefix for { diff --git a/openstack.go b/openstack.go index 6cb92a6..ad23ce4 100644 --- a/openstack.go +++ b/openstack.go @@ -224,7 +224,7 @@ func NewOpenstackOSBackendV1Auth(container string, prefix string, caCert string) func (b OpenstackOSBackend) ListObjects(prefix string) ([]Object, error) { var objects []Object - prefix = pathutil.Join(b.Prefix, prefix) + prefix = joinAndNormalizePrefix(b.Prefix, prefix) opts := &osObjects.ListOpts{ Full: true, Prefix: prefix, diff --git a/oracle.go b/oracle.go index e9e9b30..8846b7b 100644 --- a/oracle.go +++ b/oracle.go @@ -43,7 +43,6 @@ type OracleCSBackend struct { // NewOracleCSBackend creates a new instance of OracleCSBackend func NewOracleCSBackend(bucket string, prefix string, region string, compartmentId string) *OracleCSBackend { - var config common.ConfigurationProvider var err error @@ -102,7 +101,6 @@ func NewOracleCSBackend(bucket string, prefix string, region string, compartment } func createBucket(ctx context.Context, c objectstorage.ObjectStorageClient, namespace string, bucket string, compartmentId string) (string, error) { - // Create the bucket request := objectstorage.CreateBucketRequest{ NamespaceName: &namespace, @@ -115,7 +113,6 @@ func createBucket(ctx context.Context, c objectstorage.ObjectStorageClient, name _, err := c.CreateBucket(ctx, request) return bucket, err - } func getNamespace(ctx context.Context, c objectstorage.ObjectStorageClient) (string, error) { @@ -131,7 +128,7 @@ func getNamespace(ctx context.Context, c objectstorage.ObjectStorageClient) (str // ListObjects lists all objects in OCI Object Storage bucket, at prefix func (b OracleCSBackend) ListObjects(prefix string) ([]Object, error) { var objects []Object - prefix = pathutil.Join(b.Prefix, prefix) + prefix = joinAndNormalizePrefix(b.Prefix, prefix) request := objectstorage.ListObjectsRequest{ NamespaceName: &b.Namespace, @@ -181,14 +178,12 @@ func (b OracleCSBackend) GetObject(path string) (Object, error) { } rc, err := b.Client.GetObject(b.Context, request) - if err != nil { return object, err } object.LastModified = rc.LastModified.Time content, err := ioutil.ReadAll(rc.Content) - if err != nil { return object, err } @@ -198,7 +193,6 @@ func (b OracleCSBackend) GetObject(path string) (Object, error) { // PutObject uploads an object to OCI Object Storage bucket, at prefix func (b OracleCSBackend) PutObject(path string, content []byte) error { - objectname := pathutil.Join(b.Prefix, path) metadata := make(map[string]string) contentLen := int64(binary.Size(content)) @@ -219,7 +213,6 @@ func (b OracleCSBackend) PutObject(path string, content []byte) error { // DeleteObject removes an object from OCI Object Storage bucket, at prefix func (b OracleCSBackend) DeleteObject(path string) error { - objectname := pathutil.Join(b.Prefix, path) request := objectstorage.DeleteObjectRequest{ diff --git a/storage.go b/storage.go index a402f7e..4c42e3c 100644 --- a/storage.go +++ b/storage.go @@ -18,6 +18,7 @@ package storage import ( "fmt" + "path" "path/filepath" "strings" "time" @@ -97,6 +98,20 @@ func cleanPrefix(prefix string) string { return strings.Trim(prefix, "/") } +// normalizePath ensures a non-empty path ends with a slash +func normalizePath(path string) string { + if path != "" && !strings.HasSuffix(path, "/") { + path += "/" + } + return path +} + +// joinAndNormalizePrefix joins base prefix and prefix, then ensures the result ends with a slash +func joinAndNormalizePrefix(basePrefix, prefix string) string { + joined := path.Join(basePrefix, prefix) + return normalizePath(joined) +} + func removePrefixFromObjectPath(prefix string, path string) string { if prefix == "" { return path diff --git a/storage_test.go b/storage_test.go index 79cce28..faf1adf 100644 --- a/storage_test.go +++ b/storage_test.go @@ -178,6 +178,24 @@ func (suite *StorageTestSuite) TestHasSuffix() { suite.False(o2.HasExtension("tgz"), "object does not have tgz suffix") } +func (suite *StorageTestSuite) TestNormalizePath() { + suite.Equal("", normalizePath(""), "empty path normalized to empty string") + suite.Equal("/", normalizePath("/"), "path with only slash normalized to single slash") + suite.Equal("prefix/", normalizePath("prefix"), "path without trailing slash normalized to path with trailing slash") + suite.Equal("prefix/", normalizePath("prefix/"), "path with trailing slash remains unchanged") + suite.Equal("prefix/sth/", normalizePath("prefix/sth"), "path with subdirectory normalized to path with trailing slash") +} + +func (suite *StorageTestSuite) TestJoinAndNormalizePrefix() { + suite.Equal("base/prefix/", joinAndNormalizePrefix("base", "prefix"), "join base and prefix with normalization") + suite.Equal("base/", joinAndNormalizePrefix("base", ""), "join base with empty prefix") + suite.Equal("prefix/", joinAndNormalizePrefix("", "prefix"), "join empty base with prefix") + suite.Equal("", joinAndNormalizePrefix("", ""), "join empty base and prefix returns empty string") + suite.Equal("base/prefix/sub/", joinAndNormalizePrefix("base", "prefix/sub"), "join base with multi-level prefix") + suite.Equal("base/prefix/", joinAndNormalizePrefix("base/", "prefix"), "join base with trailing slash and prefix") + suite.Equal("base/prefix/", joinAndNormalizePrefix("base", "/prefix"), "join base with prefix starting with slash") +} + func (suite *StorageTestSuite) TestGetObjectSliceDiff() { now := time.Now() os1 := []Object{ diff --git a/tencent.go b/tencent.go index 5ab49d7..88d79e3 100644 --- a/tencent.go +++ b/tencent.go @@ -44,7 +44,6 @@ const ( // NewTencentCloudCOSBackend creates a new instance of TencentCloudCOSBackend func NewTencentCloudCOSBackend(bucket string, prefix string, endpoint string) *TencentCloudCOSBackend { - secretID := os.Getenv("TENCENT_CLOUD_COS_SECRET_ID") secretKey := os.Getenv("TENCENT_CLOUD_COS_SECRET_KEY") @@ -85,10 +84,9 @@ func NewTencentCloudCOSBackend(bucket string, prefix string, endpoint string) *T // ListObjects lists all objects in Tencent Cloud COS bucket, at prefix func (t TencentCloudCOSBackend) ListObjects(prefix string) ([]Object, error) { - var objects []Object - prefix = pathutil.Join(t.Prefix, prefix) + prefix = joinAndNormalizePrefix(t.Prefix, prefix) cosPrefix := prefix cosMarker := "" if cosPrefix != "" { @@ -133,7 +131,6 @@ func (t TencentCloudCOSBackend) ListObjects(prefix string) ([]Object, error) { // GetObject retrieves an object from Tencent Cloud COS bucket, at prefix func (t TencentCloudCOSBackend) GetObject(path string) (Object, error) { - var object Object object.Path = path @@ -164,7 +161,6 @@ func (t TencentCloudCOSBackend) GetObject(path string) (Object, error) { // PutObject uploads an object to Tencent Cloud COS bucket, at prefix func (t TencentCloudCOSBackend) PutObject(path string, content []byte) error { - key := pathutil.Join(t.Prefix, path) var err error @@ -176,7 +172,6 @@ func (t TencentCloudCOSBackend) PutObject(path string, content []byte) error { // DeleteObject removes an object from Tencent Cloud COS bucket, at prefix func (t TencentCloudCOSBackend) DeleteObject(path string) error { - key := pathutil.Join(t.Prefix, path) _, err := t.Object.Delete(context.Background(), key) return err