Gapra/sync input conversion

azcopy/optionsUtil.go

@@ -0,0 +1,80 @@
+package azcopy
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"runtime"
+
+	"github.com/Azure/azure-storage-azcopy/v10/common"
+)
+
+// BlockSizeInBytes converts a FLOATING POINT number of MiB, to a number of bytes
+// A non-nil error is returned if the conversion is not possible to do accurately (e.g. it comes out of a fractional number of bytes)
+// The purpose of using floating point is to allow specialist users (e.g. those who want small block sizes to tune their read IOPS)
+// to use fractions of a MiB. E.g.
+// 0.25 = 256 KiB
+// 0.015625 = 16 KiB
+func BlockSizeInBytes(rawBlockSizeInMiB float64) (int64, error) {
+	if rawBlockSizeInMiB < 0 {
+		return 0, errors.New("negative block size not allowed")
+	}
+	rawSizeInBytes := rawBlockSizeInMiB * 1024 * 1024 // internally we use bytes, but users' convenience the command line uses MiB
+	if rawSizeInBytes > math.MaxInt64 {
+		return 0, errors.New("block size too big for int64")
+	}
+	const epsilon = 0.001 // arbitrarily using a tolerance of 1000th of a byte
+	_, frac := math.Modf(rawSizeInBytes)
+	isWholeNumber := frac < epsilon || frac > 1.0-epsilon // frac is very close to 0 or 1, so rawSizeInBytes is (very close to) an integer
+	if !isWholeNumber {
+		return 0, fmt.Errorf("while fractional numbers of MiB are allowed as the block size, the fraction must result to a whole number of bytes. %.12f MiB resolves to %.3f bytes", rawBlockSizeInMiB, rawSizeInBytes)
+	}
+	return int64(math.Round(rawSizeInBytes)), nil
+}
+
+// GetPreserveInfoDefault returns the default value for the PreserveInfo option based on the current OS and FromTo.
+func GetPreserveInfoDefault(fromTo common.FromTo) bool {
+	// defaults to true for NFS-aware transfers, and SMB-aware transfers on Windows.
+	return AreBothLocationsNFSAware(fromTo) ||
+		(runtime.GOOS == "windows" && areBothLocationsSMBAware(fromTo))
+}
+
+func AreBothLocationsNFSAware(fromTo common.FromTo) bool {
+	// 1. Upload (Linux -> Azure File)
+	// 2. Download (Azure File -> Linux)
+	// 3. S2S (Azure File -> Azure File) (Works on Windows,Linux,Mac)
+	if (runtime.GOOS == "linux") &&
+		(fromTo == common.EFromTo.LocalFileNFS() || fromTo == common.EFromTo.FileNFSLocal()) {
+		return true
+	} else if fromTo == common.EFromTo.FileNFSFileNFS() {
+		return true
+	} else {
+		return false
+	}
+}
+
+func areBothLocationsSMBAware(fromTo common.FromTo) bool {
+	// 1. Upload (Windows/Linux -> Azure File)
+	// 2. Download (Azure File -> Windows/Linux)
+	// 3. S2S (Azure File -> Azure File)
+	if (runtime.GOOS == "windows" || runtime.GOOS == "linux") &&
+		(fromTo == common.EFromTo.LocalFile() || fromTo == common.EFromTo.FileLocal()) {
+		return true
+	} else if fromTo == common.EFromTo.FileFile() {
+		return true
+	} else {
+		return false
+	}
+}
+
+func areBothLocationsPOSIXAware(fromTo common.FromTo) bool {
+	// POSIX properties are stored in blob metadata-- They don't need a special persistence strategy for S2S methods.
+	switch fromTo {
+	case common.EFromTo.BlobLocal(), common.EFromTo.LocalBlob(), common.EFromTo.BlobFSLocal(), common.EFromTo.LocalBlobFS():
+		return runtime.GOOS == "linux"
+	case common.EFromTo.BlobBlob(), common.EFromTo.BlobFSBlobFS(), common.EFromTo.BlobFSBlob(), common.EFromTo.BlobBlobFS():
+		return true
+	default:
+		return false
+	}
+}

azcopy/pathUtils.go

@@ -0,0 +1,128 @@
+package azcopy
+
+import (
+	"fmt"
+	"net/url"
+	"strings"
+
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas"
+	"github.com/Azure/azure-storage-azcopy/v10/common"
+	"github.com/pkg/errors"
+)
+
+// ----- LOCATION LEVEL HANDLING -----
+type LocationLevel uint8
+
+var ELocationLevel LocationLevel = 0
+
+func (LocationLevel) Account() LocationLevel   { return 0 } // Account is never used in AzCopy, but is in testing to understand resource management.
+func (LocationLevel) Service() LocationLevel   { return 1 }
+func (LocationLevel) Container() LocationLevel { return 2 }
+func (LocationLevel) Object() LocationLevel    { return 3 } // An Object can be a directory or object.
+
+// Uses syntax to assume the "level" of a location.
+// This is typically used to
+func DetermineLocationLevel(location string, locationType common.Location, source bool) (LocationLevel, error) {
+	switch locationType {
+	// In local, there's no such thing as a service.
+	// As such, we'll treat folders as containers, and files as objects.
+	case common.ELocation.Local():
+		level := LocationLevel(ELocationLevel.Object())
+		if strings.Contains(location, "*") {
+			return ELocationLevel.Container(), nil
+		}
+
+		if strings.HasSuffix(location, "/") {
+			level = ELocationLevel.Container()
+		}
+
+		if !source {
+			return level, nil // Return the assumption.
+		}
+
+		fi, err := common.OSStat(location)
+
+		if err != nil {
+			return level, nil // Return the assumption.
+		}
+
+		if fi.IsDir() {
+			return ELocationLevel.Container(), nil
+		} else {
+			return ELocationLevel.Object(), nil
+		}
+	case common.ELocation.Benchmark():
+		return ELocationLevel.Object(), nil // we always benchmark to a subfolder, not the container root
+
+	case common.ELocation.Blob(),
+		common.ELocation.File(),
+		common.ELocation.FileNFS(),
+		common.ELocation.BlobFS(),
+		common.ELocation.S3(),
+		common.ELocation.GCP():
+		URL, err := url.Parse(location)
+
+		if err != nil {
+			return ELocationLevel.Service(), err
+		}
+
+		// GenericURLParts determines the correct resource URL parts to make use of
+		bURL := common.NewGenericResourceURLParts(*URL, locationType)
+
+		if strings.Contains(bURL.GetContainerName(), "*") && bURL.GetObjectName() != "" {
+			return ELocationLevel.Service(), errors.New("can't use a wildcarded container name and specific blob name in combination")
+		}
+
+		if bURL.GetObjectName() != "" {
+			return ELocationLevel.Object(), nil
+		} else if bURL.GetContainerName() != "" && !strings.Contains(bURL.GetContainerName(), "*") {
+			return ELocationLevel.Container(), nil
+		} else {
+			return ELocationLevel.Service(), nil
+		}
+	default: // Probably won't ever hit this
+		return ELocationLevel.Service(), fmt.Errorf("getting level of location is impossible on location %s", locationType)
+	}
+}
+
+func GetContainerName(path string, location common.Location) (string, error) {
+	switch location {
+	case common.ELocation.Local():
+		panic("attempted to get container name on local location")
+	case common.ELocation.Blob(),
+		common.ELocation.File(),
+		common.ELocation.FileNFS(),
+		common.ELocation.BlobFS():
+		bURLParts, err := sas.ParseURL(path)
+		if err != nil {
+			return "", err
+		}
+		return bURLParts.ContainerName, nil
+	case common.ELocation.S3():
+		baseURL, err := url.Parse(path)
+
+		if err != nil {
+			return "", err
+		}
+
+		s3URLParts, err := common.NewS3URLParts(*baseURL)
+
+		if err != nil {
+			return "", err
+		}
+
+		return s3URLParts.BucketName, nil
+	case common.ELocation.GCP():
+		baseURL, err := url.Parse(path)
+		if err != nil {
+			return "", err
+		}
+		gcpURLParts, err := common.NewGCPURLParts(*baseURL)
+		if err != nil {
+			return "", err
+		}
+		return gcpURLParts.BucketName, nil
+	default:
+		return "", fmt.Errorf("cannot get container name on location type %s", location.String())
+	}
+}

azcopy/sync.go

@@ -0,0 +1,76 @@
+package azcopy
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/Azure/azure-storage-azcopy/v10/common"
+)
+
+// SyncOptions contains the optional parameters for the Sync operation.
+type SyncOptions struct {
+	FromTo                  common.FromTo
+	Recursive               *bool // Default true
+	IncludeDirectoryStubs   bool
+	PreserveInfo            *bool // Default true
+	PreservePosixProperties bool
+	ForceIfReadOnly         bool
+	BlockSizeMB             float64
+	PutBlobSizeMB           float64
+	IncludePatterns         []string
+	ExcludePatterns         []string
+	ExcludePaths            []string
+	IncludeAttributes       []string
+	ExcludeAttributes       []string
+	IncludeRegex            []string
+	ExcludeRegex            []string
+	DeleteDestination       common.DeleteDestination
+	PutMd5                  bool                        // TODO: (gapra) Should we make this an enum called PutHash for None/MD5? So user can set the HashType?
+	CheckMd5                common.HashValidationOption // TODO (gapra) Same comment as above
+	S2SPreserveAccessTier   *bool                       // Default true
+	S2SPreserveBlobTags     bool
+	CpkByName               string
+	CpkByValue              bool
+	MirrorMode              bool
+	TrailingDot             common.TrailingDotOption
+	IncludeRoot             bool
+	CompareHash             common.SyncHashType
+	HashMetaDir             string
+	LocalHashStorageMode    *common.HashStorageMode // Default based on OS
+	PreservePermissions     bool
+	Hardlinks               common.HardlinkHandlingType
+
+	dryrun                           bool
+	deleteDestinationFileIfNecessary bool
+}
+
+/* AzCopy internal use only. Exposing these as setters to add a hurdle to their use. */
+
+func (s *SyncOptions) SetInternalOptions(dryrun, deleteDestinationFileIfNecessary bool) {
+	s.dryrun = dryrun
+	s.deleteDestinationFileIfNecessary = deleteDestinationFileIfNecessary
+}
+
+// Sync
+// 1. Phase 1 will implement arg processing and validation only.
+// 2. Phase 2 will implement enumerator initialization
+// 3. Phase 3 will implement the sync progress tracking
+func (c *Client) Sync(ctx context.Context, src, dest string, opts SyncOptions) (ret *Syncer, err error) {
+	// Input
+	if src == "" || dest == "" {
+		return nil, fmt.Errorf("source and destination must be specified for sync")
+	}
+
+	// AzCopy CLI sets this globally before calling Sync.
+	// If in library mode, this will not be set and we will use the user-provided handler.
+	// Note: It is not ideal that this is a global, but keeping it this way for now to avoid a larger refactor than this already is.
+	syncHandler := common.GetLifecycleMgr()
+	if syncHandler == nil {
+		syncHandler = common.NewJobUIHooks()
+		common.SetUIHooks(syncHandler)
+	}
+
+	ret, err = newSyncer(src, dest, opts)
+
+	return
+}