diff --git a/python/tank/descriptor/io_descriptor/git.py b/python/tank/descriptor/io_descriptor/git.py index 2ea2014347..23ba4dee15 100644 --- a/python/tank/descriptor/io_descriptor/git.py +++ b/python/tank/descriptor/io_descriptor/git.py @@ -8,20 +8,21 @@ # agreement to the Shotgun Pipeline Toolkit Source Code License. All rights # not expressly granted therein are reserved by Shotgun Software Inc. import os -import uuid -import shutil -import tempfile +import platform import subprocess +from time import time + from .downloadable import IODescriptorDownloadable from ... import LogManager from ...util.process import subprocess_check_output, SubprocessCalledProcessError -from ..errors import TankError +from ..errors import TankError, TankDescriptorError from ...util import filesystem from ...util import is_windows log = LogManager.get_logger(__name__) +IS_WINDOWS = platform.system() == "Windows" def _can_hide_terminal(): @@ -34,7 +35,8 @@ def _can_hide_terminal(): subprocess.STARTF_USESHOWWINDOW subprocess.SW_HIDE return True - except Exception: + except AttributeError as e: + log.debug("Terminal cant be hidden: %s" % e) return False @@ -59,7 +61,40 @@ class TankGitError(TankError): pass -class IODescriptorGit(IODescriptorDownloadable): +class _IODescriptorGitCache(type): + """Use as metaclass. Caches object instances for 2min.""" + + _instances = {} + + def __call__(cls, descriptor_dict, sg_connection, bundle_type): + now = int(time() / 100) + floored_time = now - now % 2 # Cache is valid for 2min + + if ( + descriptor_dict["type"] == "git_branch" + ): # cant fetch last commit here, too soon + version = "-".join( + filter( + None, [descriptor_dict.get("version"), descriptor_dict["branch"]] + ) + ) + else: + version = descriptor_dict["version"] + + id_ = "-".join([descriptor_dict["type"], descriptor_dict["path"], version]) + + cached_time, self = cls._instances.get(id_, (-1, None)) + if cached_time < floored_time: + log.debug( + "{} {} cache expired: cachedTime:{}".format(self, id_, cached_time) + ) + self = super().__call__(descriptor_dict, sg_connection, bundle_type) + cls._instances[id_] = (floored_time, self) + + return self + + +class IODescriptorGit(IODescriptorDownloadable, metaclass=_IODescriptorGitCache): """ Base class for git descriptors. @@ -81,11 +116,67 @@ def __init__(self, descriptor_dict, sg_connection, bundle_type): descriptor_dict, sg_connection, bundle_type ) - self._path = descriptor_dict.get("path") - # strip trailing slashes - this is so that when we build - # the name later (using os.basename) we construct it correctly. - if self._path.endswith("/") or self._path.endswith("\\"): - self._path = self._path[:-1] + _path = self._normalize_path(descriptor_dict.get("path")) + + if self._path_is_local(_path): + self._path = self._execute_git_commands( + ["git", "-C", _path, "remote", "get-url", "origin"] + ) + log.debug( + "Get remote url from local repo: {} -> {}".format(_path, self._path) + ) + filtered_local_data = { + k: v + for k, v in self._fetch_local_data(_path).items() + if k not in descriptor_dict + } + descriptor_dict.update(filtered_local_data) + else: + self._path = _path + + def is_git_available(self): + log.debug("Checking that git exists and can be executed...") + + if IS_WINDOWS: + cmd = "where" + else: + cmd = "which" + + try: + output = _check_output([cmd, "git"]) + except SubprocessCalledProcessError: + raise TankGitError( + "Cannot execute the 'git' command. Please make sure that git is " + "installed on your system and that the git executable has been added to the PATH." + ) + else: + log.debug("Git installed: %s" % output) + return True + + def _execute_git_commands(self, commands): + # first probe to check that git exists in our PATH + self.is_git_available() + + log.debug("Executing command '%s' using subprocess module." % commands) + + # It's important to pass GIT_TERMINAL_PROMPT=0 or the git subprocess will + # just hang waiting for credentials to be entered on the missing terminal. + # I would have expected Windows to give an error about stdin being close and + # aborting the git command but at least on Windows 10 that is not the case. + environ = os.environ.copy() + environ["GIT_TERMINAL_PROMPT"] = "0" + + try: + output = _check_output(commands, env=environ) + except SubprocessCalledProcessError as e: + raise TankGitError( + "Error executing git operation '%s': %s (Return code %s)" + % (commands, e.output, e.returncode) + ) + else: + output = output.strip().strip("'") + log.debug("Execution successful. stderr/stdout: '%s'" % output) + return output @LogManager.log_timing def _clone_then_execute_git_commands( @@ -124,127 +215,19 @@ def _clone_then_execute_git_commands( """ # ensure *parent* folder exists parent_folder = os.path.dirname(target_path) - - filesystem.ensure_folder_exists(parent_folder) - - # first probe to check that git exists in our PATH - log.debug("Checking that git exists and can be executed...") - try: - output = _check_output(["git", "--version"]) - except: - log.exception("Unexpected error:") - raise TankGitError( - "Cannot execute the 'git' command. Please make sure that git is " - "installed on your system and that the git executable has been added to the PATH." - ) - - log.debug("Git installed: %s" % output) + os.makedirs(parent_folder, exist_ok=True) # Make sure all git commands are correct according to the descriptor type - cmd = self._validate_git_commands( + cmd = self._get_git_clone_commands( target_path, depth=depth, ref=ref, is_latest_commit=is_latest_commit ) + self._execute_git_commands(cmd) - run_with_os_system = True + if commands: + full_commands = ["git", "-C", os.path.normpath(target_path)] + full_commands.extend(commands) - # We used to call only os.system here. On macOS and Linux this behaved correctly, - # i.e. if stdin was open you would be prompted on the terminal and if not then an - # error would be raised. This is the best we could do. - # - # However, for Windows, os.system actually pops a terminal, which is annoying, especially - # if you don't require to authenticate. To avoid this popup, we will first launch - # git through the subprocess module and instruct it to not show a terminal for the - # subprocess. - # - # If that fails, then we'll assume that it failed because credentials were required. - # Unfortunately, we can't tell why it failed. - # - # Note: We only try this workflow if we can actually hide the terminal on Windows. - # If we can't there's no point doing all of this and we should just use - # os.system. - if is_windows() and _can_hide_terminal(): - log.debug("Executing command '%s' using subprocess module." % cmd) - try: - # It's important to pass GIT_TERMINAL_PROMPT=0 or the git subprocess will - # just hang waiting for credentials to be entered on the missing terminal. - # I would have expected Windows to give an error about stdin being close and - # aborting the git command but at least on Windows 10 that is not the case. - environ = {} - environ.update(os.environ) - environ["GIT_TERMINAL_PROMPT"] = "0" - _check_output(cmd, env=environ) - - # If that works, we're done and we don't need to use os.system. - run_with_os_system = False - status = 0 - except SubprocessCalledProcessError: - log.debug("Subprocess call failed.") - - if run_with_os_system: - # Make sure path and repo path are quoted. - log.debug("Executing command '%s' using os.system" % cmd) - log.debug( - "Note: in a terminal environment, this may prompt for authentication" - ) - status = os.system(cmd) - - log.debug("Command returned exit code %s" % status) - if status != 0: - raise TankGitError( - "Error executing git operation. The git command '%s' " - "returned error code %s." % (cmd, status) - ) - log.debug("Git clone into '%s' successful." % target_path) - - # clone worked ok! Now execute git commands on this repo - - output = None - - for command in commands: - # we use git -C to specify the working directory where to execute the command - # this option was added in as part of git 1.9 - # and solves an issue with UNC paths on windows. - full_command = 'git -C "%s" %s' % (target_path, command) - log.debug("Executing '%s'" % full_command) - - try: - output = _check_output(full_command, shell=True) - - # note: it seems on windows, the result is sometimes wrapped in single quotes. - output = output.strip().strip("'") - - except SubprocessCalledProcessError as e: - raise TankGitError( - f"Error executing GIT operation '{full_command}': {e.output}" - f" (Return code {e.returncode}). " - " Supported GIT version: 1.9+." - ) - log.debug("Execution successful. stderr/stdout: '%s'" % output) - - # return the last returned stdout/stderr - return output - - def _tmp_clone_then_execute_git_commands(self, commands, depth=None, ref=None): - """ - Clone into a temp location and executes the given - list of git commands. - - For more details, see :meth:`_clone_then_execute_git_commands`. - - :param commands: list git commands to execute, e.g. ['checkout x'] - :returns: stdout and stderr of the last command executed as a string - """ - clone_tmp = os.path.join( - tempfile.gettempdir(), "sgtk_clone_%s" % uuid.uuid4().hex - ) - filesystem.ensure_folder_exists(clone_tmp) - try: - return self._clone_then_execute_git_commands( - clone_tmp, commands, depth, ref - ) - finally: - log.debug("Cleaning up temp location '%s'" % clone_tmp) - shutil.rmtree(clone_tmp, ignore_errors=True) + return self._execute_git_commands(full_commands) def get_system_name(self): """ @@ -268,10 +251,9 @@ def has_remote_access(self): can_connect = True try: log.debug("%r: Probing if a connection to git can be established..." % self) - # clone repo into temp folder - self._tmp_clone_then_execute_git_commands([], depth=1) + self._execute_git_commands(["git", "ls-remote", "--heads", self._path]) log.debug("...connection established") - except Exception as e: + except (OSError, SubprocessCalledProcessError) as e: log.debug("...could not establish connection: %s" % e) can_connect = False return can_connect @@ -293,17 +275,44 @@ def _copy(self, target_path, skip_list=None): # make sure item exists locally self.ensure_local() # copy descriptor into target. - # the skip list contains .git folders by default, so pass in [] - # to turn that restriction off. In the case of the git descriptor, - # we want to transfer this folder as well. - filesystem.copy_folder( - self.get_path(), - target_path, - # Make we do not pass none or we will be getting the default skip list. - skip_list=skip_list or [], - ) + filesystem.copy_folder(self.get_path(), target_path, skip_list=skip_list or []) + + def _normalize_path(self, path): + if path.endswith("/") or path.endswith("\\"): + new_path = path[:-1] + else: + new_path = path + + if os.path.isdir(new_path): + if not new_path.endswith(".git"): + new_path = os.path.join(new_path, ".git") + + return new_path + + def _path_is_local(self, path): + """ + Check if path value is an existing folder, and if contain a .git folder. + """ + if os.path.isdir(path): + output = self._execute_git_commands( + [ + "git", + "-C", + os.path.normpath(path), + "rev-parse", + "--git-dir", + ] + ) + if output.startswith("fatal: not a git repository"): + raise TankDescriptorError( + "Folder is not a git repository: {}".format(path) + ) + elif output == ".": + return True + + return False - def _validate_git_commands( + def _get_git_clone_commands( self, target_path, depth=None, ref=None, is_latest_commit=None ): """ @@ -323,23 +332,22 @@ def _validate_git_commands( # complications in cleanup scenarios and with file copying. We want # each repo that we clone to be completely independent on a filesystem level. log.debug("Git Cloning %r into %s" % (self, target_path)) - depth = "--depth %s" % depth if depth else "" - ref = "-b %s" % ref if ref else "" - cmd = 'git clone --no-hardlinks -q "%s" %s "%s" %s' % ( - self._path, - ref, - target_path, - depth, - ) - if self._descriptor_dict.get("type") == "git_branch": - if not is_latest_commit: - if "--depth" in cmd: - depth = "" - cmd = 'git clone --no-hardlinks -q "%s" %s "%s" %s' % ( - self._path, - ref, - target_path, - depth, - ) + + if self._descriptor_dict.get("type") == "git_branch" and not is_latest_commit: + depth = None + + cmd = ["git", "clone", "--no-hardlinks", "-q"] + + if ref: + cmd.extend(["-b", str(ref)]) + + if depth: + cmd.extend(["--depth", str(depth)]) + + cmd.append(self._path) + cmd.append(os.path.normpath(target_path)) return cmd + + def _fetch_local_data(self, path): + return {} diff --git a/python/tank/descriptor/io_descriptor/git_branch.py b/python/tank/descriptor/io_descriptor/git_branch.py index 745060213a..2c60c7ea89 100644 --- a/python/tank/descriptor/io_descriptor/git_branch.py +++ b/python/tank/descriptor/io_descriptor/git_branch.py @@ -10,7 +10,7 @@ import os import copy -from .git import IODescriptorGit, TankGitError, _check_output +from .git import IODescriptorGit, TankGitError from ..errors import TankDescriptorError from ... import LogManager @@ -19,6 +19,8 @@ except ImportError: from tank_vendor import six as sgutils +from ...util.process import SubprocessCalledProcessError + log = LogManager.get_logger(__name__) @@ -64,7 +66,7 @@ def __init__(self, descriptor_dict, sg_connection, bundle_type): """ # make sure all required fields are there self._validate_descriptor( - descriptor_dict, required=["type", "path", "version", "branch"], optional=[] + descriptor_dict, required=["type", "path", "branch"], optional=["version"] ) # call base class @@ -76,8 +78,8 @@ def __init__(self, descriptor_dict, sg_connection, bundle_type): # have a path to a repo self._sg_connection = sg_connection self._bundle_type = bundle_type - self._version = descriptor_dict.get("version") self._branch = sgutils.ensure_str(descriptor_dict.get("branch")) + self._version = descriptor_dict.get("version") or self.get_latest_commit() def __str__(self): """ @@ -94,44 +96,50 @@ def _get_bundle_cache_path(self, bundle_cache_root): :param bundle_cache_root: Bundle cache root path :return: Path to bundle cache location """ - # If the descriptor is an integer change the version to a string type - if isinstance(self._version, int): - self._version = str(self._version) - # to be MAXPATH-friendly, we only use the first seven chars - short_hash = self._version[:7] - # git@github.com:manneohrstrom/tk-hiero-publish.git -> tk-hiero-publish.git # /full/path/to/local/repo.git -> repo.git name = os.path.basename(self._path) - return os.path.join(bundle_cache_root, "gitbranch", name, short_hash) + return os.path.join( + bundle_cache_root, "gitbranch", name, self.get_short_version() + ) def get_version(self): - """ - Returns the version number string for this item, .e.g 'v1.2.3' - or the branch name 'master' - """ + """Returns the full commit sha.""" return self._version - def _is_latest_commit(self, version, branch): + def get_short_version(self): + """Returns the short commit sha.""" + return self._version[:7] + + def get_latest_commit(self): + """Fetch the latest commit on a specific branch""" + output = self._execute_git_commands( + ["git", "ls-remote", self._path, self._branch] + ) + latest_commit = output.split("\t")[0] + + if not latest_commit: + raise TankDescriptorError( + "Could not get latest commit for %s, branch %s" + % (self._path, self._branch) + ) + + return sgutils.ensure_str(latest_commit) + + def get_latest_short_commit(self): + return self.get_latest_commit()[:7] + + def _is_latest_commit(self): """ Check if the git_branch descriptor is pointing to the latest commit version. """ # first probe to check that git exists in our PATH log.debug("Checking if the version is pointing to the latest commit...") - try: - output = _check_output(["git", "ls-remote", self._path, branch]) - except: - log.exception("Unexpected error:") - raise TankGitError( - "Cannot execute the 'git' command. Please make sure that git is " - "installed on your system and that the git executable has been added to the PATH." - ) - latest_commit = output.split("\t") - short_latest_commit = latest_commit[0][:7] + short_latest_commit = self.get_latest_short_commit() - if short_latest_commit != version[:7]: + if short_latest_commit != self.get_short_version(): return False log.debug( "This version is pointing to the latest commit %s, lets enable shallow clones" @@ -140,38 +148,29 @@ def _is_latest_commit(self, version, branch): return True - def _download_local(self, destination_path): + def _download_local(self, target_path): """ - Retrieves this version to local repo. - Will exit early if app already exists local. - - This will connect to remote git repositories. - Depending on how git is configured, https repositories - requiring credentials may result in a shell opening up - requesting username and password. - - The git repo will be cloned into the local cache and - will then be adjusted to point at the relevant commit. - - :param destination_path: The destination path on disk to which - the git branch descriptor is to be downloaded to. + Downloads the data represented by the descriptor into the primary bundle + cache path. """ + log.info("Downloading {}:{}".format(self.get_system_name(), self._version)) + depth = None - is_latest_commit = self._is_latest_commit(self._version, self._branch) + is_latest_commit = self._is_latest_commit() if is_latest_commit: depth = 1 try: # clone the repo, switch to the given branch # then reset to the given commit - commands = ['checkout -q "%s"' % self._version] + commands = [f"checkout", "-q", self._version] self._clone_then_execute_git_commands( - destination_path, + target_path, commands, depth=depth, ref=self._branch, is_latest_commit=is_latest_commit, ) - except Exception as e: + except (TankGitError, OSError, SubprocessCalledProcessError) as e: raise TankDescriptorError( "Could not download %s, branch %s, " "commit %s: %s" % (self._path, self._branch, self._version, e) @@ -209,24 +208,9 @@ def get_latest_version(self, constraint_pattern=None): "Latest version will be used." % self ) - try: - # clone the repo, get the latest commit hash - # for the given branch - commands = [ - 'checkout -q "%s"' % self._branch, - "log -n 1 \"%s\" --pretty=format:'%%H'" % self._branch, - ] - git_hash = self._tmp_clone_then_execute_git_commands(commands) - - except Exception as e: - raise TankDescriptorError( - "Could not get latest commit for %s, " - "branch %s: %s" % (self._path, self._branch, e) - ) - # make a new descriptor new_loc_dict = copy.deepcopy(self._descriptor_dict) - new_loc_dict["version"] = sgutils.ensure_str(git_hash) + new_loc_dict["version"] = self.get_latest_commit() desc = IODescriptorGitBranch( new_loc_dict, self._sg_connection, self._bundle_type ) @@ -255,3 +239,16 @@ def get_latest_cached_version(self, constraint_pattern=None): else: # no cached version exists return None + + def _fetch_local_data(self, path): + version = self._execute_git_commands( + ["git", "-C", os.path.normpath(path), "rev-parse", "HEAD"] + ) + + branch = self._execute_git_commands( + ["git", "-C", os.path.normpath(path), "branch", "--show-current"] + ) + + local_data = {"version": version, "branch": branch} + log.debug("Get local repo data: {}".format(local_data)) + return local_data diff --git a/python/tank/descriptor/io_descriptor/git_tag.py b/python/tank/descriptor/io_descriptor/git_tag.py index 77dad56c8b..b4e583d95d 100644 --- a/python/tank/descriptor/io_descriptor/git_tag.py +++ b/python/tank/descriptor/io_descriptor/git_tag.py @@ -11,7 +11,7 @@ import copy import re -from .git import IODescriptorGit +from .git import IODescriptorGit, TankGitError from ..errors import TankDescriptorError from ... import LogManager @@ -20,8 +20,12 @@ except ImportError: from tank_vendor import six as sgutils +from ...util.process import SubprocessCalledProcessError + log = LogManager.get_logger(__name__) +TAG_REGEX = re.compile(".*refs/tags/([^^/]+)$") + class IODescriptorGitTag(IODescriptorGit): """ @@ -62,10 +66,25 @@ def __init__(self, descriptor_dict, sg_connection, bundle_type): # path is handled by base class - all git descriptors # have a path to a repo - self._version = descriptor_dict.get("version") self._sg_connection = sg_connection self._bundle_type = bundle_type + raw_version = descriptor_dict.get("version") + raw_version_is_latest = raw_version == "latest" + + if "x" in raw_version or raw_version_is_latest: + if raw_version_is_latest: + self._version = self._get_latest_by_pattern(None) + else: + self._version = self._get_latest_by_pattern(raw_version) + log.info( + "{}-{} resolved as {}".format( + self.get_system_name(), raw_version, self._version + ) + ) + else: + self._version = raw_version + def __str__(self): """ Human readable representation @@ -73,6 +92,16 @@ def __str__(self): # git@github.com:manneohrstrom/tk-hiero-publish.git, tag v1.2.3 return "%s, Tag %s" % (self._path, self._version) + @property + def _tags(self): + """Fetch tags if necessary.""" + try: + return self.__tags + except AttributeError: + log.info("Fetch tags for {}".format(self.get_system_name())) + self.__tags = self._fetch_tags() + return self.__tags + def _get_bundle_cache_path(self, bundle_cache_root): """ Given a cache root, compute a cache path suitable @@ -122,33 +151,21 @@ def _get_cache_paths(self): return paths def get_version(self): - """ - Returns the version number string for this item, .e.g 'v1.2.3' - """ + """Returns the tag name.""" return self._version - def _download_local(self, destination_path): + def _download_local(self, target_path): """ - Retrieves this version to local repo. - Will exit early if app already exists local. - - This will connect to remote git repositories. - Depending on how git is configured, https repositories - requiring credentials may result in a shell opening up - requesting username and password. - - The git repo will be cloned into the local cache and - will then be adjusted to point at the relevant tag. - - :param destination_path: The destination path on disk to which - the git tag descriptor is to be downloaded to. + Downloads the data represented by the descriptor into the primary bundle + cache path. """ + log.info("Downloading {}:{}".format(self.get_system_name(), self._version)) try: # clone the repo, checkout the given tag self._clone_then_execute_git_commands( - destination_path, [], depth=1, ref=self._version + target_path, [], depth=1, ref=self._version ) - except Exception as e: + except (TankGitError, OSError, SubprocessCalledProcessError) as e: raise TankDescriptorError( "Could not download %s, " "tag %s: %s" % (self._path, self._version, e) ) @@ -174,10 +191,7 @@ def get_latest_version(self, constraint_pattern=None): :returns: IODescriptorGitTag object """ - if constraint_pattern: - tag_name = self._get_latest_by_pattern(constraint_pattern) - else: - tag_name = self._get_latest_version() + tag_name = self._get_latest_by_pattern(constraint_pattern) new_loc_dict = copy.deepcopy(self._descriptor_dict) new_loc_dict["version"] = sgutils.ensure_str(tag_name) @@ -199,57 +213,47 @@ def _get_latest_by_pattern(self, pattern): - v1.2.3.x (will always return a forked version, eg. v1.2.3.2) :returns: IODescriptorGitTag object """ - git_tags = self._fetch_tags() - latest_tag = self._find_latest_tag_by_pattern(git_tags, pattern) - if latest_tag is None: - raise TankDescriptorError( - "'%s' does not have a version matching the pattern '%s'. " - "Available versions are: %s" - % (self.get_system_name(), pattern, ", ".join(git_tags)) - ) + if not pattern: + latest_tag = self._get_latest_tag() + else: + latest_tag = self._find_latest_tag_by_pattern(self._tags, pattern) + if latest_tag is None: + raise TankDescriptorError( + "'%s' does not have a version matching the pattern '%s'. " + "Available versions are: %s" + % (self.get_system_name(), pattern, ", ".join(self._tags)) + ) return latest_tag def _fetch_tags(self): - try: - # clone the repo, list all tags - # for the repository, across all branches - commands = ["ls-remote -q --tags %s" % self._path] - tags = self._tmp_clone_then_execute_git_commands(commands, depth=1).split( - "\n" - ) - regex = re.compile(".*refs/tags/([^^]*)$") - git_tags = [] - for tag in tags: - m = regex.match(sgutils.ensure_str(tag)) - if m: - git_tags.append(m.group(1)) - - except Exception as e: - raise TankDescriptorError( - "Could not get list of tags for %s: %s" % (self._path, e) - ) + output = self._execute_git_commands( + ["git", "ls-remote", "-q", "--tags", self._path] + ) - if len(git_tags) == 0: - raise TankDescriptorError( - "Git repository %s doesn't have any tags!" % self._path - ) + git_tags = [] + for line in output.splitlines(): + m = TAG_REGEX.match(sgutils.ensure_str(line)) + if m: + git_tags.append(m.group(1)) return git_tags - def _get_latest_version(self): - """ - Returns a descriptor object that represents the latest version. - :returns: IODescriptorGitTag object - """ - tags = self._fetch_tags() - latest_tag = self._find_latest_tag_by_pattern(tags, pattern=None) - if latest_tag is None: + def _get_latest_tag(self): + """Get latest tag name. Compare them as version numbers.""" + if not self._tags: raise TankDescriptorError( "Git repository %s doesn't have any tags!" % self._path ) - return latest_tag + tupled_tags = [] + for t in self._tags: + items = t.split(".") + tupled_tags.append( + tuple(int(item) if item.isdigit() else item for item in items) + ) + + return ".".join(map(str, sorted(tupled_tags)[-1])) def get_latest_cached_version(self, constraint_pattern=None): """ @@ -287,3 +291,12 @@ def get_latest_cached_version(self, constraint_pattern=None): log.debug("Latest cached version resolved to %r" % desc) return desc + + def _fetch_local_data(self, path): + version = self._execute_git_commands( + ["git", "-C", os.path.normpath(path), "describe", "--tags", "--abbrev=0"] + ) + + local_data = {"version": version} + log.debug("Get local repo data: {}".format(local_data)) + return local_data diff --git a/python/tank/util/filesystem.py b/python/tank/util/filesystem.py index c4d4320736..275b47c10f 100644 --- a/python/tank/util/filesystem.py +++ b/python/tank/util/filesystem.py @@ -270,7 +270,12 @@ def copy_folder(src, dst, folder_permissions=0o775, skip_list=None): if os.path.isdir(srcname): files.extend(copy_folder(srcname, dstname, folder_permissions)) else: + if os.path.exists(dstname) and not os.access(dstname, os.W_OK): + # if destination already exists but is readonly, change it to writable + os.chmod(dstname, stat.S_IWUSR) + shutil.copy(srcname, dstname) + files.append(srcname) # if the file extension is sh, set executable permissions if ( diff --git a/tests/descriptor_tests/test_descriptors.py b/tests/descriptor_tests/test_descriptors.py index 1538d9a1c6..f84fa1cee2 100644 --- a/tests/descriptor_tests/test_descriptors.py +++ b/tests/descriptor_tests/test_descriptors.py @@ -687,16 +687,32 @@ def test_git_branch_descriptor_commands(self): } ) self.assertEqual( - desc._io_descriptor._validate_git_commands( + desc._io_descriptor._get_git_clone_commands( target_path, depth=1, ref="master" ), - 'git clone --no-hardlinks -q "%s" %s "%s" ' - % (self.git_repo_uri, "-b master", target_path,), + [ + "git", + "clone", + "--no-hardlinks", + "-q", + "-b", + "master", + self.git_repo_uri, + target_path, + ], ) self.assertEqual( - desc._io_descriptor._validate_git_commands(target_path, ref="master"), - 'git clone --no-hardlinks -q "%s" %s "%s" ' - % (self.git_repo_uri, "-b master", target_path,), + desc._io_descriptor._get_git_clone_commands(target_path, ref="master"), + [ + "git", + "clone", + "--no-hardlinks", + "-q", + "-b", + "master", + self.git_repo_uri, + target_path, + ], ) diff --git a/tests/descriptor_tests/test_downloadables.py b/tests/descriptor_tests/test_downloadables.py index 1a9b029893..4b92311c12 100644 --- a/tests/descriptor_tests/test_downloadables.py +++ b/tests/descriptor_tests/test_downloadables.py @@ -46,6 +46,26 @@ def _raise_exception(placeholder_a="default_a", placeholder_b="default_b"): raise OSError("An unknown OSError occurred") +def onerror(func, path, exc_info): + """ + Error handler for ``shutil.rmtree``. + + If the error is due to an access error (read only file) + it attempts to add write permission and then retries. + + If the error is for another reason it re-raises the error. + + Usage : ``shutil.rmtree(path, onerror=onerror)`` + """ + import stat + + if not os.access(path, os.W_OK): + os.chmod(path, stat.S_IWUSR) + func(path) + else: + raise + + class TestDownloadableIODescriptors(ShotgunTestBase): """ Tests the ability of the descriptor to download to a path on disk. @@ -624,7 +644,9 @@ def test_descriptor_rename_error_fallbacks(self, *_): "e1c03fa", ) if os.path.exists(git_location): - shutil.move(git_location, "%s.bak.%s" % (git_location, uuid.uuid4().hex)) + # cant use `shutil.move` on readonly files on windows + shutil.copytree(git_location, "%s.bak.%s" % (git_location, uuid.uuid4().hex)) + shutil.rmtree(git_location, onerror=onerror) # make sure nothing exists self.assertFalse(os.path.exists(git_location)) @@ -676,7 +698,9 @@ def our_move_mock(src, dst): "e1c03fa", ) if os.path.exists(git_location): - shutil.move(git_location, "%s.bak.%s" % (git_location, uuid.uuid4().hex)) + # cant use `shutil.move` on readonly files on windows + shutil.copytree(git_location, "%s.bak.%s" % (git_location, uuid.uuid4().hex)) + shutil.rmtree(git_location, onerror=onerror) # make sure nothing exists self.assertFalse(os.path.exists(git_location)) @@ -712,7 +736,9 @@ def test_partial_download_handling(self): "e1c03fa", ) if os.path.exists(git_location): - shutil.move(git_location, "%s.bak.%s" % (git_location, uuid.uuid4().hex)) + # cant use `shutil.move` on readonly files on windows + shutil.copytree(git_location, "%s.bak.%s" % (git_location, uuid.uuid4().hex)) + shutil.rmtree(git_location, onerror=onerror) # make sure nothing exists self.assertFalse(os.path.exists(git_location)) diff --git a/tests/descriptor_tests/test_git.py b/tests/descriptor_tests/test_git.py index 914f311f2c..38c812847a 100644 --- a/tests/descriptor_tests/test_git.py +++ b/tests/descriptor_tests/test_git.py @@ -62,12 +62,12 @@ def test_latest(self): } desc = self._create_desc(location_dict, True) - self.assertEqual(desc.version, "30c293f29a50b1e58d2580522656695825523dba") + self.assertEqual(desc.version, "dac945d50d2bd0a828181dc3e1d31cfea2c64065") location_dict = {"type": "git", "path": self.git_repo_uri} desc = self._create_desc(location_dict, True) - self.assertEqual(desc.version, "v0.16.1") + self.assertEqual(desc.version, "v0.18.2") @skip_if_git_missing def test_tag(self): @@ -94,20 +94,20 @@ def test_tag(self): latest_desc = desc.find_latest_version() - self.assertEqual(latest_desc.version, "v0.16.1") + self.assertEqual(latest_desc.version, "v0.18.2") self.assertEqual(latest_desc.get_path(), None) latest_desc.ensure_local() - self.assertEqual(latest_desc.find_latest_cached_version().version, "v0.16.1") + self.assertEqual(latest_desc.find_latest_cached_version().version, "v0.18.2") self.assertEqual( - latest_desc.find_latest_cached_version("v0.16.x").version, "v0.16.1" + latest_desc.find_latest_cached_version("v0.18.x").version, "v0.18.2" ) self.assertEqual( latest_desc.get_path(), - os.path.join(self.bundle_cache, "git", "tk-config-default.git", "v0.16.1"), + os.path.join(self.bundle_cache, "git", "tk-config-default.git", "v0.18.2"), ) latest_desc = desc.find_latest_version("v0.15.x") @@ -176,7 +176,7 @@ def test_branch(self): latest_desc = desc.find_latest_version() self.assertEqual( - latest_desc.version, "30c293f29a50b1e58d2580522656695825523dba" + latest_desc.version, "dac945d50d2bd0a828181dc3e1d31cfea2c64065" ) self.assertEqual(latest_desc.get_path(), None) @@ -185,7 +185,7 @@ def test_branch(self): self.assertEqual( latest_desc.get_path(), os.path.join( - self.bundle_cache, "gitbranch", "tk-config-default.git", "30c293f" + self.bundle_cache, "gitbranch", "tk-config-default.git", "dac945d" ), ) @@ -212,7 +212,7 @@ def test_branch(self): latest_desc = desc.find_latest_version() self.assertEqual( - latest_desc.version, "7fa75a749c1dfdbd9ad93ee3497c7eaa8e1a488d" + latest_desc.version, "b2960d7e8cce43dda63369a6805881443b3daf17" ) self.assertEqual(latest_desc.get_path(), None) @@ -221,7 +221,7 @@ def test_branch(self): self.assertEqual( latest_desc.get_path(), os.path.join( - self.bundle_cache, "gitbranch", "tk-config-default.git", "7fa75a7" + self.bundle_cache, "gitbranch", "tk-config-default.git", "b2960d7" ), ) diff --git a/tests/descriptor_tests/test_io_descriptors.py b/tests/descriptor_tests/test_io_descriptors.py index cc635ad3de..6a6a150a3e 100644 --- a/tests/descriptor_tests/test_io_descriptors.py +++ b/tests/descriptor_tests/test_io_descriptors.py @@ -227,7 +227,7 @@ def test_git_branch_descriptor_input(self): sg = self.mockgun location = { "type": "git_branch", - "version": 6547378, + "version": "6547378", "branch": "master", "path": self.git_repo_uri, }