Publish refs from spack-packages. clone_spack handle split repo

Author: kwryankrattiger

images/protected-publish/pkg/common.py

@@ -15,6 +15,7 @@
 
 
 SPACK_REPO = "https://github.com/spack/spack"
+PACKAGES_REPO = "https://github.com/spack/spack-packages"
 
 TIMESTAMP_AND_SIZE = r"^[\d]{4}-[\d]{2}-[\d]{2}\s[\d]{2}:[\d]{2}:[\d]{2}\s+\d+\s+"
 TIMESTAMP_PATTERN = "%Y-%m-%d %H:%M:%S"
@@ -188,12 +189,16 @@ def extract_json_from_clearsig(file_path):
 # clone the matching version of spack.
 #
 # Clones the version of spack specified by ref to the root of the file system
-def clone_spack(ref: str = "develop", repo: str = SPACK_REPO, clone_dir: str = "/"):
+def clone_spack(spack_ref: str = "develop", packages_ref: str = "develop", spack_repo: str = SPACK_REPO, packages_repo: str = PACKAGES_REPO, clone_dir: str = "/"):
     spack_path = f"{clone_dir}/spack"
+    packages_path = f"{clone_dir}/spack-packages"
 
     if os.path.isdir(spack_path):
         shutil.rmtree(spack_path)
 
+    if os.path.isdir(packages_path):
+        shutil.rmtree(packages_path)
+
     owd = os.getcwd()
 
     try:
@@ -206,8 +211,35 @@ def clone_spack(ref: str = "develop", repo: str = SPACK_REPO, clone_dir: str = "
                 "1",
                 "--single-branch",
                 "--branch",
-                f"{ref}",
-                f"{repo}",
+                f"{spack_ref}",
+                f"{spack_repo}",
+                spack_path,
+            ],
+            check=True,
+        )
+        subprocess.run(
+            [
+                "git",
+                "clone",
+                "--depth",
+                "1",
+                "--single-branch",
+                "--branch",
+                f"{packages_ref}",
+                f"{packages_repo}",
+                packages_path,
+            ],
+            check=True,
+        )
+        # Configure the repo destination
+        subprocess.run(
+            [
+                "spack/bin/spack",
+                "repo",
+                "set",
+                "builtin",
+                "--destination",
+                packages_path,
             ],
             check=True,
         )

images/protected-publish/pkg/publish.py

@@ -280,13 +280,14 @@ def publish(
     print("Publishing complete")
 
     # Clone spack version appropriate to what we're publishing
-    clone_spack(ref, clone_dir=workdir)
+    clone_spack(packages_ref=ref, clone_dir=workdir)
     spack_exe = f"{workdir}/spack/bin/spack"
 
     # Can be useful for testing to clone a custom spack to somewhere other than "/"
     # clone_spack(
-    #     ref="content-addressable-tarballs-2",
-    #     repo="https://github.com/scottwittenburg/spack.git",
+    #     packages_ref=ref,
+    #     spack_ref="content-addressable-tarballs-2",
+    #     spack_repo="https://github.com/scottwittenburg/spack.git",
     #     clone_dir=workdir,
     # )
     # spack_exe = f"{workdir}/spack/bin/spack"
@@ -641,7 +642,7 @@ def main():
     parser.add_argument(
         "-r",
         "--ref",
-        default="develop",
+        action="append",
         help=(
             "A single protected ref to publish, or else 'recent', to "
             "publish any protected refs that had a pipeline recently"
@@ -690,10 +691,18 @@ def main():
 
     args = parser.parse_args()
 
-    if args.ref == "recent":
+    refs = []
+    if not args.ref:
+        refs = ["develop"]
+
+    if "recent" in args.ref:
         refs = get_recently_run_protected_refs(args.days)
-    else:
-        refs = [args.ref]
+        args.ref.remove("recent")
+
+    if args.ref:
+        refs.extend(list(args.ref))
+        print(args.ref)
+        print(refs)
 
     exceptions = []