From 85cc8fef1133851e6a2aae712cffd1ad0a737d26 Mon Sep 17 00:00:00 2001 From: Paul Hildebrandt Date: Thu, 5 Dec 2024 17:49:02 +0100 Subject: [PATCH 1/2] only update index once Signed-off-by: Paul Hildebrandt --- src/glcli.py | 14 +++---- src/registry.py | 100 +++++++++++------------------------------------- 2 files changed, 29 insertions(+), 85 deletions(-) diff --git a/src/glcli.py b/src/glcli.py index cf82d40..4f607d8 100755 --- a/src/glcli.py +++ b/src/glcli.py @@ -43,7 +43,7 @@ def cli(): ) @click.option( "--manifest_file", - default="manifest-entry.json", + default="manifests/manifest.json", help="A file where the index entry for the pushed manifest is written to.", ) @click.option( @@ -82,24 +82,24 @@ def push_manifest( help="Version of image", ) @click.option( - "--manifest_file", - default="manifest-entry.json", - help="A file where the index entry is read from.", + "--manifest_folder", + default="manifests", + help="A folder where the index entries are read from.", ) @click.option( "--insecure", default=False, help="Use HTTP to communicate with the registry", ) -def update_index(container, version, manifest_file, insecure): - """push a index entry from a file to an index""" +def update_index(container, version, manifest_folder, insecure): + """push a index entry from a list of files to an index""" container_name = f"{container}:{version}" registry = GlociRegistry( container_name=container_name, token=os.getenv("GLOCI_REGISTRY_TOKEN"), insecure=insecure, ) - registry.update_index_entries(manifest_file, version) + registry.update_index(manifest_folder) if __name__ == "__main__": diff --git a/src/registry.py b/src/registry.py index 17d02d2..9932679 100644 --- a/src/registry.py +++ b/src/registry.py @@ -196,7 +196,7 @@ def get_index(self, allowed_media_type: Optional[list[str]] = None): return index except ValueError: - logger.debug("Index not found, creating new Index!") + logger.info("Index not found, creating new Index!") return NewIndex() @ensure_container @@ -290,27 +290,6 @@ def get_manifest_by_cname( container, manifest_digest, allowed_media_type=allowed_media_type ) - def update_index(self, old_digest: Optional[str], manifest_meta_data: dict): - """ - replaces an old manifest entry with a new manifest entry - """ - index = self.get_index() - - if "manifests" not in index: - logger.debug("Index is empty") - updated = False - if old_digest is not None: - for i, manifest in enumerate(index["manifests"]): - if manifest["digest"] == old_digest: - logger.debug("Found old manifest entry") - index["manifests"][i] = manifest_meta_data - updated = True - break - if not updated: - logger.debug("Did NOT find old manifest entry") - index["manifests"].append(manifest_meta_data) - - return index def change_state(self, cname: str, version: str, architecture: str, new_state: str): manifest_container = OrasContainer( @@ -324,44 +303,6 @@ def change_state(self, cname: str, version: str, architecture: str, new_state: s logger.warning("No annotations found in manifest, init annotations now.") manifest["annotations"] = {} - def attach_layer( - self, - cname: str, - version: str, - architecture: str, - file_path: str, - media_type: str, - ): - if not os.path.exists(file_path): - exit(f"{file_path} does not exist.") - - manifest_container = OrasContainer( - f"{self.container_name}-{cname}-{architecture}" - ) - - manifest = self.get_manifest_by_cname( - self.container, cname, version, architecture - ) - - layer = self.create_layer(file_path, cname, version, architecture, media_type) - self._check_200_response(self.upload_blob(file_path, self.container, layer)) - - manifest["layers"].append(layer) - - old_manifest_digest = self.get_digest(manifest_container) - self._check_200_response(self.upload_manifest(manifest, manifest_container)) - - new_manifest_metadata = self.get_manifest_meta_data_by_cname( - self.container, cname, version, architecture - ) - new_manifest_metadata["digest"] = self.get_digest(manifest_container) - new_manifest_metadata["size"] = self.get_manifest_size(manifest_container) - new_manifest_metadata["platform"] = NewPlatform(architecture, version) - - new_index = self.update_index(old_manifest_digest, new_manifest_metadata) - self._check_200_response(self.upload_index(new_index)) - - print(f"Successfully attached {file_path} to {manifest_container}") @ensure_container def remove_container(self, container: OrasContainer): @@ -516,26 +457,29 @@ def push_image_manifest( print(json.dumps(manifest_index_metadata), file=open(manifest_file, "w")) logger.info(f"Index entry written to {manifest_file}") - # self.update_index_entries(architecture, cname, manifest_index_metadata, version) - return local_digest - def update_index_entries(self, manifest_file, version): - manifest_index_metadata = json.loads(open(manifest_file, "r").read()) - arch = manifest_index_metadata["annotations"]["architecture"] - cname = manifest_index_metadata["annotations"]["cname"] - old_manifest_meta_data = self.get_manifest_meta_data_by_cname( - self.container, cname, version, arch - ) - if old_manifest_meta_data is not None: - new_index = self.update_index( - old_manifest_meta_data["digest"], manifest_index_metadata - ) - logger.info("Replaced manifest entry") - else: - new_index = self.update_index(None, manifest_index_metadata) - logger.info("Appended manifest entry") - self._check_200_response(self.upload_index(new_index)) + def update_index(self, manifest_folder): + index = self.get_index() + new_entries = 0 + + for file in os.listdir(manifest_folder): + manifest_metadata = json.loads(open(manifest_folder + "/" + file, "r").read()) + # Skip if manifest with same digest already exists + found = False + for entry in index["manifests"]: + if entry["digest"] == manifest_metadata["digest"]: + found = True + break + if found: + logger.info(f"Skipping manifest with digest {manifest_metadata["digest"]} - already exists") + continue + index["manifests"].append(manifest_metadata) + logger.info(f"Index appended locally {manifest_metadata["annotations"]["cname"]}") + new_entries+=1 + + self._check_200_response(self.upload_index(index)) + logger.info(f"Index pushed with {new_entries} new entries") def create_layer( self, From 87bf2d8ad50caaac66d6b31d5c6a2c4b52e6da67 Mon Sep 17 00:00:00 2001 From: Paul Hildebrandt Date: Thu, 5 Dec 2024 17:58:44 +0100 Subject: [PATCH 2/2] adjust e2e tests Signed-off-by: Paul Hildebrandt --- src/registry.py | 16 ++++++++++------ tests/conftest.py | 3 +++ tests/test_e2e.py | 6 ++++-- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/registry.py b/src/registry.py index 9932679..2456c51 100644 --- a/src/registry.py +++ b/src/registry.py @@ -290,7 +290,6 @@ def get_manifest_by_cname( container, manifest_digest, allowed_media_type=allowed_media_type ) - def change_state(self, cname: str, version: str, architecture: str, new_state: str): manifest_container = OrasContainer( f"{self.container_name}-{cname}-{architecture}" @@ -303,7 +302,6 @@ def change_state(self, cname: str, version: str, architecture: str, new_state: s logger.warning("No annotations found in manifest, init annotations now.") manifest["annotations"] = {} - @ensure_container def remove_container(self, container: OrasContainer): self.delete_tag(container.manifest_url()) @@ -464,7 +462,9 @@ def update_index(self, manifest_folder): new_entries = 0 for file in os.listdir(manifest_folder): - manifest_metadata = json.loads(open(manifest_folder + "/" + file, "r").read()) + manifest_metadata = json.loads( + open(manifest_folder + "/" + file, "r").read() + ) # Skip if manifest with same digest already exists found = False for entry in index["manifests"]: @@ -472,11 +472,15 @@ def update_index(self, manifest_folder): found = True break if found: - logger.info(f"Skipping manifest with digest {manifest_metadata["digest"]} - already exists") + logger.info( + f"Skipping manifest with digest {manifest_metadata["digest"]} - already exists" + ) continue index["manifests"].append(manifest_metadata) - logger.info(f"Index appended locally {manifest_metadata["annotations"]["cname"]}") - new_entries+=1 + logger.info( + f"Index appended locally {manifest_metadata["annotations"]["cname"]}" + ) + new_entries += 1 self._check_200_response(self.upload_index(index)) logger.info(f"Index pushed with {new_entries} new entries") diff --git a/tests/conftest.py b/tests/conftest.py index a6b2629..a8deabc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -29,6 +29,7 @@ def pytest_sessionstart(session): call_command("./tests/cert/gencert.sh") print("pytest session started") call_command("./tests/data/build-test-data.sh --dummy") + call_command("mkdir -p manifests") def pytest_sessionfinish(session): @@ -37,3 +38,5 @@ def pytest_sessionfinish(session): os.remove("./tests/cert/oci-sign.crt") if os.path.isfile("./tests/cert/oci-sign.key"): os.remove("./tests/cert/oci-sign.key") + if os.path.isdir("./manifests"): + shutil.rmtree("./manifests") diff --git a/tests/test_e2e.py b/tests/test_e2e.py index e64652f..23ec650 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -41,6 +41,8 @@ def test_push_manifest_and_index(version, arch, cname): "True", "--cosign_file", "digest", + "--manifest_file", + "manifests/manifest.json", ], catch_exceptions=False, ) @@ -67,8 +69,8 @@ def test_push_manifest_and_index(version, arch, cname): version, "--insecure", "True", - "--manifest_file", - "manifest-entry.json", + "--manifest_folder", + "manifests", ], catch_exceptions=False, )