Skip to content

Only update index once #36

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
eigenpaul merged 2 commits into from
Dec 5, 2024
Merged

Only update index once #36

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
85cc8fe
only update index once
eigenpaul Dec 5, 2024
87bf2d8
adjust e2e tests
eigenpaul Dec 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions src/glcli.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ def cli():
)
@click.option(
"--manifest_file",
default="manifest-entry.json",
default="manifests/manifest.json",
help="A file where the index entry for the pushed manifest is written to.",
)
@click.option(
Expand Down Expand Up @@ -82,24 +82,24 @@ def push_manifest(
help="Version of image",
)
@click.option(
"--manifest_file",
default="manifest-entry.json",
help="A file where the index entry is read from.",
"--manifest_folder",
default="manifests",
help="A folder where the index entries are read from.",
)
@click.option(
"--insecure",
default=False,
help="Use HTTP to communicate with the registry",
)
def update_index(container, version, manifest_file, insecure):
"""push a index entry from a file to an index"""
def update_index(container, version, manifest_folder, insecure):
"""push a index entry from a list of files to an index"""
container_name = f"{container}:{version}"
registry = GlociRegistry(
container_name=container_name,
token=os.getenv("GLOCI_REGISTRY_TOKEN"),
insecure=insecure,
)
registry.update_index_entries(manifest_file, version)
registry.update_index(manifest_folder)


if __name__ == "__main__":
Expand Down
106 changes: 27 additions & 79 deletions src/registry.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ def get_index(self, allowed_media_type: Optional[list[str]] = None):
return index

except ValueError:
logger.debug("Index not found, creating new Index!")
logger.info("Index not found, creating new Index!")
return NewIndex()

@ensure_container
Expand Down Expand Up @@ -290,28 +290,6 @@ def get_manifest_by_cname(
container, manifest_digest, allowed_media_type=allowed_media_type
)

def update_index(self, old_digest: Optional[str], manifest_meta_data: dict):
"""
replaces an old manifest entry with a new manifest entry
"""
index = self.get_index()

if "manifests" not in index:
logger.debug("Index is empty")
updated = False
if old_digest is not None:
for i, manifest in enumerate(index["manifests"]):
if manifest["digest"] == old_digest:
logger.debug("Found old manifest entry")
index["manifests"][i] = manifest_meta_data
updated = True
break
if not updated:
logger.debug("Did NOT find old manifest entry")
index["manifests"].append(manifest_meta_data)

return index

def change_state(self, cname: str, version: str, architecture: str, new_state: str):
manifest_container = OrasContainer(
f"{self.container_name}-{cname}-{architecture}"
Expand All @@ -324,45 +302,6 @@ def change_state(self, cname: str, version: str, architecture: str, new_state: s
logger.warning("No annotations found in manifest, init annotations now.")
manifest["annotations"] = {}

def attach_layer(
self,
cname: str,
version: str,
architecture: str,
file_path: str,
media_type: str,
):
if not os.path.exists(file_path):
exit(f"{file_path} does not exist.")

manifest_container = OrasContainer(
f"{self.container_name}-{cname}-{architecture}"
)

manifest = self.get_manifest_by_cname(
self.container, cname, version, architecture
)

layer = self.create_layer(file_path, cname, version, architecture, media_type)
self._check_200_response(self.upload_blob(file_path, self.container, layer))

manifest["layers"].append(layer)

old_manifest_digest = self.get_digest(manifest_container)
self._check_200_response(self.upload_manifest(manifest, manifest_container))

new_manifest_metadata = self.get_manifest_meta_data_by_cname(
self.container, cname, version, architecture
)
new_manifest_metadata["digest"] = self.get_digest(manifest_container)
new_manifest_metadata["size"] = self.get_manifest_size(manifest_container)
new_manifest_metadata["platform"] = NewPlatform(architecture, version)

new_index = self.update_index(old_manifest_digest, new_manifest_metadata)
self._check_200_response(self.upload_index(new_index))

print(f"Successfully attached {file_path} to {manifest_container}")

@ensure_container
def remove_container(self, container: OrasContainer):
self.delete_tag(container.manifest_url())
Expand Down Expand Up @@ -516,26 +455,35 @@ def push_image_manifest(
print(json.dumps(manifest_index_metadata), file=open(manifest_file, "w"))
logger.info(f"Index entry written to {manifest_file}")

# self.update_index_entries(architecture, cname, manifest_index_metadata, version)

return local_digest

def update_index_entries(self, manifest_file, version):
manifest_index_metadata = json.loads(open(manifest_file, "r").read())
arch = manifest_index_metadata["annotations"]["architecture"]
cname = manifest_index_metadata["annotations"]["cname"]
old_manifest_meta_data = self.get_manifest_meta_data_by_cname(
self.container, cname, version, arch
)
if old_manifest_meta_data is not None:
new_index = self.update_index(
old_manifest_meta_data["digest"], manifest_index_metadata
def update_index(self, manifest_folder):
index = self.get_index()
new_entries = 0

for file in os.listdir(manifest_folder):
manifest_metadata = json.loads(
open(manifest_folder + "/" + file, "r").read()
)
logger.info("Replaced manifest entry")
else:
new_index = self.update_index(None, manifest_index_metadata)
logger.info("Appended manifest entry")
self._check_200_response(self.upload_index(new_index))
# Skip if manifest with same digest already exists
found = False
for entry in index["manifests"]:
if entry["digest"] == manifest_metadata["digest"]:
found = True
break
if found:
logger.info(
f"Skipping manifest with digest {manifest_metadata["digest"]} - already exists"
)
continue
index["manifests"].append(manifest_metadata)
logger.info(
f"Index appended locally {manifest_metadata["annotations"]["cname"]}"
)
new_entries += 1

self._check_200_response(self.upload_index(index))
logger.info(f"Index pushed with {new_entries} new entries")

def create_layer(
self,
Expand Down
3 changes: 3 additions & 0 deletions tests/conftest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ def pytest_sessionstart(session):
call_command("./tests/cert/gencert.sh")
print("pytest session started")
call_command("./tests/data/build-test-data.sh --dummy")
call_command("mkdir -p manifests")


def pytest_sessionfinish(session):
Expand All @@ -37,3 +38,5 @@ def pytest_sessionfinish(session):
os.remove("./tests/cert/oci-sign.crt")
if os.path.isfile("./tests/cert/oci-sign.key"):
os.remove("./tests/cert/oci-sign.key")
if os.path.isdir("./manifests"):
shutil.rmtree("./manifests")
6 changes: 4 additions & 2 deletions tests/test_e2e.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ def test_push_manifest_and_index(version, arch, cname):
"True",
"--cosign_file",
"digest",
"--manifest_file",
"manifests/manifest.json",
],
catch_exceptions=False,
)
Expand All @@ -67,8 +69,8 @@ def test_push_manifest_and_index(version, arch, cname):
version,
"--insecure",
"True",
"--manifest_file",
"manifest-entry.json",
"--manifest_folder",
"manifests",
],
catch_exceptions=False,
)
Expand Down
Loading