Skip to content

add protobuf parsing for binary #48

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Sep 19, 2025
Merged

add protobuf parsing for binary #48

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
b44b716
split versions and packages api endpoint processiong functions
benev0 Sep 11, 2025
7086e5a
update changelog
benev0 Sep 12, 2025
0973ee6
add doc comments
benev0 Sep 13, 2025
67faf12
add tests for get_repository_versions_data & get_package_data
benev0 Sep 15, 2025
edf6e0c
rename functions
benev0 Sep 15, 2025
d472444
update of regestry v2 request builder function docs
benev0 Sep 18, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
# Changelog

## v4.3.0

- Added `get_repository_versions_data`, `get_package_data`,

## v4.2.0 - 2025-08-27

- `Version`'s serde deserializer can now work with `String` as well as `str`.
Expand Down
27 changes: 23 additions & 4 deletions src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -256,9 +256,8 @@ pub fn get_repository_versions_request(
.expect("get_repository_versions_request request")
}

/// Parse a request that get the names and versions of all of the packages on
/// Parse a request that gets the names and versions of all of the packages on
/// the package registry.
///
pub fn get_repository_versions_response(
response: http::Response<Vec<u8>>,
public_key: &[u8],
Expand All @@ -274,7 +273,15 @@ pub fn get_repository_versions_response(
let mut body = Vec::new();
decoder.read_to_end(&mut body)?;

let signed = Signed::decode(body.as_slice())?;
get_repository_versions_data(&body, public_key)
}

/// Parse a signed binary message containing all of the packages on the package registry.
pub fn get_repository_versions_data(
protobuf_bytes: &Vec<u8>,
public_key: &[u8],
) -> Result<HashMap<String, Vec<Version>>, ApiError> {
let signed = Signed::decode(protobuf_bytes.as_slice())?;

let payload =
verify_payload(signed, public_key).map_err(|_| ApiError::IncorrectPayloadSignature)?;
Expand Down Expand Up @@ -339,7 +346,15 @@ pub fn get_package_response(
let mut body = Vec::new();
decoder.read_to_end(&mut body)?;

let signed = Signed::decode(body.as_slice())?;
get_package_data(&body, public_key)
}

/// Parse a signed binary message containing the information for a package in the repository.
pub fn get_package_data(
protobuf_bytes: &Vec<u8>,
public_key: &[u8],
) -> Result<Package, ApiError> {
let signed = Signed::decode(protobuf_bytes.as_slice())?;

let payload =
verify_payload(signed, public_key).map_err(|_| ApiError::IncorrectPayloadSignature)?;
Expand Down Expand Up @@ -739,6 +754,10 @@ impl ApiError {
pub fn is_not_found(&self) -> bool {
matches!(self, Self::NotFound)
}

pub fn is_invalid_protobuf(&self) -> bool {
matches!(self, Self::InvalidProtobuf(_))
}
}

/// Read a body and ensure it has the given sha256 digest.
Expand Down
295 changes: 187 additions & 108 deletions src/tests.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
// TODO: remove all the async stuff and mockito server. The library is pure now
// so it isn't needed.

use std::convert::TryFrom;
use std::{convert::TryFrom, io::Cursor};

use super::*;
use mockito::Matcher;
Expand Down Expand Up @@ -801,6 +801,116 @@ async fn publish_docs_forbidden() {
mock.assert();
}

fn expected_package_exfmt() -> Package {
Package {
name: "exfmt".to_string(),
repository: "hexpm".to_string(),
releases: vec![
Release {
version: Version::try_from("0.0.0").unwrap(),
requirements: [].into(),
retirement_status: None,
outer_checksum: vec![
82, 48, 191, 145, 92, 172, 0, 108, 238, 71, 57, 23, 101, 177, 161, 83, 91,
182, 18, 232, 249, 225, 29, 12, 246, 5, 215, 165, 32, 57, 179, 110
],
meta: (),
},
Release {
version: Version::try_from("0.1.0").unwrap(),
requirements: [].into(),
retirement_status: None,
outer_checksum: vec![
111, 246, 240, 176, 118, 229, 12, 15, 164, 61, 186, 3, 89, 106, 153, 225,
247, 52, 245, 8, 216, 139, 21, 232, 200, 16, 214, 59, 241, 188, 9, 6
],
meta: (),
},
Release {
version: Version::try_from("0.2.0").unwrap(),
requirements: [].into(),
retirement_status: None,
outer_checksum: vec![
149, 9, 192, 229, 84, 162, 110, 207, 161, 43, 31, 0, 126, 168, 14, 243, 31,
43, 195, 238, 100, 91, 78, 100, 213, 181, 101, 154, 106, 168, 170, 107
],
meta: (),
},
Release {
version: Version::try_from("0.2.1").unwrap(),
requirements: [].into(),
retirement_status: None,
outer_checksum: vec![
157, 229, 28, 212, 92, 249, 14, 240, 235, 104, 31, 12, 160, 199, 83, 195,
154, 105, 222, 37, 221, 80, 181, 183, 113, 240, 234, 107, 144, 85, 255, 65
],
meta: (),
},
Release {
version: Version::try_from("0.2.2").unwrap(),
requirements: [].into(),
retirement_status: None,
outer_checksum: vec![
112, 250, 133, 189, 183, 192, 54, 218, 115, 55, 216, 97, 204, 201, 191,
168, 250, 133, 138, 252, 202, 240, 74, 197, 228, 235, 81, 18, 241, 7, 155,
38
],
meta: (),
},
Release {
version: Version::try_from("0.2.3").unwrap(),
requirements: [].into(),
retirement_status: None,
outer_checksum: vec![
131, 20, 29, 160, 171, 124, 7, 125, 210, 88, 17, 189, 199, 49, 191, 190,
14, 162, 38, 247, 52, 176, 189, 17, 7, 188, 151, 152, 24, 64, 170, 29
],
meta: (),
},
Release {
version: Version::try_from("0.2.4").unwrap(),
requirements: [].into(),
retirement_status: None,
outer_checksum: vec![
109, 162, 185, 169, 26, 4, 62, 60, 167, 54, 182, 161, 140, 197, 75, 113,
183, 117, 247, 201, 218, 228, 14, 160, 115, 157, 196, 51, 108, 16, 96, 217
],
meta: (),
},
Release {
version: Version::try_from("0.3.0").unwrap(),
requirements: [].into(),
retirement_status: None,
outer_checksum: vec![
97, 50, 95, 212, 242, 59, 245, 177, 140, 78, 79, 180, 108, 174, 119, 176,
24, 80, 218, 152, 178, 227, 152, 242, 32, 126, 72, 67, 222, 0, 173, 170
],
meta: (),
},
Release {
version: Version::try_from("0.4.0").unwrap(),
requirements: [].into(),
retirement_status: None,
outer_checksum: vec![
246, 178, 237, 214, 217, 158, 143, 52, 130, 186, 64, 50, 94, 175, 161, 81,
68, 186, 4, 73, 53, 226, 235, 144, 209, 84, 231, 136, 165, 119, 122, 126
],
meta: (),
},
Release {
version: Version::try_from("0.5.0").unwrap(),
requirements: [].into(),
retirement_status: None,
outer_checksum: vec![
151, 86, 157, 218, 218, 131, 240, 119, 198, 216, 202, 240, 65, 17, 57, 228,
84, 252, 59, 207, 246, 49, 22, 21, 52, 47, 51, 139, 190, 9, 95, 109
],
meta: (),
}
],
}
}

#[tokio::test]
async fn get_package_ok_test() {
let response_body = std::include_bytes!("../test/package_exfmt");
Expand Down Expand Up @@ -828,113 +938,7 @@ async fn get_package_ok_test() {
.unwrap();

assert_eq!(
Package {
name: "exfmt".to_string(),
repository: "hexpm".to_string(),
releases: vec![
Release {
version: Version::try_from("0.0.0").unwrap(),
requirements: [].into(),
retirement_status: None,
outer_checksum: vec![
82, 48, 191, 145, 92, 172, 0, 108, 238, 71, 57, 23, 101, 177, 161, 83, 91,
182, 18, 232, 249, 225, 29, 12, 246, 5, 215, 165, 32, 57, 179, 110
],
meta: (),
},
Release {
version: Version::try_from("0.1.0").unwrap(),
requirements: [].into(),
retirement_status: None,
outer_checksum: vec![
111, 246, 240, 176, 118, 229, 12, 15, 164, 61, 186, 3, 89, 106, 153, 225,
247, 52, 245, 8, 216, 139, 21, 232, 200, 16, 214, 59, 241, 188, 9, 6
],
meta: (),
},
Release {
version: Version::try_from("0.2.0").unwrap(),
requirements: [].into(),
retirement_status: None,
outer_checksum: vec![
149, 9, 192, 229, 84, 162, 110, 207, 161, 43, 31, 0, 126, 168, 14, 243, 31,
43, 195, 238, 100, 91, 78, 100, 213, 181, 101, 154, 106, 168, 170, 107
],
meta: (),
},
Release {
version: Version::try_from("0.2.1").unwrap(),
requirements: [].into(),
retirement_status: None,
outer_checksum: vec![
157, 229, 28, 212, 92, 249, 14, 240, 235, 104, 31, 12, 160, 199, 83, 195,
154, 105, 222, 37, 221, 80, 181, 183, 113, 240, 234, 107, 144, 85, 255, 65
],
meta: (),
},
Release {
version: Version::try_from("0.2.2").unwrap(),
requirements: [].into(),
retirement_status: None,
outer_checksum: vec![
112, 250, 133, 189, 183, 192, 54, 218, 115, 55, 216, 97, 204, 201, 191,
168, 250, 133, 138, 252, 202, 240, 74, 197, 228, 235, 81, 18, 241, 7, 155,
38
],
meta: (),
},
Release {
version: Version::try_from("0.2.3").unwrap(),
requirements: [].into(),
retirement_status: None,
outer_checksum: vec![
131, 20, 29, 160, 171, 124, 7, 125, 210, 88, 17, 189, 199, 49, 191, 190,
14, 162, 38, 247, 52, 176, 189, 17, 7, 188, 151, 152, 24, 64, 170, 29
],
meta: (),
},
Release {
version: Version::try_from("0.2.4").unwrap(),
requirements: [].into(),
retirement_status: None,
outer_checksum: vec![
109, 162, 185, 169, 26, 4, 62, 60, 167, 54, 182, 161, 140, 197, 75, 113,
183, 117, 247, 201, 218, 228, 14, 160, 115, 157, 196, 51, 108, 16, 96, 217
],
meta: (),
},
Release {
version: Version::try_from("0.3.0").unwrap(),
requirements: [].into(),
retirement_status: None,
outer_checksum: vec![
97, 50, 95, 212, 242, 59, 245, 177, 140, 78, 79, 180, 108, 174, 119, 176,
24, 80, 218, 152, 178, 227, 152, 242, 32, 126, 72, 67, 222, 0, 173, 170
],
meta: (),
},
Release {
version: Version::try_from("0.4.0").unwrap(),
requirements: [].into(),
retirement_status: None,
outer_checksum: vec![
246, 178, 237, 214, 217, 158, 143, 52, 130, 186, 64, 50, 94, 175, 161, 81,
68, 186, 4, 73, 53, 226, 235, 144, 209, 84, 231, 136, 165, 119, 122, 126
],
meta: (),
},
Release {
version: Version::try_from("0.5.0").unwrap(),
requirements: [].into(),
retirement_status: None,
outer_checksum: vec![
151, 86, 157, 218, 218, 131, 240, 119, 198, 216, 202, 240, 65, 17, 57, 228,
84, 252, 59, 207, 246, 49, 22, 21, 52, 47, 51, 139, 190, 9, 95, 109
],
meta: (),
}
],
},
expected_package_exfmt(),
package,
);

Expand All @@ -959,6 +963,38 @@ async fn get_package_not_found() {
assert!(error.is_not_found());
}

#[tokio::test]
async fn get_package_from_bytes_ok() {
let response_body = std::include_bytes!("../test/package_exfmt");
let mut uncompressed = Vec::new();
let mut decoder = GzDecoder::new(Cursor::new(response_body));
let _ = decoder.read_to_end(&mut uncompressed).expect("failed to decompress body");

let package = crate::get_package_data(
&uncompressed,
std::include_bytes!("../test/public_key")
)
.expect("package failed to parse");

assert_eq!(
expected_package_exfmt(),
package,
);
}

#[tokio::test]
async fn get_package_from_bytes_malformed() {
// public key should not be a valid protobuf and should therefore fail
let bytes = std::include_bytes!("../test/public_key").to_vec();
let package_error = crate::get_package_data(
&bytes,
&bytes,
)
.expect_err("parsing failed to fail");

assert!(package_error.is_invalid_protobuf());
}

#[tokio::test]
async fn get_repository_versions_ok_test() {
let response_body = std::include_bytes!("../test/versions");
Expand Down Expand Up @@ -1003,6 +1039,49 @@ async fn get_repository_versions_ok_test() {
mock.assert();
}

#[tokio::test]
async fn get_repository_versions_from_bytes_ok() {
let response_body = std::include_bytes!("../test/versions");
let mut uncompressed = Vec::new();
let mut decoder = GzDecoder::new(Cursor::new(response_body));
let _ = decoder.read_to_end(&mut uncompressed).expect("failed to decompress body");

let versions = crate::get_repository_versions_data(
&uncompressed,
std::include_bytes!("../test/public_key"),
)
.expect("versions failed to parse");

assert_eq!(
&vec![
Version::parse("0.0.0").unwrap(),
Version::parse("0.1.0").unwrap(),
Version::parse("0.2.0").unwrap(),
Version::parse("0.2.1").unwrap(),
Version::parse("0.2.2").unwrap(),
Version::parse("0.2.3").unwrap(),
Version::parse("0.2.4").unwrap(),
Version::parse("0.3.0").unwrap(),
Version::parse("0.4.0").unwrap(),
Version::parse("0.5.0").unwrap(),
],
versions.get("exfmt").unwrap(),
);
}

#[tokio::test]
async fn get_repository_versions_from_bytes_malformed() {
// public key should not be a valid protobuf and should therefore fail
let bytes = std::include_bytes!("../test/public_key").to_vec();
let versions_error = crate::get_repository_versions_data(
&bytes,
&bytes,
)
.expect_err("parsing failed to fail");

assert!(versions_error.is_invalid_protobuf());
}

#[tokio::test]
async fn get_repository_tarball_ok_test() {
let config = Config::new();
Expand Down