diff --git a/.github/workflows/basic.yml b/.github/workflows/basic.yml index b89a099..65c93cf 100644 --- a/.github/workflows/basic.yml +++ b/.github/workflows/basic.yml @@ -55,20 +55,16 @@ jobs: - uses: "dtolnay/rust-toolchain@stable" - - name: "scratch test" - run: | - docker pull ghcr.io/githedgehog/testn/n-vm:0.0.4 - docker run --privileged --rm busybox:latest sh -c "echo 512 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages && cat /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages" - cargo test --package=scratch - - - name: "build" + - name: "build and test" run: | nix-channel --add https://nixos.org/channels/nixos-unstable nixpkgs nix-channel --update nix build -f default.nix testn.container docker load < ./result - docker build --tag ghcr.io/githedgehog/testn/n-vm:0.0.0 . - # TODO: push to ghcr.io (this requires plumbing the version from git into the build) + docker build --tag ghcr.io/githedgehog/testn/n-vm:v0.0.6 . + docker run --privileged --rm busybox:latest sh -c "echo 1024 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages && cat /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages" + cargo test --package=scratch + docker push ghcr.io/githedgehog/testn/n-vm:v0.0.6 - name: "Setup tmate session for debug" if: ${{ failure() && github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} diff --git a/Cargo.lock b/Cargo.lock index d2d8505..084a3a5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -882,7 +882,7 @@ dependencies = [ [[package]] name = "n-it" -version = "0.0.5" +version = "0.0.6" dependencies = [ "capctl", "nix 0.30.1", @@ -895,7 +895,7 @@ dependencies = [ [[package]] name = "n-vm" -version = "0.0.5" +version = "0.0.6" dependencies = [ "bollard", "capctl", @@ -916,7 +916,7 @@ dependencies = [ [[package]] name = "n-vm-macros" -version = "0.0.5" +version = "0.0.6" dependencies = [ "proc-macro2", "quote", diff --git a/README.md b/README.md index f6af882..3ad2e11 100644 --- a/README.md +++ b/README.md @@ -22,10 +22,11 @@ 2. Allocate some 2MiB hugepages: - If you don't already have some hugepages available, you can allocate 512 of them with the following command: + If you don't already have some hugepages available, you can allocate 1024 of + them with the following command: ```bash - echo 512 | sudo tee /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages + sudo tee /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages <<< 1024 ``` This will last until you reboot (or explicitly deallocate them). @@ -33,7 +34,7 @@ 3. Pull the current docker image: ```bash - docker pull ghcr.io/githedgehog/testn/n-vm:0.0.5 + docker pull ghcr.io/githedgehog/testn/n-vm:v0.0.6 ``` 4. Change into the repo directory and run the scratch test @@ -43,4 +44,5 @@ ``` If all goes well you should see two tests pass and two tests fail. -The tests which fail exist to illustrate the type of output we get from a failed in-vm and non in-vm test. +The tests which fail exist to illustrate the type of output we get from a failed +in-vm and non in-vm test. diff --git a/default.nix b/default.nix index 2dc027b..d08446b 100644 --- a/default.nix +++ b/default.nix @@ -51,10 +51,10 @@ let in rec { linux = pkgs.linuxManualConfig rec { - version = "6.12.47"; + version = "6.12.49"; src = fetchTarball { url = "https://cdn.kernel.org/pub/linux/kernel/v${pkgs.lib.versions.major version}.x/linux-${version}.tar.xz"; - sha256 = "sha256:0cb2hqrz1rvqsvr0s7q61525ig57l8hzgaajjcyhg3x0fqsy4avm"; + sha256 = "sha256:0nxbwcyb1shfw9s833agk32zh133xzqxpw7j4fzdskzl1x65jaws"; }; configfile = ./linux/kernel.config; inherit (pkgs.llvmPackages_21) stdenv; diff --git a/linux/kernel.config b/linux/kernel.config index f94e104..8b898ff 100644 --- a/linux/kernel.config +++ b/linux/kernel.config @@ -1,16 +1,16 @@ # # Automatically generated file; DO NOT EDIT. -# Linux/x86 6.12.47 Kernel Configuration +# Linux/x86 6.12.49 Kernel Configuration # -CONFIG_CC_VERSION_TEXT="clang version 21.1.0" +CONFIG_CC_VERSION_TEXT="clang version 21.1.1" CONFIG_GCC_VERSION=0 CONFIG_CC_IS_CLANG=y -CONFIG_CLANG_VERSION=210100 +CONFIG_CLANG_VERSION=210101 CONFIG_AS_IS_LLVM=y -CONFIG_AS_VERSION=210100 +CONFIG_AS_VERSION=210101 CONFIG_LD_VERSION=0 CONFIG_LD_IS_LLD=y -CONFIG_LLD_VERSION=210100 +CONFIG_LLD_VERSION=210101 CONFIG_RUSTC_VERSION=108900 CONFIG_RUSTC_LLVM_VERSION=200107 CONFIG_CC_CAN_LINK=y diff --git a/n-it/Cargo.toml b/n-it/Cargo.toml index 6cfaeb7..73f462f 100644 --- a/n-it/Cargo.toml +++ b/n-it/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "n-it" -version = "0.0.5" +version = "0.0.6" edition = "2024" license = "Apache-2.0" publish = false diff --git a/n-vm-macros/Cargo.toml b/n-vm-macros/Cargo.toml index 3e23cf6..aea369d 100644 --- a/n-vm-macros/Cargo.toml +++ b/n-vm-macros/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "n-vm-macros" -version = "0.0.5" +version = "0.0.6" edition = "2024" license = "Apache-2.0" publish = false diff --git a/n-vm/Cargo.toml b/n-vm/Cargo.toml index 690718e..8ec20d7 100644 --- a/n-vm/Cargo.toml +++ b/n-vm/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "n-vm" -version = "0.0.5" +version = "0.0.6" edition = "2024" license = "Apache-2.0" publish = false diff --git a/n-vm/src/lib.rs b/n-vm/src/lib.rs index 815aceb..5adce86 100644 --- a/n-vm/src/lib.rs +++ b/n-vm/src/lib.rs @@ -9,7 +9,6 @@ use bollard::secret::{ ContainerCreateBody, ContainerState, DeviceMapping, HostConfig, MountBindOptions, RestartPolicy, RestartPolicyNameEnum, }; -use cloud_hypervisor_client::SocketBasedApiClient; use cloud_hypervisor_client::apis::DefaultApi; use cloud_hypervisor_client::models::console_config::Mode; use cloud_hypervisor_client::models::{ @@ -23,7 +22,7 @@ use tokio_stream::StreamExt; use command_fds::{CommandFdExt, FdMapping}; use serde_json::StreamDeserializer; use tokio_util::bytes::{Buf, BytesMut}; -use tracing::error; +use tracing::{debug, error}; pub use n_vm_macros::in_vm; @@ -161,13 +160,28 @@ pub async fn run_in_vm(_: F) -> VmTestOutput { let virtiofsd = launch_virtiofsd("/vm.root").await; let listen = tokio::net::UnixListener::bind("/vm/vhost.vsock_123456").unwrap(); let init_system_trace = tokio::spawn(async move { + const CAPACITY_GUESS: usize = 32_768; + let mut init_system_trace = Vec::with_capacity(CAPACITY_GUESS); let (mut connection, _) = listen.accept().await.unwrap(); - let mut init_system_trace = String::with_capacity(32_768); - connection - .read_to_string(&mut init_system_trace) - .await - .unwrap(); - init_system_trace + loop { + tokio::select! { + res = connection.read_buf(&mut init_system_trace) => { + match res { + Ok(bytes) => { + if bytes == 0 { + break; + } + tokio::task::yield_now().await; + }, + Err(e) => { + error!("{e}"); + break; + }, + } + } + }; + } + String::from_utf8_lossy(&init_system_trace).to_string() }); let (_, test_name) = test_name.split_once("::").unwrap(); let config = VmConfig { @@ -175,7 +189,7 @@ pub async fn run_in_vm(_: F) -> VmTestOutput { firmware: None, kernel: Some("/bzImage".into()), cmdline: Some(format!( - "earlyprintk=ttyS0 console=ttyS0 ro rootfstype=virtiofs root=root default_hugepagesz=2M hugepagesz=2M hugepages=32 init=/bin/n-it {full_bin_name} {test_name} --exact --no-capture --format=terse" + "earlyprintk=ttyS0 console=ttyS0 ro rootfstype=virtiofs root=root default_hugepagesz=2M hugepagesz=2M hugepages=16 init=/bin/n-it {full_bin_name} {test_name} --exact --no-capture --format=terse" )), ..Default::default() }, @@ -197,7 +211,7 @@ pub async fn run_in_vm(_: F) -> VmTestOutput { ..Default::default() }), memory: Some(MemoryConfig { - size: 256 * 1024 * 1024, // 256MiB + size: 512 * 1024 * 1024, // 512MiB mergeable: Some(true), shared: Some(true), hugepages: Some(true), @@ -323,9 +337,24 @@ pub async fn run_in_vm(_: F) -> VmTestOutput { let client = Arc::new(tokio::sync::Mutex::new( cloud_hypervisor_client::socket_based_api_client(vmm_socket_path), )); - let hypervisor_event_logs_success = - tokio::spawn(watch_hypervisor(event_receiver, client.clone())); + let mut loops = 0; + loop { + match tokio::fs::try_exists(vmm_socket_path).await { + Ok(true) => break, + Ok(false) => { + loops += 1; + if loops > 100 { + panic!("failed to communicate with hypervisor: no api socket found"); + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + Err(err) => { + panic!("unable to communicate with hypervisor: {err}"); + } + } + } client.lock().await.create_vm(config).await.unwrap(); + let hypervisor_watch = tokio::spawn(watch_hypervisor(event_receiver)); let kernel_log = tokio::task::spawn(async move { let mut loops = 0; while loops < 100 { @@ -348,11 +377,31 @@ pub async fn run_in_vm(_: F) -> VmTestOutput { kernel_log }); client.lock().await.boot_vm().await.unwrap(); - let (hypervisor_events, hypervisor_verdict) = hypervisor_event_logs_success.await.unwrap(); + let init_trace = match init_system_trace.await { + Ok(log) => log, + Err(err) => { + format!("unable to join init system task: {err}") + } + }; + let (hypervisor_events, hypervisor_verdict) = hypervisor_watch.await.unwrap(); let hypervisor_output = process.wait_with_output().await.unwrap(); let kernel_log = kernel_log .await .unwrap_or_else(|err| format!("!!!KERNEL LOG MISSING!!!:\n\n{err:#?}\n\n")); + + match client.lock().await.shutdown_vm().await { + Ok(()) => {} + Err(err) => { + debug!("vm shutdown: {err}"); + } + }; + match client.lock().await.shutdown_vmm().await { + Ok(()) => {} + Err(err) => { + debug!("vmm shutdown: {err}"); + } + } + let virtiofsd = virtiofsd.wait_with_output().await.unwrap(); VmTestOutput { success: virtiofsd.status.success() @@ -361,7 +410,7 @@ pub async fn run_in_vm(_: F) -> VmTestOutput { stdout: String::from_utf8_lossy(&hypervisor_output.stdout).to_string(), stderr: String::from_utf8_lossy(&hypervisor_output.stderr).to_string(), console: kernel_log.clone(), - init_trace: init_system_trace.await.unwrap(), + init_trace, virtiofsd_stdout: String::from_utf8_lossy(virtiofsd.stdout.as_slice()).to_string(), virtiofsd_stderr: String::from_utf8_lossy(virtiofsd.stderr.as_slice()).to_string(), hypervisor_events: hypervisor_events, @@ -421,7 +470,6 @@ where async fn watch_hypervisor( receiver: tokio::net::unix::pipe::Receiver, - client: Arc>, ) -> (Vec, bool) { let decoder = AsyncJsonStreamDecoder::new(); @@ -447,16 +495,13 @@ async fn watch_hypervisor( }; } Some(Err(e)) => { - success = false; error!("{e:#?}"); + } + None => { break; } - None => {} } } - let client = client.lock().await; - client.shutdown_vm().await.unwrap(); - client.shutdown_vmm().await.unwrap(); return (hlog, success); } @@ -513,7 +558,7 @@ pub fn run_test_in_vm(_test_fn: F) -> ContainerState { entrypoint: None, cmd: Some(args), // TODO: this needs to be dynamic somehow. Not sure how to do that yet. - image: Some("ghcr.io/githedgehog/testn/n-vm:0.0.5".into()), + image: Some("ghcr.io/githedgehog/testn/n-vm:v0.0.6".into()), network_disabled: Some(true), env: Some([ "IN_TEST_CONTAINER=YES".into(), diff --git a/shell.nix b/shell.nix index 6dcbb06..d051ec0 100644 --- a/shell.nix +++ b/shell.nix @@ -1,6 +1,20 @@ { pkgs ? import { }, }: -pkgs.mkShell { - nativeBuildInputs = with pkgs; [ llvmPackages.clang ]; -} +(pkgs.buildFHSEnv { + name = "testn-shell"; + targetPkgs = + pkgs: + (with pkgs; [ + # for nix + nil + nix-prefetch-git + nixd + + # for dev + bash + docker-client + rustup + ]); + runScript = ''bash''; +}).env