diff --git a/MODULE.bazel b/MODULE.bazel index bcc595972..f0e5af273 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -165,6 +165,12 @@ use_repo( ) bazel_dep(name = "score_itf", version = "0.5.0", dev_dependency = True) +git_override( + module_name = "score_itf", + commit = "54cab4f6ebc1fb40d0b93a8c1a451a03bda77b6a", + remote = "https://github.com/eclipse-score/itf.git", +) + bazel_dep(name = "score_rules_imagefs", version = "0.0.3", dev_dependency = True) imagefs = use_extension("@score_rules_imagefs//extensions:imagefs.bzl", "imagefs", dev_dependency = True) diff --git a/quality/integration_testing/environments/dual_qemu/BUILD b/quality/integration_testing/environments/dual_qemu/BUILD new file mode 100644 index 000000000..2fd5b772f --- /dev/null +++ b/quality/integration_testing/environments/dual_qemu/BUILD @@ -0,0 +1,42 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +load("@rules_python//python:defs.bzl", "py_library") +load("@score_itf//bazel:py_itf_plugin.bzl", "py_itf_plugin") + +exports_files(["dual_qemu_config.example.json"]) + +py_library( + name = "dual_qemu_lib", + srcs = [ + "__init__.py", + "config.py", + "dual_qemu_process.py", + "ivshmem_qemu.py", + ], + imports = [".."], + deps = [ + "@score_itf//score/itf/core/process", + "@score_itf//score/itf/core/utils", + "@score_itf//score/itf/plugins/qemu", + ], +) + +# ---- ITF plugin target (used by the py_itf_test symbolic macro) ---- +py_itf_plugin( + name = "dual_qemu_plugin", + enabled_plugins = [ + "dual_qemu", + ], + py_library = ":dual_qemu_lib", + visibility = ["//:__subpackages__"], +) diff --git a/quality/integration_testing/environments/dual_qemu/README.md b/quality/integration_testing/environments/dual_qemu/README.md new file mode 100644 index 000000000..d966ef119 --- /dev/null +++ b/quality/integration_testing/environments/dual_qemu/README.md @@ -0,0 +1,97 @@ +# `dual_qemu` ITF plugin + +A pytest/ITF plugin that boots **two** QNX QEMU VMs which share a single QEMU +[`ivshmem`](https://www.qemu.org/docs/master/system/devices/ivshmem.html) region. It is a +thin extension of the upstream single-VM `qemu` plugin +(`@score_itf//score/itf/plugins/qemu`). + +## What it adds over the upstream `qemu` plugin + +| Aspect | upstream `qemu` | `dual_qemu` | +| ----------------- | ---------------------- | ----------------------------------------------------- | +| Number of VMs | 1 (`target_init`) | 2 (`target_a`, `target_b`) | +| Shared memory | none | one `ivshmem-plain` region backed by one host file | +| SSH ports | one | distinct host port per VM (e.g. `2222` / `2223`) | +| Inter-VM NIC | n/a | optional point-to-point socket NIC (off by default) | + +The shared region uses **`ivshmem-plain`** — a passive shared-memory PCI device with +**no MSI-X and no doorbell**. Cross-VM notifications travel over a separate +socket-based control plane, so the data plane only needs plain shared memory. + +## Fixtures + +| Fixture | Scope | Description | +| ------------------ | ------- | ------------------------------------------------------- | +| `target_a` | session | First VM (`QemuTarget`). | +| `target_b` | session | Second VM (`QemuTarget`). | +| `ivshmem_backend` | session | Path of the shared host backing file. | +| `config` | session | Loaded `DualQemuConfigModel` + `qemu_image`. | + +Both VMs run the upstream `pre_tests_phase` checks (ping / SSH / SFTP) before the tests +start. + +### Boot reliability + +Booting two QNX guests **concurrently** under KVM occasionally wedges the second guest +during device bring-up (it can hang at SMP AP / secondary-CPU startup under the `-kernel` +boot path, so its `sshd` never comes up and QEMU's SLIRP resets the harness connections). +To keep the test reliable the plugin: + +- boots the VMs **sequentially** — it starts a VM, waits until SSH is *stably* reachable + and runs `pre_tests_phase`, and only then starts the next one, so the two guests never + initialise their devices at the same time; +- gives each VM a single core and a **distinct NIC MAC**; +- the `dual_qemu_integration_test` macro additionally marks the test `flaky = True` so + bazel transparently retries the rare residual KVM boot hiccup. + +## CLI options + +| Option | Required | Description | +| -------------------- | -------- | -------------------------------------------------- | +| `--dual-qemu-config` | yes | JSON config (see `dual_qemu_config.example.json`). | +| `--qemu-image` | yes | Shared QEMU IFS image booted by both VMs. | + +## Configuration + +See [`dual_qemu_config.example.json`](dual_qemu_config.example.json). A config is just +**two ordinary QEMU configs** (each an upstream `QemuConfigModel`) plus an `ivshmem` +block: + +```json +{ + "ivshmem": { "size": "4M" }, + "intervm_network": { "enabled": false, "host_port": 12345 }, + "vms": [ { "...": "QemuConfigModel for VM-A" }, + { "...": "QemuConfigModel for VM-B" } ] +} +``` + +- `ivshmem.size` — size of the shared region (e.g. `4M`). +- `ivshmem.mem_path` (optional) — explicit host backing file; when empty a temporary file + is created for the session and removed on teardown. +- `intervm_network.enabled` — when `true`, adds a socket NIC where VM-A listens and VM-B + connects on `host_port` (for the future socket control plane). +- Each VM's `ssh_port` and `port_forwarding` must use **distinct host ports**. + +## Cross-VM lib-memory example + +[`example_xvm/`](example_xvm/README.md) drives the `score::memory::shared` allocator (the +one LoLa uses) across both VMs over the ivshmem BAR. The one BAR is split into two +page-aligned lib-memory regions, so the exchange is **bidirectional**: the producer creates +a forward region holding a `Vector` (the consumer opens it), and the consumer creates a +separate reverse region holding a `Map` (the producer opens it) — each container's +`OffsetPtr`s resolve despite each VM mapping the BAR at a different base. See +[`example_xvm/README.md`](example_xvm/README.md) for the details, the test target, and how +to run it manually. + +## Debugging guest boot + +Set `DUAL_QEMU_SERIAL_DIR` to capture each VM's serial console to +`/vm_serial.log` (instead of the normal `mon:stdio`). For example: + +```bash +bazel test //path/to/your:dual_qemu_test \ + --config=qnx_x86_64 --strategy=TestRunner=local \ + --test_env=DUAL_QEMU_SERIAL_DIR=/tmp/dqdbg +# then inspect /tmp/dqdbg/vm0_serial.log and /tmp/dqdbg/vm1_serial.log +``` diff --git a/quality/integration_testing/environments/dual_qemu/__init__.py b/quality/integration_testing/environments/dual_qemu/__init__.py new file mode 100644 index 000000000..6bd5acad3 --- /dev/null +++ b/quality/integration_testing/environments/dual_qemu/__init__.py @@ -0,0 +1,139 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Pytest plugin that boots **two** QNX QEMU VMs sharing a QEMU ``ivshmem`` region. + +It is a thin extension of the single-VM ``qemu`` plugin +(``@score_itf//score/itf/plugins/qemu``): it reuses ``QemuTarget`` / ``pre_tests_phase`` +and only adds (a) a second VM, (b) an ``ivshmem-plain`` device backed by one shared host +file, and (c) distinct host SSH ports per VM. + +Exposed session fixtures: + - ``target_a`` / ``target_b`` -- the two booted VMs (``QemuTarget``). + - ``ivshmem_backend`` -- path of the shared host backing file. +""" +import logging +import socket + +import pytest + +from score.itf.core.utils.bunch import Bunch + +from .config import load_configuration, parse_size +from .dual_qemu_process import DualQemuProcess + +logger = logging.getLogger(__name__) + + +def pytest_addoption(parser): + parser.addoption( + "--dual-qemu-config", + action="store", + required=True, + help="Path to a JSON file describing the two VMs and the ivshmem region.", + ) + parser.addoption( + "--qemu-image-a", + action="store", + required=True, + help="Path to the QEMU IFS image for VM-A.", + ) + parser.addoption( + "--qemu-image-b", + action="store", + required=True, + help="Path to the QEMU IFS image for VM-B.", + ) + + +@pytest.fixture(scope="session") +def config(request): + return Bunch( + dual_config=load_configuration(request.config.getoption("dual_qemu_config")), + qemu_images=[ + request.config.getoption("qemu_image_a"), + request.config.getoption("qemu_image_b"), + ], + ) + + +@pytest.fixture(scope="session") +def ivshmem_backend(config, tmp_path_factory): + """Create the single host backing file both VMs map as their ivshmem region.""" + ivshmem = config.dual_config.ivshmem + size_bytes = parse_size(ivshmem.size) + + if ivshmem.mem_path: + path = ivshmem.mem_path + else: + path = str(tmp_path_factory.mktemp("ivshmem") / "score_ivshmem") + + logger.info(f"Creating ivshmem backing file {path} ({ivshmem.size})") + with open(path, "wb") as backing_file: + backing_file.truncate(size_bytes) + + yield path + + +@pytest.fixture(scope="session") +def _targets(config, ivshmem_backend): + """Boot both VMs sequentially, verify them, and tear down in reverse order.""" + logger.info(f"Starting dual-VM tests on host: {socket.gethostname()}") + dual_config = config.dual_config + + # When the inter-VM network is enabled, VM-A hosts the socket and VM-B connects. + intervm = dual_config.intervm_network + intervm_roles = [None, None] + if intervm.enabled: + intervm_roles = [("listen", intervm.host_port), ("connect", intervm.host_port)] + + vms = dual_config.vms + + # Boot VM-A first, then VM-B. Sequential booting avoids a KVM race where two QNX + # guests initializing concurrently can wedge the second guest's device bring-up. + with DualQemuProcess( + config.qemu_images[0], + vms[0].qemu_ram_size, + vms[0].qemu_num_cores, + vm_config=vms[0], + port_forwarding=vms[0].port_forwarding, + ivshmem_path=ivshmem_backend, + ivshmem_size=dual_config.ivshmem.size, + intervm=intervm_roles[0], + vm_index=0, + ) as process_a: + with DualQemuProcess( + config.qemu_images[1], + vms[1].qemu_ram_size, + vms[1].qemu_num_cores, + vm_config=vms[1], + port_forwarding=vms[1].port_forwarding, + ivshmem_path=ivshmem_backend, + ivshmem_size=dual_config.ivshmem.size, + intervm=intervm_roles[1], + vm_index=1, + ) as process_b: + # Re-verify VM-A is still responsive (it may have gone quiet while VM-B booted). + process_a.ensure_responsive() + yield [process_a.target, process_b.target] + + +@pytest.fixture(scope="session") +def target_a(_targets): + """The first VM (VM-A) in the inter-VM shared-memory tests.""" + return _targets[0] + + +@pytest.fixture(scope="session") +def target_b(_targets): + """The second VM (VM-B) in the inter-VM shared-memory tests.""" + return _targets[1] diff --git a/quality/integration_testing/environments/dual_qemu/config.py b/quality/integration_testing/environments/dual_qemu/config.py new file mode 100644 index 000000000..ec4235148 --- /dev/null +++ b/quality/integration_testing/environments/dual_qemu/config.py @@ -0,0 +1,69 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Configuration model for the dual-QEMU (ivshmem) plugin. + +Each VM is described by the ``QemuConfigModel``, so a dual config is just two +ordinary QEMU configs plus a shared ``ivshmem`` block. +""" +import json +import logging + +from pydantic import BaseModel, ConfigDict, Field, ValidationError + +from score.itf.plugins.qemu.config import QemuConfigModel + +logger = logging.getLogger(__name__) + +_SIZE_UNITS = {"K": 1024, "M": 1024**2, "G": 1024**3, "T": 1024**4, "P": 1024**5} + + +def parse_size(size: str) -> int: + """Convert a QEMU-style size string such as ``"4M"`` into a byte count.""" + return int(size[:-1]) * _SIZE_UNITS[size[-1]] + + +class IvshmemConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + + size: str = Field(default="4M", pattern=r"^[0-9]+[KMGTP]$") + # Optional explicit host backing file. When empty, a temporary file is created + # by the ``ivshmem_backend`` fixture and removed on session teardown. + mem_path: str = "" + + +class InterVmNetwork(BaseModel): + model_config = ConfigDict(extra="forbid") + + # Optional point-to-point socket NIC between the two VMs for a socket-based control + # plane. Off by default + enabled: bool = False + host_port: int = Field(default=12345, ge=1, le=65535) + + +class DualQemuConfigModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + ivshmem: IvshmemConfig = Field(default_factory=IvshmemConfig) + intervm_network: InterVmNetwork = Field(default_factory=InterVmNetwork) + vms: list[QemuConfigModel] = Field(min_length=2, max_length=2) + + +def load_configuration(config_file: str) -> DualQemuConfigModel: + logger.info(f"Loading dual-QEMU configuration from {config_file}") + with open(config_file, "r", encoding="utf-8") as handle: + raw = json.load(handle) + try: + return DualQemuConfigModel(**raw) + except ValidationError as error: + logger.error(f"Invalid dual-QEMU configuration: {error}") + raise diff --git a/quality/integration_testing/environments/dual_qemu/dual_qemu_config.example.json b/quality/integration_testing/environments/dual_qemu/dual_qemu_config.example.json new file mode 100644 index 000000000..448150049 --- /dev/null +++ b/quality/integration_testing/environments/dual_qemu/dual_qemu_config.example.json @@ -0,0 +1,47 @@ +{ + "ivshmem": { + "size": "4M" + }, + "intervm_network": { + "enabled": false, + "host_port": 12345 + }, + "vms": [ + { + "networks": [ + { + "name": "vm_a_net", + "ip_address": "127.0.0.1", + "gateway": "127.0.0.1" + } + ], + "ssh_port": 2222, + "qemu_num_cores": "1", + "qemu_ram_size": "1G", + "port_forwarding": [ + { + "host_port": 2222, + "guest_port": 22 + } + ] + }, + { + "networks": [ + { + "name": "vm_b_net", + "ip_address": "127.0.0.1", + "gateway": "127.0.0.1" + } + ], + "ssh_port": 2223, + "qemu_num_cores": "1", + "qemu_ram_size": "1G", + "port_forwarding": [ + { + "host_port": 2223, + "guest_port": 22 + } + ] + } + ] +} diff --git a/quality/integration_testing/environments/dual_qemu/dual_qemu_process.py b/quality/integration_testing/environments/dual_qemu/dual_qemu_process.py new file mode 100644 index 000000000..78a976d8f --- /dev/null +++ b/quality/integration_testing/environments/dual_qemu/dual_qemu_process.py @@ -0,0 +1,137 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Process wrapper that extends :class:`QemuProcess` with ivshmem and boot-retry logic. + +Subclasses ``QemuProcess`` and replaces its internal ``_qemu`` with an +:class:`IvshmemQemu` instance so the VM is launched with an ``ivshmem-plain`` device. + +The ``start()`` method is self-healing: it waits for stable SSH, runs ``pre_tests_phase``, +and restarts the QEMU process up to ``max_boot_attempts`` times if sshd never comes up. +""" +import logging +import time + +from score.itf.plugins.qemu.checks import pre_tests_phase +from score.itf.plugins.qemu.qemu_process import QemuProcess +from score.itf.plugins.qemu.qemu_target import QemuTarget + +from .ivshmem_qemu import IvshmemQemu + +logger = logging.getLogger(__name__) + + +def _wait_for_ssh(target, total_timeout: int = 180, interval: int = 3, stable_successes: int = 3): + """Wait until the VM *stably* serves SSH. + + Early-boot sshd is briefly unstable, so require several consecutive successes to + avoid the ``pre_tests_phase`` (5 retries) failing in that window. + """ + deadline = time.monotonic() + total_timeout + last_error = None + consecutive = 0 + while time.monotonic() < deadline: + try: + with target.ssh(timeout=10, n_retries=1, retry_interval=1) as ssh: + if ssh.execute_command("echo ready") == 0: + consecutive += 1 + if consecutive >= stable_successes: + return + time.sleep(interval) + continue + consecutive = 0 + except Exception as ex: # pylint: disable=broad-except + last_error = ex + consecutive = 0 + time.sleep(interval) + raise TimeoutError(f"VM never became stably reachable via SSH within {total_timeout}s: {last_error}") + + +class DualQemuProcess(QemuProcess): + """A :class:`QemuProcess` subclass with ivshmem support and self-healing boot. + + Use as a context manager:: + + with DualQemuProcess(...) as process: + target = process.target + ... + """ + + def __init__( + self, + path_to_qemu_image, + available_ram, + available_cores, + vm_config, + port_forwarding=[], + ivshmem_path=None, + ivshmem_size="4M", + intervm=None, + vm_index=0, + max_boot_attempts=3, + boot_timeout=100, + ): + super().__init__(path_to_qemu_image, available_ram, available_cores, port_forwarding=port_forwarding) + # Replace the base's default Qemu with our ivshmem-capable subclass. + self._qemu = IvshmemQemu( + path_to_qemu_image, + available_ram, + available_cores, + port_forwarding=port_forwarding, + ivshmem_path=ivshmem_path, + ivshmem_size=ivshmem_size, + intervm=intervm, + vm_index=vm_index, + ) + self._vm_config = vm_config + self._max_boot_attempts = max_boot_attempts + self._boot_timeout = boot_timeout + self._target = None + + def start(self): + """Boot the VM, retrying up to ``max_boot_attempts`` times if sshd never serves.""" + last_error = None + for attempt in range(1, self._max_boot_attempts + 1): + super().start() + try: + self._target = QemuTarget(self, self._vm_config) + _wait_for_ssh(self._target, total_timeout=self._boot_timeout) + pre_tests_phase(self._target) + return self + except Exception as ex: # pylint: disable=broad-except + last_error = ex + logger.warning( + "VM boot attempt %d/%d did not reach a usable state (%s); restarting", + attempt, + self._max_boot_attempts, + ex, + ) + try: + self.stop() + except Exception: # pylint: disable=broad-except + logger.exception("Failed to stop the wedged QEMU before retrying") + raise RuntimeError( + f"VM never booted into a usable state after {self._max_boot_attempts} attempts: {last_error}" + ) + + def ensure_responsive(self, timeout: int = 30, stable_successes: int = 2): + """Re-verify the VM is still reachable; restart in place if not.""" + try: + _wait_for_ssh(self._target, total_timeout=timeout, stable_successes=stable_successes) + except Exception as ex: # pylint: disable=broad-except + logger.warning("VM went unresponsive (%s); restarting", ex) + self.restart() + + @property + def target(self): + """The ``QemuTarget`` for this VM (available after ``start()``).""" + return self._target diff --git a/quality/integration_testing/environments/dual_qemu/example_xvm/BUILD b/quality/integration_testing/environments/dual_qemu/example_xvm/BUILD new file mode 100644 index 000000000..bd0a59987 --- /dev/null +++ b/quality/integration_testing/environments/dual_qemu/example_xvm/BUILD @@ -0,0 +1,85 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +# Cross-VM lib memory over the ivshmem BAR (Create on one VM, Open on the other). +# Self-contained: BAR discovery + the custom TypedMemory provider live here, and the binary +# rendezvous through a handshake header in the last page of the BAR. See the .cpp files. +load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library") +load("@score_baselibs//score/language/safecpp:toolchain_features.bzl", "COMPILER_WARNING_FEATURES") +load("//quality/integration_testing:integration_testing.bzl", "dual_qemu_integration_test") +load("//score/mw/com/test:pkg_application.bzl", "pkg_application") + +# BAR discovery + the custom TypedMemory provider that binds a shm name to the ivshmem BAR +# physical address (shm_ctl(SHMCTL_PHYS)). This is the only piece that lets *unmodified* +# score::memory::shared map onto the BAR. +cc_library( + name = "ivshmem_lib_memory_support", + srcs = [ + "bar_discovery.cpp", + "bar_typed_memory.cpp", + ], + hdrs = [ + "bar_discovery.h", + "bar_typed_memory.h", + ], + features = COMPILER_WARNING_FEATURES + [ + "aborts_upon_exception", + ], + # The provider reaches the ivshmem shared memory through pci-server (libpci). + linkopts = select({ + "@platforms//os:qnx": ["-lpci"], + "//conditions:default": [], + }), + target_compatible_with = ["@platforms//os:qnx"], + deps = [ + # Umbrella target that exports the factory + resource; the concrete + # :shared_memory_factory target is not visible to us. + "@score_baselibs//score/memory/shared", + # Public TypedMemory interface (transitively provides user_permission.h + errno). + "@score_baselibs//score/memory/shared/typedshm/typedshm_wrapper:typedmemory", + "@score_baselibs//score/os:errno", + ], +) + +# TODO: Uncomment once https://github.com/eclipse-score/baselibs/issues/348 is resolved and +# com is bumped to a baselibs version containing the lock-free allocator fix. +# cc_binary( +# name = "lib_memory_ivshmem_xvm", +# srcs = ["lib_memory_ivshmem_xvm.cpp"], +# features = COMPILER_WARNING_FEATURES + [ +# "aborts_upon_exception", +# ], +# target_compatible_with = ["@platforms//os:qnx"], +# deps = [ +# ":ivshmem_lib_memory_support", +# "@score_baselibs//score/memory/shared", +# "@score_baselibs//score/memory/shared:managed_memory_resource", +# "@score_baselibs//score/memory/shared:map", +# "@score_baselibs//score/memory/shared:vector", +# # lib memory pulls in mw::log; a concrete recorder backend is needed at final link +# # (otherwise: undefined reference to CreateRecorderFactory()). +# "@score_baselibs//score/mw/log:console", +# ], +# ) +# +# pkg_application( +# name = "lib_memory_ivshmem_xvm-pkg", +# app_name = "lib_memory_ivshmem_xvm", +# bin = [":lib_memory_ivshmem_xvm"], +# ) +# +# dual_qemu_integration_test( +# name = "test_lib_memory_ivshmem_xvm", +# srcs = ["lib_memory_ivshmem_xvm_test.py"], +# filesystem_a = ":lib_memory_ivshmem_xvm-pkg", +# filesystem_b = ":lib_memory_ivshmem_xvm-pkg", +# ) diff --git a/quality/integration_testing/environments/dual_qemu/example_xvm/README.md b/quality/integration_testing/environments/dual_qemu/example_xvm/README.md new file mode 100644 index 000000000..9f407e20c --- /dev/null +++ b/quality/integration_testing/environments/dual_qemu/example_xvm/README.md @@ -0,0 +1,78 @@ +# Cross-VM lib-memory example + +This is the payload demo for the [`dual_qemu`](../README.md) plugin: it drives the +`score::memory::shared` allocator (the one LoLa uses) across the two VMs over the ivshmem +BAR. The single physical BAR is split into **two page-aligned lib-memory regions** plus a +handshake page — each VM *creates* its own region and *opens* the peer's, even though each VM +maps the BAR at a **different** virtual base address. The data in each region is a LoLa-style +storage root: a single object constructed *with* the memory resource, holding a shared-memory +container whose elements are allocated through the resource's `MemoryResourceProxy`. Its +internal `OffsetPtr`s resolve on the peer VM despite the different base — which is exactly the +property that makes the LoLa data plane work across VMs. + +The exchange is **bidirectional**, using two different LoLa container types in two separate +regions, so each region has a **single writer** (its creator) — no VM allocates into a +peer-created arena: + +| Direction | Region / shm name | Container | +| ---------------------- | -------------------- | ------------------------------------------------ | +| producer → consumer | region A / `/fwd` | `score::memory::shared::Vector` | +| consumer → producer | region B / `/rev` | `score::memory::shared::Map` | + +``` +BAR base ─────────────────────────────────────────────► BAR end +| region A (forward) | region B (reverse) | handshake | +| paddr+0, size=SA | paddr+SA, size=SB | last page | + producer Create()s consumer Create()s + consumer Open()s (read) producer Open()s (read) +``` + +The single binary `lib_memory_ivshmem_xvm` takes a `--role`: + +- `--role producer` (VM-A) — `Create("/fwd")` at the BAR base and `push_back` two values into + its shared `Vector`; then (after the consumer signals its region is ready) bind + `Open("/rev")` + and verify the `Map` the consumer created. +- `--role consumer` (VM-B) — binds + `Open("/fwd")`, verifies the magic, the run `nonce`, and + the vector sum (`42 + 43 == 85`); then `Create("/rev")` in the second sub-range and `emplace` + two entries into its own `Map`. + +Both print `verified` on success. + +> Two cross-VM hurdles the example has to clear: +> 1. **Separate shm namespaces** — the same name is a *different* object in each guest, so the +> reader must first bind its name to the region's BAR physical sub-range (`shm_ctl(SHMCTL_PHYS)` +> via a custom `TypedMemory` provider) *before* calling `Open()`. +> 2. **Init ordering** — ivshmem-plain has no doorbell, so the two roles rendezvous by polling +> a small handshake header placed in the **last page** of the BAR, outside both lib-memory +> arenas. Start both roles roughly together; each waits for the other. +> +> Even with single-writer regions this **still** relies on the lock-free lib-memory patch +> (`//third_party/score_baselibs:lib_memory_lockfree_alloc.patch`): the stock +> `SharedMemoryResource` control block starts with a process-shared `pthread` mutex, and +> `pthread_mutex_lock` returns `EINVAL` on the BAR's physically-mapped device memory regardless +> of contention. Splitting into two regions removes the cross-*kernel* sync requirement, not the +> device-memory one. QNX-only — the BAR is reached through `pci-server` + `mmap_device_memory()`, +> so no `/dev/ivshmem0` guest driver is needed. +> +> See [eclipse-score/baselibs#348](https://github.com/eclipse-score/baselibs/issues/348) for the +> upstream issue tracking the lock-free fix. + +## Files + +| File | Purpose | +| ----------------------------------- | -------------------------------------------------------------------- | +| `lib_memory_ivshmem_xvm.cpp` | The producer/consumer binary (handshake + Create/Open + verify). | +| `bar_discovery.{h,cpp}` | Finds the ivshmem PCI device and its shared-memory BAR. | +| `bar_typed_memory.{h,cpp}` | `TypedMemory` provider binding a shm name to the BAR phys address. | +| `lib_memory_ivshmem_xvm_test.py` | Runs both roles concurrently and asserts both report `verified`. | +| `BUILD` | Builds + packages the binary and wires `dual_qemu_integration_test`. | + +## Running it as a test + +Run with streamed (or `all`) output so both VMs' lines are shown: + +```bash +bazel test //quality/integration_testing/environments/dual_qemu/example_xvm:test_lib_memory_ivshmem_xvm \ + --config=qnx_x86_64 \ + --test_output=streamed +``` diff --git a/quality/integration_testing/environments/dual_qemu/example_xvm/bar_discovery.cpp b/quality/integration_testing/environments/dual_qemu/example_xvm/bar_discovery.cpp new file mode 100644 index 000000000..627611f36 --- /dev/null +++ b/quality/integration_testing/environments/dual_qemu/example_xvm/bar_discovery.cpp @@ -0,0 +1,130 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ +#include "quality/integration_testing/environments/dual_qemu/example_xvm/bar_discovery.h" + +#include + +#if defined(__QNXNTO__) +// The QNX PCI server header declares plain C functions but does not guard them with +// extern "C", so wrap it to avoid C++ name mangling when linking against libpci. +extern "C" { +#include +} +#include // ThreadCtl, _NTO_TCTL_IO +#endif + +namespace qemu_shared_ivshmem +{ + +#if defined(__QNXNTO__) + +bool DiscoverIvshmemBar(std::uint64_t& paddr, std::uint64_t& size) noexcept +{ + constexpr pci_vid_t kVendor = 0x1af4U; // Red Hat / virtio vendor used by ivshmem + constexpr pci_did_t kDevice = 0x1110U; // ivshmem device id + + // Reading/writing PCI configuration space from a user process is privileged on QNX. + // ThreadCtl(_NTO_TCTL_IO) requests I/O privilege; it succeeds when running as root. + if (::ThreadCtl(_NTO_TCTL_IO, nullptr) == -1) + { + std::perror("ThreadCtl(_NTO_TCTL_IO)"); + return false; + } + + const pci_bdf_t bdf = pci_device_find(0U, kVendor, kDevice, PCI_CCODE_ANY); + if (bdf == PCI_BDF_NONE) + { + std::fprintf(stderr, "ivshmem PCI device %04x:%04x not found (is pci-server running?)\n", kVendor, kDevice); + return false; + } + + pci_err_t err = PCI_ERR_OK; + const pci_devhdl_t hdl = pci_device_attach(bdf, pci_attachFlags_OWNER, &err); + if (hdl == nullptr || err != PCI_ERR_OK) + { + std::fprintf(stderr, "pci_device_attach failed (err=%d)\n", static_cast(err)); + return false; + } + + // Booting the QNX IFS directly with `-kernel` skips the BIOS PCI enumeration that would + // normally turn the device on, so enable Memory Space + Bus Master ourselves. Without + // it, CPU accesses to the physical BAR are not decoded and fault ("Memory fault"). + constexpr pci_cmd_t kMemSpaceEnable = static_cast(1U << 1); + constexpr pci_cmd_t kBusMasterEnable = static_cast(1U << 2); + pci_cmd_t cmd = 0U; + err = pci_device_read_cmd(bdf, &cmd); + if (err != PCI_ERR_OK) + { + std::fprintf(stderr, "pci_device_read_cmd failed (err=%d)\n", static_cast(err)); + (void)pci_device_detach(hdl); + return false; + } + pci_cmd_t cmd_set = 0U; + err = pci_device_write_cmd(hdl, static_cast(cmd | kMemSpaceEnable | kBusMasterEnable), &cmd_set); + if (err != PCI_ERR_OK) + { + std::fprintf(stderr, "pci_device_write_cmd failed (err=%d)\n", static_cast(err)); + (void)pci_device_detach(hdl); + return false; + } + + pci_ba_t ba[7]; + int_t nba = static_cast(sizeof(ba) / sizeof(ba[0])); + err = pci_device_read_ba(hdl, &nba, ba, pci_reqType_e_UNSPECIFIED); + if (err != PCI_ERR_OK) + { + std::fprintf(stderr, "pci_device_read_ba failed (err=%d)\n", static_cast(err)); + (void)pci_device_detach(hdl); + return false; + } + + // The shared-memory BAR is the largest memory region (BAR0 holds the small register + // block, the shared RAM lives in BAR2). + const pci_ba_t* shared = nullptr; + for (int_t i = 0; i < nba; ++i) + { + if (ba[i].type == pci_asType_e_MEM && (shared == nullptr || ba[i].size > shared->size)) + { + shared = &ba[i]; + } + } + if (shared == nullptr) + { + std::fprintf(stderr, "no memory BAR found on the ivshmem device\n"); + (void)pci_device_detach(hdl); + return false; + } + + std::fprintf(stderr, + "ivshmem: BAR%d paddr=0x%llx size=0x%llx\n", + static_cast(shared->bar_num), + static_cast(shared->addr), + static_cast(shared->size)); + + // Keep the device attached for the process lifetime; the OS releases it on exit. + paddr = static_cast(shared->addr); + size = static_cast(shared->size); + return true; +} + +#else // !__QNXNTO__ + +bool DiscoverIvshmemBar(std::uint64_t& /*paddr*/, std::uint64_t& /*size*/) noexcept +{ + std::fprintf(stderr, "DiscoverIvshmemBar is only supported on QNX\n"); + return false; +} + +#endif // __QNXNTO__ + +} // namespace qemu_shared_ivshmem diff --git a/quality/integration_testing/environments/dual_qemu/example_xvm/bar_discovery.h b/quality/integration_testing/environments/dual_qemu/example_xvm/bar_discovery.h new file mode 100644 index 000000000..c8bb04471 --- /dev/null +++ b/quality/integration_testing/environments/dual_qemu/example_xvm/bar_discovery.h @@ -0,0 +1,36 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ +#ifndef QUALITY_INTEGRATION_TESTING_ENVIRONMENTS_DUAL_QEMU_EXAMPLE_XVM_BAR_DISCOVERY_H +#define QUALITY_INTEGRATION_TESTING_ENVIRONMENTS_DUAL_QEMU_EXAMPLE_XVM_BAR_DISCOVERY_H + +#include + +namespace qemu_shared_ivshmem +{ + +// Discovers the ivshmem shared-memory PCI device (QEMU ivshmem-plain, and the +// QNX-Hypervisor `vdev ivshmem`, both present vendor 0x1af4 / device 0x1110) through +// pci-server, enables Memory-Space + Bus-Master decoding, and returns the *physical* +// address and size of its shared-memory BAR (the largest MEM BAR). +// +// Unlike the hand-rolled example, this does NOT map the BAR itself -- it only returns the +// physical address so that lib memory (SharedMemoryFactory) can map it via a typed-memory +// provider that binds a shm name to this physical address (shm_ctl(SHMCTL_PHYS)). +// +// The device is kept attached for the lifetime of the process (released by the OS on exit). +// QNX-only: on any other OS this returns false. +bool DiscoverIvshmemBar(std::uint64_t& paddr, std::uint64_t& size) noexcept; + +} // namespace qemu_shared_ivshmem + +#endif // QUALITY_INTEGRATION_TESTING_ENVIRONMENTS_DUAL_QEMU_EXAMPLE_XVM_BAR_DISCOVERY_H diff --git a/quality/integration_testing/environments/dual_qemu/example_xvm/bar_typed_memory.cpp b/quality/integration_testing/environments/dual_qemu/example_xvm/bar_typed_memory.cpp new file mode 100644 index 000000000..0e1723774 --- /dev/null +++ b/quality/integration_testing/environments/dual_qemu/example_xvm/bar_typed_memory.cpp @@ -0,0 +1,120 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ +#include "quality/integration_testing/environments/dual_qemu/example_xvm/bar_typed_memory.h" + +#include +#include // shm_open, shm_unlink, shm_ctl_special, SHMCTL_PHYS, SHMCTL_HAS_SPECIAL (QNX) +#include + +#include + +#if defined(__QNXNTO__) && (defined(__X86_64__) || defined(__x86_64__)) +// X86_64_PTE_MTYPE_WB: the "special" value that tells shm_ctl_special() to map the +// physical region Write-Back cacheable (the header documents these as being "for use +// with shm_ctl_special()"). ivshmem-plain is host-RAM backed, so WB is correct and lets +// normal loads/stores/atomics work on the BAR. +#include +#endif + +namespace qemu_shared_ivshmem +{ + +BarTypedMemory::BarTypedMemory(std::uint64_t paddr, std::uint64_t size) noexcept : paddr_{paddr}, size_{size} {} + +score::cpp::expected_blank BarTypedMemory::AllocateNamedTypedMemory( + std::size_t /*shm_size*/, + std::string shm_name, + const score::memory::shared::permission::UserPermissions& /*permissions*/) const noexcept +{ +#if defined(__QNXNTO__) + // Create the named shared-memory object... + const int fd = ::shm_open(shm_name.c_str(), O_RDWR | O_CREAT, 0666); + if (fd == -1) + { + return score::cpp::make_unexpected(score::os::Error::createFromErrno()); + } + + // ...and bind it to the ivshmem BAR physical address. After this, a plain + // shm_open(shm_name)+mmap (which the factory does next) maps the BAR, not fresh RAM. + // + // We request Write-Back caching via shm_ctl_special(X86_64_PTE_MTYPE_WB) rather than a + // plain shm_ctl(SHMCTL_PHYS) (which maps the BAR UNCACHEABLE): ivshmem-plain is host-RAM + // backed and KVM keeps it hardware-cache-coherent across guests, so WB is correct and + // lets ordinary loads/stores/atomics work on the BAR. + // + // NOTE (verified on target): WB is ACCEPTED (rc=0). The production score::memory::shared + // umbrella then runs over the BAR *only* with the lock-free lib-memory patch + // (//third_party/score_baselibs:lib_memory_lockfree_alloc.patch). The stock resource places + // a QNX process-shared pthread mutex (SharedMemoryResource::ControlBlock) at the start of the + // region and locks it; on SHMCTL_PHYS device-BAR memory that lock returns EINVAL(22) and + // aborts -- a QNX limitation on process-shared sync objects over physically-mapped device + // memory, independent of cache type. The patch replaces that mutex with a lock-free (atomic + // CAS) bump allocator, which is what lets this example map lib memory onto the BAR. +#if defined(__X86_64__) || defined(__x86_64__) + int rc = ::shm_ctl_special( + fd, SHMCTL_PHYS | SHMCTL_HAS_SPECIAL, paddr_, size_, static_cast(X86_64_PTE_MTYPE_WB)); + if (rc == -1) + { + // WB rejected (older/other QNX): fall back to the default uncacheable binding. + rc = ::shm_ctl(fd, SHMCTL_PHYS, paddr_, size_); + } +#else + const int rc = ::shm_ctl(fd, SHMCTL_PHYS, paddr_, size_); +#endif + if (rc == -1) + { + const int saved = errno; + (void)::close(fd); + (void)::shm_unlink(shm_name.c_str()); + return score::cpp::make_unexpected(score::os::Error::createFromErrno(saved)); + } + + // The object persists (name + physical binding) until Unlink(); our fd is no longer + // needed because the factory re-opens by name. + (void)::close(fd); + return {}; +#else + (void)shm_name; + return score::cpp::make_unexpected(score::os::Error::createFromErrno(ENOSYS)); +#endif +} + +score::cpp::expected BarTypedMemory::AllocateAndOpenAnonymousTypedMemory( + std::uint64_t /*shm_size*/) const noexcept +{ + // A device BAR is a fixed, named physical region; anonymous allocation is meaningless. + return score::cpp::make_unexpected(score::os::Error::createFromErrno(ENOSYS)); +} + +score::cpp::expected_blank BarTypedMemory::Unlink(std::string_view shm_name) const noexcept +{ +#if defined(__QNXNTO__) + const std::string name{shm_name}; + if (::shm_unlink(name.c_str()) == -1) + { + return score::cpp::make_unexpected(score::os::Error::createFromErrno()); + } + return {}; +#else + (void)shm_name; + return score::cpp::make_unexpected(score::os::Error::createFromErrno(ENOSYS)); +#endif +} + +score::cpp::expected BarTypedMemory::GetCreatorUid( + std::string_view /*shm_name*/) const noexcept +{ + return ::getuid(); +} + +} // namespace qemu_shared_ivshmem diff --git a/quality/integration_testing/environments/dual_qemu/example_xvm/bar_typed_memory.h b/quality/integration_testing/environments/dual_qemu/example_xvm/bar_typed_memory.h new file mode 100644 index 000000000..a2a05c626 --- /dev/null +++ b/quality/integration_testing/environments/dual_qemu/example_xvm/bar_typed_memory.h @@ -0,0 +1,68 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ +#ifndef QUALITY_INTEGRATION_TESTING_ENVIRONMENTS_DUAL_QEMU_EXAMPLE_XVM_BAR_TYPED_MEMORY_H +#define QUALITY_INTEGRATION_TESTING_ENVIRONMENTS_DUAL_QEMU_EXAMPLE_XVM_BAR_TYPED_MEMORY_H + +#include "score/memory/shared/typedshm/typedshm_wrapper/typed_memory.h" + +#include + +#include +#include +#include +#include + +namespace qemu_shared_ivshmem +{ + +// A custom lib-memory "typed memory" provider that backs a named shared-memory object with +// the ivshmem BAR instead of ordinary /dev/shmem RAM. +// +// This is the single seam that lets *unmodified* score::memory::shared map onto the BAR: +// when injected via SharedMemoryFactory::SetTypedMemoryProvider(), a subsequent +// Create(path, ..., prefer_typed_memory=true) delegates the allocation here. We bind the +// shm name to the BAR's physical address with shm_ctl(fd, SHMCTL_PHYS, paddr, size); the +// factory then does its usual shm_open(path)+mmap, which now lands on the BAR (and skips +// ftruncate/seal/permissions). +// +// Both VMs mapping the same BAR at different virtual addresses is fine because lib memory +// stores every pointer as an OffsetPtr (self-relative offset). +class BarTypedMemory final : public score::memory::shared::TypedMemory +{ + public: + // paddr/size come from DiscoverIvshmemBar(). + BarTypedMemory(std::uint64_t paddr, std::uint64_t size) noexcept; + + // Binds `shm_name` to the ivshmem BAR physical address so a later shm_open()+mmap of + // that name maps the BAR. This is the only method the factory actually needs. + score::cpp::expected_blank AllocateNamedTypedMemory( + std::size_t shm_size, + std::string shm_name, + const score::memory::shared::permission::UserPermissions& permissions) const noexcept override; + + // Anonymous typed memory is not supported for a fixed device BAR. + score::cpp::expected AllocateAndOpenAnonymousTypedMemory( + std::uint64_t shm_size) const noexcept override; + + score::cpp::expected_blank Unlink(std::string_view shm_name) const noexcept override; + + score::cpp::expected GetCreatorUid(std::string_view shm_name) const noexcept override; + + private: + std::uint64_t paddr_; + std::uint64_t size_; +}; + +} // namespace qemu_shared_ivshmem + +#endif // QUALITY_INTEGRATION_TESTING_ENVIRONMENTS_DUAL_QEMU_EXAMPLE_XVM_BAR_TYPED_MEMORY_H diff --git a/quality/integration_testing/environments/dual_qemu/example_xvm/lib_memory_ivshmem_xvm.cpp b/quality/integration_testing/environments/dual_qemu/example_xvm/lib_memory_ivshmem_xvm.cpp new file mode 100644 index 000000000..55c8e4b7d --- /dev/null +++ b/quality/integration_testing/environments/dual_qemu/example_xvm/lib_memory_ivshmem_xvm.cpp @@ -0,0 +1,544 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ + +// CROSS-VM demo: Lib memory (score::memory::shared) over the ivshmem BAR, +// with TWO separate shared-memory regions carved out of the one BAR -- each VM CREATES its own +// region and OPENS the peer's -- so the exchange is bidirectional with single-writer ownership. +// +// lib_memory_ivshmem_xvm --role producer // VM-A: Create() the forward region, read reverse +// lib_memory_ivshmem_xvm --role consumer // VM-B: Open() the forward region, Create() reverse +// +// BAR layout (the single physical BAR split into page-aligned sub-ranges): +// +// | forward region (A) | reverse region (B) | handshake | +// | paddr+0, size=SA | paddr+SA, size=SB | last page | +// producer Create()s consumer Create()s +// writes Vector writes Map +// consumer Open()s (read) producer Open()s (read) +// +// Bidirectional data sharing (two different LoLa container types, two regions): +// producer -> consumer : score::memory::shared::Vector (region A / "/fwd") +// consumer -> producer : score::memory::shared::Map (region B / "/rev") +// Each region has a SINGLE writer (its creator), so no VM allocates into a peer-created arena. +// +// Why a handshake is still needed (the two hard cross-VM problems): +// 1. Separate shm namespaces. Each QNX guest has its OWN shm object namespace: a name on +// VM-A and on VM-B are DIFFERENT objects. What makes them the same memory is that BOTH +// are bound to the SAME BAR physical sub-range via shm_ctl(SHMCTL_PHYS). So the reader +// must FIRST bind its own name to that sub-range (via the provider) and only THEN call +// SharedMemoryFactory::Open() -- Open() itself does a plain shm_open() and never consults +// the typed-memory provider. +// 2. Init ordering. A reader must not Open()/parse a region until its creator has finished +// Create(). ivshmem-plain has no doorbell, so we rendezvous by polling a small handshake +// header placed in the LAST page of the BAR (outside both lib-memory arenas). +// +// Protocol (best-effort; see caveats at the bottom): +// producer: reset handshake -> Create("/fwd")+init Vector (run `nonce`) -> publish {nonce, +// FWD_READY} -> wait for REV_READY -> bind+Open("/rev") -> verify Map -> exit. +// consumer: wait for FWD_READY -> bind+Open("/fwd") -> verify Vector -> Create("/rev")+init +// Map (echo `nonce`) -> publish REV_READY -> exit. +// +// Even though each region has a single writer, this STILL needs the lock-free lib-memory patch: +// the control block's mutex physically lives on the BAR (device memory), where pthread_mutex_lock +// returns EINVAL regardless of contention. The split removes the cross-KERNEL sync requirement, +// not the device-memory one. +// +// QNX-only (uses pci-server + mmap_device_memory + shm_ctl(SHMCTL_PHYS)). + +#include "quality/integration_testing/environments/dual_qemu/example_xvm/bar_discovery.h" +#include "quality/integration_testing/environments/dual_qemu/example_xvm/bar_typed_memory.h" + +#include "score/memory/shared/i_shared_memory_resource.h" +#include "score/memory/shared/managed_memory_resource.h" +#include "score/memory/shared/map.h" +#include "score/memory/shared/shared_memory_factory.h" +#include "score/memory/shared/vector.h" + +#include +#include +#include +#include +#include +#include +#include + +#if defined(__QNXNTO__) +#include // mmap_device_memory / munmap_device_memory +#include // getpid +#endif + +namespace +{ +using score::memory::shared::ISharedMemoryResource; +using score::memory::shared::ManagedMemoryResource; +using score::memory::shared::SharedMemoryFactory; + +// Process exit codes (0 == OK). +constexpr int kExitDiscoverFailed = 1; // no ivshmem BAR +constexpr int kExitBadArgs = 2; // missing/invalid --role +constexpr int kExitMapFailed = 3; // could not raw-map the BAR for the handshake +constexpr int kExitCreateFailed = 4; // producer: Create returned nullptr +constexpr int kExitNotTypedMemory = 5; // producer: shm_ctl(SHMCTL_PHYS) rejected +constexpr int kExitOpenFailed = 6; // consumer: bind or Open failed +constexpr int kExitVerifyFailed = 7; // consumer: content mismatch +constexpr int kExitHandshakeTimeout = 8; // peer did not rendezvous in time + +constexpr std::uint32_t kMagic = 0x00C0FFEEU; +constexpr std::uint32_t kReadyMagic = 0x0AD10ADEU; // a region has been created + filled + +constexpr char kFwdName[] = "/lola_ivshmem_fwd"; // region A: producer -> consumer (Vector) +constexpr char kRevName[] = "/lola_ivshmem_rev"; // region B: consumer -> producer (Map) +constexpr std::size_t kReserve = 4096U; // lib-memory user bytes per region +constexpr std::uint64_t kPageSize = 4096U; // shm_ctl(SHMCTL_PHYS) needs page alignment +constexpr std::size_t kHandshakeReserve = 4096U; // last page of BAR reserved for the handshake +constexpr int kPollTimeoutMs = 30000; // rendezvous timeout +constexpr int kPollStepMs = 10; + +// LoLa-style shared layout: a single "storage" root object constructed WITH the memory +// resource, holding a shared-memory container whose elements are allocated through the +// resource's MemoryResourceProxy -- exactly how lola::ServiceDataStorage owns its Map/Vector +// members. The container's internal OffsetPtrs resolve on the peer VM despite the different +// mapping base. `nonce` links this Create to the handshake so the reader can reject stale +// BAR contents from a previous run. +// +// Two separate regions, one container each, single writer per region: +// ForwardRoot lives in region A, created + filled by the producer (consumer reads it). +// ReverseRoot lives in region B, created + filled by the consumer (producer reads it). +using ValueVector = score::memory::shared::Vector; +using ResponseMap = score::memory::shared::Map; + +struct ForwardRoot +{ + explicit ForwardRoot(ManagedMemoryResource& resource) noexcept : magic{0U}, nonce{0U}, values{resource} {} + + std::uint32_t magic; + std::uint32_t nonce; + ValueVector values; // producer -> consumer +}; + +struct ReverseRoot +{ + explicit ReverseRoot(ManagedMemoryResource& resource) noexcept : magic{0U}, nonce{0U}, responses{resource} {} + + std::uint32_t magic; + std::uint32_t nonce; + ResponseMap responses; // consumer -> producer +}; + +// Page-aligned split of the one BAR into two lib-memory regions plus the handshake page. +struct BarLayout +{ + std::uint64_t fwd_paddr; + std::uint64_t fwd_size; + std::uint64_t rev_paddr; + std::uint64_t rev_size; +}; + +BarLayout ComputeLayout(std::uint64_t paddr, std::uint64_t size) noexcept +{ + // Reserve the last page for the handshake, then split the rest in two page-aligned halves. + const std::uint64_t usable = size - kHandshakeReserve; + const std::uint64_t fwd_size = (usable / 2U) & ~(kPageSize - 1U); + BarLayout layout{}; + layout.fwd_paddr = paddr; + layout.fwd_size = fwd_size; + layout.rev_paddr = paddr + fwd_size; // page-aligned: paddr and fwd_size are both page-aligned + layout.rev_size = usable - fwd_size; + return layout; +} + +// Handshake header placed at the END of the BAR (raw-mapped), never touched by lib memory. +// volatile because it is written by one VM and polled by the other with no cache-coherency +// help beyond ivshmem-plain's shared host RAM. +struct Handshake +{ + volatile std::uint32_t fwd_ready; // producer -> consumer: region A created + filled + volatile std::uint32_t rev_ready; // consumer -> producer: region B created + filled + volatile std::uint32_t nonce; // producer's run nonce, echoed into both roots + volatile std::uint32_t cross_write_done_a; // consumer wrote into region A (producer's region) + volatile std::uint32_t cross_write_done_b; // producer wrote into region B (consumer's region) +}; + +void SleepMs(int ms) noexcept +{ + struct timespec ts; + ts.tv_sec = ms / 1000; + ts.tv_nsec = static_cast(ms % 1000) * 1000000L; + (void)nanosleep(&ts, nullptr); +} + +// Raw-maps the BAR physical memory so we can reach the handshake header. Discovery has +// already requested I/O privilege and enabled the device's memory decoding. +void* MapBarRaw(std::uint64_t paddr, std::uint64_t size) noexcept +{ +#if defined(__QNXNTO__) + void* const addr = ::mmap_device_memory(nullptr, size, PROT_READ | PROT_WRITE, 0, static_cast(paddr)); + return (addr == MAP_FAILED) ? nullptr : addr; +#else + (void)paddr; + (void)size; + return nullptr; +#endif +} + +Handshake* HandshakeAt(void* raw, std::uint64_t size) noexcept +{ + // Last page of the BAR; well past the lib-memory arena (control block + kReserve). + return reinterpret_cast(static_cast(raw) + (size - kHandshakeReserve)); +} + +// Polls *word until it equals `expected` or the timeout elapses. Returns true on match. +bool WaitForWord(volatile std::uint32_t* word, std::uint32_t expected, int timeout_ms) noexcept +{ + for (int waited = 0; waited <= timeout_ms; waited += kPollStepMs) + { + if (*word == expected) + { + return true; + } + SleepMs(kPollStepMs); + } + return false; +} + +int RunProducer(const BarLayout& layout, Handshake* hs) noexcept +{ + // Reset the handshake for this run before anyone can observe a READY. + hs->rev_ready = 0U; + hs->nonce = 0U; + hs->fwd_ready = 0U; + hs->cross_write_done_a = 0U; + hs->cross_write_done_b = 0U; + std::atomic_thread_fence(std::memory_order_release); + + std::uint32_t nonce = 0U; +#if defined(__QNXNTO__) + nonce = static_cast(getpid()); +#endif + nonce = (nonce << 8) ^ static_cast(std::time(nullptr)); + if (nonce == 0U) + { + nonce = 1U; + } + + // Forward region (A): Create() our own region at the BAR base and fill its Vector. This VM + // is the SOLE writer of region A. + SharedMemoryFactory::SetTypedMemoryProvider( + std::make_shared(layout.fwd_paddr, layout.fwd_size)); + + auto fwd = SharedMemoryFactory::Create( + kFwdName, + [nonce](std::shared_ptr res) { + // Mirror lola::Skeleton: construct the root storage object WITH the resource, then + // fill its shared-memory container from region A's own arena. + ForwardRoot* const root = res->construct(*res); + root->values.reserve(2U); + root->values.push_back(42U); + root->values.push_back(43U); + root->magic = kMagic; + root->nonce = nonce; + }, + kReserve, + SharedMemoryFactory::WorldWritable{}, + /*prefer_typed_memory=*/true); + + if (fwd == nullptr) + { + std::fprintf(stderr, "producer: Create(/fwd) failed\n"); + return kExitCreateFailed; + } + if (!fwd->IsShmInTypedMemory()) + { + std::fprintf(stderr, "producer: /fwd NOT in typed memory (shm_ctl(SHMCTL_PHYS) rejected)\n"); + return kExitNotTypedMemory; + } + + // Publish: make the forward data visible before announcing it is ready. + std::atomic_thread_fence(std::memory_order_release); + hs->nonce = nonce; + hs->fwd_ready = kReadyMagic; + std::fprintf( + stderr, "producer: published forward Vector region (nonce=0x%08x), waiting for reverse region\n", nonce); + + // Wait for the consumer to Create() + fill region B. + if (!WaitForWord(&hs->rev_ready, kReadyMagic, kPollTimeoutMs)) + { + std::fprintf(stderr, "producer: timed out waiting for consumer reverse region\n"); + return kExitHandshakeTimeout; + } + std::atomic_thread_fence(std::memory_order_acquire); + + // Reverse region (B): bind our own name to region B's sub-range, then Open() and read the + // Map the consumer created. Its OffsetPtr node links resolve here despite the different base. + auto rev_provider = std::make_shared(layout.rev_paddr, layout.rev_size); + const auto bind = rev_provider->AllocateNamedTypedMemory( + static_cast(layout.rev_size), kRevName, SharedMemoryFactory::WorldWritable{}); + if (!bind.has_value()) + { + std::fprintf(stderr, "producer: failed to bind /rev to region B\n"); + return kExitOpenFailed; + } + + auto rev = SharedMemoryFactory::Open(kRevName, /*is_read_write=*/true); + if (rev == nullptr) + { + std::fprintf(stderr, "producer: Open(/rev) failed\n"); + return kExitOpenFailed; + } + + const ReverseRoot* const root = static_cast(rev->getUsableBaseAddress()); + std::uint32_t response_sum = 0U; + std::size_t response_count = 0U; + for (const auto& entry : root->responses) + { + response_sum += entry.second; + ++response_count; + } + if (root->magic != kMagic || root->nonce != nonce || response_count != 2U || response_sum != 300U) + { + std::fprintf(stderr, + "producer: reverse Map verify FAILED (magic=0x%08x nonce=0x%08x/exp0x%08x count=%zu sum=%u)\n", + root->magic, + root->nonce, + nonce, + response_count, + response_sum); + return kExitVerifyFailed; + } + + std::fprintf(stderr, "producer: resolved consumer reverse Map (count=%zu sum=%u)\n", response_count, response_sum); + + // ---- Cross-write phase: producer writes INTO region B (consumer's region) ---- + // Open region B read-write (already opened above), then emplace a new entry into the + // consumer's Map. This proves BOTH VMs can write to the SAME shared memory location. + ReverseRoot* const rev_root_rw = static_cast(rev->getUsableBaseAddress()); + rev_root_rw->responses.emplace(3U, 400U); + std::atomic_thread_fence(std::memory_order_release); + hs->cross_write_done_b = kReadyMagic; + std::fprintf(stderr, "producer: wrote entry {3,400} into consumer's region B\n"); + + // Wait for consumer to write into region A (our region). + if (!WaitForWord(&hs->cross_write_done_a, kReadyMagic, kPollTimeoutMs)) + { + std::fprintf(stderr, "producer: timed out waiting for consumer cross-write into region A\n"); + return kExitHandshakeTimeout; + } + std::atomic_thread_fence(std::memory_order_acquire); + + // Verify consumer's cross-write into region A (our Vector). + const ForwardRoot* const fwd_root = static_cast(fwd->getUsableBaseAddress()); + if (fwd_root->values.size() != 3U) + { + std::fprintf(stderr, + "producer: cross-write verify FAILED (expected 3 values, got %zu)\n", + static_cast(fwd_root->values.size())); + return kExitVerifyFailed; + } + const std::uint32_t cross_sum = fwd_root->values[0] + fwd_root->values[1] + fwd_root->values[2]; + if (cross_sum != 185U) // 42 + 43 + 100 + { + std::fprintf(stderr, "producer: cross-write verify FAILED (expected sum=185, got %u)\n", cross_sum); + return kExitVerifyFailed; + } + std::fprintf(stderr, + "producer: verified consumer cross-wrote into region A (values=[%u,%u,%u] sum=%u)\n", + fwd_root->values[0], + fwd_root->values[1], + fwd_root->values[2], + cross_sum); + std::printf("verified\n"); + return 0; +} + +int RunConsumer(const BarLayout& layout, Handshake* hs) noexcept +{ + // Wait for the producer to Create() + fill region A. + if (!WaitForWord(&hs->fwd_ready, kReadyMagic, kPollTimeoutMs)) + { + std::fprintf(stderr, "consumer: timed out waiting for producer forward region\n"); + return kExitHandshakeTimeout; + } + std::atomic_thread_fence(std::memory_order_acquire); + const std::uint32_t expected_nonce = hs->nonce; + + // Forward region (A): bind our own name to region A's sub-range, then Open() and verify the + // producer's Vector. Open() does a plain shm_open(), so this VM must bind the name first. + auto fwd_provider = std::make_shared(layout.fwd_paddr, layout.fwd_size); + const auto bind_fwd = fwd_provider->AllocateNamedTypedMemory( + static_cast(layout.fwd_size), kFwdName, SharedMemoryFactory::WorldWritable{}); + if (!bind_fwd.has_value()) + { + std::fprintf(stderr, "consumer: failed to bind /fwd to region A\n"); + return kExitOpenFailed; + } + + auto fwd = SharedMemoryFactory::Open(kFwdName, /*is_read_write=*/true); + if (fwd == nullptr) + { + std::fprintf(stderr, "consumer: Open(/fwd) failed\n"); + return kExitOpenFailed; + } + + ForwardRoot* const froot = static_cast(fwd->getUsableBaseAddress()); + std::uint32_t sum = 0U; + for (const std::uint32_t value : froot->values) + { + sum += value; + } + if (froot->magic != kMagic || froot->nonce != expected_nonce || sum != 85U) + { + std::fprintf(stderr, + "consumer: forward verify FAILED (magic=0x%08x nonce=0x%08x/exp0x%08x sum=%u)\n", + froot->magic, + froot->nonce, + expected_nonce, + sum); + return kExitVerifyFailed; + } + + // Reverse region (B): Create() our OWN region and fill its Map. This VM is the SOLE writer + // of region B; the producer only reads it. Each emplace allocates a tree node from region + // B's own arena. + SharedMemoryFactory::SetTypedMemoryProvider( + std::make_shared(layout.rev_paddr, layout.rev_size)); + + auto rev = SharedMemoryFactory::Create( + kRevName, + [expected_nonce](std::shared_ptr res) { + ReverseRoot* const root = res->construct(*res); + root->responses.emplace(1U, 100U); + root->responses.emplace(2U, 200U); + root->magic = kMagic; + root->nonce = expected_nonce; + }, + kReserve, + SharedMemoryFactory::WorldWritable{}, + /*prefer_typed_memory=*/true); + + if (rev == nullptr) + { + std::fprintf(stderr, "consumer: Create(/rev) failed\n"); + return kExitCreateFailed; + } + if (!rev->IsShmInTypedMemory()) + { + std::fprintf(stderr, "consumer: /rev NOT in typed memory (shm_ctl(SHMCTL_PHYS) rejected)\n"); + return kExitNotTypedMemory; + } + + // Publish: make the reverse Map visible before announcing it is ready. + std::atomic_thread_fence(std::memory_order_release); + hs->rev_ready = kReadyMagic; + std::fprintf(stderr, "consumer: verified forward Vector (sum=%u), published reverse Map region\n", sum); + + // ---- Cross-write phase: consumer writes INTO region A (producer's region) ---- + // The producer's Vector lives in region A. We (consumer) push_back a new value into it. + // This proves BOTH VMs can write to the SAME shared memory location. + ForwardRoot* const fwd_root_rw = static_cast(fwd->getUsableBaseAddress()); + fwd_root_rw->values.push_back(100U); + std::atomic_thread_fence(std::memory_order_release); + hs->cross_write_done_a = kReadyMagic; + std::fprintf(stderr, "consumer: wrote value 100 into producer's region A Vector\n"); + + // Wait for producer to write into region B (our region). + if (!WaitForWord(&hs->cross_write_done_b, kReadyMagic, kPollTimeoutMs)) + { + std::fprintf(stderr, "consumer: timed out waiting for producer cross-write into region B\n"); + return kExitHandshakeTimeout; + } + std::atomic_thread_fence(std::memory_order_acquire); + + // Verify producer's cross-write into region B (our Map). + const ReverseRoot* const rev_root = static_cast(rev->getUsableBaseAddress()); + std::uint32_t final_sum = 0U; + std::size_t final_count = 0U; + for (const auto& entry : rev_root->responses) + { + final_sum += entry.second; + ++final_count; + } + if (final_count != 3U || final_sum != 700U) // 100 + 200 + 400 + { + std::fprintf(stderr, + "consumer: cross-write verify FAILED (expected count=3 sum=700, got count=%zu sum=%u)\n", + final_count, + final_sum); + return kExitVerifyFailed; + } + std::fprintf( + stderr, "consumer: verified producer cross-wrote into region B (count=%zu sum=%u)\n", final_count, final_sum); + std::printf("verified\n"); + return 0; +} + +enum class Role +{ + kUnknown, + kProducer, + kConsumer +}; + +Role ParseRole(int argc, char** argv) noexcept +{ + for (int i = 1; i < argc; ++i) + { + if (std::strcmp(argv[i], "--role") == 0 && (i + 1) < argc) + { + const char* const v = argv[i + 1]; + if (std::strcmp(v, "producer") == 0) + { + return Role::kProducer; + } + if (std::strcmp(v, "consumer") == 0) + { + return Role::kConsumer; + } + } + } + return Role::kUnknown; +} +} // namespace + +int main(int argc, char** argv) +{ + const Role role = ParseRole(argc, argv); + if (role == Role::kUnknown) + { + std::fprintf(stderr, "usage: %s --role \n", argv[0]); + return kExitBadArgs; + } + + std::uint64_t paddr = 0U; + std::uint64_t size = 0U; + if (!qemu_shared_ivshmem::DiscoverIvshmemBar(paddr, size)) + { + return kExitDiscoverFailed; + } + // Need room for two lib-memory regions (control block + kReserve each) plus the handshake. + if (size <= (kHandshakeReserve + 2U * (kReserve + kPageSize))) + { + std::fprintf(stderr, "ivshmem BAR too small: 0x%llx\n", static_cast(size)); + return kExitMapFailed; + } + const BarLayout layout = ComputeLayout(paddr, size); + + void* const raw = MapBarRaw(paddr, size); + if (raw == nullptr) + { + std::fprintf(stderr, "failed to raw-map the ivshmem BAR for the handshake\n"); + return kExitMapFailed; + } + Handshake* const hs = HandshakeAt(raw, size); + + return (role == Role::kProducer) ? RunProducer(layout, hs) : RunConsumer(layout, hs); +} diff --git a/quality/integration_testing/environments/dual_qemu/example_xvm/lib_memory_ivshmem_xvm_test.py b/quality/integration_testing/environments/dual_qemu/example_xvm/lib_memory_ivshmem_xvm_test.py new file mode 100644 index 000000000..955c30f2d --- /dev/null +++ b/quality/integration_testing/environments/dual_qemu/example_xvm/lib_memory_ivshmem_xvm_test.py @@ -0,0 +1,64 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""CROSS-VM test: PRODUCTION lib memory over the ivshmem BAR (two regions). + +The one BAR is split into two page-aligned lib-memory regions plus a handshake page. VM-A runs +``--role producer`` (Create the forward ``/fwd`` region + fill a shared Vector, then Open the +consumer's ``/rev`` region and verify its Map); VM-B runs ``--role consumer`` (Open ``/fwd`` and +verify the Vector, then Create ``/rev`` + fill a shared Map). Each region has a single writer. +The two rendezvous through a handshake header in the last page of the BAR, so they must run +CONCURRENTLY. + +This proves both VMs can Create/Open their own lib-memory region in the same BAR and resolve +LoLa containers in BOTH directions despite mapping the BAR at different bases. + + bazel test //quality/integration_testing/environments/dual_qemu/example_xvm:test_lib_memory_ivshmem_xvm \\ + --config=qnx_x86_64 --test_output=streamed +""" +import logging +import threading + +logger = logging.getLogger(__name__) + +APP = "bin/lib_memory_ivshmem_xvm" +CWD = "/opt/lib_memory_ivshmem_xvm" + + +def _run(target, label, role, results): + rc, out = target.execute(f"cd {CWD} && {APP} --role {role}") + text = out.decode(errors="replace").strip() + logger.info("==================== %s (%s) ====================", label, role) + logger.info("%s (rc=%s)", text, rc) + print(f"\n[{label}] {text} (rc={rc})") + results[label] = (rc, text) + + +def test_cross_vm_lib_memory_over_bar(target_a, target_b): + """Producer Creates+fills the /fwd Vector region; consumer Creates+fills the /rev Map region.""" + results = {} + threads = [ + threading.Thread(target=_run, args=(target_a, "VM-A", "producer", results)), + threading.Thread(target=_run, args=(target_b, "VM-B", "consumer", results)), + ] + for t in threads: + t.start() + for t in threads: + t.join() + + rc_a, text_a = results["VM-A"] + rc_b, text_b = results["VM-B"] + + assert rc_a == 0, f"producer (VM-A) failed (rc={rc_a}): {text_a}" + assert rc_b == 0, f"consumer (VM-B) failed (rc={rc_b}): {text_b}" + assert "verified" in text_a, f"producer did not complete the handshake, got: {text_a!r}" + assert "verified" in text_b, f"consumer did not verify the list, got: {text_b!r}" diff --git a/quality/integration_testing/environments/dual_qemu/ivshmem_qemu.py b/quality/integration_testing/environments/dual_qemu/ivshmem_qemu.py new file mode 100644 index 000000000..c543feabb --- /dev/null +++ b/quality/integration_testing/environments/dual_qemu/ivshmem_qemu.py @@ -0,0 +1,115 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""A QEMU launcher that adds an ``ivshmem-plain`` device so VMs can share one host file. + +Subclasses the upstream :class:`Qemu` and overrides :meth:`_extra_qemu_args` to inject +the ivshmem, inter-VM NIC, and per-VM MAC arguments. +""" +import logging + +from score.itf.plugins.qemu.qemu import Qemu + +logger = logging.getLogger(__name__) + + +class IvshmemQemu(Qemu): + """A :class:`Qemu` subclass that maps a shared ``ivshmem-plain`` region.""" + + def __init__( + self, + path_to_image, + ram="1G", + cores="2", + port_forwarding=[], + ivshmem_path=None, + ivshmem_size="4M", + intervm=None, + vm_index=0, + ): + """ + :param str ivshmem_path: Host backing file shared between VMs (memory-backend-file). + :param str ivshmem_size: Size of the shared region, e.g. "4M". + :param tuple intervm: Optional ("listen"|"connect", host_port) for a point-to-point + socket NIC between the two VMs. ``None`` disables it. + :param int vm_index: Zero-based VM index, used to derive unique NIC MACs. + """ + self._ivshmem_path = ivshmem_path + self._ivshmem_size = ivshmem_size + self._intervm = intervm + self._vm_index = vm_index + self._dual_port_forwarding = port_forwarding + # Pass port_forwarding=[] to the base so it doesn't add default-MAC devices. + # We handle port forwarding ourselves in _extra_qemu_args with per-VM MACs. + super().__init__(path_to_image, ram, cores, cpu="host", port_forwarding=[]) + # Re-resolve: "host" is invalid under TCG, fall back to "max". + if self._accelerator_support == "tcg": + self._Qemu__cpu = "max" + logger.warning("Running under TCG: using -cpu max instead of host.") + + def _extra_qemu_args(self): + """Inject ivshmem, per-VM-MAC port forwarding, and inter-VM NIC arguments.""" + return self._ivshmem_args() + self._port_forwarding_with_mac_args() + self._intervm_args() + + def _ivshmem_args(self): + if not self._ivshmem_path: + return [] + # ivshmem-plain: a passive shared-memory PCI device (no MSI-X / doorbell). + # Both VMs point "mem-path" at the same host file with share=on, so the device + # BAR2 of each guest is backed by the same physical pages. + return [ + "-object", + ( + "memory-backend-file,id=ivshmem_mem," + f"mem-path={self._ivshmem_path},size={self._ivshmem_size},share=on" + ), + "-device", + "ivshmem-plain,memdev=ivshmem_mem", + ] + + def _mac_for(self, nic_id): + """Locally-administered MAC unique per (vm_index, nic_id).""" + return f"52:54:00:00:{self._vm_index:02x}:{nic_id:02x}" + + def _port_forwarding_with_mac_args(self): + """Port forwarding with per-VM unique MACs. + + The base class's ``__port_forwarding_args`` uses QEMU's default MAC, which causes + the second QNX VM's sshd to hang when two VMs run on the same host. We pass + ``port_forwarding=[]`` to the base and handle it here with unique MACs instead. + """ + result = [] + for id, forwarding in enumerate(self._dual_port_forwarding, start=1): + result.extend( + [ + "-netdev", + f"user,id=net{id},hostfwd=tcp::{forwarding.host_port}-:{forwarding.guest_port}", + "-device", + f"virtio-net-pci,netdev=net{id},mac={self._mac_for(id)}", + ] + ) + return result + + def _intervm_args(self): + if not self._intervm: + return [] + mode, host_port = self._intervm + if mode == "listen": + netdev = f"socket,id=intervm,listen=:{host_port}" + else: + netdev = f"socket,id=intervm,connect=127.0.0.1:{host_port}" + return [ + "-netdev", + netdev, + "-device", + f"virtio-net-pci,netdev=intervm,mac={self._mac_for(0x80)}", + ] diff --git a/quality/integration_testing/integration_testing.bzl b/quality/integration_testing/integration_testing.bzl index 26c2f7ad4..347d324fc 100644 --- a/quality/integration_testing/integration_testing.bzl +++ b/quality/integration_testing/integration_testing.bzl @@ -142,3 +142,95 @@ def integration_test(name, srcs, filesystem, **kwargs): env = {"DOCKER_HOST": ""}, **kwargs ) + +def dual_qemu_integration_test( + name, + srcs, + filesystem_a, + filesystem_b = None, + dual_config = Label("//quality/integration_testing/environments/dual_qemu:dual_qemu_config.example.json"), + **kwargs): + """Run an integration test on TWO QNX QEMU VMs sharing an ivshmem region. + + Mirrors `integration_test` but wires the `dual_qemu` plugin (which provides the + `target_a` / `target_b` fixtures) instead of the single-VM `qemu_plugin`. Each VM boots + its own IFS image built from `filesystem_a` / `filesystem_b`. When `filesystem_b` is not + specified both VMs share the same image. QNX-only. + + Args: + name: test target name. + srcs: pytest source files. + filesystem_a: a `pkg_*` target installed into VM-A's IFS image. + filesystem_b: a `pkg_*` target installed into VM-B's IFS image. Defaults to + ``filesystem_a`` (both VMs boot the same image). + dual_config: JSON config describing the two VMs and the ivshmem region. + **kwargs: forwarded to `py_itf_test`. + """ + if filesystem_b == None: + filesystem_b = filesystem_a + + QNX_TARGET_COMPATIBLE_WITH = select({ + "@platforms//cpu:x86_64": ["@platforms//cpu:x86_64"], + "@platforms//cpu:arm64": ["@platforms//cpu:arm64"], + }) + [ + "@platforms//os:qnx", + ] + + qemu_image_a = "_init_ifs_{}_a".format(name) + qnx_ifs( + name = qemu_image_a, + out = "init_ifs_{}_a".format(name), + build_file = "//quality/integration_testing/environments/qnx8_qemu:init_build", + srcs = [filesystem_a, "//quality/integration_testing/environments/qnx8_qemu:qnx_config"], + target_compatible_with = QNX_TARGET_COMPATIBLE_WITH, + ) + + qemu_image_b = "_init_ifs_{}_b".format(name) + qnx_ifs( + name = qemu_image_b, + out = "init_ifs_{}_b".format(name), + build_file = "//quality/integration_testing/environments/qnx8_qemu:init_build", + srcs = [filesystem_b, "//quality/integration_testing/environments/qnx8_qemu:qnx_config"], + target_compatible_with = QNX_TARGET_COMPATIBLE_WITH, + ) + + _extend_list_in_kwargs(kwargs, "data", [qemu_image_a, qemu_image_b, dual_config]) + _extend_list_in_kwargs( + kwargs, + "args", + [ + "--log-cli-level=DEBUG", + "--dual-qemu-config=$(location {})".format(dual_config), + "--qemu-image-a=$(location {})".format(qemu_image_a), + "--qemu-image-b=$(location {})".format(qemu_image_b), + ], + ) + + # Two VMs require even more resources than a single one. + if "size" not in kwargs: + kwargs["size"] = "enormous" + if "timeout" not in kwargs: + kwargs["timeout"] = "moderate" + + # Driving two real QNX guests under KVM has rare, environment-induced boot + # nondeterminism (e.g. a guest occasionally wedging during device bring-up). + # The fixtures already stagger boots and wait for stable SSH; mark the test + # flaky so bazel transparently retries such infrastructure hiccups. + if "flaky" not in kwargs: + kwargs["flaky"] = True + + _extend_list_in_kwargs_without_duplicates( + kwargs, + "target_compatible_with", + ["@score_cpp_policies//sanitizer/constraints:no_tsan"], + ) + _extend_list_in_kwargs(kwargs, "target_compatible_with", QNX_TARGET_COMPATIBLE_WITH) + + py_itf_test( + name = name, + srcs = srcs, + plugins = [ + "//quality/integration_testing/environments/dual_qemu:dual_qemu_plugin", + ], + **kwargs + )