feat(itf): add dual-QEMU ivshmem integration-test environment

MODULE.bazel

@@ -165,6 +165,12 @@ use_repo(
 )
 
 bazel_dep(name = "score_itf", version = "0.5.0", dev_dependency = True)
+git_override(
+    module_name = "score_itf",
+    commit = "54cab4f6ebc1fb40d0b93a8c1a451a03bda77b6a",
+    remote = "https://github.com/eclipse-score/itf.git",
+)
+
 bazel_dep(name = "score_rules_imagefs", version = "0.0.3", dev_dependency = True)
 
 imagefs = use_extension("@score_rules_imagefs//extensions:imagefs.bzl", "imagefs", dev_dependency = True)

quality/integration_testing/environments/dual_qemu/BUILD

@@ -0,0 +1,42 @@
+# *******************************************************************************
+# Copyright (c) 2026 Contributors to the Eclipse Foundation
+#
+# See the NOTICE file(s) distributed with this work for additional
+# information regarding copyright ownership.
+#
+# This program and the accompanying materials are made available under the
+# terms of the Apache License Version 2.0 which is available at
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# SPDX-License-Identifier: Apache-2.0
+# *******************************************************************************
+load("@rules_python//python:defs.bzl", "py_library")
+load("@score_itf//bazel:py_itf_plugin.bzl", "py_itf_plugin")
+
+exports_files(["dual_qemu_config.example.json"])
+
+py_library(
+    name = "dual_qemu_lib",
+    srcs = [
+        "__init__.py",
+        "config.py",
+        "dual_qemu_process.py",
+        "ivshmem_qemu.py",
+    ],
+    imports = [".."],
+    deps = [
+        "@score_itf//score/itf/core/process",
+        "@score_itf//score/itf/core/utils",
+        "@score_itf//score/itf/plugins/qemu",
+    ],
+)
+
+# ---- ITF plugin target (used by the py_itf_test symbolic macro) ----
+py_itf_plugin(
+    name = "dual_qemu_plugin",
+    enabled_plugins = [
+        "dual_qemu",
+    ],
+    py_library = ":dual_qemu_lib",
+    visibility = ["//:__subpackages__"],
+)

quality/integration_testing/environments/dual_qemu/README.md

@@ -0,0 +1,97 @@
+# `dual_qemu` ITF plugin
+
+A pytest/ITF plugin that boots **two** QNX QEMU VMs which share a single QEMU
+[`ivshmem`](https://www.qemu.org/docs/master/system/devices/ivshmem.html) region. It is a
+thin extension of the upstream single-VM `qemu` plugin
+(`@score_itf//score/itf/plugins/qemu`).
+
+## What it adds over the upstream `qemu` plugin
+
+| Aspect            | upstream `qemu`        | `dual_qemu`                                            |
+| ----------------- | ---------------------- | ----------------------------------------------------- |
+| Number of VMs     | 1 (`target_init`)      | 2 (`target_a`, `target_b`)                            |
+| Shared memory     | none                   | one `ivshmem-plain` region backed by one host file    |
+| SSH ports         | one                    | distinct host port per VM (e.g. `2222` / `2223`)      |
+| Inter-VM NIC      | n/a                    | optional point-to-point socket NIC (off by default)   |
+
+The shared region uses **`ivshmem-plain`** — a passive shared-memory PCI device with
+**no MSI-X and no doorbell**. Cross-VM notifications travel over a separate
+socket-based control plane, so the data plane only needs plain shared memory.
+
+## Fixtures
+
+| Fixture            | Scope   | Description                                              |
+| ------------------ | ------- | ------------------------------------------------------- |
+| `target_a`         | session | First VM (`QemuTarget`).                                 |
+| `target_b`         | session | Second VM (`QemuTarget`).                                |
+| `ivshmem_backend`  | session | Path of the shared host backing file.                   |
+| `config`           | session | Loaded `DualQemuConfigModel` + `qemu_image`.            |
+
+Both VMs run the upstream `pre_tests_phase` checks (ping / SSH / SFTP) before the tests
+start.
+
+### Boot reliability
+
+Booting two QNX guests **concurrently** under KVM occasionally wedges the second guest
+during device bring-up (it can hang at SMP AP / secondary-CPU startup under the `-kernel`
+boot path, so its `sshd` never comes up and QEMU's SLIRP resets the harness connections).
+To keep the test reliable the plugin:
+
+- boots the VMs **sequentially** — it starts a VM, waits until SSH is *stably* reachable
+  and runs `pre_tests_phase`, and only then starts the next one, so the two guests never
+  initialise their devices at the same time;
+- gives each VM a single core and a **distinct NIC MAC**;
+- the `dual_qemu_integration_test` macro additionally marks the test `flaky = True` so
+  bazel transparently retries the rare residual KVM boot hiccup.
+
+## CLI options
+
+| Option               | Required | Description                                        |
+| -------------------- | -------- | -------------------------------------------------- |
+| `--dual-qemu-config` | yes      | JSON config (see `dual_qemu_config.example.json`). |
+| `--qemu-image`       | yes      | Shared QEMU IFS image booted by both VMs.          |
+
+## Configuration
+
+See [`dual_qemu_config.example.json`](dual_qemu_config.example.json). A config is just
+**two ordinary QEMU configs** (each an upstream `QemuConfigModel`) plus an `ivshmem`
+block:
+
+```json
+{
+    "ivshmem": { "size": "4M" },
+    "intervm_network": { "enabled": false, "host_port": 12345 },
+    "vms": [ { "...": "QemuConfigModel for VM-A" },
+             { "...": "QemuConfigModel for VM-B" } ]
+}
+```
+
+- `ivshmem.size` — size of the shared region (e.g. `4M`).
+- `ivshmem.mem_path` (optional) — explicit host backing file; when empty a temporary file
+  is created for the session and removed on teardown.
+- `intervm_network.enabled` — when `true`, adds a socket NIC where VM-A listens and VM-B
+  connects on `host_port` (for the future socket control plane).
+- Each VM's `ssh_port` and `port_forwarding` must use **distinct host ports**.
+
+## Cross-VM lib-memory example
+
+[`example_xvm/`](example_xvm/README.md) drives the `score::memory::shared` allocator (the
+one LoLa uses) across both VMs over the ivshmem BAR. The one BAR is split into two
+page-aligned lib-memory regions, so the exchange is **bidirectional**: the producer creates
+a forward region holding a `Vector` (the consumer opens it), and the consumer creates a
+separate reverse region holding a `Map` (the producer opens it) — each container's
+`OffsetPtr`s resolve despite each VM mapping the BAR at a different base. See
+[`example_xvm/README.md`](example_xvm/README.md) for the details, the test target, and how
+to run it manually.
+
+## Debugging guest boot
+
+Set `DUAL_QEMU_SERIAL_DIR` to capture each VM's serial console to
+`<dir>/vm<index>_serial.log` (instead of the normal `mon:stdio`). For example:
+
+```bash
+bazel test //path/to/your:dual_qemu_test \
+    --config=qnx_x86_64 --strategy=TestRunner=local \
+    --test_env=DUAL_QEMU_SERIAL_DIR=/tmp/dqdbg
+# then inspect /tmp/dqdbg/vm0_serial.log and /tmp/dqdbg/vm1_serial.log
+```

quality/integration_testing/environments/dual_qemu/__init__.py

@@ -0,0 +1,139 @@
+# *******************************************************************************
+# Copyright (c) 2026 Contributors to the Eclipse Foundation
+#
+# See the NOTICE file(s) distributed with this work for additional
+# information regarding copyright ownership.
+#
+# This program and the accompanying materials are made available under the
+# terms of the Apache License Version 2.0 which is available at
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# SPDX-License-Identifier: Apache-2.0
+# *******************************************************************************
+"""Pytest plugin that boots **two** QNX QEMU VMs sharing a QEMU ``ivshmem`` region.
+
+It is a thin extension of the single-VM ``qemu`` plugin
+(``@score_itf//score/itf/plugins/qemu``): it reuses ``QemuTarget`` / ``pre_tests_phase``
+and only adds (a) a second VM, (b) an ``ivshmem-plain`` device backed by one shared host
+file, and (c) distinct host SSH ports per VM.
+
+Exposed session fixtures:
+    - ``target_a`` / ``target_b`` -- the two booted VMs (``QemuTarget``).
+    - ``ivshmem_backend``         -- path of the shared host backing file.
+"""
+import logging
+import socket
+
+import pytest
+
+from score.itf.core.utils.bunch import Bunch
+
+from .config import load_configuration, parse_size
+from .dual_qemu_process import DualQemuProcess
+
+logger = logging.getLogger(__name__)
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--dual-qemu-config",
+        action="store",
+        required=True,
+        help="Path to a JSON file describing the two VMs and the ivshmem region.",
+    )
+    parser.addoption(
+        "--qemu-image-a",
+        action="store",
+        required=True,
+        help="Path to the QEMU IFS image for VM-A.",
+    )
+    parser.addoption(
+        "--qemu-image-b",
+        action="store",
+        required=True,
+        help="Path to the QEMU IFS image for VM-B.",
+    )
+
+
+@pytest.fixture(scope="session")
+def config(request):
+    return Bunch(
+        dual_config=load_configuration(request.config.getoption("dual_qemu_config")),
+        qemu_images=[
+            request.config.getoption("qemu_image_a"),
+            request.config.getoption("qemu_image_b"),
+        ],
+    )
+
+
+@pytest.fixture(scope="session")
+def ivshmem_backend(config, tmp_path_factory):
+    """Create the single host backing file both VMs map as their ivshmem region."""
+    ivshmem = config.dual_config.ivshmem
+    size_bytes = parse_size(ivshmem.size)
+
+    if ivshmem.mem_path:
+        path = ivshmem.mem_path
+    else:
+        path = str(tmp_path_factory.mktemp("ivshmem") / "score_ivshmem")
+
+    logger.info(f"Creating ivshmem backing file {path} ({ivshmem.size})")
+    with open(path, "wb") as backing_file:
+        backing_file.truncate(size_bytes)
+
+    yield path
+
+
+@pytest.fixture(scope="session")
+def _targets(config, ivshmem_backend):
+    """Boot both VMs sequentially, verify them, and tear down in reverse order."""
+    logger.info(f"Starting dual-VM tests on host: {socket.gethostname()}")
+    dual_config = config.dual_config
+
+    # When the inter-VM network is enabled, VM-A hosts the socket and VM-B connects.
+    intervm = dual_config.intervm_network
+    intervm_roles = [None, None]
+    if intervm.enabled:
+        intervm_roles = [("listen", intervm.host_port), ("connect", intervm.host_port)]
+
+    vms = dual_config.vms
+
+    # Boot VM-A first, then VM-B. Sequential booting avoids a KVM race where two QNX
+    # guests initializing concurrently can wedge the second guest's device bring-up.
+    with DualQemuProcess(
+        config.qemu_images[0],
+        vms[0].qemu_ram_size,
+        vms[0].qemu_num_cores,
+        vm_config=vms[0],
+        port_forwarding=vms[0].port_forwarding,
+        ivshmem_path=ivshmem_backend,
+        ivshmem_size=dual_config.ivshmem.size,
+        intervm=intervm_roles[0],
+        vm_index=0,
+    ) as process_a:
+        with DualQemuProcess(
+            config.qemu_images[1],
+            vms[1].qemu_ram_size,
+            vms[1].qemu_num_cores,
+            vm_config=vms[1],
+            port_forwarding=vms[1].port_forwarding,
+            ivshmem_path=ivshmem_backend,
+            ivshmem_size=dual_config.ivshmem.size,
+            intervm=intervm_roles[1],
+            vm_index=1,
+        ) as process_b:
+            # Re-verify VM-A is still responsive (it may have gone quiet while VM-B booted).
+            process_a.ensure_responsive()
+            yield [process_a.target, process_b.target]
+
+
+@pytest.fixture(scope="session")
+def target_a(_targets):
+    """The first VM (VM-A) in the inter-VM shared-memory tests."""
+    return _targets[0]
+
+
+@pytest.fixture(scope="session")
+def target_b(_targets):
+    """The second VM (VM-B) in the inter-VM shared-memory tests."""
+    return _targets[1]

quality/integration_testing/environments/dual_qemu/config.py

@@ -0,0 +1,69 @@
+# *******************************************************************************
+# Copyright (c) 2026 Contributors to the Eclipse Foundation
+#
+# See the NOTICE file(s) distributed with this work for additional
+# information regarding copyright ownership.
+#
+# This program and the accompanying materials are made available under the
+# terms of the Apache License Version 2.0 which is available at
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# SPDX-License-Identifier: Apache-2.0
+# *******************************************************************************
+"""Configuration model for the dual-QEMU (ivshmem) plugin.
+
+Each VM is described by the ``QemuConfigModel``, so a dual config is just two
+ordinary QEMU configs plus a shared ``ivshmem`` block.
+"""
+import json
+import logging
+
+from pydantic import BaseModel, ConfigDict, Field, ValidationError
+
+from score.itf.plugins.qemu.config import QemuConfigModel
+
+logger = logging.getLogger(__name__)
+
+_SIZE_UNITS = {"K": 1024, "M": 1024**2, "G": 1024**3, "T": 1024**4, "P": 1024**5}
+
+
+def parse_size(size: str) -> int:
+    """Convert a QEMU-style size string such as ``"4M"`` into a byte count."""
+    return int(size[:-1]) * _SIZE_UNITS[size[-1]]
+
+
+class IvshmemConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    size: str = Field(default="4M", pattern=r"^[0-9]+[KMGTP]$")
+    # Optional explicit host backing file. When empty, a temporary file is created
+    # by the ``ivshmem_backend`` fixture and removed on session teardown.
+    mem_path: str = ""
+
+
+class InterVmNetwork(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    # Optional point-to-point socket NIC between the two VMs for a socket-based control
+    # plane. Off by default
+    enabled: bool = False
+    host_port: int = Field(default=12345, ge=1, le=65535)
+
+
+class DualQemuConfigModel(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    ivshmem: IvshmemConfig = Field(default_factory=IvshmemConfig)
+    intervm_network: InterVmNetwork = Field(default_factory=InterVmNetwork)
+    vms: list[QemuConfigModel] = Field(min_length=2, max_length=2)
+
+
+def load_configuration(config_file: str) -> DualQemuConfigModel:
+    logger.info(f"Loading dual-QEMU configuration from {config_file}")
+    with open(config_file, "r", encoding="utf-8") as handle:
+        raw = json.load(handle)
+    try:
+        return DualQemuConfigModel(**raw)
+    except ValidationError as error:
+        logger.error(f"Invalid dual-QEMU configuration: {error}")
+        raise