Add custom non-default route support

Sources/Containerization/ContainerManager.swift

@@ -94,6 +94,7 @@ public struct ContainerManager: Sendable {
             public let address: String
             public let gateway: String?
             public let macAddress: String?
+            public let routes: [Route]
 
             // `reference` isn't used concurrently.
             nonisolated(unsafe) private let reference: vmnet_network_ref
@@ -102,12 +103,14 @@ public struct ContainerManager: Sendable {
                 reference: vmnet_network_ref,
                 address: String,
                 gateway: String,
-                macAddress: String? = nil
+                macAddress: String? = nil,
+                routes: [Route] = []
             ) {
                 self.address = address
                 self.gateway = gateway
                 self.macAddress = macAddress
                 self.reference = reference
+                self.routes = routes
             }
 
             /// Returns the underlying `VZVirtioNetworkDeviceConfiguration`.

Sources/Containerization/Interface.swift

@@ -14,6 +14,19 @@
 // limitations under the License.
 //===----------------------------------------------------------------------===//
 
+/// A custom route for an interface.
+public struct Route: Sendable {
+    /// Destination network in CIDR notation (e.g., "192.168.1.0/24")
+    public var destination: String
+
+    /// Gateway IP address for this route
+    public var gateway: String
+
+    public init(destination: String, gateway: String) {
+        self.destination = destination
+        self.gateway = gateway
+    }
+}
 /// A network interface.
 public protocol Interface: Sendable {
     /// The interface IPv4 address and subnet prefix length, as a CIDR address.
@@ -25,4 +38,7 @@ public protocol Interface: Sendable {
 
     /// The interface MAC address, or nil to auto-configure the address.
     var macAddress: String? { get }
+
+    /// Custom routes to configure for this interface.
+    var routes: [Route] { get }
 }

Sources/Containerization/LinuxContainer.swift

@@ -348,6 +348,15 @@ extension LinuxContainer {
                         if let gateway = i.gateway {
                             try await agent.routeAddDefault(name: name, gateway: gateway)
                         }
+
+                        for route in i.routes {
+                            try await agent.routeAddLink(
+                                name: name,
+                                address: route.destination,
+                                gateway: route.gateway,
+                                srcAddr: ""
+                            )
+                        }
                     }
 
                     // Setup /etc/resolv.conf and /etc/hosts if asked for.

Sources/Containerization/LinuxPod.swift

@@ -339,6 +339,15 @@ extension LinuxPod {
                         if let gateway = i.gateway {
                             try await agent.routeAddDefault(name: name, gateway: gateway)
                         }
+
+                        for route in i.routes {
+                            try await agent.routeAddLink(
+                                name: name,
+                                address: route.destination,
+                                gateway: route.gateway,
+                                srcAddr: ""
+                            )
+                        }
                     }
 
                     // Setup /etc/resolv.conf if asked for

Sources/Containerization/NATInterface.swift

@@ -18,10 +18,12 @@ public struct NATInterface: Interface {
     public var address: String
     public var gateway: String?
     public var macAddress: String?
+    public var routes: [Route]
 
-    public init(address: String, gateway: String?, macAddress: String? = nil) {
+    public init(address: String, gateway: String?, macAddress: String? = nil, routes: [Route] = []) {
         self.address = address
         self.gateway = gateway
         self.macAddress = macAddress
+        self.routes = routes
     }
 }

Sources/Containerization/NATNetworkInterface.swift

@@ -29,6 +29,7 @@ public final class NATNetworkInterface: Interface, Sendable {
     public let address: String
     public let gateway: String?
     public let macAddress: String?
+    public nonisolated(unsafe) var routes: [Route]
 
     @available(macOS 26, *)
     // `reference` isn't used concurrently.
@@ -39,24 +40,28 @@ public final class NATNetworkInterface: Interface, Sendable {
         address: String,
         gateway: String?,
         reference: sending vmnet_network_ref,
-        macAddress: String? = nil
+        macAddress: String? = nil,
+        routes: [Route] = []
     ) {
         self.address = address
         self.gateway = gateway
         self.macAddress = macAddress
         self.reference = reference
+        self.routes = routes
     }
 
     @available(macOS, obsoleted: 26, message: "Use init(address:gateway:reference:macAddress:) instead")
     public init(
         address: String,
         gateway: String?,
-        macAddress: String? = nil
+        macAddress: String? = nil,
+        routes: [Route] = []
     ) {
         self.address = address
         self.gateway = gateway
         self.macAddress = macAddress
         self.reference = nil
+        self.routes = routes
     }
 }
 

Sources/Containerization/SandboxContext/SandboxContext.pb.swift

@@ -816,9 +816,20 @@ public struct Com_Apple_Containerization_Sandbox_V3_IpRouteAddLinkRequest: Senda
 
   public var srcAddr: String = String()
 
+  public var gateway: String {
+    get {return _gateway ?? String()}
+    set {_gateway = newValue}
+  }
+  /// Returns true if `gateway` has been explicitly set.
+  public var hasGateway: Bool {return self._gateway != nil}
+  /// Clears the value of `gateway`. Subsequent reads from it will return its default value.
+  public mutating func clearGateway() {self._gateway = nil}
+
   public var unknownFields = SwiftProtobuf.UnknownStorage()
 
   public init() {}
+
+  fileprivate var _gateway: String? = nil
 }
 
 public struct Com_Apple_Containerization_Sandbox_V3_IpRouteAddLinkResponse: Sendable {
@@ -2679,6 +2690,7 @@ extension Com_Apple_Containerization_Sandbox_V3_IpRouteAddLinkRequest: SwiftProt
     1: .same(proto: "interface"),
     2: .same(proto: "address"),
     3: .same(proto: "srcAddr"),
+    4: .same(proto: "gateway"),
   ]
 
   public mutating func decodeMessage<D: SwiftProtobuf.Decoder>(decoder: inout D) throws {
@@ -2690,12 +2702,17 @@ extension Com_Apple_Containerization_Sandbox_V3_IpRouteAddLinkRequest: SwiftProt
       case 1: try { try decoder.decodeSingularStringField(value: &self.interface) }()
       case 2: try { try decoder.decodeSingularStringField(value: &self.address) }()
       case 3: try { try decoder.decodeSingularStringField(value: &self.srcAddr) }()
+      case 4: try { try decoder.decodeSingularStringField(value: &self._gateway) }()
       default: break
       }
     }
   }
 
   public func traverse<V: SwiftProtobuf.Visitor>(visitor: inout V) throws {
+    // The use of inline closures is to circumvent an issue where the compiler
+    // allocates stack space for every if/case branch local when no optimizations
+    // are enabled. https://github.com/apple/swift-protobuf/issues/1034 and
+    // https://github.com/apple/swift-protobuf/issues/1182
     if !self.interface.isEmpty {
       try visitor.visitSingularStringField(value: self.interface, fieldNumber: 1)
     }
@@ -2705,13 +2722,17 @@ extension Com_Apple_Containerization_Sandbox_V3_IpRouteAddLinkRequest: SwiftProt
     if !self.srcAddr.isEmpty {
       try visitor.visitSingularStringField(value: self.srcAddr, fieldNumber: 3)
     }
+    try { if let v = self._gateway {
+      try visitor.visitSingularStringField(value: v, fieldNumber: 4)
+    } }()
     try unknownFields.traverse(visitor: &visitor)
   }
 
   public static func ==(lhs: Com_Apple_Containerization_Sandbox_V3_IpRouteAddLinkRequest, rhs: Com_Apple_Containerization_Sandbox_V3_IpRouteAddLinkRequest) -> Bool {
     if lhs.interface != rhs.interface {return false}
     if lhs.address != rhs.address {return false}
     if lhs.srcAddr != rhs.srcAddr {return false}
+    if lhs._gateway != rhs._gateway {return false}
     if lhs.unknownFields != rhs.unknownFields {return false}
     return true
   }

Sources/Containerization/SandboxContext/SandboxContext.proto

@@ -243,6 +243,7 @@ message IpRouteAddLinkRequest {
   string interface = 1;
   string address = 2;
   string srcAddr = 3;
+  optional string gateway = 4;
 }
 
 message IpRouteAddLinkResponse {}

Sources/Containerization/VirtualMachineAgent.swift

@@ -66,6 +66,7 @@ public protocol VirtualMachineAgent: Sendable {
     func down(name: String) async throws
     func addressAdd(name: String, address: String) async throws
     func routeAddDefault(name: String, gateway: String) async throws
+    func routeAddLink(name: String, address: String, gateway: String, srcAddr: String) async throws
     func configureDNS(config: DNS, location: String) async throws
     func configureHosts(config: Hosts, location: String) async throws
 
@@ -89,4 +90,7 @@ extension VirtualMachineAgent {
     public func containerStatistics(containerIDs: [String]) async throws -> [ContainerStatistics] {
         throw ContainerizationError(.unsupported, message: "containerStatistics")
     }
+    public func routeAddLink(name: String, address: String, gateway: String, srcAddr: String) async throws {
+        throw ContainerizationError(.unsupported, message: "routeAddLink")
+    }
 }

Sources/Containerization/Vminitd.swift

@@ -382,6 +382,17 @@ extension Vminitd {
             })
     }
 
+    public func routeAddLink(name: String, address: String, gateway: String, srcAddr: String = "") async throws {
+        _ = try await client.ipRouteAddLink(
+            .with {
+                $0.interface = name
+                $0.address = address
+                $0.gateway = gateway
+                $0.srcAddr = srcAddr
+            }
+        )
+    }
+
     /// Configure DNS within the sandbox's environment.
     public func configureDNS(config: DNS, location: String) async throws {
         _ = try await client.configureDns(

Sources/ContainerizationNetlink/NetlinkSession.swift

@@ -276,7 +276,8 @@ public struct NetlinkSession {
     public func routeAdd(
         interface: String,
         destinationAddress: String,
-        srcAddr: String
+        srcAddr: String,
+        gateway: String,
     ) throws {
         // ip route add [dest-cidr] dev [interface] src [src-addr] proto kernel
         let parsed = try parseCIDR(cidr: destinationAddress)
@@ -285,9 +286,12 @@ public struct NetlinkSession {
         let dstAddrAttrSize = RTAttribute.size + dstAddrBytes.count
         let srcAddrBytes = try IPv4Address(srcAddr).networkBytes
         let srcAddrAttrSize = RTAttribute.size + srcAddrBytes.count
+        let hasGateway = !gateway.isEmpty
+        let gatewayAddrBytes = hasGateway ? try IPv4Address(gateway).networkBytes : []
+        let gatewayAddrAttrSize = hasGateway ? RTAttribute.size + gatewayAddrBytes.count : 0
         let interfaceAttrSize = RTAttribute.size + MemoryLayout<UInt32>.size
         let requestSize =
-            NetlinkMessageHeader.size + RouteInfo.size + dstAddrAttrSize + srcAddrAttrSize + interfaceAttrSize
+            NetlinkMessageHeader.size + RouteInfo.size + dstAddrAttrSize + srcAddrAttrSize + interfaceAttrSize + gatewayAddrAttrSize
         var requestBuffer = [UInt8](repeating: 0, count: requestSize)
         var requestOffset = 0
 
@@ -306,7 +310,7 @@ public struct NetlinkSession {
             tos: 0,
             table: RouteTable.MAIN,
             proto: RouteProtocol.KERNEL,
-            scope: RouteScope.LINK,
+            scope: hasGateway ? RouteScope.UNIVERSE : RouteScope.LINK,
             type: RouteType.UNICAST,
             flags: 0)
         requestOffset = try requestInfo.appendBuffer(&requestBuffer, offset: requestOffset)
@@ -326,14 +330,23 @@ public struct NetlinkSession {
         let interfaceAttr = RTAttribute(len: UInt16(interfaceAttrSize), type: RouteAttributeType.OIF)
         requestOffset = try interfaceAttr.appendBuffer(&requestBuffer, offset: requestOffset)
         guard
-            let requestOffset = requestBuffer.copyIn(
+            var requestOffset = requestBuffer.copyIn(
                 as: UInt32.self,
                 value: UInt32(interfaceIndex),
                 offset: requestOffset)
         else {
             throw NetlinkDataError.sendMarshalFailure
         }
 
+        if hasGateway {
+            let gatewayAttr = RTAttribute(len: UInt16(gatewayAddrAttrSize), type: RouteAttributeType.GATEWAY)
+            requestOffset = try gatewayAttr.appendBuffer(&requestBuffer, offset: requestOffset)
+            guard let newOffset = requestBuffer.copyIn(buffer: gatewayAddrBytes, offset: requestOffset) else {
+                throw NetlinkDataError.sendMarshalFailure
+            }
+            requestOffset = newOffset
+        }
+
         guard requestOffset == requestSize else {
             throw Error.unexpectedOffset(offset: requestOffset, size: requestSize)
         }

Tests/ContainerizationNetlinkTests/NetlinkSessionTest.swift

@@ -322,7 +322,8 @@ struct NetlinkSessionTest {
         try session.routeAdd(
             interface: "eth0",
             destinationAddress: "192.168.64.0/24",
-            srcAddr: "192.168.64.3"
+            srcAddr: "192.168.64.3",
+            gateway: ""
         )
 
         #expect(mockSocket.requests.count == 2)

vminitd/Sources/vminitd/Server+GRPC.swift

@@ -789,6 +789,7 @@ extension Initd: Com_Apple_Containerization_Sandbox_V3_SandboxContextAsyncProvid
                 "interface": "\(request.interface)",
                 "address": "\(request.address)",
                 "srcAddr": "\(request.srcAddr)",
+                "gateway": "\(request.gateway)",
             ])
 
         do {
@@ -797,7 +798,8 @@ extension Initd: Com_Apple_Containerization_Sandbox_V3_SandboxContextAsyncProvid
             try session.routeAdd(
                 interface: request.interface,
                 destinationAddress: request.address,
-                srcAddr: request.srcAddr
+                srcAddr: request.srcAddr,
+                gateway: request.gateway ?? "",
             )
         } catch {
             log.error(