feat(cdp): add Network.setBlockedURLs and --block-urls request blocking

src/Config.zig

@@ -107,6 +107,7 @@ const CommonOptions = .{
     .{ .name = "user_agent", .type = ?[]const u8 },
     .{ .name = "block_private_networks", .type = bool },
     .{ .name = "block_cidrs", .type = ?[]const u8 },
+    .{ .name = "block_urls", .type = ?[]const u8 },
     .{ .name = "cookie", .type = ?[]const u8 },
     .{ .name = "cookie_jar", .type = ?[]const u8 },
     .{ .name = "storage_engine", .type = ?Storage.EngineType },
@@ -575,6 +576,14 @@ pub fn blockCidrs(self: *const Config) ?[]const u8 {
     };
 }
 
+pub fn blockedUrlPatterns(self: *const Config) ?std.mem.SplitIterator(u8, .scalar) {
+    const patterns = switch (self.mode) {
+        inline .serve, .fetch, .mcp, .agent => |opts| opts.block_urls,
+        else => unreachable,
+    } orelse return null;
+    return std.mem.splitScalar(u8, patterns, ',');
+}
+
 pub fn maxConnections(self: *const Config) u16 {
     return switch (self.mode) {
         .serve => |opts| opts.cdp_max_connections,
@@ -838,6 +847,18 @@ pub fn parseArgs(allocator: Allocator) !Config {
     return config;
 }
 
+test "Config: blockedUrlPatterns splits comma-separated patterns" {
+    var config = try Config.init(std.testing.allocator, "test", .{ .serve = .{
+        .block_urls = "*doubleclick*,*://*/*.png",
+    } });
+    defer config.deinit(std.testing.allocator);
+
+    var patterns = config.blockedUrlPatterns().?;
+    try std.testing.expectEqualStrings("*doubleclick*", patterns.next().?);
+    try std.testing.expectEqualStrings("*://*/*.png", patterns.next().?);
+    try std.testing.expectEqual(null, patterns.next());
+}
+
 pub fn validateUserAgent(ua: []const u8) !void {
     for (ua) |c| {
         if (!std.ascii.isPrint(c)) {

src/Notification.zig

@@ -250,8 +250,13 @@ pub const RequestDone = struct {
 };
 
 pub const RequestFail = struct {
+    pub const BlockedReason = enum {
+        inspector,
+    };
+
     transfer: *Transfer,
     err: anyerror,
+    blocked_reason: ?BlockedReason = null,
 };
 
 pub const RequestServedFromCache = struct {

src/cdp/domains/network.zig

@@ -28,8 +28,9 @@ const Mime = @import("../../browser/Mime.zig");
 const Notification = @import("../../Notification.zig");
 const timestamp = @import("../../datetime.zig").timestamp;
 
-const Headers = @import("../../network/HttpClient.zig").Headers;
-const Transfer = @import("../../network/HttpClient.zig").Transfer;
+const HttpClient = @import("../../network/HttpClient.zig");
+const Headers = HttpClient.Headers;
+const Transfer = HttpClient.Transfer;
 
 const CdpStorage = @import("storage.zig");
 
@@ -41,6 +42,7 @@ pub fn processMessage(cmd: *CDP.Command) !void {
         enable,
         disable,
         setCacheDisabled,
+        setBlockedURLs,
         setExtraHTTPHeaders,
         setUserAgentOverride,
         deleteCookies,
@@ -58,6 +60,7 @@ pub fn processMessage(cmd: *CDP.Command) !void {
         .enable => return enable(cmd),
         .disable => return disable(cmd),
         .setCacheDisabled => return setCacheDisabled(cmd),
+        .setBlockedURLs => return setBlockedURLs(cmd),
         .setUserAgentOverride => return @import("emulation.zig").setUserAgentOverride(cmd),
         .setExtraHTTPHeaders => return setExtraHTTPHeaders(cmd),
         .deleteCookies => return deleteCookies(cmd),
@@ -95,6 +98,16 @@ fn setCacheDisabled(cmd: *CDP.Command) !void {
     return cmd.sendResult(null, .{});
 }
 
+fn setBlockedURLs(cmd: *CDP.Command) !void {
+    const params = (try cmd.params(struct {
+        urlPatterns: []const HttpClient.BlockPattern,
+    })) orelse return error.InvalidParams;
+
+    const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
+    try bc.cdp.browser.http_client.setBlockedUrlPatterns(params.urlPatterns);
+    return cmd.sendResult(null, .{});
+}
+
 fn setExtraHTTPHeaders(cmd: *CDP.Command) !void {
     const params = (try cmd.params(struct {
         headers: std.json.ArrayHashMap([]const u8),
@@ -313,6 +326,7 @@ pub fn httpRequestFail(bc: *CDP.BrowserContext, msg: *const Notification.Request
         .type = "Ping",
         .errorText = msg.err,
         .canceled = false,
+        .blockedReason = msg.blocked_reason,
     }, .{ .session_id = session_id });
 }
 
@@ -947,3 +961,89 @@ test "cdp.Network: setCacheDisabled" {
     });
     try ctx.expectSentResult(null, .{ .id = 1 });
 }
+
+test "cdp.Network: setBlockedURLs blocks requests with inspector reason" {
+    const filter: testing.LogFilter = .init(&.{.http});
+    defer filter.deinit();
+
+    var ctx = try testing.context();
+    defer ctx.deinit();
+
+    const bc = try ctx.loadBrowserContext(.{
+        .id = "BID-BLOCK",
+        .session_id = "SID-BLOCK",
+    });
+    const page = try bc.session.createPage();
+    const client = &bc.cdp.browser.http_client;
+    defer client.setBlockedUrls(&.{}) catch unreachable;
+
+    try ctx.processMessage(.{
+        .id = 1,
+        .method = "Network.enable",
+    });
+    try ctx.processMessage(.{
+        .id = 2,
+        .method = "Network.setBlockedURLs",
+        .params = .{ .urlPatterns = &[_]HttpClient.BlockPattern{
+            .{ .urlPattern = "*://blocked.test/*", .block = true },
+        } },
+    });
+    try ctx.expectSentResult(null, .{ .id = 2 });
+
+    const ErrorContext = struct {
+        err: ?anyerror = null,
+
+        fn callback(raw: *anyopaque, err: anyerror) void {
+            const self: *@This() = @ptrCast(@alignCast(raw));
+            self.err = err;
+        }
+    };
+    var error_context: ErrorContext = .{};
+
+    try client.request(.{
+        .frame_id = page.frame_id,
+        .loader_id = 1,
+        .method = .GET,
+        .url = "https://blocked.test/script.js",
+        .cookie_jar = null,
+        .cookie_origin = "https://blocked.test/",
+        .resource_type = .script,
+        .notification = bc.session.notification,
+        .ctx = &error_context,
+        .error_callback = ErrorContext.callback,
+        .shutdown_callback = HttpClient.noopShutdown,
+    }, null);
+
+    try ctx.expectSentEvent("Network.loadingFailed", .{
+        .errorText = error.UrlBlocked,
+        .blockedReason = "inspector",
+    }, .{ .session_id = "SID-BLOCK" });
+    try testing.expectEqual(error.UrlBlocked, error_context.err.?);
+
+    try client.setBlockedUrls(&.{"*redirect-target*"});
+    error_context.err = null;
+
+    var redirect_request_id: [14]u8 = undefined;
+    _ = std.fmt.bufPrint(&redirect_request_id, "REQ-{d:0>10}", .{client.next_request_id +% 1}) catch unreachable;
+
+    try client.request(.{
+        .frame_id = page.frame_id,
+        .loader_id = 1,
+        .method = .GET,
+        .url = "http://127.0.0.1:9582/redirect-no-fragment",
+        .cookie_jar = null,
+        .cookie_origin = "http://127.0.0.1:9582/",
+        .resource_type = .script,
+        .notification = bc.session.notification,
+        .ctx = &error_context,
+        .error_callback = ErrorContext.callback,
+        .shutdown_callback = HttpClient.noopShutdown,
+    }, null);
+
+    try ctx.expectSentEvent("Network.loadingFailed", .{
+        .requestId = &redirect_request_id,
+        .errorText = error.UrlBlocked,
+        .blockedReason = "inspector",
+    }, .{ .session_id = "SID-BLOCK" });
+    try testing.expectEqual(error.UrlBlocked, error_context.err.?);
+}

src/help.zon

@@ -301,6 +301,10 @@
     \\          Prefix with '-' to allow (exempt from blocking).
     \\          e.g. --block-cidrs 10.0.0.0/8,-10.0.0.42/32
     \\          Can be combined with --block-private-networks.
+    \\  --block-urls <LIST>
+    \\          URL patterns to block, comma-separated. Patterns are matched
+    \\          case-insensitively against the full URL and '*' is a wildcard.
+    \\          e.g. --block-urls "*doubleclick*,*://*/*.png"
     \\  --block-private-networks
     \\          Block HTTP requests to private/internal IP addresses after DNS
     \\          resolution.

src/network/HttpClient.zig

@@ -35,6 +35,8 @@ const http = @import("http.zig");
 const Network = @import("Network.zig");
 const Cache = @import("cache/Cache.zig");
 const RobotsGate = @import("RobotsGate.zig");
+const UrlBlocklist = @import("UrlBlocklist.zig");
+pub const BlockPattern = UrlBlocklist.Pattern;
 
 const log = lp.log;
 const Allocator = std.mem.Allocator;
@@ -186,13 +188,29 @@ serve_mode: bool,
 obey_robots: bool,
 
 robots: RobotsGate,
+url_blocklist: ?UrlBlocklist,
 
 pub fn init(self: *Client, allocator: Allocator, network: *Network, cdp: ?*CDP) !void {
     var handles = try http.Handles.init(network.config);
     errdefer handles.deinit();
 
     const http_proxy = network.config.httpProxy();
 
+    var url_blocklist: ?UrlBlocklist = null;
+    if (network.config.blockedUrlPatterns()) |initial_patterns| {
+        var patterns: std.ArrayList([]const u8) = .empty;
+        defer patterns.deinit(allocator);
+
+        var it = initial_patterns;
+        while (it.next()) |pattern| {
+            if (pattern.len > 0) try patterns.append(allocator, pattern);
+        }
+        if (patterns.items.len > 0) {
+            url_blocklist = try UrlBlocklist.init(allocator, patterns.items);
+        }
+    }
+    errdefer if (url_blocklist) |*blocklist| blocklist.deinit();
+
     self.* = Client{
         .handles = handles,
         .network = network,
@@ -209,6 +227,7 @@ pub fn init(self: *Client, allocator: Allocator, network: *Network, cdp: ?*CDP)
         .serve_mode = network.config.mode == .serve,
         .obey_robots = network.config.obeyRobots(),
         .robots = .{ .allocator = allocator, .network = network },
+        .url_blocklist = url_blocklist,
         .arena_pool = &network.app.arena_pool,
     };
 }
@@ -237,6 +256,7 @@ pub fn deinit(self: *Client) void {
         self.allocator.free(owned);
     }
 
+    self.clearUrlBlocklist();
     self.robots.deinit();
     self.blocking_requests.deinit(self.allocator);
     self.transfers.deinit(self.allocator);
@@ -334,6 +354,42 @@ pub fn changeProxy(self: *Client, proxy: ?[:0]const u8) !void {
     self.use_proxy = self.http_proxy != null;
 }
 
+/// Replace the current URL blocklist. The caller retains ownership of
+/// `patterns`; compiled copies remain valid after the CDP command arena is
+/// released.
+pub fn setBlockedUrls(self: *Client, patterns: []const []const u8) !void {
+    const replacement: ?UrlBlocklist = if (patterns.len == 0)
+        null
+    else
+        try UrlBlocklist.init(self.allocator, patterns);
+
+    self.clearUrlBlocklist();
+    self.url_blocklist = replacement;
+}
+
+pub fn setBlockedUrlPatterns(self: *Client, patterns: []const BlockPattern) !void {
+    const replacement: ?UrlBlocklist = if (patterns.len == 0)
+        null
+    else
+        try UrlBlocklist.initPatterns(self.allocator, patterns);
+
+    self.clearUrlBlocklist();
+    self.url_blocklist = replacement;
+}
+
+fn clearUrlBlocklist(self: *Client) void {
+    if (self.url_blocklist) |*blocklist| {
+        blocklist.deinit();
+        self.url_blocklist = null;
+    }
+}
+
+fn isUrlBlocked(self: *const Client, url: []const u8, internal: bool) bool {
+    if (internal) return false;
+    const blocklist = self.url_blocklist orelse return false;
+    return blocklist.isBlocked(url);
+}
+
 pub fn newHeaders(self: *const Client) !http.Headers {
     const ua_header = self.user_agent_header_override orelse self.network.config.http_headers.user_agent_header;
     return http.Headers.init(ua_header);
@@ -779,6 +835,10 @@ fn pipeline(self: *Client, transfer: *Transfer, from: SubmitFrom) !void {
             continue :sw SubmitFrom.after_intercept;
         },
         .after_intercept => {
+            if (self.isUrlBlocked(transfer.req.url, transfer.req.internal)) {
+                log.warn(.http, "blocked url", .{ .url = transfer.req.url });
+                return transfer.failAsync(error.UrlBlocked);
+            }
             if (try self.cacheLookup(transfer)) {
                 // response came from the cache, we're done
                 return;
@@ -1366,6 +1426,14 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T
                     try transfer.handleRedirect(location.value);
                     if (!transfer.req.internal) lp.metrics.http_redirects.incr();
 
+                    if (self.isUrlBlocked(transfer.req.url, transfer.req.internal)) {
+                        log.warn(.http, "blocked url", .{ .url = transfer.req.url });
+                        self.removeConn(msg.conn);
+                        transfer._conn = null;
+                        transfer.failAsync(error.UrlBlocked);
+                        return true;
+                    }
+
                     const conn = transfer._conn.?;
 
                     try self.handles.remove(conn);
@@ -2159,6 +2227,7 @@ pub const Transfer = struct {
             self.req.notification.dispatch(.http_request_fail, &.{
                 .transfer = self,
                 .err = err,
+                .blocked_reason = if (err == error.UrlBlocked) .inspector else null,
             });
         }
 
@@ -3104,6 +3173,41 @@ fn initTestClient(client: *Client, pool: *ArenaPool) void {
     client.serve_mode = false;
     client.obey_robots = false;
     client.robots = .{ .allocator = testing.allocator, .network = undefined };
+    client.url_blocklist = null;
+}
+
+test "HttpClient: setBlockedUrls owns, replaces, and clears patterns" {
+    var pool = ArenaPool.init(testing.allocator, .{});
+    defer pool.deinit();
+
+    var client: Client = undefined;
+    initTestClient(&client, &pool);
+    defer client.clearUrlBlocklist();
+
+    var first = [_]u8{ '*', 'f', 'i', 'r', 's', 't', '*' };
+    try client.setBlockedUrls(&.{&first});
+    @memset(&first, 'x');
+
+    try testing.expect(client.url_blocklist.?.isBlocked("https://example.test/first.js"));
+    try client.setBlockedUrls(&.{"*second*"});
+    try testing.expect(!client.url_blocklist.?.isBlocked("https://example.test/first.js"));
+    try testing.expect(client.url_blocklist.?.isBlocked("https://example.test/second.js"));
+
+    try client.setBlockedUrls(&.{});
+    try testing.expectEqual(null, client.url_blocklist);
+}
+
+test "HttpClient: URL blocking exempts internal transfers" {
+    var pool = ArenaPool.init(testing.allocator, .{});
+    defer pool.deinit();
+
+    var client: Client = undefined;
+    initTestClient(&client, &pool);
+    defer client.clearUrlBlocklist();
+
+    try client.setBlockedUrls(&.{"*example.test*"});
+    try testing.expect(client.isUrlBlocked("https://example.test/script.js", false));
+    try testing.expect(!client.isUrlBlocked("https://example.test/robots.txt", true));
 }
 
 test "HttpClient: fulfillIntercepted survives a done_callback that tears down the owner" {