diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..d73a937 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,16 @@ +# Keep the build context small and stable. Without this, `COPY . .` pulls in +# .git, so every commit changes the context digest and invalidates the build +# cache even when no source file changed. +.git +.github +.gitea +.golangci.yml +.dockerignore +Dockerfile +docker-compose.yaml +dev +hack +coverage.out +go-runner +*.md +LICENSE diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index 468433d..f11cd9e 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -30,7 +30,8 @@ jobs: - name: Install golangci-lint uses: golangci/golangci-lint-action@v6 with: - version: v2.1.6 + # Must match .github/workflows/style-guide.yml. + version: v2.13.1 args: ./... - name: Run tests with coverage diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..cc9930c --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,16 @@ +# Review is required on every change: this repository decides whether personal +# data is released, so nothing here is too small to be looked at by someone else. +* @wistefan + +# The decision path itself. A change here can turn the gate into a pass-through. +/internal/plugin/ @wistefan +/internal/consent/ @wistefan +/internal/ownerresolver/ @wistefan + +# The audit trail, which is the evidence that the gate worked. +/internal/audit/ @wistefan + +# Security policy, CI gates and the release path. +/SECURITY.md @wistefan +/.github/ @wistefan +/.gitea/ @wistefan diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..67c526a --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,40 @@ +# Dependency updates. +# +# The direct dependencies had drifted years behind (testify 1.8.4 while 1.12.1 +# was current) with nothing to notice. This makes the drift a pull request +# instead of a review finding. +# +# The go-plugin-runner is deliberately grouped on its own: it is pinned at v0.5.0 +# and its transitive tree (zap 1.17, flatbuffers 2.0.0) is old, so an update to +# it is a decision to make deliberately rather than merge with the batch. A +# plugin whose runner is unmaintained is a strategic risk worth surfacing +# regularly. +version: 2 +updates: + - package-ecosystem: gomod + directory: / + schedule: + interval: weekly + open-pull-requests-limit: 5 + groups: + go-dependencies: + patterns: + - "*" + exclude-patterns: + - "github.com/apache/apisix-go-plugin-runner" + labels: + - dependencies + + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + labels: + - dependencies + + - package-ecosystem: docker + directory: / + schedule: + interval: weekly + labels: + - dependencies diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 80e8a7b..ec41049 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -22,7 +22,7 @@ structure: quality gates run on every PR and on `main`, and releases are Each release produces: -- **Container image** — `quay.io/wi_stefan/consent-plugin:` (plus `:latest` +- **Container image** — `quay.io/seamware/consent-plugin:` (plus `:latest` and `:`), multi-arch `linux/amd64,linux/arm64`. **This is the primary deployment artifact**: the APISIX deployment's init container copies `/app/go-runner` out of the image into the `ext-plugin` volume diff --git a/.github/workflows/pre-release.yml b/.github/workflows/pre-release.yml index 6af593c..5b70510 100644 --- a/.github/workflows/pre-release.yml +++ b/.github/workflows/pre-release.yml @@ -20,7 +20,7 @@ concurrency: env: REGISTRY: quay.io - REPOSITORY: wi_stefan + REPOSITORY: seamware IMAGE_NAME: consent-plugin jobs: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7c34478..8431b1d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -21,7 +21,7 @@ on: env: REGISTRY: quay.io - REPOSITORY: wi_stefan + REPOSITORY: seamware IMAGE_NAME: consent-plugin jobs: diff --git a/.github/workflows/security-analysis.yml b/.github/workflows/security-analysis.yml index d249d32..9cccea0 100644 --- a/.github/workflows/security-analysis.yml +++ b/.github/workflows/security-analysis.yml @@ -1,10 +1,37 @@ name: Security Analysis -# Reusable source-level security scanning. Findings are uploaded as SARIF to -# the GitHub Security tab; scans never block the pipeline (continue-on-error). +# Reusable source-level security scanning. +# +# The scans BLOCK the pipeline. They previously ran with continue-on-error, so a +# known-vulnerable dependency merged cleanly and the findings were informational +# only — a security gate that cannot fail is a dashboard, not a gate. SARIF is +# still uploaded to the Security tab either way. +# +# Tool versions are pinned. `@latest` made CI non-reproducible (a scanner release +# could redden an untouched PR) and put an unpinned binary in the build's trust +# boundary. Keep GOSEC_VERSION and the golangci-lint version in .gitea/workflows/ +# and .github/workflows/style-guide.yml in step, so the two pipelines cannot +# disagree about whether the code passes. +# +# A pin still has to be able to BUILD. gosec is installed from source with the +# toolchain from go.mod, so its own dependency tree must compile under that Go +# version: gosec v2.21.4 pinned golang.org/x/tools v0.25.0, which reaches into +# the internal layout of go/token via unsafe and guards it with a deliberate +# compile-time tripwire ("if the size of token.FileSet changes, this will fail to +# compile"). Go 1.26 changed that layout, so the guard fired and the step could +# never build: +# +# x/tools@v0.25.0/internal/tokeninternal/tokeninternal.go:64:9: +# invalid array length -delta * delta (constant -256 of type int64) +# +# Any pre-Go-1.26 x/tools is affected, so a scanner pin must be advanced together +# with the toolchain in go.mod. v2.29.0 builds on golang.org/x/tools v0.49.0. on: workflow_call: +env: + GOSEC_VERSION: v2.29.0 + jobs: govulncheck: runs-on: ubuntu-latest @@ -17,8 +44,12 @@ jobs: with: go-version-file: go.mod + # Verifies the module cache against go.sum before anything is built with it. + - name: Verify module checksums + run: go mod verify + - uses: golang/govulncheck-action@v1.0.4 - continue-on-error: true + id: govulncheck with: go-version-file: go.mod output-format: sarif @@ -40,11 +71,14 @@ jobs: with: go-version-file: go.mod + - name: Install gosec + run: go install github.com/securego/gosec/v2/cmd/gosec@${{ env.GOSEC_VERSION }} + + # -no-fail is deliberately NOT passed: a new finding must fail the PR. + # Suppress a reviewed finding at the call site with a #nosec comment + # carrying the reason, so the exception is visible in the diff. - name: Run gosec - continue-on-error: true - run: | - go install github.com/securego/gosec/v2/cmd/gosec@latest - gosec -fmt sarif -out gosec-results.sarif ./... + run: gosec -fmt sarif -out gosec-results.sarif ./... - uses: github/codeql-action/upload-sarif@v4 if: always() diff --git a/.github/workflows/style-guide.yml b/.github/workflows/style-guide.yml index 9ff0a9c..cfd9b73 100644 --- a/.github/workflows/style-guide.yml +++ b/.github/workflows/style-guide.yml @@ -1,6 +1,10 @@ name: Style Guide # Reusable lint check using the repository's .golangci.yml. +# +# The version is PINNED and must match .gitea/workflows/ci.yaml. With `latest` +# here and a pin there, the two pipelines could disagree about whether the same +# commit lints — and a golangci-lint release could redden an untouched PR. on: workflow_call: @@ -17,4 +21,4 @@ jobs: - uses: golangci/golangci-lint-action@v9 with: - version: latest + version: v2.13.1 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2b870df..5628b3d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -15,12 +15,18 @@ jobs: with: go-version-file: go.mod + # -coverpkg=./... attributes the integration package's coverage of the + # packages it exercises; without it those statements are discarded and the + # reported figure understates reality. - name: Run tests (race + coverage) - run: go test -race -coverprofile=coverage.out ./... + run: go test -race -coverpkg=./... -coverprofile=coverage.out ./... - name: Coverage summary run: go tool cover -func=coverage.out + - name: Enforce the coverage floor + run: make coverage-floor + - uses: actions/upload-artifact@v4 with: name: coverage diff --git a/.gitignore b/.gitignore index 5c689d1..8544e17 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,6 @@ coverage.out # OS files .DS_Store + +# Local review/working notes, not part of the repository. +review.md diff --git a/CLAUDE.md b/CLAUDE.md index a18a113..be4cf8c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,19 +1,27 @@ # consent-plugin ## Overview -An Apache APISIX Go plugin that gates access to personal data on the data -subject's **consent**. It uses the APISIX go-plugin-runner to hook into the -request/response lifecycle: the request phase (`ext-plugin-pre-req`) captures -the JWT `sub`, and the response phase (`ext-plugin-post-resp`) runs a **two-call -check** against a Prometheus-X / Visions consent-manager (resolve the subject's -`userIdentifier`, then list its consents) and allows the response only when a -granted consent exists — otherwise it replaces the response with a configurable -deny. The two phases are correlated by the Nginx `$request_id` (not the runner's -per-RPC `ID()`). The gate is coarse (allow/deny) and independent of the response -body; there is no field-level filtering. +An Apache APISIX Go plugin that gates access to personal data on the **consent +of the data owner**. It uses the APISIX go-plugin-runner to hook into the +request/response lifecycle: the request phase (`ext-plugin-pre-req`) captures the +JWT claims, and the response phase (`ext-plugin-post-resp`) asks an external +**OwnerResolver** who owns the data in the upstream payload, then runs a +**two-call check** against a Prometheus-X / Visions consent-manager per resolved +owner (resolve the owner's `userIdentifier`, then list its consents) and allows +the response only when every owner has a granted consent **for this consuming +participant**. Otherwise the response is replaced with a configurable deny. + +Ownership never comes from the requestor: the token's `sub` says who is asking, +not whose data is returned. `owner_resolver_url` is therefore required. The two +phases are correlated by the Nginx `$request_id` (not the runner's per-RPC +`ID()`). The gate is coarse (allow/deny) and independent of the response body's +shape; there is no field-level filtering. + +The plugin decodes but does **not verify** the JWT — an authentication plugin +earlier in the route is a hard prerequisite. ## Tech Stack -- Language: Go 1.21+ +- Language: Go (see `go.mod` for the pinned version) - Framework: Apache APISIX go-plugin-runner (`github.com/apache/apisix-go-plugin-runner`) - Test: Go standard `testing` package with `testify` for assertions - Build: Makefile + Docker @@ -22,29 +30,46 @@ body; there is no field-level filtering. ``` consent-plugin/ ├── CLAUDE.md # This file — AI agent codebase context -├── IMPLEMENTATION_PLAN.md # Step-by-step implementation plan -├── README.md # Project README +├── README.md # Project README (the config surface is contract) ├── Makefile # Build, test, lint targets ├── Dockerfile # Build the go-runner binary -├── go.mod # Go module definition -├── go.sum # Go dependency checksums +├── go.mod / go.sum # Go module definition and checksums ├── main.go # Entry point — registers plugin, starts runner ├── internal/ │ ├── plugin/ │ │ ├── consent.go # Plugin struct, Name(), ParseConf(), RequestFilter(), ResponseFilter() +│ │ ├── config.go # Configuration schema and validation +│ │ ├── context.go # Bounded request-context store keyed by $request_id │ │ ├── consent_test.go # Unit tests for plugin logic -│ │ └── config.go # Configuration schema struct and validation +│ │ ├── config_test.go # Unit tests for configuration +│ │ ├── config_doc_test.go # Doc-drift guard: README must document every config field +│ │ └── context_test.go # Unit tests for the context store │ ├── consent/ -│ │ ├── client.go # HTTP client for external consent API -│ │ ├── client_test.go # Unit tests for consent client -│ │ └── models.go # Request/response models for consent API +│ │ ├── client.go # Two-call consent-manager client +│ │ ├── client_test.go # Unit tests for the consent client +│ │ └── models.go # Request/response models for the consent check +│ ├── ownerresolver/ +│ │ ├── client.go # OwnerResolver /resolve client (who owns the data) +│ │ └── client_test.go # Unit tests for the resolver client +│ ├── audit/ +│ │ ├── audit.go # OTLP/HTTP access-decision audit exporter +│ │ └── audit_test.go # Unit tests for the audit exporter +│ ├── metrics/ +│ │ ├── metrics.go # Prometheus text-format exporter (decisions, latency, gauges) +│ │ └── metrics_test.go # Unit tests for the exporter +│ ├── logging/ +│ │ ├── logging.go # Leveled logging front end: redaction, sanitisation, rate limiting +│ │ └── logging_test.go # Unit tests for the logging front end │ ├── jwt/ -│ │ ├── extractor.go # JWT extraction and parsing from request headers +│ │ ├── extractor.go # JWT extraction and claim decoding (no verification) │ │ └── extractor_test.go # Unit tests for JWT extraction -│ └── filter/ -│ ├── response.go # JSON response body filtering/redaction logic -│ └── response_test.go # Unit tests for response filtering -└── docker-compose.yaml # Local dev with APISIX + plugin runner +│ └── integration/ +│ └── integration_test.go # End-to-end plugin lifecycle tests +├── dev/ +│ ├── apisix-config.yaml # APISIX config for the local stack (ext-plugin wiring) +│ ├── otel-collector.yaml # Collector config receiving the audit log +│ └── mocks/ # WireMock stubs: consent-manager, OwnerResolver, token service +└── docker-compose.yaml # Local dev with APISIX + plugin runner + mocks ``` ## Build & Test @@ -75,9 +100,39 @@ make docker-build ## Important Files - `main.go` — Entry point; registers the consent plugin and starts the runner. -- `internal/plugin/consent.go` — Core plugin: `RequestFilter` captures context (keyed by `$request_id`), `ResponseFilter` runs the two-call check and allows/denies. -- `internal/plugin/config.go` — Plugin configuration schema (consent-manager URL + prefix, `consent_key`, participant `client_id`/`client_secret` (or a static `participant_token`), optional `provider_sd`, JWT settings, deny behavior, `fail_open`). `consent_key` is **optional** (the authority's facade injects it and overrides anything sent). `consent_key`/`client_id`/`client_secret` fall back to env vars `CONSENT_KEY`/`CONSENT_CLIENT_ID`/`CONSENT_CLIENT_SECRET` (config wins) so the secret stays out of the route config; `applyEnv()` runs in `ParseConfig`. -- `internal/plugin/context.go` — Concurrent request-context store bridging the two phases, keyed by the Nginx `$request_id`. -- `internal/consent/client.go` — Consent-manager client: participant client-credentials login (`/participants/login`, token cached/refreshed) + provider-SD derivation (`/participants/me`), then the two-call check (`/users/identifier/search` + `/consents/participants/{id}`). Token/SD cache is keyed per participant with a per-entry lock, so concurrent first requests coalesce onto one login without a global lock across the HTTP call. -- `internal/audit/audit.go` — Access-decision audit emitter: exports one OTLP/HTTP log record per decision to the OTel Collector (marked `service.name=consent-access-audit` for routing). Async, batched, best-effort (bounded queue drops rather than blocking); gated by `audit_enabled` + `audit_otlp_endpoint`. `ResponseFilter` → `recordAudit` calls it. -- `go.mod` — Module path: `consent-plugin` (or as configured). +- `internal/plugin/consent.go` — Core plugin: `RequestFilter` captures context + (keyed by `$request_id`), `ResponseFilter` resolves the data owners and runs + the per-owner check. `failMode` distinguishes dependency outages (governed by + `fail_open`) from structural failures that always deny. +- `internal/plugin/config.go` — Plugin configuration schema. `owner_resolver_url` + is **required**; `fail_open` defaults to **false**. `consent_key`, + `token_service_url` and `audit_otlp_endpoint` fall back to `CONSENT_KEY`, + `CONSENT_TOKEN_SERVICE_URL` and `CONSENT_AUDIT_OTLP_ENDPOINT` (config wins) so + secrets stay out of the route config; `applyEnv()` runs in `ParseConfig`. +- `internal/plugin/config_doc_test.go` — Fails the build when the README's + configuration table and the `Config` json tags disagree in either direction. +- `internal/plugin/context.go` — Bounded request-context store bridging the two + phases, keyed by the Nginx `$request_id`: TTL, size cap, background sweep, and + size/eviction gauges. It deliberately holds no request headers. +- `internal/ownerresolver/client.go` — Client for the external OwnerResolver + `/resolve` endpoint, which answers from the DATA alone who the owners are and + whether consent is required. `parties` is for contract identification only. +- `internal/consent/client.go` — Consent-manager client: token from the + participant-local OID4VP token service (`token_service_url`, cached/refreshed + per credential identity) + provider-SD derivation (`/participants/me`), the + participant registry (`/participants`) for DID → self-description mapping, and + the two-call check (`/users/identifier/search` + `/consents/participants/{id}`). + A consent counts only if it is granted **to the named consumer** (and covers + the purpose/resource when known). +- `internal/metrics/metrics.go` — Hand-rolled Prometheus exporter (no client + dependency, like the OTLP encoder). Served from `main.go` on + `CONSENT_METRICS_ADDRESS` when set. +- `internal/logging/logging.go` — Logging front end over the runner's zap logger: + `Redact` fingerprints identifiers, `Sanitize` strips error bodies, and the + `*Every` variants rate-limit a repeated failure to one line per interval. + Nothing in the plugin calls `log.Printf` directly. +- `internal/audit/audit.go` — Access-decision audit emitter: one OTLP/HTTP log + record per decision to the OTel Collector (`service.name=consent-access-audit` + for routing). Async, batched, best-effort; gated by `audit_enabled` + + `audit_otlp_endpoint`. `ResponseFilter` → `recordAudit` calls it. +- `go.mod` — Module path: `consent-plugin`. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index db6795f..e29fe4f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,5 +1,34 @@ # Contributing +## Reporting a vulnerability + +Do not open a public issue. See [SECURITY.md](SECURITY.md) — this plugin decides +whether personal data is released, so a defect in it is handled privately first. + +## Local toolchain + +`go.mod` requires **Go 1.26**. If your system Go is older, the build tries to +fetch the toolchain automatically — and in an environment where `GOTOOLCHAIN` +cannot download (an air-gapped machine, a restricted proxy) it fails with +`toolchain not available`, which reads like "Go 1.26 does not exist" rather than +"the download was blocked". + +Either install Go 1.26+ directly, or point at a full patch version already in the +module cache and disable switching: + +```bash +ls -d "$(go env GOMODCACHE)"/golang.org/toolchain@*/ # what is cached +export PATH="$(go env GOMODCACHE)/golang.org/toolchain@v0.0.1-go1.26.7.linux-amd64/bin:$PATH" +export GOTOOLCHAIN=local +``` + +Note the bare major version (`go1.26`) is what fails; the full patch version +(`go1.26.7`) is what the cache holds. + +`golangci-lint` must match the version CI pins — see +`.github/workflows/style-guide.yml` and `.gitea/workflows/ci.yaml`, which are +kept in step with each other. + ## Pull requests - Target the `main` branch. @@ -23,7 +52,7 @@ On merge to `main`, `main.yml` re-runs the gates and calls `release.yml`, which: 1. computes the next version from the label, 2. builds, scans and pushes the multi-arch image to - `quay.io/wi_stefan/consent-plugin`, and + `quay.io/seamware/consent-plugin`, and 3. publishes a GitHub Release with the `go-runner` binaries. While a PR is open, `pre-release.yml` publishes a `…-PRE-` image and a GitHub diff --git a/Dockerfile b/Dockerfile index 0b2cdd9..d658b0e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,30 +1,43 @@ # Multi-stage build for the consent-plugin APISIX go-plugin-runner. -# Stage 1: Compile the Go binary. -# Stage 2: Copy into a minimal runtime image. +# Stage 1: compile the Go binary. +# Stage 2: copy it into a minimal runtime image. # --- Build stage --- FROM golang:1.26-alpine AS builder -RUN apk add --no-cache git - WORKDIR /build # Cache dependency downloads by copying go.mod/go.sum first. COPY go.mod go.sum ./ -RUN go mod download +RUN go mod download && go mod verify -# Copy source code and build the binary. +# Copy source code and build the binary. .dockerignore keeps .git and build +# artifacts out, so an unrelated commit does not invalidate this layer. COPY . . -RUN go build -trimpath -ldflags="-s -w" -o go-runner . +RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o go-runner . # --- Runtime stage --- -FROM alpine:3.19 +FROM alpine:3.22 RUN apk add --no-cache ca-certificates +# The runner has no reason to be root: it binds a unix socket in a directory it +# is given and makes outbound HTTP calls. Running as root only widens what a +# compromise of it reaches. The uid is fixed so a shared socket volume can be +# given predictable ownership. +RUN addgroup -g 10001 -S runner && adduser -u 10001 -S -G runner runner + WORKDIR /app COPY --from=builder /build/go-runner /app/go-runner +USER 10001:10001 + +# The runner speaks the ext-plugin protocol, not HTTP, so there is nothing to +# probe but the listener itself: a bound socket means it is accepting RPCs. A TCP +# listen address is left to the orchestrator to probe. +HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ + CMD sh -c 'case "$APISIX_LISTEN_ADDRESS" in unix:*) test -S "${APISIX_LISTEN_ADDRESS#unix:}" ;; *) exit 0 ;; esac' + # The plugin runner binary is the entrypoint. ENTRYPOINT ["/app/go-runner"] diff --git a/Makefile b/Makefile index d514548..35d6317 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ BINARY_NAME := go-runner # Docker image configuration -DOCKER_IMAGE := quay.io/wi_stefan/consent-plugin +DOCKER_IMAGE := quay.io/seamware/consent-plugin DOCKER_TAG := 0.0.1 # Go build flags @@ -13,7 +13,11 @@ GO_BUILD_FLAGS := -trimpath -ldflags="-s -w" # Coverage output file COVERAGE_FILE := coverage.out -.PHONY: build test test-cover lint license-check license-fix docker-build clean +# Minimum total statement coverage, in percent. CI and `make test-cover` fail +# below it, so a gap cannot reappear unnoticed. +COVERAGE_FLOOR := 80 + +.PHONY: build test test-cover coverage-floor lint license-check license-fix docker-build clean ## build: Compile the go-runner binary build: @@ -23,10 +27,18 @@ build: test: go test -race ./... -## test-cover: Run tests with coverage report +## test-cover: Run tests with coverage report and enforce the floor +# -coverpkg=./... is required: without it the integration package's coverage of +# internal/plugin is discarded, which understated the real figure and made the +# genuinely untested functions look like measurement noise. test-cover: - go test -race -coverprofile=$(COVERAGE_FILE) ./... + go test -race -coverpkg=./... -coverprofile=$(COVERAGE_FILE) ./... go tool cover -func=$(COVERAGE_FILE) + ./hack/coverage-floor.sh $(COVERAGE_FILE) $(COVERAGE_FLOOR) + +## coverage-floor: Assert an existing coverage profile meets COVERAGE_FLOOR +coverage-floor: + ./hack/coverage-floor.sh $(COVERAGE_FILE) $(COVERAGE_FLOOR) ## lint: Run golangci-lint lint: diff --git a/README.md b/README.md index 9dc1bc5..cdcecb2 100644 --- a/README.md +++ b/README.md @@ -6,47 +6,103 @@ An Apache APISIX Go plugin that gates access to personal data on the **consent o The plugin is attached to a route in **both** external-plugin phases: -1. **Request phase** (`ext-plugin-pre-req` → `RequestFilter`): captures the request context — method, path, and the JWT claims (notably `sub`) extracted from the configured header — and stores it keyed by the Nginx `$request_id`. -2. **Response phase** (`ext-plugin-post-resp` → `ResponseFilter`): loads that context and runs a **two-call consent check** against the consent-manager for the request subject: - - **allow** (a granted consent exists) — the response passes through unchanged. - - **deny** (no granted consent, or the subject is unknown) — the response is replaced with a configurable error status and body. +1. **Request phase** (`ext-plugin-pre-req` → `RequestFilter`): captures the request context — method, path, and the JWT claims decoded from the configured header — and stores it keyed by the Nginx `$request_id`. +2. **Response phase** (`ext-plugin-post-resp` → `ResponseFilter`): loads that context, asks the **OwnerResolver** who owns the data in the upstream response, and runs a **two-call consent check** per resolved owner: + - **allow** (every resolved owner has a granted consent for this consumer) — the response passes through unchanged. + - **deny** (any owner has no such consent, or is unknown to the consent-manager) — the response is replaced with a configurable error status and body. -The decision is a coarse allow/deny on the subject's consent and is **independent of the response body**, so an empty or non-JSON personal-data response is still gated. If the consent-manager is unreachable, or the plugin's context/credentials are missing, it applies the configured fail-open (default) or fail-closed policy. +The decision is a coarse allow/deny and is **independent of the response body's shape**, so an empty or non-JSON personal-data response is still gated. + +The payload is described to the resolver in one of three ways — `json` (parsed +and carried), `none` (there was no payload), and `opaque` (there was one but it +could not be parsed, sent with its declared content type and size). The last two +are deliberately distinct: an unreadable personal-data payload must not look to +the resolver like no payload at all. > The `$request_id` correlation is required: `ext-plugin-pre-req` and `ext-plugin-post-resp` are separate RPCs to the runner and do **not** share the runner's per-call `ID()`. +### Ownership comes from the data, never from the requestor + +`owner_resolver_url` is **required**. The access token's `sub` says who is +*asking*, not whose data is being *returned* — checking the caller's own consent +would let any subject holding a single granted consent read everyone else's +data. So the data owner is always derived from the response payload by the +OwnerResolver, and the token's claims are used only to name the **consuming +participant** for the contract lookup and to scope the consent match. + +> **The plugin does not verify the JWT signature.** It decodes the claims and +> assumes an authentication plugin earlier in the route has already validated the +> token. Attaching `consent-filter` to a route with no authentication in front of +> it is a misconfiguration: the consumer identity would be attacker-supplied. + +### Response buffering + +`ext-plugin-post-resp` forces APISIX to **buffer the entire upstream response** +before the runner sees it (`ReadBody()` is a blocking extra-info RPC over the +unix socket). Gated routes therefore do not stream, and a large response is a +memory multiplier across APISIX and the runner. Keep gated routes to bounded +responses, and note that `response_phase_timeout`, `max_owners_per_response` and +`max_resolve_body_bytes` (below) bound how long the response is held, how many +owners are checked, and how large a payload is forwarded to the resolver. + +Per-owner consent checks run concurrently (up to 8 in flight) and short-circuit +on the first denial, so latency is not the sum over owners. + ## The two-call consent check The consent-manager has no single "is there consent?" endpoint, so a check is two calls (`{consent_api_url}{consent_api_prefix}` is the base, e.g. `http://consent-manager:3000/v1`): -**1. Resolve the subject to a user identifier** — authenticated with the consent key: +**1. Resolve the owner to a user identifier** — authenticated with the consent key: ``` POST {base}/users/identifier/search Header: x-visionstrust-consent-key: -Body: { "selfDescription": "", "email": "" } -→ { "userIdentifier": "" } (404 / empty ⇒ unknown subject ⇒ deny) +Body: { "selfDescription": "", "email": "" } +→ { "userIdentifier": "" } (404 / empty ⇒ unknown owner ⇒ deny) ``` **2. List that user's consents** — authenticated with the participant JWT: ``` GET {base}/consents/participants/{userIdentifier}?receipt=true Header: Authorization: Bearer -→ { "consents": [ { "status": "granted" | "revoked" | ... } ] } +→ { "consents": [ { "status": "granted", "consumer": {...}, "purposes": [...], "data": [...] } ] } ``` -Access is **allowed** if at least one returned consent has `status == "granted"`. -The subject DID is taken from the JWT `sub` claim (so `jwt_claims_to_forward` must include `sub`) and sent as the user `email` (the consent-manager's DID-in-email convention). +Access is **allowed** only if a returned consent satisfies all of: + +- `status == "granted"`; +- it was granted to the **consuming participant** identified from the token (a + consent names one consumer; one granted to X is not authority for Y); +- it covers the **purpose**, when the resolver named one for the claim (set + `require_purpose` to deny rather than proceed unscoped when it does not); +- it covers the **data resource**, when the resolver scoped the claim to one. + +The owner DID is sent as the user `email` (the consent-manager's +DID-in-email convention). -### Participant authentication (client credentials) +### Participant authentication -Call 2 needs a **participant JWT**, and call 1 needs the **provider self-description**. Rather than pinning a static (expiring) token and a per-registration SD, the plugin obtains both from **participant client credentials**: +Call 2 needs a **participant JWT**, and call 1 needs the **provider +self-description**. The plugin holds no participant credentials of its own: it +asks the participant-local OID4VP token service (the consent-facade's +`POST /internal/tokens`) for a short-lived token by **audience name**, then +derives the provider SD from `/participants/me`: ``` -POST {base}/participants/login { "clientID": ..., "clientSecret": ... } → { "jwt": ... } (1h token, cached & refreshed) -GET {base}/participants/me Authorization: Bearer → { "selfDescriptionURL": ... } +POST {token_service_url} { "audience": "" } → { "access_token": ..., "expires_in": ... } +GET {base}/participants/me Authorization: Bearer → { "selfDescriptionURL": ... } ``` -So configuring `client_id` + `client_secret` is enough: the token is fetched (and re-fetched on expiry or a 401), and `provider_sd` is derived from `/participants/me` when not set explicitly. Tokens are cached process-wide (keyed by base URL + client id). A static `participant_token` and/or explicit `provider_sd` remain supported as overrides. +The token is cached and refreshed on expiry or a 401. The cache is keyed on the +full credential identity (base URL, host, prefix, audience, token-service URL, +provider SD, and hashes of the static token and consent key), so two routes +fronting different participants against the same consent-manager never share a +token. A static `participant_token` and/or an explicit `provider_sd` remain +supported as overrides for tests and manual runs. + +The consumer DID from the token is translated to a self-description URL via +`GET {base}/participants` (the consent-manager doubles as the participant +registry), cached for 10 minutes — and negatively for 30 seconds, so one +misconfigured DID does not re-fetch the registry on every request. ## Configuration Reference @@ -55,50 +111,85 @@ Configured via the APISIX route plugin JSON (identically on both `ext-plugin-pre | Field | Type | Required | Default | Description | |-------|------|----------|---------|-------------| | `consent_api_url` | `string` | **Yes** | — | Base URL of the consent-manager (e.g. `http://consent-manager:3000`). `http`/`https` only. | -| `consent_api_prefix` | `string` | No | `/v1` | API prefix prepended to endpoint paths (the consent-manager's `API_PREFIX`). | -| `consent_key` | `string` | No | — | Shared secret sent as `x-visionstrust-consent-key` on call 1. **Optional**: when the plugin sits behind the authority's facade, the facade injects it server-side (and overrides anything sent here). Falls back to the `CONSENT_KEY` env var. Only needed for a facade-less deployment. | -| `client_id` | `string` | Yes* | — | Participant client id; exchanged (with `client_secret`) for a participant token via `/participants/login`. Falls back to the `CONSENT_CLIENT_ID` env var. | -| `client_secret` | `string` | Yes* | — | Participant client secret. Falls back to the `CONSENT_CLIENT_SECRET` env var, so it need not sit in the route config (etcd). | -| `participant_token_ttl` | `int` | No | `3000` | Seconds a client-credentials token is cached before re-login. | -| `participant_token` | `string` | No | — | *Static* participant JWT override (legacy). Prefer `client_id`/`client_secret`. | -| `provider_sd` | `string` | No | — | Provider self-description URL for call 1. Optional: derived from `/participants/me` when unset. | -| `jwt_header_name` | `string` | No | `Authorization` | Header carrying the JWT. | -| `jwt_claims_to_forward` | `[]string` | No | `[]` | JWT claims to decode. **Must include `sub`** — the subject is resolved from it. | +| `owner_resolver_url` | `string` | **Yes** | — | The OwnerResolver `/resolve` endpoint. The data owner is resolved from the response payload; without it the plugin cannot determine whose consent to check, so the route fails to load. `http`/`https` only. | +| `owner_resolver_timeout` | `int` | No | `2000` | Per-call timeout in ms for `/resolve`. Range 1–60000. | +| `service` | `string` | No | — | Logical dataset id sent to the OwnerResolver as `resource.service`, so it can select the rule for this route. | +| `response_phase_timeout` | `int` | No | `10000` | Budget in ms for the **entire** response phase — party lookups, `/resolve`, and every per-owner consent check together. APISIX holds the buffered response for this whole time, so it is bounded independently of the per-call timeouts. Range 1–120000. | +| `require_purpose` | `bool` | No | `false` | Deny when a resolved claim names no processing purpose. Purpose matching depends on the OwnerResolver populating an optional field, so a resolver that never sets it runs with purpose scoping silently disabled; turn this on once yours emits one. `consent_purpose_unconstrained_total` counts the checks it would deny. | +| `max_resolve_body_bytes` | `int` | No | `1048576` | Maximum upstream body forwarded to the OwnerResolver. A larger body is denied rather than copied — the body is held whole, validated and marshalled again, so the peak footprint is ~3× its size per in-flight request on top of APISIX's own buffering. Range 1–104857600. | +| `max_owners_per_response` | `int` | No | `50` | Maximum distinct data owners checked for one response. A response resolving to more is denied rather than answered after an unbounded number of consent calls. Range 1–1000. | +| `consumer_claim` | `string` | No | `verifiableCredential.issuer` | Dotted claim path naming the **consuming participant**. Supports array indexing (`verifiableCredential[0].issuer`), and a bare segment landing on an array traverses its first element — a Verifiable Presentation routinely carries `verifiableCredential` as an array. Used for the contract lookup and to scope the consent match — never for ownership. | +| `consent_api_prefix` | `string` | No | `/v1` | API prefix prepended to endpoint paths (the consent-manager's `API_PREFIX`). Must start with `/`; a trailing `/` is trimmed. | +| `consent_api_host` | `string` | No | — | Overrides the HTTP `Host` header on consent-manager calls. Needed when `consent_api_url` points at an in-cluster service whose gateway route is host-scoped to the public ingress name. | | `consent_api_timeout` | `int` | No | `5000` | Per-call timeout in ms. Range 1–60000. | +| `consent_key` | `string` | No | — | Shared secret sent as `x-visionstrust-consent-key` on call 1. **Optional**: behind the authority's facade the key is injected server-side (and overrides anything sent here). Falls back to `CONSENT_KEY`. | +| `token_service_url` | `string` | Yes* | — | The participant-local OID4VP token service (the consent-facade's `POST /internal/tokens`). Falls back to `CONSENT_TOKEN_SERVICE_URL`. `http`/`https` only. | +| `token_audience` | `string` | No | `consent-manager` | Audience **name** asked of the token service — its own configured target, not a URL. | +| `participant_token_ttl` | `int` | No | `3000` | Seconds a fetched token is cached. The token service's own `expires_in` wins when shorter. Range 1–86400. | +| `participant_token` | `string` | Yes* | — | *Static*, pre-obtained participant token. An override for tests and manual runs; prefer `token_service_url`, which refreshes automatically. | +| `provider_sd` | `string` | No | — | Provider self-description URL for call 1. Derived from `/participants/me` when unset. | +| `jwt_header_name` | `string` | No | `Authorization` | Header carrying the JWT. | +| `jwt_claims_to_forward` | `[]string` | No | `[]` (all) | Claims to decode and keep. Empty decodes all. The root of `consumer_claim` is always added. | | `deny_status_code` | `int` | No | `403` | Status returned on deny. Range 100–599. | | `deny_response_body` | `string` | No | `{"error":"access denied by consent policy"}` | Body returned on deny. | | `deny_response_content_type` | `string` | No | `application/json` | `Content-Type` for deny responses. | -| `fail_open` | `bool` | No | `true` | On a consent-manager error / missing context / missing credential: `true` passes through, `false` denies. | +| `fail_open` | `bool` | No | `false` | On a **dependency failure** (resolver or consent-manager down/erroring, unreadable body): `false` denies, `true` passes through. Enabling it is logged as a warning at parse time. It does **not** apply to structural failures — no correlation id, no request context, no participant credentials, or "consent required but no owner resolved" — which always deny. | | `audit_enabled` | `bool` | No | `false` | Emit an access-decision audit event (OTLP/HTTP log) to a Collector for every decision. Async + best-effort; never affects the decision. | | `audit_otlp_endpoint` | `string` | Yes† | — | Base OTLP/HTTP endpoint of the Collector (e.g. `http://otel-collector:4318`); `/v1/logs` is appended. Falls back to `CONSENT_AUDIT_OTLP_ENDPOINT`. | +| `audit_otlp_headers` | `object` | No | — | Extra HTTP headers sent on every audit export, for a Collector that requires authentication (e.g. `{"Authorization":"Bearer ..."}`). | | `audit_service_name` | `string` | No | `consent-access-audit` | Resource `service.name` on audit records — the marker the Collector routes on to keep audit logs separate from traces. | -\* Provide **either** `client_id`+`client_secret` (recommended) **or** a static `participant_token`. None are enforced at parse time (the route still loads), but the check cannot succeed without a way to authenticate as the participant: when absent the plugin denies (unless `fail_open` is `true`). +\* Provide **either** `token_service_url` (recommended) **or** a static +`participant_token`. This is enforced at parse time: a route with neither cannot +authenticate as the participant and so cannot complete a single consent check, +and is rejected rather than loaded. † Required only when `audit_enabled` is `true`. -**Credentials via env.** `consent_key`, `client_id` and `client_secret` each fall back to an environment variable (`CONSENT_KEY`, `CONSENT_CLIENT_ID`, `CONSENT_CLIENT_SECRET`) when omitted from the route config; a value in the config always wins. The plugin runner inherits these from the APISIX container, which sources them from a Kubernetes Secret — so the participant secret need not be stored as plaintext in the route config (etcd). - -**Access audit log.** With `audit_enabled`, the plugin emits one OTLP/HTTP **log record** per decision to `audit_otlp_endpoint`, stamped with resource `service.name=` and attributes `event.domain=audit`, `consent.decision`, `consent.reason`, `enduser.id`, `http.request.method`, `url.path`, `http.request.id`. Emission is asynchronous, batched, and best-effort (a bounded queue drops rather than blocking the request path), so a slow/absent Collector never affects data access. Mark-based routing lets the Collector send these to an append-only audit sink separate from traces. +**Credentials via env.** `consent_key`, `token_service_url` and +`audit_otlp_endpoint` each fall back to an environment variable (`CONSENT_KEY`, +`CONSENT_TOKEN_SERVICE_URL`, `CONSENT_AUDIT_OTLP_ENDPOINT`) when omitted from the +route config; a value in the config always wins. The plugin runner inherits these +from the APISIX container, which sources them from a Kubernetes Secret — so +secrets need not be stored as plaintext in the route config (etcd). + +**Metrics.** Set the `CONSENT_METRICS_ADDRESS` environment variable on the +plugin runner (e.g. `:9091`) to expose Prometheus metrics on `/metrics`: +`consent_decisions_total{decision,fail_mode}` (a deny caused by an outage is +labelled apart from one caused by consent), +`consent_dependency_calls_total{dependency,outcome}`, +`consent_dependency_duration_seconds{dependency}`, +`consent_request_context_store_size`, `consent_request_contexts_evicted_total` +and `consent_audit_events_dropped_total`. Metrics are off unless the variable is +set — the runner is otherwise reached only over its unix socket, so opening a +TCP port is the deployment's decision. + +**Access audit log.** With `audit_enabled`, the plugin emits one OTLP/HTTP **log record** per **checked data owner** (so the log answers whose consent was consulted and what each said, not merely whether the response was released; a request that failed before any owner was reached is recorded once as itself) to `audit_otlp_endpoint`, stamped with resource `service.name=` and attributes `event.domain=audit`, `consent.decision`, `consent.reason`, `enduser.id`, `http.request.method`, `url.path`, `http.request.id`. Emission is asynchronous, batched, and best-effort (a bounded queue drops rather than blocking the request path), so a slow/absent Collector never affects data access. The queue is flushed when the runner exits (it returns on `SIGTERM`/`SIGINT`), so a redeploy does not discard the last flush interval of decisions. Reasons are sanitised before export (control characters collapsed, length bounded) so an upstream error body cannot reach the audit sink verbatim. Mark-based routing lets the Collector send these to an append-only audit sink separate from traces. ## APISIX Route Configuration Example ```bash -curl -X PUT http://127.0.0.1:9180/apisix/admin/routes/1 \ +# Both phases must carry the SAME configuration, so build it once. +PLUGIN_CONF='{"consent_api_url":"http://consent-manager:3000","consent_api_prefix":"/v1","owner_resolver_url":"http://owner-resolver:8080/resolve","service":"personal-profiles","token_service_url":"http://consent-facade:8080/internal/tokens","token_audience":"consent-manager","fail_open":false}' + +jq -n --arg conf "$PLUGIN_CONF" '{ + uri: "/*", + host: "data-service.example.org", + upstream: { type: "roundrobin", nodes: { "backend-service:8080": 1 } }, + plugins: { + "ext-plugin-pre-req": { conf: [ { name: "consent-filter", value: $conf } ] }, + "ext-plugin-post-resp": { conf: [ { name: "consent-filter", value: $conf } ] } + } +}' | curl -X PUT http://127.0.0.1:9180/apisix/admin/routes/1 \ -H "X-API-KEY: your-admin-api-key" \ -H "Content-Type: application/json" \ - -d '{ - "uri": "/*", - "host": "data-service.example.org", - "upstream": { "type": "roundrobin", "nodes": { "backend-service:8080": 1 } }, - "plugins": { - "ext-plugin-pre-req": { "conf": [ { "name": "consent-filter", "value": "{\"consent_api_url\":\"http://consent-manager:3000\",\"consent_api_prefix\":\"/v1\",\"jwt_claims_to_forward\":[\"sub\"],\"consent_key\":\"\",\"client_id\":\"\",\"client_secret\":\"\",\"fail_open\":false}" } ] }, - "ext-plugin-post-resp": { "conf": [ { "name": "consent-filter", "value": "{\"consent_api_url\":\"http://consent-manager:3000\",\"consent_api_prefix\":\"/v1\",\"jwt_claims_to_forward\":[\"sub\"],\"consent_key\":\"\",\"client_id\":\"\",\"client_secret\":\"\",\"fail_open\":false}" } ] } - } - }' + -d @- ``` -Both phases are required: `pre-req` captures the JWT context; `post-resp` performs the check and blocks the response. +`consent_key` is omitted above because the facade injects it; set it (or +`CONSENT_KEY`) for a facade-less deployment. Both phases are required and must +carry the **same** configuration: `pre-req` captures the token context, +`post-resp` performs the check and blocks the response. ## Build and Deployment @@ -124,11 +215,49 @@ make lint # golangci-lint docker compose up --build ``` +The stack is self-contained — APISIX + etcd, the plugin runner, a mock +consent-manager / OwnerResolver / token service, an echo upstream, and an OTel +Collector for the audit log: + | Service | Description | Ports | |---------|-------------|-------| | `etcd` | APISIX configuration store | `2379` | | `apisix` | APISIX gateway | `9080` (HTTP), `9180` (Admin API) | -| `plugin-runner` | consent-filter plugin runner | — (Unix socket) | +| `plugin-runner` | consent-filter plugin runner | — (unix socket, shared volume) | +| `mock` | consent-manager + OwnerResolver + token service stubs (`dev/mocks/`) | `8081` | +| `upstream` | echo service standing in for the personal-data API | — | +| `otel-collector` | receives the access-decision audit log | `4318` | + +The runner also serves Prometheus metrics on `9091` (`CONSENT_METRICS_ADDRESS` is +set for it in the compose file). + +Then create the gated route: + +```bash +PLUGIN_CONF='{"consent_api_url":"http://mock:8080","owner_resolver_url":"http://mock:8080/resolve","token_service_url":"http://mock:8080/internal/tokens","consent_key":"dev-consent-key","service":"dev-profiles","fail_open":false}' + +jq -n --arg conf "$PLUGIN_CONF" '{ + uri: "/*", + upstream: { type: "roundrobin", nodes: { "upstream:8080": 1 } }, + plugins: { + "ext-plugin-pre-req": { conf: [ { name: "consent-filter", value: $conf } ] }, + "ext-plugin-post-resp": { conf: [ { name: "consent-filter", value: $conf } ] } + } +}' | curl -s -X PUT http://127.0.0.1:9180/apisix/admin/routes/1 \ + -H "X-API-KEY: edd1c9f034335f136f87ad84b625c8f1" -H "Content-Type: application/json" -d @- + +# The token names the consumer the mock's consent was granted to, so this passes. +TOKEN_PAYLOAD=$(printf '{"verifiableCredential":{"issuer":"did:key:zDevConsumer"}}' | base64 -w0 | tr '+/' '-_' | tr -d '=') +curl -i http://127.0.0.1:9080/profile -H "Authorization: Bearer e30.${TOKEN_PAYLOAD}.nosig" +``` + +Change the `consumer` in `dev/mocks/consents.json` (or the `status` to +`revoked`) and restart the `mock` service to watch the same request be denied. +The `otel-collector` logs show the audit record for each decision. + +> The socket is shared through a **directory** (the `runner-socket` volume), not +> by bind-mounting the socket file: Docker creates a directory at a bind-mount +> path that does not exist yet, and the runner then cannot bind. ## Project Structure @@ -140,10 +269,14 @@ consent-plugin/ │ ├── plugin/ │ │ ├── consent.go # RequestFilter + ResponseFilter (the consent gate) │ │ ├── config.go # Configuration schema and validation -│ │ └── context.go # Request context store keyed by $request_id +│ │ └── context.go # Bounded request-context store keyed by $request_id │ ├── consent/ │ │ ├── client.go # Two-call consent-manager client │ │ └── models.go # Request/response models and Decision type +│ ├── ownerresolver/ +│ │ └── client.go # OwnerResolver /resolve client (who owns the data) +│ ├── audit/ +│ │ └── audit.go # OTLP/HTTP access-decision audit exporter │ ├── jwt/ │ │ └── extractor.go # JWT extraction and claim decoding │ └── integration/ @@ -161,7 +294,7 @@ versioning rules. Each release publishes: - a multi-arch (`linux/amd64,arm64`) image - `quay.io/wi_stefan/consent-plugin:` (also `:latest`, `:`), and + `quay.io/seamware/consent-plugin:` (also `:latest`, `:`), and - standalone `go-runner` binaries (`consent-plugin-linux-{amd64,arm64}`) on the GitHub Release. @@ -174,7 +307,7 @@ APISIX launches it as the external plugin runner: ```yaml initContainers: - name: install-consent-plugin - image: quay.io/wi_stefan/consent-plugin: + image: quay.io/seamware/consent-plugin: command: ["cp", "/app/go-runner", "/ext-plugin/go-runner"] volumeMounts: - name: ext-plugin-bin diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..d2fa3b3 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,51 @@ +# Security Policy + +`consent-plugin` gates access to personal data. A defect in it can release data +a subject never consented to, so please treat findings here as sensitive. + +## Reporting a vulnerability + +**Do not open a public issue for a suspected vulnerability.** + +Report it privately through GitHub's +[private vulnerability reporting](https://github.com/wistefan/consent-plugin/security/advisories/new) +on this repository. If that is unavailable to you, contact a maintainer listed in +[CODEOWNERS](.github/CODEOWNERS) directly. + +Please include: + +- the version, image tag or commit you tested, +- the plugin configuration in use, with secrets redacted, +- what you observed and what you expected — for a gate, "the response was + released" or "the response was denied" is the key fact, +- a minimal reproduction if you have one. + +We aim to acknowledge a report within three working days and to agree a +disclosure timeline with you before anything is published. + +## What is in scope + +Anything that changes the access decision or leaks data around it, including: + +- a response released without a granted consent from the resolved data owner, +- a consent granted to one consuming participant authorising another, +- data ownership being taken from the requestor rather than from the data, +- information about denied data reaching the client (headers, timing, counts), +- credentials or personal data reaching logs, metrics, or the audit sink in a + form that was not intended, +- a way to make the plugin fail open that the operator did not configure, +- suppressing or forging audit records. + +## What is out of scope + +- The absence of JWT signature verification. The plugin decodes the token and + relies on an authentication plugin earlier in the route; this is documented in + the README and is a deployment requirement, not a defect in the plugin. +- Findings that require an already-compromised APISIX instance or plugin runner. +- The local development stack under `dev/`, which uses fixed credentials on + purpose and is not for deployment. + +## Supported versions + +Fixes land on `main` and are published as a new release. Older tags are not +patched; please upgrade. diff --git a/dev/apisix-config.yaml b/dev/apisix-config.yaml new file mode 100644 index 0000000..4f159b9 --- /dev/null +++ b/dev/apisix-config.yaml @@ -0,0 +1,41 @@ +# APISIX configuration for the local development stack (docker-compose.yaml). +# +# The only part specific to this project is `ext-plugin.path_for_test`, which +# points APISIX at the unix socket the plugin-runner container binds. The two +# containers share it through the `runner-socket` volume: the socket must live in +# a shared DIRECTORY, because bind-mounting the socket file itself makes Docker +# create a directory at that path before either process can bind it. + +apisix: + node_listen: 9080 + enable_admin: true + proxy_mode: http + +deployment: + role: traditional + role_traditional: + config_provider: etcd + admin: + admin_key: + - name: admin + # Development only. Never reuse this key outside the local stack. + key: edd1c9f034335f136f87ad84b625c8f1 + role: admin + allow_admin: + - 0.0.0.0/0 + admin_listen: + port: 9180 + etcd: + host: + - "http://etcd:2379" + prefix: /apisix + timeout: 30 + +ext-plugin: + # The runner is started by its own container, not by APISIX, so APISIX only + # needs to know where to reach it. + path_for_test: /opt/runner/runner.sock + +plugins: + - ext-plugin-pre-req + - ext-plugin-post-resp diff --git a/dev/mocks/consents.json b/dev/mocks/consents.json new file mode 100644 index 0000000..af468a5 --- /dev/null +++ b/dev/mocks/consents.json @@ -0,0 +1,26 @@ +{ + "name": "owner consents", + "metadata": { + "comment": "GET /consents/participants/{id}?receipt=true - the owner's consents. Granted TO dev-consumer, which is what makes the check pass; change the consumer here, or the status to revoked, to watch the gate deny." + }, + "request": { + "method": "GET", + "urlPathPattern": "/v1/consents/participants/.*" + }, + "response": { + "status": 200, + "headers": { + "Content-Type": "application/json" + }, + "jsonBody": { + "consents": [ + { + "status": "granted", + "consumer": { + "selfDescriptionURL": "http://mock:8080/participants/dev-consumer" + } + } + ] + } + } +} diff --git a/dev/mocks/identifier-search.json b/dev/mocks/identifier-search.json new file mode 100644 index 0000000..49c7b5a --- /dev/null +++ b/dev/mocks/identifier-search.json @@ -0,0 +1,19 @@ +{ + "name": "identifier search", + "metadata": { + "comment": "POST /users/identifier/search - resolves the data owner DID (sent as the user email) to the provider-scoped user identifier." + }, + "request": { + "method": "POST", + "url": "/v1/users/identifier/search" + }, + "response": { + "status": 200, + "headers": { + "Content-Type": "application/json" + }, + "jsonBody": { + "userIdentifier": "dev-user-identifier" + } + } +} diff --git a/dev/mocks/owner-resolver.json b/dev/mocks/owner-resolver.json new file mode 100644 index 0000000..2746c68 --- /dev/null +++ b/dev/mocks/owner-resolver.json @@ -0,0 +1,28 @@ +{ + "name": "owner resolver", + "metadata": { + "comment": "POST /resolve - the OwnerResolver. It answers from the DATA who the owner is; here it always reports the same owner so the stack has a working happy path." + }, + "request": { + "method": "POST", + "url": "/resolve" + }, + "response": { + "status": 200, + "headers": { + "Content-Type": "application/json" + }, + "jsonBody": { + "consentRequired": true, + "claims": [ + { + "selector": { + "type": "jsonpath", + "value": "$.id" + }, + "ownerId": "did:key:zDevOwner" + } + ] + } + } +} diff --git a/dev/mocks/participants-me.json b/dev/mocks/participants-me.json new file mode 100644 index 0000000..25e6944 --- /dev/null +++ b/dev/mocks/participants-me.json @@ -0,0 +1,19 @@ +{ + "name": "provider self-description", + "metadata": { + "comment": "GET /participants/me - the provider self-description the identifier search is scoped by." + }, + "request": { + "method": "GET", + "url": "/v1/participants/me" + }, + "response": { + "status": 200, + "headers": { + "Content-Type": "application/json" + }, + "jsonBody": { + "selfDescriptionURL": "http://mock:8080/participants/dev-provider" + } + } +} diff --git a/dev/mocks/participants.json b/dev/mocks/participants.json new file mode 100644 index 0000000..1be4d59 --- /dev/null +++ b/dev/mocks/participants.json @@ -0,0 +1,22 @@ +{ + "name": "participant registry", + "metadata": { + "comment": "GET /participants - the registry that maps the consumer DID from the token to the self-description URL a contract names its parties by." + }, + "request": { + "method": "GET", + "url": "/v1/participants" + }, + "response": { + "status": 200, + "headers": { + "Content-Type": "application/json" + }, + "jsonBody": [ + { + "did": "did:key:zDevConsumer", + "selfDescriptionURL": "http://mock:8080/participants/dev-consumer" + } + ] + } +} diff --git a/dev/mocks/token-service.json b/dev/mocks/token-service.json new file mode 100644 index 0000000..917647b --- /dev/null +++ b/dev/mocks/token-service.json @@ -0,0 +1,21 @@ +{ + "name": "participant token service", + "metadata": { + "comment": "The participant-local OID4VP token service (the consent-facade's POST /internal/tokens). It mints the short-lived participant token the plugin uses for the consents lookup." + }, + "request": { + "method": "POST", + "url": "/internal/tokens" + }, + "response": { + "status": 200, + "headers": { + "Content-Type": "application/json" + }, + "jsonBody": { + "access_token": "dev-participant-token", + "token_type": "Bearer", + "expires_in": 3600 + } + } +} diff --git a/dev/otel-collector.yaml b/dev/otel-collector.yaml new file mode 100644 index 0000000..a400c40 --- /dev/null +++ b/dev/otel-collector.yaml @@ -0,0 +1,26 @@ +# OpenTelemetry Collector configuration for the local development stack. +# +# The plugin exports one OTLP/HTTP log record per checked data owner, marked with +# resource service.name=consent-access-audit. This config shows the routing that +# marker exists for: audit records go to their own pipeline, which in a real +# deployment would be an append-only sink rather than the console. + +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + +processors: + batch: {} + +exporters: + debug: + verbosity: detailed + +service: + pipelines: + logs: + receivers: [otlp] + processors: [batch] + exporters: [debug] diff --git a/docker-compose.yaml b/docker-compose.yaml index 6d651a7..0d0f083 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -2,40 +2,112 @@ # # Usage: # docker compose up --build +# # then create the gated route (see README, "Local Development with Docker Compose") # -# This starts APISIX with etcd and the Go plugin runner sidecar. - -version: "3.8" +# The stack is self-contained: APISIX and its etcd, the plugin runner, a mock +# consent-manager + OwnerResolver + token service, an upstream that returns +# personal data, and an OTel Collector to receive the access-decision audit log. +# +# Note the socket wiring. APISIX and the runner share a DIRECTORY (the +# runner-socket volume) and the socket is created inside it. Bind-mounting the +# socket file itself does not work: Docker creates a directory at that path +# before either process can bind, and the runner then fails to start. services: + # The upstream etcd image rather than bitnami/etcd, which was retired from the + # public catalog and no longer resolves — `docker compose up` failed on the + # pull. This one needs no auth-disabling env var: it is open by default, which + # is fine for a local stack and is not a deployment example. etcd: - image: bitnami/etcd:3.5 - environment: - ETCD_ENABLE_V2: "true" - ALLOW_NONE_AUTHENTICATION: "yes" - ETCD_ADVERTISE_CLIENT_URLS: "http://etcd:2379" - ETCD_LISTEN_CLIENT_URLS: "http://0.0.0.0:2379" + image: quay.io/coreos/etcd:v3.5.17 + command: + - etcd + - --name=etcd0 + - --advertise-client-urls=http://etcd:2379 + - --listen-client-urls=http://0.0.0.0:2379 ports: - "2379:2379" apisix: image: apache/apisix:3.8.0-debian depends_on: - - etcd + etcd: + condition: service_started + plugin-runner: + condition: service_healthy ports: - "9080:9080" # HTTP proxy port - "9180:9180" # Admin API port volumes: - - ./apisix-config.yaml:/usr/local/apisix/conf/config.yaml:ro - - /tmp/runner.sock:/tmp/runner.sock + - ./dev/apisix-config.yaml:/usr/local/apisix/conf/config.yaml:ro + - runner-socket:/opt/runner restart: on-failure + # A named volume is created root-owned, and the runner image runs as uid 10001, + # so hand the socket directory over before the runner starts. + socket-init: + image: alpine:3.22 + command: ["chown", "10001:10001", "/opt/runner"] + volumes: + - runner-socket:/opt/runner + plugin-runner: build: context: . dockerfile: Dockerfile + depends_on: + socket-init: + condition: service_completed_successfully + mock: + condition: service_healthy volumes: - - /tmp/runner.sock:/tmp/runner.sock + - runner-socket:/opt/runner environment: - APISIX_LISTEN_ADDRESS: "unix:/tmp/runner.sock" + APISIX_LISTEN_ADDRESS: "unix:/opt/runner/runner.sock" + # Credentials the plugin reads from the environment rather than the route + # config, mirroring how the Kubernetes deployment sources them from a Secret. + CONSENT_KEY: "dev-consent-key" + CONSENT_TOKEN_SERVICE_URL: "http://mock:8080/internal/tokens" + CONSENT_AUDIT_OTLP_ENDPOINT: "http://otel-collector:4318" + # Prometheus metrics, off unless an address is set. + CONSENT_METRICS_ADDRESS: ":9091" + ports: + - "9091:9091" # Prometheus metrics restart: on-failure + + # Mock consent-manager, OwnerResolver and participant token service. The + # stubs live in dev/mocks/ — edit consents.json to flip the gate's answer. + mock: + image: wiremock/wiremock:3.9.1 + command: ["--port", "8080", "--verbose"] + volumes: + - ./dev/mocks:/home/wiremock/mappings:ro + ports: + - "8081:8080" + # A malformed stub makes WireMock exit at startup, and without a healthcheck + # that shows up only as an unexplained deny from the gate several steps + # later. Depending services wait for this instead. + healthcheck: + test: ["CMD", "curl", "-fsS", "http://localhost:8080/__admin/health"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 5s + + # The upstream whose responses are gated. It echoes the request back as JSON, + # which is enough for the OwnerResolver stub to be asked about a payload. + upstream: + image: mendhak/http-https-echo:34 + environment: + HTTP_PORT: "8080" + + otel-collector: + image: otel/opentelemetry-collector-contrib:0.109.0 + command: ["--config=/etc/otel-collector.yaml"] + volumes: + - ./dev/otel-collector.yaml:/etc/otel-collector.yaml:ro + ports: + - "4318:4318" + +volumes: + runner-socket: diff --git a/go.mod b/go.mod index 54a0170..802360d 100644 --- a/go.mod +++ b/go.mod @@ -4,17 +4,15 @@ go 1.26 require ( github.com/apache/apisix-go-plugin-runner v0.5.0 - github.com/stretchr/testify v1.8.4 + github.com/stretchr/testify v1.12.1 ) require ( github.com/ReneKroon/ttlcache/v2 v2.4.0 // indirect - github.com/api7/ext-plugin-proto v0.6.0 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/api7/ext-plugin-proto v0.6.1 // indirect github.com/google/flatbuffers v2.0.0+incompatible // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.7.0 // indirect go.uber.org/zap v1.17.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect ) diff --git a/go.sum b/go.sum index 83992a3..0b18c2a 100644 --- a/go.sum +++ b/go.sum @@ -48,8 +48,9 @@ github.com/alvaroloes/enumer v1.1.2/go.mod h1:FxrjvuXoDAx9isTJrv4c+T410zFi0DtXIT github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/apisix-go-plugin-runner v0.5.0 h1:kg2FpLWdbrzGXwS6wc6etwZa7tpd//R9q1HMUwUEd34= github.com/apache/apisix-go-plugin-runner v0.5.0/go.mod h1:KEdzfoWik+m5JTOnYoV1X/QWat1vmcchzCjhyy68XZE= -github.com/api7/ext-plugin-proto v0.6.0 h1:xmgcKwWRiM9EpBIs1wYJ7Ife/YnLl4IL2NEy4417g60= github.com/api7/ext-plugin-proto v0.6.0/go.mod h1:8dbdAgCESeqwZ0IXirbjLbshEntmdrAX3uet+LW3jVU= +github.com/api7/ext-plugin-proto v0.6.1 h1:eQN0oHacL97ezVGWVmsRigt+ClcpgjipUq0rmW8BG4g= +github.com/api7/ext-plugin-proto v0.6.1/go.mod h1:8dbdAgCESeqwZ0IXirbjLbshEntmdrAX3uet+LW3jVU= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -76,7 +77,6 @@ github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= @@ -208,10 +208,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= @@ -245,7 +243,6 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -290,8 +287,8 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/thediveo/enumflag v0.10.1/go.mod h1:KyVhQUPzreSw85oJi2uSjFM0ODLKXBH0rPod7zc2pmI= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -327,6 +324,8 @@ go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95a go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.17.0 h1:MTjgFu6ZLKvY6Pvaqk97GlxNBuMpV4Hy/3P6tRGlI2U= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -663,7 +662,6 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= @@ -680,8 +678,6 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/hack/coverage-floor.sh b/hack/coverage-floor.sh new file mode 100755 index 0000000..9df35a6 --- /dev/null +++ b/hack/coverage-floor.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# +# Fail when total statement coverage is below a floor. +# +# Coverage was previously uploaded as a CI artifact and never asserted, so a +# whole code path could sit at 0% without anything noticing. This turns the +# number into a gate. +# +# Usage: coverage-floor.sh + +set -euo pipefail + +profile="${1:?usage: coverage-floor.sh }" +floor="${2:?usage: coverage-floor.sh }" + +total="$(go tool cover -func="${profile}" | awk '/^total:/ {gsub(/%/, "", $3); print $3}')" + +if [[ -z "${total}" ]]; then + echo "coverage-floor: could not read a total from ${profile}" >&2 + exit 1 +fi + +# awk rather than bash arithmetic: the percentages are fractional. +if awk -v total="${total}" -v floor="${floor}" 'BEGIN { exit !(total < floor) }'; then + echo "coverage-floor: total coverage ${total}% is below the ${floor}% floor" >&2 + exit 1 +fi + +echo "coverage-floor: total coverage ${total}% meets the ${floor}% floor" diff --git a/internal/audit/audit.go b/internal/audit/audit.go index a8356b4..ab52101 100644 --- a/internal/audit/audit.go +++ b/internal/audit/audit.go @@ -30,10 +30,12 @@ package audit import ( "bytes" + "consent-plugin/internal/logging" + "consent-plugin/internal/metrics" "context" "encoding/json" - "log" "net/http" + "sort" "strconv" "strings" "sync" @@ -75,6 +77,45 @@ type Config struct { ServiceName string // Timeout bounds a single export HTTP call. Zero defaults to defaultTimeout. Timeout time.Duration + // Headers are extra HTTP headers sent on every export, for a Collector that + // requires authentication (e.g. "Authorization" or a tenant header). + Headers map[string]string +} + +// key identifies the Emitter this configuration describes. Every field that +// changes the emitter's behaviour must appear in it: caching on endpoint and +// service name alone meant the first route's timeout and headers silently +// applied to every other route sharing them. +func (c Config) key() string { + parts := make([]string, 0, 3+2*len(c.Headers)) + parts = append(parts, c.Endpoint, c.serviceName(), c.Timeout.String()) + for _, name := range sortedKeys(c.Headers) { + parts = append(parts, name, c.Headers[name]) + } + return strings.Join(parts, configKeySeparator) +} + +// serviceName is the configured routing marker, or the default. +func (c Config) serviceName() string { + if c.ServiceName == "" { + return DefaultServiceName + } + return c.ServiceName +} + +// configKeySeparator joins the parts of an emitter cache key. It cannot occur in +// a URL, a service name or a header value. +const configKeySeparator = "\x00" + +// sortedKeys returns m's keys in a stable order, so an emitter key does not +// depend on map iteration order. +func sortedKeys(m map[string]string) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys } // Event is a single access decision to record. @@ -99,6 +140,7 @@ type Event struct { type Emitter struct { endpoint string serviceName string + headers map[string]string client *http.Client queue chan Event done chan struct{} @@ -112,15 +154,22 @@ var ( emitters = map[string]*Emitter{} ) +func init() { + // An attacker who can generate load can suppress the record of their own + // access by filling the queue, so the loss must be alertable, not merely + // logged every hundredth event. + metrics.RegisterCounter(metrics.AuditDroppedCounter, + "Audit events discarded because the export queue was full.", + func() float64 { return float64(Dropped()) }) +} + // Get returns a shared Emitter for cfg, creating (and starting) one on first use. -// Emitters are cached by endpoint + service name, so all routes exporting to the -// same Collector share a single background worker and connection pool. +// Emitters are cached by the full configuration (see Config.key), so all routes +// exporting to the same Collector with the same settings share a single +// background worker and connection pool, while a route configuring a different +// timeout or different headers gets its own. func Get(cfg Config) *Emitter { - sn := cfg.ServiceName - if sn == "" { - sn = DefaultServiceName - } - key := cfg.Endpoint + "|" + sn + key := cfg.key() emittersMu.Lock() defer emittersMu.Unlock() @@ -132,23 +181,45 @@ func Get(cfg Config) *Emitter { return e } +// ShutdownAll flushes and stops every emitter created so far. +// +// The runner is long-lived but not immortal: it is restarted on every redeploy, +// and without this up to defaultFlushInterval of access decisions were lost each +// time — silently, from the record that exists precisely to be complete. Wire it +// to SIGTERM/SIGINT. +func ShutdownAll() { + emittersMu.Lock() + pending := make([]*Emitter, 0, len(emitters)) + for _, e := range emitters { + pending = append(pending, e) + } + emitters = map[string]*Emitter{} + emittersMu.Unlock() + + for _, e := range pending { + e.Shutdown() + } +} + // newEmitter builds and starts an Emitter for cfg. func newEmitter(cfg Config) *Emitter { timeout := cfg.Timeout if timeout <= 0 { timeout = defaultTimeout } - serviceName := cfg.ServiceName - if serviceName == "" { - serviceName = DefaultServiceName - } + serviceName := cfg.serviceName() endpoint := strings.TrimRight(cfg.Endpoint, "/") if !strings.HasSuffix(endpoint, otlpLogsPath) { endpoint += otlpLogsPath } + headers := make(map[string]string, len(cfg.Headers)) + for name, value := range cfg.Headers { + headers[name] = value + } e := &Emitter{ endpoint: endpoint, serviceName: serviceName, + headers: headers, client: &http.Client{Timeout: timeout}, queue: make(chan Event, defaultQueueSize), done: make(chan struct{}), @@ -161,16 +232,54 @@ func newEmitter(cfg Config) *Emitter { // Emit queues ev for export. It never blocks: if the queue is full the event is // dropped and a counter is incremented (data access must not wait on the audit // pipe). Emit is safe for concurrent use. +// +// The reason is sanitised here rather than at the call site, so an upstream +// error body cannot reach the audit sink verbatim no matter which code path +// produced it. func (e *Emitter) Emit(ev Event) { + ev.Reason = SanitizeReason(ev.Reason) select { case e.queue <- ev: default: - if n := e.dropped.Add(1); n%100 == 1 { - log.Printf("[consent-filter] audit queue full, dropping event (total dropped %d)", n) + if n := e.dropped.Add(1); n%droppedLogEvery == 1 { + logging.Warnf("audit queue full, dropping event (total dropped %d)", n) } } } +// Dropped reports how many events this emitter has discarded because its queue +// was full. It is the signal that the record is incomplete: an attacker who can +// generate load can suppress the record of their own access, so this number must +// be observable rather than only logged every droppedLogEvery events. +func (e *Emitter) Dropped() uint64 { + return e.dropped.Load() +} + +// Dropped reports the total number of audit events discarded across every +// emitter. +func Dropped() uint64 { + emittersMu.Lock() + defer emittersMu.Unlock() + var total uint64 + for _, e := range emitters { + total += e.Dropped() + } + return total +} + +// droppedLogEvery rate-limits the queue-full log line. +const droppedLogEvery = 100 + +// SanitizeReason makes a decision reason safe to export: control characters +// (including the newlines of an HTML or JSON error page) are collapsed to +// spaces, and the result is truncated. +// +// Reasons are built by wrapping dependency errors, and those errors embed the +// consent-manager's response body — which can carry identifiers or other +// personal data. Exporting it verbatim would push exactly the data the audit +// pipeline exists to keep controlled into the audit sink. +func SanitizeReason(reason string) string { return logging.Sanitize(reason) } + // Shutdown stops the background worker after flushing everything still queued. // Intended for clean teardown and tests; the plugin runner is long-lived and // normally never calls it. @@ -222,25 +331,28 @@ func (e *Emitter) run() { func (e *Emitter) export(batch []Event) { body, err := json.Marshal(e.buildPayload(batch)) if err != nil { - log.Printf("[consent-filter] audit: failed to marshal %d event(s): %v", len(batch), err) + logging.Errorf("audit: failed to marshal %d event(s): %s", len(batch), logging.Sanitize(err.Error())) return } ctx, cancel := context.WithTimeout(context.Background(), e.client.Timeout) defer cancel() req, err := http.NewRequestWithContext(ctx, http.MethodPost, e.endpoint, bytes.NewReader(body)) if err != nil { - log.Printf("[consent-filter] audit: failed to build request: %v", err) + logging.Errorf("audit: failed to build the export request: %s", logging.Sanitize(err.Error())) return } req.Header.Set("Content-Type", "application/json") + for name, value := range e.headers { + req.Header.Set(name, value) + } resp, err := e.client.Do(req) if err != nil { - log.Printf("[consent-filter] audit: export to %s failed: %v", e.endpoint, err) + logging.ErrorfEvery("audit-export", "audit: export to %s failed: %s", e.endpoint, logging.Sanitize(err.Error())) return } defer func() { _ = resp.Body.Close() }() if resp.StatusCode >= http.StatusMultipleChoices { - log.Printf("[consent-filter] audit: export to %s returned HTTP %d", e.endpoint, resp.StatusCode) + logging.ErrorfEvery("audit-export-status", "audit: export to %s returned HTTP %d", e.endpoint, resp.StatusCode) } } diff --git a/internal/audit/audit_test.go b/internal/audit/audit_test.go index 75035a2..ec16520 100644 --- a/internal/audit/audit_test.go +++ b/internal/audit/audit_test.go @@ -119,3 +119,158 @@ func TestEmitNeverBlocksWhenQueueFull(t *testing.T) { } assert.GreaterOrEqual(t, e.dropped.Load(), uint64(8), "overflow beyond the queue capacity must be dropped") } + +// TestGetDistinguishesConfigurations verifies the emitter cache keys on the full +// configuration. Keying on endpoint + service name alone meant whichever route +// created the emitter first silently imposed its timeout and headers on every +// other route sharing that Collector. +func TestGetDistinguishesConfigurations(t *testing.T) { + base := Config{Endpoint: "http://collector:4318", ServiceName: "audit", Timeout: time.Second} + + tests := []struct { + name string + mutate func(cfg *Config) + }{ + {"endpoint", func(cfg *Config) { cfg.Endpoint = "http://other:4318" }}, + {"service name", func(cfg *Config) { cfg.ServiceName = "other-audit" }}, + {"timeout", func(cfg *Config) { cfg.Timeout = 5 * time.Second }}, + {"headers", func(cfg *Config) { cfg.Headers = map[string]string{"Authorization": "Bearer x"} }}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + other := base + tt.mutate(&other) + assert.NotEqual(t, base.key(), other.key(), + "configurations differing in %s must not share an emitter", tt.name) + }) + } + + t.Run("header order does not matter", func(t *testing.T) { + a := base + a.Headers = map[string]string{"A": "1", "B": "2"} + b := base + b.Headers = map[string]string{"B": "2", "A": "1"} + assert.Equal(t, a.key(), b.key()) + }) + + t.Run("an unnamed service falls back to the default", func(t *testing.T) { + named := Config{Endpoint: "http://collector:4318", ServiceName: DefaultServiceName} + unnamed := Config{Endpoint: "http://collector:4318"} + assert.Equal(t, named.key(), unnamed.key()) + }) +} + +// TestEmitSendsConfiguredHeaders verifies extra headers reach the Collector, so +// an authenticating audit sink can be used. +func TestEmitSendsConfiguredHeaders(t *testing.T) { + received := make(chan http.Header, 1) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + received <- r.Header.Clone() + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + e := newEmitter(Config{Endpoint: srv.URL, Headers: map[string]string{"Authorization": "Bearer audit-token"}}) + e.Emit(Event{Time: time.Now(), Decision: "allow"}) + e.Shutdown() + + select { + case h := <-received: + assert.Equal(t, "Bearer audit-token", h.Get("Authorization")) + assert.Equal(t, "application/json", h.Get("Content-Type")) + case <-time.After(time.Second): + t.Fatal("the Collector never received an export") + } +} + +// TestEmitSanitizesReason verifies the sanitisation happens on the way out, so +// no call site can bypass it. +func TestEmitSanitizesReason(t *testing.T) { + received := make(chan []byte, 1) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + b, _ := io.ReadAll(r.Body) + received <- b + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + e := newEmitter(Config{Endpoint: srv.URL}) + e.Emit(Event{Time: time.Now(), Decision: "deny", Reason: "upstream said:\n{\"email\":\"alice@example.org\"}"}) + e.Shutdown() + + select { + case body := <-received: + assert.NotContains(t, string(body), `\n`, "control characters must not reach the sink") + assert.Contains(t, string(body), "upstream said: ") + case <-time.After(time.Second): + t.Fatal("the Collector never received an export") + } +} + +// TestShutdownAllFlushesEveryEmitter verifies a termination flush drains all +// emitters, so a redeploy does not silently discard queued decisions. +func TestShutdownAllFlushesEveryEmitter(t *testing.T) { + var mu sync.Mutex + var records int + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var payload otlpPayload + _ = json.NewDecoder(r.Body).Decode(&payload) + mu.Lock() + for _, rl := range payload.ResourceLogs { + for _, sl := range rl.ScopeLogs { + records += len(sl.LogRecords) + } + } + mu.Unlock() + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + for _, serviceName := range []string{"audit-a", "audit-b"} { + Get(Config{Endpoint: srv.URL, ServiceName: serviceName}). + Emit(Event{Time: time.Now(), Decision: "allow", Subject: serviceName}) + } + + ShutdownAll() + + mu.Lock() + defer mu.Unlock() + assert.Equal(t, 2, records, "every emitter's queue must be flushed on shutdown") +} + +// TestDroppedIsObservable verifies the drop counter is exposed. An attacker who +// can generate load can suppress the record of their own access, so the fact +// that records were lost must be visible, not only logged occasionally. +func TestDroppedIsObservable(t *testing.T) { + e := &Emitter{queue: make(chan Event, 1), done: make(chan struct{}), stopped: make(chan struct{})} + close(e.stopped) // no worker: nothing drains the queue + + for i := 0; i < 5; i++ { + e.Emit(Event{Time: time.Now(), Decision: "allow"}) + } + + assert.Equal(t, uint64(4), e.Dropped(), "one event fits the queue, the rest are dropped and counted") +} + +// TestDroppedAggregatesAcrossEmitters verifies the package-level Dropped — the +// value the registered metric callback reads, and therefore the only thing that +// makes suppression of an access record visible — sums every emitter. +func TestDroppedAggregatesAcrossEmitters(t *testing.T) { + ShutdownAll() // start from a clean registry + t.Cleanup(ShutdownAll) + + assert.Zero(t, Dropped(), "a fresh registry has dropped nothing") + + // Two emitters whose workers are already stopped, so nothing drains them. + for _, serviceName := range []string{"audit-a", "audit-b"} { + e := Get(Config{Endpoint: "http://collector.invalid:4318", ServiceName: serviceName}) + e.Shutdown() + for i := 0; i < defaultQueueSize+3; i++ { + e.Emit(Event{Time: time.Now(), Decision: "allow"}) + } + } + + assert.Equal(t, uint64(6), Dropped(), + "three events past the queue size are dropped by each of the two emitters") +} diff --git a/internal/consent/client.go b/internal/consent/client.go index fab858e..b5aaa65 100644 --- a/internal/consent/client.go +++ b/internal/consent/client.go @@ -19,7 +19,10 @@ package consent import ( "bytes" + "consent-plugin/internal/logging" "context" + "crypto/sha256" + "encoding/hex" "encoding/json" "errors" "fmt" @@ -82,6 +85,18 @@ const ( // (HTTP 401), so a cached token should be refreshed and the call retried. var errParticipantUnauthorized = errors.New("consent client: participant token unauthorized") +// ErrNoCredentials signals that the client has no way to authenticate as the +// participant at all. It is a misconfiguration, not an availability failure: +// callers must never treat it as a transient error to be failed open on, or a +// mistyped route config becomes a silent, total bypass of the gate. +var ErrNoCredentials = errors.New("consent client: no participant_token and no token_service_url configured") + +// ErrParticipantNotRegistered signals that the participant registry answered and +// holds no participant for the requested DID. Like ErrNoCredentials it is a +// permanent condition rather than an outage: retrying will not fix it, so +// callers must not treat it as a transient error to be failed open on. +var ErrParticipantNotRegistered = errors.New("consent client: participant not registered") + // ClientConfig holds everything needed to verify consent against the // (Prometheus-X / Visions) consent-manager. type ClientConfig struct { @@ -125,8 +140,10 @@ type ClientConfig struct { // participant credentials of its own: the token service presents the // participant's verifiable credential and returns a short-lived token. GET // /participants/me then yields the provider selfDescriptionURL. Tokens are cached -// package-wide (keyed by base URL + audience) and refreshed on expiry or a 401. -// Access is allowed iff a returned consent is "granted". +// package-wide (keyed by the full credential identity, see cacheKey) and +// refreshed on expiry or a 401. Access is allowed iff a returned consent is +// "granted" AND was granted to the consuming participant named in the request +// (see hasGrantedConsent). type Client struct { baseURL string host string @@ -187,7 +204,46 @@ var ( credCache = map[string]*cacheEntry{} ) -func (c *Client) cacheKey() string { return c.baseURL + "|" + c.tokenAudience } +// credentialKeySeparator joins the components of a credential cache key. It is a +// character that cannot occur in a URL or an audience name, so no two distinct +// credential identities can produce the same joined key. +const credentialKeySeparator = "\x00" + +// cacheKey identifies the credential the cached entry belongs to. +// +// The entry holds both the participant access token and the provider +// self-description derived from it, so the key MUST cover every input that can +// change either of them - otherwise two routes fronting different participants +// but the same consent-manager share one entry, and whichever warms it first +// makes the other run its identifier search scoped to the wrong provider and its +// consents lookup as the wrong participant (silently wrong decisions in both +// directions). +// +// Secrets are hashed rather than embedded so the key can be logged or ranged +// over without leaking a token. +func (c *Client) cacheKey() string { + return strings.Join([]string{ + c.baseURL, + c.host, + c.apiPrefix, + c.tokenAudience, + c.tokenServiceURL, + c.providerSD, + hashSecret(c.staticToken), + hashSecret(c.consentKey), + }, credentialKeySeparator) +} + +// hashSecret returns a stable, non-reversible fingerprint of a secret, so it can +// distinguish cache identities without the secret itself being retained in the +// key. An empty secret maps to the empty string. +func hashSecret(secret string) string { + if secret == "" { + return "" + } + sum := sha256.Sum256([]byte(secret)) + return hex.EncodeToString(sum[:]) +} // CheckConsent runs the two-call consent verification for req.Subject, allowing // when a granted consent exists and denying otherwise. An unknown subject is a @@ -198,11 +254,16 @@ func (c *Client) CheckConsent(ctx context.Context, req ConsentRequest) (*Consent if req.Subject == "" { return &ConsentResponse{Decision: DecisionDeny, Reason: "no subject in request"}, nil } + // Without a named consumer the check degenerates into "does this subject have + // any consent at all?", which authorises the wrong agreement. Deny instead. + if req.Consumer == "" { + return &ConsentResponse{Decision: DecisionDeny, Reason: "no consuming participant identified"}, nil + } - resp, err := c.check(ctx, req.Subject, req.DataResource, false) + resp, err := c.check(ctx, req, false) if errors.Is(err, errParticipantUnauthorized) && c.staticToken == "" { // The cached token was rejected — refresh it and retry once. - resp, err = c.check(ctx, req.Subject, req.DataResource, true) + resp, err = c.check(ctx, req, true) } if errors.Is(err, errParticipantUnauthorized) { // Still unauthorized (or a static token was rejected): surface a plain error. @@ -212,15 +273,16 @@ func (c *Client) CheckConsent(ctx context.Context, req ConsentRequest) (*Consent } // check performs one full verification attempt. forceLogin refreshes a cached -// client-credentials token before use. When dataResource is non-empty the check -// is scoped: a granted consent counts only if it covers that resource. -func (c *Client) check(ctx context.Context, subject, dataResource string, forceLogin bool) (*ConsentResponse, error) { +// token before use. The check is scoped by req: a granted consent counts only if +// it was granted to req.Consumer and, when set, covers req.DataResource and +// req.Purpose. +func (c *Client) check(ctx context.Context, req ConsentRequest, forceLogin bool) (*ConsentResponse, error) { token, providerSD, err := c.credentials(ctx, forceLogin) if err != nil { return nil, err } - userIdentifier, found, err := c.resolveUserIdentifier(ctx, subject, providerSD, token) + userIdentifier, found, err := c.resolveUserIdentifier(ctx, req.Subject, providerSD, token) if err != nil { return nil, err } @@ -228,17 +290,28 @@ func (c *Client) check(ctx context.Context, subject, dataResource string, forceL return &ConsentResponse{Decision: DecisionDeny, Reason: "no user identifier for subject"}, nil } - granted, err := c.hasGrantedConsent(ctx, token, userIdentifier, dataResource) + granted, err := c.hasGrantedConsent(ctx, token, userIdentifier, req) if err != nil { return nil, err } if granted { return &ConsentResponse{Decision: DecisionAllow}, nil } - if dataResource != "" { - return &ConsentResponse{Decision: DecisionDeny, Reason: "no granted consent for resource " + dataResource}, nil + return &ConsentResponse{Decision: DecisionDeny, Reason: noGrantedConsentReason(req)}, nil +} + +// noGrantedConsentReason explains which scope the deny was decided at, so the +// audit record distinguishes "this subject consented to someone else" from +// "this subject did not consent to this resource". +func noGrantedConsentReason(req ConsentRequest) string { + reason := "no granted consent for consumer " + req.Consumer + if req.Purpose != "" { + reason += " and purpose " + req.Purpose + } + if req.DataResource != "" { + reason += " covering resource " + req.DataResource } - return &ConsentResponse{Decision: DecisionDeny, Reason: "no granted consent"}, nil + return reason } // credentials resolves the participant token and provider self-description, @@ -255,7 +328,7 @@ func (c *Client) credentials(ctx context.Context, forceFetch bool) (token, provi return c.staticToken, c.providerSD, nil } if c.staticToken == "" && c.tokenServiceURL == "" { - return "", "", fmt.Errorf("consent client: no participant_token and no token_service_url configured") + return "", "", ErrNoCredentials } // Get-or-create the per-key entry under the map lock (brief), then release it @@ -306,14 +379,50 @@ func (c *Client) credentials(ctx context.Context, forceFetch bool) (token, provi return token, providerSD, nil } +// identifierCacheTTL bounds how long a (provider, subject) -> userIdentifier +// mapping is reused. The mapping is stable for the life of the registration, so +// a short TTL is enough to collapse the repeated searches a single multi-owner +// response would otherwise make, without holding a stale identifier. +// +// Only POSITIVE results are cached. "Unknown subject" must be re-asked every +// time: a data subject can register at any moment, and remembering that they +// were unknown would keep denying them after they had consented. +const identifierCacheTTL = 60 * time.Second + +type identifierEntry struct { + userIdentifier string + expiry time.Time +} + +var ( + identifierMu sync.Mutex + identifierCache = map[string]identifierEntry{} +) + +// identifierCacheKey scopes a cached identifier to the credential identity and +// the provider it was resolved for - the identifier is provider-scoped, so it +// must never be reused across providers. +func (c *Client) identifierCacheKey(providerSD, subject string) string { + return strings.Join([]string{c.cacheKey(), providerSD, subject}, credentialKeySeparator) +} + // participantSDCacheTTL bounds how long a DID -> self-description mapping is // reused. Participants change rarely, so a generous TTL keeps the registry call // off the request path. const participantSDCacheTTL = 10 * time.Minute +// participantSDNegativeTTL bounds how long a "no such participant" answer is +// reused. Without it a single misconfigured DID re-fetches the whole participant +// list on every request; with it, a participant that is genuinely registered +// later is still picked up promptly. +const participantSDNegativeTTL = 30 * time.Second + type participantSDEntry struct { selfDescriptionURL string expiry time.Time + // unknown marks a negative result: the registry answered, and no participant + // with this DID was in it. + unknown bool } var ( @@ -336,21 +445,68 @@ type participantsResponse struct { // URL using the consent-manager's participant registry. Contracts name their // parties by self-description URL, so a DID taken from a credential must be // translated before it can be used in a contract lookup. Results are cached for -// participantSDCacheTTL. +// participantSDCacheTTL, and "no such participant" for participantSDNegativeTTL, +// so a misconfigured DID does not re-fetch the registry on every request. func (c *Client) ParticipantSelfDescriptionByDID(ctx context.Context, did string) (string, error) { if did == "" { return "", fmt.Errorf("consent client: empty participant did") } - cacheKey := c.baseURL + "|" + did + cacheKey := c.cacheKey() + credentialKeySeparator + did participantSDMu.Lock() entry, hit := participantSDCache[cacheKey] participantSDMu.Unlock() if hit && time.Now().Before(entry.expiry) { + if entry.unknown { + return "", notRegistered(did) + } return entry.selfDescriptionURL, nil } - token, _, err := c.credentials(ctx, false) + // A cached token that has since been revoked would otherwise make the mapping + // terminally fail, and with it the whole exchange — so refresh and retry once, + // exactly as CheckConsent does for the check itself. + sd, err := c.lookupParticipantSD(ctx, did, false) + if errors.Is(err, errParticipantUnauthorized) && c.staticToken == "" { + sd, err = c.lookupParticipantSD(ctx, did, true) + } + if errors.Is(err, errParticipantUnauthorized) { + return "", fmt.Errorf("consent client: participant token rejected (401) on participants lookup") + } + if err != nil { + return "", err + } + + participantSDMu.Lock() + if sd == "" { + participantSDCache[cacheKey] = participantSDEntry{unknown: true, expiry: time.Now().Add(participantSDNegativeTTL)} + } else { + participantSDCache[cacheKey] = participantSDEntry{selfDescriptionURL: sd, expiry: time.Now().Add(participantSDCacheTTL)} + } + participantSDMu.Unlock() + + if sd == "" { + return "", notRegistered(did) + } + return sd, nil +} + +// notRegistered builds the "no such participant" error, wrapping the sentinel so +// callers can tell a misconfigured DID from an unreachable registry. +// +// The DID is fingerprinted rather than embedded: this error is both logged to +// stdout and used as an audit reason, and a participant DID is an identifier +// that belongs in the audit record's own field, not in free text. +func notRegistered(did string) error { + return fmt.Errorf("%w: no participant registered for did %s", ErrParticipantNotRegistered, logging.Redact(did)) +} + +// lookupParticipantSD fetches the participant registry and returns the +// self-description URL registered for did, or "" when the registry answered but +// holds no such participant (a definite negative, not an error). forceLogin +// refreshes a cached token first. +func (c *Client) lookupParticipantSD(ctx context.Context, did string, forceLogin bool) (string, error) { + token, _, err := c.credentials(ctx, forceLogin) if err != nil { return "", err } @@ -368,7 +524,7 @@ func (c *Client) ParticipantSelfDescriptionByDID(ctx context.Context, did string return "", errParticipantUnauthorized } if status != http.StatusOK { - return "", fmt.Errorf("consent client: participants lookup returned status %d, body: %s", status, truncateBody(body)) + return "", unexpectedStatus("participants lookup", status, body) } participants, err := decodeParticipants(body) @@ -377,13 +533,10 @@ func (c *Client) ParticipantSelfDescriptionByDID(ctx context.Context, did string } for _, p := range participants { if p.DID == did && p.SelfDescriptionURL != "" { - participantSDMu.Lock() - participantSDCache[cacheKey] = participantSDEntry{selfDescriptionURL: p.SelfDescriptionURL, expiry: time.Now().Add(participantSDCacheTTL)} - participantSDMu.Unlock() return p.SelfDescriptionURL, nil } } - return "", fmt.Errorf("consent client: no participant registered for did %q", did) + return "", nil } // decodeParticipants accepts either a bare array or a {"participants": [...]} @@ -473,8 +626,7 @@ func (c *Client) fetchToken(ctx context.Context) (string, time.Duration, error) return "", 0, fmt.Errorf("consent client: failed to read token response: %w", err) } if resp.StatusCode != http.StatusOK { - return "", 0, fmt.Errorf("consent client: token service returned status %d, body: %s", - resp.StatusCode, truncateBody(body)) + return "", 0, unexpectedStatus("token service", resp.StatusCode, body) } var out tokenResponse if err := json.Unmarshal(body, &out); err != nil { @@ -507,8 +659,7 @@ func (c *Client) fetchProviderSD(ctx context.Context, token string) (string, err return "", errParticipantUnauthorized } if status != http.StatusOK { - return "", fmt.Errorf("consent client: participant lookup (/me) returned status %d, body: %s", - status, truncateBody(body)) + return "", unexpectedStatus("participant lookup (/me)", status, body) } var out meResponse if err := json.Unmarshal(body, &out); err != nil { @@ -529,11 +680,23 @@ type identifierSearchResponse struct { // user "email") to the provider-scoped user identifier. A 404 or empty identifier // means the subject is unknown (found == false). // +// A positive result is cached for identifierCacheTTL, so a response resolving to +// the same owner under several data resources searches once instead of once per +// resource. +// // It carries both the shared consent key (which the consent-manager's // consentKeyCheck validates) and the participant token as a Bearer credential, // so an authenticating facade in front of the consent-manager can validate the // participant JWT on this call too (the consent-manager ignores the Bearer here). func (c *Client) resolveUserIdentifier(ctx context.Context, subject, providerSD, token string) (identifier string, found bool, err error) { + cacheKey := c.identifierCacheKey(providerSD, subject) + identifierMu.Lock() + entry, hit := identifierCache[cacheKey] + identifierMu.Unlock() + if hit && time.Now().Before(entry.expiry) { + return entry.userIdentifier, true, nil + } + payload, err := json.Marshal(map[string]string{"selfDescription": providerSD, "email": subject}) if err != nil { return "", false, fmt.Errorf("consent client: failed to marshal identifier search: %w", err) @@ -566,35 +729,139 @@ func (c *Client) resolveUserIdentifier(ctx context.Context, subject, providerSD, return "", false, errParticipantUnauthorized } if status != http.StatusOK { - return "", false, fmt.Errorf("consent client: identifier search returned status %d, body: %s", - status, truncateBody(body)) + return "", false, unexpectedStatus("identifier search", status, body) } var out identifierSearchResponse if err := json.Unmarshal(body, &out); err != nil { return "", false, fmt.Errorf("consent client: failed to unmarshal identifier search response: %w", err) } - return out.UserIdentifier, out.UserIdentifier != "", nil + if out.UserIdentifier == "" { + return "", false, nil + } + identifierMu.Lock() + identifierCache[cacheKey] = identifierEntry{userIdentifier: out.UserIdentifier, expiry: time.Now().Add(identifierCacheTTL)} + identifierMu.Unlock() + return out.UserIdentifier, true, nil } // participantConsentsResponse is the consent-manager response to call 2. The -// ?receipt=true form returns the raw consents, each carrying its status and the -// data resources it covers. +// ?receipt=true form returns the raw consents, each carrying its status, the +// consumer it was granted to, the purposes it covers and the data resources it +// covers. type participantConsentsResponse struct { - Consents []struct { - Status string `json:"status"` - Data []struct { - Resource string `json:"resource"` - } `json:"data"` - } `json:"consents"` + Consents []consentRecord `json:"consents"` +} + +// consentRecord is the (subset of the) consent receipt the plugin decides on. +type consentRecord struct { + Status string `json:"status"` + Data []struct { + Resource string `json:"resource"` + } `json:"data"` + // Consumer / DataConsumer are the two field names the consent-manager has + // used for the participant the data is released to; either may be present. + Consumer participantRef `json:"consumer"` + DataConsumer participantRef `json:"dataConsumer"` + Purposes []struct { + ID string `json:"_id"` + Purpose string `json:"purpose"` + } `json:"purposes"` +} + +// participantRef is a participant named inside a consent record. The +// consent-manager returns it either as a bare identifier string or as an +// embedded object, so it decodes both shapes and matches on any of the +// identifiers it carries. +type participantRef struct { + ID string `json:"_id"` + DID string `json:"did"` + SelfDescriptionURL string `json:"selfDescriptionURL"` + // literal holds the value when the field was a bare string rather than an object. + literal string +} + +// UnmarshalJSON accepts either a bare identifier string or a participant object. +func (p *participantRef) UnmarshalJSON(data []byte) error { + var literal string + if err := json.Unmarshal(data, &literal); err == nil { + p.literal = literal + return nil + } + // Alias avoids recursing into this method while decoding the object form. + type participantRefObject participantRef + var obj participantRefObject + if err := json.Unmarshal(data, &obj); err != nil { + return fmt.Errorf("consent client: failed to unmarshal participant reference: %w", err) + } + *p = participantRef(obj) + return nil +} + +// matches reports whether this reference denotes the given participant identity +// (a self-description URL, but a record may name the participant by its id or +// DID instead). An empty reference matches nothing. +func (p participantRef) matches(identity string) bool { + if identity == "" { + return false + } + for _, candidate := range []string{p.SelfDescriptionURL, p.ID, p.DID, p.literal} { + if candidate != "" && candidate == identity { + return true + } + } + return false +} + +// grantedTo reports whether the consent was granted to the given consumer. +func (r consentRecord) grantedTo(consumer string) bool { + return r.Consumer.matches(consumer) || r.DataConsumer.matches(consumer) +} + +// coversPurpose reports whether the consent covers the given processing purpose. +// An empty purpose means the caller could not determine one, so the purpose is +// not part of the match. +func (r consentRecord) coversPurpose(purpose string) bool { + if purpose == "" { + return true + } + for _, p := range r.Purposes { + if p.Purpose == purpose || p.ID == purpose { + return true + } + } + return false +} + +// coversResource reports whether the consent covers the given data resource. An +// empty resource means the check is owner-level and any resource qualifies. +func (r consentRecord) coversResource(dataResource string) bool { + if dataResource == "" { + return true + } + for _, d := range r.Data { + if d.Resource == dataResource { + return true + } + } + return false } // hasGrantedConsent performs call 2: it lists the user identifier's consents as -// seen by the participant and reports whether any granted consent authorizes -// access. When dataResource is empty the check is owner-level (any granted -// consent suffices); otherwise a granted consent counts only if it covers that -// resource (dataResource ∈ consent.data[].resource). A 401 is returned as -// errParticipantUnauthorized so the caller can refresh the token and retry. -func (c *Client) hasGrantedConsent(ctx context.Context, token, userIdentifier, dataResource string) (bool, error) { +// seen by the participant and reports whether any of them authorizes THIS +// access. A consent qualifies only when all of the following hold: +// +// - its status is "granted"; +// - it was granted to req.Consumer — a consent names one consumer, and one +// granted to participant X is not authority for participant Y to read the +// same data; +// - it covers req.Purpose, when the caller could determine one; +// - it covers req.DataResource, when the check is resource-scoped. +// +// A record that names no consumer therefore never qualifies: the plugin cannot +// tell whose agreement it is, and guessing would authorise a processing purpose +// the subject never agreed to. A 401 is returned as errParticipantUnauthorized +// so the caller can refresh the token and retry. +func (c *Client) hasGrantedConsent(ctx context.Context, token, userIdentifier string, req ConsentRequest) (bool, error) { endpoint := c.endpoint(fmt.Sprintf(participantConsentsPathFmt, url.PathEscape(userIdentifier))) + "?receipt=true" httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) if err != nil { @@ -610,25 +877,19 @@ func (c *Client) hasGrantedConsent(ctx context.Context, token, userIdentifier, d return false, errParticipantUnauthorized } if status != http.StatusOK { - return false, fmt.Errorf("consent client: consents lookup returned status %d, body: %s", - status, truncateBody(body)) + return false, unexpectedStatus("consents lookup", status, body) } var out participantConsentsResponse if err := json.Unmarshal(body, &out); err != nil { return false, fmt.Errorf("consent client: failed to unmarshal consents response: %w", err) } - for _, consent := range out.Consents { - if consent.Status != grantedStatus { + for _, record := range out.Consents { + if record.Status != grantedStatus { continue } - if dataResource == "" { + if record.grantedTo(req.Consumer) && record.coversPurpose(req.Purpose) && record.coversResource(req.DataResource) { return true, nil } - for _, d := range consent.Data { - if d.Resource == dataResource { - return true, nil - } - } } return false, nil } @@ -659,13 +920,23 @@ func (c *Client) do(httpReq *http.Request) (statusCode int, body []byte, err err return resp.StatusCode, body, nil } -// maxBodyLogLength bounds error-body length in messages. -const maxBodyLogLength = 256 - -// truncateBody returns the response body as a string, truncated to maxBodyLogLength. -func truncateBody(body []byte) string { - if len(body) <= maxBodyLogLength { - return string(body) - } - return string(body[:maxBodyLogLength]) + "...(truncated)" +// unexpectedStatus builds the error for an unexpected response status from a +// dependency, and sends the response BODY to a debug log rather than into the +// error. +// +// These errors do not stay in the process: the plugin wraps them into the +// decision reason, which is exported to the audit sink and written to stdout. A +// consent-manager 500 that echoes the user identifier in its body would +// therefore land in both — and truncating it, as an earlier version did, is not +// redaction: the first surviving characters of a JSON error body are usually +// exactly the part with the identifiers in it. +// +// The error text is instead a stable, low-cardinality classification, which is +// what an audit reason wants anyway — it is queried, not read. The body is still +// available at debug level, rate-limited per operation so a failing dependency +// cannot flood the log with it. +func unexpectedStatus(operation string, status int, body []byte) error { + logging.DebugfEvery("dependency-body:"+operation, + "consent client: %s returned status %d, body: %s", operation, status, logging.Sanitize(string(body))) + return fmt.Errorf("consent client: %s returned status %d", operation, status) } diff --git a/internal/consent/client_test.go b/internal/consent/client_test.go index 7ebbe2a..5a1a7a2 100644 --- a/internal/consent/client_test.go +++ b/internal/consent/client_test.go @@ -37,6 +37,10 @@ const tokenServicePath = "/internal/tokens" // testAudience is the configured token-service target the tests ask for. const testAudience = "consent-manager" +// testConsumerSD is the self-description URL of the participant the data is +// released to — the consumer every check is scoped to. +const testConsumerSD = "http://catalog/participants/consumer" + // mockCM is a configurable mock covering the four endpoints the client uses: the // participant-local token service (/internal/tokens, served here for convenience // on the same test server), plus the consent-manager's /participants/me, @@ -48,6 +52,8 @@ type mockCM struct { statuses []string // consents statuses resourcesPerConsent [][]string // optional data[].resource per consent (index-aligned with statuses) selfDescriptionURL string // /me result + consentConsumer string // consumer the returned consents name (defaults to testConsumerSD) + consentPurposes []string // optional purposes the returned consents cover tokenStatus int // non-200 => the token service fails with this status failFirstConsents bool // first consents call 401s, then succeeds // recording @@ -120,6 +126,13 @@ func newMockCM(t *testing.T, m *mockCM) *httptest.Server { fail := m.failFirstConsents sts := append([]string(nil), m.statuses...) res := append([][]string(nil), m.resourcesPerConsent...) + consumer := m.consentConsumer + if consumer == "" { + consumer = testConsumerSD + } + // "none" makes the mock return a consent record that names no consumer. + omitConsumer := consumer == "none" + purposes := append([]string(nil), m.consentPurposes...) m.mu.Unlock() if fail && n == 1 { w.WriteHeader(http.StatusUnauthorized) @@ -128,6 +141,16 @@ func newMockCM(t *testing.T, m *mockCM) *httptest.Server { consents := make([]map[string]interface{}, 0, len(sts)) for i, s := range sts { consent := map[string]interface{}{"status": s} + if !omitConsumer { + consent["consumer"] = map[string]string{"selfDescriptionURL": consumer} + } + if len(purposes) > 0 { + ps := make([]map[string]string, 0, len(purposes)) + for _, p := range purposes { + ps = append(ps, map[string]string{"purpose": p}) + } + consent["purposes"] = ps + } if i < len(res) { data := make([]map[string]string, 0, len(res[i])) for _, r := range res[i] { @@ -146,12 +169,8 @@ func newMockCM(t *testing.T, m *mockCM) *httptest.Server { return srv } -// resetCredCache clears the package-wide credential cache between tests. -func resetCredCache() { - credCacheMu.Lock() - credCache = map[string]*cacheEntry{} - credCacheMu.Unlock() -} +// resetCredCache clears the package-wide caches between tests. +func resetCredCache() { ResetCaches() } // TestCheckConsent_HostOverride verifies the configured Host header is sent to // the consent-manager (for host-scoped gateway routes) while the connection @@ -161,7 +180,7 @@ func TestCheckConsent_HostOverride(t *testing.T) { m := &mockCM{userID: "uid-1", selfDescriptionURL: "http://provider/sd", statuses: []string{"granted"}} srv := newMockCM(t, m) c := NewClient(ClientConfig{BaseURL: srv.URL, Host: "consent-manager.dataspace-authority.org", TokenServiceURL: srv.URL + tokenServicePath, TokenAudience: testAudience, ConsentKey: "ck"}) - if _, err := c.CheckConsent(context.Background(), ConsentRequest{Subject: "alice-42"}); err != nil { + if _, err := c.CheckConsent(context.Background(), ConsentRequest{Subject: "alice-42", Consumer: testConsumerSD}); err != nil { t.Fatalf("CheckConsent: %v", err) } if m.lastHost != "consent-manager.dataspace-authority.org" { @@ -191,17 +210,17 @@ func TestCheckConsent_ResourceScoped(t *testing.T) { c := NewClient(ClientConfig{BaseURL: srv.URL, ConsentKey: "ck", TokenServiceURL: srv.URL + tokenServicePath, TokenAudience: testAudience}) // resource covered by a granted consent -> allow - resp, err := c.CheckConsent(context.Background(), ConsentRequest{Subject: "alice-42", DataResource: "urn:ngsi-ld:PersonalProfile:alice"}) + resp, err := c.CheckConsent(context.Background(), ConsentRequest{Subject: "alice-42", Consumer: testConsumerSD, DataResource: "urn:ngsi-ld:PersonalProfile:alice"}) require.NoError(t, err) assert.Equal(t, DecisionAllow, resp.Decision) // a different resource -> deny (the consent does not cover it) - resp, err = c.CheckConsent(context.Background(), ConsentRequest{Subject: "alice-42", DataResource: "urn:ngsi-ld:PersonalProfile:bob"}) + resp, err = c.CheckConsent(context.Background(), ConsentRequest{Subject: "alice-42", Consumer: testConsumerSD, DataResource: "urn:ngsi-ld:PersonalProfile:bob"}) require.NoError(t, err) assert.Equal(t, DecisionDeny, resp.Decision) // owner-level (no resource) -> allow on any granted consent - resp, err = c.CheckConsent(context.Background(), ConsentRequest{Subject: "alice-42"}) + resp, err = c.CheckConsent(context.Background(), ConsentRequest{Subject: "alice-42", Consumer: testConsumerSD}) require.NoError(t, err) assert.Equal(t, DecisionAllow, resp.Decision) } @@ -229,25 +248,43 @@ func TestCheckConsent(t *testing.T) { uid = "6a71e3567917ddaef2e2c866" ) tests := []struct { - name string - userID string - statuses []string - wantDecision Decision + name string + userID string + statuses []string + consentConsumer string // consumer the mock's consents were granted to + requestConsumer string // consumer the check is made for + wantDecision Decision }{ {name: "granted -> allow", userID: uid, statuses: []string{"granted"}, wantDecision: DecisionAllow}, {name: "one of many granted -> allow", userID: uid, statuses: []string{"revoked", "granted"}, wantDecision: DecisionAllow}, {name: "only revoked -> deny", userID: uid, statuses: []string{"revoked"}, wantDecision: DecisionDeny}, {name: "no consents -> deny", userID: uid, statuses: []string{}, wantDecision: DecisionDeny}, {name: "unknown subject (404) -> deny", userID: "", statuses: nil, wantDecision: DecisionDeny}, + { + name: "granted to another consumer -> deny", userID: uid, statuses: []string{"granted"}, + consentConsumer: "http://catalog/participants/someone-else", wantDecision: DecisionDeny, + }, + { + name: "consent naming no consumer -> deny", userID: uid, statuses: []string{"granted"}, + consentConsumer: "none", wantDecision: DecisionDeny, + }, + { + name: "no consumer in the request -> deny", userID: uid, statuses: []string{"granted"}, + requestConsumer: "none", wantDecision: DecisionDeny, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { resetCredCache() - m := &mockCM{userID: tt.userID, statuses: tt.statuses} + m := &mockCM{userID: tt.userID, statuses: tt.statuses, consentConsumer: tt.consentConsumer} srv := newMockCM(t, m) c := NewClient(ClientConfig{BaseURL: srv.URL, ConsentKey: "ck", ParticipantToken: "static-token", ProviderSD: provSD}) - resp, err := c.CheckConsent(context.Background(), ConsentRequest{Subject: subject}) + requestConsumer := testConsumerSD + if tt.requestConsumer == "none" { + requestConsumer = "" + } + resp, err := c.CheckConsent(context.Background(), ConsentRequest{Subject: subject, Consumer: requestConsumer}) require.NoError(t, err) assert.Equal(t, tt.wantDecision, resp.Decision) @@ -255,6 +292,10 @@ func TestCheckConsent(t *testing.T) { defer m.mu.Unlock() assert.Equal(t, 0, m.tokenCalls, "static token must not call the token service") assert.Equal(t, 0, m.meCalls, "static SD must not trigger /me") + if tt.requestConsumer == "none" { + assert.Equal(t, 0, m.searchCalls, "a check without a consumer must not reach the consent-manager") + return + } assert.Equal(t, "ck", m.lastConsentKey) assert.Equal(t, provSD, m.lastSearchSD) assert.Equal(t, subject, m.lastSearchEmail) @@ -273,7 +314,7 @@ func TestCheckConsent_TokenService(t *testing.T) { srv := newMockCM(t, m) c := NewClient(ClientConfig{BaseURL: srv.URL, ConsentKey: "ck", TokenServiceURL: srv.URL + tokenServicePath, TokenAudience: testAudience}) - resp, err := c.CheckConsent(context.Background(), ConsentRequest{Subject: "did:key:z"}) + resp, err := c.CheckConsent(context.Background(), ConsentRequest{Subject: "did:key:z", Consumer: testConsumerSD}) require.NoError(t, err) assert.Equal(t, DecisionAllow, resp.Decision) @@ -295,7 +336,7 @@ func TestCheckConsent_TokenAndSDCached(t *testing.T) { c := NewClient(ClientConfig{BaseURL: srv.URL, ConsentKey: "ck", TokenServiceURL: srv.URL + tokenServicePath, TokenAudience: testAudience}) for i := 0; i < 3; i++ { - resp, err := c.CheckConsent(context.Background(), ConsentRequest{Subject: "did:key:z"}) + resp, err := c.CheckConsent(context.Background(), ConsentRequest{Subject: "did:key:z", Consumer: testConsumerSD}) require.NoError(t, err) assert.Equal(t, DecisionAllow, resp.Decision) } @@ -315,7 +356,7 @@ func TestCheckConsent_401RefreshRetry(t *testing.T) { srv := newMockCM(t, m) c := NewClient(ClientConfig{BaseURL: srv.URL, ConsentKey: "ck", TokenServiceURL: srv.URL + tokenServicePath, TokenAudience: testAudience}) - resp, err := c.CheckConsent(context.Background(), ConsentRequest{Subject: "did:key:z"}) + resp, err := c.CheckConsent(context.Background(), ConsentRequest{Subject: "did:key:z", Consumer: testConsumerSD}) require.NoError(t, err) assert.Equal(t, DecisionAllow, resp.Decision) @@ -333,7 +374,7 @@ func TestCheckConsent_TokenServiceFailure(t *testing.T) { srv := newMockCM(t, m) c := NewClient(ClientConfig{BaseURL: srv.URL, ConsentKey: "ck", TokenServiceURL: srv.URL + tokenServicePath, TokenAudience: testAudience}) - _, err := c.CheckConsent(context.Background(), ConsentRequest{Subject: "did:key:z"}) + _, err := c.CheckConsent(context.Background(), ConsentRequest{Subject: "did:key:z", Consumer: testConsumerSD}) require.Error(t, err) assert.Contains(t, err.Error(), "token service returned status 404") } @@ -346,7 +387,7 @@ func TestCheckConsent_ProviderSDOverride(t *testing.T) { srv := newMockCM(t, m) c := NewClient(ClientConfig{BaseURL: srv.URL, ConsentKey: "ck", TokenServiceURL: srv.URL + tokenServicePath, TokenAudience: testAudience, ProviderSD: "http://facade/explicit"}) - resp, err := c.CheckConsent(context.Background(), ConsentRequest{Subject: "did:key:z"}) + resp, err := c.CheckConsent(context.Background(), ConsentRequest{Subject: "did:key:z", Consumer: testConsumerSD}) require.NoError(t, err) assert.Equal(t, DecisionAllow, resp.Decision) @@ -364,7 +405,7 @@ func TestCheckConsent_EmptySubject(t *testing.T) { srv := newMockCM(t, m) c := NewClient(ClientConfig{BaseURL: srv.URL, ConsentKey: "ck", TokenServiceURL: srv.URL + tokenServicePath, TokenAudience: testAudience}) - resp, err := c.CheckConsent(context.Background(), ConsentRequest{Subject: ""}) + resp, err := c.CheckConsent(context.Background(), ConsentRequest{Subject: "", Consumer: testConsumerSD}) require.NoError(t, err) assert.Equal(t, DecisionDeny, resp.Decision) m.mu.Lock() @@ -378,7 +419,7 @@ func TestCheckConsent_EmptySubject(t *testing.T) { func TestCheckConsent_MissingTokenSource(t *testing.T) { resetCredCache() c := NewClient(ClientConfig{BaseURL: "http://cm:3000", ConsentKey: "ck"}) - _, err := c.CheckConsent(context.Background(), ConsentRequest{Subject: "did:key:z"}) + _, err := c.CheckConsent(context.Background(), ConsentRequest{Subject: "did:key:z", Consumer: testConsumerSD}) require.Error(t, err) assert.Contains(t, err.Error(), "no participant_token and no token_service_url") } @@ -391,7 +432,7 @@ func TestCheckConsent_EmptyConsentKeyOmitsHeader(t *testing.T) { m := &mockCM{userID: "uid-1", statuses: []string{"granted"}} srv := newMockCM(t, m) c := NewClient(ClientConfig{BaseURL: srv.URL, ParticipantToken: "t", ProviderSD: "sd"}) - resp, err := c.CheckConsent(context.Background(), ConsentRequest{Subject: "did:key:z"}) + resp, err := c.CheckConsent(context.Background(), ConsentRequest{Subject: "did:key:z", Consumer: testConsumerSD}) require.NoError(t, err) assert.Equal(t, DecisionAllow, resp.Decision) assert.Empty(t, m.lastConsentKey, "empty consent key must not be sent as a header") @@ -414,7 +455,7 @@ func TestCheckConsent_ConcurrentTokenFetchCoalesced(t *testing.T) { defer wg.Done() // Same base URL + client id => same cache key, so the login must coalesce. c := NewClient(ClientConfig{BaseURL: srv.URL, TokenServiceURL: srv.URL + tokenServicePath, TokenAudience: testAudience}) - resp, err := c.CheckConsent(context.Background(), ConsentRequest{Subject: "did:key:z"}) + resp, err := c.CheckConsent(context.Background(), ConsentRequest{Subject: "did:key:z", Consumer: testConsumerSD}) if err != nil { errs <- err } else if resp.Decision != DecisionAllow { @@ -438,7 +479,7 @@ func TestCheckConsent_ConcurrentTokenFetchCoalesced(t *testing.T) { func TestCheckConsentTransportFailure(t *testing.T) { resetCredCache() c := NewClient(ClientConfig{BaseURL: "http://localhost:1", ConsentKey: "ck", ParticipantToken: "t", ProviderSD: "sd", TimeoutMs: MinTimeoutMs}) - _, err := c.CheckConsent(context.Background(), ConsentRequest{Subject: "did:key:z"}) + _, err := c.CheckConsent(context.Background(), ConsentRequest{Subject: "did:key:z", Consumer: testConsumerSD}) require.Error(t, err) assert.Contains(t, err.Error(), "HTTP request failed") } @@ -453,41 +494,439 @@ func TestCheckConsentContextCancellation(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() c := NewClient(ClientConfig{BaseURL: srv.URL, ConsentKey: "ck", ParticipantToken: "t", ProviderSD: "sd"}) - _, err := c.CheckConsent(ctx, ConsentRequest{Subject: "did:key:z"}) + _, err := c.CheckConsent(ctx, ConsentRequest{Subject: "did:key:z", Consumer: testConsumerSD}) require.Error(t, err) assert.Contains(t, err.Error(), "HTTP request failed") } -// TestDecisionIsValid verifies the Decision.IsValid method. -func TestDecisionIsValid(t *testing.T) { - assert.True(t, DecisionAllow.IsValid()) - assert.True(t, DecisionDeny.IsValid()) - assert.True(t, DecisionFilter.IsValid()) - assert.False(t, Decision("").IsValid()) - assert.False(t, Decision("maybe").IsValid()) +// TestCacheKeyDistinguishesCredentialIdentities verifies that two clients that +// differ in any input feeding the cached token or provider self-description get +// distinct cache keys. Sharing an entry across credential identities would make +// one route run its lookups as the wrong participant (see cacheKey). +func TestCacheKeyDistinguishesCredentialIdentities(t *testing.T) { + base := ClientConfig{ + BaseURL: "http://consent-manager:3000", + Host: "consent.example.org", + APIPrefix: "/v1", + ConsentKey: "ck-a", + ProviderSD: "http://facade/participants/a", + TokenServiceURL: "http://facade-a:8080/internal/tokens", + TokenAudience: testAudience, + } + + tests := []struct { + name string + mutate func(cfg *ClientConfig) + }{ + {"base url", func(cfg *ClientConfig) { cfg.BaseURL = "http://other-manager:3000" }}, + {"host", func(cfg *ClientConfig) { cfg.Host = "other.example.org" }}, + {"api prefix", func(cfg *ClientConfig) { cfg.APIPrefix = "/v2" }}, + {"consent key", func(cfg *ClientConfig) { cfg.ConsentKey = "ck-b" }}, + {"provider sd", func(cfg *ClientConfig) { cfg.ProviderSD = "http://facade/participants/b" }}, + {"token service url", func(cfg *ClientConfig) { cfg.TokenServiceURL = "http://facade-b:8080/internal/tokens" }}, + {"token audience", func(cfg *ClientConfig) { cfg.TokenAudience = "other-audience" }}, + {"static token", func(cfg *ClientConfig) { cfg.ParticipantToken = "static-b" }}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + other := base + tc.mutate(&other) + assert.NotEqual(t, NewClient(base).cacheKey(), NewClient(other).cacheKey(), + "clients differing in %s must not share a credential cache entry", tc.name) + }) + } + + t.Run("identical config shares an entry", func(t *testing.T) { + assert.Equal(t, NewClient(base).cacheKey(), NewClient(base).cacheKey()) + }) + + t.Run("secrets are not embedded verbatim", func(t *testing.T) { + withSecrets := base + withSecrets.ParticipantToken = "super-secret-token" + key := NewClient(withSecrets).cacheKey() + assert.NotContains(t, key, "super-secret-token") + assert.NotContains(t, key, "ck-a") + }) } -// TestConsentResponseValidate verifies the Validate method on ConsentResponse. -func TestConsentResponseValidate(t *testing.T) { - require.NoError(t, (&ConsentResponse{Decision: DecisionAllow}).Validate()) - require.NoError(t, (&ConsentResponse{Decision: DecisionDeny, Reason: "no consent"}).Validate()) +// TestCheckConsent_SeparateTokenServicesDoNotShareToken is the regression test +// for the cross-participant cache collision: two routes fronting the SAME +// consent-manager but authenticating through their own token service must each +// present their own participant token. +func TestCheckConsent_SeparateTokenServicesDoNotShareToken(t *testing.T) { + resetCredCache() - err := (&ConsentResponse{Decision: ""}).Validate() - require.Error(t, err) - assert.Contains(t, err.Error(), "decision field is empty") + var mu sync.Mutex + var consentsAuth []string - err = (&ConsentResponse{Decision: "block"}).Validate() - require.Error(t, err) - assert.Contains(t, err.Error(), "unrecognized decision") + mux := http.NewServeMux() + mux.HandleFunc("/v1/users/identifier/search", func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]string{"userIdentifier": "uid-1"}) + }) + mux.HandleFunc("/v1/consents/participants/", func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + consentsAuth = append(consentsAuth, r.Header.Get("Authorization")) + mu.Unlock() + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]interface{}{ + "consents": []map[string]interface{}{ + {"status": grantedStatus, "consumer": map[string]string{"selfDescriptionURL": testConsumerSD}}, + }, + }) + }) + consentManager := httptest.NewServer(mux) + t.Cleanup(consentManager.Close) + + // One token service per participant, each minting a distinguishable token. + newTokenService := func(token string) *httptest.Server { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]interface{}{ + "access_token": token, "token_type": "Bearer", "expires_in": 3600, + }) + })) + t.Cleanup(srv.Close) + return srv + } + + for _, token := range []string{"token-participant-a", "token-participant-b"} { + c := NewClient(ClientConfig{ + BaseURL: consentManager.URL, + ConsentKey: "ck", + ProviderSD: "http://facade/participants/" + token, + TokenServiceURL: newTokenService(token).URL, + TokenAudience: testAudience, + }) + resp, err := c.CheckConsent(context.Background(), ConsentRequest{Subject: "did:key:z", Consumer: testConsumerSD}) + require.NoError(t, err) + require.Equal(t, DecisionAllow, resp.Decision) + } + + mu.Lock() + defer mu.Unlock() + assert.Equal(t, []string{"Bearer token-participant-a", "Bearer token-participant-b"}, consentsAuth, + "each participant must authenticate with its own token, not the first one cached") +} + +// --- Participant registry lookup (the contract-side party mapping) --- + +// newParticipantRegistry starts a mock consent-manager participant registry that +// counts its calls and can 401 the first one. +func newParticipantRegistry(t *testing.T, entries []map[string]string, calls *int, fail401First *bool) *httptest.Server { + t.Helper() + var mu sync.Mutex + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + mu.Lock() + *calls++ + n := *calls + mu.Unlock() + if fail401First != nil && *fail401First && n == 1 { + w.WriteHeader(http.StatusUnauthorized) + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(entries) + })) + t.Cleanup(srv.Close) + return srv +} + +// TestParticipantSelfDescriptionByDID_401RefreshRetry verifies a rejected cached +// token is refreshed and the registry lookup retried, rather than failing +// terminally — a stale token must not take the whole exchange down. +func TestParticipantSelfDescriptionByDID_401RefreshRetry(t *testing.T) { + resetCredCache() + + var registryCalls int + fail := true + registry := newParticipantRegistry(t, + []map[string]string{{"did": "did:key:zConsumer", "selfDescriptionURL": testConsumerSD}}, + ®istryCalls, &fail) + + tokenSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]interface{}{ + "access_token": "tok", "token_type": "Bearer", "expires_in": 3600, + }) + })) + t.Cleanup(tokenSrv.Close) + + c := NewClient(ClientConfig{ + BaseURL: registry.URL, APIPrefix: "", ProviderSD: "sd", + TokenServiceURL: tokenSrv.URL, TokenAudience: testAudience, + }) + + sd, err := c.ParticipantSelfDescriptionByDID(context.Background(), "did:key:zConsumer") + require.NoError(t, err) + assert.Equal(t, testConsumerSD, sd) + assert.Equal(t, 2, registryCalls, "the 401 must trigger one refresh and retry") +} + +// TestParticipantSelfDescriptionByDID_NegativeCaching verifies an unknown DID is +// remembered as unknown, so a misconfiguration does not re-fetch the whole +// participant list on every request. +func TestParticipantSelfDescriptionByDID_NegativeCaching(t *testing.T) { + resetCredCache() + + var registryCalls int + registry := newParticipantRegistry(t, []map[string]string{}, ®istryCalls, nil) + + c := NewClient(ClientConfig{ + BaseURL: registry.URL, APIPrefix: "", ParticipantToken: "static", ProviderSD: "sd", + }) + + for i := 0; i < 3; i++ { + _, err := c.ParticipantSelfDescriptionByDID(context.Background(), "did:key:zUnregistered") + require.Error(t, err) + assert.Contains(t, err.Error(), "no participant registered") + } + assert.Equal(t, 1, registryCalls, "an unknown DID must be remembered, not re-fetched per request") +} + +// TestDecodeParticipants verifies both registry shapes are accepted: the +// consent-manager has returned a bare array and a wrapped list at different +// times, and a plugin that understands only one silently loses the ability to +// identify the consumer. +func TestDecodeParticipants(t *testing.T) { + tests := []struct { + name string + body string + want []participantListEntry + wantErr bool + }{ + { + name: "bare array", + body: `[{"did":"did:key:zA","selfDescriptionURL":"http://catalog/a"}]`, + want: []participantListEntry{{DID: "did:key:zA", SelfDescriptionURL: "http://catalog/a"}}, + }, + { + name: "wrapped list", + body: `{"participants":[{"did":"did:key:zB","selfDescriptionURL":"http://catalog/b"}]}`, + want: []participantListEntry{{DID: "did:key:zB", SelfDescriptionURL: "http://catalog/b"}}, + }, + { + name: "empty array", + body: `[]`, + want: []participantListEntry{}, + }, + { + name: "not JSON at all", + body: `gateway error`, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := decodeParticipants([]byte(tt.body)) + if tt.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} + +// TestResolveUserIdentifier_MemoisedPerSubject verifies the subject -> +// userIdentifier mapping is reused, so one response resolving to the same owner +// under several data resources searches once instead of once per resource. +func TestResolveUserIdentifier_MemoisedPerSubject(t *testing.T) { + resetCredCache() + m := &mockCM{userID: "uid-1", statuses: []string{"granted"}, resourcesPerConsent: [][]string{{"r1", "r2"}}} + srv := newMockCM(t, m) + c := NewClient(ClientConfig{BaseURL: srv.URL, ConsentKey: "ck", ParticipantToken: "static", ProviderSD: "sd"}) + + for _, resource := range []string{"r1", "r2"} { + resp, err := c.CheckConsent(context.Background(), + ConsentRequest{Subject: "did:key:zOwner", Consumer: testConsumerSD, DataResource: resource}) + require.NoError(t, err) + assert.Equal(t, DecisionAllow, resp.Decision) + } + + m.mu.Lock() + defer m.mu.Unlock() + assert.Equal(t, 1, m.searchCalls, "the same owner must be resolved to an identifier once") + assert.Equal(t, 2, m.consentsCalls, "each resource still needs its own consent decision") +} + +// TestResolveUserIdentifier_UnknownSubjectNotCached verifies an unknown subject +// is re-asked every time. A data subject can register at any moment, and +// remembering that they were unknown would keep denying them after they had +// consented. +func TestResolveUserIdentifier_UnknownSubjectNotCached(t *testing.T) { + resetCredCache() + m := &mockCM{userID: "", statuses: nil} + srv := newMockCM(t, m) + c := NewClient(ClientConfig{BaseURL: srv.URL, ConsentKey: "ck", ParticipantToken: "static", ProviderSD: "sd"}) + + for i := 0; i < 3; i++ { + resp, err := c.CheckConsent(context.Background(), + ConsentRequest{Subject: "did:key:zStranger", Consumer: testConsumerSD}) + require.NoError(t, err) + assert.Equal(t, DecisionDeny, resp.Decision) + } + + m.mu.Lock() + defer m.mu.Unlock() + assert.Equal(t, 3, m.searchCalls, "an unknown subject must not be cached as unknown") } -// TestTruncateBody verifies the body truncation helper. -func TestTruncateBody(t *testing.T) { - assert.Equal(t, "short", truncateBody([]byte("short"))) - assert.Equal(t, "", truncateBody([]byte{})) - assert.NotContains(t, truncateBody(make([]byte, maxBodyLogLength)), "...(truncated)") +// TestCheckConsent_PurposeScoped verifies the purpose leg of the consumer / +// purpose / resource match. It is load-bearing once the resolver names a +// purpose: a consent granted for "insurance quote" must not authorise a release +// for research. +func TestCheckConsent_PurposeScoped(t *testing.T) { + const ( + grantedPurpose = "insurance-quote" + otherPurpose = "research" + ) - long := truncateBody(make([]byte, maxBodyLogLength+100)) - assert.Contains(t, long, "...(truncated)") - assert.Equal(t, maxBodyLogLength+len("...(truncated)"), len(long)) + tests := []struct { + name string + consentPurposes []string + requestPurpose string + wantDecision Decision + }{ + { + name: "matching purpose allows", + consentPurposes: []string{grantedPurpose}, + requestPurpose: grantedPurpose, + wantDecision: DecisionAllow, + }, + { + name: "a different purpose denies", + consentPurposes: []string{grantedPurpose}, + requestPurpose: otherPurpose, + wantDecision: DecisionDeny, + }, + { + name: "one of several purposes matching allows", + consentPurposes: []string{otherPurpose, grantedPurpose}, + requestPurpose: grantedPurpose, + wantDecision: DecisionAllow, + }, + { + name: "a consent covering no purpose denies a purpose-scoped check", + consentPurposes: nil, + requestPurpose: grantedPurpose, + wantDecision: DecisionDeny, + }, + { + // The caller could not determine a purpose, so it is not part of the + // match; the consumer match still applies. + name: "no requested purpose leaves the check unscoped", + consentPurposes: []string{grantedPurpose}, + requestPurpose: "", + wantDecision: DecisionAllow, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + resetCredCache() + m := &mockCM{userID: "uid-1", statuses: []string{"granted"}, consentPurposes: tt.consentPurposes} + srv := newMockCM(t, m) + c := NewClient(ClientConfig{BaseURL: srv.URL, ConsentKey: "ck", ParticipantToken: "static", ProviderSD: "sd"}) + + resp, err := c.CheckConsent(context.Background(), ConsentRequest{ + Subject: "did:key:zOwner", Consumer: testConsumerSD, Purpose: tt.requestPurpose, + }) + require.NoError(t, err) + assert.Equal(t, tt.wantDecision, resp.Decision) + if tt.wantDecision == DecisionDeny { + assert.Contains(t, resp.Reason, "no granted consent") + } + }) + } +} + +// TestFetchProviderSD_Failures covers the derivation of the provider +// self-description from /participants/me. It runs before any consent check, so +// a failure here takes the whole exchange down (fail-closed) — the branches are +// worth pinning. +func TestFetchProviderSD_Failures(t *testing.T) { + tests := []struct { + name string + status int + body string + wantErr string + wantNoBody bool + }{ + { + name: "a 401 is reported as unauthorized so the caller can refresh", + status: http.StatusUnauthorized, + body: `{}`, + // CheckConsent maps the retried-and-still-401 case to this message. + wantErr: "participant token rejected (401)", + }, + { + name: "a 500 is classified without its body", + status: http.StatusInternalServerError, + body: `{"error":"boom for alice@example.org"}`, + wantErr: "participant lookup (/me) returned status 500", + }, + { + name: "an unparseable body errors", + status: http.StatusOK, + body: `not json`, + wantErr: "failed to unmarshal /me response", + }, + { + name: "an empty selfDescriptionURL errors", + status: http.StatusOK, + body: `{"selfDescriptionURL":""}`, + wantErr: "/me returned no selfDescriptionURL", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + resetCredCache() + + mux := http.NewServeMux() + mux.HandleFunc("/v1/participants/me", func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(tt.status) + _, _ = w.Write([]byte(tt.body)) + }) + mux.HandleFunc(tokenServicePath, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]interface{}{ + "access_token": "tok", "token_type": "Bearer", "expires_in": 3600, + }) + }) + srv := httptest.NewServer(mux) + t.Cleanup(srv.Close) + + c := NewClient(ClientConfig{BaseURL: srv.URL, ConsentKey: "ck", TokenServiceURL: srv.URL + tokenServicePath, TokenAudience: testAudience}) + + _, err := c.CheckConsent(context.Background(), ConsentRequest{Subject: "did:key:z", Consumer: testConsumerSD}) + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantErr) + assert.NotContains(t, err.Error(), "alice@example.org", + "a dependency response body must not travel in the error") + }) + } +} + +// TestProviderSelfDescription_StaticOverride verifies a configured provider SD +// is returned without any HTTP call at all. +func TestProviderSelfDescription_StaticOverride(t *testing.T) { + resetCredCache() + m := &mockCM{userID: "uid-1", statuses: []string{"granted"}} + srv := newMockCM(t, m) + c := NewClient(ClientConfig{BaseURL: srv.URL, ParticipantToken: "static", ProviderSD: "http://catalog/participants/static"}) + + sd, err := c.ProviderSelfDescription(context.Background()) + require.NoError(t, err) + assert.Equal(t, "http://catalog/participants/static", sd) + + m.mu.Lock() + defer m.mu.Unlock() + assert.Equal(t, 0, m.meCalls, "a static provider SD must not trigger /me") + assert.Equal(t, 0, m.tokenCalls, "a static token must not trigger the token service") } diff --git a/internal/consent/models.go b/internal/consent/models.go index 9d7dc28..ee7cdb9 100644 --- a/internal/consent/models.go +++ b/internal/consent/models.go @@ -15,14 +15,13 @@ * limitations under the License. */ -// Package consent provides an HTTP client for communicating with an external -// consent API that determines whether response data should be allowed, denied, -// or filtered based on consent policies for personal data. +// Package consent provides an HTTP client for the external consent-manager, +// which decides whether a data owner has granted the consuming participant +// consent to access their personal data. The verdict is coarse — allow or deny +// for the whole response — and is enforced by the plugin's response phase. package consent -import "fmt" - -// Decision represents the consent API's verdict on a request. +// Decision represents the consent-manager's verdict on one data owner. // It determines how the plugin handles the upstream response. type Decision string @@ -34,29 +33,13 @@ const ( // DecisionDeny indicates the response should be blocked entirely, // returning a configured error status and body to the client. DecisionDeny Decision = "deny" - - // DecisionFilter indicates the response should be modified by removing - // specific fields identified in the DeniedFields list. - DecisionFilter Decision = "filter" ) -// validDecisions is the set of recognized Decision values, used for validation. -var validDecisions = map[Decision]bool{ - DecisionAllow: true, - DecisionDeny: true, - DecisionFilter: true, -} - -// IsValid reports whether d is a recognized Decision value. -func (d Decision) IsValid() bool { - return validDecisions[d] -} - -// ConsentRequest represents the payload sent to the consent API's /check endpoint. -// It contains information about the original request and the response fields -// so the consent API can make an informed allow/deny/filter decision. +// ConsentRequest is one consent question: may this consumer be given this data +// owner's data, for this purpose and resource? type ConsentRequest struct { - // Subject is the identity of the requester, typically from the JWT "sub" claim. + // Subject is the DATA OWNER whose consent decides the access — resolved from + // the response payload by the OwnerResolver. It is never the requestor. Subject string `json:"subject"` // Resource is the request path being accessed (e.g., "/api/v1/users/123"). @@ -70,38 +53,26 @@ type ConsentRequest struct { // Empty means owner-level (any granted consent counts). DataResource string `json:"data_resource,omitempty"` - // Claims contains the forwarded JWT claims as key-value pairs. - Claims map[string]interface{} `json:"claims,omitempty"` - - // ResponseFields lists the top-level field names found in the upstream - // response body, enabling field-level consent decisions. - ResponseFields []string `json:"response_fields,omitempty"` + // Consumer identifies the participant the data is being released TO, as its + // self-description URL. It is REQUIRED: a consent is an agreement between a + // data subject and one named consumer for one named purpose, so a check that + // ignores it would let participant Y ride on a consent the subject granted to + // participant X. A check without a consumer is denied. + Consumer string `json:"consumer,omitempty"` + + // Purpose, when set, further scopes the check to the processing purpose (or + // contract) the exchange is governed by: a granted consent counts only if it + // covers this purpose. Empty means the purpose is not known — the consumer + // match still applies. + Purpose string `json:"purpose,omitempty"` } -// ConsentResponse represents the payload returned by the consent API's /check endpoint. -// It contains the consent decision and any additional information about which -// fields to remove or the reason for the decision. +// ConsentResponse is the verdict for one ConsentRequest. type ConsentResponse struct { - // Decision is the consent verdict: "allow", "deny", or "filter". + // Decision is the consent verdict: "allow" or "deny". Decision Decision `json:"decision"` - // DeniedFields lists the field names or dot-notation paths (e.g., "user.email") - // that should be removed from the response body when Decision is "filter". - DeniedFields []string `json:"denied_fields,omitempty"` - // Reason is a human-readable explanation for the consent decision, - // useful for logging and debugging. + // recorded in the audit log and useful for debugging. Reason string `json:"reason,omitempty"` } - -// Validate checks that the ConsentResponse contains a valid decision. -// Returns an error if the decision field is empty or unrecognized. -func (r *ConsentResponse) Validate() error { - if r.Decision == "" { - return fmt.Errorf("consent response validation: decision field is empty") - } - if !r.Decision.IsValid() { - return fmt.Errorf("consent response validation: unrecognized decision %q", r.Decision) - } - return nil -} diff --git a/internal/consent/reset.go b/internal/consent/reset.go new file mode 100644 index 0000000..ad27c91 --- /dev/null +++ b/internal/consent/reset.go @@ -0,0 +1,42 @@ +/* + * Copyright 2026 Seamless Middleware Technologies S.L and/or its affiliates + * and other contributors as indicated by the @author tags. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package consent + +// ResetCaches drops every package-wide cache: the participant credentials +// (token + derived provider self-description), the DID -> self-description +// mappings, and the subject -> user-identifier mappings. +// +// It exists for tests. They currently pass only because httptest allocates a +// fresh base URL per server, which happens to produce a fresh cache key; a test +// that reuses a URL, or one that runs in parallel with another, would otherwise +// see another test's token and fail in an order-dependent way. Production code +// must not call this: dropping a live token mid-flight only causes a re-fetch, +// but there is no reason to. +func ResetCaches() { + credCacheMu.Lock() + credCache = map[string]*cacheEntry{} + credCacheMu.Unlock() + + participantSDMu.Lock() + participantSDCache = map[string]participantSDEntry{} + participantSDMu.Unlock() + + identifierMu.Lock() + identifierCache = map[string]identifierEntry{} + identifierMu.Unlock() +} diff --git a/internal/integration/integration_test.go b/internal/integration/integration_test.go index 138d3d3..0861ea2 100644 --- a/internal/integration/integration_test.go +++ b/internal/integration/integration_test.go @@ -22,6 +22,7 @@ package integration import ( + "consent-plugin/internal/consent" "consent-plugin/internal/plugin" "context" "encoding/base64" @@ -32,6 +33,8 @@ import ( "net/http/httptest" "net/url" "strconv" + "strings" + "sync" "testing" pkgHTTP "github.com/apache/apisix-go-plugin-runner/pkg/http" @@ -144,15 +147,20 @@ func (r *mockResponse) ID() uint32 { return r.id } func (r *mockResponse) StatusCode() int { return http.StatusOK } func (r *mockResponse) Header() pkgHTTP.Header { return r.header } func (r *mockResponse) Var(name string) ([]byte, error) { - if name == "request_id" { + switch name { + case "request_id": return []byte(integrationReqKey(r.id)), nil + case "upstream_http_content_type": + return []byte(responseContentTypeJSON), nil } return nil, nil } func (r *mockResponse) ReadBody() ([]byte, error) { return r.body, nil } func (r *mockResponse) WriteHeader(statusCode int) { r.writtenStatus = statusCode } + +// Write appends, as the runner's Response.Write does (it writes into a buffer). func (r *mockResponse) Write(b []byte) (int, error) { - r.writtenBody = b + r.writtenBody = append(r.writtenBody, b...) return len(b), nil } @@ -196,26 +204,128 @@ func newConsentManager(t *testing.T, wantSubject, userID string, statuses []stri mux.HandleFunc("/v1/consents/participants/", func(w http.ResponseWriter, r *http.Request) { assert.Equal(t, "Bearer itest-participant-token", r.Header.Get("Authorization"), "consents lookup must carry the participant token") - consents := make([]map[string]string, 0, len(statuses)) - for _, s := range statuses { - consents = append(consents, map[string]string{"status": s}) - } w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(map[string]interface{}{"consents": consents}) + _ = json.NewEncoder(w).Encode(map[string]interface{}{"consents": consentsGrantedTo(statuses)}) }) + mux.HandleFunc("/v1/participants", participantRegistryHandler) + + return httptest.NewServer(mux) +} + +// --- OwnerResolver mock (the source of data ownership) --- + +// itestConsumerDID is the consuming participant named in the access token. +const itestConsumerDID = "did:key:zConsumer" + +// itestConsumerSD is the self-description URL the participant registry maps +// itestConsumerDID to — the consumer every consent check is scoped to. +const itestConsumerSD = "http://catalog/participants/consumer" + +// consentsGrantedTo builds consent records with the given statuses, each granted +// to the consuming participant the tests act as. +func consentsGrantedTo(statuses []string) []map[string]interface{} { + consents := make([]map[string]interface{}, 0, len(statuses)) + for _, s := range statuses { + consents = append(consents, map[string]interface{}{ + "status": s, + "consumer": map[string]string{"selfDescriptionURL": itestConsumerSD}, + }) + } + return consents +} + +// participantRegistryHandler serves the consent-manager's participant registry, +// which translates the consumer DID from the token into the self-description URL +// a contract names its parties by. +func participantRegistryHandler(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode([]map[string]string{ + {"did": itestConsumerDID, "selfDescriptionURL": itestConsumerSD}, + }) +} + +// newFailingConsentManager returns a consent-manager whose CONSENT CHECK calls +// answer with the given status code (used to exercise the fail policy). The +// participant registry still answers, so the failure under test is the check +// itself and not the preceding contract lookup. +func newFailingConsentManager(status int) *httptest.Server { + mux := http.NewServeMux() + mux.HandleFunc("/v1/participants", participantRegistryHandler) + mux.HandleFunc("/", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(status) + }) return httptest.NewServer(mux) } +// resolveEnvelope is the /resolve request the plugin sends, decoded so tests can +// assert on what the resolver was actually told. +type resolveEnvelope struct { + Resource struct { + Service string `json:"service"` + Method string `json:"method"` + Path string `json:"path"` + ContentType string `json:"contentType"` + } `json:"resource"` + Parties *struct { + Consumer string `json:"consumer"` + Provider string `json:"provider"` + } `json:"parties"` + Body *struct { + Encoding string `json:"encoding"` + Content json.RawMessage `json:"content"` + } `json:"body"` +} + +// newRecordingOwnerResolver starts a mock OwnerResolver that answers with the +// given JSON and records every envelope it received. +func newRecordingOwnerResolver(t *testing.T, reply map[string]interface{}, received *[]resolveEnvelope) *httptest.Server { + t.Helper() + var mu sync.Mutex + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var env resolveEnvelope + if err := json.NewDecoder(r.Body).Decode(&env); err != nil { + t.Errorf("failed to decode resolve request: %v", err) + } + mu.Lock() + *received = append(*received, env) + mu.Unlock() + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(reply); err != nil { + t.Errorf("failed to encode resolve response: %v", err) + } + })) +} + +// newOwnerResolver starts a mock OwnerResolver that reports the given data +// owners for every payload. With no owners, it reports that no consent is +// required. +func newOwnerResolver(t *testing.T, owners ...string) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + claims := make([]map[string]string, 0, len(owners)) + for _, o := range owners { + claims = append(claims, map[string]string{"ownerId": o}) + } + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(map[string]interface{}{ + "consentRequired": len(owners) > 0, + "claims": claims, + }); err != nil { + t.Errorf("failed to encode resolve response: %v", err) + } + })) +} + // baseConfig returns the minimal valid plugin configuration for the two-call -// check, pointing at the given consent-manager URL. -func baseConfig(consentURL string) map[string]interface{} { +// check, pointing at the given consent-manager and OwnerResolver. +func baseConfig(consentURL, resolverURL string) map[string]interface{} { return map[string]interface{}{ - "consent_api_url": consentURL, - "consent_key": "itest-consent-key", - "participant_token": "itest-participant-token", - "provider_sd": "http://consent-facade:8080/participants/org-itest", - "jwt_claims_to_forward": []string{"sub"}, + "consent_api_url": consentURL, + "owner_resolver_url": resolverURL, + "consent_key": "itest-consent-key", + "participant_token": "itest-participant-token", + "provider_sd": "http://consent-facade:8080/participants/org-itest", } } @@ -267,11 +377,15 @@ func runPluginCycle( return resp } -// consentRequest builds a GET request for a personal-data entity, carrying the -// given subject DID in the JWT "sub" claim (Authorization: Bearer ...). -func consentRequest(id uint32, subject string) *mockRequest { +// consentRequest builds a GET request for a personal-data entity. The token +// names the CONSUMER (as the embedded credential's issuer) and its "sub"; neither +// determines the data owner — the OwnerResolver does, from the response payload. +func consentRequest(id uint32, caller string) *mockRequest { h := newMockRequestHeader() - h.Set("Authorization", "Bearer "+buildMockJWT(map[string]interface{}{"sub": subject})) + h.Set("Authorization", "Bearer "+buildMockJWT(map[string]interface{}{ + "sub": caller, + "verifiableCredential": map[string]interface{}{"issuer": itestConsumerDID}, + })) return &mockRequest{ id: id, method: "GET", @@ -289,9 +403,11 @@ const defaultDenyBody = `{"error":"access denied by consent policy"}` func TestIntegration_GrantedConsentPassthrough(t *testing.T) { srv := newConsentManager(t, "did:key:zAlice", "uid-1", []string{"granted"}) defer srv.Close() + resolver := newOwnerResolver(t, "did:key:zAlice") + defer resolver.Close() - resp := runPluginCycle(t, marshalConfig(t, baseConfig(srv.URL)), - consentRequest(1, "did:key:zAlice"), []byte(`{"email":"alice@example.org"}`)) + resp := runPluginCycle(t, marshalConfig(t, baseConfig(srv.URL, resolver.URL+"/resolve")), + consentRequest(1, "did:key:zCaller"), []byte(`{"email":"alice@example.org"}`)) assert.Nil(t, resp.writtenBody, "granted consent should not modify the response") assert.Equal(t, 0, resp.writtenStatus) @@ -302,9 +418,11 @@ func TestIntegration_GrantedConsentPassthrough(t *testing.T) { func TestIntegration_NoGrantedConsentDenied(t *testing.T) { srv := newConsentManager(t, "", "uid-1", []string{"revoked"}) defer srv.Close() + resolver := newOwnerResolver(t, "did:key:zAlice") + defer resolver.Close() - resp := runPluginCycle(t, marshalConfig(t, baseConfig(srv.URL)), - consentRequest(2, "did:key:zAlice"), []byte(`{"email":"alice@example.org"}`)) + resp := runPluginCycle(t, marshalConfig(t, baseConfig(srv.URL, resolver.URL+"/resolve")), + consentRequest(2, "did:key:zCaller"), []byte(`{"email":"alice@example.org"}`)) assert.Equal(t, 403, resp.writtenStatus) assert.Equal(t, defaultDenyBody, string(resp.writtenBody)) @@ -315,9 +433,11 @@ func TestIntegration_NoGrantedConsentDenied(t *testing.T) { func TestIntegration_UnknownSubjectDenied(t *testing.T) { srv := newConsentManager(t, "", "", nil) defer srv.Close() + resolver := newOwnerResolver(t, "did:key:zStranger") + defer resolver.Close() - resp := runPluginCycle(t, marshalConfig(t, baseConfig(srv.URL)), - consentRequest(3, "did:key:zStranger"), []byte(`{"email":"x@example.org"}`)) + resp := runPluginCycle(t, marshalConfig(t, baseConfig(srv.URL, resolver.URL+"/resolve")), + consentRequest(3, "did:key:zCaller"), []byte(`{"email":"x@example.org"}`)) assert.Equal(t, 403, resp.writtenStatus) assert.Equal(t, defaultDenyBody, string(resp.writtenBody)) @@ -328,13 +448,16 @@ func TestIntegration_CustomDenyResponse(t *testing.T) { srv := newConsentManager(t, "", "uid-1", []string{"revoked"}) defer srv.Close() - cfg := baseConfig(srv.URL) + resolver := newOwnerResolver(t, "did:key:zAlice") + defer resolver.Close() + + cfg := baseConfig(srv.URL, resolver.URL+"/resolve") cfg["deny_status_code"] = 451 cfg["deny_response_body"] = `{"error":"legally restricted"}` cfg["deny_response_content_type"] = "application/json" resp := runPluginCycle(t, marshalConfig(t, cfg), - consentRequest(4, "did:key:zAlice"), []byte(`{"secret":"x"}`)) + consentRequest(4, "did:key:zCaller"), []byte(`{"secret":"x"}`)) assert.Equal(t, 451, resp.writtenStatus) assert.Equal(t, `{"error":"legally restricted"}`, string(resp.writtenBody)) @@ -344,16 +467,17 @@ func TestIntegration_CustomDenyResponse(t *testing.T) { // TestIntegration_ConsentManagerError_FailOpen verifies a consent-manager error // passes through when fail-open is set. func TestIntegration_ConsentManagerError_FailOpen(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusInternalServerError) - })) + srv := newFailingConsentManager(http.StatusInternalServerError) defer srv.Close() - cfg := baseConfig(srv.URL) + resolver := newOwnerResolver(t, "did:key:zAlice") + defer resolver.Close() + + cfg := baseConfig(srv.URL, resolver.URL+"/resolve") cfg["fail_open"] = true resp := runPluginCycle(t, marshalConfig(t, cfg), - consentRequest(5, "did:key:zAlice"), []byte(`{"data":"passes"}`)) + consentRequest(5, "did:key:zCaller"), []byte(`{"data":"passes"}`)) assert.Nil(t, resp.writtenBody, "fail-open should pass through on consent-manager error") assert.Equal(t, 0, resp.writtenStatus) @@ -362,44 +486,53 @@ func TestIntegration_ConsentManagerError_FailOpen(t *testing.T) { // TestIntegration_ConsentManagerError_FailClosed verifies a consent-manager error // is denied when fail-closed. func TestIntegration_ConsentManagerError_FailClosed(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusServiceUnavailable) - })) + srv := newFailingConsentManager(http.StatusServiceUnavailable) defer srv.Close() - cfg := baseConfig(srv.URL) + resolver := newOwnerResolver(t, "did:key:zAlice") + defer resolver.Close() + + cfg := baseConfig(srv.URL, resolver.URL+"/resolve") cfg["fail_open"] = false resp := runPluginCycle(t, marshalConfig(t, cfg), - consentRequest(6, "did:key:zAlice"), []byte(`{"data":"denied"}`)) + consentRequest(6, "did:key:zCaller"), []byte(`{"data":"denied"}`)) assert.Equal(t, 403, resp.writtenStatus) assert.Equal(t, defaultDenyBody, string(resp.writtenBody)) } -// TestIntegration_SubjectForwardedFromJWT verifies the subject from the JWT "sub" -// claim is forwarded to the consent-manager as the user email (asserted in the mock). -func TestIntegration_SubjectForwardedFromJWT(t *testing.T) { +// TestIntegration_OwnerNotRequestor verifies the subject the consent-manager is +// asked about is the RESOLVED DATA OWNER, not the caller (asserted in the mock). +func TestIntegration_OwnerNotRequestor(t *testing.T) { srv := newConsentManager(t, "did:key:zBob", "uid-bob", []string{"granted"}) defer srv.Close() + resolver := newOwnerResolver(t, "did:key:zBob") + defer resolver.Close() - resp := runPluginCycle(t, marshalConfig(t, baseConfig(srv.URL)), - consentRequest(7, "did:key:zBob"), []byte(`{"ok":true}`)) + // The caller is Alice; the resolver says the data belongs to Bob, and the + // mock asserts that Bob — not Alice — is the subject sent to the search. + resp := runPluginCycle(t, marshalConfig(t, baseConfig(srv.URL, resolver.URL+"/resolve")), + consentRequest(7, "did:key:zAlice"), []byte(`{"ok":true}`)) assert.Nil(t, resp.writtenBody) } // TestIntegration_CustomJWTHeader verifies the plugin reads the JWT from a custom -// header when configured (subject still resolves and consent is granted). +// header when configured (the resolved owner's consent is granted). func TestIntegration_CustomJWTHeader(t *testing.T) { srv := newConsentManager(t, "custom-user", "uid-c", []string{"granted"}) defer srv.Close() + resolver := newOwnerResolver(t, "custom-user") + defer resolver.Close() - cfg := baseConfig(srv.URL) + cfg := baseConfig(srv.URL, resolver.URL+"/resolve") cfg["jwt_header_name"] = "X-Auth-Token" h := newMockRequestHeader() - h.Set("X-Auth-Token", "Bearer "+buildMockJWT(map[string]interface{}{"sub": "custom-user"})) + h.Set("X-Auth-Token", "Bearer "+buildMockJWT(map[string]interface{}{ + "verifiableCredential": map[string]interface{}{"issuer": itestConsumerDID}, + })) req := &mockRequest{id: 8, method: "POST", path: []byte("/api/v1/items"), header: h} resp := runPluginCycle(t, marshalConfig(t, cfg), req, []byte(`{"created":true}`)) @@ -412,13 +545,15 @@ func TestIntegration_CustomJWTHeader(t *testing.T) { func TestIntegration_ContextCleanupAfterCycle(t *testing.T) { srv := newConsentManager(t, "", "uid-1", []string{"granted"}) defer srv.Close() + resolver := newOwnerResolver(t, "did:key:zAlice") + defer resolver.Close() const id = uint32(999) - _ = runPluginCycle(t, marshalConfig(t, baseConfig(srv.URL)), - consentRequest(id, "did:key:zAlice"), []byte(`{"data":"test"}`)) + _ = runPluginCycle(t, marshalConfig(t, baseConfig(srv.URL, resolver.URL+"/resolve")), + consentRequest(id, "did:key:zCaller"), []byte(`{"data":"test"}`)) - _, found := plugin.LoadRequestContext(integrationReqKey(id)) - assert.False(t, found, "request context should be deleted after the response cycle") + assert.Equal(t, 0, plugin.RequestContextStoreSize(), + "request context should be deleted after the response cycle") } // newConsentManagerCC starts a mock consent-manager exposing all four endpoints @@ -464,25 +599,23 @@ func newConsentManagerTokenService(t *testing.T, wantSubject, userID, selfDescri mux.HandleFunc("/v1/consents/participants/", func(w http.ResponseWriter, r *http.Request) { assert.Equal(t, "Bearer itest-token", r.Header.Get("Authorization")) - consents := make([]map[string]string, 0, len(statuses)) - for _, s := range statuses { - consents = append(consents, map[string]string{"status": s}) - } w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(map[string]interface{}{"consents": consents}) + _ = json.NewEncoder(w).Encode(map[string]interface{}{"consents": consentsGrantedTo(statuses)}) }) + mux.HandleFunc("/v1/participants", participantRegistryHandler) + return httptest.NewServer(mux) } // ccConfig is a plugin config using participant client credentials (no static // token, no explicit provider_sd — both are obtained from the consent-manager). -func tokenServiceConfig(consentURL string) map[string]interface{} { +func tokenServiceConfig(consentURL, resolverURL string) map[string]interface{} { return map[string]interface{}{ - "consent_api_url": consentURL, - "consent_key": "itest-consent-key", - "token_service_url": consentURL + "/internal/tokens", - "jwt_claims_to_forward": []string{"sub"}, + "consent_api_url": consentURL, + "owner_resolver_url": resolverURL, + "consent_key": "itest-consent-key", + "token_service_url": consentURL + "/internal/tokens", } } @@ -493,9 +626,11 @@ func TestIntegration_TokenServiceFlow(t *testing.T) { srv := newConsentManagerTokenService(t, "did:key:zAlice", "uid-1", "http://consent-facade:8080/participants/derived", []string{"granted"}) defer srv.Close() + resolver := newOwnerResolver(t, "did:key:zAlice") + defer resolver.Close() - resp := runPluginCycle(t, marshalConfig(t, tokenServiceConfig(srv.URL)), - consentRequest(20, "did:key:zAlice"), []byte(`{"ok":true}`)) + resp := runPluginCycle(t, marshalConfig(t, tokenServiceConfig(srv.URL, resolver.URL+"/resolve")), + consentRequest(20, "did:key:zCaller"), []byte(`{"ok":true}`)) assert.Nil(t, resp.writtenBody, "granted consent via client credentials should pass through") } @@ -506,10 +641,257 @@ func TestIntegration_TokenServiceDenied(t *testing.T) { srv := newConsentManagerTokenService(t, "", "uid-1", "http://consent-facade:8080/participants/derived", []string{"revoked"}) defer srv.Close() + resolver := newOwnerResolver(t, "did:key:zAlice") + defer resolver.Close() - resp := runPluginCycle(t, marshalConfig(t, tokenServiceConfig(srv.URL)), - consentRequest(21, "did:key:zAlice"), []byte(`{"secret":"x"}`)) + resp := runPluginCycle(t, marshalConfig(t, tokenServiceConfig(srv.URL, resolver.URL+"/resolve")), + consentRequest(21, "did:key:zCaller"), []byte(`{"secret":"x"}`)) assert.Equal(t, 403, resp.writtenStatus) assert.Equal(t, defaultDenyBody, string(resp.writtenBody)) } + +// --- Resolver-mode integration tests --- + +// perOwnerConsentManager starts a consent-manager whose consent status is looked +// up per data owner, so a multi-owner response can mix granted and revoked +// owners. It records the owners it was asked about, in order. +func perOwnerConsentManager(t *testing.T, statusByOwner map[string]string, asked *[]string) *httptest.Server { + t.Helper() + var mu sync.Mutex + identifiers := map[string]string{} + + mux := http.NewServeMux() + mux.HandleFunc("/v1/users/identifier/search", func(w http.ResponseWriter, r *http.Request) { + var body map[string]string + _ = json.NewDecoder(r.Body).Decode(&body) + owner := body["email"] + mu.Lock() + *asked = append(*asked, owner) + status, known := statusByOwner[owner] + if known { + identifiers["uid-"+owner] = status + } + mu.Unlock() + if !known { + w.WriteHeader(http.StatusNotFound) + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]string{"userIdentifier": "uid-" + owner}) + }) + mux.HandleFunc("/v1/consents/participants/", func(w http.ResponseWriter, r *http.Request) { + id := strings.TrimPrefix(r.URL.Path, "/v1/consents/participants/") + mu.Lock() + status := identifiers[id] + mu.Unlock() + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]interface{}{"consents": consentsGrantedTo([]string{status})}) + }) + mux.HandleFunc("/v1/participants", participantRegistryHandler) + srv := httptest.NewServer(mux) + t.Cleanup(srv.Close) + return srv +} + +// TestIntegration_Resolver drives the resolver-mode decision matrix end to end: +// a real ParseConf -> RequestFilter -> ResponseFilter against a mock resolver +// and a mock consent-manager. +func TestIntegration_Resolver(t *testing.T) { + const ( + ownerAlice = "did:key:zAlice" + ownerBob = "did:key:zBob" + ) + + tests := []struct { + name string + resolverReply map[string]interface{} + resolverCode int // non-zero => the resolver answers with this status instead + statusByOwner map[string]string + failOpen *bool + wantDenied bool + wantOwners []string // owners the consent-manager must have been asked about + }{ + { + name: "every owner granted allows", + resolverReply: resolveReply(ownerAlice, ownerBob), + statusByOwner: map[string]string{ownerAlice: "granted", ownerBob: "granted"}, + wantOwners: []string{ownerAlice, ownerBob}, + }, + { + name: "one revoked owner denies the whole response (deny_all)", + resolverReply: resolveReply(ownerAlice, ownerBob), + statusByOwner: map[string]string{ownerAlice: "granted", ownerBob: "revoked"}, + wantDenied: true, + // Which owners get asked depends on the concurrent short-circuit, so + // only the decision is asserted here. + }, + { + name: "an owner unknown to the consent-manager denies", + resolverReply: resolveReply(ownerAlice, ownerBob), + statusByOwner: map[string]string{ownerAlice: "granted"}, + wantDenied: true, + }, + { + name: "consentRequired false allows without any consent call", + resolverReply: map[string]interface{}{"consentRequired": false}, + wantOwners: nil, + }, + { + name: "consent required with no claims denies", + resolverReply: map[string]interface{}{"consentRequired": true, "claims": []map[string]string{}}, + wantDenied: true, + wantOwners: nil, + }, + { + name: "a claim with an empty owner denies", + resolverReply: map[string]interface{}{ + "consentRequired": true, + "claims": []map[string]string{{"ownerId": ""}}, + }, + wantDenied: true, + wantOwners: nil, + }, + { + name: "resolver 5xx denies by default", + resolverCode: http.StatusInternalServerError, + wantDenied: true, + wantOwners: nil, + }, + { + name: "resolver 5xx passes through with fail_open", + resolverCode: http.StatusInternalServerError, + failOpen: boolPtr(true), + wantOwners: nil, + }, + { + name: "duplicate owners are checked once", + resolverReply: resolveReply(ownerAlice, ownerAlice, ownerAlice), + statusByOwner: map[string]string{ownerAlice: "granted"}, + wantOwners: []string{ownerAlice}, + }, + } + + // Each case needs its own request id, so the context store entries cannot + // collide between subtests. + nextRequestID := uint32(100) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + consent.ResetCaches() + nextRequestID++ + + var asked []string + cm := perOwnerConsentManager(t, tt.statusByOwner, &asked) + + var resolver *httptest.Server + if tt.resolverCode != 0 { + resolver = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(tt.resolverCode) + })) + } else { + var received []resolveEnvelope + resolver = newRecordingOwnerResolver(t, tt.resolverReply, &received) + } + defer resolver.Close() + + cfg := baseConfig(cm.URL, resolver.URL+"/resolve") + if tt.failOpen != nil { + cfg["fail_open"] = *tt.failOpen + } + + resp := runPluginCycle(t, marshalConfig(t, cfg), + consentRequest(nextRequestID, "did:key:zCaller"), []byte(`{"id":"urn:entity:1"}`)) + + if tt.wantDenied { + assert.Equal(t, 403, resp.writtenStatus) + assert.Equal(t, defaultDenyBody, string(resp.writtenBody)) + } else { + assert.Nil(t, resp.writtenBody, "expected the response to pass through") + assert.Equal(t, 0, resp.writtenStatus) + } + if tt.wantOwners != nil { + // Checks run concurrently, so the order is not significant. + assert.ElementsMatch(t, tt.wantOwners, asked, "the consent-manager must be asked about exactly these owners") + } + }) + } +} + +// resolveReply builds a /resolve reply requiring consent from the given owners. +func resolveReply(owners ...string) map[string]interface{} { + claims := make([]map[string]string, 0, len(owners)) + for _, o := range owners { + claims = append(claims, map[string]string{"ownerId": o}) + } + return map[string]interface{}{"consentRequired": true, "claims": claims} +} + +// TestIntegration_ResolverReceivesPartiesAndPayload verifies what the plugin +// actually tells the resolver: the resource descriptor, the contract parties +// (resolved via the participant registry), and the upstream payload as JSON. +func TestIntegration_ResolverReceivesPartiesAndPayload(t *testing.T) { + consent.ResetCaches() + + var asked []string + cm := perOwnerConsentManager(t, map[string]string{"did:key:zAlice": "granted"}, &asked) + + var received []resolveEnvelope + resolver := newRecordingOwnerResolver(t, resolveReply("did:key:zAlice"), &received) + defer resolver.Close() + + cfg := baseConfig(cm.URL, resolver.URL+"/resolve") + cfg["service"] = "personal-profiles" + + payload := []byte(`{"id":"urn:ngsi-ld:PersonalProfile:alice"}`) + runPluginCycle(t, marshalConfig(t, cfg), consentRequest(150, "did:key:zCaller"), payload) + + require.Len(t, received, 1) + env := received[0] + assert.Equal(t, "personal-profiles", env.Resource.Service) + assert.Equal(t, "GET", env.Resource.Method) + assert.Equal(t, "/ngsi-ld/v1/entities/urn:ngsi-ld:PersonalProfile:alice", env.Resource.Path) + require.NotNil(t, env.Parties, "the contract parties must be sent, or the resolver cannot identify the contract") + assert.Equal(t, itestConsumerSD, env.Parties.Consumer, "the consumer DID must be mapped to its self-description") + assert.Equal(t, "http://consent-facade:8080/participants/org-itest", env.Parties.Provider) + require.NotNil(t, env.Body) + assert.Equal(t, "json", env.Body.Encoding) + assert.JSONEq(t, string(payload), string(env.Body.Content)) +} + +// TestIntegration_PartyResolutionFailureDenies verifies the fail-closed seam +// from H-1 end to end: when the consumer cannot be mapped to a participant, the +// resolver is never asked and the response is denied — even with fail_open, +// which must not turn a token naming an unregistered consumer into a bypass. +func TestIntegration_PartyResolutionFailureDenies(t *testing.T) { + consent.ResetCaches() + + var asked []string + cm := perOwnerConsentManager(t, map[string]string{}, &asked) + + resolverCalled := false + resolver := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + resolverCalled = true + w.Header().Set("Content-Type", "application/json") + // An unidentified-party resolve could plausibly answer "no contract + // governs this, so no consent is required" — an unconditional allow. + _ = json.NewEncoder(w).Encode(map[string]interface{}{"consentRequired": false}) + })) + defer resolver.Close() + + h := newMockRequestHeader() + h.Set("Authorization", "Bearer "+buildMockJWT(map[string]interface{}{ + "verifiableCredential": map[string]interface{}{"issuer": "did:key:zUnregisteredConsumer"}, + })) + req := &mockRequest{id: 160, method: "GET", path: []byte("/data"), header: h} + + cfg := baseConfig(cm.URL, resolver.URL+"/resolve") + cfg["fail_open"] = true + + resp := runPluginCycle(t, marshalConfig(t, cfg), req, []byte(`{"a":1}`)) + + assert.False(t, resolverCalled, "an unidentified consumer must not reach the resolver") + assert.Equal(t, 403, resp.writtenStatus, "an unidentified consumer must deny") +} + +// boolPtr returns a pointer to b, for the optional fail_open flag. +func boolPtr(b bool) *bool { return &b } diff --git a/internal/logging/logging.go b/internal/logging/logging.go new file mode 100644 index 0000000..d4967b4 --- /dev/null +++ b/internal/logging/logging.go @@ -0,0 +1,215 @@ +/* + * Copyright 2026 Seamless Middleware Technologies S.L and/or its affiliates + * and other contributors as indicated by the @author tags. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package logging is the plugin's logging front end. +// +// It exists for three reasons, each of which was a problem with calling the +// standard library's log.Printf directly: +// +// - Levels. The go-plugin-runner ships its own zap logger and configures its +// level from the runner's environment. Lines written with log.Printf bypass +// it entirely, so an operator could not raise or lower the plugin's verbosity +// at all. Everything here goes through the runner's logger. +// +// - Personal data. Log lines on the request path carry subject DIDs and +// upstream error bodies, i.e. personal data on stdout with no retention +// policy — exactly what the OTLP audit path exists to avoid. Redact turns an +// identifier into a stable fingerprint that still correlates across lines, +// and Sanitize strips an error body down to something safe to print. +// +// - Volume. A broken consent-manager produced one line per request. The +// Every variants collapse a repeated failure to one line per interval and +// report how many were suppressed. +package logging + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "strings" + "sync" + "time" + "unicode" + + runnerlog "github.com/apache/apisix-go-plugin-runner/pkg/log" +) + +// logPrefix marks every line as coming from this plugin. +const logPrefix = "[consent-filter] " + +// Debugf logs at debug level. +func Debugf(template string, args ...interface{}) { + runnerlog.Debugf(logPrefix+template, args...) +} + +// Infof logs at info level. +func Infof(template string, args ...interface{}) { + runnerlog.Infof(logPrefix+template, args...) +} + +// Warnf logs at warn level. +func Warnf(template string, args ...interface{}) { + runnerlog.Warnf(logPrefix+template, args...) +} + +// Errorf logs at error level. +func Errorf(template string, args ...interface{}) { + runnerlog.Errorf(logPrefix+template, args...) +} + +// --- rate limiting ----------------------------------------------------------- + +// suppressionInterval is how long one key stays silenced after it has logged. +// A dependency that fails for every request should cost one line per interval, +// not one line per request. +const suppressionInterval = 10 * time.Second + +type suppressionState struct { + lastLogged time.Time + suppressed uint64 +} + +var ( + suppressionMu sync.Mutex + suppressionByKey = map[string]*suppressionState{} + suppressionKeyCap = 1024 +) + +// allow reports whether key may log now, and how many lines it suppressed since +// it last did. +func allow(key string) (bool, uint64) { + suppressionMu.Lock() + defer suppressionMu.Unlock() + + state := suppressionByKey[key] + if state == nil { + // The key set is bounded: keys are compile-time constants in normal use, + // but a cap means a caller passing a variable key cannot grow the map. + if len(suppressionByKey) >= suppressionKeyCap { + return true, 0 + } + state = &suppressionState{} + suppressionByKey[key] = state + } + + now := time.Now() + if !state.lastLogged.IsZero() && now.Sub(state.lastLogged) < suppressionInterval { + state.suppressed++ + return false, 0 + } + suppressed := state.suppressed + state.suppressed = 0 + state.lastLogged = now + return true, suppressed +} + +// DebugfEvery logs at debug level at most once per suppressionInterval for the +// given key, noting how many occurrences were suppressed in between. +// +// This is where a dependency's response body belongs: useful when debugging, +// off by default, and never on a path that escapes into an audit record. +func DebugfEvery(key, template string, args ...interface{}) { + if ok, suppressed := allow(key); ok { + Debugf(template+suppressedSuffix(suppressed), args...) + } +} + +// WarnfEvery logs at warn level at most once per suppressionInterval for the +// given key, noting how many occurrences were suppressed in between. +func WarnfEvery(key, template string, args ...interface{}) { + if ok, suppressed := allow(key); ok { + Warnf(template+suppressedSuffix(suppressed), args...) + } +} + +// ErrorfEvery logs at error level at most once per suppressionInterval for the +// given key, noting how many occurrences were suppressed in between. +func ErrorfEvery(key, template string, args ...interface{}) { + if ok, suppressed := allow(key); ok { + Errorf(template+suppressedSuffix(suppressed), args...) + } +} + +// suppressedSuffix renders the count of lines that were swallowed since this key +// last logged, so the rate limiting is never silent about itself. +func suppressedSuffix(suppressed uint64) string { + if suppressed == 0 { + return "" + } + return fmt.Sprintf(" (%d further occurrence(s) suppressed)", suppressed) +} + +// ResetSuppression clears the rate-limiter state. For tests. +func ResetSuppression() { + suppressionMu.Lock() + defer suppressionMu.Unlock() + suppressionByKey = map[string]*suppressionState{} +} + +// --- redaction --------------------------------------------------------------- + +// fingerprintLength is how many hex characters of the digest identify a value. +// Eight is enough to correlate lines about the same subject within a log without +// being a usable handle on the subject themselves. +const fingerprintLength = 8 + +// redactedPrefix marks a value as a fingerprint rather than an identifier. +const redactedPrefix = "id:" + +// Redact turns an identifier (a subject DID, a participant DID) into a stable, +// non-reversible fingerprint. +// +// The identifier itself belongs in the audit record, which is exported to a +// controlled sink with a retention policy — not in stdout logs, which have +// neither. The fingerprint is stable, so lines about the same subject can still +// be correlated while debugging. +func Redact(identifier string) string { + if identifier == "" { + return "" + } + sum := sha256.Sum256([]byte(identifier)) + return redactedPrefix + hex.EncodeToString(sum[:])[:fingerprintLength] +} + +// --- sanitisation ------------------------------------------------------------ + +// maxSanitizedLength bounds a sanitised message. Longer than this and an error +// body has been spliced into it. +const maxSanitizedLength = 200 + +// sanitizedRedaction replaces the tail of an over-long message. +const sanitizedRedaction = "...(redacted)" + +// Sanitize makes an error message safe to print or export: control characters +// (including the newlines of an HTML or JSON error page) collapse to single +// spaces, and the result is truncated. +// +// Messages built by wrapping dependency errors embed the dependency's response +// body, which can carry identifiers or other personal data. +func Sanitize(message string) string { + cleaned := strings.Map(func(r rune) rune { + if unicode.IsControl(r) { + return ' ' + } + return r + }, message) + cleaned = strings.Join(strings.Fields(cleaned), " ") + if len(cleaned) > maxSanitizedLength { + return cleaned[:maxSanitizedLength-len(sanitizedRedaction)] + sanitizedRedaction + } + return cleaned +} diff --git a/internal/logging/logging_test.go b/internal/logging/logging_test.go new file mode 100644 index 0000000..779d714 --- /dev/null +++ b/internal/logging/logging_test.go @@ -0,0 +1,102 @@ +/* + * Copyright 2026 Seamless Middleware Technologies S.L and/or its affiliates + * and other contributors as indicated by the @author tags. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package logging + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +// TestSanitize verifies an error body spliced into a message cannot be printed +// or exported verbatim: control characters collapse and the result is bounded. +func TestSanitize(t *testing.T) { + tests := []struct { + name string + message string + want string + }{ + {name: "empty", message: "", want: ""}, + {name: "a plain message is untouched", message: "no granted consent", want: "no granted consent"}, + { + name: "newlines and tabs collapse to single spaces", + message: "consent check error:\n\t{\"error\":\"boom\"}\r\n", + want: `consent check error: {"error":"boom"}`, + }, + { + name: "an over-long message is truncated and marked", + message: "x" + strings.Repeat("y", 500), + want: "x" + strings.Repeat("y", maxSanitizedLength-len(sanitizedRedaction)-1) + sanitizedRedaction, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := Sanitize(tt.message) + assert.Equal(t, tt.want, got) + assert.LessOrEqual(t, len(got), maxSanitizedLength) + }) + } +} + +// TestRedact verifies an identifier becomes a stable, short, non-reversible +// fingerprint — enough to correlate lines about the same subject, not enough to +// put a subject DID in stdout. +func TestRedact(t *testing.T) { + const did = "did:key:zAliceSomeVeryLongIdentifier" + + assert.Empty(t, Redact(""), "an empty identifier stays empty") + + redacted := Redact(did) + assert.Equal(t, redacted, Redact(did), "the fingerprint must be stable") + assert.NotContains(t, redacted, did) + assert.NotContains(t, redacted, "Alice") + assert.True(t, strings.HasPrefix(redacted, redactedPrefix), "a fingerprint must be recognisable as one") + assert.Len(t, redacted, len(redactedPrefix)+fingerprintLength) + assert.NotEqual(t, redacted, Redact("did:key:zBob"), "different identifiers must differ") +} + +// TestRateLimiting verifies a repeated failure costs one line per interval and +// reports how many it swallowed. A broken consent-manager previously emitted one +// line per request. +func TestRateLimiting(t *testing.T) { + ResetSuppression() + t.Cleanup(ResetSuppression) + + const key = "test-key" + + ok, suppressed := allow(key) + assert.True(t, ok, "the first occurrence must log") + assert.Zero(t, suppressed) + + for i := 0; i < 5; i++ { + ok, _ = allow(key) + assert.False(t, ok, "occurrences within the interval must be suppressed") + } + + // A different key is limited independently. + ok, _ = allow("other-key") + assert.True(t, ok, "each key has its own budget") +} + +// TestSuppressedSuffix verifies the rate limiting is never silent about itself. +func TestSuppressedSuffix(t *testing.T) { + assert.Empty(t, suppressedSuffix(0)) + assert.Contains(t, suppressedSuffix(7), "7 further occurrence(s) suppressed") +} diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go new file mode 100644 index 0000000..ffcda56 --- /dev/null +++ b/internal/metrics/metrics.go @@ -0,0 +1,313 @@ +/* + * Copyright 2026 Seamless Middleware Technologies S.L and/or its affiliates + * and other contributors as indicated by the @author tags. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package metrics exposes the consent gate's operational signals in the +// Prometheus text format. +// +// A component that can deny production traffic had no counters at all: nothing +// said how many requests were allowed, denied or failed open, how long the +// consent-manager was taking, how large the request-context store had grown, or +// how many audit records had been dropped. Logs were the only signal, and they +// are unstructured and rate-limited. Operationally that is flying blind — a +// misconfigured route that denies everything looks exactly like a quiet one. +// +// The exporter is hand-rolled rather than pulling in a Prometheus client, for +// the same reason the OTLP encoder is: this is a sidecar-adjacent plugin whose +// dependency tree is part of its risk surface, and the text format is small. +package metrics + +import ( + "fmt" + "net/http" + "sort" + "strconv" + "strings" + "sync" + "time" +) + +// Metric names. The consent_ prefix keeps them together in a shared registry. +const ( + decisionsMetric = "consent_decisions_total" + dependencyCallsMetric = "consent_dependency_calls_total" + dependencyLatency = "consent_dependency_duration_seconds" + purposeUnconstrainedMetric = "consent_purpose_unconstrained_total" + contextStoreSizeMetric = "consent_request_context_store_size" + contextEvictedMetric = "consent_request_contexts_evicted_total" + auditDroppedMetric = "consent_audit_events_dropped_total" +) + +// Dependency names used as the "dependency" label. +const ( + DependencyConsentManager = "consent_manager" + DependencyOwnerResolver = "owner_resolver" +) + +// Call outcomes used as the "outcome" label. +const ( + OutcomeSuccess = "success" + OutcomeError = "error" +) + +// latencyBuckets are the histogram's upper bounds in seconds. They straddle the +// default per-call timeout (5s) so a dependency drifting toward it is visible +// before it starts failing. +var latencyBuckets = []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10} + +var ( + mu sync.Mutex + + // decisions counts enforced decisions by decision and by the fail mode that + // produced them, so "denied because no consent" and "denied because the + // consent-manager was down" are distinguishable — they mean opposite things + // operationally. + decisions = map[labelPair]uint64{} + + // dependencyCalls counts outbound calls by dependency and outcome. + dependencyCalls = map[labelPair]uint64{} + + // latency holds one histogram per dependency. + latency = map[string]*histogram{} + + // purposeUnconstrained counts consent checks run without a processing + // purpose to match against, i.e. checks where half of the consumer/purpose + // scoping was not actually applied. + purposeUnconstrained uint64 + + // callbacks are read at scrape time from whoever owns the number, so this + // package never has to be told when a store's size changes. + callbackMu sync.Mutex + callbacks = map[string]callbackMetric{} +) + +// Prometheus metric types used in the exposition. +const ( + metricTypeCounter = "counter" + metricTypeGauge = "gauge" +) + +// callbackMetric is a value this package does not own, read at scrape time. +type callbackMetric struct { + help string + kind string + read func() float64 +} + +// labelPair is a two-label metric key. +type labelPair struct{ first, second string } + +// histogram is a cumulative-bucket histogram. +type histogram struct { + counts []uint64 + sum float64 + total uint64 +} + +// observe records one value. +func (h *histogram) observe(value float64) { + for i, bound := range latencyBuckets { + if value <= bound { + h.counts[i]++ + } + } + h.sum += value + h.total++ +} + +// RecordDecision counts one enforced access decision. failMode names why the +// decision could not be made normally ("" for an ordinary consent verdict), so +// a deny caused by an outage is not confused with a deny caused by consent. +func RecordDecision(decision, failMode string) { + if failMode == "" { + failMode = "none" + } + mu.Lock() + defer mu.Unlock() + decisions[labelPair{decision, failMode}]++ +} + +// RecordDependencyCall records one outbound call to a dependency: its outcome +// and how long it took. +func RecordDependencyCall(dependency, outcome string, duration time.Duration) { + mu.Lock() + defer mu.Unlock() + dependencyCalls[labelPair{dependency, outcome}]++ + h := latency[dependency] + if h == nil { + h = &histogram{counts: make([]uint64, len(latencyBuckets))} + latency[dependency] = h + } + h.observe(duration.Seconds()) +} + +// RecordPurposeUnconstrained counts one consent check made without a processing +// purpose to scope it. +// +// Purpose matching depends on the OwnerResolver populating an optional field. A +// resolver whose rules never set it leaves purpose scoping entirely disabled, +// which is a silent narrowing of a compliance property — a consent granted for +// one purpose then authorises release for any other. This makes that state +// visible and alertable instead of merely documented. +func RecordPurposeUnconstrained() { + mu.Lock() + defer mu.Unlock() + purposeUnconstrained++ +} + +// RegisterGauge publishes a value that can go up and down, read at scrape time. +// The owner of the number keeps owning it; this package only asks for it. +func RegisterGauge(name, help string, read func() float64) { + registerCallback(name, help, metricTypeGauge, read) +} + +// RegisterCounter publishes a monotonically increasing value, read at scrape +// time. +// +// The distinction from RegisterGauge is not cosmetic: a `_total` series declared +// as a gauge makes `promtool check metrics` complain, and anyone reaching for +// `rate(..._total[5m])` over it is relying on an accident rather than on a +// stated contract. +func RegisterCounter(name, help string, read func() float64) { + registerCallback(name, help, metricTypeCounter, read) +} + +// registerCallback records a scrape-time value of the given Prometheus type. +func registerCallback(name, help, kind string, read func() float64) { + callbackMu.Lock() + defer callbackMu.Unlock() + callbacks[name] = callbackMetric{help: help, kind: kind, read: read} +} + +// Handler serves the metrics in the Prometheus text exposition format. +func Handler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8") + if _, err := w.Write([]byte(render())); err != nil { + // Nothing useful to do: the scraper went away mid-write. + return + } + }) +} + +// render produces the full exposition payload. +func render() string { + var out strings.Builder + + mu.Lock() + writeCounter(&out, decisionsMetric, "Access decisions enforced, by decision and fail mode.", + "decision", "fail_mode", decisions) + writeCounter(&out, dependencyCallsMetric, "Outbound calls to a dependency, by outcome.", + "dependency", "outcome", dependencyCalls) + + fmt.Fprintf(&out, "# HELP %s Consent checks run without a processing purpose to scope them.\n", purposeUnconstrainedMetric) + fmt.Fprintf(&out, "# TYPE %s counter\n", purposeUnconstrainedMetric) + fmt.Fprintf(&out, "%s %d\n", purposeUnconstrainedMetric, purposeUnconstrained) + + fmt.Fprintf(&out, "# HELP %s Duration of outbound dependency calls in seconds.\n", dependencyLatency) + fmt.Fprintf(&out, "# TYPE %s histogram\n", dependencyLatency) + for _, dependency := range sortedMapKeys(latency) { + h := latency[dependency] + for i, bound := range latencyBuckets { + // counts is already cumulative: observe increments every bucket whose + // bound is at or above the value. + fmt.Fprintf(&out, "%s_bucket{dependency=%q,le=%q} %d\n", + dependencyLatency, dependency, strconv.FormatFloat(bound, 'g', -1, 64), h.counts[i]) + } + fmt.Fprintf(&out, "%s_bucket{dependency=%q,le=\"+Inf\"} %d\n", dependencyLatency, dependency, h.total) + fmt.Fprintf(&out, "%s_sum{dependency=%q} %s\n", dependencyLatency, dependency, strconv.FormatFloat(h.sum, 'g', -1, 64)) + fmt.Fprintf(&out, "%s_count{dependency=%q} %d\n", dependencyLatency, dependency, h.total) + } + mu.Unlock() + + callbackMu.Lock() + names := make([]string, 0, len(callbacks)) + for name := range callbacks { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + metric := callbacks[name] + fmt.Fprintf(&out, "# HELP %s %s\n", name, metric.help) + fmt.Fprintf(&out, "# TYPE %s %s\n", name, metric.kind) + fmt.Fprintf(&out, "%s %s\n", name, strconv.FormatFloat(metric.read(), 'g', -1, 64)) + } + callbackMu.Unlock() + + return out.String() +} + +// writeCounter renders one two-label counter family in a stable order. +func writeCounter(out *strings.Builder, name, help, firstLabel, secondLabel string, values map[labelPair]uint64) { + fmt.Fprintf(out, "# HELP %s %s\n", name, help) + fmt.Fprintf(out, "# TYPE %s counter\n", name) + keys := make([]labelPair, 0, len(values)) + for key := range values { + keys = append(keys, key) + } + sort.Slice(keys, func(i, j int) bool { + if keys[i].first != keys[j].first { + return keys[i].first < keys[j].first + } + return keys[i].second < keys[j].second + }) + for _, key := range keys { + fmt.Fprintf(out, "%s{%s=%q,%s=%q} %d\n", name, firstLabel, key.first, secondLabel, key.second, values[key]) + } +} + +// sortedMapKeys returns a map's keys in a stable order. +func sortedMapKeys(m map[string]*histogram) []string { + keys := make([]string, 0, len(m)) + for key := range m { + keys = append(keys, key) + } + sort.Strings(keys) + return keys +} + +// Metric names published by the rest of the plugin through RegisterGauge and +// RegisterCounter. +const ( + // ContextStoreSizeGauge tracks in-flight gated requests. It goes up and down, + // so it is a gauge: in a healthy runner it returns to zero when idle, and a + // floor that keeps rising is the leak. + ContextStoreSizeGauge = contextStoreSizeMetric + + // ContextEvictedCounter counts contexts dropped because they expired or the + // store was full — requests that never reached their response phase. It only + // increases, so it is a counter. + ContextEvictedCounter = contextEvictedMetric + + // AuditDroppedCounter counts audit records lost to a full queue. An attacker + // who can generate load can suppress the record of their own access, so this + // must be alertable. It only increases, so it is a counter. + AuditDroppedCounter = auditDroppedMetric +) + +// Reset clears every metric. For tests. +func Reset() { + mu.Lock() + decisions = map[labelPair]uint64{} + dependencyCalls = map[labelPair]uint64{} + latency = map[string]*histogram{} + purposeUnconstrained = 0 + mu.Unlock() + + callbackMu.Lock() + callbacks = map[string]callbackMetric{} + callbackMu.Unlock() +} diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go new file mode 100644 index 0000000..3349f0f --- /dev/null +++ b/internal/metrics/metrics_test.go @@ -0,0 +1,122 @@ +/* + * Copyright 2026 Seamless Middleware Technologies S.L and/or its affiliates + * and other contributors as indicated by the @author tags. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package metrics + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestRenderExposition covers the whole payload: counters with their labels, the +// histogram's cumulative buckets, and gauges read at scrape time. +func TestRenderExposition(t *testing.T) { + Reset() + t.Cleanup(Reset) + + RecordDecision("allow", "") + RecordDecision("allow", "") + RecordDecision("deny", "") + // A deny caused by an outage must be distinguishable from a deny caused by + // consent: they mean opposite things operationally. + RecordDecision("deny", "by_policy") + + RecordDependencyCall(DependencyConsentManager, OutcomeSuccess, 20*time.Millisecond) + RecordDependencyCall(DependencyConsentManager, OutcomeError, 3*time.Second) + + RegisterGauge(ContextStoreSizeGauge, "Request contexts currently held.", func() float64 { return 7 }) + RegisterCounter(ContextEvictedCounter, "Request contexts evicted.", func() float64 { return 3 }) + + out := render() + + assert.Contains(t, out, `consent_decisions_total{decision="allow",fail_mode="none"} 2`) + assert.Contains(t, out, `consent_decisions_total{decision="deny",fail_mode="none"} 1`) + assert.Contains(t, out, `consent_decisions_total{decision="deny",fail_mode="by_policy"} 1`) + + assert.Contains(t, out, `consent_dependency_calls_total{dependency="consent_manager",outcome="success"} 1`) + assert.Contains(t, out, `consent_dependency_calls_total{dependency="consent_manager",outcome="error"} 1`) + + // 20ms falls in the 0.025 bucket, 3s does not; both are under +Inf. + assert.Contains(t, out, `consent_dependency_duration_seconds_bucket{dependency="consent_manager",le="0.025"} 1`) + assert.Contains(t, out, `consent_dependency_duration_seconds_bucket{dependency="consent_manager",le="5"} 2`) + assert.Contains(t, out, `consent_dependency_duration_seconds_bucket{dependency="consent_manager",le="+Inf"} 2`) + assert.Contains(t, out, `consent_dependency_duration_seconds_count{dependency="consent_manager"} 2`) + + assert.Contains(t, out, "consent_request_context_store_size 7") + assert.Contains(t, out, "# TYPE consent_decisions_total counter") + assert.Contains(t, out, "# TYPE consent_dependency_duration_seconds histogram") + + // A `_total` series must be declared a counter, or `rate()` over it is an + // accident rather than a contract; and every family needs a HELP line. + assert.Contains(t, out, "# TYPE consent_request_context_store_size gauge") + assert.Contains(t, out, "# TYPE consent_request_contexts_evicted_total counter") + assert.Contains(t, out, "# HELP consent_request_context_store_size Request contexts currently held.") + assert.Contains(t, out, "# HELP consent_request_contexts_evicted_total Request contexts evicted.") +} + +// TestGaugesAreReadAtScrapeTime verifies a gauge reflects the current value +// rather than the one at registration — the store size is the point. +func TestGaugesAreReadAtScrapeTime(t *testing.T) { + Reset() + t.Cleanup(Reset) + + size := 0 + RegisterGauge(ContextStoreSizeGauge, "Request contexts currently held.", func() float64 { return float64(size) }) + + assert.Contains(t, render(), "consent_request_context_store_size 0") + size = 42 + assert.Contains(t, render(), "consent_request_context_store_size 42") +} + +// TestHandlerServesExposition verifies the HTTP surface and its content type. +func TestHandlerServesExposition(t *testing.T) { + Reset() + t.Cleanup(Reset) + RecordDecision("deny", "always_closed") + + recorder := httptest.NewRecorder() + Handler().ServeHTTP(recorder, httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/metrics", nil)) + + require.Equal(t, http.StatusOK, recorder.Code) + assert.Contains(t, recorder.Header().Get("Content-Type"), "text/plain") + assert.Contains(t, recorder.Body.String(), `consent_decisions_total{decision="deny",fail_mode="always_closed"} 1`) +} + +// TestRenderIsStable verifies the output order does not depend on map iteration, +// so a scrape diff reflects real change. +func TestRenderIsStable(t *testing.T) { + Reset() + t.Cleanup(Reset) + + for _, decision := range []string{"deny", "allow"} { + RecordDecision(decision, "") + } + RecordDependencyCall(DependencyOwnerResolver, OutcomeSuccess, time.Millisecond) + RecordDependencyCall(DependencyConsentManager, OutcomeSuccess, time.Millisecond) + RegisterCounter(AuditDroppedCounter, "Audit events dropped.", func() float64 { return 1 }) + RegisterGauge(ContextStoreSizeGauge, "Request contexts currently held.", func() float64 { return 2 }) + + first := render() + for i := 0; i < 20; i++ { + assert.Equal(t, first, render()) + } +} diff --git a/internal/ownerresolver/client.go b/internal/ownerresolver/client.go index 5298092..bcf024c 100644 --- a/internal/ownerresolver/client.go +++ b/internal/ownerresolver/client.go @@ -24,6 +24,7 @@ package ownerresolver import ( "bytes" + "consent-plugin/internal/logging" "context" "encoding/json" "fmt" @@ -34,9 +35,24 @@ import ( // Body encodings understood by the resolver. const ( + // encodingJSON carries the payload verbatim, parsed. encodingJSON = "json" + + // encodingNone means the response carried no payload at all. encodingNone = "none" + // encodingOpaque means the response carried a payload the plugin could not + // parse as JSON. + // + // It exists because sending such a body as encodingNone made "this response + // carried a payload I could not read" indistinguishable from "this response + // had no payload". The resolver then judged ownership from the resource + // descriptor alone, so a malformed-but-personal payload — a truncated write, + // a content-type mismatch, an upstream answering XML or NDJSON on a route + // declared JSON — was released without ever being inspected. With the two + // cases separated the resolver can fail closed on the one it cannot read. + encodingOpaque = "opaque" + // DefaultTimeoutMs is the default per-call timeout for /resolve. DefaultTimeoutMs = 2000 @@ -44,25 +60,34 @@ const ( contentTypeJSON = "application/json" ) -// Selector locates a claim within the payload (mirrors the resolver contract). -type Selector struct { - Type string `json:"type"` - Value string `json:"value,omitempty"` -} - // Claim is one (owner [× dataResource]) requirement found in the data. +// +// The resolver's reply carries more than this (a selector locating the claim in +// the payload, the participant, the scheme). Only the fields the plugin acts on +// are decoded; the rest is ignored, so an unread field cannot suggest the plugin +// considers something it does not. type Claim struct { - Selector Selector `json:"selector"` - OwnerID string `json:"ownerId"` - Participant string `json:"participant,omitempty"` - DataResource string `json:"dataResource,omitempty"` + // OwnerID is the data owner whose consent decides this claim. + OwnerID string `json:"ownerId"` + + // DataResource, when set, scopes the consent match to one resource. + DataResource string `json:"dataResource,omitempty"` + + // Purpose names the processing purpose (or contract) governing this claim, + // when the resolver could identify the contract from the parties. It scopes + // the consent match: a granted consent counts only if it covers this purpose. + // Empty means the purpose is unknown and only the consumer match applies. + Purpose string `json:"purpose,omitempty"` } // Result is the OwnerResolver response. type Result struct { - ConsentRequired bool `json:"consentRequired"` - Scheme string `json:"scheme,omitempty"` - Claims []Claim `json:"claims"` + // ConsentRequired reports whether the payload needs a consent check at all. + ConsentRequired bool `json:"consentRequired"` + + // Claims are the ownership requirements found in the data. Every one must be + // satisfied for the response to be released. + Claims []Claim `json:"claims"` } type resourceDescriptor struct { @@ -73,8 +98,14 @@ type resourceDescriptor struct { } type bodyDescriptor struct { - Encoding string `json:"encoding"` - Content json.RawMessage `json:"content,omitempty"` + Encoding string `json:"encoding"` + // Content is the payload, present only for encodingJSON. + Content json.RawMessage `json:"content,omitempty"` + // ContentType is what the upstream declared, sent with encodingOpaque so the + // resolver knows what it was handed and how much it was. + ContentType string `json:"contentType,omitempty"` + // Size is the payload's length in bytes, sent with encodingOpaque. + Size int `json:"size,omitempty"` } // Parties names the exchange participants. It exists ONLY so the resolver can @@ -122,17 +153,20 @@ type Resource struct { ContentType string } -// Resolve asks the OwnerResolver about a payload. payload may be nil, in which -// case the body is sent with encoding "none" (the resolver decides from the -// resource descriptor alone). consumer, when non-empty, is forwarded so the -// resolver can find the governing contract - they are never used for ownership. -// A non-2xx response is returned as an error so the caller can apply its fail -// policy — it never means "no consent needed". +// Resolve asks the OwnerResolver about a payload. +// +// The body is described in one of three ways, and the distinction matters: +// "json" carries the payload, "none" says there was no payload, and "opaque" +// says there WAS one but it could not be parsed. Collapsing the last two would +// let an unreadable personal-data payload be judged from the resource descriptor +// alone. +// +// The parties, when known, are forwarded so the resolver can find the governing +// contract — they are never used for ownership. A non-2xx response is returned +// as an error so the caller can apply its fail policy; it never means "no +// consent needed". func (c *Client) Resolve(ctx context.Context, res Resource, p Parties, payload []byte) (Result, error) { - reqBody := &bodyDescriptor{Encoding: encodingNone} - if len(payload) > 0 && json.Valid(payload) { - reqBody = &bodyDescriptor{Encoding: encodingJSON, Content: json.RawMessage(payload)} - } + reqBody := describeBody(payload, res.ContentType) req := resolveRequest{ Resource: resourceDescriptor(res), Body: reqBody, @@ -162,7 +196,13 @@ func (c *Client) Resolve(ctx context.Context, res Resource, p Parties, payload [ return Result{}, fmt.Errorf("owner-resolver: read response: %w", err) } if resp.StatusCode != http.StatusOK { - return Result{}, fmt.Errorf("owner-resolver: status %d: %s", resp.StatusCode, truncate(body)) + // The body goes to a debug log, not into the error: this error becomes the + // plugin's decision reason, which is exported to the audit sink and + // written to stdout, and a resolver error page can echo the payload it was + // given — which is the personal data the gate exists to protect. + logging.DebugfEvery("resolver-body", + "owner-resolver: status %d, body: %s", resp.StatusCode, logging.Sanitize(string(body))) + return Result{}, fmt.Errorf("owner-resolver: status %d", resp.StatusCode) } var out Result @@ -172,10 +212,14 @@ func (c *Client) Resolve(ctx context.Context, res Resource, p Parties, payload [ return out, nil } -func truncate(b []byte) string { - const limit = 256 - if len(b) <= limit { - return string(b) +// describeBody classifies the upstream payload for the resolve envelope. +func describeBody(payload []byte, contentType string) *bodyDescriptor { + switch { + case len(payload) == 0: + return &bodyDescriptor{Encoding: encodingNone} + case json.Valid(payload): + return &bodyDescriptor{Encoding: encodingJSON, Content: json.RawMessage(payload)} + default: + return &bodyDescriptor{Encoding: encodingOpaque, ContentType: contentType, Size: len(payload)} } - return string(b[:limit]) + "...(truncated)" } diff --git a/internal/ownerresolver/client_test.go b/internal/ownerresolver/client_test.go index 0b12f0a..1c5e3b1 100644 --- a/internal/ownerresolver/client_test.go +++ b/internal/ownerresolver/client_test.go @@ -109,3 +109,110 @@ func TestResolve_Non2xxIsError(t *testing.T) { t.Fatal("expected error on non-2xx resolver response") } } + +// TestDescribeBody verifies the three body encodings stay distinguishable. +// +// A payload the plugin cannot parse used to be described exactly like no payload +// at all, so the resolver judged ownership from the resource descriptor alone +// and a malformed-but-personal response (a truncated write, a content-type +// mismatch, an upstream answering XML on a route declared JSON) was released +// without ever being inspected. +func TestDescribeBody(t *testing.T) { + tests := []struct { + name string + payload []byte + contentType string + wantEncoding string + wantContent string + wantContentType string + wantSize int + }{ + { + name: "no payload", + payload: nil, + wantEncoding: encodingNone, + }, + { + name: "empty payload is no payload", + payload: []byte{}, + wantEncoding: encodingNone, + }, + { + name: "valid JSON is carried verbatim", + payload: []byte(`{"id":"urn:entity:1"}`), + contentType: "application/json", + wantEncoding: encodingJSON, + wantContent: `{"id":"urn:entity:1"}`, + }, + { + name: "unparseable payload is opaque, not absent", + payload: []byte("alice@example.org"), + contentType: "application/xml", + wantEncoding: encodingOpaque, + wantContentType: "application/xml", + wantSize: 49, + }, + { + name: "truncated JSON is opaque, not absent", + payload: []byte(`{"id":"urn:entity`), + contentType: "application/json", + wantEncoding: encodingOpaque, + wantContentType: "application/json", + wantSize: 17, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := describeBody(tt.payload, tt.contentType) + if got.Encoding != tt.wantEncoding { + t.Fatalf("encoding = %q, want %q", got.Encoding, tt.wantEncoding) + } + if string(got.Content) != tt.wantContent { + t.Errorf("content = %q, want %q", string(got.Content), tt.wantContent) + } + if got.ContentType != tt.wantContentType { + t.Errorf("contentType = %q, want %q", got.ContentType, tt.wantContentType) + } + if got.Size != tt.wantSize { + t.Errorf("size = %d, want %d", got.Size, tt.wantSize) + } + }) + } +} + +// TestResolve_SendsOpaqueBodyForUnparseablePayload verifies the distinction +// survives onto the wire, so the resolver can act on it. +func TestResolve_SendsOpaqueBodyForUnparseablePayload(t *testing.T) { + var gotReq resolveRequest + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + raw, _ := io.ReadAll(r.Body) + _ = json.Unmarshal(raw, &gotReq) + w.Header().Set("Content-Type", contentTypeJSON) + _, _ = w.Write([]byte(`{"consentRequired":true,"claims":[{"ownerId":"did:key:zOwner"}]}`)) + })) + defer srv.Close() + + c := NewClient(srv.URL, DefaultTimeoutMs) + _, err := c.Resolve(context.Background(), + Resource{Service: "svc", Method: "GET", Path: "/p", ContentType: "application/xml"}, + Parties{Consumer: testConsumer}, + []byte("")) + if err != nil { + t.Fatalf("Resolve: %v", err) + } + + if gotReq.Body == nil { + t.Fatal("no body descriptor was sent") + } + if gotReq.Body.Encoding != encodingOpaque { + t.Errorf("encoding = %q, want %q — an unreadable payload must not look like no payload", + gotReq.Body.Encoding, encodingOpaque) + } + if gotReq.Body.ContentType != "application/xml" { + t.Errorf("contentType = %q, want application/xml", gotReq.Body.ContentType) + } + if len(gotReq.Body.Content) != 0 { + t.Errorf("an opaque body must not carry content, got %q", string(gotReq.Body.Content)) + } +} diff --git a/internal/plugin/config.go b/internal/plugin/config.go index 04c52e6..2c9ec25 100644 --- a/internal/plugin/config.go +++ b/internal/plugin/config.go @@ -15,16 +15,16 @@ * limitations under the License. */ -// Package plugin implements the APISIX consent-filter plugin that intercepts -// HTTP responses and applies consent-based filtering for personal data. package plugin import ( + "consent-plugin/internal/logging" "encoding/json" "errors" "fmt" "net/url" "os" + "strings" ) // Default values for optional configuration fields. @@ -36,6 +36,59 @@ const ( // OwnerResolver calls. DefaultOwnerResolverTimeout = 2000 + // DefaultResponsePhaseTimeout is the default budget in milliseconds for the + // ENTIRE response phase - the party lookups, the resolve call and every + // per-owner consent check together. APISIX holds the buffered response for + // this whole time, so it must be bounded independently of the per-call + // timeouts, which multiply by the number of owners. + DefaultResponsePhaseTimeout = 10000 + + // DefaultMaxOwnersPerResponse is the default cap on how many distinct data + // owners are checked for one response. A collection endpoint returning + // hundreds of entities would otherwise issue hundreds of consent checks + // before answering. + DefaultMaxOwnersPerResponse = 50 + + // MinMaxOwnersPerResponse and MaxMaxOwnersPerResponse bound the cap itself. + MinMaxOwnersPerResponse = 1 + MaxMaxOwnersPerResponse = 1000 + + // DefaultMaxResolveBodyBytes is the default cap on the upstream body + // forwarded to the OwnerResolver (1 MiB). + DefaultMaxResolveBodyBytes = 1 << 20 + + // MinMaxResolveBodyBytes and MaxMaxResolveBodyBytes bound that cap (up to + // 100 MiB, which is already well past what a gated route should return). + MinMaxResolveBodyBytes = 1 + MaxMaxResolveBodyBytes = 100 << 20 + + // MinResponsePhaseTimeout and MaxResponsePhaseTimeout bound the response-phase + // budget in milliseconds (120s is already far beyond any sane gateway timeout). + MinResponsePhaseTimeout = 1 + MaxResponsePhaseTimeout = 120000 + + // MinOwnerResolverTimeout and MaxOwnerResolverTimeout bound the per-call + // OwnerResolver timeout in milliseconds. + MinOwnerResolverTimeout = 1 + MaxOwnerResolverTimeout = 60000 + + // DefaultParticipantTokenTTL is how long, in seconds, a fetched participant + // token is cached when the route does not say. It matches the consent + // client's own default. + DefaultParticipantTokenTTL = 3000 + + // MinParticipantTokenTTL and MaxParticipantTokenTTL bound the token cache + // lifetime in seconds. The upper bound is a day — far longer than any token + // this plugin is issued — and it also keeps the value away from the range + // where `time.Duration(ttl) * time.Second` overflows into a negative + // duration, which would make every cached token instantly expired. + MinParticipantTokenTTL = 1 + MaxParticipantTokenTTL = 86400 + + // apiPrefixSeparator is the path separator an API prefix must start with. A + // prefix without it is silently concatenated into a malformed URL. + apiPrefixSeparator = "/" + // DefaultConsumerClaim is the dotted claim path holding the consuming // participant's identity. The provider's verifier embeds the presented // credential in the access token (jwtInclusion.fullInclusion), so the @@ -87,6 +140,11 @@ const ( // EnvTokenServiceURL supplies TokenServiceURL (the participant-local OID4VP // token service). + // + // #nosec G101 -- this is the NAME of an environment variable, not a + // credential; it trips the hardcoded-credentials heuristic only because it + // contains "TOKEN". The value it names is read from the environment at + // ParseConfig time (see applyEnv) and never appears in the source. EnvTokenServiceURL = "CONSENT_TOKEN_SERVICE_URL" // EnvAuditOTLPEndpoint supplies AuditOTLPEndpoint (the OTLP/HTTP Collector @@ -118,21 +176,26 @@ type Config struct { // Defaults to DefaultJWTHeaderName ("Authorization"). JWTHeaderName string `json:"jwt_header_name,omitempty"` - // JWTClaimsToForward specifies which JWT claims to send to the consent API. - // For example: ["sub", "scope"]. Must include "sub" — the consent check - // resolves the data subject from the "sub" claim. + // JWTClaimsToForward lists the JWT claims the request phase decodes and keeps. + // An empty list decodes every claim. The claims identify the CONSUMER (see + // ConsumerClaim) for the contract lookup; they never identify the data owner. JWTClaimsToForward []string `json:"jwt_claims_to_forward,omitempty"` // ConsentAPIPrefix is the consent-manager API prefix prepended to endpoint // paths. Defaults to DefaultConsentAPIPrefix ("/v1"). ConsentAPIPrefix string `json:"consent_api_prefix,omitempty"` - // OwnerResolverURL is the external OwnerResolver /resolve endpoint. When set, - // the data owner is resolved from the RESPONSE DATA (never the requestor): + // OwnerResolverURL is the external OwnerResolver /resolve endpoint (required). + // The data owner is resolved from the RESPONSE DATA, never from the requestor: // the plugin posts the payload, gets back (owner[, dataResource]) claims, and - // checks consent per owner. When empty, the plugin falls back to the legacy - // behaviour of taking the subject from the JWT. - OwnerResolverURL string `json:"owner_resolver_url,omitempty"` + // checks consent per owner. + // + // It is required because the only alternative — treating the access token's + // "sub" as the data subject — asks whether the CALLER has consented, which + // establishes no link between the caller and the data being returned and so + // gates nothing (any subject with one granted consent becomes a universal + // reader). + OwnerResolverURL string `json:"owner_resolver_url"` // ConsentAPIHost overrides the HTTP Host header sent on consent-manager // calls. Needed when ConsentAPIURL points at an in-cluster gateway service @@ -145,6 +208,42 @@ type Config struct { // OwnerResolver (defaults to DefaultOwnerResolverTimeout). OwnerResolverTimeout int `json:"owner_resolver_timeout,omitempty"` + // ResponsePhaseTimeout bounds, in milliseconds, the whole response phase: + // the party lookups, the resolve call and every per-owner consent check + // together. Without it the worst case is the per-call timeout multiplied by + // the number of owners, all while APISIX holds the buffered response. + // Defaults to DefaultResponsePhaseTimeout. + ResponsePhaseTimeout int `json:"response_phase_timeout,omitempty"` + + // RequirePurpose makes a resolved claim that names no processing purpose a + // denial instead of an unscoped check. + // + // Purpose matching depends on the OwnerResolver populating an optional field, + // so a resolver whose rules never set it silently runs with purpose scoping + // disabled — a consent granted for one purpose then authorises release for + // any other. Defaults to false, because requiring it would break every + // deployment whose resolver does not emit it yet; turn it on once yours does, + // and the property becomes enforced rather than hoped for. The + // consent_purpose_unconstrained_total metric counts the checks this would + // have denied. + RequirePurpose bool `json:"require_purpose,omitempty"` + + // MaxResolveBodyBytes caps the upstream body forwarded to the OwnerResolver. + // + // The body is read whole, validated as JSON, and marshalled again into the + // resolve envelope, so the peak footprint is roughly 3x its size per in-flight + // request — on top of APISIX's own buffering of the same response. A handful + // of concurrent large-collection responses can therefore drive the runner's + // memory well past what the response size suggests. A body above this is + // denied rather than forwarded. Defaults to DefaultMaxResolveBodyBytes. + MaxResolveBodyBytes int `json:"max_resolve_body_bytes,omitempty"` + + // MaxOwnersPerResponse caps how many distinct data owners are checked for a + // single response. A response resolving to more owners than this is denied + // rather than answered after an unbounded number of consent calls. + // Defaults to DefaultMaxOwnersPerResponse. + MaxOwnersPerResponse int `json:"max_owners_per_response,omitempty"` + // Service is the logical dataset id sent to the OwnerResolver as // resource.service, so it can select the right rule for this route. Service string `json:"service,omitempty"` @@ -174,8 +273,9 @@ type Config struct { TokenAudience string `json:"token_audience,omitempty"` // ParticipantTokenTTL caps, in seconds, how long a fetched token is cached - // (defaults to 3000s). The token service reports its own lifetime; the - // shorter of the two wins. Ignored for a static token. + // (defaults to DefaultParticipantTokenTTL). The token service reports its own + // lifetime; the shorter of the two wins. Ignored for a static token. Bounded + // by Validate, so it cannot overflow when converted to a time.Duration. ParticipantTokenTTL int `json:"participant_token_ttl,omitempty"` // ParticipantToken is an optional *static*, pre-obtained access token for the @@ -200,10 +300,12 @@ type Config struct { // Defaults to DefaultDenyResponseContentType ("application/json"). DenyResponseContentType string `json:"deny_response_content_type,omitempty"` - // FailOpen controls the behavior when the consent API is unavailable or - // returns an error. When nil or true (default), responses pass through - // on consent API errors (fail-open). When false, responses are denied - // on consent API errors (fail-closed). + // FailOpen controls what happens when a dependency is unavailable or errors. + // It defaults to FALSE (fail-closed): the failure mode of a consent gate must + // not be "release the personal data", and it must certainly not be reached by + // omitting a field. Setting it to true is a deliberate, logged decision to + // prefer availability over the gate, and even then it does not apply to + // conditions that are misconfigurations rather than outages (see failMode). FailOpen *bool `json:"fail_open,omitempty"` // AuditEnabled turns on emitting an access-decision audit event to an @@ -217,17 +319,22 @@ type Config struct { // AuditEnabled. Falls back to the EnvAuditOTLPEndpoint env var when empty. AuditOTLPEndpoint string `json:"audit_otlp_endpoint,omitempty"` + // AuditOTLPHeaders are extra HTTP headers sent on every audit export, for a + // Collector that requires authentication (e.g. {"Authorization": "Bearer ..."} + // or a tenant header). + AuditOTLPHeaders map[string]string `json:"audit_otlp_headers,omitempty"` + // AuditServiceName is the resource service.name stamped on audit records - // the marker the Collector routes on to keep audit logs separate from traces. // Empty defaults to the audit package's DefaultServiceName. AuditServiceName string `json:"audit_service_name,omitempty"` } -// IsFailOpen returns whether the plugin should fail-open when the consent API -// is unavailable. Returns true (fail-open) by default when FailOpen is nil. +// IsFailOpen returns whether the plugin should fail-open when a dependency is +// unavailable. Returns false (fail-closed) by default when FailOpen is nil. func (c *Config) IsFailOpen() bool { if c.FailOpen == nil { - return true + return false } return *c.FailOpen } @@ -244,10 +351,27 @@ func (c *Config) applyDefaults() { if c.ConsentAPIPrefix == "" { c.ConsentAPIPrefix = DefaultConsentAPIPrefix } - if c.OwnerResolverURL != "" && c.OwnerResolverTimeout == 0 { + // The prefix is concatenated directly with the endpoint path, so a trailing + // separator would produce a double slash in every URL. + if c.ConsentAPIPrefix != apiPrefixSeparator { + c.ConsentAPIPrefix = strings.TrimRight(c.ConsentAPIPrefix, apiPrefixSeparator) + } + if c.OwnerResolverTimeout == 0 { c.OwnerResolverTimeout = DefaultOwnerResolverTimeout } - if c.OwnerResolverURL != "" && c.ConsumerClaim == "" { + if c.ResponsePhaseTimeout == 0 { + c.ResponsePhaseTimeout = DefaultResponsePhaseTimeout + } + if c.MaxOwnersPerResponse == 0 { + c.MaxOwnersPerResponse = DefaultMaxOwnersPerResponse + } + if c.MaxResolveBodyBytes == 0 { + c.MaxResolveBodyBytes = DefaultMaxResolveBodyBytes + } + if c.ParticipantTokenTTL == 0 { + c.ParticipantTokenTTL = DefaultParticipantTokenTTL + } + if c.ConsumerClaim == "" { c.ConsumerClaim = DefaultConsumerClaim } if c.TokenAudience == "" { @@ -296,6 +420,28 @@ func (c *Config) Validate() error { return fmt.Errorf("config validation: consent_api_url must use http or https scheme, got %q", parsedURL.Scheme) } + // The resolver is the only source of data ownership, so a route without one + // cannot gate anything and must not load. + if c.OwnerResolverURL == "" { + return errors.New("config validation: owner_resolver_url is required — " + + "the data owner is resolved from the response data, and without a resolver the plugin cannot determine whose consent to check") + } + resolverURL, err := url.ParseRequestURI(c.OwnerResolverURL) + if err != nil { + return fmt.Errorf("config validation: owner_resolver_url is not a valid URL: %w", err) + } + if resolverURL.Scheme != schemeHTTP && resolverURL.Scheme != schemeHTTPS { + return fmt.Errorf("config validation: owner_resolver_url must use http or https scheme, got %q", resolverURL.Scheme) + } + + // The prefix is concatenated with the endpoint path rather than joined, so a + // missing leading separator silently yields a malformed URL and every call + // fails with a confusing 404. + if !strings.HasPrefix(c.ConsentAPIPrefix, apiPrefixSeparator) { + return fmt.Errorf("config validation: consent_api_prefix must start with %q, got %q", + apiPrefixSeparator, c.ConsentAPIPrefix) + } + if c.ConsentAPITimeout < MinConsentAPITimeout || c.ConsentAPITimeout > MaxConsentAPITimeout { return fmt.Errorf("config validation: consent_api_timeout must be between %d and %d, got %d", MinConsentAPITimeout, MaxConsentAPITimeout, c.ConsentAPITimeout) @@ -306,18 +452,33 @@ func (c *Config) Validate() error { MinHTTPStatusCode, MaxHTTPStatusCode, c.DenyStatusCode) } - if c.AuditEnabled && c.AuditOTLPEndpoint == "" { - return errors.New("config validation: audit_otlp_endpoint is required when audit_enabled is true") + if c.OwnerResolverTimeout < MinOwnerResolverTimeout || c.OwnerResolverTimeout > MaxOwnerResolverTimeout { + return fmt.Errorf("config validation: owner_resolver_timeout must be between %d and %d, got %d", + MinOwnerResolverTimeout, MaxOwnerResolverTimeout, c.OwnerResolverTimeout) } - if c.OwnerResolverURL != "" { - resolverURL, err := url.ParseRequestURI(c.OwnerResolverURL) - if err != nil { - return fmt.Errorf("config validation: owner_resolver_url is not a valid URL: %w", err) - } - if resolverURL.Scheme != schemeHTTP && resolverURL.Scheme != schemeHTTPS { - return fmt.Errorf("config validation: owner_resolver_url must use http or https scheme, got %q", resolverURL.Scheme) - } + if c.ResponsePhaseTimeout < MinResponsePhaseTimeout || c.ResponsePhaseTimeout > MaxResponsePhaseTimeout { + return fmt.Errorf("config validation: response_phase_timeout must be between %d and %d, got %d", + MinResponsePhaseTimeout, MaxResponsePhaseTimeout, c.ResponsePhaseTimeout) + } + + if c.MaxResolveBodyBytes < MinMaxResolveBodyBytes || c.MaxResolveBodyBytes > MaxMaxResolveBodyBytes { + return fmt.Errorf("config validation: max_resolve_body_bytes must be between %d and %d, got %d", + MinMaxResolveBodyBytes, MaxMaxResolveBodyBytes, c.MaxResolveBodyBytes) + } + + if c.ParticipantTokenTTL < MinParticipantTokenTTL || c.ParticipantTokenTTL > MaxParticipantTokenTTL { + return fmt.Errorf("config validation: participant_token_ttl must be between %d and %d, got %d", + MinParticipantTokenTTL, MaxParticipantTokenTTL, c.ParticipantTokenTTL) + } + + if c.MaxOwnersPerResponse < MinMaxOwnersPerResponse || c.MaxOwnersPerResponse > MaxMaxOwnersPerResponse { + return fmt.Errorf("config validation: max_owners_per_response must be between %d and %d, got %d", + MinMaxOwnersPerResponse, MaxMaxOwnersPerResponse, c.MaxOwnersPerResponse) + } + + if c.AuditEnabled && c.AuditOTLPEndpoint == "" { + return errors.New("config validation: audit_otlp_endpoint is required when audit_enabled is true") } if c.TokenServiceURL != "" { @@ -330,6 +491,15 @@ func (c *Config) Validate() error { } } + // Call 2 is authenticated as the participant, so a route with neither a token + // service nor a static token cannot complete a single check. Loading it + // cleanly and discovering that per request — as one log line, on the data + // path — is how a typo becomes an outage or, with fail_open, a silent bypass. + if c.TokenServiceURL == "" && c.ParticipantToken == "" { + return errors.New("config validation: one of token_service_url or participant_token is required — " + + "without a way to authenticate as the participant no consent check can succeed") + } + return nil } @@ -352,5 +522,10 @@ func ParseConfig(in []byte) (*Config, error) { return nil, err } + if conf.IsFailOpen() { + logging.Warnf("fail_open is enabled for %s — a consent-manager or resolver outage will RELEASE personal data instead of denying it", + conf.ConsentAPIURL) + } + return &conf, nil } diff --git a/internal/plugin/config_doc_test.go b/internal/plugin/config_doc_test.go new file mode 100644 index 0000000..1a35ec3 --- /dev/null +++ b/internal/plugin/config_doc_test.go @@ -0,0 +1,126 @@ +/* + * Copyright 2026 Seamless Middleware Technologies S.L and/or its affiliates + * and other contributors as indicated by the @author tags. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package plugin + +import ( + "go/ast" + "go/parser" + "go/token" + "os" + "reflect" + "regexp" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// readmePath is the README, relative to this package, that must document the +// plugin's configuration surface. +const readmePath = "../../README.md" + +// configSourcePath is the file declaring the Config struct. +const configSourcePath = "config.go" + +// configSectionHeading opens the README section holding the configuration +// table; the section ends at the next top-level heading. +const configSectionHeading = "## Configuration Reference" + +// readmeFieldPattern matches a config field named in the first column of the +// README's configuration table, e.g. "| `consent_api_url` | ...". +var readmeFieldPattern = regexp.MustCompile("(?m)^\\|\\s*`([a-z0-9_]+)`\\s*\\|") + +// TestREADMEDocumentsEveryConfigField guards against documentation drift on a +// security control. +// +// This repository has already shipped a README describing `client_id` / +// `client_secret` months after those fields were removed from Config. Because +// json.Unmarshal silently discards unknown fields, a config copied from that +// README parsed and validated cleanly and then could not authenticate at all — +// a documentation defect that was, in effect, a security defect. Drift on this +// surface needs a machine, not discipline. +func TestREADMEDocumentsEveryConfigField(t *testing.T) { + documented := documentedConfigFields(t) + declared := declaredConfigFields(t) + + for _, field := range declared { + assert.Contains(t, documented, field, + "config field %q is not documented in the README configuration table", field) + } + for field := range documented { + assert.Contains(t, declared, field, + "the README documents %q, which is not a field of Config — a reader would configure something that is silently discarded", field) + } +} + +// documentedConfigFields returns the field names appearing in the README's +// configuration table. +func documentedConfigFields(t *testing.T) map[string]bool { + t.Helper() + readme, err := os.ReadFile(readmePath) + require.NoError(t, err, "the README must be readable to check it for drift") + + // Only the configuration section counts: other tables in the README describe + // unrelated things (compose services, release artifacts). + _, section, found := strings.Cut(string(readme), configSectionHeading) + require.True(t, found, "the README must have a %q section", configSectionHeading) + if next := strings.Index(section, "\n## "); next >= 0 { + section = section[:next] + } + + fields := map[string]bool{} + for _, match := range readmeFieldPattern.FindAllStringSubmatch(section, -1) { + fields[match[1]] = true + } + require.NotEmpty(t, fields, "no configuration table found in the README") + return fields +} + +// declaredConfigFields returns the json tag names of every field of Config, read +// from the source rather than by reflection so that the check does not depend on +// a field being exported or populated. +func declaredConfigFields(t *testing.T) []string { + t.Helper() + file, err := parser.ParseFile(token.NewFileSet(), configSourcePath, nil, 0) + require.NoError(t, err) + + var fields []string + ast.Inspect(file, func(n ast.Node) bool { + typeSpec, ok := n.(*ast.TypeSpec) + if !ok || typeSpec.Name.Name != "Config" { + return true + } + structType, ok := typeSpec.Type.(*ast.StructType) + if !ok { + return false + } + for _, field := range structType.Fields.List { + if field.Tag == nil { + continue + } + tag := reflect.StructTag(strings.Trim(field.Tag.Value, "`")).Get("json") + if name := strings.Split(tag, ",")[0]; name != "" && name != "-" { + fields = append(fields, name) + } + } + return false + }) + require.NotEmpty(t, fields, "no json-tagged fields found on Config") + return fields +} diff --git a/internal/plugin/config_test.go b/internal/plugin/config_test.go index bb3abf1..a27ae19 100644 --- a/internal/plugin/config_test.go +++ b/internal/plugin/config_test.go @@ -28,7 +28,9 @@ import ( // validConfigJSON returns a minimal valid configuration JSON for testing. func validConfigJSON() map[string]interface{} { return map[string]interface{}{ - "consent_api_url": "https://consent.example.com/api", + "consent_api_url": "https://consent.example.com/api", + "owner_resolver_url": "https://owner-resolver.example.com/resolve", + "token_service_url": "https://consent-facade.example.com/internal/tokens", } } @@ -50,7 +52,7 @@ func TestParseConfig(t *testing.T) { }{ { name: "valid config with only required field applies defaults", - input: []byte(`{"consent_api_url": "https://consent.example.com/api"}`), + input: []byte(`{"consent_api_url": "https://consent.example.com/api", "owner_resolver_url": "https://owner-resolver.example.com/resolve", "token_service_url": "https://facade.example.com/internal/tokens"}`), check: func(t *testing.T, cfg *Config) { assert.Equal(t, "https://consent.example.com/api", cfg.ConsentAPIURL) assert.Equal(t, DefaultConsentAPITimeout, cfg.ConsentAPITimeout) @@ -66,6 +68,8 @@ func TestParseConfig(t *testing.T) { input: func() []byte { m := map[string]interface{}{ "consent_api_url": "http://localhost:8080/consent", + "owner_resolver_url": "http://localhost:9090/resolve", + "participant_token": "static-token", "consent_api_timeout": 10000, "jwt_header_name": "X-Auth-Token", "jwt_claims_to_forward": []string{"sub", "scope", "aud"}, @@ -78,6 +82,7 @@ func TestParseConfig(t *testing.T) { }(), check: func(t *testing.T, cfg *Config) { assert.Equal(t, "http://localhost:8080/consent", cfg.ConsentAPIURL) + assert.Equal(t, "http://localhost:9090/resolve", cfg.OwnerResolverURL) assert.Equal(t, 10000, cfg.ConsentAPITimeout) assert.Equal(t, "X-Auth-Token", cfg.JWTHeaderName) assert.Equal(t, []string{"sub", "scope", "aud"}, cfg.JWTClaimsToForward) @@ -242,7 +247,9 @@ func TestParseConfig_EnvFallback(t *testing.T) { t.Setenv(EnvConsentKey, "ck-from-env") t.Setenv(EnvTokenServiceURL, "http://facade-from-env:8080/internal/tokens") - cfg, err := ParseConfig(toJSON(t, validConfigJSON())) + in := validConfigJSON() + delete(in, "token_service_url") // so the env var is the only source + cfg, err := ParseConfig(toJSON(t, in)) require.NoError(t, err) assert.Equal(t, "ck-from-env", cfg.ConsentKey) assert.Equal(t, "http://facade-from-env:8080/internal/tokens", cfg.TokenServiceURL) @@ -351,6 +358,14 @@ func TestConfig_Validate(t *testing.T) { config: Config{ ConsentAPIURL: "https://consent.example.com", ConsentAPITimeout: DefaultConsentAPITimeout, + OwnerResolverURL: "https://owner-resolver.example.com/resolve", + OwnerResolverTimeout: DefaultOwnerResolverTimeout, + ConsentAPIPrefix: DefaultConsentAPIPrefix, + ParticipantToken: "static-token", + ParticipantTokenTTL: DefaultParticipantTokenTTL, + MaxResolveBodyBytes: DefaultMaxResolveBodyBytes, + ResponsePhaseTimeout: DefaultResponsePhaseTimeout, + MaxOwnersPerResponse: DefaultMaxOwnersPerResponse, JWTHeaderName: DefaultJWTHeaderName, DenyStatusCode: DefaultDenyStatusCode, DenyResponseBody: DefaultDenyResponseBody, @@ -366,11 +381,120 @@ func TestConfig_Validate(t *testing.T) { wantErr: true, errSubstr: "consent_api_url is required", }, + { + name: "missing owner_resolver_url fails", + config: Config{ + ConsentAPIURL: "https://consent.example.com", + ConsentAPITimeout: DefaultConsentAPITimeout, + ConsentAPIPrefix: DefaultConsentAPIPrefix, + OwnerResolverTimeout: DefaultOwnerResolverTimeout, + ParticipantTokenTTL: DefaultParticipantTokenTTL, + MaxResolveBodyBytes: DefaultMaxResolveBodyBytes, + ResponsePhaseTimeout: DefaultResponsePhaseTimeout, + MaxOwnersPerResponse: DefaultMaxOwnersPerResponse, + DenyStatusCode: DefaultDenyStatusCode, + }, + wantErr: true, + errSubstr: "owner_resolver_url is required", + }, + { + name: "an out-of-range max_resolve_body_bytes fails", + config: Config{ + ConsentAPIURL: "https://consent.example.com", + ConsentAPITimeout: DefaultConsentAPITimeout, + ConsentAPIPrefix: DefaultConsentAPIPrefix, + OwnerResolverURL: "https://owner-resolver.example.com/resolve", + OwnerResolverTimeout: DefaultOwnerResolverTimeout, + ParticipantToken: "static-token", + ParticipantTokenTTL: DefaultParticipantTokenTTL, + MaxResolveBodyBytes: MaxMaxResolveBodyBytes + 1, + ResponsePhaseTimeout: DefaultResponsePhaseTimeout, + MaxOwnersPerResponse: DefaultMaxOwnersPerResponse, + DenyStatusCode: DefaultDenyStatusCode, + }, + wantErr: true, + errSubstr: "max_resolve_body_bytes must be between", + }, + { + name: "an out-of-range participant_token_ttl fails", + config: Config{ + ConsentAPIURL: "https://consent.example.com", + ConsentAPITimeout: DefaultConsentAPITimeout, + ConsentAPIPrefix: DefaultConsentAPIPrefix, + OwnerResolverURL: "https://owner-resolver.example.com/resolve", + OwnerResolverTimeout: DefaultOwnerResolverTimeout, + ParticipantToken: "static-token", + // Large enough that `time.Duration(ttl) * time.Second` overflows + // into a negative duration, expiring every token immediately. + ParticipantTokenTTL: 1 << 60, + MaxResolveBodyBytes: DefaultMaxResolveBodyBytes, + ResponsePhaseTimeout: DefaultResponsePhaseTimeout, + MaxOwnersPerResponse: DefaultMaxOwnersPerResponse, + DenyStatusCode: DefaultDenyStatusCode, + }, + wantErr: true, + errSubstr: "participant_token_ttl must be between", + }, + { + name: "no credential source fails", + config: Config{ + ConsentAPIURL: "https://consent.example.com", + ConsentAPITimeout: DefaultConsentAPITimeout, + ConsentAPIPrefix: DefaultConsentAPIPrefix, + OwnerResolverURL: "https://owner-resolver.example.com/resolve", + OwnerResolverTimeout: DefaultOwnerResolverTimeout, + ParticipantTokenTTL: DefaultParticipantTokenTTL, + MaxResolveBodyBytes: DefaultMaxResolveBodyBytes, + ResponsePhaseTimeout: DefaultResponsePhaseTimeout, + MaxOwnersPerResponse: DefaultMaxOwnersPerResponse, + DenyStatusCode: DefaultDenyStatusCode, + }, + wantErr: true, + errSubstr: "one of token_service_url or participant_token is required", + }, + { + name: "a consent_api_prefix without a leading slash fails", + config: Config{ + ConsentAPIURL: "https://consent.example.com", + ConsentAPITimeout: DefaultConsentAPITimeout, + ConsentAPIPrefix: "v1", + OwnerResolverURL: "https://owner-resolver.example.com/resolve", + OwnerResolverTimeout: DefaultOwnerResolverTimeout, + ParticipantToken: "static-token", + ParticipantTokenTTL: DefaultParticipantTokenTTL, + MaxResolveBodyBytes: DefaultMaxResolveBodyBytes, + ResponsePhaseTimeout: DefaultResponsePhaseTimeout, + MaxOwnersPerResponse: DefaultMaxOwnersPerResponse, + DenyStatusCode: DefaultDenyStatusCode, + }, + wantErr: true, + errSubstr: "consent_api_prefix must start with", + }, + { + name: "an out-of-range owner_resolver_timeout fails", + config: Config{ + ConsentAPIURL: "https://consent.example.com", + ConsentAPITimeout: DefaultConsentAPITimeout, + ConsentAPIPrefix: DefaultConsentAPIPrefix, + OwnerResolverURL: "https://owner-resolver.example.com/resolve", + OwnerResolverTimeout: MaxOwnerResolverTimeout + 1, + ParticipantToken: "static-token", + ParticipantTokenTTL: DefaultParticipantTokenTTL, + MaxResolveBodyBytes: DefaultMaxResolveBodyBytes, + ResponsePhaseTimeout: DefaultResponsePhaseTimeout, + MaxOwnersPerResponse: DefaultMaxOwnersPerResponse, + DenyStatusCode: DefaultDenyStatusCode, + }, + wantErr: true, + errSubstr: "owner_resolver_timeout must be between", + }, { name: "negative timeout fails", config: Config{ ConsentAPIURL: "https://consent.example.com", ConsentAPITimeout: -1, + ConsentAPIPrefix: DefaultConsentAPIPrefix, + OwnerResolverURL: "https://owner-resolver.example.com/resolve", DenyStatusCode: DefaultDenyStatusCode, }, wantErr: true, @@ -398,7 +522,7 @@ func TestConsentFilter_ParseConf_Integration(t *testing.T) { p := &ConsentFilter{} t.Run("valid config returns *Config", func(t *testing.T) { - input := []byte(`{"consent_api_url": "https://consent.example.com/api"}`) + input := []byte(`{"consent_api_url": "https://consent.example.com/api", "owner_resolver_url": "https://owner-resolver.example.com/resolve", "token_service_url": "https://facade.example.com/internal/tokens"}`) conf, err := p.ParseConf(input) require.NoError(t, err) @@ -421,3 +545,95 @@ func TestConsentFilter_ParseConf_Integration(t *testing.T) { assert.Nil(t, conf) }) } + +func TestConfig_IsFailOpen(t *testing.T) { + tests := []struct { + name string + failOpen *bool + want bool + }{ + {name: "nil defaults to false (fail-closed)", failOpen: nil, want: false}, + {name: "explicitly true is fail-open", failOpen: boolPtr(true), want: true}, + {name: "explicitly false is fail-closed", failOpen: boolPtr(false), want: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := &Config{FailOpen: tt.failOpen} + assert.Equal(t, tt.want, cfg.IsFailOpen()) + }) + } +} + +// TestClaimKeysToDecode verifies which claims the request phase decodes: the +// configured forward list plus the root of the consumer-claim path, so the +// consuming participant can still be read in the response phase. +func TestClaimKeysToDecode(t *testing.T) { + tests := []struct { + name string + forward []string + consumer string + want []string + }{ + { + name: "empty forward list decodes every claim", + want: nil, + }, + { + name: "the consumer-claim root is added to the forward list", + forward: []string{"sub"}, + consumer: "verifiableCredential.issuer", + want: []string{"sub", "verifiableCredential"}, + }, + { + name: "an already-listed root is not added twice", + forward: []string{"sub", "verifiableCredential"}, + consumer: "verifiableCredential.issuer", + want: []string{"sub", "verifiableCredential"}, + }, + { + name: "a single-segment consumer claim is its own root", + forward: []string{"sub"}, + consumer: "issuer", + want: []string{"sub", "issuer"}, + }, + { + name: "no consumer claim leaves the forward list alone", + forward: []string{"sub"}, + want: []string{"sub"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := claimKeysToDecode(&Config{JWTClaimsToForward: tt.forward, ConsumerClaim: tt.consumer}) + assert.Equal(t, tt.want, got) + }) + } +} + +// TestParseConfig_PrefixNormalisation verifies a trailing separator is trimmed +// rather than concatenated into a double slash in every endpoint URL. +func TestParseConfig_PrefixNormalisation(t *testing.T) { + cases := []struct { + name string + configured string + want string + }{ + {name: "defaults when omitted", configured: "", want: DefaultConsentAPIPrefix}, + {name: "keeps a well-formed prefix", configured: "/v2", want: "/v2"}, + {name: "trims a trailing separator", configured: "/v2/", want: "/v2"}, + {name: "a bare separator is left alone", configured: "/", want: "/"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + in := validConfigJSON() + if tc.configured != "" { + in["consent_api_prefix"] = tc.configured + } + cfg, err := ParseConfig(toJSON(t, in)) + require.NoError(t, err) + assert.Equal(t, tc.want, cfg.ConsentAPIPrefix) + }) + } +} diff --git a/internal/plugin/consent.go b/internal/plugin/consent.go index 735f79a..836c3cf 100644 --- a/internal/plugin/consent.go +++ b/internal/plugin/consent.go @@ -15,19 +15,32 @@ * limitations under the License. */ -// Package plugin implements the APISIX consent-filter plugin that intercepts -// HTTP responses and applies consent-based filtering for personal data. +// Package plugin implements the APISIX consent-filter plugin. +// +// The plugin gates a personal-data response on the consent of the DATA OWNER. +// The request phase captures the token's claims; the response phase asks the +// OwnerResolver who owns the payload and checks, per owner, that the consuming +// participant has a granted consent. The verdict is coarse — the whole response +// is allowed or replaced with a denial. Despite the plugin's registered name +// there is no field-level filtering or redaction: a gate that removes fields +// silently misses the one it does not know about, while a coarse gate still +// covers an empty or non-JSON personal-data response. package plugin import ( "consent-plugin/internal/audit" "consent-plugin/internal/consent" "consent-plugin/internal/jwt" + "consent-plugin/internal/logging" + "consent-plugin/internal/metrics" "consent-plugin/internal/ownerresolver" "context" - "log" + "errors" + "fmt" "net/http" + "strconv" "strings" + "sync" "time" pkgHTTP "github.com/apache/apisix-go-plugin-runner/pkg/http" @@ -37,9 +50,6 @@ import ( // pluginName is the registered name for this plugin in APISIX configuration. const pluginName = "consent-filter" -// jwtSubjectClaim is the JWT claim key used to extract the subject identity. -const jwtSubjectClaim = "sub" - // nginxRequestIDVar is the Nginx variable ($request_id) holding a unique id // per HTTP request. Unlike the runner's per-RPC ID(), it is identical in the // RequestFilter (ext-plugin-pre-req) and ResponseFilter (ext-plugin-post-resp) @@ -47,6 +57,16 @@ const jwtSubjectClaim = "sub" // the other. const nginxRequestIDVar = "request_id" +// nginxUpstreamContentTypeVar is the Nginx variable ($upstream_http_content_type) +// holding the Content-Type the upstream answered with. +// +// It is read in preference to Response.Header() because that method lazily +// materialises the runner's header map, and the runner then reports +// HasChange() == true for the response — so merely LOOKING at a header sent +// every allowed response back to APISIX down the "this response was modified" +// path, carrying an empty header diff. Reading a variable has no such effect. +const nginxUpstreamContentTypeVar = "upstream_http_content_type" + // varReader is the subset of the runner's Request/Response interfaces that // exposes Nginx variables. Both pkgHTTP.Request and pkgHTTP.Response satisfy it. type varReader interface { @@ -90,43 +110,41 @@ func (c *ConsentFilter) ParseConf(in []byte) (interface{}, error) { return ParseConfig(in) } -// RequestFilter intercepts incoming HTTP requests to capture request context -// (headers, JWT claims, path, method) for use during response filtering. -// It extracts the JWT from the configured header, decodes the requested claims, -// captures all request headers, and stores the context keyed by request ID -// for later retrieval in ResponseFilter. +// RequestFilter intercepts incoming HTTP requests to capture the context the +// response phase needs: the method, the path, and the claims decoded from the +// configured JWT header, stored under the request's correlation key. +// +// The JWT is decoded, NOT verified (see internal/jwt): the claims are used only +// to name the consuming participant for the contract lookup, and the route MUST +// have an authentication plugin in front of this one that validates the token. func (c *ConsentFilter) RequestFilter(conf interface{}, w http.ResponseWriter, r pkgHTTP.Request) { cfg, ok := conf.(*Config) if !ok { - log.Printf("[consent-filter] RequestFilter: invalid config type, skipping request %d", r.ID()) + logging.Errorf("RequestFilter: invalid config type, skipping request %d", r.ID()) return } reqCtx := &RequestContext{ - Method: r.Method(), - Path: string(r.Path()), - Headers: make(http.Header), + Method: r.Method(), + Path: string(r.Path()), } - // Capture request headers from the request's Header view. - if srcHeaders := r.Header().View(); srcHeaders != nil { - for key, values := range srcHeaders { - reqCtx.Headers[key] = values - } - } + // Only the configured JWT header is read, and only the claims are kept. The + // full header set is deliberately not retained: it would put the caller's + // bearer token in a process-lifetime map that nothing ever reads. // Extract JWT token and decode claims from the configured header. jwtHeaderValue := r.Header().Get(cfg.JWTHeaderName) if jwtHeaderValue != "" { token, err := jwt.ExtractToken(jwtHeaderValue) if err != nil { - log.Printf("[consent-filter] RequestFilter: failed to extract JWT from header %q for request %d: %v", - cfg.JWTHeaderName, r.ID(), err) + logging.WarnfEvery("jwt-extract", "RequestFilter: failed to extract JWT from header %q: %s", + cfg.JWTHeaderName, logging.Sanitize(err.Error())) } else { claims, err := jwt.DecodeClaims(token, claimKeysToDecode(cfg)) if err != nil { - log.Printf("[consent-filter] RequestFilter: failed to decode JWT claims for request %d: %v", - r.ID(), err) + logging.WarnfEvery("jwt-decode", "RequestFilter: failed to decode JWT claims: %s", + logging.Sanitize(err.Error())) } else { reqCtx.JWTClaims = claims } @@ -137,14 +155,28 @@ func (c *ConsentFilter) RequestFilter(conf interface{}, w http.ResponseWriter, r // the runner's per-RPC ID() (which differs between pre-req and post-resp). key, ok := correlationKey(r) if !ok { - log.Printf("[consent-filter] RequestFilter: could not read %q for request %d; consent context not stored", - nginxRequestIDVar, r.ID()) + logging.ErrorfEvery("no-request-id-req", "RequestFilter: could not read %q; consent context not stored", + nginxRequestIDVar) return } StoreRequestContext(key, reqCtx) } +// responseContentType reports the Content-Type of the upstream response, +// preferring the Nginx variable so an allowed response is not marked as +// modified (see nginxUpstreamContentTypeVar). The header is only consulted when +// the variable is unavailable, which is a degraded case rather than the norm. +func responseContentType(w pkgHTTP.Response) string { + if value, err := w.Var(nginxUpstreamContentTypeVar); err == nil && len(value) > 0 { + return string(value) + } + if header := w.Header(); header != nil { + return header.Get("Content-Type") + } + return "" +} + // decisionAllow and decisionDeny are the audit-facing labels for the decision. const ( decisionAllow = "allow" @@ -153,6 +185,12 @@ const ( // responseOutcome is the result of evaluating consent for one response: the // decision to enforce plus the fields needed to record it in the audit log. +// +// checked carries one entry per data owner whose consent was actually consulted. +// Recording only the outcome answered "was this response allowed?" but not +// "whose consent was checked, and what did each say?" — which is the question an +// access-decision audit log exists to answer. On an allow it named no owner at +// all, and on a deny only the first owner to refuse. type responseOutcome struct { decision string // decisionAllow | decisionDeny reason string @@ -160,31 +198,51 @@ type responseOutcome struct { subject string resource string method string + checked []checkedOwner + // failMode names why the decision could not be reached normally, so a deny + // caused by an outage is not counted as a deny caused by consent. Empty for + // an ordinary consent verdict. + failMode string } -// ResponseFilter gates the upstream response on the data subject's consent. +// checkedOwner is one data owner's consent decision within a response. +type checkedOwner struct { + subject string + resource string + decision string + reason string +} + +// ResponseFilter gates the upstream response on the data owner's consent. // // The flow is: // 1. Correlate with the request phase and load (and delete) the stored context. -// 2. Build a ConsentRequest (the subject comes from the JWT "sub" claim). -// 3. Run the two-call consent check against the consent-manager. +// 2. Ask the OwnerResolver, from the RESPONSE DATA, whether consent is required +// and who the data owner(s) are. +// 3. Run the two-call consent check per resolved owner (deny_all: every owner +// must have a granted consent). // 4. Allow → pass the response through unchanged; deny → replace it with the // configured denial response. -// 5. On unresolved context or a consent-manager error, apply the fail policy -// (deny unless explicitly fail-open). +// 5. On unresolved context, a resolver error, or a consent-manager error, apply +// the fail policy (deny unless explicitly fail-open). +// +// The requestor's identity is NEVER used to determine ownership: the token's +// "sub" says who is asking, not whose data is being returned, so a check against +// it would let any subject holding one granted consent read everyone's data. // // Every decision is recorded to the audit sink (when enabled) before it is -// enforced. The check is a coarse allow/deny on the subject's consent and is -// independent of the response body, so — unlike a field-level filter — an empty -// or non-JSON personal-data response is still gated rather than passed through. +// enforced. The check is a coarse allow/deny and is independent of the response +// body's shape, so — unlike a field-level filter — an empty or non-JSON +// personal-data response is still gated rather than passed through. func (c *ConsentFilter) ResponseFilter(conf interface{}, w pkgHTTP.Response) { cfg, ok := conf.(*Config) if !ok { - log.Printf("[consent-filter] ResponseFilter: invalid config type, skipping request %d", w.ID()) + logging.Errorf("ResponseFilter: invalid config type, skipping request %d", w.ID()) return } outcome := c.evaluate(cfg, w) + metrics.RecordDecision(outcome.decision, outcome.failMode) recordAudit(cfg, outcome) if outcome.decision == decisionDeny { denyResponse(w, cfg) @@ -199,8 +257,8 @@ func (c *ConsentFilter) evaluate(cfg *Config, w pkgHTTP.Response) responseOutcom // Correlate with the request phase via the stable Nginx $request_id. key, ok := correlationKey(w) if !ok { - log.Printf("[consent-filter] ResponseFilter: could not read %q for request %d; cannot verify consent", nginxRequestIDVar, w.ID()) - return failOutcome(cfg, "no request correlation id", "", nil) + logging.ErrorfEvery("no-request-id-resp", "ResponseFilter: could not read %q; cannot verify consent", nginxRequestIDVar) + return failOutcome(cfg, failAlwaysClosed, "no request correlation id", "", nil) } // Load and delete stored request context (cleanup to prevent memory leaks). @@ -209,65 +267,92 @@ func (c *ConsentFilter) evaluate(cfg *Config, w pkgHTTP.Response) responseOutcom // The request phase did not capture context for this request; the // consent decision cannot be made, so honor the fail policy instead // of silently passing the response through. - log.Printf("[consent-filter] ResponseFilter: no request context found for request %s; cannot verify consent", key) - return failOutcome(cfg, "no request context", key, nil) + logging.WarnfEvery("no-request-context", "ResponseFilter: no request context found for request %s; cannot verify consent", key) + return failOutcome(cfg, failAlwaysClosed, "no request context", key, nil) } + // Resolve the data owner(s) from the response DATA and check consent per + // owner. ParseConfig guarantees a resolver is configured. consentClient := consent.NewClient(clientConfigFromCfg(cfg)) - - // Owner-resolver mode: resolve the data owner(s) from the response DATA and - // check consent per owner (never the requestor). Falls back to the legacy - // JWT-subject mode when no resolver is configured. - if cfg.OwnerResolverURL != "" { - return c.evaluateWithResolver(cfg, w, key, reqCtx, consentClient) - } - - consentReq := buildConsentRequest(reqCtx) - return checkConsent(cfg, key, consentClient, consentReq) + return c.evaluateWithResolver(cfg, w, key, reqCtx, consentClient) } // evaluateWithResolver reads the upstream body, asks the OwnerResolver who owns // the data (and whether consent is required), and enforces deny_all: every // distinct (owner, dataResource) claim must have a granted consent, or the whole -// response is denied. The requestor identity is never consulted. +// response is denied. The requestor identity is never consulted for ownership. func (c *ConsentFilter) evaluateWithResolver(cfg *Config, w pkgHTTP.Response, key string, reqCtx *RequestContext, consentClient *consent.Client) responseOutcome { + // One deadline for the whole phase. Every call below derives from it, so the + // total time APISIX holds the buffered response is bounded no matter how many + // owners the payload resolves to, and a client that has already given up + // cancels the work rather than leaving it running against the dependencies. + phaseCtx, cancelPhase := context.WithTimeout(context.Background(), time.Duration(cfg.ResponsePhaseTimeout)*time.Millisecond) + defer cancelPhase() + body, err := w.ReadBody() if err != nil { - log.Printf("[consent-filter] ResponseFilter: could not read upstream body for request %s: %v", key, err) - return failOutcome(cfg, "read upstream body: "+err.Error(), key, nil) + logging.ErrorfEvery("read-body", "ResponseFilter: could not read the upstream body for request %s: %s", key, logging.Sanitize(err.Error())) + return failOutcome(cfg, failByPolicy, "read upstream body: "+err.Error(), key, nil) } - contentType := "" - if h := w.Header(); h != nil { - contentType = h.Get("Content-Type") + if len(body) > cfg.MaxResolveBodyBytes { + // Forwarding it would copy the body twice more (json.Valid, then the + // marshalled envelope) on top of APISIX's own buffering. Deny instead: + // a body too large to examine is not a body we can vouch for. + logging.WarnfEvery("resolve-body-cap", "ResponseFilter: upstream body of %d bytes for request %s exceeds max_resolve_body_bytes=%d; denying", + len(body), key, cfg.MaxResolveBodyBytes) + return failOutcome(cfg, failAlwaysClosed, + fmt.Sprintf("upstream body of %d bytes exceeds max_resolve_body_bytes=%d", len(body), cfg.MaxResolveBodyBytes), + key, nil) } - resolverClient := ownerresolver.NewClient(cfg.OwnerResolverURL, cfg.OwnerResolverTimeout) + contentType := responseContentType(w) + // Parties are for CONTRACT identification only - never for ownership. The // token names the consumer by DID, while contracts name their parties by // self-description URL, so translate it via the participant registry. + // + // Both sides are resolved BEFORE the resolver is asked anything, and a failure + // on either is terminal. Proceeding with an empty Parties would omit the field + // from /resolve entirely, and a resolver that cannot identify a contract may + // answer consentRequired:false — which is an unconditional allow. That would + // put a fail-open seam in the middle of a fail-closed design, reachable by + // nothing more than a briefly unreachable consent-manager or a revoked token. resolveParties := ownerresolver.Parties{} - if consumerDID := consumerFromClaims(reqCtx.JWTClaims, cfg.ConsumerClaim); consumerDID != "" { - if consumerSD, sdErr := consentClient.ParticipantSelfDescriptionByDID(context.Background(), consumerDID); sdErr != nil { - log.Printf("[consent-filter] ResponseFilter: could not map consumer did %q to a participant for request %s: %v", consumerDID, key, sdErr) - } else { - resolveParties.Consumer = consumerSD - } + consumerDID, claimErr := consumerFromClaims(reqCtx.JWTClaims, cfg.ConsumerClaim) + if claimErr != nil { + logging.WarnfEvery("consumer-claim", "ResponseFilter: could not read the consuming participant for request %s: %s", key, logging.Sanitize(claimErr.Error())) + return failOutcome(cfg, failAlwaysClosed, "no consuming participant identified: "+claimErr.Error(), key, nil) } - if providerSD, sdErr := consentClient.ProviderSelfDescription(context.Background()); sdErr != nil { - log.Printf("[consent-filter] ResponseFilter: could not determine the provider self-description for request %s: %v", key, sdErr) - } else { - resolveParties.Provider = providerSD + consumerSD, sdErr := consentClient.ParticipantSelfDescriptionByDID(phaseCtx, consumerDID) + if sdErr != nil { + logging.WarnfEvery("consumer-lookup", "ResponseFilter: could not map the consumer to a participant for request %s: %s", key, logging.Sanitize(sdErr.Error())) + return failOutcome(cfg, failModeForError(sdErr), "consumer participant lookup failed: "+sdErr.Error(), key, nil) + } + // The consumer also scopes the consent match itself: a consent names the one + // participant it was granted to, so releasing data to any other participant + // on the strength of it would authorise an agreement the subject never made. + resolveParties.Consumer = consumerSD + + providerSD, sdErr := consentClient.ProviderSelfDescription(phaseCtx) + if sdErr != nil { + logging.ErrorfEvery("provider-sd", "ResponseFilter: could not determine the provider self-description for request %s: %s", key, logging.Sanitize(sdErr.Error())) + return failOutcome(cfg, failModeForError(sdErr), "provider self-description lookup failed: "+sdErr.Error(), key, nil) } - result, err := resolverClient.Resolve(context.Background(), ownerresolver.Resource{ + resolveParties.Provider = providerSD + + resolverClient := ownerresolver.NewClient(cfg.OwnerResolverURL, cfg.OwnerResolverTimeout) + resolveStarted := time.Now() + result, err := resolverClient.Resolve(phaseCtx, ownerresolver.Resource{ Service: cfg.Service, Method: reqCtx.Method, Path: reqCtx.Path, ContentType: contentType, }, resolveParties, body) + metrics.RecordDependencyCall(metrics.DependencyOwnerResolver, outcomeOf(err), time.Since(resolveStarted)) if err != nil { - log.Printf("[consent-filter] ResponseFilter: owner resolver error for request %s: %v", key, err) - return failOutcome(cfg, "owner resolver error: "+err.Error(), key, nil) + logging.ErrorfEvery("resolver-error", "ResponseFilter: owner resolver error for request %s: %s", key, logging.Sanitize(err.Error())) + return failOutcome(cfg, failByPolicy, "owner resolver error: "+err.Error(), key, nil) } if !result.ConsentRequired { @@ -275,66 +360,238 @@ func (c *ConsentFilter) evaluateWithResolver(cfg *Config, w pkgHTTP.Response, ke } if len(result.Claims) == 0 { // Consent required but no owner could be resolved — fail closed. - return failOutcome(cfg, "consent required but no data owner resolved", key, nil) + return failOutcome(cfg, failAlwaysClosed, "consent required but no data owner resolved", key, nil) + } + + claims, err := distinctClaims(result.Claims) + if err != nil { + return failOutcome(cfg, failAlwaysClosed, err.Error(), key, nil) + } + if outcome, ok := checkPurposeScoping(cfg, key, claims); !ok { + return outcome + } + + if len(claims) > cfg.MaxOwnersPerResponse { + logging.WarnfEvery("owner-cap", "ResponseFilter: %d distinct data owners for request %s exceeds max_owners_per_response=%d; denying", + len(claims), key, cfg.MaxOwnersPerResponse) + return failOutcome(cfg, failAlwaysClosed, + fmt.Sprintf("response resolves to %d data owners, above max_owners_per_response=%d", len(claims), cfg.MaxOwnersPerResponse), + key, nil) } - // deny_all: every distinct (owner, dataResource) claim must be granted. + return checkOwners(phaseCtx, cfg, key, reqCtx, consentClient, claims, consumerSD) +} + +// checkPurposeScoping reports whether every resolved claim carries a processing +// purpose, and what to do when one does not. +// +// With require_purpose set, a claim without a purpose denies: it is a policy the +// operator asked for, not an outage, so the fail policy does not apply to it. +// Otherwise the check proceeds unscoped by purpose and is counted, so a resolver +// that has silently stopped emitting purposes is visible in the metrics instead +// of quietly widening what a consent authorises. +func checkPurposeScoping(cfg *Config, key string, claims []ownerClaim) (responseOutcome, bool) { + for _, claim := range claims { + if claim.purpose != "" { + continue + } + if cfg.RequirePurpose { + logging.WarnfEvery("purpose-required", "ResponseFilter: resolved claim without a processing purpose for request %s and require_purpose is set; denying", key) + return failOutcome(cfg, failAlwaysClosed, "resolved claim without a processing purpose", key, nil), false + } + metrics.RecordPurposeUnconstrained() + logging.WarnfEvery("purpose-unconstrained", "ResponseFilter: the resolver named no processing purpose, so consent is matched on the consumer alone; set require_purpose once the resolver emits one") + } + return responseOutcome{}, true +} + +// ownerClaim is one distinct (owner, dataResource) pair to check. +type ownerClaim struct { + owner string + dataResource string + purpose string +} + +// distinctClaims collapses the resolver's claims to the distinct +// (owner, dataResource) pairs that must be checked, preserving the resolver's +// order so the reported denial is stable. A claim naming no owner is an error: +// the resolver said consent is required but not whose. +func distinctClaims(claims []ownerresolver.Claim) ([]ownerClaim, error) { type pair struct{ owner, resource string } - checked := make(map[pair]bool) - for _, claim := range result.Claims { + seen := make(map[pair]bool, len(claims)) + distinct := make([]ownerClaim, 0, len(claims)) + for _, claim := range claims { if claim.OwnerID == "" { - return failOutcome(cfg, "resolved claim without a data owner", key, nil) + return nil, errors.New("resolved claim without a data owner") } p := pair{owner: claim.OwnerID, resource: claim.DataResource} - if checked[p] { + if seen[p] { continue } - checked[p] = true + seen[p] = true + distinct = append(distinct, ownerClaim{owner: claim.OwnerID, dataResource: claim.DataResource, purpose: claim.Purpose}) + } + return distinct, nil +} - req := consent.ConsentRequest{ - Subject: claim.OwnerID, - Resource: reqCtx.Path, - Method: reqCtx.Method, - DataResource: claim.DataResource, - } - resp, err := consentClient.CheckConsent(context.Background(), req) - if err != nil { - log.Printf("[consent-filter] ResponseFilter: consent check error for request %s: %v", key, err) - return failOutcome(cfg, "consent check error: "+err.Error(), key, &req) - } - if resp.Decision != consent.DecisionAllow { - return responseOutcome{ - decision: decisionDeny, - reason: resp.Reason, - requestID: key, - subject: claim.OwnerID, - resource: resourceOrPath(claim.DataResource, reqCtx.Path), - method: reqCtx.Method, +// outcomeOf maps a call's error to the metric's outcome label. +func outcomeOf(err error) string { + if err != nil { + return metrics.OutcomeError + } + return metrics.OutcomeSuccess +} + +// maxConcurrentConsentChecks bounds how many per-owner checks are in flight at +// once. Serial checks made the response latency the sum of every owner's; an +// unbounded fan-out would instead make one response a burst against the +// consent-manager. A small fixed width keeps both bounded. +const maxConcurrentConsentChecks = 8 + +// checkOwners enforces deny_all across the resolved claims: every one must have +// a granted consent for this consumer, or the whole response is denied. +// +// Checks run concurrently up to maxConcurrentConsentChecks and short-circuit on +// the first problem — the remaining calls are cancelled, since nothing they +// could return would change the answer. The results are then reduced by +// reduceOwnerResults, which ranks them so the verdict does not depend on which +// goroutine happened to finish first. +func checkOwners(ctx context.Context, cfg *Config, key string, reqCtx *RequestContext, client *consent.Client, claims []ownerClaim, consumerSD string) responseOutcome { + results := make([]ownerCheckResult, len(claims)) + checksCtx, cancelChecks := context.WithCancel(ctx) + defer cancelChecks() + + slots := make(chan struct{}, maxConcurrentConsentChecks) + var wg sync.WaitGroup + + for i, claim := range claims { + wg.Add(1) + go func(i int, claim ownerClaim) { + defer wg.Done() + select { + case slots <- struct{}{}: + defer func() { <-slots }() + case <-checksCtx.Done(): + return } - } + + req := consent.ConsentRequest{ + Subject: claim.owner, + Resource: reqCtx.Path, + Method: reqCtx.Method, + DataResource: claim.dataResource, + Consumer: consumerSD, + Purpose: claim.purpose, + } + results[i].attempted = true + results[i].request = req + ownerResource := resourceOrPath(claim.dataResource, reqCtx.Path) + + started := time.Now() + resp, err := client.CheckConsent(checksCtx, req) + metrics.RecordDependencyCall(metrics.DependencyConsentManager, outcomeOf(err), time.Since(started)) + switch { + case err != nil: + results[i].err = err + results[i].problem = true + case resp.Decision != consent.DecisionAllow: + results[i].record = checkedOwner{ + subject: claim.owner, resource: ownerResource, decision: decisionDeny, reason: resp.Reason, + } + results[i].outcome = responseOutcome{ + decision: decisionDeny, + reason: resp.Reason, + requestID: key, + subject: claim.owner, + resource: ownerResource, + method: reqCtx.Method, + } + results[i].problem = true + default: + results[i].record = checkedOwner{ + subject: claim.owner, resource: ownerResource, decision: decisionAllow, reason: resp.Reason, + } + } + if results[i].problem { + // Nothing the other owners could say would change a deny_all + // verdict, so stop paying for their calls. + cancelChecks() + } + }(i, claim) } - return responseOutcome{decision: decisionAllow, requestID: key, resource: reqCtx.Path, method: reqCtx.Method} + wg.Wait() + + return reduceOwnerResults(cfg, key, reqCtx, ctx.Err(), results) } -// checkConsent runs a single consent check and maps it to an outcome (legacy -// JWT-subject mode). -func checkConsent(cfg *Config, key string, client *consent.Client, req consent.ConsentRequest) responseOutcome { - resp, err := client.CheckConsent(context.Background(), req) - if err != nil { - return failOutcome(cfg, "consent check error: "+err.Error(), key, &req) +// ownerCheckResult is one owner's consent check within a response. +type ownerCheckResult struct { + // outcome is the deny to enforce, set only when the owner denied. + outcome responseOutcome + // err is the dependency failure, set only when the check could not complete. + err error + // request is the check that was made, for the audit record on an error. + request consent.ConsentRequest + // record is this owner's audit entry (allow or deny). + record checkedOwner + // problem is true for a deny or an error, i.e. anything but a plain allow. + problem bool + // attempted is false for an owner whose check never started because the + // phase was already cancelled. + attempted bool +} + +// reduceOwnerResults collapses the per-owner results into the response outcome, +// enforcing deny_all. +// +// Results are ranked by DECISIVENESS first and index second. A deny is a +// definite answer; an error is the absence of one, and only the absence is +// subject to the operator's fail policy. Reducing by index alone ranked the two +// purely by position, so an error at a lower index could mask a deny at a higher +// one — and under `fail_open: true` that released data an owner had explicitly +// refused, with the audit record showing the contradiction (a per-owner deny +// alongside an enforced allow). Scanning for a deny across all results first +// removes the ordering dependency; within each pass the lowest index still wins, +// so the verdict stays deterministic rather than depending on which goroutine +// finished first. +// +// phaseErr is the response phase's own context error, which distinguishes a call +// cancelled because a sibling already denied (not a failure in itself) from one +// cancelled because the whole phase ran out of budget (which is). +func reduceOwnerResults(cfg *Config, key string, reqCtx *RequestContext, phaseErr error, results []ownerCheckResult) responseOutcome { + // Every owner that was actually consulted is recorded, whatever the verdict, + // so the audit log names them all rather than only the first refusal. + checked := make([]checkedOwner, 0, len(results)) + for _, result := range results { + if result.attempted && result.record.subject != "" { + checked = append(checked, result.record) + } } - decision := decisionDeny - if resp.Decision == consent.DecisionAllow { - decision = decisionAllow + + for _, result := range results { + if result.attempted && result.problem && result.err == nil { + outcome := result.outcome + outcome.checked = checked + return outcome + } } - return responseOutcome{ - decision: decision, - reason: resp.Reason, - requestID: key, - subject: req.Subject, - resource: req.Resource, - method: req.Method, + + for _, result := range results { + if !result.attempted || result.err == nil { + continue + } + // A call cancelled because a *different* owner already denied is not + // itself a failure; that deny was returned by the pass above. + if errors.Is(result.err, context.Canceled) && phaseErr == nil { + continue + } + logging.ErrorfEvery("consent-check", "ResponseFilter: consent check error for request %s: %s", key, logging.Sanitize(result.err.Error())) + req := result.request + outcome := failOutcome(cfg, failModeForError(result.err), "consent check error: "+result.err.Error(), key, &req) + outcome.checked = checked + return outcome } + return responseOutcome{decision: decisionAllow, requestID: key, resource: reqCtx.Path, method: reqCtx.Method, checked: checked} } // clientConfigFromCfg builds the consent-manager client config from the plugin config. @@ -348,8 +605,10 @@ func clientConfigFromCfg(cfg *Config) consent.ClientConfig { ParticipantToken: cfg.ParticipantToken, TokenServiceURL: cfg.TokenServiceURL, TokenAudience: cfg.TokenAudience, - TokenTTL: time.Duration(cfg.ParticipantTokenTTL) * time.Second, - TimeoutMs: cfg.ConsentAPITimeout, + // Safe to convert: Validate bounds ParticipantTokenTTL well below the + // point where the multiplication overflows a time.Duration. + TokenTTL: time.Duration(cfg.ParticipantTokenTTL) * time.Second, + TimeoutMs: cfg.ConsentAPITimeout, } } @@ -361,15 +620,42 @@ func resourceOrPath(dataResource, path string) string { return path } -// failOutcome builds the outcome for an unresolved consent check, applying the -// fail policy (allow when fail-open, otherwise deny). req may be nil when no +// failMode classifies why a consent decision could not be reached, because not +// every unresolved situation deserves the same policy. +type failMode int + +const ( + // failByPolicy is an availability failure of a dependency — the resolver or + // the consent-manager is down, slow, or erroring. Whether that releases the + // data is the operator's call, so cfg.FailOpen decides. + failByPolicy failMode = iota + + // failAlwaysClosed is a situation in which the plugin is structurally unable + // to gate: it cannot correlate the two phases, it never captured the request, + // it has no credentials at all, or the resolver says consent is required but + // names no owner. None of these are outages to ride out — fail_open must not + // turn a misconfiguration or a lost request into a silent bypass, so these + // always deny. + failAlwaysClosed +) + +// String names the fail mode for metrics and logs. +func (m failMode) String() string { + if m == failAlwaysClosed { + return "always_closed" + } + return "by_policy" +} + +// failOutcome builds the outcome for an unresolved consent check. mode decides +// whether the operator's fail policy applies at all. req may be nil when no // request context was captured. -func failOutcome(cfg *Config, reason, requestID string, req *consent.ConsentRequest) responseOutcome { +func failOutcome(cfg *Config, mode failMode, reason, requestID string, req *consent.ConsentRequest) responseOutcome { decision := decisionDeny - if cfg.IsFailOpen() { + if mode == failByPolicy && cfg.IsFailOpen() { decision = decisionAllow } - o := responseOutcome{decision: decision, reason: reason, requestID: requestID} + o := responseOutcome{decision: decision, reason: reason, requestID: requestID, failMode: mode.String()} if req != nil { o.subject = req.Subject o.resource = req.Resource @@ -378,56 +664,113 @@ func failOutcome(cfg *Config, reason, requestID string, req *consent.ConsentRequ return o } +// failModeForError maps a dependency error to its fail mode. A missing +// credential or a consumer that is not in the participant registry is a +// permanent misconfiguration: retrying will not fix it, so failing it open would +// not ride out an outage, it would grant that consumer standing access. Anything +// else is treated as an outage the operator's policy governs. +func failModeForError(err error) failMode { + if errors.Is(err, consent.ErrNoCredentials) || errors.Is(err, consent.ErrParticipantNotRegistered) { + return failAlwaysClosed + } + return failByPolicy +} + // recordAudit emits the decision to the audit sink when auditing is enabled. // The emit is asynchronous and best-effort, so it never affects the decision. +// +// One record is emitted per data owner whose consent was consulted, so the log +// can answer whose consent was checked and what each said — not merely whether +// the response was released. When no owner was reached (a failure before or +// during resolution) the outcome itself is recorded instead, so the request +// still appears in the record. func recordAudit(cfg *Config, outcome responseOutcome) { if !cfg.AuditEnabled { return } - audit.Get(audit.Config{ + emitter := audit.Get(audit.Config{ Endpoint: cfg.AuditOTLPEndpoint, ServiceName: cfg.AuditServiceName, Timeout: time.Duration(cfg.ConsentAPITimeout) * time.Millisecond, - }).Emit(audit.Event{ - Time: time.Now(), - RequestID: outcome.requestID, - Subject: outcome.subject, - Resource: outcome.resource, - Method: outcome.method, - Decision: outcome.decision, - Reason: outcome.reason, + Headers: cfg.AuditOTLPHeaders, }) + now := time.Now() + + if len(outcome.checked) == 0 { + emitter.Emit(audit.Event{ + Time: now, + RequestID: outcome.requestID, + Subject: outcome.subject, + Resource: outcome.resource, + Method: outcome.method, + Decision: outcome.decision, + Reason: outcome.reason, + }) + return + } + for _, checked := range outcome.checked { + emitter.Emit(audit.Event{ + Time: now, + RequestID: outcome.requestID, + Subject: checked.subject, + Resource: checked.resource, + Method: outcome.method, + Decision: checked.decision, + Reason: checked.reason, + }) + } } -// buildConsentRequest creates a ConsentRequest from the stored request context. -// The subject (used to look up consent) is taken from the JWT "sub" claim. -func buildConsentRequest(reqCtx *RequestContext) consent.ConsentRequest { - consentReq := consent.ConsentRequest{ - Resource: reqCtx.Path, - Method: reqCtx.Method, - Claims: reqCtx.JWTClaims, - } +// deniedResponseHeaderPrefixes are the only upstream response headers allowed to +// survive a denial. CORS headers describe the exchange rather than the resource, +// and dropping them would show a browser client a CORS error instead of the 403 +// it was actually given. +var deniedResponseHeaderPrefixes = []string{"Access-Control-"} - // Extract subject from JWT claims if available. - if reqCtx.JWTClaims != nil { - if sub, ok := reqCtx.JWTClaims[jwtSubjectClaim]; ok { - if subStr, ok := sub.(string); ok { - consentReq.Subject = subStr +// denyResponse replaces the upstream response with the configured denial. +// +// Every other upstream header is removed first. A denied caller must not learn +// anything about the data they were refused, and the upstream's headers say +// plenty: Set-Cookie, ETag and Last-Modified (the entity exists, and this is its +// version), Link (there are more pages), and application counters such as +// X-Total-Count or NGSILD-Results-Count (how many records matched) — a side +// channel straight around the gate. Content-Encoding and the upstream's +// Content-Length are also actively wrong once the body is replaced, so +// Content-Length is set to the deny body's own size. +func denyResponse(w pkgHTTP.Response, cfg *Config) { + body := []byte(cfg.DenyResponseBody) + + header := w.Header() + if view := header.View(); view != nil { + // Collect first: the names are read from the same map Del mutates. + names := make([]string, 0, len(view)) + for name := range view { + names = append(names, name) + } + for _, name := range names { + if !survivesDenial(name) { + header.Del(name) } } } + header.Set("Content-Type", cfg.DenyResponseContentType) + header.Set("Content-Length", strconv.Itoa(len(body))) - return consentReq + w.WriteHeader(cfg.DenyStatusCode) + if _, err := w.Write(body); err != nil { + logging.Errorf("ResponseFilter: failed to write the deny body for request %d: %s", w.ID(), logging.Sanitize(err.Error())) + } } -// denyResponse writes a denial response to the client using the configured -// status code, body, and content type. -func denyResponse(w pkgHTTP.Response, cfg *Config) { - w.Header().Set("Content-Type", cfg.DenyResponseContentType) - w.WriteHeader(cfg.DenyStatusCode) - if _, err := w.Write([]byte(cfg.DenyResponseBody)); err != nil { - log.Printf("[consent-filter] ResponseFilter: failed to write deny body for request %d: %v", w.ID(), err) +// survivesDenial reports whether an upstream response header may be kept on a +// denial. +func survivesDenial(name string) bool { + for _, prefix := range deniedResponseHeaderPrefixes { + if strings.HasPrefix(http.CanonicalHeaderKey(name), prefix) { + return true + } } + return false } // claimKeysToDecode returns the claim keys the request phase must decode: the @@ -442,7 +785,7 @@ func claimKeysToDecode(cfg *Config) []string { if cfg.ConsumerClaim == "" { return keys } - root := strings.SplitN(cfg.ConsumerClaim, claimPathSeparator, 2)[0] + root := claimPathRoot(cfg.ConsumerClaim) for _, k := range keys { if k == root { return keys @@ -451,30 +794,129 @@ func claimKeysToDecode(cfg *Config) []string { return append(keys, root) } -// claimPathSeparator separates the segments of a dotted claim path. -const claimPathSeparator = "." +// Claim-path syntax. A path is dot-separated segments, each optionally followed +// by bracketed array indices, e.g. "verifiableCredential[0].issuer". +const ( + // claimPathSeparator separates the segments of a dotted claim path. + claimPathSeparator = "." + + // claimIndexOpen and claimIndexClose bracket an explicit array index. + claimIndexOpen = "[" + claimIndexClose = "]" + + // firstElementIndex is the element used when a segment resolves to an array + // and the path names no index. + firstElementIndex = 0 +) + +// errClaimPathUnset signals that no consumer claim path is configured, as +// distinct from a configured path that did not resolve. Both deny, but only one +// is a configuration mistake worth reporting as such. +var errClaimPathUnset = errors.New("consumer_claim is not configured") // consumerFromClaims reads the consuming participant from a dotted claim path -// (e.g. "verifiableCredential.issuer"). It returns "" when the path is unset or -// does not resolve to a string - the resolver then reports that it cannot -// identify the contract, and the fail policy applies. -func consumerFromClaims(claims map[string]interface{}, path string) string { - if len(claims) == 0 || path == "" { - return "" +// (e.g. "verifiableCredential.issuer"). +// +// A Verifiable Presentation commonly carries "verifiableCredential" as a JSON +// ARRAY, so a walk that only ever descends into objects fails on an ordinary +// token — silently, returning "" with no indication of which segment gave up. +// Two forms of array traversal are therefore supported: an explicit index +// ("verifiableCredential[0].issuer"), and an implicit first element when a bare +// segment lands on an array. +// +// The error names the segment that failed, so a mistyped path is diagnosable +// rather than appearing as a consumer that simply is not there. +func consumerFromClaims(claims map[string]interface{}, path string) (string, error) { + if path == "" { + return "", errClaimPathUnset } + if len(claims) == 0 { + return "", errors.New("no claims decoded from the token") + } + var current interface{} = claims for _, segment := range strings.Split(path, claimPathSeparator) { - node, ok := current.(map[string]interface{}) - if !ok { - return "" + name, indices, err := parseClaimSegment(segment) + if err != nil { + return "", err + } + if name != "" { + node, ok := descendIntoObject(current) + if !ok { + return "", fmt.Errorf("claim path %q: %q is not an object", path, segment) + } + current, ok = node[name] + if !ok { + return "", fmt.Errorf("claim path %q: no claim %q", path, name) + } } - current, ok = node[segment] - if !ok { - return "" + for _, index := range indices { + array, ok := current.([]interface{}) + if !ok { + return "", fmt.Errorf("claim path %q: %q is not an array", path, name) + } + if index >= len(array) { + return "", fmt.Errorf("claim path %q: index %d is out of range (%d element(s))", path, index, len(array)) + } + current = array[index] } } - if s, ok := current.(string); ok { - return s + + if value, ok := current.(string); ok && value != "" { + return value, nil } - return "" + return "", fmt.Errorf("claim path %q did not resolve to a non-empty string", path) +} + +// descendIntoObject returns node as an object, stepping into the first element +// of an array first. A Verifiable Presentation's "verifiableCredential" is +// routinely an array of one, and requiring an explicit "[0]" for that common +// shape would make the default path wrong for most real tokens. +func descendIntoObject(node interface{}) (map[string]interface{}, bool) { + if array, ok := node.([]interface{}); ok { + if len(array) == 0 { + return nil, false + } + node = array[firstElementIndex] + } + object, ok := node.(map[string]interface{}) + return object, ok +} + +// parseClaimSegment splits one path segment into its claim name and any explicit +// array indices, e.g. "verifiableCredential[0]" -> ("verifiableCredential", [0]). +func parseClaimSegment(segment string) (name string, indices []int, err error) { + name, rest, found := strings.Cut(segment, claimIndexOpen) + if !found { + return segment, nil, nil + } + for rest != "" { + digits, remainder, closed := strings.Cut(rest, claimIndexClose) + if !closed { + return "", nil, fmt.Errorf("claim path segment %q: unterminated %q", segment, claimIndexOpen) + } + index, convErr := strconv.Atoi(digits) + if convErr != nil || index < 0 { + return "", nil, fmt.Errorf("claim path segment %q: %q is not an array index", segment, digits) + } + indices = append(indices, index) + if remainder == "" { + break + } + if !strings.HasPrefix(remainder, claimIndexOpen) { + return "", nil, fmt.Errorf("claim path segment %q: unexpected %q after an index", segment, remainder) + } + rest = strings.TrimPrefix(remainder, claimIndexOpen) + } + return name, indices, nil +} + +// claimPathRoot returns the first claim name in a dotted path, without any array +// index, so the request phase knows which top-level claim to decode. +func claimPathRoot(path string) string { + root := strings.SplitN(path, claimPathSeparator, 2)[0] + if name, _, found := strings.Cut(root, claimIndexOpen); found { + return name + } + return root } diff --git a/internal/plugin/consent_test.go b/internal/plugin/consent_test.go index 6783f2d..3009333 100644 --- a/internal/plugin/consent_test.go +++ b/internal/plugin/consent_test.go @@ -18,12 +18,25 @@ package plugin import ( + "consent-plugin/internal/audit" "consent-plugin/internal/consent" + "consent-plugin/internal/logging" + "consent-plugin/internal/metrics" + "consent-plugin/internal/ownerresolver" + "context" "encoding/json" + "errors" + "fmt" + "io" + "net" "net/http" "net/http/httptest" + "net/url" "strconv" + "strings" + "sync" "testing" + "time" pkgHTTP "github.com/apache/apisix-go-plugin-runner/pkg/http" "github.com/stretchr/testify/assert" @@ -43,7 +56,7 @@ func TestConsentFilter_ParseConf(t *testing.T) { }{ { name: "valid config returns parsed Config", - input: []byte(`{"consent_api_url": "https://consent.example.com"}`), + input: []byte(`{"consent_api_url": "https://consent.example.com", "owner_resolver_url": "https://resolver.example.com/resolve", "participant_token": "t"}`), wantErr: false, }, { @@ -51,6 +64,11 @@ func TestConsentFilter_ParseConf(t *testing.T) { input: []byte(`{}`), wantErr: true, }, + { + name: "missing owner_resolver_url returns error", + input: []byte(`{"consent_api_url": "https://consent.example.com"}`), + wantErr: true, + }, { name: "nil input returns error", input: nil, @@ -82,60 +100,81 @@ func TestPluginName_Constant(t *testing.T) { // --- Mock implementations for testing ResponseFilter --- // mockHeader implements pkgHTTP.Header for testing. +// mockHeader mirrors the runner's header implementation: View() returns the LIVE +// header map (not a copy), so a caller iterating it and deleting through Del +// behaves exactly as it does in production. type mockHeader struct { - headers map[string]string + headers http.Header } func newMockHeader() *mockHeader { - return &mockHeader{headers: make(map[string]string)} + return &mockHeader{headers: make(http.Header)} } -func (h *mockHeader) Set(key, value string) { h.headers[http.CanonicalHeaderKey(key)] = value } -func (h *mockHeader) Del(key string) { delete(h.headers, http.CanonicalHeaderKey(key)) } -func (h *mockHeader) Get(key string) string { return h.headers[http.CanonicalHeaderKey(key)] } -func (h *mockHeader) View() http.Header { - result := make(http.Header) - for k, v := range h.headers { - result[k] = []string{v} - } - return result -} +func (h *mockHeader) Set(key, value string) { h.headers.Set(key, value) } +func (h *mockHeader) Del(key string) { h.headers.Del(key) } +func (h *mockHeader) Get(key string) string { return h.headers.Get(key) } +func (h *mockHeader) View() http.Header { return h.headers } // mockResponse implements pkgHTTP.Response for testing. type mockResponse struct { - id uint32 - statusCode int - header *mockHeader - body []byte - readErr error + id uint32 + statusCode int + header *mockHeader + body []byte + readErr error + // headerReads counts Header() calls. The runner materialises its header map + // on the first one and then reports the response as modified, so an allowed + // response must not touch it. + headerReads int writtenBody []byte writtenStatus int + // suppressContentTypeVar makes Var() report no upstream Content-Type, to + // exercise the header fallback. + suppressContentTypeVar bool + // suppressRequestIDVar makes Var() report no $request_id, as happens when the + // two phases cannot be correlated. + suppressRequestIDVar bool } -func newMockResponse(id uint32, body []byte, contentType string) *mockResponse { +// newMockResponse builds a JSON upstream response carrying body. +func newMockResponse(id uint32, body []byte) *mockResponse { h := newMockHeader() - if contentType != "" { - h.Set("Content-Type", contentType) - } + h.Set("Content-Type", responseContentTypeJSON) return &mockResponse{id: id, header: h, body: body} } -func (r *mockResponse) ID() uint32 { return r.id } -func (r *mockResponse) StatusCode() int { return r.statusCode } -func (r *mockResponse) Header() pkgHTTP.Header { return r.header } +func (r *mockResponse) ID() uint32 { return r.id } +func (r *mockResponse) StatusCode() int { return r.statusCode } +func (r *mockResponse) Header() pkgHTTP.Header { + r.headerReads++ + return r.header +} // Var returns the Nginx request id ($request_id) derived from the mock's id so // correlationKey resolves to the same key the tests store under. func (r *mockResponse) Var(name string) ([]byte, error) { - if name == nginxRequestIDVar { + switch name { + case nginxRequestIDVar: + if r.suppressRequestIDVar { + return nil, nil + } return []byte(testReqKey(r.id)), nil + case nginxUpstreamContentTypeVar: + if r.suppressContentTypeVar { + return nil, nil + } + return []byte(responseContentTypeJSON), nil } return nil, nil } func (r *mockResponse) ReadBody() ([]byte, error) { return r.body, r.readErr } + +// Write appends, as the runner's Response.Write does (it writes into a buffer). +// A mock that replaced the body would hide a double-write regression. func (r *mockResponse) Write(b []byte) (int, error) { - r.writtenBody = b + r.writtenBody = append(r.writtenBody, b...) return len(b), nil } func (r *mockResponse) WriteHeader(statusCode int) { r.writtenStatus = statusCode } @@ -163,29 +202,103 @@ func newConsentManager(t *testing.T, userID string, statuses []string) *httptest _ = json.NewEncoder(w).Encode(map[string]string{"userIdentifier": userID}) }) mux.HandleFunc("/v1/consents/participants/", func(w http.ResponseWriter, r *http.Request) { - consents := make([]map[string]string, 0, len(statuses)) - for _, s := range statuses { - consents = append(consents, map[string]string{"status": s}) - } w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(map[string]interface{}{"consents": consents}) + _ = json.NewEncoder(w).Encode(map[string]interface{}{"consents": consentsGrantedTo(testConsumerSD, statuses)}) + }) + // The participant registry, used to translate the consumer DID from the token + // into the self-description URL a contract names its parties by. + mux.HandleFunc("/v1/participants", participantRegistryHandler) + mux.HandleFunc("/v1/participants/me", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]string{ + "selfDescriptionURL": "http://catalog/participants/provider", + }) }) return httptest.NewServer(mux) } -// newFailingConsentManager returns a consent-manager that answers every call -// with the given status code (used to exercise the fail policy). -func newFailingConsentManager(status int) *httptest.Server { - return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { +// newFailingConsentManager returns a consent-manager whose CONSENT CHECK calls +// answer 500 (used to exercise the fail policy). The participant registry still +// answers, so the failure under test is the check itself and not the preceding +// contract lookup. +func newFailingConsentManager() *httptest.Server { + mux := http.NewServeMux() + mux.HandleFunc("/v1/participants", participantRegistryHandler) + mux.HandleFunc("/", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + }) + return httptest.NewServer(mux) +} + +// participantRegistryHandler serves the consent-manager's participant registry, +// which maps the consumer DID from the token to its self-description URL. +func participantRegistryHandler(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode([]map[string]string{ + {"did": testConsumerDID, "selfDescriptionURL": testConsumerSD}, + }) +} + +// newUncalledConsentManager fails the test if a CONSENT CHECK reaches the +// consent-manager. The participant registry is still served: mapping the +// consumer DID to a self-description is part of the contract lookup that +// precedes the check, and happens even when no check is performed. +func newUncalledConsentManager(t *testing.T) *httptest.Server { + t.Helper() + mux := http.NewServeMux() + mux.HandleFunc("/v1/participants", participantRegistryHandler) + mux.HandleFunc("/", func(_ http.ResponseWriter, r *http.Request) { + t.Errorf("consent-manager must not be called for a consent check (path %s)", r.URL.Path) + }) + return httptest.NewServer(mux) +} + +// --- OwnerResolver mock (the source of data ownership) --- + +// resolverClaim is one (owner [x dataResource]) requirement in a mock /resolve reply. +type resolverClaim struct { + OwnerID string `json:"ownerId"` + DataResource string `json:"dataResource,omitempty"` +} + +// resolverResponse is the mock OwnerResolver's /resolve reply. +type resolverResponse struct { + ConsentRequired bool `json:"consentRequired"` + Claims []resolverClaim `json:"claims"` +} + +// ownedBy builds a resolve reply naming the given data owners (consent required). +func ownedBy(owners ...string) resolverResponse { + claims := make([]resolverClaim, 0, len(owners)) + for _, o := range owners { + claims = append(claims, resolverClaim{OwnerID: o}) + } + return resolverResponse{ConsentRequired: true, Claims: claims} +} + +// newOwnerResolver starts a mock OwnerResolver answering every /resolve with resp. +func newOwnerResolver(t *testing.T, resp resolverResponse) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(resp); err != nil { + t.Errorf("failed to encode resolve response: %v", err) + } + })) +} + +// newFailingOwnerResolver returns a resolver answering every call with status. +func newFailingOwnerResolver(status int) *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(status) })) } -// newUncalledConsentManager fails the test if the consent-manager is contacted. -func newUncalledConsentManager(t *testing.T) *httptest.Server { +// newUncalledOwnerResolver fails the test if the resolver is contacted. +func newUncalledOwnerResolver(t *testing.T) *httptest.Server { t.Helper() - return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - t.Errorf("consent-manager must not be called (path %s)", r.URL.Path) + return httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + t.Errorf("owner resolver must not be called (path %s)", r.URL.Path) })) } @@ -193,12 +306,19 @@ func newUncalledConsentManager(t *testing.T) *httptest.Server { func boolPtr(b bool) *bool { return &b } -// newTestConfig creates a valid plugin Config pointing at the given consent-manager. -func newTestConfig(consentAPIURL string) *Config { +// newTestConfig creates a valid plugin Config pointing at the given +// consent-manager and OwnerResolver. +func newTestConfig(consentAPIURL, resolverURL string) *Config { return &Config{ ConsentAPIURL: consentAPIURL, ConsentAPIPrefix: DefaultConsentAPIPrefix, ConsentAPITimeout: DefaultConsentAPITimeout, + OwnerResolverURL: resolverURL, + OwnerResolverTimeout: DefaultOwnerResolverTimeout, + ResponsePhaseTimeout: DefaultResponsePhaseTimeout, + MaxOwnersPerResponse: DefaultMaxOwnersPerResponse, + MaxResolveBodyBytes: DefaultMaxResolveBodyBytes, + ConsumerClaim: DefaultConsumerClaim, JWTHeaderName: DefaultJWTHeaderName, ConsentKey: "test-consent-key", ParticipantToken: "test-participant-token", @@ -209,16 +329,40 @@ func newTestConfig(consentAPIURL string) *Config { } } -// storeSubject stores a request context carrying the given subject DID. -func storeSubject(id uint32, subject string) { +// storeRequest stores a request context for the given mock id. The consuming +// participant is named in the claims; the data owner comes from the resolver. +func storeRequest(id uint32) { StoreRequestContext(testReqKey(id), &RequestContext{ - Method: "GET", - Path: "/ngsi-ld/v1/entities/urn:ngsi-ld:PersonalProfile:alice", - JWTClaims: map[string]interface{}{"sub": subject}, + Method: "GET", + Path: "/ngsi-ld/v1/entities/urn:ngsi-ld:PersonalProfile:alice", + JWTClaims: map[string]interface{}{ + "verifiableCredential": map[string]interface{}{"issuer": testConsumerDID}, + }, }) } -const testSubjectDID = "did:key:zSubject" +// testOwnerDID is the data owner a mock resolver reports. +const testOwnerDID = "did:key:zOwner" + +// testConsumerDID is the requesting participant named in the token claims. +const testConsumerDID = "did:key:zConsumer" + +// testConsumerSD is the self-description URL the participant registry maps +// testConsumerDID to — the consumer every consent check is scoped to. +const testConsumerSD = "http://catalog/participants/consumer" + +// consentsGrantedTo builds consent records with the given statuses, each granted +// to the named consuming participant. +func consentsGrantedTo(consumer string, statuses []string) []map[string]interface{} { + consents := make([]map[string]interface{}, 0, len(statuses)) + for _, s := range statuses { + consents = append(consents, map[string]interface{}{ + "status": s, + "consumer": map[string]string{"selfDescriptionURL": consumer}, + }) + } + return consents +} // --- ResponseFilter tests (coarse allow/deny gate) --- @@ -227,95 +371,211 @@ func TestConsentFilter_ResponseFilter(t *testing.T) { name string setupContext func(id uint32) consentServer func(t *testing.T) *httptest.Server - configFn func(consentURL string) *Config + resolverServer func(t *testing.T) *httptest.Server + configFn func(cfg *Config) invalidConfig bool wantWrittenBody string wantWrittenStatus int wantNoWrite bool }{ { - name: "granted consent passes the response through", - setupContext: func(id uint32) { storeSubject(id, testSubjectDID) }, - consentServer: func(t *testing.T) *httptest.Server { return newConsentManager(t, "uid-1", []string{"granted"}) }, - wantNoWrite: true, + name: "granted consent for the resolved owner passes the response through", + setupContext: storeRequest, + consentServer: func(t *testing.T) *httptest.Server { return newConsentManager(t, "uid-1", []string{"granted"}) }, + resolverServer: func(t *testing.T) *httptest.Server { return newOwnerResolver(t, ownedBy(testOwnerDID)) }, + wantNoWrite: true, }, { name: "no granted consent denies with the default response", - setupContext: func(id uint32) { storeSubject(id, testSubjectDID) }, + setupContext: storeRequest, consentServer: func(t *testing.T) *httptest.Server { return newConsentManager(t, "uid-1", []string{"revoked"}) }, + resolverServer: func(t *testing.T) *httptest.Server { return newOwnerResolver(t, ownedBy(testOwnerDID)) }, wantWrittenBody: DefaultDenyResponseBody, wantWrittenStatus: DefaultDenyStatusCode, }, { - name: "unknown subject (404 on search) denies", - setupContext: func(id uint32) { storeSubject(id, testSubjectDID) }, + name: "unknown owner (404 on search) denies", + setupContext: storeRequest, consentServer: func(t *testing.T) *httptest.Server { return newConsentManager(t, "", nil) }, + resolverServer: func(t *testing.T) *httptest.Server { return newOwnerResolver(t, ownedBy(testOwnerDID)) }, wantWrittenBody: DefaultDenyResponseBody, wantWrittenStatus: DefaultDenyStatusCode, }, { - name: "deny uses the custom status code and body", - setupContext: func(id uint32) { storeSubject(id, testSubjectDID) }, - consentServer: func(t *testing.T) *httptest.Server { return newConsentManager(t, "uid-1", []string{"revoked"}) }, - configFn: func(consentURL string) *Config { - cfg := newTestConfig(consentURL) + name: "deny uses the custom status code and body", + setupContext: storeRequest, + consentServer: func(t *testing.T) *httptest.Server { return newConsentManager(t, "uid-1", []string{"revoked"}) }, + resolverServer: func(t *testing.T) *httptest.Server { return newOwnerResolver(t, ownedBy(testOwnerDID)) }, + configFn: func(cfg *Config) { cfg.DenyStatusCode = 451 cfg.DenyResponseBody = `{"msg":"legally blocked"}` - return cfg }, wantWrittenBody: `{"msg":"legally blocked"}`, wantWrittenStatus: 451, }, { - name: "empty sub claim denies without contacting the consent-manager", - setupContext: func(id uint32) { storeSubject(id, "") }, + name: "consent not required allows without contacting the consent-manager", + setupContext: storeRequest, + consentServer: newUncalledConsentManager, + resolverServer: func(t *testing.T) *httptest.Server { + return newOwnerResolver(t, resolverResponse{ConsentRequired: false}) + }, + wantNoWrite: true, + }, + { + name: "consent required but no owner resolved denies", + setupContext: storeRequest, + consentServer: newUncalledConsentManager, + resolverServer: func(t *testing.T) *httptest.Server { + return newOwnerResolver(t, resolverResponse{ConsentRequired: true}) + }, + configFn: func(cfg *Config) { cfg.FailOpen = boolPtr(false) }, + wantWrittenBody: DefaultDenyResponseBody, + wantWrittenStatus: DefaultDenyStatusCode, + }, + { + name: "resolved claim without an owner id denies", + setupContext: storeRequest, consentServer: newUncalledConsentManager, - // empty subject => CheckConsent returns deny before any HTTP call + resolverServer: func(t *testing.T) *httptest.Server { + return newOwnerResolver(t, resolverResponse{ConsentRequired: true, Claims: []resolverClaim{{OwnerID: ""}}}) + }, + configFn: func(cfg *Config) { cfg.FailOpen = boolPtr(false) }, + wantWrittenBody: DefaultDenyResponseBody, + wantWrittenStatus: DefaultDenyStatusCode, + }, + { + name: "one denying owner denies the whole response (deny_all)", + setupContext: storeRequest, + consentServer: func(t *testing.T) *httptest.Server { return newConsentManager(t, "uid-1", []string{"revoked"}) }, + resolverServer: func(t *testing.T) *httptest.Server { return newOwnerResolver(t, ownedBy("did:key:zA", "did:key:zB")) }, + wantWrittenBody: DefaultDenyResponseBody, + wantWrittenStatus: DefaultDenyStatusCode, + }, + { + name: "resolver error denies by default (fail-closed)", + setupContext: storeRequest, + consentServer: newUncalledConsentManager, + resolverServer: func(t *testing.T) *httptest.Server { return newFailingOwnerResolver(http.StatusInternalServerError) }, + wantWrittenBody: DefaultDenyResponseBody, + wantWrittenStatus: DefaultDenyStatusCode, + }, + { + name: "resolver error with fail-open explicitly enabled passes through", + setupContext: storeRequest, + consentServer: newUncalledConsentManager, + resolverServer: func(t *testing.T) *httptest.Server { return newFailingOwnerResolver(http.StatusInternalServerError) }, + configFn: func(cfg *Config) { cfg.FailOpen = boolPtr(true) }, + wantNoWrite: true, + }, + { + name: "resolver error with fail-closed denies", + setupContext: storeRequest, + consentServer: newUncalledConsentManager, + resolverServer: func(t *testing.T) *httptest.Server { return newFailingOwnerResolver(http.StatusInternalServerError) }, + configFn: func(cfg *Config) { cfg.FailOpen = boolPtr(false) }, wantWrittenBody: DefaultDenyResponseBody, wantWrittenStatus: DefaultDenyStatusCode, }, { - name: "consent-manager error with fail-open passes through", - setupContext: func(id uint32) { storeSubject(id, testSubjectDID) }, - consentServer: func(t *testing.T) *httptest.Server { return newFailingConsentManager(http.StatusInternalServerError) }, - wantNoWrite: true, + name: "consent-manager error denies by default (fail-closed)", + setupContext: storeRequest, + consentServer: func(t *testing.T) *httptest.Server { return newFailingConsentManager() }, + resolverServer: func(t *testing.T) *httptest.Server { return newOwnerResolver(t, ownedBy(testOwnerDID)) }, + wantWrittenBody: DefaultDenyResponseBody, + wantWrittenStatus: DefaultDenyStatusCode, }, { - name: "consent-manager error with fail-closed denies", - setupContext: func(id uint32) { storeSubject(id, testSubjectDID) }, - consentServer: func(t *testing.T) *httptest.Server { return newFailingConsentManager(http.StatusInternalServerError) }, - configFn: func(consentURL string) *Config { - cfg := newTestConfig(consentURL) - cfg.FailOpen = boolPtr(false) - return cfg + name: "consent-manager error with fail-open explicitly enabled passes through", + setupContext: storeRequest, + consentServer: func(t *testing.T) *httptest.Server { return newFailingConsentManager() }, + resolverServer: func(t *testing.T) *httptest.Server { return newOwnerResolver(t, ownedBy(testOwnerDID)) }, + configFn: func(cfg *Config) { cfg.FailOpen = boolPtr(true) }, + wantNoWrite: true, + }, + { + name: "consent-manager error with fail-closed denies", + setupContext: storeRequest, + consentServer: func(t *testing.T) *httptest.Server { return newFailingConsentManager() }, + resolverServer: func(t *testing.T) *httptest.Server { return newOwnerResolver(t, ownedBy(testOwnerDID)) }, + configFn: func(cfg *Config) { cfg.FailOpen = boolPtr(false) }, + wantWrittenBody: DefaultDenyResponseBody, + wantWrittenStatus: DefaultDenyStatusCode, + }, + { + // Losing the request context is not an outage to ride out: the plugin + // cannot gate at all, so fail_open must not turn it into a bypass. + name: "missing request context denies even with fail-open enabled", + setupContext: nil, + consentServer: newUncalledConsentManager, + resolverServer: newUncalledOwnerResolver, + configFn: func(cfg *Config) { cfg.FailOpen = boolPtr(true) }, + wantWrittenBody: DefaultDenyResponseBody, + wantWrittenStatus: DefaultDenyStatusCode, + }, + { + name: "missing request context with fail-closed denies", + setupContext: nil, + consentServer: newUncalledConsentManager, + resolverServer: newUncalledOwnerResolver, + configFn: func(cfg *Config) { cfg.FailOpen = boolPtr(false) }, + wantWrittenBody: DefaultDenyResponseBody, + wantWrittenStatus: DefaultDenyStatusCode, + }, + { + name: "unresolvable consumer denies without asking the resolver", + setupContext: func(id uint32) { + StoreRequestContext(testReqKey(id), &RequestContext{ + Method: "GET", Path: "/data", + JWTClaims: map[string]interface{}{ + "verifiableCredential": map[string]interface{}{"issuer": "did:key:zNotRegistered"}, + }, + }) }, + consentServer: newUncalledConsentManager, + resolverServer: newUncalledOwnerResolver, + // A consumer that is not in the registry is a permanent condition, so + // fail_open must not grant it standing access. + configFn: func(cfg *Config) { cfg.FailOpen = boolPtr(true) }, wantWrittenBody: DefaultDenyResponseBody, wantWrittenStatus: DefaultDenyStatusCode, }, { - name: "missing request context with fail-open passes through", - setupContext: nil, - consentServer: newUncalledConsentManager, - wantNoWrite: true, + name: "no consumer claim in the token denies without asking the resolver", + setupContext: func(id uint32) { + StoreRequestContext(testReqKey(id), &RequestContext{ + Method: "GET", Path: "/data", + JWTClaims: map[string]interface{}{"sub": "did:key:zCaller"}, + }) + }, + consentServer: newUncalledConsentManager, + resolverServer: newUncalledOwnerResolver, + configFn: func(cfg *Config) { cfg.FailOpen = boolPtr(false) }, + wantWrittenBody: DefaultDenyResponseBody, + wantWrittenStatus: DefaultDenyStatusCode, }, { - name: "missing request context with fail-closed denies", - setupContext: nil, - consentServer: newUncalledConsentManager, - configFn: func(consentURL string) *Config { - cfg := newTestConfig(consentURL) - cfg.FailOpen = boolPtr(false) - return cfg + // A route with no way to authenticate as the participant is a + // misconfiguration; fail_open must not make it a silent full bypass. + name: "missing participant credentials deny even with fail-open enabled", + setupContext: storeRequest, + consentServer: newUncalledConsentManager, + resolverServer: newUncalledOwnerResolver, + configFn: func(cfg *Config) { + cfg.FailOpen = boolPtr(true) + cfg.ParticipantToken = "" + cfg.TokenServiceURL = "" }, wantWrittenBody: DefaultDenyResponseBody, wantWrittenStatus: DefaultDenyStatusCode, }, { - name: "invalid config type passes through", - setupContext: func(id uint32) { storeSubject(id, testSubjectDID) }, - consentServer: newUncalledConsentManager, - invalidConfig: true, - wantNoWrite: true, + name: "invalid config type passes through", + setupContext: storeRequest, + consentServer: newUncalledConsentManager, + resolverServer: newUncalledOwnerResolver, + invalidConfig: true, + wantNoWrite: true, }, } @@ -325,18 +585,21 @@ func TestConsentFilter_ResponseFilter(t *testing.T) { server := tt.consentServer(t) defer server.Close() + resolver := tt.resolverServer(t) + defer resolver.Close() var cfg interface{} - switch { - case tt.invalidConfig: + if tt.invalidConfig { cfg = "not-a-config" - case tt.configFn != nil: - cfg = tt.configFn(server.URL) - default: - cfg = newTestConfig(server.URL) + } else { + c := newTestConfig(server.URL, resolver.URL+"/resolve") + if tt.configFn != nil { + tt.configFn(c) + } + cfg = c } - resp := newMockResponse(1, nil, "") + resp := newMockResponse(1, []byte(`{"id":"urn:ngsi-ld:PersonalProfile:alice"}`)) if tt.setupContext != nil { tt.setupContext(resp.id) } @@ -359,104 +622,989 @@ func TestConsentFilter_ResponseFilter(t *testing.T) { } } +// responseContentTypeJSON is the Content-Type of the simulated upstream responses. +const responseContentTypeJSON = "application/json" + func TestConsentFilter_ResponseFilter_ContextCleanup(t *testing.T) { clearContextStore() server := newConsentManager(t, "uid-1", []string{"granted"}) defer server.Close() + resolver := newOwnerResolver(t, ownedBy(testOwnerDID)) + defer resolver.Close() - cfg := newTestConfig(server.URL) + cfg := newTestConfig(server.URL, resolver.URL+"/resolve") const id = uint32(200) - storeSubject(id, testSubjectDID) + storeRequest(id) - resp := newMockResponse(id, nil, "") + resp := newMockResponse(id, []byte(`{}`)) (&ConsentFilter{}).ResponseFilter(cfg, resp) - _, found := LoadRequestContext(testReqKey(id)) - assert.False(t, found, "request context should be deleted after ResponseFilter") + assert.Equal(t, 0, RequestContextStoreSize(), "request context should be deleted after ResponseFilter") } func TestConsentFilter_ResponseFilter_DenySetsContentType(t *testing.T) { clearContextStore() server := newConsentManager(t, "uid-1", []string{"revoked"}) defer server.Close() + resolver := newOwnerResolver(t, ownedBy(testOwnerDID)) + defer resolver.Close() - cfg := newTestConfig(server.URL) + cfg := newTestConfig(server.URL, resolver.URL+"/resolve") cfg.DenyResponseContentType = "text/plain" const id = uint32(201) - storeSubject(id, testSubjectDID) + storeRequest(id) - resp := newMockResponse(id, nil, "") + resp := newMockResponse(id, []byte(`{}`)) (&ConsentFilter{}).ResponseFilter(cfg, resp) assert.Equal(t, "text/plain", resp.header.Get("Content-Type")) assert.Equal(t, DefaultDenyStatusCode, resp.writtenStatus) } -func TestBuildConsentRequest(t *testing.T) { +// TestResponseFilter_ConsentScopedToConsumer is the regression test for the +// consumer scoping: the owner's consent was granted to a DIFFERENT participant, +// so it is no authority for this consumer to read the data. +func TestResponseFilter_ConsentScopedToConsumer(t *testing.T) { + clearContextStore() + + mux := http.NewServeMux() + mux.HandleFunc("/v1/users/identifier/search", func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]string{"userIdentifier": "uid-owner"}) + }) + mux.HandleFunc("/v1/consents/participants/", func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]interface{}{ + "consents": consentsGrantedTo("http://catalog/participants/someone-else", []string{"granted"}), + }) + }) + mux.HandleFunc("/v1/participants", participantRegistryHandler) + server := httptest.NewServer(mux) + defer server.Close() + + resolver := newOwnerResolver(t, ownedBy(testOwnerDID)) + defer resolver.Close() + + const id = uint32(203) + storeRequest(id) + resp := newMockResponse(id, []byte(`{"id":"urn:ngsi-ld:PersonalProfile:alice"}`)) + (&ConsentFilter{}).ResponseFilter(newTestConfig(server.URL, resolver.URL+"/resolve"), resp) + + assert.Equal(t, DefaultDenyStatusCode, resp.writtenStatus, + "a consent granted to another participant must not authorise this consumer") + assert.Equal(t, DefaultDenyResponseBody, string(resp.writtenBody)) +} + +// TestResponseFilter_OwnerNotRequestor is the regression test for the removed +// legacy mode: the consent that decides access must be the RESOLVED OWNER's, not +// the caller's. The resolver names Bob as the owner while the token's "sub" is +// Alice; the identifier search must ask about Bob. +func TestResponseFilter_OwnerNotRequestor(t *testing.T) { + clearContextStore() + + const owner = "did:key:zBob" + var searchedSubjects []string + + mux := http.NewServeMux() + mux.HandleFunc("/v1/users/identifier/search", func(w http.ResponseWriter, r *http.Request) { + var body map[string]string + _ = json.NewDecoder(r.Body).Decode(&body) + searchedSubjects = append(searchedSubjects, body["email"]) + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]string{"userIdentifier": "uid-bob"}) + }) + mux.HandleFunc("/v1/consents/participants/", func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]interface{}{"consents": consentsGrantedTo(testConsumerSD, []string{"revoked"})}) + }) + mux.HandleFunc("/v1/participants", participantRegistryHandler) + server := httptest.NewServer(mux) + defer server.Close() + + resolver := newOwnerResolver(t, ownedBy(owner)) + defer resolver.Close() + + const id = uint32(202) + // The caller is Alice; the data belongs to Bob. + StoreRequestContext(testReqKey(id), &RequestContext{ + Method: "GET", + Path: "/ngsi-ld/v1/entities/urn:ngsi-ld:PersonalProfile:bob", + JWTClaims: map[string]interface{}{ + "sub": "did:key:zAlice", + "verifiableCredential": map[string]interface{}{"issuer": testConsumerDID}, + }, + }) + + resp := newMockResponse(id, []byte(`{"id":"urn:ngsi-ld:PersonalProfile:bob"}`)) + (&ConsentFilter{}).ResponseFilter(newTestConfig(server.URL, resolver.URL+"/resolve"), resp) + + assert.Equal(t, []string{owner}, searchedSubjects, + "consent must be checked for the resolved data owner, never for the token subject") + assert.Equal(t, DefaultDenyStatusCode, resp.writtenStatus, + "the owner has no granted consent, so the caller's own consent must not unlock the data") +} + +// TestResponseFilter_OwnerCapDenies verifies a response resolving to more data +// owners than the cap is denied outright, rather than answered after an +// unbounded number of consent calls. A collection endpoint returning hundreds of +// entities is otherwise a latency and load amplifier any caller can trigger. +func TestResponseFilter_OwnerCapDenies(t *testing.T) { + clearContextStore() + + owners := make([]string, 0, 5) + for i := 0; i < 5; i++ { + owners = append(owners, fmt.Sprintf("did:key:zOwner%d", i)) + } + + server := newUncalledConsentManager(t) + defer server.Close() + resolver := newOwnerResolver(t, ownedBy(owners...)) + defer resolver.Close() + + cfg := newTestConfig(server.URL, resolver.URL+"/resolve") + cfg.MaxOwnersPerResponse = 3 + // Even with fail_open the cap must deny: it is a deliberate limit, not an outage. + cfg.FailOpen = boolPtr(true) + + const id = uint32(210) + storeRequest(id) + resp := newMockResponse(id, []byte(`{}`)) + (&ConsentFilter{}).ResponseFilter(cfg, resp) + + assert.Equal(t, DefaultDenyStatusCode, resp.writtenStatus, + "more owners than the cap must deny without running the checks") +} + +// TestResponseFilter_ResponsePhaseDeadline verifies the whole response phase is +// bounded: a consent-manager that never answers must not let APISIX hold the +// buffered response for per-call-timeout x owner-count. +func TestResponseFilter_ResponsePhaseDeadline(t *testing.T) { + clearContextStore() + + // The consent-manager never answers, so the phase budget is the only thing + // that can end the wait. + release := make(chan struct{}) + mux := http.NewServeMux() + mux.HandleFunc("/v1/participants", participantRegistryHandler) + mux.HandleFunc("/", func(_ http.ResponseWriter, r *http.Request) { + select { + case <-release: + case <-r.Context().Done(): + } + }) + server := httptest.NewServer(mux) + // Releasing the handlers must happen BEFORE Close, which waits for them. + defer server.Close() + defer close(release) + + resolver := newOwnerResolver(t, ownedBy("did:key:zA", "did:key:zB", "did:key:zC")) + defer resolver.Close() + + cfg := newTestConfig(server.URL, resolver.URL+"/resolve") + cfg.ResponsePhaseTimeout = 150 + cfg.ConsentAPITimeout = 60000 // far beyond the phase budget, so the phase budget must win + + const id = uint32(211) + storeRequest(id) + resp := newMockResponse(id, []byte(`{}`)) + + start := time.Now() + (&ConsentFilter{}).ResponseFilter(cfg, resp) + elapsed := time.Since(start) + + assert.Less(t, elapsed, 5*time.Second, "the response phase must be bounded by response_phase_timeout") + assert.Equal(t, DefaultDenyStatusCode, resp.writtenStatus, "a timed-out phase must fail closed by default") +} + +// TestDistinctClaims verifies the resolver's claims collapse to the distinct +// (owner, dataResource) pairs that actually need checking, in resolver order. +func TestDistinctClaims(t *testing.T) { tests := []struct { name string - reqCtx *RequestContext - wantReq consent.ConsentRequest + claims []ownerresolver.Claim + want []ownerClaim + wantErr bool }{ { - name: "builds request with subject from sub claim", - reqCtx: &RequestContext{ - Method: "GET", - Path: "/api/users/1", - JWTClaims: map[string]interface{}{"sub": "did:key:z42", "scope": "read"}, - }, - wantReq: consent.ConsentRequest{ - Subject: "did:key:z42", - Resource: "/api/users/1", - Method: "GET", - Claims: map[string]interface{}{"sub": "did:key:z42", "scope": "read"}, - }, + name: "duplicates collapse", + claims: []ownerresolver.Claim{{OwnerID: "a"}, {OwnerID: "a"}, {OwnerID: "b"}}, + want: []ownerClaim{{owner: "a"}, {owner: "b"}}, }, { - name: "builds request without JWT claims", - reqCtx: &RequestContext{Method: "POST", Path: "/api/data"}, - wantReq: consent.ConsentRequest{ - Resource: "/api/data", - Method: "POST", - }, + name: "same owner with different resources stays distinct", + claims: []ownerresolver.Claim{{OwnerID: "a", DataResource: "r1"}, {OwnerID: "a", DataResource: "r2"}}, + want: []ownerClaim{{owner: "a", dataResource: "r1"}, {owner: "a", dataResource: "r2"}}, }, { - name: "non-string sub claim is ignored", - reqCtx: &RequestContext{ - Method: "GET", - Path: "/api/test", - JWTClaims: map[string]interface{}{"sub": float64(123)}, - }, - wantReq: consent.ConsentRequest{ - Resource: "/api/test", - Method: "GET", - Claims: map[string]interface{}{"sub": float64(123)}, + name: "the purpose is carried through", + claims: []ownerresolver.Claim{{OwnerID: "a", Purpose: "p1"}}, + want: []ownerClaim{{owner: "a", purpose: "p1"}}, + }, + { + name: "a claim without an owner is an error", + claims: []ownerresolver.Claim{{OwnerID: "a"}, {OwnerID: ""}}, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := distinctClaims(tt.claims) + if tt.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} + +// TestDenyResponse_StripsUpstreamHeaders verifies the denial does not inherit the +// upstream's headers. A denied caller must not be told that the entity exists +// (ETag/Last-Modified), how many records matched (X-Total-Count), that more +// pages follow (Link), or be handed a session cookie — that is a side channel +// straight around the gate. Content-Length must also describe the deny body, not +// the upstream's. +func TestDenyResponse_StripsUpstreamHeaders(t *testing.T) { + clearContextStore() + server := newConsentManager(t, "uid-1", []string{"revoked"}) + defer server.Close() + resolver := newOwnerResolver(t, ownedBy(testOwnerDID)) + defer resolver.Close() + + const id = uint32(220) + storeRequest(id) + + resp := newMockResponse(id, []byte(`{"records":[1,2,3]}`)) + resp.header.Set("ETag", `"v7"`) + resp.header.Set("Last-Modified", "Wed, 27 Aug 2026 10:00:00 GMT") + resp.header.Set("Set-Cookie", "session=abc123") + resp.header.Set("Link", `; rel="next"`) + resp.header.Set("X-Total-Count", "4210") + resp.header.Set("NGSILD-Results-Count", "4210") + resp.header.Set("Content-Encoding", "gzip") + resp.header.Set("Content-Length", "19") + resp.header.Set("Access-Control-Allow-Origin", "https://app.example.org") + + (&ConsentFilter{}).ResponseFilter(newTestConfig(server.URL, resolver.URL+"/resolve"), resp) + + require.Equal(t, DefaultDenyStatusCode, resp.writtenStatus) + for _, leaked := range []string{"ETag", "Last-Modified", "Set-Cookie", "Link", "X-Total-Count", "NGSILD-Results-Count", "Content-Encoding"} { + assert.Empty(t, resp.header.Get(leaked), "%s must not survive a denial", leaked) + } + assert.Equal(t, "https://app.example.org", resp.header.Get("Access-Control-Allow-Origin"), + "CORS headers describe the exchange, not the data, and must survive so the client sees the 403") + assert.Equal(t, DefaultDenyResponseContentType, resp.header.Get("Content-Type")) + assert.Equal(t, strconv.Itoa(len(DefaultDenyResponseBody)), resp.header.Get("Content-Length"), + "Content-Length must describe the deny body, not the upstream's") + assert.Equal(t, DefaultDenyResponseBody, string(resp.writtenBody)) +} + +// TestRecordAudit_RecordsEveryCheckedOwner verifies the audit log answers whose +// consent was checked and what each said. Recording only the response outcome +// named no owner at all on an allow, and only the first refusal on a deny — +// which is not what an access-decision log is for. +func TestRecordAudit_RecordsEveryCheckedOwner(t *testing.T) { + type record struct { + subject, decision string + } + var mu sync.Mutex + var got []record + + collector := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var payload struct { + ResourceLogs []struct { + ScopeLogs []struct { + LogRecords []struct { + Attributes []struct { + Key string `json:"key"` + Value struct { + StringValue string `json:"stringValue"` + } `json:"value"` + } `json:"attributes"` + } `json:"logRecords"` + } `json:"scopeLogs"` + } `json:"resourceLogs"` + } + if err := json.NewDecoder(r.Body).Decode(&payload); err != nil { + t.Errorf("failed to decode OTLP payload: %v", err) + } + mu.Lock() + for _, rl := range payload.ResourceLogs { + for _, sl := range rl.ScopeLogs { + for _, lr := range sl.LogRecords { + var rec record + for _, attr := range lr.Attributes { + switch attr.Key { + case "enduser.id": + rec.subject = attr.Value.StringValue + case "consent.decision": + rec.decision = attr.Value.StringValue + } + } + got = append(got, rec) + } + } + } + mu.Unlock() + w.WriteHeader(http.StatusOK) + })) + defer collector.Close() + + cfg := &Config{ + AuditEnabled: true, + AuditOTLPEndpoint: collector.URL, + AuditServiceName: "consent-access-audit-test", + ConsentAPITimeout: DefaultConsentAPITimeout, + } + recordAudit(cfg, responseOutcome{ + decision: decisionDeny, + requestID: "req-1", + method: "GET", + checked: []checkedOwner{ + {subject: "did:key:zA", resource: "/r", decision: decisionAllow}, + {subject: "did:key:zB", resource: "/r", decision: decisionDeny, reason: "no granted consent"}, + }, + }) + audit.ShutdownAll() + + mu.Lock() + defer mu.Unlock() + assert.ElementsMatch(t, []record{ + {subject: "did:key:zA", decision: decisionAllow}, + {subject: "did:key:zB", decision: decisionDeny}, + }, got, "every consulted owner must appear in the audit log, not only the one that denied") +} + +// TestRecordAudit_RecordsOutcomeWhenNoOwnerReached verifies a request that +// failed before any owner was consulted still appears in the record. +func TestRecordAudit_RecordsOutcomeWhenNoOwnerReached(t *testing.T) { + var mu sync.Mutex + records := 0 + collector := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var payload struct { + ResourceLogs []struct { + ScopeLogs []struct { + LogRecords []json.RawMessage `json:"logRecords"` + } `json:"scopeLogs"` + } `json:"resourceLogs"` + } + _ = json.NewDecoder(r.Body).Decode(&payload) + mu.Lock() + for _, rl := range payload.ResourceLogs { + for _, sl := range rl.ScopeLogs { + records += len(sl.LogRecords) + } + } + mu.Unlock() + w.WriteHeader(http.StatusOK) + })) + defer collector.Close() + + cfg := &Config{ + AuditEnabled: true, + AuditOTLPEndpoint: collector.URL, + AuditServiceName: "consent-access-audit-no-owner", + ConsentAPITimeout: DefaultConsentAPITimeout, + } + recordAudit(cfg, responseOutcome{decision: decisionDeny, requestID: "req-2", reason: "owner resolver error"}) + audit.ShutdownAll() + + mu.Lock() + defer mu.Unlock() + assert.Equal(t, 1, records, "a failure before any owner was reached must still be recorded") +} + +// TestConsumerFromClaims covers the claim-path walk, including the array shapes +// a real Verifiable Presentation uses. An object-only walk returned "" on an +// ordinary VP token, which fed straight into the fail-closed party seam and made +// every such request deny for no visible reason. +func TestConsumerFromClaims(t *testing.T) { + const issuer = "did:key:zIssuer" + + objectClaims := map[string]interface{}{ + "verifiableCredential": map[string]interface{}{"issuer": issuer}, + } + arrayClaims := map[string]interface{}{ + "verifiableCredential": []interface{}{ + map[string]interface{}{"issuer": issuer}, + map[string]interface{}{"issuer": "did:key:zOther"}, + }, + } + + tests := []struct { + name string + claims map[string]interface{} + path string + want string + wantErr bool + errSubstr string + }{ + {name: "object shape", claims: objectClaims, path: "verifiableCredential.issuer", want: issuer}, + {name: "array shape traverses the first element", claims: arrayClaims, path: "verifiableCredential.issuer", want: issuer}, + {name: "explicit index", claims: arrayClaims, path: "verifiableCredential[0].issuer", want: issuer}, + {name: "explicit non-zero index", claims: arrayClaims, path: "verifiableCredential[1].issuer", want: "did:key:zOther"}, + {name: "single segment", claims: map[string]interface{}{"iss": issuer}, path: "iss", want: issuer}, + { + name: "nested arrays", + claims: map[string]interface{}{"a": []interface{}{[]interface{}{map[string]interface{}{"b": issuer}}}}, + path: "a[0][0].b", + want: issuer, + }, + { + name: "unset path is reported as unconfigured", + claims: objectClaims, path: "", + wantErr: true, errSubstr: "not configured", + }, + { + name: "no claims at all", + claims: nil, path: "verifiableCredential.issuer", + wantErr: true, errSubstr: "no claims decoded", + }, + { + name: "missing claim names the segment", + claims: objectClaims, path: "verifiableCredential.subject", + wantErr: true, errSubstr: `no claim "subject"`, + }, + { + name: "index out of range", + claims: arrayClaims, path: "verifiableCredential[9].issuer", + wantErr: true, errSubstr: "out of range", + }, + { + name: "non-string leaf", + claims: map[string]interface{}{"iss": float64(42)}, path: "iss", + wantErr: true, errSubstr: "non-empty string", + }, + { + name: "empty-string leaf", + claims: map[string]interface{}{"iss": ""}, path: "iss", + wantErr: true, errSubstr: "non-empty string", + }, + { + name: "descending into a scalar", + claims: map[string]interface{}{"iss": issuer}, path: "iss.nested", + wantErr: true, errSubstr: "is not an object", + }, + { + name: "empty array", + claims: map[string]interface{}{"vc": []interface{}{}}, path: "vc.issuer", + wantErr: true, errSubstr: "is not an object", + }, + { + name: "unterminated index", + claims: arrayClaims, path: "verifiableCredential[0.issuer", + wantErr: true, errSubstr: "unterminated", + }, + { + name: "non-numeric index", + claims: arrayClaims, path: "verifiableCredential[first].issuer", + wantErr: true, errSubstr: "not an array index", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := consumerFromClaims(tt.claims, tt.path) + if tt.wantErr { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.errSubstr) + assert.Empty(t, got) + return + } + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} + +// TestClaimPathRoot verifies the top-level claim the request phase must decode +// is found even when the path starts with an array index. +func TestClaimPathRoot(t *testing.T) { + tests := []struct{ path, want string }{ + {"verifiableCredential.issuer", "verifiableCredential"}, + {"verifiableCredential[0].issuer", "verifiableCredential"}, + {"iss", "iss"}, + {"", ""}, + } + for _, tt := range tests { + t.Run(tt.path, func(t *testing.T) { + assert.Equal(t, tt.want, claimPathRoot(tt.path)) + }) + } +} + +// TestResponseFilter_AllowDoesNotTouchHeaders verifies an allowed response never +// calls Header(). The runner materialises its header map on the first call and +// then reports HasChange() == true, so merely reading the Content-Type sent +// every gated response back to APISIX down the "this response was modified" +// path with an empty header diff. +func TestResponseFilter_AllowDoesNotTouchHeaders(t *testing.T) { + clearContextStore() + server := newConsentManager(t, "uid-1", []string{"granted"}) + defer server.Close() + resolver := newOwnerResolver(t, ownedBy(testOwnerDID)) + defer resolver.Close() + + const id = uint32(230) + storeRequest(id) + resp := newMockResponse(id, []byte(`{"id":"x"}`)) + + (&ConsentFilter{}).ResponseFilter(newTestConfig(server.URL, resolver.URL+"/resolve"), resp) + + assert.Equal(t, 0, resp.writtenStatus, "the response should have been allowed") + assert.Zero(t, resp.headerReads, "an allowed response must not materialise the header map") +} + +// TestResponseContentType covers both sources: the Nginx variable, and the +// header fallback for a deployment where the variable is unavailable. +func TestResponseContentType(t *testing.T) { + t.Run("prefers the nginx variable", func(t *testing.T) { + resp := newMockResponse(240, nil) + assert.Equal(t, responseContentTypeJSON, responseContentType(resp)) + assert.Zero(t, resp.headerReads, "the variable must be enough") + }) + + t.Run("falls back to the header", func(t *testing.T) { + resp := newMockResponse(241, nil) + resp.suppressContentTypeVar = true + resp.header.Set("Content-Type", "application/ld+json") + assert.Equal(t, "application/ld+json", responseContentType(resp)) + assert.Positive(t, resp.headerReads) + }) + + t.Run("reports nothing when neither source has it", func(t *testing.T) { + resp := newMockResponse(242, nil) + resp.suppressContentTypeVar = true + resp.header.Del("Content-Type") + assert.Empty(t, responseContentType(resp)) + }) +} + +// TestResponseFilter_BodyCapDenies verifies an oversized upstream body is denied +// rather than forwarded. The resolver call holds the body whole, validates it +// and marshals it again, so forwarding a large collection response multiplies +// the runner's memory on top of APISIX's own buffering. +func TestResponseFilter_BodyCapDenies(t *testing.T) { + clearContextStore() + + server := newUncalledConsentManager(t) + defer server.Close() + resolver := newUncalledOwnerResolver(t) + defer resolver.Close() + + cfg := newTestConfig(server.URL, resolver.URL+"/resolve") + cfg.MaxResolveBodyBytes = 32 + // The cap is a deliberate limit, not an outage, so fail_open must not lift it. + cfg.FailOpen = boolPtr(true) + + const id = uint32(250) + storeRequest(id) + resp := newMockResponse(id, []byte(`{"padding":"`+strings.Repeat("x", 64)+`"}`)) + + (&ConsentFilter{}).ResponseFilter(cfg, resp) + + assert.Equal(t, DefaultDenyStatusCode, resp.writtenStatus, + "a body too large to examine must be denied, not forwarded") +} + +// TestResponseFilter_CorrelationIdMissingInOnerPhase verifies the case where the +// two phases cannot be correlated: the request phase stored nothing (or the +// response phase cannot read $request_id), so there is no context to decide on. +// The plugin is structurally unable to gate here, so it must deny even with +// fail_open — this is not an outage to ride out. +func TestResponseFilter_CorrelationIDMissingInOnePhase(t *testing.T) { + server := newUncalledConsentManager(t) + defer server.Close() + resolver := newUncalledOwnerResolver(t) + defer resolver.Close() + + cfg := newTestConfig(server.URL, resolver.URL+"/resolve") + cfg.FailOpen = boolPtr(true) + + t.Run("response phase cannot read the correlation id", func(t *testing.T) { + clearContextStore() + resp := newMockResponse(260, []byte(`{}`)) + resp.suppressRequestIDVar = true + storeRequest(260) + + (&ConsentFilter{}).ResponseFilter(cfg, resp) + + assert.Equal(t, DefaultDenyStatusCode, resp.writtenStatus, + "without a correlation id the phases cannot be matched, so nothing can be verified") + }) + + t.Run("request phase never stored a context", func(t *testing.T) { + clearContextStore() + resp := newMockResponse(261, []byte(`{}`)) + + (&ConsentFilter{}).ResponseFilter(cfg, resp) + + assert.Equal(t, DefaultDenyStatusCode, resp.writtenStatus, + "a response whose request phase left no context must not be released") + }) +} + +// mockRequest implements pkgHTTP.Request. correlationID is what Var() reports +// for $request_id; empty means the variable is unavailable. +type mockRequest struct { + header *mockHeader + correlationID string +} + +func (r *mockRequest) ID() uint32 { return 1 } +func (r *mockRequest) SrcIP() net.IP { return net.ParseIP("127.0.0.1") } +func (r *mockRequest) Method() string { return "GET" } +func (r *mockRequest) Path() []byte { return []byte("/data") } +func (r *mockRequest) SetPath([]byte) {} +func (r *mockRequest) Header() pkgHTTP.Header { return r.header } +func (r *mockRequest) Args() url.Values { return nil } +func (r *mockRequest) Var(name string) ([]byte, error) { + if name == nginxRequestIDVar && r.correlationID != "" { + return []byte(r.correlationID), nil + } + return nil, nil +} +func (r *mockRequest) Body() ([]byte, error) { return nil, nil } +func (r *mockRequest) Context() context.Context { return context.Background() } +func (r *mockRequest) RespHeader() http.Header { return nil } + +// TestRequestFilter_CorrelationID verifies the request phase stores a context +// only when it can be correlated with the response phase. Storing one it could +// never retrieve would leak an entry per request into the context store. +func TestRequestFilter_CorrelationID(t *testing.T) { + cfg := newTestConfig("http://consent.invalid", "http://resolver.invalid/resolve") + + t.Run("no correlation id stores nothing", func(t *testing.T) { + clearContextStore() + req := &mockRequest{header: newMockHeader()} + + (&ConsentFilter{}).RequestFilter(cfg, httptest.NewRecorder(), req) + + assert.Equal(t, 0, RequestContextStoreSize(), + "a context that could never be correlated must not be stored") + }) + + t.Run("a correlation id stores the context", func(t *testing.T) { + clearContextStore() + defer clearContextStore() + req := &mockRequest{header: newMockHeader(), correlationID: "req-abc"} + + (&ConsentFilter{}).RequestFilter(cfg, httptest.NewRecorder(), req) + + stored, found := LoadAndDeleteRequestContext("req-abc") + require.True(t, found) + assert.Equal(t, "/data", stored.Path) + assert.Equal(t, "GET", stored.Method) + }) +} + +// TestReduceOwnerResults is the regression test for the ordering hole the +// concurrency work opened, exercised at the level where the ranking actually +// lives. +// +// Driving it through two concurrent HTTP checks turned out to be inherently +// racy: whichever owner finishes first cancels the other, so a sibling's deny +// can arrive as a cancellation instead — the test passed or failed on timing +// rather than on the property. Reducing scripted results is deterministic and +// pins the rule directly. +func TestReduceOwnerResults(t *testing.T) { + reqCtx := &RequestContext{Method: "GET", Path: "/data"} + + denied := func(owner string) ownerCheckResult { + return ownerCheckResult{ + attempted: true, + problem: true, + outcome: responseOutcome{ + decision: decisionDeny, reason: "no granted consent", requestID: "req-1", + subject: owner, resource: "/data", method: "GET", }, + record: checkedOwner{subject: owner, resource: "/data", decision: decisionDeny}, + } + } + allowed := func(owner string) ownerCheckResult { + return ownerCheckResult{ + attempted: true, + record: checkedOwner{subject: owner, resource: "/data", decision: decisionAllow}, + } + } + errored := func(owner string, err error) ownerCheckResult { + return ownerCheckResult{ + attempted: true, + problem: true, + err: err, + request: consent.ConsentRequest{Subject: owner, Resource: "/data", Method: "GET"}, + } + } + + dependencyErr := errors.New("consent client: consents lookup returned status 500") + + tests := []struct { + name string + failOpen bool + phaseErr error + results []ownerCheckResult + wantDecision string + wantSubject string + wantChecked int + }{ + { + name: "every owner allows", failOpen: false, + results: []ownerCheckResult{allowed("a"), allowed("b")}, + wantDecision: decisionAllow, wantChecked: 2, + }, + { + name: "one deny denies them all", failOpen: false, + results: []ownerCheckResult{allowed("a"), denied("b")}, + wantDecision: decisionDeny, wantSubject: "b", wantChecked: 2, + }, + { + // The finding: with fail_open the error path allows, so ranking the + // error first releases data owner b has explicitly refused. + name: "a deny at a higher index outranks an error at a lower one", failOpen: true, + results: []ownerCheckResult{errored("a", dependencyErr), denied("b")}, + wantDecision: decisionDeny, wantSubject: "b", wantChecked: 1, + }, + { + name: "a deny at a lower index still wins", failOpen: true, + results: []ownerCheckResult{denied("a"), errored("b", dependencyErr)}, + wantDecision: decisionDeny, wantSubject: "a", wantChecked: 1, + }, + { + name: "the lowest-indexed deny is reported", failOpen: false, + results: []ownerCheckResult{denied("a"), denied("b")}, + wantDecision: decisionDeny, wantSubject: "a", wantChecked: 2, + }, + { + name: "with no deny anywhere, fail-open allows on an error", failOpen: true, + results: []ownerCheckResult{allowed("a"), errored("b", dependencyErr)}, + wantDecision: decisionAllow, wantChecked: 1, }, + { + name: "with no deny anywhere, fail-closed denies on an error", failOpen: false, + results: []ownerCheckResult{allowed("a"), errored("b", dependencyErr)}, + wantDecision: decisionDeny, wantChecked: 1, + }, + { + // A sibling deny cancels the rest; those cancellations are not + // failures, and the deny that caused them is what gets reported. + name: "a cancellation caused by a sibling deny is ignored", failOpen: true, + results: []ownerCheckResult{errored("a", context.Canceled), denied("b")}, + wantDecision: decisionDeny, wantSubject: "b", wantChecked: 1, + }, + { + // But a cancellation from the phase deadline IS a failure. + name: "a cancellation from the phase deadline applies the fail policy", failOpen: false, + phaseErr: context.DeadlineExceeded, + results: []ownerCheckResult{errored("a", context.Canceled), allowed("b")}, + wantDecision: decisionDeny, wantChecked: 1, + }, + { + name: "owners whose check never started are ignored", failOpen: false, + results: []ownerCheckResult{allowed("a"), {}}, + wantDecision: decisionAllow, wantChecked: 1, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := &Config{FailOpen: boolPtr(tt.failOpen)} + + got := reduceOwnerResults(cfg, "req-1", reqCtx, tt.phaseErr, tt.results) + + assert.Equal(t, tt.wantDecision, got.decision) + if tt.wantSubject != "" { + assert.Equal(t, tt.wantSubject, got.subject, "the reported owner must be the deciding one") + } + assert.Len(t, got.checked, tt.wantChecked, + "every consulted owner must reach the audit log, and only those") + }) } +} + +// TestCheckOwners_ErrorStillAppliesFailPolicy verifies the reordering did not +// swallow the error path: with no deny anywhere, a dependency error is still +// what decides, and fail_open still governs it. +func TestCheckOwners_ErrorStillAppliesFailPolicy(t *testing.T) { + server := newFailingConsentManager() + defer server.Close() + resolver := newOwnerResolver(t, ownedBy(testOwnerDID)) + defer resolver.Close() + tests := []struct { + name string + failOpen bool + wantDenied bool + }{ + {name: "fail-closed denies", failOpen: false, wantDenied: true}, + {name: "fail-open passes through", failOpen: true, wantDenied: false}, + } + + // Each case needs its own request id so the context-store entries cannot + // collide between subtests. + nextRequestID := uint32(280) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.wantReq, buildConsentRequest(tt.reqCtx)) + clearContextStore() + consent.ResetCaches() + + cfg := newTestConfig(server.URL, resolver.URL+"/resolve") + cfg.FailOpen = boolPtr(tt.failOpen) + + nextRequestID++ + id := nextRequestID + storeRequest(id) + resp := newMockResponse(id, []byte(`{"id":"x"}`)) + + (&ConsentFilter{}).ResponseFilter(cfg, resp) + + if tt.wantDenied { + assert.Equal(t, DefaultDenyStatusCode, resp.writtenStatus) + return + } + assert.Equal(t, 0, resp.writtenStatus, "a dependency error with fail_open must still pass through") }) } } -func TestConfig_IsFailOpen(t *testing.T) { +// TestCheckPurposeScoping covers what happens when the resolver names no +// processing purpose for a claim. +// +// Purpose matching depends on a different service populating an optional field, +// so a resolver whose rules stop emitting it silently disables half of the +// consumer/purpose scoping — a consent granted for one purpose then authorises +// release for any other. Without require_purpose that is counted and logged; +// with it, it denies. +func TestCheckPurposeScoping(t *testing.T) { tests := []struct { - name string - failOpen *bool - want bool + name string + requirePurpose bool + failOpen bool + claims []ownerClaim + wantOK bool + wantUnconstained int }{ - {name: "nil defaults to true (fail-open)", failOpen: nil, want: true}, - {name: "explicitly true is fail-open", failOpen: boolPtr(true), want: true}, - {name: "explicitly false is fail-closed", failOpen: boolPtr(false), want: false}, + { + name: "every claim carries a purpose", + claims: []ownerClaim{{owner: "a", purpose: "insurance-quote"}, {owner: "b", purpose: "insurance-quote"}}, + wantOK: true, + }, + { + name: "a missing purpose is counted when not required", + claims: []ownerClaim{{owner: "a"}, {owner: "b", purpose: "research"}}, + wantOK: true, + wantUnconstained: 1, + }, + { + name: "every unscoped claim is counted", + claims: []ownerClaim{{owner: "a"}, {owner: "b"}}, + wantOK: true, + wantUnconstained: 2, + }, + { + name: "a missing purpose denies when required", + requirePurpose: true, + claims: []ownerClaim{{owner: "a", purpose: "research"}, {owner: "b"}}, + wantOK: false, + }, + { + // require_purpose is a policy the operator asked for, not an outage, + // so fail_open must not lift it. + name: "require_purpose denies even with fail-open", + requirePurpose: true, + failOpen: true, + claims: []ownerClaim{{owner: "a"}}, + wantOK: false, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - cfg := &Config{FailOpen: tt.failOpen} - assert.Equal(t, tt.want, cfg.IsFailOpen()) + metrics.Reset() + t.Cleanup(metrics.Reset) + logging.ResetSuppression() + + cfg := newTestConfig("http://consent.invalid", "http://resolver.invalid/resolve") + cfg.RequirePurpose = tt.requirePurpose + cfg.FailOpen = boolPtr(tt.failOpen) + + outcome, ok := checkPurposeScoping(cfg, "req-1", tt.claims) + + assert.Equal(t, tt.wantOK, ok) + if !tt.wantOK { + assert.Equal(t, decisionDeny, outcome.decision, + "require_purpose is a policy, not an outage — fail_open must not lift it") + return + } + assert.Contains(t, metricsExposition(), fmt.Sprintf("consent_purpose_unconstrained_total %d", tt.wantUnconstained)) }) } } + +// metricsExposition renders the current metrics for assertion. +func metricsExposition() string { + recorder := httptest.NewRecorder() + metrics.Handler().ServeHTTP(recorder, httptest.NewRequestWithContext(context.Background(), http.MethodGet, "/metrics", nil)) + return recorder.Body.String() +} + +// TestAuditReasonCarriesNoDependencyBody verifies a dependency's response body +// cannot reach the audit sink through the decision reason. +// +// Errors from the consent client become the reason, and the reason is exported. +// A consent-manager 500 that echoes the user's identifier in its body would +// therefore land in the audit record and on stdout — and truncating it, as an +// earlier version did, is not redaction: the surviving prefix of a JSON error +// body is usually exactly the part with the identifiers in it. The reason must +// be a stable classification instead. +func TestAuditReasonCarriesNoDependencyBody(t *testing.T) { + clearContextStore() + consent.ResetCaches() + + const leakedIdentifier = "alice@example.org" + + var mu sync.Mutex + var reasons []string + collector := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + mu.Lock() + reasons = append(reasons, string(body)) + mu.Unlock() + w.WriteHeader(http.StatusOK) + })) + defer collector.Close() + + mux := http.NewServeMux() + mux.HandleFunc("/v1/participants", participantRegistryHandler) + mux.HandleFunc("/", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + // A dependency echoing personal data in its error page. + _, _ = w.Write([]byte(`{"error":"lookup failed for ` + leakedIdentifier + `","trace":"..."}`)) + }) + server := httptest.NewServer(mux) + defer server.Close() + + resolver := newOwnerResolver(t, ownedBy(testOwnerDID)) + defer resolver.Close() + + cfg := newTestConfig(server.URL, resolver.URL+"/resolve") + cfg.AuditEnabled = true + cfg.AuditOTLPEndpoint = collector.URL + cfg.AuditServiceName = "consent-access-audit-leak-test" + + const id = uint32(290) + storeRequest(id) + (&ConsentFilter{}).ResponseFilter(cfg, newMockResponse(id, []byte(`{"id":"x"}`))) + audit.ShutdownAll() + + mu.Lock() + defer mu.Unlock() + require.NotEmpty(t, reasons, "the decision should have been audited") + for _, payload := range reasons { + assert.NotContains(t, payload, leakedIdentifier, + "a dependency response body must not reach the audit sink through the reason") + } + assert.Contains(t, strings.Join(reasons, ""), "identifier search returned status 500", + "the reason should classify the failure instead") +} diff --git a/internal/plugin/context.go b/internal/plugin/context.go index 6efc9bb..aeb4198 100644 --- a/internal/plugin/context.go +++ b/internal/plugin/context.go @@ -18,14 +18,22 @@ package plugin import ( + "consent-plugin/internal/logging" + "consent-plugin/internal/metrics" "fmt" - "net/http" + "sort" "sync" + "time" ) -// RequestContext holds the captured request information that is needed -// during response filtering. It is stored during RequestFilter and -// retrieved during ResponseFilter. +// RequestContext holds the captured request information the response phase +// needs. It is stored during RequestFilter and retrieved during ResponseFilter. +// +// It deliberately holds no request headers. The only thing the response phase +// needs from the request is the method, the path and the decoded claims; keeping +// a copy of every header would mean retaining the Authorization bearer token for +// the lifetime of the entry, which is precisely the wrong thing to leak when an +// entry outlives its request. type RequestContext struct { // Method is the HTTP method of the original request (e.g., "GET", "POST"). Method string @@ -33,76 +41,233 @@ type RequestContext struct { // Path is the URI path of the original request. Path string - // Headers contains the HTTP headers from the original request. - Headers http.Header - // JWTClaims holds the decoded JWT claims extracted from the configured header. // The map keys are claim names and values are the claim values. JWTClaims map[string]interface{} } -// requestContextStore is a package-level concurrent-safe store that maps a -// stable per-request key to its captured RequestContext. This bridges the -// RequestFilter and ResponseFilter phases, which APISIX invokes as two -// separate RPC calls (ext-plugin-pre-req and ext-plugin-post-resp). +// Bounds on the request-context store. The store bridges two phases of the same +// HTTP request, so an entry is normally live for the duration of one upstream +// call. Anything still present well after that belongs to a request whose +// response phase will never run. +const ( + // RequestContextTTL is how long an entry may live before the janitor evicts + // it. It must comfortably exceed the upstream response time of a gated route; + // evicting too early only means the response phase finds no context and + // (fail-closed) denies. + RequestContextTTL = 60 * time.Second + + // requestContextSweepInterval is how often the janitor evicts expired entries. + requestContextSweepInterval = 10 * time.Second + + // MaxRequestContexts caps how many entries the store may hold. The cap is the + // backstop against an unauthenticated memory-exhaustion primitive: a client + // that opens requests and aborts before the response leaks one entry each. + MaxRequestContexts = 100_000 + + // contextEvictionBatch is how many entries an overflow evicts at once. + // + // Freeing a single slot per overflow meant a full store paid a whole-map scan + // on EVERY subsequent request, serialised behind the store's mutex — and the + // store only reaches the cap under the leak or abort-flood the cap exists to + // contain. The O(n) path was therefore guaranteed to engage exactly when load + // was already pathological, converting an unbounded memory leak into an + // unbounded latency cliff. Evicting a batch amortises the scan over the whole + // batch, so the cost is paid once per contextEvictionBatch requests instead + // of once per request. + contextEvictionBatch = MaxRequestContexts / 100 +) + +// storedRequestContext is one entry plus the time it was stored, which is what +// makes expiry possible. +type storedRequestContext struct { + ctx *RequestContext + storedAt time.Time +} + +// requestContextStore maps a stable per-request key to its captured +// RequestContext. This bridges the RequestFilter and ResponseFilter phases, +// which APISIX invokes as two separate RPC calls (ext-plugin-pre-req and +// ext-plugin-post-resp). // // The key MUST be stable across those two phases for the same HTTP request. // The runner's per-RPC id (Request.ID()/Response.ID()) is NOT stable between // them, so the Nginx `$request_id` variable is used instead (see // correlationKey in consent.go). -var requestContextStore sync.Map +// +// Entries are normally removed by LoadAndDeleteRequestContext in the response +// phase. That phase does not always run — the client disconnects, the upstream +// times out, an earlier plugin short-circuits the request, ext-plugin-post-resp +// is not attached to the route — and the runner is a long-lived process, so +// without a TTL and a cap the map grows monotonically until the runner is OOM +// killed. Both are enforced here. +var ( + requestContextMu sync.Mutex + requestContextStore = map[string]storedRequestContext{} + // contextsEvicted counts entries removed because they expired or because the + // store was full, i.e. requests whose response phase never ran. A number that + // climbs in production means requests are being lost, or the store is being + // driven deliberately. + contextsEvicted uint64 + janitorOnce sync.Once +) + +func init() { + // Publish the store's size and eviction count so a leak is observable rather + // than only inferable from memory growth. + metrics.RegisterGauge(metrics.ContextStoreSizeGauge, + "Request contexts currently held, i.e. gated requests in flight.", + func() float64 { return float64(RequestContextStoreSize()) }) + metrics.RegisterCounter(metrics.ContextEvictedCounter, + "Request contexts evicted because they expired or the store was full.", + func() float64 { return float64(RequestContextsEvicted()) }) +} + +// startContextJanitor launches the background sweep exactly once. It is started +// lazily from the first Store so that importing the package (as tests and the +// runner registration do) never leaves a goroutine running for nothing. +func startContextJanitor() { + janitorOnce.Do(func() { + go func() { + ticker := time.NewTicker(requestContextSweepInterval) + defer ticker.Stop() + for range ticker.C { + if n := sweepRequestContexts(time.Now()); n > 0 { + logging.Warnf("request-context store: evicted %d expired entr(ies), %d remaining", + n, RequestContextStoreSize()) + } + } + }() + }) +} // StoreRequestContext saves a RequestContext for the given request key. // It overwrites any previously stored context for the same key. +// +// The store is bounded: when it is full, expired entries are swept first and, +// failing that, a batch of the oldest entries is evicted, so a new request is +// never refused service by a leak from an older one. func StoreRequestContext(requestKey string, ctx *RequestContext) { - requestContextStore.Store(requestKey, ctx) + startContextJanitor() + + requestContextMu.Lock() + defer requestContextMu.Unlock() + + now := time.Now() + if len(requestContextStore) >= MaxRequestContexts { + if _, replacing := requestContextStore[requestKey]; !replacing { + evictForSpaceLocked(now) + } + } + requestContextStore[requestKey] = storedRequestContext{ctx: ctx, storedAt: now} } -// LoadRequestContext retrieves the stored RequestContext for the given -// request key. Returns the context and true if found, or nil and false -// if no context exists for that key. -func LoadRequestContext(requestKey string) (*RequestContext, bool) { - val, ok := requestContextStore.Load(requestKey) - if !ok { - return nil, false +// evictForSpaceLocked makes room in a full store: expired entries first and, if +// everything is still live, a batch of the oldest. Callers must hold +// requestContextMu. +// +// Both paths scan the map, which is why they free many slots rather than one: +// the next contextEvictionBatch requests then find room without scanning at all. +func evictForSpaceLocked(now time.Time) { + if n := sweepLocked(now); n > 0 { + logging.WarnfEvery("context-store-full", "request-context store full (%d), evicted %d expired entr(ies)", MaxRequestContexts, n) + return + } + if n := evictOldestLocked(contextEvictionBatch); n > 0 { + logging.ErrorfEvery("context-store-overflow", + "request-context store full (%d) with no expired entries, evicted the %d oldest", MaxRequestContexts, n) } +} - ctx, ok := val.(*RequestContext) - if !ok { - return nil, false +// evictOldestLocked removes up to batch of the oldest entries and returns how +// many it removed. Callers must hold requestContextMu. +func evictOldestLocked(batch int) int { + if batch <= 0 || len(requestContextStore) == 0 { + return 0 + } + type aged struct { + key string + storedAt time.Time + } + entries := make([]aged, 0, len(requestContextStore)) + for key, entry := range requestContextStore { + entries = append(entries, aged{key: key, storedAt: entry.storedAt}) + } + sort.Slice(entries, func(i, j int) bool { return entries[i].storedAt.Before(entries[j].storedAt) }) + + if batch > len(entries) { + batch = len(entries) + } + for _, entry := range entries[:batch] { + delete(requestContextStore, entry.key) } + contextsEvicted += uint64(batch) + return batch +} + +// sweepRequestContexts removes every entry stored more than RequestContextTTL +// before now and returns how many were removed. +func sweepRequestContexts(now time.Time) int { + requestContextMu.Lock() + defer requestContextMu.Unlock() + return sweepLocked(now) +} + +// sweepLocked is sweepRequestContexts for a caller already holding the mutex. +func sweepLocked(now time.Time) int { + evicted := 0 + for key, entry := range requestContextStore { + if now.Sub(entry.storedAt) > RequestContextTTL { + delete(requestContextStore, key) + evicted++ + } + } + contextsEvicted += uint64(evicted) + return evicted +} - return ctx, true +// RequestContextStoreSize reports how many request contexts are currently held. +// It is the gauge that makes a leak observable: in a healthy runner it tracks +// the number of in-flight gated requests and returns to zero when idle. +func RequestContextStoreSize() int { + requestContextMu.Lock() + defer requestContextMu.Unlock() + return len(requestContextStore) } -// DeleteRequestContext removes the stored RequestContext for the given -// request key. This should be called after the context has been consumed -// during ResponseFilter to prevent memory leaks. -func DeleteRequestContext(requestKey string) { - requestContextStore.Delete(requestKey) +// RequestContextsEvicted reports how many contexts have been evicted because +// they expired or the store was full — i.e. how many requests never reached +// their response phase. +func RequestContextsEvicted() uint64 { + requestContextMu.Lock() + defer requestContextMu.Unlock() + return contextsEvicted } // LoadAndDeleteRequestContext atomically loads and removes the stored -// RequestContext for the given request key. This is the preferred method -// for consuming context during ResponseFilter as it combines retrieval -// and cleanup in a single operation. +// RequestContext for the given request key. This is how the response phase +// consumes a context: retrieval and cleanup in a single operation. An entry that +// has outlived RequestContextTTL is reported as absent (and removed), so a +// stale context can never decide a fresh request. func LoadAndDeleteRequestContext(requestKey string) (*RequestContext, bool) { - val, ok := requestContextStore.LoadAndDelete(requestKey) + requestContextMu.Lock() + defer requestContextMu.Unlock() + + entry, ok := requestContextStore[requestKey] if !ok { return nil, false } - - ctx, ok := val.(*RequestContext) - if !ok { + delete(requestContextStore, requestKey) + if time.Since(entry.storedAt) > RequestContextTTL { + contextsEvicted++ return nil, false } - - return ctx, true + return entry.ctx, true } // String returns a human-readable representation of the RequestContext, // useful for logging and debugging. func (rc *RequestContext) String() string { - return fmt.Sprintf("RequestContext{Method: %s, Path: %s, Claims: %d, Headers: %d}", - rc.Method, rc.Path, len(rc.JWTClaims), len(rc.Headers)) + return fmt.Sprintf("RequestContext{Method: %s, Path: %s, Claims: %d}", + rc.Method, rc.Path, len(rc.JWTClaims)) } diff --git a/internal/plugin/context_bench_test.go b/internal/plugin/context_bench_test.go new file mode 100644 index 0000000..02ce52a --- /dev/null +++ b/internal/plugin/context_bench_test.go @@ -0,0 +1,68 @@ +/* + * Copyright 2026 Seamless Middleware Technologies S.L and/or its affiliates + * and other contributors as indicated by the @author tags. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package plugin + +import ( + "fmt" + "testing" + "time" +) + +// fillContextStore populates the store with n live entries of ascending age. +func fillContextStore(n int) { + now := time.Now() + requestContextMu.Lock() + defer requestContextMu.Unlock() + requestContextStore = make(map[string]storedRequestContext, n) + for i := 0; i < n; i++ { + requestContextStore[fmt.Sprintf("live-%d", i)] = storedRequestContext{ + ctx: &RequestContext{Path: "/x"}, + storedAt: now.Add(time.Duration(i) * time.Microsecond), + } + } +} + +// BenchmarkStoreRequestContext measures the ordinary path: a store with room. +func BenchmarkStoreRequestContext(b *testing.B) { + fillContextStore(0) + b.Cleanup(clearContextStore) + + ctx := &RequestContext{Method: "GET", Path: "/data"} + for i := 0; b.Loop(); i++ { + StoreRequestContext(fmt.Sprintf("key-%d", i), ctx) + } +} + +// BenchmarkStoreRequestContextAtCap measures the overflow path, which is the +// one that matters: the store only reaches its cap under the leak or the +// abort-flood the cap exists to contain, so this is the cost the gate pays +// exactly when it is already under pressure. +// +// Evicting one entry per overflow made every request at the cap pay two +// whole-map scans under the store's mutex. Evicting a batch amortises that over +// contextEvictionBatch requests, so the per-request cost here should stay close +// to the uncontended case above rather than scaling with MaxRequestContexts. +func BenchmarkStoreRequestContextAtCap(b *testing.B) { + fillContextStore(MaxRequestContexts) + b.Cleanup(clearContextStore) + + ctx := &RequestContext{Method: "GET", Path: "/data"} + for i := 0; b.Loop(); i++ { + StoreRequestContext(fmt.Sprintf("overflow-%d", i), ctx) + } +} diff --git a/internal/plugin/context_test.go b/internal/plugin/context_test.go index 1eba047..d8ceeb8 100644 --- a/internal/plugin/context_test.go +++ b/internal/plugin/context_test.go @@ -19,9 +19,10 @@ package plugin import ( "fmt" - "net/http" + "reflect" "sync" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -29,154 +30,200 @@ import ( // clearContextStore resets the package-level request context store between tests. func clearContextStore() { - requestContextStore.Range(func(key, _ interface{}) bool { - requestContextStore.Delete(key) - return true - }) + requestContextMu.Lock() + defer requestContextMu.Unlock() + requestContextStore = map[string]storedRequestContext{} + contextsEvicted = 0 } -func TestStoreAndLoadRequestContext(t *testing.T) { - clearContextStore() - defer clearContextStore() +// storeAt stores a context as if it had been captured at the given time, so +// expiry can be exercised without waiting for it. +func storeAt(key string, ctx *RequestContext, at time.Time) { + requestContextMu.Lock() + defer requestContextMu.Unlock() + requestContextStore[key] = storedRequestContext{ctx: ctx, storedAt: at} +} +func TestStoreAndLoadAndDeleteRequestContext(t *testing.T) { tests := []struct { name string requestID uint32 ctx *RequestContext }{ { - name: "store and load basic context", + name: "context with claims", requestID: 1, ctx: &RequestContext{ Method: "GET", - Path: "/api/users", - Headers: http.Header{"Authorization": []string{"Bearer token1"}}, - JWTClaims: map[string]interface{}{"sub": "user-1"}, + Path: "/api/users/123", + JWTClaims: map[string]interface{}{"sub": "did:key:z42"}, }, }, { - name: "store and load context with empty claims", + name: "context without claims", requestID: 2, - ctx: &RequestContext{ - Method: "POST", - Path: "/api/data", - Headers: http.Header{}, - JWTClaims: map[string]interface{}{}, - }, - }, - { - name: "store and load context with nil claims", - requestID: 3, - ctx: &RequestContext{ - Method: "DELETE", - Path: "/api/users/123", - Headers: nil, - JWTClaims: nil, - }, + ctx: &RequestContext{Method: "POST", Path: "/api/data"}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + clearContextStore() + defer clearContextStore() + StoreRequestContext(testReqKey(tt.requestID), tt.ctx) + assert.Equal(t, 1, RequestContextStoreSize()) - loaded, ok := LoadRequestContext(testReqKey(tt.requestID)) - require.True(t, ok, "expected context to be found") + loaded, ok := LoadAndDeleteRequestContext(testReqKey(tt.requestID)) + require.True(t, ok) assert.Equal(t, tt.ctx, loaded) + + _, ok = LoadAndDeleteRequestContext(testReqKey(tt.requestID)) + assert.False(t, ok, "consuming a context must remove it") + assert.Equal(t, 0, RequestContextStoreSize()) }) } } -func TestLoadRequestContext_NotFound(t *testing.T) { +func TestLoadAndDeleteRequestContext_NotFound(t *testing.T) { clearContextStore() defer clearContextStore() - const nonExistentID uint32 = 99999 - loaded, ok := LoadRequestContext(testReqKey(nonExistentID)) - assert.False(t, ok, "expected context not to be found") + loaded, ok := LoadAndDeleteRequestContext(testReqKey(999)) + assert.False(t, ok) assert.Nil(t, loaded) } -func TestDeleteRequestContext(t *testing.T) { +func TestStoreRequestContext_OverwritesExisting(t *testing.T) { clearContextStore() defer clearContextStore() - const requestID uint32 = 10 - ctx := &RequestContext{ - Method: "GET", - Path: "/api/test", - } + const requestID = uint32(10) + StoreRequestContext(testReqKey(requestID), &RequestContext{Method: "GET", Path: "/first"}) + StoreRequestContext(testReqKey(requestID), &RequestContext{Method: "POST", Path: "/second"}) - StoreRequestContext(testReqKey(requestID), ctx) + assert.Equal(t, 1, RequestContextStoreSize(), "overwriting must not grow the store") - // Verify it's stored. - loaded, ok := LoadRequestContext(testReqKey(requestID)) + loaded, ok := LoadAndDeleteRequestContext(testReqKey(requestID)) require.True(t, ok) - assert.Equal(t, ctx, loaded) + assert.Equal(t, "/second", loaded.Path) +} - // Delete it. - DeleteRequestContext(testReqKey(requestID)) +// TestRequestContext_ExpiredIsNotServed verifies an entry that outlived the TTL +// is reported as absent (and removed) rather than deciding a fresh request. +func TestRequestContext_ExpiredIsNotServed(t *testing.T) { + clearContextStore() + defer clearContextStore() - // Verify it's gone. - loaded, ok = LoadRequestContext(testReqKey(requestID)) - assert.False(t, ok) + key := testReqKey(20) + storeAt(key, &RequestContext{Method: "GET", Path: "/stale"}, time.Now().Add(-RequestContextTTL-time.Second)) + + loaded, ok := LoadAndDeleteRequestContext(key) + assert.False(t, ok, "an expired context must not be served") assert.Nil(t, loaded) + assert.Equal(t, 0, RequestContextStoreSize()) + assert.Equal(t, uint64(1), RequestContextsEvicted()) } -func TestLoadAndDeleteRequestContext(t *testing.T) { +// TestSweepRequestContexts verifies the janitor evicts exactly the entries whose +// response phase never ran, leaving live ones alone. Without it, every aborted +// request leaks an entry for the lifetime of the runner. +func TestSweepRequestContexts(t *testing.T) { clearContextStore() defer clearContextStore() - const requestID uint32 = 20 - ctx := &RequestContext{ - Method: "PUT", - Path: "/api/resource", - JWTClaims: map[string]interface{}{"sub": "user-20"}, - } + now := time.Now() + storeAt("expired-1", &RequestContext{Path: "/a"}, now.Add(-RequestContextTTL-time.Second)) + storeAt("expired-2", &RequestContext{Path: "/b"}, now.Add(-2*RequestContextTTL)) + storeAt("live", &RequestContext{Path: "/c"}, now) - StoreRequestContext(testReqKey(requestID), ctx) + assert.Equal(t, 2, sweepRequestContexts(now)) + assert.Equal(t, 1, RequestContextStoreSize()) + assert.Equal(t, uint64(2), RequestContextsEvicted()) - // LoadAndDelete should return the context. - loaded, ok := LoadAndDeleteRequestContext(testReqKey(requestID)) + loaded, ok := LoadAndDeleteRequestContext("live") require.True(t, ok) - assert.Equal(t, ctx, loaded) - - // Subsequent load should fail — already deleted. - loaded, ok = LoadRequestContext(testReqKey(requestID)) - assert.False(t, ok) - assert.Nil(t, loaded) + assert.Equal(t, "/c", loaded.Path) } -func TestLoadAndDeleteRequestContext_NotFound(t *testing.T) { +// TestStoreRequestContext_EnforcesCap verifies a full store makes room instead +// of growing without bound — the backstop against a client that opens requests +// and aborts before the response phase. +// +// It also pins the amortisation: an overflow evicts a BATCH, so the whole-map +// scan is paid once per contextEvictionBatch requests rather than on every +// request. Freeing one slot at a time turned the cap from a memory bound into a +// latency cliff, engaging precisely under the flood the cap exists to contain. +func TestStoreRequestContext_EnforcesCap(t *testing.T) { clearContextStore() defer clearContextStore() - const nonExistentID uint32 = 88888 - loaded, ok := LoadAndDeleteRequestContext(testReqKey(nonExistentID)) - assert.False(t, ok) - assert.Nil(t, loaded) + now := time.Now() + requestContextMu.Lock() + for i := 0; i < MaxRequestContexts; i++ { + // All live, so no sweep can free anything: the oldest must be evicted. + requestContextStore[fmt.Sprintf("live-%d", i)] = storedRequestContext{ + ctx: &RequestContext{Path: "/x"}, + storedAt: now.Add(time.Duration(i) * time.Millisecond), + } + } + requestContextMu.Unlock() + + StoreRequestContext("newest", &RequestContext{Path: "/new"}) + + assert.Equal(t, MaxRequestContexts-contextEvictionBatch+1, RequestContextStoreSize(), + "an overflow must evict a batch, not a single entry") + assert.Equal(t, uint64(contextEvictionBatch), RequestContextsEvicted()) + + // The batch taken is the oldest one. + _, ok := LoadAndDeleteRequestContext("live-0") + assert.False(t, ok, "the oldest entries are the ones evicted") + _, ok = LoadAndDeleteRequestContext(fmt.Sprintf("live-%d", contextEvictionBatch-1)) + assert.False(t, ok, "the whole oldest batch is evicted") + _, ok = LoadAndDeleteRequestContext(fmt.Sprintf("live-%d", contextEvictionBatch)) + assert.True(t, ok, "entries beyond the batch survive") + _, ok = LoadAndDeleteRequestContext("newest") + assert.True(t, ok, "the new request must still be served") } -func TestStoreRequestContext_OverwritesExisting(t *testing.T) { +// TestStoreRequestContext_EvictionIsAmortised verifies the requests following an +// overflow are served from the headroom the batch freed, without evicting (and +// therefore without scanning) again. +func TestStoreRequestContext_EvictionIsAmortised(t *testing.T) { clearContextStore() defer clearContextStore() - const requestID uint32 = 30 - original := &RequestContext{ - Method: "GET", - Path: "/original", + now := time.Now() + requestContextMu.Lock() + for i := 0; i < MaxRequestContexts; i++ { + requestContextStore[fmt.Sprintf("live-%d", i)] = storedRequestContext{ + ctx: &RequestContext{Path: "/x"}, + storedAt: now.Add(time.Duration(i) * time.Millisecond), + } } - replacement := &RequestContext{ - Method: "POST", - Path: "/replacement", + requestContextMu.Unlock() + + StoreRequestContext("overflow", &RequestContext{Path: "/new"}) + evictedAfterFirst := RequestContextsEvicted() + require.Equal(t, uint64(contextEvictionBatch), evictedAfterFirst) + + // The batch freed contextEvictionBatch slots and one was consumed by the + // store above, so this many more fit without any further eviction. + for i := 0; i < contextEvictionBatch-1; i++ { + StoreRequestContext(fmt.Sprintf("after-%d", i), &RequestContext{Path: "/y"}) } - StoreRequestContext(testReqKey(requestID), original) - StoreRequestContext(testReqKey(requestID), replacement) + assert.Equal(t, evictedAfterFirst, RequestContextsEvicted(), + "requests within the freed headroom must not trigger another scan") + assert.Equal(t, MaxRequestContexts, RequestContextStoreSize()) +} - loaded, ok := LoadRequestContext(testReqKey(requestID)) - require.True(t, ok) - assert.Equal(t, replacement, loaded) +// TestRequestContext_HoldsNoHeaders pins the property that made a leaked entry a +// credential leak: the context must not retain the request's Authorization +// header (or any other). +func TestRequestContext_HoldsNoHeaders(t *testing.T) { + rc := RequestContext{} + assert.Equal(t, 3, reflectFieldCount(rc), "RequestContext must hold only Method, Path and JWTClaims") } func TestRequestContext_String(t *testing.T) { @@ -190,20 +237,14 @@ func TestRequestContext_String(t *testing.T) { ctx: &RequestContext{ Method: "GET", Path: "/api/users", - Headers: http.Header{"Auth": []string{"val"}}, - JWTClaims: map[string]interface{}{"sub": "u1", "scope": "read"}, + JWTClaims: map[string]interface{}{"sub": "u", "scope": "read"}, }, - expected: "RequestContext{Method: GET, Path: /api/users, Claims: 2, Headers: 1}", + expected: "RequestContext{Method: GET, Path: /api/users, Claims: 2}", }, { - name: "empty context", - ctx: &RequestContext{ - Method: "", - Path: "", - Headers: nil, - JWTClaims: nil, - }, - expected: "RequestContext{Method: , Path: , Claims: 0, Headers: 0}", + name: "empty context", + ctx: &RequestContext{}, + expected: "RequestContext{Method: , Path: , Claims: 0}", }, } @@ -214,84 +255,35 @@ func TestRequestContext_String(t *testing.T) { } } -func TestConcurrentStoreAndLoad(t *testing.T) { +func TestConcurrentStoreLoadAndDelete(t *testing.T) { clearContextStore() defer clearContextStore() const goroutineCount = 100 var wg sync.WaitGroup - // Concurrently store contexts with different IDs. for i := uint32(0); i < goroutineCount; i++ { wg.Add(1) go func(id uint32) { defer wg.Done() - ctx := &RequestContext{ + StoreRequestContext(testReqKey(id), &RequestContext{ Method: "GET", Path: fmt.Sprintf("/api/resource/%d", id), JWTClaims: map[string]interface{}{"sub": fmt.Sprintf("user-%d", id)}, - } - StoreRequestContext(testReqKey(id), ctx) - }(i) - } - wg.Wait() - - // Concurrently load and verify all contexts. - for i := uint32(0); i < goroutineCount; i++ { - wg.Add(1) - go func(id uint32) { - defer wg.Done() - loaded, ok := LoadRequestContext(testReqKey(id)) - assert.True(t, ok, "context for ID %d should exist", id) - if ok { - expectedPath := fmt.Sprintf("/api/resource/%d", id) - assert.Equal(t, expectedPath, loaded.Path) - } - }(i) - } - wg.Wait() - - // Concurrently delete all contexts. - for i := uint32(0); i < goroutineCount; i++ { - wg.Add(1) - go func(id uint32) { - defer wg.Done() - DeleteRequestContext(testReqKey(id)) + }) }(i) } wg.Wait() + assert.Equal(t, int(goroutineCount), RequestContextStoreSize()) - // Verify all contexts are gone. - for i := uint32(0); i < goroutineCount; i++ { - _, ok := LoadRequestContext(testReqKey(i)) - assert.False(t, ok, "context for ID %d should have been deleted", i) - } -} - -func TestConcurrentLoadAndDelete(t *testing.T) { - clearContextStore() - defer clearContextStore() - - const goroutineCount = 100 - var wg sync.WaitGroup - - // Pre-populate the store. - for i := uint32(0); i < goroutineCount; i++ { - StoreRequestContext(testReqKey(i), &RequestContext{ - Method: "GET", - Path: fmt.Sprintf("/api/%d", i), - }) - } - - // Concurrently LoadAndDelete — each ID should be successfully loaded - // exactly once across all goroutines. + // Each id must be loaded successfully exactly once across all goroutines. results := make([]bool, goroutineCount) for i := uint32(0); i < goroutineCount; i++ { wg.Add(1) go func(id uint32) { defer wg.Done() - _, ok := LoadAndDeleteRequestContext(testReqKey(id)) - results[id] = ok + loaded, ok := LoadAndDeleteRequestContext(testReqKey(id)) + results[id] = ok && loaded.Path == fmt.Sprintf("/api/resource/%d", id) }(i) } wg.Wait() @@ -299,10 +291,10 @@ func TestConcurrentLoadAndDelete(t *testing.T) { for i := uint32(0); i < goroutineCount; i++ { assert.True(t, results[i], "LoadAndDelete should succeed for ID %d", i) } + assert.Equal(t, 0, RequestContextStoreSize()) +} - // Verify everything is deleted. - for i := uint32(0); i < goroutineCount; i++ { - _, ok := LoadRequestContext(testReqKey(i)) - assert.False(t, ok, "context for ID %d should have been deleted", i) - } +// reflectFieldCount returns how many fields a struct value has. +func reflectFieldCount(v interface{}) int { + return reflect.TypeOf(v).NumField() } diff --git a/main.go b/main.go index 87413ba..39c31fc 100644 --- a/main.go +++ b/main.go @@ -21,12 +21,80 @@ package main import ( + "consent-plugin/internal/audit" + "consent-plugin/internal/logging" + "consent-plugin/internal/metrics" + // Import the plugin package to trigger init() registration. _ "consent-plugin/internal/plugin" + "errors" + "net/http" + "os" + "time" "github.com/apache/apisix-go-plugin-runner/pkg/runner" - // Import the plugin package to trigger init() registration. ) +// EnvMetricsAddress is the listen address for the Prometheus metrics endpoint +// (e.g. ":9091"). Metrics are off unless it is set: the runner is normally +// reached only over its unix socket, so opening a TCP port is an explicit +// decision for the deployment to make. +const EnvMetricsAddress = "CONSENT_METRICS_ADDRESS" + +// metricsPath is where the metrics are exposed. +const metricsPath = "/metrics" + +// metricsServerTimeout bounds a metrics request, so a stuck scraper cannot hold +// a connection open indefinitely. +const metricsServerTimeout = 10 * time.Second + func main() { - runner.Run(runner.RunnerConfig{}) + go serveMetrics() + + runAndFlush(func() { runner.Run(runner.RunnerConfig{}) }, audit.ShutdownAll) +} + +// runAndFlush runs the plugin runner to completion and only then flushes the +// audit queue. +// +// The ordering is the whole point, and it is easy to get wrong. The runner +// installs its own signal.Notify for SIGINT/SIGTERM and returns from Run as soon +// as one arrives. An earlier version of this file waited for the same signal in +// a goroutine of its own and flushed there — but Go delivers a signal to every +// registered channel at once, so the runner's handler returned, main returned, +// and the process exited while the flush was still draining its queue and +// waiting on an HTTP export. The flush was present and essentially never +// completed: exactly the loss it was added to prevent, now wearing the +// appearance of being handled, and invisible to the dropped-events counter +// because these were not queue-overflow drops. +// +// Since Run already blocks until the signal, the flush belongs after it, +// synchronously, where nothing can exit out from under it — and the plugin needs +// no signal handler of its own. +func runAndFlush(run, flush func()) { + run() + flush() +} + +// serveMetrics exposes the plugin's Prometheus metrics when an address is +// configured. A component that can deny production traffic should not be +// observable only through unstructured logs. +func serveMetrics() { + address := os.Getenv(EnvMetricsAddress) + if address == "" { + return + } + mux := http.NewServeMux() + mux.Handle(metricsPath, metrics.Handler()) + server := &http.Server{ + Addr: address, + Handler: mux, + ReadHeaderTimeout: metricsServerTimeout, + ReadTimeout: metricsServerTimeout, + WriteTimeout: metricsServerTimeout, + } + logging.Infof("serving metrics on %s%s", address, metricsPath) + if err := server.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + // The gate keeps working without metrics; do not take the runner down. + logging.Errorf("metrics endpoint stopped: %s", logging.Sanitize(err.Error())) + } } diff --git a/main_test.go b/main_test.go new file mode 100644 index 0000000..f1af53c --- /dev/null +++ b/main_test.go @@ -0,0 +1,61 @@ +/* + * Copyright 2026 Seamless Middleware Technologies S.L and/or its affiliates + * and other contributors as indicated by the @author tags. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestRunAndFlushOrdering pins the ordering the audit flush depends on: it must +// happen after the runner has returned, not concurrently with it. +// +// The previous wiring ran the flush in its own goroutine waiting on the same +// signal the runner waits on, so the process exited before the flush finished +// and the queued records were lost. Nothing in the suite noticed, because a +// flush that never completes still compiles and still passes every test that +// only checks it was called. +func TestRunAndFlushOrdering(t *testing.T) { + var sequence []string + + runAndFlush( + func() { sequence = append(sequence, "run") }, + func() { sequence = append(sequence, "flush") }, + ) + + require.Len(t, sequence, 2) + assert.Equal(t, []string{"run", "flush"}, sequence, + "the audit flush must run after the runner returns, or the process exits with the queue undrained") +} + +// TestRunAndFlushFlushesAfterRunBlocks verifies the flush waits for a runner +// that returns only when it is good and ready — the real runner blocks until +// SIGTERM. +func TestRunAndFlushFlushesAfterRunBlocks(t *testing.T) { + runnerReturned := false + flushSawReturn := false + + runAndFlush( + func() { runnerReturned = true }, + func() { flushSawReturn = runnerReturned }, + ) + + assert.True(t, flushSawReturn, "the flush must not start until the runner has returned") +}