From 3a8afa67cd9566ea570a9f520554d2b3c9b8047f Mon Sep 17 00:00:00 2001 From: Chamal Abeywardhana Date: Tue, 11 Aug 2026 11:40:04 +1000 Subject: [PATCH 1/7] Remove unnecessary pull-secret from DVO metrics requests The DVO client was attaching the cluster pull-secret as an Authorization header over plaintext HTTP to the unauthenticated DVO /metrics endpoint, unnecessarily exposing the cloud.openshift.com credential on the pod network. Remove the auth round-tripper and pull-secret retrieval from the DVO client since the metrics endpoint does not require authentication. Add unit tests for the DVO client including a regression guard to ensure no Authorization header is sent. Fixes: ROSAENG-61337 Claude AI assisted --- pkg/dvo/builder.go | 11 +---- pkg/dvo/client.go | 28 ++--------- pkg/dvo/client_test.go | 97 +++++++++++++++++++++++++++++++++++++++ pkg/dvo/dvo_suite_test.go | 13 ++++++ 4 files changed, 116 insertions(+), 33 deletions(-) create mode 100644 pkg/dvo/client_test.go create mode 100644 pkg/dvo/dvo_suite_test.go diff --git a/pkg/dvo/builder.go b/pkg/dvo/builder.go index a970b550..27fe4169 100644 --- a/pkg/dvo/builder.go +++ b/pkg/dvo/builder.go @@ -1,11 +1,9 @@ package dvo import ( - "fmt" "net/http" "github.com/openshift/managed-upgrade-operator/pkg/metrics" - "github.com/openshift/managed-upgrade-operator/util" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -32,15 +30,8 @@ func (dcb *dvoClientBuilder) New(c client.Client) (DvoClient, error) { return nil, err } - // Fetch the cluster AccessToken - accessToken, err := util.GetAccessToken(c) - if err != nil { - return nil, fmt.Errorf("failed to retrieve cluster access token") - } - - // Set up the HTTP client using the token httpClient := http.Client{ - Transport: &dvoRoundTripper{authorization: *accessToken}, + Transport: newDvoTransport(), } // Create and return a new instance of dvoClient diff --git a/pkg/dvo/client.go b/pkg/dvo/client.go index d59b7715..d8dc2208 100644 --- a/pkg/dvo/client.go +++ b/pkg/dvo/client.go @@ -8,8 +8,6 @@ import ( "time" "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/openshift/managed-upgrade-operator/util" ) const ( @@ -17,11 +15,6 @@ const ( METRICS_API_PATH = "/metrics" ) -var ( - // ErrClusterIdNotFound is an error describing the cluster ID can not be found - ErrClusterIdNotFound = fmt.Errorf("OCM did not return a valid cluster ID: pull-secret may be invalid OR cluster's owner is disabled/banned in OCM") -) - // DvoClient enables an implementation of a DVO client //go:generate mockgen -destination=mocks/client.go -package=mocks github.com/openshift/managed-upgrade-operator/pkg/dvo DvoClient @@ -38,26 +31,15 @@ type dvoClient struct { httpClient http.Client } -type dvoRoundTripper struct { - authorization util.AccessToken -} - -func (drt *dvoRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { - authVal := fmt.Sprintf("AccessToken %s:%s", drt.authorization.ClusterId, drt.authorization.PullSecret) - req.Header.Add("Authorization", authVal) - transport := http.Transport{ - // Configure proxy support for Routes mode (when DVO is accessed externally) - // Respects HTTP_PROXY, HTTPS_PROXY, and NO_PROXY environment variables +func newDvoTransport() *http.Transport { + return &http.Transport{ Proxy: http.ProxyFromEnvironment, - - // Configure timeouts for reliable DVO communication DialContext: (&net.Dialer{ - Timeout: 30 * time.Second, // Maximum time to establish TCP connection - KeepAlive: 30 * time.Second, // TCP keep-alive probe interval + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, }).DialContext, - TLSHandshakeTimeout: 30 * time.Second, // Maximum time for TLS handshake (increased from 5s for proxy environments) + TLSHandshakeTimeout: 30 * time.Second, } - return transport.RoundTrip(req) } func (c *dvoClient) GetMetrics() ([]byte, error) { diff --git a/pkg/dvo/client_test.go b/pkg/dvo/client_test.go new file mode 100644 index 00000000..68820b1a --- /dev/null +++ b/pkg/dvo/client_test.go @@ -0,0 +1,97 @@ +package dvo + +import ( + "fmt" + "net/http" + "net/http/httptest" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("DVO Client", func() { + var ( + testServer *httptest.Server + client *dvoClient + ) + + AfterEach(func() { + if testServer != nil { + testServer.Close() + } + }) + + Context("GetMetrics", func() { + It("returns metrics on a successful response", func() { + expectedBody := `# HELP deployment_validation_operator_total Total deployments checked +# TYPE deployment_validation_operator_total counter +deployment_validation_operator_total 42 +` + testServer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + Expect(r.URL.Path).To(Equal(METRICS_API_PATH)) + Expect(r.Method).To(Equal(http.MethodGet)) + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, expectedBody) + })) + + client = &dvoClient{ + dvoBaseUrl: testServer.Listener.Addr().String(), + httpClient: *testServer.Client(), + } + + body, err := client.GetMetrics() + Expect(err).To(BeNil()) + Expect(string(body)).To(Equal(expectedBody)) + }) + + It("does not send an Authorization header", func() { + testServer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + Expect(r.Header.Get("Authorization")).To(BeEmpty()) + w.WriteHeader(http.StatusOK) + })) + + client = &dvoClient{ + dvoBaseUrl: testServer.Listener.Addr().String(), + httpClient: *testServer.Client(), + } + + _, err := client.GetMetrics() + Expect(err).To(BeNil()) + }) + + It("returns an error when the server is unreachable", func() { + client = &dvoClient{ + dvoBaseUrl: "127.0.0.1:1", + httpClient: http.Client{Transport: newDvoTransport()}, + } + + _, err := client.GetMetrics() + Expect(err).ToNot(BeNil()) + }) + + It("returns the body even on non-200 status codes", func() { + testServer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + fmt.Fprint(w, "internal error") + })) + + client = &dvoClient{ + dvoBaseUrl: testServer.Listener.Addr().String(), + httpClient: *testServer.Client(), + } + + body, err := client.GetMetrics() + Expect(err).To(BeNil()) + Expect(string(body)).To(Equal("internal error")) + }) + }) + + Context("newDvoTransport", func() { + It("returns a transport with proxy support configured", func() { + transport := newDvoTransport() + Expect(transport).ToNot(BeNil()) + Expect(transport.Proxy).ToNot(BeNil()) + Expect(transport.TLSHandshakeTimeout.Seconds()).To(Equal(float64(30))) + }) + }) +}) diff --git a/pkg/dvo/dvo_suite_test.go b/pkg/dvo/dvo_suite_test.go new file mode 100644 index 00000000..eb0a0dac --- /dev/null +++ b/pkg/dvo/dvo_suite_test.go @@ -0,0 +1,13 @@ +package dvo + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestDvo(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "DVO Client Suite") +} From 56e5ab29b13aec0851cc5e24d92d9e4dbafe6354 Mon Sep 17 00:00:00 2001 From: Chamal Abeywardhana Date: Thu, 20 Aug 2026 14:52:00 +1000 Subject: [PATCH 2/7] Adding capability to override to DVO metrics SVC url for local testing --- pkg/dvo/builder.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pkg/dvo/builder.go b/pkg/dvo/builder.go index 27fe4169..21280a15 100644 --- a/pkg/dvo/builder.go +++ b/pkg/dvo/builder.go @@ -2,12 +2,14 @@ package dvo import ( "net/http" + "os" "github.com/openshift/managed-upgrade-operator/pkg/metrics" "sigs.k8s.io/controller-runtime/pkg/client" ) // DvoClientBuilder enables implementation of a DVO client. +// //go:generate mockgen -destination=mocks/builder.go -package=mocks github.com/openshift/managed-upgrade-operator/pkg/dvo DvoClientBuilder type DvoClientBuilder interface { New(c client.Client) (DvoClient, error) @@ -26,6 +28,15 @@ func (dcb *dvoClientBuilder) New(c client.Client) (DvoClient, error) { // Get the service URL for the deployment-validation-operator-metrics service svcURL, err := metrics.NetworkTarget(c, "openshift-deployment-validation-operator", "deployment-validation-operator-metrics", "http-metrics") + + // For local run of MUO via `make run`, it's expected that DVO_SVC_URL is set, after port-forwarding DVO metrics service + // e.g. port-forward command: `oc port-forward svc/deployment-validation-operator-metrics 53083:8383 -n openshift-deployment-validation-operator` + // e.g. dvoSVCULR: 127.0.0.1:53083 + dvoSVCURL := os.Getenv("DVO_SVC_URL") + if dvoSVCURL != "" { + svcURL = dvoSVCURL + } + if err != nil { return nil, err } From 6afcf16c8fbb2fc9c8947134c82d346fc636ea9a Mon Sep 17 00:00:00 2001 From: Chamal Abeywardhana Date: Fri, 21 Aug 2026 13:05:16 +1000 Subject: [PATCH 3/7] Documentation update --- development/port-forwards | 8 +++++--- docs/development.md | 34 +++++++++++++++++----------------- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/development/port-forwards b/development/port-forwards index ed12c21b..d813df86 100755 --- a/development/port-forwards +++ b/development/port-forwards @@ -15,7 +15,7 @@ do echo "example: ${0##} \$CONTEXT" echo exit 1 - fi + fi done OC=$(which oc) @@ -32,6 +32,8 @@ then sudo -- sh -c "echo 127.0.0.1 prometheus-k8s.openshift-monitoring.svc.cluster.local alertmanager-main.openshift-monitoring.svc.cluster.local >> /etc/hosts" fi -# Setup prometheus and alertmanager port-forwards +# Setup prometheus, alertmanager and DVO metrics port-forwards. +# Note: DVO metrics port 53083 can be chaned to any local port avaiable. If changed, make sure to include the correct port when run while true; do $OC port-forward -n openshift-monitoring svc/prometheus-k8s 9091:9091;done & -while true; do $OC port-forward -n openshift-monitoring svc/alertmanager-main 9094:9094;done & \ No newline at end of file +while true; do $OC port-forward -n openshift-monitoring svc/alertmanager-main 9094:9094;done & +while true; do $OC port-forward -n openshift-deployment-validation-operator svc/deployment-validation-operator-metrics 53083:8383;done & diff --git a/docs/development.md b/docs/development.md index fc32f83a..ab1cb48d 100644 --- a/docs/development.md +++ b/docs/development.md @@ -157,15 +157,15 @@ Regardless of how you choose to run the operator, before doing so ensure the `Up $ oc create -f deploy/crds/upgrade.managed.openshift.io_upgradeconfigs_crd.yaml ``` -MUO by defaults uses in the internal services to contact prometheus and alertmanager. This enables the use of a firewall to prevent egress calls however increases local development complexity slightly. +MUO by defaults uses in the internal services to contact prometheus and alertmanager. This enables the use of a firewall to prevent egress calls however increases local development complexity slightly. -There are now three main modes that MUO can be ran in. +There are now three main modes that MUO can be ran in. -1. Run in a container in cluster. -2. Run locally using port-forwards and `/etc/hosts` entries to replicate production environment. -3. Run locally using Routes to access services. This is not true production however is the most simple for local development. +1. Run in a container in cluster. +2. Run locally using port-forwards and `/etc/hosts` entries to replicate production environment. +3. Run locally using Routes to access services. This is not true production however is the most simple for local development. -Modes 2 and 3 can be executed via the `Makefile` optionally setting the `$OPERATOR_NAMESPACE` as explored in the next section. +Modes 2 and 3 can be executed via the `Makefile` optionally setting the `$OPERATOR_NAMESPACE` as explored in the next section. ``` run Wrapper around operator sdk run. Requires OPERATOR_NAMESPACE to be set. See run-standard for defaults. @@ -204,7 +204,7 @@ example: ./development/port-forwards $CONTEXT $ ./development/port-forwards $CONTEXT ``` -The operator can then be ran as follows. +The operator can then be ran as follows. ``` $ oc login $(oc get infrastructures cluster -o json | jq -r '.status.apiServerURL') --token $(oc -n openshift-managed-upgrade-operator serviceaccounts get-token managed-upgrade-operator) @@ -217,25 +217,25 @@ You don't have any projects. Contact your system administrator to request a proj Then if you are using the standard namespace ``` -$ make run-standard +OPERATOR_NAMESPACE="openshift-managed-upgrade-operator" WATCH_NAMESPACE="" DVO_SVC_URL="127.0.0.1:53083" go run ./main.go ``` -Else you can provide your own. +Else you can provide your own. ``` -$ OPERATOR_NAMESPACE=managed-upgrade-operator make run +OPERATOR_NAMESPACE="openshift-managed-upgrade-operator-test" WATCH_NAMESPACE="" DVO_SVC_URL="127.0.0.1:53083" go run ./main.go ``` ### Run using cluster routes -Run locally using standard namespace and cluster routes. +Run locally using standard namespace and cluster routes. ``` $ make run-standard-routes ``` -Run locally using custom namespace and cluster routes. +Run locally using custom namespace and cluster routes. ``` $ OPERATOR_NAMESPACE=managed-upgrade-operator make run-routes @@ -255,8 +255,8 @@ $ make docker-build IMG=quay.io//managed-upgrade-operator:latest podman push quay.io//managed-upgrade-operator:latest ``` -- Login to `oc` [as admin](https://github.com/openshift/ops-sop/blob/master/v4/howto/break-glass-kubeadmin.md#for-clusters-with-public-api) - +- Login to `oc` [as admin](https://github.com/openshift/ops-sop/blob/master/v4/howto/break-glass-kubeadmin.md#for-clusters-with-public-api) + - Ensure no other instances of managed-upgrade-operator are actively running on your cluster, as they may conflict. If MUO is already deployed on the cluster scale the deployment down to 0: ```shell @@ -341,12 +341,12 @@ $ oc apply -f test/deploy/upgrade.managed.openshift.io_v1alpha1_upgradeconfig_cr ```shell oc get upgrade -n test-managed-upgrade-operator -``` +``` - Inspect `upgradeConfig`: ```shell -oc describe upgrade -n test-managed-upgrade-operator managed-upgrade-config +oc describe upgrade -n test-managed-upgrade-operator managed-upgrade-config ``` - It can be useful to monitor the events in `test-managed-upgrade-operator` namespace during the upgrade: @@ -396,4 +396,4 @@ python hack/maintenance-update.py # To update deps to a specific Openshift release python hack/maintenance-update.py --release release-4.19 -``` \ No newline at end of file +``` From 788d8c9e5311ddbbf9adff7480fef4fa117ff34f Mon Sep 17 00:00:00 2001 From: Chamal Abeywardhana Date: Fri, 21 Aug 2026 13:10:52 +1000 Subject: [PATCH 4/7] boilerplate update --- boilerplate/generated-includes.mk | 5 ----- build/Dockerfile | 2 +- build/Dockerfile.olm-registry | 2 +- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/boilerplate/generated-includes.mk b/boilerplate/generated-includes.mk index 451ff8c2..0347cf30 100644 --- a/boilerplate/generated-includes.mk +++ b/boilerplate/generated-includes.mk @@ -2,8 +2,3 @@ # This file automatically includes any *.mk files in your subscribed # conventions. Please ensure your base Makefile includes only this file. include boilerplate/_lib/boilerplate.mk -include boilerplate/openshift/golang-osd-operator/csv-generate/csv-generate.mk -include boilerplate/openshift/golang-osd-operator/project.mk -include boilerplate/openshift/golang-osd-operator/standard.mk -include boilerplate/openshift/golang-osd-e2e/project.mk -include boilerplate/openshift/golang-osd-e2e/standard.mk diff --git a/build/Dockerfile b/build/Dockerfile index 19a6ffef..5579fe46 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -8,7 +8,7 @@ COPY . . RUN make go-build #### -FROM registry.access.redhat.com/ubi9/ubi-minimal:9.8-1786380870 +FROM registry.access.redhat.com/ubi9/ubi-minimal:9.8-1786987521 ENV USER_UID=1001 \ USER_NAME=managed-upgrade-operator diff --git a/build/Dockerfile.olm-registry b/build/Dockerfile.olm-registry index 85f60aad..2750d914 100644 --- a/build/Dockerfile.olm-registry +++ b/build/Dockerfile.olm-registry @@ -4,7 +4,7 @@ COPY ${SAAS_OPERATOR_DIR} manifests RUN initializer --permissive # ubi-micro does not work for clusters with fips enabled unless we make OpenSSL available -FROM registry.access.redhat.com/ubi9/ubi-minimal:9.8-1786380870 +FROM registry.access.redhat.com/ubi9/ubi-minimal:latest COPY --from=builder /bin/registry-server /bin/registry-server COPY --from=builder /bin/grpc_health_probe /bin/grpc_health_probe From c26c0747e1e835509432e86d4bc3564d3cea7b68 Mon Sep 17 00:00:00 2001 From: Chamal Abeywardhana Date: Fri, 21 Aug 2026 13:38:00 +1000 Subject: [PATCH 5/7] boilerplate update --- ...rator-agentic-sdlc-check-pull-request.yaml | 2 +- boilerplate/_data/last-boilerplate-commit | 2 +- boilerplate/generated-includes.mk | 5 ++ .../gangway-bridge-template.yml | 73 +++++++++++++++---- .../openshift/golang-osd-operator/codecov.sh | 23 +++--- build/Dockerfile.olm-registry | 2 +- test/e2e/gangway-bridge-template.yml | 73 +++++++++++++++---- 7 files changed, 138 insertions(+), 42 deletions(-) diff --git a/.tekton/managed-upgrade-operator-agentic-sdlc-check-pull-request.yaml b/.tekton/managed-upgrade-operator-agentic-sdlc-check-pull-request.yaml index 6528fe43..b2a5d6e4 100644 --- a/.tekton/managed-upgrade-operator-agentic-sdlc-check-pull-request.yaml +++ b/.tekton/managed-upgrade-operator-agentic-sdlc-check-pull-request.yaml @@ -43,7 +43,7 @@ spec: - name: url value: https://github.com/openshift/boilerplate - name: revision - value: 56243bd0598738bea9eb26bc6260df023a414063 + value: f94ee72e7bb0b6c7ad60262874b000cda3cc45d3 - name: pathInRepo value: pipelines/agentic-sdlc-check/pipeline.yaml status: {} diff --git a/boilerplate/_data/last-boilerplate-commit b/boilerplate/_data/last-boilerplate-commit index 19e99239..afbcd8aa 100644 --- a/boilerplate/_data/last-boilerplate-commit +++ b/boilerplate/_data/last-boilerplate-commit @@ -1 +1 @@ -56243bd0598738bea9eb26bc6260df023a414063 +f94ee72e7bb0b6c7ad60262874b000cda3cc45d3 diff --git a/boilerplate/generated-includes.mk b/boilerplate/generated-includes.mk index 0347cf30..451ff8c2 100644 --- a/boilerplate/generated-includes.mk +++ b/boilerplate/generated-includes.mk @@ -2,3 +2,8 @@ # This file automatically includes any *.mk files in your subscribed # conventions. Please ensure your base Makefile includes only this file. include boilerplate/_lib/boilerplate.mk +include boilerplate/openshift/golang-osd-operator/csv-generate/csv-generate.mk +include boilerplate/openshift/golang-osd-operator/project.mk +include boilerplate/openshift/golang-osd-operator/standard.mk +include boilerplate/openshift/golang-osd-e2e/project.mk +include boilerplate/openshift/golang-osd-e2e/standard.mk diff --git a/boilerplate/openshift/golang-osd-e2e/gangway-bridge-template.yml b/boilerplate/openshift/golang-osd-e2e/gangway-bridge-template.yml index ec701e6d..410d18cb 100644 --- a/boilerplate/openshift/golang-osd-e2e/gangway-bridge-template.yml +++ b/boilerplate/openshift/golang-osd-e2e/gangway-bridge-template.yml @@ -12,7 +12,13 @@ parameters: description: Seconds between status polls - name: TIMEOUT value: "7200" - description: Maximum seconds to wait for job completion + description: Maximum seconds to wait per attempt for job completion + - name: MAX_RETRIES + value: "1" + description: Number of times to retry the Prow job on failure before reporting failure + - name: ACTIVE_DEADLINE + value: "14430" + description: Kubernetes Job deadline in seconds (should exceed TIMEOUT * (MAX_RETRIES + 1)) - name: JOB_ENVS value: "" description: Comma-separated KEY=VALUE pairs passed to the Prow job @@ -29,7 +35,7 @@ objects: name: gangway-bridge-${IMAGE_TAG}-${JOBID} spec: backoffLimit: 0 - activeDeadlineSeconds: ${{TIMEOUT}} + activeDeadlineSeconds: ${{ACTIVE_DEADLINE}} template: spec: automountServiceAccountToken: false @@ -46,6 +52,13 @@ objects: [[ "${TIMEOUT}" =~ ^[1-9][0-9]*$ ]] || { log "ERROR: TIMEOUT must be a positive integer"; exit 1; } [[ "${POLL_INTERVAL}" =~ ^[1-9][0-9]*$ ]] || { log "ERROR: POLL_INTERVAL must be a positive integer"; exit 1; } + [[ "${MAX_RETRIES}" =~ ^[0-9]+$ ]] || { log "ERROR: MAX_RETRIES must be a non-negative integer"; exit 1; } + + REQUIRED_DEADLINE=$(( (MAX_RETRIES + 1) * TIMEOUT + (MAX_RETRIES * 30) )) + if [[ "${ACTIVE_DEADLINE}" -lt "${REQUIRED_DEADLINE}" ]]; then + log "ERROR: ACTIVE_DEADLINE (${ACTIVE_DEADLINE}s) is less than the minimum required for ${MAX_RETRIES} retries with TIMEOUT=${TIMEOUT}s (need at least ${REQUIRED_DEADLINE}s)" + exit 1 + fi BODY='{"job_execution_type":"1"}' if [[ -n "${JOB_ENVS:-}" ]]; then @@ -53,21 +66,47 @@ objects: BODY=$(jq -cn --argjson e "$ENVS" '{"job_execution_type":"1","pod_spec_options":{"envs":$e}}') fi - RESP=$(curl -sfSL --retry 3 --retry-delay 10 -X POST -H "Authorization: Bearer ${GANGWAY_TOKEN}" -H "Content-Type: application/json" -d "${BODY}" "${GW}/${JOB_NAME}") - ID=$(echo "$RESP" | jq -re .id) - PROW_URL="https://prow.ci.openshift.org/view/gs/test-platform-results/logs/${JOB_NAME}/${ID}" - log "Triggered ${JOB_NAME} -> ${ID}" - log "Prow logs: ${PROW_URL}" + trigger_and_poll() { + if ! RESP=$(curl -sfSL --max-time 60 --retry 3 --retry-delay 10 -X POST -H "Authorization: Bearer ${GANGWAY_TOKEN}" -H "Content-Type: application/json" -d "${BODY}" "${GW}/${JOB_NAME}"); then + log "Failed to trigger ${JOB_NAME}" + return 1 + fi + if ! ID=$(echo "$RESP" | jq -re .id); then + log "Gangway did not return a valid execution ID" + return 1 + fi + PROW_URL="https://prow.ci.openshift.org/view/gs/test-platform-results/logs/${JOB_NAME}/${ID}" + log "Triggered ${JOB_NAME} -> ${ID}" + log "Prow logs: ${PROW_URL}" + + END=$((SECONDS + ${TIMEOUT})) + while [[ $SECONDS -lt $END ]]; do + sleep "${POLL_INTERVAL}" + S=$(curl -sfSL --max-time 30 -H "Authorization: Bearer ${GANGWAY_TOKEN}" "${GW}/${ID}" | jq -r .job_status) || S=UNKNOWN + log "${S} ($((SECONDS))s)" + case $S in + SUCCESS) log "Prow logs: ${PROW_URL}"; return 0;; + FAILURE|ABORTED|ERROR) log "Prow logs: ${PROW_URL}"; return 1;; + esac + done + log "Prow logs: ${PROW_URL}" + log "Timeout"; return 1 + } - END=$((SECONDS + ${TIMEOUT})) - while [[ $SECONDS -lt $END ]]; do - sleep "${POLL_INTERVAL}" - S=$(curl -sfSL -H "Authorization: Bearer ${GANGWAY_TOKEN}" "${GW}/${ID}" | jq -r .job_status) || S=UNKNOWN - log "${S} ($((SECONDS))s)" - case $S in SUCCESS) log "Prow logs: ${PROW_URL}"; exit 0;; FAILURE|ABORTED|ERROR) log "Prow logs: ${PROW_URL}"; exit 1;; esac + ATTEMPT=0 + while true; do + ATTEMPT=$((ATTEMPT + 1)) + log "Attempt ${ATTEMPT} of $((MAX_RETRIES + 1))" + if trigger_and_poll; then + exit 0 + fi + if [[ $ATTEMPT -gt $MAX_RETRIES ]]; then + log "All attempts exhausted" + exit 1 + fi + log "Retrying in 30s..." + sleep 30 done - log "Prow logs: ${PROW_URL}" - log "Timeout"; exit 1 env: - name: JOB_NAME value: ${JOB_NAME} @@ -82,6 +121,10 @@ objects: value: ${TIMEOUT} - name: JOB_ENVS value: ${JOB_ENVS} + - name: MAX_RETRIES + value: ${MAX_RETRIES} + - name: ACTIVE_DEADLINE + value: ${ACTIVE_DEADLINE} resources: requests: cpu: "50m" diff --git a/boilerplate/openshift/golang-osd-operator/codecov.sh b/boilerplate/openshift/golang-osd-operator/codecov.sh index 8fc79bd0..95fb5a90 100755 --- a/boilerplate/openshift/golang-osd-operator/codecov.sh +++ b/boilerplate/openshift/golang-osd-operator/codecov.sh @@ -23,11 +23,11 @@ rm -f "${COVER_PROFILE}.tmp" # Configure the git refs and job link based on how the job was triggered via prow if [[ "${JOB_TYPE}" == "presubmit" ]]; then echo "detected PR code coverage job for #${PULL_NUMBER}" - REF_FLAGS="-P ${PULL_NUMBER} -C ${PULL_PULL_SHA}" + REF_FLAGS="--pr ${PULL_NUMBER} --commit-sha ${PULL_PULL_SHA}" JOB_LINK="${CI_SERVER_URL}/pr-logs/pull/${REPO_OWNER}_${REPO_NAME}/${PULL_NUMBER}/${JOB_NAME}/${BUILD_ID}" elif [[ "${JOB_TYPE}" == "postsubmit" ]]; then echo "detected branch code coverage job for ${PULL_BASE_REF}" - REF_FLAGS="-B ${PULL_BASE_REF} -C ${PULL_BASE_SHA}" + REF_FLAGS="--branch ${PULL_BASE_REF} --commit-sha ${PULL_BASE_SHA}" JOB_LINK="${CI_SERVER_URL}/logs/${JOB_NAME}/${BUILD_ID}" elif [[ "${JOB_TYPE}" == "local" ]]; then echo "coverage report available at ${COVER_PROFILE}" @@ -43,12 +43,17 @@ export CI_BUILD_ID="${JOB_NAME}" export CI_JOB_ID="${BUILD_ID}" if [[ "${JOB_TYPE}" != "local" ]]; then - if [[ -z "${ARTIFACT_DIR:-}" ]] || [[ ! -d "${ARTIFACT_DIR}" ]] || [[ ! -w "${ARTIFACT_DIR}" ]]; then - echo '${ARTIFACT_DIR} must be set for non-local jobs, and must point to a writable directory' >&2 - exit 1 - fi - curl -sS https://codecov.io/bash -o "${ARTIFACT_DIR}/codecov.sh" - bash <(cat "${ARTIFACT_DIR}/codecov.sh") -Z -K -f "${COVER_PROFILE}" -r "${REPO_OWNER}/${REPO_NAME}" ${REF_FLAGS} + CODECOV_VERSION="${CODECOV_VERSION:-v11.3.1}" + CODECOV_SHA256="${CODECOV_SHA256:-ca1d64196d2d34771084afe76ea657d581bf628e31d993ff8e52ea09cc88a56d}" + CODECOV_BIN="$(mktemp -d)/codecov" + + curl -sSfL "https://github.com/codecov/codecov-cli/releases/download/${CODECOV_VERSION}/codecovcli_linux" \ + -o "${CODECOV_BIN}" + echo "${CODECOV_SHA256} ${CODECOV_BIN}" | sha256sum -c - + chmod +x "${CODECOV_BIN}" + + "${CODECOV_BIN}" upload-process --fail-on-error --git-service github \ + --file "${COVER_PROFILE}" --slug "${REPO_OWNER}/${REPO_NAME}" ${REF_FLAGS} else - bash <(curl -s https://codecov.io/bash) -Z -K -f "${COVER_PROFILE}" -r "${REPO_OWNER}/${REPO_NAME}" ${REF_FLAGS} + echo "coverage report available at ${COVER_PROFILE} (no upload in local mode)" fi diff --git a/build/Dockerfile.olm-registry b/build/Dockerfile.olm-registry index 2750d914..ef9c2588 100644 --- a/build/Dockerfile.olm-registry +++ b/build/Dockerfile.olm-registry @@ -4,7 +4,7 @@ COPY ${SAAS_OPERATOR_DIR} manifests RUN initializer --permissive # ubi-micro does not work for clusters with fips enabled unless we make OpenSSL available -FROM registry.access.redhat.com/ubi9/ubi-minimal:latest +FROM registry.access.redhat.com/ubi9/ubi-minimal:9.8-1786987521 COPY --from=builder /bin/registry-server /bin/registry-server COPY --from=builder /bin/grpc_health_probe /bin/grpc_health_probe diff --git a/test/e2e/gangway-bridge-template.yml b/test/e2e/gangway-bridge-template.yml index ec701e6d..410d18cb 100644 --- a/test/e2e/gangway-bridge-template.yml +++ b/test/e2e/gangway-bridge-template.yml @@ -12,7 +12,13 @@ parameters: description: Seconds between status polls - name: TIMEOUT value: "7200" - description: Maximum seconds to wait for job completion + description: Maximum seconds to wait per attempt for job completion + - name: MAX_RETRIES + value: "1" + description: Number of times to retry the Prow job on failure before reporting failure + - name: ACTIVE_DEADLINE + value: "14430" + description: Kubernetes Job deadline in seconds (should exceed TIMEOUT * (MAX_RETRIES + 1)) - name: JOB_ENVS value: "" description: Comma-separated KEY=VALUE pairs passed to the Prow job @@ -29,7 +35,7 @@ objects: name: gangway-bridge-${IMAGE_TAG}-${JOBID} spec: backoffLimit: 0 - activeDeadlineSeconds: ${{TIMEOUT}} + activeDeadlineSeconds: ${{ACTIVE_DEADLINE}} template: spec: automountServiceAccountToken: false @@ -46,6 +52,13 @@ objects: [[ "${TIMEOUT}" =~ ^[1-9][0-9]*$ ]] || { log "ERROR: TIMEOUT must be a positive integer"; exit 1; } [[ "${POLL_INTERVAL}" =~ ^[1-9][0-9]*$ ]] || { log "ERROR: POLL_INTERVAL must be a positive integer"; exit 1; } + [[ "${MAX_RETRIES}" =~ ^[0-9]+$ ]] || { log "ERROR: MAX_RETRIES must be a non-negative integer"; exit 1; } + + REQUIRED_DEADLINE=$(( (MAX_RETRIES + 1) * TIMEOUT + (MAX_RETRIES * 30) )) + if [[ "${ACTIVE_DEADLINE}" -lt "${REQUIRED_DEADLINE}" ]]; then + log "ERROR: ACTIVE_DEADLINE (${ACTIVE_DEADLINE}s) is less than the minimum required for ${MAX_RETRIES} retries with TIMEOUT=${TIMEOUT}s (need at least ${REQUIRED_DEADLINE}s)" + exit 1 + fi BODY='{"job_execution_type":"1"}' if [[ -n "${JOB_ENVS:-}" ]]; then @@ -53,21 +66,47 @@ objects: BODY=$(jq -cn --argjson e "$ENVS" '{"job_execution_type":"1","pod_spec_options":{"envs":$e}}') fi - RESP=$(curl -sfSL --retry 3 --retry-delay 10 -X POST -H "Authorization: Bearer ${GANGWAY_TOKEN}" -H "Content-Type: application/json" -d "${BODY}" "${GW}/${JOB_NAME}") - ID=$(echo "$RESP" | jq -re .id) - PROW_URL="https://prow.ci.openshift.org/view/gs/test-platform-results/logs/${JOB_NAME}/${ID}" - log "Triggered ${JOB_NAME} -> ${ID}" - log "Prow logs: ${PROW_URL}" + trigger_and_poll() { + if ! RESP=$(curl -sfSL --max-time 60 --retry 3 --retry-delay 10 -X POST -H "Authorization: Bearer ${GANGWAY_TOKEN}" -H "Content-Type: application/json" -d "${BODY}" "${GW}/${JOB_NAME}"); then + log "Failed to trigger ${JOB_NAME}" + return 1 + fi + if ! ID=$(echo "$RESP" | jq -re .id); then + log "Gangway did not return a valid execution ID" + return 1 + fi + PROW_URL="https://prow.ci.openshift.org/view/gs/test-platform-results/logs/${JOB_NAME}/${ID}" + log "Triggered ${JOB_NAME} -> ${ID}" + log "Prow logs: ${PROW_URL}" + + END=$((SECONDS + ${TIMEOUT})) + while [[ $SECONDS -lt $END ]]; do + sleep "${POLL_INTERVAL}" + S=$(curl -sfSL --max-time 30 -H "Authorization: Bearer ${GANGWAY_TOKEN}" "${GW}/${ID}" | jq -r .job_status) || S=UNKNOWN + log "${S} ($((SECONDS))s)" + case $S in + SUCCESS) log "Prow logs: ${PROW_URL}"; return 0;; + FAILURE|ABORTED|ERROR) log "Prow logs: ${PROW_URL}"; return 1;; + esac + done + log "Prow logs: ${PROW_URL}" + log "Timeout"; return 1 + } - END=$((SECONDS + ${TIMEOUT})) - while [[ $SECONDS -lt $END ]]; do - sleep "${POLL_INTERVAL}" - S=$(curl -sfSL -H "Authorization: Bearer ${GANGWAY_TOKEN}" "${GW}/${ID}" | jq -r .job_status) || S=UNKNOWN - log "${S} ($((SECONDS))s)" - case $S in SUCCESS) log "Prow logs: ${PROW_URL}"; exit 0;; FAILURE|ABORTED|ERROR) log "Prow logs: ${PROW_URL}"; exit 1;; esac + ATTEMPT=0 + while true; do + ATTEMPT=$((ATTEMPT + 1)) + log "Attempt ${ATTEMPT} of $((MAX_RETRIES + 1))" + if trigger_and_poll; then + exit 0 + fi + if [[ $ATTEMPT -gt $MAX_RETRIES ]]; then + log "All attempts exhausted" + exit 1 + fi + log "Retrying in 30s..." + sleep 30 done - log "Prow logs: ${PROW_URL}" - log "Timeout"; exit 1 env: - name: JOB_NAME value: ${JOB_NAME} @@ -82,6 +121,10 @@ objects: value: ${TIMEOUT} - name: JOB_ENVS value: ${JOB_ENVS} + - name: MAX_RETRIES + value: ${MAX_RETRIES} + - name: ACTIVE_DEADLINE + value: ${ACTIVE_DEADLINE} resources: requests: cpu: "50m" From 3d1b49ce5a9c5b74042621bc41d3078d67af6269 Mon Sep 17 00:00:00 2001 From: Chamal Abeywardhana Date: Mon, 24 Aug 2026 08:32:27 +1000 Subject: [PATCH 6/7] Moved the local running instructions from code to docs --- docs/development.md | 9 ++++++++- pkg/dvo/builder.go | 5 +---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/development.md b/docs/development.md index ab1cb48d..2d81f654 100644 --- a/docs/development.md +++ b/docs/development.md @@ -213,11 +213,18 @@ Logged into "https://$API_URL:6443" as "system:serviceaccount:openshift-managed- You don't have any projects. Contact your system administrator to request a project. ``` +Note: +> You might need to remove the webhook _sre-regular-user-validation_ +```oc delete validatingwebhookconfigurations sre-regular-user-validation``` \ +> For local run of MUO via `make run`, it's expected that DVO_SVC_URL is set, after port-forwarding DVO metrics service \ +>>e.g. port-forward command: `oc port-forward svc/deployment-validation-operator-metrics 53083:8383 -n openshift-deployment-validation-operator` \ +>>e.g. dvoSVCULR: 127.0.0.1:53083 Then if you are using the standard namespace ``` -OPERATOR_NAMESPACE="openshift-managed-upgrade-operator" WATCH_NAMESPACE="" DVO_SVC_URL="127.0.0.1:53083" go run ./main.go +export DVO_SVC_URL="127.0.0.1:53083" +make run ``` Else you can provide your own. diff --git a/pkg/dvo/builder.go b/pkg/dvo/builder.go index 21280a15..f7667656 100644 --- a/pkg/dvo/builder.go +++ b/pkg/dvo/builder.go @@ -28,10 +28,7 @@ func (dcb *dvoClientBuilder) New(c client.Client) (DvoClient, error) { // Get the service URL for the deployment-validation-operator-metrics service svcURL, err := metrics.NetworkTarget(c, "openshift-deployment-validation-operator", "deployment-validation-operator-metrics", "http-metrics") - - // For local run of MUO via `make run`, it's expected that DVO_SVC_URL is set, after port-forwarding DVO metrics service - // e.g. port-forward command: `oc port-forward svc/deployment-validation-operator-metrics 53083:8383 -n openshift-deployment-validation-operator` - // e.g. dvoSVCULR: 127.0.0.1:53083 + // Override svcURL for DVO, when environment variable is set dvoSVCURL := os.Getenv("DVO_SVC_URL") if dvoSVCURL != "" { svcURL = dvoSVCURL From 03bcfd7466175cc640ad559d680640cef07a9f01 Mon Sep 17 00:00:00 2001 From: Chamal Abeywardhana Date: Mon, 24 Aug 2026 08:53:02 +1000 Subject: [PATCH 7/7] renamed the newDvoTransport function to dvoTransport --- pkg/dvo/builder.go | 2 +- pkg/dvo/client.go | 2 +- pkg/dvo/client_test.go | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/dvo/builder.go b/pkg/dvo/builder.go index f7667656..d32f0977 100644 --- a/pkg/dvo/builder.go +++ b/pkg/dvo/builder.go @@ -39,7 +39,7 @@ func (dcb *dvoClientBuilder) New(c client.Client) (DvoClient, error) { } httpClient := http.Client{ - Transport: newDvoTransport(), + Transport: dvoTransport(), } // Create and return a new instance of dvoClient diff --git a/pkg/dvo/client.go b/pkg/dvo/client.go index d8dc2208..516f8169 100644 --- a/pkg/dvo/client.go +++ b/pkg/dvo/client.go @@ -31,7 +31,7 @@ type dvoClient struct { httpClient http.Client } -func newDvoTransport() *http.Transport { +func dvoTransport() *http.Transport { return &http.Transport{ Proxy: http.ProxyFromEnvironment, DialContext: (&net.Dialer{ diff --git a/pkg/dvo/client_test.go b/pkg/dvo/client_test.go index 68820b1a..888227b6 100644 --- a/pkg/dvo/client_test.go +++ b/pkg/dvo/client_test.go @@ -62,7 +62,7 @@ deployment_validation_operator_total 42 It("returns an error when the server is unreachable", func() { client = &dvoClient{ dvoBaseUrl: "127.0.0.1:1", - httpClient: http.Client{Transport: newDvoTransport()}, + httpClient: http.Client{Transport: dvoTransport()}, } _, err := client.GetMetrics() @@ -86,9 +86,9 @@ deployment_validation_operator_total 42 }) }) - Context("newDvoTransport", func() { + Context("dvoTransport", func() { It("returns a transport with proxy support configured", func() { - transport := newDvoTransport() + transport := dvoTransport() Expect(transport).ToNot(BeNil()) Expect(transport.Proxy).ToNot(BeNil()) Expect(transport.TLSHandshakeTimeout.Seconds()).To(Equal(float64(30)))