From 9840ab809535d3aef9ba5be6da4d860960aae379 Mon Sep 17 00:00:00 2001 From: Christian Jonas Date: Tue, 14 Jul 2026 15:30:59 -0400 Subject: [PATCH 01/13] add leg to coordinator --- .github/workflows/load-test-coordinator.yml | 26 ++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/.github/workflows/load-test-coordinator.yml b/.github/workflows/load-test-coordinator.yml index 5f66aaa50..56511fe6c 100644 --- a/.github/workflows/load-test-coordinator.yml +++ b/.github/workflows/load-test-coordinator.yml @@ -54,10 +54,30 @@ jobs: SHA="$(git rev-parse HEAD)" PR="$(gh pr list --repo "${{ github.repository }}" --state open \ --base main --head "$REF" --json number --jq '.[0].number // ""' 2>/dev/null || true)" + + # go-bench baseline: latest published release, falls back to the highest v* tag. + BASELINE="$(gh api "repos/${{ github.repository }}/releases/latest" --jq .tag_name 2>/dev/null || true)" + if [ -z "$BASELINE" ]; then + BASELINE="$(git ls-remote --tags --refs origin 'v[0-9]*' \ + | sed 's|.*refs/tags/||' | sort -V | tail -n 1 || true)" + fi + [ -n "$BASELINE" ] || echo "::warning::no baseline release found; the go-bench leg will fail" + echo "go-bench baseline: ${BASELINE:-}" + + # Per-leg sizing rides in the roster; ec2-leg.yml receives it verbatim. + LEGS="$(jq -nc --arg baseline "$BASELINE" '[ + {label: "Apply-load ingestion", run_label: "apply-load", + script: "cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/ingest-load-test/run-load-test.sh", + instance_type: "c5.2xlarge", root_volume_gb: 500, budget_minutes: 225, extra_env: ""}, + {label: "Go endpoint benchmarks", run_label: "go-bench", + script: "cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/go-bench/run-go-bench.sh", + instance_type: "c5.2xlarge", root_volume_gb: 50, budget_minutes: 90, + extra_env: ("export BASELINE_REF=" + $baseline)} + ]')" { echo "target_sha=$SHA" echo "pr_number=$PR" - echo 'legs=[{"label":"Apply-load ingestion","run_label":"apply-load","script":"cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/ingest-load-test/run-load-test.sh"}]' + echo "legs=$LEGS" } >> "$GITHUB_OUTPUT" # The fan-out: one matrix entry per leg. @@ -74,6 +94,10 @@ jobs: target_ref: ${{ needs.plan.outputs.target_sha }} run_label: ${{ matrix.leg.run_label }} leg_script: ${{ matrix.leg.script }} + instance_type: ${{ matrix.leg.instance_type }} + root_volume_gb: ${{ matrix.leg.root_volume_gb }} + budget_minutes: ${{ matrix.leg.budget_minutes }} + extra_env: ${{ matrix.leg.extra_env }} report: name: Aggregate + report From 421229e5d9bdbad49c3ebb8e71c250b14d9c68dd Mon Sep 17 00:00:00 2001 From: Christian Jonas Date: Tue, 14 Jul 2026 15:32:02 -0400 Subject: [PATCH 02/13] fix tx test benchmark panic --- cmd/stellar-rpc/internal/db/transaction_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cmd/stellar-rpc/internal/db/transaction_test.go b/cmd/stellar-rpc/internal/db/transaction_test.go index c04ffe239..a22e33f0c 100644 --- a/cmd/stellar-rpc/internal/db/transaction_test.go +++ b/cmd/stellar-rpc/internal/db/transaction_test.go @@ -381,13 +381,14 @@ func BenchmarkTransactionFetch(b *testing.B) { require.NoError(b, write.Commit(lcms[len(lcms)-1], nil)) reader := NewTransactionReader(log, db, passphrase) - randoms := make([]int, b.N) - for i := 0; b.Loop(); i++ { + // fixed-size pool of pre-drawn indices (b.Loop owns the iteration count) + randoms := make([]int, 4096) + for i := range randoms { randoms[i] = rand.Intn(len(lcms)) } for i := 0; b.Loop(); i++ { - r := randoms[i] + r := randoms[i%len(randoms)] tx, err := reader.GetTransaction(ctx, lcms[r].TransactionHash(0)) require.NoError(b, err) assert.Equal(b, r%2 == 0, tx.Successful) From 2c1c969bba879c5be32a2545aaee21c2d35f8d24 Mon Sep 17 00:00:00 2001 From: Christian Jonas Date: Tue, 14 Jul 2026 15:55:57 -0400 Subject: [PATCH 03/13] shorten shell script header comments --- .../infrastructure/perf-eval/backfill-test/run-backfill.sh | 7 +++---- .../perf-eval/ingest-load-test/run-load-test.sh | 7 +++---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/backfill-test/run-backfill.sh b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/backfill-test/run-backfill.sh index e43a5b766..2193e35f8 100644 --- a/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/backfill-test/run-backfill.sh +++ b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/backfill-test/run-backfill.sh @@ -1,7 +1,6 @@ -# Backfill ingestion leg. Concatenated after bootstrap-common.sh in the rendered -# EC2 user-data, so it relies on that file's env, helpers, bootstrap_box, and -# run_leg. It hands off to `runner instantiate`, which builds stellar-rpc and -# times a backfill run against the pubnet datastore. The other half, `runner +# Backfill ingestion leg. Relies on bootstrap-common.sh's env, helpers, +# bootstrap_box, and run_leg. It hands off to `runner instantiate` to build +# stellar-rpc and time a backfill run against the datalake. The other half, `runner # gather`, polls S3 for the result object. LEG_TITLE="Backfill ingestion" diff --git a/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/ingest-load-test/run-load-test.sh b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/ingest-load-test/run-load-test.sh index d08fccdb0..e98347fac 100644 --- a/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/ingest-load-test/run-load-test.sh +++ b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/ingest-load-test/run-load-test.sh @@ -1,7 +1,6 @@ -# Apply-load ingestion leg. Concatenated after bootstrap-common.sh in the -# rendered EC2 user-data, so it relies on that file's env, helpers, bootstrap_box, -# and run_leg. It hands off to the leg runner, which streams the corpus from S3 -# and runs the ingest benchmark. +# Apply-load ingestion leg. Relies on bootstrap-common.sh's env, helpers, +# bootstrap_box, and run_leg. It hands off to the leg runner, which streams +# the corpus from S3 and runs the ingest benchmark. LEG_TITLE="Ingest load test" log "clearing stale apply-load state" From bb93fdc08f305b072f889ed43a3453b3edaa44cd Mon Sep 17 00:00:00 2001 From: Christian Jonas Date: Tue, 14 Jul 2026 16:08:17 -0400 Subject: [PATCH 04/13] add go bench script --- .../infrastructure/perf-eval/go-bench/run-go-bench.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/go-bench/run-go-bench.sh diff --git a/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/go-bench/run-go-bench.sh b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/go-bench/run-go-bench.sh new file mode 100644 index 000000000..23df3bcbb --- /dev/null +++ b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/go-bench/run-go-bench.sh @@ -0,0 +1,11 @@ +# Go endpoint benchmark leg. Relies on bootstrap-common.sh's env, helpers, +# bootstrap_box, and run_leg. The runner benches the candidate checkout against +# BASELINE_REF and publishes a benchstat comparison. +LEG_TITLE="Go endpoint benchmarks" + +log "clearing stale go-bench state" +rm -rf "$WORK_DIR/stellar-rpc-baseline" +rm -f /tmp/baseline.txt /tmp/candidate.txt /tmp/benchstat.txt /tmp/bench-results.json + +bootstrap_box +run_leg ./cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/go-bench/runner From 0928edbfafbd0c6b50162a488c57cb07683be116 Mon Sep 17 00:00:00 2001 From: Christian Jonas Date: Tue, 14 Jul 2026 16:40:16 -0400 Subject: [PATCH 05/13] extract common env getter from leg instantiates --- .../backfill-test/runner/instantiate.go | 28 +++++++--------- .../perf-eval/harness/harness.go | 14 ++++++++ .../ingest-load-test/runner/instantiate.go | 32 ++++++++----------- 3 files changed, 40 insertions(+), 34 deletions(-) diff --git a/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/backfill-test/runner/instantiate.go b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/backfill-test/runner/instantiate.go index 0c71efae4..39d1a0fc9 100644 --- a/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/backfill-test/runner/instantiate.go +++ b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/backfill-test/runner/instantiate.go @@ -34,25 +34,20 @@ var backfillDoneRe = regexp.MustCompile(`Backfill process complete, ledgers \[(\ // runs a timed backfill, then publishes the verdict. func instantiate(ctx context.Context) error { var ( - bucket = harness.Env("BUCKET", "stellar-rpc-ci-load-test") - region = harness.Env("REGION", "us-east-1") - workDir = harness.Env("WORK_DIR", "/data") - resultsFile = harness.Env("RESULTS_FILE", "/tmp/results.md") - resultKey = os.Getenv("RESULT_KEY") - targetSHA = os.Getenv("TARGET_SHA") - runID = harness.Env("RUN_ID", "manual") + env = harness.GetEnv() // ~1 day by default for cheap test runs; the full week is 120960. retention = harness.Env("HISTORY_RETENTION_WINDOW", "17280") deadline = harness.Env("BACKFILL_DEADLINE", "4h") - binaryPath = filepath.Join(workDir, "stellar-rpc-bin") // built here (the repo checkout is in WORK_DIR) + binaryPath = filepath.Join(env["WORK_DIR"], "stellar-rpc-bin") // built here (the repo checkout is in WORK_DIR) ) repoRoot, err := os.Getwd() if err != nil { return err } bail := func(format string, args ...any) error { - return harness.BailInstance(resultsFile, "Backfill ingestion", runID, targetSHA, fmt.Sprintf(format, args...)) + return harness.BailInstance(env["RESULTS_FILE"], "Backfill ingestion", env["RUN_ID"], env["TARGET_SHA"], + fmt.Sprintf(format, args...)) } want, err := strconv.Atoi(retention) // used to compare against ingested ledgers below @@ -60,11 +55,11 @@ func instantiate(ctx context.Context) error { return bail("parsing retention window %q: %v", retention, err) } - awsCfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(region)) + awsCfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(env["REGION"])) if err != nil { return bail("loading AWS config: %v", err) } - fetch := &harness.S3Fetcher{Client: s3.NewFromConfig(awsCfg), Bucket: bucket} + fetch := &harness.S3Fetcher{Client: s3.NewFromConfig(awsCfg), Bucket: env["BUCKET"]} if err := fetch.FetchVerified(ctx, "core/stellar-core.zst", corePath, true, "stellar-core"); err != nil { return bail("%v", err) @@ -83,12 +78,12 @@ func instantiate(ctx context.Context) error { } // fetch + write core config from SDK - coreCfg := filepath.Join(workDir, "captive-core-pubnet.cfg") + coreCfg := filepath.Join(env["WORK_DIR"], "captive-core-pubnet.cfg") if err := os.WriteFile(coreCfg, ledgerbackend.PubnetDefaultConfig, 0o644); err != nil { return bail("writing captive-core config: %v", err) } - cfgPath, err := renderConfig(repoRoot, workDir, coreCfg, retention) + cfgPath, err := renderConfig(repoRoot, env["WORK_DIR"], coreCfg, retention) if err != nil { return bail("rendering config: %v", err) } @@ -108,12 +103,13 @@ func instantiate(ctx context.Context) error { } logger.Infof("backfill complete: %d ledgers [%d -> %d] in %s", ingested, lo, hi, elapsed.Round(time.Second)) - md := renderMarkdown(targetSHA, retention, lo, hi, ingested, elapsed) - if err := os.WriteFile(resultsFile, []byte(md), 0o644); err != nil { + md := renderMarkdown(env["TARGET_SHA"], retention, lo, hi, ingested, elapsed) + if err := os.WriteFile(env["RESULTS_FILE"], []byte(md), 0o644); err != nil { return bail("writing results: %v", err) } if err := harness.PublishResult( - ctx, fetch.Client, bucket, resultKey, "ok", runID, targetSHA, resultsFile, ""); err != nil { + ctx, fetch.Client, env["BUCKET"], env["RESULT_KEY"], "ok", env["RUN_ID"], env["TARGET_SHA"], + env["RESULTS_FILE"], ""); err != nil { return bail("publishing result: %v", err) } return nil diff --git a/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/harness/harness.go b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/harness/harness.go index 13d3c7883..a124ff996 100644 --- a/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/harness/harness.go +++ b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/harness/harness.go @@ -24,6 +24,20 @@ import ( supportlog "github.com/stellar/go-stellar-sdk/support/log" ) +// GetEnv reads the common leg environment. +func GetEnv() map[string]string { + return map[string]string{ + "BUCKET": Env("BUCKET", "stellar-rpc-ci-load-test"), + "REGION": Env("REGION", "us-east-1"), + "WORK_DIR": Env("WORK_DIR", "/data"), + "RESULTS_FILE": Env("RESULTS_FILE", "/tmp/results.md"), + "RESULT_KEY": os.Getenv("RESULT_KEY"), + "TARGET_SHA": os.Getenv("TARGET_SHA"), + "RUN_ID": Env("RUN_ID", "manual"), + "REPO": Env("REPO", "stellar/stellar-rpc"), + } +} + // NewLogger returns an Info-level logger (supportlog.New starts at WARN). Each // leg's runner uses one for its own messages. func NewLogger() *supportlog.Entry { diff --git a/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/ingest-load-test/runner/instantiate.go b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/ingest-load-test/runner/instantiate.go index c1a21b7d6..88f494839 100644 --- a/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/ingest-load-test/runner/instantiate.go +++ b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/ingest-load-test/runner/instantiate.go @@ -26,15 +26,9 @@ var ( // from S3, runs the benchmark, and writes the ok/fail verdict. func instantiate(ctx context.Context) error { var ( - bucket = harness.Env("BUCKET", "stellar-rpc-ci-load-test") - region = harness.Env("REGION", "us-east-1") - workDir = harness.Env("WORK_DIR", "/data") - goldenDB = harness.Env("GOLDEN_DB", filepath.Join(workDir, "golden.sqlite")) - resultsFile = harness.Env("RESULTS_FILE", "/tmp/results.md") + env = harness.GetEnv() + goldenDB = harness.Env("GOLDEN_DB", filepath.Join(env["WORK_DIR"], "golden.sqlite")) benchResults = harness.Env("BENCH_RESULTS", "/tmp/bench-results.json") - resultKey = os.Getenv("RESULT_KEY") - targetSHA = os.Getenv("TARGET_SHA") - runID = harness.Env("RUN_ID", "manual") ) repoRoot, err := os.Getwd() @@ -42,14 +36,15 @@ func instantiate(ctx context.Context) error { return err } bail := func(format string, args ...any) error { - return harness.BailInstance(resultsFile, "Ingest load test", runID, targetSHA, fmt.Sprintf(format, args...)) + return harness.BailInstance(env["RESULTS_FILE"], "Ingest load test", env["RUN_ID"], env["TARGET_SHA"], + fmt.Sprintf(format, args...)) } - awsCfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(region)) + awsCfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(env["REGION"])) if err != nil { return bail("loading AWS config: %v", err) } - fetch := &harness.S3Fetcher{Client: s3.NewFromConfig(awsCfg), Bucket: bucket} + fetch := &harness.S3Fetcher{Client: s3.NewFromConfig(awsCfg), Bucket: env["BUCKET"]} bundlePaths, goldenFetchSecs, err := fetchCorpus(ctx, fetch, goldenDB) if err != nil { @@ -69,10 +64,10 @@ func instantiate(ctx context.Context) error { "LOADTEST_INGEST_DEADLINE=" + harness.Env("LOADTEST_INGEST_DEADLINE", "150m"), "LOADTEST_SQLITE_PATH=" + goldenDB, "PERF_RESULTS_PATH=" + benchResults, - "PERF_RESULTS_MD_PATH=" + resultsFile, - "PERF_TARGET_SHA=" + targetSHA, - "PERF_RUN_ID=" + runID, - "PERF_REPO=" + harness.Env("REPO", "stellar/stellar-rpc"), + "PERF_RESULTS_MD_PATH=" + env["RESULTS_FILE"], + "PERF_TARGET_SHA=" + env["TARGET_SHA"], + "PERF_RUN_ID=" + env["RUN_ID"], + "PERF_REPO=" + env["REPO"], fmt.Sprintf("PERF_GOLDEN_FETCH_SECONDS=%d", goldenFetchSecs), "STELLAR_RPC_INTEGRATION_TESTS_ENABLED=true", } @@ -82,12 +77,13 @@ func instantiate(ctx context.Context) error { return bail("benchmark failed:\n%v", err) } - if fi, err := os.Stat(resultsFile); err != nil || fi.Size() == 0 { - return bail("benchmark succeeded but did not emit %s", resultsFile) + if fi, err := os.Stat(env["RESULTS_FILE"]); err != nil || fi.Size() == 0 { + return bail("benchmark succeeded but did not emit %s", env["RESULTS_FILE"]) } logger.Infof("results ready; publishing verdict") if err := harness.PublishResult( - ctx, fetch.Client, bucket, resultKey, "ok", runID, targetSHA, resultsFile, benchResults, + ctx, fetch.Client, env["BUCKET"], env["RESULT_KEY"], "ok", env["RUN_ID"], env["TARGET_SHA"], + env["RESULTS_FILE"], benchResults, ); err != nil { return bail("publishing result: %v", err) } From 3e894576de9e4326eaca8331bed7bc2f319a2cf4 Mon Sep 17 00:00:00 2001 From: Christian Jonas Date: Tue, 14 Jul 2026 17:00:39 -0400 Subject: [PATCH 06/13] inline RequireEnv into gather.go --- .../perf-eval/harness/gather.go | 48 +++++++++---------- .../perf-eval/harness/harness.go | 15 ------ .../ingest-load-test/runner/instantiate.go | 3 +- 3 files changed, 26 insertions(+), 40 deletions(-) diff --git a/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/harness/gather.go b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/harness/gather.go index 759e44b28..9f89da9f8 100644 --- a/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/harness/gather.go +++ b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/harness/gather.go @@ -22,33 +22,33 @@ const commandWaitTimeout = 60 * time.Second // and relays the result as step outputs. On timeout it writes a debug comment // instead. Used by every leg's runner. func Gather(ctx context.Context) error { - vals, err := RequireEnv("INSTANCE_ID", "AWS_REGION", - "RESULTS_TIMEOUT", "POLL_INTERVAL", "GITHUB_OUTPUT", "DEBUG_LOG_LINES", "DEBUG_LOG_EVERY_POLLS", - "BUCKET", "RESULT_KEY", "RUN_ID") - if err != nil { - return err - } - instanceID, region, githubOutput := vals[0], vals[1], vals[4] - bucket, resultKey, runID := vals[7], vals[8], vals[9] - - resultsTimeoutSec, err := strconv.Atoi(vals[2]) - if err != nil { - return fmt.Errorf("RESULTS_TIMEOUT: %w", err) - } - pollIntervalSec, err := strconv.Atoi(vals[3]) - if err != nil { - return fmt.Errorf("POLL_INTERVAL: %w", err) + envStr := map[string]string{} + var missing []string + for _, k := range []string{ + "INSTANCE_ID", "AWS_REGION", "RESULTS_TIMEOUT", "POLL_INTERVAL", "GITHUB_OUTPUT", + "DEBUG_LOG_LINES", "DEBUG_LOG_EVERY_POLLS", "BUCKET", "RESULT_KEY", "RUN_ID", + } { + if envStr[k] = os.Getenv(k); envStr[k] == "" { + missing = append(missing, k) + } } - debugLogLines, err := strconv.Atoi(vals[5]) - if err != nil { - return fmt.Errorf("DEBUG_LOG_LINES: %w", err) + if len(missing) > 0 { + return fmt.Errorf("missing required env: %s", strings.Join(missing, ", ")) } - debugEveryPolls, err := strconv.Atoi(vals[6]) - if err != nil { - return fmt.Errorf("DEBUG_LOG_EVERY_POLLS: %w", err) + instanceID, region, githubOutput := envStr["INSTANCE_ID"], envStr["AWS_REGION"], envStr["GITHUB_OUTPUT"] + bucket, resultKey, runID := envStr["BUCKET"], envStr["RESULT_KEY"], envStr["RUN_ID"] + + envInt := map[string]int{} + for _, k := range []string{"RESULTS_TIMEOUT", "POLL_INTERVAL", "DEBUG_LOG_LINES", "DEBUG_LOG_EVERY_POLLS"} { + n, err := strconv.Atoi(envStr[k]) + if err != nil { + return fmt.Errorf("%s: %w", k, err) + } + envInt[k] = n } - resultsTimeout := time.Duration(resultsTimeoutSec) * time.Second - pollInterval := time.Duration(pollIntervalSec) * time.Second + debugLogLines, debugEveryPolls := envInt["DEBUG_LOG_LINES"], envInt["DEBUG_LOG_EVERY_POLLS"] + resultsTimeout := time.Duration(envInt["RESULTS_TIMEOUT"]) * time.Second + pollInterval := time.Duration(envInt["POLL_INTERVAL"]) * time.Second awsCfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(region)) if err != nil { diff --git a/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/harness/harness.go b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/harness/harness.go index a124ff996..134848982 100644 --- a/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/harness/harness.go +++ b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/harness/harness.go @@ -64,21 +64,6 @@ func Env(key, def string) string { return def } -// RequireEnv returns the values of keys in order, erroring with every unset one. -func RequireEnv(keys ...string) ([]string, error) { - vals := make([]string, len(keys)) - var missing []string - for i, k := range keys { - if vals[i] = os.Getenv(k); vals[i] == "" { - missing = append(missing, k) - } - } - if len(missing) > 0 { - return nil, fmt.Errorf("missing required env: %s", strings.Join(missing, ", ")) - } - return vals, nil -} - // appendOutputs appends lines to the GitHub Actions step-output file. func appendOutputs(path string, lines ...string) error { f, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0o644) diff --git a/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/ingest-load-test/runner/instantiate.go b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/ingest-load-test/runner/instantiate.go index 88f494839..9cb03fbce 100644 --- a/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/ingest-load-test/runner/instantiate.go +++ b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/ingest-load-test/runner/instantiate.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path/filepath" + "strconv" "strings" "time" @@ -68,7 +69,7 @@ func instantiate(ctx context.Context) error { "PERF_TARGET_SHA=" + env["TARGET_SHA"], "PERF_RUN_ID=" + env["RUN_ID"], "PERF_REPO=" + env["REPO"], - fmt.Sprintf("PERF_GOLDEN_FETCH_SECONDS=%d", goldenFetchSecs), + "PERF_GOLDEN_FETCH_SECONDS=" + strconv.Itoa(goldenFetchSecs), "STELLAR_RPC_INTEGRATION_TESTS_ENABLED=true", } if err := harness.RunStreaming(ctx, repoRoot, benchEnv, 80, From bf012088a7409ae6c9b01519b137d02a71af1616 Mon Sep 17 00:00:00 2001 From: Christian Jonas Date: Wed, 15 Jul 2026 12:24:50 -0400 Subject: [PATCH 07/13] add bench runner package --- .../perf-eval/go-bench/runner/instantiate.go | 276 ++++++++++++++++++ .../perf-eval/go-bench/runner/main.go | 10 + 2 files changed, 286 insertions(+) create mode 100644 cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/go-bench/runner/instantiate.go create mode 100644 cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/go-bench/runner/main.go diff --git a/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/go-bench/runner/instantiate.go b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/go-bench/runner/instantiate.go new file mode 100644 index 000000000..678aa4c66 --- /dev/null +++ b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/go-bench/runner/instantiate.go @@ -0,0 +1,276 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "os" + "os/exec" + "path" + "path/filepath" + "strconv" + "strings" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" + + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/harness" +) + +const legTitle = "Go endpoint benchmarks" + +// auditedBenches is the roster of benches for this test (excludes benchmarks +// that are integration tests). +var auditedBenches = []struct { + pkg string + benches []string +}{ + {"cmd/stellar-rpc/internal/methods", []string{ + "BenchmarkGetLedgers", "BenchmarkGetEventsTopicFilters", "BenchmarkGetEvents", + "BenchmarkJSONTransactions", "BenchmarkGetProtocolVersion", + }}, + {"cmd/stellar-rpc/internal/preflight", []string{"BenchmarkGetPreflight"}}, + {"cmd/stellar-rpc/internal/feewindow", []string{"BenchmarkComputeFeeDistribution"}}, + {"cmd/stellar-rpc/internal/db", []string{ + "BenchmarkGetLedgerRange", "BenchmarkBatchGetLedgers", "BenchmarkTransactionFetch", + }}, +} + +// instantiate is the instance half after the bootstrap, which runs the benches +// and writes the results file to be published to S3. +func instantiate(ctx context.Context) error { + var ( + env = harness.GetEnv() + baselineRef = os.Getenv("BASELINE_REF") + countStr = harness.Env("BENCH_COUNT", "10") + + baselineOut = "/tmp/baseline.txt" + candidateOut = "/tmp/candidate.txt" + benchstatOut = "/tmp/benchstat.txt" + benchResults = "/tmp/bench-results.json" + ) + + repoRoot, err := os.Getwd() + if err != nil { + return err + } + bail := func(format string, args ...any) error { + return harness.BailInstance(env["RESULTS_FILE"], legTitle, env["RUN_ID"], env["TARGET_SHA"], + fmt.Sprintf(format, args...)) + } + + count, err := strconv.Atoi(countStr) + if err != nil || count < 1 { + return bail("invalid BENCH_COUNT %q", countStr) + } + if baselineRef == "" { + return bail("BASELINE_REF unset; the coordinator resolves it from the latest release") + } + + awsCfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(env["REGION"])) + if err != nil { + return bail("loading AWS config: %v", err) + } + s3Client := s3.NewFromConfig(awsCfg) + + baselineDir := filepath.Join(env["WORK_DIR"], "stellar-rpc-baseline") + logger.Infof("checking out baseline %s into %s", baselineRef, baselineDir) + baselineSHA, err := checkoutBaseline(ctx, baselineDir, env["REPO"], baselineRef) + if err != nil { + return bail("checking out baseline %s: %v", baselineRef, err) + } + + for _, dir := range []string{baselineDir, repoRoot} { + logger.Infof("building rpc libs in %s", dir) + if err := harness.RunStreaming(ctx, dir, nil, 40, "make", "build-libs"); err != nil { + return bail("make build-libs failed in %s: %v", dir, err) + } + } + + logger.Infof("running audited benchmarks (count=%d) on baseline %s", count, baselineRef) + baselineFails := runSuite(ctx, baselineDir, baselineOut, count) + logger.Infof("running audited benchmarks (count=%d) on candidate %s", count, env["TARGET_SHA"]) + candidateFails := runSuite(ctx, repoRoot, candidateOut, count) + + logger.Infof("comparing with benchstat") + if err := runBenchstat(ctx, baselineOut, candidateOut, benchstatOut); err != nil { + return bail("benchstat failed: %v", err) + } + benchstat, err := os.ReadFile(benchstatOut) + if err != nil { + return bail("reading benchstat output: %v", err) + } + + uploadRawLogs(ctx, s3Client, env["BUCKET"], env["RESULT_KEY"], + map[string]string{"baseline.txt": baselineOut, "candidate.txt": candidateOut, "benchstat.txt": benchstatOut}) + + report := benchReport{ + BaselineRef: baselineRef, + BaselineSHA: baselineSHA, + TargetSHA: env["TARGET_SHA"], + Count: count, + Benchstat: string(benchstat), + BaselineFails: baselineFails, + CandidateFails: candidateFails, + RawLogsPrefix: "s3://" + env["BUCKET"] + "/" + path.Dir(env["RESULT_KEY"]) + "/", + } + if err := os.WriteFile(env["RESULTS_FILE"], []byte(renderMarkdown(report)), 0o644); err != nil { + return bail("writing results markdown: %v", err) + } + // The shell wrapper publishes the fail result from resultsFile on non-zero exit. + if len(candidateFails) > 0 { + return fmt.Errorf("candidate benchmarks failed in %s", strings.Join(candidateFails, ", ")) + } + + logger.Infof("results ready; publishing verdict") + if err := publishOK(ctx, s3Client, report, env, benchResults); err != nil { + return bail("publishing result: %v", err) + } + return nil +} + +// publishOK writes the bench metadata and publishes the ok result object. +func publishOK( + ctx context.Context, client *s3.Client, r benchReport, env map[string]string, benchResults string, +) error { + meta, err := json.Marshal(r) + if err != nil { + return fmt.Errorf("marshaling bench metadata: %w", err) + } + if err := os.WriteFile(benchResults, meta, 0o644); err != nil { + return fmt.Errorf("writing bench metadata: %w", err) + } + return harness.PublishResult( + ctx, client, env["BUCKET"], env["RESULT_KEY"], "ok", env["RUN_ID"], r.TargetSHA, env["RESULTS_FILE"], benchResults) +} + +// checkoutBaseline shallow-fetches ref (tag, branch, or SHA) from repo into dir +// and returns the checked-out SHA. +func checkoutBaseline(ctx context.Context, dir, repo, ref string) (string, error) { + if err := os.MkdirAll(dir, 0o755); err != nil { + return "", err + } + for _, args := range [][]string{ + {"init", "-q"}, + {"remote", "add", "origin", "https://github.com/" + repo + ".git"}, + {"fetch", "--depth", "1", "origin", ref}, + {"checkout", "--detach", "FETCH_HEAD"}, + } { + if err := harness.RunStreaming(ctx, dir, nil, 20, "git", args...); err != nil { + return "", fmt.Errorf("git %s: %w", strings.Join(args, " "), err) + } + } + out, err := exec.CommandContext(ctx, "git", "-C", dir, "rev-parse", "HEAD").Output() + if err != nil { + return "", fmt.Errorf("git rev-parse: %w", err) + } + return strings.TrimSpace(string(out)), nil +} + +// runSuite runs the audited benchmarks of each roster package in dir with +// identical flags, teeing each package's stdout to outFile while streaming to +// the box log. Returns the packages that failed. +func runSuite(ctx context.Context, dir, outFile string, count int) []string { + var failed []string + for _, entry := range auditedBenches { + // Open per iteration (append) so a defer isn't held across the whole loop. + f, err := os.OpenFile(outFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + logger.Warnf("opening %s for %s: %v", outFile, entry.pkg, err) + failed = append(failed, entry.pkg) + continue + } + cmd := exec.CommandContext(ctx, "go", + "test", "-run", "^$", + "-bench", "^("+strings.Join(entry.benches, "|")+")$", + "-benchmem", "-count", strconv.Itoa(count), "-timeout", "30m", + "./"+entry.pkg+"/") + cmd.Dir = dir + cmd.Stdout = io.MultiWriter(f, os.Stderr) + cmd.Stderr = os.Stderr + err = cmd.Run() + f.Close() + if err != nil { + logger.Warnf("bench run failed for %s in %s: %v", entry.pkg, dir, err) + failed = append(failed, entry.pkg) + } + } + return failed +} + +// runBenchstat compares the two bench outputs into outFile, teeing benchstat's +// stdout there while streaming to the box log. +func runBenchstat(ctx context.Context, baselineOut, candidateOut, outFile string) error { + f, err := os.OpenFile(outFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + return err + } + defer f.Close() + cmd := exec.CommandContext(ctx, "go", "run", "golang.org/x/perf/cmd/benchstat@latest", + filepath.Base(baselineOut), filepath.Base(candidateOut)) + cmd.Dir = filepath.Dir(baselineOut) + cmd.Stdout = io.MultiWriter(f, os.Stderr) + cmd.Stderr = os.Stderr + return cmd.Run() +} + +// uploadRawLogs best-effort copies the raw bench outputs next to the result +// object, so the comment can stay a summary. +func uploadRawLogs(ctx context.Context, client *s3.Client, bucket, resultKey string, files map[string]string) { + if resultKey == "" { + return + } + prefix := path.Dir(resultKey) + for name, p := range files { + body, err := os.ReadFile(p) + if err != nil { + logger.Warnf("skipping raw log upload of %s: %v", p, err) + continue + } + key := prefix + "/" + name + if _, err := client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &key, + Body: bytes.NewReader(body), + ContentType: aws.String("text/plain"), + }); err != nil { + logger.Warnf("uploading s3://%s/%s: %v", bucket, key, err) + } + } +} + +// benchReport is everything the comparison markdown is rendered from. +type benchReport struct { + BaselineRef string `json:"baselineRef"` + BaselineSHA string `json:"baselineSha"` + TargetSHA string `json:"targetSha"` + Count int `json:"count"` + BaselineFails []string `json:"baselineFails,omitempty"` // roster packages whose baseline bench run failed + CandidateFails []string `json:"candidateFails,omitempty"` // roster packages whose candidate bench run failed + Benchstat string `json:"-"` + RawLogsPrefix string `json:"-"` // s3:// prefix holding the raw logs, "" on local runs +} + +// renderMarkdown renders the leg's comment section: the refs compared + flags +// + any per-package failures + the benchstat output in a drop-down. +func renderMarkdown(r benchReport) string { + var b strings.Builder + fmt.Fprintf(&b, "**Baseline** `%s` (`%s`) vs **candidate** `%s` ā€” `-benchmem -count=%d`, "+ + "both refs sequentially on one box.\n", + r.BaselineRef, r.BaselineSHA[:min(12, len(r.BaselineSHA))], r.TargetSHA[:min(12, len(r.TargetSHA))], r.Count) + for _, pkg := range r.CandidateFails { + fmt.Fprintf(&b, "\nāŒ Candidate bench run failed in `%s`; see the box log.\n", pkg) + } + for _, pkg := range r.BaselineFails { + fmt.Fprintf(&b, "\nāŒ Baseline bench run failed in `%s`; its rows lack a base column.\n", pkg) + } + fmt.Fprintf(&b, "\n
\nbenchstat: baseline vs candidate\n\n```\n%s\n```\n\n
\n", + strings.TrimRight(r.Benchstat, "\n")) + if r.RawLogsPrefix != "" { + fmt.Fprintf(&b, "\nRaw benchmark logs: `%s`\n", r.RawLogsPrefix) + } + return b.String() +} diff --git a/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/go-bench/runner/main.go b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/go-bench/runner/main.go new file mode 100644 index 000000000..6deb81cc8 --- /dev/null +++ b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/go-bench/runner/main.go @@ -0,0 +1,10 @@ +// Command runner runs the audited endpoint Go benchmarks on the box for the +// release candidate and a baseline release, and publishes a benchstat +// comparison of the two. +package main + +import "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/harness" + +var logger = harness.NewLogger() + +func main() { harness.Run(instantiate) } From 1e6560c12a4778d7c78d80d21fe7498b005e26e0 Mon Sep 17 00:00:00 2001 From: Christian Jonas Date: Fri, 17 Jul 2026 14:47:10 -0400 Subject: [PATCH 08/13] patch minor uploadRawLogs bug --- .github/workflows/load-test-coordinator.yml | 2 +- .../perf-eval/go-bench/runner/instantiate.go | 31 ++++++++++++++----- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/.github/workflows/load-test-coordinator.yml b/.github/workflows/load-test-coordinator.yml index f20137f6b..4355f3378 100644 --- a/.github/workflows/load-test-coordinator.yml +++ b/.github/workflows/load-test-coordinator.yml @@ -11,7 +11,7 @@ defaults: on: push: - branches: [release/**] + branches: [release/**, go-bench-test] workflow_dispatch: inputs: target_ref: diff --git a/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/go-bench/runner/instantiate.go b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/go-bench/runner/instantiate.go index 678aa4c66..a9f0bc353 100644 --- a/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/go-bench/runner/instantiate.go +++ b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/go-bench/runner/instantiate.go @@ -10,6 +10,7 @@ import ( "os/exec" "path" "path/filepath" + "sort" "strconv" "strings" @@ -104,9 +105,13 @@ func instantiate(ctx context.Context) error { return bail("reading benchstat output: %v", err) } - uploadRawLogs(ctx, s3Client, env["BUCKET"], env["RESULT_KEY"], + uploaded := uploadRawLogs(ctx, s3Client, env["BUCKET"], env["RESULT_KEY"], map[string]string{"baseline.txt": baselineOut, "candidate.txt": candidateOut, "benchstat.txt": benchstatOut}) + var rawLogsPrefix string + if len(uploaded) > 0 { + rawLogsPrefix = "s3://" + env["BUCKET"] + "/" + path.Dir(env["RESULT_KEY"]) + "/" + } report := benchReport{ BaselineRef: baselineRef, BaselineSHA: baselineSHA, @@ -115,7 +120,8 @@ func instantiate(ctx context.Context) error { Benchstat: string(benchstat), BaselineFails: baselineFails, CandidateFails: candidateFails, - RawLogsPrefix: "s3://" + env["BUCKET"] + "/" + path.Dir(env["RESULT_KEY"]) + "/", + RawLogsPrefix: rawLogsPrefix, + RawLogs: uploaded, } if err := os.WriteFile(env["RESULTS_FILE"], []byte(renderMarkdown(report)), 0o644); err != nil { return bail("writing results markdown: %v", err) @@ -218,11 +224,15 @@ func runBenchstat(ctx context.Context, baselineOut, candidateOut, outFile string } // uploadRawLogs best-effort copies the raw bench outputs next to the result -// object, so the comment can stay a summary. -func uploadRawLogs(ctx context.Context, client *s3.Client, bucket, resultKey string, files map[string]string) { +// object, so the comment can stay a summary. Returns the names of the files +// that actually landed sorted for a stable comment. +func uploadRawLogs( + ctx context.Context, client *s3.Client, bucket, resultKey string, files map[string]string, +) []string { if resultKey == "" { - return + return nil } + var uploaded []string prefix := path.Dir(resultKey) for name, p := range files { body, err := os.ReadFile(p) @@ -238,8 +248,12 @@ func uploadRawLogs(ctx context.Context, client *s3.Client, bucket, resultKey str ContentType: aws.String("text/plain"), }); err != nil { logger.Warnf("uploading s3://%s/%s: %v", bucket, key, err) + continue } + uploaded = append(uploaded, name) } + sort.Strings(uploaded) + return uploaded } // benchReport is everything the comparison markdown is rendered from. @@ -251,7 +265,8 @@ type benchReport struct { BaselineFails []string `json:"baselineFails,omitempty"` // roster packages whose baseline bench run failed CandidateFails []string `json:"candidateFails,omitempty"` // roster packages whose candidate bench run failed Benchstat string `json:"-"` - RawLogsPrefix string `json:"-"` // s3:// prefix holding the raw logs, "" on local runs + RawLogsPrefix string `json:"-"` // s3:// prefix holding the raw logs, "" when none uploaded + RawLogs []string `json:"-"` // names of the raw logs that actually uploaded } // renderMarkdown renders the leg's comment section: the refs compared + flags @@ -269,8 +284,8 @@ func renderMarkdown(r benchReport) string { } fmt.Fprintf(&b, "\n
\nbenchstat: baseline vs candidate\n\n```\n%s\n```\n\n
\n", strings.TrimRight(r.Benchstat, "\n")) - if r.RawLogsPrefix != "" { - fmt.Fprintf(&b, "\nRaw benchmark logs: `%s`\n", r.RawLogsPrefix) + if len(r.RawLogs) > 0 { + fmt.Fprintf(&b, "\nRaw benchmark logs (`%s`): %s\n", r.RawLogsPrefix, strings.Join(r.RawLogs, ", ")) } return b.String() } From e4f105124fab82852feb4854b0c730a95c4f8520 Mon Sep 17 00:00:00 2001 From: Christian Jonas Date: Fri, 17 Jul 2026 18:15:59 -0400 Subject: [PATCH 09/13] use deny list instead of include list for test names --- .../perf-eval/go-bench/runner/instantiate.go | 82 +++++++++---------- 1 file changed, 38 insertions(+), 44 deletions(-) diff --git a/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/go-bench/runner/instantiate.go b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/go-bench/runner/instantiate.go index a9f0bc353..0afc2d6fd 100644 --- a/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/go-bench/runner/instantiate.go +++ b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/go-bench/runner/instantiate.go @@ -23,21 +23,9 @@ import ( const legTitle = "Go endpoint benchmarks" -// auditedBenches is the roster of benches for this test (excludes benchmarks -// that are integration tests). -var auditedBenches = []struct { - pkg string - benches []string -}{ - {"cmd/stellar-rpc/internal/methods", []string{ - "BenchmarkGetLedgers", "BenchmarkGetEventsTopicFilters", "BenchmarkGetEvents", - "BenchmarkJSONTransactions", "BenchmarkGetProtocolVersion", - }}, - {"cmd/stellar-rpc/internal/preflight", []string{"BenchmarkGetPreflight"}}, - {"cmd/stellar-rpc/internal/feewindow", []string{"BenchmarkComputeFeeDistribution"}}, - {"cmd/stellar-rpc/internal/db", []string{ - "BenchmarkGetLedgerRange", "BenchmarkBatchGetLedgers", "BenchmarkTransactionFetch", - }}, +// benchDenylist is the set of benchmarks to exclude from the suite. +var benchDenylist = []string{ + "BenchmarkGetLedgerEntries", // is an integration test } // instantiate is the instance half after the bootstrap, which runs the benches @@ -176,35 +164,42 @@ func checkoutBaseline(ctx context.Context, dir, repo, ref string) (string, error return strings.TrimSpace(string(out)), nil } -// runSuite runs the audited benchmarks of each roster package in dir with -// identical flags, teeing each package's stdout to outFile while streaming to -// the box log. Returns the packages that failed. +// runSuite runs every benchmark in the module in dir except benchDenylist. +// Returns the packages go test reported as failed (empty on success). func runSuite(ctx context.Context, dir, outFile string, count int) []string { - var failed []string - for _, entry := range auditedBenches { - // Open per iteration (append) so a defer isn't held across the whole loop. - f, err := os.OpenFile(outFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) - if err != nil { - logger.Warnf("opening %s for %s: %v", outFile, entry.pkg, err) - failed = append(failed, entry.pkg) - continue + f, err := os.OpenFile(outFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + logger.Warnf("opening %s: %v", outFile, err) + return []string{""} + } + defer f.Close() + var buf bytes.Buffer + cmd := exec.CommandContext(ctx, "go", "test", "-run", "^$", "-bench", ".", + "-skip", "^("+strings.Join(benchDenylist, "|")+")$", + "-benchmem", "-count", strconv.Itoa(count), "-timeout", "30m", + "./...") + cmd.Dir = dir + cmd.Stdout, cmd.Stderr = io.MultiWriter(f, &buf, os.Stderr), os.Stderr + if err := cmd.Run(); err != nil { + failed := parseFailedPkgs(buf.String()) + if len(failed) == 0 { + failed = []string{""} } - cmd := exec.CommandContext(ctx, "go", - "test", "-run", "^$", - "-bench", "^("+strings.Join(entry.benches, "|")+")$", - "-benchmem", "-count", strconv.Itoa(count), "-timeout", "30m", - "./"+entry.pkg+"/") - cmd.Dir = dir - cmd.Stdout = io.MultiWriter(f, os.Stderr) - cmd.Stderr = os.Stderr - err = cmd.Run() - f.Close() - if err != nil { - logger.Warnf("bench run failed for %s in %s: %v", entry.pkg, dir, err) - failed = append(failed, entry.pkg) + logger.Warnf("bench run failed in %s: %v (packages: %s)", dir, err, strings.Join(failed, ", ")) + return failed + } + return nil +} + +// parseFailedPkgs allows us to see which packages failed in the bench suite. +func parseFailedPkgs(out string) []string { + var pkgs []string + for line := range strings.SplitSeq(out, "\n") { + if fields := strings.Fields(line); len(fields) >= 2 && fields[0] == "FAIL" { + pkgs = append(pkgs, fields[1]) } } - return failed + return pkgs } // runBenchstat compares the two bench outputs into outFile, teeing benchstat's @@ -218,8 +213,7 @@ func runBenchstat(ctx context.Context, baselineOut, candidateOut, outFile string cmd := exec.CommandContext(ctx, "go", "run", "golang.org/x/perf/cmd/benchstat@latest", filepath.Base(baselineOut), filepath.Base(candidateOut)) cmd.Dir = filepath.Dir(baselineOut) - cmd.Stdout = io.MultiWriter(f, os.Stderr) - cmd.Stderr = os.Stderr + cmd.Stdout, cmd.Stderr = io.MultiWriter(f, os.Stderr), os.Stderr return cmd.Run() } @@ -262,8 +256,8 @@ type benchReport struct { BaselineSHA string `json:"baselineSha"` TargetSHA string `json:"targetSha"` Count int `json:"count"` - BaselineFails []string `json:"baselineFails,omitempty"` // roster packages whose baseline bench run failed - CandidateFails []string `json:"candidateFails,omitempty"` // roster packages whose candidate bench run failed + BaselineFails []string `json:"baselineFails,omitempty"` // packages whose baseline bench run failed + CandidateFails []string `json:"candidateFails,omitempty"` // packages whose candidate bench run failed Benchstat string `json:"-"` RawLogsPrefix string `json:"-"` // s3:// prefix holding the raw logs, "" when none uploaded RawLogs []string `json:"-"` // names of the raw logs that actually uploaded From 00b3733b8796132b80a21e04073474dfad68c4be Mon Sep 17 00:00:00 2001 From: Christian Jonas Date: Mon, 20 Jul 2026 14:41:08 -0400 Subject: [PATCH 10/13] fix s3 upload bug for bench test --- .github/workflows/load-test-coordinator.yml | 2 +- .../perf-eval/bootstrap-common.sh | 5 ++-- .../perf-eval/go-bench/runner/instantiate.go | 23 +++++++++++++------ 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/.github/workflows/load-test-coordinator.yml b/.github/workflows/load-test-coordinator.yml index 4355f3378..3e10ccb53 100644 --- a/.github/workflows/load-test-coordinator.yml +++ b/.github/workflows/load-test-coordinator.yml @@ -77,7 +77,7 @@ jobs: [ {"label":"Apply-load ingestion","run_label":"apply-load","script":"cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/ingest-load-test/run-load-test.sh","budget":225}, {"label":"Backfill ingestion","run_label":"backfill","script":"cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/backfill-test/run-backfill.sh","budget":345}, - {"label":"Go endpoint benchmarks","run_label":"go-bench","script":"cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/go-bench/run-go-bench.sh","budget":90,"root_volume_gb":50,"extra_env":"export BASELINE_REF=$BASELINE"} + {"label":"Go endpoint benchmarks","run_label":"go-bench","script":"cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/go-bench/run-go-bench.sh","budget":120,"root_volume_gb":50,"extra_env":"export BASELINE_REF=$BASELINE"} ] EOF )" diff --git a/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/bootstrap-common.sh b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/bootstrap-common.sh index 3dac90f5b..b3bbc3c4c 100644 --- a/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/bootstrap-common.sh +++ b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/bootstrap-common.sh @@ -64,12 +64,13 @@ bail() { } trap 'bail "unhandled error at line $LINENO while running: $BASH_COMMAND"' ERR -# persists the full box log to S3 even if instance terminated +# persists the full box log to S3 even if instance terminated. TERM is trapped +# too: a hard terminate mid-run (leg timeout) never reaches the EXIT trap. upload_box_log() { [ -n "$BUCKET" ] && [ -n "$RESULT_KEY" ] || return 0 aws s3 cp /var/log/user-data.log "s3://$BUCKET/${RESULT_KEY%/*}/user-data.log" >/dev/null 2>&1 || true } -trap upload_box_log EXIT +trap upload_box_log EXIT TERM # bootstrap_box installs the build toolchain and checks out TARGET_SHA into # $WORK_DIR/stellar-rpc, leaving the shell cd'd at the repo root. Generic across diff --git a/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/go-bench/runner/instantiate.go b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/go-bench/runner/instantiate.go index 0afc2d6fd..dac984147 100644 --- a/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/go-bench/runner/instantiate.go +++ b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/go-bench/runner/instantiate.go @@ -1,6 +1,7 @@ package main import ( + "bufio" "bytes" "context" "encoding/json" @@ -165,6 +166,7 @@ func checkoutBaseline(ctx context.Context, dir, repo, ref string) (string, error } // runSuite runs every benchmark in the module in dir except benchDenylist. +// Only stderr (tool/compile errors, low-volume) streams to the log. // Returns the packages go test reported as failed (empty on success). func runSuite(ctx context.Context, dir, outFile string, count int) []string { f, err := os.OpenFile(outFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) @@ -173,15 +175,14 @@ func runSuite(ctx context.Context, dir, outFile string, count int) []string { return []string{""} } defer f.Close() - var buf bytes.Buffer cmd := exec.CommandContext(ctx, "go", "test", "-run", "^$", "-bench", ".", "-skip", "^("+strings.Join(benchDenylist, "|")+")$", "-benchmem", "-count", strconv.Itoa(count), "-timeout", "30m", "./...") cmd.Dir = dir - cmd.Stdout, cmd.Stderr = io.MultiWriter(f, &buf, os.Stderr), os.Stderr + cmd.Stdout, cmd.Stderr = f, os.Stderr if err := cmd.Run(); err != nil { - failed := parseFailedPkgs(buf.String()) + failed := parseFailedPkgs(outFile) if len(failed) == 0 { failed = []string{""} } @@ -191,11 +192,19 @@ func runSuite(ctx context.Context, dir, outFile string, count int) []string { return nil } -// parseFailedPkgs allows us to see which packages failed in the bench suite. -func parseFailedPkgs(out string) []string { +// parseFailedPkgs scans a bench output file for go test's FAIL lines. +func parseFailedPkgs(outFile string) []string { + f, err := os.Open(outFile) + if err != nil { + logger.Warnf("reading %s for FAIL lines: %v", outFile, err) + return nil + } + defer f.Close() var pkgs []string - for line := range strings.SplitSeq(out, "\n") { - if fields := strings.Fields(line); len(fields) >= 2 && fields[0] == "FAIL" { + sc := bufio.NewScanner(f) + sc.Buffer(make([]byte, 64*1024), 1024*1024) + for sc.Scan() { + if fields := strings.Fields(sc.Text()); len(fields) >= 2 && fields[0] == "FAIL" { pkgs = append(pkgs, fields[1]) } } From eac6a5f57d511c36881ffcc22ecffa48557b6781 Mon Sep 17 00:00:00 2001 From: Christian Jonas Date: Mon, 20 Jul 2026 17:56:14 -0400 Subject: [PATCH 11/13] stop testing BenchmarkTransactionFetch --- .github/workflows/load-test-coordinator.yml | 6 +++--- .../infrastructure/perf-eval/go-bench/runner/instantiate.go | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/load-test-coordinator.yml b/.github/workflows/load-test-coordinator.yml index 3e10ccb53..a3bb22fc9 100644 --- a/.github/workflows/load-test-coordinator.yml +++ b/.github/workflows/load-test-coordinator.yml @@ -72,11 +72,11 @@ jobs: echo "go-bench baseline: ${BASELINE:-}" # Unquoted heredoc so $BASELINE expands into the go-bench entry; - # jq -c compacts and validates. + # Temporarily disabled while iterating on go-bench (restore before merge): + # {"label":"Apply-load ingestion","run_label":"apply-load","script":"cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/ingest-load-test/run-load-test.sh","budget":225}, + # {"label":"Backfill ingestion","run_label":"backfill","script":"cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/backfill-test/run-backfill.sh","budget":345}, LEGS="$(jq -c . < Date: Mon, 27 Jul 2026 14:27:51 -0400 Subject: [PATCH 12/13] remove testing/scaffolding changes --- .github/workflows/load-test-coordinator.yml | 7 +++---- .../infrastructure/perf-eval/go-bench/run-go-bench.sh | 2 +- .../perf-eval/go-bench/runner/instantiate.go | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 5 files changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/workflows/load-test-coordinator.yml b/.github/workflows/load-test-coordinator.yml index a3bb22fc9..1b7240e01 100644 --- a/.github/workflows/load-test-coordinator.yml +++ b/.github/workflows/load-test-coordinator.yml @@ -11,7 +11,7 @@ defaults: on: push: - branches: [release/**, go-bench-test] + branches: [release/**] workflow_dispatch: inputs: target_ref: @@ -72,11 +72,10 @@ jobs: echo "go-bench baseline: ${BASELINE:-}" # Unquoted heredoc so $BASELINE expands into the go-bench entry; - # Temporarily disabled while iterating on go-bench (restore before merge): - # {"label":"Apply-load ingestion","run_label":"apply-load","script":"cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/ingest-load-test/run-load-test.sh","budget":225}, - # {"label":"Backfill ingestion","run_label":"backfill","script":"cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/backfill-test/run-backfill.sh","budget":345}, LEGS="$(jq -c . < Date: Mon, 27 Jul 2026 14:30:39 -0400 Subject: [PATCH 13/13] revert irrelevant go.mod/sum changes --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a560cbb75..d2a250e3e 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/creachadair/jrpc2 v1.3.3 github.com/fsouza/fake-gcs-server v1.49.2 github.com/go-chi/chi v4.1.2+incompatible - github.com/mattn/go-sqlite3 v1.14.48 + github.com/mattn/go-sqlite3 v1.14.17 github.com/montanaflynn/stats v0.7.1 github.com/pelletier/go-toml v1.9.5 github.com/prometheus/client_golang v1.23.2 diff --git a/go.sum b/go.sum index a5f335f41..9a94100f5 100644 --- a/go.sum +++ b/go.sum @@ -352,8 +352,8 @@ github.com/markbates/oncer v1.0.0/go.mod h1:Z59JA581E9GP6w96jai+TGqafHPW+cPfRxz2 github.com/markbates/safe v1.0.1 h1:yjZkbvRM6IzKj9tlu/zMJLS0n/V351OZWRnF3QfaUxI= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= -github.com/mattn/go-sqlite3 v1.14.48 h1:7XHIgl0a8HwOaiK4E47ozLkST78rR9+OtNGx27D/TFs= -github.com/mattn/go-sqlite3 v1.14.48/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= +github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM= +github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= github.com/minio/minio-go/v7 v7.0.71 h1:No9XfOKTYi6i0GnBj+WZwD8WP5GZfL7n7GOjRqCdAjA=