diff --git a/.github/workflows/load-test-coordinator.yml b/.github/workflows/load-test-coordinator.yml index 1610f76f2..1b7240e01 100644 --- a/.github/workflows/load-test-coordinator.yml +++ b/.github/workflows/load-test-coordinator.yml @@ -62,14 +62,28 @@ jobs: PR="$(gh pr list --repo "${{ github.repository }}" --state open \ --head "$REF" --json number --jq '.[0].number // ""' 2>/dev/null || true)" fi - LEGS='[ + # go-bench baseline: latest published release, falls back to the highest v* tag. + BASELINE="$(gh api "repos/${{ github.repository }}/releases/latest" --jq .tag_name 2>/dev/null || true)" + if [ -z "$BASELINE" ]; then + BASELINE="$(git ls-remote --tags --refs origin 'v[0-9]*' \ + | sed 's|.*refs/tags/||' | sort -V | tail -n 1 || true)" + fi + [ -n "$BASELINE" ] || echo "::warning::no baseline release found; the go-bench leg will fail" + echo "go-bench baseline: ${BASELINE:-}" + + # Unquoted heredoc so $BASELINE expands into the go-bench entry; + LEGS="$(jq -c . <> "$GITHUB_OUTPUT" # The fan-out: one matrix entry per leg. @@ -87,6 +101,9 @@ jobs: run_label: ${{ matrix.leg.run_label }} leg_script: ${{ matrix.leg.script }} budget_minutes: ${{ matrix.leg.budget }} + # optional per-leg fields; fallbacks mirror the ec2-leg.yml defaults + root_volume_gb: ${{ matrix.leg.root_volume_gb || 500 }} + extra_env: ${{ matrix.leg.extra_env || '' }} report: name: Aggregate + report diff --git a/cmd/stellar-rpc/internal/db/transaction_test.go b/cmd/stellar-rpc/internal/db/transaction_test.go index c04ffe239..a22e33f0c 100644 --- a/cmd/stellar-rpc/internal/db/transaction_test.go +++ b/cmd/stellar-rpc/internal/db/transaction_test.go @@ -381,13 +381,14 @@ func BenchmarkTransactionFetch(b *testing.B) { require.NoError(b, write.Commit(lcms[len(lcms)-1], nil)) reader := NewTransactionReader(log, db, passphrase) - randoms := make([]int, b.N) - for i := 0; b.Loop(); i++ { + // fixed-size pool of pre-drawn indices (b.Loop owns the iteration count) + randoms := make([]int, 4096) + for i := range randoms { randoms[i] = rand.Intn(len(lcms)) } for i := 0; b.Loop(); i++ { - r := randoms[i] + r := randoms[i%len(randoms)] tx, err := reader.GetTransaction(ctx, lcms[r].TransactionHash(0)) require.NoError(b, err) assert.Equal(b, r%2 == 0, tx.Successful) diff --git a/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/backfill-test/run-backfill.sh b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/backfill-test/run-backfill.sh index e43a5b766..2193e35f8 100644 --- a/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/backfill-test/run-backfill.sh +++ b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/backfill-test/run-backfill.sh @@ -1,7 +1,6 @@ -# Backfill ingestion leg. Concatenated after bootstrap-common.sh in the rendered -# EC2 user-data, so it relies on that file's env, helpers, bootstrap_box, and -# run_leg. It hands off to `runner instantiate`, which builds stellar-rpc and -# times a backfill run against the pubnet datastore. The other half, `runner +# Backfill ingestion leg. Relies on bootstrap-common.sh's env, helpers, +# bootstrap_box, and run_leg. It hands off to `runner instantiate` to build +# stellar-rpc and time a backfill run against the datalake. The other half, `runner # gather`, polls S3 for the result object. LEG_TITLE="Backfill ingestion" diff --git a/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/backfill-test/runner/instantiate.go b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/backfill-test/runner/instantiate.go index 0c71efae4..39d1a0fc9 100644 --- a/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/backfill-test/runner/instantiate.go +++ b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/backfill-test/runner/instantiate.go @@ -34,25 +34,20 @@ var backfillDoneRe = regexp.MustCompile(`Backfill process complete, ledgers \[(\ // runs a timed backfill, then publishes the verdict. func instantiate(ctx context.Context) error { var ( - bucket = harness.Env("BUCKET", "stellar-rpc-ci-load-test") - region = harness.Env("REGION", "us-east-1") - workDir = harness.Env("WORK_DIR", "/data") - resultsFile = harness.Env("RESULTS_FILE", "/tmp/results.md") - resultKey = os.Getenv("RESULT_KEY") - targetSHA = os.Getenv("TARGET_SHA") - runID = harness.Env("RUN_ID", "manual") + env = harness.GetEnv() // ~1 day by default for cheap test runs; the full week is 120960. retention = harness.Env("HISTORY_RETENTION_WINDOW", "17280") deadline = harness.Env("BACKFILL_DEADLINE", "4h") - binaryPath = filepath.Join(workDir, "stellar-rpc-bin") // built here (the repo checkout is in WORK_DIR) + binaryPath = filepath.Join(env["WORK_DIR"], "stellar-rpc-bin") // built here (the repo checkout is in WORK_DIR) ) repoRoot, err := os.Getwd() if err != nil { return err } bail := func(format string, args ...any) error { - return harness.BailInstance(resultsFile, "Backfill ingestion", runID, targetSHA, fmt.Sprintf(format, args...)) + return harness.BailInstance(env["RESULTS_FILE"], "Backfill ingestion", env["RUN_ID"], env["TARGET_SHA"], + fmt.Sprintf(format, args...)) } want, err := strconv.Atoi(retention) // used to compare against ingested ledgers below @@ -60,11 +55,11 @@ func instantiate(ctx context.Context) error { return bail("parsing retention window %q: %v", retention, err) } - awsCfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(region)) + awsCfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(env["REGION"])) if err != nil { return bail("loading AWS config: %v", err) } - fetch := &harness.S3Fetcher{Client: s3.NewFromConfig(awsCfg), Bucket: bucket} + fetch := &harness.S3Fetcher{Client: s3.NewFromConfig(awsCfg), Bucket: env["BUCKET"]} if err := fetch.FetchVerified(ctx, "core/stellar-core.zst", corePath, true, "stellar-core"); err != nil { return bail("%v", err) @@ -83,12 +78,12 @@ func instantiate(ctx context.Context) error { } // fetch + write core config from SDK - coreCfg := filepath.Join(workDir, "captive-core-pubnet.cfg") + coreCfg := filepath.Join(env["WORK_DIR"], "captive-core-pubnet.cfg") if err := os.WriteFile(coreCfg, ledgerbackend.PubnetDefaultConfig, 0o644); err != nil { return bail("writing captive-core config: %v", err) } - cfgPath, err := renderConfig(repoRoot, workDir, coreCfg, retention) + cfgPath, err := renderConfig(repoRoot, env["WORK_DIR"], coreCfg, retention) if err != nil { return bail("rendering config: %v", err) } @@ -108,12 +103,13 @@ func instantiate(ctx context.Context) error { } logger.Infof("backfill complete: %d ledgers [%d -> %d] in %s", ingested, lo, hi, elapsed.Round(time.Second)) - md := renderMarkdown(targetSHA, retention, lo, hi, ingested, elapsed) - if err := os.WriteFile(resultsFile, []byte(md), 0o644); err != nil { + md := renderMarkdown(env["TARGET_SHA"], retention, lo, hi, ingested, elapsed) + if err := os.WriteFile(env["RESULTS_FILE"], []byte(md), 0o644); err != nil { return bail("writing results: %v", err) } if err := harness.PublishResult( - ctx, fetch.Client, bucket, resultKey, "ok", runID, targetSHA, resultsFile, ""); err != nil { + ctx, fetch.Client, env["BUCKET"], env["RESULT_KEY"], "ok", env["RUN_ID"], env["TARGET_SHA"], + env["RESULTS_FILE"], ""); err != nil { return bail("publishing result: %v", err) } return nil diff --git a/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/bootstrap-common.sh b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/bootstrap-common.sh index 3dac90f5b..b3bbc3c4c 100644 --- a/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/bootstrap-common.sh +++ b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/bootstrap-common.sh @@ -64,12 +64,13 @@ bail() { } trap 'bail "unhandled error at line $LINENO while running: $BASH_COMMAND"' ERR -# persists the full box log to S3 even if instance terminated +# persists the full box log to S3 even if instance terminated. TERM is trapped +# too: a hard terminate mid-run (leg timeout) never reaches the EXIT trap. upload_box_log() { [ -n "$BUCKET" ] && [ -n "$RESULT_KEY" ] || return 0 aws s3 cp /var/log/user-data.log "s3://$BUCKET/${RESULT_KEY%/*}/user-data.log" >/dev/null 2>&1 || true } -trap upload_box_log EXIT +trap upload_box_log EXIT TERM # bootstrap_box installs the build toolchain and checks out TARGET_SHA into # $WORK_DIR/stellar-rpc, leaving the shell cd'd at the repo root. Generic across diff --git a/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/go-bench/run-go-bench.sh b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/go-bench/run-go-bench.sh new file mode 100644 index 000000000..f8ef6e7fb --- /dev/null +++ b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/go-bench/run-go-bench.sh @@ -0,0 +1,11 @@ +# Go endpoint benchmark leg. Relies on bootstrap-common.sh's env, helpers, +# bootstrap_box, and run_leg. The runner benches the candidate checkout against +# BASELINE_REF and publishes a benchstat comparison. +LEG_TITLE="Go endpoint benchmarks" + +log "clearing stale go-bench state" +rm -rf "$WORK_DIR/stellar-rpc-baseline" +rm -f /tmp/baseline.txt /tmp/candidate.txt /tmp/benchstat.txt /tmp/bench-results.json + +bootstrap_box +run_leg ./cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/go-bench/runner diff --git a/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/go-bench/runner/instantiate.go b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/go-bench/runner/instantiate.go new file mode 100644 index 000000000..b1330f5f9 --- /dev/null +++ b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/go-bench/runner/instantiate.go @@ -0,0 +1,295 @@ +package main + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "os" + "os/exec" + "path" + "path/filepath" + "sort" + "strconv" + "strings" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" + + "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/harness" +) + +const legTitle = "Go endpoint benchmarks" + +// benchDenylist is the set of benchmarks to exclude from the suite. +var benchDenylist = []string{ + "BenchmarkGetLedgerEntries", // is an integration test + "BenchmarkTransactionFetch", // is broken in current baseline release +} + +// instantiate is the instance half after the bootstrap, which runs the benches +// and writes the results file to be published to S3. +func instantiate(ctx context.Context) error { + var ( + env = harness.GetEnv() + baselineRef = os.Getenv("BASELINE_REF") + countStr = harness.Env("BENCH_COUNT", "10") + + baselineOut = "/tmp/baseline.txt" + candidateOut = "/tmp/candidate.txt" + benchstatOut = "/tmp/benchstat.txt" + benchResults = "/tmp/bench-results.json" + ) + + repoRoot, err := os.Getwd() + if err != nil { + return err + } + bail := func(format string, args ...any) error { + return harness.BailInstance(env["RESULTS_FILE"], legTitle, env["RUN_ID"], env["TARGET_SHA"], + fmt.Sprintf(format, args...)) + } + + count, err := strconv.Atoi(countStr) + if err != nil || count < 1 { + return bail("invalid BENCH_COUNT %q", countStr) + } + if baselineRef == "" { + return bail("BASELINE_REF unset; the coordinator resolves it from the latest release") + } + + awsCfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(env["REGION"])) + if err != nil { + return bail("loading AWS config: %v", err) + } + s3Client := s3.NewFromConfig(awsCfg) + + baselineDir := filepath.Join(env["WORK_DIR"], "stellar-rpc-baseline") + logger.Infof("checking out baseline %s into %s", baselineRef, baselineDir) + baselineSHA, err := checkoutBaseline(ctx, baselineDir, env["REPO"], baselineRef) + if err != nil { + return bail("checking out baseline %s: %v", baselineRef, err) + } + + for _, dir := range []string{baselineDir, repoRoot} { + logger.Infof("building rpc libs in %s", dir) + if err := harness.RunStreaming(ctx, dir, nil, 40, "make", "build-libs"); err != nil { + return bail("make build-libs failed in %s: %v", dir, err) + } + } + + logger.Infof("running audited benchmarks (count=%d) on baseline %s", count, baselineRef) + baselineFails := runSuite(ctx, baselineDir, baselineOut, count) + logger.Infof("running audited benchmarks (count=%d) on candidate %s", count, env["TARGET_SHA"]) + candidateFails := runSuite(ctx, repoRoot, candidateOut, count) + + logger.Infof("comparing with benchstat") + if err := runBenchstat(ctx, baselineOut, candidateOut, benchstatOut); err != nil { + return bail("benchstat failed: %v", err) + } + benchstat, err := os.ReadFile(benchstatOut) + if err != nil { + return bail("reading benchstat output: %v", err) + } + + uploaded := uploadRawLogs(ctx, s3Client, env["BUCKET"], env["RESULT_KEY"], + map[string]string{"baseline.txt": baselineOut, "candidate.txt": candidateOut, "benchstat.txt": benchstatOut}) + + var rawLogsPrefix string + if len(uploaded) > 0 { + rawLogsPrefix = "s3://" + env["BUCKET"] + "/" + path.Dir(env["RESULT_KEY"]) + "/" + } + report := benchReport{ + BaselineRef: baselineRef, + BaselineSHA: baselineSHA, + TargetSHA: env["TARGET_SHA"], + Count: count, + Benchstat: string(benchstat), + BaselineFails: baselineFails, + CandidateFails: candidateFails, + RawLogsPrefix: rawLogsPrefix, + RawLogs: uploaded, + } + if err := os.WriteFile(env["RESULTS_FILE"], []byte(renderMarkdown(report)), 0o644); err != nil { + return bail("writing results markdown: %v", err) + } + // The shell wrapper publishes the fail result from resultsFile on non-zero exit. + if len(candidateFails) > 0 { + return fmt.Errorf("candidate benchmarks failed in %s", strings.Join(candidateFails, ", ")) + } + + logger.Infof("results ready; publishing verdict") + if err := publishOK(ctx, s3Client, report, env, benchResults); err != nil { + return bail("publishing result: %v", err) + } + return nil +} + +// publishOK writes the bench metadata and publishes the ok result object. +func publishOK( + ctx context.Context, client *s3.Client, r benchReport, env map[string]string, benchResults string, +) error { + meta, err := json.Marshal(r) + if err != nil { + return fmt.Errorf("marshaling bench metadata: %w", err) + } + if err := os.WriteFile(benchResults, meta, 0o644); err != nil { + return fmt.Errorf("writing bench metadata: %w", err) + } + return harness.PublishResult( + ctx, client, env["BUCKET"], env["RESULT_KEY"], "ok", env["RUN_ID"], r.TargetSHA, env["RESULTS_FILE"], benchResults) +} + +// checkoutBaseline shallow-fetches ref (tag, branch, or SHA) from repo into dir +// and returns the checked-out SHA. +func checkoutBaseline(ctx context.Context, dir, repo, ref string) (string, error) { + if err := os.MkdirAll(dir, 0o755); err != nil { + return "", err + } + for _, args := range [][]string{ + {"init", "-q"}, + {"remote", "add", "origin", "https://github.com/" + repo + ".git"}, + {"fetch", "--depth", "1", "origin", ref}, + {"checkout", "--detach", "FETCH_HEAD"}, + } { + if err := harness.RunStreaming(ctx, dir, nil, 20, "git", args...); err != nil { + return "", fmt.Errorf("git %s: %w", strings.Join(args, " "), err) + } + } + out, err := exec.CommandContext(ctx, "git", "-C", dir, "rev-parse", "HEAD").Output() + if err != nil { + return "", fmt.Errorf("git rev-parse: %w", err) + } + return strings.TrimSpace(string(out)), nil +} + +// runSuite runs every benchmark in the module in dir except benchDenylist. +// Only stderr (tool/compile errors, low-volume) streams to the log. +// Returns the packages go test reported as failed (empty on success). +func runSuite(ctx context.Context, dir, outFile string, count int) []string { + f, err := os.OpenFile(outFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + logger.Warnf("opening %s: %v", outFile, err) + return []string{""} + } + defer f.Close() + cmd := exec.CommandContext(ctx, "go", "test", "-run", "^$", "-bench", ".", + "-skip", "^("+strings.Join(benchDenylist, "|")+")$", + "-benchmem", "-count", strconv.Itoa(count), "-timeout", "30m", + "./...") + cmd.Dir = dir + cmd.Stdout, cmd.Stderr = f, os.Stderr + if err := cmd.Run(); err != nil { + failed := parseFailedPkgs(outFile) + if len(failed) == 0 { + failed = []string{""} + } + logger.Warnf("bench run failed in %s: %v (packages: %s)", dir, err, strings.Join(failed, ", ")) + return failed + } + return nil +} + +// parseFailedPkgs scans a bench output file for go test's FAIL lines. +func parseFailedPkgs(outFile string) []string { + f, err := os.Open(outFile) + if err != nil { + logger.Warnf("reading %s for FAIL lines: %v", outFile, err) + return nil + } + defer f.Close() + var pkgs []string + sc := bufio.NewScanner(f) + sc.Buffer(make([]byte, 64*1024), 1024*1024) + for sc.Scan() { + if fields := strings.Fields(sc.Text()); len(fields) >= 2 && fields[0] == "FAIL" { + pkgs = append(pkgs, fields[1]) + } + } + return pkgs +} + +// runBenchstat compares the two bench outputs into outFile, teeing benchstat's +// stdout there while streaming to the box log. +func runBenchstat(ctx context.Context, baselineOut, candidateOut, outFile string) error { + f, err := os.OpenFile(outFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + return err + } + defer f.Close() + cmd := exec.CommandContext(ctx, "go", "run", "golang.org/x/perf/cmd/benchstat@latest", + filepath.Base(baselineOut), filepath.Base(candidateOut)) + cmd.Dir = filepath.Dir(baselineOut) + cmd.Stdout, cmd.Stderr = io.MultiWriter(f, os.Stderr), os.Stderr + return cmd.Run() +} + +// uploadRawLogs best-effort copies the raw bench outputs next to the result +// object, so the comment can stay a summary. Returns the names of the files +// that actually landed sorted for a stable comment. +func uploadRawLogs( + ctx context.Context, client *s3.Client, bucket, resultKey string, files map[string]string, +) []string { + if resultKey == "" { + return nil + } + var uploaded []string + prefix := path.Dir(resultKey) + for name, p := range files { + body, err := os.ReadFile(p) + if err != nil { + logger.Warnf("skipping raw log upload of %s: %v", p, err) + continue + } + key := prefix + "/" + name + if _, err := client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &key, + Body: bytes.NewReader(body), + ContentType: aws.String("text/plain"), + }); err != nil { + logger.Infof("uploading s3://%s/%s: %v", bucket, key, err) + continue + } + uploaded = append(uploaded, name) + } + sort.Strings(uploaded) + return uploaded +} + +// benchReport is everything the comparison markdown is rendered from. +type benchReport struct { + BaselineRef string `json:"baselineRef"` + BaselineSHA string `json:"baselineSha"` + TargetSHA string `json:"targetSha"` + Count int `json:"count"` + BaselineFails []string `json:"baselineFails,omitempty"` // packages whose baseline bench run failed + CandidateFails []string `json:"candidateFails,omitempty"` // packages whose candidate bench run failed + Benchstat string `json:"-"` + RawLogsPrefix string `json:"-"` // s3:// prefix holding the raw logs, "" when none uploaded + RawLogs []string `json:"-"` // names of the raw logs that actually uploaded +} + +// renderMarkdown renders the leg's comment section: the refs compared + flags +// + any per-package failures + the benchstat output in a drop-down. +func renderMarkdown(r benchReport) string { + var b strings.Builder + fmt.Fprintf(&b, "**Baseline** `%s` (`%s`) vs **candidate** `%s` ā€” `-benchmem -count=%d`, "+ + "both refs sequentially on one box.\n", + r.BaselineRef, r.BaselineSHA[:min(12, len(r.BaselineSHA))], r.TargetSHA[:min(12, len(r.TargetSHA))], r.Count) + for _, pkg := range r.CandidateFails { + fmt.Fprintf(&b, "\nāŒ Candidate bench run failed in `%s`; see the box log.\n", pkg) + } + for _, pkg := range r.BaselineFails { + fmt.Fprintf(&b, "\nāŒ Baseline bench run failed in `%s`; its rows lack a base column.\n", pkg) + } + fmt.Fprintf(&b, "\n
\nbenchstat: baseline vs candidate\n\n```\n%s\n```\n\n
\n", + strings.TrimRight(r.Benchstat, "\n")) + if len(r.RawLogs) > 0 { + fmt.Fprintf(&b, "\nRaw benchmark logs (`%s`): %s\n", r.RawLogsPrefix, strings.Join(r.RawLogs, ", ")) + } + return b.String() +} diff --git a/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/go-bench/runner/main.go b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/go-bench/runner/main.go new file mode 100644 index 000000000..6deb81cc8 --- /dev/null +++ b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/go-bench/runner/main.go @@ -0,0 +1,10 @@ +// Command runner runs the audited endpoint Go benchmarks on the box for the +// release candidate and a baseline release, and publishes a benchstat +// comparison of the two. +package main + +import "github.com/stellar/stellar-rpc/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/harness" + +var logger = harness.NewLogger() + +func main() { harness.Run(instantiate) } diff --git a/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/harness/gather.go b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/harness/gather.go index 759e44b28..9f89da9f8 100644 --- a/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/harness/gather.go +++ b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/harness/gather.go @@ -22,33 +22,33 @@ const commandWaitTimeout = 60 * time.Second // and relays the result as step outputs. On timeout it writes a debug comment // instead. Used by every leg's runner. func Gather(ctx context.Context) error { - vals, err := RequireEnv("INSTANCE_ID", "AWS_REGION", - "RESULTS_TIMEOUT", "POLL_INTERVAL", "GITHUB_OUTPUT", "DEBUG_LOG_LINES", "DEBUG_LOG_EVERY_POLLS", - "BUCKET", "RESULT_KEY", "RUN_ID") - if err != nil { - return err - } - instanceID, region, githubOutput := vals[0], vals[1], vals[4] - bucket, resultKey, runID := vals[7], vals[8], vals[9] - - resultsTimeoutSec, err := strconv.Atoi(vals[2]) - if err != nil { - return fmt.Errorf("RESULTS_TIMEOUT: %w", err) - } - pollIntervalSec, err := strconv.Atoi(vals[3]) - if err != nil { - return fmt.Errorf("POLL_INTERVAL: %w", err) + envStr := map[string]string{} + var missing []string + for _, k := range []string{ + "INSTANCE_ID", "AWS_REGION", "RESULTS_TIMEOUT", "POLL_INTERVAL", "GITHUB_OUTPUT", + "DEBUG_LOG_LINES", "DEBUG_LOG_EVERY_POLLS", "BUCKET", "RESULT_KEY", "RUN_ID", + } { + if envStr[k] = os.Getenv(k); envStr[k] == "" { + missing = append(missing, k) + } } - debugLogLines, err := strconv.Atoi(vals[5]) - if err != nil { - return fmt.Errorf("DEBUG_LOG_LINES: %w", err) + if len(missing) > 0 { + return fmt.Errorf("missing required env: %s", strings.Join(missing, ", ")) } - debugEveryPolls, err := strconv.Atoi(vals[6]) - if err != nil { - return fmt.Errorf("DEBUG_LOG_EVERY_POLLS: %w", err) + instanceID, region, githubOutput := envStr["INSTANCE_ID"], envStr["AWS_REGION"], envStr["GITHUB_OUTPUT"] + bucket, resultKey, runID := envStr["BUCKET"], envStr["RESULT_KEY"], envStr["RUN_ID"] + + envInt := map[string]int{} + for _, k := range []string{"RESULTS_TIMEOUT", "POLL_INTERVAL", "DEBUG_LOG_LINES", "DEBUG_LOG_EVERY_POLLS"} { + n, err := strconv.Atoi(envStr[k]) + if err != nil { + return fmt.Errorf("%s: %w", k, err) + } + envInt[k] = n } - resultsTimeout := time.Duration(resultsTimeoutSec) * time.Second - pollInterval := time.Duration(pollIntervalSec) * time.Second + debugLogLines, debugEveryPolls := envInt["DEBUG_LOG_LINES"], envInt["DEBUG_LOG_EVERY_POLLS"] + resultsTimeout := time.Duration(envInt["RESULTS_TIMEOUT"]) * time.Second + pollInterval := time.Duration(envInt["POLL_INTERVAL"]) * time.Second awsCfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(region)) if err != nil { diff --git a/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/harness/harness.go b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/harness/harness.go index 13d3c7883..134848982 100644 --- a/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/harness/harness.go +++ b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/harness/harness.go @@ -24,6 +24,20 @@ import ( supportlog "github.com/stellar/go-stellar-sdk/support/log" ) +// GetEnv reads the common leg environment. +func GetEnv() map[string]string { + return map[string]string{ + "BUCKET": Env("BUCKET", "stellar-rpc-ci-load-test"), + "REGION": Env("REGION", "us-east-1"), + "WORK_DIR": Env("WORK_DIR", "/data"), + "RESULTS_FILE": Env("RESULTS_FILE", "/tmp/results.md"), + "RESULT_KEY": os.Getenv("RESULT_KEY"), + "TARGET_SHA": os.Getenv("TARGET_SHA"), + "RUN_ID": Env("RUN_ID", "manual"), + "REPO": Env("REPO", "stellar/stellar-rpc"), + } +} + // NewLogger returns an Info-level logger (supportlog.New starts at WARN). Each // leg's runner uses one for its own messages. func NewLogger() *supportlog.Entry { @@ -50,21 +64,6 @@ func Env(key, def string) string { return def } -// RequireEnv returns the values of keys in order, erroring with every unset one. -func RequireEnv(keys ...string) ([]string, error) { - vals := make([]string, len(keys)) - var missing []string - for i, k := range keys { - if vals[i] = os.Getenv(k); vals[i] == "" { - missing = append(missing, k) - } - } - if len(missing) > 0 { - return nil, fmt.Errorf("missing required env: %s", strings.Join(missing, ", ")) - } - return vals, nil -} - // appendOutputs appends lines to the GitHub Actions step-output file. func appendOutputs(path string, lines ...string) error { f, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0o644) diff --git a/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/ingest-load-test/run-load-test.sh b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/ingest-load-test/run-load-test.sh index d08fccdb0..e98347fac 100644 --- a/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/ingest-load-test/run-load-test.sh +++ b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/ingest-load-test/run-load-test.sh @@ -1,7 +1,6 @@ -# Apply-load ingestion leg. Concatenated after bootstrap-common.sh in the -# rendered EC2 user-data, so it relies on that file's env, helpers, bootstrap_box, -# and run_leg. It hands off to the leg runner, which streams the corpus from S3 -# and runs the ingest benchmark. +# Apply-load ingestion leg. Relies on bootstrap-common.sh's env, helpers, +# bootstrap_box, and run_leg. It hands off to the leg runner, which streams +# the corpus from S3 and runs the ingest benchmark. LEG_TITLE="Ingest load test" log "clearing stale apply-load state" diff --git a/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/ingest-load-test/runner/instantiate.go b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/ingest-load-test/runner/instantiate.go index c1a21b7d6..9cb03fbce 100644 --- a/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/ingest-load-test/runner/instantiate.go +++ b/cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/ingest-load-test/runner/instantiate.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path/filepath" + "strconv" "strings" "time" @@ -26,15 +27,9 @@ var ( // from S3, runs the benchmark, and writes the ok/fail verdict. func instantiate(ctx context.Context) error { var ( - bucket = harness.Env("BUCKET", "stellar-rpc-ci-load-test") - region = harness.Env("REGION", "us-east-1") - workDir = harness.Env("WORK_DIR", "/data") - goldenDB = harness.Env("GOLDEN_DB", filepath.Join(workDir, "golden.sqlite")) - resultsFile = harness.Env("RESULTS_FILE", "/tmp/results.md") + env = harness.GetEnv() + goldenDB = harness.Env("GOLDEN_DB", filepath.Join(env["WORK_DIR"], "golden.sqlite")) benchResults = harness.Env("BENCH_RESULTS", "/tmp/bench-results.json") - resultKey = os.Getenv("RESULT_KEY") - targetSHA = os.Getenv("TARGET_SHA") - runID = harness.Env("RUN_ID", "manual") ) repoRoot, err := os.Getwd() @@ -42,14 +37,15 @@ func instantiate(ctx context.Context) error { return err } bail := func(format string, args ...any) error { - return harness.BailInstance(resultsFile, "Ingest load test", runID, targetSHA, fmt.Sprintf(format, args...)) + return harness.BailInstance(env["RESULTS_FILE"], "Ingest load test", env["RUN_ID"], env["TARGET_SHA"], + fmt.Sprintf(format, args...)) } - awsCfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(region)) + awsCfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(env["REGION"])) if err != nil { return bail("loading AWS config: %v", err) } - fetch := &harness.S3Fetcher{Client: s3.NewFromConfig(awsCfg), Bucket: bucket} + fetch := &harness.S3Fetcher{Client: s3.NewFromConfig(awsCfg), Bucket: env["BUCKET"]} bundlePaths, goldenFetchSecs, err := fetchCorpus(ctx, fetch, goldenDB) if err != nil { @@ -69,11 +65,11 @@ func instantiate(ctx context.Context) error { "LOADTEST_INGEST_DEADLINE=" + harness.Env("LOADTEST_INGEST_DEADLINE", "150m"), "LOADTEST_SQLITE_PATH=" + goldenDB, "PERF_RESULTS_PATH=" + benchResults, - "PERF_RESULTS_MD_PATH=" + resultsFile, - "PERF_TARGET_SHA=" + targetSHA, - "PERF_RUN_ID=" + runID, - "PERF_REPO=" + harness.Env("REPO", "stellar/stellar-rpc"), - fmt.Sprintf("PERF_GOLDEN_FETCH_SECONDS=%d", goldenFetchSecs), + "PERF_RESULTS_MD_PATH=" + env["RESULTS_FILE"], + "PERF_TARGET_SHA=" + env["TARGET_SHA"], + "PERF_RUN_ID=" + env["RUN_ID"], + "PERF_REPO=" + env["REPO"], + "PERF_GOLDEN_FETCH_SECONDS=" + strconv.Itoa(goldenFetchSecs), "STELLAR_RPC_INTEGRATION_TESTS_ENABLED=true", } if err := harness.RunStreaming(ctx, repoRoot, benchEnv, 80, @@ -82,12 +78,13 @@ func instantiate(ctx context.Context) error { return bail("benchmark failed:\n%v", err) } - if fi, err := os.Stat(resultsFile); err != nil || fi.Size() == 0 { - return bail("benchmark succeeded but did not emit %s", resultsFile) + if fi, err := os.Stat(env["RESULTS_FILE"]); err != nil || fi.Size() == 0 { + return bail("benchmark succeeded but did not emit %s", env["RESULTS_FILE"]) } logger.Infof("results ready; publishing verdict") if err := harness.PublishResult( - ctx, fetch.Client, bucket, resultKey, "ok", runID, targetSHA, resultsFile, benchResults, + ctx, fetch.Client, env["BUCKET"], env["RESULT_KEY"], "ok", env["RUN_ID"], env["TARGET_SHA"], + env["RESULTS_FILE"], benchResults, ); err != nil { return bail("publishing result: %v", err) }