fix(tools/talis): finalize fibre setup race fixes

Three follow-up bugs surfaced from the PR #3303 follow-up
verification run on a 3-validator AWS Fibre cluster:

- aws.go: CreateAWSInstances exited 0 even when individual
  instance launches failed, so `talis up` lied about success
  and downstream steps proceeded against a partial cluster.
  Returns a joined error now so failure cascades stop early.

- download.go: sshExec used cmd.CombinedOutput, mixing SSH
  warnings (the "Warning: Permanently added '...'..." chatter
  on stderr) into bytes the caller hands to fmt.Sscanf("%d").
  The CLI-side providers cross-check parsed those warnings
  as 0 and looped until its 5-min deadline even though a
  direct SSH query showed all 3 providers registered. Switch
  to cmd.Output() (stdout only) and add `-q -o LogLevel=ERROR`
  to silence the chatter for any caller that does combine
  streams.

- fibre_setup.go: the per-validator escrow verification used
  `celestia-appd query fibre escrow` which doesn't exist —
  the actual subcommand is `escrow-account`. The query
  errored on every retry, the grep for "amount" never
  matched, and the script wedged on the 3-min deadline
  reporting `FATAL: fibre-0 escrow not present`. Switch to
  `escrow-account` and key on `"found":true` (the explicit
  existence flag in the response). Also wrap the fibre-0
  deposit-to-escrow itself in a retry loop matching set-host
  — same `--yes`-returns-before-inclusion silent-failure
  mode bit it. fibre-1..N stay best-effort.

Author: walldiss

tools/talis/aws.go

@@ -350,16 +350,28 @@ func CreateAWSInstances(ctx context.Context, insts []Instance, sshKey, keyName s
 		close(results)
 	}()
 
-	var created []Instance
+	var (
+		created  []Instance
+		failures []string
+	)
 	for res := range results {
 		if res.err != nil {
 			fmt.Printf("❌ %s failed after %v %v\n", res.inst.Name, res.timeRequired, res.err)
+			failures = append(failures, fmt.Sprintf("%s: %v", res.inst.Name, res.err))
 		} else {
 			created = append(created, res.inst)
 			fmt.Printf("✅ %s is up (public=%s) in %v\n", res.inst.Name, res.inst.PublicIP, res.timeRequired)
 		}
 		fmt.Printf("---- Progress: %d/%d\n", len(created), total)
 	}
+	if len(failures) > 0 {
+		// Surface partial-failure as an error so `talis up` exits
+		// non-zero; without this, downstream genesis runs against a
+		// half-provisioned config and fails much later with confusing
+		// "X has no public IP yet" messages.
+		return created, fmt.Errorf("%d/%d instance(s) failed to launch: %s",
+			len(failures), total, strings.Join(failures, "; "))
+	}
 	return created, nil
 }
 

tools/talis/download.go

@@ -193,16 +193,26 @@ func compressAndDownload(table, localPath, user, host, sshKeyPath string) error
 	return nil
 }
 
-// sshExec runs a command on a remote host via SSH and returns the combined output.
+// sshExec runs a command on a remote host via SSH and returns stdout only.
+//
+// We intentionally do NOT use CombinedOutput here. ssh prints connection
+// chatter ("Warning: Permanently added '...' to the list of known hosts.")
+// on stderr, and a previous `CombinedOutput` revision caused
+// `fmt.Sscanf(out, "%d")` parses to silently return 0 because the leading
+// stderr line had no digits. Capturing only stdout keeps numeric output
+// parseable; -q + LogLevel=ERROR further suppresses the chatter for any
+// caller that does combine streams.
 func sshExec(user, host, sshKeyPath, command string) ([]byte, error) {
 	cmd := exec.Command("ssh",
+		"-q",
+		"-o", "LogLevel=ERROR",
 		"-o", "StrictHostKeyChecking=no",
 		"-o", "UserKnownHostsFile=/dev/null",
 		"-i", sshKeyPath,
 		fmt.Sprintf("%s@%s", user, host),
 		command,
 	)
-	return cmd.CombinedOutput()
+	return cmd.Output()
 }
 
 func sftpDownload(remotePath, localPath, user, host, sshKeyPath string) error {

tools/talis/fibre_setup.go

@@ -58,25 +58,24 @@ func setupFibreCmd() *cobra.Command {
 				// the CLI level (`--yes` returns the txhash before block
 				// inclusion), but the tx never lands. Polling explicitly
 				// avoids the `sleep 10` heuristic that used to be here.
-				sb.WriteString(fmt.Sprintf(
-					"echo 'waiting for chain to produce first block...'\n"+
-						"DEADLINE=$(( $(date +%%s) + 300 ))\n"+
-						"while true; do\n"+
-						"  H=$(celestia-appd status --chain-id %s 2>/dev/null | "+
-						"      grep -oE '\"latest_block_height\":\"[0-9]+\"' | "+
-						"      grep -oE '[0-9]+' | head -1)\n"+
-						"  if [ -n \"$H\" ] && [ \"$H\" -gt 0 ]; then\n"+
-						"    echo \"chain is at height $H\"\n"+
-						"    break\n"+
-						"  fi\n"+
-						"  if [ $(date +%%s) -gt $DEADLINE ]; then\n"+
-						"    echo 'FATAL: chain never produced a block within 5m' >&2\n"+
-						"    exit 1\n"+
-						"  fi\n"+
-						"  sleep 3\n"+
+				sb.WriteString(
+					"echo 'waiting for chain to produce first block...'\n" +
+						"DEADLINE=$(( $(date +%s) + 300 ))\n" +
+						"while true; do\n" +
+						"  H=$(celestia-appd status 2>/dev/null | " +
+						"      grep -oE '\"latest_block_height\":\"[0-9]+\"' | " +
+						"      grep -oE '[0-9]+' | head -1)\n" +
+						"  if [ -n \"$H\" ] && [ \"$H\" -gt 0 ]; then\n" +
+						"    echo \"chain is at height $H\"\n" +
+						"    break\n" +
+						"  fi\n" +
+						"  if [ $(date +%s) -gt $DEADLINE ]; then\n" +
+						"    echo 'FATAL: chain never produced a block within 5m' >&2\n" +
+						"    exit 1\n" +
+						"  fi\n" +
+						"  sleep 3\n" +
 						"done\n",
-					cfg.ChainID,
-				))
+				)
 
 				// 1. Register fibre host address. Plain `host:port` form —
 				// x/valaddr requires it; the gRPC client dials it via the
@@ -113,44 +112,54 @@ func setupFibreCmd() *cobra.Command {
 					cfg.ChainID,
 				))
 
-				// 2. Deposit escrow for each fibre worker account
-				for i := range fibreAccounts {
-					keyName := fmt.Sprintf("fibre-%d", i)
-					sb.WriteString(fmt.Sprintf(
-						"celestia-appd tx fibre deposit-to-escrow %s "+
-							"--from %s --keyring-backend=test --home .celestia-app "+
-							"--chain-id %s --fees %s --yes\n",
-						escrowAmount,
-						keyName,
-						cfg.ChainID, fees,
-					))
-				}
-
-				// 3. Verify the FIRST fibre account's escrow actually
-				// landed before we let the tmux session exit. If even
-				// fibre-0 isn't funded, every Fibre upload from the
-				// runner fails with `escrow account not found for
-				// signer …` — same silent-failure mode as set-host.
-				// Other accounts (fibre-1..N) are funded best-effort:
-				// the runner only signs with fibre-0 by default.
+				// 2. Deposit escrow for fibre-0 inside a retry loop.
+				// Same silent-failure mode as set-host: `--yes` returns
+				// the txhash before inclusion, so a single bounced tx
+				// (mempool full, signer not yet propagated, …) leaves
+				// the runner failing every upload with
+				// `escrow account not found for signer …`. fibre-0 is
+				// the one the runner actually signs with by default,
+				// so it's the only one we hard-block on.
 				sb.WriteString(fmt.Sprintf(
 					"FIBRE0_ADDR=$(celestia-appd keys show fibre-0 --keyring-backend test --home .celestia-app -a)\n"+
-						"DEADLINE=$(( $(date +%%s) + 180 ))\n"+
+						"DEADLINE=$(( $(date +%%s) + 300 ))\n"+
 						"while true; do\n"+
-						"  if celestia-appd query fibre escrow \"$FIBRE0_ADDR\" --chain-id %s -o json 2>/dev/null \\\n"+
-						"     | grep -q '\"amount\"'; then\n"+
+						"  celestia-appd tx fibre deposit-to-escrow %s "+
+						"--from fibre-0 --keyring-backend=test --home .celestia-app "+
+						"--chain-id %s --fees %s --yes >/dev/null 2>&1 || true\n"+
+						"  sleep 6\n"+
+						"  if celestia-appd query fibre escrow-account \"$FIBRE0_ADDR\" --chain-id %s -o json 2>/dev/null \\\n"+
+						"     | grep -q '\"found\":true'; then\n"+
 						"    echo \"escrow confirmed for fibre-0 ($FIBRE0_ADDR)\"\n"+
 						"    break\n"+
 						"  fi\n"+
 						"  if [ $(date +%%s) -gt $DEADLINE ]; then\n"+
-						"    echo \"FATAL: fibre-0 escrow not present after 3m\" >&2\n"+
+						"    echo \"FATAL: fibre-0 escrow did not land after 5m\" >&2\n"+
 						"    exit 1\n"+
 						"  fi\n"+
-						"  sleep 5\n"+
+						"  echo 'fibre-0 escrow pending, retrying...'\n"+
 						"done\n",
+					escrowAmount,
+					cfg.ChainID, fees,
 					cfg.ChainID,
 				))
 
+				// 3. Best-effort fund fibre-1..N. The runner only signs
+				// with fibre-0 by default, so a missing one of these
+				// doesn't block uploads — they exist as headroom for
+				// future signer rotation.
+				for i := 1; i < fibreAccounts; i++ {
+					keyName := fmt.Sprintf("fibre-%d", i)
+					sb.WriteString(fmt.Sprintf(
+						"celestia-appd tx fibre deposit-to-escrow %s "+
+							"--from %s --keyring-backend=test --home .celestia-app "+
+							"--chain-id %s --fees %s --yes\n",
+						escrowAmount,
+						keyName,
+						cfg.ChainID, fees,
+					))
+				}
+
 				script := sb.String()
 
 				sem <- struct{}{}