feat(db-bigquery): accept the service account key as a string, from the environment

jest.config.ts

@@ -29,6 +29,13 @@ const defaultConfig: Config = {
   },
 };
 
+// What the `connector-unit` project claims, excluded from the backend
+// projects that would otherwise also match it.
+const connectorUnitIgnored = [
+  ...(defaultConfig.testPathIgnorePatterns ?? []),
+  '\\.unit\\.spec\\.ts$',
+];
+
 const config: Config = {
   moduleFileExtensions: ['js', 'jsx', 'ts', 'tsx'],
   setupFilesAfterEnv: ['<rootDir>/test/jest.setup.ts', 'jest-expect-message'],
@@ -102,6 +109,21 @@ const config: Config = {
         ],
       },
     },
+    {
+      // Connector tests that need no warehouse — `*.unit.spec.ts` under the
+      // db-* packages. They live in the same directories as the tests that do
+      // need one, so without this project they would run only in that
+      // backend's credentialed CI job: a hermetic test of BigQuery config
+      // parsing would sit out every PR that didn't touch BigQuery, which is
+      // exactly the PR that breaks it. The db-* projects below exclude the
+      // same pattern, so each file still runs in exactly one project — a file
+      // matched by two would appear twice in `jest --listTests` and fail
+      // scripts/ci-test-sanity-check.sh.
+      ...defaultConfig,
+      displayName: 'connector-unit',
+      roots: ['<rootDir>/packages/'],
+      testMatch: ['<rootDir>/packages/malloy-db-*/**/*.unit.spec.ts'],
+    },
     {
       ...defaultConfig,
       displayName: 'db-all',
@@ -110,6 +132,7 @@ const config: Config = {
     {
       ...defaultConfig,
       displayName: 'db-bigquery',
+      testPathIgnorePatterns: connectorUnitIgnored,
       roots: [
         '<rootDir>/packages/malloy-db-bigquery/',
         '<rootDir>/test/src/databases/bigquery/',
@@ -155,6 +178,7 @@ const config: Config = {
     {
       ...defaultConfig,
       displayName: 'db-publisher',
+      testPathIgnorePatterns: connectorUnitIgnored,
       roots: ['<rootDir>/packages/malloy-db-publisher/'],
     },
     {
@@ -170,6 +194,7 @@ const config: Config = {
     {
       ...defaultConfig,
       displayName: 'db-databricks',
+      testPathIgnorePatterns: connectorUnitIgnored,
       roots: ['<rootDir>/packages/malloy-db-databricks/'],
     },
   ],

package.json

@@ -48,7 +48,7 @@
     "typecheck": "tsc --build test/tsconfig.json",
     "precheck": "npm run typecheck && npm run test-duckdb",
     "test-mssql-via-duckdb": "MALLOY_DATABASE=mssql_via_duckdb JEST_SILENT_REPORTER_SHOW_PATHS=true jest --selectProjects db-all --reporters jest-silent-reporter summary",
-    "ci-core": "MALLOY_DATABASES=duckdb,postgres JEST_SILENT_REPORTER_SHOW_PATHS=true jest --reporters --selectProjects malloy-core malloy-render malloy-render-test --reporters jest-silent-reporter summary --runInBand",
+    "ci-core": "MALLOY_DATABASES=duckdb,postgres JEST_SILENT_REPORTER_SHOW_PATHS=true jest --reporters --selectProjects malloy-core malloy-render malloy-render-test connector-unit --reporters jest-silent-reporter summary --runInBand",
     "ci-bigquery": "MALLOY_DATABASE=bigquery JEST_SILENT_REPORTER_SHOW_PATHS=true jest --selectProjects db-all db-bigquery --reporters jest-silent-reporter summary",
     "ci-duckdb-wasm": "MALLOY_DATABASE=duckdb_wasm JEST_SILENT_REPORTER_SHOW_PATHS=true jest --selectProjects db-all db-duckdb --reporters jest-silent-reporter summary --runInBand",
     "ci-duckdb": "MALLOY_DATABASE=duckdb JEST_SILENT_REPORTER_SHOW_PATHS=true jest --selectProjects db-all db-duckdb db-duckdb-core simple-builder --reporters jest-silent-reporter summary",

packages/malloy-db-bigquery/src/bigquery_connection.ts

@@ -78,6 +78,8 @@ interface BigQueryConnectionOptions extends ConnectionConfig {
   projectId?: string;
   serviceAccountKeyPath?: string;
   serviceAccountKey?: {[key: string]: ConnectionParameterValue};
+  /** The key file's contents, as a JSON string or base64-encoded JSON. */
+  serviceAccountKeyJson?: string;
   location?: string;
   maximumBytesBilled?: string;
   timeoutMs?: string;
@@ -87,6 +89,85 @@ interface BigQueryConnectionOptions extends ConnectionConfig {
   setupSQL?: string;
 }
 
+type JsonObject = {[key: string]: ConnectionParameterValue};
+
+function isJsonObject(value: unknown): value is JsonObject {
+  return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
+/**
+ * The two credential shapes the SDK's `credentials` option accepts: a service
+ * account key, and an external account (workload identity federation) config,
+ * which carries no key of its own. Anything else — `{}`, a config file, half a
+ * key — is rejected here rather than at the first query, where it arrives as
+ * "the incoming JSON object does not contain a client_email field" and names
+ * nothing that would lead back to this property.
+ */
+function isCredentialObject(value: JsonObject): boolean {
+  return (
+    (typeof value['client_email'] === 'string' &&
+      typeof value['private_key'] === 'string') ||
+    value['type'] === 'external_account'
+  );
+}
+
+function isProbablyBase64(text: string): boolean {
+  // `{` is not in the base64 alphabet, so JSON can never be mistaken for an
+  // encoding of itself. Everything else is treated as base64 and allowed to
+  // fail the parse below, which keeps the check to the one thing it can be
+  // certain about.
+  return !text.startsWith('{');
+}
+
+/**
+ * A service account key that arrives as a *string* rather than as structured
+ * config, in either of the two encodings a deployment can produce.
+ *
+ * `serviceAccountKey` is a `json`-typed property, and a json-typed slot takes
+ * its value literally: an `{env: "..."}` reference is never resolved in one,
+ * because reference indirection into structured config is exactly what the
+ * config compiler refuses. A deployment whose credentials live in an
+ * environment variable — the normal shape for a server — therefore cannot
+ * reach that slot at all, and the literal `{env: "..."}` object it does reach
+ * the SDK with fails as "the incoming JSON object does not contain a
+ * client_email field". This is a string slot, so a reference resolves and the
+ * value arrives here to be parsed.
+ *
+ * Base64 is accepted because a JSON object full of quotes and braces survives
+ * a shell, a CI secret editor, and a `.env` file poorly; base64 is one token
+ * that survives all three. Which one arrived is detected rather than declared,
+ * so a deployment that switches encodings doesn't also have to edit config.
+ *
+ * Nothing from `text` reaches the error message. It is a private key, and
+ * JSON.parse's own SyntaxError quotes the input it choked on.
+ */
+function credentialsFromJson(text: string): JsonObject {
+  // Buffer's base64 decoder drops characters outside the alphabet rather than
+  // rejecting them, so a mangled value decodes to garbage instead of throwing.
+  // The parse below is what catches that, which is why one message has to
+  // cover both encodings: at this point either could have been intended.
+  const json = isProbablyBase64(text)
+    ? Buffer.from(text, 'base64').toString('utf8')
+    : text;
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(json);
+  } catch {
+    throw new Error(
+      'serviceAccountKeyJson is neither JSON nor base64-encoded JSON. It ' +
+        'must hold the entire service account key file.'
+    );
+  }
+  if (!isJsonObject(parsed) || !isCredentialObject(parsed)) {
+    throw new Error(
+      'serviceAccountKeyJson parsed but is not a service account key: it ' +
+        'has no client_email and private_key. It must hold the entire key ' +
+        'file, not a fragment of one.'
+    );
+  }
+  return parsed;
+}
+
 // BigQuery label grammar: keys and values are lowercase, <=63 chars, and
 // [a-z0-9_-]; keys must start with a lowercase letter. Values are transformed
 // to fit (lowercase, disallowed chars -> '_', truncate); a key that can't be
@@ -439,11 +520,28 @@ export class BigQueryConnection
     if (typeof arg === 'string') {
       this.name = arg;
     } else {
-      const {name, client_email, private_key, serviceAccountKey, ...args} = arg;
+      // Every key-bearing property is destructured out of `args`, so a key
+      // never lands in `this.config` — only the credentials object the SDK
+      // needs does.
+      const {
+        name,
+        client_email,
+        private_key,
+        serviceAccountKey,
+        serviceAccountKeyJson,
+        ...args
+      } = arg;
       this.name = name;
       config = args;
+      // Trimmed before it is looked at: a value that came through a here-doc,
+      // a `$(cat key.json)`, or a secret-store copy tends to carry a trailing
+      // newline, and both the encoding sniff and the emptiness check below
+      // would otherwise read it as content.
+      const keyJson = serviceAccountKeyJson?.trim();
       if (serviceAccountKey) {
         config.credentials = serviceAccountKey;
+      } else if (keyJson) {
+        config.credentials = credentialsFromJson(keyJson);
       } else if (client_email || private_key) {
         config.credentials = {
           client_email,