feat: Dataflow template for importing Aerospike backups into Bigtable

.github/workflows/aerospike-bigtable-migration-tools-ci.yaml

@@ -0,0 +1,54 @@
+name: Aerospike Bigtable Migration Tools CI
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - "aerospike-bigtable-migration-tools/**"
+  pull_request:
+    paths:
+      - "aerospike-bigtable-migration-tools/**"
+
+jobs:
+  ci:
+    name: Check the Aerospike Bigtable Migration Tools code
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: C formatter
+        working-directory: aerospike-bigtable-migration-tools
+        run: |
+          shopt -s globstar
+          clang-format --dry-run --Werror -style=file -assume-filename=.clang-format **/*.c
+      - name: Java formatter
+        working-directory: aerospike-bigtable-migration-tools
+        run: mvn -B -ntp -Pall spotless:check
+      # TODO: test other submodules (e.g., backup-loader), which require dependencies compiled as described in the Dockerfile.
+      - name: Java unit tests
+        working-directory: aerospike-bigtable-migration-tools
+        run: mvn -B -ntp test -pl adapter,replicator
+      # Replicator's integration tests need to be provided with Kafka Connect Bigtable Sink .jar
+      - name: Compile sink .jar
+        working-directory: kafka-connect-bigtable-sink
+        run: mvn -B -ntp clean package -pl sink -DskipUnitTests -DskipIntegrationTests
+      - name: Create integration_test_plugins dir and necessary subdirectories
+        working-directory: aerospike-bigtable-migration-tools
+        run: mkdir -p replicator/integration_test_plugins/sink
+      - name: Copy sink .jar into directory used by replicator's integration tests
+        run: cp kafka-connect-bigtable-sink/sink/target/sink*.jar aerospike-bigtable-migration-tools/replicator/integration_test_plugins/sink
+      - name: Show integration_test_plugins dir contents
+        working-directory: aerospike-bigtable-migration-tools
+        run: tree --du replicator/integration_test_plugins
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@v2
+        with:
+          version: latest
+          install_components: beta,bigtable
+      - name: Integration tests
+        working-directory: aerospike-bigtable-migration-tools
+        run: |
+          set -euo pipefail
+          gcloud beta emulators bigtable start --host-port=0.0.0.0:8086 &
+          mvn -B -ntp verify -pl adapter,replicator -DskipUnitTests

aerospike-bigtable-migration-tools/.clang-format

@@ -0,0 +1,57 @@
+# taken from: https://github.com/googleapis/google-cloud-cpp/blob/main/.clang-format
+# Use the Google style in this project.
+BasedOnStyle: Google
+
+# Some folks prefer to write "int& foo" while others prefer "int &foo".  The
+# Google Style Guide only asks for consistency within a project, we chose
+# "int& foo" for this project:
+DerivePointerAlignment: false
+PointerAlignment: Left
+
+# The Google Style Guide only asks for consistency w.r.t. "east const" vs.
+# "const west" alignment of cv-qualifiers. In this project we use "east const".
+QualifierAlignment: Right
+
+IncludeBlocks: Merge
+IncludeCategories:
+- Regex: '^\"google/cloud/internal/disable_deprecation_warnings.inc\"$'
+  Priority: -1
+# System and C-language headers should go last. These expressions may miss a few
+# cases, we can always add more.  Or we can conservative
+- Regex: '^<[A-Za-z0-9_]*\.h>$'
+  Priority: 10000
+- Regex: '^<sys/[A-Za-z0-9_]*\.h>$'
+  Priority: 10000
+# Matches common headers first, but sorts them after project includes
+- Regex: '^\"google/cloud/(internal/|grpc_utils/|testing_util/|[^/]+\.h)'
+  Priority: 1000
+- Regex: '^\"google/cloud/'  # project includes should sort first
+  Priority: 500
+- Regex: '^\"generator/'     # project includes should sort first
+  Priority: 500
+- Regex: '^\"generator/internal/' # project internals second
+  Priority: 1000
+- Regex: '^\"generator/testing/'  # testing helpers third
+  Priority: 1100
+- Regex: '^\"'         # And then includes from other projects or the system
+  Priority: 1500
+- Regex: '^<grpc/'
+  Priority: 2000
+- Regex: '^<google/*'
+  Priority: 3000
+- Regex: '^<.*/.*'
+  Priority: 4000
+- Regex: '^<.*.hpp>'
+  Priority: 4000
+- Regex: '^<[^/]*>'
+  Priority: 5000
+
+# Format raw string literals with a `pb` or `proto` tag as proto.
+RawStringFormats:
+- Language: TextProto
+  Delimiters:
+  - 'pb'
+  - 'proto'
+  BasedOnStyle: Google
+
+CommentPragmas: '(@copydoc|@copybrief|@see|@overload|@snippet)'

aerospike-bigtable-migration-tools/.clang-tidy

@@ -0,0 +1,148 @@
+---
+# taken from: https://github.com/googleapis/google-cloud-cpp/blob/main/.clang-tidy
+# Configure clang-tidy for this project.
+
+# Here is an explanation for why some of the checks are disabled:
+#
+#  -google-readability-namespace-comments: the *_CLIENT_NS is a macro, and
+#      clang-tidy fails to match it against the initial value.
+#
+#  -modernize-use-trailing-return-type: clang-tidy recommends using
+#      `auto Foo() -> std::string { return ...; }`, we think the code is less
+#      readable in this form.
+#
+#  --modernize-concat-nested-namespaces: clang-tidy recommends
+#      `namespace google::cloud {}` over `namespace google { namespace cloud { } }`
+#      We need to support C++14, which does not supported nested namespaces.
+#
+#  --modernize-use-nodiscard: clang-tidy recommends adding a nodiscard annotation
+#      to functions where the return value should not be ignored.
+#      We need to support C++14, which does not supported the annotation.
+#
+#  -modernize-return-braced-init-list: We think removing typenames and using
+#      only braced-init can hurt readability.
+#
+#  -modernize-avoid-c-arrays: We only use C arrays when they seem to be the
+#      right tool for the job, such as `char foo[] = "hello"`. In these cases,
+#      avoiding C arrays often makes the code less readable, and std::array is
+#      not a drop-in replacement because it doesn't deduce the size.
+#
+#  -modernize-type-traits: clang-tidy recommands using c++17 style variable
+#      templates. We will enable this check after we moved to c++17.
+#
+#  -modernize-unary-static-assert: clang-tidy asks removing empty string in
+#      static_assert(), the check is only applicable for c++17 and later code.
+#      We will enable this check after we moved to c++17.
+#
+#  -performance-move-const-arg: This warning requires the developer to
+#      know/care more about the implementation details of types/functions than
+#      should be necessary. For example, `A a; F(std::move(a));` will trigger a
+#      warning IFF `A` is a trivial type (and therefore the move is
+#      meaningless). It would also warn if `F` accepts by `const&`, which is
+#      another detail that the caller need not care about.
+#
+#  -performance-avoid-endl: we would like to turn this on, but there are too
+#      many legitimate uses in our samples.
+#
+#  -performance-enum-size: Smaller enums may or not may be faster, it depends on
+#      the architechture. If data size was a consideration, we might decide to
+#      enable the warnings.
+#
+#  -readability-redundant-declaration: A friend declaration inside a class
+#      counts as a declaration, so if we also declare that friend outside the
+#      class in order to document it as part of the public API, that will
+#      trigger a redundant declaration warning from this check.
+#
+#  -readability-avoid-return-with-void-value: We believe this is idiomatic
+#    and saves typing, and the intent is obvious.
+#
+#  -readability-function-cognitive-complexity: too many false positives with
+#      clang-tidy-12. We need to disable this check in macros, and that setting
+#      only appears in clang-tidy-13.
+#
+#  -bugprone-narrowing-conversions: too many false positives around
+#      `std::size_t`  vs. `*::difference_type`.
+#
+#  -bugprone-easily-swappable-parameters: too many false positives.
+#
+#  -bugprone-implicit-widening-of-multiplication-result: too many false positives.
+#     Almost any expression of the form `2 * variable` or `long x = a_int * b_int;`
+#     generates an error.
+#
+#  -bugprone-unchecked-optional-access: too many false positives in tests.
+#     Despite what the documentation says, this warning appears after
+#     `ASSERT_TRUE(variable)` or `ASSERT_TRUE(variable.has_value())`.
+#
+#   TODO(#14162): Enable clang-tidy checks. We initially omitted these checks
+#   because they require large cleanup efforts or were blocking the clang-tidy
+#   X update.
+Checks: >
+  -*,
+  abseil-*,
+  bugprone-*,
+  google-*,
+  misc-*,
+  modernize-*,
+  performance-*,
+  portability-*,
+  readability-*,
+  -google-readability-braces-around-statements,
+  -google-readability-namespace-comments,
+  -google-runtime-references,
+  -misc-non-private-member-variables-in-classes,
+  -misc-const-correctness,
+  -misc-include-cleaner,
+  -modernize-return-braced-init-list,
+  -modernize-use-trailing-return-type,
+  -modernize-concat-nested-namespaces,
+  -modernize-use-nodiscard,
+  -modernize-avoid-c-arrays,
+  -modernize-type-traits,
+  -modernize-unary-static-assert,
+  -performance-move-const-arg,
+  -performance-avoid-endl,
+  -performance-enum-size,
+  -readability-braces-around-statements,
+  -readability-identifier-length,
+  -readability-magic-numbers,
+  -readability-named-parameter,
+  -readability-redundant-declaration,
+  -readability-avoid-return-with-void-value,
+  -readability-function-cognitive-complexity,
+  -bugprone-narrowing-conversions,
+  -bugprone-easily-swappable-parameters,
+  -bugprone-inc-dec-in-conditions,
+  -bugprone-implicit-widening-of-multiplication-result,
+  -bugprone-unchecked-optional-access,
+  -bugprone-unused-local-non-trivial-variable,
+  -bugprone-unused-return-value
+
+# Turn all the warnings from the checks above into errors.
+WarningsAsErrors: "*"
+
+HeaderFilterRegex: "(google/cloud/|generator/).*\\.h$"
+
+CheckOptions:
+  - { key: readability-identifier-naming.NamespaceCase,          value: lower_case }
+  - { key: readability-identifier-naming.ClassCase,              value: CamelCase  }
+  - { key: readability-identifier-naming.StructCase,             value: CamelCase  }
+  - { key: readability-identifier-naming.TemplateParameterCase,  value: CamelCase  }
+  - { key: readability-identifier-naming.FunctionCase,           value: aNy_CasE  }
+  - { key: readability-identifier-naming.VariableCase,           value: lower_case }
+  - { key: readability-identifier-naming.ClassMemberCase,        value: lower_case }
+  - { key: readability-identifier-naming.ClassMemberSuffix,      value: _          }
+  - { key: readability-identifier-naming.PrivateMemberSuffix,    value: _          }
+  - { key: readability-identifier-naming.ProtectedMemberSuffix,  value: _          }
+  - { key: readability-identifier-naming.EnumConstantCase,         value: CamelCase }
+  - { key: readability-identifier-naming.EnumConstantPrefix,       value: k         }
+  - { key: readability-identifier-naming.ConstexprVariableCase,    value: CamelCase }
+  - { key: readability-identifier-naming.ConstexprVariablePrefix,  value: k         }
+  - { key: readability-identifier-naming.GlobalConstantCase,       value: CamelCase }
+  - { key: readability-identifier-naming.GlobalConstantPrefix,     value: k         }
+  - { key: readability-identifier-naming.MemberConstantCase,       value: CamelCase }
+  - { key: readability-identifier-naming.MemberConstantPrefix,     value: k         }
+  - { key: readability-identifier-naming.StaticConstantCase,       value: CamelCase }
+  - { key: readability-identifier-naming.StaticConstantPrefix,     value: k         }
+  - { key: readability-implicit-bool-conversion.AllowIntegerConditions,  value: 1   }
+  - { key: readability-implicit-bool-conversion.AllowPointerConditions,  value: 1   }
+  - { key: readability-function-cognitive-complexity.IgnoreMacros,  value: 1   }

aerospike-bigtable-migration-tools/.devcontainer/devcontainer.json

@@ -0,0 +1,48 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+{
+  "name": "aerospike-migration-tools",
+  "build": {
+    "dockerfile": "../Dockerfile",
+    "target": "builder"
+  },
+  "customizations": {
+    "vscode": {
+      "settings": {
+        "C_Cpp.default.includePath": [
+          "/usr/lib/jvm/java-17-openjdk-amd64/include",
+          "/usr/lib/jvm/java-17-openjdk-amd64/include/linux",
+          "/app/include",
+          "/app/modules/c-client/modules/common/src/include",
+          "/app/modules/c-client/src/include",
+          "/app/modules/secret-agent-client/src/include",
+          "/usr/local/include"
+        ]
+      },
+      "extensions": [
+        "streetsidesoftware.code-spell-checker",
+        "ms-vscode.cpptools",
+        "vscjava.vscode-java-pack",
+        "redhat.vscode-xml",
+        "nefrob.vscode-just-syntax"
+      ]
+    }
+  },
+  "postCreateCommand": "ln -s /app /workspaces/aerospike-migration-tools/app",
+  "runArgs": [
+    "--network=host",
+    "--cap-add=SYS_PTRACE"
+  ],
+  "remoteUser": "root"
+}

aerospike-bigtable-migration-tools/.gitignore

@@ -0,0 +1,5 @@
+target
+.idea
+app
+cp.txt
+.mvn

aerospike-bigtable-migration-tools/.mdformat.toml

@@ -0,0 +1,3 @@
+wrap = 80           # wrap all .md files to 80 characters - possible values: {"keep", "no", INTEGER}
+number = false      # possible values: {false, true}
+end_of_line = "lf"  # possible values: {"lf", "crlf"}