Rewrite the data-loader in go and allow the data-laoder to be modified

Author: johscheuer

Dockerfile

@@ -38,13 +38,6 @@ COPY kubectl-fdb/ kubectl-fdb/
 # Build
 RUN CGO_ENABLED=1 GOOS=linux GOARCH=amd64 GO111MODULE=on make manager plugin-go
 
-# Create user and group here since we don't have the tools
-# in distroless
-RUN groupadd --gid 4059 fdb && \
-	useradd --gid 4059 --uid 4059 --create-home --shell /bin/bash fdb && \
-	mkdir -p /var/log/fdb && \
-	touch /var/log/fdb/.keep
-
 FROM docker.io/rockylinux/rockylinux:9.5-minimal
 
 ARG FDB_VERSION
@@ -63,11 +56,15 @@ RUN set -eux && \
     rpm -i foundationdb-clients-${FDB_VERSION}-1.el7.x86_64.rpm --excludepath=/usr/bin --excludepath=/usr/lib/foundationdb/backup_agent && \
     rm foundationdb-clients-${FDB_VERSION}-1.el7.x86_64.rpm foundationdb-clients-${FDB_VERSION}-1.el7.x86_64.rpm.sha256
 
-COPY --from=builder /etc/passwd /etc/passwd
-COPY --from=builder /etc/group /etc/group
+# Create user and group here since we don't have the tools
+# in distroless
+RUN groupadd --gid 4059 fdb && \
+	useradd --gid 4059 --uid 4059 --create-home --shell /bin/bash fdb && \
+	mkdir -p /var/log/fdb && \
+	touch /var/log/fdb/.keep \
+
 COPY --chown=fdb:fdb --from=builder /workspace/bin/manager .
 COPY --chown=fdb:fdb --from=builder /workspace/bin/kubectl-fdb /usr/local/bin/kubectl-fdb
-COPY --chown=fdb:fdb --from=builder /var/log/fdb/.keep /var/log/fdb/.keep
 
 # Set to the numeric UID of fdb user to satisfy PodSecurityPolices which enforce runAsNonRoot
 USER 4059

e2e/fixtures/fdb_data_loader.go

@@ -25,11 +25,13 @@ import (
 	"context"
 	"errors"
 	"io"
-	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	"log"
 	"text/template"
 	"time"
 
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/utils/pointer"
+
 	"github.com/onsi/gomega"
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -65,7 +67,7 @@ spec:
         name: {{ .Name }}
         # This configuration will load ~1GB per data loader.
         args:
-        - --keys=1000000
+        - --keys={{ .Keys }}
         - --batch-size=50
         - --value-size=1000
         env:
@@ -85,6 +87,8 @@ spec:
             value: /var/dynamic/fdb/primary/lib
           - name: FDB_NETWORK_OPTION_TRACE_LOG_GROUP
             value: {{ .Name }}
+          - name: FDB_NETWORK_OPTION_TRACE_ENABLE
+            value: "/tmp/fdb-trace-logs"
           - name: FDB_NETWORK_OPTION_EXTERNAL_CLIENT_DIRECTORY
             value: /var/dynamic/fdb/libs
           - name: PYTHONUNBUFFERED
@@ -97,6 +101,8 @@ spec:
           - name: fdb-certs
             mountPath: /tmp/fdb-certs
             readOnly: true
+          - name: fdb-logs
+            mountPath: /tmp/fdb-trace-logs
         resources:
          requests:
            cpu: "1"
@@ -161,11 +167,13 @@ spec:
                 path: fdb.cluster
         - name: fdb-libs
           emptyDir: {}
+        - name: fdb-logs
+          emptyDir: {}
         - name: fdb-certs
           secret:
             secretName: {{ .SecretName }}`
 
-	// For now we only load 2GB into the cluster, we can increase this later if we want.
+	// For now, we only load 2GB into the cluster, we can increase this later if we want.
 	dataLoaderJobUnifiedImage = `apiVersion: batch/v1
 kind: Job
 metadata:
@@ -185,7 +193,7 @@ spec:
         name: {{ .Name }}
         # This configuration will load ~1GB per data loader.
         args:
-        - --keys=1000000
+        - --keys={{ .Keys }}
         - --batch-size=50
         - --value-size=1000
         env:
@@ -201,6 +209,8 @@ spec:
           # Consider remove this option once 6.3 is no longer being used.
           - name: FDB_NETWORK_OPTION_IGNORE_EXTERNAL_CLIENT_FAILURES
             value: ""
+          - name: FDB_NETWORK_OPTION_TRACE_ENABLE
+            value: "/tmp/fdb-trace-logs"
           - name: LD_LIBRARY_PATH
             value: /var/dynamic/fdb
           - name: FDB_NETWORK_OPTION_TRACE_LOG_GROUP
@@ -217,6 +227,8 @@ spec:
           - name: fdb-certs
             mountPath: /tmp/fdb-certs
             readOnly: true
+          - name: fdb-logs
+            mountPath: /tmp/fdb-trace-logs
         resources:
          requests:
            cpu: "1"
@@ -278,12 +290,14 @@ spec:
                 path: fdb.cluster
         - name: fdb-libs
           emptyDir: {}
+        - name: fdb-logs
+          emptyDir: {}
         - name: fdb-certs
           secret:
             secretName: {{ .SecretName }}`
 )
 
-// dataLoaderConfig represents the configuration of the Dataloader Job.
+// dataLoaderConfig represents the configuration of the data-loader Job.
 type dataLoaderConfig struct {
 	// Name of the data loader Job.
 	Name string
@@ -298,27 +312,31 @@ type dataLoaderConfig struct {
 	// SecretName represents the Kubernetes secret that contains the certificates for communicating with the FoundationDB
 	// cluster.
 	SecretName string
+	// Keys defines how many keys should be written by the data loader.
+	Keys int
 }
 
-func (factory *Factory) getDataLoaderConfig(cluster *FdbCluster) *dataLoaderConfig {
+func (factory *Factory) getDataLoaderConfig(cluster *FdbCluster, keys *int) *dataLoaderConfig {
+	log.Println("keys:", keys, "deref:", pointer.IntDeref(keys, 1000000))
 	return &dataLoaderConfig{
 		Name:            dataLoaderName,
 		Image:           factory.GetDataLoaderImage(),
 		Namespace:       cluster.Namespace(),
 		SidecarVersions: factory.GetSidecarConfigs(),
 		ClusterName:     cluster.Name(),
 		SecretName:      factory.GetSecretName(),
+		Keys:            pointer.IntDeref(keys, 1000000),
 	}
 }
 
 // CreateDataLoaderIfAbsent will create the data loader for the provided cluster and load some random data into the cluster.
 func (factory *Factory) CreateDataLoaderIfAbsent(cluster *FdbCluster) {
-	factory.CreateDataLoaderIfAbsentWithWait(cluster, true)
+	factory.CreateDataLoaderIfAbsentWithWait(cluster, nil, true)
 }
 
 // CreateDataLoaderIfAbsentWithWait will create the data loader for the provided cluster and load some random data into the cluster.
 // If wait is true, the method will wait until the data loader has finished.
-func (factory *Factory) CreateDataLoaderIfAbsentWithWait(cluster *FdbCluster, wait bool) {
+func (factory *Factory) CreateDataLoaderIfAbsentWithWait(cluster *FdbCluster, keys *int, wait bool) {
 	if !factory.options.enableDataLoading {
 		return
 	}
@@ -330,7 +348,8 @@ func (factory *Factory) CreateDataLoaderIfAbsentWithWait(cluster *FdbCluster, wa
 	t, err := template.New("dataLoaderJob").Parse(dataLoaderJobTemplate)
 	gomega.Expect(err).NotTo(gomega.HaveOccurred())
 	buf := bytes.Buffer{}
-	gomega.Expect(t.Execute(&buf, factory.getDataLoaderConfig(cluster))).NotTo(gomega.HaveOccurred())
+	log.Println("keys:", keys)
+	gomega.Expect(t.Execute(&buf, factory.getDataLoaderConfig(cluster, keys))).NotTo(gomega.HaveOccurred())
 	decoder := yamlutil.NewYAMLOrJSONDecoder(&buf, 100000)
 	for {
 		var rawObj runtime.RawExtension

e2e/test_operator_ha/operator_ha_test.go

@@ -307,7 +307,7 @@ var _ = Describe("Operator HA tests", Label("e2e", "pr"), func() {
 					})
 
 				// TODO (johscheuer): Allow to have this as a long running task until the test is done.
-				factory.CreateDataLoaderIfAbsentWithWait(fdbCluster.GetPrimary(), false)
+				factory.CreateDataLoaderIfAbsentWithWait(fdbCluster.GetPrimary(), nil, false)
 
 				time.Sleep(1 * time.Minute)
 				log.Println("replacedPod", replacedPod.Name, "useLocalitiesForExclusion", fdbCluster.GetPrimary().GetCluster().UseLocalitiesForExclusion())
@@ -415,7 +415,7 @@ var _ = Describe("Operator HA tests", Label("e2e", "pr"), func() {
 					})
 
 				// TODO (johscheuer): Allow to have this as a long running task until the test is done.
-				factory.CreateDataLoaderIfAbsentWithWait(fdbCluster.GetPrimary(), false)
+				factory.CreateDataLoaderIfAbsentWithWait(fdbCluster.GetPrimary(), nil, false)
 
 				time.Sleep(1 * time.Minute)
 				log.Println("replacedPod", replacedPod.Name, "useLocalitiesForExclusion", fdbCluster.GetPrimary().GetCluster().UseLocalitiesForExclusion())

main.go

@@ -47,7 +47,7 @@ func init() {
 }
 
 func main() {
-	fdb.MustAPIVersion(620)
+	fdb.MustAPIVersion(710)
 	operatorOpts := setup.Options{}
 	operatorOpts.BindFlags(flag.CommandLine)
 

sample-apps/data-loader/Dockerfile

@@ -1,21 +1,55 @@
-FROM python:3.9-slim
+ARG FDB_VERSION=7.1.67
+ARG FDB_WEBSITE=https://github.com/apple/foundationdb/releases/download
 
-COPY app.py /usr/local/bin
+# Build the manager binary
+FROM docker.io/library/golang:1.23.7 AS builder
 
-RUN pip install foundationdb==6.2.10
-RUN groupadd --gid 4059 fdb && \
-	useradd --gid 4059 --uid 4059 --shell /usr/sbin/nologin fdb
+ARG FDB_VERSION
+ARG FDB_WEBSITE
+ARG TAG="latest"
 
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends curl && \
-    curl -Ls https://github.com/krallin/tini/releases/download/v0.19.0/tini-amd64 -o tini  && \
-    echo "93dcc18adc78c65a028a84799ecf8ad40c936fdfc5f2a57b1acda5a8117fa82c  tini" > tini-sha.txt && \
-    sha256sum --quiet -c tini-sha.txt && \
-    chmod +x tini && \
-    mv tini /usr/bin/ && \
-    rm -rf /tmp/*
+RUN set -eux && \
+    curl --fail -L "${FDB_WEBSITE}/${FDB_VERSION}/foundationdb-clients_${FDB_VERSION}-1_amd64.deb" -o foundationdb-clients_${FDB_VERSION}-1_amd64.deb && \
+    curl --fail -L "${FDB_WEBSITE}/${FDB_VERSION}/foundationdb-clients_${FDB_VERSION}-1_amd64.deb.sha256" -o foundationdb-clients_${FDB_VERSION}-1_amd64.deb.sha256 && \
+    sha256sum -c foundationdb-clients_${FDB_VERSION}-1_amd64.deb.sha256 && \
+    dpkg -i foundationdb-clients_${FDB_VERSION}-1_amd64.deb && \
+    rm foundationdb-clients_${FDB_VERSION}-1_amd64.deb foundationdb-clients_${FDB_VERSION}-1_amd64.deb.sha256
+
+WORKDIR /workspace
+# Copy the Go Modules manifests
+COPY go.mod go.mod
+COPY go.sum go.sum
+# cache deps before building and copying source so that we don't need to re-download as much
+# and so that source changes don't invalidate our downloaded layer
+RUN go mod download -x
+
+# Copy the go source
+COPY main.go main.go
+
+# Build
+RUN CGO_ENABLED=1 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -o /workspace/bin/data-loader main.go
+
+FROM docker.io/rockylinux/rockylinux:9.5-minimal
+
+ARG FDB_VERSION
+ARG FDB_WEBSITE
+
+VOLUME /usr/lib/fdb
+
+WORKDIR /
+
+RUN set -eux && \
+    curl --fail -L "${FDB_WEBSITE}/${FDB_VERSION}/foundationdb-clients-${FDB_VERSION}-1.el7.x86_64.rpm" -o foundationdb-clients-${FDB_VERSION}-1.el7.x86_64.rpm && \
+    curl --fail -L "${FDB_WEBSITE}/${FDB_VERSION}/foundationdb-clients-${FDB_VERSION}-1.el7.x86_64.rpm.sha256" -o foundationdb-clients-${FDB_VERSION}-1.el7.x86_64.rpm.sha256 && \
+    microdnf install -y glibc pkg-config && \
+    microdnf clean all && \
+    sha256sum -c foundationdb-clients-${FDB_VERSION}-1.el7.x86_64.rpm.sha256 && \
+    rpm -i foundationdb-clients-${FDB_VERSION}-1.el7.x86_64.rpm --excludepath=/usr/bin --excludepath=/usr/lib/foundationdb/backup_agent && \
+    rm foundationdb-clients-${FDB_VERSION}-1.el7.x86_64.rpm foundationdb-clients-${FDB_VERSION}-1.el7.x86_64.rpm.sha256
+
+COPY --chown=fdb:fdb --from=builder /workspace/bin/data-loader /usr/local/bin/data-loader
 
 # Set to the numeric UID of fdb user to satisfy PodSecurityPolices which enforce runAsNonRoot
 USER 4059
 
-ENTRYPOINT [ "/usr/bin/tini", "-g", "--", "python", "/usr/local/bin/app.py" ]
+ENTRYPOINT ["/usr/local/bin/data-loader"]

sample-apps/data-loader/app.py

@@ -0,0 +1,9 @@
+module github.com/FoundationDB/fdb-data-loader
+
+go 1.22.7
+
+require (
+	// Binding version for 7.1.67
+	github.com/apple/foundationdb/bindings/go v0.0.0-20250115161953-f1ab8147ed1c
+	github.com/google/uuid v1.6.0
+)

sample-apps/data-loader/go.mod

@@ -0,0 +1,4 @@
+github.com/apple/foundationdb/bindings/go v0.0.0-20250115161953-f1ab8147ed1c h1:Nnun3T50beIpO6YKDZInHuZMgnNtYJafSlQAvkXwOWc=
+github.com/apple/foundationdb/bindings/go v0.0.0-20250115161953-f1ab8147ed1c/go.mod h1:OMVSB21p9+xQUIqlGizHPZfjK+SHws1ht+ZytVDoz9U=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=

sample-apps/data-loader/go.sum

@@ -24,15 +24,20 @@ spec:
             mountPath: /var/dynamic-conf
       initContainers:
         - name: foundationdb-kubernetes-init
-          image: foundationdb/foundationdb-kubernetes-sidecar:6.3.3-1
+          image: foundationdb/fdb-kubernetes-monitor:7.1.57
           args:
             - "--copy-file"
             - "fdb.cluster"
             - "--copy-library"
-            - "6.3"
+            - "7.1"
             - "--copy-library"
-            - "6.2"
-            - "--init-mode"
+            - "6.3"
+            - "--mode"
+            - "init"
+            - "--input-dir"
+            - "/var/dynamic-conf"
+            - "--output-dir"
+            - "/var/output-files"
             - "--require-not-empty"
             - "fdb.cluster"
           volumeMounts: