Add new job type for generating manifests using image-builder-cli [HMS-9049]

.gitlab-ci.yml

@@ -590,6 +590,13 @@ API:
           - aws/rhel-10.1-ga-aarch64
         INTERNAL_NETWORK: ["true"]
 
+      # single test config for experimental feature
+      - IMAGE_TYPE: ["aws"]
+        RUNNER:
+          - aws/rhel-10.1-ga-x86_64
+        IMAGE_BUILDER_EXPERIMENTAL: "image-builder-manifest-generation=1"
+        INTERNAL_NETWORK: ["true"]
+
 API-module-hotfixes:
   stage: test
   extends: .terraform

cmd/osbuild-composer/composer.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/osbuild/images/pkg/depsolvednf"
 	"github.com/osbuild/images/pkg/distrofactory"
+	"github.com/osbuild/images/pkg/experimentalflags"
 	"github.com/osbuild/images/pkg/reporegistry"
 	"github.com/osbuild/osbuild-composer/internal/auth"
 	"github.com/osbuild/osbuild-composer/internal/cloudapi"
@@ -173,6 +174,12 @@ func (c *Composer) InitAPI(cert, key string, enableTLS bool, enableMTLS bool, en
 		TenantProviderFields: c.config.Koji.JWTTenantProviderFields,
 	}
 
+	// handle experimental image-builder manifest generation option using the
+	// experimentalflags pkg from osbuild/images.
+	if experimentalflags.Bool("image-builder-manifest-generation") {
+		config.ImageBuilderManifestGeneration = true
+	}
+
 	c.api = cloudapi.NewServer(c.workers, c.distros, c.repos, config)
 
 	if !enableTLS {

cmd/osbuild-worker/export_test.go

@@ -4,6 +4,7 @@ var (
 	WorkerClientErrorFrom         = workerClientErrorFrom
 	MakeJobErrorFromOsbuildOutput = makeJobErrorFromOsbuildOutput
 	Main                          = main
+	ParseManifestPipelines        = parseManifestPipelines
 )
 
 func MockRun(new func()) (restore func()) {

cmd/osbuild-worker/jobimpl-image-builder-manifest.go

@@ -0,0 +1,136 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/osbuild/osbuild-composer/internal/common"
+	"github.com/osbuild/osbuild-composer/internal/worker"
+	"github.com/osbuild/osbuild-composer/internal/worker/clienterrors"
+	"github.com/sirupsen/logrus"
+)
+
+type ImageBuilderManifestJobImpl struct {
+	RepositoryMTLSConfig *RepositoryMTLSConfig
+}
+
+func (impl *ImageBuilderManifestJobImpl) Run(job worker.Job) error {
+	logWithId := logrus.WithField("jobId", job.Id().String())
+
+	result := &worker.ImageBuilderManifestJobResult{
+		Manifest: nil,
+		ManifestInfo: worker.ManifestInfo{
+			OSBuildComposerVersion: common.BuildVersion(),
+			// TODO: add image-builder version fields when we get
+			// machine-readable version output from image-builder-cli
+		},
+	}
+
+	defer func() {
+		err := job.Finish(&result)
+		if err != nil {
+			logWithId.Errorf("Error reporting job result: %v", err)
+		}
+	}()
+
+	var args worker.ImageBuilderManifestJob
+	err := job.Args(&args)
+	if err != nil {
+		return err
+	}
+
+	if impl.RepositoryMTLSConfig != nil {
+		for repoi, repo := range args.Args.Repositories {
+			for _, baseurlstr := range repo.BaseURLs {
+				match, err := impl.RepositoryMTLSConfig.CompareBaseURL(baseurlstr)
+				if err != nil {
+					result.JobError = clienterrors.New(clienterrors.ErrorInvalidRepositoryURL, "Repository URL is malformed", err.Error())
+					return err
+				}
+				if match {
+					impl.RepositoryMTLSConfig.SetupRepoSSL(&args.Args.Repositories[repoi])
+				}
+			}
+		}
+	}
+
+	manifest, err := worker.RunImageBuilderManifest(args.Args, args.ExtraEnv, os.Stderr)
+	if err != nil {
+		result.JobError = workerClientErrorFrom(err, logWithId)
+	}
+	result.Manifest = manifest
+
+	pipelineNames, err := parseManifestPipelines(manifest)
+	if err != nil {
+		// Not a critical failure. Log the error and continue. This will cause
+		// the job log to be unordered.
+		// For Koji builds, it will also fail to create package listings.
+		logWithId.Warningf("failed to parse manifest for pipeline names: %v", err)
+	}
+	result.ManifestInfo.PipelineNames = pipelineNames
+
+	return nil
+}
+
+// Parse the raw manifest into an incomplete representation of the manifest
+// structure in order to retrieve the pipeline names in order. Any pipeline
+// names that appear under the 'build' property of another pipeline are added
+// to the build pipelines list. The rest are added to the payload pipelines.
+func parseManifestPipelines(rawManifest []byte) (*worker.PipelineNames, error) {
+	if len(rawManifest) == 0 {
+		return nil, fmt.Errorf("manifest is empty")
+	}
+
+	// Partial manifest structure for extracting pipeline names.
+	type partialManifest struct {
+		Version   string `json:"version"`
+		Pipelines []struct {
+			Name  string `json:"name"`
+			Build string `json:"build"`
+		} `json:"pipelines"`
+	}
+
+	var pm partialManifest
+	if err := json.Unmarshal(rawManifest, &pm); err != nil {
+		return nil, err
+	}
+
+	if pm.Version != "2" {
+		return nil, fmt.Errorf("unexpected manifest version: %s != 2", pm.Version)
+	}
+
+	if len(pm.Pipelines) == 0 {
+		return nil, fmt.Errorf("no pipelines found")
+	}
+
+	// Collect all pipeline names in order and save build pipeline names in a
+	// map so we can split them later.
+	var allPipelineNames []string
+	buildPipelineNames := make(map[string]bool)
+	for _, pl := range pm.Pipelines {
+		allPipelineNames = append(allPipelineNames, pl.Name)
+		if pl.Build != "" {
+			// The build property is in the form "name:build", where "build" is
+			// the name of the build pipeline.
+			parts := strings.SplitN(pl.Build, ":", 2)
+			if len(parts) != 2 {
+				return nil, fmt.Errorf("unexpected pipeline build property format: %s", pl.Build)
+			}
+			buildPipelineNames[parts[1]] = true
+		}
+	}
+
+	var pipelineNames worker.PipelineNames
+	for _, plName := range allPipelineNames {
+		if buildPipelineNames[plName] {
+			pipelineNames.Build = append(pipelineNames.Build, plName)
+			continue
+		}
+
+		pipelineNames.Payload = append(pipelineNames.Payload, plName)
+	}
+
+	return &pipelineNames, nil
+}

cmd/osbuild-worker/jobimpl-image-builder-manifest_test.go

@@ -0,0 +1,218 @@
+package main_test
+
+import (
+	"testing"
+
+	main "github.com/osbuild/osbuild-composer/cmd/osbuild-worker"
+	"github.com/osbuild/osbuild-composer/internal/worker"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestParseManifestPipelines(t *testing.T) {
+	type testCase struct {
+		Manifest []byte
+
+		ExpectedPipelines *worker.PipelineNames
+		ExpectedError     string
+	}
+
+	testCases := map[string]testCase{
+		"empty": {
+			ExpectedError: "manifest is empty",
+		},
+		"bad-json": {
+			Manifest:      []byte(`{not json}`),
+			ExpectedError: "invalid character 'n' looking for beginning of object key string",
+		},
+		"bad-manifest": {
+			Manifest:      []byte(`{"unknown-key": "value"}`),
+			ExpectedError: `unexpected manifest version:  != 2`,
+		},
+		"bad-version": {
+			Manifest:      []byte(`{"version": "42"}`),
+			ExpectedError: `unexpected manifest version: 42 != 2`,
+		},
+		"no-pipelines": {
+			Manifest:      []byte(`{"version": "2"}`),
+			ExpectedError: `no pipelines found`,
+		},
+		"bad-build": {
+			Manifest: []byte(`{
+	"version": "2",
+	"pipelines": [
+		{
+			"name": "build-pipeline"
+		},
+		{
+			"name": "root-tree",
+			"build": "build-pipeline"
+		}
+	],
+	"sources": []
+}`),
+			ExpectedError: `unexpected pipeline build property format: build-pipeline`,
+		},
+
+		"simple": {
+			Manifest: []byte(`{
+	"version": "2",
+	"pipelines": [
+		{
+			"name": "pipeline"
+		}
+	],
+	"sources": []
+}`),
+			ExpectedPipelines: &worker.PipelineNames{
+				Payload: []string{"pipeline"},
+			},
+		},
+
+		"with-build": {
+			Manifest: []byte(`{
+	"version": "2",
+	"pipelines": [
+		{
+			"name": "build-pipeline"
+		},
+		{
+			"name": "root-tree",
+			"build": "name:build-pipeline"
+		},
+		{
+			"name": "image",
+			"build": "name:build-pipeline"
+		}
+	],
+	"sources": []
+}`),
+			ExpectedPipelines: &worker.PipelineNames{
+				Build:   []string{"build-pipeline"},
+				Payload: []string{"root-tree", "image"},
+			},
+		},
+
+		"real-simplified": { // Real manifest, but simplified by removing stage options, inputs, and sources
+			Manifest: []byte(`{
+  "version": "2",
+  "pipelines": [
+    {
+      "name": "build",
+      "runner": "org.osbuild.fedora42",
+      "stages": [
+        {
+          "type": "org.osbuild.rpm",
+          "inputs": {},
+          "options": {}
+        },
+        {
+          "type": "org.osbuild.selinux",
+          "options": {}
+        }
+      ]
+    },
+    {
+      "name": "os",
+      "build": "name:build",
+      "stages": [
+        {
+          "type": "org.osbuild.rpm",
+          "inputs": {},
+          "options": {}
+        },
+        {
+          "type": "org.osbuild.fix-bls",
+          "options": {}
+        },
+        {
+          "type": "org.osbuild.locale",
+          "options": {
+            "language": "C.UTF-8"
+          }
+        },
+        {
+          "type": "org.osbuild.hostname",
+          "options": {}
+        },
+        {
+          "type": "org.osbuild.timezone",
+          "options": {}
+        },
+        {
+          "type": "org.osbuild.machine-id",
+          "options": {}
+        }
+      ]
+    },
+    {
+      "name": "archive",
+      "build": "name:build",
+      "stages": [
+        {
+          "type": "org.osbuild.tar",
+          "inputs": {},
+          "options": {}
+        }
+      ]
+    },
+    {
+      "name": "xz",
+      "build": "name:build",
+      "stages": [
+        {
+          "type": "org.osbuild.xz",
+          "inputs": {},
+          "options": {}
+        }
+      ]
+    }
+  ],
+  "sources": {
+    "org.osbuild.librepo": {}
+  }
+}`),
+			ExpectedPipelines: &worker.PipelineNames{
+				Build:   []string{"build"},
+				Payload: []string{"os", "archive", "xz"},
+			},
+		},
+		"unknown-build": {
+			// The build property refers to a pipeline that doesn't exist. This
+			// manifest is invalid, so it will fail to validate in osbuild, but
+			// our function will work and ignore the invalid reference. This
+			// means the "build-pipeline" will not be identified as a build
+			// pipeline but instead be added to the Payload list along with
+			// "root-tree".
+			Manifest: []byte(`{
+	"version": "2",
+	"pipelines": [
+		{
+			"name": "build-pipeline"
+		},
+		{
+			"name": "root-tree",
+			"build": "build:not-a-pipeline"
+		}
+	],
+	"sources": []
+}`),
+			ExpectedPipelines: &worker.PipelineNames{
+				Payload: []string{"build-pipeline", "root-tree"},
+			},
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			assert := assert.New(t)
+			plNames, err := main.ParseManifestPipelines(tc.Manifest)
+			if tc.ExpectedError != "" {
+				assert.EqualError(err, tc.ExpectedError)
+				return
+			}
+
+			assert.NoError(err)
+			assert.Equal(tc.ExpectedPipelines, plNames)
+		})
+	}
+}