Support YAML only configuration

cmd/amazon-cloudwatch-agent/amazon-cloudwatch-agent.go

@@ -434,12 +434,19 @@ func mergeConfigs(configPaths []string) (*confmap.Conf, error) {
 		for _, configPath := range configPaths {
 			loaders = append(loaders, confmap.NewFileLoader(configPath))
 		}
-		result := confmap.New()
+		var result *confmap.Conf
 		for _, loader := range loaders {
 			conf, err := loader.Load()
 			if err != nil {
+				if errors.Is(err, os.ErrNotExist) {
+					log.Printf("D! Skipping non-existent OTEL config: %s", loader.ID())
+					continue
+				}
 				return nil, fmt.Errorf("failed to load OTEL configs: %w", err)
 			}
+			if result == nil {
+				result = confmap.New()
+			}
 			if err = result.Merge(conf); err != nil {
 				return nil, fmt.Errorf("failed to merge OTEL configs: %w", err)
 			}

cmd/amazon-cloudwatch-agent/amazon-cloudwatch-agent_test.go

@@ -89,9 +89,17 @@ service:
 		wantErr     bool
 	}{
 		"WithInvalidFile": {
-			input:   []string{filepath.Join("not", "a", "file"), filepath.Join("testdata", "base.yaml")},
+			input:   []string{filepath.Join("testdata", "invalid.yaml"), filepath.Join("testdata", "base.yaml")},
 			wantErr: true,
 		},
+		"WithAllMissingFiles": {
+			input: []string{filepath.Join("not", "a", "file"), filepath.Join("also", "not", "a", "file")},
+			want:  nil,
+		},
+		"WithMissingFile": {
+			input: []string{filepath.Join("not", "a", "file"), filepath.Join("testdata", "base.yaml")},
+			want:  mustLoadFromFile(t, filepath.Join("testdata", "base.yaml")),
+		},
 		"WithNoMerge": {
 			input:   []string{filepath.Join("testdata", "base.yaml")},
 			wantErr: false,

cmd/amazon-cloudwatch-agent/testdata/invalid.yaml

@@ -0,0 +1 @@
+invalid: yaml: content: [

internal/merge/confmap/load.go

@@ -13,6 +13,7 @@ import (
 
 type Loader interface {
 	Load() (*Conf, error)
+	ID() string
 }
 
 type FileLoader struct {
@@ -23,6 +24,10 @@ func NewFileLoader(path string) *FileLoader {
 	return &FileLoader{path: path}
 }
 
+func (f *FileLoader) ID() string {
+	return f.path
+}
+
 func (f *FileLoader) Load() (*Conf, error) {
 	// Clean the path before using it.
 	content, err := os.ReadFile(filepath.Clean(f.path))
@@ -41,6 +46,10 @@ func NewByteLoader(id string, content []byte) *ByteLoader {
 	return &ByteLoader{id: id, content: content}
 }
 
+func (b *ByteLoader) ID() string {
+	return b.id
+}
+
 func (b *ByteLoader) Load() (*Conf, error) {
 	var rawConf map[string]any
 	if err := yaml.Unmarshal(b.content, &rawConf); err != nil {

internal/merge/confmap/load_test.go

@@ -12,6 +12,7 @@ import (
 
 func TestFileLoader(t *testing.T) {
 	loader := NewFileLoader(filepath.Join("not", "a", "file"))
+	assert.Equal(t, filepath.Join("not", "a", "file"), loader.ID())
 	got, err := loader.Load()
 	assert.Error(t, err)
 	assert.Nil(t, got)
@@ -26,6 +27,7 @@ func TestByteLoader(t *testing.T) {
   nop/1:
 `
 	loader := NewByteLoader("invalid-yaml", []byte("string"))
+	assert.Equal(t, "invalid-yaml", loader.ID())
 	got, err := loader.Load()
 	assert.Error(t, err)
 	assert.Nil(t, got)

tool/translator/translator.go

@@ -23,6 +23,7 @@ import (
 const (
 	exitErrorMessage   = "configuration validation first phase failed. Agent version: %v. Verify the JSON input is only using features supported by this version"
 	exitSuccessMessage = "Configuration validation first phase succeeded"
+	exitSkipMessage    = "Configuration validation first phase skipped. No JSON files found"
 	version            = "1.0"
 	envConfigFileName  = "env-config.json"
 	yamlConfigFileName = "amazon-cloudwatch-agent.yaml"
@@ -98,12 +99,23 @@ func (ct *ConfigTranslator) Translate() (err error) {
 		}
 	}()
 
+	tomlConfigPath := cmdutil.GetTomlConfigPath(ct.ctx.OutputTomlFilePath())
+	tomlConfigDir := filepath.Dir(tomlConfigPath)
+	yamlConfigPath := filepath.Join(tomlConfigDir, yamlConfigFileName)
+
 	mergedJSONConfigMap, err := cmdutil.GenerateMergedJsonConfigMap(ct.ctx)
-	if err != nil {
+	onlyYAML := errors.Is(err, cmdutil.ErrOnlyYAML)
+	if err != nil && !onlyYAML {
 		return fmt.Errorf("failed to generate merged json config: %v", err)
 	}
+	if onlyYAML {
+		log.Println(exitSkipMessage)
+		// YAML-only configs still require a generated TOML for agent and logging configuration.
+		// TOML translation requires a non-nil map.
+		mergedJSONConfigMap = map[string]any{}
+	}
 
-	if !ct.ctx.RunInContainer() {
+	if !onlyYAML && !ct.ctx.RunInContainer() {
 		current, err := user.Current()
 		if err == nil && current.Name == "****" {
 			runAsUser, err := userutil.DetectRunAsUser(mergedJSONConfigMap)
@@ -114,24 +126,30 @@ func (ct *ConfigTranslator) Translate() (err error) {
 		}
 	}
 
-	tomlConfigPath := cmdutil.GetTomlConfigPath(ct.ctx.OutputTomlFilePath())
-	tomlConfigDir := filepath.Dir(tomlConfigPath)
-	yamlConfigPath := filepath.Join(tomlConfigDir, yamlConfigFileName)
 	tomlConfig, err := cmdutil.TranslateJsonMapToTomlConfig(mergedJSONConfigMap)
 	if err != nil {
 		return fmt.Errorf("failed to generate TOML configuration validation content: %v", err)
 	}
-	yamlConfig, err := cmdutil.TranslateJsonMapToYamlConfig(mergedJSONConfigMap)
-	if err != nil && !errors.Is(err, pipeline.ErrNoPipelines) {
-		return fmt.Errorf("failed to generate YAML configuration validation content: %v", err)
+	var yamlConfig any
+	if !onlyYAML {
+		yamlConfig, err = cmdutil.TranslateJsonMapToYamlConfig(mergedJSONConfigMap)
+		if err != nil && !errors.Is(err, pipeline.ErrNoPipelines) {
+			return fmt.Errorf("failed to generate YAML configuration validation content: %v", err)
+		}
 	}
 	if err = cmdutil.ConfigToTomlFile(tomlConfig, tomlConfigPath); err != nil {
 		return fmt.Errorf("failed to create the configuration TOML validation file: %v", err)
 	}
-	if err = cmdutil.ConfigToYamlFile(yamlConfig, yamlConfigPath); err != nil {
-		return fmt.Errorf("failed to create the configuration YAML validation file: %v", err)
+	if yamlConfig != nil {
+		if err = cmdutil.ConfigToYamlFile(yamlConfig, yamlConfigPath); err != nil {
+			return fmt.Errorf("failed to create the configuration YAML validation file: %v", err)
+		}
+	} else {
+		_ = os.Remove(yamlConfigPath)
+	}
+	if !onlyYAML {
+		log.Println(exitSuccessMessage)
 	}
-	log.Println(exitSuccessMessage)
 
 	envConfigPath := filepath.Join(tomlConfigDir, envConfigFileName)
 	cmdutil.TranslateJsonMapToEnvConfigFile(mergedJSONConfigMap, envConfigPath)

tool/translator/translator_test.go

@@ -0,0 +1,43 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: MIT
+
+package translator
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/aws/amazon-cloudwatch-agent/translator"
+	translatorcontext "github.com/aws/amazon-cloudwatch-agent/translator/context"
+	translatorutil "github.com/aws/amazon-cloudwatch-agent/translator/util"
+)
+
+func TestTranslate_OnlyYAML(t *testing.T) {
+	orig := translatorutil.DetectRegion
+	translatorutil.DetectRegion = func(string, map[string]string) (string, string) {
+		return "us-east-1", "mock"
+	}
+	defer func() { translatorutil.DetectRegion = orig }()
+
+	translator.ResetMessages()
+	translatorcontext.ResetContext()
+
+	tmpDir := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(tmpDir, "config.yaml"), nil, 0600))
+
+	tomlPath := filepath.Join(tmpDir, "output.toml")
+	// Pre-create the YAML output file to verify it gets removed.
+	yamlPath := filepath.Join(tmpDir, yamlConfigFileName)
+	require.NoError(t, os.WriteFile(yamlPath, nil, 0600))
+
+	ct, err := NewConfigTranslator("linux", "", tmpDir, tomlPath, "ec2", "", "default")
+	require.NoError(t, err)
+
+	assert.NoError(t, ct.Translate())
+	assert.FileExists(t, tomlPath)
+	assert.NoFileExists(t, yamlPath)
+}

translator/cmdutil/translatorutil.go

@@ -4,6 +4,7 @@
 package cmdutil
 
 import (
+	"errors"
 	"fmt"
 	"log"
 	"os"
@@ -36,6 +37,8 @@ const (
 	defaultTomlConfigName    = "CWAgent.conf"
 )
 
+var ErrOnlyYAML = errors.New("only YAML files detected")
+
 // TranslateJsonMapToEnvConfigFile populates env-config.json based on the input json config.
 func TranslateJsonMapToEnvConfigFile(jsonConfigValue map[string]interface{}, envConfigPath string) {
 	if envConfigPath == "" {
@@ -115,6 +118,7 @@ func GenerateMergedJsonConfigMap(ctx *context.Context) (map[string]interface{},
 	// for the append operation when the existing file name and new .tmp file name have diff
 	// only for the ".tmp" suffix, i.e. it is override operation even it says append.
 	var jsonConfigMapMap = make(map[string]map[string]interface{})
+	var foundYAML bool
 
 	if ctx.MultiConfig() == "append" || ctx.MultiConfig() == "remove" {
 		// backwards compatible for the old json config file
@@ -163,6 +167,7 @@ func GenerateMergedJsonConfigMap(ctx *context.Context) (map[string]interface{},
 				ext = filepath.Ext(key)
 				// skip .yaml files
 				if ext == constants.FileSuffixYAML {
+					foundYAML = true
 					return nil
 				}
 				if ctx.MultiConfig() == "default" || ctx.MultiConfig() == "append" {
@@ -176,6 +181,7 @@ func GenerateMergedJsonConfigMap(ctx *context.Context) (map[string]interface{},
 				}
 			} else if ext == constants.FileSuffixYAML {
 				// skip .yaml files
+				foundYAML = true
 				return nil
 			} else {
 				// non .tmp / existing files
@@ -208,6 +214,9 @@ func GenerateMergedJsonConfigMap(ctx *context.Context) (map[string]interface{},
 			}
 			jsonConfigMapMap[config.CWConfigContent] = jm
 		}
+		if foundYAML && len(jsonConfigMapMap) == 0 {
+			return nil, ErrOnlyYAML
+		}
 	}
 
 	defaultConfig, err := translatorUtil.GetDefaultJsonConfigMap(ctx.Os(), ctx.Mode())

translator/cmdutil/translatorutil_test.go

@@ -6,13 +6,15 @@ package cmdutil
 import (
 	"encoding/json"
 	"os"
-	"path"
+	"path/filepath"
 	"regexp"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	"github.com/aws/amazon-cloudwatch-agent/cfg/envconfig"
+	translatorcontext "github.com/aws/amazon-cloudwatch-agent/translator/context"
 	"github.com/aws/amazon-cloudwatch-agent/translator/util"
 )
 
@@ -24,7 +26,7 @@ func TestTranslateJsonMapToEnvConfigFile(t *testing.T) {
 			"aws_sdk_log_level": "loglevel",
 		},
 	}
-	envConfigPath := path.Join(t.TempDir(), "env-config.json")
+	envConfigPath := filepath.Join(t.TempDir(), "env-config.json")
 	expectedFile := "testdata/env-config.json"
 
 	TranslateJsonMapToEnvConfigFile(jsonConfigValue, envConfigPath)
@@ -233,3 +235,80 @@ func checkIfSchemaValidateAsExpected(t *testing.T, jsonInputPath string, shouldS
 		assert.False(t, shouldSuccess, "It should pass the schemaValidation!")
 	}
 }
+
+func TestGenerateMergedJsonConfigMap_OnlyYAML(t *testing.T) {
+	testCases := map[string]string{
+		"WithYAMLFile":    "config.yaml",
+		"WithYAMLTmpFile": "config.yaml.tmp",
+	}
+	for name, file := range testCases {
+		t.Run(name, func(t *testing.T) {
+			tmpDir := t.TempDir()
+			require.NoError(t, os.WriteFile(filepath.Join(tmpDir, file), nil, 0600))
+
+			translatorcontext.ResetContext()
+			ctx := translatorcontext.CurrentContext()
+			ctx.SetInputJsonDirPath(tmpDir)
+			ctx.SetMultiConfig("default")
+
+			got, err := GenerateMergedJsonConfigMap(ctx)
+
+			assert.ErrorIs(t, err, ErrOnlyYAML)
+			assert.Nil(t, got)
+		})
+	}
+}
+
+func TestGenerateMergedJsonConfigMap_MixedJSONAndYAML(t *testing.T) {
+	for _, mode := range []string{"append", "remove"} {
+		t.Run(mode, func(t *testing.T) {
+			tmpDir := t.TempDir()
+			require.NoError(t, os.WriteFile(filepath.Join(tmpDir, "config.json"), []byte(`{"agent":{"debug":true}}`), 0600))
+			require.NoError(t, os.WriteFile(filepath.Join(tmpDir, "otel.yaml"), nil, 0600))
+
+			translatorcontext.ResetContext()
+			ctx := translatorcontext.CurrentContext()
+			ctx.SetInputJsonDirPath(tmpDir)
+			ctx.SetMultiConfig(mode)
+
+			result, err := GenerateMergedJsonConfigMap(ctx)
+
+			assert.NoError(t, err)
+			assert.NotNil(t, result)
+		})
+	}
+}
+
+func TestGenerateMergedJsonConfigMap_DefaultModeWithTmpJSONAndYAML(t *testing.T) {
+	tmpDir := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(tmpDir, "config.json.tmp"), []byte(`{"agent":{"debug":true}}`), 0600))
+	require.NoError(t, os.WriteFile(filepath.Join(tmpDir, "otel.yaml"), nil, 0600))
+
+	translatorcontext.ResetContext()
+	ctx := translatorcontext.CurrentContext()
+	ctx.SetInputJsonDirPath(tmpDir)
+	ctx.SetMultiConfig("default")
+
+	result, err := GenerateMergedJsonConfigMap(ctx)
+
+	assert.NoError(t, err)
+	assert.NotNil(t, result)
+}
+
+func TestGenerateMergedJsonConfigMap_EnvVarJSONWithYAML(t *testing.T) {
+	tmpDir := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(tmpDir, "otel.yaml"), nil, 0600))
+
+	t.Setenv(envconfig.RunInContainer, envconfig.TrueValue)
+	t.Setenv(envconfig.CWConfigContent, `{"agent":{"debug":true}}`)
+
+	translatorcontext.ResetContext()
+	ctx := translatorcontext.CurrentContext()
+	ctx.SetInputJsonDirPath(tmpDir)
+	ctx.SetMultiConfig("default")
+
+	result, err := GenerateMergedJsonConfigMap(ctx)
+
+	assert.NoError(t, err)
+	assert.NotNil(t, result)
+}