diff --git a/README.md b/README.md index a04708b..228daf5 100644 --- a/README.md +++ b/README.md @@ -271,6 +271,18 @@ limits: min: 1 ``` +### JSON Source (via URL) + +The JSON parser loads configuration from a URL, supporting `file://`, `http://`, and `https://` schemes. Schemeless paths are treated as local files. + +First, define a URL option: + +```go +var jsonConfigURL = zfg.URL("json.config", "file:///path/to/your/config.json", "URL for JSON configuration") +// Or for an optional remote config: +// var remoteJsonConfigURL = zfg.URL("json.remote", "", "Optional remote JSON config URL (http/https)") +``` + **Example Go config:** ```go @@ -278,6 +290,16 @@ zfg.Str("group.option", "", "hierarchical usage") zfg.Ints("numbers", nil, "slice of server configs") zfg.Map("limits", nil, "map of limits") ``` +Then, add it to `zerocfg.Parse`: + +```go +err := zerocfg.Parse( + // ... other providers + json.New(jsonConfigURL), +) +``` + +If the `json.config` URL option is not provided (or its default is empty), the JSON provider will be skipped without error. ## Advanced Usage diff --git a/json/parse.go b/json/parse.go new file mode 100644 index 0000000..330d3f1 --- /dev/null +++ b/json/parse.go @@ -0,0 +1,150 @@ +package json + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strings" +) + +type Parser struct { + sourceURL **url.URL + + conv func(any) string + awaited map[string]bool +} + +func New(sourceURL **url.URL) *Parser { + return &Parser{sourceURL: sourceURL} +} + +func (p *Parser) Type() string { + if p.sourceURL == nil || *(p.sourceURL) == nil { + return "json" + } + return fmt.Sprintf("json[%s]", (*(p.sourceURL)).String()) +} + +func fetchJSONData(u *url.URL) ([]byte, error) { + if u == nil { + return nil, fmt.Errorf("source URL is nil") + } + + var reader io.ReadCloser + var err error + + switch u.Scheme { + case "http", "https": + resp, httpErr := http.Get(u.String()) + if httpErr != nil { + return nil, fmt.Errorf("http get failed for %s: %w", u.String(), httpErr) + } + if resp.StatusCode != http.StatusOK { + resp.Body.Close() + return nil, fmt.Errorf("http status %s for %s", resp.Status, u.String()) + } + reader = resp.Body + case "file": + path := u.Path + if u.Host != "" && (os.PathSeparator == '\\' && strings.HasPrefix(path, "/")) { + path = "//" + u.Host + path + } else if u.Host != "" { + path = u.Host + path + } + reader, err = os.Open(path) + if err != nil { + return nil, fmt.Errorf("file open failed for %s: %w", path, err) + } + default: + if u.Scheme == "" && u.Path != "" { + reader, err = os.Open(u.Path) + if err != nil { + return nil, fmt.Errorf("local file open failed for %s: %w", u.Path, err) + } + } else { + return nil, fmt.Errorf("unsupported URL scheme: %q for %s", u.Scheme, u.String()) + } + } + defer reader.Close() + + data, err := io.ReadAll(reader) + if err != nil { + return nil, fmt.Errorf("read failed for %s: %w", u.String(), err) + } + return data, nil +} + +func (p *Parser) Parse(keys map[string]bool, conv func(any) string) (found, unknown map[string]string, err error) { + if p.sourceURL == nil || *(p.sourceURL) == nil { + return make(map[string]string), make(map[string]string), nil + } + + currentURL := *(p.sourceURL) + + data, err := fetchJSONData(currentURL) + if err != nil { + return nil, nil, fmt.Errorf("fetch json data from %s: %w", currentURL.String(), err) + } + + p.conv = conv + p.awaited = keys + + return p.parse(data) +} + +func (p *Parser) parse(data []byte) (found, unknown map[string]string, err error) { + if len(data) == 0 { + return nil, nil, fmt.Errorf("unmarshal json: empty json input") + } + var settings any + + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.UseNumber() + + if err = decoder.Decode(&settings); err != nil { + return nil, nil, fmt.Errorf("unmarshal json: %w", err) + } + + settingsMap, ok := settings.(map[string]any) + if !ok { + return nil, nil, fmt.Errorf("json root is not an object") + } + + found, unknown = p.flatten(settingsMap) + return found, unknown, nil +} + +func (p *Parser) flatten(settings map[string]any) (found, unknown map[string]string) { + found, unknown = make(map[string]string), make(map[string]string) + p.flattenDFS(settings, "", found, unknown) + return found, unknown +} + +func (p *Parser) flattenDFS(m map[string]any, prefix string, found, unknown map[string]string) { + for k, v := range m { + if v == nil { + continue + } + + newKey := k + if prefix != "" { + newKey = prefix + "." + k + } + + if p.awaited[newKey] { + found[newKey] = p.conv(v) + continue + } + + if subMap, ok := v.(map[string]any); ok { + p.flattenDFS(subMap, newKey, found, unknown) + continue + } + + unknown[newKey] = p.conv(v) + } +} diff --git a/json/parse_test.go b/json/parse_test.go new file mode 100644 index 0000000..151a2d6 --- /dev/null +++ b/json/parse_test.go @@ -0,0 +1,227 @@ +package json_test + +import ( + "net/url" + "os" + "path/filepath" + "strings" + "testing" + + zfg "github.com/chaindead/zerocfg" + "github.com/chaindead/zerocfg/json" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func fileURL(t *testing.T, path string) *url.URL { + absPath, err := filepath.Abs(path) + require.NoError(t, err) + u := &url.URL{ + Scheme: "file", + Path: filepath.ToSlash(absPath), + } + return u +} + +func TestParse(t *testing.T) { + tests := []struct { + name string + input string + awaited map[string]bool + found map[string]string + unknown map[string]string + wantErr bool + errText string + }{ + { + name: "simple key-value", + input: `{"str": "name", "int": 1}`, + awaited: map[string]bool{ + "str": true, + }, + found: map[string]string{ + "str": `name`, + }, + unknown: map[string]string{ + "int": `1`, + }, + }, + { + name: "nested", + input: `{ + "host": "localhost", + "database": { + "port": 5432, + "credentials": { + "username": "admin" + } + } + }`, + awaited: map[string]bool{ + "database.credentials.username": true, + }, + found: map[string]string{ + "database.credentials.username": `admin`, + }, + unknown: map[string]string{ + "host": `localhost`, + "database.port": `5432`, + }, + }, + { + name: "array", + input: `{"tags": ["a", "b"]}`, + awaited: map[string]bool{ + "tags": true, + }, + found: map[string]string{ + "tags": `["a","b"]`, + }, + unknown: map[string]string{}, + }, + { + name: "map", + input: `{"options": {"k1": 1, "k2": "v2"}}`, + awaited: map[string]bool{ + "options": true, + }, + found: map[string]string{ + "options": `{"k1":1,"k2":"v2"}`, + }, + unknown: map[string]string{}, + }, + { + name: "null value is skipped", + input: `{"key1": "value1", "key2": null, "key3": 123}`, + awaited: map[string]bool{ + "key1": true, + "key2": true, + "key3": true, + }, + found: map[string]string{ + "key1": `value1`, + "key3": `123`, + }, + unknown: map[string]string{}, + }, + { + name: "json root is not an object - array", + input: `[1, 2, 3]`, + wantErr: true, + errText: "json root is not an object", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.found == nil { + tt.found = map[string]string{} + } + if tt.unknown == nil { + tt.unknown = map[string]string{} + } + + var u *url.URL + if tt.input != "" { + tmpFile := tempFile(t, tt.input) + u = fileURL(t, tmpFile) + } + urlPtr := &u + + p := json.New(urlPtr) + + found, unknown, err := p.Parse(tt.awaited, zfg.ToString) + + if tt.wantErr { + assert.Error(t, err) + if tt.errText != "" { + assert.True(t, strings.Contains(err.Error(), tt.errText), "Error message mismatch: expected to contain '%s', got '%s'", tt.errText, err.Error()) + } + } else { + require.NoError(t, err) + assert.Equal(t, tt.found, found, "Found map mismatch") + assert.Equal(t, tt.unknown, unknown, "Unknown map mismatch") + } + }) + } +} + +func TestParse_MalformedError(t *testing.T) { + tmpFile := tempFile(t, `{"invalid": "json",}`) + u := fileURL(t, tmpFile) + urlPtr := &u + p := json.New(urlPtr) + + _, _, err := p.Parse(map[string]bool{}, zfg.ToString) + assert.Error(t, err) + assert.Contains(t, err.Error(), "unmarshal json") +} + +func TestParse_EmptyInputError(t *testing.T) { + tmpFile := tempFile(t, ``) + u := fileURL(t, tmpFile) + urlPtr := &u + p := json.New(urlPtr) + + _, _, err := p.Parse(map[string]bool{}, zfg.ToString) + assert.Error(t, err) + assert.Contains(t, err.Error(), "empty json input") +} + +func TestParse_FileNotExistError(t *testing.T) { + nonExistentPath := filepath.Join(t.TempDir(), "no_such_file.json") + u := fileURL(t, nonExistentPath) + urlPtr := &u + p := json.New(urlPtr) + + _, _, err := p.Parse(map[string]bool{"some.key": true}, zfg.ToString) + require.Error(t, err) + assert.Contains(t, err.Error(), "fetch json data") + assert.True(t, strings.Contains(err.Error(), "no such file") || strings.Contains(err.Error(), "cannot find the path"), "Error message: %v", err) +} + +func TestParse_NilURL(t *testing.T) { + var nilURL *url.URL + urlPtr := &nilURL + p := json.New(urlPtr) + found, unknown, err := p.Parse(map[string]bool{}, zfg.ToString) + require.NoError(t, err) + assert.Empty(t, found) + assert.Empty(t, unknown) +} + +func TestParse_UnsupportedScheme(t *testing.T) { + u, err := url.Parse("ftp://example.com/file.json") + require.NoError(t, err) + urlPtr := &u + p := json.New(urlPtr) + _, _, parseErr := p.Parse(map[string]bool{}, zfg.ToString) + require.Error(t, parseErr) + assert.Contains(t, parseErr.Error(), "unsupported URL scheme: \"ftp\"") +} + +func TestParse_SchemelessPathAsFile(t *testing.T) { + tmpFileContent := `{"schemeless": "ok"}` + tmpFilePath := tempFile(t, tmpFileContent) + + u := &url.URL{Path: tmpFilePath} + urlPtr := &u + p := json.New(urlPtr) + + found, _, err := p.Parse(map[string]bool{"schemeless": true}, zfg.ToString) + require.NoError(t, err) + require.Equal(t, map[string]string{"schemeless": "ok"}, found) +} + +func tempFile(t *testing.T, data string) string { + f, err := os.CreateTemp(t.TempDir(), "test-*.json") + require.NoError(t, err) + + if data != "" { + _, err = f.WriteString(data) + require.NoError(t, err) + } + name := f.Name() + require.NoError(t, f.Close()) + return name +} diff --git a/z_test.go b/z_test.go index 798d04f..14c8494 100644 --- a/z_test.go +++ b/z_test.go @@ -2,6 +2,7 @@ package zerocfg import ( "net" + neturl "net/url" "reflect" "strings" "testing" @@ -152,13 +153,43 @@ func Test_ValueOk(t *testing.T) { }, map[string]any{"float": 1., "str": "val"}) }, }, + { + varType: "url", + init: func() (reg func() any, val any, src map[string]any) { + defaultURLStr := "http://default.com/path" + valStr := "https://example.com/another?query=1" + + parsedStdURL, err := neturl.Parse(valStr) + require.NoError(t, err) + expectedVal := urlValue(*parsedStdURL) + + reg = func() any { + return URL(name, defaultURLStr, "url description") + } + return reg, expectedVal, map[string]any{name: valStr} + }, + }, + { + varType: "url empty", + init: func() (reg func() any, val any, src map[string]any) { + defaultURLStr := "http://default.com/path" + valStr := "" + + expectedVal := urlValue{} + + reg = func() any { + return URL(name, defaultURLStr, "url description") + } + return reg, expectedVal, map[string]any{name: valStr} + }, + }, } dereference := func(t *testing.T, v any) any { val := reflect.ValueOf(v) require.True(t, val.Kind() == reflect.Ptr, "val must be a pointer") - - return val.Elem().Interface() + elem := val.Elem() + return elem.Interface() } for _, tt := range tests { @@ -172,21 +203,48 @@ func Test_ValueOk(t *testing.T) { require.NoError(t, err) actual := dereference(t, ptr) - require.EqualValues(t, expected, actual) + + if tt.varType == "url" || tt.varType == "url empty" { + actualURLValue, okActual := actual.(urlValue) + require.True(t, okActual, "actual should be urlValue") + expectedURLValue, okExpected := expected.(urlValue) + require.True(t, okExpected, "expected should be urlValue") + + actualStdURL := neturl.URL(actualURLValue) + expectedStdURL := neturl.URL(expectedURLValue) + require.Equal(t, expectedStdURL.String(), actualStdURL.String()) + } else { + require.EqualValues(t, expected, actual) + } // check Set and ToString is compatible node, ok := c.vs[name] require.True(t, ok) - err = node.Value.Set(ToString(actual)) + stringRepresentation := ToString(actual) + + err = node.Value.Set(stringRepresentation) require.NoError(t, err) updatedActual := dereference(t, ptr) - require.Equal(t, actual, updatedActual) + + if tt.varType == "url" || tt.varType == "url empty" { + updatedActualURLValue, okUpdated := updatedActual.(urlValue) + require.True(t, okUpdated) + actualURLValue, okActual := actual.(urlValue) + require.True(t, okActual) + + updatedStdURL := neturl.URL(updatedActualURLValue) + actualStdURL := neturl.URL(actualURLValue) + require.Equal(t, actualStdURL.String(), updatedStdURL.String()) + + } else { + require.Equal(t, actual, updatedActual) + } // check type name - awaitedType := strings.Split(tt.varType, " ")[0] - require.Equal(t, awaitedType, node.Value.Type()) + cleanVarType := strings.Split(tt.varType, " ")[0] + require.Equal(t, cleanVarType, node.Value.Type()) }) } } diff --git a/z_url.go b/z_url.go new file mode 100644 index 0000000..a173946 --- /dev/null +++ b/z_url.go @@ -0,0 +1,56 @@ +package zerocfg + +import ( + "fmt" + "net/url" +) + +type urlValue url.URL + +func newURLValue(val urlValue, p *urlValue) Value { + return p +} + +func (u *urlValue) Set(s string) error { + if s == "" { + *u = urlValue{} + return nil + } + parsedURL, err := url.Parse(s) + if err != nil { + return fmt.Errorf("parsing URL %q: %w", s, err) + } + if parsedURL == nil { + *u = urlValue{} + return nil + } + *u = urlValue(*parsedURL) + return nil +} + +func (u *urlValue) Type() string { + return "url" +} + +func (u *urlValue) String() string { + if u == nil { + return "" + } + tempOriginalURL := url.URL(*u) + return tempOriginalURL.String() +} + +func URL(name string, defValStr string, desc string, opts ...OptNode) *urlValue { + var initialStruct urlValue + + if defValStr != "" { + parsedURL, err := url.Parse(defValStr) + if err != nil { + panic(fmt.Sprintf("invalid default URL value %q for %q: %v", defValStr, name, err)) + } + if parsedURL != nil { + initialStruct = urlValue(*parsedURL) + } + } + return Any(name, initialStruct, desc, newURLValue, opts...) +}