From 551c8c0f01df2d212fa992155808db13390709b7 Mon Sep 17 00:00:00 2001 From: Tom Wright <935867+TomWright@users.noreply.github.com> Date: Fri, 22 May 2026 12:14:32 +0100 Subject: [PATCH 1/2] Fix YAML output escaping emoji and supplementary-plane Unicode (#437) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Post-process yaml.Marshal output to unescape \UNNNNNNNN sequences back to UTF-8 for supplementary-plane characters (U+10000–U+10FFFF). The upstream yaml library's isPrintable does not handle 4-byte UTF-8, causing all such characters to be escaped. --- parsing/yaml/yaml_test.go | 16 ++++++++++++ parsing/yaml/yaml_writer.go | 50 ++++++++++++++++++++++++++++++++++++- 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/parsing/yaml/yaml_test.go b/parsing/yaml/yaml_test.go index f87ac49f..d80d5306 100644 --- a/parsing/yaml/yaml_test.go +++ b/parsing/yaml/yaml_test.go @@ -674,6 +674,22 @@ single: 'Tom' `, }.run) + t.Run("emoji preserved as UTF-8", rwTestCase{ + in: "US: \"\U0001F1FA\U0001F1F8\"\n", + }.run) + + t.Run("simple emoji preserved", rwTestCase{ + in: "face: \"\U0001F600\"\n", + }.run) + + t.Run("mixed ASCII and emoji", rwTestCase{ + in: "msg: \"hello \U0001F30D world\"\n", + }.run) + + t.Run("supplementary plane unicode", rwTestCase{ + in: "clef: \"\U0001D11E\"\ncjk: \"\U00020000\"\nmath: \"\U0001D54F\"\n", + }.run) + t.Run("yaml expansion budget resets per document", func(t *testing.T) { reader, err := parsing.Format("yaml").NewReader(parsing.DefaultReaderOptions()) if err != nil { diff --git a/parsing/yaml/yaml_writer.go b/parsing/yaml/yaml_writer.go index c56e6138..1b838fa0 100644 --- a/parsing/yaml/yaml_writer.go +++ b/parsing/yaml/yaml_writer.go @@ -1,7 +1,10 @@ package yaml import ( + "bytes" + "encoding/hex" "fmt" + "unicode/utf8" "github.com/tomwright/dasel/v3/model" "github.com/tomwright/dasel/v3/parsing" @@ -29,7 +32,52 @@ func (j *yamlWriter) Write(value *model.Value) ([]byte, error) { if err != nil { return nil, err } - return yaml.Marshal(res) + out, err := yaml.Marshal(res) + if err != nil { + return nil, err + } + return unescapeYAMLUnicode(out), nil +} + +// unescapeYAMLUnicode replaces \UNNNNNNNN escape sequences (8 hex digits) with +// the corresponding UTF-8 bytes. The yaml library incorrectly escapes +// supplementary-plane characters (U+10000–U+10FFFF) because its isPrintable +// function does not handle 4-byte UTF-8 sequences. +func unescapeYAMLUnicode(data []byte) []byte { + marker := []byte(`\U`) + var result []byte + for { + idx := bytes.Index(data, marker) + if idx == -1 { + break + } + // Need exactly 8 hex digits after \U + if idx+10 > len(data) { + result = append(result, data[:idx+2]...) + data = data[idx+2:] + continue + } + hexBytes := data[idx+2 : idx+10] + decoded, err := hex.DecodeString(string(hexBytes)) + if err != nil || len(decoded) != 4 { + result = append(result, data[:idx+2]...) + data = data[idx+2:] + continue + } + r := rune(decoded[0])<<24 | rune(decoded[1])<<16 | rune(decoded[2])<<8 | rune(decoded[3]) + if r < 0x10000 || r > 0x10FFFF || !utf8.ValidRune(r) { + result = append(result, data[:idx+10]...) + data = data[idx+10:] + continue + } + result = append(result, data[:idx]...) + var buf [4]byte + n := utf8.EncodeRune(buf[:], r) + result = append(result, buf[:n]...) + data = data[idx+10:] + } + result = append(result, data...) + return result } func (yv *yamlValue) ToNode() (*yaml.Node, error) { From 6c88cc9dead3eb21e4dec498e5ee50d54615f437 Mon Sep 17 00:00:00 2001 From: Tom Wright <935867+TomWright@users.noreply.github.com> Date: Fri, 22 May 2026 12:16:25 +0100 Subject: [PATCH 2/2] Update CHANGELOG with YAML unicode fix --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 332ac639..a7dd4b2a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- Fix YAML output escaping emoji and supplementary-plane Unicode characters (U+10000–U+10FFFF) to `\UNNNNNNNN` sequences instead of preserving UTF-8 ([#437](https://github.com/TomWright/dasel/issues/437)). + ## [v3.11.0] - 2026-05-19 ### Added