Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

## [Unreleased]

### Fixed

- Fix YAML output escaping emoji and supplementary-plane Unicode characters (U+10000–U+10FFFF) to `\UNNNNNNNN` sequences instead of preserving UTF-8 ([#437](https://github.com/TomWright/dasel/issues/437)).

## [v3.11.0] - 2026-05-19

### Added
Expand Down
16 changes: 16 additions & 0 deletions parsing/yaml/yaml_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -674,6 +674,22 @@ single: 'Tom'
`,
}.run)

t.Run("emoji preserved as UTF-8", rwTestCase{
in: "US: \"\U0001F1FA\U0001F1F8\"\n",
}.run)

t.Run("simple emoji preserved", rwTestCase{
in: "face: \"\U0001F600\"\n",
}.run)

t.Run("mixed ASCII and emoji", rwTestCase{
in: "msg: \"hello \U0001F30D world\"\n",
}.run)

t.Run("supplementary plane unicode", rwTestCase{
in: "clef: \"\U0001D11E\"\ncjk: \"\U00020000\"\nmath: \"\U0001D54F\"\n",
}.run)

t.Run("yaml expansion budget resets per document", func(t *testing.T) {
reader, err := parsing.Format("yaml").NewReader(parsing.DefaultReaderOptions())
if err != nil {
Expand Down
50 changes: 49 additions & 1 deletion parsing/yaml/yaml_writer.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
package yaml

import (
"bytes"
"encoding/hex"
"fmt"
"unicode/utf8"

"github.com/tomwright/dasel/v3/model"
"github.com/tomwright/dasel/v3/parsing"
Expand Down Expand Up @@ -29,7 +32,52 @@ func (j *yamlWriter) Write(value *model.Value) ([]byte, error) {
if err != nil {
return nil, err
}
return yaml.Marshal(res)
out, err := yaml.Marshal(res)
if err != nil {
return nil, err
}
return unescapeYAMLUnicode(out), nil
}

// unescapeYAMLUnicode replaces \UNNNNNNNN escape sequences (8 hex digits) with
// the corresponding UTF-8 bytes. The yaml library incorrectly escapes
// supplementary-plane characters (U+10000–U+10FFFF) because its isPrintable
// function does not handle 4-byte UTF-8 sequences.
func unescapeYAMLUnicode(data []byte) []byte {
marker := []byte(`\U`)
var result []byte
for {
idx := bytes.Index(data, marker)
if idx == -1 {
break
}
// Need exactly 8 hex digits after \U
if idx+10 > len(data) {
result = append(result, data[:idx+2]...)
data = data[idx+2:]
continue
}
hexBytes := data[idx+2 : idx+10]
decoded, err := hex.DecodeString(string(hexBytes))
if err != nil || len(decoded) != 4 {
result = append(result, data[:idx+2]...)
data = data[idx+2:]
continue
}
r := rune(decoded[0])<<24 | rune(decoded[1])<<16 | rune(decoded[2])<<8 | rune(decoded[3])
if r < 0x10000 || r > 0x10FFFF || !utf8.ValidRune(r) {
result = append(result, data[:idx+10]...)
data = data[idx+10:]
continue
}
result = append(result, data[:idx]...)
var buf [4]byte
n := utf8.EncodeRune(buf[:], r)
result = append(result, buf[:n]...)
data = data[idx+10:]
}
result = append(result, data...)
return result
}

func (yv *yamlValue) ToNode() (*yaml.Node, error) {
Expand Down
Loading