Snowflake: decode octal/hex/unicode string-literal escapes (#17)

* Snowflake: decode octal/hex/unicode string-literal escapes

Decode \ooo, \xhh and \uhhhh as Unicode code points, keep \0 as NUL,
collapse unrecognized letters (\a, \Z) to the bare letter, and raise a
tokenizer error on malformed \x / \u — gated behind a new
supports_snowflake_string_literal_escapes dialect capability so other
backslash-escaping dialects keep MySQL/BigQuery semantics.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

* Satisfy rust-1.97 clippy on main (useless_borrows_in_formatting, needless_return_with_question_mark)

Pre-existing lints in ast/parser/tests/examples surfaced by the stable
toolchain advancing to rust-1.97 (main last passed CI on the older
stable). Mechanical fixes only; no behaviour change. Unrelated to the
escape-parity commit but required to get this PR's lint job green.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

---------

Co-authored-by: wojpadlo <wojciech.padlo@localstack.cloud>
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>

Author: 3 people

examples/cli.rs

@@ -71,9 +71,9 @@ $ cargo run --example cli - [--dialectname]
             .expect("failed to read from stdin");
         String::from_utf8(buf).expect("stdin content wasn't valid utf8")
     } else {
-        println!("Parsing from file '{}' using {:?}", &filename, dialect);
+        println!("Parsing from file '{}' using {:?}", filename, dialect);
         fs::read_to_string(&filename)
-            .unwrap_or_else(|_| panic!("Unable to read the file {}", &filename))
+            .unwrap_or_else(|_| panic!("Unable to read the file {}", filename))
     };
     let without_bom = if contents.chars().next().unwrap() as u64 != 0xfeff {
         contents.as_str()

src/ast/ddl.rs

@@ -4735,7 +4735,7 @@ impl fmt::Display for AlterTable {
         if self.only {
             write!(f, "ONLY ")?;
         }
-        write!(f, "{} ", &self.name)?;
+        write!(f, "{} ", self.name)?;
         if let Some(cluster) = &self.on_cluster {
             write!(f, "ON CLUSTER {cluster} ")?;
         }

src/ast/mod.rs

@@ -13462,7 +13462,7 @@ impl fmt::Display for AlterUser {
         let has_props = !self.set_props.options.is_empty();
         if has_props {
             write!(f, " SET")?;
-            write!(f, " {}", &self.set_props)?;
+            write!(f, " {}", self.set_props)?;
         }
         if !self.unset_props.is_empty() {
             write!(f, " UNSET {}", display_comma_separated(&self.unset_props))?;

src/ast/query.rs

@@ -3530,7 +3530,7 @@ pub struct LockClause {
 
 impl fmt::Display for LockClause {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "FOR {}", &self.lock_type)?;
+        write!(f, "FOR {}", self.lock_type)?;
         if let Some(ref of) = self.of {
             write!(f, " OF {of}")?;
         }

src/dialect/mod.rs

@@ -283,6 +283,18 @@ pub trait Dialect: Debug + Any {
         false
     }
 
+    /// Determine whether the dialect decodes Snowflake's extended backslash
+    /// escape set inside single-quoted string literals: octal `\ooo`, hex
+    /// `\xhh` and unicode `\uhhhh` (each yielding a Unicode code point), with
+    /// unrecognized letters such as `\a` / `\Z` collapsing to the bare letter
+    /// and malformed `\x` / `\u` raising a tokenizer error. This differs from
+    /// the MySQL/BigQuery-style mapping used by
+    /// [`Self::supports_string_literal_backslash_escape`], so a dialect that
+    /// enables this must also enable that one.
+    fn supports_snowflake_string_literal_escapes(&self) -> bool {
+        false
+    }
+
     /// Determine whether the dialect strips the backslash when escaping LIKE wildcards (%, _).
     ///
     /// [MySQL] has a special case when escaping single quoted strings which leaves these unescaped
@@ -2016,6 +2028,10 @@ mod tests {
                 self.0.supports_string_literal_backslash_escape()
             }
 
+            fn supports_snowflake_string_literal_escapes(&self) -> bool {
+                self.0.supports_snowflake_string_literal_escapes()
+            }
+
             fn supports_filter_during_aggregation(&self) -> bool {
                 self.0.supports_filter_during_aggregation()
             }

src/dialect/snowflake.rs

@@ -172,6 +172,11 @@ impl Dialect for SnowflakeDialect {
         true
     }
 
+    // See https://docs.snowflake.com/en/sql-reference/data-types-text#escape-sequences-in-single-quoted-string-constants
+    fn supports_snowflake_string_literal_escapes(&self) -> bool {
+        true
+    }
+
     fn supports_within_after_array_aggregation(&self) -> bool {
         true
     }

src/parser/mod.rs

@@ -5010,7 +5010,7 @@ impl<'a> Parser<'a> {
         if self.parse_keyword(expected) {
             Ok(self.get_current_token().clone())
         } else {
-            self.expected_ref(format!("{:?}", &expected).as_str(), self.peek_token_ref())
+            self.expected_ref(format!("{:?}", expected).as_str(), self.peek_token_ref())
         }
     }
 
@@ -5023,7 +5023,7 @@ impl<'a> Parser<'a> {
         if self.parse_keyword(expected) {
             Ok(())
         } else {
-            self.expected_ref(format!("{:?}", &expected).as_str(), self.peek_token_ref())
+            self.expected_ref(format!("{:?}", expected).as_str(), self.peek_token_ref())
         }
     }
 
@@ -13974,7 +13974,7 @@ impl<'a> Parser<'a> {
                 }
                 Token::EOF => break,
                 token => {
-                    return Err(ParserError::ParserError(format!(
+                    Err(ParserError::ParserError(format!(
                         "Unexpected token in identifier: {token}"
                     )))?;
                 }
@@ -17298,7 +17298,7 @@ impl<'a> Parser<'a> {
                 where_clause = Some(self.parse_expr()?);
             } else {
                 let tok = self.peek_token_ref();
-                return parser_err!(
+                parser_err!(
                     format!(
                         "Expected one of DIMENSIONS, METRICS, FACTS or WHERE, got {}",
                         tok.token

src/tokenizer.rs

@@ -2287,17 +2287,21 @@ impl<'a> Tokenizer<'a> {
 
                     num_consecutive_quotes = 0;
 
-                    if let Some(next) = chars.peek() {
+                    if let Some(&next) = chars.peek() {
                         if !self.unescape
                             || (self.dialect.ignores_wildcard_escapes()
-                                && (*next == '%' || *next == '_'))
+                                && (next == '%' || next == '_'))
                         {
                             // In no-escape mode, the given query has to be saved completely
                             // including backslashes. Similarly, with ignore_like_wildcard_escapes,
                             // the backslash is not stripped.
                             s.push(ch);
-                            s.push(*next);
+                            s.push(next);
                             chars.next(); // consume next
+                        } else if self.dialect.supports_snowflake_string_literal_escapes() {
+                            let escape_loc = chars.location();
+                            chars.next(); // consume the escape indicator
+                            s.push(self.unescape_snowflake_escape(next, chars, escape_loc)?);
                         } else {
                             let n = match next {
                                 '0' => '\0',
@@ -2308,7 +2312,7 @@ impl<'a> Tokenizer<'a> {
                                 'r' => '\r',
                                 't' => '\t',
                                 'Z' => '\u{1a}',
-                                _ => *next,
+                                _ => next,
                             };
                             s.push(n);
                             chars.next(); // consume next
@@ -2331,6 +2335,174 @@ impl<'a> Tokenizer<'a> {
         self.tokenizer_error(error_loc, "Unterminated string literal")
     }
 
+    /// Decode a single Snowflake backslash escape (the backslash and the
+    /// `indicator` char have already been consumed). Octal `\ooo`, hex `\xhh`
+    /// and unicode `\uhhhh` each yield a Unicode code point; recognized control
+    /// letters map to their control character; every other letter collapses to
+    /// itself. Malformed `\x` / `\u` raise a tokenizer error, matching real
+    /// Snowflake.
+    fn unescape_snowflake_escape(
+        &self,
+        indicator: char,
+        chars: &mut State,
+        loc: Location,
+    ) -> Result<char, TokenizerError> {
+        match indicator {
+            'x' => {
+                let mut digits = String::new();
+                while digits.len() < 2 {
+                    match chars.peek() {
+                        Some(c) if c.is_ascii_hexdigit() => {
+                            if let Some(c) = chars.next() {
+                                digits.push(c);
+                            }
+                        }
+                        _ => break,
+                    }
+                }
+                if digits.len() < 2 {
+                    return self.tokenizer_error(
+                        loc,
+                        format!(
+                            "Invalid hex escape sequence '\\x{digits}'; should be exactly 2 digits."
+                        ),
+                    );
+                }
+                self.code_point(&digits, 16, loc)
+            }
+            'u' => self.unescape_unicode(chars, loc),
+            // Octal `\ooo`: up to three octal digits, but a third digit is only
+            // consumed when the running value stays within a single byte
+            // (<= 0xFF); otherwise it is left as a literal character.
+            '0'..='7' => {
+                let mut value = indicator.to_digit(8).unwrap_or(0);
+                // Second octal digit always fits in a byte (max 0o77 = 63).
+                if let Some(d) = chars.peek().and_then(|c| c.to_digit(8)) {
+                    value = value * 8 + d;
+                    chars.next();
+                    // Third digit only when the byte does not overflow.
+                    if let Some(d) = chars.peek().and_then(|c| c.to_digit(8)) {
+                        if value * 8 + d <= 0xFF {
+                            value = value * 8 + d;
+                            chars.next();
+                        }
+                    }
+                }
+                char::from_u32(value).map_or_else(
+                    || self.tokenizer_error(loc, "Invalid octal escape sequence."),
+                    Ok,
+                )
+            }
+            'b' => Ok('\u{8}'),
+            'f' => Ok('\u{c}'),
+            'n' => Ok('\n'),
+            'r' => Ok('\r'),
+            't' => Ok('\t'),
+            other => Ok(other),
+        }
+    }
+
+    fn code_point(&self, digits: &str, radix: u32, loc: Location) -> Result<char, TokenizerError> {
+        u32::from_str_radix(digits, radix)
+            .ok()
+            .and_then(char::from_u32)
+            .map_or_else(|| self.tokenizer_error(loc, "Invalid escape sequence."), Ok)
+    }
+
+    /// Decode a `\uhhhh` escape (the `\u` has already been consumed). A high
+    /// surrogate must be followed by a `\uhhhh` low surrogate and combines with
+    /// it into an astral code point; an unpaired high or low surrogate is an
+    /// error, matching real Snowflake.
+    fn unescape_unicode(&self, chars: &mut State, loc: Location) -> Result<char, TokenizerError> {
+        let value = self.read_hex4(chars, loc)?;
+        if (0xD800..=0xDBFF).contains(&value) {
+            return match self.read_low_surrogate(chars) {
+                Some(low) => {
+                    let cp = 0x1_0000 + ((value - 0xD800) << 10) + (low - 0xDC00);
+                    char::from_u32(cp)
+                        .map_or_else(|| self.tokenizer_error(loc, "Invalid escape sequence."), Ok)
+                }
+                None => self.tokenizer_error(
+                    loc,
+                    format!(
+                        "Invalid Unicode string literal; high surrogate '\\u{value:04X}' \
+                         must be followed by a low surrogate ('\\uDC00'-'\\uDFFF')."
+                    ),
+                ),
+            };
+        }
+        if (0xDC00..=0xDFFF).contains(&value) {
+            return self.tokenizer_error(
+                loc,
+                format!(
+                    "Invalid Unicode string literal; low surrogate '\\u{value:04X}' \
+                     must be preceded by a high surrogate ('\\uD800'-'\\uDBFF')."
+                ),
+            );
+        }
+        char::from_u32(value)
+            .map_or_else(|| self.tokenizer_error(loc, "Invalid escape sequence."), Ok)
+    }
+
+    /// Read exactly four hex digits as a `u32`, erroring like Snowflake when
+    /// fewer than four are present.
+    fn read_hex4(&self, chars: &mut State, loc: Location) -> Result<u32, TokenizerError> {
+        let mut digits = String::new();
+        while digits.len() < 4 {
+            match chars.peek() {
+                Some(c) if c.is_ascii_hexdigit() => {
+                    if let Some(c) = chars.next() {
+                        digits.push(c);
+                    }
+                }
+                _ => break,
+            }
+        }
+        if digits.len() < 4 {
+            return self.tokenizer_error(
+                loc,
+                format!(
+                    "Invalid unicode escape sequence '\\u{digits}'; should be exactly 4 digits."
+                ),
+            );
+        }
+        u32::from_str_radix(&digits, 16).map_or_else(
+            |_| self.tokenizer_error(loc, "Invalid unicode escape sequence."),
+            Ok,
+        )
+    }
+
+    /// Consume a trailing `\uhhhh` low surrogate, returning its value when
+    /// present and in range. Consumes greedily on the error path — the caller
+    /// aborts the whole token, so partial consumption is irrelevant.
+    fn read_low_surrogate(&self, chars: &mut State) -> Option<u32> {
+        if chars.peek() != Some(&'\\') {
+            return None;
+        }
+        chars.next();
+        if chars.peek() != Some(&'u') {
+            return None;
+        }
+        chars.next();
+        let mut digits = String::new();
+        while digits.len() < 4 {
+            match chars.peek() {
+                Some(c) if c.is_ascii_hexdigit() => {
+                    if let Some(c) = chars.next() {
+                        digits.push(c);
+                    }
+                }
+                _ => break,
+            }
+        }
+        if digits.len() < 4 {
+            return None;
+        }
+        u32::from_str_radix(&digits, 16)
+            .ok()
+            .filter(|low| (0xDC00..=0xDFFF).contains(low))
+    }
+
     fn tokenize_multiline_comment(
         &self,
         chars: &mut State,
@@ -3878,10 +4050,13 @@ mod tests {
             (r#"'%a\'%b'"#, r#"%a\'%b"#, r#"%a'%b"#),
             (r#"'a\'\'b\'c\'d'"#, r#"a\'\'b\'c\'d"#, r#"a''b'c'd"#),
             (r#"'\\'"#, r#"\\"#, r#"\"#),
+            // Snowflake keeps `\0` as NUL, maps the control-letter escapes, and
+            // collapses unrecognized letters (`\a` -> a, `\Z` -> Z) to the bare
+            // letter rather than a control character.
             (
                 r#"'\0\a\b\f\n\r\t\Z'"#,
                 r#"\0\a\b\f\n\r\t\Z"#,
-                "\0\u{7}\u{8}\u{c}\n\r\t\u{1a}",
+                "\0a\u{8}\u{c}\n\r\tZ",
             ),
             (r#"'\"'"#, r#"\""#, "\""),
             (r#"'\\a\\b\'c'"#, r#"\\a\\b\'c"#, r#"\a\b'c"#),
@@ -3890,6 +4065,20 @@ mod tests {
             (r#"'\q'"#, r#"\q"#, r#"q"#),
             (r#"'\%\_'"#, r#"\%\_"#, r#"%_"#),
             (r#"'\\%\\_'"#, r#"\\%\\_"#, r#"\%\_"#),
+            // Hex `\xhh` and unicode `\uhhhh` yield a code point; octal `\ooo`
+            // takes a third digit only when the byte does not overflow.
+            (r#"'\x41'"#, r#"\x41"#, "A"),
+            (r#"'\u0041'"#, r#"\u0041"#, "A"),
+            (r#"'\u00e9'"#, r#"\u00e9"#, "\u{e9}"),
+            (r#"'\ud83d\ude00'"#, r#"\ud83d\ude00"#, "\u{1f600}"),
+            (r#"'\xff'"#, r#"\xff"#, "\u{ff}"),
+            (r#"'A'"#, r#"A"#, "A"),
+            (r#"'é'"#, r#"é"#, "\u{e9}"),
+            (r#"'ꯍ'"#, r#"ꯍ"#, "\u{abcd}"),
+            (r#"'\101'"#, r#"\101"#, "A"),
+            (r#"'\1012'"#, r#"\1012"#, "A2"),
+            (r#"'\777'"#, r#"\777"#, "\u{3f}7"),
+            (r#"'\400'"#, r#"\400"#, "\u{20}0"),
         ] {
             let tokens = Tokenizer::new(&dialect, sql)
                 .with_unescape(false)