From 8da3a37910ec4ff13150a2f9162cf1d2a86a838b Mon Sep 17 00:00:00 2001 From: Stefan Date: Fri, 23 May 2025 01:03:25 +0200 Subject: [PATCH 1/2] feat: number scientific notation --- core/expression/src/functions/internal.rs | 14 ++++++++++++-- core/expression/src/lexer/lexer.rs | 22 ++++++++++++++++++++++ core/expression/src/parser/parser.rs | 4 +++- core/expression/src/variable/conv.rs | 16 ++++++++++------ core/expression/src/variable/de.rs | 13 +++++++------ 5 files changed, 54 insertions(+), 15 deletions(-) diff --git a/core/expression/src/functions/internal.rs b/core/expression/src/functions/internal.rs index abce92ad..78086040 100644 --- a/core/expression/src/functions/internal.rs +++ b/core/expression/src/functions/internal.rs @@ -559,7 +559,12 @@ pub(crate) mod imp { let a = args.var(0)?; let val = match a { V::Number(n) => *n, - V::String(str) => Decimal::from_str_exact(str.trim()).context("Invalid number")?, + V::String(str) => { + let s = str.trim(); + Decimal::from_str_exact(s) + .or_else(|_| Decimal::from_scientific(s)) + .context("Invalid number")? + } V::Bool(b) => match *b { true => Decimal::ONE, false => Decimal::ZERO, @@ -574,7 +579,12 @@ pub(crate) mod imp { let a = args.var(0)?; let is_ok = match a { V::Number(_) => true, - V::String(str) => Decimal::from_str_exact(str.trim()).is_ok(), + V::String(str) => { + let s = str.trim(); + Decimal::from_str_exact(s) + .or_else(|_| Decimal::from_scientific(s)) + .is_ok() + } _ => false, }; diff --git a/core/expression/src/lexer/lexer.rs b/core/expression/src/lexer/lexer.rs index f2a73af0..c9d60466 100644 --- a/core/expression/src/lexer/lexer.rs +++ b/core/expression/src/lexer/lexer.rs @@ -220,6 +220,28 @@ impl<'arena, 'self_ref> Scanner<'arena, 'self_ref> { end = e; } + if let Some((e_pos, _)) = self.cursor.next_if(|c| c == 'e') { + end = e_pos; + + if let Some((sign_pos, _)) = self.cursor.next_if(|c| c == '+' || c == '-') { + end = sign_pos; + } + + let mut has_exponent_digits = false; + while let Some((exp_pos, _)) = self.cursor.next_if(|c| is_token_type!(c, "digit")) { + end = exp_pos; + has_exponent_digits = true; + } + + if !has_exponent_digits { + while self.cursor.position() > e_pos { + self.cursor.back(); + } + + end = e_pos - 1; + } + } + self.push(Token { kind: TokenKind::Number, span: (start as u32, (end + 1) as u32), diff --git a/core/expression/src/parser/parser.rs b/core/expression/src/parser/parser.rs index 1732a9ab..e42e40c0 100644 --- a/core/expression/src/parser/parser.rs +++ b/core/expression/src/parser/parser.rs @@ -236,7 +236,9 @@ impl<'arena, 'token_ref, Flavor> Parser<'arena, 'token_ref, Flavor> { }); }; - let Ok(decimal) = Decimal::from_str_exact(token.value) else { + let Ok(decimal) = + Decimal::from_str_exact(token.value).or_else(|_| Decimal::from_scientific(token.value)) + else { return self.error(AstNodeError::InvalidNumber { number: afmt!(self, "{}", token.value), span: token.span, diff --git a/core/expression/src/variable/conv.rs b/core/expression/src/variable/conv.rs index 08e0242b..2de12db1 100644 --- a/core/expression/src/variable/conv.rs +++ b/core/expression/src/variable/conv.rs @@ -12,9 +12,11 @@ impl From for Variable { match value { Value::Null => Variable::Null, Value::Bool(b) => Variable::Bool(b), - Value::Number(n) => { - Variable::Number(Decimal::from_str_exact(n.as_str()).expect("Allowed number")) - } + Value::Number(n) => Variable::Number( + Decimal::from_str_exact(n.as_str()) + .or_else(|_| Decimal::from_scientific(n.as_str())) + .expect("Allowed number"), + ), Value::String(s) => Variable::String(Rc::from(s.as_str())), Value::Array(arr) => { Variable::from_array(arr.into_iter().map(Variable::from).collect()) @@ -33,9 +35,11 @@ impl From<&Value> for Variable { match value { Value::Null => Variable::Null, Value::Bool(b) => Variable::Bool(*b), - Value::Number(n) => { - Variable::Number(Decimal::from_str_exact(n.as_str()).expect("Allowed number")) - } + Value::Number(n) => Variable::Number( + Decimal::from_str_exact(n.as_str()) + .or_else(|_| Decimal::from_scientific(n.as_str())) + .expect("Allowed number"), + ), Value::String(s) => Variable::String(Rc::from(s.as_str())), Value::Array(arr) => Variable::from_array(arr.iter().map(Variable::from).collect()), Value::Object(obj) => Variable::from_object( diff --git a/core/expression/src/variable/de.rs b/core/expression/src/variable/de.rs index e619e82e..81066f37 100644 --- a/core/expression/src/variable/de.rs +++ b/core/expression/src/variable/de.rs @@ -91,13 +91,14 @@ impl<'de> Visitor<'de> for VariableVisitor { map.next_entry_seed(PhantomData::>, VariableDeserializer)? { if first && key.deref() == NUMBER_TOKEN { + let str = value + .as_str() + .ok_or_else(|| Error::custom("failed to deserialize number"))?; + return Ok(Variable::Number( - Decimal::from_str_exact( - value - .as_str() - .ok_or_else(|| Error::custom("failed to deserialize number"))?, - ) - .map_err(|_| Error::custom("invalid number"))?, + Decimal::from_str_exact(str) + .or_else(|_| Decimal::from_scientific(str)) + .map_err(|_| Error::custom("invalid number"))?, )); } From ec9634f4d0804fc3afaedd202ea9ab9091734179 Mon Sep 17 00:00:00 2001 From: Stefan Date: Fri, 23 May 2025 11:29:36 +0200 Subject: [PATCH 2/2] add tests --- core/expression/tests/data/standard.csv | 85 ++++++++++++++++++++++++- 1 file changed, 84 insertions(+), 1 deletion(-) diff --git a/core/expression/tests/data/standard.csv b/core/expression/tests/data/standard.csv index dd6d087e..6f03b62c 100644 --- a/core/expression/tests/data/standard.csv +++ b/core/expression/tests/data/standard.csv @@ -465,4 +465,87 @@ trunc(7.999, 0);; 7 trunc(7.999, 1);; 7.9 trunc(7.999, 2);; 7.99 trunc(-7.444, 2);; -7.44 -trunc(-7.999, 2);; -7.99 \ No newline at end of file +trunc(-7.999, 2);; -7.99 + +# Scientific notation - basic parsing +1e5;;100000 +2e3;;2000 +5e0;;5 +1e-2;;0.01 +2.5e2;;250 +3.14e1;;31.4 +1.23e-3;;0.00123 +-1e3;;-1000 +-2.5e-2;;-0.025 + +# Scientific notation - arithmetic operations +1e2 + 1e1;;110 +2e3 - 5e2;;1500 +3e2 * 2e1;;6000 +1e4 / 2e2;;50 +1e2 + 50;;150 +1e3 - 100;;900 +2e2 * 3;;600 +1e3 / 10;;100 + +# Scientific notation - comparisons +1e3 == 1000;;true +1e3 != 999;;true +1e2 > 50;;true +1e2 < 200;;true +1e2 >= 100;;true +1e2 <= 100;;true +2.5e2 == 250;;true +1.5e-2 == 0.015;;true + +# Scientific notation - type conversions +string(1e3);;'1000' +string(2.5e2);;'250' +string(1.23e-3);;'0.00123' +number('1e3');;1000 +number('2.5e2');;250 +number('1.23e-3');;0.00123 +isNumeric('1e3');;true +isNumeric('2.5e-2');;true +isNumeric('1.23e-3');;true + +# Scientific notation - array operations +sum([1e2, 2e2, 3e2]);;600 +max([1e1, 2e1, 3e1]);;30 +min([1e1, 2e1, 3e1]);;10 +avg([1e2, 2e2, 3e2]);;200 +contains([1e2, 2e2, 3e2], 200);;true +map([1e1, 2e1, 3e1], # * 2);;[20, 40, 60] +filter([1e1, 2e1, 3e1, 4e1], # > 25);;[30, 40] + +# Scientific notation - edge cases +1e+3;;1000 +1e-0;;1 +0e5;;0 +1.0e2;;100 +10e1;;100 +0.1e3;;100 +1.5e+2;;150 +2e-1;;0.2 + +# Scientific notation - mixed with regular numbers +1e2 + 50.5;;150.5 +1000 - 1e2;;900 +15 * 1e2;;1500 +1e3 / 2.5;;400 + +# Scientific notation - complex expressions +(1e2 + 2e2) * 1e1;;3000 +1e3 / (2e1 + 3e1);;20 +abs(-1e2);;100 +floor(1.23e2);;123 +ceil(1.23e2);;123 +round(1.234e2);;123 + +# Scientific notation - in functions +1e1 ^ 2;;100 + +# Scientific notation - template strings +`Value: ${1e3}`;; 'Value: 1000' +`Scientific: ${2.5e2}`;; 'Scientific: 250' +`Negative: ${-1e2}`;; 'Negative: -100' \ No newline at end of file