ravenrs_sheet/formula/
lexer.rs

1//! Tokenizer for Excel formulas (A1 style, canonical or selected UI separators).
2//!
3//! Token spans are byte offsets into the source, which the editor uses to colour references.
4
5use crate::base::CellError;
6
7use crate::formula::ast::TrimMode;
8
9#[derive(Clone, Debug, PartialEq)]
10pub enum Tok {
11    Number(f64),
12    Text(String),
13    Bool(bool),
14    Error(CellError),
15    /// A reference-looking or name-looking word, possibly sheet-qualified: `A1`, `$B$2`, `A:A`
16    /// is lexed as Word(A) Colon Word(A); `Sheet1!A1` → `Sheet(Some("Sheet1"))` then `Word`.
17    Word(String),
18    /// Sheet prefix `Sheet1!`, `'My Sheet'!`, `Sheet1:Sheet3!` (first, last).
19    Sheet(String, Option<String>),
20    /// Function name (word immediately followed by `(`), the `(` is consumed.
21    Func(String),
22    /// Structured reference body after a table name or bare: the raw text inside `[...]`.
23    Struct(String),
24    Op(&'static str),
25    LParen,
26    RParen,
27    LBrace,
28    RBrace,
29    Comma,
30    Semicolon,
31    Colon,
32    /// Trim reference range operator `.:`, `:.` or `.:.`.
33    Trim(TrimMode),
34    Space,
35    Eof,
36}
37
38#[derive(Clone, Debug, PartialEq)]
39pub struct Token {
40    pub tok: Tok,
41    pub start: usize,
42    pub end: usize,
43}
44
45#[derive(Clone, Debug, PartialEq, thiserror::Error)]
46#[error("{msg} at {pos}")]
47pub struct LexError {
48    pub msg: String,
49    pub pos: usize,
50}
51
52fn is_word_char(c: char) -> bool {
53    c.is_alphanumeric() || matches!(c, '_' | '.' | '$' | '\\' | '?')
54}
55
56/// `A1`, `$A$1`, `A`, `$A`, `1`, `$1`: a word that can end a range, so `A1.:` is a trim
57/// reference and not a name ending in a dot.
58fn is_range_end(w: &str) -> bool {
59    let s = w.strip_prefix('$').unwrap_or(w);
60    let (letters, rest) = s.split_at(s.find(|c: char| !c.is_ascii_alphabetic()).unwrap_or(s.len()));
61    let digits = if letters.is_empty() { rest } else { rest.strip_prefix('$').unwrap_or(rest) };
62    letters.len() <= 3 && if digits.is_empty() { !letters.is_empty() && rest.is_empty() } else { digits.bytes().all(|b| b.is_ascii_digit()) }
63}
64
65/// Whether `c` can start the second end of a range after `:.` (`A1:.E10`, `A:.$C`, `1:.5`).
66fn starts_range_end(c: Option<char>) -> bool {
67    c.is_some_and(|c| c.is_ascii_alphanumeric() || c == '$')
68}
69
70fn byte_pos(src: &str, chars: &[(usize, char)], index: usize) -> usize {
71    chars.get(index).map(|c| c.0).unwrap_or(src.len())
72}
73
74fn char_at(chars: &[(usize, char)], index: usize) -> Option<char> {
75    chars.get(index).map(|c| c.1)
76}
77
78fn emit(src: &str, chars: &[(usize, char)], out: &mut Vec<Token>, tok: Tok, start: usize, end_index: usize) {
79    out.push(Token { tok, start, end: byte_pos(src, chars, end_index) });
80}
81
82fn quoted(src: &str, chars: &[(usize, char)], i: usize, start: usize, out: &mut Vec<Token>) -> Result<usize, LexError> {
83    let quote = char_at(chars, i).unwrap_or_default();
84    let mut value = String::new();
85    let mut j = i + 1;
86    loop {
87        match char_at(chars, j) {
88            None => {
89                let msg = if quote == '"' { "unterminated text" } else { "unterminated sheet name" };
90                return Err(LexError { msg: msg.into(), pos: start });
91            }
92            Some(c) if c == quote && char_at(chars, j + 1) == Some(quote) => {
93                value.push(quote);
94                j += 2;
95            }
96            Some(c) if c == quote => {
97                j += 1;
98                break;
99            }
100            Some(c) => {
101                value.push(c);
102                j += 1;
103            }
104        }
105    }
106    if quote == '"' {
107        emit(src, chars, out, Tok::Text(value), start, j);
108        return Ok(j);
109    }
110    if char_at(chars, j) != Some('!') {
111        return Err(LexError { msg: "expected ! after sheet name".into(), pos: byte_pos(src, chars, j) });
112    }
113    let (first, last) = match value.split_once(':') {
114        Some((first, last)) => (first.to_string(), Some(last.to_string())),
115        None => (value, None),
116    };
117    emit(src, chars, out, Tok::Sheet(first, last), start, j + 1);
118    Ok(j + 1)
119}
120
121fn error_literal(src: &str, chars: &[(usize, char)], i: usize, start: usize, out: &mut Vec<Token>) -> Result<usize, LexError> {
122    let rest = src.get(start..).unwrap_or("");
123    let error = CellError::ALL
124        .iter()
125        .copied()
126        .find(|error| rest.get(..error.as_str().len()).is_some_and(|text| text.eq_ignore_ascii_case(error.as_str())))
127        .ok_or_else(|| LexError { msg: "unknown error literal".into(), pos: start })?;
128    let end = i + error.as_str().chars().count();
129    emit(src, chars, out, Tok::Error(error), start, end);
130    Ok(end)
131}
132
133fn structured_ref(src: &str, chars: &[(usize, char)], i: usize, start: usize, out: &mut Vec<Token>) -> Result<usize, LexError> {
134    let mut depth = 0;
135    let mut j = i;
136    loop {
137        match char_at(chars, j) {
138            None => return Err(LexError { msg: "unbalanced [".into(), pos: start }),
139            Some('\'') => j += 2,
140            Some('[') => {
141                depth += 1;
142                j += 1;
143            }
144            Some(']') => {
145                depth -= 1;
146                j += 1;
147                if depth == 0 {
148                    break;
149                }
150            }
151            Some(_) => j += 1,
152        }
153    }
154    let inner = src.get(byte_pos(src, chars, i + 1)..byte_pos(src, chars, j.saturating_sub(1))).unwrap_or("").to_string();
155    emit(src, chars, out, Tok::Struct(inner), start, j);
156    Ok(j)
157}
158
159fn number(
160    src: &str,
161    chars: &[(usize, char)],
162    locale: crate::formula::FormulaLocale,
163    i: usize,
164    start: usize,
165    out: &mut Vec<Token>,
166) -> Result<usize, LexError> {
167    let ch = |index| char_at(chars, index);
168    let mut j = i;
169    while ch(j).is_some_and(|c| c.is_ascii_digit()) {
170        j += 1;
171    }
172    let is_row_ref = {
173        let mut k = j;
174        if ch(k) == Some('.') && ch(k + 1) == Some(':') {
175            k += 1;
176        }
177        ch(k) == Some(':') && {
178            k += 1;
179            if ch(k) == Some('.') {
180                k += 1;
181            }
182            if ch(k) == Some('$') {
183                k += 1;
184            }
185            ch(k).is_some_and(|c| c.is_ascii_digit())
186        }
187    };
188    if is_row_ref {
189        let word = chars.get(i..j).map(|s| s.iter().map(|c| c.1).collect()).unwrap_or_default();
190        emit(src, chars, out, Tok::Word(word), start, j);
191        return Ok(j);
192    }
193    if ch(j) == Some(locale.decimal_separator()) {
194        j += 1;
195        while ch(j).is_some_and(|c| c.is_ascii_digit()) {
196            j += 1;
197        }
198    }
199    if matches!(ch(j), Some('e' | 'E')) {
200        let mut exponent = j + 1;
201        if matches!(ch(exponent), Some('+' | '-')) {
202            exponent += 1;
203        }
204        if ch(exponent).is_some_and(|c| c.is_ascii_digit()) {
205            while ch(exponent).is_some_and(|c| c.is_ascii_digit()) {
206                exponent += 1;
207            }
208            j = exponent;
209        }
210    }
211    let text = src.get(start..byte_pos(src, chars, j)).unwrap_or("");
212    let value: f64 =
213        text.replace(locale.decimal_separator(), ".").parse().map_err(|_| LexError { msg: format!("bad number `{text}`"), pos: start })?;
214    if !value.is_finite() {
215        return Err(LexError { msg: format!("number too large `{text}`"), pos: start });
216    }
217    emit(src, chars, out, Tok::Number(value), start, j);
218    Ok(j)
219}
220
221fn word(
222    src: &str,
223    chars: &[(usize, char)],
224    locale: crate::formula::FormulaLocale,
225    array_depth: usize,
226    i: usize,
227    start: usize,
228    out: &mut Vec<Token>,
229) -> usize {
230    let ch = |index| char_at(chars, index);
231    let mut j = i;
232    while ch(j).is_some_and(|c| is_word_char(c) && !(array_depth > 0 && c == locale.array_column_separator())) {
233        j += 1;
234    }
235    let value: String = chars.get(i..j).map(|s| s.iter().map(|c| c.1).collect()).unwrap_or_default();
236    if ch(j) == Some('!') {
237        emit(src, chars, out, Tok::Sheet(value, None), start, j + 1);
238        j + 1
239    } else if ch(j) == Some(':') && sheet_range_end(chars, locale, array_depth, &value, j, out) {
240        let mut end = j + 1;
241        while ch(end).is_some_and(|c| is_word_char(c) && !(array_depth > 0 && c == locale.array_column_separator())) {
242            end += 1;
243        }
244        let last: String = chars.get(j + 1..end).map(|s| s.iter().map(|c| c.1).collect()).unwrap_or_default();
245        emit(src, chars, out, Tok::Sheet(value, Some(last)), start, end + 1);
246        end + 1
247    } else if ch(j) == Some('(') {
248        emit(src, chars, out, Tok::Func(value), start, j + 1);
249        j + 1
250    } else if value.eq_ignore_ascii_case("TRUE") && ch(j) != Some('[') {
251        emit(src, chars, out, Tok::Bool(true), start, j);
252        j
253    } else if value.eq_ignore_ascii_case("FALSE") && ch(j) != Some('[') {
254        emit(src, chars, out, Tok::Bool(false), start, j);
255        j
256    } else if value.len() > 1 && value.ends_with('.') && ch(j) == Some(':') && is_range_end(value.get(..value.len() - 1).unwrap_or("")) {
257        emit(src, chars, out, Tok::Word(value[..value.len() - 1].to_string()), start, j - 1);
258        j - 1
259    } else {
260        emit(src, chars, out, Tok::Word(value), start, j);
261        j
262    }
263}
264
265fn sheet_range_end(
266    chars: &[(usize, char)],
267    locale: crate::formula::FormulaLocale,
268    array_depth: usize,
269    value: &str,
270    colon: usize,
271    out: &[Token],
272) -> bool {
273    let ch = |index| char_at(chars, index);
274    let mut end = colon + 1;
275    while ch(end).is_some_and(|c| is_word_char(c) && !(array_depth > 0 && c == locale.array_column_separator())) {
276        end += 1;
277    }
278    let repeats_sheet = is_range_end(value)
279        && matches!(out.last(), Some(Token { tok: Tok::Sheet(previous, None), .. })
280            if chars.get(colon + 1..end).is_some_and(|s| crate::formula::same_name(&s.iter().map(|c| c.1).collect::<String>(), previous)));
281    end > colon + 1 && ch(end) == Some('!') && !repeats_sheet
282}
283
284/// Tokenizes formula text (without the leading `=`).
285pub fn tokenize(src: &str) -> Result<Vec<Token>, LexError> {
286    tokenize_locale(src, crate::formula::FormulaLocale::En)
287}
288
289/// Tokenizes UI syntax while retaining byte spans in the original text.
290pub fn tokenize_locale(src: &str, locale: crate::formula::FormulaLocale) -> Result<Vec<Token>, LexError> {
291    let chars: Vec<(usize, char)> = src.char_indices().collect();
292    let n = chars.len();
293    let pos_of = |i: usize| chars.get(i).map(|c| c.0).unwrap_or(src.len());
294    let ch = |i: usize| chars.get(i).map(|c| c.1);
295    let mut out = Vec::new();
296    let mut i = 0;
297    let mut array_depth = 0usize;
298    while i < n {
299        let start = pos_of(i);
300        let Some(c) = ch(i) else { break };
301        let push = |out: &mut Vec<Token>, tok: Tok, end_i: usize| out.push(Token { tok, start, end: pos_of(end_i) });
302        match c {
303            ' ' | '\t' | '\n' | '\r' => {
304                let mut j = i;
305                while matches!(ch(j), Some(' ' | '\t' | '\n' | '\r')) {
306                    j += 1;
307                }
308                push(&mut out, Tok::Space, j);
309                i = j;
310            }
311            '"' | '\'' => i = quoted(src, &chars, i, start, &mut out)?,
312            '#' if out.last().is_some_and(|t: &Token| t.end == start && matches!(t.tok, Tok::Word(_))) => {
313                // Spill range operator right after a reference: `A1#`.
314                push(&mut out, Tok::Op("#"), i + 1);
315                i += 1;
316            }
317            '#' => {
318                i = error_literal(src, &chars, i, start, &mut out)?;
319            }
320            '[' => {
321                i = structured_ref(src, &chars, i, start, &mut out)?;
322            }
323            '.' if ch(i + 1) == Some(':') => {
324                // Trim reference `A1.:E10` / `A1.:.E10` (the word before stopped at the dot).
325                if ch(i + 2) == Some('.') && starts_range_end(ch(i + 3)) {
326                    push(&mut out, Tok::Trim(TrimMode::All), i + 3);
327                    i += 3;
328                } else {
329                    push(&mut out, Tok::Trim(TrimMode::Leading), i + 2);
330                    i += 2;
331                }
332            }
333            c if c.is_ascii_digit() || (c == locale.decimal_separator() && ch(i + 1).is_some_and(|d| d.is_ascii_digit())) => {
334                i = number(src, &chars, locale, i, start, &mut out)?;
335            }
336            '(' => {
337                push(&mut out, Tok::LParen, i + 1);
338                i += 1;
339            }
340            ')' => {
341                push(&mut out, Tok::RParen, i + 1);
342                i += 1;
343            }
344            '{' => {
345                array_depth += 1;
346                push(&mut out, Tok::LBrace, i + 1);
347                i += 1;
348            }
349            '}' => {
350                array_depth = array_depth.saturating_sub(1);
351                push(&mut out, Tok::RBrace, i + 1);
352                i += 1;
353            }
354            c if (array_depth == 0 && c == locale.list_separator()) || (array_depth > 0 && c == locale.array_column_separator()) => {
355                push(&mut out, Tok::Comma, i + 1);
356                i += 1;
357            }
358            ';' => {
359                push(&mut out, Tok::Semicolon, i + 1);
360                i += 1;
361            }
362            ':' => {
363                if ch(i + 1) == Some('.') && starts_range_end(ch(i + 2)) {
364                    push(&mut out, Tok::Trim(TrimMode::Trailing), i + 2);
365                    i += 2;
366                } else {
367                    push(&mut out, Tok::Colon, i + 1);
368                    i += 1;
369                }
370            }
371            '+' | '-' | '*' | '/' | '^' | '&' | '%' | '@' | '=' => {
372                let op = match c {
373                    '+' => "+",
374                    '-' => "-",
375                    '*' => "*",
376                    '/' => "/",
377                    '^' => "^",
378                    '&' => "&",
379                    '%' => "%",
380                    '@' => "@",
381                    _ => "=",
382                };
383                push(&mut out, Tok::Op(op), i + 1);
384                i += 1;
385            }
386            '<' | '>' => {
387                let two = ch(i + 1);
388                let (op, len) = match (c, two) {
389                    ('<', Some('=')) => ("<=", 2),
390                    ('<', Some('>')) => ("<>", 2),
391                    ('>', Some('=')) => (">=", 2),
392                    ('<', _) => ("<", 1),
393                    _ => (">", 1),
394                };
395                push(&mut out, Tok::Op(op), i + len);
396                i += len;
397            }
398            c if is_word_char(c) => i = word(src, &chars, locale, array_depth, i, start, &mut out),
399            other => return Err(LexError { msg: format!("unexpected character `{other}`"), pos: start }),
400        }
401    }
402    out.push(Token { tok: Tok::Eof, start: src.len(), end: src.len() });
403    Ok(out)
404}
ravenrs_sheet/formula/lexer.rs

ravenrs_sheet/formula/
lexer.rs
