ifc_lite_core/parser/
tokenizer.rs

1// This Source Code Form is subject to the terms of the Mozilla Public
2// License, v. 2.0. If a copy of the MPL was not distributed with this
3// file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
5//! nom-combinator tokenizer for STEP/IFC entity lines.
6//!
7//! Zero-copy tokenization: string-like tokens borrow their original bytes.
8
9use nom::{
10    branch::alt,
11    bytes::complete::take_while1,
12    character::complete::{char, digit1, one_of},
13    combinator::{map, map_res, opt, recognize},
14    multi::separated_list0,
15    sequence::{delimited, pair, preceded, tuple},
16    IResult,
17};
18
19use crate::error::{Error, Result};
20use crate::generated::IfcType;
21use crate::step_encoding::step_string_literal_end;
22
23/// STEP/IFC token.
24///
25/// String-like tokens borrow their original bytes. Decode them only at a
26/// user-facing boundary so malformed real-world encodings cannot invalidate
27/// the structural parser.
28#[derive(Debug, Clone, PartialEq)]
29pub enum Token<'a> {
30    /// Entity reference: #123
31    EntityRef(u32),
32    /// String literal: 'text'
33    String(&'a [u8]),
34    /// Integer: 42
35    Integer(i64),
36    /// Float: 3.14
37    Float(f64),
38    /// Enum: .TRUE., .FALSE., .UNKNOWN.
39    Enum(&'a [u8]),
40    /// List: (1, 2, 3)
41    List(Vec<Token<'a>>),
42    /// Typed value: IFCPARAMETERVALUE(0.), IFCBOOLEAN(.T.)
43    TypedValue(&'a [u8], Vec<Token<'a>>),
44    /// Null value: $
45    Null,
46    /// Asterisk (derived value): *
47    Derived,
48}
49
50/// Parse entity reference: #123
51fn entity_ref(input: &[u8]) -> IResult<&[u8], Token<'_>> {
52    map(
53        preceded(char('#'), map_res(digit1, lexical_core::parse::<u32>)),
54        Token::EntityRef,
55    )(input)
56}
57
58/// Parse string literal: 'text' or "text"
59///
60/// An apostrophe literal ends where the shared rule says it does
61/// ([`step_string_literal_end`]): `''` is an escaped quote and the operand of
62/// a `\S\` PAGE directive is text. Ending at that operand left the rest of
63/// `'Para \S\' 5'` as unparseable input, so the whole entity failed to
64/// decode (#7589). The token borrows the content between the quotes, still
65/// escaped; consumers un-double `''` and decode backslash escapes.
66fn string_literal(input: &[u8]) -> IResult<&[u8], Token<'_>> {
67    match input.first() {
68        Some(b'\'') => match step_string_literal_end(input, 0) {
69            Some(past_close) => Ok((
70                &input[past_close..],
71                Token::String(&input[1..past_close - 1]),
72            )),
73            None => Err(nom::Err::Error(nom::error::Error::new(
74                input,
75                nom::error::ErrorKind::Char,
76            ))),
77        },
78        _ => map(
79            delimited(char('"'), double_quoted_content, char('"')),
80            Token::String,
81        )(input),
82    }
83}
84
85/// Content of a `"..."` literal up to its closing `"`; a doubled `""` stays
86/// inside. Uses memchr for SIMD-accelerated quote searching.
87#[inline]
88fn double_quoted_content(input: &[u8]) -> IResult<&[u8], &[u8]> {
89    let mut pos = 0;
90    while let Some(found) = memchr::memchr(b'"', &input[pos..]) {
91        let idx = pos + found;
92        if input.get(idx + 1) == Some(&b'"') {
93            pos = idx + 2;
94            continue;
95        }
96        return Ok((&input[idx..], &input[..idx]));
97    }
98    Err(nom::Err::Error(nom::error::Error::new(
99        input,
100        nom::error::ErrorKind::Char,
101    )))
102}
103
104/// Parse integer: 42, -42, +42
105///
106/// ISO 10303-21 spells `INTEGER` as `[ SIGN ] DIGIT { DIGIT }` with
107/// `SIGN = '+' | '-'`, so a leading `+` is as legal as a leading `-`. Accepting
108/// only `-` dropped the WHOLE entity, not just the attribute: every caller of
109/// `parse_entity` discards a record that fails to tokenize
110/// (`let Ok(..) = .. else { continue }`). The exponent sign below already
111/// accepted both, `fast_float2::parse_partial` (the fast reader in `decoder.rs`)
112/// accepts `+`, and the TypeScript half reads values with `parseFloat`, so the
113/// same file used to decode differently in the browser and in wasm.
114/// Uses lexical-core for 10x faster parsing
115#[inline]
116fn integer(input: &[u8]) -> IResult<&[u8], Token<'_>> {
117    map_res(recognize(tuple((opt(one_of("+-")), digit1))), |s: &[u8]| {
118        lexical_core::parse::<i64>(s)
119            .map(Token::Integer)
120            .map_err(|_| "parse error")
121    })(input)
122}
123
124/// Parse float: 3.14, -3.14, +3.14, 1.5E-10, 0., 1.
125/// IFC allows floats like "0." without decimal digits
126///
127/// `REAL` carries the same `[ SIGN ]` as `INTEGER`; see [`integer`] for why a
128/// rejected sign costs the whole entity.
129/// Uses lexical-core for 10x faster parsing
130#[inline]
131fn float(input: &[u8]) -> IResult<&[u8], Token<'_>> {
132    map_res(
133        recognize(tuple((
134            opt(one_of("+-")),
135            digit1,
136            char('.'),
137            opt(digit1), // Made optional to support "0." format
138            opt(tuple((one_of("eE"), opt(one_of("+-")), digit1))),
139        ))),
140        |s: &[u8]| {
141            lexical_core::parse::<f64>(s)
142                .map(Token::Float)
143                .map_err(|_| "parse error")
144        },
145    )(input)
146}
147
148/// Parse enum: .TRUE., .FALSE., .UNKNOWN., .ELEMENT.
149fn enum_value(input: &[u8]) -> IResult<&[u8], Token<'_>> {
150    map(
151        delimited(
152            char('.'),
153            take_while1(|c: u8| c.is_ascii_alphanumeric() || c == b'_'),
154            char('.'),
155        ),
156        Token::Enum,
157    )(input)
158}
159
160/// Parse null: $
161fn null(input: &[u8]) -> IResult<&[u8], Token<'_>> {
162    map(char('$'), |_| Token::Null)(input)
163}
164
165/// Parse derived: *
166fn derived(input: &[u8]) -> IResult<&[u8], Token<'_>> {
167    map(char('*'), |_| Token::Derived)(input)
168}
169
170/// Maximum nesting depth for token recursion (list and typed-value bodies).
171///
172/// Each `(` in the input bumps depth by one. Real-world IFC entities rarely
173/// nest beyond 5-10 levels; 256 leaves comfortable headroom while keeping
174/// the stack bounded against pathological inputs.
175const MAX_NESTING_DEPTH: u32 = 256;
176
177/// Parse typed value: IFCPARAMETERVALUE(0.), IFCBOOLEAN(.T.)
178///
179/// The type name and its `(` are usually adjacent, but a STEP writer's line
180/// wrap can land exactly between them - measured on a real file where 39 of
181/// 175 `IFCSURFACESTYLERENDERING` entities wrap as `…MEASURE\r\n(1.)`. Without
182/// `ws` here, `char('(')` sees `\r` and the whole entity fails to parse, which
183/// silently dropped it from every full-file walk (`decode_at_uncached` returns
184/// `Err`, and every caller's `let Ok(..) = .. else { continue }` skips it) -
185/// the entity was never malformed, just wrapped where the parser assumed it
186/// never would be.
187///
188/// An EMPTY argument list also needs `ws` on both sides of the parens
189/// (`IFCLABEL(\r\n)`), not just before `(`: `separated_list0` matches zero
190/// items without consuming anything, so with a bare `char(')')` any
191/// whitespace or comment before the `)` left it unconsumed and the whole
192/// typed value failed to parse (#3789).
193fn typed_value_at_depth(input: &[u8], depth: u32) -> IResult<&[u8], Token<'_>> {
194    map(
195        pair(
196            // Type name (all caps with optional numbers/underscores)
197            take_while1(|c: u8| c.is_ascii_alphanumeric() || c == b'_'),
198            // Arguments
199            delimited(
200                pair(ws, char('(')),
201                preceded(
202                    ws,
203                    separated_list0(delimited(ws, char(','), ws), move |i| {
204                        token_at_depth(i, depth)
205                    }),
206                ),
207                pair(ws, char(')')),
208            ),
209        ),
210        |(type_name, args)| Token::TypedValue(type_name, args),
211    )(input)
212}
213
214/// Skip whitespace AND `/* ... */` comments -- 10303-21 allows a comment
215/// anywhere whitespace is allowed, including between the attributes this
216/// tokenizer decodes. Without this, `/* rev; b */ $` was not trivia: no `alt`
217/// branch in `token_at_depth` starts with `/`, so a comment before a value
218/// made the whole entity fail to parse rather than reading the value clean.
219///
220/// Reuses `skip_step_trivia`, the matched pair of the scanners' trivia skip
221/// (`step-lexing.ts`'s `skipTrivia` on the TS side). An unterminated comment
222/// consumes to end of input rather than failing this always-succeeds
223/// combinator; the `alt` that follows then has nothing left to match and the
224/// entity fails to parse, which is the existing behaviour for corrupt input.
225fn ws(input: &[u8]) -> IResult<&[u8], ()> {
226    let end = super::lexical::skip_step_trivia(input, 0).unwrap_or(input.len());
227    Ok((&input[end..], ()))
228}
229
230/// Parse a token with optional surrounding whitespace
231/// Optimized ordering: test cheapest patterns first (single-char markers)
232fn token(input: &[u8]) -> IResult<&[u8], Token<'_>> {
233    token_at_depth(input, 0)
234}
235
236fn token_at_depth(input: &[u8], depth: u32) -> IResult<&[u8], Token<'_>> {
237    if depth > MAX_NESTING_DEPTH {
238        return Err(nom::Err::Failure(nom::error::Error::new(
239            input,
240            nom::error::ErrorKind::TooLarge,
241        )));
242    }
243    delimited(
244        ws,
245        alt((
246            // Single-char markers first (O(1) check)
247            null,       // $
248            derived,    // *
249            entity_ref, // # + digits
250            // Then by complexity
251            enum_value,     // .XXX.
252            string_literal, // 'xxx'
253            move |i| list_at_depth(i, depth + 1), // (...)
254            // Numbers: float before integer since float includes '.'
255            float,
256            integer,
257            // IFCPARAMETERVALUE(0.) - most expensive, last
258            move |i| typed_value_at_depth(i, depth + 1),
259        )),
260        ws,
261    )(input)
262}
263
264/// Parse list: (1, 2, 3) or nested lists
265/// Test-only wrapper for the depth-0 entry into list.
266#[cfg(test)]
267fn list(input: &[u8]) -> IResult<&[u8], Token<'_>> {
268    list_at_depth(input, 0)
269}
270
271/// An EMPTY list (`( )`, `(\r\n)`, `(/* empty */)`) needs `ws` explicitly
272/// around the empty `separated_list0`: it matches zero items without
273/// consuming anything, so a bare `char(')')` failed to parse past any
274/// whitespace or comment left between the parens (#3789, same shape as
275/// `typed_value_at_depth`'s empty-args fix).
276fn list_at_depth(input: &[u8], depth: u32) -> IResult<&[u8], Token<'_>> {
277    map(
278        delimited(
279            pair(char('('), ws),
280            separated_list0(delimited(ws, char(','), ws), move |i| {
281                token_at_depth(i, depth)
282            }),
283            pair(ws, char(')')),
284        ),
285        Token::List,
286    )(input)
287}
288
289/// Parse a complete entity line from raw IFC bytes.
290/// Example: #123=IFCWALL('guid','owner',$,$,'name',$,$,$);
291// The nom `IResult` parser tuple type is intentionally explicit here; factoring
292// it into a `type` alias would obscure the parser combinator structure.
293#[allow(clippy::type_complexity)]
294pub fn parse_entity<'a, T>(input: &'a T) -> Result<(u32, IfcType, Vec<Token<'a>>)>
295where
296    T: AsRef<[u8]> + ?Sized,
297{
298    let input = input.as_ref();
299    let result: IResult<&[u8], (u32, &[u8], Vec<Token>)> = tuple((
300        // Entity ID: #123
301        delimited(
302            ws,
303            preceded(char('#'), map_res(digit1, lexical_core::parse::<u32>)),
304            ws,
305        ),
306        // Equals sign
307        preceded(
308            char('='),
309            // Entity type: IFCWALL
310            delimited(
311                ws,
312                take_while1(|c: u8| c.is_ascii_alphanumeric() || c == b'_'),
313                ws,
314            ),
315        ),
316        // Arguments: ('guid', 'owner', ...). `ws` around the empty
317        // `separated_list0` handles an EMPTY argument list separated from
318        // its `)` by whitespace or a comment (`#1=IFCX(\r\n);`), the same
319        // shape as `list_at_depth`'s and `typed_value_at_depth`'s fix
320        // (#3789): with zero items `separated_list0` consumes nothing, so
321        // a bare `char(')')` failed to parse past it.
322        delimited(
323            pair(char('('), ws),
324            separated_list0(delimited(ws, char(','), ws), token),
325            tuple((ws, char(')'), ws, char(';'))),
326        ),
327    ))(input);
328
329    match result {
330        Ok((_, (id, type_str, args))) => {
331            let type_str = std::str::from_utf8(type_str)
332                .map_err(|_| Error::parse(0, "Entity type is not ASCII/UTF-8"))?;
333            let ifc_type = IfcType::from_str(type_str);
334            Ok((id, ifc_type, args))
335        }
336        Err(e) => Err(Error::parse(0, format!("Failed to parse entity: {}", e))),
337    }
338}
339
340#[cfg(test)]
341#[path = "tokenizer_tests.rs"]
342mod tokenizer_tests;
ifc_lite_core/parser/tokenizer.rs

ifc_lite_core/parser/
tokenizer.rs
