Struct LexerSpec 

pub struct LexerSpec {
    pub lexemes: Vec<LexemeSpec>,
    pub regex_builder: RegexBuilder,
    pub no_forcing: bool,
    pub allow_initial_skip: bool,
    pub num_extra_lexemes: usize,
    pub skip_by_class: Vec<LexemeIdx>,
    pub current_class: LexemeClass,
    pub special_token_rx: Option<ExprRef>,
    pub has_stop: bool,
    pub has_max_tokens: bool,
    pub has_temperature: bool,
    pub grammar_warnings: Vec<(String, usize)>,
    /* private fields */
}

Fields

lexemes: Vec<LexemeSpec>

regex_builder: RegexBuilder

no_forcing: bool

allow_initial_skip: bool

num_extra_lexemes: usize

skip_by_class: Vec<LexemeIdx>

current_class: LexemeClass

special_token_rx: Option<ExprRef>

has_stop: bool

has_max_tokens: bool

has_temperature: bool

grammar_warnings: Vec<(String, usize)>

Implementations

impl LexerSpec

pub fn new() -> Result<LexerSpec, Error>

pub fn render_warnings(&self) -> Vec<String>

pub fn can_rollback(&self) -> bool

pub fn check_rollback(&self) -> Result<(), Error>

pub fn has_forced_bytes(&self, lex_spec: &LexemeSpec, bytes: &[u8]) -> bool

Check if the lexeme always matches bytes.

pub fn setup_lexeme_class( &mut self, skip: RegexAst, ) -> Result<LexemeClass, Error>

pub fn alloc_lexeme_set(&self) -> LexemeSet

pub fn alloc_grammar_set(&self) -> SimpleVob

pub fn lexeme_set(&self, cond: impl Fn(&LexemeSpec) -> bool) -> LexemeSet

pub fn all_lexemes(&self) -> LexemeSet

pub fn lazy_lexemes(&self) -> LexemeSet

pub fn eos_ending_lexemes(&self) -> LexemeSet

pub fn token_range_lexemes(&self, possible: &LexemeSet) -> Vec<&LexemeSpec>

pub fn is_nullable(&self, idx: LexemeIdx) -> bool

pub fn to_regex_vec(&self, limits: &mut ParserLimits) -> Result<RegexVec, Error>

pub fn add_rx_and_stop( &mut self, name: String, body_rx: RegexAst, stop_rx: RegexAst, lazy: bool, max_tokens: usize, is_suffix: bool, ) -> Result<LexemeIdx, Error>

pub fn add_simple_literal( &mut self, name: String, literal: &str, contextual: bool, ) -> Result<LexemeIdx, Error>

pub fn add_special_token( &mut self, name: String, token_ranges: Vec<RangeInclusive<u32>>, ) -> Result<LexemeIdx, Error>

pub fn add_greedy_lexeme( &mut self, name: String, rx: RegexAst, contextual: bool, json_options: Option<JsonQuoteOptions>, max_tokens: usize, ) -> Result<LexemeIdx, Error>

pub fn add_extra_lexemes(&mut self, extra_lexemes: &[String])

pub fn extra_lexeme(&self, idx: usize) -> LexemeIdx

pub fn dbg_lexeme_set(&self, vob: &LexemeSet) -> String

pub fn lexeme_spec(&self, idx: LexemeIdx) -> &LexemeSpec

pub fn cost(&self) -> u64

pub fn skip_id(&self, class: LexemeClass) -> LexemeIdx

pub fn lexeme_def_to_string(&self, idx: LexemeIdx) -> String

pub fn dbg_lexeme_set_ext(&self, vob: &SimpleVob) -> String

Trait Implementations

impl Clone for LexerSpec

fn clone(&self) -> LexerSpec

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for LexerSpec

fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error>

Formats the value using the given formatter.