Struct TokenParser

pub struct TokenParser {
    pub token_env: Arc<dyn TokenizerEnv + Sync>,
    pub parser: Parser,
    pub compute_mask_start_time: Instant,
    pub last_bias_time: Duration,
    pub inference_caps: InferenceCapabilities,
    pub logger: Logger,
    pub limits: ParserLimits,
    pub bias_computer: Arc<dyn BiasComputer>,
    /* private fields */
}

Fields

token_env: Arc<dyn TokenizerEnv + Sync>

parser: Parser

compute_mask_start_time: Instant

last_bias_time: Duration

inference_caps: InferenceCapabilities

logger: Logger

limits: ParserLimits

bias_computer: Arc<dyn BiasComputer>

Implementations

impl TokenParser

pub fn from_llguidance_json( token_env: Arc<dyn TokenizerEnv + Sync>, top_grammar: TopLevelGrammar, logger: Logger, inference_caps: InferenceCapabilities, limits: ParserLimits, extra_lexemes: Vec<String>, ) -> Result<TokenParser, Error>

pub fn deep_clone(&self) -> TokenParser

pub fn stop_reason(&self) -> StopReason

pub fn is_fresh(&self) -> bool

pub fn parser_stats(&self) -> &ParserStats

pub fn last_step_stats(&self) -> &ParserStats

pub fn max_step_stats(&self) -> &ParserStats

pub fn num_tokens(&self) -> usize

pub fn final_bytes(&self) -> &[u8]

pub fn is_accepting(&mut self) -> bool

pub fn bytes_since(&self, idx: usize) -> &[u8]

pub fn start_without_prompt(&mut self)

pub fn process_prompt(&mut self, prompt: Vec<u32>) -> Vec<u32>

pub fn error_message(&self) -> Option<String>

pub fn validate_token(&mut self, token: u32) -> Result<bool, Error>

pub fn validate_tokens_raw(&mut self, tokens: &[u32]) -> Result<usize, Error>

Returns how many of the passed tokens can be accepted by the parser. It does not tokenize forced bytes, so will accept non-canonical tokenizations. If called with more than one token, it may ignore max_tokens constraints.

pub fn compute_mask(&mut self) -> Result<SimpleVob, Error>

pub fn temperature(&self) -> Option<f32>

pub fn consume_token(&mut self, token: u32) -> Result<usize, Error>

Extend the current state of the parser with given token. Returns number of tokens to backtrack if any.

pub fn check_stop(&mut self) -> Result<bool, Error>

Check whether the current parser state forces the sequence to stop. If so, puts the parser in stop state and returns true. Otherwise, returns false. This generally should be called after consume_token().

pub fn compute_ff_tokens(&mut self) -> Vec<u32>

Check if there are any tokens to fast-forward, forced by the current parser state.

pub fn consume_ff_tokens(&mut self) -> Result<Vec<u32>, Error>

Compute and then consume fast-forward tokens.

Trait Implementations

impl Clone for TokenParser

fn clone(&self) -> TokenParser

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.