Struct TokTrie

pub struct TokTrie { /* private fields */ }

Implementations

impl TokTrie

pub const SPECIAL_TOKEN_MARKER: u8 = 255u8

pub const MAX_DBG_TOKENS: usize = 200usize

pub fn from(info: &TokRxInfo, words: &Vec<Vec<u8>>) -> TokTrie

pub fn with_eos_token(&self, eos_token: u32) -> TokTrie

pub fn with_info(&self, info: TokRxInfo) -> TokTrie

pub fn build_chat_mode_trie(&self) -> TokTrie

pub fn info(&self) -> &TokRxInfo

pub fn special_token(&self, tok: SpecialToken) -> u32

pub fn eos_token(&self) -> u32

pub fn vocab_size(&self) -> usize

pub fn alloc_token_set(&self) -> SimpleVob

pub fn singleton_token_set(&self, tok: u32) -> SimpleVob

pub fn token_set_dbg(&self, ts: &SimpleVob) -> String

pub fn alloc_logits(&self) -> Vec<f32>

pub fn test_trace_tokens(&self, toks: &[u32]) -> String

pub fn tokens_dbg(&self, toks: &[u32]) -> String

pub fn token_dbg(&self, idx: u32) -> String

pub fn token_str(&self, idx: u32) -> String

pub fn token(&self, idx: u32) -> &[u8]

pub fn decode(&self, tokens: &[u32]) -> Vec<u8>

pub fn decode_raw(&self, tokens: &[u32]) -> Vec<u8>

pub fn decode_str(&self, tokens: &[u32]) -> String

pub fn get_special_token(&self, name: &str) -> Option<u32>

pub fn get_special_tokens(&self) -> Vec<u32>

pub fn greedy_tokenize(&self, bytes: &[u8]) -> Vec<u32>

pub fn tokenize_with_greedy_fallback( &self, s: &[u8], str_tokenize: impl FnOnce(&str) -> Vec<u32>, ) -> Vec<u32>

pub fn has_extensions(&self, bytes: &[u8]) -> bool

pub fn token_id(&self, bytes: &[u8]) -> Option<u32>

pub fn prefix_token_id(&self, bytes: &[u8]) -> (u32, usize)

pub fn from_bytes(bytes: &[u8]) -> TokTrie

pub fn max_token_len(&self) -> usize

pub fn serialize(&self) -> Vec<u8>

pub fn root(&self) -> &TrieNode

pub fn check_against(&self, tokens: &Vec<Vec<u8>>)

pub fn child_at_byte<'a>( &'a self, n: &'a TrieNode, byte: u8, ) -> Option<&'a TrieNode>

pub fn all_subtokens(&self, bytes: &[u8]) -> Vec<u32>

pub fn node_children(&self, n: &TrieNode) -> NodeChildren<'_>

pub fn child_at_bytes<'a>( &'a self, n: &'a TrieNode, bytes: &[u8], ) -> Option<&'a TrieNode>

pub fn token_id_at_bytes(&self, bytes: &[u8]) -> Option<u32>

pub fn compute_bias(&self, r: &mut impl Recognizer, logits: &mut SimpleVob)

pub fn compute_bias_ext( &self, r: &mut impl Recognizer, logits: &mut SimpleVob, start: &[u8], )

pub fn apply_duplicates(&self, logits: &mut SimpleVob)

pub fn append_tokens( &self, r: &mut impl Recognizer, ts: &[u32], ) -> Result<(), Error>

pub fn append_token(&self, r: &mut impl Recognizer, t: u32) -> Result<(), Error>

pub fn token_allowed(&self, r: &mut impl Recognizer, t: u32) -> bool

pub fn chop_tokens( &self, r: &mut impl Recognizer, tokens: &[u32], ) -> (usize, usize)

Return how many tokens and bytes need to chopped off tokens, so that we do not limit all possible future tokenizations matching the recognizer.

pub fn has_valid_extensions( &self, r: &mut impl Recognizer, start: &[u8], ) -> bool

Check if add_bias() would have returned any tokens.

pub fn add_bias( &self, r: &mut impl Recognizer, toks: &mut SimpleVob, start: &[u8], )

pub fn sorted_tokens(&self) -> Vec<(u32, Vec<u8>)>

pub fn trie_stats(&self) -> String

Trait Implementations

impl Clone for TokTrie

fn clone(&self) -> TokTrie

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.