mistralrs::llguidance::ffi

Function llg_new_tokenizer

pub extern "C" fn llg_new_tokenizer(
    tok_init: &LlgTokenizerInit,
    error_string: *mut i8,
    error_string_len: usize,
) -> *mut LlgTokenizer
Expand description

Construct a new tokenizer from the given TokenizerInit