diffusion_rs_common

Function load_bpe_tokenizer

source 
pub fn load_bpe_tokenizer(
    vocab_file: &FileData,
    merges_file: &FileData,
    src: &ModelSource,
) -> Result<Tokenizer>