Enum ModelSelected

pub enum ModelSelected {
    Toml {
        file: String,
    },
    Plain {
        model_id: String,
        tokenizer_json: Option<String>,
        arch: Option<NormalLoaderType>,
        dtype: ModelDType,
        topology: Option<String>,
        organization: Option<IsqOrganization>,
        write_uqff: Option<PathBuf>,
        from_uqff: Option<PathBuf>,
        imatrix: Option<PathBuf>,
        calibration_file: Option<PathBuf>,
        max_seq_len: usize,
        max_batch_size: usize,
    },
    XLora {
        model_id: Option<String>,
        tokenizer_json: Option<String>,
        xlora_model_id: String,
        order: String,
        tgt_non_granular_index: Option<usize>,
        arch: Option<NormalLoaderType>,
        dtype: ModelDType,
        topology: Option<String>,
        write_uqff: Option<PathBuf>,
        from_uqff: Option<PathBuf>,
        max_seq_len: usize,
        max_batch_size: usize,
    },
    Lora {
        model_id: Option<String>,
        tokenizer_json: Option<String>,
        adapters_model_id: String,
        order: String,
        arch: Option<NormalLoaderType>,
        dtype: ModelDType,
        topology: Option<String>,
        write_uqff: Option<PathBuf>,
        from_uqff: Option<PathBuf>,
        max_seq_len: usize,
        max_batch_size: usize,
    },
    GGUF {
        tok_model_id: Option<String>,
        quantized_model_id: String,
        quantized_filename: String,
        dtype: ModelDType,
        topology: Option<String>,
        max_seq_len: usize,
        max_batch_size: usize,
    },
    XLoraGGUF {
        tok_model_id: Option<String>,
        quantized_model_id: String,
        quantized_filename: String,
        xlora_model_id: String,
        order: String,
        tgt_non_granular_index: Option<usize>,
        dtype: ModelDType,
        topology: Option<String>,
        max_seq_len: usize,
        max_batch_size: usize,
    },
    LoraGGUF {
        tok_model_id: Option<String>,
        quantized_model_id: String,
        quantized_filename: String,
        adapters_model_id: String,
        order: String,
        dtype: ModelDType,
        topology: Option<String>,
        max_seq_len: usize,
        max_batch_size: usize,
    },
    GGML {
        tok_model_id: String,
        tokenizer_json: Option<String>,
        quantized_model_id: String,
        quantized_filename: String,
        gqa: usize,
        dtype: ModelDType,
        topology: Option<String>,
        max_seq_len: usize,
        max_batch_size: usize,
    },
    XLoraGGML {
        tok_model_id: Option<String>,
        tokenizer_json: Option<String>,
        quantized_model_id: String,
        quantized_filename: String,
        xlora_model_id: String,
        order: String,
        tgt_non_granular_index: Option<usize>,
        gqa: usize,
        dtype: ModelDType,
        topology: Option<String>,
        max_seq_len: usize,
        max_batch_size: usize,
    },
    LoraGGML {
        tok_model_id: Option<String>,
        tokenizer_json: Option<String>,
        quantized_model_id: String,
        quantized_filename: String,
        adapters_model_id: String,
        order: String,
        gqa: usize,
        dtype: ModelDType,
        topology: Option<String>,
        max_seq_len: usize,
        max_batch_size: usize,
    },
    VisionPlain {

        model_id: String,
        tokenizer_json: Option<String>,
        arch: VisionLoaderType,
        dtype: ModelDType,
        topology: Option<String>,
        write_uqff: Option<PathBuf>,
        from_uqff: Option<PathBuf>,
        max_edge: Option<u32>,
        calibration_file: Option<PathBuf>,
        imatrix: Option<PathBuf>,
        max_seq_len: usize,
        max_batch_size: usize,
        max_num_images: usize,
        max_image_length: usize,
    
},
    DiffusionPlain {
        model_id: String,
        arch: DiffusionLoaderType,
        dtype: ModelDType,
    },
}

Variants

Toml

Select the model from a toml file

Fields

file: String

.toml file containing the selector configuration.

Plain

Select a plain model, without quantization or adapters

Fields

model_id: String

Model ID to load from. This may be a HF hub repo or a local path.

tokenizer_json: Option<String>

Path to local tokenizer.json file. If this is specified it is used over any remote file.

arch: Option<NormalLoaderType>

The architecture of the model.

dtype: ModelDType

Model data type. Defaults to auto.

topology: Option<String>

Path to a topology YAML file.

organization: Option<IsqOrganization>

ISQ organization: default or moqe (Mixture of Quantized Experts: https://arxiv.org/abs/2310.02410).

write_uqff: Option<PathBuf>

UQFF path to write to.

from_uqff: Option<PathBuf>

UQFF path to load from. If provided, this takes precedence over applying ISQ.

imatrix: Option<PathBuf>

.imatrix file to enhance GGUF quantizations with. Incompatible with --calibration-file/-c

calibration_file: Option<PathBuf>

Generate and utilize an imatrix to enhance GGUF quantizations. Incompatible with --imatrix/-i

max_seq_len: usize

Maximum prompt sequence length to expect for this model. This affects automatic device mapping but is not a hard limit.

max_batch_size: usize

Maximum prompt batch size to expect for this model. This affects automatic device mapping but is not a hard limit.

XLora

Select an X-LoRA architecture

Fields

model_id: Option<String>

Force a base model ID to load from instead of using the ordering file. This may be a HF hub repo or a local path.

tokenizer_json: Option<String>

Path to local tokenizer.json file. If this is specified it is used over any remote file.

xlora_model_id: String

Model ID to load X-LoRA from. This may be a HF hub repo or a local path.

order: String

Ordering JSON file

tgt_non_granular_index: Option<usize>

Index of completion tokens to generate scalings up until. If this is 1, then there will be one completion token generated before it is cached. This makes the maximum running sequences 1.

arch: Option<NormalLoaderType>

The architecture of the model.

dtype: ModelDType

Model data type. Defaults to auto.

topology: Option<String>

Path to a topology YAML file.

write_uqff: Option<PathBuf>

UQFF path to write to.

from_uqff: Option<PathBuf>

UQFF path to load from. If provided, this takes precedence over applying ISQ.

max_seq_len: usize

Maximum prompt sequence length to expect for this model. This affects automatic device mapping but is not a hard limit.

max_batch_size: usize

Maximum prompt batch size to expect for this model. This affects automatic device mapping but is not a hard limit.

Lora

Select a LoRA architecture

Fields

model_id: Option<String>

Force a base model ID to load from instead of using the ordering file. This may be a HF hub repo or a local path.

tokenizer_json: Option<String>

Path to local tokenizer.json file. If this is specified it is used over any remote file.

adapters_model_id: String

Model ID to load LoRA from. This may be a HF hub repo or a local path.

order: String

Ordering JSON file

arch: Option<NormalLoaderType>

The architecture of the model.

dtype: ModelDType

Model data type. Defaults to auto.

topology: Option<String>

Path to a topology YAML file.

write_uqff: Option<PathBuf>

UQFF path to write to.

from_uqff: Option<PathBuf>

UQFF path to load from. If provided, this takes precedence over applying ISQ.

max_seq_len: usize

Maximum prompt sequence length to expect for this model. This affects automatic device mapping but is not a hard limit.

max_batch_size: usize

Maximum prompt batch size to expect for this model. This affects automatic device mapping but is not a hard limit.

GGUF

Select a GGUF model.

Fields

tok_model_id: Option<String>

tok_model_id is the local or remote model ID where you can find a tokenizer_config.json file. If the chat_template is specified, then it will be treated as a path and used over remote files, removing all remote accesses.

quantized_model_id: String

Quantized model ID to find the quantized_filename. This may be a HF hub repo or a local path.

quantized_filename: String

Quantized filename(s). May be a single filename, or use a delimiter of “ “ (a single space) for multiple files.

dtype: ModelDType

Model data type. Defaults to auto.

topology: Option<String>

Path to a topology YAML file.

max_seq_len: usize

Maximum prompt sequence length to expect for this model. This affects automatic device mapping but is not a hard limit.

max_batch_size: usize

Maximum prompt batch size to expect for this model. This affects automatic device mapping but is not a hard limit.

XLoraGGUF

Select a GGUF model with X-LoRA.

Fields

tok_model_id: Option<String>

tok_model_id is the local or remote model ID where you can find a tokenizer_config.json file. If the chat_template is specified, then it will be treated as a path and used over remote files, removing all remote accesses.

quantized_model_id: String

Quantized model ID to find the quantized_filename. This may be a HF hub repo or a local path.

quantized_filename: String

Quantized filename(s). May be a single filename, or use a delimiter of “ “ (a single space) for multiple files.

xlora_model_id: String

Model ID to load X-LoRA from. This may be a HF hub repo or a local path.

order: String

Ordering JSON file

tgt_non_granular_index: Option<usize>

Index of completion tokens to generate scalings up until. If this is 1, then there will be one completion token generated before it is cached. This makes the maximum running sequences 1.

dtype: ModelDType

Model data type. Defaults to auto.

topology: Option<String>

Path to a topology YAML file.

max_seq_len: usize

Maximum prompt sequence length to expect for this model. This affects automatic device mapping but is not a hard limit.

max_batch_size: usize

Maximum prompt batch size to expect for this model. This affects automatic device mapping but is not a hard limit.

LoraGGUF

Select a GGUF model with LoRA.

Fields

tok_model_id: Option<String>

tok_model_id is the local or remote model ID where you can find a tokenizer_config.json file. If the chat_template is specified, then it will be treated as a path and used over remote files, removing all remote accesses.

quantized_model_id: String

Quantized model ID to find the quantized_filename. This may be a HF hub repo or a local path.

quantized_filename: String

Quantized filename(s). May be a single filename, or use a delimiter of “ “ (a single space) for multiple files.

adapters_model_id: String

Model ID to load LoRA from. This may be a HF hub repo or a local path.

order: String

Ordering JSON file

dtype: ModelDType

Model data type. Defaults to auto.

topology: Option<String>

Path to a topology YAML file.

max_seq_len: usize

Maximum prompt sequence length to expect for this model. This affects automatic device mapping but is not a hard limit.

max_batch_size: usize

Maximum prompt batch size to expect for this model. This affects automatic device mapping but is not a hard limit.

GGML

Select a GGML model.

Fields

tok_model_id: String

Model ID to load the tokenizer from. This may be a HF hub repo or a local path.

tokenizer_json: Option<String>

Path to local tokenizer.json file. If this is specified it is used over any remote file.

quantized_model_id: String

Quantized model ID to find the quantized_filename. This may be a HF hub repo or a local path.

quantized_filename: String

Quantized filename.

gqa: usize

GQA value

dtype: ModelDType

Model data type. Defaults to auto.

topology: Option<String>

Path to a topology YAML file.

max_seq_len: usize

Maximum prompt sequence length to expect for this model. This affects automatic device mapping but is not a hard limit.

max_batch_size: usize

Maximum prompt batch size to expect for this model. This affects automatic device mapping but is not a hard limit.

XLoraGGML

Select a GGML model with X-LoRA.

Fields

tok_model_id: Option<String>

Model ID to load the tokenizer from. This may be a HF hub repo or a local path.

tokenizer_json: Option<String>

Path to local tokenizer.json file. If this is specified it is used over any remote file.

quantized_model_id: String

Quantized model ID to find the quantized_filename. This may be a HF hub repo or a local path.

quantized_filename: String

Quantized filename.

xlora_model_id: String

Model ID to load X-LoRA from. This may be a HF hub repo or a local path.

order: String

Ordering JSON file

tgt_non_granular_index: Option<usize>

Index of completion tokens to generate scalings up until. If this is 1, then there will be one completion token generated before it is cached. This makes the maximum running sequences 1.

gqa: usize

GQA value

dtype: ModelDType

Model data type. Defaults to auto.

topology: Option<String>

Path to a topology YAML file.

max_seq_len: usize

Maximum prompt sequence length to expect for this model. This affects automatic device mapping but is not a hard limit.

max_batch_size: usize

Maximum prompt batch size to expect for this model. This affects automatic device mapping but is not a hard limit.

LoraGGML

Select a GGML model with LoRA.

Fields

tok_model_id: Option<String>

Model ID to load the tokenizer from. This may be a HF hub repo or a local path.

tokenizer_json: Option<String>

Path to local tokenizer.json file. If this is specified it is used over any remote file.

quantized_model_id: String

Quantized model ID to find the quantized_filename. This may be a HF hub repo or a local path.

quantized_filename: String

Quantized filename.

adapters_model_id: String

Model ID to load LoRA from. This may be a HF hub repo or a local path.

order: String

Ordering JSON file

gqa: usize

GQA value

dtype: ModelDType

Model data type. Defaults to auto.

topology: Option<String>

Path to a topology YAML file.

max_seq_len: usize

Maximum prompt sequence length to expect for this model. This affects automatic device mapping but is not a hard limit.

max_batch_size: usize

Maximum prompt batch size to expect for this model. This affects automatic device mapping but is not a hard limit.

VisionPlain

Select a vision plain model, without quantization or adapters

Fields

model_id: String

Model ID to load from. This may be a HF hub repo or a local path.

tokenizer_json: Option<String>

Path to local tokenizer.json file. If this is specified it is used over any remote file.

arch: VisionLoaderType

The architecture of the model.

dtype: ModelDType

Model data type. Defaults to auto.

topology: Option<String>

Path to a topology YAML file.

write_uqff: Option<PathBuf>

UQFF path to write to.

from_uqff: Option<PathBuf>

UQFF path to load from. If provided, this takes precedence over applying ISQ.

max_edge: Option<u32>

Automatically resize and pad images to this maximum edge length. Aspect ratio is preserved. This is only supported on the Qwen2-VL and Idefics models. Others handle this internally.

calibration_file: Option<PathBuf>

Generate and utilize an imatrix to enhance GGUF quantizations.

imatrix: Option<PathBuf>

.cimatrix file to enhance GGUF quantizations with. This must be a .cimatrix file.

max_seq_len: usize

Maximum prompt sequence length to expect for this model. This affects automatic device mapping but is not a hard limit.

max_batch_size: usize

Maximum prompt batch size to expect for this model. This affects automatic device mapping but is not a hard limit.

max_num_images: usize

Maximum prompt number of images to expect for this model. This affects automatic device mapping but is not a hard limit.

max_image_length: usize

Maximum expected image size will have this edge length on both edges. This affects automatic device mapping but is not a hard limit.

DiffusionPlain

Select a diffusion plain model, without quantization or adapters

Fields

model_id: String

Model ID to load from. This may be a HF hub repo or a local path.

arch: DiffusionLoaderType

The architecture of the model.

dtype: ModelDType

Model data type. Defaults to auto.

Trait Implementations

impl Debug for ModelSelected

fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error>

Formats the value using the given formatter.

impl FromArgMatches for ModelSelected

fn from_arg_matches( __clap_arg_matches: &ArgMatches, ) -> Result<ModelSelected, Error>

Instantiate Self from [ArgMatches], parsing the arguments as needed.

fn from_arg_matches_mut( __clap_arg_matches: &mut ArgMatches, ) -> Result<ModelSelected, Error>

Instantiate Self from [ArgMatches], parsing the arguments as needed.

fn update_from_arg_matches( &mut self, __clap_arg_matches: &ArgMatches, ) -> Result<(), Error>

Assign values from ArgMatches to self.

fn update_from_arg_matches_mut<'b>( &mut self, __clap_arg_matches: &mut ArgMatches, ) -> Result<(), Error>

Assign values from ArgMatches to self.

impl Subcommand for ModelSelected

fn augment_subcommands<'b>(__clap_app: Command) -> Command

Append to [Command] so it can instantiate Self via [FromArgMatches::from_arg_matches_mut]

fn augment_subcommands_for_update<'b>(__clap_app: Command) -> Command

Append to [Command] so it can instantiate self via [FromArgMatches::update_from_arg_matches_mut]

fn has_subcommand(__clap_name: &str) -> bool

Test whether Self can parse a specific subcommand