Enum QuantMethodConfig

pub enum QuantMethodConfig {
    Gptq {
        bits: i32,
        use_exllama: bool,
        q_weight: Tensor,
        gptq_qzeros: Option<Tensor>,
        gptq_scales: Tensor,
        g_idx: Option<Tensor>,
        bias: Option<Tensor>,
        workspace: Option<Tensor>,
        is_marlin: bool,
    },
    Gguf {
        q_weight: Arc<QTensor>,
        b: Option<Tensor>,
    },
    Unquantized(Linear),
    Hqq {
        tensor: Tensor,
        bits: HqqBits,
        group_size: NonZeroUsize,
        axis: HqqAxis,
        optimization_steps: Option<usize>,
        round_zeros: Option<bool>,
        channel_wise: Option<bool>,
        bias: Option<Tensor>,
    },
    Dummy,
    FP8 {
        lin: Linear,
        dtype: DType,
    },
    Bnb {
        weight: Tensor,
        bias: Option<Tensor>,
        params: BnbQuantParmas,
        quant_ty: BnbQuantType,
    },
    BlockwiseFP8 {
        weight: Tensor,
        weight_scale_inv: Tensor,
        bias: Option<Tensor>,
        dequant_dtype: DType,
        weight_block_size: Vec<usize>,
    },
}

Variants

Gptq

Fields

bits: i32

use_exllama: bool

q_weight: Tensor

gptq_qzeros: Option<Tensor>

gptq_scales: Tensor

g_idx: Option<Tensor>

bias: Option<Tensor>

workspace: Option<Tensor>

is_marlin: bool

Gguf

Fields

q_weight: Arc<QTensor>

b: Option<Tensor>

Unquantized(Linear)

Hqq

Fields

tensor: Tensor

bits: HqqBits

group_size: NonZeroUsize

axis: HqqAxis

optimization_steps: Option<usize>

round_zeros: Option<bool>

channel_wise: Option<bool>

bias: Option<Tensor>

Dummy

FP8

Fields

lin: Linear

dtype: DType

Bnb

Fields

weight: Tensor

bias: Option<Tensor>

params: BnbQuantParmas

quant_ty: BnbQuantType

BlockwiseFP8

Fields

weight: Tensor

weight_scale_inv: Tensor

bias: Option<Tensor>

dequant_dtype: DType

weight_block_size: Vec<usize>

Trait Implementations

impl Clone for QuantMethodConfig

fn clone(&self) -> QuantMethodConfig

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for QuantMethodConfig

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.