Enum BnbLinear

pub enum BnbLinear {
    Fp4Nf4 {
        weight: Tensor,
        bias: Option<Tensor>,
        params: BnbQuantParmas,
        quant_ty: BnbQuantType,
    },
    Int8 {
        weight: Tensor,
        scb: Tensor,
        bias: Option<Tensor>,
    },
}

Variants

Fp4Nf4

Fields

weight: Tensor

bias: Option<Tensor>

params: BnbQuantParmas

quant_ty: BnbQuantType

Int8

Fields

weight: Tensor

scb: Tensor

bias: Option<Tensor>

Implementations

impl BnbLinear

pub fn linear_b( in_dim: usize, out_dim: usize, bias: bool, vb: VarBuilder<'_>, ) -> Result<Self>

Trait Implementations

impl Debug for BnbLinear

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl QuantMethod for BnbLinear

fn new(method: QuantMethodConfig) -> Result<Self>

where Self: Sized,

fn dequantize_w(&self, out_ty: DType) -> Result<Tensor>

fn forward(&self, xs: &Tensor) -> Result<Tensor>

Compute matmul of self and a. self should contain the weights.

fn quantized_act_type(&self) -> Option<DType>

If a quantized method, return the activation dtype.

fn to_device(&self, dev: &Device) -> Result<Arc<dyn QuantMethod>>

Cast this layer to the given device.

fn size_in_bytes(&self) -> Result<usize>

fn device(&self) -> Device

fn forward_autocast(&self, a: &Tensor) -> Result<Tensor>

Compute matmul of self and a. self should contain the weights. Automatically cast to required quantization actiation type and back

fn forward_via_half(&self, a: &Tensor) -> Result<Tensor>

Compute matmul of self and a. self should contain the weights. This may go via half precision if it is supported.