Struct BnbLinear

pub struct BnbLinear { /* private fields */ }

Implementations

impl BnbLinear

pub fn linear_b( _in_dim: usize, out_dim: usize, bias: bool, vb: VarBuilder<'_>, ) -> Result<Self>

Trait Implementations

impl Debug for BnbLinear

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl QuantMethod for BnbLinear

fn new(method: QuantMethodConfig) -> Result<Self>

where Self: Sized,

fn dequantize_w(&self) -> Result<Tensor>

fn forward(&self, xs: &Tensor) -> Result<Tensor>

Compute matmul of self and a. self should contain the weights.

fn quantized_act_type(&self) -> Option<DType>

If a quantized method, return the activation dtype.

fn add_delta_w(&self, _delta: &Tensor) -> Result<Arc<dyn QuantMethod>>

Add a delta weight from LoRA to the weights. This should be prescaled with alpha.

fn dtype_and_device(&self) -> (DType, Device)

Weight dtype and device

fn get_bias_mut(&mut self) -> Option<&mut Tensor>

If the quant is backed by a qmatmul.

fn apply_isq( self: Arc<Self>, _dtype: Option<IsqType>, _device: Device, _n_quantized: &AtomicUsize, _imatrix_weight: Option<Vec<f32>>, ) -> Result<Arc<dyn QuantMethod>>

If the quant is backed by a qmatmul.

fn get_max_isq_cpu_threads(&self, _dtype: IsqType) -> Option<NonZeroUsize>

fn maybe_to_gguf_quant(self: Arc<Self>) -> Result<Arc<dyn QuantMethod>>

Convert to an equivalent gguf quantization, if applicable.

fn forward_autocast(&self, a: &Tensor) -> Result<Tensor>

Compute matmul of self and a. self should contain the weights. Automatically cast to required quantization actiation type and back

fn forward_via_half(&self, a: &Tensor) -> Result<Tensor>

Compute matmul of self and a. self should contain the weights. This may go via half precision if it is supported.

fn unquant_weight_bias(&self) -> Option<(Tensor, Option<Tensor>)>

fn begin_track_stats(&mut self) -> Result<()>

Begin tracking stats into an ImatrixLayerStats

fn end_track_stats(&self) -> Result<Tensor>

End tracking stats into an ImatrixLayerStats. Returns the computed imatrix.

impl QuantizedSerde for BnbLinear

fn isq_serde_supported(&self) -> bool

fn name(&self) -> &'static str

fn serialize(&self) -> Result<Cow<'_, [u8]>>

fn deserialize( _data: Cow<'_, [u8]>, _device: &Device, ) -> Result<Arc<dyn QuantMethod>>

where Self: Sized,