Struct RotatingKvCache

pub struct RotatingKvCache { /* private fields */ }

Implementations

impl RotatingKvCache

pub fn new(dim: usize, max_seq_len: usize) -> Self

pub fn k_cache(&self) -> &RotatingCache

pub fn v_cache(&self) -> &RotatingCache

pub fn k_cache_mut(&mut self) -> &mut RotatingCache

pub fn v_cache_mut(&mut self) -> &mut RotatingCache

pub fn k(&self) -> Result<Option<Tensor>>

pub fn v(&self) -> Result<Option<Tensor>>

pub fn append(&mut self, k: &Tensor, v: &Tensor) -> Result<(Tensor, Tensor)>

pub fn offset(&self) -> usize

pub fn current_seq_len(&self) -> usize

pub fn attn_mask( &self, seq_len: usize, device: &Device, ) -> Result<Option<Tensor>>

pub fn reset(&mut self)

Trait Implementations

impl Clone for RotatingKvCache

fn clone(&self) -> RotatingKvCache

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for RotatingKvCache

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.