pub struct SpeculativeLoader {
pub target: Box<dyn Loader>,
pub draft: Box<dyn Loader>,
pub config: SpeculativeConfig,
}
Expand description
A loader for a speculative pipeline using 2 Loader
s.
Fields§
§target: Box<dyn Loader>
§draft: Box<dyn Loader>
§config: SpeculativeConfig
Trait Implementations§
source§impl Loader for SpeculativeLoader
impl Loader for SpeculativeLoader
source§fn load_model_from_hf(
&self,
revision: Option<String>,
token_source: TokenSource,
dtype: &dyn TryIntoDType,
device: &Device,
silent: bool,
mapper: DeviceMapMetadata,
in_situ_quant: Option<IsqType>,
paged_attn_config: Option<PagedAttentionConfig>,
) -> Result<Arc<Mutex<dyn Pipeline + Sync + Send>>, Error>
fn load_model_from_hf( &self, revision: Option<String>, token_source: TokenSource, dtype: &dyn TryIntoDType, device: &Device, silent: bool, mapper: DeviceMapMetadata, in_situ_quant: Option<IsqType>, paged_attn_config: Option<PagedAttentionConfig>, ) -> Result<Arc<Mutex<dyn Pipeline + Sync + Send>>, Error>
If
revision
is None, then it defaults to main
.
If dtype
is None, then it defaults to the model default (usually BF16).
If model is not found on HF, will attempt to resolve locally.source§fn load_model_from_path(
&self,
paths: &Box<dyn ModelPaths>,
dtype: &dyn TryIntoDType,
device: &Device,
silent: bool,
mapper: DeviceMapMetadata,
in_situ_quant: Option<IsqType>,
paged_attn_config: Option<PagedAttentionConfig>,
) -> Result<Arc<Mutex<dyn Pipeline + Sync + Send>>, Error>
fn load_model_from_path( &self, paths: &Box<dyn ModelPaths>, dtype: &dyn TryIntoDType, device: &Device, silent: bool, mapper: DeviceMapMetadata, in_situ_quant: Option<IsqType>, paged_attn_config: Option<PagedAttentionConfig>, ) -> Result<Arc<Mutex<dyn Pipeline + Sync + Send>>, Error>
Load a model from the specified paths.
Also initializes
DEBUG
.fn get_id(&self) -> String
fn get_kind(&self) -> ModelKind
Auto Trait Implementations§
impl Freeze for SpeculativeLoader
impl !RefUnwindSafe for SpeculativeLoader
impl Send for SpeculativeLoader
impl Sync for SpeculativeLoader
impl Unpin for SpeculativeLoader
impl !UnwindSafe for SpeculativeLoader
Blanket Implementations§
source§impl<T> BorrowMut<T> for Twhere
T: ?Sized,
impl<T> BorrowMut<T> for Twhere
T: ?Sized,
source§fn borrow_mut(&mut self) -> &mut T
fn borrow_mut(&mut self) -> &mut T
Mutably borrows from an owned value. Read more
§impl<T> Downcast for Twhere
T: AsAny + ?Sized,
impl<T> Downcast for Twhere
T: AsAny + ?Sized,
§fn downcast_ref<T>(&self) -> Option<&T>where
T: AsAny,
fn downcast_ref<T>(&self) -> Option<&T>where
T: AsAny,
Forward to the method defined on the type
Any
.§fn downcast_mut<T>(&mut self) -> Option<&mut T>where
T: AsAny,
fn downcast_mut<T>(&mut self) -> Option<&mut T>where
T: AsAny,
Forward to the method defined on the type
Any
.§impl<T> Instrument for T
impl<T> Instrument for T
§fn instrument(self, span: Span) -> Instrumented<Self>
fn instrument(self, span: Span) -> Instrumented<Self>
§fn in_current_span(self) -> Instrumented<Self>
fn in_current_span(self) -> Instrumented<Self>
source§impl<T> IntoEither for T
impl<T> IntoEither for T
source§fn into_either(self, into_left: bool) -> Either<Self, Self> ⓘ
fn into_either(self, into_left: bool) -> Either<Self, Self> ⓘ
Converts
self
into a Left
variant of Either<Self, Self>
if into_left
is true
.
Converts self
into a Right
variant of Either<Self, Self>
otherwise. Read moresource§fn into_either_with<F>(self, into_left: F) -> Either<Self, Self> ⓘ
fn into_either_with<F>(self, into_left: F) -> Either<Self, Self> ⓘ
Converts
self
into a Left
variant of Either<Self, Self>
if into_left(&self)
returns true
.
Converts self
into a Right
variant of Either<Self, Self>
otherwise. Read more