pub struct TextSpeculativeBuilder { /* private fields */ }Implementations§
Source§impl TextSpeculativeBuilder
impl TextSpeculativeBuilder
Sourcepub fn new(
target: TextModelBuilder,
draft: TextModelBuilder,
speculative_config: SpeculativeConfig,
) -> Result<Self>
pub fn new( target: TextModelBuilder, draft: TextModelBuilder, speculative_config: SpeculativeConfig, ) -> Result<Self>
Create a builder for a speculative decoding pipeline.
- PagedAttention settings are ignored as our impl of speculative decoding does not support this yet.
- Prefix caching settings are ignored as our impl of speculative decoding does not support this yet.
Otherwise, scheduling parameters such as max_num_seqs are sourced from the target model.
pub async fn build(self) -> Result<Model>
Auto Trait Implementations§
impl Freeze for TextSpeculativeBuilder
impl RefUnwindSafe for TextSpeculativeBuilder
impl Send for TextSpeculativeBuilder
impl Sync for TextSpeculativeBuilder
impl Unpin for TextSpeculativeBuilder
impl UnwindSafe for TextSpeculativeBuilder
Blanket Implementations§
Source§impl<T> BorrowMut<T> for Twhere
T: ?Sized,
impl<T> BorrowMut<T> for Twhere
T: ?Sized,
Source§fn borrow_mut(&mut self) -> &mut T
fn borrow_mut(&mut self) -> &mut T
Mutably borrows from an owned value. Read more
§impl<T> Downcast for Twhere
T: AsAny + ?Sized,
impl<T> Downcast for Twhere
T: AsAny + ?Sized,
§fn downcast_ref<T>(&self) -> Option<&T>where
T: AsAny,
fn downcast_ref<T>(&self) -> Option<&T>where
T: AsAny,
Forward to the method defined on the type
Any.§fn downcast_mut<T>(&mut self) -> Option<&mut T>where
T: AsAny,
fn downcast_mut<T>(&mut self) -> Option<&mut T>where
T: AsAny,
Forward to the method defined on the type
Any.§impl<T> Instrument for T
impl<T> Instrument for T
§fn instrument(self, span: Span) -> Instrumented<Self>
fn instrument(self, span: Span) -> Instrumented<Self>
§fn in_current_span(self) -> Instrumented<Self>
fn in_current_span(self) -> Instrumented<Self>
Source§impl<T> IntoEither for T
impl<T> IntoEither for T
Source§fn into_either(self, into_left: bool) -> Either<Self, Self> ⓘ
fn into_either(self, into_left: bool) -> Either<Self, Self> ⓘ
Converts
self into a Left variant of Either<Self, Self>
if into_left is true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read moreSource§fn into_either_with<F>(self, into_left: F) -> Either<Self, Self> ⓘ
fn into_either_with<F>(self, into_left: F) -> Either<Self, Self> ⓘ
Converts
self into a Left variant of Either<Self, Self>
if into_left(&self) returns true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read more