Struct AnyMoeLoader

pub struct AnyMoeLoader {
    pub target: Box<dyn Loader>,
    pub config: AnyMoeConfig,
    pub path: String,
    pub prefix: String,
    pub mlp: String,
    pub model_ids: Vec<String>,
    pub layers: Vec<usize>,
}

Fields

target: Box<dyn Loader>

config: AnyMoeConfig

path: String

prefix: String

mlp: String

model_ids: Vec<String>

layers: Vec<usize>

Trait Implementations

impl Loader for AnyMoeLoader

fn load_model_from_hf( &self, revision: Option<String>, token_source: TokenSource, dtype: &dyn TryIntoDType, device: &Device, silent: bool, mapper: DeviceMapMetadata, in_situ_quant: Option<IsqType>, paged_attn_config: Option<PagedAttentionConfig>, ) -> Result<Arc<Mutex<dyn Pipeline + Sync + Send>>, Error>

If revision is None, then it defaults to main. If dtype is None, then it defaults to the model default (usually BF16). If model is not found on HF, will attempt to resolve locally.

fn load_model_from_path( &self, paths: &Box<dyn ModelPaths>, dtype: &dyn TryIntoDType, device: &Device, silent: bool, mapper: DeviceMapMetadata, in_situ_quant: Option<IsqType>, paged_attn_config: Option<PagedAttentionConfig>, ) -> Result<Arc<Mutex<dyn Pipeline + Sync + Send>>, Error>

Load a model from the specified paths. Also initializes DEBUG.

fn get_id(&self) -> String

fn get_kind(&self) -> ModelKind