mistralrs_core/vision_models/qwen2vl/
config.rs

1// https://github.com/huggingface/transformers/blob/f2c388e3f946862f657acc1e21b272ec946fc66c/src/transformers/models/qwen2_vl/configuration_qwen2_vl.py
2
3use mistralrs_quant::QuantizedConfig;
4
5use crate::layers::Activation;
6
7use crate::serde_default_fn;
8
9serde_default_fn!(Activation, default_vision_hidden_act, Activation::QuickGelu);
10serde_default_fn!(usize, default_in_channels, 3);
11
12#[derive(Debug, Clone, serde::Deserialize)]
13pub struct VisionConfig {
14    pub depth: usize,
15    pub embed_dim: usize,
16    pub hidden_size: usize,
17    #[serde(default = "default_vision_hidden_act")]
18    pub hidden_act: Activation,
19    pub mlp_ratio: f64,
20    pub num_heads: usize,
21    #[serde(default = "default_in_channels")]
22    pub in_channels: usize,
23    pub patch_size: usize,
24    pub spatial_merge_size: usize,
25    pub temporal_patch_size: usize,
26}
27
28#[derive(Debug, Clone, serde::Deserialize)]
29pub struct MRopeScaling {
30    pub mrope_section: Vec<usize>,
31}
32
33#[derive(Debug, Clone, serde::Deserialize)]
34pub struct Config {
35    pub vocab_size: usize,
36    pub hidden_size: usize,
37    pub intermediate_size: usize,
38    pub num_hidden_layers: usize,
39    pub num_attention_heads: usize,
40    pub num_key_value_heads: usize,
41    pub hidden_act: Activation,
42    pub max_position_embeddings: usize,
43    pub rms_norm_eps: f64,
44    pub tie_word_embeddings: bool,
45    pub rope_theta: f64,
46    pub use_sliding_window: bool,
47    pub sliding_window: Option<usize>,
48    pub vision_config: VisionConfig,
49    pub rope_scaling: MRopeScaling,
50    pub quantization_config: Option<QuantizedConfig>,
51    pub image_token_id: u32,
52    pub video_token_id: u32,
53    // pub vision_start_token_id: usize,
54    // pub max_window_layers: usize,
55}
mistralrs_core/vision_models/qwen2vl/config.rs

mistralrs_core/vision_models/qwen2vl/
config.rs
