mistralrs_core/amoe/
inputs.rs

1use std::{fs::File, path::Path};
2
3use csv::Reader;
4use serde::Deserialize;
5
6pub struct AnyMoeTrainingResult {
7    pub steps: usize,
8    /// One for each gating layer
9    pub final_loss: Vec<f32>,
10}
11
12#[derive(Deserialize, Debug)]
13pub struct AnyMoeTrainingInputRow {
14    pub prompt: String,
15    pub expert: usize,
16    pub image_urls: Option<Vec<String>>,
17}
18
19#[derive(Deserialize, Debug)]
20pub struct AnyMoeTrainingInputs {
21    rows: Vec<AnyMoeTrainingInputRow>,
22}
23
24impl AnyMoeTrainingInputs {
25    /// From a CSV file with the mandatory columns `prompt` (String), `expert` (usize), and the optional
26    /// column `image_urls` (`Vec<String>`).
27    pub fn from_csv<P: AsRef<Path>>(file: P) -> anyhow::Result<Self> {
28        let file = File::open(file)?;
29        let mut reader = Reader::from_reader(file);
30        let mut rows = Vec::new();
31        for result in reader.deserialize() {
32            let row: AnyMoeTrainingInputRow = result?;
33            rows.push(row);
34        }
35        Ok(Self { rows })
36    }
37
38    /// From a JSON file with the top-level key being `rows` (array), which contains objects with the
39    /// keys `prompt` (String), `expert` (usize), `image_urls` (Option<Vec<String>>).
40    pub fn from_json<P: AsRef<Path>>(file: P) -> anyhow::Result<Self> {
41        let file = File::open(file)?;
42        Ok(serde_json::from_reader(file)?)
43    }
44
45    pub fn len(&self) -> usize {
46        self.rows.len()
47    }
48
49    pub fn into_inner(self) -> Vec<AnyMoeTrainingInputRow> {
50        self.rows
51    }
52}
mistralrs_core/amoe/inputs.rs

mistralrs_core/amoe/
inputs.rs
