mistralrs_server_core/
openai.rs

1//! ## OpenAI compatible functionality.
2
3use std::{collections::HashMap, ops::Deref};
4
5use either::Either;
6use mistralrs_core::{
7    ImageGenerationResponseFormat, LlguidanceGrammar, Tool, ToolChoice, ToolType, WebSearchOptions,
8};
9use serde::{Deserialize, Serialize};
10use serde_json::Value;
11use utoipa::{
12    openapi::{schema::SchemaType, ArrayBuilder, ObjectBuilder, OneOfBuilder, RefOr, Schema, Type},
13    PartialSchema, ToSchema,
14};
15
16/// Inner content structure for messages that can be either a string or key-value pairs
17#[derive(Debug, Clone, Deserialize, Serialize)]
18pub struct MessageInnerContent(
19    #[serde(with = "either::serde_untagged")] pub Either<String, HashMap<String, String>>,
20);
21
22// The impl Deref was preventing the Derive ToSchema and #[schema] macros from
23// properly working, so manually impl ToSchema
24impl PartialSchema for MessageInnerContent {
25    fn schema() -> RefOr<Schema> {
26        RefOr::T(message_inner_content_schema())
27    }
28}
29
30impl ToSchema for MessageInnerContent {
31    fn schemas(
32        schemas: &mut Vec<(
33            String,
34            utoipa::openapi::RefOr<utoipa::openapi::schema::Schema>,
35        )>,
36    ) {
37        schemas.push((
38            MessageInnerContent::name().into(),
39            MessageInnerContent::schema(),
40        ));
41    }
42}
43
44impl Deref for MessageInnerContent {
45    type Target = Either<String, HashMap<String, String>>;
46    fn deref(&self) -> &Self::Target {
47        &self.0
48    }
49}
50
51/// Function for MessageInnerContent Schema generation to handle `Either`
52fn message_inner_content_schema() -> Schema {
53    Schema::OneOf(
54        OneOfBuilder::new()
55            // Either::Left - simple string
56            .item(Schema::Object(
57                ObjectBuilder::new()
58                    .schema_type(SchemaType::Type(Type::String))
59                    .build(),
60            ))
61            // Either::Right - object with string values
62            .item(Schema::Object(
63                ObjectBuilder::new()
64                    .schema_type(SchemaType::Type(Type::Object))
65                    .additional_properties(Some(RefOr::T(Schema::Object(
66                        ObjectBuilder::new()
67                            .schema_type(SchemaType::Type(Type::String))
68                            .build(),
69                    ))))
70                    .build(),
71            ))
72            .build(),
73    )
74}
75
76/// Message content that can be either simple text or complex structured content
77#[derive(Debug, Clone, Deserialize, Serialize)]
78pub struct MessageContent(
79    #[serde(with = "either::serde_untagged")]
80    Either<String, Vec<HashMap<String, MessageInnerContent>>>,
81);
82
83// The impl Deref was preventing the Derive ToSchema and #[schema] macros from
84// properly working, so manually impl ToSchema
85impl PartialSchema for MessageContent {
86    fn schema() -> RefOr<Schema> {
87        RefOr::T(message_content_schema())
88    }
89}
90
91impl ToSchema for MessageContent {
92    fn schemas(
93        schemas: &mut Vec<(
94            String,
95            utoipa::openapi::RefOr<utoipa::openapi::schema::Schema>,
96        )>,
97    ) {
98        schemas.push((MessageContent::name().into(), MessageContent::schema()));
99    }
100}
101
102impl MessageContent {
103    /// Create a new MessageContent from a string
104    pub fn from_text(text: String) -> Self {
105        MessageContent(Either::Left(text))
106    }
107
108    /// Extract text from MessageContent
109    pub fn to_text(&self) -> Option<String> {
110        match &self.0 {
111            Either::Left(text) => Some(text.clone()),
112            Either::Right(parts) => {
113                // For complex content, try to extract text from parts
114                let mut text_parts = Vec::new();
115                for part in parts {
116                    for (key, value) in part {
117                        if key == "text" {
118                            if let Either::Left(text) = &**value {
119                                text_parts.push(text.clone());
120                            }
121                        }
122                    }
123                }
124                if text_parts.is_empty() {
125                    None
126                } else {
127                    Some(text_parts.join(" "))
128                }
129            }
130        }
131    }
132}
133
134impl Deref for MessageContent {
135    type Target = Either<String, Vec<HashMap<String, MessageInnerContent>>>;
136    fn deref(&self) -> &Self::Target {
137        &self.0
138    }
139}
140
141/// Function for MessageContent Schema generation to handle `Either`
142fn message_content_schema() -> Schema {
143    Schema::OneOf(
144        OneOfBuilder::new()
145            .item(Schema::Object(
146                ObjectBuilder::new()
147                    .schema_type(SchemaType::Type(Type::String))
148                    .build(),
149            ))
150            .item(Schema::Array(
151                ArrayBuilder::new()
152                    .items(RefOr::T(Schema::Object(
153                        ObjectBuilder::new()
154                            .schema_type(SchemaType::Type(Type::Object))
155                            .additional_properties(Some(RefOr::Ref(
156                                utoipa::openapi::Ref::from_schema_name("MessageInnerContent"),
157                            )))
158                            .build(),
159                    )))
160                    .build(),
161            ))
162            .build(),
163    )
164}
165
166/// Represents a function call made by the assistant
167///
168/// When using tool calling, this structure contains the details of a function
169/// that the model has decided to call, including the function name and its parameters.
170#[derive(Clone, Debug, serde::Deserialize, serde::Serialize, ToSchema)]
171pub struct FunctionCalled {
172    /// The name of the function to call
173    pub name: String,
174    /// The function arguments
175    #[serde(alias = "arguments")]
176    pub parameters: String,
177}
178
179/// Represents a tool call made by the assistant
180///
181/// This structure wraps a function call with its type information.
182#[derive(Clone, Debug, serde::Deserialize, serde::Serialize, ToSchema)]
183pub struct ToolCall {
184    /// The type of tool being called
185    #[serde(rename = "type")]
186    pub tp: ToolType,
187    ///  The function call details
188    pub function: FunctionCalled,
189}
190
191/// Represents a single message in a conversation
192///
193/// ### Examples
194///
195/// ```ignore
196/// use either::Either;
197/// use mistralrs_server_core::openai::{Message, MessageContent};
198///
199/// // User message
200/// let user_msg = Message {
201///     content: Some(MessageContent(Either::Left("What's 2+2?".to_string()))),
202///     role: "user".to_string(),
203///     name: None,
204///     tool_calls: None,
205/// };
206///
207/// // System message
208/// let system_msg = Message {
209///     content: Some(MessageContent(Either::Left("You are a helpful assistant.".to_string()))),
210///     role: "system".to_string(),
211///     name: None,
212///     tool_calls: None,
213/// };
214/// ```
215#[derive(Debug, Clone, Deserialize, Serialize, ToSchema)]
216pub struct Message {
217    /// The message content
218    pub content: Option<MessageContent>,
219    /// The role of the message sender ("user", "assistant", "system", "tool", etc.)
220    pub role: String,
221    pub name: Option<String>,
222    /// Optional list of tool calls
223    pub tool_calls: Option<Vec<ToolCall>>,
224}
225
226/// Stop token configuration for generation
227///
228/// Defines when the model should stop generating text, either with a single
229/// stop token or multiple possible stop sequences.
230#[derive(Debug, Clone, Deserialize, Serialize, ToSchema)]
231#[serde(untagged)]
232pub enum StopTokens {
233    ///  Multiple possible stop sequences
234    Multi(Vec<String>),
235    /// Single stop sequence
236    Single(String),
237}
238
239/// Default value helper
240fn default_false() -> bool {
241    false
242}
243
244/// Default value helper
245fn default_1usize() -> usize {
246    1
247}
248
249/// Default value helper
250fn default_720usize() -> usize {
251    720
252}
253
254/// Default value helper
255fn default_1280usize() -> usize {
256    1280
257}
258
259/// Default value helper
260fn default_model() -> String {
261    "default".to_string()
262}
263
264/// Default value helper
265fn default_response_format() -> ImageGenerationResponseFormat {
266    ImageGenerationResponseFormat::Url
267}
268
269/// Grammar specification for structured generation
270///
271/// Defines different types of grammars that can be used to constrain model output,
272/// ensuring it follows specific formats or structures.
273///
274/// ### Examples
275///
276/// ```ignore
277/// use mistralrs_server_core::openai::Grammar;
278///
279/// // Regex grammar for phone numbers
280/// let phone_regex = Grammar::Regex(r"\d{3}-\d{3}-\d{4}".to_string());
281///
282/// // JSON schema for structured data
283/// let json_schema = Grammar::JsonSchema(serde_json::json!({
284///     "type": "object",
285///     "properties": {
286///         "name": {"type": "string"},
287///         "age": {"type": "integer"}
288///     },
289///     "required": ["name", "age"]
290/// }));
291///
292/// // Lark grammar for arithmetic expressions
293/// let lark_grammar = Grammar::Lark(r#"
294///     ?start: expr
295///     expr: term ("+" term | "-" term)*
296///     term: factor ("*" factor | "/" factor)*
297///     factor: NUMBER | "(" expr ")"
298///     %import common.NUMBER
299/// "#.to_string());
300/// ```
301#[derive(Debug, Clone, Deserialize, Serialize)]
302#[serde(tag = "type", content = "value")]
303pub enum Grammar {
304    /// Regular expression grammar
305    #[serde(rename = "regex")]
306    Regex(String),
307    /// JSON schema grammar
308    #[serde(rename = "json_schema")]
309    JsonSchema(serde_json::Value),
310    /// LLGuidance grammar
311    #[serde(rename = "llguidance")]
312    Llguidance(LlguidanceGrammar),
313    /// Lark parser grammar
314    #[serde(rename = "lark")]
315    Lark(String),
316}
317
318// Implement ToSchema manually to handle `LlguidanceGrammar`
319impl PartialSchema for Grammar {
320    fn schema() -> RefOr<Schema> {
321        RefOr::T(Schema::OneOf(
322            OneOfBuilder::new()
323                .item(create_grammar_variant_schema(
324                    "regex",
325                    Schema::Object(
326                        ObjectBuilder::new()
327                            .schema_type(SchemaType::Type(Type::String))
328                            .build(),
329                    ),
330                ))
331                .item(create_grammar_variant_schema(
332                    "json_schema",
333                    Schema::Object(
334                        ObjectBuilder::new()
335                            .schema_type(SchemaType::Type(Type::Object))
336                            .build(),
337                    ),
338                ))
339                .item(create_grammar_variant_schema(
340                    "llguidance",
341                    llguidance_schema(),
342                ))
343                .item(create_grammar_variant_schema(
344                    "lark",
345                    Schema::Object(
346                        ObjectBuilder::new()
347                            .schema_type(SchemaType::Type(Type::String))
348                            .build(),
349                    ),
350                ))
351                .build(),
352        ))
353    }
354}
355
356impl ToSchema for Grammar {
357    fn schemas(
358        schemas: &mut Vec<(
359            String,
360            utoipa::openapi::RefOr<utoipa::openapi::schema::Schema>,
361        )>,
362    ) {
363        schemas.push((Grammar::name().into(), Grammar::schema()));
364    }
365}
366
367/// Helper function to create a grammar variant schema
368fn create_grammar_variant_schema(type_value: &str, value_schema: Schema) -> Schema {
369    Schema::Object(
370        ObjectBuilder::new()
371            .schema_type(SchemaType::Type(Type::Object))
372            .property(
373                "type",
374                RefOr::T(Schema::Object(
375                    ObjectBuilder::new()
376                        .schema_type(SchemaType::Type(Type::String))
377                        .enum_values(Some(vec![serde_json::Value::String(
378                            type_value.to_string(),
379                        )]))
380                        .build(),
381                )),
382            )
383            .property("value", RefOr::T(value_schema))
384            .required("type")
385            .required("value")
386            .build(),
387    )
388}
389
390/// Helper function to generate LLGuidance schema
391fn llguidance_schema() -> Schema {
392    let grammar_with_lexer_schema = Schema::Object(
393        ObjectBuilder::new()
394            .schema_type(SchemaType::Type(Type::Object))
395            .property(
396                "name",
397                RefOr::T(Schema::Object(
398                    ObjectBuilder::new()
399                        .schema_type(SchemaType::from_iter([Type::String, Type::Null]))
400                        .description(Some(
401                            "The name of this grammar, can be used in GenGrammar nodes",
402                        ))
403                        .build(),
404                )),
405            )
406            .property(
407                "json_schema",
408                RefOr::T(Schema::Object(
409                    ObjectBuilder::new()
410                        .schema_type(SchemaType::from_iter([Type::Object, Type::Null]))
411                        .description(Some("The JSON schema that the grammar should generate"))
412                        .build(),
413                )),
414            )
415            .property(
416                "lark_grammar",
417                RefOr::T(Schema::Object(
418                    ObjectBuilder::new()
419                        .schema_type(SchemaType::from_iter([Type::String, Type::Null]))
420                        .description(Some("The Lark grammar that the grammar should generate"))
421                        .build(),
422                )),
423            )
424            .description(Some("Grammar configuration with lexer settings"))
425            .build(),
426    );
427
428    Schema::Object(
429        ObjectBuilder::new()
430            .schema_type(SchemaType::Type(Type::Object))
431            .property(
432                "grammars",
433                RefOr::T(Schema::Array(
434                    ArrayBuilder::new()
435                        .items(RefOr::T(grammar_with_lexer_schema))
436                        .description(Some("List of grammar configurations"))
437                        .build(),
438                )),
439            )
440            .property(
441                "max_tokens",
442                RefOr::T(Schema::Object(
443                    ObjectBuilder::new()
444                        .schema_type(SchemaType::from_iter([Type::Integer, Type::Null]))
445                        .description(Some("Maximum number of tokens to generate"))
446                        .build(),
447                )),
448            )
449            .required("grammars")
450            .description(Some("Top-level grammar configuration for LLGuidance"))
451            .build(),
452    )
453}
454
455/// JSON Schema for structured responses
456#[derive(Debug, Clone, Deserialize, Serialize, ToSchema)]
457pub struct JsonSchemaResponseFormat {
458    pub name: String,
459    pub schema: serde_json::Value,
460}
461
462/// Response format for model output
463#[derive(Debug, Clone, Deserialize, Serialize, ToSchema)]
464#[serde(tag = "type")]
465pub enum ResponseFormat {
466    /// Free-form text response
467    #[serde(rename = "text")]
468    Text,
469    /// Structured response following a JSON schema
470    #[serde(rename = "json_schema")]
471    JsonSchema {
472        json_schema: JsonSchemaResponseFormat,
473    },
474}
475
476/// Chat completion request following OpenAI's specification
477#[derive(Debug, Clone, Deserialize, Serialize, ToSchema)]
478pub struct ChatCompletionRequest {
479    #[schema(
480        schema_with = messages_schema,
481        example = json!(vec![Message{content:Some(MessageContent{0: either::Left(("Why did the crab cross the road?".to_string()))}), role:"user".to_string(), name: None, tool_calls: None}])
482    )]
483    #[serde(with = "either::serde_untagged")]
484    pub messages: Either<Vec<Message>, String>,
485    #[schema(example = "mistral")]
486    #[serde(default = "default_model")]
487    pub model: String,
488    #[schema(example = json!(Option::None::<HashMap<u32, f32>>))]
489    pub logit_bias: Option<HashMap<u32, f32>>,
490    #[serde(default = "default_false")]
491    #[schema(example = false)]
492    pub logprobs: bool,
493    #[schema(example = json!(Option::None::<usize>))]
494    pub top_logprobs: Option<usize>,
495    #[schema(example = 256)]
496    #[serde(alias = "max_completion_tokens")]
497    pub max_tokens: Option<usize>,
498    #[serde(rename = "n")]
499    #[serde(default = "default_1usize")]
500    #[schema(example = 1)]
501    pub n_choices: usize,
502    #[schema(example = json!(Option::None::<f32>))]
503    pub presence_penalty: Option<f32>,
504    #[schema(example = json!(Option::None::<f32>))]
505    pub frequency_penalty: Option<f32>,
506    #[serde(rename = "stop")]
507    #[schema(example = json!(Option::None::<StopTokens>))]
508    pub stop_seqs: Option<StopTokens>,
509    #[schema(example = 0.7)]
510    pub temperature: Option<f64>,
511    #[schema(example = json!(Option::None::<f64>))]
512    pub top_p: Option<f64>,
513    #[schema(example = true)]
514    pub stream: Option<bool>,
515    #[schema(example = json!(Option::None::<Vec<Tool>>))]
516    pub tools: Option<Vec<Tool>>,
517    #[schema(example = json!(Option::None::<ToolChoice>))]
518    pub tool_choice: Option<ToolChoice>,
519    #[schema(example = json!(Option::None::<ResponseFormat>))]
520    pub response_format: Option<ResponseFormat>,
521    #[schema(example = json!(Option::None::<WebSearchOptions>))]
522    pub web_search_options: Option<WebSearchOptions>,
523
524    // mistral.rs additional
525    #[schema(example = json!(Option::None::<usize>))]
526    pub top_k: Option<usize>,
527    #[schema(example = json!(Option::None::<Grammar>))]
528    pub grammar: Option<Grammar>,
529    #[schema(example = json!(Option::None::<f64>))]
530    pub min_p: Option<f64>,
531    #[schema(example = json!(Option::None::<f32>))]
532    pub dry_multiplier: Option<f32>,
533    #[schema(example = json!(Option::None::<f32>))]
534    pub dry_base: Option<f32>,
535    #[schema(example = json!(Option::None::<usize>))]
536    pub dry_allowed_length: Option<usize>,
537    #[schema(example = json!(Option::None::<String>))]
538    pub dry_sequence_breakers: Option<Vec<String>>,
539    #[schema(example = json!(Option::None::<bool>))]
540    pub enable_thinking: Option<bool>,
541}
542
543/// Function for ChatCompletionRequest.messages Schema generation to handle `Either`
544fn messages_schema() -> Schema {
545    Schema::OneOf(
546        OneOfBuilder::new()
547            .item(Schema::Array(
548                ArrayBuilder::new()
549                    .items(RefOr::Ref(utoipa::openapi::Ref::from_schema_name(
550                        "Message",
551                    )))
552                    .build(),
553            ))
554            .item(Schema::Object(
555                ObjectBuilder::new()
556                    .schema_type(SchemaType::Type(Type::String))
557                    .build(),
558            ))
559            .build(),
560    )
561}
562
563/// Model information metadata about an available mode
564#[derive(Debug, Serialize, ToSchema)]
565pub struct ModelObject {
566    pub id: String,
567    pub object: &'static str,
568    pub created: u64,
569    pub owned_by: &'static str,
570    /// Whether tools are available through MCP or tool callbacks
571    #[serde(skip_serializing_if = "Option::is_none")]
572    pub tools_available: Option<bool>,
573    /// Number of tools available from MCP servers
574    #[serde(skip_serializing_if = "Option::is_none")]
575    pub mcp_tools_count: Option<usize>,
576    /// Number of connected MCP servers
577    #[serde(skip_serializing_if = "Option::is_none")]
578    pub mcp_servers_connected: Option<usize>,
579}
580
581/// Collection of available models
582#[derive(Debug, Serialize, ToSchema)]
583pub struct ModelObjects {
584    pub object: &'static str,
585    pub data: Vec<ModelObject>,
586}
587
588/// Legacy OpenAI compatible text completion request
589#[derive(Debug, Clone, Deserialize, Serialize, ToSchema)]
590pub struct CompletionRequest {
591    #[schema(example = "mistral")]
592    #[serde(default = "default_model")]
593    pub model: String,
594    #[schema(example = "Say this is a test.")]
595    pub prompt: String,
596    #[schema(example = 1)]
597    pub best_of: Option<usize>,
598    #[serde(rename = "echo")]
599    #[serde(default = "default_false")]
600    #[schema(example = false)]
601    pub echo_prompt: bool,
602    #[schema(example = json!(Option::None::<f32>))]
603    pub presence_penalty: Option<f32>,
604    #[schema(example = json!(Option::None::<f32>))]
605    pub frequency_penalty: Option<f32>,
606    #[schema(example = json!(Option::None::<HashMap<u32, f32>>))]
607    pub logit_bias: Option<HashMap<u32, f32>>,
608    #[schema(example = json!(Option::None::<usize>))]
609    pub logprobs: Option<usize>,
610    #[schema(example = 16)]
611    #[serde(alias = "max_completion_tokens")]
612    pub max_tokens: Option<usize>,
613    #[serde(rename = "n")]
614    #[serde(default = "default_1usize")]
615    #[schema(example = 1)]
616    pub n_choices: usize,
617    #[serde(rename = "stop")]
618    #[schema(example = json!(Option::None::<StopTokens>))]
619    pub stop_seqs: Option<StopTokens>,
620    pub stream: Option<bool>,
621    #[schema(example = 0.7)]
622    pub temperature: Option<f64>,
623    #[schema(example = json!(Option::None::<f64>))]
624    pub top_p: Option<f64>,
625    #[schema(example = json!(Option::None::<String>))]
626    pub suffix: Option<String>,
627    #[serde(rename = "user")]
628    pub _user: Option<String>,
629    #[schema(example = json!(Option::None::<Vec<Tool>>))]
630    pub tools: Option<Vec<Tool>>,
631    #[schema(example = json!(Option::None::<ToolChoice>))]
632    pub tool_choice: Option<ToolChoice>,
633
634    // mistral.rs additional
635    #[schema(example = json!(Option::None::<usize>))]
636    pub top_k: Option<usize>,
637    #[schema(example = json!(Option::None::<Grammar>))]
638    pub grammar: Option<Grammar>,
639    #[schema(example = json!(Option::None::<f64>))]
640    pub min_p: Option<f64>,
641    #[schema(example = json!(Option::None::<f32>))]
642    pub dry_multiplier: Option<f32>,
643    #[schema(example = json!(Option::None::<f32>))]
644    pub dry_base: Option<f32>,
645    #[schema(example = json!(Option::None::<usize>))]
646    pub dry_allowed_length: Option<usize>,
647    #[schema(example = json!(Option::None::<String>))]
648    pub dry_sequence_breakers: Option<Vec<String>>,
649}
650
651/// Image generation request
652#[derive(Debug, Clone, Deserialize, Serialize, ToSchema)]
653pub struct ImageGenerationRequest {
654    #[schema(example = "mistral")]
655    #[serde(default = "default_model")]
656    pub model: String,
657    #[schema(example = "Draw a picture of a majestic, snow-covered mountain.")]
658    pub prompt: String,
659    #[serde(rename = "n")]
660    #[serde(default = "default_1usize")]
661    #[schema(example = 1)]
662    pub n_choices: usize,
663    #[serde(default = "default_response_format")]
664    pub response_format: ImageGenerationResponseFormat,
665    #[serde(default = "default_720usize")]
666    #[schema(example = 720)]
667    pub height: usize,
668    #[serde(default = "default_1280usize")]
669    #[schema(example = 1280)]
670    pub width: usize,
671}
672
673/// Audio format options for speech generation responses.
674#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default, ToSchema)]
675#[serde(rename_all = "lowercase")]
676pub enum AudioResponseFormat {
677    /// Widely compatible, lossy compression, good for web streaming
678    #[default]
679    Mp3,
680    /// Good compression efficiency, ideal for real-time communication
681    Opus,
682    /// High-quality lossy compression, commonly used in mobile applications
683    Aac,
684    /// Lossless compression, larger file sizes but good audio quality
685    Flac,
686    /// Uncompressed, largest file sizes but maximum compatibility
687    Wav,
688    ///  Raw audio data, requires additional format specification
689    Pcm,
690}
691
692impl AudioResponseFormat {
693    /// Generate the appropriate MIME content type string for this audio format.
694    pub fn audio_content_type(
695        &self,
696        pcm_rate: usize,
697        pcm_channels: usize,
698        pcm_format: &'static str,
699    ) -> String {
700        let content_type = match &self {
701            AudioResponseFormat::Mp3 => "audio/mpeg".to_string(),
702            AudioResponseFormat::Opus => "audio/ogg; codecs=opus".to_string(),
703            AudioResponseFormat::Aac => "audio/aac".to_string(),
704            AudioResponseFormat::Flac => "audio/flac".to_string(),
705            AudioResponseFormat::Wav => "audio/wav".to_string(),
706            AudioResponseFormat::Pcm => format!("audio/pcm; codecs=1; format={pcm_format}"),
707        };
708
709        format!("{content_type}; rate={pcm_rate}; channels={pcm_channels}")
710    }
711}
712
713/// Speech generation request
714#[derive(Debug, Clone, Deserialize, Serialize, ToSchema)]
715pub struct SpeechGenerationRequest {
716    /// The TTS model to use for audio generation.
717    #[schema(example = "nari-labs/Dia-1.6B")]
718    #[serde(default = "default_model")]
719    pub model: String,
720    /// The text content to convert to speech.
721    #[schema(
722        example = "[S1] Dia is an open weights text to dialogue model. [S2] You get full control over scripts and voices. [S1] Wow. Amazing. (laughs) [S2] Try it now on Git hub or Hugging Face."
723    )]
724    pub input: String,
725    // `voice` and `instructions` are ignored.
726    /// The desired audio format for the generated speech.
727    #[schema(example = "mp3")]
728    pub response_format: AudioResponseFormat,
729}
730
731/// Helper type for messages field in ResponsesCreateRequest
732#[derive(Debug, Clone, Deserialize, Serialize)]
733#[serde(untagged)]
734pub enum ResponsesMessages {
735    Messages(Vec<Message>),
736    String(String),
737}
738
739impl ResponsesMessages {
740    pub fn into_either(self) -> Either<Vec<Message>, String> {
741        match self {
742            ResponsesMessages::Messages(msgs) => Either::Left(msgs),
743            ResponsesMessages::String(s) => Either::Right(s),
744        }
745    }
746}
747
748impl PartialSchema for ResponsesMessages {
749    fn schema() -> RefOr<Schema> {
750        RefOr::T(messages_schema())
751    }
752}
753
754impl ToSchema for ResponsesMessages {
755    fn schemas(
756        schemas: &mut Vec<(
757            String,
758            utoipa::openapi::RefOr<utoipa::openapi::schema::Schema>,
759        )>,
760    ) {
761        schemas.push((
762            ResponsesMessages::name().into(),
763            ResponsesMessages::schema(),
764        ));
765    }
766}
767
768/// Response creation request
769#[derive(Debug, Clone, Deserialize, Serialize, ToSchema)]
770pub struct ResponsesCreateRequest {
771    #[schema(example = "mistral")]
772    #[serde(default = "default_model")]
773    pub model: String,
774    pub input: ResponsesMessages,
775    #[schema(example = json!(Option::None::<String>))]
776    pub instructions: Option<String>,
777    #[schema(example = json!(Option::None::<Vec<String>>))]
778    pub modalities: Option<Vec<String>>,
779    #[schema(example = json!(Option::None::<String>))]
780    pub previous_response_id: Option<String>,
781    #[schema(example = json!(Option::None::<HashMap<u32, f32>>))]
782    pub logit_bias: Option<HashMap<u32, f32>>,
783    #[serde(default = "default_false")]
784    #[schema(example = false)]
785    pub logprobs: bool,
786    #[schema(example = json!(Option::None::<usize>))]
787    pub top_logprobs: Option<usize>,
788    #[schema(example = 256)]
789    #[serde(alias = "max_completion_tokens", alias = "max_output_tokens")]
790    pub max_tokens: Option<usize>,
791    #[serde(rename = "n")]
792    #[serde(default = "default_1usize")]
793    #[schema(example = 1)]
794    pub n_choices: usize,
795    #[schema(example = json!(Option::None::<f32>))]
796    pub presence_penalty: Option<f32>,
797    #[schema(example = json!(Option::None::<f32>))]
798    pub frequency_penalty: Option<f32>,
799    #[serde(rename = "stop")]
800    #[schema(example = json!(Option::None::<StopTokens>))]
801    pub stop_seqs: Option<StopTokens>,
802    #[schema(example = 0.7)]
803    pub temperature: Option<f64>,
804    #[schema(example = json!(Option::None::<f64>))]
805    pub top_p: Option<f64>,
806    #[schema(example = false)]
807    pub stream: Option<bool>,
808    #[schema(example = json!(Option::None::<Vec<Tool>>))]
809    pub tools: Option<Vec<Tool>>,
810    #[schema(example = json!(Option::None::<ToolChoice>))]
811    pub tool_choice: Option<ToolChoice>,
812    #[schema(example = json!(Option::None::<ResponseFormat>))]
813    pub response_format: Option<ResponseFormat>,
814    #[schema(example = json!(Option::None::<WebSearchOptions>))]
815    pub web_search_options: Option<WebSearchOptions>,
816    #[schema(example = json!(Option::None::<Value>))]
817    pub metadata: Option<Value>,
818    #[schema(example = json!(Option::None::<bool>))]
819    pub output_token_details: Option<bool>,
820    #[schema(example = json!(Option::None::<bool>))]
821    pub parallel_tool_calls: Option<bool>,
822    #[schema(example = json!(Option::None::<bool>))]
823    pub store: Option<bool>,
824    #[schema(example = json!(Option::None::<usize>))]
825    pub max_tool_calls: Option<usize>,
826    #[schema(example = json!(Option::None::<bool>))]
827    pub reasoning_enabled: Option<bool>,
828    #[schema(example = json!(Option::None::<usize>))]
829    pub reasoning_max_tokens: Option<usize>,
830    #[schema(example = json!(Option::None::<usize>))]
831    pub reasoning_top_logprobs: Option<usize>,
832    #[schema(example = json!(Option::None::<Vec<String>>))]
833    pub truncation: Option<HashMap<String, Value>>,
834
835    // mistral.rs additional
836    #[schema(example = json!(Option::None::<usize>))]
837    pub top_k: Option<usize>,
838    #[schema(example = json!(Option::None::<Grammar>))]
839    pub grammar: Option<Grammar>,
840    #[schema(example = json!(Option::None::<f64>))]
841    pub min_p: Option<f64>,
842    #[schema(example = json!(Option::None::<f32>))]
843    pub dry_multiplier: Option<f32>,
844    #[schema(example = json!(Option::None::<f32>))]
845    pub dry_base: Option<f32>,
846    #[schema(example = json!(Option::None::<usize>))]
847    pub dry_allowed_length: Option<usize>,
848    #[schema(example = json!(Option::None::<String>))]
849    pub dry_sequence_breakers: Option<Vec<String>>,
850    #[schema(example = json!(Option::None::<bool>))]
851    pub enable_thinking: Option<bool>,
852}
853
854/// Response object
855#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
856pub struct ResponsesObject {
857    pub id: String,
858    pub object: &'static str,
859    pub created_at: f64,
860    pub model: String,
861    pub status: String,
862    pub output: Vec<ResponsesOutput>,
863    pub output_text: Option<String>,
864    pub usage: Option<ResponsesUsage>,
865    pub error: Option<ResponsesError>,
866    pub metadata: Option<Value>,
867    pub instructions: Option<String>,
868    pub incomplete_details: Option<ResponsesIncompleteDetails>,
869}
870
871/// Response usage information
872#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
873pub struct ResponsesUsage {
874    pub input_tokens: usize,
875    pub output_tokens: usize,
876    pub total_tokens: usize,
877    pub input_tokens_details: Option<ResponsesInputTokensDetails>,
878    pub output_tokens_details: Option<ResponsesOutputTokensDetails>,
879}
880
881/// Input tokens details
882#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
883pub struct ResponsesInputTokensDetails {
884    pub audio_tokens: Option<usize>,
885    pub cached_tokens: Option<usize>,
886    pub image_tokens: Option<usize>,
887    pub text_tokens: Option<usize>,
888}
889
890/// Output tokens details
891#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
892pub struct ResponsesOutputTokensDetails {
893    pub audio_tokens: Option<usize>,
894    pub text_tokens: Option<usize>,
895    pub reasoning_tokens: Option<usize>,
896}
897
898/// Response error
899#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
900pub struct ResponsesError {
901    #[serde(rename = "type")]
902    pub error_type: String,
903    pub message: String,
904}
905
906/// Incomplete details for incomplete responses
907#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
908pub struct ResponsesIncompleteDetails {
909    pub reason: String,
910}
911
912/// Response output item
913#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
914pub struct ResponsesOutput {
915    pub id: String,
916    #[serde(rename = "type")]
917    pub output_type: String,
918    pub role: String,
919    pub status: Option<String>,
920    pub content: Vec<ResponsesContent>,
921}
922
923/// Response content item
924#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
925pub struct ResponsesContent {
926    #[serde(rename = "type")]
927    pub content_type: String,
928    pub text: Option<String>,
929    pub annotations: Option<Vec<ResponsesAnnotation>>,
930}
931
932/// Response annotation
933#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
934pub struct ResponsesAnnotation {
935    #[serde(rename = "type")]
936    pub annotation_type: String,
937    pub text: String,
938    pub start_index: usize,
939    pub end_index: usize,
940}
941
942/// Response streaming chunk
943#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
944pub struct ResponsesChunk {
945    pub id: String,
946    pub object: &'static str,
947    pub created_at: f64,
948    pub model: String,
949    pub chunk_type: String,
950    pub delta: Option<ResponsesDelta>,
951    pub usage: Option<ResponsesUsage>,
952    pub metadata: Option<Value>,
953}
954
955/// Response delta for streaming
956#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
957pub struct ResponsesDelta {
958    pub output: Option<Vec<ResponsesDeltaOutput>>,
959    pub status: Option<String>,
960}
961
962/// Response delta output item
963#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
964pub struct ResponsesDeltaOutput {
965    pub id: String,
966    #[serde(rename = "type")]
967    pub output_type: String,
968    pub content: Option<Vec<ResponsesDeltaContent>>,
969}
970
971/// Response delta content item
972#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
973pub struct ResponsesDeltaContent {
974    #[serde(rename = "type")]
975    pub content_type: String,
976    pub text: Option<String>,
977}
mistralrs_server_core/openai.rs

mistralrs_server_core/
openai.rs
