mistralrs 

1from .mistralrs import *
2
3__doc__ = mistralrs.__doc__
4if hasattr(mistralrs, "__all__"):
5    __all__ = mistralrs.__all__
class Runner:

An object wrapping the underlying Rust system to handle requests and process conversations.

def send_chat_completion_request(self, /, request, model_id=None):

Send an OpenAI API compatible request, returning the result.

def send_completion_request(self, /, request, model_id=None):

Send an OpenAI API compatible request, returning the result.

def generate_image( self, /, prompt, response_format, height=720, width=1280, model_id=None):

Generate an image.

def generate_audio(self, /, prompt, model_id=None):

Generate audio.

def send_re_isq(self, /, dtype, model_id=None):

Send a request to re-ISQ the model. If the model was loaded as GGUF or GGML then nothing will happen.

def tokenize_text(self, /, text, add_special_tokens, enable_thinking, model_id=None):

Tokenize some text, returning raw tokens.

def detokenize_text(self, /, tokens, skip_special_tokens, model_id=None):

Detokenize some tokens, returning text.

def list_models(self, /):

List all available model IDs in multi-model mode.

def get_default_model_id(self, /):

Get the default model ID in multi-model mode.

def set_default_model_id(self, /, model_id):

Set the default model ID in multi-model mode.

def remove_model(self, /, model_id):

Remove a model by ID in multi-model mode.

def send_chat_completion_request_to_model(self, /, request, model_id):

Send an OpenAI API compatible request to a specific model, returning the result.

def send_completion_request_to_model(self, /, request, model_id):

Send an OpenAI API compatible completion request to a specific model, returning the result.

class MultiModelRunner:

A multi-model runner that provides a cleaner interface for managing multiple models. This wraps the existing Runner and provides model-specific methods.

def send_chat_completion_request_to_model(self, /, request, model_id):

Send a chat completion request to a specific model.

def send_completion_request_to_model(self, /, request, model_id):

Send a completion request to a specific model.

def list_models(self, /):

List all available model IDs.

def get_default_model_id(self, /):

Get the default model ID.

def set_default_model_id(self, /, model_id):

Set the default model ID.

def remove_model(self, /, model_id):

Remove a model by ID.

def send_chat_completion_request(self, /, request, model_id=None):

Send a chat completion request to the default model.

def send_completion_request(self, /, request, model_id=None):

Send a completion request to the default model.

def generate_image( self, /, prompt, response_format, height=720, width=1280, model_id=None):

Generate an image using the default model.

def generate_audio(self, /, prompt, model_id=None):

Generate audio using the default model.

def send_re_isq(self, /, dtype, model_id=None):

Send a request to re-ISQ the default model.

def tokenize_text(self, /, text, add_special_tokens, enable_thinking, model_id=None):

Tokenize some text using the default model.

def detokenize_text(self, /, tokens, skip_special_tokens, model_id=None):

Detokenize some tokens using the default model.

def inner(self, /):

Get a copy of the underlying Runner instance.

class Which:
Plain = <class 'builtins.Which_Plain'>
XLora = <class 'builtins.Which_XLora'>
Lora = <class 'builtins.Which_Lora'>
GGUF = <class 'builtins.Which_GGUF'>
XLoraGGUF = <class 'builtins.Which_XLoraGGUF'>
LoraGGUF = <class 'builtins.Which_LoraGGUF'>
GGML = <class 'builtins.Which_GGML'>
XLoraGGML = <class 'builtins.Which_XLoraGGML'>
LoraGGML = <class 'builtins.Which_LoraGGML'>
VisionPlain = <class 'builtins.Which_VisionPlain'>
DiffusionPlain = <class 'builtins.Which_DiffusionPlain'>
Speech = <class 'builtins.Which_Speech'>
class ChatCompletionRequest:

An OpenAI API compatible chat completion request.

class CompletionRequest:

An OpenAI API compatible completion request.

class Architecture:
class VisionArchitecture:
class DiffusionArchitecture:
class AnyMoeConfig:
class AnyMoeExpertType:
FineTuned = <class 'builtins.AnyMoeExpertType_FineTuned'>
LoraAdapter = <class 'builtins.AnyMoeExpertType_LoraAdapter'>
class ToolChoice:
NoTools = ToolChoice.NoTools
class SpeechGenerationResponse:
rate
channels
pcm
class SpeechLoaderType:
class ResponseMessage:

Chat completion response message.

role
tool_calls
content
class Delta:

Delta in content for streaming response.

role
content
tool_calls
class ResponseLogprob:

A logprob with the top logprobs for this token.

token
bytes
logprob
top_logprobs
class Logprobs:

Logprobs per token.

content
class Choice:

Chat completion choice.

message
index
finish_reason
logprobs
class ChunkChoice:

Chat completion streaming chunk choice.

index
finish_reason
logprobs
delta
class Usage:

OpenAI compatible (superset) usage during a request.

prompt_tokens
total_time_sec
completion_tokens
total_prompt_time_sec
total_completion_time_sec
avg_prompt_tok_per_sec
total_tokens
avg_tok_per_sec
avg_compl_tok_per_sec
class ChatCompletionResponse:

An OpenAI compatible chat completion response.

system_fingerprint
object
usage
choices
id
created
model
class ChatCompletionChunkResponse:

Chat completion streaming request chunk.

system_fingerprint
usage
id
choices
object
model
created
class CompletionChoice:

Completion request choice.

finish_reason
text
index
logprobs
class CompletionResponse:

An OpenAI compatible completion response.

model
object
id
system_fingerprint
choices
usage
created
class TopLogprob:

Top-n logprobs element

logprob
token
bytes
class ModelDType:

DType for the model.

If the model is quantized, this is ignored so it is reasonable to use the [Default] impl.

Note: When using Auto, fallback pattern is: BF16 -> F16 -> 32

class ImageGenerationResponseFormat:

Image generation response format

class McpServerSourcePy:

MCP server source configuration for different transport types

Http = <class 'builtins.McpServerSourcePy_Http'>
Process = <class 'builtins.McpServerSourcePy_Process'>
WebSocket = <class 'builtins.McpServerSourcePy_WebSocket'>
class McpServerConfigPy:

Configuration for an individual MCP server

source
resources
tool_prefix
id
enabled
bearer_token
name
class McpClientConfigPy:

Configuration for MCP client integration

max_concurrent_calls
servers
tool_timeout_secs
auto_register_tools