mistralrs_server_core::cached_responses

Trait ResponseCache

Source 
pub trait ResponseCache: Send + Sync {
    // Required methods
    fn store_response(
        &self,
        id: String,
        response: ResponsesObject,
    ) -> Result<()>;
    fn get_response(&self, id: &str) -> Result<Option<ResponsesObject>>;
    fn delete_response(&self, id: &str) -> Result<bool>;
    fn store_chunks(
        &self,
        id: String,
        chunks: Vec<ResponsesChunk>,
    ) -> Result<()>;
    fn get_chunks(&self, id: &str) -> Result<Option<Vec<ResponsesChunk>>>;
    fn store_conversation_history(
        &self,
        id: String,
        messages: Vec<Message>,
    ) -> Result<()>;
    fn get_conversation_history(&self, id: &str) -> Result<Option<Vec<Message>>>;
}
Expand description

Trait for caching responses

Required Methods§

Source

fn store_response(&self, id: String, response: ResponsesObject) -> Result<()>

Store a response object with the given ID

Source

fn get_response(&self, id: &str) -> Result<Option<ResponsesObject>>

Retrieve a response object by ID

Source

fn delete_response(&self, id: &str) -> Result<bool>

Delete a response object by ID

Source

fn store_chunks(&self, id: String, chunks: Vec<ResponsesChunk>) -> Result<()>

Store streaming chunks for a response

Source

fn get_chunks(&self, id: &str) -> Result<Option<Vec<ResponsesChunk>>>

Retrieve streaming chunks for a response

Source

fn store_conversation_history( &self, id: String, messages: Vec<Message>, ) -> Result<()>

Store conversation history for a response

Source

fn get_conversation_history(&self, id: &str) -> Result<Option<Vec<Message>>>

Retrieve conversation history for a response

Implementors§