ResponseCache

mistralrs_server_core::cached_responses

Trait ResponseCache 

Source 
pub trait ResponseCache: Send + Sync {
    // Required methods
    fn store_response(
        &self,
        id: String,
        response: ResponseResource,
    ) -> Result<()>;
    fn get_response(&self, id: &str) -> Result<Option<ResponseResource>>;
    fn delete_response(&self, id: &str) -> Result<bool>;
    fn store_conversation_history(
        &self,
        id: String,
        messages: Vec<Message>,
    ) -> Result<()>;
    fn get_conversation_history(&self, id: &str) -> Result<Option<Vec<Message>>>;
}
Expand description

Trait for caching responses

Required Methods§

Source

fn store_response(&self, id: String, response: ResponseResource) -> Result<()>

Store a response object with the given ID

Source

fn get_response(&self, id: &str) -> Result<Option<ResponseResource>>

Retrieve a response object by ID

Source

fn delete_response(&self, id: &str) -> Result<bool>

Delete a response object by ID

Source

fn store_conversation_history( &self, id: String, messages: Vec<Message>, ) -> Result<()>

Store conversation history for a response

Source

fn get_conversation_history(&self, id: &str) -> Result<Option<Vec<Message>>>

Retrieve conversation history for a response

Implementors§