From 00265fd135a132dca16d2b1f12c66867f3b16aa2 Mon Sep 17 00:00:00 2001 From: Tobias Schoofs Date: Thu, 15 Feb 2024 21:47:55 +0000 Subject: [PATCH 01/12] [feat/modelmanager] routes in their own module --- crates/edgen_server/src/lib.rs | 29 ++----------------- crates/edgen_server/src/routes.rs | 47 +++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 27 deletions(-) create mode 100644 crates/edgen_server/src/routes.rs diff --git a/crates/edgen_server/src/lib.rs b/crates/edgen_server/src/lib.rs index ccfe1c2..7053b53 100644 --- a/crates/edgen_server/src/lib.rs +++ b/crates/edgen_server/src/lib.rs @@ -20,7 +20,6 @@ use std::process::exit; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; -use axum::Router; use tower_http::cors::CorsLayer; use futures::executor::block_on; @@ -44,6 +43,7 @@ pub mod graceful_shutdown; mod llm; mod model; pub mod openai_shim; +mod routes; pub mod status; pub mod util; mod whisper; @@ -189,32 +189,7 @@ async fn run_server(args: &cli::Serve) -> Result { ) .await; - let http_app = Router::new() - // -- AI endpoints ----------------------------------------------------- - // ---- Chat ----------------------------------------------------------- - .route( - "/v1/chat/completions", - axum::routing::post(openai_shim::chat_completions), - ) - // ---- Audio ---------------------------------------------------------- - .route( - "/v1/audio/transcriptions", - axum::routing::post(openai_shim::create_transcription), - ) - // -- AI status endpoints ---------------------------------------------- - // ---- Chat ----------------------------------------------------------- - .route( - "/v1/chat/completions/status", - axum::routing::get(status::chat_completions_status), - ) - // ---- Audio ---------------------------------------------------------- - .route( - "/v1/audio/transcriptions/status", - axum::routing::get(status::audio_transcriptions_status), - ) - // -- Miscellaneous services ------------------------------------------- - .route("/v1/misc/version", axum::routing::get(misc::edgen_version)) - .layer(CorsLayer::permissive()); + let http_app = routes::routes().layer(CorsLayer::permissive()); let uri_vector = if !args.uri.is_empty() { info!("Overriding default URI"); diff --git a/crates/edgen_server/src/routes.rs b/crates/edgen_server/src/routes.rs new file mode 100644 index 0000000..5be3338 --- /dev/null +++ b/crates/edgen_server/src/routes.rs @@ -0,0 +1,47 @@ +/* Copyright 2023- The Binedge, Lda team. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! Contains all routes served by Edgen + +use axum::Router; + +use crate::misc; +use crate::openai_shim; +use crate::status; + +pub fn routes() -> Router { + Router::new() + // -- AI endpoints ----------------------------------------------------- + // ---- Chat ----------------------------------------------------------- + .route( + "/v1/chat/completions", + axum::routing::post(openai_shim::chat_completions), + ) + // ---- Audio ---------------------------------------------------------- + .route( + "/v1/audio/transcriptions", + axum::routing::post(openai_shim::create_transcription), + ) + // -- AI status endpoints ---------------------------------------------- + // ---- Chat ----------------------------------------------------------- + .route( + "/v1/chat/completions/status", + axum::routing::get(status::chat_completions_status), + ) + // ---- Audio ---------------------------------------------------------- + .route( + "/v1/audio/transcriptions/status", + axum::routing::get(status::audio_transcriptions_status), + ) + // -- Miscellaneous services ------------------------------------------- + .route("/v1/misc/version", axum::routing::get(misc::edgen_version)) +} From 21c56419e8ce89ca8306564380bb552d92f78ab1 Mon Sep 17 00:00:00 2001 From: Tobias Schoofs Date: Thu, 15 Feb 2024 21:58:56 +0000 Subject: [PATCH 02/12] [feat/modelmanager] routes added --- crates/edgen_server/src/lib.rs | 1 + crates/edgen_server/src/model_man.rs | 25 +++++++++++++++++++++++++ crates/edgen_server/src/routes.rs | 11 +++++++++++ 3 files changed, 37 insertions(+) create mode 100644 crates/edgen_server/src/model_man.rs diff --git a/crates/edgen_server/src/lib.rs b/crates/edgen_server/src/lib.rs index 7053b53..b46dd6e 100644 --- a/crates/edgen_server/src/lib.rs +++ b/crates/edgen_server/src/lib.rs @@ -42,6 +42,7 @@ pub mod error; pub mod graceful_shutdown; mod llm; mod model; +mod model_man; pub mod openai_shim; mod routes; pub mod status; diff --git a/crates/edgen_server/src/model_man.rs b/crates/edgen_server/src/model_man.rs new file mode 100644 index 0000000..222804d --- /dev/null +++ b/crates/edgen_server/src/model_man.rs @@ -0,0 +1,25 @@ +/* Copyright 2023- The Binedge, Lda team. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use axum::response::{IntoResponse, Json, Response}; + +pub async fn list_models() -> Response { + Json(true).into_response() +} + +pub async fn retrieve_model() -> Response { + Json(true).into_response() +} + +pub async fn delete_model() -> Response { + Json(true).into_response() +} diff --git a/crates/edgen_server/src/routes.rs b/crates/edgen_server/src/routes.rs index 5be3338..3c4a237 100644 --- a/crates/edgen_server/src/routes.rs +++ b/crates/edgen_server/src/routes.rs @@ -15,6 +15,7 @@ use axum::Router; use crate::misc; +use crate::model_man; use crate::openai_shim; use crate::status; @@ -44,4 +45,14 @@ pub fn routes() -> Router { ) // -- Miscellaneous services ------------------------------------------- .route("/v1/misc/version", axum::routing::get(misc::edgen_version)) + // -- Model Manager ---------------------------------------------------- + .route("/v1/models", axum::routing::get(model_man::list_models)) + .route( + "/v1/models/{model}", + axum::routing::get(model_man::retrieve_model), + ) + .route( + "/v1/models/{model}", + axum::routing::delete(model_man::delete_model), + ) } From 20de7bc0cfd229d2b6ee3f18ddf0aa6cd4118e72 Mon Sep 17 00:00:00 2001 From: Tobias Schoofs Date: Thu, 15 Feb 2024 22:55:11 +0000 Subject: [PATCH 03/12] [feat/modelmanager] parse model entry --- crates/edgen_server/src/model_man.rs | 133 +++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) diff --git a/crates/edgen_server/src/model_man.rs b/crates/edgen_server/src/model_man.rs index 222804d..f56252f 100644 --- a/crates/edgen_server/src/model_man.rs +++ b/crates/edgen_server/src/model_man.rs @@ -23,3 +23,136 @@ pub async fn retrieve_model() -> Response { pub async fn delete_model() -> Response { Json(true).into_response() } + +#[derive(Debug, PartialEq)] +enum ParseError { + MissingDashes, + NotaModel, + NoOwner, + NoRepo, +} + +fn parse_model_entry(model_string: &str) -> Result<(String, String), ParseError> { + let vs = model_string.split("--").collect::>(); + + if vs.len() < 3 { + return Err(ParseError::MissingDashes); + } + + if vs[0] != "models" { + return Err(ParseError::NotaModel); + } + + // the owner is always the second + // if the original owner contained double dashes + // we won't found him + let owner = vs[1].to_string(); + if owner.is_empty() { + return Err(ParseError::NoOwner); + } + + let repo = if vs.len() > 3 { + vs[2..].join("--") + } else { + vs[2].to_string() + }; + if repo.is_empty() { + return Err(ParseError::NoRepo); + } + + Ok((owner, repo)) +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn parse_simple_valid() { + assert_eq!( + parse_model_entry("models--TheBloke--TinyLlama-1.1B-Chat-v1.0-GGUF"), + Ok(( + "TheBloke".to_string(), + "TinyLlama-1.1B-Chat-v1.0-GGUF".to_string(), + )) + ); + } + + #[test] + fn parse_dashes_in_repo_valid() { + assert_eq!( + parse_model_entry("models--TheBloke--TinyLlama--1.1B--Chat--v1.0--GGUF"), + Ok(( + "TheBloke".to_string(), + "TinyLlama--1.1B--Chat--v1.0--GGUF".to_string(), + )) + ); + } + + #[test] + fn parse_dashes_in_owner_valid() { + assert_eq!( + parse_model_entry("models--The--Bloke--TinyLlama--1.1B--Chat--v1.0--GGUF"), + Ok(( + "The".to_string(), + "Bloke--TinyLlama--1.1B--Chat--v1.0--GGUF".to_string(), + )) + ); + } + + #[test] + fn fail_dashes_in_owner() { + assert_ne!( + parse_model_entry("models--The--Bloke--TinyLlama--1.1B--Chat--v1.0--GGUF"), + Ok(( + "TheBloke".to_string(), + "TinyLlama--1.1B--Chat--v1.0--GGUF".to_string(), + )) + ); + } + + #[test] + fn fail_does_not_start_with_model() { + assert_eq!( + parse_model_entry("datasets--TheBloke--TinyLlama-1.1B-Chat-v1.0-GGUF"), + Err(ParseError::NotaModel) + ); + } + + #[test] + fn fail_no_dashes_between_owner_and_repo() { + assert_eq!( + parse_model_entry("models--TheBloke-TinyLlama-1.1B-Chat-v1.0-GGUF"), + Err(ParseError::MissingDashes) + ); + } + + #[test] + fn fail_no_dashes_after_owner() { + assert_eq!( + parse_model_entry("models--TheBloke"), + Err(ParseError::MissingDashes) + ); + } + + #[test] + fn fail_no_repo() { + assert_eq!( + parse_model_entry("models--TheBloke--"), + Err(ParseError::NoRepo) + ); + } + + #[test] + fn fail_no_owner() { + assert_eq!(parse_model_entry("models----"), Err(ParseError::NoOwner)); + } + + #[test] + fn fail_no_model() { + assert_eq!( + parse_model_entry("--TheBlock--whatever"), + Err(ParseError::NotaModel) + ); + } +} From a4a07a22cd4ad03116a15451da4d1b4ef00cef38 Mon Sep 17 00:00:00 2001 From: Tobias Schoofs Date: Fri, 16 Feb 2024 10:22:05 +0000 Subject: [PATCH 04/12] [feat/modelmanager] roundtrip id <-> path --- crates/edgen_server/src/model_man.rs | 263 ++++++++++++++++++++++++--- 1 file changed, 234 insertions(+), 29 deletions(-) diff --git a/crates/edgen_server/src/model_man.rs b/crates/edgen_server/src/model_man.rs index f56252f..8a9c58c 100644 --- a/crates/edgen_server/src/model_man.rs +++ b/crates/edgen_server/src/model_man.rs @@ -10,7 +10,16 @@ * limitations under the License. */ +use std::fmt; +use std::fmt::Display; +use std::path::{Path, PathBuf}; +use std::time::{SystemTime, SystemTimeError}; + use axum::response::{IntoResponse, Json, Response}; +use serde::{Deserialize, Serialize}; +use thiserror; +use tracing::{error, info, warn}; +use utoipa::ToSchema; pub async fn list_models() -> Response { Json(true).into_response() @@ -24,19 +33,107 @@ pub async fn delete_model() -> Response { Json(true).into_response() } -#[derive(Debug, PartialEq)] +#[derive(ToSchema, Deserialize, Serialize, Debug, PartialEq, Eq)] +pub struct ModelDesc { + id: String, + created: u64, + object: String, + owned_by: String, +} + +#[derive(Debug, thiserror::Error)] +enum PathError { + Generic(String), + ParseError(#[from] ParseError), + IOError(#[from] std::io::Error), + TimeError(#[from] SystemTimeError), +} + +impl Display for PathError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}", self) + } +} + +async fn path_to_model_desc(path: &Path) -> Result { + let f = path + .file_name() + .ok_or(PathError::Generic("empty path".to_string()))?; + let model = f + .to_str() + .ok_or(PathError::Generic("invalid file name".to_string()))?; + let (owner, repo) = parse_path(model)?; + let metadata = tokio::fs::metadata(path).await?; + if !metadata.is_dir() { + return Err(PathError::Generic("not a directory".to_string())); + }; + let tp = match metadata.created() { + Ok(n) => n, + Err(_) => SystemTime::UNIX_EPOCH, // unknown + }; + + let created = tp.duration_since(SystemTime::UNIX_EPOCH)?.as_secs(); + + Ok(ModelDesc { + id: to_model_id(&owner, &repo), + created: created, + object: "model".to_string(), + owned_by: owner.to_string(), + }) +} + +fn to_model_id(owner: &str, repo: &str) -> String { + format!("{}/{}", owner, repo) +} + +fn model_id_to_path(id: &str) -> Result { + let (owner, repo) = parse_model_id(id)?; + let s = format!("models--{}--{}", owner, repo); + Ok(PathBuf::from(s)) +} + +#[derive(Debug, PartialEq, thiserror::Error)] enum ParseError { - MissingDashes, + MissingSeparator, NotaModel, NoOwner, NoRepo, } -fn parse_model_entry(model_string: &str) -> Result<(String, String), ParseError> { +impl Display for ParseError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}", self) + } +} + +fn parse_model_id(id: &str) -> Result<(String, String), ParseError> { + let vs = id.split("/").collect::>(); + if vs.len() < 2 { + return Err(ParseError::MissingSeparator); + } + + let owner = vs[0].to_string(); + if owner.is_empty() { + return Err(ParseError::NoOwner); + } + + let repo = if vs.len() > 2 { + vs[1..].join("/") + } else { + vs[1].to_string() + }; + if repo.is_empty() { + return Err(ParseError::NoRepo); + } + + Ok((owner, repo)) +} + +fn parse_path(model_string: &str) -> Result<(String, String), ParseError> { let vs = model_string.split("--").collect::>(); if vs.len() < 3 { - return Err(ParseError::MissingDashes); + return Err(ParseError::MissingSeparator); } if vs[0] != "models" { @@ -66,11 +163,14 @@ fn parse_model_entry(model_string: &str) -> Result<(String, String), ParseError> #[cfg(test)] mod test { use super::*; + use std::ffi::OsStr; + use std::path::Path; + // --- Parse Model Id ------------------------------------------------------------------------- #[test] - fn parse_simple_valid() { + fn parse_simple_model_id_valid() { assert_eq!( - parse_model_entry("models--TheBloke--TinyLlama-1.1B-Chat-v1.0-GGUF"), + parse_model_id("TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF"), Ok(( "TheBloke".to_string(), "TinyLlama-1.1B-Chat-v1.0-GGUF".to_string(), @@ -79,9 +179,93 @@ mod test { } #[test] - fn parse_dashes_in_repo_valid() { + fn parse_model_id_slashes_in_repo() { + assert_eq!( + parse_model_id("TheBloke/TinyLlama/1.1B/Chat/v1.0-GGUF"), + Ok(( + "TheBloke".to_string(), + "TinyLlama/1.1B/Chat/v1.0-GGUF".to_string(), + )) + ); + } + + #[test] + fn parse_model_id_slashes_in_owner_valid() { assert_eq!( - parse_model_entry("models--TheBloke--TinyLlama--1.1B--Chat--v1.0--GGUF"), + parse_model_id("The/Bloke/TinyLlama-1.1B-Chat-v1.0-GGUF"), + Ok(( + "The".to_string(), + "Bloke/TinyLlama-1.1B-Chat-v1.0-GGUF".to_string(), + )) + ); + } + + #[test] + fn fail_model_id_slashes_in_owner_valid() { + assert_ne!( + parse_model_id("The/Bloke/TinyLlama-1.1B-Chat-v1.0-GGUF"), + Ok(( + "TheBloke".to_string(), + "TinyLlama-1.1B-Chat-v1.0-GGUF".to_string(), + )) + ); + } + + #[test] + fn fail_model_id_no_slashes_between_owner_and_repo() { + assert_eq!( + parse_model_id("The-Bloke-TinyLlama-1.1B-Chat-v1.0-GGUF"), + Err(ParseError::MissingSeparator) + ); + } + + #[test] + fn fail_model_id_no_slashes_after_owner() { + assert_eq!( + parse_model_id("The-Bloke"), + Err(ParseError::MissingSeparator) + ); + } + + #[test] + fn fail_model_id_no_repo() { + assert_eq!(parse_model_id("The-Bloke/"), Err(ParseError::NoRepo)); + } + + #[test] + fn fail_model_id_no_owner() { + assert_eq!( + parse_model_id("/The-Bloke-TinyLlama-1.1B-Chat-v1.0-GGUF"), + Err(ParseError::NoOwner) + ); + } + + #[test] + fn fail_model_id_nothing() { + assert_eq!(parse_model_id("/"), Err(ParseError::NoOwner)); + } + + #[test] + fn fail_model_id_even_less() { + assert_eq!(parse_model_id(""), Err(ParseError::MissingSeparator)); + } + + // --- Parse Model Entry ---------------------------------------------------------------------- + #[test] + fn parse_path_simple_valid() { + assert_eq!( + parse_path("models--TheBloke--TinyLlama-1.1B-Chat-v1.0-GGUF"), + Ok(( + "TheBloke".to_string(), + "TinyLlama-1.1B-Chat-v1.0-GGUF".to_string(), + )) + ); + } + + #[test] + fn parse_path_dashes_in_repo_valid() { + assert_eq!( + parse_path("models--TheBloke--TinyLlama--1.1B--Chat--v1.0--GGUF"), Ok(( "TheBloke".to_string(), "TinyLlama--1.1B--Chat--v1.0--GGUF".to_string(), @@ -90,9 +274,9 @@ mod test { } #[test] - fn parse_dashes_in_owner_valid() { + fn parse_path_dashes_in_owner_valid() { assert_eq!( - parse_model_entry("models--The--Bloke--TinyLlama--1.1B--Chat--v1.0--GGUF"), + parse_path("models--The--Bloke--TinyLlama--1.1B--Chat--v1.0--GGUF"), Ok(( "The".to_string(), "Bloke--TinyLlama--1.1B--Chat--v1.0--GGUF".to_string(), @@ -101,9 +285,9 @@ mod test { } #[test] - fn fail_dashes_in_owner() { + fn fail_path_dashes_in_owner() { assert_ne!( - parse_model_entry("models--The--Bloke--TinyLlama--1.1B--Chat--v1.0--GGUF"), + parse_path("models--The--Bloke--TinyLlama--1.1B--Chat--v1.0--GGUF"), Ok(( "TheBloke".to_string(), "TinyLlama--1.1B--Chat--v1.0--GGUF".to_string(), @@ -112,47 +296,68 @@ mod test { } #[test] - fn fail_does_not_start_with_model() { + fn fail_path_does_not_start_with_model() { assert_eq!( - parse_model_entry("datasets--TheBloke--TinyLlama-1.1B-Chat-v1.0-GGUF"), + parse_path("datasets--TheBloke--TinyLlama-1.1B-Chat-v1.0-GGUF"), Err(ParseError::NotaModel) ); } #[test] - fn fail_no_dashes_between_owner_and_repo() { + fn fail_path_no_dashes_between_owner_and_repo() { assert_eq!( - parse_model_entry("models--TheBloke-TinyLlama-1.1B-Chat-v1.0-GGUF"), - Err(ParseError::MissingDashes) + parse_path("models--TheBloke-TinyLlama-1.1B-Chat-v1.0-GGUF"), + Err(ParseError::MissingSeparator) ); } #[test] - fn fail_no_dashes_after_owner() { + fn fail_path_no_dashes_after_owner() { assert_eq!( - parse_model_entry("models--TheBloke"), - Err(ParseError::MissingDashes) + parse_path("models--TheBloke"), + Err(ParseError::MissingSeparator) ); } #[test] - fn fail_no_repo() { - assert_eq!( - parse_model_entry("models--TheBloke--"), - Err(ParseError::NoRepo) - ); + fn fail_path_no_repo() { + assert_eq!(parse_path("models--TheBloke--"), Err(ParseError::NoRepo)); } #[test] - fn fail_no_owner() { - assert_eq!(parse_model_entry("models----"), Err(ParseError::NoOwner)); + fn fail_path_no_owner() { + assert_eq!(parse_path("models----"), Err(ParseError::NoOwner)); } #[test] - fn fail_no_model() { + fn fail_path_no_model() { assert_eq!( - parse_model_entry("--TheBlock--whatever"), + parse_path("--TheBlock--whatever"), Err(ParseError::NotaModel) ); } + + #[test] + fn fail_path_nothing() { + assert_eq!(parse_path(""), Err(ParseError::MissingSeparator)); + } + + // --- Roundtrip ------------------------------------------------------------------------------ + #[test] + fn simple_roundtrip() { + let paths = vec![ + "models--TheBloke--TinyLlama-1.1B-Chat-v1.0-GGUF", + "models--The--Bloke--TinyLlama--1.1B--Chat--v1.0--GGUF", + "models--TheBloke--TinyLlama--1.1B--Chat--v1.0--GGUF", + ]; + for path in paths.into_iter() { + let (owner, repo) = parse_path(path).unwrap(); + let id = to_model_id(&owner, &repo); + let pb = model_id_to_path(&id).unwrap(); + let round = pb.as_path().to_str().unwrap(); + assert_eq!(path, round); + } + } + + // --- path to desc --------------------------------------------------------------------------- } From 43b7d0ef61957b38067c6f01561893f3abda6e0f Mon Sep 17 00:00:00 2001 From: Tobias Schoofs Date: Fri, 16 Feb 2024 12:59:28 +0000 Subject: [PATCH 05/12] [feat/modelmanager] list models in dir --- Cargo.lock | 1 + crates/edgen_server/Cargo.toml | 1 + crates/edgen_server/src/model_man.rs | 85 ++++++++++++++++++++++++++++ 3 files changed, 87 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 69d8e5c..ac7adb9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1446,6 +1446,7 @@ dependencies = [ "serde_derive", "serde_json", "serde_yaml", + "tempfile", "testcontainers", "thiserror", "time", diff --git a/crates/edgen_server/Cargo.toml b/crates/edgen_server/Cargo.toml index 6ed02d4..cf55f54 100644 --- a/crates/edgen_server/Cargo.toml +++ b/crates/edgen_server/Cargo.toml @@ -43,3 +43,4 @@ testcontainers = "0.15.0" [dev-dependencies] levenshtein = "1.0.5" +tempfile = { workspace = true } diff --git a/crates/edgen_server/src/model_man.rs b/crates/edgen_server/src/model_man.rs index 8a9c58c..8311fd4 100644 --- a/crates/edgen_server/src/model_man.rs +++ b/crates/edgen_server/src/model_man.rs @@ -55,6 +55,38 @@ impl Display for PathError { } } +async fn list_models_in_dir(path: &Path) -> Result, PathError> { + let mut v = vec![]; + let es = tokio::fs::read_dir(path).await; + if es.is_err() { + warn!("model manager: cannot read directory {:?} ({:?})", path, es); + return Err(PathError::IOError(es.unwrap_err())); + }; + let mut es = es.unwrap(); + loop { + let e = es.next_entry().await; + if e.is_err() { + warn!("model manager: cannot get entry: {:?}", e); + continue; + } + let tmp = e.unwrap(); + if tmp.is_none() { + break; + } + let tmp = tmp.unwrap(); + match path_to_model_desc(tmp.path().as_path()).await { + Ok(m) => v.push(m), + Err(e) => { + warn!( + "model manager: invalid entry in directory {:?}: {:?}", + path, e + ); + } + } + } + Ok(v) +} + async fn path_to_model_desc(path: &Path) -> Result { let f = path .file_name() @@ -165,6 +197,9 @@ mod test { use super::*; use std::ffi::OsStr; use std::path::Path; + use std::time::SystemTime; + + use tempfile; // --- Parse Model Id ------------------------------------------------------------------------- #[test] @@ -360,4 +395,54 @@ mod test { } // --- path to desc --------------------------------------------------------------------------- + #[tokio::test] + async fn test_list_models_in_dir() { + let bloke = "TheBloke"; + let the = "The"; + let r1 = "TinyLlama-1.1B-Chat-v1.0-GGUF"; + let r2 = "Bloke--TinyLlama-1.1B-Chat-v1.0-GGUF"; + let r3 = "TinyLlama--1.1B--Chat--v1.0--GGUF"; + let f1 = format!("models--{}--{}", bloke, r1); + let f2 = format!("models--{}--{}", the, r2); + let f3 = format!("models--{}--{}", bloke, r3); + let f4 = "invisible".to_string(); + let f5 = "models--TheBlokeInvisible".to_string(); + let f6 = "tmp".to_string(); + + let temp = tempfile::tempdir().expect("cannot create tempfile"); + + let recent = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_secs() + - 2; // careful with leap seconds + + std::fs::create_dir(temp.path().join(&f1)).expect(&format!("cannot create dir {:?}", f1)); + std::fs::create_dir(temp.path().join(&f2)).expect(&format!("cannot create dir {:?}", f2)); + std::fs::create_dir(temp.path().join(&f3)).expect(&format!("cannot create dir {:?}", f3)); + std::fs::create_dir(temp.path().join(&f4)).expect(&format!("cannot create dir {:?}", f4)); + std::fs::create_dir(temp.path().join(&f5)).expect(&format!("cannot create dir {:?}", f5)); + std::fs::create_dir(temp.path().join(&f6)).expect(&format!("cannot create dir {:?}", f6)); + + let result = list_models_in_dir(temp.path()) + .await + .expect("cannot list directory"); + + assert_eq!(result.len(), 3); + + println!("recent is {}", recent); + for m in result { + assert_eq!(m.object, "model"); + if m.owned_by != the { + assert_eq!(m.owned_by, bloke); + } + if m.id != format!("{}/{}", bloke, r1) && m.id != format!("{}/{}", bloke, r3) { + assert_eq!(m.id, format!("{}/{}", the, r2)); + } + println!("{:?}", m); + + let d = m.created.checked_sub(recent).unwrap(); + assert!(d <= 3); + } + } } From 2fe8c9b3028fe1adfc2e379e7573ea84618abc2b Mon Sep 17 00:00:00 2001 From: Tobias Schoofs Date: Fri, 16 Feb 2024 14:56:49 +0000 Subject: [PATCH 06/12] [feat/modelmanager] models endpoint is working --- crates/edgen_server/src/model_man.rs | 49 ++++++++++++++++++++++++---- 1 file changed, 43 insertions(+), 6 deletions(-) diff --git a/crates/edgen_server/src/model_man.rs b/crates/edgen_server/src/model_man.rs index 8311fd4..5301e6f 100644 --- a/crates/edgen_server/src/model_man.rs +++ b/crates/edgen_server/src/model_man.rs @@ -15,14 +15,20 @@ use std::fmt::Display; use std::path::{Path, PathBuf}; use std::time::{SystemTime, SystemTimeError}; +use axum::http::StatusCode; use axum::response::{IntoResponse, Json, Response}; use serde::{Deserialize, Serialize}; use thiserror; use tracing::{error, info, warn}; use utoipa::ToSchema; +use edgen_core::settings; + pub async fn list_models() -> Response { - Json(true).into_response() + match list_all_models().await { + Ok(v) => Json(v).into_response(), + Err(e) => internal_server_error(&format!("model manager: cannot list models: {:?}", e)), + } } pub async fn retrieve_model() -> Response { @@ -33,6 +39,11 @@ pub async fn delete_model() -> Response { Json(true).into_response() } +fn internal_server_error(msg: &str) -> Response { + warn!("[ERROR] {}", msg); + StatusCode::INTERNAL_SERVER_ERROR.into_response() +} + #[derive(ToSchema, Deserialize, Serialize, Debug, PartialEq, Eq)] pub struct ModelDesc { id: String, @@ -55,8 +66,32 @@ impl Display for PathError { } } -async fn list_models_in_dir(path: &Path) -> Result, PathError> { +async fn list_all_models() -> Result, PathError> { + let completions_dir = settings::SETTINGS + .read() + .await + .read() + .await + .chat_completions_models_dir + .trim() + .to_string(); + + let transcriptions_dir = settings::SETTINGS + .read() + .await + .read() + .await + .chat_completions_models_dir + .trim() + .to_string(); + let mut v = vec![]; + list_models_in_dir(Path::new(&completions_dir), &mut v).await?; + list_models_in_dir(Path::new(&transcriptions_dir), &mut v).await?; + Ok(v) +} + +async fn list_models_in_dir(path: &Path, v: &mut Vec) -> Result<(), PathError> { let es = tokio::fs::read_dir(path).await; if es.is_err() { warn!("model manager: cannot read directory {:?} ({:?})", path, es); @@ -84,7 +119,7 @@ async fn list_models_in_dir(path: &Path) -> Result, PathError> { } } } - Ok(v) + Ok(()) } async fn path_to_model_desc(path: &Path) -> Result { @@ -424,14 +459,16 @@ mod test { std::fs::create_dir(temp.path().join(&f5)).expect(&format!("cannot create dir {:?}", f5)); std::fs::create_dir(temp.path().join(&f6)).expect(&format!("cannot create dir {:?}", f6)); - let result = list_models_in_dir(temp.path()) + let mut v = vec![]; + + let _ = list_models_in_dir(temp.path(), &mut v) .await .expect("cannot list directory"); - assert_eq!(result.len(), 3); + assert_eq!(v.len(), 3); println!("recent is {}", recent); - for m in result { + for m in v { assert_eq!(m.object, "model"); if m.owned_by != the { assert_eq!(m.owned_by, bloke); From b2c4f9a51d171312f3d2aa990530a37d0fd6e19d Mon Sep 17 00:00:00 2001 From: Tobias Schoofs Date: Fri, 16 Feb 2024 16:25:46 +0000 Subject: [PATCH 07/12] [feat/modelmanager] all endpoints are working --- crates/edgen_server/src/model_man.rs | 127 +++++++++++++++++++++++---- crates/edgen_server/src/routes.rs | 4 +- 2 files changed, 112 insertions(+), 19 deletions(-) diff --git a/crates/edgen_server/src/model_man.rs b/crates/edgen_server/src/model_man.rs index 5301e6f..4645d7a 100644 --- a/crates/edgen_server/src/model_man.rs +++ b/crates/edgen_server/src/model_man.rs @@ -15,11 +15,12 @@ use std::fmt::Display; use std::path::{Path, PathBuf}; use std::time::{SystemTime, SystemTimeError}; +use axum::extract; use axum::http::StatusCode; use axum::response::{IntoResponse, Json, Response}; use serde::{Deserialize, Serialize}; use thiserror; -use tracing::{error, info, warn}; +use tracing::warn; use utoipa::ToSchema; use edgen_core::settings; @@ -31,16 +32,27 @@ pub async fn list_models() -> Response { } } -pub async fn retrieve_model() -> Response { - Json(true).into_response() +pub async fn retrieve_model(extract::Path(id): extract::Path) -> Response { + match model_id_to_desc(&id).await { + Ok(d) => Json(d).into_response(), + Err(e) => { + internal_server_error(&format!("model manager: cannot get model {}: {:?}", id, e)) + } + } } -pub async fn delete_model() -> Response { - Json(true).into_response() +pub async fn delete_model(extract::Path(id): extract::Path) -> Response { + match remove_model(&id).await { + Ok(()) => StatusCode::NO_CONTENT.into_response(), + Err(e) => internal_server_error(&format!( + "model manager: cannot delete model {}: {:?}", + id, e + )), + } } fn internal_server_error(msg: &str) -> Response { - warn!("[ERROR] {}", msg); + warn!("{}", msg); StatusCode::INTERNAL_SERVER_ERROR.into_response() } @@ -55,6 +67,7 @@ pub struct ModelDesc { #[derive(Debug, thiserror::Error)] enum PathError { Generic(String), + ModelNotFound, ParseError(#[from] ParseError), IOError(#[from] std::io::Error), TimeError(#[from] SystemTimeError), @@ -67,28 +80,36 @@ impl Display for PathError { } async fn list_all_models() -> Result, PathError> { - let completions_dir = settings::SETTINGS + let completions_dir = chat_completions_dir().await; + let transcriptions_dir = audio_transcriptions_dir().await; + + let mut v = vec![]; + + list_models_in_dir(Path::new(&completions_dir), &mut v).await?; + list_models_in_dir(Path::new(&transcriptions_dir), &mut v).await?; + Ok(v) +} + +async fn chat_completions_dir() -> String { + settings::SETTINGS .read() .await .read() .await .chat_completions_models_dir .trim() - .to_string(); + .to_string() +} - let transcriptions_dir = settings::SETTINGS +async fn audio_transcriptions_dir() -> String { + settings::SETTINGS .read() .await .read() .await - .chat_completions_models_dir + .audio_transcriptions_models_dir .trim() - .to_string(); - - let mut v = vec![]; - list_models_in_dir(Path::new(&completions_dir), &mut v).await?; - list_models_in_dir(Path::new(&transcriptions_dir), &mut v).await?; - Ok(v) + .to_string() } async fn list_models_in_dir(path: &Path, v: &mut Vec) -> Result<(), PathError> { @@ -122,6 +143,32 @@ async fn list_models_in_dir(path: &Path, v: &mut Vec) -> Result<(), P Ok(()) } +async fn model_id_to_desc(id: &str) -> Result { + let path = search_model(id).await?; + path_to_model_desc(path.as_path()).await +} + +async fn search_model(id: &str) -> Result { + let model = model_id_to_path(id)?; + let dir = chat_completions_dir().await; + let path = Path::new(&dir).join(&model); + if path.is_dir() { + return Ok(path); + } + let dir = audio_transcriptions_dir().await; + let path = Path::new(&dir).join(&model); + if path.is_dir() { + return Ok(path); + } + Err(PathError::ModelNotFound) +} + +async fn remove_model(id: &str) -> Result<(), PathError> { + let model = search_model(id).await?; + let _ = tokio::fs::remove_dir_all(model).await?; + Ok(()) +} + async fn path_to_model_desc(path: &Path) -> Result { let f = path .file_name() @@ -230,12 +277,20 @@ fn parse_path(model_string: &str) -> Result<(String, String), ParseError> { #[cfg(test)] mod test { use super::*; - use std::ffi::OsStr; use std::path::Path; use std::time::SystemTime; use tempfile; + async fn init_settings_for_test() { + settings::SETTINGS + .write() + .await + .init() + .await + .expect("Failed to initialise settings"); + } + // --- Parse Model Id ------------------------------------------------------------------------- #[test] fn parse_simple_model_id_valid() { @@ -482,4 +537,42 @@ mod test { assert!(d <= 3); } } + + #[tokio::test] + // this test should go to integration tests! + async fn test_delete_model() { + let owner = "TheFaker"; + let repo = "my-faked-model-v1-GGUF"; + let model = format!("models--{}--{}", owner, repo); + let id = format!("{}/{}", owner, repo); + + init_settings_for_test().await; + + let dir = chat_completions_dir().await; + + // let temp = tempfile::tempdir().expect("cannot create tempfile"); + + let recent = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_secs() + - 2; // careful with leap seconds + + let dir = Path::new(&dir).join(&model); + std::fs::create_dir(&dir).expect(&format!("cannot create model {:?}", dir)); + + let m = model_id_to_desc(&id).await; + assert!(m.is_ok(), "cannot get model"); + let m = m.unwrap(); + + assert_eq!(m.object, "model"); + assert_eq!(m.owned_by, owner); + assert_eq!(m.id, id); + let d = m.created.checked_sub(recent).unwrap(); + assert!(d <= 3); + + let result = remove_model(&id).await; + assert!(result.is_ok()); + assert!(!dir.exists()); + } } diff --git a/crates/edgen_server/src/routes.rs b/crates/edgen_server/src/routes.rs index 3c4a237..b1557e1 100644 --- a/crates/edgen_server/src/routes.rs +++ b/crates/edgen_server/src/routes.rs @@ -48,11 +48,11 @@ pub fn routes() -> Router { // -- Model Manager ---------------------------------------------------- .route("/v1/models", axum::routing::get(model_man::list_models)) .route( - "/v1/models/{model}", + "/v1/models/:model", axum::routing::get(model_man::retrieve_model), ) .route( - "/v1/models/{model}", + "/v1/models/:model", axum::routing::delete(model_man::delete_model), ) } From 94df3b8d4903997d05751ff1391602c95d0d1ecc Mon Sep 17 00:00:00 2001 From: Tobias Schoofs Date: Sun, 18 Feb 2024 14:39:20 +0000 Subject: [PATCH 08/12] [feat/modelmanager] integration test modelmanager added, model_man module public --- crates/edgen_core/src/settings.rs | 24 ++ crates/edgen_server/src/lib.rs | 2 +- crates/edgen_server/src/misc.rs | 2 +- crates/edgen_server/src/model_man.rs | 107 ++------ crates/edgen_server/src/routes.rs | 4 +- crates/edgen_server/tests/common/mod.rs | 105 ++++++++ .../edgen_server/tests/modelmanager_tests.rs | 238 ++++++++++++++++++ crates/edgen_server/tests/settings_tests.rs | 1 + 8 files changed, 399 insertions(+), 84 deletions(-) create mode 100644 crates/edgen_server/tests/modelmanager_tests.rs diff --git a/crates/edgen_core/src/settings.rs b/crates/edgen_core/src/settings.rs index 51826ba..f4b6f0e 100644 --- a/crates/edgen_core/src/settings.rs +++ b/crates/edgen_core/src/settings.rs @@ -95,6 +95,30 @@ fn build_config_file_path() -> PathBuf { config_dir.join(Path::new(&filename)) } +/// Helper to get the chat completions model directory. +pub async fn chat_completions_dir() -> String { + SETTINGS + .read() + .await + .read() + .await + .chat_completions_models_dir + .trim() + .to_string() +} + +/// Helper to get the audio transcriptions model directory. +pub async fn audio_transcriptions_dir() -> String { + SETTINGS + .read() + .await + .read() + .await + .audio_transcriptions_models_dir + .trim() + .to_string() +} + #[derive(Error, Debug, Serialize)] pub enum SettingsError { #[error("failed to read the settings file: {0}")] diff --git a/crates/edgen_server/src/lib.rs b/crates/edgen_server/src/lib.rs index b46dd6e..88b057f 100644 --- a/crates/edgen_server/src/lib.rs +++ b/crates/edgen_server/src/lib.rs @@ -42,7 +42,7 @@ pub mod error; pub mod graceful_shutdown; mod llm; mod model; -mod model_man; +pub mod model_man; pub mod openai_shim; mod routes; pub mod status; diff --git a/crates/edgen_server/src/misc.rs b/crates/edgen_server/src/misc.rs index 7e5aa77..c041c46 100644 --- a/crates/edgen_server/src/misc.rs +++ b/crates/edgen_server/src/misc.rs @@ -36,7 +36,7 @@ pub struct Version { build: String, } -/// GET `/v1/version`: returns the current version of edgend. +/// GET `/v1/misc/version`: returns the current version of edgend. /// /// The version is returned as json value with major, minor and patch as integer /// and build as string (which may be empty). diff --git a/crates/edgen_server/src/model_man.rs b/crates/edgen_server/src/model_man.rs index 4645d7a..b4f8747 100644 --- a/crates/edgen_server/src/model_man.rs +++ b/crates/edgen_server/src/model_man.rs @@ -10,6 +10,8 @@ * limitations under the License. */ +//! Model Manager Endpoints + use std::fmt; use std::fmt::Display; use std::path::{Path, PathBuf}; @@ -25,6 +27,9 @@ use utoipa::ToSchema; use edgen_core::settings; +/// GET `/v1/models`: returns a list of model descriptors for all models in all model directories. +/// +/// For any error, the endpoint returns "internal server error". pub async fn list_models() -> Response { match list_all_models().await { Ok(v) => Json(v).into_response(), @@ -32,6 +37,9 @@ pub async fn list_models() -> Response { } } +/// GET `/v1/models{:id}`: returns the model descriptor for the model indicated by 'id'. +/// +/// For any error, the endpoint returns "internal server error". pub async fn retrieve_model(extract::Path(id): extract::Path) -> Response { match model_id_to_desc(&id).await { Ok(d) => Json(d).into_response(), @@ -41,6 +49,10 @@ pub async fn retrieve_model(extract::Path(id): extract::Path) -> Respons } } +/// DELETE `/v1/models{:id}`: deletes the model indicated by 'id'. +/// +/// Returns 204 (NO CONTENT) on success. +/// For any error, the endpoint returns "internal server error". pub async fn delete_model(extract::Path(id): extract::Path) -> Response { match remove_model(&id).await { Ok(()) => StatusCode::NO_CONTENT.into_response(), @@ -56,12 +68,17 @@ fn internal_server_error(msg: &str) -> Response { StatusCode::INTERNAL_SERVER_ERROR.into_response() } +/// Model Descriptor #[derive(ToSchema, Deserialize, Serialize, Debug, PartialEq, Eq)] pub struct ModelDesc { - id: String, - created: u64, - object: String, - owned_by: String, + /// model Id + pub id: String, + /// when the file was created + pub created: u64, + /// object type, always 'model' + pub object: String, + /// repo owner + pub owned_by: String, } #[derive(Debug, thiserror::Error)] @@ -80,8 +97,8 @@ impl Display for PathError { } async fn list_all_models() -> Result, PathError> { - let completions_dir = chat_completions_dir().await; - let transcriptions_dir = audio_transcriptions_dir().await; + let completions_dir = settings::chat_completions_dir().await; + let transcriptions_dir = settings::audio_transcriptions_dir().await; let mut v = vec![]; @@ -90,28 +107,6 @@ async fn list_all_models() -> Result, PathError> { Ok(v) } -async fn chat_completions_dir() -> String { - settings::SETTINGS - .read() - .await - .read() - .await - .chat_completions_models_dir - .trim() - .to_string() -} - -async fn audio_transcriptions_dir() -> String { - settings::SETTINGS - .read() - .await - .read() - .await - .audio_transcriptions_models_dir - .trim() - .to_string() -} - async fn list_models_in_dir(path: &Path, v: &mut Vec) -> Result<(), PathError> { let es = tokio::fs::read_dir(path).await; if es.is_err() { @@ -123,7 +118,7 @@ async fn list_models_in_dir(path: &Path, v: &mut Vec) -> Result<(), P let e = es.next_entry().await; if e.is_err() { warn!("model manager: cannot get entry: {:?}", e); - continue; + break; } let tmp = e.unwrap(); if tmp.is_none() { @@ -150,12 +145,12 @@ async fn model_id_to_desc(id: &str) -> Result { async fn search_model(id: &str) -> Result { let model = model_id_to_path(id)?; - let dir = chat_completions_dir().await; + let dir = settings::chat_completions_dir().await; let path = Path::new(&dir).join(&model); if path.is_dir() { return Ok(path); } - let dir = audio_transcriptions_dir().await; + let dir = settings::audio_transcriptions_dir().await; let path = Path::new(&dir).join(&model); if path.is_dir() { return Ok(path); @@ -256,7 +251,7 @@ fn parse_path(model_string: &str) -> Result<(String, String), ParseError> { // the owner is always the second // if the original owner contained double dashes - // we won't found him + // we won't find him let owner = vs[1].to_string(); if owner.is_empty() { return Err(ParseError::NoOwner); @@ -277,20 +272,10 @@ fn parse_path(model_string: &str) -> Result<(String, String), ParseError> { #[cfg(test)] mod test { use super::*; - use std::path::Path; use std::time::SystemTime; use tempfile; - async fn init_settings_for_test() { - settings::SETTINGS - .write() - .await - .init() - .await - .expect("Failed to initialise settings"); - } - // --- Parse Model Id ------------------------------------------------------------------------- #[test] fn parse_simple_model_id_valid() { @@ -537,42 +522,4 @@ mod test { assert!(d <= 3); } } - - #[tokio::test] - // this test should go to integration tests! - async fn test_delete_model() { - let owner = "TheFaker"; - let repo = "my-faked-model-v1-GGUF"; - let model = format!("models--{}--{}", owner, repo); - let id = format!("{}/{}", owner, repo); - - init_settings_for_test().await; - - let dir = chat_completions_dir().await; - - // let temp = tempfile::tempdir().expect("cannot create tempfile"); - - let recent = SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH) - .unwrap() - .as_secs() - - 2; // careful with leap seconds - - let dir = Path::new(&dir).join(&model); - std::fs::create_dir(&dir).expect(&format!("cannot create model {:?}", dir)); - - let m = model_id_to_desc(&id).await; - assert!(m.is_ok(), "cannot get model"); - let m = m.unwrap(); - - assert_eq!(m.object, "model"); - assert_eq!(m.owned_by, owner); - assert_eq!(m.id, id); - let d = m.created.checked_sub(recent).unwrap(); - assert!(d <= 3); - - let result = remove_model(&id).await; - assert!(result.is_ok()); - assert!(!dir.exists()); - } } diff --git a/crates/edgen_server/src/routes.rs b/crates/edgen_server/src/routes.rs index b1557e1..687c32b 100644 --- a/crates/edgen_server/src/routes.rs +++ b/crates/edgen_server/src/routes.rs @@ -43,8 +43,6 @@ pub fn routes() -> Router { "/v1/audio/transcriptions/status", axum::routing::get(status::audio_transcriptions_status), ) - // -- Miscellaneous services ------------------------------------------- - .route("/v1/misc/version", axum::routing::get(misc::edgen_version)) // -- Model Manager ---------------------------------------------------- .route("/v1/models", axum::routing::get(model_man::list_models)) .route( @@ -55,4 +53,6 @@ pub fn routes() -> Router { "/v1/models/:model", axum::routing::delete(model_man::delete_model), ) + // -- Miscellaneous services ------------------------------------------- + .route("/v1/misc/version", axum::routing::get(misc::edgen_version)) } diff --git a/crates/edgen_server/tests/common/mod.rs b/crates/edgen_server/tests/common/mod.rs index cc01f49..81faa58 100644 --- a/crates/edgen_server/tests/common/mod.rs +++ b/crates/edgen_server/tests/common/mod.rs @@ -32,6 +32,7 @@ pub const TRANSCRIPTIONS_URL: &str = "/transcriptions"; pub const STATUS_URL: &str = "/status"; pub const MISC_URL: &str = "/misc"; pub const VERSION_URL: &str = "/version"; +pub const MODELS_URL: &str = "/models"; pub const CHAT_COMPLETIONS_BODY: &str = r#" { @@ -51,6 +52,7 @@ pub const CHAT_COMPLETIONS_BODY: &str = r#" "#; pub const BACKUP_DIR: &str = "env_backup"; +pub const CONFIG_BACKUP_DIR: &str = "config_backup"; pub const MY_MODEL_FILES: &str = "my_models"; #[derive(Debug, PartialEq, Eq)] @@ -102,6 +104,39 @@ where } } +// Backup config only before running 'f'; +// restore config, even if 'f' panicks. +pub fn with_save_config(f: F) +where + F: FnOnce() + panic::UnwindSafe, +{ + println!("with save config!"); + + backup_config().unwrap(); + + println!("=============="); + println!("STARTING TESTS"); + println!("=============="); + + let r = panic::catch_unwind(f); + + println!("==========="); + println!("TESTS READY"); + println!("==========="); + + let _ = match restore_config() { + Ok(_) => (), + Err(e) => { + panic!("Panic! Cannot restore your config: {:?}", e); + } + }; + + match r { + Err(e) => panic::resume_unwind(e), + Ok(_) => (), + } +} + // Start edgen before running 'f' pub fn with_edgen(f: F) where @@ -135,6 +170,18 @@ where }); } +// Backup config directories) +// and start edgen before running 'f'; +// restore config, even if 'f' or edgen panick. +pub fn with_save_config_edgen(f: F) +where + F: FnOnce() + panic::UnwindSafe, +{ + with_save_config(|| { + with_edgen(f); + }); +} + pub fn test_message(msg: &str) { println!("=== Test {}", msg); } @@ -411,3 +458,61 @@ fn restore_env() -> Result<(), io::Error> { Ok(()) } + +fn backup_config() -> Result<(), BackupError> { + println!("backing up"); + + let backup_dir = Path::new(CONFIG_BACKUP_DIR); + if backup_dir.exists() { + let msg = format!( + "directory {} exists! + This means an earlier test run did not finish correctly. \ + Restore your environment manually.", + CONFIG_BACKUP_DIR, + ); + eprintln!("{}", msg); + return Err(BackupError::Unfinished); + } + + println!("config dir: {:?}", settings::PROJECT_DIRS.config_dir()); + + fs::create_dir(&backup_dir)?; + + let cnfg = settings::PROJECT_DIRS.config_dir(); + let cnfg_bkp = backup_dir.join("config"); + + if cnfg.exists() { + println!("config bkp: {:?}", cnfg_bkp); + copy_dir(&cnfg, &cnfg_bkp)?; + fs::remove_dir_all(&cnfg)?; + } else { + println!("config {:?} does not exist", cnfg); + } + + Ok(()) +} + +fn restore_config() -> Result<(), io::Error> { + println!("restoring"); + + let backup_dir = Path::new(CONFIG_BACKUP_DIR); + + let cnfg = settings::PROJECT_DIRS.config_dir(); + let cnfg_bkp = backup_dir.join("config"); + + if cnfg.exists() { + fs::remove_dir_all(&cnfg)?; + } + + if cnfg_bkp.exists() { + println!("{:?} -> {:?}", cnfg_bkp, cnfg); + copy_dir(&cnfg_bkp, &cnfg)?; + } else { + println!("config bkp {:?} does not exist", cnfg_bkp); + } + + println!("removing {:?}", backup_dir); + fs::remove_dir_all(&backup_dir)?; + + Ok(()) +} diff --git a/crates/edgen_server/tests/modelmanager_tests.rs b/crates/edgen_server/tests/modelmanager_tests.rs new file mode 100644 index 0000000..c839346 --- /dev/null +++ b/crates/edgen_server/tests/modelmanager_tests.rs @@ -0,0 +1,238 @@ +use std::path; +use std::path::{Path, PathBuf}; +use std::time::SystemTime; + +use futures::executor::block_on; +use reqwest::{blocking, StatusCode}; + +use edgen_core::settings; +use edgen_server::model_man::ModelDesc; + +#[allow(dead_code)] +mod common; + +#[test] +fn test_modelmanager() { + common::with_save_config_edgen(|| { + pass_always(); + + // ================================ + config_exists(); + + // endpoints reachable + connect_to_server_test(); + + let my_models_dir = format!( + "{}{}{}", + common::CONFIG_BACKUP_DIR, + path::MAIN_SEPARATOR, + common::MY_MODEL_FILES, + ); + + let new_chat_completions_dir = my_models_dir.clone() + + &format!( + "{}{}{}{}", + path::MAIN_SEPARATOR, + "chat", + path::MAIN_SEPARATOR, + "completions", + ); + + let new_audio_transcriptions_dir = my_models_dir.clone() + + &format!( + "{}{}{}{}", + path::MAIN_SEPARATOR, + "audio", + path::MAIN_SEPARATOR, + "transcriptions", + ); + + set_model_dir(common::Endpoint::ChatCompletions, &new_chat_completions_dir); + + set_model_dir( + common::Endpoint::AudioTranscriptions, + &new_audio_transcriptions_dir, + ); + + make_dirs(); + + test_list_models(); + test_delete_model(); + }) +} + +fn pass_always() { + common::test_message("pass always"); + assert!(true); +} + +fn config_exists() { + common::test_message("config exists"); + assert!(settings::PROJECT_DIRS.config_dir().exists()); + assert!(settings::CONFIG_FILE.exists()); +} + +// exercise the edgen version endpoint to make sure the server is reachable. +fn connect_to_server_test() { + common::test_message("connect to server"); + assert!(match blocking::get(common::make_url(&[ + common::BASE_URL, + common::MISC_URL, + common::VERSION_URL + ])) { + Err(e) => { + eprintln!("cannot connect: {:?}", e); + false + } + Ok(v) => { + println!("have: '{}'", v.text().unwrap()); + true + } + }); +} + +// edit the config file: set another model dir for the indicated endpoint. +fn set_model_dir(ep: common::Endpoint, model_dir: &str) { + common::test_message(&format!("set {} model directory to {}", ep, model_dir,)); + + let mut config = common::get_config().unwrap(); + + match &ep { + common::Endpoint::ChatCompletions => { + config.chat_completions_models_dir = model_dir.to_string(); + } + common::Endpoint::AudioTranscriptions => { + config.audio_transcriptions_models_dir = model_dir.to_string(); + } + } + common::write_config(&config).unwrap(); + + println!("pausing for 4 secs to make sure the config file has been updated"); + std::thread::sleep(std::time::Duration::from_secs(4)); +} + +// actually create the model dirs before using them +fn make_dirs() { + let dir = block_on(async { settings::chat_completions_dir().await }); + std::fs::create_dir_all(&dir).expect("cannot create chat completions model dir"); + + assert!(PathBuf::from(&dir).exists()); + + let dir = block_on(async { settings::audio_transcriptions_dir().await }); + std::fs::create_dir_all(&dir).expect("cannot create audio transcriptions model dir"); + + assert!(PathBuf::from(&dir).exists()); +} + +fn test_list_models() { + common::test_message("list models"); + + let bloke = "TheBloke"; + let the = "The"; + let r1 = "TinyLlama-1.1B-Chat-v1.0-GGUF"; + let r2 = "Bloke--TinyLlama-1.1B-Chat-v1.0-GGUF"; + let r3 = "TinyLlama--1.1B--Chat--v1.0--GGUF"; + let f1 = format!("models--{}--{}", bloke, r1); + let f2 = format!("models--{}--{}", the, r2); + let f3 = format!("models--{}--{}", bloke, r3); + let f4 = "invisible".to_string(); + let f5 = "models--TheBlokeInvisible".to_string(); + let f6 = "tmp".to_string(); + + let recent = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_secs() + - 2; // careful with leap seconds + + let dir1 = block_on(async { settings::audio_transcriptions_dir().await }); + let dir1 = PathBuf::from(&dir1); + + let dir2 = block_on(async { settings::chat_completions_dir().await }); + let dir2 = PathBuf::from(&dir2); + + std::fs::create_dir(dir1.join(&f1)).expect(&format!("cannot create dir {:?}", f1)); + std::fs::create_dir(dir2.join(&f2)).expect(&format!("cannot create dir {:?}", f2)); + std::fs::create_dir(dir1.join(&f3)).expect(&format!("cannot create dir {:?}", f3)); + std::fs::create_dir(dir1.join(&f4)).expect(&format!("cannot create dir {:?}", f4)); + std::fs::create_dir(dir2.join(&f5)).expect(&format!("cannot create dir {:?}", f5)); + std::fs::create_dir(dir1.join(&f6)).expect(&format!("cannot create dir {:?}", f6)); + + // --- get model descriptor + let res = blocking::get(common::make_url(&[common::BASE_URL, common::MODELS_URL])) + .expect("models get endpoint failed"); + assert!(res.status().is_success(), "models failed"); + let v: Vec = res.json().expect("cannot convert to model descs"); + + assert_eq!(v.len(), 3); + + println!("recent is {}", recent); + for m in v { + assert_eq!(m.object, "model"); + if m.owned_by != the { + assert_eq!(m.owned_by, bloke); + } + if m.id != format!("{}/{}", bloke, r1) && m.id != format!("{}/{}", bloke, r3) { + assert_eq!(m.id, format!("{}/{}", the, r2)); + } + println!("{:?}", m); + + let d = m.created.checked_sub(recent).unwrap(); + assert!(d <= 3); + } +} + +fn test_delete_model() { + common::test_message("delete model"); + + let owner = "TheFaker"; + let repo = "my-faked-model-v1-GGUF"; + let model = format!("models--{}--{}", owner, repo); + let id = format!("{}/{}", owner, repo); + let id_url = format!("{}%2f{}", owner, repo); + + let recent = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_secs() + - 2; // careful with leap seconds + + let dir = block_on(async { settings::chat_completions_dir().await }); + + let dir = Path::new(&dir).join(&model); + std::fs::create_dir(&dir).expect(&format!("cannot create model {:?}", dir)); + + // --- get model descriptor + let res = blocking::get(common::make_url(&[ + common::BASE_URL, + common::MODELS_URL, + "/", + &id_url, + ])) + .expect("models get endpoint failed"); + + assert!(res.status().is_success(), "models failed"); + let m: ModelDesc = res.json().expect("cannot convert to model desc"); + + println!("model descriptor: {:?}", m); + assert_eq!(m.object, "model"); + assert_eq!(m.owned_by, owner); + assert_eq!(m.id, id); + let d = m.created.checked_sub(recent).unwrap(); + assert!(d <= 3); + + // --- delete model + println!("delete model"); + let res = blocking::Client::new() + .delete(common::make_url(&[ + common::BASE_URL, + common::MODELS_URL, + "/", + &id_url, + ])) + .send() + .expect("models delete endpoint failed"); + + assert_eq!(res.status(), StatusCode::NO_CONTENT); + assert!(!dir.exists()); +} diff --git a/crates/edgen_server/tests/settings_tests.rs b/crates/edgen_server/tests/settings_tests.rs index f0d74da..3bffac4 100644 --- a/crates/edgen_server/tests/settings_tests.rs +++ b/crates/edgen_server/tests/settings_tests.rs @@ -6,6 +6,7 @@ use reqwest::blocking; use edgen_core::settings; use edgen_server::status; +#[allow(dead_code)] mod common; #[test] From b2c8c3a4cac033b5c12d69aa48a403b923a84605 Mon Sep 17 00:00:00 2001 From: Tobias Schoofs Date: Sun, 18 Feb 2024 19:06:41 +0000 Subject: [PATCH 09/12] [feat/modelmanager] remove duplicated code --- crates/edgen_server/tests/common/mod.rs | 105 +++++++++++++ .../edgen_server/tests/modelmanager_tests.rs | 62 +------- crates/edgen_server/tests/settings_tests.rs | 138 ++---------------- 3 files changed, 120 insertions(+), 185 deletions(-) diff --git a/crates/edgen_server/tests/common/mod.rs b/crates/edgen_server/tests/common/mod.rs index 81faa58..bd7fc69 100644 --- a/crates/edgen_server/tests/common/mod.rs +++ b/crates/edgen_server/tests/common/mod.rs @@ -238,6 +238,111 @@ pub fn reset_config() { edgen_server::config_reset().unwrap(); } +pub fn pass_always() { + test_message("pass always"); + assert!(true); +} + +pub fn config_exists() { + test_message("config exists"); + assert!(settings::PROJECT_DIRS.config_dir().exists()); + assert!(settings::CONFIG_FILE.exists()); +} + +pub fn data_exists() { + test_message("data exists"); + let data = settings::PROJECT_DIRS.data_dir(); + println!("exists: {:?}", data); + assert!(data.exists()); + + let models = data.join("models"); + println!("exists: {:?}", models); + assert!(models.exists()); + + let chat = models.join("chat"); + println!("exists: {:?}", chat); + assert!(models.exists()); + + let completions = chat.join("completions"); + println!("exists: {:?}", completions); + assert!(completions.exists()); + + let audio = models.join("audio"); + println!("exists: {:?}", audio); + assert!(audio.exists()); + + let transcriptions = audio.join("transcriptions"); + println!("exists: {:?}", transcriptions); + assert!(transcriptions.exists()); +} + +// edit the config file: set another model dir for the indicated endpoint. +pub fn set_model_dir(ep: Endpoint, model_dir: &str) { + test_message(&format!("set {} model directory to {}", ep, model_dir,)); + + let mut config = get_config().unwrap(); + + match &ep { + Endpoint::ChatCompletions => { + config.chat_completions_models_dir = model_dir.to_string(); + } + Endpoint::AudioTranscriptions => { + config.audio_transcriptions_models_dir = model_dir.to_string(); + } + } + write_config(&config).unwrap(); + + println!("pausing for 4 secs to make sure the config file has been updated"); + std::thread::sleep(std::time::Duration::from_secs(4)); +} + +// edit the config file: set another model name and repo for the indicated endpoint. +pub fn set_model(ep: Endpoint, model_name: &str, model_repo: &str) { + test_message(&format!("set {} model to {}", ep, model_name,)); + + let mut config = get_config().unwrap(); + + match &ep { + Endpoint::ChatCompletions => { + config.chat_completions_model_name = model_name.to_string(); + config.chat_completions_model_repo = model_repo.to_string(); + } + Endpoint::AudioTranscriptions => { + config.audio_transcriptions_model_name = model_name.to_string(); + config.audio_transcriptions_model_repo = model_repo.to_string(); + } + } + write_config(&config).unwrap(); + + println!("pausing for 4 secs to make sure the config file has been updated"); + std::thread::sleep(std::time::Duration::from_secs(4)); + let url = match ep { + Endpoint::ChatCompletions => make_url(&[BASE_URL, CHAT_URL, COMPLETIONS_URL, STATUS_URL]), + Endpoint::AudioTranscriptions => { + make_url(&[BASE_URL, AUDIO_URL, TRANSCRIPTIONS_URL, STATUS_URL]) + } + }; + let stat: status::AIStatus = blocking::get(url).unwrap().json().unwrap(); + assert_eq!(stat.active_model, model_name); +} + +// exercise the edgen version endpoint to make sure the server is reachable. +pub fn connect_to_server_test() { + test_message("connect to server"); + assert!( + match blocking::get(make_url(&[BASE_URL, MISC_URL, VERSION_URL])) { + Err(e) => { + eprintln!("cannot connect: {:?}", e); + false + } + Ok(v) => { + println!("have: '{}'", v.text().unwrap()); + true + } + } + ); +} + // spawn a thread to send a request to the indicated endpoint. // This allows the caller to perform another task in the caller thread. pub fn spawn_request(ep: Endpoint, body: String) -> thread::JoinHandle { diff --git a/crates/edgen_server/tests/modelmanager_tests.rs b/crates/edgen_server/tests/modelmanager_tests.rs index c839346..a2bacfe 100644 --- a/crates/edgen_server/tests/modelmanager_tests.rs +++ b/crates/edgen_server/tests/modelmanager_tests.rs @@ -14,13 +14,11 @@ mod common; #[test] fn test_modelmanager() { common::with_save_config_edgen(|| { - pass_always(); + common::pass_always(); - // ================================ - config_exists(); + common::config_exists(); - // endpoints reachable - connect_to_server_test(); + common::connect_to_server_test(); let my_models_dir = format!( "{}{}{}", @@ -47,9 +45,9 @@ fn test_modelmanager() { "transcriptions", ); - set_model_dir(common::Endpoint::ChatCompletions, &new_chat_completions_dir); + common::set_model_dir(common::Endpoint::ChatCompletions, &new_chat_completions_dir); - set_model_dir( + common::set_model_dir( common::Endpoint::AudioTranscriptions, &new_audio_transcriptions_dir, ); @@ -61,56 +59,6 @@ fn test_modelmanager() { }) } -fn pass_always() { - common::test_message("pass always"); - assert!(true); -} - -fn config_exists() { - common::test_message("config exists"); - assert!(settings::PROJECT_DIRS.config_dir().exists()); - assert!(settings::CONFIG_FILE.exists()); -} - -// exercise the edgen version endpoint to make sure the server is reachable. -fn connect_to_server_test() { - common::test_message("connect to server"); - assert!(match blocking::get(common::make_url(&[ - common::BASE_URL, - common::MISC_URL, - common::VERSION_URL - ])) { - Err(e) => { - eprintln!("cannot connect: {:?}", e); - false - } - Ok(v) => { - println!("have: '{}'", v.text().unwrap()); - true - } - }); -} - -// edit the config file: set another model dir for the indicated endpoint. -fn set_model_dir(ep: common::Endpoint, model_dir: &str) { - common::test_message(&format!("set {} model directory to {}", ep, model_dir,)); - - let mut config = common::get_config().unwrap(); - - match &ep { - common::Endpoint::ChatCompletions => { - config.chat_completions_models_dir = model_dir.to_string(); - } - common::Endpoint::AudioTranscriptions => { - config.audio_transcriptions_models_dir = model_dir.to_string(); - } - } - common::write_config(&config).unwrap(); - - println!("pausing for 4 secs to make sure the config file has been updated"); - std::thread::sleep(std::time::Duration::from_secs(4)); -} - // actually create the model dirs before using them fn make_dirs() { let dir = block_on(async { settings::chat_completions_dir().await }); diff --git a/crates/edgen_server/tests/settings_tests.rs b/crates/edgen_server/tests/settings_tests.rs index 3bffac4..ab0c5ba 100644 --- a/crates/edgen_server/tests/settings_tests.rs +++ b/crates/edgen_server/tests/settings_tests.rs @@ -3,9 +3,6 @@ use std::path; use reqwest::blocking; -use edgen_core::settings; -use edgen_server::status; - #[allow(dead_code)] mod common; @@ -49,16 +46,16 @@ fn fake_test() { fn test_battery() { common::with_save_edgen(|| { // make sure everything is right - pass_always(); + common::pass_always(); // ================================ common::test_message("SCENARIO 1"); // ================================ - config_exists(); - data_exists(); + common::config_exists(); + common::data_exists(); // endpoints reachable - connect_to_server_test(); + common::connect_to_server_test(); chat_completions_status_reachable(); audio_transcriptions_status_reachable(); @@ -67,12 +64,12 @@ fn test_battery() { common::test_message("SCENARIO 2"); // ================================ // set small models, so we don't need to download too much - set_model( + common::set_model( common::Endpoint::ChatCompletions, common::SMALL_LLM_NAME, common::SMALL_LLM_REPO, ); - set_model( + common::set_model( common::Endpoint::AudioTranscriptions, common::SMALL_WHISPER_NAME, common::SMALL_WHISPER_REPO, @@ -114,9 +111,9 @@ fn test_battery() { "transcriptions", ); - set_model_dir(common::Endpoint::ChatCompletions, &new_chat_completions_dir); + common::set_model_dir(common::Endpoint::ChatCompletions, &new_chat_completions_dir); - set_model_dir( + common::set_model_dir( common::Endpoint::AudioTranscriptions, &new_audio_transcriptions_dir, ); @@ -148,12 +145,12 @@ fn test_battery() { // ================================ test_config_reset(); - set_model( + common::set_model( common::Endpoint::ChatCompletions, common::SMALL_LLM_NAME, common::SMALL_LLM_REPO, ); - set_model( + common::set_model( common::Endpoint::AudioTranscriptions, common::SMALL_WHISPER_NAME, common::SMALL_WHISPER_REPO, @@ -168,63 +165,6 @@ fn test_battery() { }) } -fn pass_always() { - common::test_message("pass always"); - assert!(true); -} - -// exercise the edgen version endpoint to make sure the server is reachable. -fn connect_to_server_test() { - common::test_message("connect to server"); - assert!(match blocking::get(common::make_url(&[ - common::BASE_URL, - common::MISC_URL, - common::VERSION_URL - ])) { - Err(e) => { - eprintln!("cannot connect: {:?}", e); - false - } - Ok(v) => { - println!("have: '{}'", v.text().unwrap()); - true - } - }); -} - -fn config_exists() { - common::test_message("config exists"); - assert!(settings::PROJECT_DIRS.config_dir().exists()); - assert!(settings::CONFIG_FILE.exists()); -} - -fn data_exists() { - common::test_message("data exists"); - let data = settings::PROJECT_DIRS.data_dir(); - println!("exists: {:?}", data); - assert!(data.exists()); - - let models = data.join("models"); - println!("exists: {:?}", models); - assert!(models.exists()); - - let chat = models.join("chat"); - println!("exists: {:?}", chat); - assert!(models.exists()); - - let completions = chat.join("completions"); - println!("exists: {:?}", completions); - assert!(completions.exists()); - - let audio = models.join("audio"); - println!("exists: {:?}", audio); - assert!(audio.exists()); - - let transcriptions = audio.join("transcriptions"); - println!("exists: {:?}", transcriptions); - assert!(transcriptions.exists()); -} - fn chat_completions_status_reachable() { common::test_message("chat completions status is reachable"); assert!(match blocking::get(common::make_url(&[ @@ -263,64 +203,6 @@ fn audio_transcriptions_status_reachable() { }); } -// edit the config file: set another model name and repo for the indicated endpoint. -fn set_model(ep: common::Endpoint, model_name: &str, model_repo: &str) { - common::test_message(&format!("set {} model to {}", ep, model_name,)); - - let mut config = common::get_config().unwrap(); - - match &ep { - common::Endpoint::ChatCompletions => { - config.chat_completions_model_name = model_name.to_string(); - config.chat_completions_model_repo = model_repo.to_string(); - } - common::Endpoint::AudioTranscriptions => { - config.audio_transcriptions_model_name = model_name.to_string(); - config.audio_transcriptions_model_repo = model_repo.to_string(); - } - } - common::write_config(&config).unwrap(); - - println!("pausing for 4 secs to make sure the config file has been updated"); - std::thread::sleep(std::time::Duration::from_secs(4)); - let url = match ep { - common::Endpoint::ChatCompletions => common::make_url(&[ - common::BASE_URL, - common::CHAT_URL, - common::COMPLETIONS_URL, - common::STATUS_URL, - ]), - common::Endpoint::AudioTranscriptions => common::make_url(&[ - common::BASE_URL, - common::AUDIO_URL, - common::TRANSCRIPTIONS_URL, - common::STATUS_URL, - ]), - }; - let stat: status::AIStatus = blocking::get(url).unwrap().json().unwrap(); - assert_eq!(stat.active_model, model_name); -} - -// edit the config file: set another model dir for the indicated endpoint. -fn set_model_dir(ep: common::Endpoint, model_dir: &str) { - common::test_message(&format!("set {} model directory to {}", ep, model_dir,)); - - let mut config = common::get_config().unwrap(); - - match &ep { - common::Endpoint::ChatCompletions => { - config.chat_completions_models_dir = model_dir.to_string(); - } - common::Endpoint::AudioTranscriptions => { - config.audio_transcriptions_models_dir = model_dir.to_string(); - } - } - common::write_config(&config).unwrap(); - - println!("pausing for 4 secs to make sure the config file has been updated"); - std::thread::sleep(std::time::Duration::from_secs(4)); -} - fn test_config_reset() { common::test_message("test resetting config"); common::reset_config(); From 9f78c6e99e43492b9417bb0a8c6323879eddb361 Mon Sep 17 00:00:00 2001 From: Tobias Schoofs Date: Mon, 19 Feb 2024 08:59:52 +0000 Subject: [PATCH 10/12] [feat/modelmanager] minor improvements and documentation --- crates/edgen_server/tests/common/mod.rs | 51 +++++++++++---------- crates/edgen_server/tests/settings_tests.rs | 2 + 2 files changed, 30 insertions(+), 23 deletions(-) diff --git a/crates/edgen_server/tests/common/mod.rs b/crates/edgen_server/tests/common/mod.rs index bd7fc69..e1ac337 100644 --- a/crates/edgen_server/tests/common/mod.rs +++ b/crates/edgen_server/tests/common/mod.rs @@ -71,8 +71,8 @@ impl Display for Endpoint { } } -// Backup environment (config and model directories) before running 'f'; -// restore environment, even if 'f' panicks. +/// Backup environment (config and model directories) before running 'f'; +/// restore environment, even if 'f' panicks. pub fn with_save_env(f: F) where F: FnOnce() + panic::UnwindSafe, @@ -104,8 +104,8 @@ where } } -// Backup config only before running 'f'; -// restore config, even if 'f' panicks. +/// Backup config only before running 'f'; +/// restore config, even if 'f' panicks. pub fn with_save_config(f: F) where F: FnOnce() + panic::UnwindSafe, @@ -137,7 +137,7 @@ where } } -// Start edgen before running 'f' +/// Start edgen before running 'f' pub fn with_edgen(f: F) where F: FnOnce() + panic::UnwindSafe, @@ -158,9 +158,9 @@ where f(); } -// Backup environment (config and model directories) -// and start edgen before running 'f'; -// restore environment, even if 'f' or edgen panick. +/// Backup environment (config and model directories) +/// and start edgen before running 'f'; +/// restore environment, even if 'f' or edgen panick. pub fn with_save_edgen(f: F) where F: FnOnce() + panic::UnwindSafe, @@ -170,9 +170,9 @@ where }); } -// Backup config directories) -// and start edgen before running 'f'; -// restore config, even if 'f' or edgen panick. +/// Backup config directory +/// and start edgen before running 'f'; +/// restore config, even if 'f' or edgen panick. pub fn with_save_config_edgen(f: F) where F: FnOnce() + panic::UnwindSafe, @@ -186,6 +186,11 @@ pub fn test_message(msg: &str) { println!("=== Test {}", msg); } +pub fn pass_always() { + test_message("pass always"); + assert!(true); +} + pub fn make_url(v: &[&str]) -> String { let mut s = "".to_string(); for e in v { @@ -238,11 +243,6 @@ pub fn reset_config() { edgen_server::config_reset().unwrap(); } -pub fn pass_always() { - test_message("pass always"); - assert!(true); -} - pub fn config_exists() { test_message("config exists"); assert!(settings::PROJECT_DIRS.config_dir().exists()); @@ -276,7 +276,7 @@ pub fn data_exists() { assert!(transcriptions.exists()); } -// edit the config file: set another model dir for the indicated endpoint. +/// Edit the config file: set another model dir for the indicated endpoint. pub fn set_model_dir(ep: Endpoint, model_dir: &str) { test_message(&format!("set {} model directory to {}", ep, model_dir,)); @@ -296,7 +296,8 @@ pub fn set_model_dir(ep: Endpoint, model_dir: &str) { std::thread::sleep(std::time::Duration::from_secs(4)); } -// edit the config file: set another model name and repo for the indicated endpoint. +/// Edit the config file: set another model name and repo for the indicated endpoint. +/// Use the status endpoint to check whether the model was updated. pub fn set_model(ep: Endpoint, model_name: &str, model_repo: &str) { test_message(&format!("set {} model to {}", ep, model_name,)); @@ -316,6 +317,7 @@ pub fn set_model(ep: Endpoint, model_name: &str, model_repo: &str) { println!("pausing for 4 secs to make sure the config file has been updated"); std::thread::sleep(std::time::Duration::from_secs(4)); + let url = match ep { Endpoint::ChatCompletions => make_url(&[BASE_URL, CHAT_URL, COMPLETIONS_URL, STATUS_URL]), Endpoint::AudioTranscriptions => { @@ -326,7 +328,7 @@ pub fn set_model(ep: Endpoint, model_name: &str, model_repo: &str) { assert_eq!(stat.active_model, model_name); } -// exercise the edgen version endpoint to make sure the server is reachable. +/// Exercise the edgen version endpoint to make sure the server is reachable. pub fn connect_to_server_test() { test_message("connect to server"); assert!( @@ -336,6 +338,7 @@ pub fn connect_to_server_test() { false } Ok(v) => { + assert!(v.status().is_success()); println!("have: '{}'", v.text().unwrap()); true } @@ -343,8 +346,8 @@ pub fn connect_to_server_test() { ); } -// spawn a thread to send a request to the indicated endpoint. -// This allows the caller to perform another task in the caller thread. +/// Spawn a thread to send a request to the indicated endpoint. +/// This allows the caller to perform another task in the caller thread. pub fn spawn_request(ep: Endpoint, body: String) -> thread::JoinHandle { match ep { Endpoint::ChatCompletions => spawn_chat_completions_request(body), @@ -409,7 +412,7 @@ pub fn spawn_audio_transcriptions_request() -> thread::JoinHandle { }) } -// Assert that a download is ongoing and download progress is reported. +/// Assert that a download is ongoing and download progress is reported. pub fn assert_download(endpoint: &str) { println!("requesting status of {}", endpoint); @@ -442,7 +445,7 @@ pub fn assert_download(endpoint: &str) { assert_eq!(stat.download_progress, 100); } -// Assert that *no* download is ongoing. +/// Assert that *no* download is ongoing. pub fn assert_no_download(endpoint: &str) { println!("requesting status of {}", endpoint); @@ -480,6 +483,7 @@ impl From> for BackupError { } } +// backup environment: config and data fn backup_env() -> Result<(), BackupError> { println!("backing up"); @@ -525,6 +529,7 @@ fn backup_env() -> Result<(), BackupError> { Ok(()) } +// restore environment: config and data fn restore_env() -> Result<(), io::Error> { println!("restoring"); diff --git a/crates/edgen_server/tests/settings_tests.rs b/crates/edgen_server/tests/settings_tests.rs index ab0c5ba..4dbb942 100644 --- a/crates/edgen_server/tests/settings_tests.rs +++ b/crates/edgen_server/tests/settings_tests.rs @@ -178,6 +178,7 @@ fn chat_completions_status_reachable() { false } Ok(v) => { + assert!(v.status().is_success()); println!("have: '{}'", v.text().unwrap()); true } @@ -197,6 +198,7 @@ fn audio_transcriptions_status_reachable() { false } Ok(v) => { + assert!(v.status().is_success()); println!("have: '{}'", v.text().unwrap()); true } From 3357c44c2b5ff82ada59ff63b30fc6b96cf94a55 Mon Sep 17 00:00:00 2001 From: Tobias Schoofs Date: Mon, 19 Feb 2024 10:49:35 +0000 Subject: [PATCH 11/12] [feat/modelmanager] delete_model returns DeletionStatus --- crates/edgen_server/src/model_man.rs | 22 +++++++++++++++---- .../edgen_server/tests/modelmanager_tests.rs | 9 +++++--- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/crates/edgen_server/src/model_man.rs b/crates/edgen_server/src/model_man.rs index b4f8747..4dc8ca6 100644 --- a/crates/edgen_server/src/model_man.rs +++ b/crates/edgen_server/src/model_man.rs @@ -51,11 +51,10 @@ pub async fn retrieve_model(extract::Path(id): extract::Path) -> Respons /// DELETE `/v1/models{:id}`: deletes the model indicated by 'id'. /// -/// Returns 204 (NO CONTENT) on success. /// For any error, the endpoint returns "internal server error". pub async fn delete_model(extract::Path(id): extract::Path) -> Response { match remove_model(&id).await { - Ok(()) => StatusCode::NO_CONTENT.into_response(), + Ok(d) => Json(d).into_response(), Err(e) => internal_server_error(&format!( "model manager: cannot delete model {}: {:?}", id, e @@ -81,6 +80,17 @@ pub struct ModelDesc { pub owned_by: String, } +/// Model Deletion Status +#[derive(ToSchema, Deserialize, Serialize, Debug, PartialEq, Eq)] +pub struct ModelDeletionStatus { + /// model Id + pub id: String, + /// object type, always 'model' + pub object: String, + /// repo owner + pub deleted: bool, +} + #[derive(Debug, thiserror::Error)] enum PathError { Generic(String), @@ -158,10 +168,14 @@ async fn search_model(id: &str) -> Result { Err(PathError::ModelNotFound) } -async fn remove_model(id: &str) -> Result<(), PathError> { +async fn remove_model(id: &str) -> Result { let model = search_model(id).await?; let _ = tokio::fs::remove_dir_all(model).await?; - Ok(()) + Ok(ModelDeletionStatus { + id: id.to_string(), + object: "model".to_string(), + deleted: true, + }) } async fn path_to_model_desc(path: &Path) -> Result { diff --git a/crates/edgen_server/tests/modelmanager_tests.rs b/crates/edgen_server/tests/modelmanager_tests.rs index a2bacfe..0394388 100644 --- a/crates/edgen_server/tests/modelmanager_tests.rs +++ b/crates/edgen_server/tests/modelmanager_tests.rs @@ -3,10 +3,10 @@ use std::path::{Path, PathBuf}; use std::time::SystemTime; use futures::executor::block_on; -use reqwest::{blocking, StatusCode}; +use reqwest::blocking; use edgen_core::settings; -use edgen_server::model_man::ModelDesc; +use edgen_server::model_man::{ModelDeletionStatus, ModelDesc}; #[allow(dead_code)] mod common; @@ -181,6 +181,9 @@ fn test_delete_model() { .send() .expect("models delete endpoint failed"); - assert_eq!(res.status(), StatusCode::NO_CONTENT); + assert!(res.status().is_success()); + let m: ModelDeletionStatus = res.json().expect("cannot convert to model deletion status"); + assert!(m.deleted); + assert_eq!(m.id, id); assert!(!dir.exists()); } From 3ebba177472061b4f572699c8172fff0d62aca0b Mon Sep 17 00:00:00 2001 From: Tobias Schoofs Date: Mon, 19 Feb 2024 11:02:17 +0000 Subject: [PATCH 12/12] [feat/modelmanager] simple python test for models --- tests/test_models.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100755 tests/test_models.py diff --git a/tests/test_models.py b/tests/test_models.py new file mode 100755 index 0000000..254968a --- /dev/null +++ b/tests/test_models.py @@ -0,0 +1,18 @@ + +from edgen import Edgen, APIConnectionError +from edgen.resources.misc import Version +import pytest +import subprocess + +client = Edgen() + +def test_models(): + try: + models = client.models.list() + except APIConnectionError: + pytest.fail("No connection. Is edgen running?") + + assert(type(models) is list) + +if __name__ == "__main__": + test_models()