From 3f8d00aec8d2036ef4110608204df78489c064b1 Mon Sep 17 00:00:00 2001 From: Sebastian Estrella <2049686+sestrella@users.noreply.github.com> Date: Fri, 19 Apr 2024 21:25:10 -0500 Subject: [PATCH 01/22] Enable Rust language --- devenv.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/devenv.nix b/devenv.nix index d247908..956466e 100644 --- a/devenv.nix +++ b/devenv.nix @@ -59,6 +59,8 @@ in languages.elm.enable = true; + languages.rust.enable = true; + services.opensearch.enable = !config.container.isBuilding; services.postgres.enable = !config.container.isBuilding; services.caddy.enable = true; From 70f37031a60062bb2bd0d4271ab2dd12b55bf6ee Mon Sep 17 00:00:00 2001 From: Sebastian Estrella <2049686+sestrella@users.noreply.github.com> Date: Fri, 19 Apr 2024 21:33:31 -0500 Subject: [PATCH 02/22] Initial scaffolding --- backend-rs/Cargo.toml | 8 ++++++++ backend-rs/src/main.rs | 3 +++ 2 files changed, 11 insertions(+) create mode 100644 backend-rs/Cargo.toml create mode 100644 backend-rs/src/main.rs diff --git a/backend-rs/Cargo.toml b/backend-rs/Cargo.toml new file mode 100644 index 0000000..c068025 --- /dev/null +++ b/backend-rs/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "backend-rs" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] diff --git a/backend-rs/src/main.rs b/backend-rs/src/main.rs new file mode 100644 index 0000000..e7a11a9 --- /dev/null +++ b/backend-rs/src/main.rs @@ -0,0 +1,3 @@ +fn main() { + println!("Hello, world!"); +} From 6f4ea8e148e403266b34b323dd7202dd7ac50b84 Mon Sep 17 00:00:00 2001 From: Sebastian Estrella <2049686+sestrella@users.noreply.github.com> Date: Fri, 19 Apr 2024 21:42:05 -0500 Subject: [PATCH 03/22] Install axum --- backend-rs/.gitignore | 20 ++++++++++++++++++++ backend-rs/Cargo.toml | 1 + 2 files changed, 21 insertions(+) create mode 100644 backend-rs/.gitignore diff --git a/backend-rs/.gitignore b/backend-rs/.gitignore new file mode 100644 index 0000000..389159a --- /dev/null +++ b/backend-rs/.gitignore @@ -0,0 +1,20 @@ +# Created by https://www.toptal.com/developers/gitignore/api/rust +# Edit at https://www.toptal.com/developers/gitignore?templates=rust + +### Rust ### +# Generated by Cargo +# will have compiled files and executables +debug/ +target/ + +# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries +# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html +Cargo.lock + +# These are backup files generated by rustfmt +**/*.rs.bk + +# MSVC Windows builds of rustc generate these, which store debugging information +*.pdb + +# End of https://www.toptal.com/developers/gitignore/api/rust diff --git a/backend-rs/Cargo.toml b/backend-rs/Cargo.toml index c068025..36642d0 100644 --- a/backend-rs/Cargo.toml +++ b/backend-rs/Cargo.toml @@ -6,3 +6,4 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +axum = "0.7" From ee3982eb30d9cec7f9a24c1d24c848f25404a9f9 Mon Sep 17 00:00:00 2001 From: Sebastian Estrella <2049686+sestrella@users.noreply.github.com> Date: Fri, 19 Apr 2024 21:46:29 -0500 Subject: [PATCH 04/22] Axum hello world example --- backend-rs/Cargo.toml | 1 + backend-rs/src/main.rs | 12 ++++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/backend-rs/Cargo.toml b/backend-rs/Cargo.toml index 36642d0..9593353 100644 --- a/backend-rs/Cargo.toml +++ b/backend-rs/Cargo.toml @@ -7,3 +7,4 @@ edition = "2021" [dependencies] axum = "0.7" +tokio = { version = "1.37", features = ["full"] } diff --git a/backend-rs/src/main.rs b/backend-rs/src/main.rs index e7a11a9..ff05d1c 100644 --- a/backend-rs/src/main.rs +++ b/backend-rs/src/main.rs @@ -1,3 +1,11 @@ -fn main() { - println!("Hello, world!"); +use axum::{routing::get, Router}; + +#[tokio::main] +async fn main() { + // build our application with a single route + let app = Router::new().route("/", get(|| async { "Hello, World!" })); + + // run our app with hyper, listening globally on port 3000 + let listener = tokio::net::TcpListener::bind("0.0.0.0:3000").await.unwrap(); + axum::serve(listener, app).await.unwrap(); } From d39834fde6d886010e53dbc7b122d59c5bdc032c Mon Sep 17 00:00:00 2001 From: Sebastian Estrella <2049686+sestrella@users.noreply.github.com> Date: Fri, 19 Apr 2024 22:12:45 -0500 Subject: [PATCH 05/22] Basic route setup for /api --- backend-rs/src/main.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/backend-rs/src/main.rs b/backend-rs/src/main.rs index ff05d1c..72f8b84 100644 --- a/backend-rs/src/main.rs +++ b/backend-rs/src/main.rs @@ -1,9 +1,15 @@ -use axum::{routing::get, Router}; +use axum::{ + routing::{get, post}, + Router, +}; #[tokio::main] async fn main() { // build our application with a single route - let app = Router::new().route("/", get(|| async { "Hello, World!" })); + let api = Router::new() + .route("/flake", get(|| async { "Hello, World!" })) + .route("/publish", post(|| async { "Hello, World!" })); + let app = Router::new().nest("/api", api); // run our app with hyper, listening globally on port 3000 let listener = tokio::net::TcpListener::bind("0.0.0.0:3000").await.unwrap(); From 3cced06d2cc899486d88f41585a0d9011d24a04f Mon Sep 17 00:00:00 2001 From: Sebastian Estrella <2049686+sestrella@users.noreply.github.com> Date: Sun, 21 Apr 2024 11:47:12 -0500 Subject: [PATCH 06/22] Extract handlers to separate functions --- backend-rs/src/main.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/backend-rs/src/main.rs b/backend-rs/src/main.rs index 72f8b84..85d4860 100644 --- a/backend-rs/src/main.rs +++ b/backend-rs/src/main.rs @@ -7,11 +7,19 @@ use axum::{ async fn main() { // build our application with a single route let api = Router::new() - .route("/flake", get(|| async { "Hello, World!" })) - .route("/publish", post(|| async { "Hello, World!" })); + .route("/flake", get(get_flake)) + .route("/publish", post(post_publish)); let app = Router::new().nest("/api", api); // run our app with hyper, listening globally on port 3000 let listener = tokio::net::TcpListener::bind("0.0.0.0:3000").await.unwrap(); axum::serve(listener, app).await.unwrap(); } + +async fn get_flake() -> &'static str { + "Flake" +} + +async fn post_publish() -> &'static str { + "Publish" +} From f45be4b6ba28139c5c04b963b423a489eac3ae89 Mon Sep 17 00:00:00 2001 From: Sebastian Estrella <2049686+sestrella@users.noreply.github.com> Date: Sun, 21 Apr 2024 12:00:50 -0500 Subject: [PATCH 07/22] Add AppState to store OpenSearch client --- backend-rs/Cargo.toml | 1 + backend-rs/src/main.rs | 17 +++++++++++++++-- devenv.nix | 4 ++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/backend-rs/Cargo.toml b/backend-rs/Cargo.toml index 9593353..d10fcbf 100644 --- a/backend-rs/Cargo.toml +++ b/backend-rs/Cargo.toml @@ -7,4 +7,5 @@ edition = "2021" [dependencies] axum = "0.7" +opensearch = "2.2" tokio = { version = "1.37", features = ["full"] } diff --git a/backend-rs/src/main.rs b/backend-rs/src/main.rs index 85d4860..79a556a 100644 --- a/backend-rs/src/main.rs +++ b/backend-rs/src/main.rs @@ -1,14 +1,26 @@ +use std::sync::Arc; + use axum::{ + extract::State, routing::{get, post}, Router, }; +use opensearch::OpenSearch; + +struct AppState { + opensearch: OpenSearch, +} #[tokio::main] async fn main() { // build our application with a single route + let state = Arc::new(AppState { + opensearch: OpenSearch::default(), + }); let api = Router::new() .route("/flake", get(get_flake)) - .route("/publish", post(post_publish)); + .route("/publish", post(post_publish)) + .with_state(state); let app = Router::new().nest("/api", api); // run our app with hyper, listening globally on port 3000 @@ -16,7 +28,8 @@ async fn main() { axum::serve(listener, app).await.unwrap(); } -async fn get_flake() -> &'static str { +async fn get_flake(State(state): State>) -> &'static str { + let opensearch = &state.opensearch; "Flake" } diff --git a/devenv.nix b/devenv.nix index 956466e..28cb98c 100644 --- a/devenv.nix +++ b/devenv.nix @@ -40,6 +40,10 @@ in pkgs.cloudflared pkgs.openapi-generator-cli pkgs.nodePackages.pyright + ] ++ lib.optionals pkgs.stdenv.isDarwin [ + pkgs.darwin.CF + pkgs.darwin.Security + pkgs.darwin.configd ]; # https://github.com/cachix/devenv/pull/745 From 573ebea58893282b23153b33ebc81b33f8aed189 Mon Sep 17 00:00:00 2001 From: Sebastian Estrella <2049686+sestrella@users.noreply.github.com> Date: Sun, 21 Apr 2024 16:38:15 -0500 Subject: [PATCH 08/22] Install sqlx --- backend-rs/Cargo.toml | 1 + devenv.lock | 39 +++++++++++++++++++++++++++++++++++++++ devenv.nix | 6 +++++- devenv.yaml | 5 +++++ 4 files changed, 50 insertions(+), 1 deletion(-) diff --git a/backend-rs/Cargo.toml b/backend-rs/Cargo.toml index d10fcbf..60634e9 100644 --- a/backend-rs/Cargo.toml +++ b/backend-rs/Cargo.toml @@ -8,4 +8,5 @@ edition = "2021" [dependencies] axum = "0.7" opensearch = "2.2" +sqlx = { version = "0.7", features = ["runtime-tokio", "postgres"] } tokio = { version = "1.37", features = ["full"] } diff --git a/devenv.lock b/devenv.lock index a8e423b..bb57917 100644 --- a/devenv.lock +++ b/devenv.lock @@ -17,6 +17,27 @@ "type": "github" } }, + "fenix": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ], + "rust-analyzer-src": "rust-analyzer-src" + }, + "locked": { + "lastModified": 1713680591, + "owner": "nix-community", + "repo": "fenix", + "rev": "19aaa94a73cc670a4d87e84f0909966cd8f8cd79", + "treeHash": "3f8a7cd37625d251717495bae0839ad3afd0d2b9", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "fenix", + "type": "github" + } + }, "flake-compat": { "flake": false, "locked": { @@ -182,12 +203,30 @@ "root": { "inputs": { "devenv": "devenv", + "fenix": "fenix", "mk-shell-bin": "mk-shell-bin", "nix2container": "nix2container", "nixpkgs": "nixpkgs", "pre-commit-hooks": "pre-commit-hooks" } }, + "rust-analyzer-src": { + "flake": false, + "locked": { + "lastModified": 1713628977, + "owner": "rust-lang", + "repo": "rust-analyzer", + "rev": "55d9a533b309119c8acd13061581b43ae8840823", + "treeHash": "435e9c646978f38b8389e483b4d4fb15ba932cad", + "type": "github" + }, + "original": { + "owner": "rust-lang", + "ref": "nightly", + "repo": "rust-analyzer", + "type": "github" + } + }, "systems": { "locked": { "lastModified": 1681028828, diff --git a/devenv.nix b/devenv.nix index 28cb98c..14a71ac 100644 --- a/devenv.nix +++ b/devenv.nix @@ -63,7 +63,11 @@ in languages.elm.enable = true; - languages.rust.enable = true; + languages.rust = { + enable = true; + # https://github.com/launchbadge/sqlx/blob/main/FAQ.md#what-versions-of-rust-does-sqlx-support-what-is-sqlxs-msrv + channel = "stable"; + }; services.opensearch.enable = !config.container.isBuilding; services.postgres.enable = !config.container.isBuilding; diff --git a/devenv.yaml b/devenv.yaml index 8ca8144..f9a126c 100644 --- a/devenv.yaml +++ b/devenv.yaml @@ -8,3 +8,8 @@ inputs: follows: nixpkgs mk-shell-bin: url: github:rrbutani/nix-mk-shell-bin + fenix: + url: github:nix-community/fenix + inputs: + nixpkgs: + follows: nixpkgs From 4aef8cf01e827d389c995da63d5314bb0490aa80 Mon Sep 17 00:00:00 2001 From: Sebastian Estrella <2049686+sestrella@users.noreply.github.com> Date: Sun, 21 Apr 2024 16:59:10 -0500 Subject: [PATCH 09/22] Add PgPool to AppState --- backend-rs/src/main.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/backend-rs/src/main.rs b/backend-rs/src/main.rs index 79a556a..18cafd5 100644 --- a/backend-rs/src/main.rs +++ b/backend-rs/src/main.rs @@ -6,16 +6,24 @@ use axum::{ Router, }; use opensearch::OpenSearch; +use sqlx::{postgres::PgPoolOptions, PgPool}; struct AppState { opensearch: OpenSearch, + pool: PgPool, } #[tokio::main] async fn main() { + // TODO: read PG and OS host names from env variables // build our application with a single route + let pool = PgPoolOptions::new() + .connect("postgres://localhost:5432") + .await + .unwrap(); let state = Arc::new(AppState { opensearch: OpenSearch::default(), + pool, }); let api = Router::new() .route("/flake", get(get_flake)) From 79438e15acf777493eb544a76caa4121bfd2adae Mon Sep 17 00:00:00 2001 From: Sebastian Estrella <2049686+sestrella@users.noreply.github.com> Date: Sun, 21 Apr 2024 21:35:44 -0500 Subject: [PATCH 10/22] First call to OpenSearch --- backend-rs/Cargo.toml | 2 ++ backend-rs/src/main.rs | 53 +++++++++++++++++++++++++++++++++++++----- 2 files changed, 49 insertions(+), 6 deletions(-) diff --git a/backend-rs/Cargo.toml b/backend-rs/Cargo.toml index 60634e9..0f68e9a 100644 --- a/backend-rs/Cargo.toml +++ b/backend-rs/Cargo.toml @@ -8,5 +8,7 @@ edition = "2021" [dependencies] axum = "0.7" opensearch = "2.2" +serde = "1.0" +serde_json = "1.0" sqlx = { version = "0.7", features = ["runtime-tokio", "postgres"] } tokio = { version = "1.37", features = ["full"] } diff --git a/backend-rs/src/main.rs b/backend-rs/src/main.rs index 18cafd5..c140ddd 100644 --- a/backend-rs/src/main.rs +++ b/backend-rs/src/main.rs @@ -1,11 +1,13 @@ -use std::sync::Arc; +use std::{collections::HashMap, sync::Arc}; use axum::{ - extract::State, + extract::{Query, State}, + response::{ErrorResponse, IntoResponse}, routing::{get, post}, Router, }; -use opensearch::OpenSearch; +use opensearch::{OpenSearch, SearchParts}; +use serde_json::json; use sqlx::{postgres::PgPoolOptions, PgPool}; struct AppState { @@ -13,6 +15,20 @@ struct AppState { pool: PgPool, } +enum AppError {} + +impl IntoResponse for AppError { + fn into_response(self) -> axum::response::Response { + todo!() + } +} + +impl From for AppError { + fn from(value: opensearch::Error) -> Self { + todo!() + } +} + #[tokio::main] async fn main() { // TODO: read PG and OS host names from env variables @@ -36,9 +52,34 @@ async fn main() { axum::serve(listener, app).await.unwrap(); } -async fn get_flake(State(state): State>) -> &'static str { - let opensearch = &state.opensearch; - "Flake" +async fn get_flake( + State(state): State>, + Query(params): Query>, +) -> Result<(), AppError> { + if let Some(q) = params.get("q") { + let response = &state + .opensearch + .search(SearchParts::Index(&["opensearch_index"])) + .size(10) + .body(json!({ + "query": { + "multi_match": { + "query": q, + "fuzziness": "AUTO", + "fields": [ + "description^2", + "readme", + "outputs", + "repo^2", + "owner^2", + ], + } + } + })) + .send() + .await?; + } + Ok(()) } async fn post_publish() -> &'static str { From 6ce40789bc50d9c228c81c930fbdcba6d7044d86 Mon Sep 17 00:00:00 2001 From: Sebastian Estrella <2049686+sestrella@users.noreply.github.com> Date: Sun, 21 Apr 2024 21:46:51 -0500 Subject: [PATCH 11/22] Extract hits from OpenSearch response --- backend-rs/src/main.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/backend-rs/src/main.rs b/backend-rs/src/main.rs index c140ddd..683346a 100644 --- a/backend-rs/src/main.rs +++ b/backend-rs/src/main.rs @@ -2,12 +2,12 @@ use std::{collections::HashMap, sync::Arc}; use axum::{ extract::{Query, State}, - response::{ErrorResponse, IntoResponse}, + response::IntoResponse, routing::{get, post}, Router, }; use opensearch::{OpenSearch, SearchParts}; -use serde_json::json; +use serde_json::{json, Value}; use sqlx::{postgres::PgPoolOptions, PgPool}; struct AppState { @@ -77,7 +77,16 @@ async fn get_flake( } })) .send() + .await? + .json::() .await?; + let hits = response["hits"]["hits"] + .as_array() + // TODO: Remove this unwrap + .unwrap() + .into_iter() + // TODO: Transform _id to int + .map(|hit| (&hit["_id"], &hit["_score"])); } Ok(()) } From fdd03244a210db9dcf424febb31fad22e95ecde6 Mon Sep 17 00:00:00 2001 From: Sebastian Estrella <2049686+sestrella@users.noreply.github.com> Date: Sun, 21 Apr 2024 21:49:12 -0500 Subject: [PATCH 12/22] Wrap OpenSearch errors inside AppError --- backend-rs/src/main.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/backend-rs/src/main.rs b/backend-rs/src/main.rs index 683346a..0adca49 100644 --- a/backend-rs/src/main.rs +++ b/backend-rs/src/main.rs @@ -15,7 +15,9 @@ struct AppState { pool: PgPool, } -enum AppError {} +enum AppError { + OpenSearchError(opensearch::Error), +} impl IntoResponse for AppError { fn into_response(self) -> axum::response::Response { @@ -25,7 +27,7 @@ impl IntoResponse for AppError { impl From for AppError { fn from(value: opensearch::Error) -> Self { - todo!() + AppError::OpenSearchError(value) } } From 62850dd322fe3c5082c2f779de9da880017b14d7 Mon Sep 17 00:00:00 2001 From: Sebastian Estrella <2049686+sestrella@users.noreply.github.com> Date: Sun, 21 Apr 2024 22:27:27 -0500 Subject: [PATCH 13/22] Add release struct query DB using data from OS --- backend-rs/src/main.rs | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/backend-rs/src/main.rs b/backend-rs/src/main.rs index 0adca49..f8da705 100644 --- a/backend-rs/src/main.rs +++ b/backend-rs/src/main.rs @@ -31,6 +31,12 @@ impl From for AppError { } } +#[derive(sqlx::FromRow)] +struct Release { + id: i64, + readme: String, +} + #[tokio::main] async fn main() { // TODO: read PG and OS host names from env variables @@ -82,13 +88,15 @@ async fn get_flake( .await? .json::() .await?; - let hits = response["hits"]["hits"] - .as_array() - // TODO: Remove this unwrap - .unwrap() - .into_iter() - // TODO: Transform _id to int - .map(|hit| (&hit["_id"], &hit["_score"])); + // TODO: Remove this unwrap, use fold or map to create the HashMap + let mut hits: HashMap = HashMap::new(); + for hit in response["hits"]["hits"].as_array().unwrap() { + hits.insert(hit["_id"].to_string(), hit["_score"].to_string()); + } + sqlx::query_as::<_, Release>("SELECT * FROM release WHERE id IN (?)") + .bind(hits.keys().cloned().collect::>()) + .fetch_all(&state.pool) + .await; } Ok(()) } From bb00fcc08007ae41fdcb08d01d732d1df86964e1 Mon Sep 17 00:00:00 2001 From: Sebastian Estrella <2049686+sestrella@users.noreply.github.com> Date: Sun, 21 Apr 2024 23:09:55 -0500 Subject: [PATCH 14/22] Return releases sorted by score --- backend-rs/src/main.rs | 43 +++++++++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/backend-rs/src/main.rs b/backend-rs/src/main.rs index f8da705..3c303dd 100644 --- a/backend-rs/src/main.rs +++ b/backend-rs/src/main.rs @@ -4,7 +4,7 @@ use axum::{ extract::{Query, State}, response::IntoResponse, routing::{get, post}, - Router, + Json, Router, }; use opensearch::{OpenSearch, SearchParts}; use serde_json::{json, Value}; @@ -17,12 +17,7 @@ struct AppState { enum AppError { OpenSearchError(opensearch::Error), -} - -impl IntoResponse for AppError { - fn into_response(self) -> axum::response::Response { - todo!() - } + SqlxError(sqlx::Error), } impl From for AppError { @@ -31,7 +26,19 @@ impl From for AppError { } } -#[derive(sqlx::FromRow)] +impl From for AppError { + fn from(value: sqlx::Error) -> Self { + AppError::SqlxError(value) + } +} + +impl IntoResponse for AppError { + fn into_response(self) -> axum::response::Response { + todo!() + } +} + +#[derive(sqlx::FromRow, serde::Serialize)] struct Release { id: i64, readme: String, @@ -63,7 +70,7 @@ async fn main() { async fn get_flake( State(state): State>, Query(params): Query>, -) -> Result<(), AppError> { +) -> Result>, AppError> { if let Some(q) = params.get("q") { let response = &state .opensearch @@ -89,16 +96,22 @@ async fn get_flake( .json::() .await?; // TODO: Remove this unwrap, use fold or map to create the HashMap - let mut hits: HashMap = HashMap::new(); + let mut hits: HashMap = HashMap::new(); for hit in response["hits"]["hits"].as_array().unwrap() { - hits.insert(hit["_id"].to_string(), hit["_score"].to_string()); + // TODO: properly handle errors + hits.insert( + hit["_id"].as_i64().unwrap(), + hit["_score"].as_i64().unwrap(), + ); } - sqlx::query_as::<_, Release>("SELECT * FROM release WHERE id IN (?)") - .bind(hits.keys().cloned().collect::>()) + let mut releases = sqlx::query_as::<_, Release>("SELECT * FROM release WHERE id IN (?)") + .bind(hits.keys().cloned().collect::>()) .fetch_all(&state.pool) - .await; + .await?; + releases.sort_by(|a, b| hits[&b.id].cmp(&hits[&a.id])); + return Ok(Json(releases)); } - Ok(()) + todo!() } async fn post_publish() -> &'static str { From 35f926eb3da0e9e47d6ad8e1d51fca34d36b8cb9 Mon Sep 17 00:00:00 2001 From: Sebastian Estrella <2049686+sestrella@users.noreply.github.com> Date: Sun, 21 Apr 2024 23:25:43 -0500 Subject: [PATCH 15/22] Return releases orderd by created_at when q param is not present --- backend-rs/src/main.rs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/backend-rs/src/main.rs b/backend-rs/src/main.rs index 3c303dd..786c3a3 100644 --- a/backend-rs/src/main.rs +++ b/backend-rs/src/main.rs @@ -71,7 +71,7 @@ async fn get_flake( State(state): State>, Query(params): Query>, ) -> Result>, AppError> { - if let Some(q) = params.get("q") { + let releases = if let Some(q) = params.get("q") { let response = &state .opensearch .search(SearchParts::Index(&["opensearch_index"])) @@ -104,14 +104,19 @@ async fn get_flake( hit["_score"].as_i64().unwrap(), ); } + // TODO: This query is actually a join between different tables let mut releases = sqlx::query_as::<_, Release>("SELECT * FROM release WHERE id IN (?)") .bind(hits.keys().cloned().collect::>()) .fetch_all(&state.pool) .await?; releases.sort_by(|a, b| hits[&b.id].cmp(&hits[&a.id])); - return Ok(Json(releases)); - } - todo!() + releases + } else { + sqlx::query_as::<_, Release>("SELECT * FROM release ORDER BY created_at LIMIT 100") + .fetch_all(&state.pool) + .await? + }; + return Ok(Json(releases)); } async fn post_publish() -> &'static str { From ae3c59ec894a7ff5c654280407e331ab37053755 Mon Sep 17 00:00:00 2001 From: Sebastian Estrella <2049686+sestrella@users.noreply.github.com> Date: Sun, 21 Apr 2024 23:45:09 -0500 Subject: [PATCH 16/22] Add struct to match Python response --- backend-rs/src/main.rs | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/backend-rs/src/main.rs b/backend-rs/src/main.rs index 786c3a3..33ef016 100644 --- a/backend-rs/src/main.rs +++ b/backend-rs/src/main.rs @@ -38,7 +38,14 @@ impl IntoResponse for AppError { } } -#[derive(sqlx::FromRow, serde::Serialize)] +#[derive(serde::Serialize)] +struct GetFlakeResponse { + releases: Vec, + count: usize, + query: Option, +} + +#[derive(serde::Serialize, sqlx::FromRow)] struct Release { id: i64, readme: String, @@ -70,8 +77,9 @@ async fn main() { async fn get_flake( State(state): State>, Query(params): Query>, -) -> Result>, AppError> { - let releases = if let Some(q) = params.get("q") { +) -> Result, AppError> { + let query = params.get("q"); + let releases = if let Some(q) = query { let response = &state .opensearch .search(SearchParts::Index(&["opensearch_index"])) @@ -116,7 +124,13 @@ async fn get_flake( .fetch_all(&state.pool) .await? }; - return Ok(Json(releases)); + let count = releases.len(); + return Ok(Json(GetFlakeResponse { + releases, + count, + // TODO: Try to avoid using cloned() + query: query.cloned(), + })); } async fn post_publish() -> &'static str { From e165967cb6958fa6ed49e8beea241da35d379127 Mon Sep 17 00:00:00 2001 From: Sebastian Estrella <2049686+sestrella@users.noreply.github.com> Date: Mon, 22 Apr 2024 06:27:43 -0500 Subject: [PATCH 17/22] Update query and change return data type for get_flake --- backend-rs/src/main.rs | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/backend-rs/src/main.rs b/backend-rs/src/main.rs index 33ef016..c635405 100644 --- a/backend-rs/src/main.rs +++ b/backend-rs/src/main.rs @@ -40,15 +40,21 @@ impl IntoResponse for AppError { #[derive(serde::Serialize)] struct GetFlakeResponse { - releases: Vec, + releases: Vec, count: usize, query: Option, } #[derive(serde::Serialize, sqlx::FromRow)] -struct Release { +struct FlakeRelease { + #[serde(skip_serializing)] id: i64, - readme: String, + owner: String, + repo: String, + version: String, + description: String, + // TODO: Change to DateTime? + created_at: String, } #[tokio::main] @@ -113,14 +119,25 @@ async fn get_flake( ); } // TODO: This query is actually a join between different tables - let mut releases = sqlx::query_as::<_, Release>("SELECT * FROM release WHERE id IN (?)") - .bind(hits.keys().cloned().collect::>()) - .fetch_all(&state.pool) - .await?; + let mut releases = sqlx::query_as::<_, FlakeRelease>( + "SELECT release.id AS id, \ + githubowner.name AS owner, \ + githubrepo.name AS repo, \ + release.version AS version, \ + release.description AS description, \ + release.created_at AS created_at \ + FROM release \ + INNER JOIN githubrepo ON githubrepo.id = release.repo_id \ + INNER JOIN githubowner ON githubowner.id = githubrepo.owner_id \ + WHERE release.id IN (?)", + ) + .bind(hits.keys().cloned().collect::>()) + .fetch_all(&state.pool) + .await?; releases.sort_by(|a, b| hits[&b.id].cmp(&hits[&a.id])); releases } else { - sqlx::query_as::<_, Release>("SELECT * FROM release ORDER BY created_at LIMIT 100") + sqlx::query_as::<_, FlakeRelease>("SELECT * FROM release ORDER BY created_at LIMIT 100") .fetch_all(&state.pool) .await? }; From fc325093749d3404196aac7f5efce74580121f03 Mon Sep 17 00:00:00 2001 From: Sebastian Estrella <2049686+sestrella@users.noreply.github.com> Date: Mon, 22 Apr 2024 06:29:17 -0500 Subject: [PATCH 18/22] Add TODO note --- backend-rs/src/main.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/backend-rs/src/main.rs b/backend-rs/src/main.rs index c635405..0a68cd3 100644 --- a/backend-rs/src/main.rs +++ b/backend-rs/src/main.rs @@ -137,6 +137,7 @@ async fn get_flake( releases.sort_by(|a, b| hits[&b.id].cmp(&hits[&a.id])); releases } else { + // TODO: Update this query sqlx::query_as::<_, FlakeRelease>("SELECT * FROM release ORDER BY created_at LIMIT 100") .fetch_all(&state.pool) .await? From 81f3e4656311bf11730f35b11fdb96c0e7fdc99c Mon Sep 17 00:00:00 2001 From: Sebastian Estrella <2049686+sestrella@users.noreply.github.com> Date: Tue, 23 Apr 2024 05:56:06 -0500 Subject: [PATCH 19/22] Update release query --- backend-rs/src/main.rs | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/backend-rs/src/main.rs b/backend-rs/src/main.rs index 0a68cd3..4fc56f2 100644 --- a/backend-rs/src/main.rs +++ b/backend-rs/src/main.rs @@ -137,8 +137,19 @@ async fn get_flake( releases.sort_by(|a, b| hits[&b.id].cmp(&hits[&a.id])); releases } else { - // TODO: Update this query - sqlx::query_as::<_, FlakeRelease>("SELECT * FROM release ORDER BY created_at LIMIT 100") + sqlx::query_as::<_, FlakeRelease>( + "SELECT release.id AS id, \ + githubowner.name AS owner, \ + githubrepo.name AS repo, \ + release.version AS version, \ + release.description AS description, \ + release.created_at AS created_at \ + FROM release \ + INNER JOIN githubrepo ON githubrepo.id = release.repo_id \ + INNER JOIN githubowner ON githubowner.id = githubrepo.owner_id \ + ORDER BY release.created_at LIMIT 100" + + ) .fetch_all(&state.pool) .await? }; From 7b43c34bba677b9b59234112901a510a0eb6adc4 Mon Sep 17 00:00:00 2001 From: Sebastian Estrella <2049686+sestrella@users.noreply.github.com> Date: Mon, 6 May 2024 01:27:45 -0500 Subject: [PATCH 20/22] First tests --- backend-rs/Cargo.toml | 4 ++ backend-rs/src/main.rs | 101 +++++++++++++++++++++++++++++++++++------ devenv.lock | 11 +++-- devenv.nix | 1 + devenv.yaml | 2 +- 5 files changed, 98 insertions(+), 21 deletions(-) diff --git a/backend-rs/Cargo.toml b/backend-rs/Cargo.toml index 0f68e9a..d6da7f7 100644 --- a/backend-rs/Cargo.toml +++ b/backend-rs/Cargo.toml @@ -12,3 +12,7 @@ serde = "1.0" serde_json = "1.0" sqlx = { version = "0.7", features = ["runtime-tokio", "postgres"] } tokio = { version = "1.37", features = ["full"] } + +[dev-dependencies] +http-body-util = "0.1" +tower = "0.4" diff --git a/backend-rs/src/main.rs b/backend-rs/src/main.rs index 4fc56f2..188d788 100644 --- a/backend-rs/src/main.rs +++ b/backend-rs/src/main.rs @@ -2,11 +2,12 @@ use std::{collections::HashMap, sync::Arc}; use axum::{ extract::{Query, State}, + http::StatusCode, response::IntoResponse, routing::{get, post}, Json, Router, }; -use opensearch::{OpenSearch, SearchParts}; +use opensearch::{indices::IndicesCreateParts, OpenSearch, SearchParts}; use serde_json::{json, Value}; use sqlx::{postgres::PgPoolOptions, PgPool}; @@ -34,7 +35,11 @@ impl From for AppError { impl IntoResponse for AppError { fn into_response(self) -> axum::response::Response { - todo!() + let body = match self { + AppError::OpenSearchError(error) => error.to_string(), + AppError::SqlxError(error) => error.to_string(), + }; + (StatusCode::INTERNAL_SERVER_ERROR, Json(body)).into_response() } } @@ -69,15 +74,23 @@ async fn main() { opensearch: OpenSearch::default(), pool, }); - let api = Router::new() - .route("/flake", get(get_flake)) - .route("/publish", post(post_publish)) - .with_state(state); - let app = Router::new().nest("/api", api); - + // TODO: check if index exist before creating one + let _ = state + .opensearch + .indices() + .create(IndicesCreateParts::Index("flakes")) + .send() + .await; // run our app with hyper, listening globally on port 3000 let listener = tokio::net::TcpListener::bind("0.0.0.0:3000").await.unwrap(); - axum::serve(listener, app).await.unwrap(); + axum::serve(listener, app(state)).await.unwrap(); +} + +fn app(state: Arc) -> Router { + let api = Router::new() + .route("/flake", get(get_flake)) + .route("/publish", post(post_publish)); + Router::new().nest("/api", api).with_state(state) } async fn get_flake( @@ -88,7 +101,7 @@ async fn get_flake( let releases = if let Some(q) = query { let response = &state .opensearch - .search(SearchParts::Index(&["opensearch_index"])) + .search(SearchParts::Index(&["flakes"])) .size(10) .body(json!({ "query": { @@ -129,7 +142,7 @@ async fn get_flake( FROM release \ INNER JOIN githubrepo ON githubrepo.id = release.repo_id \ INNER JOIN githubowner ON githubowner.id = githubrepo.owner_id \ - WHERE release.id IN (?)", + WHERE release.id IN ($1)", ) .bind(hits.keys().cloned().collect::>()) .fetch_all(&state.pool) @@ -147,11 +160,10 @@ async fn get_flake( FROM release \ INNER JOIN githubrepo ON githubrepo.id = release.repo_id \ INNER JOIN githubowner ON githubowner.id = githubrepo.owner_id \ - ORDER BY release.created_at LIMIT 100" - + ORDER BY release.created_at DESC LIMIT 100", ) - .fetch_all(&state.pool) - .await? + .fetch_all(&state.pool) + .await? }; let count = releases.len(); return Ok(Json(GetFlakeResponse { @@ -165,3 +177,62 @@ async fn get_flake( async fn post_publish() -> &'static str { "Publish" } + +#[cfg(test)] +mod tests { + use std::env; + + use super::*; + use axum::body::Body; + use axum::http::{Request, StatusCode}; + use http_body_util::BodyExt; + use sqlx::postgres::PgConnectOptions; + use tower::ServiceExt; + + #[tokio::test] + async fn test_get_flake_with_params() { + let host = env::var("PGHOST").unwrap().to_string(); + let opts = PgConnectOptions::new().host(&host); + let pool = PgPoolOptions::new().connect_with(opts).await.unwrap(); + let state = Arc::new(AppState { + opensearch: OpenSearch::default(), + pool, + }); + let app = app(state); + let response = app + .oneshot( + Request::builder() + .uri("/api/flake?q=search") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let body = response.into_body().collect().await.unwrap().to_bytes(); + let body: Value = serde_json::from_slice(&body).unwrap(); + println!("#{body}"); + // assert_eq!(response.status(), StatusCode::OK); + } + + #[tokio::test] + async fn test_get_flake_without_params() { + let host = env::var("PGHOST").unwrap().to_string(); + let opts = PgConnectOptions::new().host(&host); + let pool = PgPoolOptions::new().connect_with(opts).await.unwrap(); + let state = Arc::new(AppState { + opensearch: OpenSearch::default(), + pool, + }); + let app = app(state); + let response = app + .oneshot( + Request::builder() + .uri("/api/flake") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + } +} diff --git a/devenv.lock b/devenv.lock index bb57917..938e814 100644 --- a/devenv.lock +++ b/devenv.lock @@ -25,15 +25,16 @@ "rust-analyzer-src": "rust-analyzer-src" }, "locked": { - "lastModified": 1713680591, - "owner": "nix-community", + "lastModified": 1714397108, + "owner": "sandydoo", "repo": "fenix", - "rev": "19aaa94a73cc670a4d87e84f0909966cd8f8cd79", - "treeHash": "3f8a7cd37625d251717495bae0839ad3afd0d2b9", + "rev": "111b2aea9d2ac261c6f0a4aac4bde6c0e8c5bb3d", + "treeHash": "1bd45fad5c728d297601fbfc51771f56488e2beb", "type": "github" }, "original": { - "owner": "nix-community", + "owner": "sandydoo", + "ref": "patch-rust-analyzer-preview", "repo": "fenix", "type": "github" } diff --git a/devenv.nix b/devenv.nix index 14a71ac..1d0d219 100644 --- a/devenv.nix +++ b/devenv.nix @@ -44,6 +44,7 @@ in pkgs.darwin.CF pkgs.darwin.Security pkgs.darwin.configd + pkgs.darwin.dyld ]; # https://github.com/cachix/devenv/pull/745 diff --git a/devenv.yaml b/devenv.yaml index f9a126c..f652952 100644 --- a/devenv.yaml +++ b/devenv.yaml @@ -9,7 +9,7 @@ inputs: mk-shell-bin: url: github:rrbutani/nix-mk-shell-bin fenix: - url: github:nix-community/fenix + url: github:sandydoo/fenix/patch-rust-analyzer-preview inputs: nixpkgs: follows: nixpkgs From 1451c7d7fef1bd45cf3467c234b57e371e41efed Mon Sep 17 00:00:00 2001 From: Sebastian Estrella <2049686+sestrella@users.noreply.github.com> Date: Mon, 10 Jun 2024 15:35:59 -0500 Subject: [PATCH 21/22] Read PgPool connection string from env variable --- backend-rs/.gitignore | 3 +++ backend-rs/Cargo.toml | 1 + backend-rs/src/main.rs | 11 +++++------ 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/backend-rs/.gitignore b/backend-rs/.gitignore index 389159a..976e7ba 100644 --- a/backend-rs/.gitignore +++ b/backend-rs/.gitignore @@ -18,3 +18,6 @@ Cargo.lock *.pdb # End of https://www.toptal.com/developers/gitignore/api/rust + +# dotenv +.env diff --git a/backend-rs/Cargo.toml b/backend-rs/Cargo.toml index d6da7f7..a8a1663 100644 --- a/backend-rs/Cargo.toml +++ b/backend-rs/Cargo.toml @@ -7,6 +7,7 @@ edition = "2021" [dependencies] axum = "0.7" +dotenv = "0.15.0" opensearch = "2.2" serde = "1.0" serde_json = "1.0" diff --git a/backend-rs/src/main.rs b/backend-rs/src/main.rs index 188d788..4940d68 100644 --- a/backend-rs/src/main.rs +++ b/backend-rs/src/main.rs @@ -1,4 +1,4 @@ -use std::{collections::HashMap, sync::Arc}; +use std::{collections::HashMap, env, sync::Arc}; use axum::{ extract::{Query, State}, @@ -9,7 +9,7 @@ use axum::{ }; use opensearch::{indices::IndicesCreateParts, OpenSearch, SearchParts}; use serde_json::{json, Value}; -use sqlx::{postgres::PgPoolOptions, PgPool}; +use sqlx::postgres::{PgPool, PgPoolOptions}; struct AppState { opensearch: OpenSearch, @@ -66,10 +66,9 @@ struct FlakeRelease { async fn main() { // TODO: read PG and OS host names from env variables // build our application with a single route - let pool = PgPoolOptions::new() - .connect("postgres://localhost:5432") - .await - .unwrap(); + dotenv::dotenv().ok(); + let database_url = env::var("DATABASE_URL").unwrap(); + let pool = PgPoolOptions::new().connect(&database_url).await.unwrap(); let state = Arc::new(AppState { opensearch: OpenSearch::default(), pool, From 033dfe425e616fc771bcf1f518cd195965e730b8 Mon Sep 17 00:00:00 2001 From: Sebastian Estrella <2049686+sestrella@users.noreply.github.com> Date: Mon, 10 Jun 2024 16:04:49 -0500 Subject: [PATCH 22/22] Setup tracing --- backend-rs/Cargo.toml | 7 +++++-- backend-rs/src/main.rs | 12 +++++++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/backend-rs/Cargo.toml b/backend-rs/Cargo.toml index a8a1663..aaf25f2 100644 --- a/backend-rs/Cargo.toml +++ b/backend-rs/Cargo.toml @@ -6,13 +6,16 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -axum = "0.7" -dotenv = "0.15.0" +axum = { version = "0.7", features = ["tracing"] } +dotenv = "0.15" opensearch = "2.2" serde = "1.0" serde_json = "1.0" sqlx = { version = "0.7", features = ["runtime-tokio", "postgres"] } tokio = { version = "1.37", features = ["full"] } +tower-http = { version = "0.5", features = ["trace"] } +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } [dev-dependencies] http-body-util = "0.1" diff --git a/backend-rs/src/main.rs b/backend-rs/src/main.rs index 4940d68..730c759 100644 --- a/backend-rs/src/main.rs +++ b/backend-rs/src/main.rs @@ -10,6 +10,9 @@ use axum::{ use opensearch::{indices::IndicesCreateParts, OpenSearch, SearchParts}; use serde_json::{json, Value}; use sqlx::postgres::{PgPool, PgPoolOptions}; +use tower_http::trace::TraceLayer; +use tracing_subscriber::{fmt, EnvFilter}; +use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; struct AppState { opensearch: OpenSearch, @@ -67,6 +70,10 @@ async fn main() { // TODO: read PG and OS host names from env variables // build our application with a single route dotenv::dotenv().ok(); + tracing_subscriber::registry() + .with(fmt::layer()) + .with(EnvFilter::from_default_env()) + .init(); let database_url = env::var("DATABASE_URL").unwrap(); let pool = PgPoolOptions::new().connect(&database_url).await.unwrap(); let state = Arc::new(AppState { @@ -89,7 +96,10 @@ fn app(state: Arc) -> Router { let api = Router::new() .route("/flake", get(get_flake)) .route("/publish", post(post_publish)); - Router::new().nest("/api", api).with_state(state) + Router::new() + .nest("/api", api) + .layer(TraceLayer::new_for_http()) + .with_state(state) } async fn get_flake(