From 02fdaacf6b9e0b671ecbbab7345329aeaec0735c Mon Sep 17 00:00:00 2001 From: luc Date: Sun, 13 Oct 2024 22:26:07 -0400 Subject: [PATCH] feat: Integrate process-compose and services-flake for improved service management - Add process-compose-flake and services-flake as new inputs - Implement a new process-compose configuration for managing services - Integrate Ollama LLM service with configurable models - Refactor PostgreSQL setup to use services-flake - Add a new topos-service.nix for defining the Topos service - Update flake.nix to use the new service configurations - Modify .env_dev to use 127.0.0.1 instead of localhost for POSTGRES_HOST - Simplify devShells configuration - Remove manual setup hooks for PostgreSQL and Kafka These changes aim to provide a more robust and manageable service configuration, leveraging process-compose for better service orchestration and dependency management. --- .env_dev | 2 +- flake.lock | 34 +++- flake.nix | 442 +++++++++++++++++++++++----------------------- topos-service.nix | 28 +++ 4 files changed, 280 insertions(+), 226 deletions(-) create mode 100644 topos-service.nix diff --git a/.env_dev b/.env_dev index c1268cb..75ae375 100644 --- a/.env_dev +++ b/.env_dev @@ -11,5 +11,5 @@ SUPABASE_KEY= POSTGRES_DB=test_topos_db POSTGRES_USER=username POSTGRES_PASSWORD=your_password -POSTGRES_HOST=localhost +POSTGRES_HOST=127.0.0.1 POSTGRES_PORT=5432 diff --git a/flake.lock b/flake.lock index 1ab5a19..9567a9d 100644 --- a/flake.lock +++ b/flake.lock @@ -109,11 +109,43 @@ "type": "github" } }, + "process-compose-flake": { + "locked": { + "lastModified": 1728868941, + "narHash": "sha256-yEMzxZfy+EE9gSqn++SyZeAVHXYupFT8Wyf99Z/CXXU=", + "owner": "Platonic-Systems", + "repo": "process-compose-flake", + "rev": "29301aec92d73c9b075fcfd06a6fb18665bfe6b5", + "type": "github" + }, + "original": { + "owner": "Platonic-Systems", + "repo": "process-compose-flake", + "type": "github" + } + }, "root": { "inputs": { "flake-parts": "flake-parts", "nixpkgs": "nixpkgs", - "poetry2nix": "poetry2nix" + "poetry2nix": "poetry2nix", + "process-compose-flake": "process-compose-flake", + "services-flake": "services-flake" + } + }, + "services-flake": { + "locked": { + "lastModified": 1728811751, + "narHash": "sha256-IrwycNtt6jxJGCi+QJ8Bbzt9flg0vNeGLAR0KBbj4a8=", + "owner": "juspay", + "repo": "services-flake", + "rev": "e9f663036f3b1b1a12b0f136628ef93a8be92443", + "type": "github" + }, + "original": { + "owner": "juspay", + "repo": "services-flake", + "type": "github" } }, "systems": { diff --git a/flake.nix b/flake.nix index c970d8f..543c318 100644 --- a/flake.nix +++ b/flake.nix @@ -8,251 +8,245 @@ url = "github:nix-community/poetry2nix"; inputs.nixpkgs.follows = "nixpkgs"; }; + process-compose-flake.url = "github:Platonic-Systems/process-compose-flake"; + services-flake.url = "github:juspay/services-flake"; }; - outputs = { self, nixpkgs, flake-parts, poetry2nix }@inputs: + outputs = { self, nixpkgs, flake-parts, poetry2nix, process-compose-flake, services-flake }@inputs: flake-parts.lib.mkFlake { inherit inputs; } { - imports = [ ]; - systems = [ "x86_64-linux" "aarch64-darwin" ]; - perSystem = { pkgs, system, ... }: - let - pkgs = import nixpkgs { - inherit system; - overlays = [ - inputs.poetry2nix.overlays.default - (final: prev: { - toposPoetryEnv = final.callPackage toposPoetryEnv { }; - pythonPackagesExtensions = prev.pythonPackagesExtensions ++ [ - (python-final: python-prev: { - pystray = python-final.callPackage ./overlays/pystray/default.nix { }; - }) + imports = [ inputs.process-compose-flake.flakeModule ]; + systems = [ "x86_64-linux" "aarch64-darwin" ]; + perSystem = { self', pkgs, system, lib, ... }: + let + pkgs = import nixpkgs { + inherit system; + overlays = [ + inputs.poetry2nix.overlays.default + (final: prev: { + toposPoetryEnv = final.callPackage toposPoetryEnv { }; + pythonPackagesExtensions = prev.pythonPackagesExtensions ++ [ + (python-final: python-prev: { + pystray = python-final.callPackage ./overlays/pystray/default.nix { }; + }) + ]; + }) ]; - }) - ]; - }; - # see https://github.com/nix-community/poetry2nix/tree/master#api for more functions and examples. - #TODO: Figure out how to add setuptools to all the packages which need it, this is currently not working as expected. - overrides = pkgs.poetry2nix.overrides.withDefaults (final: super: - pkgs.lib.mapAttrs - (attr: systems: super.${attr}.overridePythonAttrs - (old: { - nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ map (a: final.${a}) systems; - })) - { - # https://github.com/nix-community/poetry2nix/blob/master/docs/edgecases.md#modulenotfounderror-no-module-named-packagename - package = [ "setuptools" ]; - } - ); - toposPoetryEnv = pkgs.poetry2nix.mkPoetryEnv { - projectDir = self; - preferWheels = true; - inherit overrides; - }; - - envFile = pkgs.writeText "env_dev" (builtins.readFile ./.env_dev); - configFile = pkgs.copyPathToStore ./config.yaml; - yq = pkgs.yq-go; - - # Note: This only loads the settings from the repos config file - # if one is not already set in the user's .config directory. - toposSetupHook = '' - export TOPOS_CONFIG_PATH="$HOME/.config/topos/config.yaml" - mkdir -p "$(dirname "$TOPOS_CONFIG_PATH")" - if [ ! -f "$TOPOS_CONFIG_PATH" ]; then - echo "Creating new config file at $TOPOS_CONFIG_PATH" - echo "# Topos Configuration" > "$TOPOS_CONFIG_PATH" - ${yq}/bin/yq eval ${configFile} | while IFS= read -r line; do - echo "$line" >> "$TOPOS_CONFIG_PATH" - done - echo "Config file created at $TOPOS_CONFIG_PATH" - else - echo "Config file already exists at $TOPOS_CONFIG_PATH" - fi - ''; - - postgresSetupHook = '' - # SETUP POSTGRES SERVER - echo "Loading environment variables from Nix store" - export $(cat ${envFile} | xargs) - - # Define PGDATA and LOGFILE based on environment variables - export PGDATA=$(pwd)/pgdata - LOGFILE=$PGDATA/postgresql.log - - echo "Initializing PostgreSQL data directory at $PGDATA" - - echo "PGDATA: $PGDATA" - if [ ! -d "$PGDATA" ]; then - mkdir -p "$PGDATA" - initdb -D "$PGDATA" - fi - - echo "Stopping any existing PostgreSQL server..." - pg_ctl -D "$PGDATA" stop -s -m fast || echo "No existing server to stop." + }; - echo "Starting PostgreSQL server..." - pg_ctl -D "$PGDATA" -l $LOGFILE start -w + # see https://github.com/nix-community/poetry2nix/tree/master#api for more functions and examples. + #TODO: Figure out how to add setuptools to all the packages which need it, this is currently not working as expected. + overrides = pkgs.poetry2nix.overrides.withDefaults (final: super: + pkgs.lib.mapAttrs + (attr: systems: super.${attr}.overridePythonAttrs + (old: { + nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ map (a: final.${a}) systems; + })) + { + # https://github.com/nix-community/poetry2nix/blob/master/docs/edgecases.md#modulenotfounderror-no-module-named-packagename + package = [ "setuptools" ]; + } + ); + toposPoetryEnv = pkgs.poetry2nix.mkPoetryEnv { + projectDir = self; + preferWheels = true; + inherit overrides; + }; - # Wait for PostgreSQL to start - for i in {1..10}; do - if pg_isready -q; then - break + envFile = pkgs.writeText "env_dev" (builtins.readFile ./.env_dev); + parseEnvFile = envFile: + let + content = builtins.readFile envFile; + lines = lib.filter (l: l != "" && !lib.hasPrefix "#" l) (lib.splitString "\n" content); + parseLine = l: + let + parts = lib.splitString "=" l; + in + { name = lib.head parts; value = lib.concatStringsSep "=" (lib.tail parts); }; + in + builtins.listToAttrs (map parseLine lines); + envVars = parseEnvFile ./.env_dev; + + configFile = pkgs.copyPathToStore ./config.yaml; + yq = pkgs.yq-go; + + # Note: This only loads the settings from the repos config file + # if one is not already set in the user's .config directory. + toposSetupHook = '' + export $(cat ${envFile} | xargs) + export TOPOS_CONFIG_PATH="$HOME/.config/topos/config.yaml" + mkdir -p "$(dirname "$TOPOS_CONFIG_PATH")" + if [ ! -f "$TOPOS_CONFIG_PATH" ]; then + echo "Creating new config file at $TOPOS_CONFIG_PATH" + echo "# Topos Configuration" > "$TOPOS_CONFIG_PATH" + ${yq}/bin/yq eval ${configFile} | while IFS= read -r line; do + echo "$line" >> "$TOPOS_CONFIG_PATH" + done + echo "Config file created at $TOPOS_CONFIG_PATH" + else + echo "Config file already exists at $TOPOS_CONFIG_PATH" fi - echo "Waiting for PostgreSQL to start..." - sleep 1 - done - - if ! pg_isready -q; then - echo "Failed to start PostgreSQL. Check the logs at $LOGFILE" - exit 1 - fi - - # Create the database if it doesn't exist - if ! psql -lqt | cut -d \| -f 1 | grep -qw "$POSTGRES_DB"; then - createdb "$POSTGRES_DB" - fi - - # Create the user if they don't exist - if ! psql -tAc "SELECT 1 FROM pg_roles WHERE rolname='$POSTGRES_USER'" | grep -q 1; then - createuser -s "$POSTGRES_USER" - fi - - # Set up the test database, role, and tables - psql -v ON_ERROR_STOP=1 -d $POSTGRES_DB <