Merge pull request #14 from jonnyjohnson1/jonny/posgres-schema-update

Jonny/postgres schema update

.env_dev

@@ -9,7 +9,7 @@ ONE_API_API_KEY="sk-oneapi.local123"
 SUPABASE_URL=
 SUPABASE_KEY=
 POSTGRES_DB=test_topos_db
-POSTGRES_USER=username
-POSTGRES_PASSWORD=your_password
+POSTGRES_USER=jonny
+POSTGRES_PASSWORD=1234589034
 POSTGRES_HOST=127.0.0.1
 POSTGRES_PORT=5432

.env_template

@@ -0,0 +1,15 @@
+NEO4J_URI="bolt://localhost:7687"
+NEO4J_USER="neo4j"
+NEO4J_PASSWORD="password"
+NEO4J_TEST_DATABASE="neo4j"
+NEO4J_SHOWROOM_DATABASE="neo4j"
+JWT_SECRET="terces_tj"
+OPEN_AI_API_KEY="sk-openai.com123"
+ONE_API_API_KEY="sk-oneapi.local123"
+SUPABASE_URL=
+SUPABASE_KEY=
+POSTGRES_DB=test_topos_db
+POSTGRES_USER=username
+POSTGRES_PASSWORD=your_password_here
+POSTGRES_HOST=127.0.0.1
+POSTGRES_PORT=5432

.gitignore

@@ -112,4 +112,6 @@ topos/cloudbuild.yaml
 dockerfile
 
 topos.app
-/pgdata
+/pgdata
+/data
+/result

README.md

@@ -2,15 +2,30 @@
   <img src="https://github.com/jonnyjohnson1/topos-cli/blob/main/topos/assets/topos_blk_rounded.png" style="width: 70px; height: 70px;" alt="Private LLMs" />
 </p>
 <p align="center">
-  <em>Private AI Backend Service</em>
+  <em>Private, Personal AI Backend Service</em>
 </p>
 
 ---
 
 # Topos
-A personal intelligence service, using your own computer to power private conversations with friends, family, and coworkers, collect/store your own private data, and use AI privately. Runs  great with the [chat arena](https://github.com/jonnyjohnson1/chat-arena) app available on desktop and mobile.
+Topos is a private, personal AI and database management service.
+It comes as an easy-to-install server for your AI apps to connect to.
 
-Tech: ollama, postgres, FastAPI, nixOS, huggingface-transformers
+It is a personal intelligence service, using your own computer to power private conversations with friends, family, and coworkers, collect/store your own private data, and use AI privately. 
+
+Apps Using Topos:
+- [chat arena](https://github.com/jonnyjohnson1/chat-arena) #desktop #mobile
+
+Tech: nixOS, ollama, postgres, FastAPI, huggingface-transformers
+
+<p align="center">
+  <img src="https://github.com/jonnyjohnson1/topos-cli/blob/main/topos/assets/tui.png" style="zoom:67%;" alt="Terminal User Interface" />
+</p>
+<p align="center">
+  <em>Runs the Terminal User Interface provided by [F1bonacc1](https://github.com/F1bonacc1/process-compose)</em>
+</p>
+
+---
 
 ## (MacOS) Easy Install With .dmg
 *(Experimental)*: This is new, and should work on most MacOS machines!
@@ -46,6 +61,14 @@ run built binary
 ./result/bin/topos
 ```
 
+(You might also try this)
+```
+nix build --extra-experimental-features nix-command --extra-experimental-features flakes --show-trace
+```
+```
+./result/bin/services-flake-topos
+```
+
 ### Dev Shell
 ```
 nix develop
@@ -57,6 +80,13 @@ topos run
 nix develop .#poetry
 ```
 
+## Install Tips
+
+### POSTGRES 
+- If postgres is already running, running the bin fails, shut it down first.
+- Nix will manage postgres' start/stop function itself when you use it, but if you have started the database elsewhere, it won't be able to manage it, and will fail to start up.
+
+
 ## Install Instructions
 requires `brew install just`
 requires `brew install poetry`

config.yaml

@@ -1 +1 @@
-active_spacy_model: en_core_web_trf
+active_spacy_model: en_core_web_sm

default.nix

@@ -42,23 +42,43 @@ in pkgs.mkShell {
 
     # Set up the test database, role, and tables
     psql -d $POSTGRES_DB <<SQL | tee -a $LOGFILE
-    CREATE TABLE IF NOT EXISTS conversation_cache (
-        conv_id TEXT PRIMARY KEY,
-        message_data JSONB NOT NULL
+    
+    -- Create the conversation table
+    CREATE TABLE IF NOT EXISTS conversation (
+        message_id VARCHAR PRIMARY KEY,
+        conv_id VARCHAR NOT NULL,
+        userid VARCHAR NOT NULL,
+        timestamp TIMESTAMP NOT NULL,
+        name VARCHAR,
+        role VARCHAR NOT NULL,
+        message TEXT NOT NULL
     );
 
-    CREATE TABLE IF NOT EXISTS entities (
-        id TEXT PRIMARY KEY,
-        label TEXT NOT NULL,
-        properties JSONB
+    -- Create the utterance_token_info table
+    CREATE TABLE IF NOT EXISTS utterance_token_info (
+        message_id VARCHAR PRIMARY KEY,
+        conv_id VARCHAR NOT NULL,
+        userid VARCHAR NOT NULL,
+        name VARCHAR,
+        role VARCHAR NOT NULL,
+        timestamp TIMESTAMP NOT NULL,
+        ents JSONB
     );
 
-    CREATE TABLE IF NOT EXISTS relations (
-        source_id TEXT,
-        relation_type TEXT,
-        target_id TEXT,
-        properties JSONB,
-        PRIMARY KEY (source_id, relation_type, target_id)
+    -- Create the utterance_text_info table
+    CREATE TABLE IF NOT EXISTS utterance_text_info (
+        message_id VARCHAR PRIMARY KEY,
+        conv_id VARCHAR NOT NULL,
+        userid VARCHAR NOT NULL,
+        name VARCHAR,
+        role VARCHAR NOT NULL,
+        timestamp TIMESTAMP NOT NULL,
+        moderator JSONB,
+        mod_label VARCHAR,
+        tern_sent JSONB,
+        tern_label VARCHAR,
+        emo_27 JSONB,
+        emo_27_label VARCHAR
     );
 
     CREATE ROLE $POSTGRES_USER WITH LOGIN PASSWORD '$POSTGRES_PASSWORD';

flake.nix

@@ -68,7 +68,7 @@
 
             configFile = pkgs.copyPathToStore ./config.yaml;
             yq = pkgs.yq-go;
-
+            
             kafkaPreStartup = ''
             echo "Kafka is ready. Creating topic..."
             ${pkgs.apacheKafka}/bin/kafka-topics.sh --create --topic chat_topic --bootstrap-server localhost:9092 --partitions 1 --replication-factor 1 --if-not-exists
@@ -135,24 +135,42 @@
                           CREATE USER ${envVars.POSTGRES_USER} WITH SUPERUSER PASSWORD '${envVars.POSTGRES_PASSWORD}';
                         '';
                         after = ''
-                          CREATE TABLE IF NOT EXISTS conversation_cache (
-                              conv_id TEXT PRIMARY KEY,
-                              message_data JSONB NOT NULL
-                          );
-
-                          CREATE TABLE IF NOT EXISTS entities (
-                              id TEXT PRIMARY KEY,
-                              label TEXT NOT NULL,
-                              properties JSONB
-                          );
-
-                          CREATE TABLE IF NOT EXISTS relations (
-                              source_id TEXT,
-                              relation_type TEXT,
-                              target_id TEXT,
-                              properties JSONB,
-                              PRIMARY KEY (source_id, relation_type, target_id)
-                          );
+                          CREATE TABLE IF NOT EXISTS conversation (
+                            message_id VARCHAR PRIMARY KEY,
+                            conv_id VARCHAR NOT NULL,
+                            userid VARCHAR NOT NULL,
+                            timestamp TIMESTAMP NOT NULL,
+                            name VARCHAR,
+                            role VARCHAR NOT NULL,
+                            message TEXT NOT NULL
+                        );
+
+                        -- Create the utterance_token_info table
+                        CREATE TABLE IF NOT EXISTS utterance_token_info (
+                            message_id VARCHAR PRIMARY KEY,
+                            conv_id VARCHAR NOT NULL,
+                            userid VARCHAR NOT NULL,
+                            name VARCHAR,
+                            role VARCHAR NOT NULL,
+                            timestamp TIMESTAMP NOT NULL,
+                            ents JSONB
+                        );
+
+                        -- Create the utterance_text_info table
+                        CREATE TABLE IF NOT EXISTS utterance_text_info (
+                            message_id VARCHAR PRIMARY KEY,
+                            conv_id VARCHAR NOT NULL,
+                            userid VARCHAR NOT NULL,
+                            name VARCHAR,
+                            role VARCHAR NOT NULL,
+                            timestamp TIMESTAMP NOT NULL,
+                            moderator JSONB,
+                            mod_label VARCHAR,
+                            tern_sent JSONB,
+                            tern_label VARCHAR,
+                            emo_27 JSONB,
+                            emo_27_label VARCHAR
+                        );
 
                           GRANT ALL PRIVILEGES ON DATABASE ${envVars.POSTGRES_DB} TO ${envVars.POSTGRES_USER};
                           GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO ${envVars.POSTGRES_USER};

justfile

@@ -22,3 +22,8 @@ python:
 
 dmg:
     create-dmg topos.app --volicon "topos/assets/topos_blk_rounded.png" --icon "topos/assets/topos_blk_rounded.png"
+
+stoppg:
+    export PGDATA=$(pwd)/pgdata
+    echo "Stopping any existing PostgreSQL server..."
+    pg_ctl -D "$PGDATA" stop || echo "No existing server to stop."

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "topos"
-version = "0.2.2"
+version = "0.2.3"
 description = "The official Python client for Topos."
 authors = ["Dialogues <[email protected]>"]
 license = "MIT"
@@ -43,9 +43,9 @@ pystray = "0.19.5"
 supabase = "^2.6.0"
 psycopg2-binary = "^2.9.9"
 en-core-web-sm = {url = "https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl"}
-en-core-web-lg = {url = "https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.8.0/en_core_web_lg-3.8.0-py3-none-any.whl"}
-en-core-web-md = {url = "https://github.com/explosion/spacy-models/releases/download/en_core_web_md-3.8.0/en_core_web_md-3.8.0-py3-none-any.whl"}
-en-core-web-trf = {url = "https://github.com/explosion/spacy-models/releases/download/en_core_web_trf-3.8.0/en_core_web_trf-3.8.0-py3-none-any.whl"}
+# en-core-web-lg = {url = "https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.8.0/en_core_web_lg-3.8.0-py3-none-any.whl"}
+# en-core-web-md = {url = "https://github.com/explosion/spacy-models/releases/download/en_core_web_md-3.8.0/en_core_web_md-3.8.0-py3-none-any.whl"}
+# en-core-web-trf = {url = "https://github.com/explosion/spacy-models/releases/download/en_core_web_trf-3.8.0/en_core_web_trf-3.8.0-py3-none-any.whl"}
 [tool.poetry.group.dev.dependencies]
 pytest = "^7.4.3"
 pytest-asyncio = "^0.23.2"

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = topos
-version = 0.2.1
+version = 0.2.3
 author = Jonny Johnson
 author_email = [email protected]
 description = For interacting with Topos tooling

setup.py

@@ -2,7 +2,7 @@
 
 setup(
     name='topos',
-    version='0.2.1',
+    version='0.2.3',
     packages=find_packages(),
     entry_points={
         'console_scripts': [