Merge pull request #10 from dbazhenov/pg_init

Postgres Init DB fix

Author: dbazhenov

cmd/dataset/main.go

@@ -37,6 +37,7 @@ func main() {
 		checkConnectSettings()
 
 		// The main process of getting data from GitHub API and store it into MySQL, PostgreSQL, MongoDB databases.
+
 		fetchGitHubData(app.Config)
 
 		// Delay before the next start (Defined by the DELAY_MINUTES parameter)
@@ -88,6 +89,13 @@ func checkConnectSettings() {
 
 func fetchGitHubData(envVars app.EnvVars) {
 
+	if envVars.Postgres.ConnectionStatus != "Connected" &&
+		envVars.MySQL.ConnectionStatus != "Connected" &&
+		envVars.MongoDB.ConnectionStatus != "Connected" {
+		log.Printf("Error: No database connections are established.")
+		return
+	}
+
 	report := helperReportStart()
 
 	// Get all the repositories of the organization.
@@ -612,35 +620,6 @@ func MongoDBprocessPulls(envVars app.EnvVars, allRepos []*github.Repository, all
 
 	log.Printf("Databases: MongoDB: Start")
 
-	admin_db := client.Database("admin") // используем базу данных admin
-
-	// Check and set profiling level
-	var profilingLevel bson.M
-	err = admin_db.RunCommand(ctx, bson.D{{Key: "profile", Value: -1}}).Decode(&profilingLevel)
-	if err != nil {
-		log.Printf("MongoDB: Get Profiling Level Error: %s", err)
-		return err
-	}
-
-	if profilingLevel["was"] == int32(0) {
-		err = admin_db.RunCommand(ctx, bson.D{{Key: "profile", Value: 1}}).Err()
-		if err != nil {
-			log.Printf("MongoDB: Set Profiling Level Error: %s", err)
-			return err
-		}
-	}
-
-	// Set operation profiling
-	err = admin_db.RunCommand(ctx, bson.D{
-		{Key: "profile", Value: 2},
-		{Key: "slowms", Value: 200},
-		{Key: "sampleRate", Value: 1.0},
-	}).Err()
-	if err != nil {
-		log.Printf("MongoDB: Set Operation Profiling Error: %s", err)
-		return err
-	}
-
 	db := client.Database(envVars.MongoDB.DB)
 	dbCollectionRepos := db.Collection("repositories")
 	dbCollectionPulls := db.Collection("pulls")

data/init/mongodb/init.js

@@ -1 +1 @@
-db.setProfilingLevel(1);
+db.setProfilingLevel(2, {slowms: 0})

data/init/mongodb/mongod.conf

@@ -1,4 +1,4 @@
 operationProfiling:
   mode: all
   slowOpThresholdMs: 200
-  rateLimit: 100
+  rateLimit: 100 # (Only available with Percona Server for MongoDB.)

data/init/postgresql/init.sql

@@ -22,20 +22,21 @@ CREATE TABLE IF NOT EXISTS github.pulls (
     id BIGINT NOT NULL,
     repo VARCHAR(255) NOT NULL,
     data JSON,
-    PRIMARY KEY (id, repo),
-    INDEX idx_id (id),
-    INDEX idx_repo (repo)
+    PRIMARY KEY (id, repo)
 );
 
 CREATE TABLE IF NOT EXISTS github.pulls_test (
     id BIGINT NOT NULL,
     repo VARCHAR(255) NOT NULL,
     data JSON,
-    PRIMARY KEY (id, repo),
-    INDEX idx_id (id),
-    INDEX idx_repo (repo)
+    PRIMARY KEY (id, repo)
 );
 
+CREATE INDEX idx_id_pulls ON github.pulls (id);
+CREATE INDEX idx_repo_pulls ON github.pulls (repo);
+CREATE INDEX idx_id_pulls_test ON github.pulls_test (id);
+CREATE INDEX idx_repo_pulls_test ON github.pulls_test (repo);
+
 CREATE TABLE IF NOT EXISTS github.reports_runs (
     id SERIAL PRIMARY KEY,
     data JSONB

internal/databases/postgres/postgres.go

@@ -291,17 +291,13 @@ func InitDB(connection_string string) error {
 			id BIGINT NOT NULL,
 			repo VARCHAR(255) NOT NULL,
 			data JSON,
-			PRIMARY KEY (id, repo),
-			INDEX idx_id (id),
-			INDEX idx_repo (repo)
+			PRIMARY KEY (id, repo)
 		);
 		CREATE TABLE IF NOT EXISTS github.pulls_test (
 			id BIGINT NOT NULL,
 			repo VARCHAR(255) NOT NULL,
 			data JSON,
-			PRIMARY KEY (id, repo),
-			INDEX idx_id (id),
-			INDEX idx_repo (repo)
+			PRIMARY KEY (id, repo)
 		);
         CREATE TABLE IF NOT EXISTS github.reports_runs (
             id SERIAL PRIMARY KEY,
@@ -311,6 +307,10 @@ func InitDB(connection_string string) error {
             id SERIAL PRIMARY KEY,
             data JSONB
         );
+		CREATE INDEX idx_id_pulls ON github.pulls (id);
+		CREATE INDEX idx_repo_pulls ON github.pulls (repo);
+		CREATE INDEX idx_id_pulls_test ON github.pulls_test (id);
+		CREATE INDEX idx_repo_pulls_test ON github.pulls_test (repo);
     `
 	err = executeSQL(newDB, schemaSQL)
 	if err != nil {

k8s/demo-app/values.yaml

@@ -72,10 +72,10 @@ resources:
   dataset:
     requests:
       memory: "4Gi"
-      cpu: "1"
+      cpu: "3"
     limits:
       memory: "4Gi"
-      cpu: "1"
+      cpu: "3"
   load:
     requests:
       memory: "2Gi"