From 1ebea1e74eb703b78b40f993d573bb5b23fb3dda Mon Sep 17 00:00:00 2001 From: Joshua Rodriguez Date: Fri, 8 Nov 2024 15:19:45 -0800 Subject: [PATCH] Runs a terraform destroy on modules while deleting. Some rearangement and cleanup of the install/uninstall code --- backend/internal/database/models/models.go | 32 +++- backend/internal/database/operations.go | 154 ++++++++++++---- backend/internal/database/setup.go | 12 +- backend/internal/processes/installer.go | 106 ++++++----- backend/internal/processes/uninstaller.go | 204 ++------------------- backend/internal/terraform/management.go | 23 ++- backend/internal/terraform/runner.go | 62 ++++++- backend/internal/web/api.go | 70 ++----- backend/types/marketplace.go | 13 ++ src/components/ApplicationPanelItem.svelte | 48 ++--- src/components/common/Modal.svelte | 2 +- 11 files changed, 355 insertions(+), 371 deletions(-) diff --git a/backend/internal/database/models/models.go b/backend/internal/database/models/models.go index 955bb5b..0baf1a3 100644 --- a/backend/internal/database/models/models.go +++ b/backend/internal/database/models/models.go @@ -1,8 +1,11 @@ package models import ( + "errors" "gorm.io/gorm" "time" + "database/sql/driver" + "encoding/json" ) type CoreConfig struct { @@ -45,7 +48,7 @@ type Deployment struct { CreationDate time.Time } -type InstalledMarketplaceApplication struct { +type InstalledMarketplaceApplicationDB struct { gorm.Model Name string DeploymentName string @@ -53,4 +56,29 @@ type InstalledMarketplaceApplication struct { Source string Status string PackageName string -} \ No newline at end of file + TerraformModuleName string + Variables JSON `json:"variables"` + AdvancedValues JSON `json:"advanced_values"` +} + +type JSON json.RawMessage + +func (j JSON) Value() (driver.Value, error) { + if len(j) == 0 { + return nil, nil + } + return json.RawMessage(j).MarshalJSON() +} + +func (j *JSON) Scan(value interface{}) error { + if value == nil { + *j = nil + return nil + } + s, ok := value.([]byte) + if !ok { + return errors.New("Invalid Scan Source") + } + *j = append((*j)[0:0], s...) + return nil +} diff --git a/backend/internal/database/operations.go b/backend/internal/database/operations.go index 6d6bea0..f703027 100644 --- a/backend/internal/database/operations.go +++ b/backend/internal/database/operations.go @@ -1,11 +1,13 @@ package database import ( + "encoding/json" "errors" "fmt" log "github.com/sirupsen/logrus" "github.com/unity-sds/unity-management-console/backend/internal/application/config" "github.com/unity-sds/unity-management-console/backend/internal/database/models" + "github.com/unity-sds/unity-management-console/backend/types" "gorm.io/gorm" "gorm.io/gorm/clause" ) @@ -134,19 +136,6 @@ func (g GormDatastore) FetchDeploymentIDByName(deploymentID string) (uint, error return deployment.ID, nil } -func (g GormDatastore) GetInstalledApplicationByName(name string) (*models.InstalledMarketplaceApplication, error) { - var application models.InstalledMarketplaceApplication - result := g.db.Where("name = ?", name).Where("status != 'UNINSTALLED'").First(&application) - if result.Error != nil { - if errors.Is(result.Error, gorm.ErrRecordNotFound) { - return nil, nil - } - - log.WithError(result.Error).Error("Error finding application") - return nil, result.Error - } - return &application, nil -} func (g GormDatastore) FetchDeploymentIDByApplicationName(deploymentName string) (uint, error) { var application models.Application @@ -191,13 +180,55 @@ func (g GormDatastore) FetchAllApplicationStatus() ([]models.Deployment, error) return deployments, nil } -func (g GormDatastore) FetchAllInstalledMarketplaceApplications() ([]models.InstalledMarketplaceApplication, error) { - var applications []models.InstalledMarketplaceApplication - result := g.db.Find(&applications) +func parseMarketplaceApplication(dbApp *models.InstalledMarketplaceApplicationDB) (*types.InstalledMarketplaceApplication, error) { + typeApp := &types.InstalledMarketplaceApplication{ + Name: dbApp.Name, + DeploymentName: dbApp.DeploymentName, + Version: dbApp.Version, + Source: dbApp.Source, + Status: dbApp.Status, + PackageName: dbApp.PackageName, + TerraformModuleName: dbApp.TerraformModuleName, + Variables: make(map[string]string), + AdvancedValues: make(types.AdvancedValue), + } + + // Convert Variables JSON to map + if dbApp.Variables != nil { + if err := json.Unmarshal([]byte(dbApp.Variables), &typeApp.Variables); err != nil { + log.WithError(err).Error("Failed to unmarshal Variables") + return nil, err + } + } + + // Convert AdvancedValues JSON to map + if dbApp.AdvancedValues != nil { + if err := json.Unmarshal([]byte(dbApp.AdvancedValues), &typeApp.AdvancedValues); err != nil { + log.WithError(err).Error("Failed to unmarshal AdvancedValues") + return nil, err + } + } + + return typeApp, nil +} + +func (g GormDatastore) FetchAllInstalledMarketplaceApplications() ([]*types.InstalledMarketplaceApplication, error) { + var dbApps []models.InstalledMarketplaceApplicationDB + result := g.db.Find(&dbApps) if result.Error != nil { return nil, result.Error } - return applications, nil + + typeApps := make([]*types.InstalledMarketplaceApplication, 0, len(dbApps)) + for _, dbApp := range dbApps { + typeApp, err := parseMarketplaceApplication(&dbApp) + if err != nil { + return nil, err + } + typeApps = append(typeApps, typeApp) + } + + return typeApps, nil } func (g GormDatastore) FetchAllApplicationStatusByDeployment(deploymentid uint) ([]models.Application, error) { @@ -259,18 +290,48 @@ func (g GormDatastore) RemoveApplicationByName(deploymentName string, applicatio return nil } -func (g GormDatastore) StoreInstalledMarketplaceApplication(model models.InstalledMarketplaceApplication) error { - if err := g.db.Save(&model).Error; err != nil { - // Handle error for Save +func (g GormDatastore) StoreInstalledMarketplaceApplication(application *types.InstalledMarketplaceApplication) error { + // Convert types model to database model + dbApp := &models.InstalledMarketplaceApplicationDB{ + Name: application.Name, + DeploymentName: application.DeploymentName, + Version: application.Version, + Source: application.Source, + Status: application.Status, + PackageName: application.PackageName, + TerraformModuleName: application.TerraformModuleName, + } + + // Convert Variables map to JSON + if application.Variables != nil { + varsJSON, err := json.Marshal(application.Variables) + if err != nil { + log.WithError(err).Error("Failed to marshal Variables") + return err + } + dbApp.Variables = models.JSON(varsJSON) + } + + // Convert AdvancedValues map to JSON + if application.AdvancedValues != nil { + advJSON, err := json.Marshal(application.AdvancedValues) + if err != nil { + log.WithError(err).Error("Failed to marshal AdvancedValues") + return err + } + dbApp.AdvancedValues = models.JSON(advJSON) + } + + if err := g.db.Save(dbApp).Error; err != nil { log.WithError(err).Error("Problem saving record to database") return err } return nil } -func (g GormDatastore) GetInstalledMarketplaceApplicationStatusByName(appName string, deploymentName string) (*models.InstalledMarketplaceApplication, error) { - var application models.InstalledMarketplaceApplication - err := g.db.Where("Name = ? AND deployment_name = ?", appName, deploymentName).First(&application).Error +func (g GormDatastore) GetInstalledMarketplaceApplication(appName string, deploymentName string) (*types.InstalledMarketplaceApplication, error) { + var dbApp models.InstalledMarketplaceApplicationDB + err := g.db.Where("Name = ? AND deployment_name = ?", appName, deploymentName).First(&dbApp).Error if errors.Is(err, gorm.ErrRecordNotFound) { return nil, nil } @@ -278,25 +339,56 @@ func (g GormDatastore) GetInstalledMarketplaceApplicationStatusByName(appName st log.WithError(err).Error("Problem getting application status") return nil, err } - return &application, nil + + return parseMarketplaceApplication(&dbApp) } -func (g GormDatastore) UpdateInstalledMarketplaceApplicationStatusByName(appName string, deploymentName string, status string) error { - var app models.InstalledMarketplaceApplication - g.db.Where("name = ? AND deployment_name = ?", appName, deploymentName).First(&app) - app.Status = status +func (g GormDatastore) UpdateInstalledMarketplaceApplication(application *types.InstalledMarketplaceApplication) error { + // Find existing record + var existingApp models.InstalledMarketplaceApplicationDB + if err := g.db.Where("name = ? AND deployment_name = ?", application.Name, application.DeploymentName).First(&existingApp).Error; err != nil { + log.WithError(err).Error("Problem finding existing application") + return err + } + + // Update fields + existingApp.Version = application.Version + existingApp.Source = application.Source + existingApp.Status = application.Status + existingApp.PackageName = application.PackageName + existingApp.TerraformModuleName = application.TerraformModuleName + + // Convert Variables map to JSON + if application.Variables != nil { + varsJSON, err := json.Marshal(application.Variables) + if err != nil { + log.WithError(err).Error("Failed to marshal Variables") + return err + } + existingApp.Variables = models.JSON(varsJSON) + } - if err := g.db.Save(&app).Error; err != nil { - // Handle error for Save + // Convert AdvancedValues map to JSON + if application.AdvancedValues != nil { + advJSON, err := json.Marshal(application.AdvancedValues) + if err != nil { + log.WithError(err).Error("Failed to marshal AdvancedValues") + return err + } + existingApp.AdvancedValues = models.JSON(advJSON) + } + + if err := g.db.Save(&existingApp).Error; err != nil { log.WithError(err).Error("Problem saving record to database") return err } return nil } + func (g GormDatastore) RemoveInstalledMarketplaceApplication(appName string, deploymentName string) error { - if err := g.db.Where("name = ? AND deployment_name = ?", appName, deploymentName).Delete(&models.InstalledMarketplaceApplication{}).Error; err != nil { + if err := g.db.Where("name = ? AND deployment_name = ?", appName, deploymentName).Delete(&models.InstalledMarketplaceApplicationDB{}).Error; err != nil { return err } return nil diff --git a/backend/internal/database/setup.go b/backend/internal/database/setup.go index bc8ff65..d8e5450 100644 --- a/backend/internal/database/setup.go +++ b/backend/internal/database/setup.go @@ -5,6 +5,7 @@ import ( "github.com/unity-sds/unity-management-console/backend/internal/application" "github.com/unity-sds/unity-management-console/backend/internal/application/config" "github.com/unity-sds/unity-management-console/backend/internal/database/models" + "github.com/unity-sds/unity-management-console/backend/types" "gorm.io/driver/sqlite" "gorm.io/gorm" ) @@ -101,7 +102,7 @@ func NewGormDatastore() (Datastore, error) { return nil, err } - err = db.AutoMigrate(&models.InstalledMarketplaceApplication{}) + err = db.AutoMigrate(&models.InstalledMarketplaceApplicationDB{}) if err != nil { return nil, err } @@ -127,10 +128,9 @@ type Datastore interface { RemoveDeploymentByName(name string) error RemoveApplicationByName(deploymentName string, applicationName string) error FetchDeploymentIDByApplicationName(deploymentName string) (uint, error) - GetInstalledApplicationByName(name string) (*models.InstalledMarketplaceApplication, error) - StoreInstalledMarketplaceApplication(model models.InstalledMarketplaceApplication) error - UpdateInstalledMarketplaceApplicationStatusByName(appName string,displayName string, status string) error - GetInstalledMarketplaceApplicationStatusByName(appName string, displayName string) (*models.InstalledMarketplaceApplication, error) - FetchAllInstalledMarketplaceApplications() ([]models.InstalledMarketplaceApplication, error) + StoreInstalledMarketplaceApplication(application *types.InstalledMarketplaceApplication) error + GetInstalledMarketplaceApplication(appName string, displayName string) (*types.InstalledMarketplaceApplication, error) + FetchAllInstalledMarketplaceApplications() ([]*types.InstalledMarketplaceApplication, error) RemoveInstalledMarketplaceApplication(appName string, deploymentName string) error + UpdateInstalledMarketplaceApplication(application *types.InstalledMarketplaceApplication) error } diff --git a/backend/internal/processes/installer.go b/backend/internal/processes/installer.go index e2b585f..b7d17c0 100644 --- a/backend/internal/processes/installer.go +++ b/backend/internal/processes/installer.go @@ -8,17 +8,17 @@ import ( "github.com/unity-sds/unity-management-console/backend/internal/application/config" "github.com/unity-sds/unity-management-console/backend/internal/aws" "github.com/unity-sds/unity-management-console/backend/internal/database" - "github.com/unity-sds/unity-management-console/backend/internal/database/models" "github.com/unity-sds/unity-management-console/backend/internal/terraform" "github.com/unity-sds/unity-management-console/backend/internal/websocket" "github.com/unity-sds/unity-management-console/backend/types" "github.com/zclconf/go-cty/cty" + "math/rand" "os" "os/exec" "path/filepath" "regexp" "strings" - // "time" + "time" ) func fetchMandatoryVars() ([]terraform.Varstruct, error) { @@ -36,32 +36,44 @@ func fetchMandatoryVars() ([]terraform.Varstruct, error) { } - - -func startApplicationInstallTerraform(appConfig *config.AppConfig, location string, installParams *types.ApplicationInstallParams, meta *marketplace.MarketplaceMetadata, db database.Datastore) { - log.Errorf("Application name is: %s", installParams.Name) - terraform.AddApplicationToStack(appConfig, location, meta, installParams, db) - execute(db, appConfig, meta, installParams) +func startApplicationInstallTerraform(appConfig *config.AppConfig, location string, application *types.InstalledMarketplaceApplication, meta *marketplace.MarketplaceMetadata, db database.Datastore) { + log.Errorf("Application name is: %s", application.Name) + terraform.AddApplicationToStack(appConfig, location, meta, application, db) + executeTerraformInstall(db, appConfig, meta, application) } func InstallMarketplaceApplication(appConfig *config.AppConfig, location string, installParams *types.ApplicationInstallParams, meta *marketplace.MarketplaceMetadata, db database.Datastore, sync bool) error { if meta.Backend == "terraform" { - application := models.InstalledMarketplaceApplication{ - Name: installParams.Name, - Version: installParams.Version, - DeploymentName: installParams.DeploymentName, - PackageName: meta.Name, - Source: meta.Package, - Status: "STAGED", + + rand.Seed(time.Now().UnixNano()) + chars := "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + randomChars := make([]byte, 5) + for i, v := range rand.Perm(52)[:5] { + randomChars[i] = chars[v] + } + log.Info("Appending block to body") + terraformModuleName := fmt.Sprintf("%s-%s", installParams.DeploymentName, string(randomChars)) + + application := &types.InstalledMarketplaceApplication{ + Name: installParams.Name, + Version: installParams.Version, + DeploymentName: installParams.DeploymentName, + PackageName: meta.Name, + Source: meta.Package, + Status: "STAGED", + TerraformModuleName: terraformModuleName, + Variables: installParams.Variables, } db.StoreInstalledMarketplaceApplication(application) - db.UpdateInstalledMarketplaceApplicationStatusByName(installParams.Name, installParams.DeploymentName, "INSTALLING") + + application.Status = "INSTALLING" + db.UpdateInstalledMarketplaceApplication(application) if sync { - startApplicationInstallTerraform(appConfig, location, installParams, meta, db) + startApplicationInstallTerraform(appConfig, location, application, meta, db) } else { - go startApplicationInstallTerraform(appConfig, location, installParams, meta, db) + go startApplicationInstallTerraform(appConfig, location, application, meta, db) } @@ -72,11 +84,7 @@ func InstallMarketplaceApplication(appConfig *config.AppConfig, location string, } } - - - - -func execute(db database.Datastore, appConfig *config.AppConfig, meta *marketplace.MarketplaceMetadata, installParams *types.ApplicationInstallParams) error { +func executeTerraformInstall(db database.Datastore, appConfig *config.AppConfig, meta *marketplace.MarketplaceMetadata, application *types.InstalledMarketplaceApplication) error { // Create install_logs directory if it doesn't exist logDir := filepath.Join(appConfig.Workdir, "install_logs") if err := os.MkdirAll(logDir, 0755); err != nil && !os.IsExist(err) { @@ -90,34 +98,38 @@ func execute(db database.Datastore, appConfig *config.AppConfig, meta *marketpla // return err //} //terraform.WriteTFVars(m, appConfig) - err := runPreInstall(appConfig, meta, installParams) + err := runPreInstall(appConfig, meta, application) if err != nil { return err } - db.UpdateInstalledMarketplaceApplicationStatusByName(installParams.Name, installParams.DeploymentName, "INSTALLING") + application.Status = "INSTALLING" + db.UpdateInstalledMarketplaceApplication(application) fetchAllApplications(db) - logfile := filepath.Join(logDir, fmt.Sprintf("%s_%s_install_log", installParams.Name, installParams.DeploymentName)) + logfile := filepath.Join(logDir, fmt.Sprintf("%s_%s_install_log", application.Name, application.DeploymentName)) err = terraform.RunTerraformLogOutToFile(appConfig, logfile, executor, "") if err != nil { - db.UpdateInstalledMarketplaceApplicationStatusByName(installParams.Name, installParams.DeploymentName, "FAILED") - fetchAllApplications(db) + application.Status = "FAILED" + db.UpdateInstalledMarketplaceApplication(application) return err } - db.UpdateInstalledMarketplaceApplicationStatusByName(installParams.Name, installParams.DeploymentName, "INSTALLED") + + application.Status = "INSTALLED" + db.UpdateInstalledMarketplaceApplication(application) fetchAllApplications(db) - err = runPostInstallNew(appConfig, meta, installParams) + err = runPostInstallNew(appConfig, meta, application) if err != nil { - db.UpdateInstalledMarketplaceApplicationStatusByName(installParams.Name, installParams.DeploymentName, "POSTINSTALL FAILED") + application.Status = "POSTINSTALL FAILED" + db.UpdateInstalledMarketplaceApplication(application) fetchAllApplications(db) - return err } - db.UpdateInstalledMarketplaceApplicationStatusByName(installParams.Name, installParams.DeploymentName, "COMPLETE") + application.Status = "COMPLETE" + db.UpdateInstalledMarketplaceApplication(application) fetchAllApplications(db) return nil @@ -157,7 +169,7 @@ func runPostInstall(appConfig *config.AppConfig, meta *marketplace.MarketplaceMe return nil } -func runPostInstallNew(appConfig *config.AppConfig, meta *marketplace.MarketplaceMetadata, installParams *types.ApplicationInstallParams) error { +func runPostInstallNew(appConfig *config.AppConfig, meta *marketplace.MarketplaceMetadata, application *types.InstalledMarketplaceApplication) error { if meta.PostInstall != "" { //TODO UNPIN ME @@ -179,10 +191,10 @@ func runPostInstallNew(appConfig *config.AppConfig, meta *marketplace.Marketplac // log.Infof("Adding environment variable: %s = %s", cleanKey, v) // cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", cleanKey, v)) // } - cmd.Env = append(cmd.Env, fmt.Sprintf("DEPLOYMENTNAME=%s", installParams.DeploymentName)) + cmd.Env = append(cmd.Env, fmt.Sprintf("DEPLOYMENTNAME=%s", application.DeploymentName)) cmd.Env = append(cmd.Env, fmt.Sprintf("WORKDIR=%s", appConfig.Workdir)) - cmd.Env = append(cmd.Env, fmt.Sprintf("DISPLAYNAME=%s", installParams.DisplayName)) - cmd.Env = append(cmd.Env, fmt.Sprintf("APPNAME=%s", installParams.Name)) + cmd.Env = append(cmd.Env, fmt.Sprintf("DISPLAYNAME=%s", application.DeploymentName)) + cmd.Env = append(cmd.Env, fmt.Sprintf("APPNAME=%s", application.Name)) if err := cmd.Run(); err != nil { log.WithError(err).Error("Error running post install script") @@ -192,8 +204,7 @@ func runPostInstallNew(appConfig *config.AppConfig, meta *marketplace.Marketplac return nil } - -func runPreInstall(appConfig *config.AppConfig, meta *marketplace.MarketplaceMetadata, installParams *types.ApplicationInstallParams) error { +func runPreInstall(appConfig *config.AppConfig, meta *marketplace.MarketplaceMetadata, application *types.InstalledMarketplaceApplication) error { if meta.PreInstall != "" { // TODO UNPIN ME path := filepath.Join(appConfig.Workdir, "terraform", "modules", meta.Name, meta.Version, meta.WorkDirectory, meta.PreInstall) @@ -214,10 +225,10 @@ func runPreInstall(appConfig *config.AppConfig, meta *marketplace.MarketplaceMet // log.Infof("Adding environment variable: %s = %s", cleanKey, v) // cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", cleanKey, v)) // } - cmd.Env = append(cmd.Env, fmt.Sprintf("DEPLOYMENTNAME=%s", installParams.DeploymentName)) + cmd.Env = append(cmd.Env, fmt.Sprintf("DEPLOYMENTNAME=%s", application.DeploymentName)) cmd.Env = append(cmd.Env, fmt.Sprintf("WORKDIR=%s", appConfig.Workdir)) - cmd.Env = append(cmd.Env, fmt.Sprintf("DISPLAYNAME=%s", installParams.DisplayName)) - cmd.Env = append(cmd.Env, fmt.Sprintf("APPNAME=%s", installParams.Name)) + cmd.Env = append(cmd.Env, fmt.Sprintf("DISPLAYNAME=%s", application.DeploymentName)) + cmd.Env = append(cmd.Env, fmt.Sprintf("APPNAME=%s", application.Name)) if err := cmd.Run(); err != nil { log.WithError(err).Error("Error running pre install script") return err @@ -226,10 +237,9 @@ func runPreInstall(appConfig *config.AppConfig, meta *marketplace.MarketplaceMet return nil } - func TriggerInstall(store database.Datastore, applicationInstallParams *types.ApplicationInstallParams, conf *config.AppConfig, sync bool) error { // First check if this application is already installed. - existingApplication, err := store.GetInstalledMarketplaceApplicationStatusByName(applicationInstallParams.Name, applicationInstallParams.DeploymentName) + existingApplication, err := store.GetInstalledMarketplaceApplication(applicationInstallParams.Name, applicationInstallParams.DeploymentName) if err != nil { log.WithError(err).Error("Error finding applications") return errors.New("Unable to search applcation list") @@ -260,7 +270,9 @@ func TriggerInstall(store database.Datastore, applicationInstallParams *types.Ap func TriggerUninstall(wsManager *websocket.WebSocketManager, userid string, store database.Datastore, received *marketplace.Uninstall, conf *config.AppConfig) error { if received.All == true { return UninstallAll(conf, wsManager, userid, received) - } else { - return UninstallApplication(received.Application, received.DeploymentName, received.DisplayName, conf, store, wsManager, userid) - } + } + // else { + // return UninstallApplication(received.Application, received.DeploymentName, received.DisplayName, conf, store, wsManager, userid) + // } + return nil } diff --git a/backend/internal/processes/uninstaller.go b/backend/internal/processes/uninstaller.go index a87c612..118ede2 100644 --- a/backend/internal/processes/uninstaller.go +++ b/backend/internal/processes/uninstaller.go @@ -10,12 +10,13 @@ import ( "github.com/unity-sds/unity-management-console/backend/internal/database" "github.com/unity-sds/unity-management-console/backend/internal/terraform" "github.com/unity-sds/unity-management-console/backend/internal/websocket" + "github.com/unity-sds/unity-management-console/backend/types" "io/ioutil" "os" "path" // "strconv" - "strings" "fmt" + "strings" ) type UninstallPayload struct { @@ -33,11 +34,11 @@ func UninstallAll(conf *config.AppConfig, conn *websocket.WebSocketManager, user //return err } - if (received.DeleteBucket) { + if received.DeleteBucket { err = aws.DeleteS3Bucket(conf.BucketName) if err != nil { log.WithError(err).Error("FAILED TO REMOVE S3 BUCKET") - } + } } err = aws.DeleteStateTable(conf.InstallPrefix) @@ -49,177 +50,26 @@ func UninstallAll(conf *config.AppConfig, conn *websocket.WebSocketManager, user return nil } -func UninstallApplication(appname string, deploymentname string, displayname string, conf *config.AppConfig, store database.Datastore, conn *websocket.WebSocketManager, userid string) error { - executor := &terraform.RealTerraformExecutor{} - - filepath := path.Join(conf.Workdir, "workspace") - - files, err := ioutil.ReadDir(filepath) - if err != nil { - return err - } - - for _, file := range files { - log.Infof("Checking file %s has prefix: %s", file.Name(), appname) - if strings.HasPrefix(file.Name(), appname) { - log.Infof("File was a match") - // Open the file - f, err := os.Open(path.Join(filepath, file.Name())) - if err != nil { - return err - } - - // Read comments at the top - scanner := bufio.NewScanner(f) - metadata := make(map[string]string) - for scanner.Scan() { - line := scanner.Text() - if !strings.HasPrefix(line, "#") { - break - } - // Parsing the comments - parts := strings.SplitN(strings.TrimPrefix(line, "# "), ": ", 2) - if len(parts) == 2 { - key := parts[0] - value := strings.TrimSpace(parts[1]) - metadata[key] = value - } - } - f.Close() - - // Check applicationName from the comments and delete the file if it matches - log.Infof("Check if appname %s == %s", metadata["applicationName"], displayname) - if metadata["applicationName"] == displayname { - p := path.Join(filepath, file.Name()) - log.Infof("Attempting to delete file: %s", p) - err = os.Remove(p) - if err != nil { - id, err := store.FetchDeploymentIDByName(deploymentname) - log.WithError(err).Error("Failed to fetch deployment ID by name when removing application") - err = store.UpdateApplicationStatus(id, appname, displayname, "UNINSTALL FAILED") - log.WithError(err).Error("Failed to update application status removing application") - return err - } - err := store.RemoveApplicationByName(deploymentname, appname) - if err != nil { - id, err := store.FetchDeploymentIDByName(deploymentname) - log.WithError(err).Error("Failed to fetch deployment ID by name when removing application") - err = store.UpdateApplicationStatus(id, appname, displayname, "UNINSTALL FAILED") - log.WithError(err).Error("Failed to update application status removing application") - return err - } - err = fetchAllApplications(store) - if err != nil { - return err - } - err = terraform.RunTerraform(conf, conn, userid, executor, "") - if err != nil { - return err - } - return nil - } - } - } - - return nil -} - -func UninstallApplicationNew(appname string, deploymentname string, displayname string, conf *config.AppConfig, store database.Datastore) error { +func UninstallApplication(application *types.InstalledMarketplaceApplication, conf *config.AppConfig, store database.Datastore) error { // Create uninstall_logs directory if it doesn't exist logDir := path.Join(conf.Workdir, "uninstall_logs") if err := os.MkdirAll(logDir, 0755); err != nil && !os.IsExist(err) { return fmt.Errorf("failed to create install_logs directory: %w", err) } + logfile := path.Join(logDir, fmt.Sprintf("%s_%s_uninstall_log", application.Name, application.DeploymentName)) executor := &terraform.RealTerraformExecutor{} - filepath := path.Join(conf.Workdir, "workspace") + application.Status = "UNINSTALLING" + store.UpdateInstalledMarketplaceApplication(application) - files, err := ioutil.ReadDir(filepath) + // Run a terraform destroy on the module to be uninstalled + err := terraform.DestroyTerraformModule(conf, logfile, executor, application.TerraformModuleName) if err != nil { return err } - for _, file := range files { - log.Infof("Checking file %s has prefix: %s", file.Name(), appname) - if strings.HasPrefix(file.Name(), appname) { - log.Infof("File was a match") - // Open the file - f, err := os.Open(path.Join(filepath, file.Name())) - if err != nil { - return err - } - - // Read comments at the top - scanner := bufio.NewScanner(f) - metadata := make(map[string]string) - for scanner.Scan() { - line := scanner.Text() - if !strings.HasPrefix(line, "#") { - break - } - // Parsing the comments - parts := strings.SplitN(strings.TrimPrefix(line, "# "), ": ", 2) - if len(parts) == 2 { - key := parts[0] - value := strings.TrimSpace(parts[1]) - metadata[key] = value - } - } - f.Close() - - // Check applicationName from the comments and delete the file if it matches - log.Infof("Check if appname %s == %s", metadata["applicationName"], displayname) - if metadata["applicationName"] == displayname { - p := path.Join(filepath, file.Name()) - log.Infof("Attempting to delete file: %s", p) - err = os.Remove(p) - if err != nil { - id, err := store.FetchDeploymentIDByName(deploymentname) - log.WithError(err).Error("Failed to fetch deployment ID by name when removing application") - err = store.UpdateApplicationStatus(id, appname, displayname, "UNINSTALL FAILED") - log.WithError(err).Error("Failed to update application status removing application") - return err - } - logfile := path.Join(logDir, fmt.Sprintf("%s_uninstall_log", deploymentname)) - err = terraform.RunTerraformLogOutToFile(conf, logfile, executor, "") - if err != nil { - log.WithError(err).Error("Failed to uninstall application") - return err - } - - // err := store.RemoveApplicationByName(deploymentname, appname) - // if err != nil { - // id, err := store.FetchDeploymentIDByName(deploymentname) - // log.WithError(err).Error("Failed to fetch deployment ID by name when removing application") - // err = store.UpdateApplicationStatus(id, appname, displayname, "UNINSTALL FAILED") - // log.WithError(err).Error("Failed to update application status removing application") - // return err - // } - id, err := store.FetchDeploymentIDByName(deploymentname) - err = store.UpdateApplicationStatus(id, appname, displayname, "UNINSTALLED") - err = fetchAllApplications(store) - if err != nil { - return err - } - - return nil - } - } - } - - return nil -} - -func UninstallApplicationNewV2(appName string, version string, deploymentName string, conf *config.AppConfig, store database.Datastore) error { - // Create uninstall_logs directory if it doesn't exist - logDir := path.Join(conf.Workdir, "uninstall_logs") - if err := os.MkdirAll(logDir, 0755); err != nil && !os.IsExist(err) { - return fmt.Errorf("failed to create install_logs directory: %w", err) - } - - executor := &terraform.RealTerraformExecutor{} - + // Find and delete the module files in our TF workspace filepath := path.Join(conf.Workdir, "workspace") files, err := ioutil.ReadDir(filepath) @@ -228,8 +78,8 @@ func UninstallApplicationNewV2(appName string, version string, deploymentName st } for _, file := range files { - log.Infof("Checking file %s has prefix: %s", file.Name(), appName) - if strings.HasPrefix(file.Name(), appName) { + log.Infof("Checking file %s has prefix: %s", file.Name(), application.Name) + if strings.HasPrefix(file.Name(), application.Name) { log.Infof("File was a match") // Open the file f, err := os.Open(path.Join(filepath, file.Name())) @@ -256,37 +106,23 @@ func UninstallApplicationNewV2(appName string, version string, deploymentName st f.Close() // Check applicationName from the comments and delete the file if it matches - log.Infof("Check if appname %s == %s", metadata["applicationName"], deploymentName) - if metadata["applicationName"] == deploymentName { + log.Infof("Check if appname %s == %s", metadata["applicationName"], application.DeploymentName) + if metadata["applicationName"] == application.DeploymentName { p := path.Join(filepath, file.Name()) log.Infof("Attempting to delete file: %s", p) err = os.Remove(p) - // if err != nil { - // log.WithError(err).Error("Failed to fetch deployment ID by name when removing application") - // err = store.UpdateApplicationStatus(id, appname, deploymentName, "UNINSTALL FAILED") - // log.WithError(err).Error("Failed to update application status removing application") - // return err - // } - store.UpdateInstalledMarketplaceApplicationStatusByName(appName, deploymentName, "STARTING UNINSTALL") - logfile := path.Join(logDir, fmt.Sprintf("%s_%s_uninstall_log", appName, deploymentName)) + + // Run terraform apply on modified workspace + logfile := path.Join(logDir, fmt.Sprintf("%s_%s_uninstall_log", application.Name, application.DeploymentName)) err = terraform.RunTerraformLogOutToFile(conf, logfile, executor, "") if err != nil { log.WithError(err).Error("Failed to uninstall application") return err } - // err = store.RemoveInstalledMarketplaceApplicationByName(appName) + application.Status = "UNINSTALLED" + store.UpdateInstalledMarketplaceApplication(application) - // err := store.RemoveApplicationByName(deploymentname, appname) - // if err != nil { - // id, err := store.FetchDeploymentIDByName(deploymentname) - // log.WithError(err).Error("Failed to fetch deployment ID by name when removing application") - // err = store.UpdateApplicationStatus(id, appname, deploymentName, "UNINSTALL FAILED") - // log.WithError(err).Error("Failed to update application status removing application") - // return err - // } - // id, err := store.FetchDeploymentIDByName(deploymentname) - err = store.UpdateInstalledMarketplaceApplicationStatusByName(appName, deploymentName, "UNINSTALLED") err = fetchAllApplications(store) if err != nil { return err diff --git a/backend/internal/terraform/management.go b/backend/internal/terraform/management.go index 6cb2e4e..38e9946 100644 --- a/backend/internal/terraform/management.go +++ b/backend/internal/terraform/management.go @@ -82,7 +82,6 @@ func convertToCty(data interface{}) cty.Value { return cty.NilVal } - func parseAdvancedVariables(advancedVars *types.AdvancedValue, cloudenv *map[string]cty.Value) { if advancedVars == nil { return @@ -157,20 +156,19 @@ func appendBlockToBody(body *hclwrite.Body, blockType string, labels []string, s } } - // AddApplicationToStack adds the given application configuration to the stack. // It takes care of creating the necessary workspace directory, generating the // HCL file, and writing the required attributes. -func AddApplicationToStack(appConfig *config.AppConfig, location string, meta *marketplace.MarketplaceMetadata, installParams *types.ApplicationInstallParams, db database.Datastore) error { - log.Infof("Adding application to stack. Location: %v, meta %v, install: %v, deploymentID: %v", location, meta, installParams) +func AddApplicationToStack(appConfig *config.AppConfig, location string, meta *marketplace.MarketplaceMetadata, application *types.InstalledMarketplaceApplication, db database.Datastore) error { + log.Infof("Adding application to stack. Location: %v, meta %v, install: %v, module ID: %s", location, meta, application.TerraformModuleName) rand.Seed(time.Now().UnixNano()) // s := GenerateRandomString(8) hclFile := hclwrite.NewEmptyFile() directory := filepath.Join(appConfig.Workdir, "workspace") - log.Errorf("Application name: %s", installParams.Name) - filename := fmt.Sprintf("%v-%v.tf", installParams.Name, installParams.DeploymentName) + log.Errorf("Application name: %s", application.Name) + filename := fmt.Sprintf("%v-%v.tf", application.Name, application.DeploymentName) log.Errorf("Creating file with the name: %s", filename) tfFile, err := createFile(directory, filename, 0755) @@ -190,11 +188,11 @@ func AddApplicationToStack(appConfig *config.AppConfig, location string, meta *m } log.Info("Generating header") - generateMetadataHeader(rootBody, u.String(), meta.Name, installParams.DeploymentName, installParams.Version, "admin") + generateMetadataHeader(rootBody, u.String(), meta.Name, application.DeploymentName, application.Version, "admin") log.Info("adding attributes") attributes := map[string]cty.Value{ - "deployment_name": cty.StringVal(installParams.DeploymentName), + "deployment_name": cty.StringVal(application.DeploymentName), "tags": cty.MapValEmpty(cty.String), // Example of setting an empty map "project": cty.StringVal(appConfig.Project), "venue": cty.StringVal(appConfig.Venue), @@ -202,14 +200,14 @@ func AddApplicationToStack(appConfig *config.AppConfig, location string, meta *m } log.Info("Organising variable replacement") - if installParams.Variables != nil { - for key, element := range installParams.Variables { + if application.Variables != nil { + for key, element := range application.Variables { log.Infof("Adding variable: %s, %s", key, element) attributes[key] = cty.StringVal(element) } } log.Info("Parsing advanced vars") - parseAdvancedVariables(&installParams.AdvancedValues, &attributes) + parseAdvancedVariables(&application.AdvancedValues, &attributes) rand.Seed(time.Now().UnixNano()) chars := "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" randomChars := make([]byte, 5) @@ -217,7 +215,7 @@ func AddApplicationToStack(appConfig *config.AppConfig, location string, meta *m randomChars[i] = chars[v] } log.Info("Appending block to body") - appendBlockToBody(rootBody, "module", []string{fmt.Sprintf("%s-%s", installParams.DeploymentName, string(randomChars))}, path, attributes) + appendBlockToBody(rootBody, "module", []string{application.TerraformModuleName}, path, attributes) log.Info("Writing hcl file.") _, err = tfFile.Write(hclFile.Bytes()) @@ -241,6 +239,7 @@ func lookUpVariablePointer(element string, inst *marketplace.Install) (string, e return "", nil } + func lookUpFromDependencies(element string, inst *marketplace.Install_Applications) (string, error) { deps := inst.Dependencies for k, v := range deps { diff --git a/backend/internal/terraform/runner.go b/backend/internal/terraform/runner.go index 3dcb525..708c28c 100644 --- a/backend/internal/terraform/runner.go +++ b/backend/internal/terraform/runner.go @@ -25,6 +25,7 @@ type TerraformExecutor interface { Init(context.Context, ...tfexec.InitOption) error Plan(context.Context, ...tfexec.PlanOption) (bool, error) Apply(context.Context, ...tfexec.ApplyOption) error + Destroy(context.Context, ...tfexec.DestroyOption) error SetStdout(io.Writer) SetStderr(io.Writer) SetLogger(*log.Logger) @@ -56,6 +57,10 @@ func (r *RealTerraformExecutor) Apply(ctx context.Context, opts ...tfexec.ApplyO return r.tf.Apply(ctx, opts...) } +func (r *RealTerraformExecutor) Destroy(ctx context.Context, opts ...tfexec.DestroyOption) error { + return r.tf.Destroy(ctx, opts...) +} + func (r *RealTerraformExecutor) SetStdout(w io.Writer) { r.tf.SetStdout(w) } @@ -268,6 +273,57 @@ func RunTerraformLogOutToFile(appconf *config.AppConfig, logfile string, executo return nil } + +func DestroyTerraformModule(appconf *config.AppConfig, logfile string, executor TerraformExecutor, moduleName string) error { + bucket := fmt.Sprintf("bucket=%s", appconf.BucketName) + key := fmt.Sprintf("key=%s-%s-tfstate", appconf.Project, appconf.Venue) + region := fmt.Sprintf("region=%s", appconf.AWSRegion) + + p := filepath.Join(appconf.Workdir, "workspace") + tf, err := executor.NewTerraform(p, "/usr/local/bin/terraform") + if err != nil { + log.Fatalf("error running Terraform: %s", err) + } + + // Open the log file in append mode + logfileHandle, err := os.OpenFile(logfile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + log.Fatalf("error opening log file: %s", err) + } + defer logfileHandle.Close() + + // Create a multi-writer that writes to both file and stdout + writerStdout := io.MultiWriter(os.Stdout, logfileHandle) + writerStderr := io.MultiWriter(os.Stderr, logfileHandle) + + tf.SetStdout(writerStdout) + tf.SetStderr(writerStderr) + tf.SetLogger(log.StandardLogger()) + + logfileHandle.WriteString("Starting Terraform") + + err = executor.Init(context.Background(), tfexec.Upgrade(true), tfexec.BackendConfig(bucket), tfexec.BackendConfig(key), tfexec.BackendConfig(region)) + + if err != nil { + log.WithError(err).Error("error initialising terraform") + logfileHandle.WriteString(fmt.Sprintf("Error initialising terraform: %s\n", err)) + return err + } + + logfileHandle.WriteString("Waiting 60 seconds for the state to settle\n") + time.Sleep(60 * time.Second) + + err = executor.Destroy(context.Background(), tfexec.Target(fmt.Sprintf("module.%s", moduleName))) + if err != nil { + log.WithError(err).Error("error running terraform destroy") + logfileHandle.WriteString(fmt.Sprintf("Error running destroy: %s\n", err)) + return err + } + + logfileHandle.WriteString("Terraform destroy completed successfully\n") + return nil +} + func DestroyAllTerraform(appconf *config.AppConfig, wsmgr *ws.WebSocketManager, id string, executor TerraformExecutor) error { p := filepath.Join(appconf.Workdir, "workspace") @@ -289,7 +345,7 @@ func DestroyAllTerraform(appconf *config.AppConfig, wsmgr *ws.WebSocketManager, userid: id, wsmgr: wsmgr, level: "ERROR", - } + } writerStdout = io.MultiWriter(os.Stdout, wwsWriter) writerStderr = io.MultiWriter(os.Stderr, wwserrWriter) } @@ -310,7 +366,7 @@ func DestroyAllTerraform(appconf *config.AppConfig, wsmgr *ws.WebSocketManager, wrap := marketplace.UnityWebsocketMessage{Content: &om} b, _ := proto.Marshal(&wrap) wsmgr.SendMessageToUserID(id, b) - return err + return err } } @@ -322,7 +378,7 @@ func DestroyAllTerraform(appconf *config.AppConfig, wsmgr *ws.WebSocketManager, om := marketplace.UnityWebsocketMessage_Simplemessage{Simplemessage: &message} wrap := marketplace.UnityWebsocketMessage{Content: &om} b, _ := proto.Marshal(&wrap) - wsmgr.SendMessageToUserID(id, b) + wsmgr.SendMessageToUserID(id, b) } return nil } diff --git a/backend/internal/web/api.go b/backend/internal/web/api.go index 5cdd062..5a4493c 100644 --- a/backend/internal/web/api.go +++ b/backend/internal/web/api.go @@ -79,40 +79,15 @@ func handleApplicationInstall(appConfig config.AppConfig, db database.Datastore) log.Errorf("Got JSON: %v", applicationInstallParams) - // First check if this application is already installed. - existingApplication, err := db.GetInstalledMarketplaceApplicationStatusByName(applicationInstallParams.Name, applicationInstallParams.DeploymentName) - if err != nil { - log.WithError(err).Error("Error finding applications") - c.JSON(http.StatusInternalServerError, gin.H{"error": "Unable to search applcation list"}) - } - - if existingApplication != nil && existingApplication.Status != "UNINSTALLED" { - errMsg := fmt.Sprintf("Application with name %s already exists. Status: %s", applicationInstallParams.Name, existingApplication.Status) - c.JSON(http.StatusInternalServerError, gin.H{"error": errMsg}) - return - } - - // OK to start installing, get the metadata for this application - metadata, err := processes.FetchMarketplaceMetadata(applicationInstallParams.Name, applicationInstallParams.Version, &appConfig) - if err != nil { - log.Errorf("Unable to fetch metadata for application: %s, %v", applicationInstallParams.Name, err) - errMsg := fmt.Sprintf("Unable to fetch package metatadata: %v", err) - c.JSON(http.StatusBadRequest, gin.H{"error": errMsg}) - return - } + // Kick off install process in async mode. Errors will come back from the initial checks, otherwise we can return OK to user. + err = processes.TriggerInstall(db, &applicationInstallParams, &appConfig, false) - // Install the application package files - location, err := processes.FetchPackage(&metadata, &appConfig) if err != nil { - log.Errorf("Unable to fetch package for application: %s, %v", applicationInstallParams.Name, err) - c.JSON(http.StatusInternalServerError, gin.H{"error": "Unable to fetch package"}) + c.JSON(http.StatusBadRequest, gin.H{"error": err}) return } - - err = processes.InstallMarketplaceApplication(&appConfig, location, &applicationInstallParams, &metadata, db, false) c.Status(http.StatusOK) - } } @@ -158,10 +133,17 @@ func handleGetInstallLogs(appConfig config.AppConfig, db database.Datastore, uni func handleUninstallApplication(appConfig config.AppConfig, db database.Datastore) func(c *gin.Context) { return func(c *gin.Context) { appName := c.Param("appName") - version := c.Param("version") deploymentName := c.Param("deploymentName") - go processes.UninstallApplicationNewV2(appName, version, deploymentName, &conf, db) + app, err := db.GetInstalledMarketplaceApplication(appName, deploymentName) + + if err != nil { + log.Errorf("Installed application not found: %v", err) + c.Status(http.StatusNotFound) + return + } + + go processes.UninstallApplication(app, &conf, db) } } @@ -169,7 +151,7 @@ func handleGetApplicationInstallStatusByName(appConfig config.AppConfig, db data return func(c *gin.Context) { appName := c.Param("appName") deploymentName := c.Param("deploymentName") - app, err := db.GetInstalledMarketplaceApplicationStatusByName(appName, deploymentName) + app, err := db.GetInstalledMarketplaceApplication(appName, deploymentName) if err != nil { log.Errorf("Error reading application status: %v", err) @@ -198,7 +180,7 @@ func handleDeleteApplication(appConfig config.AppConfig, db database.Datastore) appName := c.Param("appName") deploymentName := c.Param("deploymentName") - existingApplication, err := db.GetInstalledMarketplaceApplicationStatusByName(appName, deploymentName) + existingApplication, err := db.GetInstalledMarketplaceApplication(appName, deploymentName) if existingApplication == nil { log.Errorf("Unable to find application %s and deployment %s", appName, deploymentName) c.JSON(http.StatusUnprocessableEntity, gin.H{"error": "Application or deployment name doesn't exist."}) @@ -213,27 +195,3 @@ func handleDeleteApplication(appConfig config.AppConfig, db database.Datastore) c.Status(http.StatusOK) } } - -// func handleGetAPICall(appConfig config.AppConfig) gin.HandlerFunc { -// fn := func(c *gin.Context) { -// switch endpoint := c.Param("endpoint"); endpoint { -// case "health_checks": -// handleHealthChecks(c, appConfig) -// default: -// handleNoRoute(c) -// } -// } -// return gin.HandlerFunc(fn) -// } - -// func handlePostAPICall(appConfig config.AppConfig) gin.HandlerFunc { -// fn := func(c *gin.Context) { -// switch endpoint := c.Param("endpoint"); endpoint { -// case "uninstall": -// handleUninstall(c, appConfig) -// default: -// handleNoRoute(c) -// } -// } -// return gin.HandlerFunc(fn) -// } diff --git a/backend/types/marketplace.go b/backend/types/marketplace.go index 7b725d4..5ad3ac9 100644 --- a/backend/types/marketplace.go +++ b/backend/types/marketplace.go @@ -10,3 +10,16 @@ type ApplicationInstallParams struct { Variables map[string]string AdvancedValues AdvancedValue } + + +type InstalledMarketplaceApplication struct { + Name string + DeploymentName string + Version string + Source string + Status string + PackageName string + TerraformModuleName string + Variables map[string]string + AdvancedValues AdvancedValue +} \ No newline at end of file diff --git a/src/components/ApplicationPanelItem.svelte b/src/components/ApplicationPanelItem.svelte index d6b132e..a5c80db 100644 --- a/src/components/ApplicationPanelItem.svelte +++ b/src/components/ApplicationPanelItem.svelte @@ -1,5 +1,5 @@