From 92ac47ac0b4695a7952d2ed78dd388287b1d9808 Mon Sep 17 00:00:00 2001 From: Jonathan Meeks Date: Fri, 4 Apr 2025 17:50:53 -0500 Subject: [PATCH 01/13] initial cli interface to steampipe cmds --- .gitignore | 3 + cmd/ctrlc/root/sync/steampipe/steampipe.go | 108 +++++++++++++++++++++ cmd/ctrlc/root/sync/sync.go | 5 +- 3 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 .gitignore create mode 100644 cmd/ctrlc/root/sync/steampipe/steampipe.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..138410f --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ + +.idea/ +bin/ \ No newline at end of file diff --git a/cmd/ctrlc/root/sync/steampipe/steampipe.go b/cmd/ctrlc/root/sync/steampipe/steampipe.go new file mode 100644 index 0000000..56bc8e4 --- /dev/null +++ b/cmd/ctrlc/root/sync/steampipe/steampipe.go @@ -0,0 +1,108 @@ +package steampipe + +import ( + "fmt" + "os" + + "github.com/MakeNowJust/heredoc/v2" + "github.com/charmbracelet/log" + "github.com/spf13/cobra" +) + +type SteampipeClient struct { + ExecCmd string `default:"steampipe"` +} + +func NewSteampipeClient() *SteampipeClient { + return &SteampipeClient{} +} + +func (c *SteampipeClient) ListResourceGroups() ([]string, error) { + // Simulate fetching resource groups from Steampipe + resourceGroups := []string{"group1", "group2", "group3"} + return resourceGroups, nil +} + +func (c *SteampipeClient) SendResourcesFromGroup(resourceGroup string) ([]string, error) { + // Simulate sending resources from a specific group to Ctrlplane + resourceGroups := []string{resourceGroup} + return resourceGroups, nil +} + +func NewSyncSteampipeCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "steampipe ", + Short: "Subcommands for integrating steampipe with Ctrlplane", + Example: heredoc.Doc(` + $ ctrlc sync steampipe list # Show which resourceGroups are available + $ ctrlc sync steampipe send # Send to Ctrlplane the resource info for all resourceGroups + `), + } + + cmd.AddCommand(NewSyncSteampipeListCmd()) + cmd.AddCommand(NewSyncSteampipeSendCmd()) + + return cmd +} + +func NewSyncSteampipeListCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "list", + Short: "List all available resourceGroups", + RunE: func(cmd *cobra.Command, args []string) error { + log.Info("Listing all available resourceGroups") + + client := NewSteampipeClient() + + resourceGroups, err := client.ListResourceGroups() + if err != nil { + return err + } + + for _, group := range resourceGroups { + fmt.Println(group) + } + + return nil + }, + } + + return cmd +} + +func NewSyncSteampipeSendCmd() *cobra.Command { + var resourceGroup string + + cmd := &cobra.Command{ + Use: "send", + Short: "Send resource info to Ctrlplane", + PreRunE: func(cmd *cobra.Command, args []string) error { + if resourceGroup == "" { + return fmt.Errorf("resource-group must be provided") + } + return nil + }, + RunE: func(cmd *cobra.Command, args []string) error { + log.Info("Send all available resources") + + client := NewSteampipeClient() + + resourceGroups, err := client.SendResourcesFromGroup(resourceGroup) + if err != nil { + return err + } + + for _, group := range resourceGroups { + fmt.Println(group) + } + + return nil + }, + } + + cmd.Flags().StringVarP(&resourceGroup, "resource-group", "r", os.Getenv("STEAMPIPE_RESOURCE_GROUP"), "The resource group name") + + cmd.MarkFlagRequired("resource-group") + + return cmd +} diff --git a/cmd/ctrlc/root/sync/sync.go b/cmd/ctrlc/root/sync/sync.go index 9f8ace3..15886f7 100644 --- a/cmd/ctrlc/root/sync/sync.go +++ b/cmd/ctrlc/root/sync/sync.go @@ -3,6 +3,7 @@ package sync import ( "github.com/MakeNowJust/heredoc/v2" "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/clickhouse" + "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/steampipe" "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/tailscale" "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/terraform" "github.com/ctrlplanedev/cli/internal/cliutil" @@ -19,6 +20,7 @@ func NewSyncCmd() *cobra.Command { $ ctrlc sync tfe --interval 5m # Run every 5 minutes $ ctrlc sync tailscale --interval 1h # Run every hour $ ctrlc sync clickhouse # Run once + $ ctrlc sync steampipe # Run once `), } @@ -27,6 +29,7 @@ func NewSyncCmd() *cobra.Command { cmd.AddCommand(cliutil.AddIntervalSupport(terraform.NewSyncTerraformCmd(), "")) cmd.AddCommand(cliutil.AddIntervalSupport(tailscale.NewSyncTailscaleCmd(), "")) cmd.AddCommand(cliutil.AddIntervalSupport(clickhouse.NewSyncClickhouseCmd(), "")) + cmd.AddCommand(steampipe.NewSyncSteampipeCmd()) return cmd -} \ No newline at end of file +} From 40879c199d0db8ac043e176c716f633fc62b8218 Mon Sep 17 00:00:00 2001 From: Jonathan Meeks Date: Fri, 4 Apr 2025 21:33:19 -0500 Subject: [PATCH 02/13] first pass at listing resource groups --- cmd/ctrlc/root/sync/steampipe/client.go | 12 +++ cmd/ctrlc/root/sync/steampipe/list.go | 85 ++++++++++++++++++++++ cmd/ctrlc/root/sync/steampipe/send.go | 10 +++ cmd/ctrlc/root/sync/steampipe/steampipe.go | 28 +------ 4 files changed, 111 insertions(+), 24 deletions(-) create mode 100644 cmd/ctrlc/root/sync/steampipe/client.go create mode 100644 cmd/ctrlc/root/sync/steampipe/list.go create mode 100644 cmd/ctrlc/root/sync/steampipe/send.go diff --git a/cmd/ctrlc/root/sync/steampipe/client.go b/cmd/ctrlc/root/sync/steampipe/client.go new file mode 100644 index 0000000..509f197 --- /dev/null +++ b/cmd/ctrlc/root/sync/steampipe/client.go @@ -0,0 +1,12 @@ +package steampipe + +type SteampipeClient struct { + ExecCmd string `default:"steampipe"` +} + + +func NewSteampipeClient() *SteampipeClient { + return &SteampipeClient{} +} + + diff --git a/cmd/ctrlc/root/sync/steampipe/list.go b/cmd/ctrlc/root/sync/steampipe/list.go new file mode 100644 index 0000000..e8216d8 --- /dev/null +++ b/cmd/ctrlc/root/sync/steampipe/list.go @@ -0,0 +1,85 @@ +package steampipe + +import ( + "encoding/json" + "fmt" + "os/exec" + "strings" + + "github.com/charmbracelet/log" +) + +type SteampipePluginList struct { + Installed []PluginInfo `json:"installed"` + Failed *interface{} `json:"failed"` + Warnings *interface{} `json:"warnings"` +} + +type PluginInfo struct { + Name string `json:"name"` + Version string `json:"version"` + Connections []string `json:"connections"` +} + +type ResourceGroup struct { + ConnectionType string + Name string + ResourceType string +} + +func (c *SteampipeClient) ListResourceGroups() ([]ResourceGroup, error) { + + cmd := exec.Command("steampipe", "plugin", "list", "--output", "json") + output, err := cmd.CombinedOutput() + if err != nil { + return nil, fmt.Errorf("failed to execute steampipe command: %w", err) + } + + // Simulate processing the output (e.g., parsing JSON) + // fmt.Printf("Command output: %s\n", output) + + pluginList := &SteampipePluginList{} + err = json.Unmarshal(output, pluginList) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal plugin list: %w", err) + } + + if pluginList.Failed != nil { + log.Errorf("Resource Group list failures: %v", pluginList.Failed) + } + + if pluginList.Warnings != nil { + log.Errorf("Resource Group list warnings: %v", pluginList.Warnings) + } + + resourceGroups := make([]ResourceGroup, 0) + + for _, plugin := range pluginList.Installed { + connectionType := getConnectionType(plugin.Name) + fmt.Printf("%s.*:[resource]\n", connectionType) + resourceGroups = append(resourceGroups, ResourceGroup{ + ConnectionType: connectionType, + Name: "*", + ResourceType: "*", + }) + for _, connection := range plugin.Connections { + // if pluginName == connection name, then it's covered by `pluginName.*` + if connection != connectionType { + fmt.Printf("%s.%s:[resource]\n", connectionType, connection) + resourceGroups = append(resourceGroups, ResourceGroup{ + ConnectionType: connectionType, + Name: connection, + ResourceType: "*", + }) + } + } + } + + return resourceGroups, nil +} + +func getConnectionType(pluginName string) string { + sansVersion := strings.Split(pluginName, "@")[0] + delimitedName := strings.Split(sansVersion, "/") + return delimitedName[len(delimitedName)-1] +} diff --git a/cmd/ctrlc/root/sync/steampipe/send.go b/cmd/ctrlc/root/sync/steampipe/send.go new file mode 100644 index 0000000..edf3c3f --- /dev/null +++ b/cmd/ctrlc/root/sync/steampipe/send.go @@ -0,0 +1,10 @@ +package steampipe + + + + +func (c *SteampipeClient) SendResourcesFromGroup(resourceGroup string) ([]string, error) { + // Simulate sending resources from a specific group to Ctrlplane + resourceGroups := []string{resourceGroup} + return resourceGroups, nil +} diff --git a/cmd/ctrlc/root/sync/steampipe/steampipe.go b/cmd/ctrlc/root/sync/steampipe/steampipe.go index 56bc8e4..1f3f7bb 100644 --- a/cmd/ctrlc/root/sync/steampipe/steampipe.go +++ b/cmd/ctrlc/root/sync/steampipe/steampipe.go @@ -9,26 +9,6 @@ import ( "github.com/spf13/cobra" ) -type SteampipeClient struct { - ExecCmd string `default:"steampipe"` -} - -func NewSteampipeClient() *SteampipeClient { - return &SteampipeClient{} -} - -func (c *SteampipeClient) ListResourceGroups() ([]string, error) { - // Simulate fetching resource groups from Steampipe - resourceGroups := []string{"group1", "group2", "group3"} - return resourceGroups, nil -} - -func (c *SteampipeClient) SendResourcesFromGroup(resourceGroup string) ([]string, error) { - // Simulate sending resources from a specific group to Ctrlplane - resourceGroups := []string{resourceGroup} - return resourceGroups, nil -} - func NewSyncSteampipeCmd() *cobra.Command { cmd := &cobra.Command{ Use: "steampipe ", @@ -54,14 +34,14 @@ func NewSyncSteampipeListCmd() *cobra.Command { client := NewSteampipeClient() - resourceGroups, err := client.ListResourceGroups() + _, err := client.ListResourceGroups() if err != nil { return err } - for _, group := range resourceGroups { - fmt.Println(group) - } + // for _, group := range resourceGroups { + // fmt.Println(group) + // } return nil }, From 78612e16faf442ffa371eddff85f8778bf097e6d Mon Sep 17 00:00:00 2001 From: Jonathan Meeks Date: Sat, 5 Apr 2025 08:52:48 -0500 Subject: [PATCH 03/13] steampipe resource provider Description ----------- `steampipe` may be used as a resource-provider for a myriad of cloud services. This first pass will target resources from: `aws`, `azure`, `gcp`, `github`, and `tfe` (Terraform Cloud). --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 138410f..175a8a5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .idea/ -bin/ \ No newline at end of file +bin/ +*.session.sql \ No newline at end of file From 1c888ecd8bd1f6e4a038ed7c7a66d6ba66552d09 Mon Sep 17 00:00:00 2001 From: Jonathan Meeks Date: Sun, 6 Apr 2025 11:15:20 -0500 Subject: [PATCH 04/13] connect to steampipe with postgres connection, not cli --- cmd/ctrlc/root/sync/steampipe/client.go | 41 +++++- cmd/ctrlc/root/sync/steampipe/list.go | 138 ++++++++++-------- cmd/ctrlc/root/sync/steampipe/registry/aws.go | 24 +++ .../root/sync/steampipe/registry/azure.go | 12 ++ .../root/sync/steampipe/registry/component.go | 20 +++ cmd/ctrlc/root/sync/steampipe/registry/gcp.go | 16 ++ .../sync/steampipe/registry/kubernetes.go | 32 ++++ .../root/sync/steampipe/registry/registry.go | 25 ++++ cmd/ctrlc/root/sync/steampipe/steampipe.go | 32 +++- go.mod | 1 + go.sum | 2 + 11 files changed, 274 insertions(+), 69 deletions(-) create mode 100644 cmd/ctrlc/root/sync/steampipe/registry/aws.go create mode 100644 cmd/ctrlc/root/sync/steampipe/registry/azure.go create mode 100644 cmd/ctrlc/root/sync/steampipe/registry/component.go create mode 100644 cmd/ctrlc/root/sync/steampipe/registry/gcp.go create mode 100644 cmd/ctrlc/root/sync/steampipe/registry/kubernetes.go create mode 100644 cmd/ctrlc/root/sync/steampipe/registry/registry.go diff --git a/cmd/ctrlc/root/sync/steampipe/client.go b/cmd/ctrlc/root/sync/steampipe/client.go index 509f197..25c2c96 100644 --- a/cmd/ctrlc/root/sync/steampipe/client.go +++ b/cmd/ctrlc/root/sync/steampipe/client.go @@ -1,12 +1,47 @@ package steampipe +import ( + "database/sql" + "fmt" + + _ "github.com/lib/pq" +) + type SteampipeClient struct { - ExecCmd string `default:"steampipe"` + Connection string `default:"connection"` + db *sql.DB } +func NewSteampipeClient(spConnection string) (*SteampipeClient, error) { + client := &SteampipeClient{ + Connection: spConnection, + } + + // Initialize database connection + db, err := sql.Open("postgres", spConnection) + if err != nil { + return nil, fmt.Errorf("failed to open database connection: %w", err) + } -func NewSteampipeClient() *SteampipeClient { - return &SteampipeClient{} + // Test the connection + if err := db.Ping(); err != nil { + db.Close() + return nil, fmt.Errorf("failed to connect to database: %w", err) + } + + client.db = db + return client, nil } +// Close closes the database connection +func (c *SteampipeClient) Close() error { + if c.db != nil { + return c.db.Close() + } + return nil +} +// GetDB returns the underlying database connection +func (c *SteampipeClient) GetDB() *sql.DB { + return c.db +} diff --git a/cmd/ctrlc/root/sync/steampipe/list.go b/cmd/ctrlc/root/sync/steampipe/list.go index e8216d8..bf02b49 100644 --- a/cmd/ctrlc/root/sync/steampipe/list.go +++ b/cmd/ctrlc/root/sync/steampipe/list.go @@ -1,85 +1,105 @@ package steampipe import ( - "encoding/json" "fmt" - "os/exec" - "strings" + "sort" - "github.com/charmbracelet/log" + "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/steampipe/registry" ) -type SteampipePluginList struct { - Installed []PluginInfo `json:"installed"` - Failed *interface{} `json:"failed"` - Warnings *interface{} `json:"warnings"` +type Connection struct { + Name string + Type string + ResourceType string } -type PluginInfo struct { - Name string `json:"name"` - Version string `json:"version"` - Connections []string `json:"connections"` +// ForeignTable represents a row from information_schema.foreign_tables +type ForeignTable struct { + ForeignTableCatalog string + ForeignTableSchema string + ForeignTableName string + ForeignServerName string } -type ResourceGroup struct { - ConnectionType string - Name string - ResourceType string -} - -func (c *SteampipeClient) ListResourceGroups() ([]ResourceGroup, error) { +func (c *SteampipeClient) ListResourceGroups() ([]Connection, error) { + resourceGroups := make([]Connection, 0) - cmd := exec.Command("steampipe", "plugin", "list", "--output", "json") - output, err := cmd.CombinedOutput() + foreignTables, err := c.GetSteampipeForeignTables() if err != nil { - return nil, fmt.Errorf("failed to execute steampipe command: %w", err) + return nil, fmt.Errorf("failed to get foreign tables: %w", err) } - // Simulate processing the output (e.g., parsing JSON) - // fmt.Printf("Command output: %s\n", output) - - pluginList := &SteampipePluginList{} - err = json.Unmarshal(output, pluginList) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal plugin list: %w", err) + for _, foreignTable := range foreignTables { + component, ok := registry.GetAccessInfo(foreignTable.ForeignTableName) + if !ok { + continue + } + connectionName := foreignTable.ForeignTableSchema + if component.ConnectionType == connectionName { + connectionName = "*" + } + resourceGroups = append(resourceGroups, Connection{ + Name: connectionName, + Type: component.ConnectionType, + ResourceType: component.ResourceType, + }) } - if pluginList.Failed != nil { - log.Errorf("Resource Group list failures: %v", pluginList.Failed) + fmt.Printf("%-30s %-20s %-30s\n", "connection-name", "connection-type", "resource-type") + fmt.Printf("%-30s %-20s %-30s\n", "------------------------------", "--------------------", "--------------------------------") + sort.Slice(resourceGroups, func(i, j int) bool { + if resourceGroups[i].Name == resourceGroups[j].Name { + if resourceGroups[i].Type == resourceGroups[j].Type { + return resourceGroups[i].ResourceType < resourceGroups[j].ResourceType + } + return resourceGroups[i].Type < resourceGroups[j].Type + } + return resourceGroups[i].Name < resourceGroups[j].Name + }) + for _, rg := range resourceGroups { + fmt.Printf("%-30s %-20s %-30s\n", rg.Name, rg.Type, rg.ResourceType) } - if pluginList.Warnings != nil { - log.Errorf("Resource Group list warnings: %v", pluginList.Warnings) - } + return resourceGroups, nil +} - resourceGroups := make([]ResourceGroup, 0) +// GetSteampipeForeignTables returns all foreign tables where foreign_table_catalog is 'steampipe' +func (c *SteampipeClient) GetSteampipeForeignTables() ([]ForeignTable, error) { + query := ` + SELECT + foreign_table_catalog, + foreign_table_schema, + foreign_table_name, + foreign_server_name + FROM information_schema.foreign_tables + WHERE foreign_table_catalog = 'steampipe' + ORDER BY foreign_table_schema, foreign_table_name + ` - for _, plugin := range pluginList.Installed { - connectionType := getConnectionType(plugin.Name) - fmt.Printf("%s.*:[resource]\n", connectionType) - resourceGroups = append(resourceGroups, ResourceGroup{ - ConnectionType: connectionType, - Name: "*", - ResourceType: "*", - }) - for _, connection := range plugin.Connections { - // if pluginName == connection name, then it's covered by `pluginName.*` - if connection != connectionType { - fmt.Printf("%s.%s:[resource]\n", connectionType, connection) - resourceGroups = append(resourceGroups, ResourceGroup{ - ConnectionType: connectionType, - Name: connection, - ResourceType: "*", - }) - } + rows, err := c.db.Query(query) + if err != nil { + return nil, fmt.Errorf("failed to query foreign tables: %w", err) + } + defer rows.Close() + + var tables []ForeignTable + for rows.Next() { + var table ForeignTable + err := rows.Scan( + &table.ForeignTableCatalog, + &table.ForeignTableSchema, + &table.ForeignTableName, + &table.ForeignServerName, + ) + if err != nil { + return nil, fmt.Errorf("failed to scan foreign table row: %w", err) } + tables = append(tables, table) } - return resourceGroups, nil -} + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("error iterating foreign table rows: %w", err) + } -func getConnectionType(pluginName string) string { - sansVersion := strings.Split(pluginName, "@")[0] - delimitedName := strings.Split(sansVersion, "/") - return delimitedName[len(delimitedName)-1] + return tables, nil } diff --git a/cmd/ctrlc/root/sync/steampipe/registry/aws.go b/cmd/ctrlc/root/sync/steampipe/registry/aws.go new file mode 100644 index 0000000..36020c8 --- /dev/null +++ b/cmd/ctrlc/root/sync/steampipe/registry/aws.go @@ -0,0 +1,24 @@ +package registry + +var awsComponents = []SteampipeAccessInfo{ + { + ConnectionType: "aws", + ResourceType: "eks_cluster", + }, + { + ConnectionType: "aws", + ResourceType: "vpc", + }, + { + ConnectionType: "aws", + ResourceType: "rds_db_cluster", + }, + { + ConnectionType: "aws", + ResourceType: "rds_db_instance", + }, + { + ConnectionType: "aws", + ResourceType: "elasticache_cluster", + }, +} diff --git a/cmd/ctrlc/root/sync/steampipe/registry/azure.go b/cmd/ctrlc/root/sync/steampipe/registry/azure.go new file mode 100644 index 0000000..0fbe446 --- /dev/null +++ b/cmd/ctrlc/root/sync/steampipe/registry/azure.go @@ -0,0 +1,12 @@ +package registry + +var azureComponents = []SteampipeAccessInfo{ + { + ConnectionType: "azure", + ResourceType: "kubernetes_cluster", + }, + { + ConnectionType: "azure", + ResourceType: "sql_database", + }, +} diff --git a/cmd/ctrlc/root/sync/steampipe/registry/component.go b/cmd/ctrlc/root/sync/steampipe/registry/component.go new file mode 100644 index 0000000..0300770 --- /dev/null +++ b/cmd/ctrlc/root/sync/steampipe/registry/component.go @@ -0,0 +1,20 @@ +package registry + +import "fmt" + +type SteampipeAccessInfo struct { + ConnectionType string + ResourceType string + TablePrefix string +} + +func (c *SteampipeAccessInfo) GetTablePrefix() string { + if c.TablePrefix == "" { + return c.ConnectionType + } + return c.TablePrefix +} + +func (c *SteampipeAccessInfo) TableName() string { + return fmt.Sprintf("%s_%s", c.GetTablePrefix(), c.ResourceType) +} diff --git a/cmd/ctrlc/root/sync/steampipe/registry/gcp.go b/cmd/ctrlc/root/sync/steampipe/registry/gcp.go new file mode 100644 index 0000000..0375d36 --- /dev/null +++ b/cmd/ctrlc/root/sync/steampipe/registry/gcp.go @@ -0,0 +1,16 @@ +package registry + +var gcpComponents = []SteampipeAccessInfo{ + { + ConnectionType: "gcp", + ResourceType: "kubernetes_cluster", + }, + { + ConnectionType: "gcp", + ResourceType: "sql_database", + }, + { + ConnectionType: "gcp", + ResourceType: "sql_database_instance", + }, +} diff --git a/cmd/ctrlc/root/sync/steampipe/registry/kubernetes.go b/cmd/ctrlc/root/sync/steampipe/registry/kubernetes.go new file mode 100644 index 0000000..e553013 --- /dev/null +++ b/cmd/ctrlc/root/sync/steampipe/registry/kubernetes.go @@ -0,0 +1,32 @@ +package registry + +var kubernetesComponents = []SteampipeAccessInfo{ + { + ConnectionType: "kubernetes", + ResourceType: "namespace", + }, + { + ConnectionType: "kubernetes", + ResourceType: "pod", + }, + { + ConnectionType: "kubernetes", + ResourceType: "node", + }, + { + ConnectionType: "kubernetes", + ResourceType: "service", + }, + { + ConnectionType: "kubernetes", + ResourceType: "deployment", + }, + { + ConnectionType: "kubernetes", + ResourceType: "job", + }, + { + ConnectionType: "kubernetes", + ResourceType: "cronjob", + }, +} diff --git a/cmd/ctrlc/root/sync/steampipe/registry/registry.go b/cmd/ctrlc/root/sync/steampipe/registry/registry.go new file mode 100644 index 0000000..7fa4b2b --- /dev/null +++ b/cmd/ctrlc/root/sync/steampipe/registry/registry.go @@ -0,0 +1,25 @@ +package registry + +func buildRegistry() []SteampipeAccessInfo { + components := []SteampipeAccessInfo{} + components = append(components, kubernetesComponents...) + components = append(components, awsComponents...) + components = append(components, gcpComponents...) + components = append(components, azureComponents...) + return components +} + +func buildTableNameRegistry() map[string]SteampipeAccessInfo { + tableNameRegistry := make(map[string]SteampipeAccessInfo) + for _, component := range buildRegistry() { + tableNameRegistry[component.TableName()] = component + } + return tableNameRegistry +} + +var tableNameRegistry = buildTableNameRegistry() + +func GetAccessInfo(tableName string) (SteampipeAccessInfo, bool) { + accessInfo, ok := tableNameRegistry[tableName] + return accessInfo, ok +} diff --git a/cmd/ctrlc/root/sync/steampipe/steampipe.go b/cmd/ctrlc/root/sync/steampipe/steampipe.go index 1f3f7bb..d3fc5ca 100644 --- a/cmd/ctrlc/root/sync/steampipe/steampipe.go +++ b/cmd/ctrlc/root/sync/steampipe/steampipe.go @@ -14,8 +14,8 @@ func NewSyncSteampipeCmd() *cobra.Command { Use: "steampipe ", Short: "Subcommands for integrating steampipe with Ctrlplane", Example: heredoc.Doc(` - $ ctrlc sync steampipe list # Show which resourceGroups are available - $ ctrlc sync steampipe send # Send to Ctrlplane the resource info for all resourceGroups + $ ctrlc sync steampipe list # Show which resource providers are available + $ ctrlc sync steampipe send # Send to Ctrlplane the resource info for all resourceGroups `), } @@ -26,15 +26,22 @@ func NewSyncSteampipeCmd() *cobra.Command { } func NewSyncSteampipeListCmd() *cobra.Command { + var spConnection string + cmd := &cobra.Command{ Use: "list", - Short: "List all available resourceGroups", + Short: "List all available resource-providers", RunE: func(cmd *cobra.Command, args []string) error { log.Info("Listing all available resourceGroups") - client := NewSteampipeClient() + client, err := NewSteampipeClient(spConnection) + if err != nil { + log.Error("Failed to create Steampipe client", "error", err) + return err + } + defer client.Close() - _, err := client.ListResourceGroups() + _, err = client.ListResourceGroups() if err != nil { return err } @@ -47,11 +54,15 @@ func NewSyncSteampipeListCmd() *cobra.Command { }, } + cmd.Flags().StringVarP(&spConnection, "steampipe-connection", "c", "", "The steampipe postgresql connection string to use") + cmd.MarkFlagRequired("steampipe-connection") + return cmd } func NewSyncSteampipeSendCmd() *cobra.Command { var resourceGroup string + var spConnection string cmd := &cobra.Command{ Use: "send", @@ -63,9 +74,14 @@ func NewSyncSteampipeSendCmd() *cobra.Command { return nil }, RunE: func(cmd *cobra.Command, args []string) error { - log.Info("Send all available resources") + log.Info("Sending resource groups to Ctrlplane") - client := NewSteampipeClient() + client, err := NewSteampipeClient(spConnection) + if err != nil { + log.Error("Failed to create Steampipe client", "error", err) + return err + } + defer client.Close() resourceGroups, err := client.SendResourcesFromGroup(resourceGroup) if err != nil { @@ -81,8 +97,10 @@ func NewSyncSteampipeSendCmd() *cobra.Command { } cmd.Flags().StringVarP(&resourceGroup, "resource-group", "r", os.Getenv("STEAMPIPE_RESOURCE_GROUP"), "The resource group name") + cmd.Flags().StringVarP(&spConnection, "steampipe-connection", "c", "", "The steampipe postgresql connection string to use") cmd.MarkFlagRequired("resource-group") + cmd.MarkFlagRequired("steampipe-connection") return cmd } diff --git a/go.mod b/go.mod index 7f17a6c..387d5ec 100644 --- a/go.mod +++ b/go.mod @@ -56,6 +56,7 @@ require ( github.com/invopop/yaml v0.3.1 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/lib/pq v1.10.9 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect diff --git a/go.sum b/go.sum index 6393431..0a04a0f 100644 --- a/go.sum +++ b/go.sum @@ -131,6 +131,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= From 5cb81d1a0a27e76d92192715299eac4848205e36 Mon Sep 17 00:00:00 2001 From: Jonathan Meeks Date: Mon, 7 Apr 2025 20:26:20 -0500 Subject: [PATCH 05/13] steampipe wip --- cmd/ctrlc/root/sync/steampipe/list.go | 86 +++++++++++++------ cmd/ctrlc/root/sync/steampipe/registry/aws.go | 30 ++++++- cmd/ctrlc/root/sync/steampipe/steampipe.go | 50 +++++++++-- go.mod | 1 + go.sum | 3 + 5 files changed, 135 insertions(+), 35 deletions(-) diff --git a/cmd/ctrlc/root/sync/steampipe/list.go b/cmd/ctrlc/root/sync/steampipe/list.go index bf02b49..0801c4a 100644 --- a/cmd/ctrlc/root/sync/steampipe/list.go +++ b/cmd/ctrlc/root/sync/steampipe/list.go @@ -8,9 +8,20 @@ import ( ) type Connection struct { - Name string - Type string - ResourceType string + Name string + Type string + SteampipeResource SteampipeResource + CtrlPlaneResource CtrlPlaneResource +} + +type CtrlPlaneResource struct { + Id string + Type string +} + +type SteampipeResource struct { + TableName string + ConnectionName string } // ForeignTable represents a row from information_schema.foreign_tables @@ -21,8 +32,23 @@ type ForeignTable struct { ForeignServerName string } -func (c *SteampipeClient) ListResourceGroups() ([]Connection, error) { - resourceGroups := make([]Connection, 0) +// SortedConnections implements sort.Interface for []Connection based on Name, Type, and CtrlPlaneResource.Type +type SortedConnections []Connection + +func (c SortedConnections) Len() int { return len(c) } +func (c SortedConnections) Swap(i, j int) { c[i], c[j] = c[j], c[i] } +func (c SortedConnections) Less(i, j int) bool { + if c[i].Name == c[j].Name { + if c[i].Type == c[j].Type { + return c[i].CtrlPlaneResource.Type < c[j].CtrlPlaneResource.Type + } + return c[i].Type < c[j].Type + } + return c[i].Name < c[j].Name +} + +func (c *SteampipeClient) ListConnections() ([]Connection, error) { + connections := make([]Connection, 0) foreignTables, err := c.GetSteampipeForeignTables() if err != nil { @@ -30,37 +56,43 @@ func (c *SteampipeClient) ListResourceGroups() ([]Connection, error) { } for _, foreignTable := range foreignTables { + + spResource := SteampipeResource{ + TableName: foreignTable.ForeignTableName, + ConnectionName: foreignTable.ForeignTableSchema, + } + component, ok := registry.GetAccessInfo(foreignTable.ForeignTableName) if !ok { continue } - connectionName := foreignTable.ForeignTableSchema - if component.ConnectionType == connectionName { - connectionName = "*" + + cpResource := CtrlPlaneResource{ + Id: "", + Type: component.ResourceType, } - resourceGroups = append(resourceGroups, Connection{ - Name: connectionName, - Type: component.ConnectionType, - ResourceType: component.ResourceType, - }) - } - fmt.Printf("%-30s %-20s %-30s\n", "connection-name", "connection-type", "resource-type") - fmt.Printf("%-30s %-20s %-30s\n", "------------------------------", "--------------------", "--------------------------------") - sort.Slice(resourceGroups, func(i, j int) bool { - if resourceGroups[i].Name == resourceGroups[j].Name { - if resourceGroups[i].Type == resourceGroups[j].Type { - return resourceGroups[i].ResourceType < resourceGroups[j].ResourceType - } - return resourceGroups[i].Type < resourceGroups[j].Type + // Connection name defaults to the schema name, but for the case + // where the schema name is the same as the .... TODO: fix this descr + name := foreignTable.ForeignTableSchema + if component.ConnectionType == name { + name = "*" } - return resourceGroups[i].Name < resourceGroups[j].Name - }) - for _, rg := range resourceGroups { - fmt.Printf("%-30s %-20s %-30s\n", rg.Name, rg.Type, rg.ResourceType) + + conn := Connection{ + Name: name, + Type: component.ConnectionType, + SteampipeResource: spResource, + CtrlPlaneResource: cpResource, + } + + connections = append(connections, conn) } - return resourceGroups, nil + // Replace the sort.Slice with sort.Sort using our custom sort.Interface + sort.Sort(SortedConnections(connections)) + + return connections, nil } // GetSteampipeForeignTables returns all foreign tables where foreign_table_catalog is 'steampipe' diff --git a/cmd/ctrlc/root/sync/steampipe/registry/aws.go b/cmd/ctrlc/root/sync/steampipe/registry/aws.go index 36020c8..9bf50b4 100644 --- a/cmd/ctrlc/root/sync/steampipe/registry/aws.go +++ b/cmd/ctrlc/root/sync/steampipe/registry/aws.go @@ -3,7 +3,35 @@ package registry var awsComponents = []SteampipeAccessInfo{ { ConnectionType: "aws", - ResourceType: "eks_cluster", + /* + IsType: func(resourceType string) bool { + return resourceType == "eks_cluster" + }, + Convert func(res) ResourceObj { + return ResourceObj{ + Name : fooo, + Version: "kubernetes/v1", + Kind: "ClusterApi", + Config: map[string]interface{}{ + "auth": map[string]interface{} { + method: "aws/eks", + region: "us-east-1", + cluster: "default", + accountId: "123456789012", + }, + status: "active", + name: "default", + }, + Identifier: "my-arn", + Metadata: map[string]interface{}{ + "bunch": "of", + "misc": "stuff", + "not": "excludeed", + }, + + }, + */ + ResourceType: "eks_cluster", }, { ConnectionType: "aws", diff --git a/cmd/ctrlc/root/sync/steampipe/steampipe.go b/cmd/ctrlc/root/sync/steampipe/steampipe.go index d3fc5ca..89da4d7 100644 --- a/cmd/ctrlc/root/sync/steampipe/steampipe.go +++ b/cmd/ctrlc/root/sync/steampipe/steampipe.go @@ -6,10 +6,22 @@ import ( "github.com/MakeNowJust/heredoc/v2" "github.com/charmbracelet/log" + "github.com/ctrlplanedev/cli/internal/api" + "github.com/olekukonko/tablewriter" "github.com/spf13/cobra" + "github.com/spf13/viper" ) func NewSyncSteampipeCmd() *cobra.Command { + apiURL := viper.GetString("url") + apiKey := viper.GetString("api-key") + workspaceId := viper.GetString("workspace") + + ctrlplaneClient, err := api.NewAPIKeyClientWithResponses(apiURL, apiKey) + if err != nil { + return fmt.Errorf("failed to create API client: %w", err) + } + cmd := &cobra.Command{ Use: "steampipe ", Short: "Subcommands for integrating steampipe with Ctrlplane", @@ -41,22 +53,46 @@ func NewSyncSteampipeListCmd() *cobra.Command { } defer client.Close() - _, err = client.ListResourceGroups() + connections, err := client.ListConnections() if err != nil { return err } - // for _, group := range resourceGroups { - // fmt.Println(group) - // } + // Create a new table writer + table := tablewriter.NewWriter(os.Stdout) + + // Set headers + table.SetHeader([]string{"Resource ID", "Resource Type", "Connection Name", "Steampipe Table"}) + + // Add rows + for _, conn := range connections { + table.Append([]string{ + conn.CtrlPlaneResource.Id, + conn.CtrlPlaneResource.Type, + conn.Name, + conn.SteampipeResource.TableName, + }) + } + + // Set table properties + table.SetAutoWrapText(false) + table.SetAutoFormatHeaders(true) + table.SetHeaderAlignment(tablewriter.ALIGN_LEFT) + table.SetAlignment(tablewriter.ALIGN_LEFT) + table.SetCenterSeparator("") + table.SetColumnSeparator("|") + table.SetRowSeparator("") + table.SetBorder(true) + table.SetTablePadding("\t") + table.SetNoWhiteSpace(true) + + // Render the table + table.Render() return nil }, } - cmd.Flags().StringVarP(&spConnection, "steampipe-connection", "c", "", "The steampipe postgresql connection string to use") - cmd.MarkFlagRequired("steampipe-connection") - return cmd } diff --git a/go.mod b/go.mod index 387d5ec..0270199 100644 --- a/go.mod +++ b/go.mod @@ -70,6 +70,7 @@ require ( github.com/muesli/termenv v0.15.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/oapi-codegen/oapi-codegen/v2 v2.4.1 // indirect + github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/perimeterx/marshmallow v1.1.5 // indirect github.com/pkg/errors v0.9.1 // indirect diff --git a/go.sum b/go.sum index 0a04a0f..5b448ee 100644 --- a/go.sum +++ b/go.sum @@ -143,6 +143,7 @@ github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxec github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= @@ -171,6 +172,8 @@ github.com/oapi-codegen/oapi-codegen/v2 v2.4.1 h1:ykgG34472DWey7TSjd8vIfNykXgjOg github.com/oapi-codegen/oapi-codegen/v2 v2.4.1/go.mod h1:N5+lY1tiTDV3V1BeHtOxeWXHoPVeApvsvjJqegfoaz8= github.com/oapi-codegen/runtime v1.1.1 h1:EXLHh0DXIJnWhdRPN2w4MXAzFyE4CskzhNLUmtpMYro= github.com/oapi-codegen/runtime v1.1.1/go.mod h1:SK9X900oXmPWilYR5/WKPzt3Kqxn/uS/+lbpREv+eCg= +github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= +github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= From fbf8334983369a8c0e28a25c1acf05d16394e6a6 Mon Sep 17 00:00:00 2001 From: jonathan meeks Date: Mon, 7 Apr 2025 22:29:37 -0500 Subject: [PATCH 06/13] steampipe adapter transition, wip --- .../steampipe/{registry => adapter}/aws.go | 4 +- .../steampipe/{registry => adapter}/azure.go | 4 +- .../steampipe/{registry => adapter}/gcp.go | 4 +- .../{registry => adapter}/kubernetes.go | 4 +- .../root/sync/steampipe/adapter/model.go | 25 ++++ .../{registry => adapter}/registry.go | 14 +- cmd/ctrlc/root/sync/steampipe/fetch.go | 52 +++++++ cmd/ctrlc/root/sync/steampipe/list.go | 137 ------------------ .../root/sync/steampipe/registry/component.go | 20 --- cmd/ctrlc/root/sync/steampipe/send.go | 10 -- cmd/ctrlc/root/sync/steampipe/steampipe.go | 78 ++++++++-- 11 files changed, 159 insertions(+), 193 deletions(-) rename cmd/ctrlc/root/sync/steampipe/{registry => adapter}/aws.go (94%) rename cmd/ctrlc/root/sync/steampipe/{registry => adapter}/azure.go (70%) rename cmd/ctrlc/root/sync/steampipe/{registry => adapter}/gcp.go (78%) rename cmd/ctrlc/root/sync/steampipe/{registry => adapter}/kubernetes.go (87%) create mode 100644 cmd/ctrlc/root/sync/steampipe/adapter/model.go rename cmd/ctrlc/root/sync/steampipe/{registry => adapter}/registry.go (56%) create mode 100644 cmd/ctrlc/root/sync/steampipe/fetch.go delete mode 100644 cmd/ctrlc/root/sync/steampipe/list.go delete mode 100644 cmd/ctrlc/root/sync/steampipe/registry/component.go delete mode 100644 cmd/ctrlc/root/sync/steampipe/send.go diff --git a/cmd/ctrlc/root/sync/steampipe/registry/aws.go b/cmd/ctrlc/root/sync/steampipe/adapter/aws.go similarity index 94% rename from cmd/ctrlc/root/sync/steampipe/registry/aws.go rename to cmd/ctrlc/root/sync/steampipe/adapter/aws.go index 9bf50b4..de312d6 100644 --- a/cmd/ctrlc/root/sync/steampipe/registry/aws.go +++ b/cmd/ctrlc/root/sync/steampipe/adapter/aws.go @@ -1,6 +1,6 @@ -package registry +package adapter -var awsComponents = []SteampipeAccessInfo{ +var awsAdapters = []SteampipeAdapter{ { ConnectionType: "aws", /* diff --git a/cmd/ctrlc/root/sync/steampipe/registry/azure.go b/cmd/ctrlc/root/sync/steampipe/adapter/azure.go similarity index 70% rename from cmd/ctrlc/root/sync/steampipe/registry/azure.go rename to cmd/ctrlc/root/sync/steampipe/adapter/azure.go index 0fbe446..7ed0910 100644 --- a/cmd/ctrlc/root/sync/steampipe/registry/azure.go +++ b/cmd/ctrlc/root/sync/steampipe/adapter/azure.go @@ -1,6 +1,6 @@ -package registry +package adapter -var azureComponents = []SteampipeAccessInfo{ +var azureComponents = []SteampipeAdapter{ { ConnectionType: "azure", ResourceType: "kubernetes_cluster", diff --git a/cmd/ctrlc/root/sync/steampipe/registry/gcp.go b/cmd/ctrlc/root/sync/steampipe/adapter/gcp.go similarity index 78% rename from cmd/ctrlc/root/sync/steampipe/registry/gcp.go rename to cmd/ctrlc/root/sync/steampipe/adapter/gcp.go index 0375d36..24353a2 100644 --- a/cmd/ctrlc/root/sync/steampipe/registry/gcp.go +++ b/cmd/ctrlc/root/sync/steampipe/adapter/gcp.go @@ -1,6 +1,6 @@ -package registry +package adapter -var gcpComponents = []SteampipeAccessInfo{ +var gcpComponents = []SteampipeAdapter{ { ConnectionType: "gcp", ResourceType: "kubernetes_cluster", diff --git a/cmd/ctrlc/root/sync/steampipe/registry/kubernetes.go b/cmd/ctrlc/root/sync/steampipe/adapter/kubernetes.go similarity index 87% rename from cmd/ctrlc/root/sync/steampipe/registry/kubernetes.go rename to cmd/ctrlc/root/sync/steampipe/adapter/kubernetes.go index e553013..a165bd1 100644 --- a/cmd/ctrlc/root/sync/steampipe/registry/kubernetes.go +++ b/cmd/ctrlc/root/sync/steampipe/adapter/kubernetes.go @@ -1,6 +1,6 @@ -package registry +package adapter -var kubernetesComponents = []SteampipeAccessInfo{ +var kubernetesComponents = []SteampipeAdapter{ { ConnectionType: "kubernetes", ResourceType: "namespace", diff --git a/cmd/ctrlc/root/sync/steampipe/adapter/model.go b/cmd/ctrlc/root/sync/steampipe/adapter/model.go new file mode 100644 index 0000000..58bda46 --- /dev/null +++ b/cmd/ctrlc/root/sync/steampipe/adapter/model.go @@ -0,0 +1,25 @@ +package adapter + +import ( + "fmt" + "github.com/ctrlplanedev/cli/internal/api" +) + +type SteampipeAdapter struct { + ConnectionType string + ResourceType string + TablePrefix string + Translate func(data map[string]interface{}) api.AgentResource + IsCompatible func(data map[string]interface{}) bool +} + +func (c *SteampipeAdapter) GetTablePrefix() string { + if c.TablePrefix == "" { + return c.ConnectionType + } + return c.TablePrefix +} + +func (c *SteampipeAdapter) TableName() string { + return fmt.Sprintf("%s_%s", c.GetTablePrefix(), c.ResourceType) +} diff --git a/cmd/ctrlc/root/sync/steampipe/registry/registry.go b/cmd/ctrlc/root/sync/steampipe/adapter/registry.go similarity index 56% rename from cmd/ctrlc/root/sync/steampipe/registry/registry.go rename to cmd/ctrlc/root/sync/steampipe/adapter/registry.go index 7fa4b2b..7950511 100644 --- a/cmd/ctrlc/root/sync/steampipe/registry/registry.go +++ b/cmd/ctrlc/root/sync/steampipe/adapter/registry.go @@ -1,16 +1,16 @@ -package registry +package adapter -func buildRegistry() []SteampipeAccessInfo { - components := []SteampipeAccessInfo{} +func buildRegistry() []SteampipeAdapter { + components := []SteampipeAdapter{} components = append(components, kubernetesComponents...) - components = append(components, awsComponents...) + components = append(components, awsAdapters...) components = append(components, gcpComponents...) components = append(components, azureComponents...) return components } -func buildTableNameRegistry() map[string]SteampipeAccessInfo { - tableNameRegistry := make(map[string]SteampipeAccessInfo) +func buildTableNameRegistry() map[string]SteampipeAdapter { + tableNameRegistry := make(map[string]SteampipeAdapter) for _, component := range buildRegistry() { tableNameRegistry[component.TableName()] = component } @@ -19,7 +19,7 @@ func buildTableNameRegistry() map[string]SteampipeAccessInfo { var tableNameRegistry = buildTableNameRegistry() -func GetAccessInfo(tableName string) (SteampipeAccessInfo, bool) { +func GetAccessInfo(tableName string) (SteampipeAdapter, bool) { accessInfo, ok := tableNameRegistry[tableName] return accessInfo, ok } diff --git a/cmd/ctrlc/root/sync/steampipe/fetch.go b/cmd/ctrlc/root/sync/steampipe/fetch.go new file mode 100644 index 0000000..560f7d2 --- /dev/null +++ b/cmd/ctrlc/root/sync/steampipe/fetch.go @@ -0,0 +1,52 @@ +package steampipe + +import ( + "fmt" + "github.com/ctrlplanedev/cli/internal/api" +) + +func (c *SteampipeClient) Fetch(tableName string) ([]api.AgentResource, error) { + resources := make([]api.AgentResource, 0) + + results, err := c.SelectAll(tableName) + if err != nil { + return nil, fmt.Errorf("failed to fetch from %s table: %w", tableName, err) + } + + for _, result := range results { + + fmt.Println(result) + + resources = append(resources, api.AgentResource{}) + } + + return resources, nil +} + +// SelectAll returns all foreign tables where foreign_table_catalog is 'steampipe' +func (c *SteampipeClient) SelectAll(tableName string) ([]map[string]interface{}, error) { + results := make([]map[string]interface{}, 0) + + query := fmt.Sprintf("SELECT * FROM %s", tableName) + + rows, err := c.db.Query(query) + if err != nil { + return nil, fmt.Errorf("failed to query %s table: %w", tableName, err) + } + defer rows.Close() + + for rows.Next() { + var row = make(map[string]interface{}) + err := rows.Scan(row) + if err != nil { + return nil, fmt.Errorf("failed to scan %s table row: %w", tableName, err) + } + results = append(results, row) + } + + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("error iterating %s table rows: %w", tableName, err) + } + + return results, nil +} diff --git a/cmd/ctrlc/root/sync/steampipe/list.go b/cmd/ctrlc/root/sync/steampipe/list.go deleted file mode 100644 index 0801c4a..0000000 --- a/cmd/ctrlc/root/sync/steampipe/list.go +++ /dev/null @@ -1,137 +0,0 @@ -package steampipe - -import ( - "fmt" - "sort" - - "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/steampipe/registry" -) - -type Connection struct { - Name string - Type string - SteampipeResource SteampipeResource - CtrlPlaneResource CtrlPlaneResource -} - -type CtrlPlaneResource struct { - Id string - Type string -} - -type SteampipeResource struct { - TableName string - ConnectionName string -} - -// ForeignTable represents a row from information_schema.foreign_tables -type ForeignTable struct { - ForeignTableCatalog string - ForeignTableSchema string - ForeignTableName string - ForeignServerName string -} - -// SortedConnections implements sort.Interface for []Connection based on Name, Type, and CtrlPlaneResource.Type -type SortedConnections []Connection - -func (c SortedConnections) Len() int { return len(c) } -func (c SortedConnections) Swap(i, j int) { c[i], c[j] = c[j], c[i] } -func (c SortedConnections) Less(i, j int) bool { - if c[i].Name == c[j].Name { - if c[i].Type == c[j].Type { - return c[i].CtrlPlaneResource.Type < c[j].CtrlPlaneResource.Type - } - return c[i].Type < c[j].Type - } - return c[i].Name < c[j].Name -} - -func (c *SteampipeClient) ListConnections() ([]Connection, error) { - connections := make([]Connection, 0) - - foreignTables, err := c.GetSteampipeForeignTables() - if err != nil { - return nil, fmt.Errorf("failed to get foreign tables: %w", err) - } - - for _, foreignTable := range foreignTables { - - spResource := SteampipeResource{ - TableName: foreignTable.ForeignTableName, - ConnectionName: foreignTable.ForeignTableSchema, - } - - component, ok := registry.GetAccessInfo(foreignTable.ForeignTableName) - if !ok { - continue - } - - cpResource := CtrlPlaneResource{ - Id: "", - Type: component.ResourceType, - } - - // Connection name defaults to the schema name, but for the case - // where the schema name is the same as the .... TODO: fix this descr - name := foreignTable.ForeignTableSchema - if component.ConnectionType == name { - name = "*" - } - - conn := Connection{ - Name: name, - Type: component.ConnectionType, - SteampipeResource: spResource, - CtrlPlaneResource: cpResource, - } - - connections = append(connections, conn) - } - - // Replace the sort.Slice with sort.Sort using our custom sort.Interface - sort.Sort(SortedConnections(connections)) - - return connections, nil -} - -// GetSteampipeForeignTables returns all foreign tables where foreign_table_catalog is 'steampipe' -func (c *SteampipeClient) GetSteampipeForeignTables() ([]ForeignTable, error) { - query := ` - SELECT - foreign_table_catalog, - foreign_table_schema, - foreign_table_name, - foreign_server_name - FROM information_schema.foreign_tables - WHERE foreign_table_catalog = 'steampipe' - ORDER BY foreign_table_schema, foreign_table_name - ` - - rows, err := c.db.Query(query) - if err != nil { - return nil, fmt.Errorf("failed to query foreign tables: %w", err) - } - defer rows.Close() - - var tables []ForeignTable - for rows.Next() { - var table ForeignTable - err := rows.Scan( - &table.ForeignTableCatalog, - &table.ForeignTableSchema, - &table.ForeignTableName, - &table.ForeignServerName, - ) - if err != nil { - return nil, fmt.Errorf("failed to scan foreign table row: %w", err) - } - tables = append(tables, table) - } - - if err := rows.Err(); err != nil { - return nil, fmt.Errorf("error iterating foreign table rows: %w", err) - } - - return tables, nil -} diff --git a/cmd/ctrlc/root/sync/steampipe/registry/component.go b/cmd/ctrlc/root/sync/steampipe/registry/component.go deleted file mode 100644 index 0300770..0000000 --- a/cmd/ctrlc/root/sync/steampipe/registry/component.go +++ /dev/null @@ -1,20 +0,0 @@ -package registry - -import "fmt" - -type SteampipeAccessInfo struct { - ConnectionType string - ResourceType string - TablePrefix string -} - -func (c *SteampipeAccessInfo) GetTablePrefix() string { - if c.TablePrefix == "" { - return c.ConnectionType - } - return c.TablePrefix -} - -func (c *SteampipeAccessInfo) TableName() string { - return fmt.Sprintf("%s_%s", c.GetTablePrefix(), c.ResourceType) -} diff --git a/cmd/ctrlc/root/sync/steampipe/send.go b/cmd/ctrlc/root/sync/steampipe/send.go deleted file mode 100644 index edf3c3f..0000000 --- a/cmd/ctrlc/root/sync/steampipe/send.go +++ /dev/null @@ -1,10 +0,0 @@ -package steampipe - - - - -func (c *SteampipeClient) SendResourcesFromGroup(resourceGroup string) ([]string, error) { - // Simulate sending resources from a specific group to Ctrlplane - resourceGroups := []string{resourceGroup} - return resourceGroups, nil -} diff --git a/cmd/ctrlc/root/sync/steampipe/steampipe.go b/cmd/ctrlc/root/sync/steampipe/steampipe.go index 89da4d7..0ea76b3 100644 --- a/cmd/ctrlc/root/sync/steampipe/steampipe.go +++ b/cmd/ctrlc/root/sync/steampipe/steampipe.go @@ -13,26 +13,82 @@ import ( ) func NewSyncSteampipeCmd() *cobra.Command { + var resourceProvider string + var spConnection string + var spTable string + apiURL := viper.GetString("url") apiKey := viper.GetString("api-key") workspaceId := viper.GetString("workspace") - ctrlplaneClient, err := api.NewAPIKeyClientWithResponses(apiURL, apiKey) - if err != nil { - return fmt.Errorf("failed to create API client: %w", err) - } - cmd := &cobra.Command{ - Use: "steampipe ", + Use: "steampipe", Short: "Subcommands for integrating steampipe with Ctrlplane", Example: heredoc.Doc(` - $ ctrlc sync steampipe list # Show which resource providers are available - $ ctrlc sync steampipe send # Send to Ctrlplane the resource info for all resourceGroups + $ ctrlc sync steampipe -r resource-provider -c steampipe-connection -t steampipe-table `), + RunE: func(cmd *cobra.Command, args []string) error { + spClient, err := NewSteampipeClient(spConnection) + if err != nil { + log.Error("Failed to create Steampipe spClient", "error", err) + return err + } + + cpClient, err := api.NewAPIKeyClientWithResponses(apiURL, apiKey) + if err != nil { + return fmt.Errorf("failed to create API spClient: %w", err) + return err + } + + defer spClient.Close() + + connections, err := spClient.Fetch() + if err != nil { + return err + } + + // Create a new table writer + table := tablewriter.NewWriter(os.Stdout) + + // Set headers + table.SetHeader([]string{"Resource ID", "Resource Type", "Connection Name", "Steampipe Table"}) + + // Add rows + for _, conn := range connections { + table.Append([]string{ + conn.CtrlPlaneResource.Id, + conn.CtrlPlaneResource.Type, + conn.Name, + conn.SteampipeResource.TableName, + }) + } + + // Set table properties + table.SetAutoWrapText(false) + table.SetAutoFormatHeaders(true) + table.SetHeaderAlignment(tablewriter.ALIGN_LEFT) + table.SetAlignment(tablewriter.ALIGN_LEFT) + table.SetCenterSeparator("") + table.SetColumnSeparator("|") + table.SetRowSeparator("") + table.SetBorder(true) + table.SetTablePadding("\t") + table.SetNoWhiteSpace(true) + + // Render the table + table.Render() + + return nil + }, } - cmd.AddCommand(NewSyncSteampipeListCmd()) - cmd.AddCommand(NewSyncSteampipeSendCmd()) + cmd.Flags().StringVarP(&resourceProvider, "resource-provider", "r", os.Getenv("RESOURCE_PROVIDER"), "The resource group name") + cmd.Flags().StringVarP(&spConnection, "steampipe-connection", "c", os.Getenv("STEAMPIPE_CONNECTION"), "The steampipe postgresql connection string to use") + cmd.Flags().StringVarP(&spTable, "steampipe-table", "t", os.Getenv("STEAMPIPE_TABLE"), "The steampipe postgresql table to select from") + + cmd.MarkFlagRequired("resource-provider") + cmd.MarkFlagRequired("steampipe-connection") + cmd.MarkFlagRequired("steampipe-table") return cmd } @@ -53,7 +109,7 @@ func NewSyncSteampipeListCmd() *cobra.Command { } defer client.Close() - connections, err := client.ListConnections() + connections, err := client.Fetch() if err != nil { return err } From 3a30a0c0a1af5d14dd4288a55fa389804065e2a8 Mon Sep 17 00:00:00 2001 From: jonathan meeks Date: Tue, 8 Apr 2025 09:08:48 -0500 Subject: [PATCH 07/13] steampipe adapter transition, wip --- cmd/ctrlc/root/sync/steampipe/steampipe.go | 168 ++++----------------- 1 file changed, 27 insertions(+), 141 deletions(-) diff --git a/cmd/ctrlc/root/sync/steampipe/steampipe.go b/cmd/ctrlc/root/sync/steampipe/steampipe.go index 0ea76b3..5dcc522 100644 --- a/cmd/ctrlc/root/sync/steampipe/steampipe.go +++ b/cmd/ctrlc/root/sync/steampipe/steampipe.go @@ -1,19 +1,20 @@ package steampipe import ( + "context" "fmt" + "net/http" "os" "github.com/MakeNowJust/heredoc/v2" "github.com/charmbracelet/log" "github.com/ctrlplanedev/cli/internal/api" - "github.com/olekukonko/tablewriter" "github.com/spf13/cobra" "github.com/spf13/viper" ) func NewSyncSteampipeCmd() *cobra.Command { - var resourceProvider string + var providerName string var spConnection string var spTable string @@ -28,171 +29,56 @@ func NewSyncSteampipeCmd() *cobra.Command { $ ctrlc sync steampipe -r resource-provider -c steampipe-connection -t steampipe-table `), RunE: func(cmd *cobra.Command, args []string) error { + var err error + var provider *api.ResourceProvider + var providerResp *http.Response + + ctx := context.Background() + spClient, err := NewSteampipeClient(spConnection) if err != nil { - log.Error("Failed to create Steampipe spClient", "error", err) - return err + return fmt.Errorf("Failed to create Steampipe Connection: %w", err) } cpClient, err := api.NewAPIKeyClientWithResponses(apiURL, apiKey) + if err != nil { return fmt.Errorf("failed to create API spClient: %w", err) - return err } defer spClient.Close() - connections, err := spClient.Fetch() + resources, err := spClient.Fetch(spTable) if err != nil { return err } - // Create a new table writer - table := tablewriter.NewWriter(os.Stdout) - - // Set headers - table.SetHeader([]string{"Resource ID", "Resource Type", "Connection Name", "Steampipe Table"}) - - // Add rows - for _, conn := range connections { - table.Append([]string{ - conn.CtrlPlaneResource.Id, - conn.CtrlPlaneResource.Type, - conn.Name, - conn.SteampipeResource.TableName, - }) + log.Infof("Resource count %d", len(resources)) + if len(resources) > 0 { + provider, err = api.NewResourceProvider(cpClient, workspaceId, providerName) + if err != nil { + return fmt.Errorf("failed to create resource provider: %w", err) + } + providerResp, err = provider.UpsertResource(ctx, resources) + if providerResp != nil { + log.Info("upsert resources response ", "status", providerResp.Status) + } + if err != nil { + return fmt.Errorf("failed to upsert resources: %w", err) + } } - // Set table properties - table.SetAutoWrapText(false) - table.SetAutoFormatHeaders(true) - table.SetHeaderAlignment(tablewriter.ALIGN_LEFT) - table.SetAlignment(tablewriter.ALIGN_LEFT) - table.SetCenterSeparator("") - table.SetColumnSeparator("|") - table.SetRowSeparator("") - table.SetBorder(true) - table.SetTablePadding("\t") - table.SetNoWhiteSpace(true) - - // Render the table - table.Render() - return nil }, } - cmd.Flags().StringVarP(&resourceProvider, "resource-provider", "r", os.Getenv("RESOURCE_PROVIDER"), "The resource group name") + cmd.Flags().StringVarP(&providerName, "resource-provider-name", "r", os.Getenv("RESOURCE_PROVIDER"), "The resource group name") cmd.Flags().StringVarP(&spConnection, "steampipe-connection", "c", os.Getenv("STEAMPIPE_CONNECTION"), "The steampipe postgresql connection string to use") cmd.Flags().StringVarP(&spTable, "steampipe-table", "t", os.Getenv("STEAMPIPE_TABLE"), "The steampipe postgresql table to select from") - cmd.MarkFlagRequired("resource-provider") + cmd.MarkFlagRequired("resource-provider-name") cmd.MarkFlagRequired("steampipe-connection") cmd.MarkFlagRequired("steampipe-table") return cmd } - -func NewSyncSteampipeListCmd() *cobra.Command { - var spConnection string - - cmd := &cobra.Command{ - Use: "list", - Short: "List all available resource-providers", - RunE: func(cmd *cobra.Command, args []string) error { - log.Info("Listing all available resourceGroups") - - client, err := NewSteampipeClient(spConnection) - if err != nil { - log.Error("Failed to create Steampipe client", "error", err) - return err - } - defer client.Close() - - connections, err := client.Fetch() - if err != nil { - return err - } - - // Create a new table writer - table := tablewriter.NewWriter(os.Stdout) - - // Set headers - table.SetHeader([]string{"Resource ID", "Resource Type", "Connection Name", "Steampipe Table"}) - - // Add rows - for _, conn := range connections { - table.Append([]string{ - conn.CtrlPlaneResource.Id, - conn.CtrlPlaneResource.Type, - conn.Name, - conn.SteampipeResource.TableName, - }) - } - - // Set table properties - table.SetAutoWrapText(false) - table.SetAutoFormatHeaders(true) - table.SetHeaderAlignment(tablewriter.ALIGN_LEFT) - table.SetAlignment(tablewriter.ALIGN_LEFT) - table.SetCenterSeparator("") - table.SetColumnSeparator("|") - table.SetRowSeparator("") - table.SetBorder(true) - table.SetTablePadding("\t") - table.SetNoWhiteSpace(true) - - // Render the table - table.Render() - - return nil - }, - } - - return cmd -} - -func NewSyncSteampipeSendCmd() *cobra.Command { - var resourceGroup string - var spConnection string - - cmd := &cobra.Command{ - Use: "send", - Short: "Send resource info to Ctrlplane", - PreRunE: func(cmd *cobra.Command, args []string) error { - if resourceGroup == "" { - return fmt.Errorf("resource-group must be provided") - } - return nil - }, - RunE: func(cmd *cobra.Command, args []string) error { - log.Info("Sending resource groups to Ctrlplane") - - client, err := NewSteampipeClient(spConnection) - if err != nil { - log.Error("Failed to create Steampipe client", "error", err) - return err - } - defer client.Close() - - resourceGroups, err := client.SendResourcesFromGroup(resourceGroup) - if err != nil { - return err - } - - for _, group := range resourceGroups { - fmt.Println(group) - } - - return nil - }, - } - - cmd.Flags().StringVarP(&resourceGroup, "resource-group", "r", os.Getenv("STEAMPIPE_RESOURCE_GROUP"), "The resource group name") - cmd.Flags().StringVarP(&spConnection, "steampipe-connection", "c", "", "The steampipe postgresql connection string to use") - - cmd.MarkFlagRequired("resource-group") - cmd.MarkFlagRequired("steampipe-connection") - - return cmd -} From 58953c60b2e70f88a1c254b80c947157e184f955 Mon Sep 17 00:00:00 2001 From: Jonathan Meeks Date: Tue, 8 Apr 2025 18:40:09 -0500 Subject: [PATCH 08/13] reworked adapters to ApiResource --- cmd/ctrlc/root/sync/steampipe/adapter/aws.go | 101 ++++++++++-------- .../root/sync/steampipe/adapter/azure.go | 16 +-- cmd/ctrlc/root/sync/steampipe/adapter/gcp.go | 24 ++--- .../root/sync/steampipe/adapter/kubernetes.go | 56 +++++----- .../root/sync/steampipe/adapter/model.go | 53 +++++++-- .../root/sync/steampipe/adapter/registry.go | 44 +++++--- cmd/ctrlc/root/sync/steampipe/fetch.go | 94 +++++++++++++--- cmd/ctrlc/root/sync/steampipe/steampipe.go | 31 +++--- internal/api/resource_provider.go | 12 ++- 9 files changed, 280 insertions(+), 151 deletions(-) diff --git a/cmd/ctrlc/root/sync/steampipe/adapter/aws.go b/cmd/ctrlc/root/sync/steampipe/adapter/aws.go index de312d6..6074556 100644 --- a/cmd/ctrlc/root/sync/steampipe/adapter/aws.go +++ b/cmd/ctrlc/root/sync/steampipe/adapter/aws.go @@ -1,52 +1,61 @@ package adapter +import "github.com/ctrlplanedev/cli/internal/api" + var awsAdapters = []SteampipeAdapter{ { - ConnectionType: "aws", - /* - IsType: func(resourceType string) bool { - return resourceType == "eks_cluster" - }, - Convert func(res) ResourceObj { - return ResourceObj{ - Name : fooo, - Version: "kubernetes/v1", - Kind: "ClusterApi", - Config: map[string]interface{}{ - "auth": map[string]interface{} { - method: "aws/eks", - region: "us-east-1", - cluster: "default", - accountId: "123456789012", - }, - status: "active", - name: "default", - }, - Identifier: "my-arn", - Metadata: map[string]interface{}{ - "bunch": "of", - "misc": "stuff", - "not": "excludeed", - }, - - }, - */ - ResourceType: "eks_cluster", - }, - { - ConnectionType: "aws", - ResourceType: "vpc", - }, - { - ConnectionType: "aws", - ResourceType: "rds_db_cluster", - }, - { - ConnectionType: "aws", - ResourceType: "rds_db_instance", - }, - { - ConnectionType: "aws", - ResourceType: "elasticache_cluster", + Table: "aws_eks_cluster", + Translate: func(data *map[string]interface{}) (api.AgentResource, bool) { + var ok bool + var resource api.AgentResource + var tags map[string]string + + resource = api.AgentResource{ + Config: make(map[string]interface{}), + Metadata: make(map[string]string), + } + + if resource.Identifier, ok = getValue[string](data, []string{"arn"}); !ok { + return resource, false + } + + if resource.Name, ok = getValue[string](data, []string{"name"}); !ok { + return resource, false + } + + if tags, ok = getValue[map[string]string](data, []string{"tags"}); ok { + for key, value := range tags { + resource.Metadata[key] = value + } + } else { + return resource, false + } + + if resource.Version, ok = getValue[string](data, []string{"version"}); !ok { + return resource, false + } + + if resource.Kind, ok = getValue[string](data, []string{"kind"}); !ok { + return resource, false + } + + return resource, true + }, }, + //{ + // Plugin: "aws", + // ResourceType: "vpc", + //}, + //{ + // Plugin: "aws", + // ResourceType: "rds_db_cluster", + //}, + //{ + // Plugin: "aws", + // ResourceType: "rds_db_instance", + //}, + //{ + // Plugin: "aws", + // ResourceType: "elasticache_cluster", + //}, } diff --git a/cmd/ctrlc/root/sync/steampipe/adapter/azure.go b/cmd/ctrlc/root/sync/steampipe/adapter/azure.go index 7ed0910..48ac266 100644 --- a/cmd/ctrlc/root/sync/steampipe/adapter/azure.go +++ b/cmd/ctrlc/root/sync/steampipe/adapter/azure.go @@ -1,12 +1,12 @@ package adapter var azureComponents = []SteampipeAdapter{ - { - ConnectionType: "azure", - ResourceType: "kubernetes_cluster", - }, - { - ConnectionType: "azure", - ResourceType: "sql_database", - }, + //{ + // Plugin: "azure", + // ResourceType: "kubernetes_cluster", + //}, + //{ + // Plugin: "azure", + // ResourceType: "sql_database", + //}, } diff --git a/cmd/ctrlc/root/sync/steampipe/adapter/gcp.go b/cmd/ctrlc/root/sync/steampipe/adapter/gcp.go index 24353a2..f64b2dd 100644 --- a/cmd/ctrlc/root/sync/steampipe/adapter/gcp.go +++ b/cmd/ctrlc/root/sync/steampipe/adapter/gcp.go @@ -1,16 +1,16 @@ package adapter var gcpComponents = []SteampipeAdapter{ - { - ConnectionType: "gcp", - ResourceType: "kubernetes_cluster", - }, - { - ConnectionType: "gcp", - ResourceType: "sql_database", - }, - { - ConnectionType: "gcp", - ResourceType: "sql_database_instance", - }, + //{ + // Plugin: "gcp", + // ResourceType: "kubernetes_cluster", + //}, + //{ + // Plugin: "gcp", + // ResourceType: "sql_database", + //}, + //{ + // Plugin: "gcp", + // ResourceType: "sql_database_instance", + //}, } diff --git a/cmd/ctrlc/root/sync/steampipe/adapter/kubernetes.go b/cmd/ctrlc/root/sync/steampipe/adapter/kubernetes.go index a165bd1..8a11da1 100644 --- a/cmd/ctrlc/root/sync/steampipe/adapter/kubernetes.go +++ b/cmd/ctrlc/root/sync/steampipe/adapter/kubernetes.go @@ -1,32 +1,32 @@ package adapter var kubernetesComponents = []SteampipeAdapter{ - { - ConnectionType: "kubernetes", - ResourceType: "namespace", - }, - { - ConnectionType: "kubernetes", - ResourceType: "pod", - }, - { - ConnectionType: "kubernetes", - ResourceType: "node", - }, - { - ConnectionType: "kubernetes", - ResourceType: "service", - }, - { - ConnectionType: "kubernetes", - ResourceType: "deployment", - }, - { - ConnectionType: "kubernetes", - ResourceType: "job", - }, - { - ConnectionType: "kubernetes", - ResourceType: "cronjob", - }, + //{ + // Plugin: "kubernetes", + // ResourceType: "namespace", + //}, + //{ + // Plugin: "kubernetes", + // ResourceType: "pod", + //}, + //{ + // Plugin: "kubernetes", + // ResourceType: "node", + //}, + //{ + // Plugin: "kubernetes", + // ResourceType: "service", + //}, + //{ + // Plugin: "kubernetes", + // ResourceType: "deployment", + //}, + //{ + // Plugin: "kubernetes", + // ResourceType: "job", + //}, + //{ + // Plugin: "kubernetes", + // ResourceType: "cronjob", + //}, } diff --git a/cmd/ctrlc/root/sync/steampipe/adapter/model.go b/cmd/ctrlc/root/sync/steampipe/adapter/model.go index 58bda46..5f38958 100644 --- a/cmd/ctrlc/root/sync/steampipe/adapter/model.go +++ b/cmd/ctrlc/root/sync/steampipe/adapter/model.go @@ -2,24 +2,55 @@ package adapter import ( "fmt" + "github.com/charmbracelet/log" "github.com/ctrlplanedev/cli/internal/api" ) type SteampipeAdapter struct { - ConnectionType string - ResourceType string - TablePrefix string - Translate func(data map[string]interface{}) api.AgentResource - IsCompatible func(data map[string]interface{}) bool + Table string + Translate func(data *map[string]interface{}) (api.AgentResource, bool) } -func (c *SteampipeAdapter) GetTablePrefix() string { - if c.TablePrefix == "" { - return c.ConnectionType +func getValue[T any](data interface{}, keys []string) (T, bool) { + var zero T // Default zero value for the type T + var exists bool + var value interface{} + + if len(keys) == 0 { + if value, ok := data.(T); ok { + return value, true + } + return zero, false + } + + currentKey := keys[0] + switch casted := data.(type) { + case map[string]interface{}: + value, exists = casted[currentKey] + case []interface{}: + if index, ok := parseIndex(currentKey); ok && index >= 0 && index < len(casted) { + value = casted[index] + } else { + log.Warn("could not find index in array") + return zero, false + } + } + if !exists { + return zero, false } - return c.TablePrefix + + if len(keys) == 1 { + if typedValue, ok := value.(T); ok { + return typedValue, true + } + return zero, false + } + + return getValue[T](value, keys[1:]) } -func (c *SteampipeAdapter) TableName() string { - return fmt.Sprintf("%s_%s", c.GetTablePrefix(), c.ResourceType) +func parseIndex(key string) (int, bool) { + var index int + _, err := fmt.Sscanf(key, "[%d]", &index) + return index, err == nil } diff --git a/cmd/ctrlc/root/sync/steampipe/adapter/registry.go b/cmd/ctrlc/root/sync/steampipe/adapter/registry.go index 7950511..a2764d0 100644 --- a/cmd/ctrlc/root/sync/steampipe/adapter/registry.go +++ b/cmd/ctrlc/root/sync/steampipe/adapter/registry.go @@ -1,25 +1,35 @@ package adapter -func buildRegistry() []SteampipeAdapter { - components := []SteampipeAdapter{} - components = append(components, kubernetesComponents...) - components = append(components, awsAdapters...) - components = append(components, gcpComponents...) - components = append(components, azureComponents...) - return components -} +import ( + "strings" +) -func buildTableNameRegistry() map[string]SteampipeAdapter { - tableNameRegistry := make(map[string]SteampipeAdapter) - for _, component := range buildRegistry() { - tableNameRegistry[component.TableName()] = component +func buildRegistry() map[string]*SteampipeAdapter { + registry := make(map[string]*SteampipeAdapter) + for _, pluginAdapters := range [][]SteampipeAdapter{kubernetesComponents, awsAdapters, gcpComponents, azureComponents} { + for _, adapter := range pluginAdapters { + registry[adapter.Table] = &adapter + } } - return tableNameRegistry + return registry } -var tableNameRegistry = buildTableNameRegistry() +var registry = buildRegistry() + +func SelectAdapter(table string) *SteampipeAdapter { + var adapter *SteampipeAdapter + var ok bool + tableName := stripSchema(table) + + if adapter, ok = registry[tableName]; !ok { + return nil + } + return adapter +} -func GetAccessInfo(tableName string) (SteampipeAdapter, bool) { - accessInfo, ok := tableNameRegistry[tableName] - return accessInfo, ok +func stripSchema(table string) string { + if strings.Contains(table, ".") { + return strings.Split(table, ".")[1] + } + return table } diff --git a/cmd/ctrlc/root/sync/steampipe/fetch.go b/cmd/ctrlc/root/sync/steampipe/fetch.go index 560f7d2..e668fd6 100644 --- a/cmd/ctrlc/root/sync/steampipe/fetch.go +++ b/cmd/ctrlc/root/sync/steampipe/fetch.go @@ -1,31 +1,52 @@ package steampipe import ( + "database/sql" + "encoding/json" "fmt" + "github.com/charmbracelet/log" + "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/steampipe/adapter" "github.com/ctrlplanedev/cli/internal/api" ) -func (c *SteampipeClient) Fetch(tableName string) ([]api.AgentResource, error) { +func (c *SteampipeClient) Fetch(table string) ([]api.AgentResource, error) { + var resource api.AgentResource + var ok bool + + ad := adapter.SelectAdapter(table) + if ad == nil { + return nil, fmt.Errorf("no adapter found for table %s", table) + } + resources := make([]api.AgentResource, 0) - results, err := c.SelectAll(tableName) + results, err := c.SelectAll(table) if err != nil { - return nil, fmt.Errorf("failed to fetch from %s table: %w", tableName, err) + return nil, fmt.Errorf("failed to fetch from %s table: %w", table, err) + } + + if len(results) == 0 { + return nil, nil } for _, result := range results { fmt.Println(result) - resources = append(resources, api.AgentResource{}) + if resource, ok = ad.Translate(result); ok { + resources = append(resources, resource) + } } return resources, nil } // SelectAll returns all foreign tables where foreign_table_catalog is 'steampipe' -func (c *SteampipeClient) SelectAll(tableName string) ([]map[string]interface{}, error) { - results := make([]map[string]interface{}, 0) +func (c *SteampipeClient) SelectAll(tableName string) ([]*map[string]interface{}, error) { + var columns []string + var err error + + results := make([]*map[string]interface{}, 0) query := fmt.Sprintf("SELECT * FROM %s", tableName) @@ -33,15 +54,22 @@ func (c *SteampipeClient) SelectAll(tableName string) ([]map[string]interface{}, if err != nil { return nil, fmt.Errorf("failed to query %s table: %w", tableName, err) } - defer rows.Close() - - for rows.Next() { - var row = make(map[string]interface{}) - err := rows.Scan(row) + defer func(rows *sql.Rows) { + err := rows.Close() if err != nil { - return nil, fmt.Errorf("failed to scan %s table row: %w", tableName, err) + log.Errorf("failed to close rows: %v", err) } - results = append(results, row) + }(rows) + + columns, err = rows.Columns() + if err != nil { + return nil, fmt.Errorf("failed to get columns from %s table: %w", tableName, err) + } + + var row map[string]interface{} + for rows.Next() { + row, err = toJsonObj(rows, columns) + results = append(results, &row) } if err := rows.Err(); err != nil { @@ -50,3 +78,43 @@ func (c *SteampipeClient) SelectAll(tableName string) ([]map[string]interface{}, return results, nil } + +// toJsonObj converts a row from the database to a JSON object +// It assumes that a next row is available! +func toJsonObj(nextRow *sql.Rows, columns []string) (map[string]interface{}, error) { + // Create values and associate ptrs to them for row scanning + values := make([]interface{}, len(columns)) + valuePtrs := make([]interface{}, len(columns)) + for i := range columns { + valuePtrs[i] = &values[i] + } + jsonObj := make(map[string]interface{}) + jsonArr := make([]interface{}, 0) + + err := nextRow.Scan(valuePtrs...) + if err != nil { + return nil, fmt.Errorf("failed to scan row: %w", err) + } + result := make(map[string]interface{}) + for i, col := range columns { + if values[i] == nil { + continue + } + switch v := values[i].(type) { + case []byte: + if err = json.Unmarshal(v, &jsonObj); err == nil { + values[i] = jsonObj + break + } + if err = json.Unmarshal(v, &jsonArr); err == nil { + values[i] = jsonArr + break + } + values[i] = string(v) + default: + values[i] = v + } + result[col] = values[i] + } + return result, nil +} diff --git a/cmd/ctrlc/root/sync/steampipe/steampipe.go b/cmd/ctrlc/root/sync/steampipe/steampipe.go index 5dcc522..b273705 100644 --- a/cmd/ctrlc/root/sync/steampipe/steampipe.go +++ b/cmd/ctrlc/root/sync/steampipe/steampipe.go @@ -15,8 +15,8 @@ import ( func NewSyncSteampipeCmd() *cobra.Command { var providerName string - var spConnection string - var spTable string + var connection string + var table string apiURL := viper.GetString("url") apiKey := viper.GetString("api-key") @@ -32,30 +32,33 @@ func NewSyncSteampipeCmd() *cobra.Command { var err error var provider *api.ResourceProvider var providerResp *http.Response + var resources []api.AgentResource ctx := context.Background() - spClient, err := NewSteampipeClient(spConnection) + steampipe, err := NewSteampipeClient(connection) if err != nil { - return fmt.Errorf("Failed to create Steampipe Connection: %w", err) + return fmt.Errorf("failed to create steampipe connection: %w", err) } + defer func(steampipe *SteampipeClient) { + err := steampipe.Close() + if err != nil { + log.Errorf("failed to close steampipe connection: %v", err) + } + }(steampipe) - cpClient, err := api.NewAPIKeyClientWithResponses(apiURL, apiKey) - + apiClient, err := api.NewAPIKeyClientWithResponses(apiURL, apiKey) if err != nil { - return fmt.Errorf("failed to create API spClient: %w", err) + return fmt.Errorf("failed to create API client: %w", err) } - defer spClient.Close() - - resources, err := spClient.Fetch(spTable) - if err != nil { + if resources, err = steampipe.Fetch(table); err != nil { return err } log.Infof("Resource count %d", len(resources)) if len(resources) > 0 { - provider, err = api.NewResourceProvider(cpClient, workspaceId, providerName) + provider, err = api.NewResourceProvider(apiClient, workspaceId, providerName) if err != nil { return fmt.Errorf("failed to create resource provider: %w", err) } @@ -73,8 +76,8 @@ func NewSyncSteampipeCmd() *cobra.Command { } cmd.Flags().StringVarP(&providerName, "resource-provider-name", "r", os.Getenv("RESOURCE_PROVIDER"), "The resource group name") - cmd.Flags().StringVarP(&spConnection, "steampipe-connection", "c", os.Getenv("STEAMPIPE_CONNECTION"), "The steampipe postgresql connection string to use") - cmd.Flags().StringVarP(&spTable, "steampipe-table", "t", os.Getenv("STEAMPIPE_TABLE"), "The steampipe postgresql table to select from") + cmd.Flags().StringVarP(&connection, "steampipe-connection", "c", os.Getenv("STEAMPIPE_CONNECTION"), "The steampipe postgresql connection string to use") + cmd.Flags().StringVarP(&table, "steampipe-table", "t", os.Getenv("STEAMPIPE_TABLE"), "The steampipe postgresql table to select from") cmd.MarkFlagRequired("resource-provider-name") cmd.MarkFlagRequired("steampipe-connection") diff --git a/internal/api/resource_provider.go b/internal/api/resource_provider.go index 5905ffa..40630a3 100644 --- a/internal/api/resource_provider.go +++ b/internal/api/resource_provider.go @@ -15,13 +15,21 @@ func NewResourceProvider(client *ClientWithResponses, workspaceId string, name s log.Debug("Upserting resource provider", "workspaceId", workspaceId, "name", name) resp, err := client.UpsertResourceProviderWithResponse( ctx, workspaceId, name) + var respBody string + if resp != nil && resp.Body != nil { + respBody = string(resp.Body) + } + var respCode int + if resp != nil { + respCode = resp.StatusCode() + } if err != nil { log.Error("Failed to upsert resource provider", "error", err, "workspaceId", workspaceId, "name", name, - "status", resp.StatusCode, - "body", string(resp.Body)) + "status", respCode, + "body", respBody) return nil, fmt.Errorf("failed to upsert resource provider: %w", err) } From aacdeb97afec42263f77cecfb9544b94be5b7c51 Mon Sep 17 00:00:00 2001 From: jonathan meeks Date: Tue, 8 Apr 2025 22:37:17 -0500 Subject: [PATCH 09/13] steampipe adapter aws_eks, mostly working --- cmd/ctrlc/root/sync/steampipe/adapter/aws.go | 95 ++++++++++++++----- .../root/sync/steampipe/adapter/model.go | 6 ++ cmd/ctrlc/root/sync/steampipe/fetch.go | 2 +- 3 files changed, 78 insertions(+), 25 deletions(-) diff --git a/cmd/ctrlc/root/sync/steampipe/adapter/aws.go b/cmd/ctrlc/root/sync/steampipe/adapter/aws.go index 6074556..a63667e 100644 --- a/cmd/ctrlc/root/sync/steampipe/adapter/aws.go +++ b/cmd/ctrlc/root/sync/steampipe/adapter/aws.go @@ -6,40 +6,87 @@ var awsAdapters = []SteampipeAdapter{ { Table: "aws_eks_cluster", Translate: func(data *map[string]interface{}) (api.AgentResource, bool) { - var ok bool - var resource api.AgentResource + var name string + var clusterArn string var tags map[string]string + var accountId string + var region string + var certificateAuthorityData string + var status string + var version string + var endpoint string + var roleArn string + var platformVersion string - resource = api.AgentResource{ - Config: make(map[string]interface{}), - Metadata: make(map[string]string), - } + var zero api.AgentResource = api.AgentResource{} + var ok bool - if resource.Identifier, ok = getValue[string](data, []string{"arn"}); !ok { - return resource, false + if clusterArn, ok = getPathValue[string](data, "arn"); !ok { + return zero, false } - - if resource.Name, ok = getValue[string](data, []string{"name"}); !ok { - return resource, false + if name, ok = getPathValue[string](data, "name"); !ok { + return zero, false } - - if tags, ok = getValue[map[string]string](data, []string{"tags"}); ok { - for key, value := range tags { - resource.Metadata[key] = value - } - } else { - return resource, false + if tags, ok = getPathValue[map[string]string](data, "tags"); !ok { + return zero, false } - - if resource.Version, ok = getValue[string](data, []string{"version"}); !ok { - return resource, false + if accountId, ok = getPathValue[string](data, "account_id"); !ok { + return zero, false + } + if region, ok = getPathValue[string](data, "region"); !ok { + return zero, false + } + if certificateAuthorityData, ok = getPathValue[string](data, "certificate_authority.Data"); !ok { + return zero, false + } + if status, ok = getPathValue[string](data, "status"); !ok { + return zero, false + } + if version, ok = getPathValue[string](data, "version"); !ok { + return zero, false + } + if endpoint, ok = getPathValue[string](data, "endpoint"); !ok { + return zero, false + } + if roleArn, ok = getPathValue[string](data, "role_arn"); !ok { + return zero, false + } + if platformVersion, ok = getPathValue[string](data, "platform_version"); !ok { + return zero, false } - if resource.Kind, ok = getValue[string](data, []string{"kind"}); !ok { - return resource, false + metadata := map[string]string{ + "aws/account-id": accountId, + "aws/arn": clusterArn, + "aws/eks-role-arn": roleArn, + "aws/region": region, + "aws/platform-version": platformVersion, + } + for k, v := range tags { + metadata[k] = v } - return resource, true + return api.AgentResource{ + Identifier: clusterArn, + Name: name, + Version: version, + Kind: "ClusterAPI", + Config: map[string]interface{}{ + "auth": map[string]string{ + "method": "aws/eks", + "region": region, + "accountId": accountId, + "clusterName": name, + }, + "name": name, + "server": map[string]string{ + "endpoint": endpoint, + "certificationAuthorityData": certificateAuthorityData, + }, + "status": status, + }, + Metadata: metadata, + }, true }, }, //{ diff --git a/cmd/ctrlc/root/sync/steampipe/adapter/model.go b/cmd/ctrlc/root/sync/steampipe/adapter/model.go index 5f38958..220502d 100644 --- a/cmd/ctrlc/root/sync/steampipe/adapter/model.go +++ b/cmd/ctrlc/root/sync/steampipe/adapter/model.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/charmbracelet/log" "github.com/ctrlplanedev/cli/internal/api" + "strings" ) type SteampipeAdapter struct { @@ -11,6 +12,11 @@ type SteampipeAdapter struct { Translate func(data *map[string]interface{}) (api.AgentResource, bool) } +func getPathValue[T any](data *map[string]interface{}, path string) (T, bool) { + keys := strings.Split(path, ".") + return getValue[T](data, keys) +} + func getValue[T any](data interface{}, keys []string) (T, bool) { var zero T // Default zero value for the type T var exists bool diff --git a/cmd/ctrlc/root/sync/steampipe/fetch.go b/cmd/ctrlc/root/sync/steampipe/fetch.go index e668fd6..121bfb4 100644 --- a/cmd/ctrlc/root/sync/steampipe/fetch.go +++ b/cmd/ctrlc/root/sync/steampipe/fetch.go @@ -31,7 +31,7 @@ func (c *SteampipeClient) Fetch(table string) ([]api.AgentResource, error) { for _, result := range results { - fmt.Println(result) + //fmt.Println(result) if resource, ok = ad.Translate(result); ok { resources = append(resources, resource) From cc6510b7490c016f2035270a010e493a56ec24c9 Mon Sep 17 00:00:00 2001 From: jonathan meeks Date: Wed, 9 Apr 2025 09:21:15 -0500 Subject: [PATCH 10/13] refactoring path-like traversal --- cmd/ctrlc/root/sync/steampipe/adapter/aws.go | 64 +++++++++++++++---- .../root/sync/steampipe/adapter/model.go | 57 ++++++----------- .../root/sync/steampipe/adapter/registry.go | 2 + cmd/ctrlc/root/sync/steampipe/fetch.go | 2 + 4 files changed, 76 insertions(+), 49 deletions(-) diff --git a/cmd/ctrlc/root/sync/steampipe/adapter/aws.go b/cmd/ctrlc/root/sync/steampipe/adapter/aws.go index a63667e..6ea46ea 100644 --- a/cmd/ctrlc/root/sync/steampipe/adapter/aws.go +++ b/cmd/ctrlc/root/sync/steampipe/adapter/aws.go @@ -1,6 +1,10 @@ package adapter -import "github.com/ctrlplanedev/cli/internal/api" +import ( + "encoding/json" + "github.com/charmbracelet/log" + "github.com/ctrlplanedev/cli/internal/api" +) var awsAdapters = []SteampipeAdapter{ { @@ -19,39 +23,73 @@ var awsAdapters = []SteampipeAdapter{ var platformVersion string var zero api.AgentResource = api.AgentResource{} - var ok bool + var err error + var entityName = "aws_eks_cluster" - if clusterArn, ok = getPathValue[string](data, "arn"); !ok { + if clusterArn, err = getPathValue[string](data, "arn"); err != nil { + log.Infof("could not find %s in %s: %s", "arn", entityName, err.Error()) + dataStr, _ := json.Marshal(data) + log.Debugf("data: %s", dataStr) return zero, false } - if name, ok = getPathValue[string](data, "name"); !ok { + if name, err = getPathValue[string](data, "name"); err != nil { + log.Infof("could not find %s in %s: %s", "name", entityName, err.Error()) + dataStr, _ := json.Marshal(data) + log.Debugf("data: %s", dataStr) return zero, false } - if tags, ok = getPathValue[map[string]string](data, "tags"); !ok { + if tags, err = getPathValue[map[string]string](data, "tags"); err != nil { + log.Infof("could not find %s in %s: %s", "tags", entityName, err.Error()) + dataStr, _ := json.Marshal(data) + log.Debugf("data: %s", dataStr) return zero, false } - if accountId, ok = getPathValue[string](data, "account_id"); !ok { + if accountId, err = getPathValue[string](data, "account_id"); err != nil { + log.Infof("could not find %s in %s: %s", "account_id", entityName, err.Error()) + dataStr, _ := json.Marshal(data) + log.Debugf("data: %s", dataStr) return zero, false } - if region, ok = getPathValue[string](data, "region"); !ok { + if region, err = getPathValue[string](data, "region"); err != nil { + log.Infof("could not find %s in %s: %s", "region", entityName, err.Error()) + dataStr, _ := json.Marshal(data) + log.Debugf("data: %s", dataStr) return zero, false } - if certificateAuthorityData, ok = getPathValue[string](data, "certificate_authority.Data"); !ok { + if certificateAuthorityData, err = getPathValue[string](data, "certificate_authority.Data"); err != nil { + log.Infof("could not find %s in %s: %s", "certificate_authority.Data", entityName, err.Error()) + dataStr, _ := json.Marshal(data) + log.Debugf("data: %s", dataStr) return zero, false } - if status, ok = getPathValue[string](data, "status"); !ok { + if status, err = getPathValue[string](data, "status"); err != nil { + log.Infof("could not find %s in %s: %s", "status", entityName, err.Error()) + dataStr, _ := json.Marshal(data) + log.Debugf("data: %s", dataStr) return zero, false } - if version, ok = getPathValue[string](data, "version"); !ok { + if version, err = getPathValue[string](data, "version"); err != nil { + log.Infof("could not find %s in %s: %s", "version", entityName, err.Error()) + dataStr, _ := json.Marshal(data) + log.Debugf("data: %s", dataStr) return zero, false } - if endpoint, ok = getPathValue[string](data, "endpoint"); !ok { + if endpoint, err = getPathValue[string](data, "endpoint"); err != nil { + log.Infof("could not find %s in %s: %s", "endpoint", entityName, err.Error()) + dataStr, _ := json.Marshal(data) + log.Debugf("data: %s", dataStr) return zero, false } - if roleArn, ok = getPathValue[string](data, "role_arn"); !ok { + if roleArn, err = getPathValue[string](data, "role_arn"); err != nil { + log.Infof("could not find %s in %s: %s", "role_arn", entityName, err.Error()) + dataStr, _ := json.Marshal(data) + log.Debugf("data: %s", dataStr) return zero, false } - if platformVersion, ok = getPathValue[string](data, "platform_version"); !ok { + if platformVersion, err = getPathValue[string](data, "platform_version"); err != nil { + log.Infof("could not find %s in %s: %s", "platform_version", entityName, err.Error()) + dataStr, _ := json.Marshal(data) + log.Debugf("data: %s", dataStr) return zero, false } diff --git a/cmd/ctrlc/root/sync/steampipe/adapter/model.go b/cmd/ctrlc/root/sync/steampipe/adapter/model.go index 220502d..f0e89d0 100644 --- a/cmd/ctrlc/root/sync/steampipe/adapter/model.go +++ b/cmd/ctrlc/root/sync/steampipe/adapter/model.go @@ -2,7 +2,6 @@ package adapter import ( "fmt" - "github.com/charmbracelet/log" "github.com/ctrlplanedev/cli/internal/api" "strings" ) @@ -12,47 +11,33 @@ type SteampipeAdapter struct { Translate func(data *map[string]interface{}) (api.AgentResource, bool) } -func getPathValue[T any](data *map[string]interface{}, path string) (T, bool) { +func getPathValue[T any](data *map[string]interface{}, path string) (T, error) { keys := strings.Split(path, ".") - return getValue[T](data, keys) -} - -func getValue[T any](data interface{}, keys []string) (T, bool) { var zero T // Default zero value for the type T - var exists bool - var value interface{} - - if len(keys) == 0 { - if value, ok := data.(T); ok { - return value, true + var value interface{} = data + var ok bool + + for _, key := range keys { + switch casted := value.(type) { + case map[string]interface{}: + if value, ok = casted[key]; !ok { + return zero, fmt.Errorf("missing value for %s in path %s", key, path) + } + case []interface{}: + if index, ok := parseIndex(key); ok && index >= 0 && index < len(casted) { + value = casted[index] + } else { + return zero, fmt.Errorf("invalid index %s in path %s", key, path) + } + default: + return zero, fmt.Errorf("type mismatch for %s in path %s, expected %T, got %T", key, path, zero, value) } - return zero, false } - currentKey := keys[0] - switch casted := data.(type) { - case map[string]interface{}: - value, exists = casted[currentKey] - case []interface{}: - if index, ok := parseIndex(currentKey); ok && index >= 0 && index < len(casted) { - value = casted[index] - } else { - log.Warn("could not find index in array") - return zero, false - } - } - if !exists { - return zero, false + if finalValue, ok := value.(T); ok { + return finalValue, nil } - - if len(keys) == 1 { - if typedValue, ok := value.(T); ok { - return typedValue, true - } - return zero, false - } - - return getValue[T](value, keys[1:]) + return zero, fmt.Errorf("type mismatch for at %s, expected %T, got %T", path, zero, value) } func parseIndex(key string) (int, bool) { diff --git a/cmd/ctrlc/root/sync/steampipe/adapter/registry.go b/cmd/ctrlc/root/sync/steampipe/adapter/registry.go index a2764d0..1aa7146 100644 --- a/cmd/ctrlc/root/sync/steampipe/adapter/registry.go +++ b/cmd/ctrlc/root/sync/steampipe/adapter/registry.go @@ -1,6 +1,7 @@ package adapter import ( + "github.com/charmbracelet/log" "strings" ) @@ -22,6 +23,7 @@ func SelectAdapter(table string) *SteampipeAdapter { tableName := stripSchema(table) if adapter, ok = registry[tableName]; !ok { + log.Warnf("could not find adapter for table %s", tableName) return nil } return adapter diff --git a/cmd/ctrlc/root/sync/steampipe/fetch.go b/cmd/ctrlc/root/sync/steampipe/fetch.go index 121bfb4..4453e53 100644 --- a/cmd/ctrlc/root/sync/steampipe/fetch.go +++ b/cmd/ctrlc/root/sync/steampipe/fetch.go @@ -25,6 +25,8 @@ func (c *SteampipeClient) Fetch(table string) ([]api.AgentResource, error) { return nil, fmt.Errorf("failed to fetch from %s table: %w", table, err) } + log.Infof("steampipe '%s' query returned %d rows", table, len(results)) + if len(results) == 0 { return nil, nil } From 03b42315ec49216280dd94e508525ccc86deb332 Mon Sep 17 00:00:00 2001 From: Jonathan Meeks Date: Wed, 9 Apr 2025 18:22:28 -0500 Subject: [PATCH 11/13] adapters for gcp, aws and az(only k8s) --- cmd/ctrlc/root/sync/steampipe/adapter/aws.go | 146 ------------------ .../root/sync/steampipe/adapter/aws/ec2.go | 66 ++++++++ .../root/sync/steampipe/adapter/aws/eks.go | 77 +++++++++ .../root/sync/steampipe/adapter/aws/rds.go | 71 +++++++++ .../root/sync/steampipe/adapter/aws/vpc.go | 52 +++++++ .../root/sync/steampipe/adapter/azure.go | 12 -- .../root/sync/steampipe/adapter/azure/aks.go | 76 +++++++++ cmd/ctrlc/root/sync/steampipe/adapter/gcp.go | 16 -- .../sync/steampipe/adapter/gcp/compute.go | 68 ++++++++ .../root/sync/steampipe/adapter/gcp/gke.go | 83 ++++++++++ .../root/sync/steampipe/adapter/gcp/sql.go | 73 +++++++++ .../root/sync/steampipe/adapter/kubernetes.go | 32 ---- .../root/sync/steampipe/adapter/model.go | 47 ------ .../sync/steampipe/adapter/model/adapter.go | 23 +++ .../sync/steampipe/adapter/model/metadata.go | 28 ++++ .../sync/steampipe/adapter/model/sql_row.go | 76 +++++++++ .../root/sync/steampipe/adapter/registry.go | 33 ++-- cmd/ctrlc/root/sync/steampipe/fetch.go | 38 +++-- cmd/ctrlc/root/sync/steampipe/steampipe.go | 10 +- 19 files changed, 746 insertions(+), 281 deletions(-) delete mode 100644 cmd/ctrlc/root/sync/steampipe/adapter/aws.go create mode 100644 cmd/ctrlc/root/sync/steampipe/adapter/aws/ec2.go create mode 100644 cmd/ctrlc/root/sync/steampipe/adapter/aws/eks.go create mode 100644 cmd/ctrlc/root/sync/steampipe/adapter/aws/rds.go create mode 100644 cmd/ctrlc/root/sync/steampipe/adapter/aws/vpc.go delete mode 100644 cmd/ctrlc/root/sync/steampipe/adapter/azure.go create mode 100644 cmd/ctrlc/root/sync/steampipe/adapter/azure/aks.go delete mode 100644 cmd/ctrlc/root/sync/steampipe/adapter/gcp.go create mode 100644 cmd/ctrlc/root/sync/steampipe/adapter/gcp/compute.go create mode 100644 cmd/ctrlc/root/sync/steampipe/adapter/gcp/gke.go create mode 100644 cmd/ctrlc/root/sync/steampipe/adapter/gcp/sql.go delete mode 100644 cmd/ctrlc/root/sync/steampipe/adapter/kubernetes.go delete mode 100644 cmd/ctrlc/root/sync/steampipe/adapter/model.go create mode 100644 cmd/ctrlc/root/sync/steampipe/adapter/model/adapter.go create mode 100644 cmd/ctrlc/root/sync/steampipe/adapter/model/metadata.go create mode 100644 cmd/ctrlc/root/sync/steampipe/adapter/model/sql_row.go diff --git a/cmd/ctrlc/root/sync/steampipe/adapter/aws.go b/cmd/ctrlc/root/sync/steampipe/adapter/aws.go deleted file mode 100644 index 6ea46ea..0000000 --- a/cmd/ctrlc/root/sync/steampipe/adapter/aws.go +++ /dev/null @@ -1,146 +0,0 @@ -package adapter - -import ( - "encoding/json" - "github.com/charmbracelet/log" - "github.com/ctrlplanedev/cli/internal/api" -) - -var awsAdapters = []SteampipeAdapter{ - { - Table: "aws_eks_cluster", - Translate: func(data *map[string]interface{}) (api.AgentResource, bool) { - var name string - var clusterArn string - var tags map[string]string - var accountId string - var region string - var certificateAuthorityData string - var status string - var version string - var endpoint string - var roleArn string - var platformVersion string - - var zero api.AgentResource = api.AgentResource{} - var err error - var entityName = "aws_eks_cluster" - - if clusterArn, err = getPathValue[string](data, "arn"); err != nil { - log.Infof("could not find %s in %s: %s", "arn", entityName, err.Error()) - dataStr, _ := json.Marshal(data) - log.Debugf("data: %s", dataStr) - return zero, false - } - if name, err = getPathValue[string](data, "name"); err != nil { - log.Infof("could not find %s in %s: %s", "name", entityName, err.Error()) - dataStr, _ := json.Marshal(data) - log.Debugf("data: %s", dataStr) - return zero, false - } - if tags, err = getPathValue[map[string]string](data, "tags"); err != nil { - log.Infof("could not find %s in %s: %s", "tags", entityName, err.Error()) - dataStr, _ := json.Marshal(data) - log.Debugf("data: %s", dataStr) - return zero, false - } - if accountId, err = getPathValue[string](data, "account_id"); err != nil { - log.Infof("could not find %s in %s: %s", "account_id", entityName, err.Error()) - dataStr, _ := json.Marshal(data) - log.Debugf("data: %s", dataStr) - return zero, false - } - if region, err = getPathValue[string](data, "region"); err != nil { - log.Infof("could not find %s in %s: %s", "region", entityName, err.Error()) - dataStr, _ := json.Marshal(data) - log.Debugf("data: %s", dataStr) - return zero, false - } - if certificateAuthorityData, err = getPathValue[string](data, "certificate_authority.Data"); err != nil { - log.Infof("could not find %s in %s: %s", "certificate_authority.Data", entityName, err.Error()) - dataStr, _ := json.Marshal(data) - log.Debugf("data: %s", dataStr) - return zero, false - } - if status, err = getPathValue[string](data, "status"); err != nil { - log.Infof("could not find %s in %s: %s", "status", entityName, err.Error()) - dataStr, _ := json.Marshal(data) - log.Debugf("data: %s", dataStr) - return zero, false - } - if version, err = getPathValue[string](data, "version"); err != nil { - log.Infof("could not find %s in %s: %s", "version", entityName, err.Error()) - dataStr, _ := json.Marshal(data) - log.Debugf("data: %s", dataStr) - return zero, false - } - if endpoint, err = getPathValue[string](data, "endpoint"); err != nil { - log.Infof("could not find %s in %s: %s", "endpoint", entityName, err.Error()) - dataStr, _ := json.Marshal(data) - log.Debugf("data: %s", dataStr) - return zero, false - } - if roleArn, err = getPathValue[string](data, "role_arn"); err != nil { - log.Infof("could not find %s in %s: %s", "role_arn", entityName, err.Error()) - dataStr, _ := json.Marshal(data) - log.Debugf("data: %s", dataStr) - return zero, false - } - if platformVersion, err = getPathValue[string](data, "platform_version"); err != nil { - log.Infof("could not find %s in %s: %s", "platform_version", entityName, err.Error()) - dataStr, _ := json.Marshal(data) - log.Debugf("data: %s", dataStr) - return zero, false - } - - metadata := map[string]string{ - "aws/account-id": accountId, - "aws/arn": clusterArn, - "aws/eks-role-arn": roleArn, - "aws/region": region, - "aws/platform-version": platformVersion, - } - for k, v := range tags { - metadata[k] = v - } - - return api.AgentResource{ - Identifier: clusterArn, - Name: name, - Version: version, - Kind: "ClusterAPI", - Config: map[string]interface{}{ - "auth": map[string]string{ - "method": "aws/eks", - "region": region, - "accountId": accountId, - "clusterName": name, - }, - "name": name, - "server": map[string]string{ - "endpoint": endpoint, - "certificationAuthorityData": certificateAuthorityData, - }, - "status": status, - }, - Metadata: metadata, - }, true - }, - }, - //{ - // Plugin: "aws", - // ResourceType: "vpc", - //}, - //{ - // Plugin: "aws", - // ResourceType: "rds_db_cluster", - //}, - //{ - // Plugin: "aws", - // ResourceType: "rds_db_instance", - //}, - //{ - // Plugin: "aws", - // ResourceType: "elasticache_cluster", - //}, -} diff --git a/cmd/ctrlc/root/sync/steampipe/adapter/aws/ec2.go b/cmd/ctrlc/root/sync/steampipe/adapter/aws/ec2.go new file mode 100644 index 0000000..0be963e --- /dev/null +++ b/cmd/ctrlc/root/sync/steampipe/adapter/aws/ec2.go @@ -0,0 +1,66 @@ +package aws + +import ( + "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/steampipe/adapter/model" + "github.com/ctrlplanedev/cli/internal/api" +) + +const ec2Table = "aws_ec2_instance" + +var EC2 model.SteampipeAdapter = &model.SteampipeAdapterStruct{ + Table: ec2Table, + Translate: func(data map[string]interface{}) (api.AgentResource, bool) { + var entityName = ec2Table + var sqlRow model.SqlRow = model.SqlRow{ + EntityName: entityName, + Data: data, + } + + var name string + var clusterArn string + var tags model.Tags + var accountId string + var region string + var status string + var endpoint string + + var zero api.AgentResource = api.AgentResource{} + + if !model.GetRequiredValue[string](sqlRow, "arn", &clusterArn) || + !model.GetRequiredValue[string](sqlRow, "instance_id", &name) || + !model.GetOptionalValue[model.Tags](sqlRow, "tags", &tags) || + !model.GetRequiredValue[string](sqlRow, "account_id", &accountId) || + !model.GetRequiredValue[string](sqlRow, "region", ®ion) || + !model.GetRequiredValue[string](sqlRow, "instance_state", &status) || + !model.GetRequiredValue[string](sqlRow, "private_dns_name", &endpoint) { + return zero, false + } + + metadata := model.BuildMetadata(map[string]string{ + "aws/account-id": accountId, + "aws/arn": clusterArn, + "aws/region": region, + }).AppendTags(tags) + + return api.AgentResource{ + Identifier: clusterArn, + Name: name, + Version: "compute/v1", + Kind: "Compute", + Config: map[string]interface{}{ + "auth": map[string]string{ + "method": "aws/ec2", + "region": region, + "accountId": accountId, + "instanceId": name, + }, + "name": name, + "server": map[string]string{ + "endpoint": endpoint, + }, + "status": status, + }, + Metadata: metadata, + }, true + }, +} diff --git a/cmd/ctrlc/root/sync/steampipe/adapter/aws/eks.go b/cmd/ctrlc/root/sync/steampipe/adapter/aws/eks.go new file mode 100644 index 0000000..f7f7cd3 --- /dev/null +++ b/cmd/ctrlc/root/sync/steampipe/adapter/aws/eks.go @@ -0,0 +1,77 @@ +package aws + +import ( + "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/steampipe/adapter/model" + "github.com/ctrlplanedev/cli/internal/api" +) + +const eksTable = "aws_eks_cluster" + +var EKS model.SteampipeAdapter = &model.SteampipeAdapterStruct{ + Table: eksTable, + Translate: func(data map[string]interface{}) (api.AgentResource, bool) { + var entityName = eksTable + var sqlRow model.SqlRow = model.SqlRow{ + EntityName: entityName, + Data: data, + } + + var name string + var arn string + var tags model.Tags + var accountId string + var region string + var certificateAuthorityData string + var status string + var version string + var endpoint string + var roleArn string + var platformVersion string + + var zero api.AgentResource = api.AgentResource{} + + if !model.GetRequiredValue[string](sqlRow, "arn", &arn) || + !model.GetRequiredValue[string](sqlRow, "name", &name) || + !model.GetOptionalValue[model.Tags](sqlRow, "tags", &tags) || + !model.GetRequiredValue[string](sqlRow, "account_id", &accountId) || + !model.GetRequiredValue[string](sqlRow, "region", ®ion) || + !model.GetRequiredValue[string](sqlRow, "certificate_authority.Data", &certificateAuthorityData) || + !model.GetRequiredValue[string](sqlRow, "status", &status) || + !model.GetRequiredValue[string](sqlRow, "version", &version) || + !model.GetRequiredValue[string](sqlRow, "endpoint", &endpoint) || + !model.GetRequiredValue[string](sqlRow, "role_arn", &roleArn) || + !model.GetRequiredValue[string](sqlRow, "platform_version", &platformVersion) { + return zero, false + } + + metadata := model.BuildMetadata(map[string]string{ + "aws/account-id": accountId, + "aws/arn": arn, + "aws/eks-role-arn": roleArn, + "aws/region": region, + "aws/platform-version": platformVersion, + }).AppendTags(tags) + + return api.AgentResource{ + Identifier: arn, + Name: name, + Version: "kubernetes/v1", + Kind: "ClusterAPI", + Config: map[string]interface{}{ + "auth": map[string]string{ + "method": "aws/eks", + "region": region, + "accountId": accountId, + "clusterName": name, + }, + "name": name, + "server": map[string]string{ + "endpoint": endpoint, + "certificationAuthorityData": certificateAuthorityData, + }, + "status": status, + }, + Metadata: metadata, + }, true + }, +} diff --git a/cmd/ctrlc/root/sync/steampipe/adapter/aws/rds.go b/cmd/ctrlc/root/sync/steampipe/adapter/aws/rds.go new file mode 100644 index 0000000..a1bb1e5 --- /dev/null +++ b/cmd/ctrlc/root/sync/steampipe/adapter/aws/rds.go @@ -0,0 +1,71 @@ +package aws + +import ( + "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/steampipe/adapter/model" + "github.com/ctrlplanedev/cli/internal/api" +) + +const rdsTable = "aws_rds_db_cluster" + +var RDS model.SteampipeAdapter = &model.SteampipeAdapterStruct{ + Table: rdsTable, + Translate: func(data map[string]interface{}) (api.AgentResource, bool) { + var entityName = rdsTable + var sqlRow model.SqlRow = model.SqlRow{ + EntityName: entityName, + Data: data, + } + + var name string + var dbName string + var arn string + var tags model.Tags + var accountId string + var region string + var status string + var version string + var endpoint string + + var zero api.AgentResource = api.AgentResource{} + + if !model.GetRequiredValue[string](sqlRow, "arn", &arn) || + !model.GetRequiredValue[string](sqlRow, "db_cluster_identifier", &name) || + !model.GetRequiredValue[string](sqlRow, "database_name", &dbName) || + !model.GetOptionalValue[model.Tags](sqlRow, "tags", &tags) || + !model.GetRequiredValue[string](sqlRow, "account_id", &accountId) || + !model.GetRequiredValue[string](sqlRow, "region", ®ion) || + !model.GetRequiredValue[string](sqlRow, "status", &status) || + !model.GetRequiredValue[string](sqlRow, "engine_version", &version) || + !model.GetRequiredValue[string](sqlRow, "endpoint", &endpoint) { + return zero, false + } + + metadata := model.BuildMetadata(map[string]string{ + "aws/account-id": accountId, + "aws/arn": arn, + "aws/region": region, + "aws/engine-version": version, + }).AppendTags(tags) + + return api.AgentResource{ + Identifier: arn, + Name: name, + Version: "database/v1", + Kind: "Database", + Config: map[string]interface{}{ + "auth": map[string]string{ + "method": "aws/rds", + "region": region, + "accountId": accountId, + "clusterName": name, + }, + "name": name, + "server": map[string]string{ + "endpoint": endpoint, + }, + "status": status, + }, + Metadata: metadata, + }, true + }, +} diff --git a/cmd/ctrlc/root/sync/steampipe/adapter/aws/vpc.go b/cmd/ctrlc/root/sync/steampipe/adapter/aws/vpc.go new file mode 100644 index 0000000..832ceae --- /dev/null +++ b/cmd/ctrlc/root/sync/steampipe/adapter/aws/vpc.go @@ -0,0 +1,52 @@ +package aws + +import ( + "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/steampipe/adapter/model" + "github.com/ctrlplanedev/cli/internal/api" +) + +const vpcTable = "aws_vpc" + +var VPC model.SteampipeAdapter = &model.SteampipeAdapterStruct{ + Table: vpcTable, + Translate: func(data map[string]interface{}) (api.AgentResource, bool) { + var entityName = vpcTable + var sqlRow model.SqlRow = model.SqlRow{ + EntityName: entityName, + Data: data, + } + + var name string + var arn string + var tags model.Tags + var accountId string + var region string + + var zero api.AgentResource = api.AgentResource{} + + if !model.GetRequiredValue[string](sqlRow, "arn", &arn) || + !model.GetRequiredValue[string](sqlRow, "vpc_id", &name) || + !model.GetOptionalValue[model.Tags](sqlRow, "tags", &tags) || + !model.GetRequiredValue[string](sqlRow, "account_id", &accountId) || + !model.GetRequiredValue[string](sqlRow, "region", ®ion) { + return zero, false + } + + metadata := model.BuildMetadata(map[string]string{ + "aws/account-id": accountId, + "aws/arn": arn, + "aws/region": region, + }).AppendTags(tags) + + return api.AgentResource{ + Identifier: arn, + Name: name, + Version: "vpc/v1", + Kind: "VPC", + Config: map[string]interface{}{ + "name": name, + }, + Metadata: metadata, + }, true + }, +} diff --git a/cmd/ctrlc/root/sync/steampipe/adapter/azure.go b/cmd/ctrlc/root/sync/steampipe/adapter/azure.go deleted file mode 100644 index 48ac266..0000000 --- a/cmd/ctrlc/root/sync/steampipe/adapter/azure.go +++ /dev/null @@ -1,12 +0,0 @@ -package adapter - -var azureComponents = []SteampipeAdapter{ - //{ - // Plugin: "azure", - // ResourceType: "kubernetes_cluster", - //}, - //{ - // Plugin: "azure", - // ResourceType: "sql_database", - //}, -} diff --git a/cmd/ctrlc/root/sync/steampipe/adapter/azure/aks.go b/cmd/ctrlc/root/sync/steampipe/adapter/azure/aks.go new file mode 100644 index 0000000..033f5bf --- /dev/null +++ b/cmd/ctrlc/root/sync/steampipe/adapter/azure/aks.go @@ -0,0 +1,76 @@ +package azure + +import ( + "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/steampipe/adapter/model" + "github.com/ctrlplanedev/cli/internal/api" +) + +const aksTable = "azure_kubernetes_cluster" + +var AKS model.SteampipeAdapter = &model.SteampipeAdapterStruct{ + Table: aksTable, + Translate: func(data map[string]interface{}) (api.AgentResource, bool) { + var entityName = aksTable + var sqlRow model.SqlRow = model.SqlRow{ + EntityName: entityName, + Data: data, + } + + var name string + var id string + var tags model.Tags + var subscriptionId string + var location string + var status string + var version string + var autoscalerProfile map[string]interface{} + var autoscaler string + + var zero api.AgentResource = api.AgentResource{} + + if !model.GetRequiredValue[string](sqlRow, "id", &id) || + !model.GetRequiredValue[string](sqlRow, "name", &name) || + !model.GetOptionalValue[model.Tags](sqlRow, "tags", &tags) || + !model.GetRequiredValue[string](sqlRow, "subscription_id", &subscriptionId) || + !model.GetRequiredValue[string](sqlRow, "location", &location) || + !model.GetOptionalValue[string](sqlRow, "power_state.code", &status) || + !model.GetRequiredValue[string](sqlRow, "kubernetes_version", &version) || + !model.GetRequiredValue[map[string]interface{}](sqlRow, "auto_scaler_profile", &autoscalerProfile) { + return zero, false + } + + if autoscalerProfile != nil { + autoscaler = "true" + } else { + autoscaler = "false" + } + + metadata := model.BuildMetadata(map[string]string{ + "ctrlplane/external-id": id, + "azure/id": id, + "azure/location": location, + "azure/subscription-id": subscriptionId, + "kubernetes/version": version, + "kubernetes/autoscaler-enabled": autoscaler, + }).AppendTags(tags) + + return api.AgentResource{ + Identifier: id, + Name: name, + Version: "kubernetes/v1", + Kind: "ClusterAPI", + Config: map[string]interface{}{ + "auth": map[string]string{ + "method": "azure/aks", + "location": location, + "subscriptionId": subscriptionId, + "clusterName": name, + }, + "name": name, + "server": map[string]string{}, + "status": status, + }, + Metadata: metadata, + }, true + }, +} diff --git a/cmd/ctrlc/root/sync/steampipe/adapter/gcp.go b/cmd/ctrlc/root/sync/steampipe/adapter/gcp.go deleted file mode 100644 index f64b2dd..0000000 --- a/cmd/ctrlc/root/sync/steampipe/adapter/gcp.go +++ /dev/null @@ -1,16 +0,0 @@ -package adapter - -var gcpComponents = []SteampipeAdapter{ - //{ - // Plugin: "gcp", - // ResourceType: "kubernetes_cluster", - //}, - //{ - // Plugin: "gcp", - // ResourceType: "sql_database", - //}, - //{ - // Plugin: "gcp", - // ResourceType: "sql_database_instance", - //}, -} diff --git a/cmd/ctrlc/root/sync/steampipe/adapter/gcp/compute.go b/cmd/ctrlc/root/sync/steampipe/adapter/gcp/compute.go new file mode 100644 index 0000000..e081867 --- /dev/null +++ b/cmd/ctrlc/root/sync/steampipe/adapter/gcp/compute.go @@ -0,0 +1,68 @@ +package gcp + +import ( + "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/steampipe/adapter/model" + "github.com/ctrlplanedev/cli/internal/api" + "strconv" +) + +const computeTable = "gcp_compute_instance" + +var Compute model.SteampipeAdapter = &model.SteampipeAdapterStruct{ + Table: computeTable, + Translate: func(data map[string]interface{}) (api.AgentResource, bool) { + var entityName = computeTable + var sqlRow model.SqlRow = model.SqlRow{ + EntityName: entityName, + Data: data, + } + + var name string + var id int64 + var tags model.Tags + var project string + var location string + var status string + var selfLink string + + var zero api.AgentResource = api.AgentResource{} + + if !model.GetRequiredValue[int64](sqlRow, "id", &id) || + !model.GetRequiredValue[string](sqlRow, "name", &name) || + !model.GetOptionalValue[model.Tags](sqlRow, "tags", &tags) || + !model.GetRequiredValue[string](sqlRow, "project", &project) || + !model.GetRequiredValue[string](sqlRow, "location", &location) || + !model.GetRequiredValue[string](sqlRow, "status", &status) || + !model.GetRequiredValue[string](sqlRow, "self_link", &selfLink) { + return zero, false + } + + metadata := model.BuildMetadata(map[string]string{ + "ctrlplane/external-id": strconv.FormatInt(id, 10), + "google/account-id": project, + "google/id": strconv.FormatInt(id, 10), + "google/location": location, + "google/project": project, + "google/self-link": selfLink, + }).AppendTags(tags) + + return api.AgentResource{ + Identifier: strconv.FormatInt(id, 10), + Name: name, + Version: "compute/v1", + Kind: "Compute", + Config: map[string]interface{}{ + "auth": map[string]string{ + "method": "google/compute", + "location": location, + "project": project, + "name": name, + }, + "name": name, + "server": map[string]string{}, + "status": status, + }, + Metadata: metadata, + }, true + }, +} diff --git a/cmd/ctrlc/root/sync/steampipe/adapter/gcp/gke.go b/cmd/ctrlc/root/sync/steampipe/adapter/gcp/gke.go new file mode 100644 index 0000000..2333b75 --- /dev/null +++ b/cmd/ctrlc/root/sync/steampipe/adapter/gcp/gke.go @@ -0,0 +1,83 @@ +package gcp + +import ( + "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/steampipe/adapter/model" + "github.com/ctrlplanedev/cli/internal/api" + "strconv" +) + +const gkeTable = "gcp_kubernetes_cluster" + +var GKE model.SteampipeAdapter = &model.SteampipeAdapterStruct{ + Table: gkeTable, + Translate: func(data map[string]interface{}) (api.AgentResource, bool) { + var entityName = gkeTable + var sqlRow model.SqlRow = model.SqlRow{ + EntityName: entityName, + Data: data, + } + + var name string + var id string + var tags model.Tags + var project string + var location string + var certificateAuthorityData string + var status string + var version string + var endpoint string + var selfLink string + var autopilot bool + var autoscaling string + + var zero api.AgentResource = api.AgentResource{} + + if !model.GetRequiredValue[string](sqlRow, "id", &id) || + !model.GetRequiredValue[string](sqlRow, "name", &name) || + !model.GetOptionalValue[model.Tags](sqlRow, "tags", &tags) || + !model.GetRequiredValue[string](sqlRow, "project", &project) || + !model.GetRequiredValue[string](sqlRow, "location", &location) || + !model.GetRequiredValue[string](sqlRow, "master_auth.clusterCaCertificate", &certificateAuthorityData) || + !model.GetRequiredValue[string](sqlRow, "status", &status) || + !model.GetRequiredValue[string](sqlRow, "current_master_version", &version) || + !model.GetRequiredValue[string](sqlRow, "endpoint", &endpoint) || + !model.GetRequiredValue[string](sqlRow, "self_link", &selfLink) || + !model.GetRequiredValue[bool](sqlRow, "autopilot_enabled", &autopilot) || + !model.GetOptionalValue[string](sqlRow, "autoscaling.autoscalingProfile", &autoscaling) { + return zero, false + } + + metadata := model.BuildMetadata(map[string]string{ + "ctrlplane/external-id": id, + "google/account-id": project, + "google/id": id, + "google/location": location, + "google/project": project, + "google/self-link": selfLink, + "google/autopilot": strconv.FormatBool(autopilot), + "kubernetes/autoscaling": autoscaling, + }).AppendTags(tags) + + return api.AgentResource{ + Identifier: id, + Name: name, + Version: "kubernetes/v1", + Kind: "ClusterAPI", + Config: map[string]interface{}{ + "auth": map[string]string{ + "method": "google/gke", + "location": location, + "project": project, + "clusterName": name, + }, + "name": name, + "server": map[string]string{ + "endpoint": endpoint, + "certificationAuthorityData": certificateAuthorityData, + }, + "status": status, + }, + Metadata: metadata, + }, true + }, +} diff --git a/cmd/ctrlc/root/sync/steampipe/adapter/gcp/sql.go b/cmd/ctrlc/root/sync/steampipe/adapter/gcp/sql.go new file mode 100644 index 0000000..89d7f21 --- /dev/null +++ b/cmd/ctrlc/root/sync/steampipe/adapter/gcp/sql.go @@ -0,0 +1,73 @@ +package gcp + +import ( + "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/steampipe/adapter/model" + "github.com/ctrlplanedev/cli/internal/api" +) + +const sqlTable = "gcp_sql_database_instance" + +var SQL model.SteampipeAdapter = &model.SteampipeAdapterStruct{ + Table: sqlTable, + Translate: func(data map[string]interface{}) (api.AgentResource, bool) { + var entityName = sqlTable + var sqlRow model.SqlRow = model.SqlRow{ + EntityName: entityName, + Data: data, + } + + var name string + var id string + var tags model.Tags + var project string + var location string + var sslCert string + var status string + var version string + var selfLink string + + var zero api.AgentResource = api.AgentResource{} + + if !model.GetRequiredValue[string](sqlRow, "connection_name", &id) || + !model.GetRequiredValue[string](sqlRow, "name", &name) || + !model.GetOptionalValue[model.Tags](sqlRow, "tags", &tags) || + !model.GetRequiredValue[string](sqlRow, "project", &project) || + !model.GetRequiredValue[string](sqlRow, "location", &location) || + !model.GetRequiredValue[string](sqlRow, "ssl_configuration.cert", &sslCert) || + !model.GetRequiredValue[string](sqlRow, "state", &status) || + !model.GetRequiredValue[string](sqlRow, "database_installed_version", &version) || + !model.GetRequiredValue[string](sqlRow, "self_link", &selfLink) { + return zero, false + } + + metadata := model.BuildMetadata(map[string]string{ + "ctrlplane/external-id": id, + "google/account-id": project, + "google/id": id, + "google/location": location, + "google/project": project, + "google/self-link": selfLink, + }).AppendTags(tags) + + return api.AgentResource{ + Identifier: id, + Name: name, + Version: "database/v1", + Kind: "Database", + Config: map[string]interface{}{ + "auth": map[string]string{ + "method": "google/sql", + "location": location, + "project": project, + "name": name, + }, + "name": name, + "server": map[string]string{ + "sslCert": sslCert, + }, + "status": status, + }, + Metadata: metadata, + }, true + }, +} diff --git a/cmd/ctrlc/root/sync/steampipe/adapter/kubernetes.go b/cmd/ctrlc/root/sync/steampipe/adapter/kubernetes.go deleted file mode 100644 index 8a11da1..0000000 --- a/cmd/ctrlc/root/sync/steampipe/adapter/kubernetes.go +++ /dev/null @@ -1,32 +0,0 @@ -package adapter - -var kubernetesComponents = []SteampipeAdapter{ - //{ - // Plugin: "kubernetes", - // ResourceType: "namespace", - //}, - //{ - // Plugin: "kubernetes", - // ResourceType: "pod", - //}, - //{ - // Plugin: "kubernetes", - // ResourceType: "node", - //}, - //{ - // Plugin: "kubernetes", - // ResourceType: "service", - //}, - //{ - // Plugin: "kubernetes", - // ResourceType: "deployment", - //}, - //{ - // Plugin: "kubernetes", - // ResourceType: "job", - //}, - //{ - // Plugin: "kubernetes", - // ResourceType: "cronjob", - //}, -} diff --git a/cmd/ctrlc/root/sync/steampipe/adapter/model.go b/cmd/ctrlc/root/sync/steampipe/adapter/model.go deleted file mode 100644 index f0e89d0..0000000 --- a/cmd/ctrlc/root/sync/steampipe/adapter/model.go +++ /dev/null @@ -1,47 +0,0 @@ -package adapter - -import ( - "fmt" - "github.com/ctrlplanedev/cli/internal/api" - "strings" -) - -type SteampipeAdapter struct { - Table string - Translate func(data *map[string]interface{}) (api.AgentResource, bool) -} - -func getPathValue[T any](data *map[string]interface{}, path string) (T, error) { - keys := strings.Split(path, ".") - var zero T // Default zero value for the type T - var value interface{} = data - var ok bool - - for _, key := range keys { - switch casted := value.(type) { - case map[string]interface{}: - if value, ok = casted[key]; !ok { - return zero, fmt.Errorf("missing value for %s in path %s", key, path) - } - case []interface{}: - if index, ok := parseIndex(key); ok && index >= 0 && index < len(casted) { - value = casted[index] - } else { - return zero, fmt.Errorf("invalid index %s in path %s", key, path) - } - default: - return zero, fmt.Errorf("type mismatch for %s in path %s, expected %T, got %T", key, path, zero, value) - } - } - - if finalValue, ok := value.(T); ok { - return finalValue, nil - } - return zero, fmt.Errorf("type mismatch for at %s, expected %T, got %T", path, zero, value) -} - -func parseIndex(key string) (int, bool) { - var index int - _, err := fmt.Sscanf(key, "[%d]", &index) - return index, err == nil -} diff --git a/cmd/ctrlc/root/sync/steampipe/adapter/model/adapter.go b/cmd/ctrlc/root/sync/steampipe/adapter/model/adapter.go new file mode 100644 index 0000000..b868a39 --- /dev/null +++ b/cmd/ctrlc/root/sync/steampipe/adapter/model/adapter.go @@ -0,0 +1,23 @@ +package model + +import ( + "github.com/ctrlplanedev/cli/internal/api" +) + +type SteampipeAdapterStruct struct { + Table string + Translate func(data map[string]interface{}) (api.AgentResource, bool) +} + +func (a *SteampipeAdapterStruct) EntityName() string { + return a.Table +} + +func (a *SteampipeAdapterStruct) ToApiResource(row SqlRow) (api.AgentResource, bool) { + return a.Translate(row.Data) +} + +type SteampipeAdapter interface { + EntityName() string + ToApiResource(row SqlRow) (api.AgentResource, bool) +} diff --git a/cmd/ctrlc/root/sync/steampipe/adapter/model/metadata.go b/cmd/ctrlc/root/sync/steampipe/adapter/model/metadata.go new file mode 100644 index 0000000..326acab --- /dev/null +++ b/cmd/ctrlc/root/sync/steampipe/adapter/model/metadata.go @@ -0,0 +1,28 @@ +package model + +func BuildMetadata(data map[string]string) Metadata { + var result = make(Metadata) + for k, v := range data { + result[k] = v + } + return result +} + +type Metadata map[string]string + +type Tags = map[string]interface{} + +func (m Metadata) AppendTags(tags Tags) Metadata { + for k, v := range tags { + var strVal string + var ok bool + if v != nil { + if strVal, ok = v.(string); ok { + m[k] = strVal + } + } else { + m[k] = "" + } + } + return m +} diff --git a/cmd/ctrlc/root/sync/steampipe/adapter/model/sql_row.go b/cmd/ctrlc/root/sync/steampipe/adapter/model/sql_row.go new file mode 100644 index 0000000..9d836cb --- /dev/null +++ b/cmd/ctrlc/root/sync/steampipe/adapter/model/sql_row.go @@ -0,0 +1,76 @@ +package model + +import ( + "encoding/json" + "fmt" + "github.com/charmbracelet/log" + "strings" +) + +type SqlRow struct { + EntityName string + Data map[string]interface{} +} + +func GetRequiredValue[T any](row SqlRow, path string, valuePtr *T) bool { + return GetRowValue[T](row, path, valuePtr, true) +} + +func GetOptionalValue[T any](row SqlRow, path string, valuePtr *T) bool { + return GetRowValue[T](row, path, valuePtr, false) +} + +func GetRowValue[T any](row SqlRow, path string, valuePtr *T, required bool) bool { + var err error + var result T + if result, err = getPathValue[T](row.Data, path); err != nil { + if required { + log.Infof("%s -> %s: %s", row.EntityName, path, err.Error()) + if log.GetLevel() >= log.DebugLevel { + dataStr, _ := json.Marshal(row.Data) + log.Debugf("data: %s", dataStr) + } + return false + } + } + *valuePtr = result + return true + +} + +const PathSeparator = "." + +func getPathValue[T any](data map[string]interface{}, path string) (T, error) { + keys := strings.Split(path, PathSeparator) + var zero T // Default zero value for the type T + var value interface{} = data + var ok bool + + for _, key := range keys { + switch casted := value.(type) { + case map[string]interface{}: + if value, ok = casted[key]; !ok { + return zero, fmt.Errorf("missing value for %s in path %s", key, path) + } + case []interface{}: + if index, ok := parseIndex(key); ok && index >= 0 && index < len(casted) { + value = casted[index] + } else { + return zero, fmt.Errorf("invalid index %s in path %s", key, path) + } + default: + return zero, fmt.Errorf("type mismatch for %s in path %s, expected %T, got %T", key, path, zero, value) + } + } + + if finalValue, ok := value.(T); ok { + return finalValue, nil + } + return zero, fmt.Errorf("type mismatch for at %s, expected %T, got %T", path, zero, value) +} + +func parseIndex(key string) (int, bool) { + var index int + _, err := fmt.Sscanf(key, "[%d]", &index) + return index, err == nil +} diff --git a/cmd/ctrlc/root/sync/steampipe/adapter/registry.go b/cmd/ctrlc/root/sync/steampipe/adapter/registry.go index 1aa7146..c1a64d8 100644 --- a/cmd/ctrlc/root/sync/steampipe/adapter/registry.go +++ b/cmd/ctrlc/root/sync/steampipe/adapter/registry.go @@ -2,31 +2,44 @@ package adapter import ( "github.com/charmbracelet/log" + "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/steampipe/adapter/aws" + "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/steampipe/adapter/azure" + "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/steampipe/adapter/gcp" + "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/steampipe/adapter/model" "strings" ) -func buildRegistry() map[string]*SteampipeAdapter { - registry := make(map[string]*SteampipeAdapter) - for _, pluginAdapters := range [][]SteampipeAdapter{kubernetesComponents, awsAdapters, gcpComponents, azureComponents} { - for _, adapter := range pluginAdapters { - registry[adapter.Table] = &adapter - } +var adapters = []model.SteampipeAdapter{ + aws.EC2, + aws.EKS, + aws.RDS, + aws.VPC, + gcp.GKE, + gcp.SQL, + gcp.Compute, + azure.AKS, +} + +func buildRegistry() map[string]model.SteampipeAdapter { + registry := make(map[string]model.SteampipeAdapter) + for _, adapter := range adapters { + registry[adapter.EntityName()] = adapter } return registry } var registry = buildRegistry() -func SelectAdapter(table string) *SteampipeAdapter { - var adapter *SteampipeAdapter +func SelectAdapter(table string) model.SteampipeAdapter { + var result model.SteampipeAdapter var ok bool tableName := stripSchema(table) - if adapter, ok = registry[tableName]; !ok { + if result, ok = registry[tableName]; !ok { log.Warnf("could not find adapter for table %s", tableName) return nil } - return adapter + return result } func stripSchema(table string) string { diff --git a/cmd/ctrlc/root/sync/steampipe/fetch.go b/cmd/ctrlc/root/sync/steampipe/fetch.go index 4453e53..8d4af25 100644 --- a/cmd/ctrlc/root/sync/steampipe/fetch.go +++ b/cmd/ctrlc/root/sync/steampipe/fetch.go @@ -6,10 +6,11 @@ import ( "fmt" "github.com/charmbracelet/log" "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/steampipe/adapter" + "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/steampipe/adapter/model" "github.com/ctrlplanedev/cli/internal/api" ) -func (c *SteampipeClient) Fetch(table string) ([]api.AgentResource, error) { +func (c *SteampipeClient) DoSync(table string) ([]api.AgentResource, error) { var resource api.AgentResource var ok bool @@ -20,22 +21,31 @@ func (c *SteampipeClient) Fetch(table string) ([]api.AgentResource, error) { resources := make([]api.AgentResource, 0) - results, err := c.SelectAll(table) + jsonObjRows, err := c.SelectAll(table) if err != nil { return nil, fmt.Errorf("failed to fetch from %s table: %w", table, err) } - log.Infof("steampipe '%s' query returned %d rows", table, len(results)) + log.Infof("steampipe '%s' query returned %d rows", table, len(jsonObjRows)) - if len(results) == 0 { + if len(jsonObjRows) == 0 { return nil, nil } - for _, result := range results { + var sqlRow model.SqlRow = model.SqlRow{ + EntityName: table, + Data: nil, + } + + for _, jsonObj := range jsonObjRows { - //fmt.Println(result) + sqlRow.Data = jsonObj - if resource, ok = ad.Translate(result); ok { + if resource, ok = ad.ToApiResource(sqlRow); ok { + if log.GetLevel() >= log.DebugLevel { + payloadStr, _ := json.Marshal(resource) + log.Debugf("%s", payloadStr) + } resources = append(resources, resource) } } @@ -44,11 +54,11 @@ func (c *SteampipeClient) Fetch(table string) ([]api.AgentResource, error) { } // SelectAll returns all foreign tables where foreign_table_catalog is 'steampipe' -func (c *SteampipeClient) SelectAll(tableName string) ([]*map[string]interface{}, error) { +func (c *SteampipeClient) SelectAll(tableName string) ([]map[string]interface{}, error) { var columns []string var err error - results := make([]*map[string]interface{}, 0) + results := make([]map[string]interface{}, 0) query := fmt.Sprintf("SELECT * FROM %s", tableName) @@ -70,8 +80,10 @@ func (c *SteampipeClient) SelectAll(tableName string) ([]*map[string]interface{} var row map[string]interface{} for rows.Next() { - row, err = toJsonObj(rows, columns) - results = append(results, &row) + if row, err = toJsonObj(rows, columns); err != nil { + return nil, fmt.Errorf("failed to convert row to JSON object: %w", err) + } + results = append(results, row) } if err := rows.Err(); err != nil { @@ -90,8 +102,6 @@ func toJsonObj(nextRow *sql.Rows, columns []string) (map[string]interface{}, err for i := range columns { valuePtrs[i] = &values[i] } - jsonObj := make(map[string]interface{}) - jsonArr := make([]interface{}, 0) err := nextRow.Scan(valuePtrs...) if err != nil { @@ -104,10 +114,12 @@ func toJsonObj(nextRow *sql.Rows, columns []string) (map[string]interface{}, err } switch v := values[i].(type) { case []byte: + jsonObj := make(map[string]interface{}) if err = json.Unmarshal(v, &jsonObj); err == nil { values[i] = jsonObj break } + jsonArr := make([]interface{}, 0) if err = json.Unmarshal(v, &jsonArr); err == nil { values[i] = jsonArr break diff --git a/cmd/ctrlc/root/sync/steampipe/steampipe.go b/cmd/ctrlc/root/sync/steampipe/steampipe.go index b273705..a118294 100644 --- a/cmd/ctrlc/root/sync/steampipe/steampipe.go +++ b/cmd/ctrlc/root/sync/steampipe/steampipe.go @@ -18,10 +18,6 @@ func NewSyncSteampipeCmd() *cobra.Command { var connection string var table string - apiURL := viper.GetString("url") - apiKey := viper.GetString("api-key") - workspaceId := viper.GetString("workspace") - cmd := &cobra.Command{ Use: "steampipe", Short: "Subcommands for integrating steampipe with Ctrlplane", @@ -34,6 +30,10 @@ func NewSyncSteampipeCmd() *cobra.Command { var providerResp *http.Response var resources []api.AgentResource + apiURL := viper.GetString("url") + apiKey := viper.GetString("api-key") + workspaceId := viper.GetString("workspace") + ctx := context.Background() steampipe, err := NewSteampipeClient(connection) @@ -52,7 +52,7 @@ func NewSyncSteampipeCmd() *cobra.Command { return fmt.Errorf("failed to create API client: %w", err) } - if resources, err = steampipe.Fetch(table); err != nil { + if resources, err = steampipe.DoSync(table); err != nil { return err } From b176128b63dafd57d4f5726d963ccfe8c94ba000 Mon Sep 17 00:00:00 2001 From: Jonathan Meeks Date: Thu, 10 Apr 2025 14:18:16 -0500 Subject: [PATCH 12/13] move to jsonschema based definitions --- CtrlcSteampipe-2025-04-10-191519.mmd | 52 + .../root/sync/steampipe/adapter/aws/ec2.go | 130 ++- .../root/sync/steampipe/adapter/aws/eks.go | 150 +-- .../root/sync/steampipe/adapter/aws/rds.go | 138 +-- .../adapter/aws/schema/ec2_row.schema.json | 48 + .../aws/schema/examples/ec2_row_ex1.json | 1031 +++++++++++++++++ .../root/sync/steampipe/adapter/aws/vpc.go | 100 +- .../root/sync/steampipe/adapter/azure/aks.go | 148 +-- .../sync/steampipe/adapter/gcp/compute.go | 132 +-- .../root/sync/steampipe/adapter/gcp/gke.go | 162 +-- .../root/sync/steampipe/adapter/gcp/sql.go | 142 +-- .../sync/steampipe/adapter/model/adapter.go | 14 +- .../sync/steampipe/adapter/model/metadata.go | 20 +- .../sync/steampipe/adapter/model/sql_row.go | 80 +- .../root/sync/steampipe/adapter/registry.go | 16 +- cmd/ctrlc/root/sync/steampipe/client.go | 10 +- cmd/ctrlc/root/sync/steampipe/fetch.go | 41 +- cmd/ctrlc/root/sync/steampipe/steampipe.go | 2 +- go.mod | 16 +- go.sum | 103 +- 20 files changed, 1784 insertions(+), 751 deletions(-) create mode 100644 CtrlcSteampipe-2025-04-10-191519.mmd create mode 100644 cmd/ctrlc/root/sync/steampipe/adapter/aws/schema/ec2_row.schema.json create mode 100644 cmd/ctrlc/root/sync/steampipe/adapter/aws/schema/examples/ec2_row_ex1.json diff --git a/CtrlcSteampipe-2025-04-10-191519.mmd b/CtrlcSteampipe-2025-04-10-191519.mmd new file mode 100644 index 0000000..24bf939 --- /dev/null +++ b/CtrlcSteampipe-2025-04-10-191519.mmd @@ -0,0 +1,52 @@ +flowchart TD + subgraph s1["Steampipe + Processor"] + F1{"Next Row"} + OG1(["SQL Row + map[string]interface{}"]) + STR1(["SQL Row + JSON String"]) + V1{"Validate"} + SCH1(["SQL Row + JSON Schema"]) + GO1(["SQL Row + AwsEc2Row{}"]) + end + subgraph ad["Adapter"] + M2{"Translate"} + GO2(["Resource + AwsEc2Resource{}"]) + STR2(["Resource + JSON String"]) + V2{"Validate"} + SCH2(["Resource + JSON Schema"]) + end + subgraph sync["sync"] + M3{"Collect"} + GO3(["AgentResource + AgentResource{}"]) + end + SPDB[("SteampipeDB")] -- each row --> F1 + F1 -- build --> OG1 + OG1 -- marshal --> STR1 + STR1 -.-> V1 + V1 -.-> STR1 & SCH1 + STR1 -- unmarshal --> GO1 + GO1 --> M2 + M2 --> GO2 + GO2 -- marshal --> STR2 + STR2 -.-> V2 + V2 -.-> STR2 & SCH2 + note["extension of
AgentResource schema"] -.-> GO2 & SCH2 + GO3 -.-> note + M3 -- unmarshal --> GO3 + STR2 --> M3 + ctrlplane["CtrlPlane API"] + + note@{ shape: notch-rect} + ctrlplane@{ shape: h-cyl} + + sync ==> ctrlplane + + diff --git a/cmd/ctrlc/root/sync/steampipe/adapter/aws/ec2.go b/cmd/ctrlc/root/sync/steampipe/adapter/aws/ec2.go index 0be963e..b69fcd4 100644 --- a/cmd/ctrlc/root/sync/steampipe/adapter/aws/ec2.go +++ b/cmd/ctrlc/root/sync/steampipe/adapter/aws/ec2.go @@ -1,66 +1,108 @@ package aws import ( + _ "embed" + "encoding/json" + "github.com/charmbracelet/log" "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/steampipe/adapter/model" - "github.com/ctrlplanedev/cli/internal/api" + "github.com/xeipuuv/gojsonschema" ) const ec2Table = "aws_ec2_instance" -var EC2 model.SteampipeAdapter = &model.SteampipeAdapterStruct{ - Table: ec2Table, - Translate: func(data map[string]interface{}) (api.AgentResource, bool) { - var entityName = ec2Table - var sqlRow model.SqlRow = model.SqlRow{ - EntityName: entityName, - Data: data, - } +//go:embed schema/ec2_row.schema.json +var ec2RowSchema string + +var rowSchemaLoader = gojsonschema.NewStringLoader(ec2RowSchema) + +type Ec2Row struct { + Arn string `json:"arn"` + InstanceID string `json:"instance_id"` + AccountID string `json:"account_id"` + Region string `json:"region"` + InstanceState string `json:"instance_state"` + PrivateDNSName string `json:"private_dns_name" default:""` + Tags map[string]string `json:"tags"` +} + +type Ec2Resource struct { + Config Ec2ResConfig `json:"config"` + Identifier string `json:"identifier"` + Kind string `json:"kind"` + Metadata map[string]string `json:"metadata"` + Name string `json:"name"` + Version string `json:"version"` +} + +type Ec2ResConfig struct { + Auth Ec2ResConfigAuth `json:"auth"` + Name string `json:"name"` + Server Ec2ResConfigServer `json:"server"` + Status string `json:"status"` +} - var name string - var clusterArn string - var tags model.Tags - var accountId string - var region string - var status string - var endpoint string +type Ec2ResConfigAuth struct { + Method string `json:"method"` + Region string `json:"region"` + AccountId string `json:"accountId"` + InstanceId string `json:"instanceId"` +} - var zero api.AgentResource = api.AgentResource{} +type Ec2ResConfigServer struct { + Endpoint string `json:"endpoint"` +} - if !model.GetRequiredValue[string](sqlRow, "arn", &clusterArn) || - !model.GetRequiredValue[string](sqlRow, "instance_id", &name) || - !model.GetOptionalValue[model.Tags](sqlRow, "tags", &tags) || - !model.GetRequiredValue[string](sqlRow, "account_id", &accountId) || - !model.GetRequiredValue[string](sqlRow, "region", ®ion) || - !model.GetRequiredValue[string](sqlRow, "instance_state", &status) || - !model.GetRequiredValue[string](sqlRow, "private_dns_name", &endpoint) { - return zero, false +type Ec2ResRequiredMetadata struct { + AccountID string `json:"aws/account-id"` + Arn string `json:"aws/arn"` + Region string `json:"aws/region"` +} + +var EC2 model.SteampipeAdapter = &model.SteampipeAdapterStruct{ + Table: ec2Table, + Convert: func(rowJsonStr string) (string, bool) { + // Validate row json schema and build into type + row, ok := model.ValidateAndUnmarshal[Ec2Row](rowSchemaLoader, rowJsonStr) + if !ok { + return "", false } - metadata := model.BuildMetadata(map[string]string{ - "aws/account-id": accountId, - "aws/arn": clusterArn, - "aws/region": region, - }).AppendTags(tags) + // Add required metadata and merge with tags + metadata := make(model.Metadata) + metadata["aws/account-id"] = row.AccountID + metadata["aws/arn"] = row.Arn + metadata["aws/region"] = row.Region + metadata = metadata.AppendTags(row.Tags) - return api.AgentResource{ - Identifier: clusterArn, - Name: name, + // Build resource specific to the resource type + result := Ec2Resource{ + Identifier: row.Arn, + Name: row.InstanceID, Version: "compute/v1", Kind: "Compute", - Config: map[string]interface{}{ - "auth": map[string]string{ - "method": "aws/ec2", - "region": region, - "accountId": accountId, - "instanceId": name, + Config: Ec2ResConfig{ + Auth: Ec2ResConfigAuth{ + Method: "aws/ec2", + Region: row.Region, + AccountId: row.AccountID, + InstanceId: row.InstanceID, }, - "name": name, - "server": map[string]string{ - "endpoint": endpoint, + Name: row.InstanceID, + Server: Ec2ResConfigServer{ + Endpoint: row.PrivateDNSName, }, - "status": status, + Status: row.InstanceState, }, Metadata: metadata, - }, true + } + + var resultJson []byte + var err error + if resultJson, err = json.Marshal(result); err != nil { + log.Errorf("failed to marshal EC2 resource: %v") + return "", false + } + + return string(resultJson), true }, } diff --git a/cmd/ctrlc/root/sync/steampipe/adapter/aws/eks.go b/cmd/ctrlc/root/sync/steampipe/adapter/aws/eks.go index f7f7cd3..40b2566 100644 --- a/cmd/ctrlc/root/sync/steampipe/adapter/aws/eks.go +++ b/cmd/ctrlc/root/sync/steampipe/adapter/aws/eks.go @@ -1,77 +1,77 @@ package aws -import ( - "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/steampipe/adapter/model" - "github.com/ctrlplanedev/cli/internal/api" -) - -const eksTable = "aws_eks_cluster" - -var EKS model.SteampipeAdapter = &model.SteampipeAdapterStruct{ - Table: eksTable, - Translate: func(data map[string]interface{}) (api.AgentResource, bool) { - var entityName = eksTable - var sqlRow model.SqlRow = model.SqlRow{ - EntityName: entityName, - Data: data, - } - - var name string - var arn string - var tags model.Tags - var accountId string - var region string - var certificateAuthorityData string - var status string - var version string - var endpoint string - var roleArn string - var platformVersion string - - var zero api.AgentResource = api.AgentResource{} - - if !model.GetRequiredValue[string](sqlRow, "arn", &arn) || - !model.GetRequiredValue[string](sqlRow, "name", &name) || - !model.GetOptionalValue[model.Tags](sqlRow, "tags", &tags) || - !model.GetRequiredValue[string](sqlRow, "account_id", &accountId) || - !model.GetRequiredValue[string](sqlRow, "region", ®ion) || - !model.GetRequiredValue[string](sqlRow, "certificate_authority.Data", &certificateAuthorityData) || - !model.GetRequiredValue[string](sqlRow, "status", &status) || - !model.GetRequiredValue[string](sqlRow, "version", &version) || - !model.GetRequiredValue[string](sqlRow, "endpoint", &endpoint) || - !model.GetRequiredValue[string](sqlRow, "role_arn", &roleArn) || - !model.GetRequiredValue[string](sqlRow, "platform_version", &platformVersion) { - return zero, false - } - - metadata := model.BuildMetadata(map[string]string{ - "aws/account-id": accountId, - "aws/arn": arn, - "aws/eks-role-arn": roleArn, - "aws/region": region, - "aws/platform-version": platformVersion, - }).AppendTags(tags) - - return api.AgentResource{ - Identifier: arn, - Name: name, - Version: "kubernetes/v1", - Kind: "ClusterAPI", - Config: map[string]interface{}{ - "auth": map[string]string{ - "method": "aws/eks", - "region": region, - "accountId": accountId, - "clusterName": name, - }, - "name": name, - "server": map[string]string{ - "endpoint": endpoint, - "certificationAuthorityData": certificateAuthorityData, - }, - "status": status, - }, - Metadata: metadata, - }, true - }, -} +//import ( +// "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/steampipe/adapter/model" +// "github.com/ctrlplanedev/cli/internal/api" +//) +// +//const eksTable = "aws_eks_cluster" +// +//var EKS model.SteampipeAdapter = &model.SteampipeAdapterStruct{ +// Table: eksTable, +// Translate: func(data map[string]interface{}) (api.AgentResource, bool) { +// var entityName = eksTable +// var sqlRow model.SqlRow = model.SqlRow{ +// EntityName: entityName, +// Data: data, +// } +// +// var name string +// var arn string +// var tags model.Tags +// var accountId string +// var region string +// var certificateAuthorityData string +// var status string +// var version string +// var endpoint string +// var roleArn string +// var platformVersion string +// +// var zero api.AgentResource = api.AgentResource{} +// +// if !model.GetRequiredValue[string](sqlRow, "arn", &arn) || +// !model.GetRequiredValue[string](sqlRow, "name", &name) || +// !model.GetOptionalValue[model.Tags](sqlRow, "tags", &tags) || +// !model.GetRequiredValue[string](sqlRow, "account_id", &accountId) || +// !model.GetRequiredValue[string](sqlRow, "region", ®ion) || +// !model.GetRequiredValue[string](sqlRow, "certificate_authority.Data", &certificateAuthorityData) || +// !model.GetRequiredValue[string](sqlRow, "status", &status) || +// !model.GetRequiredValue[string](sqlRow, "version", &version) || +// !model.GetRequiredValue[string](sqlRow, "endpoint", &endpoint) || +// !model.GetRequiredValue[string](sqlRow, "role_arn", &roleArn) || +// !model.GetRequiredValue[string](sqlRow, "platform_version", &platformVersion) { +// return zero, false +// } +// +// metadata := model.BuildMetadata(map[string]string{ +// "aws/account-id": accountId, +// "aws/arn": arn, +// "aws/eks-role-arn": roleArn, +// "aws/region": region, +// "aws/platform-version": platformVersion, +// }).AppendTags(tags) +// +// return api.AgentResource{ +// Identifier: arn, +// Name: name, +// Version: "kubernetes/v1", +// Kind: "ClusterAPI", +// Config: map[string]interface{}{ +// "auth": map[string]string{ +// "method": "aws/eks", +// "region": region, +// "accountId": accountId, +// "clusterName": name, +// }, +// "name": name, +// "server": map[string]string{ +// "endpoint": endpoint, +// "certificationAuthorityData": certificateAuthorityData, +// }, +// "status": status, +// }, +// Metadata: metadata, +// }, true +// }, +//} diff --git a/cmd/ctrlc/root/sync/steampipe/adapter/aws/rds.go b/cmd/ctrlc/root/sync/steampipe/adapter/aws/rds.go index a1bb1e5..c6ab48c 100644 --- a/cmd/ctrlc/root/sync/steampipe/adapter/aws/rds.go +++ b/cmd/ctrlc/root/sync/steampipe/adapter/aws/rds.go @@ -1,71 +1,71 @@ package aws -import ( - "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/steampipe/adapter/model" - "github.com/ctrlplanedev/cli/internal/api" -) - -const rdsTable = "aws_rds_db_cluster" - -var RDS model.SteampipeAdapter = &model.SteampipeAdapterStruct{ - Table: rdsTable, - Translate: func(data map[string]interface{}) (api.AgentResource, bool) { - var entityName = rdsTable - var sqlRow model.SqlRow = model.SqlRow{ - EntityName: entityName, - Data: data, - } - - var name string - var dbName string - var arn string - var tags model.Tags - var accountId string - var region string - var status string - var version string - var endpoint string - - var zero api.AgentResource = api.AgentResource{} - - if !model.GetRequiredValue[string](sqlRow, "arn", &arn) || - !model.GetRequiredValue[string](sqlRow, "db_cluster_identifier", &name) || - !model.GetRequiredValue[string](sqlRow, "database_name", &dbName) || - !model.GetOptionalValue[model.Tags](sqlRow, "tags", &tags) || - !model.GetRequiredValue[string](sqlRow, "account_id", &accountId) || - !model.GetRequiredValue[string](sqlRow, "region", ®ion) || - !model.GetRequiredValue[string](sqlRow, "status", &status) || - !model.GetRequiredValue[string](sqlRow, "engine_version", &version) || - !model.GetRequiredValue[string](sqlRow, "endpoint", &endpoint) { - return zero, false - } - - metadata := model.BuildMetadata(map[string]string{ - "aws/account-id": accountId, - "aws/arn": arn, - "aws/region": region, - "aws/engine-version": version, - }).AppendTags(tags) - - return api.AgentResource{ - Identifier: arn, - Name: name, - Version: "database/v1", - Kind: "Database", - Config: map[string]interface{}{ - "auth": map[string]string{ - "method": "aws/rds", - "region": region, - "accountId": accountId, - "clusterName": name, - }, - "name": name, - "server": map[string]string{ - "endpoint": endpoint, - }, - "status": status, - }, - Metadata: metadata, - }, true - }, -} +//import ( +// "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/steampipe/adapter/model" +// "github.com/ctrlplanedev/cli/internal/api" +//) +// +//const rdsTable = "aws_rds_db_cluster" +// +//var RDS model.SteampipeAdapter = &model.SteampipeAdapterStruct{ +// Table: rdsTable, +// Translate: func(data map[string]interface{}) (api.AgentResource, bool) { +// var entityName = rdsTable +// var sqlRow model.SqlRow = model.SqlRow{ +// EntityName: entityName, +// Data: data, +// } +// +// var name string +// var dbName string +// var arn string +// var tags model.Tags +// var accountId string +// var region string +// var status string +// var version string +// var endpoint string +// +// var zero api.AgentResource = api.AgentResource{} +// +// if !model.GetRequiredValue[string](sqlRow, "arn", &arn) || +// !model.GetRequiredValue[string](sqlRow, "db_cluster_identifier", &name) || +// !model.GetRequiredValue[string](sqlRow, "database_name", &dbName) || +// !model.GetOptionalValue[model.Tags](sqlRow, "tags", &tags) || +// !model.GetRequiredValue[string](sqlRow, "account_id", &accountId) || +// !model.GetRequiredValue[string](sqlRow, "region", ®ion) || +// !model.GetRequiredValue[string](sqlRow, "status", &status) || +// !model.GetRequiredValue[string](sqlRow, "engine_version", &version) || +// !model.GetRequiredValue[string](sqlRow, "endpoint", &endpoint) { +// return zero, false +// } +// +// metadata := model.BuildMetadata(map[string]string{ +// "aws/account-id": accountId, +// "aws/arn": arn, +// "aws/region": region, +// "aws/engine-version": version, +// }).AppendTags(tags) +// +// return api.AgentResource{ +// Identifier: arn, +// Name: name, +// Version: "database/v1", +// Kind: "Database", +// Config: map[string]interface{}{ +// "auth": map[string]string{ +// "method": "aws/rds", +// "region": region, +// "accountId": accountId, +// "clusterName": name, +// }, +// "name": name, +// "server": map[string]string{ +// "endpoint": endpoint, +// }, +// "status": status, +// }, +// Metadata: metadata, +// }, true +// }, +//} diff --git a/cmd/ctrlc/root/sync/steampipe/adapter/aws/schema/ec2_row.schema.json b/cmd/ctrlc/root/sync/steampipe/adapter/aws/schema/ec2_row.schema.json new file mode 100644 index 0000000..e2f14c0 --- /dev/null +++ b/cmd/ctrlc/root/sync/steampipe/adapter/aws/schema/ec2_row.schema.json @@ -0,0 +1,48 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "AwsEc2SqlRow", + "type": "object", + "additionalProperties": true, + "required": [ + "arn", + "instance_id", + "account_id", + "region", + "instance_state", + "private_dns_name", + "tags" + ], + "properties": { + "arn": { + "type": "string", + "description": "EC2 ARN" + }, + "instance_id": { + "type": "string", + "description": "Using the instance_id as the 'name' for an EC2 instance" + }, + "account_id": { + "type": "string", + "description": "AWS Account ID" + }, + "region": { + "type": "string", + "description": "AWS Region" + }, + "instance_state": { + "type": "string", + "description": "EC2 instance status" + }, + "private_dns_name": { + "type": "string", + "description": "Private DNS name of the EC2 instance" + }, + "tags": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "description": "All tags on EC2 instance itself" + } + } +} diff --git a/cmd/ctrlc/root/sync/steampipe/adapter/aws/schema/examples/ec2_row_ex1.json b/cmd/ctrlc/root/sync/steampipe/adapter/aws/schema/examples/ec2_row_ex1.json new file mode 100644 index 0000000..11bf49a --- /dev/null +++ b/cmd/ctrlc/root/sync/steampipe/adapter/aws/schema/examples/ec2_row_ex1.json @@ -0,0 +1,1031 @@ +{ + "$schema": "../ec2_row.schema.json", + "_ctx": { + "connection_name": "aws", + "steampipe": { + "sdk_version": "5.11.4" + } + }, + "account_id": "830241207209", + "akas": [ + "arn:aws:ec2:us-east-1:830241207209:instance/i-01288b6098e18be7b" + ], + "ami_launch_index": 0, + "architecture": "x86_64", + "arn": "arn:aws:ec2:us-east-1:830241207209:instance/i-01288b6098e18be7b", + "block_device_mappings": [ + { + "DeviceName": "/dev/xvda", + "Ebs": { + "AssociatedResource": null, + "AttachTime": "2025-04-01T00:02:43Z", + "DeleteOnTermination": true, + "Status": "attached", + "VolumeId": "vol-08d70dc58536f3b25", + "VolumeOwnerId": null + } + } + ], + "boot_mode": "", + "capacity_reservation_specification": { + "CapacityReservationPreference": "open", + "CapacityReservationTarget": null + }, + "client_token": "fleet-f186b826-5197-c19c-8410-860ad9dbf3a5-0", + "cpu_options_core_count": 2, + "cpu_options_threads_per_core": 2, + "current_instance_boot_mode": "legacy-bios", + "disable_api_termination": false, + "ebs_optimized": false, + "ena_support": true, + "enclave_options": { + "Enabled": false + }, + "hibernation_options": { + "Configured": false + }, + "hypervisor": "xen", + "iam_instance_profile_arn": "arn:aws:iam::830241207209:instance-profile/eks-68c946c0-3a40-ec02-6c53-1c319cac08e5", + "iam_instance_profile_id": "AIPA4CTR3OOU47HXDN4ZA", + "image_id": "ami-03fcb7c9c1ca36b4d", + "instance_id": "i-01288b6098e18be7b", + "instance_initiated_shutdown_behavior": "stop", + "instance_lifecycle": "", + "instance_state": "running", + "instance_status": { + "AvailabilityZone": "us-east-1b", + "Events": null, + "InstanceId": "i-01288b6098e18be7b", + "InstanceState": { + "Code": 16, + "Name": "running" + }, + "InstanceStatus": { + "Details": [ + { + "ImpairedSince": null, + "Name": "reachability", + "Status": "passed" + } + ], + "Status": "ok" + }, + "OutpostArn": null, + "SystemStatus": { + "Details": [ + { + "ImpairedSince": null, + "Name": "reachability", + "Status": "passed" + } + ], + "Status": "ok" + } + }, + "instance_type": "r6i.xlarge", + "launch_template_data": { + "BlockDeviceMappings": [ + { + "DeviceName": "/dev/xvda", + "Ebs": { + "DeleteOnTermination": true, + "Encrypted": true, + "Iops": 3000, + "KmsKeyId": "arn:aws:kms:us-east-1:830241207209:key/244aa299-ec84-4817-9d0e-01eb0dd099a3", + "SnapshotId": "snap-06f117083705d83dd", + "Throughput": 125, + "VolumeSize": 100, + "VolumeType": "gp3" + }, + "NoDevice": null, + "VirtualName": null + } + ], + "CapacityReservationSpecification": { + "CapacityReservationPreference": "open", + "CapacityReservationTarget": null + }, + "CpuOptions": { + "AmdSevSnp": "", + "CoreCount": 2, + "ThreadsPerCore": 2 + }, + "CreditSpecification": null, + "DisableApiStop": false, + "DisableApiTermination": false, + "EbsOptimized": false, + "ElasticGpuSpecifications": null, + "ElasticInferenceAccelerators": null, + "EnclaveOptions": { + "Enabled": false + }, + "HibernationOptions": { + "Configured": false + }, + "IamInstanceProfile": { + "Arn": "arn:aws:iam::830241207209:instance-profile/eks-68c946c0-3a40-ec02-6c53-1c319cac08e5", + "Name": null + }, + "ImageId": "ami-03fcb7c9c1ca36b4d", + "InstanceInitiatedShutdownBehavior": "stop", + "InstanceMarketOptions": null, + "InstanceRequirements": null, + "InstanceType": "r6i.xlarge", + "KernelId": null, + "KeyName": null, + "LicenseSpecifications": null, + "MaintenanceOptions": { + "AutoRecovery": "default" + }, + "MetadataOptions": { + "HttpEndpoint": "enabled", + "HttpProtocolIpv6": "disabled", + "HttpPutResponseHopLimit": 2, + "HttpTokens": "required", + "InstanceMetadataTags": "disabled", + "State": "" + }, + "Monitoring": { + "Enabled": true + }, + "NetworkInterfaces": [ + { + "AssociateCarrierIpAddress": null, + "AssociatePublicIpAddress": null, + "ConnectionTrackingSpecification": null, + "DeleteOnTermination": true, + "Description": "", + "DeviceIndex": 0, + "EnaSrdSpecification": null, + "Groups": [ + "sg-04e83501310692e69", + "sg-0f52e5a544842390f" + ], + "InterfaceType": "interface", + "Ipv4PrefixCount": null, + "Ipv4Prefixes": [], + "Ipv6AddressCount": null, + "Ipv6Addresses": [], + "Ipv6PrefixCount": null, + "Ipv6Prefixes": [], + "NetworkCardIndex": 0, + "NetworkInterfaceId": null, + "PrimaryIpv6": null, + "PrivateIpAddress": null, + "PrivateIpAddresses": [ + { + "Primary": true, + "PrivateIpAddress": "10.10.11.57" + }, + { + "Primary": false, + "PrivateIpAddress": "10.10.11.10" + }, + { + "Primary": false, + "PrivateIpAddress": "10.10.11.43" + }, + { + "Primary": false, + "PrivateIpAddress": "10.10.11.233" + }, + { + "Primary": false, + "PrivateIpAddress": "10.10.11.206" + }, + { + "Primary": false, + "PrivateIpAddress": "10.10.11.163" + }, + { + "Primary": false, + "PrivateIpAddress": "10.10.11.94" + }, + { + "Primary": false, + "PrivateIpAddress": "10.10.11.223" + }, + { + "Primary": false, + "PrivateIpAddress": "10.10.11.188" + }, + { + "Primary": false, + "PrivateIpAddress": "10.10.11.210" + }, + { + "Primary": false, + "PrivateIpAddress": "10.10.11.83" + }, + { + "Primary": false, + "PrivateIpAddress": "10.10.11.208" + }, + { + "Primary": false, + "PrivateIpAddress": "10.10.11.87" + }, + { + "Primary": false, + "PrivateIpAddress": "10.10.11.117" + }, + { + "Primary": false, + "PrivateIpAddress": "10.10.11.53" + } + ], + "SecondaryPrivateIpAddressCount": null, + "SubnetId": "subnet-0f0d619a86b69c67c" + }, + { + "AssociateCarrierIpAddress": null, + "AssociatePublicIpAddress": null, + "ConnectionTrackingSpecification": null, + "DeleteOnTermination": true, + "Description": "aws-K8S-i-01288b6098e18be7b", + "DeviceIndex": 1, + "EnaSrdSpecification": null, + "Groups": [ + "sg-04e83501310692e69", + "sg-0f52e5a544842390f" + ], + "InterfaceType": "interface", + "Ipv4PrefixCount": null, + "Ipv4Prefixes": [], + "Ipv6AddressCount": null, + "Ipv6Addresses": [], + "Ipv6PrefixCount": null, + "Ipv6Prefixes": [], + "NetworkCardIndex": 0, + "NetworkInterfaceId": null, + "PrimaryIpv6": null, + "PrivateIpAddress": null, + "PrivateIpAddresses": [ + { + "Primary": true, + "PrivateIpAddress": "10.10.11.201" + }, + { + "Primary": false, + "PrivateIpAddress": "10.10.11.73" + }, + { + "Primary": false, + "PrivateIpAddress": "10.10.11.110" + }, + { + "Primary": false, + "PrivateIpAddress": "10.10.11.47" + }, + { + "Primary": false, + "PrivateIpAddress": "10.10.11.12" + }, + { + "Primary": false, + "PrivateIpAddress": "10.10.11.172" + }, + { + "Primary": false, + "PrivateIpAddress": "10.10.11.66" + }, + { + "Primary": false, + "PrivateIpAddress": "10.10.11.134" + }, + { + "Primary": false, + "PrivateIpAddress": "10.10.11.26" + }, + { + "Primary": false, + "PrivateIpAddress": "10.10.11.31" + }, + { + "Primary": false, + "PrivateIpAddress": "10.10.11.28" + }, + { + "Primary": false, + "PrivateIpAddress": "10.10.11.18" + }, + { + "Primary": false, + "PrivateIpAddress": "10.10.11.114" + }, + { + "Primary": false, + "PrivateIpAddress": "10.10.11.150" + }, + { + "Primary": false, + "PrivateIpAddress": "10.10.11.180" + } + ], + "SecondaryPrivateIpAddressCount": null, + "SubnetId": "subnet-0f0d619a86b69c67c" + }, + { + "AssociateCarrierIpAddress": null, + "AssociatePublicIpAddress": null, + "ConnectionTrackingSpecification": null, + "DeleteOnTermination": true, + "Description": "aws-K8S-i-01288b6098e18be7b", + "DeviceIndex": 2, + "EnaSrdSpecification": null, + "Groups": [ + "sg-04e83501310692e69", + "sg-0f52e5a544842390f" + ], + "InterfaceType": "interface", + "Ipv4PrefixCount": null, + "Ipv4Prefixes": [], + "Ipv6AddressCount": null, + "Ipv6Addresses": [], + "Ipv6PrefixCount": null, + "Ipv6Prefixes": [], + "NetworkCardIndex": 0, + "NetworkInterfaceId": null, + "PrimaryIpv6": null, + "PrivateIpAddress": null, + "PrivateIpAddresses": [ + { + "Primary": true, + "PrivateIpAddress": "10.10.11.170" + }, + { + "Primary": false, + "PrivateIpAddress": "10.10.11.202" + }, + { + "Primary": false, + "PrivateIpAddress": "10.10.11.171" + }, + { + "Primary": false, + "PrivateIpAddress": "10.10.11.139" + }, + { + "Primary": false, + "PrivateIpAddress": "10.10.11.232" + }, + { + "Primary": false, + "PrivateIpAddress": "10.10.11.105" + }, + { + "Primary": false, + "PrivateIpAddress": "10.10.11.98" + }, + { + "Primary": false, + "PrivateIpAddress": "10.10.11.32" + }, + { + "Primary": false, + "PrivateIpAddress": "10.10.11.96" + }, + { + "Primary": false, + "PrivateIpAddress": "10.10.11.6" + }, + { + "Primary": false, + "PrivateIpAddress": "10.10.11.37" + }, + { + "Primary": false, + "PrivateIpAddress": "10.10.11.157" + }, + { + "Primary": false, + "PrivateIpAddress": "10.10.11.243" + }, + { + "Primary": false, + "PrivateIpAddress": "10.10.11.112" + }, + { + "Primary": false, + "PrivateIpAddress": "10.10.11.183" + } + ], + "SecondaryPrivateIpAddressCount": null, + "SubnetId": "subnet-0f0d619a86b69c67c" + } + ], + "Placement": { + "Affinity": null, + "AvailabilityZone": "us-east-1b", + "GroupId": null, + "GroupName": "", + "HostId": null, + "HostResourceGroupArn": null, + "PartitionNumber": null, + "SpreadDomain": null, + "Tenancy": "default" + }, + "PrivateDnsNameOptions": { + "EnableResourceNameDnsAAAARecord": false, + "EnableResourceNameDnsARecord": false, + "HostnameType": "ip-name" + }, + "RamDiskId": null, + "SecurityGroupIds": null, + "SecurityGroups": null, + "TagSpecifications": [ + { + "ResourceType": "instance", + "Tags": [ + { + "Key": "TerraformModule", + "Value": "terraform-aws-wandb/module/app_eks" + }, + { + "Key": "TerraformNamespace", + "Value": "wandb-odaia" + }, + { + "Key": "Name", + "Value": "wandb-odaia-b" + }, + { + "Key": "eks:nodegroup-name", + "Value": "wandb-odaia-b20241015000738841700000005" + }, + { + "Key": "env", + "Value": "managed-install" + }, + { + "Key": "GithubRepo", + "Value": "wandb" + }, + { + "Key": "kubernetes.io/cluster/wandb-odaia", + "Value": "owned" + }, + { + "Key": "customer-ns", + "Value": "wandb-odaia" + }, + { + "Key": "eks:cluster-name", + "Value": "wandb-odaia" + }, + { + "Key": "k8s.io/cluster-autoscaler/enabled", + "Value": "true" + }, + { + "Key": "k8s.io/cluster-autoscaler/wandb-odaia", + "Value": "owned" + }, + { + "Key": "GithubOrg", + "Value": "terraform-aws-wandb" + } + ] + } + ], + "UserData": "TUlNRS1WZXJzaW9uOiAxLjANCkNvbnRlbnQtVHlwZTogbXVsdGlwYXJ0L21peGVkOyBib3VuZGFyeT0iLy8iDQoNCi0tLy8NCkNvbnRlbnQtVHJhbnNmZXItRW5jb2Rpbmc6IDdiaXQNCkNvbnRlbnQtVHlwZTogdGV4dC94LXNoZWxsc2NyaXB0DQpNaW1lLVZlcnNpb246IDEuMA0KDQojIS9iaW4vYmFzaCAtZQoKIyBTZXQgYm9vdHN0cmFwIGVudgpwcmludGYgJyMhL2Jpbi9iYXNoCmV4cG9ydCBBRERJVElPTkFMX0tVQkVMRVRfRVhUUkFfQVJHUz0iLS1zeXN0ZW0tcmVzZXJ2ZWQ9Y3B1PTcwbSxtZW1vcnk9MTAwTWksZXBoZW1lcmFsLXN0b3JhZ2U9NzUwTWkscGlkPTUwMCIKJyA+IC9ldGMvcHJvZmlsZS5kL2Vrcy1ib290c3RyYXAtZW52LnNoCgojIFNvdXJjZSBleHRyYSBlbnZpcm9ubWVudCB2YXJpYWJsZXMgaW4gYm9vdHN0cmFwIHNjcmlwdApzZWQgLWkgJy9ec2V0IC1vIGVycmV4aXQvYVxcbnNvdXJjZSAvZXRjL3Byb2ZpbGUuZC9la3MtYm9vdHN0cmFwLWVudi5zaCcgL2V0Yy9la3MvYm9vdHN0cmFwLnNoCgojIE1lcmdlIEFERElUSU9OQUxfS1VCRUxFVF9FWFRSQV9BUkdTIGludG8gS1VCRUxFVF9FWFRSQV9BUkdTCnNlZCAtaSAncy9eS1VCRUxFVF9FWFRSQV9BUkdTPSIke0tVQkVMRVRfRVhUUkFfQVJHUzotfS9LVUJFTEVUX0VYVFJBX0FSR1M9IiR7S1VCRUxFVF9FWFRSQV9BUkdTOi19ICR7QURESVRJT05BTF9LVUJFTEVUX0VYVFJBX0FSR1N9LycgL2V0Yy9la3MvYm9vdHN0cmFwLnNoCgojIFVzZXIgc3VwcGxpZWQgcHJlIHVzZXJkYXRhCgoNCi0tLy8NCkNvbnRlbnQtVHlwZTogdGV4dC94LXNoZWxsc2NyaXB0OyBjaGFyc2V0PSJ1cy1hc2NpaSINCiMhL2Jpbi9iYXNoDQpzZXQgLWV4DQpCNjRfQ0xVU1RFUl9DQT1MUzB0TFMxQ1JVZEpUaUJEUlZKVVNVWkpRMEZVUlMwdExTMHRDazFKU1VSQ1ZFTkRRV1V5WjBGM1NVSkJaMGxKVUU1cFkyeHZTVE01ZVRoM1JGRlpTa3R2V2tsb2RtTk9RVkZGVEVKUlFYZEdWRVZVVFVKRlIwRXhWVVVLUVhoTlMyRXpWbWxhV0VwMVdsaFNiR042UVdWR2R6QjVUa1JCZWsxVVJYaE9SRlUwVFhwYVlVWjNNSHBPUkVGNlRVUnJlRTVVUVhwTmVscGhUVUpWZUFwRmVrRlNRbWRPVmtKQlRWUkRiWFF4V1cxV2VXSnRWakJhV0UxM1oyZEZhVTFCTUVkRFUzRkhVMGxpTTBSUlJVSkJVVlZCUVRSSlFrUjNRWGRuWjBWTENrRnZTVUpCVVVNNWRVNU9TVTVwUVRkUU1IcE1OV3R6WjA0MWJ6aFVWMHBTYWxScVZ6QndiVzU2TVVaSlZqaElXRUY2ZW05WE9GTlBSamRUWm5nemNHa0tVWFp5Y0hCa1JURnJSMGQyWW14SFoxQjFNMUpqTmpNeFJTdHdZMlJCU1hwdmVUZFVibGQyZW1KRmJWVXdSRGRJWVd0UVZHZFJabVJpZFZGTGFWSXdZZ3BSTkZKaGMwMW9kR3hIVm1nMFQzQnRZM05uZEU5SGRtTm5UMFpXTlRCS2FIVnNiRTloZFdSb2VqTkdLMU52YjJka1ZHVlRNM2RpWnpGSFZHaFJlazg1Q2taalYxaHRkamxYVTFGUk16ZDFkMUJuZDJGdmRYRkdlRmcwZFdaT2RUUnZiMmxNZUc0eVFqWkpNbEpTYldsTmVHOVNRMFpQTm5CSlNXdzNkblJOZDNVS1RuSTJRelZTVDFOT2JISXZja1JGTTBReVpsTm9WME50YVhkMFlrWkhVSFoyZWt0SVRHRlNhRGM0TkZKdmEwbElSVTE0TUcxNmVIZHlUblJ4U1VwUFJ3cEdUWEZOTmxKRVdXZzFjMWw1TTNsVGJ6bE5Ra0ZDWlVWTWJsWnFRV2ROUWtGQlIycFhWRUpZVFVFMFIwRXhWV1JFZDBWQ0wzZFJSVUYzU1VOd1JFRlFDa0puVGxaSVVrMUNRV1k0UlVKVVFVUkJVVWd2VFVJd1IwRXhWV1JFWjFGWFFrSlVaMnBET1dJd2MwdzNjM0ZFTkdGWFZWaExURTFQTVRaa1JIWlVRVllLUW1kT1ZraFNSVVZFYWtGTloyZHdjbVJYU214amJUVnNaRWRXZWsxQk1FZERVM0ZIVTBsaU0wUlJSVUpEZDFWQlFUUkpRa0ZSUWtkNFRIVXJhRkUzYmdwU01FcGphUzlVZDJOV1ppOVhjRmhDUWxSMmRtdHVNM040WVdkVVExSnJjWFZPWkRoVk1GUXlZMHd5WWxKWGR6RlBWek5FTUc1Q09VSm1NbE01TWxaSUNtNWphVTlITlRSSlZXaFVORE5RUzBkbE1WbHBha054TUU0M2FtTjRRalUyZDBoU1JrbzJRVzlDVW10VWRWZ3pielJ6WWtKdUswMXJPREowV2tremMwNEthV3MwWm5kdmNHSnRhVTkyVldscVRsZFNSVXhDZUVKSGFUWmhhbFJUVmtwSmJuVkRWazA1Ylcxd1RXVlBOMUJ4YW1OcVdXMVBNRGxMVVVWUFprSnRXUW92UkRFelVUQnlabTk2YUM5Q1dIQldNbGRLUVUxR2VEaHBVM2xZY2s1SlMyd3ZhVzlyZEhVNVltcERTa3hWY2tKYVdWbG5UMHAzYUc1R05WYzNjbWM0Q25kaEwydG9Tbkl6UVVGTU1IQjJOM1pUY25OVWRYRm9WelZXUXl0U1NFUlpNbEEwVTBsWVlUaHFUR3hSVmtjdmVYVmtNemRKS3psb1YyeHBabXN4Tm5BS1dtOUlkM2gzZUZCS1MzTkpDaTB0TFMwdFJVNUVJRU5GVWxSSlJrbERRVlJGTFMwdExTMEsNCkFQSV9TRVJWRVJfVVJMPWh0dHBzOi8vQUEzOUI0NDA4QzY5MjE1QjY1OTUwRDI3NEYxQTYzMzQuZ3I3LnVzLWVhc3QtMS5la3MuYW1hem9uYXdzLmNvbQ0KSzhTX0NMVVNURVJfRE5TX0lQPTE3Mi4yMC4wLjEwDQovZXRjL2Vrcy9ib290c3RyYXAuc2ggd2FuZGItb2RhaWEgLS1rdWJlbGV0LWV4dHJhLWFyZ3MgJy0tbm9kZS1sYWJlbHM9ZWtzLmFtYXpvbmF3cy5jb20vbm9kZWdyb3VwLWltYWdlPWFtaS0wM2ZjYjdjOWMxY2EzNmI0ZCxla3MuYW1hem9uYXdzLmNvbS9zb3VyY2VMYXVuY2hUZW1wbGF0ZVZlcnNpb249NSxla3MuYW1hem9uYXdzLmNvbS9jYXBhY2l0eVR5cGU9T05fREVNQU5ELGVrcy5hbWF6b25hd3MuY29tL25vZGVncm91cD13YW5kYi1vZGFpYS1iMjAyNDEwMTUwMDA3Mzg4NDE3MDAwMDAwMDUsZWtzLmFtYXpvbmF3cy5jb20vc291cmNlTGF1bmNoVGVtcGxhdGVJZD1sdC0wZjIyNWYwYzk0YTViOGEyNiAtLW1heC1wb2RzPTU4JyAtLWI2NC1jbHVzdGVyLWNhICRCNjRfQ0xVU1RFUl9DQSAtLWFwaXNlcnZlci1lbmRwb2ludCAkQVBJX1NFUlZFUl9VUkwgLS1kbnMtY2x1c3Rlci1pcCAkSzhTX0NMVVNURVJfRE5TX0lQIC0tdXNlLW1heC1wb2RzIGZhbHNlDQoNCg0KLS0vLy0tDQo=" + }, + "launch_time": "2025-04-01T00:02:43Z", + "maintenance_options": { + "AutoRecovery": "default" + }, + "metadata_options": { + "HttpEndpoint": "enabled", + "HttpProtocolIpv6": "disabled", + "HttpPutResponseHopLimit": 2, + "HttpTokens": "required", + "InstanceMetadataTags": "disabled", + "State": "applied" + }, + "monitoring_state": "enabled", + "network_interfaces": [ + { + "Association": null, + "Attachment": { + "AttachTime": "2025-04-01T00:02:43Z", + "AttachmentId": "eni-attach-0fa0d3162b90ad65f", + "DeleteOnTermination": true, + "DeviceIndex": 0, + "EnaSrdSpecification": null, + "NetworkCardIndex": 0, + "Status": "attached" + }, + "ConnectionTrackingConfiguration": null, + "Description": "", + "Groups": [ + { + "GroupId": "sg-04e83501310692e69", + "GroupName": "wandb-odaia-primary-workers" + }, + { + "GroupId": "sg-0f52e5a544842390f", + "GroupName": "wandb-odaia20240311145836066400000018" + } + ], + "InterfaceType": "interface", + "Ipv4Prefixes": null, + "Ipv6Addresses": [], + "Ipv6Prefixes": null, + "MacAddress": "12:c8:b5:a6:bb:39", + "NetworkInterfaceId": "eni-0fafbd414a122d5b0", + "OwnerId": "830241207209", + "PrivateDnsName": "ip-10-10-11-57.ec2.internal", + "PrivateIpAddress": "10.10.11.57", + "PrivateIpAddresses": [ + { + "Association": null, + "Primary": true, + "PrivateDnsName": "ip-10-10-11-57.ec2.internal", + "PrivateIpAddress": "10.10.11.57" + }, + { + "Association": null, + "Primary": false, + "PrivateDnsName": "ip-10-10-11-10.ec2.internal", + "PrivateIpAddress": "10.10.11.10" + }, + { + "Association": null, + "Primary": false, + "PrivateDnsName": "ip-10-10-11-43.ec2.internal", + "PrivateIpAddress": "10.10.11.43" + }, + { + "Association": null, + "Primary": false, + "PrivateDnsName": "ip-10-10-11-233.ec2.internal", + "PrivateIpAddress": "10.10.11.233" + }, + { + "Association": null, + "Primary": false, + "PrivateDnsName": "ip-10-10-11-206.ec2.internal", + "PrivateIpAddress": "10.10.11.206" + }, + { + "Association": null, + "Primary": false, + "PrivateDnsName": "ip-10-10-11-163.ec2.internal", + "PrivateIpAddress": "10.10.11.163" + }, + { + "Association": null, + "Primary": false, + "PrivateDnsName": "ip-10-10-11-94.ec2.internal", + "PrivateIpAddress": "10.10.11.94" + }, + { + "Association": null, + "Primary": false, + "PrivateDnsName": "ip-10-10-11-223.ec2.internal", + "PrivateIpAddress": "10.10.11.223" + }, + { + "Association": null, + "Primary": false, + "PrivateDnsName": "ip-10-10-11-188.ec2.internal", + "PrivateIpAddress": "10.10.11.188" + }, + { + "Association": null, + "Primary": false, + "PrivateDnsName": "ip-10-10-11-210.ec2.internal", + "PrivateIpAddress": "10.10.11.210" + }, + { + "Association": null, + "Primary": false, + "PrivateDnsName": "ip-10-10-11-83.ec2.internal", + "PrivateIpAddress": "10.10.11.83" + }, + { + "Association": null, + "Primary": false, + "PrivateDnsName": "ip-10-10-11-208.ec2.internal", + "PrivateIpAddress": "10.10.11.208" + }, + { + "Association": null, + "Primary": false, + "PrivateDnsName": "ip-10-10-11-87.ec2.internal", + "PrivateIpAddress": "10.10.11.87" + }, + { + "Association": null, + "Primary": false, + "PrivateDnsName": "ip-10-10-11-117.ec2.internal", + "PrivateIpAddress": "10.10.11.117" + }, + { + "Association": null, + "Primary": false, + "PrivateDnsName": "ip-10-10-11-53.ec2.internal", + "PrivateIpAddress": "10.10.11.53" + } + ], + "SourceDestCheck": true, + "Status": "in-use", + "SubnetId": "subnet-0f0d619a86b69c67c", + "VpcId": "vpc-0b78c58bce36a4ad3" + }, + { + "Association": null, + "Attachment": { + "AttachTime": "2025-04-01T00:05:30Z", + "AttachmentId": "eni-attach-06104d186c23d0709", + "DeleteOnTermination": true, + "DeviceIndex": 1, + "EnaSrdSpecification": null, + "NetworkCardIndex": 0, + "Status": "attached" + }, + "ConnectionTrackingConfiguration": null, + "Description": "aws-K8S-i-01288b6098e18be7b", + "Groups": [ + { + "GroupId": "sg-04e83501310692e69", + "GroupName": "wandb-odaia-primary-workers" + }, + { + "GroupId": "sg-0f52e5a544842390f", + "GroupName": "wandb-odaia20240311145836066400000018" + } + ], + "InterfaceType": "interface", + "Ipv4Prefixes": null, + "Ipv6Addresses": [], + "Ipv6Prefixes": null, + "MacAddress": "12:c8:6c:76:c4:fd", + "NetworkInterfaceId": "eni-040f9558e9322de53", + "OwnerId": "830241207209", + "PrivateDnsName": "ip-10-10-11-201.ec2.internal", + "PrivateIpAddress": "10.10.11.201", + "PrivateIpAddresses": [ + { + "Association": null, + "Primary": true, + "PrivateDnsName": "ip-10-10-11-201.ec2.internal", + "PrivateIpAddress": "10.10.11.201" + }, + { + "Association": null, + "Primary": false, + "PrivateDnsName": "ip-10-10-11-73.ec2.internal", + "PrivateIpAddress": "10.10.11.73" + }, + { + "Association": null, + "Primary": false, + "PrivateDnsName": "ip-10-10-11-110.ec2.internal", + "PrivateIpAddress": "10.10.11.110" + }, + { + "Association": null, + "Primary": false, + "PrivateDnsName": "ip-10-10-11-47.ec2.internal", + "PrivateIpAddress": "10.10.11.47" + }, + { + "Association": null, + "Primary": false, + "PrivateDnsName": "ip-10-10-11-12.ec2.internal", + "PrivateIpAddress": "10.10.11.12" + }, + { + "Association": null, + "Primary": false, + "PrivateDnsName": "ip-10-10-11-172.ec2.internal", + "PrivateIpAddress": "10.10.11.172" + }, + { + "Association": null, + "Primary": false, + "PrivateDnsName": "ip-10-10-11-66.ec2.internal", + "PrivateIpAddress": "10.10.11.66" + }, + { + "Association": null, + "Primary": false, + "PrivateDnsName": "ip-10-10-11-134.ec2.internal", + "PrivateIpAddress": "10.10.11.134" + }, + { + "Association": null, + "Primary": false, + "PrivateDnsName": "ip-10-10-11-26.ec2.internal", + "PrivateIpAddress": "10.10.11.26" + }, + { + "Association": null, + "Primary": false, + "PrivateDnsName": "ip-10-10-11-31.ec2.internal", + "PrivateIpAddress": "10.10.11.31" + }, + { + "Association": null, + "Primary": false, + "PrivateDnsName": "ip-10-10-11-28.ec2.internal", + "PrivateIpAddress": "10.10.11.28" + }, + { + "Association": null, + "Primary": false, + "PrivateDnsName": "ip-10-10-11-18.ec2.internal", + "PrivateIpAddress": "10.10.11.18" + }, + { + "Association": null, + "Primary": false, + "PrivateDnsName": "ip-10-10-11-114.ec2.internal", + "PrivateIpAddress": "10.10.11.114" + }, + { + "Association": null, + "Primary": false, + "PrivateDnsName": "ip-10-10-11-150.ec2.internal", + "PrivateIpAddress": "10.10.11.150" + }, + { + "Association": null, + "Primary": false, + "PrivateDnsName": "ip-10-10-11-180.ec2.internal", + "PrivateIpAddress": "10.10.11.180" + } + ], + "SourceDestCheck": true, + "Status": "in-use", + "SubnetId": "subnet-0f0d619a86b69c67c", + "VpcId": "vpc-0b78c58bce36a4ad3" + }, + { + "Association": null, + "Attachment": { + "AttachTime": "2025-04-01T00:09:48Z", + "AttachmentId": "eni-attach-0b9d4a70e4659a04c", + "DeleteOnTermination": true, + "DeviceIndex": 2, + "EnaSrdSpecification": null, + "NetworkCardIndex": 0, + "Status": "attached" + }, + "ConnectionTrackingConfiguration": null, + "Description": "aws-K8S-i-01288b6098e18be7b", + "Groups": [ + { + "GroupId": "sg-04e83501310692e69", + "GroupName": "wandb-odaia-primary-workers" + }, + { + "GroupId": "sg-0f52e5a544842390f", + "GroupName": "wandb-odaia20240311145836066400000018" + } + ], + "InterfaceType": "interface", + "Ipv4Prefixes": null, + "Ipv6Addresses": [], + "Ipv6Prefixes": null, + "MacAddress": "12:6a:6b:f1:53:3f", + "NetworkInterfaceId": "eni-0389354afdd8385f0", + "OwnerId": "830241207209", + "PrivateDnsName": "ip-10-10-11-170.ec2.internal", + "PrivateIpAddress": "10.10.11.170", + "PrivateIpAddresses": [ + { + "Association": null, + "Primary": true, + "PrivateDnsName": "ip-10-10-11-170.ec2.internal", + "PrivateIpAddress": "10.10.11.170" + }, + { + "Association": null, + "Primary": false, + "PrivateDnsName": "ip-10-10-11-202.ec2.internal", + "PrivateIpAddress": "10.10.11.202" + }, + { + "Association": null, + "Primary": false, + "PrivateDnsName": "ip-10-10-11-171.ec2.internal", + "PrivateIpAddress": "10.10.11.171" + }, + { + "Association": null, + "Primary": false, + "PrivateDnsName": "ip-10-10-11-139.ec2.internal", + "PrivateIpAddress": "10.10.11.139" + }, + { + "Association": null, + "Primary": false, + "PrivateDnsName": "ip-10-10-11-232.ec2.internal", + "PrivateIpAddress": "10.10.11.232" + }, + { + "Association": null, + "Primary": false, + "PrivateDnsName": "ip-10-10-11-105.ec2.internal", + "PrivateIpAddress": "10.10.11.105" + }, + { + "Association": null, + "Primary": false, + "PrivateDnsName": "ip-10-10-11-98.ec2.internal", + "PrivateIpAddress": "10.10.11.98" + }, + { + "Association": null, + "Primary": false, + "PrivateDnsName": "ip-10-10-11-32.ec2.internal", + "PrivateIpAddress": "10.10.11.32" + }, + { + "Association": null, + "Primary": false, + "PrivateDnsName": "ip-10-10-11-96.ec2.internal", + "PrivateIpAddress": "10.10.11.96" + }, + { + "Association": null, + "Primary": false, + "PrivateDnsName": "ip-10-10-11-6.ec2.internal", + "PrivateIpAddress": "10.10.11.6" + }, + { + "Association": null, + "Primary": false, + "PrivateDnsName": "ip-10-10-11-37.ec2.internal", + "PrivateIpAddress": "10.10.11.37" + }, + { + "Association": null, + "Primary": false, + "PrivateDnsName": "ip-10-10-11-157.ec2.internal", + "PrivateIpAddress": "10.10.11.157" + }, + { + "Association": null, + "Primary": false, + "PrivateDnsName": "ip-10-10-11-243.ec2.internal", + "PrivateIpAddress": "10.10.11.243" + }, + { + "Association": null, + "Primary": false, + "PrivateDnsName": "ip-10-10-11-112.ec2.internal", + "PrivateIpAddress": "10.10.11.112" + }, + { + "Association": null, + "Primary": false, + "PrivateDnsName": "ip-10-10-11-183.ec2.internal", + "PrivateIpAddress": "10.10.11.183" + } + ], + "SourceDestCheck": true, + "Status": "in-use", + "SubnetId": "subnet-0f0d619a86b69c67c", + "VpcId": "vpc-0b78c58bce36a4ad3" + } + ], + "partition": "aws", + "placement_availability_zone": "us-east-1b", + "placement_group_name": "", + "placement_tenancy": "default", + "platform": "", + "platform_details": "Linux/UNIX", + "private_dns_name": "ip-10-10-11-57.ec2.internal", + "private_dns_name_options": { + "EnableResourceNameDnsAAAARecord": false, + "EnableResourceNameDnsARecord": false, + "HostnameType": "ip-name" + }, + "private_ip_address": "10.10.11.57", + "product_codes": [], + "public_dns_name": "", + "region": "us-east-1", + "root_device_name": "/dev/xvda", + "root_device_type": "ebs", + "security_groups": [ + { + "GroupId": "sg-04e83501310692e69", + "GroupName": "wandb-odaia-primary-workers" + }, + { + "GroupId": "sg-0f52e5a544842390f", + "GroupName": "wandb-odaia20240311145836066400000018" + } + ], + "source_dest_check": true, + "sp_connection_name": "aws", + "sp_ctx": { + "connection_name": "aws", + "steampipe": { + "sdk_version": "5.11.4" + } + }, + "sriov_net_support": "simple", + "state_code": 16, + "state_transition_reason": "", + "state_transition_time": "2025-04-01T00:02:43Z", + "subnet_id": "subnet-0f0d619a86b69c67c", + "tags": { + "GithubOrg": "terraform-aws-wandb", + "GithubRepo": "wandb", + "Name": "wandb-odaia-b", + "TerraformModule": "terraform-aws-wandb/module/app_eks", + "TerraformNamespace": "wandb-odaia", + "aws:autoscaling:groupName": "eks-wandb-odaia-b20241015000738841700000005-68c946c0-3a40-ec02-6c53-1c319cac08e5", + "aws:ec2:fleet-id": "fleet-f186b826-5197-c19c-8410-860ad9dbf3a5", + "aws:ec2launchtemplate:id": "lt-0625cbe472e9ca046", + "aws:ec2launchtemplate:version": "6", + "aws:eks:cluster-name": "wandb-odaia", + "customer-ns": "wandb-odaia", + "eks:cluster-name": "wandb-odaia", + "eks:nodegroup-name": "wandb-odaia-b20241015000738841700000005", + "env": "managed-install", + "k8s.io/cluster-autoscaler/enabled": "true", + "k8s.io/cluster-autoscaler/wandb-odaia": "owned", + "kubernetes.io/cluster/wandb-odaia": "owned" + }, + "tags_src": [ + { + "Key": "TerraformModule", + "Value": "terraform-aws-wandb/module/app_eks" + }, + { + "Key": "TerraformNamespace", + "Value": "wandb-odaia" + }, + { + "Key": "Name", + "Value": "wandb-odaia-b" + }, + { + "Key": "eks:nodegroup-name", + "Value": "wandb-odaia-b20241015000738841700000005" + }, + { + "Key": "env", + "Value": "managed-install" + }, + { + "Key": "GithubRepo", + "Value": "wandb" + }, + { + "Key": "kubernetes.io/cluster/wandb-odaia", + "Value": "owned" + }, + { + "Key": "customer-ns", + "Value": "wandb-odaia" + }, + { + "Key": "aws:ec2launchtemplate:version", + "Value": "6" + }, + { + "Key": "eks:cluster-name", + "Value": "wandb-odaia" + }, + { + "Key": "k8s.io/cluster-autoscaler/enabled", + "Value": "true" + }, + { + "Key": "aws:autoscaling:groupName", + "Value": "eks-wandb-odaia-b20241015000738841700000005-68c946c0-3a40-ec02-6c53-1c319cac08e5" + }, + { + "Key": "aws:ec2:fleet-id", + "Value": "fleet-f186b826-5197-c19c-8410-860ad9dbf3a5" + }, + { + "Key": "k8s.io/cluster-autoscaler/wandb-odaia", + "Value": "owned" + }, + { + "Key": "aws:ec2launchtemplate:id", + "Value": "lt-0625cbe472e9ca046" + }, + { + "Key": "GithubOrg", + "Value": "terraform-aws-wandb" + }, + { + "Key": "aws:eks:cluster-name", + "Value": "wandb-odaia" + } + ], + "title": "wandb-odaia-b", + "usage_operation": "RunInstances", + "usage_operation_update_time": "2025-04-01T00:02:43Z", + "user_data": "MIME-Version: 1.0Content-Type: multipart/mixed; boundary=\"//\"--//Content-Transfer-Encoding: 7bitContent-Type: text/x-shellscriptMime-Version: 1.0#!/bin/bash -e# Set bootstrap envprintf '#!/bin/bashexport ADDITIONAL_KUBELET_EXTRA_ARGS=\"--system-reserved=cpu=70m,memory=100Mi,ephemeral-storage=750Mi,pid=500\"' \u003e /etc/profile.d/eks-bootstrap-env.sh# Source extra environment variables in bootstrap scriptsed -i '/^set -o errexit/a\\\\nsource /etc/profile.d/eks-bootstrap-env.sh' /etc/eks/bootstrap.sh# Merge ADDITIONAL_KUBELET_EXTRA_ARGS into KUBELET_EXTRA_ARGSsed -i 's/^KUBELET_EXTRA_ARGS=\"${KUBELET_EXTRA_ARGS:-}/KUBELET_EXTRA_ARGS=\"${KUBELET_EXTRA_ARGS:-} ${ADDITIONAL_KUBELET_EXTRA_ARGS}/' /etc/eks/bootstrap.sh# User supplied pre userdata--//Content-Type: text/x-shellscript; charset=\"us-ascii\"#!/bin/bashset -exB64_CLUSTER_CA=LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURCVENDQWUyZ0F3SUJBZ0lJUE5pY2xvSTM5eTh3RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB5TkRBek1URXhORFU0TXpaYUZ3MHpOREF6TURreE5UQXpNelphTUJVeApFekFSQmdOVkJBTVRDbXQxWW1WeWJtVjBaWE13Z2dFaU1BMEdDU3FHU0liM0RRRUJBUVVBQTRJQkR3QXdnZ0VLCkFvSUJBUUM5dU5OSU5pQTdQMHpMNWtzZ041bzhUV0pSalRqVzBwbW56MUZJVjhIWEF6em9XOFNPRjdTZngzcGkKUXZycHBkRTFrR0d2YmxHZ1B1M1JjNjMxRStwY2RBSXpveTdUbld2emJFbVUwRDdIYWtQVGdRZmRidVFLaVIwYgpRNFJhc01odGxHVmg0T3BtY3NndE9HdmNnT0ZWNTBKaHVsbE9hdWRoejNGK1Nvb2dkVGVTM3diZzFHVGhRek85CkZjV1htdjlXU1FRMzd1d1Bnd2FvdXFGeFg0dWZOdTRvb2lMeG4yQjZJMlJSbWlNeG9SQ0ZPNnBJSWw3dnRNd3UKTnI2QzVST1NObHIvckRFM0QyZlNoV0NtaXd0YkZHUHZ2ektITGFSaDc4NFJva0lIRU14MG16eHdyTnRxSUpPRwpGTXFNNlJEWWg1c1l5M3lTbzlNQkFCZUVMblZqQWdNQkFBR2pXVEJYTUE0R0ExVWREd0VCL3dRRUF3SUNwREFQCkJnTlZIUk1CQWY4RUJUQURBUUgvTUIwR0ExVWREZ1FXQkJUZ2pDOWIwc0w3c3FENGFXVVhLTE1PMTZkRHZUQVYKQmdOVkhSRUVEakFNZ2dwcmRXSmxjbTVsZEdWek1BMEdDU3FHU0liM0RRRUJDd1VBQTRJQkFRQkd4THUraFE3bgpSMEpjaS9Ud2NWZi9XcFhCQlR2dmtuM3N4YWdUQ1JrcXVOZDhVMFQyY0wyYlJXdzFPVzNEMG5COUJmMlM5MlZICm5jaU9HNTRJVWhUNDNQS0dlMVlpakNxME43amN4QjU2d0hSRko2QW9CUmtUdVgzbzRzYkJuK01rODJ0Wkkzc04KaWs0ZndvcGJtaU92VWlqTldSRUxCeEJHaTZhalRTVkpJbnVDVk05bW1wTWVPN1BxamNqWW1PMDlLUUVPZkJtWQovRDEzUTByZm96aC9CWHBWMldKQU1GeDhpU3lYck5JS2wvaW9rdHU5YmpDSkxVckJaWVlnT0p3aG5GNVc3cmc4CndhL2toSnIzQUFMMHB2N3ZTcnNUdXFoVzVWQytSSERZMlA0U0lYYThqTGxRVkcveXVkMzdJKzloV2xpZmsxNnAKWm9Id3h3eFBKS3NJCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0KAPI_SERVER_URL=https://AA39B4408C69215B65950D274F1A6334.gr7.us-east-1.eks.amazonaws.comK8S_CLUSTER_DNS_IP=172.20.0.10/etc/eks/bootstrap.sh wandb-odaia --kubelet-extra-args '--node-labels=eks.amazonaws.com/nodegroup-image=ami-03fcb7c9c1ca36b4d,eks.amazonaws.com/sourceLaunchTemplateVersion=5,eks.amazonaws.com/capacityType=ON_DEMAND,eks.amazonaws.com/nodegroup=wandb-odaia-b20241015000738841700000005,eks.amazonaws.com/sourceLaunchTemplateId=lt-0f225f0c94a5b8a26 --max-pods=58' --b64-cluster-ca $B64_CLUSTER_CA --apiserver-endpoint $API_SERVER_URL --dns-cluster-ip $K8S_CLUSTER_DNS_IP --use-max-pods false--//--", + "virtualization_type": "hvm", + "vpc_id": "vpc-0b78c58bce36a4ad3" +} diff --git a/cmd/ctrlc/root/sync/steampipe/adapter/aws/vpc.go b/cmd/ctrlc/root/sync/steampipe/adapter/aws/vpc.go index 832ceae..980283c 100644 --- a/cmd/ctrlc/root/sync/steampipe/adapter/aws/vpc.go +++ b/cmd/ctrlc/root/sync/steampipe/adapter/aws/vpc.go @@ -1,52 +1,52 @@ package aws -import ( - "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/steampipe/adapter/model" - "github.com/ctrlplanedev/cli/internal/api" -) - -const vpcTable = "aws_vpc" - -var VPC model.SteampipeAdapter = &model.SteampipeAdapterStruct{ - Table: vpcTable, - Translate: func(data map[string]interface{}) (api.AgentResource, bool) { - var entityName = vpcTable - var sqlRow model.SqlRow = model.SqlRow{ - EntityName: entityName, - Data: data, - } - - var name string - var arn string - var tags model.Tags - var accountId string - var region string - - var zero api.AgentResource = api.AgentResource{} - - if !model.GetRequiredValue[string](sqlRow, "arn", &arn) || - !model.GetRequiredValue[string](sqlRow, "vpc_id", &name) || - !model.GetOptionalValue[model.Tags](sqlRow, "tags", &tags) || - !model.GetRequiredValue[string](sqlRow, "account_id", &accountId) || - !model.GetRequiredValue[string](sqlRow, "region", ®ion) { - return zero, false - } - - metadata := model.BuildMetadata(map[string]string{ - "aws/account-id": accountId, - "aws/arn": arn, - "aws/region": region, - }).AppendTags(tags) - - return api.AgentResource{ - Identifier: arn, - Name: name, - Version: "vpc/v1", - Kind: "VPC", - Config: map[string]interface{}{ - "name": name, - }, - Metadata: metadata, - }, true - }, -} +//import ( +// "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/steampipe/adapter/model" +// "github.com/ctrlplanedev/cli/internal/api" +//) +// +//const vpcTable = "aws_vpc" +// +//var VPC model.SteampipeAdapter = &model.SteampipeAdapterStruct{ +// Table: vpcTable, +// Translate: func(data map[string]interface{}) (api.AgentResource, bool) { +// var entityName = vpcTable +// var sqlRow model.SqlRow = model.SqlRow{ +// EntityName: entityName, +// Data: data, +// } +// +// var name string +// var arn string +// var tags model.Tags +// var accountId string +// var region string +// +// var zero api.AgentResource = api.AgentResource{} +// +// if !model.GetRequiredValue[string](sqlRow, "arn", &arn) || +// !model.GetRequiredValue[string](sqlRow, "vpc_id", &name) || +// !model.GetOptionalValue[model.Tags](sqlRow, "tags", &tags) || +// !model.GetRequiredValue[string](sqlRow, "account_id", &accountId) || +// !model.GetRequiredValue[string](sqlRow, "region", ®ion) { +// return zero, false +// } +// +// metadata := model.BuildMetadata(map[string]string{ +// "aws/account-id": accountId, +// "aws/arn": arn, +// "aws/region": region, +// }).AppendTags(tags) +// +// return api.AgentResource{ +// Identifier: arn, +// Name: name, +// Version: "vpc/v1", +// Kind: "VPC", +// Config: map[string]interface{}{ +// "name": name, +// }, +// Metadata: metadata, +// }, true +// }, +//} diff --git a/cmd/ctrlc/root/sync/steampipe/adapter/azure/aks.go b/cmd/ctrlc/root/sync/steampipe/adapter/azure/aks.go index 033f5bf..aec551f 100644 --- a/cmd/ctrlc/root/sync/steampipe/adapter/azure/aks.go +++ b/cmd/ctrlc/root/sync/steampipe/adapter/azure/aks.go @@ -1,76 +1,76 @@ package azure -import ( - "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/steampipe/adapter/model" - "github.com/ctrlplanedev/cli/internal/api" -) - -const aksTable = "azure_kubernetes_cluster" - -var AKS model.SteampipeAdapter = &model.SteampipeAdapterStruct{ - Table: aksTable, - Translate: func(data map[string]interface{}) (api.AgentResource, bool) { - var entityName = aksTable - var sqlRow model.SqlRow = model.SqlRow{ - EntityName: entityName, - Data: data, - } - - var name string - var id string - var tags model.Tags - var subscriptionId string - var location string - var status string - var version string - var autoscalerProfile map[string]interface{} - var autoscaler string - - var zero api.AgentResource = api.AgentResource{} - - if !model.GetRequiredValue[string](sqlRow, "id", &id) || - !model.GetRequiredValue[string](sqlRow, "name", &name) || - !model.GetOptionalValue[model.Tags](sqlRow, "tags", &tags) || - !model.GetRequiredValue[string](sqlRow, "subscription_id", &subscriptionId) || - !model.GetRequiredValue[string](sqlRow, "location", &location) || - !model.GetOptionalValue[string](sqlRow, "power_state.code", &status) || - !model.GetRequiredValue[string](sqlRow, "kubernetes_version", &version) || - !model.GetRequiredValue[map[string]interface{}](sqlRow, "auto_scaler_profile", &autoscalerProfile) { - return zero, false - } - - if autoscalerProfile != nil { - autoscaler = "true" - } else { - autoscaler = "false" - } - - metadata := model.BuildMetadata(map[string]string{ - "ctrlplane/external-id": id, - "azure/id": id, - "azure/location": location, - "azure/subscription-id": subscriptionId, - "kubernetes/version": version, - "kubernetes/autoscaler-enabled": autoscaler, - }).AppendTags(tags) - - return api.AgentResource{ - Identifier: id, - Name: name, - Version: "kubernetes/v1", - Kind: "ClusterAPI", - Config: map[string]interface{}{ - "auth": map[string]string{ - "method": "azure/aks", - "location": location, - "subscriptionId": subscriptionId, - "clusterName": name, - }, - "name": name, - "server": map[string]string{}, - "status": status, - }, - Metadata: metadata, - }, true - }, -} +//import ( +// "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/steampipe/adapter/model" +// "github.com/ctrlplanedev/cli/internal/api" +//) +// +//const aksTable = "azure_kubernetes_cluster" +// +//var AKS model.SteampipeAdapter = &model.SteampipeAdapterStruct{ +// Table: aksTable, +// Translate: func(data map[string]interface{}) (api.AgentResource, bool) { +// var entityName = aksTable +// var sqlRow model.SqlRow = model.SqlRow{ +// EntityName: entityName, +// Data: data, +// } +// +// var name string +// var id string +// var tags model.Tags +// var subscriptionId string +// var location string +// var status string +// var version string +// var autoscalerProfile map[string]interface{} +// var autoscaler string +// +// var zero api.AgentResource = api.AgentResource{} +// +// if !model.GetRequiredValue[string](sqlRow, "id", &id) || +// !model.GetRequiredValue[string](sqlRow, "name", &name) || +// !model.GetOptionalValue[model.Tags](sqlRow, "tags", &tags) || +// !model.GetRequiredValue[string](sqlRow, "subscription_id", &subscriptionId) || +// !model.GetRequiredValue[string](sqlRow, "location", &location) || +// !model.GetOptionalValue[string](sqlRow, "power_state.code", &status) || +// !model.GetRequiredValue[string](sqlRow, "kubernetes_version", &version) || +// !model.GetRequiredValue[map[string]interface{}](sqlRow, "auto_scaler_profile", &autoscalerProfile) { +// return zero, false +// } +// +// if autoscalerProfile != nil { +// autoscaler = "true" +// } else { +// autoscaler = "false" +// } +// +// metadata := model.BuildMetadata(map[string]string{ +// "ctrlplane/external-id": id, +// "azure/id": id, +// "azure/location": location, +// "azure/subscription-id": subscriptionId, +// "kubernetes/version": version, +// "kubernetes/autoscaler-enabled": autoscaler, +// }).AppendTags(tags) +// +// return api.AgentResource{ +// Identifier: id, +// Name: name, +// Version: "kubernetes/v1", +// Kind: "ClusterAPI", +// Config: map[string]interface{}{ +// "auth": map[string]string{ +// "method": "azure/aks", +// "location": location, +// "subscriptionId": subscriptionId, +// "clusterName": name, +// }, +// "name": name, +// "server": map[string]string{}, +// "status": status, +// }, +// Metadata: metadata, +// }, true +// }, +//} diff --git a/cmd/ctrlc/root/sync/steampipe/adapter/gcp/compute.go b/cmd/ctrlc/root/sync/steampipe/adapter/gcp/compute.go index e081867..69c2c34 100644 --- a/cmd/ctrlc/root/sync/steampipe/adapter/gcp/compute.go +++ b/cmd/ctrlc/root/sync/steampipe/adapter/gcp/compute.go @@ -1,68 +1,68 @@ package gcp -import ( - "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/steampipe/adapter/model" - "github.com/ctrlplanedev/cli/internal/api" - "strconv" -) - -const computeTable = "gcp_compute_instance" - -var Compute model.SteampipeAdapter = &model.SteampipeAdapterStruct{ - Table: computeTable, - Translate: func(data map[string]interface{}) (api.AgentResource, bool) { - var entityName = computeTable - var sqlRow model.SqlRow = model.SqlRow{ - EntityName: entityName, - Data: data, - } - - var name string - var id int64 - var tags model.Tags - var project string - var location string - var status string - var selfLink string - - var zero api.AgentResource = api.AgentResource{} - - if !model.GetRequiredValue[int64](sqlRow, "id", &id) || - !model.GetRequiredValue[string](sqlRow, "name", &name) || - !model.GetOptionalValue[model.Tags](sqlRow, "tags", &tags) || - !model.GetRequiredValue[string](sqlRow, "project", &project) || - !model.GetRequiredValue[string](sqlRow, "location", &location) || - !model.GetRequiredValue[string](sqlRow, "status", &status) || - !model.GetRequiredValue[string](sqlRow, "self_link", &selfLink) { - return zero, false - } - - metadata := model.BuildMetadata(map[string]string{ - "ctrlplane/external-id": strconv.FormatInt(id, 10), - "google/account-id": project, - "google/id": strconv.FormatInt(id, 10), - "google/location": location, - "google/project": project, - "google/self-link": selfLink, - }).AppendTags(tags) - - return api.AgentResource{ - Identifier: strconv.FormatInt(id, 10), - Name: name, - Version: "compute/v1", - Kind: "Compute", - Config: map[string]interface{}{ - "auth": map[string]string{ - "method": "google/compute", - "location": location, - "project": project, - "name": name, - }, - "name": name, - "server": map[string]string{}, - "status": status, - }, - Metadata: metadata, - }, true - }, -} +//import ( +// "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/steampipe/adapter/model" +// "github.com/ctrlplanedev/cli/internal/api" +// "strconv" +//) +// +//const computeTable = "gcp_compute_instance" +// +//var Compute model.SteampipeAdapter = &model.SteampipeAdapterStruct{ +// Table: computeTable, +// Translate: func(data map[string]interface{}) (api.AgentResource, bool) { +// var entityName = computeTable +// var sqlRow model.SqlRow = model.SqlRow{ +// EntityName: entityName, +// Data: data, +// } +// +// var name string +// var id int64 +// var tags model.Tags +// var project string +// var location string +// var status string +// var selfLink string +// +// var zero api.AgentResource = api.AgentResource{} +// +// if !model.GetRequiredValue[int64](sqlRow, "id", &id) || +// !model.GetRequiredValue[string](sqlRow, "name", &name) || +// !model.GetOptionalValue[model.Tags](sqlRow, "tags", &tags) || +// !model.GetRequiredValue[string](sqlRow, "project", &project) || +// !model.GetRequiredValue[string](sqlRow, "location", &location) || +// !model.GetRequiredValue[string](sqlRow, "status", &status) || +// !model.GetRequiredValue[string](sqlRow, "self_link", &selfLink) { +// return zero, false +// } +// +// metadata := model.BuildMetadata(map[string]string{ +// "ctrlplane/external-id": strconv.FormatInt(id, 10), +// "google/account-id": project, +// "google/id": strconv.FormatInt(id, 10), +// "google/location": location, +// "google/project": project, +// "google/self-link": selfLink, +// }).AppendTags(tags) +// +// return api.AgentResource{ +// Identifier: strconv.FormatInt(id, 10), +// Name: name, +// Version: "compute/v1", +// Kind: "Compute", +// Config: map[string]interface{}{ +// "auth": map[string]string{ +// "method": "google/compute", +// "location": location, +// "project": project, +// "name": name, +// }, +// "name": name, +// "server": map[string]string{}, +// "status": status, +// }, +// Metadata: metadata, +// }, true +// }, +//} diff --git a/cmd/ctrlc/root/sync/steampipe/adapter/gcp/gke.go b/cmd/ctrlc/root/sync/steampipe/adapter/gcp/gke.go index 2333b75..2a8f112 100644 --- a/cmd/ctrlc/root/sync/steampipe/adapter/gcp/gke.go +++ b/cmd/ctrlc/root/sync/steampipe/adapter/gcp/gke.go @@ -1,83 +1,83 @@ package gcp -import ( - "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/steampipe/adapter/model" - "github.com/ctrlplanedev/cli/internal/api" - "strconv" -) - -const gkeTable = "gcp_kubernetes_cluster" - -var GKE model.SteampipeAdapter = &model.SteampipeAdapterStruct{ - Table: gkeTable, - Translate: func(data map[string]interface{}) (api.AgentResource, bool) { - var entityName = gkeTable - var sqlRow model.SqlRow = model.SqlRow{ - EntityName: entityName, - Data: data, - } - - var name string - var id string - var tags model.Tags - var project string - var location string - var certificateAuthorityData string - var status string - var version string - var endpoint string - var selfLink string - var autopilot bool - var autoscaling string - - var zero api.AgentResource = api.AgentResource{} - - if !model.GetRequiredValue[string](sqlRow, "id", &id) || - !model.GetRequiredValue[string](sqlRow, "name", &name) || - !model.GetOptionalValue[model.Tags](sqlRow, "tags", &tags) || - !model.GetRequiredValue[string](sqlRow, "project", &project) || - !model.GetRequiredValue[string](sqlRow, "location", &location) || - !model.GetRequiredValue[string](sqlRow, "master_auth.clusterCaCertificate", &certificateAuthorityData) || - !model.GetRequiredValue[string](sqlRow, "status", &status) || - !model.GetRequiredValue[string](sqlRow, "current_master_version", &version) || - !model.GetRequiredValue[string](sqlRow, "endpoint", &endpoint) || - !model.GetRequiredValue[string](sqlRow, "self_link", &selfLink) || - !model.GetRequiredValue[bool](sqlRow, "autopilot_enabled", &autopilot) || - !model.GetOptionalValue[string](sqlRow, "autoscaling.autoscalingProfile", &autoscaling) { - return zero, false - } - - metadata := model.BuildMetadata(map[string]string{ - "ctrlplane/external-id": id, - "google/account-id": project, - "google/id": id, - "google/location": location, - "google/project": project, - "google/self-link": selfLink, - "google/autopilot": strconv.FormatBool(autopilot), - "kubernetes/autoscaling": autoscaling, - }).AppendTags(tags) - - return api.AgentResource{ - Identifier: id, - Name: name, - Version: "kubernetes/v1", - Kind: "ClusterAPI", - Config: map[string]interface{}{ - "auth": map[string]string{ - "method": "google/gke", - "location": location, - "project": project, - "clusterName": name, - }, - "name": name, - "server": map[string]string{ - "endpoint": endpoint, - "certificationAuthorityData": certificateAuthorityData, - }, - "status": status, - }, - Metadata: metadata, - }, true - }, -} +//import ( +// "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/steampipe/adapter/model" +// "github.com/ctrlplanedev/cli/internal/api" +// "strconv" +//) +// +//const gkeTable = "gcp_kubernetes_cluster" +// +//var GKE model.SteampipeAdapter = &model.SteampipeAdapterStruct{ +// Table: gkeTable, +// Translate: func(data map[string]interface{}) (api.AgentResource, bool) { +// var entityName = gkeTable +// var sqlRow model.SqlRow = model.SqlRow{ +// EntityName: entityName, +// Data: data, +// } +// +// var name string +// var id string +// var tags model.Tags +// var project string +// var location string +// var certificateAuthorityData string +// var status string +// var version string +// var endpoint string +// var selfLink string +// var autopilot bool +// var autoscaling string +// +// var zero api.AgentResource = api.AgentResource{} +// +// if !model.GetRequiredValue[string](sqlRow, "id", &id) || +// !model.GetRequiredValue[string](sqlRow, "name", &name) || +// !model.GetOptionalValue[model.Tags](sqlRow, "tags", &tags) || +// !model.GetRequiredValue[string](sqlRow, "project", &project) || +// !model.GetRequiredValue[string](sqlRow, "location", &location) || +// !model.GetRequiredValue[string](sqlRow, "master_auth.clusterCaCertificate", &certificateAuthorityData) || +// !model.GetRequiredValue[string](sqlRow, "status", &status) || +// !model.GetRequiredValue[string](sqlRow, "current_master_version", &version) || +// !model.GetRequiredValue[string](sqlRow, "endpoint", &endpoint) || +// !model.GetRequiredValue[string](sqlRow, "self_link", &selfLink) || +// !model.GetRequiredValue[bool](sqlRow, "autopilot_enabled", &autopilot) || +// !model.GetOptionalValue[string](sqlRow, "autoscaling.autoscalingProfile", &autoscaling) { +// return zero, false +// } +// +// metadata := model.BuildMetadata(map[string]string{ +// "ctrlplane/external-id": id, +// "google/account-id": project, +// "google/id": id, +// "google/location": location, +// "google/project": project, +// "google/self-link": selfLink, +// "google/autopilot": strconv.FormatBool(autopilot), +// "kubernetes/autoscaling": autoscaling, +// }).AppendTags(tags) +// +// return api.AgentResource{ +// Identifier: id, +// Name: name, +// Version: "kubernetes/v1", +// Kind: "ClusterAPI", +// Config: map[string]interface{}{ +// "auth": map[string]string{ +// "method": "google/gke", +// "location": location, +// "project": project, +// "clusterName": name, +// }, +// "name": name, +// "server": map[string]string{ +// "endpoint": endpoint, +// "certificationAuthorityData": certificateAuthorityData, +// }, +// "status": status, +// }, +// Metadata: metadata, +// }, true +// }, +//} diff --git a/cmd/ctrlc/root/sync/steampipe/adapter/gcp/sql.go b/cmd/ctrlc/root/sync/steampipe/adapter/gcp/sql.go index 89d7f21..6436964 100644 --- a/cmd/ctrlc/root/sync/steampipe/adapter/gcp/sql.go +++ b/cmd/ctrlc/root/sync/steampipe/adapter/gcp/sql.go @@ -1,73 +1,73 @@ package gcp -import ( - "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/steampipe/adapter/model" - "github.com/ctrlplanedev/cli/internal/api" -) - -const sqlTable = "gcp_sql_database_instance" - -var SQL model.SteampipeAdapter = &model.SteampipeAdapterStruct{ - Table: sqlTable, - Translate: func(data map[string]interface{}) (api.AgentResource, bool) { - var entityName = sqlTable - var sqlRow model.SqlRow = model.SqlRow{ - EntityName: entityName, - Data: data, - } - - var name string - var id string - var tags model.Tags - var project string - var location string - var sslCert string - var status string - var version string - var selfLink string - - var zero api.AgentResource = api.AgentResource{} - - if !model.GetRequiredValue[string](sqlRow, "connection_name", &id) || - !model.GetRequiredValue[string](sqlRow, "name", &name) || - !model.GetOptionalValue[model.Tags](sqlRow, "tags", &tags) || - !model.GetRequiredValue[string](sqlRow, "project", &project) || - !model.GetRequiredValue[string](sqlRow, "location", &location) || - !model.GetRequiredValue[string](sqlRow, "ssl_configuration.cert", &sslCert) || - !model.GetRequiredValue[string](sqlRow, "state", &status) || - !model.GetRequiredValue[string](sqlRow, "database_installed_version", &version) || - !model.GetRequiredValue[string](sqlRow, "self_link", &selfLink) { - return zero, false - } - - metadata := model.BuildMetadata(map[string]string{ - "ctrlplane/external-id": id, - "google/account-id": project, - "google/id": id, - "google/location": location, - "google/project": project, - "google/self-link": selfLink, - }).AppendTags(tags) - - return api.AgentResource{ - Identifier: id, - Name: name, - Version: "database/v1", - Kind: "Database", - Config: map[string]interface{}{ - "auth": map[string]string{ - "method": "google/sql", - "location": location, - "project": project, - "name": name, - }, - "name": name, - "server": map[string]string{ - "sslCert": sslCert, - }, - "status": status, - }, - Metadata: metadata, - }, true - }, -} +//import ( +// "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/steampipe/adapter/model" +// "github.com/ctrlplanedev/cli/internal/api" +//) +// +//const sqlTable = "gcp_sql_database_instance" +// +//var SQL model.SteampipeAdapter = &model.SteampipeAdapterStruct{ +// Table: sqlTable, +// Translate: func(data map[string]interface{}) (api.AgentResource, bool) { +// var entityName = sqlTable +// var sqlRow model.SqlRow = model.SqlRow{ +// EntityName: entityName, +// Data: data, +// } +// +// var name string +// var id string +// var tags model.Tags +// var project string +// var location string +// var sslCert string +// var status string +// var version string +// var selfLink string +// +// var zero api.AgentResource = api.AgentResource{} +// +// if !model.GetRequiredValue[string](sqlRow, "connection_name", &id) || +// !model.GetRequiredValue[string](sqlRow, "name", &name) || +// !model.GetOptionalValue[model.Tags](sqlRow, "tags", &tags) || +// !model.GetRequiredValue[string](sqlRow, "project", &project) || +// !model.GetRequiredValue[string](sqlRow, "location", &location) || +// !model.GetRequiredValue[string](sqlRow, "ssl_configuration.cert", &sslCert) || +// !model.GetRequiredValue[string](sqlRow, "state", &status) || +// !model.GetRequiredValue[string](sqlRow, "database_installed_version", &version) || +// !model.GetRequiredValue[string](sqlRow, "self_link", &selfLink) { +// return zero, false +// } +// +// metadata := model.BuildMetadata(map[string]string{ +// "ctrlplane/external-id": id, +// "google/account-id": project, +// "google/id": id, +// "google/location": location, +// "google/project": project, +// "google/self-link": selfLink, +// }).AppendTags(tags) +// +// return api.AgentResource{ +// Identifier: id, +// Name: name, +// Version: "database/v1", +// Kind: "Database", +// Config: map[string]interface{}{ +// "auth": map[string]string{ +// "method": "google/sql", +// "location": location, +// "project": project, +// "name": name, +// }, +// "name": name, +// "server": map[string]string{ +// "sslCert": sslCert, +// }, +// "status": status, +// }, +// Metadata: metadata, +// }, true +// }, +//} diff --git a/cmd/ctrlc/root/sync/steampipe/adapter/model/adapter.go b/cmd/ctrlc/root/sync/steampipe/adapter/model/adapter.go index b868a39..f0b330b 100644 --- a/cmd/ctrlc/root/sync/steampipe/adapter/model/adapter.go +++ b/cmd/ctrlc/root/sync/steampipe/adapter/model/adapter.go @@ -1,23 +1,19 @@ package model -import ( - "github.com/ctrlplanedev/cli/internal/api" -) - type SteampipeAdapterStruct struct { - Table string - Translate func(data map[string]interface{}) (api.AgentResource, bool) + Table string + Convert func(jsonRow string) (string, bool) } func (a *SteampipeAdapterStruct) EntityName() string { return a.Table } -func (a *SteampipeAdapterStruct) ToApiResource(row SqlRow) (api.AgentResource, bool) { - return a.Translate(row.Data) +func (a *SteampipeAdapterStruct) ToResourceJson(sqlRow SqlRow) (string, bool) { + return a.Convert(sqlRow.Json) } type SteampipeAdapter interface { EntityName() string - ToApiResource(row SqlRow) (api.AgentResource, bool) + ToResourceJson(sqlRow SqlRow) (string, bool) } diff --git a/cmd/ctrlc/root/sync/steampipe/adapter/model/metadata.go b/cmd/ctrlc/root/sync/steampipe/adapter/model/metadata.go index 326acab..43236d7 100644 --- a/cmd/ctrlc/root/sync/steampipe/adapter/model/metadata.go +++ b/cmd/ctrlc/root/sync/steampipe/adapter/model/metadata.go @@ -1,28 +1,12 @@ package model -func BuildMetadata(data map[string]string) Metadata { - var result = make(Metadata) - for k, v := range data { - result[k] = v - } - return result -} - type Metadata map[string]string -type Tags = map[string]interface{} +type Tags = map[string]string func (m Metadata) AppendTags(tags Tags) Metadata { for k, v := range tags { - var strVal string - var ok bool - if v != nil { - if strVal, ok = v.(string); ok { - m[k] = strVal - } - } else { - m[k] = "" - } + m[k] = v } return m } diff --git a/cmd/ctrlc/root/sync/steampipe/adapter/model/sql_row.go b/cmd/ctrlc/root/sync/steampipe/adapter/model/sql_row.go index 9d836cb..4fd6d42 100644 --- a/cmd/ctrlc/root/sync/steampipe/adapter/model/sql_row.go +++ b/cmd/ctrlc/root/sync/steampipe/adapter/model/sql_row.go @@ -4,73 +4,43 @@ import ( "encoding/json" "fmt" "github.com/charmbracelet/log" + "github.com/xeipuuv/gojsonschema" "strings" ) type SqlRow struct { EntityName string - Data map[string]interface{} + Json string } -func GetRequiredValue[T any](row SqlRow, path string, valuePtr *T) bool { - return GetRowValue[T](row, path, valuePtr, true) -} - -func GetOptionalValue[T any](row SqlRow, path string, valuePtr *T) bool { - return GetRowValue[T](row, path, valuePtr, false) -} - -func GetRowValue[T any](row SqlRow, path string, valuePtr *T, required bool) bool { - var err error - var result T - if result, err = getPathValue[T](row.Data, path); err != nil { - if required { - log.Infof("%s -> %s: %s", row.EntityName, path, err.Error()) - if log.GetLevel() >= log.DebugLevel { - dataStr, _ := json.Marshal(row.Data) - log.Debugf("data: %s", dataStr) - } - return false - } +// ValidateAndUnmarshal ValidateAndMarshal is a convenience function to validate and unmarshal JSON to T. +// It assumes the schemaLoader and T are compatible. +func ValidateAndUnmarshal[T any](schemaLoader gojsonschema.JSONLoader, jsonStr string) (T, bool) { + var zero T + rowJsonLoader := gojsonschema.NewStringLoader(jsonStr) + + // Perform validation + result, err := gojsonschema.Validate(schemaLoader, rowJsonLoader) + if err != nil { + fmt.Printf("Error validating JSON: %s\n", err) + return zero, false } - *valuePtr = result - return true -} - -const PathSeparator = "." - -func getPathValue[T any](data map[string]interface{}, path string) (T, error) { - keys := strings.Split(path, PathSeparator) - var zero T // Default zero value for the type T - var value interface{} = data - var ok bool - - for _, key := range keys { - switch casted := value.(type) { - case map[string]interface{}: - if value, ok = casted[key]; !ok { - return zero, fmt.Errorf("missing value for %s in path %s", key, path) - } - case []interface{}: - if index, ok := parseIndex(key); ok && index >= 0 && index < len(casted) { - value = casted[index] - } else { - return zero, fmt.Errorf("invalid index %s in path %s", key, path) - } - default: - return zero, fmt.Errorf("type mismatch for %s in path %s, expected %T, got %T", key, path, zero, value) + // Check validation result + if !result.Valid() { + var errors []string + for _, desc := range result.Errors() { + errors = append(errors, desc.String()) } + log.Errorf("Invalid JSON:\n%s", strings.Join(errors, "\n")) + return zero, false } - if finalValue, ok := value.(T); ok { - return finalValue, nil + var row T + if err := json.Unmarshal([]byte(jsonStr), &row); err != nil { + log.Errorf("Failed to unmarshal JSON: %s", err) + return zero, false } - return zero, fmt.Errorf("type mismatch for at %s, expected %T, got %T", path, zero, value) -} -func parseIndex(key string) (int, bool) { - var index int - _, err := fmt.Sscanf(key, "[%d]", &index) - return index, err == nil + return row, true } diff --git a/cmd/ctrlc/root/sync/steampipe/adapter/registry.go b/cmd/ctrlc/root/sync/steampipe/adapter/registry.go index c1a64d8..a45c72d 100644 --- a/cmd/ctrlc/root/sync/steampipe/adapter/registry.go +++ b/cmd/ctrlc/root/sync/steampipe/adapter/registry.go @@ -3,21 +3,19 @@ package adapter import ( "github.com/charmbracelet/log" "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/steampipe/adapter/aws" - "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/steampipe/adapter/azure" - "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/steampipe/adapter/gcp" "github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/steampipe/adapter/model" "strings" ) var adapters = []model.SteampipeAdapter{ aws.EC2, - aws.EKS, - aws.RDS, - aws.VPC, - gcp.GKE, - gcp.SQL, - gcp.Compute, - azure.AKS, + //aws.EKS, + //aws.RDS, + //aws.VPC, + //gcp.GKE, + //gcp.SQL, + //gcp.Compute, + //azure.AKS, } func buildRegistry() map[string]model.SteampipeAdapter { diff --git a/cmd/ctrlc/root/sync/steampipe/client.go b/cmd/ctrlc/root/sync/steampipe/client.go index 25c2c96..eb4a09c 100644 --- a/cmd/ctrlc/root/sync/steampipe/client.go +++ b/cmd/ctrlc/root/sync/steampipe/client.go @@ -7,13 +7,13 @@ import ( _ "github.com/lib/pq" ) -type SteampipeClient struct { +type Client struct { Connection string `default:"connection"` db *sql.DB } -func NewSteampipeClient(spConnection string) (*SteampipeClient, error) { - client := &SteampipeClient{ +func NewSteampipeClient(spConnection string) (*Client, error) { + client := &Client{ Connection: spConnection, } @@ -34,7 +34,7 @@ func NewSteampipeClient(spConnection string) (*SteampipeClient, error) { } // Close closes the database connection -func (c *SteampipeClient) Close() error { +func (c *Client) Close() error { if c.db != nil { return c.db.Close() } @@ -42,6 +42,6 @@ func (c *SteampipeClient) Close() error { } // GetDB returns the underlying database connection -func (c *SteampipeClient) GetDB() *sql.DB { +func (c *Client) GetDB() *sql.DB { return c.db } diff --git a/cmd/ctrlc/root/sync/steampipe/fetch.go b/cmd/ctrlc/root/sync/steampipe/fetch.go index 8d4af25..2b7e96e 100644 --- a/cmd/ctrlc/root/sync/steampipe/fetch.go +++ b/cmd/ctrlc/root/sync/steampipe/fetch.go @@ -10,8 +10,8 @@ import ( "github.com/ctrlplanedev/cli/internal/api" ) -func (c *SteampipeClient) DoSync(table string) ([]api.AgentResource, error) { - var resource api.AgentResource +func (c *Client) DoSync(table string) ([]api.AgentResource, error) { + var jsResource string var ok bool ad := adapter.SelectAdapter(table) @@ -21,32 +21,39 @@ func (c *SteampipeClient) DoSync(table string) ([]api.AgentResource, error) { resources := make([]api.AgentResource, 0) - jsonObjRows, err := c.SelectAll(table) + jsonRows, err := c.SelectAll(table) if err != nil { return nil, fmt.Errorf("failed to fetch from %s table: %w", table, err) } - log.Infof("steampipe '%s' query returned %d rows", table, len(jsonObjRows)) + log.Infof("steampipe '%s' query returned %d rows", table, len(jsonRows)) - if len(jsonObjRows) == 0 { + if len(jsonRows) == 0 { return nil, nil } var sqlRow model.SqlRow = model.SqlRow{ EntityName: table, - Data: nil, + Json: "", } - for _, jsonObj := range jsonObjRows { + for _, jsRow := range jsonRows { - sqlRow.Data = jsonObj + sqlRow.Json = jsRow - if resource, ok = ad.ToApiResource(sqlRow); ok { + log.Debugf("SqlRow '%s' JSON \n%s", table, jsRow) + + if jsResource, ok = ad.ToResourceJson(sqlRow); ok { if log.GetLevel() >= log.DebugLevel { - payloadStr, _ := json.Marshal(resource) - log.Debugf("%s", payloadStr) + payloadStr, _ := json.Marshal(jsResource) + log.Debugf("Resource JSON \n%s", payloadStr) + } + var apiResource api.AgentResource + if err = json.Unmarshal([]byte(jsResource), &apiResource); err != nil { + log.Errorf("Failed to unmarshal json resource '%s': %s", jsResource, err) + } else { + resources = append(resources, apiResource) } - resources = append(resources, resource) } } @@ -54,11 +61,11 @@ func (c *SteampipeClient) DoSync(table string) ([]api.AgentResource, error) { } // SelectAll returns all foreign tables where foreign_table_catalog is 'steampipe' -func (c *SteampipeClient) SelectAll(tableName string) ([]map[string]interface{}, error) { +func (c *Client) SelectAll(tableName string) ([]string, error) { var columns []string var err error - results := make([]map[string]interface{}, 0) + results := make([]string, 0) query := fmt.Sprintf("SELECT * FROM %s", tableName) @@ -79,11 +86,15 @@ func (c *SteampipeClient) SelectAll(tableName string) ([]map[string]interface{}, } var row map[string]interface{} + var rowJson []byte for rows.Next() { if row, err = toJsonObj(rows, columns); err != nil { return nil, fmt.Errorf("failed to convert row to JSON object: %w", err) } - results = append(results, row) + if rowJson, err = json.Marshal(row); err != nil { + return nil, fmt.Errorf("failed row to JSON: %w", err) + } + results = append(results, string(rowJson)) } if err := rows.Err(); err != nil { diff --git a/cmd/ctrlc/root/sync/steampipe/steampipe.go b/cmd/ctrlc/root/sync/steampipe/steampipe.go index a118294..3a6ccf4 100644 --- a/cmd/ctrlc/root/sync/steampipe/steampipe.go +++ b/cmd/ctrlc/root/sync/steampipe/steampipe.go @@ -40,7 +40,7 @@ func NewSyncSteampipeCmd() *cobra.Command { if err != nil { return fmt.Errorf("failed to create steampipe connection: %w", err) } - defer func(steampipe *SteampipeClient) { + defer func(steampipe *Client) { err := steampipe.Close() if err != nil { log.Errorf("failed to close steampipe connection: %v", err) diff --git a/go.mod b/go.mod index 0270199..d611bd9 100644 --- a/go.mod +++ b/go.mod @@ -12,12 +12,14 @@ require ( github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.3 github.com/hashicorp/go-tfe v1.73.1 + github.com/lib/pq v1.10.9 github.com/mitchellh/go-homedir v1.1.0 github.com/moby/term v0.5.0 github.com/oapi-codegen/runtime v1.1.1 github.com/spf13/cobra v1.8.1 github.com/spf13/viper v1.19.0 github.com/tailscale/tailscale-client-go/v2 v2.0.0-20241217012816-8143c7dc1766 + github.com/xeipuuv/gojsonschema v1.2.0 gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.32.1 k8s.io/apimachinery v0.32.1 @@ -30,11 +32,9 @@ require ( github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/charmbracelet/lipgloss v0.10.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/dprotaso/go-yit v0.0.0-20220510233725-9ba8df137936 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect - github.com/getkin/kin-openapi v0.127.0 // indirect github.com/go-logfmt/logfmt v0.6.0 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect @@ -53,10 +53,8 @@ require ( github.com/hashicorp/hcl v1.0.0 // indirect github.com/hashicorp/jsonapi v1.3.2 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/invopop/yaml v0.3.1 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/lib/pq v1.10.9 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect @@ -65,31 +63,26 @@ require ( github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect github.com/muesli/reflow v0.3.0 // indirect github.com/muesli/termenv v0.15.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/oapi-codegen/oapi-codegen/v2 v2.4.1 // indirect - github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect - github.com/perimeterx/marshmallow v1.1.5 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect - github.com/speakeasy-api/openapi-overlay v0.9.0 // indirect github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/tailscale/hujson v0.0.0-20220506213045-af5ed07155e5 // indirect - github.com/vmware-labs/yaml-jsonpath v0.3.2 // indirect github.com/x448/float16 v0.8.4 // indirect + github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect + github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect - golang.org/x/mod v0.21.0 // indirect golang.org/x/net v0.30.0 // indirect golang.org/x/oauth2 v0.23.0 // indirect golang.org/x/sync v0.8.0 // indirect @@ -97,7 +90,6 @@ require ( golang.org/x/term v0.25.0 // indirect golang.org/x/text v0.19.0 // indirect golang.org/x/time v0.7.0 // indirect - golang.org/x/tools v0.26.0 // indirect google.golang.org/protobuf v1.35.1 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/go.sum b/go.sum index 5b448ee..6b55970 100644 --- a/go.sum +++ b/go.sum @@ -14,9 +14,6 @@ github.com/charmbracelet/lipgloss v0.10.0 h1:KWeXFSexGcfahHX+54URiZGkBFazf70JNMt github.com/charmbracelet/lipgloss v0.10.0/go.mod h1:Wig9DSfvANsxqkRsqj6x87irdy123SR4dOXlKa91ciE= github.com/charmbracelet/log v0.4.0 h1:G9bQAcx8rWA2T3pWvx7YtPTPwgqpk7D68BX21IRW8ZM= github.com/charmbracelet/log v0.4.0/go.mod h1:63bXt/djrizTec0l11H20t8FDSvA4CRZJ1KH22MdptM= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= @@ -25,23 +22,16 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dprotaso/go-yit v0.0.0-20191028211022-135eb7262960/go.mod h1:9HQzr9D/0PGwMEbC3d5AB7oi67+h4TsQqItC1GVYG58= -github.com/dprotaso/go-yit v0.0.0-20220510233725-9ba8df137936 h1:PRxIJD8XjimM5aTknUK9w6DHLDox2r2M3DI4i2pnd3w= -github.com/dprotaso/go-yit v0.0.0-20220510233725-9ba8df137936/go.mod h1:ttYvX5qlB+mlV1okblJqcSMtR4c52UKxDiX9GRBS8+Q= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= -github.com/getkin/kin-openapi v0.127.0 h1:Mghqi3Dhryf3F8vR370nN67pAERW+3a95vomb3MAREY= -github.com/getkin/kin-openapi v0.127.0/go.mod h1:OZrfXzUfGrNbsKj+xmFBx6E5c6yH3At/tAKSc2UszXM= github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= @@ -54,29 +44,15 @@ github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -85,7 +61,6 @@ github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -110,12 +85,8 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/jsonapi v1.3.2 h1:gP3fX2ZT7qXi+PbwieptzkspIohO2kCSiBUvUTBAbMs= github.com/hashicorp/jsonapi v1.3.2/go.mod h1:kWfdn49yCjQvbpnvY1dxxAuAFzISwrrMDQOcu6NsFoM= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/invopop/yaml v0.3.1 h1:f0+ZpmhfBSS4MhG+4HYseMdJhoeeopbSKbq5Rpeelso= -github.com/invopop/yaml v0.3.1/go.mod h1:PMOp3nn4/12yEZUFfmOuNHJsZToEEOwoWsT+D81KkeA= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -123,7 +94,6 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -143,7 +113,6 @@ github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxec github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= @@ -158,40 +127,20 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= -github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo= github.com/muesli/termenv v0.15.2/go.mod h1:Epx+iuz8sNs7mNKhxzH4fWXGNpZwUaJKRS1noLXviQ8= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= -github.com/oapi-codegen/oapi-codegen/v2 v2.4.1 h1:ykgG34472DWey7TSjd8vIfNykXgjOgYJZoQbKfEeY/Q= -github.com/oapi-codegen/oapi-codegen/v2 v2.4.1/go.mod h1:N5+lY1tiTDV3V1BeHtOxeWXHoPVeApvsvjJqegfoaz8= github.com/oapi-codegen/runtime v1.1.1 h1:EXLHh0DXIJnWhdRPN2w4MXAzFyE4CskzhNLUmtpMYro= github.com/oapi-codegen/runtime v1.1.1/go.mod h1:SK9X900oXmPWilYR5/WKPzt3Kqxn/uS/+lbpREv+eCg= -github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= -github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= -github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= -github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= -github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= -github.com/perimeterx/marshmallow v1.1.5 h1:a2LALqQ1BlHM8PZblsDdidgv1mWi1DgC2UmX50IvK2s= -github.com/perimeterx/marshmallow v1.1.5/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -208,11 +157,8 @@ github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6ke github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= -github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= -github.com/speakeasy-api/openapi-overlay v0.9.0 h1:Wrz6NO02cNlLzx1fB093lBlYxSI54VRhy1aSutx0PQg= -github.com/speakeasy-api/openapi-overlay v0.9.0/go.mod h1:f5FloQrHA7MsxYg9djzMD5h6dxrHjVVByWKh7an8TRc= github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= @@ -229,8 +175,6 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= @@ -243,10 +187,14 @@ github.com/tailscale/hujson v0.0.0-20220506213045-af5ed07155e5 h1:erxeiTyq+nw4Cz github.com/tailscale/hujson v0.0.0-20220506213045-af5ed07155e5/go.mod h1:DFSS3NAGHthKo1gTlmEcSBiZrRJXi28rLNd/1udP1c8= github.com/tailscale/tailscale-client-go/v2 v2.0.0-20241217012816-8143c7dc1766 h1:1uj3+ZbxA1U1GH6VGTTRcwK8ywKjZJZBy/v22lqJm04= github.com/tailscale/tailscale-client-go/v2 v2.0.0-20241217012816-8143c7dc1766/go.mod h1:i/MSgQ71kdyh1Wdp50XxrIgtsyO4uZ2SZSPd83lGKHM= -github.com/vmware-labs/yaml-jsonpath v0.3.2 h1:/5QKeCBGdsInyDCyVNLbXyilb61MXGi9NP674f9Hobk= -github.com/vmware-labs/yaml-jsonpath v0.3.2/go.mod h1:U6whw1z03QyqgWdgXxvVnQ90zN1BWz5V+51Ewf8k+rQ= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= @@ -260,52 +208,30 @@ golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= -golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24= golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= @@ -313,7 +239,6 @@ golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= @@ -321,35 +246,19 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20191026110619-0b21df46bc1d/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 532d0866bd225504b3be455290dc4572bc2d25db Mon Sep 17 00:00:00 2001 From: Jonathan Meeks Date: Fri, 11 Apr 2025 17:31:54 -0500 Subject: [PATCH 13/13] tweaks to ec2 resource --- .../root/sync/steampipe/adapter/aws/ec2.go | 26 ++++++++++--------- .../sync/steampipe/adapter/model/adapter.go | 8 +++--- cmd/ctrlc/root/sync/steampipe/fetch.go | 4 +-- cmd/ctrlc/root/sync/steampipe/steampipe.go | 9 ++++++- internal/api/resource_provider.go | 13 +++++----- 5 files changed, 35 insertions(+), 25 deletions(-) diff --git a/cmd/ctrlc/root/sync/steampipe/adapter/aws/ec2.go b/cmd/ctrlc/root/sync/steampipe/adapter/aws/ec2.go index b69fcd4..20acf70 100644 --- a/cmd/ctrlc/root/sync/steampipe/adapter/aws/ec2.go +++ b/cmd/ctrlc/root/sync/steampipe/adapter/aws/ec2.go @@ -26,12 +26,13 @@ type Ec2Row struct { } type Ec2Resource struct { - Config Ec2ResConfig `json:"config"` - Identifier string `json:"identifier"` - Kind string `json:"kind"` - Metadata map[string]string `json:"metadata"` - Name string `json:"name"` - Version string `json:"version"` + WorkspaceId string `json:"workspaceId"` + Config Ec2ResConfig `json:"config"` + Identifier string `json:"identifier"` + Kind string `json:"kind"` + Metadata map[string]string `json:"metadata"` + Name string `json:"name"` + Version string `json:"version"` } type Ec2ResConfig struct { @@ -60,7 +61,7 @@ type Ec2ResRequiredMetadata struct { var EC2 model.SteampipeAdapter = &model.SteampipeAdapterStruct{ Table: ec2Table, - Convert: func(rowJsonStr string) (string, bool) { + Convert: func(workspaceId string, rowJsonStr string) (string, bool) { // Validate row json schema and build into type row, ok := model.ValidateAndUnmarshal[Ec2Row](rowSchemaLoader, rowJsonStr) if !ok { @@ -76,10 +77,11 @@ var EC2 model.SteampipeAdapter = &model.SteampipeAdapterStruct{ // Build resource specific to the resource type result := Ec2Resource{ - Identifier: row.Arn, - Name: row.InstanceID, - Version: "compute/v1", - Kind: "Compute", + WorkspaceId: workspaceId, + Identifier: row.Arn, + Name: row.InstanceID, + Version: "vm/v1", + Kind: "VM", Config: Ec2ResConfig{ Auth: Ec2ResConfigAuth{ Method: "aws/ec2", @@ -99,7 +101,7 @@ var EC2 model.SteampipeAdapter = &model.SteampipeAdapterStruct{ var resultJson []byte var err error if resultJson, err = json.Marshal(result); err != nil { - log.Errorf("failed to marshal EC2 resource: %v") + log.Errorf("failed to marshal EC2 resource: %v", err) return "", false } diff --git a/cmd/ctrlc/root/sync/steampipe/adapter/model/adapter.go b/cmd/ctrlc/root/sync/steampipe/adapter/model/adapter.go index f0b330b..f8df9bc 100644 --- a/cmd/ctrlc/root/sync/steampipe/adapter/model/adapter.go +++ b/cmd/ctrlc/root/sync/steampipe/adapter/model/adapter.go @@ -2,18 +2,18 @@ package model type SteampipeAdapterStruct struct { Table string - Convert func(jsonRow string) (string, bool) + Convert func(workspaceId string, jsonRow string) (string, bool) } func (a *SteampipeAdapterStruct) EntityName() string { return a.Table } -func (a *SteampipeAdapterStruct) ToResourceJson(sqlRow SqlRow) (string, bool) { - return a.Convert(sqlRow.Json) +func (a *SteampipeAdapterStruct) ToResourceJson(workspaceId string, sqlRow SqlRow) (string, bool) { + return a.Convert(workspaceId, sqlRow.Json) } type SteampipeAdapter interface { EntityName() string - ToResourceJson(sqlRow SqlRow) (string, bool) + ToResourceJson(workspaceId string, sqlRow SqlRow) (string, bool) } diff --git a/cmd/ctrlc/root/sync/steampipe/fetch.go b/cmd/ctrlc/root/sync/steampipe/fetch.go index 2b7e96e..39c3cae 100644 --- a/cmd/ctrlc/root/sync/steampipe/fetch.go +++ b/cmd/ctrlc/root/sync/steampipe/fetch.go @@ -10,7 +10,7 @@ import ( "github.com/ctrlplanedev/cli/internal/api" ) -func (c *Client) DoSync(table string) ([]api.AgentResource, error) { +func (c *Client) DoSync(workspaceId string, table string) ([]api.AgentResource, error) { var jsResource string var ok bool @@ -43,7 +43,7 @@ func (c *Client) DoSync(table string) ([]api.AgentResource, error) { log.Debugf("SqlRow '%s' JSON \n%s", table, jsRow) - if jsResource, ok = ad.ToResourceJson(sqlRow); ok { + if jsResource, ok = ad.ToResourceJson(workspaceId, sqlRow); ok { if log.GetLevel() >= log.DebugLevel { payloadStr, _ := json.Marshal(jsResource) log.Debugf("Resource JSON \n%s", payloadStr) diff --git a/cmd/ctrlc/root/sync/steampipe/steampipe.go b/cmd/ctrlc/root/sync/steampipe/steampipe.go index 3a6ccf4..a819791 100644 --- a/cmd/ctrlc/root/sync/steampipe/steampipe.go +++ b/cmd/ctrlc/root/sync/steampipe/steampipe.go @@ -3,6 +3,7 @@ package steampipe import ( "context" "fmt" + "io" "net/http" "os" @@ -52,7 +53,7 @@ func NewSyncSteampipeCmd() *cobra.Command { return fmt.Errorf("failed to create API client: %w", err) } - if resources, err = steampipe.DoSync(table); err != nil { + if resources, err = steampipe.DoSync(workspaceId, table); err != nil { return err } @@ -65,6 +66,12 @@ func NewSyncSteampipeCmd() *cobra.Command { providerResp, err = provider.UpsertResource(ctx, resources) if providerResp != nil { log.Info("upsert resources response ", "status", providerResp.Status) + bodyContent, err := io.ReadAll(providerResp.Body) + if err != nil { + log.Errorf("failed to read response body: %v", err) + } + log.Info("upsert resources response ", "body", string(bodyContent)) + providerResp.Body.Close() } if err != nil { return fmt.Errorf("failed to upsert resources: %w", err) diff --git a/internal/api/resource_provider.go b/internal/api/resource_provider.go index 40630a3..430eaac 100644 --- a/internal/api/resource_provider.go +++ b/internal/api/resource_provider.go @@ -63,12 +63,13 @@ type ResourceProvider struct { } type AgentResource struct { - Config map[string]interface{} `json:"config"` - Identifier string `json:"identifier"` - Kind string `json:"kind"` - Metadata map[string]string `json:"metadata"` - Name string `json:"name"` - Version string `json:"version"` + WorkspaceId string `json:"workspaceId"` + Config map[string]interface{} `json:"config"` + Identifier string `json:"identifier"` + Kind string `json:"kind"` + Metadata map[string]string `json:"metadata"` + Name string `json:"name"` + Version string `json:"version"` } func (r *ResourceProvider) UpsertResource(ctx context.Context, resources []AgentResource) (*http.Response, error) {