Steampipe-sync

.gitignore

@@ -0,0 +1,4 @@
+
+.idea/
+bin/
+*.session.sql

CtrlcSteampipe-2025-04-10-191519.mmd

@@ -0,0 +1,52 @@
+flowchart TD
+ subgraph s1["Steampipe 
+  Processor"]
+        F1{"Next Row"}
+        OG1(["SQL Row
+    map[string]interface{}"])
+        STR1(["SQL Row
+    JSON String"])
+        V1{"Validate"}
+        SCH1(["SQL Row
+    JSON Schema"])
+        GO1(["SQL Row
+    AwsEc2Row{}"])
+  end
+ subgraph ad["Adapter"]
+        M2{"Translate"}
+        GO2(["Resource
+    AwsEc2Resource{}"])
+        STR2(["Resource
+    JSON String"])
+        V2{"Validate"}
+        SCH2(["Resource
+    JSON Schema"])
+  end
+ subgraph sync["sync"]
+        M3{"Collect"}
+        GO3(["AgentResource
+  AgentResource{}"])
+  end
+    SPDB[("SteampipeDB")] -- each row --> F1
+    F1 -- build --> OG1
+    OG1 -- marshal --> STR1
+    STR1 -.-> V1
+    V1 -.-> STR1 & SCH1
+    STR1 -- unmarshal --> GO1
+    GO1 --> M2
+    M2 --> GO2
+    GO2 -- marshal --> STR2
+    STR2 -.-> V2
+    V2 -.-> STR2 & SCH2
+    note["extension of <br/>AgentResource schema"] -.-> GO2 & SCH2
+    GO3 -.-> note
+    M3 -- unmarshal --> GO3
+    STR2 --> M3
+    ctrlplane["CtrlPlane API"]
+
+    note@{ shape: notch-rect}
+    ctrlplane@{ shape: h-cyl}
+
+    sync ==> ctrlplane
+
+

cmd/ctrlc/root/sync/steampipe/adapter/aws/ec2.go

@@ -0,0 +1,110 @@
+package aws
+
+import (
+	_ "embed"
+	"encoding/json"
+	"github.com/charmbracelet/log"
+	"github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/steampipe/adapter/model"
+	"github.com/xeipuuv/gojsonschema"
+)
+
+const ec2Table = "aws_ec2_instance"
+
+//go:embed schema/ec2_row.schema.json
+var ec2RowSchema string
+
+var rowSchemaLoader = gojsonschema.NewStringLoader(ec2RowSchema)
+
+type Ec2Row struct {
+	Arn            string            `json:"arn"`
+	InstanceID     string            `json:"instance_id"`
+	AccountID      string            `json:"account_id"`
+	Region         string            `json:"region"`
+	InstanceState  string            `json:"instance_state"`
+	PrivateDNSName string            `json:"private_dns_name" default:""`
+	Tags           map[string]string `json:"tags"`
+}
+
+type Ec2Resource struct {
+	WorkspaceId string            `json:"workspaceId"`
+	Config      Ec2ResConfig      `json:"config"`
+	Identifier  string            `json:"identifier"`
+	Kind        string            `json:"kind"`
+	Metadata    map[string]string `json:"metadata"`
+	Name        string            `json:"name"`
+	Version     string            `json:"version"`
+}
+
+type Ec2ResConfig struct {
+	Auth   Ec2ResConfigAuth   `json:"auth"`
+	Name   string             `json:"name"`
+	Server Ec2ResConfigServer `json:"server"`
+	Status string             `json:"status"`
+}
+
+type Ec2ResConfigAuth struct {
+	Method     string `json:"method"`
+	Region     string `json:"region"`
+	AccountId  string `json:"accountId"`
+	InstanceId string `json:"instanceId"`
+}
+
+type Ec2ResConfigServer struct {
+	Endpoint string `json:"endpoint"`
+}
+
+type Ec2ResRequiredMetadata struct {
+	AccountID string `json:"aws/account-id"`
+	Arn       string `json:"aws/arn"`
+	Region    string `json:"aws/region"`
+}
+
+var EC2 model.SteampipeAdapter = &model.SteampipeAdapterStruct{
+	Table: ec2Table,
+	Convert: func(workspaceId string, rowJsonStr string) (string, bool) {
+		// Validate row json schema and build into type
+		row, ok := model.ValidateAndUnmarshal[Ec2Row](rowSchemaLoader, rowJsonStr)
+		if !ok {
+			return "", false
+		}
+
+		// Add required metadata and merge with tags
+		metadata := make(model.Metadata)
+		metadata["aws/account-id"] = row.AccountID
+		metadata["aws/arn"] = row.Arn
+		metadata["aws/region"] = row.Region
+		metadata = metadata.AppendTags(row.Tags)
+
+		// Build resource specific to the resource type
+		result := Ec2Resource{
+			WorkspaceId: workspaceId,
+			Identifier:  row.Arn,
+			Name:        row.InstanceID,
+			Version:     "vm/v1",
+			Kind:        "VM",
+			Config: Ec2ResConfig{
+				Auth: Ec2ResConfigAuth{
+					Method:     "aws/ec2",
+					Region:     row.Region,
+					AccountId:  row.AccountID,
+					InstanceId: row.InstanceID,
+				},
+				Name: row.InstanceID,
+				Server: Ec2ResConfigServer{
+					Endpoint: row.PrivateDNSName,
+				},
+				Status: row.InstanceState,
+			},
+			Metadata: metadata,
+		}
+
+		var resultJson []byte
+		var err error
+		if resultJson, err = json.Marshal(result); err != nil {
+			log.Errorf("failed to marshal EC2 resource: %v", err)
+			return "", false
+		}
+
+		return string(resultJson), true
+	},
+}

cmd/ctrlc/root/sync/steampipe/adapter/aws/eks.go

@@ -0,0 +1,77 @@
+package aws
+
+//import (
+//	"github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/steampipe/adapter/model"
+//	"github.com/ctrlplanedev/cli/internal/api"
+//)
+//
+//const eksTable = "aws_eks_cluster"
+//
+//var EKS model.SteampipeAdapter = &model.SteampipeAdapterStruct{
+//	Table: eksTable,
+//	Translate: func(data map[string]interface{}) (api.AgentResource, bool) {
+//		var entityName = eksTable
+//		var sqlRow model.SqlRow = model.SqlRow{
+//			EntityName: entityName,
+//			Data:       data,
+//		}
+//
+//		var name string
+//		var arn string
+//		var tags model.Tags
+//		var accountId string
+//		var region string
+//		var certificateAuthorityData string
+//		var status string
+//		var version string
+//		var endpoint string
+//		var roleArn string
+//		var platformVersion string
+//
+//		var zero api.AgentResource = api.AgentResource{}
+//
+//		if !model.GetRequiredValue[string](sqlRow, "arn", &arn) ||
+//			!model.GetRequiredValue[string](sqlRow, "name", &name) ||
+//			!model.GetOptionalValue[model.Tags](sqlRow, "tags", &tags) ||
+//			!model.GetRequiredValue[string](sqlRow, "account_id", &accountId) ||
+//			!model.GetRequiredValue[string](sqlRow, "region", &region) ||
+//			!model.GetRequiredValue[string](sqlRow, "certificate_authority.Data", &certificateAuthorityData) ||
+//			!model.GetRequiredValue[string](sqlRow, "status", &status) ||
+//			!model.GetRequiredValue[string](sqlRow, "version", &version) ||
+//			!model.GetRequiredValue[string](sqlRow, "endpoint", &endpoint) ||
+//			!model.GetRequiredValue[string](sqlRow, "role_arn", &roleArn) ||
+//			!model.GetRequiredValue[string](sqlRow, "platform_version", &platformVersion) {
+//			return zero, false
+//		}
+//
+//		metadata := model.BuildMetadata(map[string]string{
+//			"aws/account-id":       accountId,
+//			"aws/arn":              arn,
+//			"aws/eks-role-arn":     roleArn,
+//			"aws/region":           region,
+//			"aws/platform-version": platformVersion,
+//		}).AppendTags(tags)
+//
+//		return api.AgentResource{
+//			Identifier: arn,
+//			Name:       name,
+//			Version:    "kubernetes/v1",
+//			Kind:       "ClusterAPI",
+//			Config: map[string]interface{}{
+//				"auth": map[string]string{
+//					"method":      "aws/eks",
+//					"region":      region,
+//					"accountId":   accountId,
+//					"clusterName": name,
+//				},
+//				"name": name,
+//				"server": map[string]string{
+//					"endpoint":                   endpoint,
+//					"certificationAuthorityData": certificateAuthorityData,
+//				},
+//				"status": status,
+//			},
+//			Metadata: metadata,
+//		}, true
+//	},
+//}

cmd/ctrlc/root/sync/steampipe/adapter/aws/rds.go

@@ -0,0 +1,71 @@
+package aws
+
+//import (
+//	"github.com/ctrlplanedev/cli/cmd/ctrlc/root/sync/steampipe/adapter/model"
+//	"github.com/ctrlplanedev/cli/internal/api"
+//)
+//
+//const rdsTable = "aws_rds_db_cluster"
+//
+//var RDS model.SteampipeAdapter = &model.SteampipeAdapterStruct{
+//	Table: rdsTable,
+//	Translate: func(data map[string]interface{}) (api.AgentResource, bool) {
+//		var entityName = rdsTable
+//		var sqlRow model.SqlRow = model.SqlRow{
+//			EntityName: entityName,
+//			Data:       data,
+//		}
+//
+//		var name string
+//		var dbName string
+//		var arn string
+//		var tags model.Tags
+//		var accountId string
+//		var region string
+//		var status string
+//		var version string
+//		var endpoint string
+//
+//		var zero api.AgentResource = api.AgentResource{}
+//
+//		if !model.GetRequiredValue[string](sqlRow, "arn", &arn) ||
+//			!model.GetRequiredValue[string](sqlRow, "db_cluster_identifier", &name) ||
+//			!model.GetRequiredValue[string](sqlRow, "database_name", &dbName) ||
+//			!model.GetOptionalValue[model.Tags](sqlRow, "tags", &tags) ||
+//			!model.GetRequiredValue[string](sqlRow, "account_id", &accountId) ||
+//			!model.GetRequiredValue[string](sqlRow, "region", &region) ||
+//			!model.GetRequiredValue[string](sqlRow, "status", &status) ||
+//			!model.GetRequiredValue[string](sqlRow, "engine_version", &version) ||
+//			!model.GetRequiredValue[string](sqlRow, "endpoint", &endpoint) {
+//			return zero, false
+//		}
+//
+//		metadata := model.BuildMetadata(map[string]string{
+//			"aws/account-id":     accountId,
+//			"aws/arn":            arn,
+//			"aws/region":         region,
+//			"aws/engine-version": version,
+//		}).AppendTags(tags)
+//
+//		return api.AgentResource{
+//			Identifier: arn,
+//			Name:       name,
+//			Version:    "database/v1",
+//			Kind:       "Database",
+//			Config: map[string]interface{}{
+//				"auth": map[string]string{
+//					"method":      "aws/rds",
+//					"region":      region,
+//					"accountId":   accountId,
+//					"clusterName": name,
+//				},
+//				"name": name,
+//				"server": map[string]string{
+//					"endpoint": endpoint,
+//				},
+//				"status": status,
+//			},
+//			Metadata: metadata,
+//		}, true
+//	},
+//}

cmd/ctrlc/root/sync/steampipe/adapter/aws/schema/ec2_row.schema.json

@@ -0,0 +1,48 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "title": "AwsEc2SqlRow",
+  "type": "object",
+  "additionalProperties": true,
+  "required": [
+    "arn",
+    "instance_id",
+    "account_id",
+    "region",
+    "instance_state",
+    "private_dns_name",
+    "tags"
+  ],
+  "properties": {
+    "arn": {
+      "type": "string",
+      "description": "EC2 ARN"
+    },
+    "instance_id": {
+      "type": "string",
+      "description": "Using the instance_id as the 'name' for an EC2 instance"
+    },
+    "account_id": {
+      "type": "string",
+      "description": "AWS Account ID"
+    },
+    "region": {
+      "type": "string",
+      "description": "AWS Region"
+    },
+    "instance_state": {
+      "type": "string",
+      "description": "EC2 instance status"
+    },
+    "private_dns_name": {
+      "type": "string",
+      "description": "Private DNS name of the EC2 instance"
+    },
+    "tags": {
+      "type": "object",
+      "additionalProperties": {
+        "type": "string"
+      },
+      "description": "All tags on EC2 instance itself"
+    }
+  }
+}