Periodically capture resource metrics for every sandbox

packages/envd/internal/host/metrics.go

@@ -9,9 +9,9 @@ import (
 )
 
 type Metrics struct {
-	CPU       float64 `json:"cpu_pct"`   // Percent rounded to 2 decimal places
-	Mem       uint64  `json:"mem_bytes"` // Total virtual memory in bytes
-	Timestamp int64   `json:"ts"`        // Unix Timestamp in UTC
+	CPUPercent float64 `json:"cpu_pct"` // Percent rounded to 2 decimal places
+	MemMiB     uint64  `json:"mem_mib"` // Total virtual memory in MiB
+	Timestamp  int64   `json:"ts"`      // Unix Timestamp in UTC
 }
 
 func GetMetrics() (*Metrics, error) {
@@ -20,6 +20,8 @@ func GetMetrics() (*Metrics, error) {
 		return nil, err
 	}
 
+	memMB := v.Total / 1024 / 1024
+
 	cpuPcts, err := cpu.Percent(0, false)
 	if err != nil {
 		return nil, err
@@ -32,8 +34,8 @@ func GetMetrics() (*Metrics, error) {
 	}
 
 	return &Metrics{
-		CPU:       cpuPctRounded,
-		Mem:       v.Total,
-		Timestamp: time.Now().UTC().Unix(),
+		CPUPercent: cpuPctRounded,
+		MemMiB:     memMB,
+		Timestamp:  time.Now().UTC().Unix(),
 	}, nil
 }

packages/envd/main.go

@@ -32,7 +32,7 @@ const (
 
 var (
 	// These vars are automatically set by goreleaser.
-	Version = "0.1.2"
+	Version = "0.1.3"
 
 	debug bool
 	port  int64

packages/orchestrator/internal/sandbox/checks.go

@@ -2,19 +2,37 @@ package sandbox
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
 	"time"
 
 	"github.com/e2b-dev/infra/packages/shared/pkg/consts"
 	"github.com/e2b-dev/infra/packages/shared/pkg/utils"
+	"golang.org/x/mod/semver"
+)
+
+const (
+	healthCheckInterval      = 10 * time.Second
+	metricsCheckInterval     = 5 * time.Second
+	minEnvdVersionForMetrcis = "0.1.3"
 )
 
 func (s *Sandbox) logHeathAndUsage(ctx *utils.LockableCancelableContext) {
+	healthTicker := time.NewTicker(healthCheckInterval)
+	metricsTicker := time.NewTicker(metricsCheckInterval)
+	defer func() {
+		healthTicker.Stop()
+		metricsTicker.Stop()
+	}()
+
+	// Get metrics on sandbox startup
+	go s.LogMetrics(ctx)
+
 	for {
 		select {
-		case <-time.After(10 * time.Second):
+		case <-healthTicker.C:
 			childCtx, cancel := context.WithTimeout(ctx, time.Second)
 
 			ctx.Lock()
@@ -30,6 +48,8 @@ func (s *Sandbox) logHeathAndUsage(ctx *utils.LockableCancelableContext) {
 				s.Logger.CPUUsage(stats.CPUCount)
 				s.Logger.MemoryUsage(stats.MemoryMB)
 			}
+		case <-metricsTicker.C:
+			s.LogMetrics(ctx)
 		case <-ctx.Done():
 			return
 		}
@@ -65,3 +85,55 @@ func (s *Sandbox) Healthcheck(ctx context.Context, alwaysReport bool) {
 		return
 	}
 }
+
+func (s *Sandbox) GetMetrics(ctx context.Context) (SandboxMetrics, error) {
+	address := fmt.Sprintf("http://%s:%d/metrics", s.slot.HostIP(), consts.DefaultEnvdServerPort)
+
+	request, err := http.NewRequestWithContext(ctx, "GET", address, nil)
+	if err != nil {
+		return SandboxMetrics{}, err
+	}
+
+	response, err := httpClient.Do(request)
+	if err != nil {
+		return SandboxMetrics{}, err
+	}
+	defer response.Body.Close()
+
+	if response.StatusCode != http.StatusOK {
+		err = fmt.Errorf("unexpected status code: %d", response.StatusCode)
+		return SandboxMetrics{}, err
+	}
+
+	var metrics SandboxMetrics
+	err = json.NewDecoder(response.Body).Decode(&metrics)
+	if err != nil {
+		return SandboxMetrics{}, err
+	}
+
+	return metrics, nil
+}
+
+func (s *Sandbox) LogMetrics(ctx context.Context) {
+	if isGTEVersion(s.Sandbox.EnvdVersion, minEnvdVersionForMetrcis) {
+		metrics, err := s.GetMetrics(ctx)
+		if err != nil {
+			s.Logger.Warnf("failed to get metrics: %s", err)
+		} else {
+			s.Logger.CPUPct(metrics.CPUPercent)
+			s.Logger.MemMB(metrics.MemMiB)
+		}
+	}
+}
+
+func isGTEVersion(curVersion, minVersion string) bool {
+	if len(curVersion) > 0 && curVersion[0] != 'v' {
+		curVersion = "v" + curVersion
+	}
+
+	if !semver.IsValid(curVersion) {
+		return false
+	}
+
+	return semver.Compare(curVersion, minVersion) >= 0
+}

packages/orchestrator/internal/sandbox/metrics.go

@@ -0,0 +1,7 @@
+package sandbox
+
+type SandboxMetrics struct {
+	CPUPercent float64 `json:"cpu_pct"` // Percent rounded to 2 decimal places
+	MemMiB     uint64  `json:"mem_mib"` // Total virtual memory in MiB
+	Timestamp  int64   `json:"ts"`      // Unix Timestamp in UTC
+}

packages/orchestrator/internal/sandbox/sandbox.go

@@ -12,7 +12,6 @@ import (
 	consul "github.com/hashicorp/consul/api"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/trace"
-	"golang.org/x/mod/semver"
 
 	"github.com/e2b-dev/infra/packages/orchestrator/internal/dns"
 	"github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/uffd"
@@ -29,10 +28,11 @@ const (
 	kernelMountDir = "/fc-vm"
 	kernelName     = "vmlinux.bin"
 	fcBinaryName   = "firecracker"
+	newEnvdVersion = "v0.1.1"
 )
 
 var httpClient = http.Client{
-	Timeout: 5 * time.Second,
+	Timeout: time.Second,
 }
 
 type Sandbox struct {
@@ -245,7 +245,7 @@ func NewSandbox(
 	telemetry.ReportEvent(childCtx, "ensuring clock sync")
 
 	// Sync envds.
-	if semver.Compare(fmt.Sprintf("v%s", config.EnvdVersion), "v0.1.1") >= 0 {
+	if isGTEVersion(config.EnvdVersion, newEnvdVersion) {
 		err = instance.initEnvd(ctx, tracer, config.EnvVars)
 		if err != nil {
 			return nil, fmt.Errorf("failed to init new envd: %w", err)

packages/orchestrator/internal/server/sandboxes.go

@@ -145,7 +145,9 @@ func (s *server) Delete(ctx context.Context, in *orchestrator.SandboxRequest) (*
 		return nil, status.New(codes.NotFound, errMsg.Error()).Err()
 	}
 
+	// Check health metrics before stopping the sandbox
 	sbx.Healthcheck(ctx, true)
+	sbx.LogMetrics(ctx)
 
 	childSpan.SetAttributes(
 		attribute.String("env.id", sbx.Sandbox.TemplateID),

packages/shared/pkg/logs/logger.go

@@ -68,7 +68,7 @@ type SandboxLogger struct {
 	teamID                string
 	cpuMax                int32
 	cpuWasAboveTreshold   atomic.Bool
-	memoryMBMax           int32
+	memoryMiBMax          int32
 	memoryWasAbove        atomic.Int32
 	healthCheckWasFailing atomic.Bool
 }
@@ -83,13 +83,13 @@ func NewSandboxLogger(
 ) *SandboxLogger {
 	sbxLogExporter := getSandboxLogExporter()
 	return &SandboxLogger{
-		exporter:    sbxLogExporter,
-		instanceID:  instanceID,
-		internal:    internal,
-		envID:       envID,
-		teamID:      teamID,
-		cpuMax:      cpuMax,
-		memoryMBMax: memoryMax,
+		exporter:     sbxLogExporter,
+		instanceID:   instanceID,
+		internal:     internal,
+		envID:        envID,
+		teamID:       teamID,
+		cpuMax:       cpuMax,
+		memoryMiBMax: memoryMax,
 	}
 }
 
@@ -107,7 +107,7 @@ func (l *SandboxLogger) GetInternalLogger() *SandboxLogger {
 		return l
 	}
 
-	return NewSandboxLogger(l.instanceID, l.envID, l.teamID, l.cpuMax, l.memoryMBMax, true)
+	return NewSandboxLogger(l.instanceID, l.envID, l.teamID, l.cpuMax, l.memoryMiBMax, true)
 }
 
 func (l *SandboxLogger) Errorf(
@@ -165,20 +165,40 @@ func (l *SandboxLogger) CPUUsage(cpu float64) {
 
 func (l *SandboxLogger) MemoryUsage(memoryMB float64) {
 	// Cap at memoryMBMax
-	memoryMB = math.Min(memoryMB, float64(l.memoryMBMax))
-	if memoryMB > memoryUsageThreshold*float64(l.memoryMBMax) && int32(memoryMB) > l.memoryWasAbove.Load() {
+	memoryMB = math.Min(memoryMB, float64(l.memoryMiBMax))
+	if memoryMB > memoryUsageThreshold*float64(l.memoryMiBMax) && int32(memoryMB) > l.memoryWasAbove.Load() {
 		l.memoryWasAbove.Store(int32(memoryMB))
 		l.exporter.logger.Warn().
 			Str("instanceID", l.instanceID).
 			Str("envID", l.envID).
 			Str("teamID", l.teamID).
 			Float64("memoryMBUsed", memoryMB).
-			Int32("memoryMBTotal", l.memoryMBMax).
-			Msgf("Sandbox memory used %d %% of RAM", int(memoryMB/float64(l.memoryMBMax)*100))
+			Int32("memoryMBTotal", l.memoryMiBMax).
+			Msgf("Sandbox memory used %d %% of RAM", int(memoryMB/float64(l.memoryMiBMax)*100))
 		return
 	}
 }
 
+func (l *SandboxLogger) CPUPct(cpuPct float64) {
+	l.exporter.logger.Info().
+		Str("instanceID", l.instanceID).
+		Str("envID", l.envID).
+		Str("teamID", l.teamID).
+		Float64("cpuPct", cpuPct).
+		Int32("cpuCount", l.cpuMax).
+		Msg("CPU usage")
+}
+
+func (l *SandboxLogger) MemMiB(memMiB uint64) {
+	l.exporter.logger.Info().
+		Str("instanceID", l.instanceID).
+		Str("envID", l.envID).
+		Str("teamID", l.teamID).
+		Uint64("memMiBUsed", memMiB).
+		Int32("memoryMiBTotal", l.memoryMiBMax).
+		Msg("Memory usage")
+}
+
 func (l *SandboxLogger) Healthcheck(ok bool, alwaysReport bool) {
 	if !ok && !l.healthCheckWasFailing.Load() {
 		l.healthCheckWasFailing.Store(true)