Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Periodically capture resource metrics for every sandbox #216

Open
wants to merge 27 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 18 commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
749fbf2
make metrics struct more readable and bump env version
0div Dec 5, 2024
882f847
update orchestrator checks to use ticker and check metrics at differe…
0div Dec 5, 2024
c1c4994
add logger methods to log cpu percent and memory MB
0div Dec 5, 2024
203e525
log metrics on start and on end; reduce httpClient timeout to one second
0div Dec 5, 2024
290bbd3
Merge branch 'main' of https://github.com/e2b-dev/infra into periodic…
0div Dec 6, 2024
419b581
check if envd version is correct on every metrics log attempt
0div Dec 6, 2024
ff936ff
make CPU and MEM logger send event
0div Dec 6, 2024
f49461a
address PR comments
0div Dec 6, 2024
5ccbb3d
Update packages/envd/internal/host/metrics.go
0div Dec 6, 2024
2baee66
Update packages/envd/internal/host/metrics.go
0div Dec 6, 2024
805b77d
fix old method name
0div Dec 6, 2024
3c0343f
Merge branch 'main' of https://github.com/e2b-dev/infra into periodic…
0div Dec 6, 2024
c705e1c
remove legacu stats logic
0div Dec 6, 2024
9c94a61
specify platform/arch in docker image tf resrource to avoid docker pu…
0div Dec 6, 2024
5f859c6
report more memory data for debugging purposes; attempt to calculte f…
0div Dec 7, 2024
5df1d59
Update packages/shared/pkg/logs/logger.go
0div Dec 9, 2024
d1df2ac
Update packages/shared/pkg/logs/logger.go
0div Dec 9, 2024
1eb4d77
Update packages/shared/pkg/logs/logger.go
0div Dec 9, 2024
d7589b2
Merge branch 'main' of https://github.com/e2b-dev/infra into periodic…
0div Dec 9, 2024
27e9859
address PR review comments
0div Dec 9, 2024
c1294a1
Merge branch 'main' of https://github.com/e2b-dev/infra into periodic…
0div Dec 16, 2024
bbd42e6
add metrics endpoint to api spec and generate
0div Dec 17, 2024
9f44dc9
unmarshall log line into metric
0div Dec 18, 2024
4736cdf
fix merge conflicts
0div Jan 17, 2025
562e22c
Merge branch 'main' of https://github.com/e2b-dev/infra into periodic…
0div Jan 17, 2025
11880e6
Merge branch 'main' of https://github.com/e2b-dev/infra into periodic…
0div Jan 18, 2025
a3034c1
* add cpuTotal to spec
0div Jan 18, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions packages/docker-reverse-proxy/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ data "docker_registry_image" "docker_reverse_proxy_image" {
resource "docker_image" "docker_reverse_proxy_image" {
name = data.docker_registry_image.docker_reverse_proxy_image.name
pull_triggers = [data.docker_registry_image.docker_reverse_proxy_image.sha256_digest]
platform = "linux/amd64"
}

resource "google_service_account" "docker_registry_service_account" {
Expand Down
17 changes: 11 additions & 6 deletions packages/envd/internal/host/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,10 @@ import (
)

type Metrics struct {
CPU float64 `json:"cpu_pct"` // Percent rounded to 2 decimal places
Mem uint64 `json:"mem_bytes"` // Total virtual memory in bytes
Timestamp int64 `json:"ts"` // Unix Timestamp in UTC
Timestamp int64 `json:"ts"` // Unix Timestamp in UTC
CPUPercent float64 `json:"cpu_pct"` // Percent rounded to 2 decimal places
MemTotalMiB uint64 `json:"mem_total_mib"` // Total virtual memory in MiB
MemUsedMiB uint64 `json:"mem_used_mib"` // Used virtual memory in MiB
}

func GetMetrics() (*Metrics, error) {
Expand All @@ -20,6 +21,9 @@ func GetMetrics() (*Metrics, error) {
return nil, err
}

memUsedMiB := v.Used / 1024 / 1024
memTotalMiB := v.Total / 1024 / 1024

cpuPcts, err := cpu.Percent(0, false)
if err != nil {
return nil, err
Expand All @@ -32,8 +36,9 @@ func GetMetrics() (*Metrics, error) {
}

return &Metrics{
CPU: cpuPctRounded,
Mem: v.Total,
Timestamp: time.Now().UTC().Unix(),
Timestamp: time.Now().UTC().Unix(),
CPUPercent: cpuPctRounded,
MemUsedMiB: memUsedMiB,
MemTotalMiB: memTotalMiB,
}, nil
}
2 changes: 1 addition & 1 deletion packages/envd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ const (

var (
// These vars are automatically set by goreleaser.
Version = "0.1.2"
Version = "0.1.3"

debug bool
port int64
Expand Down
82 changes: 73 additions & 9 deletions packages/orchestrator/internal/sandbox/checks.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,34 +2,46 @@ package sandbox

import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"time"

"github.com/e2b-dev/infra/packages/shared/pkg/consts"
"github.com/e2b-dev/infra/packages/shared/pkg/utils"
"golang.org/x/mod/semver"
)

const (
healthCheckInterval = 10 * time.Second
metricsCheckInterval = 5 * time.Second
minEnvdVersionForMetrcis = "0.1.3"
)

func (s *Sandbox) logHeathAndUsage(ctx *utils.LockableCancelableContext) {
healthTicker := time.NewTicker(healthCheckInterval)
metricsTicker := time.NewTicker(metricsCheckInterval)
defer func() {
healthTicker.Stop()
metricsTicker.Stop()
}()

// Get metrics on sandbox startup
go s.LogMetrics(ctx)
0div marked this conversation as resolved.
Show resolved Hide resolved

for {
select {
case <-time.After(10 * time.Second):
case <-healthTicker.C:
childCtx, cancel := context.WithTimeout(ctx, time.Second)

ctx.Lock()
s.Healthcheck(childCtx, false)
ctx.Unlock()

cancel()

stats, err := s.stats.getStats()
if err != nil {
s.Logger.Warnf("failed to get stats: %s", err)
} else {
s.Logger.CPUUsage(stats.CPUCount)
s.Logger.MemoryUsage(stats.MemoryMB)
}
case <-metricsTicker.C:
s.LogMetrics(ctx)
case <-ctx.Done():
return
}
Expand Down Expand Up @@ -65,3 +77,55 @@ func (s *Sandbox) Healthcheck(ctx context.Context, alwaysReport bool) {
return
}
}

func (s *Sandbox) GetMetrics(ctx context.Context) (SandboxMetrics, error) {
address := fmt.Sprintf("http://%s:%d/metrics", s.slot.HostIP(), consts.DefaultEnvdServerPort)

request, err := http.NewRequestWithContext(ctx, "GET", address, nil)
if err != nil {
return SandboxMetrics{}, err
}

response, err := httpClient.Do(request)
if err != nil {
return SandboxMetrics{}, err
}
defer response.Body.Close()

if response.StatusCode != http.StatusOK {
err = fmt.Errorf("unexpected status code: %d", response.StatusCode)
return SandboxMetrics{}, err
}

var metrics SandboxMetrics
err = json.NewDecoder(response.Body).Decode(&metrics)
if err != nil {
return SandboxMetrics{}, err
}

return metrics, nil
}

func (s *Sandbox) LogMetrics(ctx context.Context) {
if isGTEVersion(s.Sandbox.EnvdVersion, minEnvdVersionForMetrcis) {
metrics, err := s.GetMetrics(ctx)
if err != nil {
s.Logger.Warnf("failed to get metrics: %s", err)
} else {
s.Logger.CPUPct(metrics.CPUPercent)
s.Logger.MemMiB(metrics.MemTotalMiB, metrics.MemUsedMiB)
}
}
}

func isGTEVersion(curVersion, minVersion string) bool {
if len(curVersion) > 0 && curVersion[0] != 'v' {
curVersion = "v" + curVersion
}

if !semver.IsValid(curVersion) {
return false
}

return semver.Compare(curVersion, minVersion) >= 0
}
8 changes: 8 additions & 0 deletions packages/orchestrator/internal/sandbox/metrics.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
package sandbox

type SandboxMetrics struct {
Timestamp int64 `json:"ts"` // Unix Timestamp in UTC
CPUPercent float64 `json:"cpu_pct"` // Percent rounded to 2 decimal places
MemTotalMiB uint64 `json:"mem_total_mib"` // Total virtual memory in MiB
MemUsedMiB uint64 `json:"mem_used_mib"` // Used virtual memory in MiB
}
13 changes: 3 additions & 10 deletions packages/orchestrator/internal/sandbox/sandbox.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ import (
consul "github.com/hashicorp/consul/api"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"golang.org/x/mod/semver"

"github.com/e2b-dev/infra/packages/orchestrator/internal/dns"
"github.com/e2b-dev/infra/packages/orchestrator/internal/sandbox/uffd"
Expand All @@ -29,10 +28,11 @@ const (
kernelMountDir = "/fc-vm"
kernelName = "vmlinux.bin"
fcBinaryName = "firecracker"
newEnvdVersion = "v0.1.1"
)

var httpClient = http.Client{
Timeout: 5 * time.Second,
Timeout: time.Second,
jakubno marked this conversation as resolved.
Show resolved Hide resolved
}

type Sandbox struct {
Expand All @@ -51,7 +51,6 @@ type Sandbox struct {

slot IPSlot
Logger *logs.SandboxLogger
stats *SandboxStats
}

func fcBinaryPath(fcVersion string) string {
Expand Down Expand Up @@ -198,11 +197,6 @@ func NewSandbox(

telemetry.ReportEvent(childCtx, "initialized FC")

stats := newSandboxStats(int32(fc.pid))
if err != nil {
return nil, fmt.Errorf("failed to create stats: %w", err)
}

healthcheckCtx := utils.NewLockableCancelableContext(context.Background())

instance := &Sandbox{
Expand All @@ -215,7 +209,6 @@ func NewSandbox(
networkPool: networkPool,
EndAt: endAt,
Logger: logger,
stats: stats,
stopOnce: sync.OnceValue(func() error {
var uffdErr error
if fcUffd != nil {
Expand Down Expand Up @@ -245,7 +238,7 @@ func NewSandbox(
telemetry.ReportEvent(childCtx, "ensuring clock sync")

// Sync envds.
if semver.Compare(fmt.Sprintf("v%s", config.EnvdVersion), "v0.1.1") >= 0 {
if isGTEVersion(config.EnvdVersion, newEnvdVersion) {
err = instance.initEnvd(ctx, tracer, config.EnvVars)
if err != nil {
return nil, fmt.Errorf("failed to init new envd: %w", err)
Expand Down
148 changes: 0 additions & 148 deletions packages/orchestrator/internal/sandbox/stats.go

This file was deleted.

2 changes: 2 additions & 0 deletions packages/orchestrator/internal/server/sandboxes.go
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,9 @@ func (s *server) Delete(ctx context.Context, in *orchestrator.SandboxRequest) (*
return nil, status.New(codes.NotFound, errMsg.Error()).Err()
}

// Check health metrics before stopping the sandbox
sbx.Healthcheck(ctx, true)
sbx.LogMetrics(ctx)
0div marked this conversation as resolved.
Show resolved Hide resolved

childSpan.SetAttributes(
attribute.String("env.id", sbx.Sandbox.TemplateID),
Expand Down
Loading