From a9d2e92266d4a8ee718011deeee3a50da2cfedb6 Mon Sep 17 00:00:00 2001 From: Drew Stinnett Date: Tue, 27 Aug 2024 12:05:25 -0400 Subject: [PATCH 01/15] Testing goreleaser --- .goreleaser.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.goreleaser.yaml b/.goreleaser.yaml index e7d72f2..b4ba104 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -2,6 +2,7 @@ # # SPDX-License-Identifier: Apache-2.0 +version: 2 builds: - goos: @@ -10,3 +11,12 @@ builds: - amd64 - arm - arm64 +nfpms: + - vendor: rivosinc + maintainer: abhinavDhulipala + formats: + - apk + - deb + - rpm + - termux.deb + - archlinux \ No newline at end of file From 0f38d02cf424231cc9a292cbb9c3d3a5665a8c43 Mon Sep 17 00:00:00 2001 From: Drew Stinnett Date: Wed, 28 Aug 2024 07:50:46 -0400 Subject: [PATCH 02/15] Adding packaging pieces --- .gitignore | 1 + .goreleaser.yaml | 16 +++++++-- README.md | 36 ++++++++++++++++++++- packaging/postinstall.sh | 4 +++ packaging/postremove.sh | 3 ++ packaging/preremove.sh | 3 ++ packaging/prometheus-slurm-exporter.service | 10 ++++++ 7 files changed, 70 insertions(+), 3 deletions(-) create mode 100755 packaging/postinstall.sh create mode 100755 packaging/postremove.sh create mode 100755 packaging/preremove.sh create mode 100644 packaging/prometheus-slurm-exporter.service diff --git a/.gitignore b/.gitignore index 5137776..8b1f490 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,4 @@ tmp* coverage.html coverage.out .DS_Store +dist/ diff --git a/.goreleaser.yaml b/.goreleaser.yaml index b4ba104..91275d1 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -12,8 +12,20 @@ builds: - arm - arm64 nfpms: - - vendor: rivosinc - maintainer: abhinavDhulipala + - + vendor: rivosinc + maintainer: abhinavDhulipal + contents: + - src: packaging/prometheus-slurm-exporter.service + dst: /usr/lib/systemd/system/prometheus-slurm-exporter.service + file_info: + mode: 0644 + group: root + owner: root + scripts: + postinstall: "packaging/postinstall.sh" + preremove: "packaging/preremove.sh" + postremove: "packaging/postremove.sh" formats: - apk - deb diff --git a/README.md b/README.md index f0854c9..1908f6b 100644 --- a/README.md +++ b/README.md @@ -153,6 +153,40 @@ Env vars can be sepcified in a `.env` file, while using the `just` | LOGLEVEL | info | Log Level: debug, info, warn, error | | CLI_TIMEOUT | 10. | # seconds before the exporter terminates command. | +### RPM/DEB Packages + +You can download RPM or DEB versions from the [Releases](releases/) tab. The +packages are configured to use systemd to start and stop the service. + +Configuring the systemd service + +`$ systemctl edit prometheus-slurm-exporter.service` + +```text +### Editing /etc/systemd/system/prometheus-slurm-exporter.service.d/override.conf +### Anything between here and the comment below will become the new contents of the file + +[Service] +Environment="PATH=/opt/slurm/bin" +Environment="POLL_INTERVAL=300" +Environment="CLI_TIMEOUT=60" +Environment="LOGLEVEL=debug" + +### Lines below this comment will be discarded + +### /usr/lib/systemd/system/prometheus-slurm-exporter.service +# [Unit] +# Description=Prometheus SLURM Exporter +# +# [Service] +# ExecStart=/usr/bin/prometheus-slurm-exporter +# Restart=always +# RestartSec=15 +# +# [Install] +# WantedBy=multi-user.target +``` + ### Future work -Add scheduler info, slurmrestd support, package binary into apt, rpm packages, and docker +Add scheduler info, slurmrestd support, package binary into docker diff --git a/packaging/postinstall.sh b/packaging/postinstall.sh new file mode 100755 index 0000000..aecee03 --- /dev/null +++ b/packaging/postinstall.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +systemctl daemon-reload +systemctl enable --now prometheus-slurm-exporter.service diff --git a/packaging/postremove.sh b/packaging/postremove.sh new file mode 100755 index 0000000..ab20f7b --- /dev/null +++ b/packaging/postremove.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +systemctl daemon-reload diff --git a/packaging/preremove.sh b/packaging/preremove.sh new file mode 100755 index 0000000..a3d9b19 --- /dev/null +++ b/packaging/preremove.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +systemctl disable prometheus-slurm-exporter.service || /bin/true diff --git a/packaging/prometheus-slurm-exporter.service b/packaging/prometheus-slurm-exporter.service new file mode 100644 index 0000000..d8b5c0e --- /dev/null +++ b/packaging/prometheus-slurm-exporter.service @@ -0,0 +1,10 @@ +[Unit] +Description=Prometheus SLURM Exporter + +[Service] +ExecStart=/usr/bin/prometheus-slurm-exporter +Restart=always +RestartSec=15 + +[Install] +WantedBy=multi-user.target From ecd9692955015d330b3e8775acce45e798774512 Mon Sep 17 00:00:00 2001 From: Drew Stinnett Date: Tue, 27 Aug 2024 12:05:25 -0400 Subject: [PATCH 03/15] Testing goreleaser Adding packaging pieces --- .gitignore | 1 + .goreleaser.yaml | 22 +++++++++++++ README.md | 36 ++++++++++++++++++++- packaging/postinstall.sh | 4 +++ packaging/postremove.sh | 3 ++ packaging/preremove.sh | 3 ++ packaging/prometheus-slurm-exporter.service | 10 ++++++ 7 files changed, 78 insertions(+), 1 deletion(-) create mode 100755 packaging/postinstall.sh create mode 100755 packaging/postremove.sh create mode 100755 packaging/preremove.sh create mode 100644 packaging/prometheus-slurm-exporter.service diff --git a/.gitignore b/.gitignore index 5137776..8b1f490 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,4 @@ tmp* coverage.html coverage.out .DS_Store +dist/ diff --git a/.goreleaser.yaml b/.goreleaser.yaml index e7d72f2..91275d1 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -2,6 +2,7 @@ # # SPDX-License-Identifier: Apache-2.0 +version: 2 builds: - goos: @@ -10,3 +11,24 @@ builds: - amd64 - arm - arm64 +nfpms: + - + vendor: rivosinc + maintainer: abhinavDhulipal + contents: + - src: packaging/prometheus-slurm-exporter.service + dst: /usr/lib/systemd/system/prometheus-slurm-exporter.service + file_info: + mode: 0644 + group: root + owner: root + scripts: + postinstall: "packaging/postinstall.sh" + preremove: "packaging/preremove.sh" + postremove: "packaging/postremove.sh" + formats: + - apk + - deb + - rpm + - termux.deb + - archlinux \ No newline at end of file diff --git a/README.md b/README.md index f0854c9..1908f6b 100644 --- a/README.md +++ b/README.md @@ -153,6 +153,40 @@ Env vars can be sepcified in a `.env` file, while using the `just` | LOGLEVEL | info | Log Level: debug, info, warn, error | | CLI_TIMEOUT | 10. | # seconds before the exporter terminates command. | +### RPM/DEB Packages + +You can download RPM or DEB versions from the [Releases](releases/) tab. The +packages are configured to use systemd to start and stop the service. + +Configuring the systemd service + +`$ systemctl edit prometheus-slurm-exporter.service` + +```text +### Editing /etc/systemd/system/prometheus-slurm-exporter.service.d/override.conf +### Anything between here and the comment below will become the new contents of the file + +[Service] +Environment="PATH=/opt/slurm/bin" +Environment="POLL_INTERVAL=300" +Environment="CLI_TIMEOUT=60" +Environment="LOGLEVEL=debug" + +### Lines below this comment will be discarded + +### /usr/lib/systemd/system/prometheus-slurm-exporter.service +# [Unit] +# Description=Prometheus SLURM Exporter +# +# [Service] +# ExecStart=/usr/bin/prometheus-slurm-exporter +# Restart=always +# RestartSec=15 +# +# [Install] +# WantedBy=multi-user.target +``` + ### Future work -Add scheduler info, slurmrestd support, package binary into apt, rpm packages, and docker +Add scheduler info, slurmrestd support, package binary into docker diff --git a/packaging/postinstall.sh b/packaging/postinstall.sh new file mode 100755 index 0000000..aecee03 --- /dev/null +++ b/packaging/postinstall.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +systemctl daemon-reload +systemctl enable --now prometheus-slurm-exporter.service diff --git a/packaging/postremove.sh b/packaging/postremove.sh new file mode 100755 index 0000000..ab20f7b --- /dev/null +++ b/packaging/postremove.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +systemctl daemon-reload diff --git a/packaging/preremove.sh b/packaging/preremove.sh new file mode 100755 index 0000000..a3d9b19 --- /dev/null +++ b/packaging/preremove.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +systemctl disable prometheus-slurm-exporter.service || /bin/true diff --git a/packaging/prometheus-slurm-exporter.service b/packaging/prometheus-slurm-exporter.service new file mode 100644 index 0000000..d8b5c0e --- /dev/null +++ b/packaging/prometheus-slurm-exporter.service @@ -0,0 +1,10 @@ +[Unit] +Description=Prometheus SLURM Exporter + +[Service] +ExecStart=/usr/bin/prometheus-slurm-exporter +Restart=always +RestartSec=15 + +[Install] +WantedBy=multi-user.target From f0dfa289154316153cfe205c729ee4c8755f7f2c Mon Sep 17 00:00:00 2001 From: Drew Stinnett Date: Wed, 28 Aug 2024 08:11:54 -0400 Subject: [PATCH 04/15] Adding license bits --- packaging/postinstall.sh | 3 +++ packaging/postremove.sh | 3 +++ packaging/preremove.sh | 3 +++ packaging/prometheus-slurm-exporter.service | 3 +++ 4 files changed, 12 insertions(+) diff --git a/packaging/postinstall.sh b/packaging/postinstall.sh index aecee03..134c768 100755 --- a/packaging/postinstall.sh +++ b/packaging/postinstall.sh @@ -1,4 +1,7 @@ #!/bin/bash +# SPDX-FileCopyrightText: 2023 Rivos Inc. +# +# SPDX-License-Identifier: Apache-2.0 systemctl daemon-reload systemctl enable --now prometheus-slurm-exporter.service diff --git a/packaging/postremove.sh b/packaging/postremove.sh index ab20f7b..c19b379 100755 --- a/packaging/postremove.sh +++ b/packaging/postremove.sh @@ -1,3 +1,6 @@ #!/bin/bash +# SPDX-FileCopyrightText: 2023 Rivos Inc. +# +# SPDX-License-Identifier: Apache-2.0 systemctl daemon-reload diff --git a/packaging/preremove.sh b/packaging/preremove.sh index a3d9b19..207e0e0 100755 --- a/packaging/preremove.sh +++ b/packaging/preremove.sh @@ -1,3 +1,6 @@ #!/bin/bash +# SPDX-FileCopyrightText: 2023 Rivos Inc. +# +# SPDX-License-Identifier: Apache-2.0 systemctl disable prometheus-slurm-exporter.service || /bin/true diff --git a/packaging/prometheus-slurm-exporter.service b/packaging/prometheus-slurm-exporter.service index d8b5c0e..42ca665 100644 --- a/packaging/prometheus-slurm-exporter.service +++ b/packaging/prometheus-slurm-exporter.service @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: 2023 Rivos Inc. +# +# SPDX-License-Identifier: Apache-2.0 [Unit] Description=Prometheus SLURM Exporter From 309c3027701394f4b4da9e0b02ec63eec163a2e1 Mon Sep 17 00:00:00 2001 From: Drew Stinnett Date: Wed, 28 Aug 2024 08:13:40 -0400 Subject: [PATCH 05/15] Fixing newline --- .goreleaser.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 91275d1..de6e898 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -31,4 +31,5 @@ nfpms: - deb - rpm - termux.deb - - archlinux \ No newline at end of file + - archlinux + From 43b84735331b1f65214084f9a7a51ed96f478a95 Mon Sep 17 00:00:00 2001 From: Drew Stinnett Date: Wed, 28 Aug 2024 08:16:43 -0400 Subject: [PATCH 06/15] Fixing whitespace --- .goreleaser.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.goreleaser.yaml b/.goreleaser.yaml index de6e898..84656f1 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -1,7 +1,6 @@ # SPDX-FileCopyrightText: 2023 Rivos Inc. # # SPDX-License-Identifier: Apache-2.0 - version: 2 builds: - @@ -12,7 +11,7 @@ builds: - arm - arm64 nfpms: - - + - vendor: rivosinc maintainer: abhinavDhulipal contents: @@ -32,4 +31,3 @@ nfpms: - rpm - termux.deb - archlinux - From 6e7dada3ed0e4c3e7b972511ba555576d85c6a49 Mon Sep 17 00:00:00 2001 From: abhinavDhulipala <46908860+abhinavDhulipala@users.noreply.github.com> Date: Wed, 28 Aug 2024 12:40:16 -0700 Subject: [PATCH 07/15] [ci] archive static templates with binary (#95) * archive static templates with binary * update docs --- .gitignore | 1 + .goreleaser.yaml | 10 ++++++++-- README.md | 13 +++++++------ exporter/trace.go | 8 +++++++- 4 files changed, 23 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 5137776..70942a3 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,4 @@ tmp* coverage.html coverage.out .DS_Store +dist diff --git a/.goreleaser.yaml b/.goreleaser.yaml index e7d72f2..c0590f8 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -2,11 +2,17 @@ # # SPDX-License-Identifier: Apache-2.0 +version: 2 builds: - - - goos: + - goos: - linux goarch: - amd64 - arm - arm64 + +archives: + - name_template: '{{ .ProjectName }}_{{ .Os }}_{{ .Arch }}' + files: + - src: ./exporter/templates/* + dst: templates diff --git a/README.md b/README.md index f0854c9..6d197c4 100644 --- a/README.md +++ b/README.md @@ -147,12 +147,13 @@ $ curl localhost:9092/metrics | grep "# HELP" ### Exporter Env Var Docs Env vars can be sepcified in a `.env` file, while using the `just` -| Var | Default Value | Purpose | -|---------------|---------------|-----------------------------------------------------------------------------| -| POLL_LIMIT | 10 | # of seconds to wait before polling slurmctl again (client-side throttling) | -| LOGLEVEL | info | Log Level: debug, info, warn, error | -| CLI_TIMEOUT | 10. | # seconds before the exporter terminates command. | +| Var | Default Value | Purpose | +|-----------------|---------------|-----------------------------------------------------------------------------| +| POLL_LIMIT | 10 | # of seconds to wait before polling slurmctl again (client-side throttling) | +| LOGLEVEL | info | Log Level: debug, info, warn, error | +| CLI_TIMEOUT | 10. | # seconds before the exporter terminates command. | +| TRACE_ROOT_PATH | "cwd" | path to ./templates directory where html files are located | ### Future work -Add scheduler info, slurmrestd support, package binary into apt, rpm packages, and docker +slurmrestd support diff --git a/exporter/trace.go b/exporter/trace.go index 642acaa..227b8b3 100644 --- a/exporter/trace.go +++ b/exporter/trace.go @@ -11,6 +11,7 @@ import ( "io/fs" "log" "net/http" + "os" "path/filepath" "sync" "text/template" @@ -106,8 +107,13 @@ type TraceCollector struct { func NewTraceCollector(config *Config) *TraceCollector { traceConfig := config.TraceConf + templateRootDir := "." + // path to look for the /templates directory. Defaults to cwd + if path, ok := os.LookupEnv("TRACE_ROOT_PATH"); ok { + templateRootDir = path + } traceDir := "" - err := filepath.WalkDir(".", func(path string, d fs.DirEntry, err error) error { + err := filepath.WalkDir(templateRootDir, func(path string, d fs.DirEntry, err error) error { if err == nil && d.IsDir() && d.Name() == templateDirName { traceDir = path return nil From aac8ae0df8c511a0c99996f73095044a7f27ac1f Mon Sep 17 00:00:00 2001 From: Drew Stinnett Date: Thu, 29 Aug 2024 09:18:29 -0400 Subject: [PATCH 08/15] Oops forgot conflict --- README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/README.md b/README.md index b9090d6..f20642e 100644 --- a/README.md +++ b/README.md @@ -190,8 +190,4 @@ Environment="LOGLEVEL=debug" ### Future work -<<<<<<< HEAD -Add scheduler info, slurmrestd support, package binary into docker -======= slurmrestd support ->>>>>>> upstream/main From 5d4374658c2d406eeffd1549a6192c4640a59cf1 Mon Sep 17 00:00:00 2001 From: Drew Stinnett Date: Thu, 29 Aug 2024 09:34:07 -0400 Subject: [PATCH 09/15] Adding trace templates to distro package --- .goreleaser.yaml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 89d40e5..8563bca 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -10,8 +10,7 @@ builds: - arm - arm64 nfpms: - - - vendor: rivosinc + - vendor: rivosinc maintainer: abhinavDhulipal contents: - src: packaging/prometheus-slurm-exporter.service @@ -20,6 +19,12 @@ nfpms: mode: 0644 group: root owner: root + - src: exporter/templates + dst: /usr/share/prometheus-slurm-exporter/templates + file_info: + mode: 0644 + group: root + owner: root scripts: postinstall: "packaging/postinstall.sh" preremove: "packaging/preremove.sh" From 2c9e0eeb93d017e7e652f3c7f50ba84fdf002e8d Mon Sep 17 00:00:00 2001 From: Drew Stinnett Date: Thu, 29 Aug 2024 09:35:13 -0400 Subject: [PATCH 10/15] Fixing spacing --- .goreleaser.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 8563bca..04bce86 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -40,4 +40,3 @@ archives: files: - src: ./exporter/templates/* dst: templates - From 3afae44c23d0baad51282343ffecccc79e79d9c4 Mon Sep 17 00:00:00 2001 From: abhinavDhulipala <46908860+abhinavDhulipala@users.noreply.github.com> Date: Thu, 29 Aug 2024 17:57:00 -0700 Subject: [PATCH 11/15] [exporter/jobs] fix JobMetric Fetcher cache bug (#99) * fix and cover cli fetcher cache bug * refactor test utils into seperate file * cover json job fetcher --- exporter/jobs.go | 119 ++++++++++++++++---------------- exporter/jobs_test.go | 152 +++++++++++++++++++++++++++++++++-------- exporter/mock_utils.go | 71 +++++++++++++++++++ exporter/utils.go | 31 --------- exporter/utils_test.go | 26 ------- 5 files changed, 255 insertions(+), 144 deletions(-) create mode 100644 exporter/mock_utils.go diff --git a/exporter/jobs.go b/exporter/jobs.go index c671b42..4a5d3ba 100644 --- a/exporter/jobs.go +++ b/exporter/jobs.go @@ -55,13 +55,28 @@ type JobJsonFetcher struct { errCounter prometheus.Counter } -func (jjf *JobJsonFetcher) FetchMetrics() ([]JobMetric, error) { +func (jjf *JobJsonFetcher) fetch() ([]JobMetric, error) { data, err := jjf.scraper.FetchRawBytes() if err != nil { jjf.errCounter.Inc() return nil, err } - return jjf.cache.FetchOrThrottle(func() ([]JobMetric, error) { return parseJobMetrics(data) }) + var squeue squeueResponse + err = json.Unmarshal(data, &squeue) + if err != nil { + slog.Error("Unmarshaling node metrics %q", err) + return nil, err + } + for _, j := range squeue.Jobs { + for _, resource := range j.JobResources.AllocNodes { + resource.Mem *= 1e9 + } + } + return squeue.Jobs, nil +} + +func (jjf *JobJsonFetcher) FetchMetrics() ([]JobMetric, error) { + return jjf.cache.FetchOrThrottle(jjf.fetch) } func (jjf *JobJsonFetcher) ScrapeDuration() time.Duration { @@ -78,65 +93,11 @@ type JobCliFallbackFetcher struct { errCounter prometheus.Counter } -func (jcf *JobCliFallbackFetcher) FetchMetrics() ([]JobMetric, error) { - data, err := jcf.scraper.FetchRawBytes() +func (jcf *JobCliFallbackFetcher) fetch() ([]JobMetric, error) { + squeue, err := jcf.scraper.FetchRawBytes() if err != nil { - jcf.errCounter.Inc() return nil, err } - return jcf.cache.FetchOrThrottle(func() ([]JobMetric, error) { return parseCliFallback(data, jcf.errCounter) }) -} - -func (jcf *JobCliFallbackFetcher) ScrapeDuration() time.Duration { - return jcf.scraper.Duration() -} - -func (jcf *JobCliFallbackFetcher) ScrapeError() prometheus.Counter { - return jcf.errCounter -} - -func totalAllocMem(resource *JobResource) float64 { - var allocMem float64 - for _, node := range resource.AllocNodes { - allocMem += node.Mem - } - return allocMem -} - -func parseJobMetrics(jsonJobList []byte) ([]JobMetric, error) { - var squeue squeueResponse - err := json.Unmarshal(jsonJobList, &squeue) - if err != nil { - slog.Error("Unmarshaling node metrics %q", err) - return nil, err - } - for _, j := range squeue.Jobs { - for _, resource := range j.JobResources.AllocNodes { - resource.Mem *= 1e9 - } - } - return squeue.Jobs, nil -} - -type NAbleTime struct{ time.Time } - -// report beginning of time in the case of N/A -func (nat *NAbleTime) UnmarshalJSON(data []byte) error { - var tString string - if err := json.Unmarshal(data, &tString); err != nil { - return err - } - nullSet := map[string]struct{}{"N/A": {}, "NONE": {}} - if _, ok := nullSet[tString]; ok { - nat.Time = time.Time{} - return nil - } - t, err := time.Parse("2006-01-02T15:04:05", tString) - nat.Time = t - return err -} - -func parseCliFallback(squeue []byte, errorCounter prometheus.Counter) ([]JobMetric, error) { jobMetrics := make([]JobMetric, 0) // clean input squeue = bytes.TrimSpace(squeue) @@ -159,13 +120,13 @@ func parseCliFallback(squeue []byte, errorCounter prometheus.Counter) ([]JobMetr } if err := json.Unmarshal(line, &metric); err != nil { slog.Error(fmt.Sprintf("squeue fallback parse error: failed on line %d `%s`", i, line)) - errorCounter.Inc() + jcf.errCounter.Inc() continue } mem, err := MemToFloat(metric.Mem) if err != nil { slog.Error(fmt.Sprintf("squeue fallback parse error: failed on line %d `%s` with err `%q`", i, line, err)) - errorCounter.Inc() + jcf.errCounter.Inc() continue } openapiJobMetric := JobMetric{ @@ -185,6 +146,44 @@ func parseCliFallback(squeue []byte, errorCounter prometheus.Counter) ([]JobMetr return jobMetrics, nil } +func (jcf *JobCliFallbackFetcher) FetchMetrics() ([]JobMetric, error) { + return jcf.cache.FetchOrThrottle(jcf.fetch) +} + +func (jcf *JobCliFallbackFetcher) ScrapeDuration() time.Duration { + return jcf.scraper.Duration() +} + +func (jcf *JobCliFallbackFetcher) ScrapeError() prometheus.Counter { + return jcf.errCounter +} + +func totalAllocMem(resource *JobResource) float64 { + var allocMem float64 + for _, node := range resource.AllocNodes { + allocMem += node.Mem + } + return allocMem +} + +type NAbleTime struct{ time.Time } + +// report beginning of time in the case of N/A +func (nat *NAbleTime) UnmarshalJSON(data []byte) error { + var tString string + if err := json.Unmarshal(data, &tString); err != nil { + return err + } + nullSet := map[string]struct{}{"N/A": {}, "NONE": {}} + if _, ok := nullSet[tString]; ok { + nat.Time = time.Time{} + return nil + } + t, err := time.Parse("2006-01-02T15:04:05", tString) + nat.Time = t + return err +} + type UserJobMetric struct { stateJobCount map[string]float64 totalJobCount float64 diff --git a/exporter/jobs_test.go b/exporter/jobs_test.go index d77eed7..91bf8c9 100644 --- a/exporter/jobs_test.go +++ b/exporter/jobs_test.go @@ -44,11 +44,14 @@ func TestNewJobsController(t *testing.T) { func TestParseJobMetrics(t *testing.T) { assert := assert.New(t) - fixture, err := MockJobInfoScraper.FetchRawBytes() - assert.Nil(err) - jms, err := parseJobMetrics(fixture) - assert.Nil(err) - assert.NotEmpty(jms) + scraper := &MockScraper{fixture: "fixtures/squeue_out.json"} + fetcher := &JobJsonFetcher{ + scraper: scraper, + cache: NewAtomicThrottledCache[JobMetric](100), + errCounter: prometheus.NewCounter(prometheus.CounterOpts{}), + } + jms, err := fetcher.fetch() + assert.NoError(err) // test parse of single job var job *JobMetric for _, m := range jms { @@ -63,22 +66,27 @@ func TestParseJobMetrics(t *testing.T) { func TestParseCliFallback(t *testing.T) { assert := assert.New(t) - fetcher := MockScraper{fixture: "fixtures/squeue_fallback.txt"} - data, err := fetcher.FetchRawBytes() - assert.Nil(err) - counter := prometheus.NewCounter(prometheus.CounterOpts{Name: "errors"}) - metrics, err := parseCliFallback(data, counter) + cliFallbackFetcher := &JobCliFallbackFetcher{ + scraper: &MockScraper{fixture: "fixtures/squeue_fallback.txt"}, + cache: NewAtomicThrottledCache[JobMetric](100), + errCounter: prometheus.NewCounter(prometheus.CounterOpts{Name: "errors"}), + } + metrics, err := cliFallbackFetcher.fetch() assert.Nil(err) assert.NotEmpty(metrics) - assert.Equal(2., CollectCounterValue(counter)) + assert.Equal(2., CollectCounterValue(cliFallbackFetcher.errCounter)) } func TestUserJobMetric(t *testing.T) { // setup assert := assert.New(t) - fixture, err := MockJobInfoScraper.FetchRawBytes() - assert.Nil(err) - jms, err := parseJobMetrics(fixture) + scraper := &MockScraper{fixture: "fixtures/squeue_out.json"} + fetcher := &JobJsonFetcher{ + scraper: scraper, + cache: NewAtomicThrottledCache[JobMetric](100), + errCounter: prometheus.NewCounter(prometheus.CounterOpts{}), + } + jms, err := fetcher.fetch() assert.Nil(err) //test @@ -158,9 +166,13 @@ func TestJobCollect_Fallback(t *testing.T) { func TestParsePartitionJobMetrics(t *testing.T) { assert := assert.New(t) - fixture, err := MockJobInfoScraper.FetchRawBytes() - assert.Nil(err) - jms, err := parseJobMetrics(fixture) + scraper := &MockScraper{fixture: "fixtures/squeue_out.json"} + fetcher := &JobJsonFetcher{ + scraper: scraper, + cache: NewAtomicThrottledCache[JobMetric](100), + errCounter: prometheus.NewCounter(prometheus.CounterOpts{}), + } + jms, err := fetcher.fetch() assert.Nil(err) partitionJobMetrics := parsePartitionJobMetrics(jms) @@ -169,9 +181,13 @@ func TestParsePartitionJobMetrics(t *testing.T) { func TestParsePartMetrics(t *testing.T) { assert := assert.New(t) - fixture, err := MockJobInfoScraper.FetchRawBytes() - assert.Nil(err) - jms, err := parseJobMetrics(fixture) + scraper := &MockScraper{fixture: "fixtures/squeue_out.json"} + fetcher := &JobJsonFetcher{ + scraper: scraper, + cache: NewAtomicThrottledCache[JobMetric](100), + errCounter: prometheus.NewCounter(prometheus.CounterOpts{}), + } + jms, err := fetcher.fetch() assert.Nil(err) featureMetrics := parseFeatureMetric(jms) @@ -222,15 +238,97 @@ func TestNAbleTimeJson_NA(t *testing.T) { func TestParseCliFallbackEmpty(t *testing.T) { assert := assert.New(t) - counter := prometheus.NewCounter(prometheus.CounterOpts{ - Name: "validation_counter", - }) - metrics, err := parseCliFallback([]byte(""), counter) + scraper := &StringByteScraper{msg: ""} + cliFallbackFetcher := &JobCliFallbackFetcher{ + scraper: scraper, + cache: NewAtomicThrottledCache[JobMetric](100), + errCounter: prometheus.NewCounter(prometheus.CounterOpts{Name: "errors"}), + } + metrics, err := cliFallbackFetcher.fetch() assert.NoError(err) assert.Empty(metrics) - assert.Zero(CollectCounterValue(counter)) - metrics, err = parseCliFallback([]byte("\n "), counter) + assert.Zero(CollectCounterValue(cliFallbackFetcher.errCounter)) + assert.Equal(1, scraper.Callcount) + scraper.msg = "\n" + metrics, err = cliFallbackFetcher.fetch() assert.NoError(err) assert.Empty(metrics) - assert.Zero(CollectCounterValue(counter)) + assert.Zero(CollectCounterValue(cliFallbackFetcher.errCounter)) + assert.Equal(2, scraper.Callcount) +} + +func TestCliJobFetcherCacheHit(t *testing.T) { + assert := assert.New(t) + scraper := &MockScraper{fixture: "fixtures/squeue_fallback.txt"} + cliFallbackFetcher := &JobCliFallbackFetcher{ + scraper: scraper, + cache: NewAtomicThrottledCache[JobMetric](100), + errCounter: prometheus.NewCounter(prometheus.CounterOpts{Name: "errors"}), + } + metrics, err := cliFallbackFetcher.FetchMetrics() + assert.NotEmpty(metrics) + assert.NoError(err) + assert.Equal(1, scraper.CallCount) + metrics, err = cliFallbackFetcher.FetchMetrics() + assert.NotEmpty(metrics) + assert.NoError(err) + // assert cache hit + assert.Equal(1, scraper.CallCount) +} + +func TestCliJobFetcherCacheMiss(t *testing.T) { + assert := assert.New(t) + scraper := &MockScraper{fixture: "fixtures/squeue_fallback.txt"} + cliFallbackFetcher := &JobCliFallbackFetcher{ + scraper: scraper, + cache: NewAtomicThrottledCache[JobMetric](0), + errCounter: prometheus.NewCounter(prometheus.CounterOpts{Name: "errors"}), + } + metrics, err := cliFallbackFetcher.FetchMetrics() + assert.NotEmpty(metrics) + assert.NoError(err) + assert.Equal(1, scraper.CallCount) + metrics, err = cliFallbackFetcher.FetchMetrics() + assert.NotEmpty(metrics) + assert.NoError(err) + // assert cache hit + assert.Equal(2, scraper.CallCount) +} + +func TestJsonJobFetcherCacheHit(t *testing.T) { + assert := assert.New(t) + scraper := &MockScraper{fixture: "fixtures/squeue_out.json"} + cliFallbackFetcher := &JobJsonFetcher{ + scraper: scraper, + cache: NewAtomicThrottledCache[JobMetric](100), + errCounter: prometheus.NewCounter(prometheus.CounterOpts{Name: "errors"}), + } + metrics, err := cliFallbackFetcher.FetchMetrics() + assert.NotEmpty(metrics) + assert.NoError(err) + assert.Equal(1, scraper.CallCount) + metrics, err = cliFallbackFetcher.FetchMetrics() + assert.NotEmpty(metrics) + assert.NoError(err) + // assert cache hit + assert.Equal(1, scraper.CallCount) +} + +func TestJsonJobFetcherCacheMiss(t *testing.T) { + assert := assert.New(t) + scraper := &MockScraper{fixture: "fixtures/squeue_out.json"} + cliFallbackFetcher := &JobJsonFetcher{ + scraper: scraper, + cache: NewAtomicThrottledCache[JobMetric](0), + errCounter: prometheus.NewCounter(prometheus.CounterOpts{Name: "errors"}), + } + metrics, err := cliFallbackFetcher.FetchMetrics() + assert.NotEmpty(metrics) + assert.NoError(err) + assert.Equal(1, scraper.CallCount) + metrics, err = cliFallbackFetcher.FetchMetrics() + assert.NotEmpty(metrics) + assert.NoError(err) + // assert cache hit + assert.Equal(2, scraper.CallCount) } diff --git a/exporter/mock_utils.go b/exporter/mock_utils.go new file mode 100644 index 0000000..3d2a1d6 --- /dev/null +++ b/exporter/mock_utils.go @@ -0,0 +1,71 @@ +// SPDX-FileCopyrightText: 2023 Rivos Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +package exporter + +import ( + "bytes" + "errors" + "os" + "time" +) + +type MockFetchErrored struct{} + +func (f *MockFetchErrored) FetchRawBytes() ([]byte, error) { + return nil, errors.New("mock fetch error") +} + +func (f *MockFetchErrored) Duration() time.Duration { + return 1 +} + +// implements SlurmByteScraper by pulling fixtures instead +// used exclusively for testing +type MockScraper struct { + fixture string + duration time.Duration + CallCount int +} + +func (f *MockScraper) FetchRawBytes() ([]byte, error) { + defer func(t time.Time) { + f.duration = time.Since(t) + }(time.Now()) + f.CallCount++ + file, err := os.ReadFile(f.fixture) + if err != nil { + return nil, err + } + // allow commenting in text files + sep := []byte("\n") + lines := bytes.Split(file, sep) + filtered := make([][]byte, 0) + for _, line := range lines { + if !bytes.HasPrefix(line, []byte("#")) { + filtered = append(filtered, line) + } + } + return bytes.Join(filtered, sep), nil +} + +func (f *MockScraper) Duration() time.Duration { + return f.duration +} + +// implements SlurmByteScraper by emmiting string payload instead +// used exclusively for testing +type StringByteScraper struct { + msg string + Callcount int +} + +func (es *StringByteScraper) FetchRawBytes() ([]byte, error) { + es.Callcount++ + return []byte(es.msg), nil +} + +func (es *StringByteScraper) Duration() time.Duration { + return time.Duration(1) +} diff --git a/exporter/utils.go b/exporter/utils.go index 5e18502..ffe0f04 100644 --- a/exporter/utils.go +++ b/exporter/utils.go @@ -133,37 +133,6 @@ func NewCliScraper(args ...string) *CliScraper { } } -// implements SlurmByteScraper by pulling fixtures instead -// used exclusively for testing -type MockScraper struct { - fixture string - duration time.Duration -} - -func (f *MockScraper) FetchRawBytes() ([]byte, error) { - defer func(t time.Time) { - f.duration = time.Since(t) - }(time.Now()) - file, err := os.ReadFile(f.fixture) - if err != nil { - return nil, err - } - // allow commenting in text files - sep := []byte("\n") - lines := bytes.Split(file, sep) - filtered := make([][]byte, 0) - for _, line := range lines { - if !bytes.HasPrefix(line, []byte("#")) { - filtered = append(filtered, line) - } - } - return bytes.Join(filtered, sep), nil -} - -func (f *MockScraper) Duration() time.Duration { - return f.duration -} - // convert slurm mem string to float64 bytes func MemToFloat(mem string) (float64, error) { if num, err := strconv.ParseFloat(mem, 64); err == nil { diff --git a/exporter/utils_test.go b/exporter/utils_test.go index b98c12d..012c56e 100644 --- a/exporter/utils_test.go +++ b/exporter/utils_test.go @@ -5,7 +5,6 @@ package exporter import ( - "errors" "fmt" "math" "math/rand" @@ -34,31 +33,6 @@ func generateRandString(n int) string { return string(randBytes) } -// used to ensure the fetch function was called -type MockFetchTriggered struct { - msg []byte - called bool -} - -func (f *MockFetchTriggered) Fetch() ([]byte, error) { - f.called = true - return f.msg, nil -} - -func (f *MockFetchTriggered) Duration() time.Duration { - return 1 -} - -type MockFetchErrored struct{} - -func (f *MockFetchErrored) FetchRawBytes() ([]byte, error) { - return nil, errors.New("mock fetch error") -} - -func (f *MockFetchErrored) Duration() time.Duration { - return 1 -} - func TestCliFetcher(t *testing.T) { assert := assert.New(t) cliFetcher := NewCliScraper("ls") From 7d462368e631a6ea74bc7dc9413fd7fae8b3ceb0 Mon Sep 17 00:00:00 2001 From: Drew Stinnett Date: Tue, 3 Sep 2024 09:18:37 -0400 Subject: [PATCH 12/15] Testing goreleaser and re-adding license --- .goreleaser.yaml | 5 ++--- packaging/postinstall.sh | 3 +++ packaging/postremove.sh | 3 +++ packaging/preremove.sh | 3 +++ packaging/prometheus-slurm-exporter.service | 3 +++ 5 files changed, 14 insertions(+), 3 deletions(-) diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 91275d1..8d2b990 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -12,8 +12,7 @@ builds: - arm - arm64 nfpms: - - - vendor: rivosinc + - vendor: rivosinc maintainer: abhinavDhulipal contents: - src: packaging/prometheus-slurm-exporter.service @@ -31,4 +30,4 @@ nfpms: - deb - rpm - termux.deb - - archlinux \ No newline at end of file + - archlinux diff --git a/packaging/postinstall.sh b/packaging/postinstall.sh index aecee03..134c768 100755 --- a/packaging/postinstall.sh +++ b/packaging/postinstall.sh @@ -1,4 +1,7 @@ #!/bin/bash +# SPDX-FileCopyrightText: 2023 Rivos Inc. +# +# SPDX-License-Identifier: Apache-2.0 systemctl daemon-reload systemctl enable --now prometheus-slurm-exporter.service diff --git a/packaging/postremove.sh b/packaging/postremove.sh index ab20f7b..c19b379 100755 --- a/packaging/postremove.sh +++ b/packaging/postremove.sh @@ -1,3 +1,6 @@ #!/bin/bash +# SPDX-FileCopyrightText: 2023 Rivos Inc. +# +# SPDX-License-Identifier: Apache-2.0 systemctl daemon-reload diff --git a/packaging/preremove.sh b/packaging/preremove.sh index a3d9b19..207e0e0 100755 --- a/packaging/preremove.sh +++ b/packaging/preremove.sh @@ -1,3 +1,6 @@ #!/bin/bash +# SPDX-FileCopyrightText: 2023 Rivos Inc. +# +# SPDX-License-Identifier: Apache-2.0 systemctl disable prometheus-slurm-exporter.service || /bin/true diff --git a/packaging/prometheus-slurm-exporter.service b/packaging/prometheus-slurm-exporter.service index d8b5c0e..42ca665 100644 --- a/packaging/prometheus-slurm-exporter.service +++ b/packaging/prometheus-slurm-exporter.service @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: 2023 Rivos Inc. +# +# SPDX-License-Identifier: Apache-2.0 [Unit] Description=Prometheus SLURM Exporter From b6c27ff3d3d053f3908953fbc30a7fde8c728b9e Mon Sep 17 00:00:00 2001 From: Drew Stinnett Date: Tue, 3 Sep 2024 09:22:58 -0400 Subject: [PATCH 13/15] Fixing whitespace --- .goreleaser.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 8d2b990..dc54b95 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -1,7 +1,6 @@ # SPDX-FileCopyrightText: 2023 Rivos Inc. # # SPDX-License-Identifier: Apache-2.0 - version: 2 builds: - From fec2288ce94c7256d6091969ed985adcea0304b0 Mon Sep 17 00:00:00 2001 From: Drew Stinnett Date: Tue, 3 Sep 2024 09:24:42 -0400 Subject: [PATCH 14/15] [ci] archive static templates with binary (#95) * archive static templates with binary * update docs --- .gitignore | 2 +- .goreleaser.yaml | 14 ++++++++++++-- README.md | 13 +++++++------ exporter/trace.go | 8 +++++++- 4 files changed, 27 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index 8b1f490..70942a3 100644 --- a/.gitignore +++ b/.gitignore @@ -24,4 +24,4 @@ tmp* coverage.html coverage.out .DS_Store -dist/ +dist diff --git a/.goreleaser.yaml b/.goreleaser.yaml index dc54b95..04bce86 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -3,8 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 version: 2 builds: - - - goos: + - goos: - linux goarch: - amd64 @@ -20,6 +19,12 @@ nfpms: mode: 0644 group: root owner: root + - src: exporter/templates + dst: /usr/share/prometheus-slurm-exporter/templates + file_info: + mode: 0644 + group: root + owner: root scripts: postinstall: "packaging/postinstall.sh" preremove: "packaging/preremove.sh" @@ -30,3 +35,8 @@ nfpms: - rpm - termux.deb - archlinux +archives: + - name_template: '{{ .ProjectName }}_{{ .Os }}_{{ .Arch }}' + files: + - src: ./exporter/templates/* + dst: templates diff --git a/README.md b/README.md index 1908f6b..f20642e 100644 --- a/README.md +++ b/README.md @@ -147,11 +147,12 @@ $ curl localhost:9092/metrics | grep "# HELP" ### Exporter Env Var Docs Env vars can be sepcified in a `.env` file, while using the `just` -| Var | Default Value | Purpose | -|---------------|---------------|-----------------------------------------------------------------------------| -| POLL_LIMIT | 10 | # of seconds to wait before polling slurmctl again (client-side throttling) | -| LOGLEVEL | info | Log Level: debug, info, warn, error | -| CLI_TIMEOUT | 10. | # seconds before the exporter terminates command. | +| Var | Default Value | Purpose | +|-----------------|---------------|-----------------------------------------------------------------------------| +| POLL_LIMIT | 10 | # of seconds to wait before polling slurmctl again (client-side throttling) | +| LOGLEVEL | info | Log Level: debug, info, warn, error | +| CLI_TIMEOUT | 10. | # seconds before the exporter terminates command. | +| TRACE_ROOT_PATH | "cwd" | path to ./templates directory where html files are located | ### RPM/DEB Packages @@ -189,4 +190,4 @@ Environment="LOGLEVEL=debug" ### Future work -Add scheduler info, slurmrestd support, package binary into docker +slurmrestd support diff --git a/exporter/trace.go b/exporter/trace.go index 642acaa..227b8b3 100644 --- a/exporter/trace.go +++ b/exporter/trace.go @@ -11,6 +11,7 @@ import ( "io/fs" "log" "net/http" + "os" "path/filepath" "sync" "text/template" @@ -106,8 +107,13 @@ type TraceCollector struct { func NewTraceCollector(config *Config) *TraceCollector { traceConfig := config.TraceConf + templateRootDir := "." + // path to look for the /templates directory. Defaults to cwd + if path, ok := os.LookupEnv("TRACE_ROOT_PATH"); ok { + templateRootDir = path + } traceDir := "" - err := filepath.WalkDir(".", func(path string, d fs.DirEntry, err error) error { + err := filepath.WalkDir(templateRootDir, func(path string, d fs.DirEntry, err error) error { if err == nil && d.IsDir() && d.Name() == templateDirName { traceDir = path return nil From 09270ee14426f960f700ded1e5714ed87051d635 Mon Sep 17 00:00:00 2001 From: Drew Stinnett Date: Tue, 3 Sep 2024 09:09:22 -0400 Subject: [PATCH 15/15] Adding in trace path detection logic --- exporter/trace.go | 26 ++++++++++++++++++++++++-- exporter/trace_test.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/exporter/trace.go b/exporter/trace.go index 227b8b3..9e74852 100644 --- a/exporter/trace.go +++ b/exporter/trace.go @@ -22,8 +22,10 @@ import ( ) // cleanup on add if greater than this threshold -const cleanupThreshold uint64 = 1_000 -const templateDirName string = "templates" +const ( + cleanupThreshold uint64 = 1_000 + templateDirName string = "templates" +) // store a jobs published proc stats type TraceInfo struct { @@ -200,3 +202,23 @@ func (c *TraceCollector) uploadTrace(w http.ResponseWriter, r *http.Request) { } } } + +// detectTracePath returns the trace_root path based on the following criteria: +// 1. If TRACE_ROOT_PATH is specified, search that directory. If we don't find a templates dir, let's panic and crash the program. +// 2. If TRACE_ROOT_PATH isn't specified, we can search cwd and /usr/share/prometheus-slurm-exporter. +func detectTracePath() string { + templateRootDir := "" + if path, ok := os.LookupEnv("TRACE_ROOT_PATH"); ok { + templateRootDir = path + if _, err := os.Stat(filepath.Join(templateRootDir, templateDirName)); err != nil { + panic("TRACE_ROOT_PATH must include a directory called: " + templateDirName) + } + return templateRootDir + } + for _, p := range []string{".", "/usr/share/prometheus-slurm-exporter"} { + if _, err := os.Stat(filepath.Join(p, templateDirName)); err == nil { + return p + } + } + return "" +} diff --git a/exporter/trace_test.go b/exporter/trace_test.go index c60e577..8244e03 100644 --- a/exporter/trace_test.go +++ b/exporter/trace_test.go @@ -10,11 +10,13 @@ import ( "net/http" "net/http/httptest" "os" + "path/filepath" "testing" "time" "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestAtomicFetcher_Cleanup(t *testing.T) { @@ -26,6 +28,7 @@ func TestAtomicFetcher_Cleanup(t *testing.T) { fetcher.cleanup() assert.Contains(fetcher.Info, int64(10)) } + func TestAtomicFetcher_Add(t *testing.T) { assert := assert.New(t) fetcher := NewAtomicProFetcher(10) @@ -202,3 +205,28 @@ func TestPython3Wrapper(t *testing.T) { json.Unmarshal(wrapperOut, &info) assert.Equal(int64(10), info.JobId) } + +func TestDetectTraceRootPath_Env(t *testing.T) { + os.Clearenv() + testDir := t.TempDir() + t.Setenv("TRACE_ROOT_PATH", testDir) + // Ensure that the function panics if given a TRACE_ROOT_PATh with no 'templates' subdirectory + assert.PanicsWithValue(t, "TRACE_ROOT_PATH must include a directory called: templates", func() { detectTracePath() }) + require.NoError(t, os.Mkdir(filepath.Join(testDir, templateDirName), 0o700)) + + // Now that we have a 'templates' subdir, it should no longer panic + assert.Equal(t, testDir, detectTracePath()) +} + +func TestDetectTraceRootPath_Default(t *testing.T) { + os.Clearenv() + testDir := t.TempDir() + os.Chdir(testDir) + + // Should come back empty if since we don't yet have a 'templates' subdir + assert.Equal(t, detectTracePath(), "") + require.NoError(t, os.Mkdir(filepath.Join(testDir, templateDirName), 0o700)) + + // Now that we have 'templates' subdir, cwd is a valid path + assert.Equal(t, detectTracePath(), ".") +}