Adding CSV support

README.md

@@ -3,8 +3,7 @@
 hperf is a tool for active measurements of the maximum achievable bandwidth between N peers, measuring RX/TX bandwidth for each peers.
 
 ## What is hperf for
-Hperf was made to test networks in large infrastructure. It's highly scalable and cabaple of running parallel tests over
-a long period of time. 
+Hperf was made to test networks in large infrastructure. It's highly scalable and cabaple of running parallel tests over a long period of time. 
 
 ## Common use cases
 - Debugging link/nic MTU issues
@@ -19,18 +18,19 @@ a long period of time.
 The binary can act as both client and server.
 
 ### Client
-The client part of hperf is responsible for orchestrating the servers. Its only job is to send commands to the
-servers and receive incremental stats updates. It can be executed from any machine that can talk to the servers.
+The client part of hperf is responsible for orchestrating the servers. Its only job is to send commands to the servers and receive incremental stats updates. It can be executed from any machine that can talk to the servers.
 
 ### Servers
-Servers are the machines we are testing. To launch the hperf command in servers mode, simply use the `server` command:
-NOTE: `server` is the only command you can execute on the servers. All other commands are executed from the client.
+Servers are the machines we are testing. To launch the hperf command in servers mode use the `server` command:
+
 ```bash
 $ ./hperf server --help
 ```
+
 This command will start an API and websocket on the given `--address` and save test results to `--storage-path`. 
 
 WARNING: do not expose `--address` to the internet
+<b>NOTE: if the `--address` is not the same as your external IP addres used for communications between servers then you need to set `--real-ip`, otherwise the server will report internal IPs in the stats and it will run the test against itself, causing invalid results.</b>
 
 ### The listen command
 Hperf can run tests without a specific `client` needing to be constantly connected. Once the `client` has started a test, the `client` can 
@@ -57,13 +57,14 @@ go install github.com/minio/hperf/cmd/hperf@latest
 
 ### Server
 Run server with default settings:
-NOTE: this will place all test result files in the same directory.
+NOTE: this will place all test result files in the same directory and use 0.0.0.0 as bind ip. We do not recommend this for larger tests. 
 ```bash
 $ ./hperf server
 ```
-Run the server with custom `--address` and `--storage-path`
+
+Run the server with custom `--address`, `--real-ip` and `--storage-path`
 ```bash
-$ ./hperf server --address 10.10.2.10:5000 --storage-path /tmp/hperf/
+$ ./hperf server --address 10.10.2.10:5000 --real-ip 150.150.20.2 --storage-path /tmp/hperf/
 ```
 
 ### Client 
@@ -85,6 +86,12 @@ host per file line.
 NOTE: Be careful not to re-use the ID's if you care about fetching results at a later date.
 
 ```bash
+# listen in on a running test
+./hperf listen --hosts 1.1.1.{1...100} --id [my_test_id]
+
+# stop a running test
+./hperf stop --hosts 1.1.1.{1...100} --id [my_test_id]
+
 # download test results
 ./hperf download --hosts 1.1.1.{1...100} --id [my_test_id] --file /tmp/test.out
 
@@ -93,16 +100,14 @@ NOTE: Be careful not to re-use the ID's if you care about fetching results at a
 # analyze test results with full print output
 ./hperf analyze --file /tmp/test.out --print-stats --print-errors
 
-# listen in on a running test
-./hperf listen --hosts 1.1.1.{1...100} --id [my_test_id]
-
-# stop a running test
-./hperf stop --hosts 1.1.1.{1...100} --id [my_test_id]
+# Generate a .csv file from a .json test file
+./hperf csv --file /tmp/test.out
 ```
 
 ## Analysis
-The analyze command will print statistics for the 10th and 90th percentiles and all datapoints in between. 
-The format used is:
+The analyze command will print statistics for the 10th and 90th percentiles and all datapoints in between. Additionally, you can use the `--print-stats` and `--print-erros` flags for a more verbose output.
+
+The analysis will show:
  - 10th percentile: total, low, avarage, high
  - in between: total, low, avarage, high
  - 90th percentile: total, low, avarage, high
@@ -144,4 +149,23 @@ $ ./hperf latency --hosts file:./hosts --id http-test-2 --duration 360 --concurr
 --bufferSize 1000 --payloadSize 1000
 ```
 
+# Full test scenario with analysis and csv export
+## On the server
+```bash
+$ ./hperf server --address 10.10.2.10:5000 --real-ip 150.150.20.2 --storage-path /tmp/hperf/
+```
+
+## The client
+
+
+
+
+
+
+
+
+
+
+
+
 

client/client.go

@@ -21,12 +21,14 @@ import (
 	"bufio"
 	"bytes"
 	"context"
+	"encoding/csv"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"math"
 	"net/http"
 	"os"
+	"reflect"
 	"runtime/debug"
 	"slices"
 	"strconv"
@@ -662,3 +664,65 @@ func updateBracketStats(b []int64, dp shared.DP) {
 		b[4] = dp.RMSH
 	}
 }
+
+func MakeCSV(ctx context.Context, c shared.Config) (err error) {
+	byteValue, err := os.ReadFile(c.File)
+	if err != nil {
+		return err
+	}
+
+	file, err := os.Create(c.File + ".csv")
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	fb := bytes.NewBuffer(byteValue)
+	scanner := bufio.NewScanner(fb)
+
+	writer := csv.NewWriter(file)
+	defer writer.Flush()
+	if err := writer.Write(getStructFields(new(shared.DP))); err != nil {
+		return err
+	}
+
+	for scanner.Scan() {
+		b := scanner.Bytes()
+		if bytes.Contains(b, []byte("Error")) {
+			continue
+		}
+		dp := new(shared.DP)
+		err = json.Unmarshal(b, dp)
+		if err != nil {
+			return err
+		}
+
+		if err := writer.Write(dpToSlice(dp)); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// Function to get field names of the struct
+func getStructFields(s interface{}) []string {
+	t := reflect.TypeOf(s).Elem()
+	fields := make([]string, t.NumField())
+	for i := 0; i < t.NumField(); i++ {
+		fields[i] = t.Field(i).Tag.Get("json")
+		if fields[i] == "" {
+			fields[i] = t.Field(i).Name
+		}
+	}
+	return fields
+}
+
+func dpToSlice(dp *shared.DP) (data []string) {
+	v := reflect.ValueOf(dp).Elem()
+	data = make([]string, v.NumField())
+	for i := 0; i < v.NumField(); i++ {
+		data[i] = fmt.Sprintf("%v", v.Field(i).Interface())
+	}
+	return
+}

cmd/hperf/csv.go

@@ -0,0 +1,54 @@
+// Copyright (c) 2015-2024 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"github.com/minio/cli"
+	"github.com/minio/hperf/client"
+)
+
+var csvCMD = cli.Command{
+	Name:   "csv",
+	Usage:  "Transform a test file to csv file",
+	Action: runCSV,
+	Flags: []cli.Flag{
+		fileFlag,
+	},
+	CustomHelpTemplate: `NAME:
+  {{.HelpName}} - {{.Usage}}
+
+USAGE:
+  {{.HelpName}} [FLAGS]
+
+FLAGS:
+  {{range .VisibleFlags}}{{.}}
+  {{end}}
+EXAMPLES:
+  1. Transform a test file to csv file:
+    {{.Prompt}} {{.HelpName}} --file /tmp/output-file
+`,
+}
+
+func runCSV(ctx *cli.Context) error {
+	config, err := parseConfig(ctx)
+	if err != nil {
+		return err
+	}
+
+	return client.MakeCSV(GlobalContext, *config)
+}

cmd/hperf/main.go

@@ -169,6 +169,7 @@ var (
 	Commands = []cli.Command{
 		analyzeCMD,
 		bandwidthCMD,
+		csvCMD,
 		deleteCMD,
 		latencyCMD,
 		listenCMD,