From 4c1c202f96320b09e8a866f4886a4da3b32a7a52 Mon Sep 17 00:00:00 2001 From: Nathaniel Waisbrot Date: Sun, 31 Dec 2023 12:44:34 -0500 Subject: [PATCH] add a command for collecting regular metrics --- birdweather/forStation.go | 30 ++++--- birdweather/generated.go | 146 ++++++++++++++++++++++++++++++++++ birdweather/genqlient.graphql | 15 ++++ birdweather/genqlient.yaml | 4 + cmd/hourlyMetrics.go | 39 +++++++++ cmd/send.go | 7 +- cmd/templatetest.go | 8 +- email/template.go | 16 +--- go.sum | 2 + main.go | 2 +- metrics/influx.go | 21 ++++- metrics/root.go | 10 ++- structs/root.go | 22 +++++ 13 files changed, 285 insertions(+), 37 deletions(-) create mode 100644 cmd/hourlyMetrics.go create mode 100644 structs/root.go diff --git a/birdweather/forStation.go b/birdweather/forStation.go index 92e5db3..9a80bde 100644 --- a/birdweather/forStation.go +++ b/birdweather/forStation.go @@ -6,26 +6,19 @@ import ( "github.com/Khan/genqlient/graphql" "github.com/waisbrot/birdweather_daily_email/metrics" + "github.com/waisbrot/birdweather_daily_email/structs" ) -type BirdCount struct { - Name string - SciName string - ImageURL string - ImageCredit string - Count int -} - -func BirdsForStation(stationid string) (string, []BirdCount) { +func BirdsForStation(stationid string) (string, []structs.BirdCount) { client := graphql.NewClient("https://app.birdweather.com/graphql", http.DefaultClient) counts, err := dailyCounts(context.Background(), client, stationid) if err != nil { panic(err) } - var result = []BirdCount{} + var result = []structs.BirdCount{} for _, count := range counts.Station.TopSpecies { - var bc BirdCount + var bc structs.BirdCount bc.Name = count.Species.CommonName bc.SciName = count.Species.ScientificName bc.ImageURL = count.Species.ImageUrl @@ -37,3 +30,18 @@ func BirdsForStation(stationid string) (string, []BirdCount) { metrics.RecordFetch(counts.Station.Name, len(result)) return counts.Station.Name, result } + +func RecordCountsForStationPastMinutes(stationId string, minutes int) { + client := graphql.NewClient("https://app.birdweather.com/graphql", http.DefaultClient) + duration := InputDuration{ + Count: minutes, + Unit: "minute", + } + counts, err := hourlyCounts(context.Background(), client, stationId, duration) + if err != nil { + panic(err) + } + for _, count := range counts.Station.TopSpecies { + metrics.RecordBird(counts.Station.Name, count.Species.CommonName, count.Breakdown.AlmostCertain) + } +} diff --git a/birdweather/generated.go b/birdweather/generated.go index e9a9e8e..0399d63 100644 --- a/birdweather/generated.go +++ b/birdweather/generated.go @@ -4,10 +4,35 @@ package birdweather import ( "context" + "time" "github.com/Khan/genqlient/graphql" ) +// A time period (e.g. last 24 hours) or explicit date duration. +type InputDuration struct { + // Number of units of time + Count int `json:"count"` + // Unit of time (hour/day/week/month/year) + Unit string `json:"unit"` + // From date + From time.Time `json:"from"` + // To date + To time.Time `json:"to"` +} + +// GetCount returns InputDuration.Count, and is useful for accessing the field via an interface. +func (v *InputDuration) GetCount() int { return v.Count } + +// GetUnit returns InputDuration.Unit, and is useful for accessing the field via an interface. +func (v *InputDuration) GetUnit() string { return v.Unit } + +// GetFrom returns InputDuration.From, and is useful for accessing the field via an interface. +func (v *InputDuration) GetFrom() time.Time { return v.From } + +// GetTo returns InputDuration.To, and is useful for accessing the field via an interface. +func (v *InputDuration) GetTo() time.Time { return v.To } + // __dailyCountsInput is used internally by genqlient type __dailyCountsInput struct { StationId string `json:"stationId"` @@ -16,6 +41,18 @@ type __dailyCountsInput struct { // GetStationId returns __dailyCountsInput.StationId, and is useful for accessing the field via an interface. func (v *__dailyCountsInput) GetStationId() string { return v.StationId } +// __hourlyCountsInput is used internally by genqlient +type __hourlyCountsInput struct { + StationId string `json:"stationId"` + TimePeriod InputDuration `json:"timePeriod"` +} + +// GetStationId returns __hourlyCountsInput.StationId, and is useful for accessing the field via an interface. +func (v *__hourlyCountsInput) GetStationId() string { return v.StationId } + +// GetTimePeriod returns __hourlyCountsInput.TimePeriod, and is useful for accessing the field via an interface. +func (v *__hourlyCountsInput) GetTimePeriod() InputDuration { return v.TimePeriod } + // dailyCountsResponse is returned by dailyCounts on success. type dailyCountsResponse struct { Station dailyCountsStation `json:"station"` @@ -112,6 +149,70 @@ func (v *dailyCountsStationTopSpeciesSpeciesCountSpecies) GetImageCredit() strin // GetImageUrl returns dailyCountsStationTopSpeciesSpeciesCountSpecies.ImageUrl, and is useful for accessing the field via an interface. func (v *dailyCountsStationTopSpeciesSpeciesCountSpecies) GetImageUrl() string { return v.ImageUrl } +// hourlyCountsResponse is returned by hourlyCounts on success. +type hourlyCountsResponse struct { + Station hourlyCountsStation `json:"station"` +} + +// GetStation returns hourlyCountsResponse.Station, and is useful for accessing the field via an interface. +func (v *hourlyCountsResponse) GetStation() hourlyCountsStation { return v.Station } + +// hourlyCountsStation includes the requested fields of the GraphQL type Station. +// The GraphQL type's documentation follows. +// +// A BirdWeather station (either real or virtual). +type hourlyCountsStation struct { + // Station name + Name string `json:"name"` + TopSpecies []hourlyCountsStationTopSpeciesSpeciesCount `json:"topSpecies"` +} + +// GetName returns hourlyCountsStation.Name, and is useful for accessing the field via an interface. +func (v *hourlyCountsStation) GetName() string { return v.Name } + +// GetTopSpecies returns hourlyCountsStation.TopSpecies, and is useful for accessing the field via an interface. +func (v *hourlyCountsStation) GetTopSpecies() []hourlyCountsStationTopSpeciesSpeciesCount { + return v.TopSpecies +} + +// hourlyCountsStationTopSpeciesSpeciesCount includes the requested fields of the GraphQL type SpeciesCount. +type hourlyCountsStationTopSpeciesSpeciesCount struct { + Breakdown hourlyCountsStationTopSpeciesSpeciesCountBreakdown `json:"breakdown"` + Species hourlyCountsStationTopSpeciesSpeciesCountSpecies `json:"species"` +} + +// GetBreakdown returns hourlyCountsStationTopSpeciesSpeciesCount.Breakdown, and is useful for accessing the field via an interface. +func (v *hourlyCountsStationTopSpeciesSpeciesCount) GetBreakdown() hourlyCountsStationTopSpeciesSpeciesCountBreakdown { + return v.Breakdown +} + +// GetSpecies returns hourlyCountsStationTopSpeciesSpeciesCount.Species, and is useful for accessing the field via an interface. +func (v *hourlyCountsStationTopSpeciesSpeciesCount) GetSpecies() hourlyCountsStationTopSpeciesSpeciesCountSpecies { + return v.Species +} + +// hourlyCountsStationTopSpeciesSpeciesCountBreakdown includes the requested fields of the GraphQL type SpeciesCountBreakdown. +type hourlyCountsStationTopSpeciesSpeciesCountBreakdown struct { + // Count of almost certain detections + AlmostCertain int `json:"almostCertain"` +} + +// GetAlmostCertain returns hourlyCountsStationTopSpeciesSpeciesCountBreakdown.AlmostCertain, and is useful for accessing the field via an interface. +func (v *hourlyCountsStationTopSpeciesSpeciesCountBreakdown) GetAlmostCertain() int { + return v.AlmostCertain +} + +// hourlyCountsStationTopSpeciesSpeciesCountSpecies includes the requested fields of the GraphQL type Species. +type hourlyCountsStationTopSpeciesSpeciesCountSpecies struct { + // Common name + CommonName string `json:"commonName"` +} + +// GetCommonName returns hourlyCountsStationTopSpeciesSpeciesCountSpecies.CommonName, and is useful for accessing the field via an interface. +func (v *hourlyCountsStationTopSpeciesSpeciesCountSpecies) GetCommonName() string { + return v.CommonName +} + // The query or mutation executed by dailyCounts. const dailyCounts_Operation = ` query dailyCounts ($stationId: ID!) { @@ -160,3 +261,48 @@ func dailyCounts( return &data, err } + +// The query or mutation executed by hourlyCounts. +const hourlyCounts_Operation = ` +query hourlyCounts ($stationId: ID!, $timePeriod: InputDuration!) { + station(id: $stationId) { + name + topSpecies(limit: 20, period: $timePeriod) { + breakdown { + almostCertain + } + species { + commonName + } + } + } +} +` + +func hourlyCounts( + ctx context.Context, + client graphql.Client, + stationId string, + timePeriod InputDuration, +) (*hourlyCountsResponse, error) { + req := &graphql.Request{ + OpName: "hourlyCounts", + Query: hourlyCounts_Operation, + Variables: &__hourlyCountsInput{ + StationId: stationId, + TimePeriod: timePeriod, + }, + } + var err error + + var data hourlyCountsResponse + resp := &graphql.Response{Data: &data} + + err = client.MakeRequest( + ctx, + req, + resp, + ) + + return &data, err +} diff --git a/birdweather/genqlient.graphql b/birdweather/genqlient.graphql index 8654c6b..272e123 100644 --- a/birdweather/genqlient.graphql +++ b/birdweather/genqlient.graphql @@ -17,3 +17,18 @@ query dailyCounts($stationId: ID!) { } } } + +query hourlyCounts($stationId: ID!, $timePeriod: InputDuration!) { + station(id: $stationId) { + name + topSpecies(limit: 20, period: $timePeriod) { + breakdown { + almostCertain + } + species { + commonName + } + } + } +} + diff --git a/birdweather/genqlient.yaml b/birdweather/genqlient.yaml index 6df07d6..09f390d 100644 --- a/birdweather/genqlient.yaml +++ b/birdweather/genqlient.yaml @@ -5,3 +5,7 @@ operations: - genqlient.graphql generated: generated.go package: birdweather + +bindings: + ISO8601Date: + type: time.Time \ No newline at end of file diff --git a/cmd/hourlyMetrics.go b/cmd/hourlyMetrics.go new file mode 100644 index 0000000..59e3bad --- /dev/null +++ b/cmd/hourlyMetrics.go @@ -0,0 +1,39 @@ +/* +Copyright © 2023 NAME HERE +*/ +package cmd + +import ( + "fmt" + + "github.com/spf13/cobra" + "github.com/spf13/viper" + "github.com/waisbrot/birdweather_daily_email/birdweather" +) + +// hourlyMetricsCmd represents the hourlyMetrics command +var hourlyMetricsCmd = &cobra.Command{ + Use: "hourlyMetrics", + Short: "Fetch and push bird metrics", + Long: ``, + Run: func(cmd *cobra.Command, args []string) { + stationIds := viper.GetIntSlice("stations") + for _, stationId := range stationIds { + birdweather.RecordCountsForStationPastMinutes(fmt.Sprint(stationId), 30) + } + }, +} + +func init() { + rootCmd.AddCommand(hourlyMetricsCmd) + + // Here you will define your flags and configuration settings. + + // Cobra supports Persistent Flags which will work for this command + // and all subcommands, e.g.: + // hourlyMetricsCmd.PersistentFlags().String("foo", "", "A help for foo") + + // Cobra supports local flags which will only run when this command + // is called directly, e.g.: + // hourlyMetricsCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") +} diff --git a/cmd/send.go b/cmd/send.go index 7a52d0b..e11b06b 100644 --- a/cmd/send.go +++ b/cmd/send.go @@ -11,6 +11,7 @@ import ( "github.com/spf13/viper" "github.com/waisbrot/birdweather_daily_email/birdweather" "github.com/waisbrot/birdweather_daily_email/email" + "github.com/waisbrot/birdweather_daily_email/structs" ) // sendCmd represents the send command @@ -24,11 +25,11 @@ Cobra is a CLI library for Go that empowers applications. This application is a tool to generate the needed files to quickly create a Cobra application.`, Run: func(cmd *cobra.Command, args []string) { - stations := []email.StationTemplate{} + stations := []structs.StationTemplate{} stationIds := viper.GetIntSlice("stations") for _, stationId := range stationIds { stationName, counts := birdweather.BirdsForStation(fmt.Sprint(stationId)) - templ := email.StationTemplate{} + templ := structs.StationTemplate{} templ.Name = stationName templ.Id = stationId templ.Counts = counts @@ -41,7 +42,7 @@ to quickly create a Cobra application.`, } yesterday := time.Now() yesterday = yesterday.Add(time.Hour * -24) - templ := email.EmailTemplate{ + templ := structs.EmailTemplate{ Day: yesterday.Weekday(), Stations: stations, } diff --git a/cmd/templatetest.go b/cmd/templatetest.go index a20b0ce..cb2aebb 100644 --- a/cmd/templatetest.go +++ b/cmd/templatetest.go @@ -9,8 +9,8 @@ import ( "time" "github.com/spf13/cobra" - "github.com/waisbrot/birdweather_daily_email/birdweather" "github.com/waisbrot/birdweather_daily_email/email" + "github.com/waisbrot/birdweather_daily_email/structs" ) // templatetestCmd represents the templatetest command @@ -25,13 +25,13 @@ This application is a tool to generate the needed files to quickly create a Cobra application.`, Run: func(cmd *cobra.Command, args []string) { fmt.Println("templatetest called") - templ := email.EmailTemplate{ + templ := structs.EmailTemplate{ Day: time.Sunday, - Stations: []email.StationTemplate{ + Stations: []structs.StationTemplate{ { Name: "Example station", Id: 42, - Counts: []birdweather.BirdCount{ + Counts: []structs.BirdCount{ { Name: "Chickadee", SciName: "Dee-dee-dee", diff --git a/email/template.go b/email/template.go index e1e4478..18bb15a 100644 --- a/email/template.go +++ b/email/template.go @@ -5,23 +5,11 @@ import ( "html/template" "io" "path" - "time" "github.com/spf13/viper" - "github.com/waisbrot/birdweather_daily_email/birdweather" + "github.com/waisbrot/birdweather_daily_email/structs" ) -type EmailTemplate struct { - Day time.Weekday - Stations []StationTemplate -} - -type StationTemplate struct { - Name string - Id int - Counts []birdweather.BirdCount -} - func readTemplate() *template.Template { templateFile := viper.GetString("email.template") base := path.Base(templateFile) @@ -32,7 +20,7 @@ func readTemplate() *template.Template { return template } -func RenderTemplate(variables EmailTemplate) io.Reader { +func RenderTemplate(variables structs.EmailTemplate) io.Reader { template := readTemplate() buffer := new(bytes.Buffer) err := template.Execute(buffer, variables) diff --git a/go.sum b/go.sum index 7c5e68e..48921da 100644 --- a/go.sum +++ b/go.sum @@ -468,6 +468,7 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -682,6 +683,7 @@ golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y= golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= +golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/main.go b/main.go index 5e17826..a3dce30 100644 --- a/main.go +++ b/main.go @@ -23,7 +23,7 @@ func main() { fmt.Fprintf(os.Stderr, "Error reading config file `%s`: %v\n", configFile, err) os.Exit(1) } - metrics.Init() + metrics.Init(os.Args[1:]) metrics.RecordInvoked() cmd.Execute() metrics.Finish() diff --git a/metrics/influx.go b/metrics/influx.go index e91928c..dc77836 100644 --- a/metrics/influx.go +++ b/metrics/influx.go @@ -14,8 +14,16 @@ var influxClient influxdb.Client var writeAPI influxapi.WriteAPIBlocking var invokeTime time.Time -func initInflux() { - influxdb.DefaultOptions().AddDefaultTag("application", "birdweather_digest") +func initInflux(args []string) { + var command string + if len(args) > 0 { + command = args[0] + } else { + command = "" + } + influxdb.DefaultOptions(). + AddDefaultTag("application", "birdweather_digest"). + AddDefaultTag("command", command) influxClient = influxdb.NewClient(viper.GetString("influx.url"), viper.GetString("influx.token")) writeAPI = influxClient.WriteAPIBlocking(viper.GetString("influx.org"), viper.GetString("influx.bucket")) produceMetrics = true @@ -56,3 +64,12 @@ func recordInfluxEmail(recipientCount int, bodyLength int) { SetTime(time.Now()) writePoint(p) } + +func recordInfluxBird(stationName string, birdName string, count int) { + p := influxdb.NewPointWithMeasurement("bird"). + AddTag("station", stationName). + AddTag("name", birdName). + AddField("count", count). + SetTime(time.Now()) + writePoint(p) +} diff --git a/metrics/root.go b/metrics/root.go index e54beeb..7a8ccf3 100644 --- a/metrics/root.go +++ b/metrics/root.go @@ -9,9 +9,9 @@ import ( var produceMetrics bool = false -func Init() { +func Init(args []string) { if viper.IsSet("influx.url") { - initInflux() + initInflux(args) } else { fmt.Fprintf(os.Stderr, "No influx.url defined. Skipping metrics production.") } @@ -40,3 +40,9 @@ func RecordEmail(recipientCount int, bodyLength int) { recordInfluxEmail(recipientCount, bodyLength) } } + +func RecordBird(stationName string, birdName string, count int) { + if produceMetrics { + recordInfluxBird(stationName, birdName, count) + } +} diff --git a/structs/root.go b/structs/root.go new file mode 100644 index 0000000..29da938 --- /dev/null +++ b/structs/root.go @@ -0,0 +1,22 @@ +package structs + +import "time" + +type BirdCount struct { + Name string + SciName string + ImageURL string + ImageCredit string + Count int +} + +type EmailTemplate struct { + Day time.Weekday + Stations []StationTemplate +} + +type StationTemplate struct { + Name string + Id int + Counts []BirdCount +}