From bcefc0d8891cd0aee3c8639e974c3d3134186623 Mon Sep 17 00:00:00 2001 From: Dmitri Goutnik Date: Mon, 18 Jul 2022 08:29:29 -0500 Subject: [PATCH] grep: add -s/-e options to limit searched logs date range --- README.md | 11 ++++++- cache/cache.go | 4 +++ cache/directory.go | 9 ++++-- clean.go | 17 ++++++----- fetch.go | 19 ++++++------- fetch/maillist.go | 8 +++--- grep.go | 71 +++++++++++++++++++++++++++++----------------- main.go | 20 +++++++++++++ stats.go | 14 ++++----- 9 files changed, 112 insertions(+), 61 deletions(-) diff --git a/README.md b/README.md index b3bb1c8..bf8a997 100644 --- a/README.md +++ b/README.md @@ -24,8 +24,11 @@ Commands (pass -h for command help): fetch download fallout logs grep search fallout logs clean clean log cache + stats show cache statistics ``` +##### Fetching failure logs: + ``` usage: fallout fetch [-h] [-D days] [-A date] [-N count] [-b builder[,builder]] [-c category[,category]] [-o origin[,origin]] [-n name[,name]] @@ -42,8 +45,10 @@ Options: -n name,... download only logs for these port names ``` +###### Searching: + ``` -usage: fallout grep [-hFOl] [-A count] [-B count] [-C count] [-b builder[,builder]] [-c category[,category]] [-o origin[,origin]] [-n name[,name]] query [query ...] +usage: fallout grep [-hFOl] [-A count] [-B count] [-C count] [-b builder[,builder]] [-c category[,category]] [-o origin[,origin]] [-n name[,name]] [-s since] [-e before] [-j jobs] query [query ...] Search cached fallout logs. @@ -59,9 +64,13 @@ Options: -c category,... limit search only to these categories -o origin,... limit search only to these origins -n name,... limit search only to these port names + -s since list only failures since this date or date-time, in RFC-3339 format + -e before list only failures before this date or date-time, in RFC-3339 format -j jobs number of parallel jobs, -j1 outputs sorted results (default: 8) ``` +##### Cleaning the cache: + ``` usage: fallout clean [-hx] [-D days] [-A date] diff --git a/cache/cache.go b/cache/cache.go index 8764685..109bbe6 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -62,6 +62,10 @@ type Filter struct { Origins []string // Allowed port names, partial names are ok. Names []string + // Allow logs only since this timestamp. + Since time.Time + // Allow logs only before this timestamp. + Before time.Time } // Walker is the cache walker interface. diff --git a/cache/directory.go b/cache/directory.go index 2382abd..cbb470b 100644 --- a/cache/directory.go +++ b/cache/directory.go @@ -292,6 +292,9 @@ func (w *DirectoryWalker) walkOrigin(builder, origin string, rch chan Entry, ech ech <- err continue } + if ts.Before(w.filter.Since) || !w.filter.Before.IsZero() && ts.After(w.filter.Before) { + continue + } e, err := newEntry(w.cache, builder, origin, ts) if err != nil { ech <- err @@ -311,15 +314,15 @@ func loadTimestamp(path string) time.Time { var zero time.Time if buf, err := os.ReadFile(filepath.Join(path, cacheTimestampName)); err == nil { if ts, err := time.Parse(cacheTimestampFormat, string(buf)); err == nil { - return ts.UTC() + return ts } } return zero } func (c *Directory) updateTimestamp(ts time.Time) { - if ts.UTC().After(c.timestamp) { - c.timestamp = ts.UTC() + if ts.After(c.timestamp) { + c.timestamp = ts _ = os.WriteFile(filepath.Join(c.path, cacheTimestampName), []byte(ts.Format(cacheTimestampFormat)), 0664) } } diff --git a/clean.go b/clean.go index 0f47f90..0d79e4e 100644 --- a/clean.go +++ b/clean.go @@ -32,7 +32,7 @@ const defaultCleanDaysLimit = 30 var ( cleanDateLimit = time.Now().UTC().AddDate(0, 0, -defaultCleanDaysLimit) - allClean bool + cleanAll bool ) func showCleanUsage() { @@ -64,22 +64,21 @@ func runClean(args []string) int { showCleanUsage() os.Exit(0) case 'x': - allClean = true + cleanAll = true case 'D': v, err := opt.Int() if err != nil { - errExit(err.Error()) + errExit("-D: %s", err) } cleanDateLimit = time.Now().UTC().AddDate(0, 0, -v) case 'A': - t, err := time.Parse(dateFormat, opt.String()) + t, err := parseDateTime(opt.String()) if err != nil { - errExit(err.Error()) - } - if t.After(time.Now().UTC()) { - errExit("date in the future: %s", t.Format(dateFormat)) + errExit("-A: %s", err) } cleanDateLimit = t + default: + panic("unhandled option: -" + string(opt.Opt)) } } @@ -88,7 +87,7 @@ func runClean(args []string) int { errExit("error initializing cache: %s", err) } - if allClean { + if cleanAll { fmt.Printf("Removing %s\n", c.Path()) if err := c.Remove(); err != nil { errExit("error removing cache: %s", err) diff --git a/fetch.go b/fetch.go index 8fadc8b..67c230c 100644 --- a/fetch.go +++ b/fetch.go @@ -41,7 +41,7 @@ const defaultFetchDaysLimit = 7 var ( fetchCountLimit int fetchDateLimit = time.Now().UTC().AddDate(0, 0, -defaultFetchDaysLimit) - onlyNew = true + fetchOnlyNew = true ) func showFetchUsage() { @@ -75,20 +75,17 @@ func runFetch(args []string) int { case 'D': v, err := opt.Int() if err != nil { - errExit(err.Error()) + errExit("-D: %s", err) } fetchDateLimit = time.Now().UTC().AddDate(0, 0, -v) - onlyNew = false + fetchOnlyNew = false case 'A': - t, err := time.Parse(dateFormat, opt.String()) + t, err := parseDateTime(opt.String()) if err != nil { - errExit(err.Error()) - } - if t.After(time.Now().UTC()) { - errExit("date in the future: %s", t.Format(dateFormat)) + errExit("-A: %s", err) } fetchDateLimit = t - onlyNew = false + fetchOnlyNew = false case 'N': v, err := opt.Int() if err != nil { @@ -133,7 +130,7 @@ func runFetch(args []string) int { Origins: origins, Names: names, } - if onlyNew && !c.Timestamp().IsZero() { + if fetchOnlyNew && !c.Timestamp().IsZero() { fflt.After = c.Timestamp() } @@ -143,7 +140,7 @@ func runFetch(args []string) int { return false, err } if e.Exists() { - if !onlyNew { + if !fetchOnlyNew { fmt.Fprintf(os.Stdout, "%s (cached)\n", res) } return true, nil diff --git a/fetch/maillist.go b/fetch/maillist.go index 5692316..5a0b458 100644 --- a/fetch/maillist.go +++ b/fetch/maillist.go @@ -109,8 +109,8 @@ func (f *Maillist) fetchMaillist(ctx context.Context, qfn QueryFunc, rch chan *R ech <- err return } - mi := ts.UTC().Year()*100 + int(ts.UTC().Month()) - ma := f.filter.After.UTC().Year()*100 + int(f.filter.After.UTC().Month()) + mi := ts.Year()*100 + int(ts.Month()) + ma := f.filter.After.Year()*100 + int(f.filter.After.Month()) if mi < ma { cancel() // link is to the month before "After", stop return @@ -197,7 +197,7 @@ func (f *Maillist) fetchMaillist(ctx context.Context, qfn QueryFunc, rch chan *R ech <- err return } - if ts.UTC().Before(f.filter.After.UTC()) { + if ts.Before(f.filter.After) { return // timestamp is before "After", skip } @@ -213,7 +213,7 @@ func (f *Maillist) fetchMaillist(ctx context.Context, qfn QueryFunc, rch chan *R resMap[url] = &Result{ Builder: builder, Origin: origin, - Timestamp: ts.UTC(), + Timestamp: ts, URL: url, } } diff --git a/grep.go b/grep.go index b24d236..bbdcf85 100644 --- a/grep.go +++ b/grep.go @@ -7,6 +7,7 @@ import ( "io" "os" "runtime" + "time" "github.com/dmgk/fallout/cache" "github.com/dmgk/fallout/format" @@ -16,7 +17,7 @@ import ( ) var grepUsageTmpl = template.Must(template.New("usage-grep").Parse(` -usage: {{.progname}} grep [-hFOl] [-A count] [-B count] [-C count] [-b builder[,builder]] [-c category[,category]] [-o origin[,origin]] [-n name[,name]] query [query ...] +usage: {{.progname}} grep [-hFOl] [-A count] [-B count] [-C count] [-b builder[,builder]] [-c category[,category]] [-o origin[,origin]] [-n name[,name]] [-s since] [-e before] [-j jobs] query [query ...] Search cached fallout logs. @@ -32,6 +33,8 @@ Options: -c category,... limit search only to these categories -o origin,... limit search only to these origins -n name,... limit search only to these port names + -s since list only failures since this date or date-time, in RFC-3339 format + -e before list only failures before this date or date-time, in RFC-3339 format -j jobs number of parallel jobs, -j1 outputs sorted results (default: {{.maxJobs}}) `[1:])) @@ -42,18 +45,20 @@ var grepCmd = command{ } var ( - queryIsRegexp = true - ored bool - filenamesOnly bool - contextAfter int - contextBefore int - maxJobs = runtime.NumCPU() + grepQueryIsRegexp = true + grepOr bool + grepFilenamesOnly bool + grepContextAfter int + grepContextBefore int + grepSince time.Time + grepBefore time.Time + grepMaxJobs = runtime.NumCPU() ) func showGrepUsage() { err := grepUsageTmpl.Execute(os.Stdout, map[string]any{ "progname": progname, - "maxJobs": maxJobs, + "maxJobs": grepMaxJobs, }) if err != nil { panic(fmt.Sprintf("error executing template %s: %v", grepUsageTmpl.Name(), err)) @@ -61,7 +66,7 @@ func showGrepUsage() { } func runGrep(args []string) int { - opts, err := getopt.NewArgv("hFOlA:B:C:b:c:o:n:j:", argsWithDefaults(args, "FALLOUT_GREP_OPTS")) + opts, err := getopt.NewArgv("hFOlA:B:C:b:c:o:n:s:e:j:", argsWithDefaults(args, "FALLOUT_GREP_OPTS")) if err != nil { panic(fmt.Sprintf("error creating options parser: %s", err)) } @@ -77,30 +82,30 @@ func runGrep(args []string) int { showGrepUsage() os.Exit(0) case 'F': - queryIsRegexp = false + grepQueryIsRegexp = false case 'O': - ored = true + grepOr = true case 'l': - filenamesOnly = true + grepFilenamesOnly = true case 'A': v, err := opt.Int() if err != nil { - errExit(err.Error()) + errExit("-A: %s", err) } - contextAfter = v + grepContextAfter = v case 'B': v, err := opt.Int() if err != nil { - errExit(err.Error()) + errExit("-B: %s", err) } - contextBefore = v + grepContextBefore = v case 'C': v, err := opt.Int() if err != nil { - errExit(err.Error()) + errExit("-C: %s", err) } - contextBefore = v - contextAfter = v + grepContextBefore = v + grepContextAfter = v case 'b': builders = splitOptions(opt.String()) case 'c': @@ -109,6 +114,18 @@ func runGrep(args []string) int { origins = splitOptions(opt.String()) case 'n': names = splitOptions(opt.String()) + case 's': + t, err := parseDateTime(opt.String()) + if err != nil { + errExit("-s: %s", err) + } + grepSince = t + case 'e': + t, err := parseDateTime(opt.String()) + if err != nil { + errExit("-e: %s", err) + } + grepBefore = t case 'j': v, err := opt.Int() if err != nil { @@ -117,7 +134,7 @@ func runGrep(args []string) int { if v <= 0 { v = 1 } - maxJobs = v + grepMaxJobs = v default: panic("unhandled option: -" + string(opt.Opt)) } @@ -143,12 +160,14 @@ func runGrep(args []string) int { Categories: categories, Origins: origins, Names: names, + Since: grepSince, + Before: grepBefore, } w := c.Walker(cflt) // list only log filenames if no queries were provided if len(opts.Args()) == 0 { - filenamesOnly = true + grepFilenamesOnly = true // no need to actually grep if no queries were provided and only filenames were requested // simple cache walk is enough and also will output results ordered by builder/origin/timestamp @@ -170,9 +189,9 @@ func runGrep(args []string) int { fm := initFormatter() gopt := &grep.Options{ - ContextAfter: contextAfter, - ContextBefore: contextBefore, - QueryIsRegexp: queryIsRegexp, + ContextAfter: grepContextAfter, + ContextBefore: grepContextBefore, + QueryIsRegexp: grepQueryIsRegexp, } gfn := func(entry cache.Entry, res []*grep.Match, err error) error { if err != nil { @@ -181,7 +200,7 @@ func runGrep(args []string) int { return fm.Format(entry, res) } - if err := g.Grep(gopt, opts.Args(), gfn, maxJobs); err != nil { + if err := g.Grep(gopt, opts.Args(), gfn, grepMaxJobs); err != nil { errExit("grep error: %s", err) return 1 } @@ -200,7 +219,7 @@ func initFormatter() format.Formatter { format.SetColors(colors) } } - if filenamesOnly { + if grepFilenamesOnly { flags |= format.FfilenamesOnly } diff --git a/main.go b/main.go index 5945c66..33b78bc 100644 --- a/main.go +++ b/main.go @@ -1,10 +1,12 @@ package main import ( + "errors" "fmt" "html/template" "os" "strings" + "time" "unicode" "github.com/dmgk/fallout/format" @@ -154,6 +156,24 @@ func splitOptions(s string) []string { }) } +func parseDateTime(s string) (time.Time, error) { + var zero time.Time + formats := []string{ + "2006-01-02", + "2006-01-02T15", + "2006-01-02T15:04", + "2006-01-02T15:04:05", + "2006-01-02T15:04:05Z", + time.RFC3339, + } + for _, f := range formats { + if ts, err := time.Parse(f, s); err == nil { + return ts, nil + } + } + return zero, errors.New("invalid date or datetime") +} + func formatSize(size int64) string { const suffixes = "KMG" if size < 1000 { diff --git a/stats.go b/stats.go index 06fe36e..7b628db 100644 --- a/stats.go +++ b/stats.go @@ -33,13 +33,13 @@ func showStatsUsage() { } var statsTmpl = template.Must(template.New("stats-output").Parse(` -Cache size: {{.logsSize}} -Latest log: {{.latestTimestamp}} -Earliest log: {{.earliestTimestamp}} -Builders: {{.buildersCount}} -Failing ports: {{.originsCount}} -Logs: {{.logsCount}} -Most failures: {{.topBuilderName}} ({{.topBuilderCount}} ports) +Cache size: {{.logsSize}} +Latest log: {{.latestTimestamp}} +Oldest log: {{.earliestTimestamp}} +Builders: {{.buildersCount}} +Failing ports: {{.originsCount}} +Logs: {{.logsCount}} +Most failures: {{.topBuilderName}} ({{.topBuilderCount}} ports) `[1:])) func runStats(args []string) int {