feat: add dry run + fix searchQueryIssues

Author: mmatur

cmd/run/command.go

@@ -9,6 +9,7 @@ import (
 const (
 	flagLogLevel                  = "log-level"
 	flagGitHubToken               = "github-token"
+	flagDryRun                    = "dry-run"
 	flagPluginURL                 = "plugin-url"
 	flagGithubSearchQueries       = "github-search-queries"
 	flagGithubSearchQueriesIssues = "github-search-queries-issues"
@@ -41,6 +42,12 @@ func Command() *cli.Command {
 				EnvVars:  []string{strcase.ToSNAKE(flagGitHubToken)},
 				Required: true,
 			},
+			&cli.BoolFlag{
+				Name:    flagDryRun,
+				Usage:   "Dry run mode.",
+				EnvVars: []string{strcase.ToSNAKE(flagDryRun)},
+				Value:   true,
+			},
 			&cli.StringFlag{
 				Name:     flagPluginURL,
 				Usage:    "Plugin Service URL",

cmd/run/config.go

@@ -10,6 +10,8 @@ type Config struct {
 	GithubToken string
 	PluginURL   string
 
+	DryRun bool
+
 	GithubSearchQueries       []string
 	GithubSearchQueriesIssues []string
 
@@ -20,8 +22,9 @@ func buildConfig(cliCtx *cli.Context) Config {
 	return Config{
 		GithubToken:               cliCtx.String(flagGitHubToken),
 		PluginURL:                 cliCtx.String(flagPluginURL),
+		DryRun:                    cliCtx.Bool(flagDryRun),
 		GithubSearchQueries:       cliCtx.StringSlice(flagGithubSearchQueries),
-		GithubSearchQueriesIssues: cliCtx.StringSlice(flagGithubSearchQueries),
+		GithubSearchQueriesIssues: cliCtx.StringSlice(flagGithubSearchQueriesIssues),
 		Tracing: tracer.Config{
 			Address:     cliCtx.String(flagTracingAddress),
 			Insecure:    cliCtx.Bool(flagTracingInsecure),

cmd/run/run.go

@@ -34,7 +34,7 @@ func run(ctx context.Context, cfg Config) error {
 		srcs = &sources.GoProxy{Client: gpClient}
 	}
 
-	scrapper := core.NewScrapper(ghClient, gpClient, pgClient, srcs, tTracer, cfg.GithubSearchQueries, cfg.GithubSearchQueriesIssues)
+	scrapper := core.NewScrapper(ghClient, gpClient, pgClient, cfg.DryRun, srcs, tTracer, cfg.GithubSearchQueries, cfg.GithubSearchQueriesIssues)
 
 	return scrapper.Run(ctx)
 }

pkg/core/scrapper.go

@@ -66,6 +66,8 @@ type Scrapper struct {
 	gp *goproxy.Client
 	pg pluginClient
 
+	dryRun bool
+
 	searchQueries       []string
 	searchQueriesIssues []string
 
@@ -76,12 +78,14 @@ type Scrapper struct {
 }
 
 // NewScrapper creates a new Scrapper instance.
-func NewScrapper(gh *github.Client, gp *goproxy.Client, pgClient pluginClient, sources Sources, tracer trace.Tracer, searchQueries, searchQueriesIssues []string) *Scrapper {
+func NewScrapper(gh *github.Client, gp *goproxy.Client, pgClient pluginClient, dryRun bool, sources Sources, tracer trace.Tracer, searchQueries, searchQueriesIssues []string) *Scrapper {
 	return &Scrapper{
 		gh: gh,
 		gp: gp,
 		pg: pgClient,
 
+		dryRun: dryRun,
+
 		searchQueries:       searchQueries,
 		searchQueriesIssues: searchQueriesIssues,
 
@@ -141,6 +145,13 @@ func (s *Scrapper) Run(ctx context.Context) error {
 				Title: github.String(issueTitle),
 				Body:  github.String(safeIssueBody(err)),
 			}
+
+			if s.dryRun {
+				logger.Info().Msg("Dry run, not creating the issue")
+				logger.Debug().Interface("issue", issue).Send()
+				continue
+			}
+
 			_, _, err = s.gh.Issues.Create(ctx, repository.GetOwner().GetLogin(), repository.GetName(), issue)
 			if err != nil {
 				span.RecordError(err)
@@ -150,6 +161,12 @@ func (s *Scrapper) Run(ctx context.Context) error {
 			continue
 		}
 
+		if s.dryRun {
+			logger.Info().Msg("Dry run, not storing the plugin")
+			logger.Debug().Interface("data", data).Send()
+			continue
+		}
+
 		err = s.store(logger.WithContext(ctx), data)
 		if err != nil {
 			span.RecordError(err)

pkg/core/scrapper_test.go

@@ -162,7 +162,7 @@ func TestScrapper_store(t *testing.T) {
 		t.Run(test.desc, func(t *testing.T) {
 			t.Parallel()
 
-			scrapper := NewScrapper(nil, nil, test.pgClient, nil, noop.NewTracerProvider().Tracer("test"), nil, nil)
+			scrapper := NewScrapper(nil, nil, test.pgClient, true, nil, noop.NewTracerProvider().Tracer("test"), nil, nil)
 
 			data := &plugin.Plugin{Name: "test"}
 			err := scrapper.store(context.Background(), data)
@@ -334,7 +334,7 @@ func TestScrapper_process(t *testing.T) {
 	gpClient := goproxy.NewClient("")
 	srcs := &sources.GitHub{Client: ghClient}
 
-	scrapper := NewScrapper(ghClient, gpClient, pgClient, srcs, noop.NewTracerProvider().Tracer("test"), []string{"topic:traefik-plugin language:Go archived:false is:public"}, []string{"is:open is:issue is:public author:traefiker"})
+	scrapper := NewScrapper(ghClient, gpClient, pgClient, true, srcs, noop.NewTracerProvider().Tracer("test"), []string{"topic:traefik-plugin language:Go archived:false is:public"}, []string{"is:open is:issue is:public author:traefiker"})
 
 	repository, _, err := ghClient.Repositories.Get(ctx, owner, repo)
 	require.NoError(t, err)
@@ -359,7 +359,7 @@ func TestScrapper_process_all(t *testing.T) {
 	gpClient := goproxy.NewClient("")
 	srcs := &sources.GitHub{Client: ghClient}
 
-	scrapper := NewScrapper(ghClient, gpClient, pgClient, srcs, noop.NewTracerProvider().Tracer("test"), []string{"topic:traefik-plugin language:Go archived:false is:public"}, []string{"is:open is:issue is:public author:traefiker"})
+	scrapper := NewScrapper(ghClient, gpClient, pgClient, true, srcs, noop.NewTracerProvider().Tracer("test"), []string{"topic:traefik-plugin language:Go archived:false is:public"}, []string{"is:open is:issue is:public author:traefiker"})
 
 	reposWithExistingIssue, err := scrapper.searchReposWithExistingIssue(ctx)
 	require.NoError(t, err)