fix: update the table building

This fixes some misconfiguration that resulted in some empty tables, it also cleans up the calling mechanism to build the database

Author: ozym

cmd/deltadb/main.go

@@ -2,31 +2,19 @@ package main
 
 import (
 	"context"
-	"database/sql"
 	"flag"
 	"fmt"
 	"log"
-	"net/url"
 	"os"
 	"time"
 
 	"github.com/GeoNet/delta"
-	"github.com/GeoNet/delta/meta/sqlite"
-	"github.com/GeoNet/delta/resp"
-
-	_ "modernc.org/sqlite"
 )
 
 type Settings struct {
-	debug bool // output more operational info
-
 	base string // base directory of delta files on disk
 	resp string // base directory of resp files on disk
-
-	db       string // name of the database file
-	response string // name of the database response table
-
-	init bool // should the database be updated
+	path string // name of the database file
 }
 
 func main() {
@@ -35,7 +23,7 @@ func main() {
 
 	flag.Usage = func() {
 		fmt.Fprintf(os.Stderr, "\n")
-		fmt.Fprintf(os.Stderr, "Build a DELTA Sqlite DB file\n")
+		fmt.Fprintf(os.Stderr, "Build and initialise a DELTA Sqlite database\n")
 		fmt.Fprintf(os.Stderr, "\n")
 		fmt.Fprintf(os.Stderr, "Usage:\n")
 		fmt.Fprintf(os.Stderr, "\n")
@@ -47,12 +35,9 @@ func main() {
 		fmt.Fprintf(os.Stderr, "\n")
 	}
 
-	flag.BoolVar(&settings.debug, "debug", false, "add extra operational info")
-	flag.BoolVar(&settings.init, "init", false, "initialise the database if a file on disk")
-	flag.StringVar(&settings.base, "base", "", "base directory of delta files on disk")
-	flag.StringVar(&settings.resp, "resp", "", "base directory of resp files on disk")
-	flag.StringVar(&settings.db, "db", "", "name of the database file on disk")
-	flag.StringVar(&settings.response, "response", "Response", "optional database response table name to use")
+	flag.StringVar(&settings.base, "base", "", "base directory of delta files on disk, default uses embedded files")
+	flag.StringVar(&settings.resp, "resp", "", "base directory of resp files on disk, default uses embedded files")
+	flag.StringVar(&settings.path, "path", "", "name of the database file on disk, default is to use memory only")
 
 	flag.Parse()
 
@@ -64,49 +49,30 @@ func main() {
 		log.Fatal(err)
 	}
 
-	// resp recovers the response files
-	files, err := resp.ListBase(settings.resp)
+	// recover any response files
+	files, err := delta.NewResp(settings.resp)
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	values := make(map[string]string)
-	for _, f := range files {
-		lookup, err := resp.LookupBase(settings.resp, f)
-		if err != nil {
-			log.Fatal(err)
-		}
-		values[f] = string(lookup)
-	}
-
-	path := ":memory:"
-	if settings.db != "" {
-		path = settings.db
-	}
-
-	opts := url.Values{}
-	opts.Set("_time_format", "sqlite")
-	opts.Set("_foreign_keys", "on")
-	if settings.db != "" && !settings.init {
-		opts.Set("mode", "ro")
-	}
-
-	db, err := sql.Open("sqlite", fmt.Sprintf("file:%s?%s", path, url.QueryEscape(opts.Encode())))
+	// open the database file handle
+	db, err := delta.NewDB(settings.path)
 	if err != nil {
-		log.Fatalf("unable to open database %s: %v", path, err)
+		log.Fatalf("unable to open database %q: %v", settings.path, err)
 	}
 	defer db.Close()
 
-	if settings.db == "" || settings.init {
+	start := time.Now()
 
-		// insert extra response files
-		extra := set.KeyValue(settings.response, "Response", "XML", values)
+	// initialise the database
+	if err := db.Init(ctx, set, files...); err != nil {
+		log.Fatalf("unable to init database: %v", err)
+	}
 
-		log.Println("initialise database")
-		start := time.Now()
-		if err := sqlite.New(db).Init(ctx, set.TableList(extra)); err != nil {
-			log.Fatalf("unable to run database exec: %v", err)
-		}
-		log.Printf("database initialised in %s", time.Since(start).String())
+	switch {
+	case settings.path != "":
+		log.Printf("successfully initialised %q in %s", settings.path, time.Since(start).Truncate(time.Millisecond).String())
+	default:
+		log.Printf("successfully initialised memory in %s", time.Since(start).Truncate(time.Millisecond).String())
 	}
 }

delta.go

@@ -1,14 +1,24 @@
 package delta
 
 import (
+	"context"
+	"database/sql"
 	"embed"
+	"fmt"
 	"io/fs"
+	"net/url"
 	"os"
 	"sync"
 
+	_ "modernc.org/sqlite"
+
 	"github.com/GeoNet/delta/meta"
+	"github.com/GeoNet/delta/meta/sqlite"
+	"github.com/GeoNet/delta/resp"
 )
 
+const ResponseName = "Response"
+
 //go:embed assets/*.csv
 //go:embed install/*.csv
 //go:embed network/*.csv
@@ -51,3 +61,56 @@ func NewBase(base string) (*meta.Set, error) {
 	}
 	return New()
 }
+
+// NewResp returns a slice of response file names.
+func NewResp(base string) ([]string, error) {
+	return resp.ListBase(base)
+}
+
+// DB is a wrapper for an SQL DB pointer.
+type DB struct {
+	*sql.DB
+}
+
+// NewDB returns a DB pointer, a non-empty path is used as a file name, otherwise the database is generated in memory.
+func NewDB(path ...string) (*DB, error) {
+
+	file := ":memory:"
+	for _, p := range path {
+		file = p
+	}
+
+	opts := url.Values{}
+	opts.Set("_time_format", "sqlite")
+	opts.Set("_foreign_keys", "on")
+
+	db, err := sql.Open("sqlite", fmt.Sprintf("file:%s?%s", file, url.QueryEscape(opts.Encode())))
+	if err != nil {
+		return nil, err
+	}
+
+	return &DB{DB: db}, nil
+}
+
+// Init updates the contents of the given database with contents from the given delta base and response paths.
+// If these are empty then the default compiled in versions will be used.
+func (db *DB) Init(ctx context.Context, set *meta.Set, files ...string) error {
+
+	values := make(map[string]string)
+	for _, file := range files {
+		lookup, err := resp.Lookup(file)
+		if err != nil {
+			return err
+		}
+		values[file] = string(lookup)
+	}
+
+	// insert any extra response files
+	extra := set.KeyValue(ResponseName, "Response", "XML", values)
+
+	if err := sqlite.New(db.DB).Init(ctx, set.TableList(extra)); err != nil {
+		return err
+	}
+
+	return nil
+}

delta_test.go

@@ -1,6 +1,7 @@
 package delta
 
 import (
+	"context"
 	"testing"
 )
 
@@ -9,3 +10,26 @@ func TestDelta(t *testing.T) {
 		t.Fatal(err)
 	}
 }
+
+func TestDB(t *testing.T) {
+
+	set, err := NewBase("")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	files, err := NewResp("")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	db, err := NewDB()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db.Close()
+
+	if err := db.Init(context.Background(), set, files...); err != nil {
+		t.Fatal(err)
+	}
+}