v1.6

.gitignore

@@ -17,4 +17,7 @@ pushcsv
 
 \.DS_Store
 
-*.nupkg
+*.nupkg
+
+# test env
+test/logs

README.md

@@ -1,3 +1,5 @@
+<img src="art/logo.svg" width="128"/>
+
 # pushcsv
 Push csv/tsv data to database
 

internal/cli/args.go

@@ -17,6 +17,7 @@ type Options struct {
 	Debug      bool
 	Trace      bool
 	DryRun     bool
+	Compact    bool
 }
 
 func NewOptions(argv []string) *Options {
@@ -50,6 +51,7 @@ func NewOptions(argv []string) *Options {
 	debug, _ := opts.Bool("--debug")
 	trace, _ := opts.Bool("--trace")
 	dryRun, _ := opts.Bool("--dry-run")
+	compact, _ := opts.Bool("--compact")
 
 	return &Options{
 		Opts:       &opts,
@@ -62,5 +64,6 @@ func NewOptions(argv []string) *Options {
 		Debug:      debug,
 		Trace:      trace,
 		DryRun:     dryRun,
+		Compact:    compact,
 	}
 }

internal/cli/usage.go

@@ -3,21 +3,21 @@ package cli
 import (
 	"fmt"
 	"os"
-	"strings"
 )
 
-const version string = "1.4"
+const version string = "1.6"
 const help string = `Push csv/tsv data to database
 
 Usage:
-	pushcsv <db_uri> <table> <file> [--headers] [--purge] [--map=<field_maps>]... [--debug] [--trace] [--dry-run]
+	pushcsv <db_uri> <table> <file> [--headers] [--purge] [--compact] [--map=<field_maps>]... [--debug] [--trace] [--dry-run]
 	pushcsv <db_uri> <table> --purge [--debug] [--trace] [--dry-run]
 
 Options:
 	-h --help            show this screen
-	--version            show version
+	-V --version         show version
 	--headers            when first line of csv file is column headers
 	--purge              purge all exiting data from table or collection before pushing new data
+	--compact            compact after pushing data (sql vacuum, mongodb compact)
 	--map=<field_maps>   map a source column header to table field [from:to]
 	                     if mapping is used, all columns missing a map will be ignored,
 	                     using mapping is a great way to selectively push data
@@ -41,16 +41,16 @@ Examples:
 	pushcsv sqlite3:data.db users ~/users.csv
 `
 
-var printHelpAndExit = func(err error, usage string) {
+var printHelpAndExit = func(err error, docoptMessage string) {
 	if err != nil {
+		// if err occured print full help
+		// docopt only includes usage section in its message
 		fmt.Fprintln(os.Stderr, help)
 		os.Exit(1)
 	} else {
-		if strings.Index(usage, version) == 0 {
-			fmt.Println(usage)
-		} else {
-			fmt.Println(help)
-		}
+		// otherwise print whatever docopt says
+		// e.g. reporting version
+		fmt.Println(docoptMessage)
 		os.Exit(0)
 	}
 }

internal/datafile/datafile.go

@@ -67,10 +67,10 @@ func prepareUTF8(file *os.File, logger *cli.Logger) ([]byte, error) {
 		return b[3:], nil
 	} else if bytes.Equal(b[0:2], BOM_UTF16) {
 		logger.Debug("Input file encoding detected as UTF16 (LittleEndian)")
-		return DecodeUTF16(b, false)
+		return DecodeUTF16(b[2:], false)
 	} else if bytes.Equal(b[0:2], BOM_UTF16_BE) {
 		logger.Debug("Input file encoding detected as UTF16 BigEndian")
-		return DecodeUTF16(b, true)
+		return DecodeUTF16(b[2:], true)
 	} else {
 		logger.Debug("Input file encoding detected as UTF8")
 		return b, nil

internal/persistance/genericsql.go

@@ -103,6 +103,18 @@ func (w GenericSQLWriter) Write(tableData *datafile.TableData) (*Result, error)
 		}
 	}
 
+	// vacuum table if requested
+	if w.CompactAfterWrite {
+		w.Logger.Debug("vacuuming table")
+		if !w.DryRun {
+			_, pErr := db.Exec(
+				fmt.Sprintf("VACUUM %s;", tableData.Name))
+			if pErr != nil {
+				return nil, pErr
+			}
+		}
+	}
+
 	// commit transaction
 	w.Logger.Debug("commiting transaction")
 	if !w.DryRun {

internal/persistance/mongo.go

@@ -75,7 +75,8 @@ func (w MongoDBWriter) Write(tableData *datafile.TableData) (*Result, error) {
 	w.Logger.Trace(session)
 
 	w.Logger.Debug("getting target collection")
-	c := session.DB(dialinfo.Database).C(tableData.Name)
+	db := session.DB(dialinfo.Database)
+	c := db.C(tableData.Name)
 	w.Logger.Trace(c)
 
 	w.Logger.Debug("opening bulk operation")
@@ -106,6 +107,21 @@ func (w MongoDBWriter) Write(tableData *datafile.TableData) (*Result, error) {
 		if txnErr != nil {
 			return nil, txnErr
 		}
+
+		// compact collection if requested
+		if w.CompactAfterWrite {
+			w.Logger.Debug("compacting collection")
+			if !w.DryRun {
+				compactErr := db.Run(
+					bson.D{{Name: "compact", Value: tableData.Name}},
+					nil)
+				if compactErr != nil {
+					return nil, compactErr
+				}
+				w.Logger.Debug("not implemented. need compacting support in mongodb driver")
+			}
+		}
+
 		w.Logger.Debug("preparing report")
 		return &Result{
 			Message: fmt.Sprintf(

internal/persistance/writer.go

@@ -16,10 +16,11 @@ type Result struct {
 }
 
 type DatabaseWriter struct {
-	Config           *DatabaseConfig
-	PurgeBeforeWrite bool
-	DryRun           bool
-	Logger           *cli.Logger
+	Config            *DatabaseConfig
+	PurgeBeforeWrite  bool
+	CompactAfterWrite bool
+	DryRun            bool
+	Logger            *cli.Logger
 }
 
 type Writer interface {
@@ -30,10 +31,11 @@ type Writer interface {
 
 func NewWriter(dbConfig *DatabaseConfig, options *cli.Options, logger *cli.Logger) (Writer, error) {
 	w := &DatabaseWriter{
-		Config:           dbConfig,
-		PurgeBeforeWrite: options.Purge,
-		DryRun:           options.DryRun,
-		Logger:           logger,
+		Config:            dbConfig,
+		PurgeBeforeWrite:  options.Purge,
+		CompactAfterWrite: options.Compact,
+		DryRun:            options.DryRun,
+		Logger:            logger,
 	}
 	if dbConfig.Backend == Postgres {
 		return GenericSQLWriter{*w}, nil

test/cases/test-all.sh

@@ -0,0 +1,23 @@
+# all test cases for pushcsv
+# TESTCONFIG is setup with pushcsv arguments (db rui and table) by each
+# database test script
+# -h argument requests to use --headers argument on test cases e.g. mongodb
+# tests need headers for object property names
+
+# report result of last command
+# $1 is test name
+function report() {
+    if [ $? -eq 0 ]; then
+        echo -e $PASS $1
+    else
+        echo -e $FAIL $1
+    fi
+}
+
+export -f report
+
+# run tests
+$TESTCASES/test-encoding.sh $*
+$TESTCASES/test-pushall.sh  $*
+$TESTCASES/test-pushmap.sh  $*
+$TESTCASES/test-purge.sh    $*

test/cases/test-encoding.sh

@@ -0,0 +1,22 @@
+echo "testing support for various csv encoding..."
+
+TESTARGS="--debug --trace >>$LOGFILE 2>&1"
+if [[ $3 == "-h" ]]; then
+    TESTARGS="--headers "$TESTARGS
+fi
+
+# test pushing utf-8 csv input
+eval pushcsv $1 $2 $ASSETS/test-utf8.csv $TESTARGS
+report "utf-8 test"
+
+# test pushing utf-8 with BOM csv input
+eval pushcsv $1 $2 $ASSETS/test-utf8bom.csv $TESTARGS
+report "utf-8 BOM test"
+
+# test pushing utf-16 csv input
+eval pushcsv $1 $2 $ASSETS/test-utf16.csv $TESTARGS
+report "utf-16 test"
+
+# test pushing utf-16 with BOM csv input
+eval pushcsv $1 $2 $ASSETS/test-utf16be.csv $TESTARGS
+report "utf-16 BOM test"

test/cases/test-purge.sh

@@ -0,0 +1 @@
+echo "testing purging existing db objects..."

test/cases/test-pushall.sh

@@ -0,0 +1 @@
+echo "testing pushing all fields to database..."

test/cases/test-pushmap.sh

@@ -0,0 +1 @@
+echo "testing pushing mapped fields to database..."

test/dbs/test-monogdb.sh

@@ -0,0 +1,45 @@
+#!
+echo "running pushcsv tests on mongodb..."
+
+# mongodb root account
+ROOTU="mdbroot"
+ROOTP="mdbrootpass"
+
+# mongodb user for this test
+TESTU="pushcsvtestuser"
+TESTP="testpass"
+
+# database and collection for this test
+TESTDB="pushcsvtestdb"
+TESTCOL="testcollection"
+
+printf $PROGRESSPRINT "pulling docker images and starting a container..."
+# pull docker container and create the root user
+docker run -d  --name pushcsvtestmongo \
+               -p 127.0.0.1:27017:27017/tcp \
+               -e MONGO_INITDB_ROOT_USERNAME=$ROOTU -e MONGO_INITDB_ROOT_PASSWORD=$ROOTP mongo \
+                 >>$LOGFILE 2>&1 
+# wait for container to start
+sleep 2
+
+printf $PROGRESSPRINT "configuring test db..."
+# report mongodb version
+echo "mongo --version" | docker exec -i pushcsvtestmongo bash   >>$LOGFILE 2>&1 
+sleep 2
+
+# switch to test db and add test user
+echo "use $TESTDB
+db.createUser({user: \"$TESTU\", pwd: \"$TESTP\", roles: [\"dbOwner\"]})
+exit
+" | docker exec -i pushcsvtestmongo mongo -u $ROOTU -p $ROOTP   >>$LOGFILE 2>&1 
+sleep 2
+
+printf $PROGRESSPRINT
+
+# export the db config and run tests
+$TESTCASES/test-all.sh "mongodb://$TESTU:$TESTP@localhost:27017/$TESTDB" $TESTCOL -h
+
+printf $PROGRESSPRINT "cleaning up..."
+docker stop pushcsvtestmongo   >>$LOGFILE 2>&1 
+docker rm pushcsvtestmongo  >>$LOGFILE 2>&1 
+docker rmi mongo >>$LOGFILE 2>&1 

test/dbs/test-mysql.sh

@@ -0,0 +1,2 @@
+#!
+echo "running pushcsv tests on mysql..."

test/dbs/test-postgres.sh

@@ -0,0 +1,2 @@
+#!
+echo "running pushcsv tests on postgresql..."

test/dbs/test-sqlite.sh

@@ -0,0 +1,11 @@
+#!
+echo "running pushcsv tests on sqlite3..."
+
+# TABLENAME="sqlitetesttable"
+# TESTDB=$TESTPATH/sqlitetestdb.db
+
+# # export the db config and run tests
+# $TESTCASES/test-all.sh "sqlite3:$TESTDB" $TABLENAME
+
+# printf $PROGRESSPRINT "cleaning up..."
+# # rm $TESTDB

test/dbs/test-sqlserver.sh

@@ -0,0 +1,2 @@
+#!
+echo "running pushcsv tests on ms-sql-server..."

test/env/setup-env.sh

@@ -0,0 +1,6 @@
+#!
+echo "setting up test environment..."
+cat /dev/null > $LOGFILE
+echo "log file "$LOGFILE
+
+echo "running on "$(uname -a)

test/env/teardown-env.sh

@@ -0,0 +1,2 @@
+#!
+echo "tearing down test environment..."

test/run-tests.sh

@@ -0,0 +1,31 @@
+#! /bin/bash
+# globals
+export TESTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
+export TESTENV=$TESTPATH/env
+export DBTESTS=$TESTPATH/dbs
+export TESTCASES=$TESTPATH/cases
+export ASSETS=$TESTPATH/assets
+export LOGPATH=$TESTPATH/logs
+
+# green PASS
+export PASS="[ \033[32mPASS\033[39m ]"
+# red FAIL
+export FAIL="[ \033[31mFAIL\033[39m ]"
+export PROGRESSPRINT="%-120s\r"
+
+# get HEAD commit hash for this test
+export HEADHASH=`git describe --always`
+export LOGFILE=$LOGPATH/$HEADHASH"_test.log"
+
+# setup
+$TESTENV/setup-env.sh      | tee -a $LOGFILE
+
+# db tests
+$DBTESTS/test-postgres.sh  | tee -a $LOGFILE
+$DBTESTS/test-monogdb.sh   | tee -a $LOGFILE
+$DBTESTS/test-mysql.sh     | tee -a $LOGFILE
+$DBTESTS/test-sqlserver.sh | tee -a $LOGFILE
+$DBTESTS/test-sqlite.sh    | tee -a $LOGFILE
+
+# teardown
+$TESTENV/teardown-env.sh   | tee -a $LOGFILE