all the changes in betrfs .4

benchmarks/application/git/run-all.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+rm git.csv
+echo "fs, op, time.s" > git.csv
+
+for fs in nilfs2 xfs zfs ftfs
+do
+	for i in {0..4}
+	do
+		../../setup-"$fs".sh
+		./run-git-clone-diff.sh $fs git.csv
+		../../cleanup-fs.sh
+	done
+done

benchmarks/application/git/run-git-clone-diff.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+. ../../fs-info.sh
+. ../../.ismounted
+support=$HOME/$repo/benchmarks/support-files
+
+if [ ! -e $support/$clone_repo ] ; then
+    cd $support; git clone https://github.com/torvalds/linux.git; cd -
+fi
+
+if [ -e $mntpnt/$clone_repo ]; then
+    echo "Stop! $mntpnt/$repo already exists."
+    echo "Please delete and run again."
+    exit 17
+fi
+
+sudo ../../clear-fs-caches.sh
+
+cd $mntpnt
+clonetime=$(TIMEFORMAT="%R"; (time (git clone $support/$clone_repo 2> /dev/null)) 2>&1)
+cd -
+
+sudo ../../clear-fs-caches.sh
+
+cd $mntpnt/$clone_repo
+difftime=$(TIMEFORMAT="%R"; (time (git diff --patch v4.7 v4.14 > patch)) 2>&1)
+cd -
+
+echo "$1", clone, $clonetime >> $2
+echo "$1", diff,  $difftime >> $2

benchmarks/application/git/run-git-clone.sh

@@ -5,7 +5,7 @@
 support=$HOME/$repo/benchmarks/support-files
 
 if [ ! -e $support/$clone_repo ] ; then
-    cd $support; git clone [email protected]:$clone_repo.git; cd -
+    cd $support; git clone https://github.com/torvalds/linux.git; cd -
 fi
 
 if [ -e $mntpnt/$clone_repo ]; then

benchmarks/application/git/run-git-diff.sh

@@ -7,16 +7,14 @@ support=$HOME/$repo/benchmarks/support-files
 
 if [ ! -e $mntpnt/$clone_repo ]; then
     if [ ! -e $support/$clone_repo ] ; then
-	cd $support; git clone [email protected]:$clone_repo.git; cd -
+	cd $support; git clone https://github.com/torvalds/linux.git; cd -
     fi
     cd $mntpnt; git clone $support/$clone_repo; cd -
 fi
 
 sudo ../../clear-fs-caches.sh
 
+cd $mntpnt/$clone_repo
+result=/usr/bin/time -f '%e' git diff --patch v4.7 v4.14 > patch
 
-# first commit where file system compiles vs current commit (8/31/14)
-
-cd $mntpnt/$clone_repo; time git diff --patch  6b93052128d62bf0e87620db78a27269eb00ccbf 637d301e7dea09f10d11b7570e86de3f06f2470c > patch
-
-#cd $mntpnt/$repo; time git diff --quiet  6b93052128d62bf0e87620db78a27269eb00ccbf 637d301e7dea09f10d11b7570e86de3f06f2470c
+echo "$1 diff $result" >> $2

benchmarks/application/rsync/run-rsync.sh

@@ -17,8 +17,8 @@ sudo ../../clear-fs-caches.sh
 #echo 'Copy from home to benchmark partition'
 #sudo perf record -g -- rsync --stats -r -t -S -h $support/$src $mntpnt/$dst
 #strace -c -T rsync --stats -r -t -S -h $support/$src $mntpnt/$dst 2&> strace.txt
-#rsync --stats -r -t -S -h $support/$src $mntpnt/$dst
-rsync --stats -r -t -W -h --inplace $support/$src $mntpnt/$dst
+rsync --stats -r -t -S -h $support/$src $mntpnt/$dst
+#rsync --stats -r -t -W -h --inplace $support/$src $mntpnt/$dst
 
 sudo ../../clear-fs-caches.sh
 

benchmarks/cleanup-fs.sh

@@ -13,6 +13,11 @@ if [[ $fstype == "ext4" || $fstype == "btrfs" || $fstype == "xfs" ]]
 then
     umount $mntpnt
     exit 0
+elif [[ $fstype == "nilfs2" ]]
+then
+    sudo nilfs-clean -q
+    umount $mntpnt
+    exit 0
 elif [[ $fstype == "ftfs" ]]
 then
     umount $mntpnt

benchmarks/clear-fs-caches.sh

@@ -21,10 +21,15 @@ if [[ $fstype == "ext4" || $fstype == "btrfs" || $fstype == "xfs" ]]
 then
     umount $mntpnt
     mount -t $fstype $sb_dev $mntpnt
+elif [[ $fstype == "nilfs2" ]]
+then
+    nilfs-clean -q
+    umount $mntpnt
+    mount -t $fstype $sb_dev $mntpnt
 elif [[ $fstype == "zfs" ]]
 then
-    echo "what do we do with zfs?"
-    exit 0
+    zfs umount $mntpnt
+    zfs mount datastore/files
 elif [[ $fstype == "ftfs" ]]
 then
     echo "removing $module and mounting/unmounting ftfs file system"

benchmarks/fs-info.sh

@@ -1,9 +1,13 @@
-sb_dev=/dev/sda4
+#On Beakers /dev/sda is the fast connector and has the SSD
+#  and the root fs is installed on the HDD on /dev/sdb
+
+#sb_dev=/dev/sdb3
+sb_dev=/dev/sdb1
 dummy_file=dummy.dev
 dummy_dev=/dev/loop0
 mntpnt=/mnt/benchmark
-southbound_module=/home/ftfs/ft-index/ftfs/ftfs.ko
-module=/home/ftfs/ft-index/filesystem/ftfs.ko
+southbound_module=/home/betrfs/ft-index/ftfs/ftfs.ko
+module=/home/betrfs/ft-index/filesystem/ftfs.ko
 repo=ft-index
-clone_repo=ft-index-snapshot
+clone_repo=linux
 circle_size=128

benchmarks/macro/mailserver/imap-test/imap-punish-new.py

@@ -0,0 +1,149 @@
+#!/usr/bin/python
+
+#import getpass
+import imaplib
+import time
+import sys
+import getopt
+import random
+import os
+import types
+
+SERVER = "localhost"
+USER = "ftfstest"
+PASS = "ftfstest"
+
+def usage() :
+    print sys.argv[0] + \
+''': [-t # num threads (1)]
+         [ -w # write operation fraction out of 100 (0)]
+         [ -m # mailboxes (shared among threads) (1)]
+         [ -n # operations (per thread) (1000)]
+         [ -h Print this message]
+'''
+
+# Get command line options
+try :
+    opts, args = getopt.getopt(sys.argv[1:], 'hn:m:t:w:x:s')
+except :
+    usage()
+    sys.exit(2)
+
+opt_threads = 4
+opt_writes = 50
+opt_mailboxes = 10
+opt_ops = 1000
+opt_setup = 0
+boxsize = 2500
+
+for o, a in opts :
+    if o == '-h' :
+        usage()
+        sys.exit(2)
+    if o == '-n' :
+        opt_ops = int(a)
+    if o == '-m' :
+        opt_mailboxes = int(a)        
+    if o == '-t' :
+        opt_threads = int(a)
+    if o == '-w' :
+        opt_writes = int(a)
+    if o == '-s' :
+        opt_setup = 1
+    if o == '-x' :
+	boxsize = int(a)
+
+print "IMAP Punish: %d threads, %d operations, %d mailboxes, %d %% writes" % (opt_threads, opt_ops, opt_mailboxes, opt_writes)
+
+if (opt_setup) :
+    # Initialize each mailbox to contain 3,000 messages
+    M = imaplib.IMAP4_SSL(SERVER)
+    M.login(USER, PASS)
+    for i in range (opt_mailboxes) :
+        name = "box%d" % i
+        M.delete(name)
+        M.create(name)
+        print "Making box %s" % name
+        for j in range (boxsize) :
+            M.append(name, None, None, "I love to go a-wandering, along the mountain tracks...")
+    M.logout()
+    sys.exit(0)
+
+#print "Done making boxes.  Making %d threads" % opt_threads
+
+t1 = time.time()
+
+# Create (fork) threads
+for i in range(opt_threads) :
+    if 0 == os.fork() :
+        # Child
+        random.seed()
+
+        bsize = boxsize
+        M = imaplib.IMAP4_SSL(SERVER)
+        M.login(USER, PASS)
+        box = "box%d" % i
+        M.select(box)
+
+        for j in range(opt_ops) :
+            rand_op = 0
+            # Select an operation
+            if opt_writes != 0 :
+                rand_op = random.randint(1, 100)
+                if (rand_op <= opt_writes) :
+                    # Choose the specific type of write
+                    # 1 = change flag
+                    # 2 = delete
+                    # 3 = append
+                    rand_op = random.randint(1, 3) 
+                else :
+                    rand_op = 0
+            # Select a random uid
+            num = random.randint(1, bsize)
+
+            #print "Thread %d, op %d, num %d, box %s" % (i, rand_op, num, box)
+
+            if rand_op == 0 :
+                typ, data = M.fetch(num, '(RFC822 FLAGS)')
+            elif rand_op == 1 : 
+                typ, data = M.fetch(num, '(RFC822 FLAGS)')
+                flagged = 0
+                if type(data[0]) is types.NoneType :
+                    continue
+
+                for flag in imaplib.ParseFlags(data[0][0]):
+                    if (flag == '\Flagged') :
+                        flagged = 1
+                if flagged :
+                    M.store(num, '-FLAGS', '(\\FLAGGED)')
+                else :
+                    M.store(num, '+FLAGS', '(\\FLAGGED)')
+
+            elif rand_op == 2:
+                M.store(num, '+FLAGS', '(\\Deleted)')
+                M.expunge()
+                bsize = bsize - 1;
+
+            elif rand_op == 3:
+                M.append(box, None, None, "I love to go a-wandering, along the mountain tracks...")
+                bsize = bsize + 1
+
+        M.logout()
+        sys.exit(0)
+
+for i in range(opt_threads) :
+    os.wait()
+
+t2 = time.time()
+
+print "This experiment took %f seconds" % (t2 - t1)
+
+# Empty out the mailbox
+if 0:
+    M = imaplib.IMAP4_SSL(SERVER)
+    M.login(USER, PASS)
+    for i in range (opt_mailboxes) :
+        name = "box%d" % i
+        M.delete(name)
+    M.logout()
+

benchmarks/macro/mailserver/imap-test/run-test.sh

@@ -1,3 +1,4 @@
+sudo dovecot stop
 cd ~/ft-index/benchmarks/
 sudo ./cleanup-fs.sh
 sudo ./setup-ftfs.sh

benchmarks/micro/circlingtradeoff/circle-dirscan.sh

@@ -8,7 +8,7 @@ cnt=0
 for i in 2048 1024 512 256 128 64 32 16 8 4 2 1 0
 #for i in 2048 
 do
-	FT_HOMEDIR=/home/ftfs/ft-index
+	FT_HOMEDIR=/home/betrfs/frwlock-fixup/ft-index
 	. $FT_HOMEDIR/benchmarks/fs-info.sh
 	sudo $FT_HOMEDIR/benchmarks/cleanup-fs.sh
 	sudo $FT_HOMEDIR/benchmarks/setup-ftfs-parameterized.sh $i 

benchmarks/micro/circlingtradeoff/circle-rename.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+TIMEFORMAT="%R"
+
+declare -a result
+declare -a resultmv
+cnt=0
+
+FT_HOMEDIR=/home/ftfs/ft-index
+. $FT_HOMEDIR/benchmarks/fs-info.sh
+sudo $FT_HOMEDIR/benchmarks/setup-ftfs.sh  10000
+
+echo "Copying linux src"	
+cp -r $FT_HOMEDIR/benchmarks/support-files/linux-3.11.10 $mntpnt
+
+dirs=($(find $mntpnt/linux-3.11.10 | shuf | head -100))
+
+#echo "Array: ${dirs[@]}"
+
+sudo $FT_HOMEDIR/benchmarks/cleanup-fs.sh
+
+for i in 2048 1024 512 256 128 64 32 16 8 4 2 1 
+do
+	FT_HOMEDIR=/home/ftfs/ft-index
+	. $FT_HOMEDIR/benchmarks/fs-info.sh
+	sudo $FT_HOMEDIR/benchmarks/setup-ftfs.sh  $i
+
+#	if [ -d $mntpnt/linux-3.11.10 ]; then
+#	:
+#	else
+#	. $FT_HOMEDIR/benchmarks/micro/prepare-support-file.sh
+#	fi
+
+	echo "Copying linux src"	
+	cp -r $FT_HOMEDIR/benchmarks/support-files/linux-3.11.10 $mntpnt
+
+	sudo $FT_HOMEDIR/benchmarks/clear-fs-caches.sh $i
+
+	echo "Circle Size :$i"
+
+	for j in $(seq 0 99)
+	do
+		echo "Moving ${dirs[$j]}"
+		result[$j]=$( { time mv ${dirs[$j]} $mntpnt/ >/dev/null; } 2>&1 )
+		echo "Time: ${result[$j]}"
+	done
+
+	total=$( IFS="+"; bc <<< "${result[*]}" )
+	echo "Total time for rename: $total"
+
+	resultmv[$cnt]=$total
+	cnt=$((cnt+1))
+
+	sudo $FT_HOMEDIR/benchmarks/cleanup-fs.sh
+done
+
+echo "mv: ${resultmv[@]}"