Skip to content

Commit 3014194

Browse files
kevinakevina
committed
Code to create graphs for issue #40.
Use "make benchmark" to run.
1 parent c33fc95 commit 3014194

20 files changed

+376
-16
lines changed

‎Makefile

+5
Original file line numberDiff line numberDiff line change
@@ -12,3 +12,8 @@ deps: gx
1212
gx-go rewrite
1313
go get -t ./...
1414

15+
benchmark:
16+
gx --verbose install --global
17+
gx-go rewrite
18+
cd leveldb && ./run-benchmark.sh
19+
cd flatfs && ./run-benchmark.sh

‎flatfs/flatfs.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,9 @@ import (
1414

1515
"github.com/ipfs/go-datastore"
1616
"github.com/ipfs/go-datastore/query"
17-
"github.com/jbenet/go-os-rename"
17+
"gx/ipfs/QmaeRR9SpXumU5tYLRkq6x6pfMe8qKzxn4ujBpsTJ2zQG7/go-os-rename"
1818

19-
logging "github.com/ipfs/go-log"
19+
logging "gx/ipfs/QmNQynaz7qfriSUJkiEZUrm2Wen1u3Kj9goZzWtrPyu7XR/go-log"
2020
)
2121

2222
var log = logging.Logger("flatfs")

‎flatfs/flatfs_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ import (
1313
"github.com/ipfs/go-datastore/query"
1414
dstest "github.com/ipfs/go-datastore/test"
1515

16-
rand "github.com/dustin/randbo"
16+
rand "gx/ipfs/QmYvsG72GsfLgUeSojXArjnU6L4Wmwk7wuAxtNLuyXcc1T/randbo"
1717
)
1818

1919
func tempdir(t testing.TB) (path string, cleanup func()) {

‎flatfs/main/main.go

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
package main
2+
3+
import db "github.com/ipfs/go-datastore/flatfs"
4+
5+
func main() {
6+
db.TimeQuery()
7+
}

‎flatfs/plot.sh

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#!/bin/sh
2+
3+
sort -nk2 res | awk '$1 == "direct" {$2 = "direct"}; 1' > res2
4+
5+
gnuplot <<EOF
6+
set terminal png size 800,500
7+
set grid lw 0.25
8+
set output "plot.png"
9+
set style fill solid 0.25 border -1
10+
set style boxplot outliers pointtype 7
11+
set style data boxplot
12+
set title "flatfs query key-only 10,000 entries"
13+
set ylabel "time (ms)"
14+
set xlabel "buffer size"
15+
16+
plot "res2" using (1):3:(0.5):2 notitle
17+
EOF
18+
19+

‎flatfs/run-benchmark.sh

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../leveldb/run-benchmark.sh

‎flatfs/time_query.go

+131
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
package flatfs
2+
3+
import (
4+
"io/ioutil"
5+
"os"
6+
"math/rand"
7+
"time"
8+
"fmt"
9+
"path/filepath"
10+
"strings"
11+
12+
ds "github.com/ipfs/go-datastore"
13+
dsq "github.com/ipfs/go-datastore/query"
14+
)
15+
16+
var N = 10000
17+
18+
func TimeQuery() {
19+
rand.Seed(time.Now().UTC().UnixNano())
20+
path, err := ioutil.TempDir("/tmp", "flatdb_")
21+
if err != nil {
22+
panic(err)
23+
}
24+
25+
d, err := New(path, 5, false)
26+
if err != nil {
27+
panic(err)
28+
}
29+
defer func() {
30+
os.RemoveAll(path)
31+
d.Close()
32+
}()
33+
for n := 0; n < N; n++ {
34+
err := d.Put(ds.NewKey(RandomString(52)), []byte(RandomString(1000)))
35+
if err != nil {
36+
panic(err)
37+
}
38+
}
39+
40+
//TestQuery(d)
41+
//TestQuery(d)
42+
//TestQuery(d)
43+
// TestDirect(d)
44+
// TestIterator(d)
45+
// TestQuery(d)
46+
//TestDirect(d)
47+
//TestDirect(d)
48+
//TestDirect(d)
49+
// TestQuery(d)
50+
// TestDirect(d)
51+
// TestQuery(d)
52+
// TestDirect(d)
53+
// TestQuery(d)
54+
// TestDirect(d)
55+
// TestDirect(d)
56+
// TestDirect(d)
57+
// TestIterator(d)
58+
// TestIterator(d)
59+
// TestIterator(d)
60+
// TestIterator(d)
61+
// TestQuery(d)
62+
// TestQuery(d)
63+
// TestQuery(d)
64+
for i := 0; i <= 100; i++ {
65+
dsq.KeysOnlyBufSize = 1 << uint(rand.Intn(15))
66+
if dsq.KeysOnlyBufSize <= 8192 {
67+
TestQuery(d)
68+
} else {
69+
TestDirect(d)
70+
}
71+
}
72+
}
73+
74+
func TestQuery(d *Datastore) {
75+
start := time.Now()
76+
rs, err := d.Query(dsq.Query{KeysOnly: true})
77+
if err != nil {
78+
panic(err)
79+
}
80+
i := 0
81+
for r := range rs.Next() {
82+
i += int(r.Key[0])
83+
}
84+
elapsed := time.Since(start)
85+
//fmt.Printf("i = %d\n", i)
86+
fmt.Printf("query %d %f\n", dsq.KeysOnlyBufSize, elapsed.Seconds()*1000)
87+
}
88+
89+
func TestDirect(fs *Datastore) {
90+
start := time.Now()
91+
i := 0
92+
err := filepath.Walk(fs.path, func(path string, info os.FileInfo, err error) error {
93+
if err != nil {
94+
log.Errorf("Walk func in Query got error: %v", err)
95+
return err
96+
}
97+
98+
if !info.Mode().IsRegular() || strings.HasPrefix(info.Name(), ".") {
99+
return nil
100+
}
101+
102+
key, ok := fs.decode(info.Name())
103+
if !ok {
104+
log.Warning("failed to decode entry in flatfs")
105+
return nil
106+
}
107+
108+
i += int(key.String()[0])
109+
110+
return nil
111+
})
112+
if err != nil {
113+
log.Warning("walk failed: ", err)
114+
}
115+
elapsed := time.Since(start)
116+
//fmt.Printf("i = %d\n", i)
117+
fmt.Printf("direct %d %f\n", dsq.KeysOnlyBufSize, elapsed.Seconds()*1000)
118+
119+
}
120+
121+
122+
123+
func RandomString(strlen int) string {
124+
const chars = "abcdefghijklmnopqrstuvwxyz0123456789"
125+
result := make([]byte, strlen)
126+
for i := 0; i < strlen; i++ {
127+
result[i] = chars[rand.Intn(len(chars))]
128+
}
129+
return string(result)
130+
}
131+

‎key.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import (
44
"path"
55
"strings"
66

7-
"github.com/satori/go.uuid"
7+
"gx/ipfs/QmcyaFHbyiZfoX5GTpcqqCPYmbjYNAhRDekXSJPFHdYNSV/go.uuid"
88

99
dsq "github.com/ipfs/go-datastore/query"
1010
)

‎leveldb/datastore.go

+50-4
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,11 @@ package leveldb
33
import (
44
ds "github.com/ipfs/go-datastore"
55
dsq "github.com/ipfs/go-datastore/query"
6-
"github.com/jbenet/goprocess"
7-
"github.com/syndtr/goleveldb/leveldb"
8-
"github.com/syndtr/goleveldb/leveldb/opt"
9-
"github.com/syndtr/goleveldb/leveldb/util"
6+
"gx/ipfs/QmQopLATEYMNg7dVqZRNDfeE2S1yKy8zrRh5xnYiuqeZBn/goprocess"
7+
"gx/ipfs/QmbBhyDKsY4mbY6xsKt3qu9Y7FPvMJ6qbD8AMjYYvPRw1g/goleveldb/leveldb"
8+
"gx/ipfs/QmbBhyDKsY4mbY6xsKt3qu9Y7FPvMJ6qbD8AMjYYvPRw1g/goleveldb/leveldb/iterator"
9+
"gx/ipfs/QmbBhyDKsY4mbY6xsKt3qu9Y7FPvMJ6qbD8AMjYYvPRw1g/goleveldb/leveldb/opt"
10+
"gx/ipfs/QmbBhyDKsY4mbY6xsKt3qu9Y7FPvMJ6qbD8AMjYYvPRw1g/goleveldb/leveldb/util"
1011
)
1112

1213
type datastore struct {
@@ -157,4 +158,49 @@ func (d *datastore) Close() (err error) {
157158
return d.DB.Close()
158159
}
159160

161+
type IterQuery struct {
162+
Prefix string // namespaces the query to results whose keys have Prefix
163+
KeysOnly bool // return only keys.
164+
}
165+
166+
func (d *datastore) Iterate(prefix string) Iterator {
167+
if prefix == "" {
168+
return &iter{d.DB.NewIterator(nil, nil)}
169+
} else {
170+
return &iter{d.DB.NewIterator(util.BytesPrefix([]byte(prefix)), nil)}
171+
}
172+
}
173+
174+
type Iterator interface {
175+
Next() bool
176+
Key() string
177+
Value() interface{}
178+
Error() error
179+
Close() error
180+
}
181+
182+
type iter struct {
183+
iter iterator.Iterator
184+
}
185+
186+
// func (i *iter) Next() bool {
187+
// more := i.iter.Next()
188+
// if !more {
189+
// i.Result = dsq.Result{dsq.Entry{"", nil}, nil}
190+
// return false
191+
// } else if i.KeysOnly {
192+
// i.Result = dsq.Result{dsq.Entry{string(i.iter.Key()), nil}, nil}
193+
// return true
194+
// } else {
195+
// i.Result = dsq.Result{dsq.Entry{string(i.iter.Key()), i.iter.Value()}, nil}
196+
// return true
197+
// }
198+
// }
199+
200+
func (i *iter) Next() bool { return i.iter.Next() }
201+
func (i *iter) Key() string { return string(i.iter.Key()) }
202+
func (i *iter) Value() interface{} { return i.iter.Value() }
203+
func (i *iter) Error() error { return nil }
204+
func (i *iter) Close() error { i.iter.Release(); return nil }
205+
160206
func (d *datastore) IsThreadSafe() {}

‎leveldb/ds_test.go

+4-1
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ import (
44
"io/ioutil"
55
"os"
66
"testing"
7+
"math/rand"
8+
"time"
79

810
ds "github.com/ipfs/go-datastore"
911
dsq "github.com/ipfs/go-datastore/query"
@@ -25,7 +27,7 @@ var testcases = map[string]string{
2527
//
2628
// d, close := newDS(t)
2729
// defer close()
28-
func newDS(t *testing.T) (*datastore, func()) {
30+
func newDS(t testing.TB) (*datastore, func()) {
2931
path, err := ioutil.TempDir("/tmp", "testing_leveldb_")
3032
if err != nil {
3133
t.Fatal(err)
@@ -122,3 +124,4 @@ func expectMatches(t *testing.T, expect []string, actualR dsq.Results) {
122124
}
123125
}
124126
}
127+

‎leveldb/main/main.go

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
package main
2+
3+
import leveldb "github.com/ipfs/go-datastore/leveldb"
4+
5+
func main() {
6+
leveldb.TimeQuery()
7+
}

‎leveldb/plot.sh

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#!/bin/sh
2+
3+
sort -nk2 res | awk '$1 == "direct" {$2 = "direct"}; 1' > res2
4+
5+
gnuplot <<EOF
6+
set terminal png size 800,500
7+
set grid lw 0.25
8+
set output "plot.png"
9+
set style fill solid 0.25 border -1
10+
set style boxplot outliers pointtype 7
11+
set style data boxplot
12+
set title "leveldb query key-only 100,000 entries"
13+
set ylabel "time (ms)"
14+
set xlabel "buffer size"
15+
16+
plot "res2" using (1):3:(0.5):2 notitle
17+
EOF
18+
19+

‎leveldb/run-benchmark.sh

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
#!/bin/sh
2+
3+
set -x
4+
5+
go run main/main.go > res
6+
./plot.sh

0 commit comments

Comments
 (0)