diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 00000000..ab8da59d --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,3 @@ +Tendermint Go-DB Copyright (C) 2015 All in Bits, Inc + +Released under the Apache2.0 license diff --git a/README.md b/README.md new file mode 100644 index 00000000..ca5ab33f --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +TODO: syndtr/goleveldb should be replaced with actual LevelDB instance diff --git a/c_level_db.go b/c_level_db.go new file mode 100644 index 00000000..33a78009 --- /dev/null +++ b/c_level_db.go @@ -0,0 +1,152 @@ +// +build gcc + +package db + +import ( + "fmt" + "path" + + "github.com/jmhodges/levigo" + + . "github.com/tendermint/go-common" +) + +func init() { + dbCreator := func(name string, dir string) (DB, error) { + return NewCLevelDB(name, dir) + } + registerDBCreator(LevelDBBackendStr, dbCreator, true) + registerDBCreator(CLevelDBBackendStr, dbCreator, false) +} + +type CLevelDB struct { + db *levigo.DB + ro *levigo.ReadOptions + wo *levigo.WriteOptions + woSync *levigo.WriteOptions +} + +func NewCLevelDB(name string, dir string) (*CLevelDB, error) { + dbPath := path.Join(dir, name+".db") + + opts := levigo.NewOptions() + opts.SetCache(levigo.NewLRUCache(1 << 30)) + opts.SetCreateIfMissing(true) + db, err := levigo.Open(dbPath, opts) + if err != nil { + return nil, err + } + ro := levigo.NewReadOptions() + wo := levigo.NewWriteOptions() + woSync := levigo.NewWriteOptions() + woSync.SetSync(true) + database := &CLevelDB{ + db: db, + ro: ro, + wo: wo, + woSync: woSync, + } + return database, nil +} + +func (db *CLevelDB) Get(key []byte) []byte { + res, err := db.db.Get(db.ro, key) + if err != nil { + PanicCrisis(err) + } + return res +} + +func (db *CLevelDB) Set(key []byte, value []byte) { + err := db.db.Put(db.wo, key, value) + if err != nil { + PanicCrisis(err) + } +} + +func (db *CLevelDB) SetSync(key []byte, value []byte) { + err := db.db.Put(db.woSync, key, value) + if err != nil { + PanicCrisis(err) + } +} + +func (db *CLevelDB) Delete(key []byte) { + err := db.db.Delete(db.wo, key) + if err != nil { + PanicCrisis(err) + } +} + +func (db *CLevelDB) DeleteSync(key []byte) { + err := db.db.Delete(db.woSync, key) + if err != nil { + PanicCrisis(err) + } +} + +func (db *CLevelDB) DB() *levigo.DB { + return db.db +} + +func (db *CLevelDB) Close() { + db.db.Close() + db.ro.Close() + db.wo.Close() + db.woSync.Close() +} + +func (db *CLevelDB) Print() { + iter := db.db.NewIterator(db.ro) + defer iter.Close() + for iter.Seek(nil); iter.Valid(); iter.Next() { + key := iter.Key() + value := iter.Value() + fmt.Printf("[%X]:\t[%X]\n", key, value) + } +} + +func (db *CLevelDB) Stats() map[string]string { + // TODO: Find the available properties for the C LevelDB implementation + keys := []string{} + + stats := make(map[string]string) + for _, key := range keys { + str, err := db.db.GetProperty(key) + if err == nil { + stats[key] = str + } + } + return stats +} + +func (db *CLevelDB) Iterator() Iterator { + return db.db.NewIterator(nil, nil) +} + +func (db *CLevelDB) NewBatch() Batch { + batch := levigo.NewWriteBatch() + return &cLevelDBBatch{db, batch} +} + +//-------------------------------------------------------------------------------- + +type cLevelDBBatch struct { + db *CLevelDB + batch *levigo.WriteBatch +} + +func (mBatch *cLevelDBBatch) Set(key, value []byte) { + mBatch.batch.Put(key, value) +} + +func (mBatch *cLevelDBBatch) Delete(key []byte) { + mBatch.batch.Delete(key) +} + +func (mBatch *cLevelDBBatch) Write() { + err := mBatch.db.db.Write(mBatch.db.wo, mBatch.batch) + if err != nil { + PanicCrisis(err) + } +} diff --git a/c_level_db_test.go b/c_level_db_test.go new file mode 100644 index 00000000..b9016149 --- /dev/null +++ b/c_level_db_test.go @@ -0,0 +1,86 @@ +// +build gcc + +package db + +import ( + "bytes" + "fmt" + "testing" + + . "github.com/tendermint/go-common" +) + +func BenchmarkRandomReadsWrites2(b *testing.B) { + b.StopTimer() + + numItems := int64(1000000) + internal := map[int64]int64{} + for i := 0; i < int(numItems); i++ { + internal[int64(i)] = int64(0) + } + db, err := NewCLevelDB(Fmt("test_%x", RandStr(12)), "") + if err != nil { + b.Fatal(err.Error()) + return + } + + fmt.Println("ok, starting") + b.StartTimer() + + for i := 0; i < b.N; i++ { + // Write something + { + idx := (int64(RandInt()) % numItems) + internal[idx] += 1 + val := internal[idx] + idxBytes := int642Bytes(int64(idx)) + valBytes := int642Bytes(int64(val)) + //fmt.Printf("Set %X -> %X\n", idxBytes, valBytes) + db.Set( + idxBytes, + valBytes, + ) + } + // Read something + { + idx := (int64(RandInt()) % numItems) + val := internal[idx] + idxBytes := int642Bytes(int64(idx)) + valBytes := db.Get(idxBytes) + //fmt.Printf("Get %X -> %X\n", idxBytes, valBytes) + if val == 0 { + if !bytes.Equal(valBytes, nil) { + b.Errorf("Expected %X for %v, got %X", + nil, idx, valBytes) + break + } + } else { + if len(valBytes) != 8 { + b.Errorf("Expected length 8 for %v, got %X", + idx, valBytes) + break + } + valGot := bytes2Int64(valBytes) + if val != valGot { + b.Errorf("Expected %v for %v, got %v", + val, idx, valGot) + break + } + } + } + } + + db.Close() +} + +/* +func int642Bytes(i int64) []byte { + buf := make([]byte, 8) + binary.BigEndian.PutUint64(buf, uint64(i)) + return buf +} + +func bytes2Int64(buf []byte) int64 { + return int64(binary.BigEndian.Uint64(buf)) +} +*/ diff --git a/db.go b/db.go new file mode 100644 index 00000000..b8849909 --- /dev/null +++ b/db.go @@ -0,0 +1,60 @@ +package db + +import . "github.com/tendermint/go-common" + +type DB interface { + Get([]byte) []byte + Set([]byte, []byte) + SetSync([]byte, []byte) + Delete([]byte) + DeleteSync([]byte) + Close() + NewBatch() Batch + + // For debugging + Print() + Iterator() Iterator + Stats() map[string]string +} + +type Batch interface { + Set(key, value []byte) + Delete(key []byte) + Write() +} + +type Iterator interface { + Next() bool + + Key() []byte + Value() []byte +} + +//----------------------------------------------------------------------------- + +const ( + LevelDBBackendStr = "leveldb" // legacy, defaults to goleveldb. + CLevelDBBackendStr = "cleveldb" + GoLevelDBBackendStr = "goleveldb" + MemDBBackendStr = "memdb" +) + +type dbCreator func(name string, dir string) (DB, error) + +var backends = map[string]dbCreator{} + +func registerDBCreator(backend string, creator dbCreator, force bool) { + _, ok := backends[backend] + if !force && ok { + return + } + backends[backend] = creator +} + +func NewDB(name string, backend string, dir string) DB { + db, err := backends[backend](name, dir) + if err != nil { + PanicSanity(Fmt("Error initializing DB: %v", err)) + } + return db +} diff --git a/go_level_db.go b/go_level_db.go new file mode 100644 index 00000000..1b4a937c --- /dev/null +++ b/go_level_db.go @@ -0,0 +1,147 @@ +package db + +import ( + "fmt" + "path" + + "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/errors" + "github.com/syndtr/goleveldb/leveldb/opt" + + . "github.com/tendermint/go-common" +) + +func init() { + dbCreator := func(name string, dir string) (DB, error) { + return NewGoLevelDB(name, dir) + } + registerDBCreator(LevelDBBackendStr, dbCreator, false) + registerDBCreator(GoLevelDBBackendStr, dbCreator, false) +} + +type GoLevelDB struct { + db *leveldb.DB +} + +func NewGoLevelDB(name string, dir string) (*GoLevelDB, error) { + dbPath := path.Join(dir, name+".db") + db, err := leveldb.OpenFile(dbPath, nil) + if err != nil { + return nil, err + } + database := &GoLevelDB{db: db} + return database, nil +} + +func (db *GoLevelDB) Get(key []byte) []byte { + res, err := db.db.Get(key, nil) + if err != nil { + if err == errors.ErrNotFound { + return nil + } else { + PanicCrisis(err) + } + } + return res +} + +func (db *GoLevelDB) Set(key []byte, value []byte) { + err := db.db.Put(key, value, nil) + if err != nil { + PanicCrisis(err) + } +} + +func (db *GoLevelDB) SetSync(key []byte, value []byte) { + err := db.db.Put(key, value, &opt.WriteOptions{Sync: true}) + if err != nil { + PanicCrisis(err) + } +} + +func (db *GoLevelDB) Delete(key []byte) { + err := db.db.Delete(key, nil) + if err != nil { + PanicCrisis(err) + } +} + +func (db *GoLevelDB) DeleteSync(key []byte) { + err := db.db.Delete(key, &opt.WriteOptions{Sync: true}) + if err != nil { + PanicCrisis(err) + } +} + +func (db *GoLevelDB) DB() *leveldb.DB { + return db.db +} + +func (db *GoLevelDB) Close() { + db.db.Close() +} + +func (db *GoLevelDB) Print() { + str, _ := db.db.GetProperty("leveldb.stats") + fmt.Printf("%v\n", str) + + iter := db.db.NewIterator(nil, nil) + for iter.Next() { + key := iter.Key() + value := iter.Value() + fmt.Printf("[%X]:\t[%X]\n", key, value) + } +} + +func (db *GoLevelDB) Stats() map[string]string { + keys := []string{ + "leveldb.num-files-at-level{n}", + "leveldb.stats", + "leveldb.sstables", + "leveldb.blockpool", + "leveldb.cachedblock", + "leveldb.openedtables", + "leveldb.alivesnaps", + "leveldb.aliveiters", + } + + stats := make(map[string]string) + for _, key := range keys { + str, err := db.db.GetProperty(key) + if err == nil { + stats[key] = str + } + } + return stats +} + +func (db *GoLevelDB) Iterator() Iterator { + return db.db.NewIterator(nil, nil) +} + +func (db *GoLevelDB) NewBatch() Batch { + batch := new(leveldb.Batch) + return &goLevelDBBatch{db, batch} +} + +//-------------------------------------------------------------------------------- + +type goLevelDBBatch struct { + db *GoLevelDB + batch *leveldb.Batch +} + +func (mBatch *goLevelDBBatch) Set(key, value []byte) { + mBatch.batch.Put(key, value) +} + +func (mBatch *goLevelDBBatch) Delete(key []byte) { + mBatch.batch.Delete(key) +} + +func (mBatch *goLevelDBBatch) Write() { + err := mBatch.db.db.Write(mBatch.batch, nil) + if err != nil { + PanicCrisis(err) + } +} diff --git a/go_level_db_test.go b/go_level_db_test.go new file mode 100644 index 00000000..24b64734 --- /dev/null +++ b/go_level_db_test.go @@ -0,0 +1,83 @@ +package db + +import ( + "bytes" + "encoding/binary" + "fmt" + "testing" + + . "github.com/tendermint/go-common" +) + +func BenchmarkRandomReadsWrites(b *testing.B) { + b.StopTimer() + + numItems := int64(1000000) + internal := map[int64]int64{} + for i := 0; i < int(numItems); i++ { + internal[int64(i)] = int64(0) + } + db, err := NewGoLevelDB(Fmt("test_%x", RandStr(12)), "") + if err != nil { + b.Fatal(err.Error()) + return + } + + fmt.Println("ok, starting") + b.StartTimer() + + for i := 0; i < b.N; i++ { + // Write something + { + idx := (int64(RandInt()) % numItems) + internal[idx] += 1 + val := internal[idx] + idxBytes := int642Bytes(int64(idx)) + valBytes := int642Bytes(int64(val)) + //fmt.Printf("Set %X -> %X\n", idxBytes, valBytes) + db.Set( + idxBytes, + valBytes, + ) + } + // Read something + { + idx := (int64(RandInt()) % numItems) + val := internal[idx] + idxBytes := int642Bytes(int64(idx)) + valBytes := db.Get(idxBytes) + //fmt.Printf("Get %X -> %X\n", idxBytes, valBytes) + if val == 0 { + if !bytes.Equal(valBytes, nil) { + b.Errorf("Expected %X for %v, got %X", + nil, idx, valBytes) + break + } + } else { + if len(valBytes) != 8 { + b.Errorf("Expected length 8 for %v, got %X", + idx, valBytes) + break + } + valGot := bytes2Int64(valBytes) + if val != valGot { + b.Errorf("Expected %v for %v, got %v", + val, idx, valGot) + break + } + } + } + } + + db.Close() +} + +func int642Bytes(i int64) []byte { + buf := make([]byte, 8) + binary.BigEndian.PutUint64(buf, uint64(i)) + return buf +} + +func bytes2Int64(buf []byte) int64 { + return int64(binary.BigEndian.Uint64(buf)) +} diff --git a/mem_db.go b/mem_db.go new file mode 100644 index 00000000..28662429 --- /dev/null +++ b/mem_db.go @@ -0,0 +1,159 @@ +package db + +import ( + "fmt" + "sync" +) + +func init() { + registerDBCreator(MemDBBackendStr, func(name string, dir string) (DB, error) { + return NewMemDB(), nil + }, false) +} + +type MemDB struct { + mtx sync.Mutex + db map[string][]byte +} + +func NewMemDB() *MemDB { + database := &MemDB{db: make(map[string][]byte)} + return database +} + +func (db *MemDB) Get(key []byte) []byte { + db.mtx.Lock() + defer db.mtx.Unlock() + return db.db[string(key)] +} + +func (db *MemDB) Set(key []byte, value []byte) { + db.mtx.Lock() + defer db.mtx.Unlock() + db.db[string(key)] = value +} + +func (db *MemDB) SetSync(key []byte, value []byte) { + db.mtx.Lock() + defer db.mtx.Unlock() + db.db[string(key)] = value +} + +func (db *MemDB) Delete(key []byte) { + db.mtx.Lock() + defer db.mtx.Unlock() + delete(db.db, string(key)) +} + +func (db *MemDB) DeleteSync(key []byte) { + db.mtx.Lock() + defer db.mtx.Unlock() + delete(db.db, string(key)) +} + +func (db *MemDB) Close() { + db.mtx.Lock() + defer db.mtx.Unlock() + db = nil +} + +func (db *MemDB) Print() { + db.mtx.Lock() + defer db.mtx.Unlock() + for key, value := range db.db { + fmt.Printf("[%X]:\t[%X]\n", []byte(key), value) + } +} + +func (db *MemDB) Stats() map[string]string { + stats := make(map[string]string) + stats["database.type"] = "memDB" + return stats +} + +type memDBIterator struct { + last int + keys []string + db *MemDB +} + +func newMemDBIterator() *memDBIterator { + return &memDBIterator{} +} + +func (it *memDBIterator) Next() bool { + if it.last >= len(it.keys) { + return false + } + it.last++ + return true +} + +func (it *memDBIterator) Key() []byte { + return []byte(it.keys[it.last]) +} + +func (it *memDBIterator) Value() []byte { + return it.db.Get(it.Key()) +} + +func (db *MemDB) Iterator() Iterator { + it := newMemDBIterator() + it.db = db + it.last = -1 + + db.mtx.Lock() + defer db.mtx.Unlock() + + // unfortunately we need a copy of all of the keys + for key, _ := range db.db { + it.keys = append(it.keys, key) + } + return it +} + +func (db *MemDB) NewBatch() Batch { + return &memDBBatch{db, nil} +} + +//-------------------------------------------------------------------------------- + +type memDBBatch struct { + db *MemDB + ops []operation +} + +type opType int + +const ( + opTypeSet = 1 + opTypeDelete = 2 +) + +type operation struct { + opType + key []byte + value []byte +} + +func (mBatch *memDBBatch) Set(key, value []byte) { + mBatch.ops = append(mBatch.ops, operation{opTypeSet, key, value}) +} + +func (mBatch *memDBBatch) Delete(key []byte) { + mBatch.ops = append(mBatch.ops, operation{opTypeDelete, key, nil}) +} + +func (mBatch *memDBBatch) Write() { + mBatch.db.mtx.Lock() + defer mBatch.db.mtx.Unlock() + + for _, op := range mBatch.ops { + if op.opType == opTypeSet { + mBatch.db.db[string(op.key)] = op.value + } else if op.opType == opTypeDelete { + delete(mBatch.db.db, string(op.key)) + } + } + +}