Merge remote-tracking branch 'db/master' into unstable

LICENSE.md

@@ -0,0 +1,3 @@
+Tendermint Go-DB Copyright (C) 2015 All in Bits, Inc
+
+Released under the Apache2.0 license

README.md

@@ -0,0 +1 @@
+TODO: syndtr/goleveldb should be replaced with actual LevelDB instance

c_level_db.go

@@ -0,0 +1,152 @@
+// +build gcc
+
+package db
+
+import (
+	"fmt"
+	"path"
+
+	"github.com/jmhodges/levigo"
+
+	. "github.com/tendermint/go-common"
+)
+
+func init() {
+	dbCreator := func(name string, dir string) (DB, error) {
+		return NewCLevelDB(name, dir)
+	}
+	registerDBCreator(LevelDBBackendStr, dbCreator, true)
+	registerDBCreator(CLevelDBBackendStr, dbCreator, false)
+}
+
+type CLevelDB struct {
+	db     *levigo.DB
+	ro     *levigo.ReadOptions
+	wo     *levigo.WriteOptions
+	woSync *levigo.WriteOptions
+}
+
+func NewCLevelDB(name string, dir string) (*CLevelDB, error) {
+	dbPath := path.Join(dir, name+".db")
+
+	opts := levigo.NewOptions()
+	opts.SetCache(levigo.NewLRUCache(1 << 30))
+	opts.SetCreateIfMissing(true)
+	db, err := levigo.Open(dbPath, opts)
+	if err != nil {
+		return nil, err
+	}
+	ro := levigo.NewReadOptions()
+	wo := levigo.NewWriteOptions()
+	woSync := levigo.NewWriteOptions()
+	woSync.SetSync(true)
+	database := &CLevelDB{
+		db:     db,
+		ro:     ro,
+		wo:     wo,
+		woSync: woSync,
+	}
+	return database, nil
+}
+
+func (db *CLevelDB) Get(key []byte) []byte {
+	res, err := db.db.Get(db.ro, key)
+	if err != nil {
+		PanicCrisis(err)
+	}
+	return res
+}
+
+func (db *CLevelDB) Set(key []byte, value []byte) {
+	err := db.db.Put(db.wo, key, value)
+	if err != nil {
+		PanicCrisis(err)
+	}
+}
+
+func (db *CLevelDB) SetSync(key []byte, value []byte) {
+	err := db.db.Put(db.woSync, key, value)
+	if err != nil {
+		PanicCrisis(err)
+	}
+}
+
+func (db *CLevelDB) Delete(key []byte) {
+	err := db.db.Delete(db.wo, key)
+	if err != nil {
+		PanicCrisis(err)
+	}
+}
+
+func (db *CLevelDB) DeleteSync(key []byte) {
+	err := db.db.Delete(db.woSync, key)
+	if err != nil {
+		PanicCrisis(err)
+	}
+}
+
+func (db *CLevelDB) DB() *levigo.DB {
+	return db.db
+}
+
+func (db *CLevelDB) Close() {
+	db.db.Close()
+	db.ro.Close()
+	db.wo.Close()
+	db.woSync.Close()
+}
+
+func (db *CLevelDB) Print() {
+	iter := db.db.NewIterator(db.ro)
+	defer iter.Close()
+	for iter.Seek(nil); iter.Valid(); iter.Next() {
+		key := iter.Key()
+		value := iter.Value()
+		fmt.Printf("[%X]:\t[%X]\n", key, value)
+	}
+}
+
+func (db *CLevelDB) Stats() map[string]string {
+	// TODO: Find the available properties for the C LevelDB implementation
+	keys := []string{}
+
+	stats := make(map[string]string)
+	for _, key := range keys {
+		str, err := db.db.GetProperty(key)
+		if err == nil {
+			stats[key] = str
+		}
+	}
+	return stats
+}
+
+func (db *CLevelDB) Iterator() Iterator {
+	return db.db.NewIterator(nil, nil)
+}
+
+func (db *CLevelDB) NewBatch() Batch {
+	batch := levigo.NewWriteBatch()
+	return &cLevelDBBatch{db, batch}
+}
+
+//--------------------------------------------------------------------------------
+
+type cLevelDBBatch struct {
+	db    *CLevelDB
+	batch *levigo.WriteBatch
+}
+
+func (mBatch *cLevelDBBatch) Set(key, value []byte) {
+	mBatch.batch.Put(key, value)
+}
+
+func (mBatch *cLevelDBBatch) Delete(key []byte) {
+	mBatch.batch.Delete(key)
+}
+
+func (mBatch *cLevelDBBatch) Write() {
+	err := mBatch.db.db.Write(mBatch.db.wo, mBatch.batch)
+	if err != nil {
+		PanicCrisis(err)
+	}
+}

c_level_db_test.go

@@ -0,0 +1,86 @@
+// +build gcc
+
+package db
+
+import (
+	"bytes"
+	"fmt"
+	"testing"
+
+	. "github.com/tendermint/go-common"
+)
+
+func BenchmarkRandomReadsWrites2(b *testing.B) {
+	b.StopTimer()
+
+	numItems := int64(1000000)
+	internal := map[int64]int64{}
+	for i := 0; i < int(numItems); i++ {
+		internal[int64(i)] = int64(0)
+	}
+	db, err := NewCLevelDB(Fmt("test_%x", RandStr(12)), "")
+	if err != nil {
+		b.Fatal(err.Error())
+		return
+	}
+
+	fmt.Println("ok, starting")
+	b.StartTimer()
+
+	for i := 0; i < b.N; i++ {
+		// Write something
+		{
+			idx := (int64(RandInt()) % numItems)
+			internal[idx] += 1
+			val := internal[idx]
+			idxBytes := int642Bytes(int64(idx))
+			valBytes := int642Bytes(int64(val))
+			//fmt.Printf("Set %X -> %X\n", idxBytes, valBytes)
+			db.Set(
+				idxBytes,
+				valBytes,
+			)
+		}
+		// Read something
+		{
+			idx := (int64(RandInt()) % numItems)
+			val := internal[idx]
+			idxBytes := int642Bytes(int64(idx))
+			valBytes := db.Get(idxBytes)
+			//fmt.Printf("Get %X -> %X\n", idxBytes, valBytes)
+			if val == 0 {
+				if !bytes.Equal(valBytes, nil) {
+					b.Errorf("Expected %X for %v, got %X",
+						nil, idx, valBytes)
+					break
+				}
+			} else {
+				if len(valBytes) != 8 {
+					b.Errorf("Expected length 8 for %v, got %X",
+						idx, valBytes)
+					break
+				}
+				valGot := bytes2Int64(valBytes)
+				if val != valGot {
+					b.Errorf("Expected %v for %v, got %v",
+						val, idx, valGot)
+					break
+				}
+			}
+		}
+	}
+
+	db.Close()
+}
+
+/*
+func int642Bytes(i int64) []byte {
+	buf := make([]byte, 8)
+	binary.BigEndian.PutUint64(buf, uint64(i))
+	return buf
+}
+
+func bytes2Int64(buf []byte) int64 {
+	return int64(binary.BigEndian.Uint64(buf))
+}
+*/

db.go

@@ -0,0 +1,60 @@
+package db
+
+import . "github.com/tendermint/go-common"
+
+type DB interface {
+	Get([]byte) []byte
+	Set([]byte, []byte)
+	SetSync([]byte, []byte)
+	Delete([]byte)
+	DeleteSync([]byte)
+	Close()
+	NewBatch() Batch
+
+	// For debugging
+	Print()
+	Iterator() Iterator
+	Stats() map[string]string
+}
+
+type Batch interface {
+	Set(key, value []byte)
+	Delete(key []byte)
+	Write()
+}
+
+type Iterator interface {
+	Next() bool
+
+	Key() []byte
+	Value() []byte
+}
+
+//-----------------------------------------------------------------------------
+
+const (
+	LevelDBBackendStr   = "leveldb" // legacy, defaults to goleveldb.
+	CLevelDBBackendStr  = "cleveldb"
+	GoLevelDBBackendStr = "goleveldb"
+	MemDBBackendStr     = "memdb"
+)
+
+type dbCreator func(name string, dir string) (DB, error)
+
+var backends = map[string]dbCreator{}
+
+func registerDBCreator(backend string, creator dbCreator, force bool) {
+	_, ok := backends[backend]
+	if !force && ok {
+		return
+	}
+	backends[backend] = creator
+}
+
+func NewDB(name string, backend string, dir string) DB {
+	db, err := backends[backend](name, dir)
+	if err != nil {
+		PanicSanity(Fmt("Error initializing DB: %v", err))
+	}
+	return db
+}

go_level_db.go

@@ -0,0 +1,147 @@
+package db
+
+import (
+	"fmt"
+	"path"
+
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/errors"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+
+	. "github.com/tendermint/go-common"
+)
+
+func init() {
+	dbCreator := func(name string, dir string) (DB, error) {
+		return NewGoLevelDB(name, dir)
+	}
+	registerDBCreator(LevelDBBackendStr, dbCreator, false)
+	registerDBCreator(GoLevelDBBackendStr, dbCreator, false)
+}
+
+type GoLevelDB struct {
+	db *leveldb.DB
+}
+
+func NewGoLevelDB(name string, dir string) (*GoLevelDB, error) {
+	dbPath := path.Join(dir, name+".db")
+	db, err := leveldb.OpenFile(dbPath, nil)
+	if err != nil {
+		return nil, err
+	}
+	database := &GoLevelDB{db: db}
+	return database, nil
+}
+
+func (db *GoLevelDB) Get(key []byte) []byte {
+	res, err := db.db.Get(key, nil)
+	if err != nil {
+		if err == errors.ErrNotFound {
+			return nil
+		} else {
+			PanicCrisis(err)
+		}
+	}
+	return res
+}
+
+func (db *GoLevelDB) Set(key []byte, value []byte) {
+	err := db.db.Put(key, value, nil)
+	if err != nil {
+		PanicCrisis(err)
+	}
+}
+
+func (db *GoLevelDB) SetSync(key []byte, value []byte) {
+	err := db.db.Put(key, value, &opt.WriteOptions{Sync: true})
+	if err != nil {
+		PanicCrisis(err)
+	}
+}
+
+func (db *GoLevelDB) Delete(key []byte) {
+	err := db.db.Delete(key, nil)
+	if err != nil {
+		PanicCrisis(err)
+	}
+}
+
+func (db *GoLevelDB) DeleteSync(key []byte) {
+	err := db.db.Delete(key, &opt.WriteOptions{Sync: true})
+	if err != nil {
+		PanicCrisis(err)
+	}
+}
+
+func (db *GoLevelDB) DB() *leveldb.DB {
+	return db.db
+}
+
+func (db *GoLevelDB) Close() {
+	db.db.Close()
+}
+
+func (db *GoLevelDB) Print() {
+	str, _ := db.db.GetProperty("leveldb.stats")
+	fmt.Printf("%v\n", str)
+
+	iter := db.db.NewIterator(nil, nil)
+	for iter.Next() {
+		key := iter.Key()
+		value := iter.Value()
+		fmt.Printf("[%X]:\t[%X]\n", key, value)
+	}
+}
+
+func (db *GoLevelDB) Stats() map[string]string {
+	keys := []string{
+		"leveldb.num-files-at-level{n}",
+		"leveldb.stats",
+		"leveldb.sstables",
+		"leveldb.blockpool",
+		"leveldb.cachedblock",
+		"leveldb.openedtables",
+		"leveldb.alivesnaps",
+		"leveldb.aliveiters",
+	}
+
+	stats := make(map[string]string)
+	for _, key := range keys {
+		str, err := db.db.GetProperty(key)
+		if err == nil {
+			stats[key] = str
+		}
+	}
+	return stats
+}
+
+func (db *GoLevelDB) Iterator() Iterator {
+	return db.db.NewIterator(nil, nil)
+}
+
+func (db *GoLevelDB) NewBatch() Batch {
+	batch := new(leveldb.Batch)
+	return &goLevelDBBatch{db, batch}
+}
+
+//--------------------------------------------------------------------------------
+
+type goLevelDBBatch struct {
+	db    *GoLevelDB
+	batch *leveldb.Batch
+}
+
+func (mBatch *goLevelDBBatch) Set(key, value []byte) {
+	mBatch.batch.Put(key, value)
+}
+
+func (mBatch *goLevelDBBatch) Delete(key []byte) {
+	mBatch.batch.Delete(key)
+}
+
+func (mBatch *goLevelDBBatch) Write() {
+	err := mBatch.db.db.Write(mBatch.batch, nil)
+	if err != nil {
+		PanicCrisis(err)
+	}
+}

go_level_db_test.go

@@ -0,0 +1,83 @@
+package db
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"testing"
+
+	. "github.com/tendermint/go-common"
+)
+
+func BenchmarkRandomReadsWrites(b *testing.B) {
+	b.StopTimer()
+
+	numItems := int64(1000000)
+	internal := map[int64]int64{}
+	for i := 0; i < int(numItems); i++ {
+		internal[int64(i)] = int64(0)
+	}
+	db, err := NewGoLevelDB(Fmt("test_%x", RandStr(12)), "")
+	if err != nil {
+		b.Fatal(err.Error())
+		return
+	}
+
+	fmt.Println("ok, starting")
+	b.StartTimer()
+
+	for i := 0; i < b.N; i++ {
+		// Write something
+		{
+			idx := (int64(RandInt()) % numItems)
+			internal[idx] += 1
+			val := internal[idx]
+			idxBytes := int642Bytes(int64(idx))
+			valBytes := int642Bytes(int64(val))
+			//fmt.Printf("Set %X -> %X\n", idxBytes, valBytes)
+			db.Set(
+				idxBytes,
+				valBytes,
+			)
+		}
+		// Read something
+		{
+			idx := (int64(RandInt()) % numItems)
+			val := internal[idx]
+			idxBytes := int642Bytes(int64(idx))
+			valBytes := db.Get(idxBytes)
+			//fmt.Printf("Get %X -> %X\n", idxBytes, valBytes)
+			if val == 0 {
+				if !bytes.Equal(valBytes, nil) {
+					b.Errorf("Expected %X for %v, got %X",
+						nil, idx, valBytes)
+					break
+				}
+			} else {
+				if len(valBytes) != 8 {
+					b.Errorf("Expected length 8 for %v, got %X",
+						idx, valBytes)
+					break
+				}
+				valGot := bytes2Int64(valBytes)
+				if val != valGot {
+					b.Errorf("Expected %v for %v, got %v",
+						val, idx, valGot)
+					break
+				}
+			}
+		}
+	}
+
+	db.Close()
+}
+
+func int642Bytes(i int64) []byte {
+	buf := make([]byte, 8)
+	binary.BigEndian.PutUint64(buf, uint64(i))
+	return buf
+}
+
+func bytes2Int64(buf []byte) int64 {
+	return int64(binary.BigEndian.Uint64(buf))
+}

mem_db.go

@@ -0,0 +1,159 @@
+package db
+
+import (
+	"fmt"
+	"sync"
+)
+
+func init() {
+	registerDBCreator(MemDBBackendStr, func(name string, dir string) (DB, error) {
+		return NewMemDB(), nil
+	}, false)
+}
+
+type MemDB struct {
+	mtx sync.Mutex
+	db  map[string][]byte
+}
+
+func NewMemDB() *MemDB {
+	database := &MemDB{db: make(map[string][]byte)}
+	return database
+}
+
+func (db *MemDB) Get(key []byte) []byte {
+	db.mtx.Lock()
+	defer db.mtx.Unlock()
+	return db.db[string(key)]
+}
+
+func (db *MemDB) Set(key []byte, value []byte) {
+	db.mtx.Lock()
+	defer db.mtx.Unlock()
+	db.db[string(key)] = value
+}
+
+func (db *MemDB) SetSync(key []byte, value []byte) {
+	db.mtx.Lock()
+	defer db.mtx.Unlock()
+	db.db[string(key)] = value
+}
+
+func (db *MemDB) Delete(key []byte) {
+	db.mtx.Lock()
+	defer db.mtx.Unlock()
+	delete(db.db, string(key))
+}
+
+func (db *MemDB) DeleteSync(key []byte) {
+	db.mtx.Lock()
+	defer db.mtx.Unlock()
+	delete(db.db, string(key))
+}
+
+func (db *MemDB) Close() {
+	db.mtx.Lock()
+	defer db.mtx.Unlock()
+	db = nil
+}
+
+func (db *MemDB) Print() {
+	db.mtx.Lock()
+	defer db.mtx.Unlock()
+	for key, value := range db.db {
+		fmt.Printf("[%X]:\t[%X]\n", []byte(key), value)
+	}
+}
+
+func (db *MemDB) Stats() map[string]string {
+	stats := make(map[string]string)
+	stats["database.type"] = "memDB"
+	return stats
+}
+
+type memDBIterator struct {
+	last int
+	keys []string
+	db   *MemDB
+}
+
+func newMemDBIterator() *memDBIterator {
+	return &memDBIterator{}
+}
+
+func (it *memDBIterator) Next() bool {
+	if it.last >= len(it.keys) {
+		return false
+	}
+	it.last++
+	return true
+}
+
+func (it *memDBIterator) Key() []byte {
+	return []byte(it.keys[it.last])
+}
+
+func (it *memDBIterator) Value() []byte {
+	return it.db.Get(it.Key())
+}
+
+func (db *MemDB) Iterator() Iterator {
+	it := newMemDBIterator()
+	it.db = db
+	it.last = -1
+
+	db.mtx.Lock()
+	defer db.mtx.Unlock()
+
+	// unfortunately we need a copy of all of the keys
+	for key, _ := range db.db {
+		it.keys = append(it.keys, key)
+	}
+	return it
+}
+
+func (db *MemDB) NewBatch() Batch {
+	return &memDBBatch{db, nil}
+}
+
+//--------------------------------------------------------------------------------
+
+type memDBBatch struct {
+	db  *MemDB
+	ops []operation
+}
+
+type opType int
+
+const (
+	opTypeSet    = 1
+	opTypeDelete = 2
+)
+
+type operation struct {
+	opType
+	key   []byte
+	value []byte
+}
+
+func (mBatch *memDBBatch) Set(key, value []byte) {
+	mBatch.ops = append(mBatch.ops, operation{opTypeSet, key, value})
+}
+
+func (mBatch *memDBBatch) Delete(key []byte) {
+	mBatch.ops = append(mBatch.ops, operation{opTypeDelete, key, nil})
+}
+
+func (mBatch *memDBBatch) Write() {
+	mBatch.db.mtx.Lock()
+	defer mBatch.db.mtx.Unlock()
+
+	for _, op := range mBatch.ops {
+		if op.opType == opTypeSet {
+			mBatch.db.db[string(op.key)] = op.value
+		} else if op.opType == opTypeDelete {
+			delete(mBatch.db.db, string(op.key))
+		}
+	}
+
+}