Skip to content

Commit

Permalink
Managed Transactions
Browse files Browse the repository at this point in the history 
- Allow a way to specify the read and commit timestamp for a transaction. This would be useful for Dgraph.
- Add tests for managed transactions.

Various cleanup:

- Don't return an error while creating a transaction.
- Make Entry struct private, now that BatchSet is no longer public.
- Put all errors in one var block.
  • Loading branch information
@manishrjain
manishrjain committed Oct 2, 2017
1 parent ffaaa66 commit a6860cb
Show file tree
Hide file tree
Showing 9 changed files with 235 additions and 174 deletions.
60 changes: 30 additions & 30 deletions errors.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,43 +22,31 @@ import (
"github.com/pkg/errors"
)

// ErrInvalidDir is returned when Badger cannot find the directory
// from where it is supposed to load the key-value store.
var ErrInvalidDir = errors.New("Invalid Dir, directory does not exist")

// ErrValueLogSize is returned when opt.ValueLogFileSize option is not within the valid
// range.
var ErrValueLogSize = errors.New("Invalid ValueLogFileSize, must be between 1MB and 2GB")

// ErrKeyNotFound is returned when key isn't found on a txn.Get.
var ErrKeyNotFound = errors.New("Key not found")
var (
// ErrInvalidDir is returned when Badger cannot find the directory
// from where it is supposed to load the key-value store.
ErrInvalidDir = errors.New("Invalid Dir, directory does not exist")

// ErrTxnTooBig is returned if too many writes are fit into a single transaction.
var ErrTxnTooBig = errors.New("Txn is too big to fit into one request.")
// ErrValueLogSize is returned when opt.ValueLogFileSize option is not within the valid
// range.
ErrValueLogSize = errors.New("Invalid ValueLogFileSize, must be between 1MB and 2GB")

// ErrConflict is returned when a transaction conflicts with another transaction. This can happen if
// the read rows had been updated concurrently by another transaction.
var ErrConflict = errors.New("Transaction Conflict. Please retry.")
// ErrKeyNotFound is returned when key isn't found on a txn.Get.
ErrKeyNotFound = errors.New("Key not found")

// ErrReadOnlyTxn is returned if an update function is called on a read-only transaction.
var ErrReadOnlyTxn = errors.New("No sets or deletes are allowed in a read-only transaction.")
// ErrTxnTooBig is returned if too many writes are fit into a single transaction.
ErrTxnTooBig = errors.New("Txn is too big to fit into one request.")

// ErrEmptyKey is returned if an empty key is passed on an update function.
var ErrEmptyKey = errors.New("Key cannot be empty.")
// ErrConflict is returned when a transaction conflicts with another transaction. This can happen if
// the read rows had been updated concurrently by another transaction.
ErrConflict = errors.New("Transaction Conflict. Please retry.")

const maxKeySize = 1 << 20
// ErrReadOnlyTxn is returned if an update function is called on a read-only transaction.
ErrReadOnlyTxn = errors.New("No sets or deletes are allowed in a read-only transaction.")

func exceedsMaxKeySizeError(key []byte) error {
return errors.Errorf("Key with size %d exceeded %dMB limit. Key:\n%s",
len(key), maxKeySize<<20, hex.Dump(key[:1<<10]))
}

func exceedsMaxValueSizeError(value []byte, maxValueSize int64) error {
return errors.Errorf("Value with size %d exceeded ValueLogFileSize (%dMB). Key:\n%s",
len(value), maxValueSize<<20, hex.Dump(value[:1<<10]))
}
// ErrEmptyKey is returned if an empty key is passed on an update function.
ErrEmptyKey = errors.New("Key cannot be empty.")

var (
// ErrRetry is returned when a log file containing the value is not found.
// This usually indicates that it may have been garbage collected, and the
// operation needs to be retried.
Expand All @@ -80,3 +68,15 @@ var (
// ErrInvalidRequest is returned if the user request is invalid.
ErrInvalidRequest = errors.New("Invalid request")
)

const maxKeySize = 1 << 20

func exceedsMaxKeySizeError(key []byte) error {
return errors.Errorf("Key with size %d exceeded %dMB limit. Key:\n%s",
len(key), maxKeySize<<20, hex.Dump(key[:1<<10]))
}

func exceedsMaxValueSizeError(value []byte, maxValueSize int64) error {
return errors.Errorf("Value with size %d exceeded ValueLogFileSize (%dMB). Key:\n%s",
len(value), maxValueSize<<20, hex.Dump(value[:1<<10]))
}
10 changes: 5 additions & 5 deletions kv.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -217,7 +217,7 @@ func NewKV(optParam *Options) (out *KV, err error) {
}

first := true
fn := func(e Entry, vp valuePointer) error { // Function for replaying.
fn := func(e entry, vp valuePointer) error { // Function for replaying.
if first {
out.elog.Printf("First key=%s\n", e.Key)
}
Expand Down Expand Up @@ -503,7 +503,7 @@ var requestPool = sync.Pool{
},
}

func (s *KV) shouldWriteValueToLSM(e Entry) bool {
func (s *KV) shouldWriteValueToLSM(e entry) bool {
return len(e.Value) < s.opt.ValueThreshold
}

Expand Down Expand Up @@ -646,7 +646,7 @@ func (s *KV) doWrites(lc *y.Closer) {
}
}

func (s *KV) sendToWriteCh(entries []*Entry) (*request, error) {
func (s *KV) sendToWriteCh(entries []*entry) (*request, error) {
var count, size int64
for _, e := range entries {
size += int64(s.opt.estimateSize(e))
Expand All @@ -671,7 +671,7 @@ func (s *KV) sendToWriteCh(entries []*Entry) (*request, error) {
// batchSet applies a list of badger.Entry. If a request level error occurs it
// will be returned.
// Check(kv.BatchSet(entries))
func (s *KV) batchSet(entries []*Entry) error {
func (s *KV) batchSet(entries []*entry) error {
req, err := s.sendToWriteCh(entries)
if err != nil {
return err
Expand All @@ -690,7 +690,7 @@ func (s *KV) batchSet(entries []*Entry) error {
// err := kv.BatchSetAsync(entries, func(err error)) {
// Check(err)
// }
func (s *KV) batchSetAsync(entries []*Entry, f func(error)) error {
func (s *KV) batchSetAsync(entries []*entry, f func(error)) error {
req, err := s.sendToWriteCh(entries)
if err != nil {
return err
Expand Down
76 changes: 27 additions & 49 deletions kv_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,22 +59,19 @@ func getItemValue(t *testing.T, item *KVItem) (val []byte) {
}

func txnSet(t *testing.T, kv *KV, key []byte, val []byte, meta byte) {
txn, err := kv.NewTransaction(true)
require.NoError(t, err)
txn := kv.NewTransaction(true)
require.NoError(t, txn.Set(key, val, meta))
require.NoError(t, txn.Commit(nil))
}

func txnDelete(t *testing.T, kv *KV, key []byte) {
txn, err := kv.NewTransaction(true)
require.NoError(t, err)
txn := kv.NewTransaction(true)
require.NoError(t, txn.Delete(key))
require.NoError(t, txn.Commit(nil))
}

func txnGet(t *testing.T, kv *KV, key []byte) (KVItem, error) {
txn, err := kv.NewTransaction(false)
require.NoError(t, err)
txn := kv.NewTransaction(false)
return txn.Get(key)
}

Expand Down Expand Up @@ -121,7 +118,7 @@ func TestConcurrentWrite(t *testing.T) {
opt.PrefetchSize = 10
opt.PrefetchValues = true

txn, err := kv.NewTransaction(true)
txn := kv.NewTransaction(true)
it := txn.NewIterator(opt)
defer it.Close()
var i, j int
Expand Down Expand Up @@ -242,18 +239,15 @@ func TestGetMore(t *testing.T) {
// n := 500000
n := 10000
m := 49 // Increasing would cause ErrTxnTooBig
fmt.Println("writing")
for i := 0; i < n; i += m {
txn, err := kv.NewTransaction(true)
require.NoError(t, err)
txn := kv.NewTransaction(true)
for j := i; j < i+m && j < n; j++ {
require.NoError(t, txn.Set(data(j), data(j), 0))
}
require.NoError(t, txn.Commit(nil))
}
require.NoError(t, kv.validate())

fmt.Println("retrieving")
for i := 0; i < n; i++ {
item, err := txnGet(t, kv, data(i))
if err != nil {
Expand All @@ -263,10 +257,8 @@ func TestGetMore(t *testing.T) {
}

// Overwrite
fmt.Println("overwriting")
for i := 0; i < n; i += m {
txn, err := kv.NewTransaction(true)
require.NoError(t, err)
txn := kv.NewTransaction(true)
for j := i; j < i+m && j < n; j++ {
require.NoError(t, txn.Set(data(j),
// Use a long value that will certainly exceed value threshold.
Expand All @@ -277,7 +269,6 @@ func TestGetMore(t *testing.T) {
}
require.NoError(t, kv.validate())

fmt.Println("testing")
for i := 0; i < n; i++ {
k := []byte(fmt.Sprintf("%09d", i))
expectedValue := fmt.Sprintf("zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz%09d", i)
Expand All @@ -294,8 +285,7 @@ func TestGetMore(t *testing.T) {
fmt.Printf("wanted=%q Item: %s\n", k, item.ToString())
fmt.Printf("on re-run, got version: %+v\n", vs)

txn, err := kv.NewTransaction(false)
require.NoError(t, err)
txn := kv.NewTransaction(false)
itr := txn.NewIterator(DefaultIteratorOptions)
for itr.Seek(k0); itr.Valid(); itr.Next() {
item := itr.Item()
Expand All @@ -314,8 +304,7 @@ func TestGetMore(t *testing.T) {
if (i % 10000) == 0 {
fmt.Printf("Deleting i=%d\n", i)
}
txn, err := kv.NewTransaction(true)
require.NoError(t, err)
txn := kv.NewTransaction(true)
for j := i; j < i+m && j < n; j++ {
require.NoError(t, txn.Delete([]byte(fmt.Sprintf("%09d", j))))
}
Expand All @@ -331,7 +320,6 @@ func TestGetMore(t *testing.T) {
item, err := txnGet(t, kv, []byte(k))
require.Equal(t, ErrKeyNotFound, err, "wanted=%q item=%s\n", k, item.ToString())
}
fmt.Println("Done and closing")
}

// Put a lot of data to move some data to disk.
Expand All @@ -352,10 +340,9 @@ func TestExistsMore(t *testing.T) {
m := 49
for i := 0; i < n; i += m {
if (i % 1000) == 0 {
fmt.Printf("Putting i=%d\n", i)
t.Logf("Putting i=%d\n", i)
}
txn, err := kv.NewTransaction(true)
require.NoError(t, err)
txn := kv.NewTransaction(true)
for j := i; j < i+m && j < n; j++ {
require.NoError(t, txn.Set([]byte(fmt.Sprintf("%09d", j)),
[]byte(fmt.Sprintf("%09d", j)),
Expand All @@ -381,8 +368,7 @@ func TestExistsMore(t *testing.T) {
if (i % 1000) == 0 {
fmt.Printf("Deleting i=%d\n", i)
}
txn, err := kv.NewTransaction(true)
require.NoError(t, err)
txn := kv.NewTransaction(true)
for j := i; j < i+m && j < n; j++ {
require.NoError(t, txn.Delete([]byte(fmt.Sprintf("%09d", j))))
}
Expand Down Expand Up @@ -428,8 +414,7 @@ func TestIterate2Basic(t *testing.T) {
opt.PrefetchValues = true
opt.PrefetchSize = 10

txn, err := kv.NewTransaction(false)
require.NoError(t, err)
txn := kv.NewTransaction(false)
it := txn.NewIterator(opt)
{
var count int
Expand Down Expand Up @@ -535,14 +520,12 @@ func TestIterateDeleted(t *testing.T) {

iterOpt := DefaultIteratorOptions
iterOpt.PrefetchValues = false
txn, err := ps.NewTransaction(false)
require.NoError(t, err)
txn := ps.NewTransaction(false)
idxIt := txn.NewIterator(iterOpt)
defer idxIt.Close()

count := 0
txn2, err := ps.NewTransaction(true)
require.NoError(t, err)
txn2 := ps.NewTransaction(true)
prefix := []byte("Key")
for idxIt.Seek(prefix); idxIt.Valid(); idxIt.Next() {
key := idxIt.Item().Key()
Expand All @@ -559,8 +542,7 @@ func TestIterateDeleted(t *testing.T) {

for _, prefetch := range [...]bool{true, false} {
t.Run(fmt.Sprintf("Prefetch=%t", prefetch), func(t *testing.T) {
txn, err := ps.NewTransaction(false)
require.NoError(t, err)
txn := ps.NewTransaction(false)
iterOpt = DefaultIteratorOptions
iterOpt.PrefetchValues = prefetch
idxIt = txn.NewIterator(iterOpt)
Expand Down Expand Up @@ -646,13 +628,12 @@ func TestBigKeyValuePairs(t *testing.T) {
bigV := make([]byte, opt.ValueLogFileSize+1)
small := make([]byte, 10)

txn, err := kv.NewTransaction(true)
txn := kv.NewTransaction(true)
require.Regexp(t, regexp.MustCompile("Key.*exceeded"), txn.Set(bigK, small, 0))
txn, err = kv.NewTransaction(true)
txn = kv.NewTransaction(true)
require.Regexp(t, regexp.MustCompile("Value.*exceeded"), txn.Set(small, bigV, 0))

txn, err = kv.NewTransaction(true)
require.NoError(t, err)
txn = kv.NewTransaction(true)
require.NoError(t, txn.Set(small, small, 0x00))
require.Regexp(t, regexp.MustCompile("Key.*exceeded"), txn.Set(bigK, bigV, 0x00))

Expand All @@ -678,9 +659,9 @@ func TestIteratorPrefetchSize(t *testing.T) {

n := 100
for i := 0; i < n; i++ {
if (i % 10) == 0 {
t.Logf("Put i=%d\n", i)
}
// if (i % 10) == 0 {
// t.Logf("Put i=%d\n", i)
// }
txnSet(t, kv, bkey(i), bval(i), byte(i%127))
}

Expand All @@ -690,8 +671,7 @@ func TestIteratorPrefetchSize(t *testing.T) {
opt.PrefetchSize = prefetchSize

var count int
txn, err := kv.NewTransaction(false)
require.NoError(t, err)
txn := kv.NewTransaction(false)
it := txn.NewIterator(opt)
{
t.Log("Starting first basic iteration")
Expand Down Expand Up @@ -724,11 +704,10 @@ func TestSetIfAbsentAsync(t *testing.T) {

n := 1000
for i := 0; i < n; i++ {
if (i % 10) == 0 {
t.Logf("Put i=%d\n", i)
}
txn, err := kv.NewTransaction(true)
require.NoError(t, err)
// if (i % 10) == 0 {
// t.Logf("Put i=%d\n", i)
// }
txn := kv.NewTransaction(true)
_, err = txn.Get(bkey(i))
require.Equal(t, ErrKeyNotFound, err)
require.NoError(t, txn.Set(bkey(i), nil, byte(i%127)))
Expand All @@ -740,8 +719,7 @@ func TestSetIfAbsentAsync(t *testing.T) {
require.NoError(t, err)

opt := DefaultIteratorOptions
txn, err := kv.NewTransaction(false)
require.NoError(t, err)
txn := kv.NewTransaction(false)
var count int
it := txn.NewIterator(opt)
{
Expand Down
4 changes: 2 additions & 2 deletions options.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,6 @@ var DefaultOptions = Options{
ValueThreshold: 20,
}

func (opt *Options) estimateSize(entry *Entry) int {
return entry.estimateSize(opt.ValueThreshold)
func (opt *Options) estimateSize(e *entry) int {
return e.estimateSize(opt.ValueThreshold)
}
Loading

0 comments on commit a6860cb

Please sign in to comment.