feat(compaction): Support Lmax to Lmax compaction (dgraph-io#1615)

This PR adds support for Lmax to Lmax compaction. These compactions are useful
when the last level of the LSM Tree contains stale data that might not get cleaned up
in the normal compaction process.
The Lmax Compactions can be enabled by setting opt.LmaxCompaction=true.

badger/cmd/info.go

@@ -256,8 +256,9 @@ func tableInfo(dir, valueDir string, db *badger.DB) {
 	y.Check(err)
 
 	fmt.Println()
-	fmt.Println("SSTable [Li, Id, Total Keys including internal keys] " +
-		"[Compression Ratio, Uncompressed Size, Index Size, BF Size] " +
+	// Total keys includes the internal keys as well.
+	fmt.Println("SSTable [Li, Id, Total Keys] " +
+		"[Compression Ratio, StaleData Ratio, Uncompressed Size, Index Size, BF Size] " +
 		"[Left Key, Version -> Right Key, Version]")
 	totalIndex := uint64(0)
 	totalBloomFilter := uint64(0)
@@ -268,9 +269,11 @@ func tableInfo(dir, valueDir string, db *badger.DB) {
 
 		compressionRatio := float64(t.UncompressedSize) /
 			float64(getInfo(fileInfos, t.ID)-int64(t.IndexSz))
-		fmt.Printf("SSTable [L%d, %03d, %07d] [%.2f, %s, %s, %s] [%20X, v%d -> %20X, v%d]\n",
-			t.Level, t.ID, t.KeyCount, compressionRatio, hbytes(int64(t.UncompressedSize)),
-			hbytes(int64(t.IndexSz)), hbytes(int64(t.BloomFilterSize)), lk, lt, rk, rt)
+		staleDataRatio := float64(t.StaleDataSize) / float64(t.UncompressedSize)
+		fmt.Printf("SSTable [L%d, %03d, %07d] [%.2f, %.2f, %s, %s, %s] [%20X, v%d -> %20X, v%d]\n",
+			t.Level, t.ID, t.KeyCount, compressionRatio, staleDataRatio,
+			hbytes(int64(t.UncompressedSize)), hbytes(int64(t.IndexSz)),
+			hbytes(int64(t.BloomFilterSize)), lk, lt, rk, rt)
 		totalIndex += uint64(t.IndexSz)
 		totalBloomFilter += uint64(t.BloomFilterSize)
 		totalCompressionRatio += compressionRatio

compaction.go

@@ -50,6 +50,10 @@ func (r keyRange) equals(dst keyRange) bool {
 }
 
 func (r *keyRange) extend(kr keyRange) {
+	// TODO(ibrahim): Is this needed?
+	if kr.isEmpty() {
+		return
+	}
 	if r.isEmpty() {
 		*r = kr
 	}
@@ -69,6 +73,11 @@ func (r keyRange) overlapsWith(dst keyRange) bool {
 	if r.isEmpty() {
 		return true
 	}
+	// TODO(ibrahim): Do you need this?
+	// Empty dst doesn't overlap with anything.
+	if dst.isEmpty() {
+		return false
+	}
 	if r.inf || dst.inf {
 		return true
 	}
@@ -176,7 +185,7 @@ func (cs *compactStatus) compareAndAdd(_ thisAndNextLevelRLocked, cd compactDef)
 	defer cs.Unlock()
 
 	tl := cd.thisLevel.level
-	y.AssertTruef(tl < len(cs.levels)-1, "Got level %d. Max levels: %d", tl, len(cs.levels))
+	y.AssertTruef(tl < len(cs.levels), "Got level %d. Max levels: %d", tl, len(cs.levels))
 	thisLevel := cs.levels[cd.thisLevel.level]
 	nextLevel := cs.levels[cd.nextLevel.level]
 
@@ -205,14 +214,17 @@ func (cs *compactStatus) delete(cd compactDef) {
 	defer cs.Unlock()
 
 	tl := cd.thisLevel.level
-	y.AssertTruef(tl < len(cs.levels)-1, "Got level %d. Max levels: %d", tl, len(cs.levels))
+	y.AssertTruef(tl < len(cs.levels), "Got level %d. Max levels: %d", tl, len(cs.levels))
 
 	thisLevel := cs.levels[cd.thisLevel.level]
 	nextLevel := cs.levels[cd.nextLevel.level]
 
 	thisLevel.delSize -= cd.thisSize
 	found := thisLevel.remove(cd.thisRange)
-	if !cd.nextRange.isEmpty() {
+	// The following check makes sense only if we're compacting more than one
+	// table. In case of the max level, we might rewrite a single table to
+	// remove stale data.
+	if cd.thisLevel != cd.nextLevel && !cd.nextRange.isEmpty() {
 		found = nextLevel.remove(cd.nextRange) && found
 	}
 

db.go

@@ -1888,9 +1888,10 @@ func (db *DB) LevelsToString() string {
 	for _, li := range levels {
 		b.WriteString(fmt.Sprintf(
 			"Level %d [%s]: NumTables: %02d. Size: %s of %s. Score: %.2f->%.2f"+
-				" Target FileSize: %s\n",
+				" StaleData: %s Target FileSize: %s\n",
 			li.Level, base(li.IsBaseLevel), li.NumTables,
-			h(li.Size), h(li.TargetSize), li.Score, li.Adjusted, h(li.TargetFileSize)))
+			h(li.Size), h(li.TargetSize), li.Score, li.Adjusted, h(li.StaleDatSize),
+			h(li.TargetFileSize)))
 	}
 	b.WriteString("Level Done\n")
 	return b.String()

fb/flatbuffer.fbs

@@ -23,6 +23,7 @@ table TableIndex {
   key_count:uint32;
   uncompressed_size:uint32;
   on_disk_size:uint32;
+  stale_data_size:uint32;
 }
 
 table BlockOffset {

go.sum

@@ -77,7 +77,6 @@ github.com/spf13/pflag v0.0.0-20170417173400-9e4c21054fa1/go.mod h1:DYY7MBk1bdzu
 github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
-github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
@@ -105,7 +104,6 @@ golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=

level_handler.go

@@ -32,15 +32,26 @@ type levelHandler struct {
 	// For level >= 1, tables are sorted by key ranges, which do not overlap.
 	// For level 0, tables are sorted by time.
 	// For level 0, newest table are at the back. Compact the oldest one first, which is at the front.
-	tables    []*table.Table
-	totalSize int64
+	tables         []*table.Table
+	totalSize      int64
+	totalStaleSize int64
 
 	// The following are initialized once and const.
 	level    int
 	strLevel string
 	db       *DB
 }
 
+func (s *levelHandler) isLastLevel() bool {
+	return s.level == s.db.opt.MaxLevels-1
+}
+
+func (s *levelHandler) getTotalStaleSize() int64 {
+	s.RLock()
+	defer s.RUnlock()
+	return s.totalStaleSize
+}
+
 func (s *levelHandler) getTotalSize() int64 {
 	s.RLock()
 	defer s.RUnlock()
@@ -54,8 +65,9 @@ func (s *levelHandler) initTables(tables []*table.Table) {
 
 	s.tables = tables
 	s.totalSize = 0
+	s.totalStaleSize = 0
 	for _, t := range tables {
-		s.totalSize += t.Size()
+		s.addSize(t)
 	}
 
 	if s.level == 0 {
@@ -89,7 +101,7 @@ func (s *levelHandler) deleteTables(toDel []*table.Table) error {
 			newTables = append(newTables, t)
 			continue
 		}
-		s.totalSize -= t.Size()
+		s.subtractSize(t)
 	}
 	s.tables = newTables
 
@@ -117,12 +129,12 @@ func (s *levelHandler) replaceTables(toDel, toAdd []*table.Table) error {
 			newTables = append(newTables, t)
 			continue
 		}
-		s.totalSize -= t.Size()
+		s.subtractSize(t)
 	}
 
 	// Increase totalSize first.
 	for _, t := range toAdd {
-		s.totalSize += t.Size()
+		s.addSize(t)
 		t.IncrRef()
 		newTables = append(newTables, t)
 	}
@@ -145,7 +157,7 @@ func (s *levelHandler) addTable(t *table.Table) {
 	s.Lock()
 	defer s.Unlock()
 
-	s.totalSize += t.Size() // Increase totalSize first.
+	s.addSize(t) // Increase totalSize first.
 	t.IncrRef()
 	s.tables = append(s.tables, t)
 }
@@ -191,11 +203,22 @@ func (s *levelHandler) tryAddLevel0Table(t *table.Table) bool {
 
 	s.tables = append(s.tables, t)
 	t.IncrRef()
-	s.totalSize += t.Size()
+	s.addSize(t)
 
 	return true
 }
 
+// This should be called while holding the lock on the level.
+func (s *levelHandler) addSize(t *table.Table) {
+	s.totalSize += t.Size()
+	s.totalStaleSize += int64(t.StaleDataSize())
+}
+
+// This should be called while holding the lock on the level.
+func (s *levelHandler) subtractSize(t *table.Table) {
+	s.totalSize -= t.Size()
+	s.totalStaleSize -= int64(t.StaleDataSize())
+}
 func (s *levelHandler) numTables() int {
 	s.RLock()
 	defer s.RUnlock()