forked from dgraph-io/badger
-
Notifications
You must be signed in to change notification settings - Fork 0
/
discard.go
168 lines (146 loc) · 4.01 KB
/
discard.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
/*
* Copyright 2020 Dgraph Labs, Inc. and Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package badger
import (
"encoding/binary"
"os"
"path/filepath"
"sort"
"sync"
"github.com/0x5459/badger/v4/y"
"github.com/dgraph-io/ristretto/z"
)
// discardStats keeps track of the amount of data that could be discarded for
// a given logfile.
type discardStats struct {
sync.Mutex
*z.MmapFile
opt Options
nextEmptySlot int
}
const discardFname string = "DISCARD"
func InitDiscardStats(opt Options) (*discardStats, error) {
fname := filepath.Join(opt.ValueDir, discardFname)
// 1GB file can store 67M discard entries. Each entry is 16 bytes.
mf, err := z.OpenMmapFile(fname, os.O_CREATE|os.O_RDWR, 1<<20)
lf := &discardStats{
MmapFile: mf,
opt: opt,
}
if err == z.NewFile {
// We don't need to zero out the entire 1GB.
lf.zeroOut()
} else if err != nil {
return nil, y.Wrapf(err, "while opening file: %s\n", discardFname)
}
for slot := 0; slot < lf.maxSlot(); slot++ {
if lf.get(16*slot) == 0 {
lf.nextEmptySlot = slot
break
}
}
sort.Sort(lf)
opt.Infof("Discard stats nextEmptySlot: %d\n", lf.nextEmptySlot)
return lf, nil
}
func (lf *discardStats) Len() int {
return lf.nextEmptySlot
}
func (lf *discardStats) Less(i, j int) bool {
return lf.get(16*i) < lf.get(16*j)
}
func (lf *discardStats) Swap(i, j int) {
left := lf.Data[16*i : 16*i+16]
right := lf.Data[16*j : 16*j+16]
var tmp [16]byte
copy(tmp[:], left)
copy(left, right)
copy(right, tmp[:])
}
// offset is not slot.
func (lf *discardStats) get(offset int) uint64 {
return binary.BigEndian.Uint64(lf.Data[offset : offset+8])
}
func (lf *discardStats) set(offset int, val uint64) {
binary.BigEndian.PutUint64(lf.Data[offset:offset+8], val)
}
// zeroOut would zero out the next slot.
func (lf *discardStats) zeroOut() {
lf.set(lf.nextEmptySlot*16, 0)
lf.set(lf.nextEmptySlot*16+8, 0)
}
func (lf *discardStats) maxSlot() int {
return len(lf.Data) / 16
}
// Update would update the discard stats for the given file id. If discard is
// 0, it would return the current value of discard for the file. If discard is
// < 0, it would set the current value of discard to zero for the file.
func (lf *discardStats) Update(fidu uint32, discard int64) int64 {
fid := uint64(fidu)
lf.Lock()
defer lf.Unlock()
idx := sort.Search(lf.nextEmptySlot, func(slot int) bool {
return lf.get(slot*16) >= fid
})
if idx < lf.nextEmptySlot && lf.get(idx*16) == fid {
off := idx*16 + 8
curDisc := lf.get(off)
if discard == 0 {
return int64(curDisc)
}
if discard < 0 {
lf.set(off, 0)
return 0
}
lf.set(off, curDisc+uint64(discard))
return int64(curDisc + uint64(discard))
}
if discard <= 0 {
// No need to add a new entry.
return 0
}
// Could not find the fid. Add the entry.
idx = lf.nextEmptySlot
lf.set(idx*16, uint64(fid))
lf.set(idx*16+8, uint64(discard))
// Move to next slot.
lf.nextEmptySlot++
for lf.nextEmptySlot >= lf.maxSlot() {
y.Check(lf.Truncate(2 * int64(len(lf.Data))))
}
lf.zeroOut()
sort.Sort(lf)
return int64(discard)
}
func (lf *discardStats) Iterate(f func(fid, stats uint64)) {
for slot := 0; slot < lf.nextEmptySlot; slot++ {
idx := 16 * slot
f(lf.get(idx), lf.get(idx+8))
}
}
// MaxDiscard returns the file id with maximum discard bytes.
func (lf *discardStats) MaxDiscard() (uint32, int64) {
lf.Lock()
defer lf.Unlock()
var maxFid, maxVal uint64
lf.Iterate(func(fid, val uint64) {
if maxVal < val {
maxVal = val
maxFid = fid
}
})
return uint32(maxFid), int64(maxVal)
}