opt(memory): Use z.Calloc for allocating KVList #1563
Conversation
stream.go
Outdated
| @@ -175,17 +203,21 @@ func (st *Stream) produceKVs(ctx context.Context, threadId int) error { | |||
| streamId := atomic.AddUint32(&st.nextStreamId, 1) | |||
|
|
|||
| outList := new(pb.KVList) | |||
| outList.AllocatorId = st.newAllocator(threadId).Ref | |||
There was a problem hiding this comment.
Initialize outList using a composite literal to simplify the code. as specified in Effective Go
stream.go
Outdated
| @@ -175,6 +202,7 @@ func (st *Stream) produceKVs(ctx context.Context, threadId int) error { | |||
| streamId := atomic.AddUint32(&st.nextStreamId, 1) | |||
|
|
|||
| outList := new(pb.KVList) | |||
| outList.AllocatorId = st.newAllocator(threadId).Ref | |||
There was a problem hiding this comment.
Initialize outList using a composite literal to simplify the code. as specified in Effective Go
Add an --encryption-key-file flag to the badger stream tool to support encrypted databases. Changes: * Add --encryption-key-file flag * Set block and index cache sizes to fix panic when running stream.
| @@ -175,6 +206,7 @@ func (st *Stream) produceKVs(ctx context.Context, threadId int) error { | |||
| streamId := atomic.AddUint32(&st.nextStreamId, 1) | |||
|
|
|||
| outList := new(pb.KVList) | |||
| outList.AllocRef = st.newAllocator(threadId).Ref | |||
There was a problem hiding this comment.
Initialize outList using a composite literal to simplify the code. as specified in Effective Go
| @@ -50,7 +52,10 @@ type collector struct { | |||
| } | |||
|
|
|||
| func (c *collector) Send(list *bpb.KVList) error { | |||
| c.kv = append(c.kv, list.Kv...) | |||
| for _, kv := range list.Kv { | |||
| cp := proto.Clone(kv).(*bpb.KV) | |||
There was a problem hiding this comment.
Unchecked type assertions can cause panics. Check for success with the x, ok := y.(type) style.
| @@ -175,6 +206,7 @@ func (st *Stream) produceKVs(ctx context.Context, threadId int) error { | |||
| streamId := atomic.AddUint32(&st.nextStreamId, 1) | |||
|
|
|||
| outList := new(pb.KVList) | |||
| outList.AllocRef = st.newAllocator(threadId).Ref | |||
There was a problem hiding this comment.
Initialize outList using a composite literal to simplify the code. as specified in Effective Go
| @@ -175,6 +206,7 @@ func (st *Stream) produceKVs(ctx context.Context, threadId int) error { | |||
| streamId := atomic.AddUint32(&st.nextStreamId, 1) | |||
|
|
|||
| outList := new(pb.KVList) | |||
| outList.AllocRef = st.newAllocator(threadId).Ref | |||
There was a problem hiding this comment.
Initialize outList using a composite literal to simplify the code. as specified in Effective Go
| @@ -50,7 +52,10 @@ type collector struct { | |||
| } | |||
|
|
|||
| func (c *collector) Send(list *bpb.KVList) error { | |||
| c.kv = append(c.kv, list.Kv...) | |||
| for _, kv := range list.Kv { | |||
| cp := proto.Clone(kv).(*bpb.KV) | |||
There was a problem hiding this comment.
Unchecked type assertions can cause panics. Check for success with the x, ok := y.(type) style.
| @@ -175,6 +206,7 @@ func (st *Stream) produceKVs(ctx context.Context, threadId int) error { | |||
| streamId := atomic.AddUint32(&st.nextStreamId, 1) | |||
|
|
|||
| outList := new(pb.KVList) | |||
| outList.AllocRef = st.newAllocator(threadId).Ref | |||
There was a problem hiding this comment.
Initialize outList using a composite literal to simplify the code. as specified in Effective Go
| } | ||
| return nil | ||
| // Mark the stream as done. | ||
| outList.Kv = append(outList.Kv, &pb.KV{ |
There was a problem hiding this comment.
Initialize outList using a composite literal to simplify the code. as specified in Effective Go
| } | ||
| return nil | ||
| // Mark the stream as done. | ||
| outList.Kv = append(outList.Kv, &pb.KV{ |
There was a problem hiding this comment.
Initialize outList using a composite literal to simplify the code. as specified in Effective Go
KVs can take up a lot of memory in the stream framework. With this change, we allocate them using z.Allocator and release them after the final KVList is sent out.
This change is