diff options
author | Teodor Sigaev | 2015-09-02 17:08:58 +0000 |
---|---|---|
committer | Teodor Sigaev | 2015-09-02 17:08:58 +0000 |
commit | 30bb26b5e04cff911db960801e32e5f57045eb61 (patch) | |
tree | b05b945313ce910e999818a1194b2e88183b32f7 | |
parent | 075ab425bdbb4bb2de41624305f20885cae7ba7e (diff) |
Allow usage of huge maintenance_work_mem for GIN build.
Currently, in-memory posting list during GIN build process is limited 1GB
because of using repalloc. The patch replaces call of repalloc to repalloc_huge.
It increases limit of posting list from 180 millions
(1GB / sizeof(ItemPointerData)) to 4 billions limited by maxcount/count fields
in GinEntryAccumulator and subsequent calls. Check added.
Also, fix accounting of allocatedMemory during build to prevent integer
overflow with maintenance_work_mem > 4GB.
Robert Abraham <[email protected]> with additions by me
-rw-r--r-- | src/backend/access/gin/ginbulk.c | 10 | ||||
-rw-r--r-- | src/backend/access/gin/ginfast.c | 2 | ||||
-rw-r--r-- | src/backend/access/gin/gininsert.c | 2 | ||||
-rw-r--r-- | src/include/access/gin_private.h | 2 |
4 files changed, 12 insertions, 4 deletions
diff --git a/src/backend/access/gin/ginbulk.c b/src/backend/access/gin/ginbulk.c index e40e5be36b..5a8546998d 100644 --- a/src/backend/access/gin/ginbulk.c +++ b/src/backend/access/gin/ginbulk.c @@ -14,6 +14,8 @@ #include "postgres.h" +#include <limits.h> + #include "access/gin_private.h" #include "utils/datum.h" #include "utils/memutils.h" @@ -36,10 +38,16 @@ ginCombineData(RBNode *existing, const RBNode *newdata, void *arg) */ if (eo->count >= eo->maxcount) { + if (eo->maxcount > INT_MAX) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("posting list is too long"), + errhint("Reduce maintenance_work_mem"))); + accum->allocatedMemory -= GetMemoryChunkSpace(eo->list); eo->maxcount *= 2; eo->list = (ItemPointerData *) - repalloc(eo->list, sizeof(ItemPointerData) * eo->maxcount); + repalloc_huge(eo->list, sizeof(ItemPointerData) * eo->maxcount); accum->allocatedMemory += GetMemoryChunkSpace(eo->list); } diff --git a/src/backend/access/gin/ginfast.c b/src/backend/access/gin/ginfast.c index c5732c3dc0..54e593d8a9 100644 --- a/src/backend/access/gin/ginfast.c +++ b/src/backend/access/gin/ginfast.c @@ -814,7 +814,7 @@ ginInsertCleanup(GinState *ginstate, */ if (GinPageGetOpaque(page)->rightlink == InvalidBlockNumber || (GinPageHasFullRow(page) && - (accum.allocatedMemory >= maintenance_work_mem * 1024L))) + (accum.allocatedMemory >= (Size)maintenance_work_mem * 1024L))) { ItemPointerData *list; uint32 nlist; diff --git a/src/backend/access/gin/gininsert.c b/src/backend/access/gin/gininsert.c index fc44f0205d..49e91853c3 100644 --- a/src/backend/access/gin/gininsert.c +++ b/src/backend/access/gin/gininsert.c @@ -281,7 +281,7 @@ ginBuildCallback(Relation index, HeapTuple htup, Datum *values, &htup->t_self); /* If we've maxed out our available memory, dump everything to the index */ - if (buildstate->accum.allocatedMemory >= maintenance_work_mem * 1024L) + if (buildstate->accum.allocatedMemory >= (Size)maintenance_work_mem * 1024L) { ItemPointerData *list; Datum key; diff --git a/src/include/access/gin_private.h b/src/include/access/gin_private.h index 5095fc1dba..acbe36a741 100644 --- a/src/include/access/gin_private.h +++ b/src/include/access/gin_private.h @@ -903,7 +903,7 @@ typedef struct GinEntryAccumulator typedef struct { GinState *ginstate; - long allocatedMemory; + Size allocatedMemory; GinEntryAccumulator *entryallocator; uint32 eas_used; RBTree *tree; |