Add option to specify segment size in blocks

The tests don't have much coverage of segment related code, as we don't create
large enough tables. To make it easier to test these paths, add a new option
specifying the segment size in blocks.

Set the new option to 6 blocks in one of the CI tasks. Smaller numbers
currently fail one of the tests, for understandable reasons.

While at it, fix some segment size related issues in the meson build.

Author: Andres Freund <[email protected]>
Discussion: https://postgr.es/m/[email protected]

Author: anarazel

.cirrus.yml

@@ -323,6 +323,7 @@ task:
           ./configure \
             --enable-cassert --enable-debug --enable-tap-tests \
             --enable-nls \
+            --with-segsize-blocks=8 \
             \
             ${LINUX_CONFIGURE_FEATURES} \
             \
@@ -491,6 +492,7 @@ task:
       -Dextra_lib_dirs=${brewpath}/lib \
       -Dcassert=true \
       -Dssl=openssl -Duuid=e2fs -Ddtrace=auto \
+      -Dsegsize_blocks=6 \
       -DPG_TEST_EXTRA="$PG_TEST_EXTRA" \
       build
 

configure

@@ -842,6 +842,7 @@ enable_dtrace
 enable_tap_tests
 with_blocksize
 with_segsize
+with_segsize_blocks
 with_wal_blocksize
 with_CC
 with_llvm
@@ -1551,6 +1552,8 @@ Optional Packages:
   --with-blocksize=BLOCKSIZE
                           set table block size in kB [8]
   --with-segsize=SEGSIZE  set table segment size in GB [1]
+  --with-segsize-blocks=SEGSIZE_BLOCKS
+                          set table segment size in blocks [0]
   --with-wal-blocksize=BLOCKSIZE
                           set WAL block size in kB [8]
   --with-CC=CMD           set compiler (deprecated)
@@ -3731,8 +3734,6 @@ _ACEOF
 #
 # Relation segment size
 #
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for segment size" >&5
-$as_echo_n "checking for segment size... " >&6; }
 
 
 
@@ -3756,12 +3757,52 @@ else
 fi
 
 
-# this expression is set up to avoid unnecessary integer overflow
-# blocksize is already guaranteed to be a factor of 1024
-RELSEG_SIZE=`expr '(' 1024 / ${blocksize} ')' '*' ${segsize} '*' 1024`
-test $? -eq 0 || exit 1
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: ${segsize}GB" >&5
+
+
+
+# Check whether --with-segsize-blocks was given.
+if test "${with_segsize_blocks+set}" = set; then :
+  withval=$with_segsize_blocks;
+  case $withval in
+    yes)
+      as_fn_error $? "argument required for --with-segsize-blocks option" "$LINENO" 5
+      ;;
+    no)
+      as_fn_error $? "argument required for --with-segsize-blocks option" "$LINENO" 5
+      ;;
+    *)
+      segsize_blocks=$withval
+      ;;
+  esac
+
+else
+  segsize_blocks=0
+fi
+
+
+
+# If --with-segsize-blocks is non-zero, it is used, --with-segsize
+# otherwise. segsize-blocks is only really useful for developers wanting to
+# test segment related code. Warn if both are used.
+if test $segsize_blocks -ne 0 -a $segsize -ne 1; then
+  { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: both --with-segsize and --with-segsize-blocks specified, --with-segsize-blocks wins" >&5
+$as_echo "$as_me: WARNING: both --with-segsize and --with-segsize-blocks specified, --with-segsize-blocks wins" >&2;}
+fi
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for segment size" >&5
+$as_echo_n "checking for segment size... " >&6; }
+if test $segsize_blocks -eq 0; then
+  # this expression is set up to avoid unnecessary integer overflow
+  # blocksize is already guaranteed to be a factor of 1024
+  RELSEG_SIZE=`expr '(' 1024 / ${blocksize} ')' '*' ${segsize} '*' 1024`
+  test $? -eq 0 || exit 1
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: ${segsize}GB" >&5
 $as_echo "${segsize}GB" >&6; }
+else
+  RELSEG_SIZE=$segsize_blocks
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: ${RELSEG_SIZE} blocks" >&5
+$as_echo "${RELSEG_SIZE} blocks" >&6; }
+fi
 
 
 cat >>confdefs.h <<_ACEOF
@@ -15450,9 +15491,11 @@ _ACEOF
 
 
 
-# If we don't have largefile support, can't handle segsize >= 2GB.
-if test "$ac_cv_sizeof_off_t" -lt 8 -a "$segsize" != "1"; then
-   as_fn_error $? "Large file support is not enabled. Segment size cannot be larger than 1GB." "$LINENO" 5
+# If we don't have largefile support, can't handle segment size >= 2GB.
+if test "$ac_cv_sizeof_off_t" -lt 8; then
+  if expr $RELSEG_SIZE '*' $blocksize '>=' 2 '*' 1024 '*' 1024; then
+    as_fn_error $? "Large file support is not enabled. Segment size cannot be larger than 1GB." "$LINENO" 5
+  fi
 fi
 
 # The cast to long int works around a bug in the HP C Compiler

configure.ac

@@ -285,15 +285,31 @@ AC_DEFINE_UNQUOTED([BLCKSZ], ${BLCKSZ}, [
 #
 # Relation segment size
 #
-AC_MSG_CHECKING([for segment size])
 PGAC_ARG_REQ(with, segsize, [SEGSIZE], [set table segment size in GB [1]],
              [segsize=$withval],
              [segsize=1])
-# this expression is set up to avoid unnecessary integer overflow
-# blocksize is already guaranteed to be a factor of 1024
-RELSEG_SIZE=`expr '(' 1024 / ${blocksize} ')' '*' ${segsize} '*' 1024`
-test $? -eq 0 || exit 1
-AC_MSG_RESULT([${segsize}GB])
+PGAC_ARG_REQ(with, segsize-blocks, [SEGSIZE_BLOCKS], [set table segment size in blocks [0]],
+             [segsize_blocks=$withval],
+             [segsize_blocks=0])
+
+# If --with-segsize-blocks is non-zero, it is used, --with-segsize
+# otherwise. segsize-blocks is only really useful for developers wanting to
+# test segment related code. Warn if both are used.
+if test $segsize_blocks -ne 0 -a $segsize -ne 1; then
+  AC_MSG_WARN([both --with-segsize and --with-segsize-blocks specified, --with-segsize-blocks wins])
+fi
+
+AC_MSG_CHECKING([for segment size])
+if test $segsize_blocks -eq 0; then
+  # this expression is set up to avoid unnecessary integer overflow
+  # blocksize is already guaranteed to be a factor of 1024
+  RELSEG_SIZE=`expr '(' 1024 / ${blocksize} ')' '*' ${segsize} '*' 1024`
+  test $? -eq 0 || exit 1
+  AC_MSG_RESULT([${segsize}GB])
+else
+  RELSEG_SIZE=$segsize_blocks
+  AC_MSG_RESULT([${RELSEG_SIZE} blocks])
+fi
 
 AC_DEFINE_UNQUOTED([RELSEG_SIZE], ${RELSEG_SIZE}, [
  RELSEG_SIZE is the maximum number of blocks allowed in one disk file.
@@ -1733,9 +1749,11 @@ fi
 dnl Check for largefile support (must be after AC_SYS_LARGEFILE)
 AC_CHECK_SIZEOF([off_t])
 
-# If we don't have largefile support, can't handle segsize >= 2GB.
-if test "$ac_cv_sizeof_off_t" -lt 8 -a "$segsize" != "1"; then
-   AC_MSG_ERROR([Large file support is not enabled. Segment size cannot be larger than 1GB.])
+# If we don't have largefile support, can't handle segment size >= 2GB.
+if test "$ac_cv_sizeof_off_t" -lt 8; then
+  if expr $RELSEG_SIZE '*' $blocksize '>=' 2 '*' 1024 '*' 1024; then
+    AC_MSG_ERROR([Large file support is not enabled. Segment size cannot be larger than 1GB.])
+  fi
 fi
 
 AC_CHECK_SIZEOF([bool], [],

doc/src/sgml/installation.sgml

@@ -1681,6 +1681,20 @@ build-postgresql:
         </para>
        </listitem>
       </varlistentry>
+
+      <varlistentry>
+       <term><option>--with-segsize-blocks=SEGSIZE_BLOCKS</option></term>
+       <listitem>
+        <para>
+         Specify the segment size in blocks. If both
+         <option>--with-segsize</option> and this option are specified, this
+         option wins.
+
+         This option is only for developers, to test segment related code.
+        </para>
+       </listitem>
+      </varlistentry>
+
      </variablelist>
 
    </sect3>
@@ -3097,6 +3111,20 @@ ninja install
        </para>
       </listitem>
      </varlistentry>
+
+      <varlistentry>
+       <term><option>-Dsegsize_blocks=SEGSIZE_BLOCKS</option></term>
+       <listitem>
+        <para>
+         Specify the segment size in blocks. If both
+         <option>-Dsegsize</option> and this option are specified, this option
+         wins.
+
+         This option is only for developers, to test segment related code.
+        </para>
+       </listitem>
+      </varlistentry>
+
     </variablelist>
    </sect3>
   </sect2>

meson.build

@@ -419,7 +419,19 @@ meson_bin = find_program(meson_binpath, native: true)
 
 cdata.set('USE_ASSERT_CHECKING', get_option('cassert') ? 1 : false)
 
-cdata.set('BLCKSZ', get_option('blocksize').to_int() * 1024, description:
+blocksize = get_option('blocksize').to_int() * 1024
+
+if get_option('segsize_blocks') != 0
+  if get_option('segsize') != 1
+    warning('both segsize and segsize_blocks specified, segsize_blocks wins')
+  endif
+
+  segsize = get_option('segsize_blocks')
+else
+  segsize = (get_option('segsize') * 1024 * 1024 * 1024) / blocksize
+endif
+
+cdata.set('BLCKSZ', blocksize, description:
 '''Size of a disk block --- this also limits the size of a tuple. You can set
    it bigger if you need bigger tuples (although TOAST should reduce the need
    to have large tuples, since fields can be spread across multiple tuples).
@@ -429,7 +441,7 @@ cdata.set('BLCKSZ', get_option('blocksize').to_int() * 1024, description:
    Changing BLCKSZ requires an initdb.''')
 
 cdata.set('XLOG_BLCKSZ', get_option('wal_blocksize').to_int() * 1024)
-cdata.set('RELSEG_SIZE', get_option('segsize') * 131072)
+cdata.set('RELSEG_SIZE', segsize)
 cdata.set('DEF_PGPORT', get_option('pgport'))
 cdata.set_quoted('DEF_PGPORT_STR', get_option('pgport').to_string())
 cdata.set_quoted('PG_KRB_SRVNAM', get_option('krb_srvnam'))
@@ -3132,9 +3144,11 @@ if meson.version().version_compare('>=0.57')
 
   summary(
     {
-      'data block size': cdata.get('BLCKSZ'),
-      'WAL block size': cdata.get('XLOG_BLCKSZ') / 1024,
-      'segment size': cdata.get('RELSEG_SIZE') / 131072,
+      'data block size': '@0@ kB'.format(cdata.get('BLCKSZ') / 1024),
+      'WAL block size': '@0@ kB'.format(cdata.get('XLOG_BLCKSZ') / 1024),
+      'segment size': get_option('segsize_blocks') != 0 ?
+        '@0@ blocks'.format(cdata.get('RELSEG_SIZE')) :
+        '@0@ GB'.format(get_option('segsize')),
     },
     section: 'Data layout',
   )

meson_options.txt

@@ -13,6 +13,9 @@ option('wal_blocksize', type : 'combo',
 option('segsize', type : 'integer', value : 1,
   description : '''Segment size, in gigabytes''')
 
+option('segsize_blocks', type : 'integer', value: 0,
+  description : '''Segment size, in blocks''')
+
 
 # Miscellaneous options