chore: install docker built extensions on AMI

.github/workflows/ami-release.yml

@@ -11,7 +11,14 @@ on:
 
 jobs:
   build:
-    runs-on: [self-hosted, X64]
+    strategy:
+      matrix:
+        include:
+          - runner: arm-runner
+            arch: arm64
+            ubuntu: focal
+            mcpu: neoverse-n1
+    runs-on: ${{ matrix.runner }}
     timeout-minutes: 150
     permissions:
       contents: write
@@ -20,7 +27,60 @@ jobs:
 
     steps:
       - name: Checkout Repo
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
+
+      - id: args
+        uses: mikefarah/yq@master
+        with:
+          cmd: yq 'to_entries | map(select(.value|type == "!!str")) |  map(.key + "=" + .value) | join("\n")' 'ansible/vars.yml'
+      - run: docker context create builders
+      - uses: docker/setup-buildx-action@v2
+        with:
+          endpoint: builders
+      - uses: docker/build-push-action@v3
+        with:
+          push: false
+          load: true
+          build-args: |
+            ${{ steps.args.outputs.result }}
+          target: extensions
+          tags: supabase/postgres:extensions
+          platforms: linux/${{ matrix.arch }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+      - name: Extract built packages
+        run: |
+          mkdir -p /tmp/extensions ansible/files/extensions
+          docker save supabase/postgres:extensions | tar xv -C /tmp/extensions
+          for layer in /tmp/extensions/*/layer.tar; do
+            tar xvf "$layer" -C ansible/files/extensions --strip-components 1
+          done
+
+      - id: version
+        run: echo "${{ steps.args.outputs.result }}" | grep "postgresql" >> "$GITHUB_OUTPUT"
+      - name: Build Postgres deb
+        uses: docker/build-push-action@v3
+        with:
+          push: false
+          load: true
+          file: docker/Dockerfile
+          target: pg-deb
+          build-args: |
+            ubuntu_release="${{ matrix.ubuntu }}"
+            postgresql_major="${{ steps.version.outputs.postgresql_major }}"
+            postgresql_release="${{ steps.version.outputs.postgresql_release }}"
+            CPPFLAGS="-mcpu=${{ matrix.mcpu }}"
+          tags: supabase/postgres:deb
+          platforms: linux/${{ matrix.arch }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+      - name: Extract Postgres deb
+        run: |
+          mkdir -p /tmp/build ansible/files/postgres
+          docker save supabase/postgres:deb | tar xv -C /tmp/build
+          for layer in /tmp/build/*/layer.tar; do
+            tar xvf "$layer" -C ansible/files/postgres --strip-components 1
+          done
 
       - name: Build AMI
         run: |
@@ -50,7 +110,7 @@ jobs:
       - name: Upload pg binaries to s3 staging
         run: |
           aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz
-    
+
       - name: configure aws credentials - prod
         uses: aws-actions/configure-aws-credentials@v1
         with:
@@ -68,7 +128,7 @@ jobs:
       - name: Upload pg binaries to s3 prod
         run: |
           aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz
-    
+
       - name: Create release
         uses: softprops/action-gh-release@v1
         with:

CONTRIBUTING.md

@@ -0,0 +1,46 @@
+# Welcome to Supabase Postgres contributing guide
+
+## Adding a new extension
+
+Extensions can either be built from source or installed through a debian package. In general, you want to add the installation commands for your extension to the [Dockerfile](Dockerfile) following the steps below.
+
+1. Create a [build stage](Dockerfile#L777) named after your extension.
+2. Add build args that specify the extension's [release version](Dockerfile#L37).
+3. If your extension is published as a package, download it to `/tmp/<name>.deb` using the [ADD command](Dockerfile#L705).
+4. If you need to build the extensions from source, use [checkinstall](Dockerfile#L791) to create a `/tmp/<name>.deb` package.
+5. Copy your extension's package from build stage to [extensions stage](Dockerfile#L851).
+
+Here's a minimal example:
+
+```dockerfile
+ARG pg_graphql_release=1.1.0
+
+####################
+# 19-pg_graphql.yml
+####################
+FROM base as pg_graphql
+# Download package archive
+ARG pg_graphql_release
+ADD "https://github.com/supabase/pg_graphql/releases/download/v${pg_graphql_release}/pg_graphql-v${pg_graphql_release}-pg${postgresql_major}-${TARGETARCH}-linux-gnu.deb" \
+    /tmp/pg_graphql.deb
+
+####################
+# Collect extension packages
+####################
+FROM scratch as extensions
+COPY --from=pg_graphql /tmp/*.deb /tmp/
+```
+
+Using this process maximises the effectiveness of Docker layer caching, which significantly speeds up our CI builds.
+
+## Testing an extension
+
+Extensions can be tested automatically using pgTAP. Start by creating a new file in [migrations/tests/extensions](migrations/tests/extensions). For example:
+
+```sql
+BEGIN;
+create extension if not exists wrappers with schema "extensions";
+ROLLBACK;
+```
+
+This test will be run as part of CI to check that your extension can be enabled successfully from the final Docker image.

ansible/playbook.yml

@@ -30,9 +30,6 @@
     - name: Install Postgres from source
       import_tasks: tasks/setup-postgres.yml
 
-    - name: Install Postgres extensions
-      import_tasks: tasks/setup-extensions.yml
-
     - name: Install PgBouncer
       import_tasks: tasks/setup-pgbouncer.yml
       tags:
@@ -79,7 +76,7 @@
       become: yes
       become_user: postgres
       shell:
-        cmd: /usr/bin/pg_ctl -D /var/lib/postgresql/data start
+        cmd: /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data start
       when: ebssurrogate_mode
 
     - name: Adjust APT update intervals
@@ -138,25 +135,14 @@
         update_cache: yes
         cache_valid_time: 3600
 
-    # Put PG binaries in a directory under $PATH
-    - name: Find all files in /usr/lib/postgresql/bin
-      find:
-        paths: /usr/lib/postgresql/bin
-      register: postgresql_bin
-
     - name: Clean out build dependencies
       import_tasks: tasks/clean-build-dependencies.yml
 
-    - name: Create symbolic links for Postgres binaries to /usr/bin/
-      become: yes
-      shell:
-        cmd: "for fl in /usr/lib/postgresql/bin/* ; do ln -sf $fl /usr/bin/$(basename $fl) ; done"
-
     - name: Restart Postgres Database without Systemd
       become: yes
       become_user: postgres
       shell:
-        cmd: /usr/bin/pg_ctl -D /var/lib/postgresql/data restart "-o -c shared_preload_libraries='pg_tle'"
+        cmd: /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data restart -o "-c shared_preload_libraries='pg_tle'"
       when: ebssurrogate_mode
 
     - name: Run migrations
@@ -168,7 +154,7 @@
       become: yes
       become_user: postgres
       shell:
-        cmd: /usr/bin/pg_ctl -D /var/lib/postgresql/data stop
+        cmd: /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data stop
       when: ebssurrogate_mode
 
     - name: Run unit tests

ansible/tasks/setup-docker.yml

@@ -0,0 +1,82 @@
+- name: Copy extension packages
+  copy:
+    src: files/extensions/
+    dest: /tmp/extensions/
+
+# Builtin apt module does not support wildcard for deb paths
+- name: Install extensions
+  shell: |
+    set -e
+    apt-get update
+    apt-get install -y --no-install-recommends /tmp/extensions/*.deb
+
+- name: pg_cron - set cron.database_name
+  become: yes
+  lineinfile:
+    path: /etc/postgresql/postgresql.conf
+    state: present
+    line: cron.database_name = 'postgres'
+
+- name: pgsodium - determine postgres bin directory
+  shell: pg_config --bindir
+  register: pg_bindir_output
+- set_fact:
+    pg_bindir: "{{ pg_bindir_output.stdout }}"
+
+- name: pgsodium - set pgsodium.getkey_script
+  become: yes
+  lineinfile:
+    path: /etc/postgresql/postgresql.conf
+    state: present
+    # script is expected to be placed by finalization tasks for different target platforms
+    line: pgsodium.getkey_script= '{{ pg_bindir }}/pgsodium_getkey.sh'
+
+- name: auto_explain - set auto_explain.log_min_duration
+  become: yes
+  lineinfile:
+    path: /etc/postgresql/postgresql.conf
+    state: present
+    line: auto_explain.log_min_duration = 10s
+
+# supautils
+- name: supautils - add supautils to session_preload_libraries
+  become: yes
+  replace:
+    path: /etc/postgresql/postgresql.conf
+    regexp: "#session_preload_libraries = ''"
+    replace: session_preload_libraries = 'supautils'
+
+- name: supautils - write custom supautils.conf
+  template:
+    src: "files/postgresql_config/supautils.conf.j2"
+    dest: /etc/postgresql-custom/supautils.conf
+    mode: 0664
+    owner: postgres
+    group: postgres
+
+- name: supautils - copy extension custom scripts
+  copy:
+    src: files/postgresql_extension_custom_scripts/
+    dest: /etc/postgresql-custom/extension-custom-scripts
+  become: yes
+
+- name: supautils - chown extension custom scripts
+  file:
+    mode: 0775
+    owner: postgres
+    group: postgres
+    path: /etc/postgresql-custom/extension-custom-scripts
+    recurse: yes
+  become: yes
+
+- name: supautils - include /etc/postgresql-custom/supautils.conf in postgresql.conf
+  become: yes
+  replace:
+    path: /etc/postgresql/postgresql.conf
+    regexp: "#include = '/etc/postgresql-custom/supautils.conf'"
+    replace: "include = '/etc/postgresql-custom/supautils.conf'"
+
+- name: Cleanup - extension packages
+  file:
+    path: /tmp/extensions
+    state: absent