postgres: Get around include/extre-filerefs issue #432

docs/develop/troubleshoot.rst

@@ -6,6 +6,8 @@ jinja2.exceptions.TemplateNotFound
 
 If you ``{% include %}`` a file, this error might be raised. To resolve the issue, add the included file to the ``extra_filerefs`` list in the ``Saltfile`` file (`Salt issue <https://github.com/saltstack/salt/issues/21370>`__).
 
+Alternatively, instead of including shared content in a service-specific file, rewrite the configuration so that the shared content is the template file, and the service-specific content is Pillar data.
+
 Jinja caching issues
 --------------------
 

docs/develop/update/postgres.rst

@@ -98,30 +98,42 @@ Configure PostgreSQL
       postgres:
         configuration: False
 
-#. Put your configuration file in the `salt/postgres/files/conf <https://github.com/open-contracting/deploy/tree/main/salt/postgres/files/conf>`__ directory. To use the base configuration, insert ``{% include 'postgres/files/conf/shared.include' %}`` at the top of the file.
+#. Put your configuration template in the `salt/postgres/files/conf <https://github.com/open-contracting/deploy/tree/main/salt/postgres/files/conf>`__ directory. In most cases, you should use the ``shared`` configuration template.
 
 #. Set ``postgres.configuration`` in the server's Pillar file:
 
    .. code-block:: yaml
-      :emphasize-lines: 2
+      :emphasize-lines: 2-6
 
       postgres:
-        configuration: kingfisher-process1
+        configuration:
+          name: kingfisher-process1
+          source: shared
+          context:
+            mykey: myvalue
+
+   The keys of the ``context`` mapping are made available as variables in the configuration template.
 
-#. If you use the base configuration:
+#. If you use the ``shared`` configuration template, under the ``context`` mapping:
 
+   -  If you need more connections, set ``max_connections`` (100, default).
    -  Set ``storage`` to either ``ssd`` (solid-state drive, default) or ``hdd`` (hard disk drive).
    -  Set ``type`` to either ``oltp`` (online transaction processing, default) or ``dw`` (data warehouse).
-   -  If you need more connections, set ``max_connections``.
+   -  Set ``content`` to add content to the configuration file.
 
    .. code-block:: yaml
       :emphasize-lines: 3-5
 
       postgres:
-        configuration: registry
-        storage: hdd
-        type: oltp
-        max_connections: 200
+        configuration:
+          name: registry
+          source: shared
+          context:
+            max_connections: 300
+            storage: hdd
+            type: oltp
+            content: |
+              max_wal_size = 10GB
 
 #. Set ``vm.nr_hugepages`` in the server's Pillar file, following `PostgreSQL's instructions <https://www.postgresql.org/docs/current/kernel-resources.html#LINUX-HUGE-PAGES>`__:
 

pillar/kingfisher_common.sls

@@ -1,5 +1,6 @@
 postgres:
   backup:
+    # Need to sync with `--stanza` in the main server's Pillar file.
     stanza: kingfisher
     retention_full: 4
     s3_bucket: ocp-db-backup

pillar/kingfisher_process.sls

@@ -45,9 +45,42 @@ postgres:
   # If the replica becomes unavailable, we can temporarily enable public access.
   # public_access: True
   version: 11
-  configuration: kingfisher-process1
-  storage: ssd
-  type: oltp
+  configuration:
+    name: kingfisher-process1
+    source: shared
+    context:
+      storage: ssd
+      type: oltp
+      content: |
+        # https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-MAX-WAL-SIZE
+        # https://github.com/open-contracting/deploy/issues/158
+        max_wal_size = 10GB
+
+        # https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-WAL-KEEP-SEGMENTS
+        # https://github.com/open-contracting/deploy/issues/158
+        wal_keep_segments = 20
+
+        # https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-SYNCHRONOUS-COMMIT
+        synchronous_commit = local
+
+        # https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-SYNCHRONOUS-STANDBY-NAMES
+        synchronous_standby_names = 'pgslave001'
+
+        ### pgBackRest
+        # https://pgbackrest.org/user-guide.html#quickstart/configure-archiving
+
+        # https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-WAL-LEVEL
+        wal_level = logical
+
+        # https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-MODE
+        archive_mode = on
+
+        # https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-ARCHIVE-COMMAND
+        # https://pgbackrest.org/user-guide.html#async-archiving/async-archive-push
+        archive_command = 'pgbackrest --stanza=kingfisher archive-push %p'
+
+        # https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-MAX-WAL-SENDERS
+        max_wal_senders = 5
   replica_ipv4:
     - 148.251.183.230
   replica_ipv6:

pillar/kingfisher_replica.sls

@@ -26,9 +26,28 @@ prometheus:
 postgres:
   public_access: True
   version: 11
-  configuration: kingfisher-replica1
-  storage: ssd
-  type: dw
+  configuration:
+    name: kingfisher-replica1
+    source: shared
+    context:
+      storage: ssd
+      type: dw
+      content: |
+        # Must be the same value as on the main server, otherwise "hot standby is not possible because max_worker_processes =
+        # 12 is a lower setting than on the master server (its value was 16)"
+        max_worker_processes = 16
+
+        # https://www.postgresql.org/docs/current/runtime-config-resource.html#GUC-MAX-PREPARED-TRANSACTIONS
+        # The main server has a manually-edited postgresql.conf file from long ago, which sets this to 100.
+        max_prepared_transactions = 100
+
+        # https://www.postgresql.org/docs/current/runtime-config-resource.html#GUC-TEMP-FILE-LIMIT
+        # https://github.com/open-contracting/deploy/issues/272
+        temp_file_limit = 1TB
+
+        # https://www.postgresql.org/docs/current/hot-standby.html
+        # Avoid the cancellation of long-running queries, by users like OCCRP.
+        hot_standby_feedback = on
   backup:
     configuration: kingfisher-replica1
     process_max: 6

pillar/redash.sls

@@ -24,7 +24,12 @@ postgres:
   version: 13
   # Docker containers don't use localhost to connect to the host's PostgreSQL service. Public access is controlled using Linode's firewall.
   public_access: True
-  configuration: redash
+  configuration:
+    name: redash
+    source: shared
+    context:
+      content: |
+        data_directory = '/var/lib/postgresql/13/main'
 
 docker:
   user: deployer

pillar/registry.sls

@@ -67,11 +67,19 @@ postgres:
   version: 12
   # Public access allows Docker connections. Hetzner's firewall prevents non-local connections.
   public_access: true
-  configuration: registry
-  storage: hdd
-  type: oltp
-  # We need a lot of connections for all the workers and threads.
-  max_connections: 300  # oltp at https://pgtune.leopard.in.ua
+  configuration:
+    name: registry
+    source: shared
+    context:
+      # We need a lot of connections for all the workers and threads.
+      max_connections: 300  # oltp at https://pgtune.leopard.in.ua
+      storage: hdd
+      type: oltp
+      content: |
+        data_directory = '/data/storage/postgresql/12/main'
+
+        # Avoid "checkpoints are occurring too frequently" due to intense writes (default 1GB).
+        max_wal_size = 10GB
 
 docker:
   user: deployer

salt/postgres/files/conf/shared.include → salt/postgres/files/conf/shared.conf

@@ -2,12 +2,10 @@
 # https://pgtune.leopard.in.ua/
 # https://github.com/jberkus/annotated.conf
 
-{%- if 'max_connections' in pillar.postgres %}
-{%- set max_connections = pillar.postgres.max_connections %}
-{%- else %}
+{%- if max_connections is not defined %}
 {%- set max_connections = 100 %}
 {%- endif %}
-{%- if pillar.postgres.get('type', 'oltp') == 'dw' %}
+{%- if type is defined and type == 'dw' %}
 {%- set max_parallel_workers_per_gather = grains.num_cpus // 2 %}
 {%- else %}
 {%- set max_parallel_workers_per_gather = [4, grains.num_cpus // 2] | min %}
@@ -56,7 +54,7 @@ max_stack_depth = 6MB
 # https://www.postgresql.org/docs/current/runtime-config-resource.html#GUC-EFFECTIVE-IO-CONCURRENCY
 # "SSDs … can often process many concurrent requests, so the best value might be in the hundreds."
 # https://github.com/le0pard/pgtune/blob/a002a699e77426d5c5221645df999eec7fe92472/webpack/selectors/configuration.js#L262
-effective_io_concurrency = {% if pillar.postgres.get('storage', 'ssd') == 'hdd' %}2{% else %}200{% endif %}
+effective_io_concurrency = {% if storage is defined and storage == 'hdd' %}2{% else %}200{% endif %}
 
 # https://www.postgresql.org/docs/current/runtime-config-resource.html#GUC-MAX-WORKER-PROCESSES
 max_worker_processes = {{ grains.num_cpus }}
@@ -85,7 +83,7 @@ effective_cache_size = {{ ( grains.mem_total * 3 ) // 4 }}MB  # 75%
 # https://www.postgresql.org/docs/current/runtime-config-query.html#GUC-RANDOM-PAGE-COST
 # https://wiki.postgresql.org/wiki/Tuning_Your_PostgreSQL_Server#random_page_cost
 # https://github.com/jfcoz/postgresqltuner/blob/312d19764d24698d764cc6846b9e6efb6347a153/postgresqltuner.pl#L867
-random_page_cost = {% if pillar.postgres.get('storage', 'ssd') == 'hdd' %}2.0{% else %}1.0{% endif %}
+random_page_cost = {% if storage is defined and storage == 'hdd' %}2.0{% else %}1.0{% endif %}
 
 ### Planner Method Configuration
 
@@ -116,3 +114,7 @@ shared_preload_libraries = 'pg_stat_statements'
 
 # https://www.postgresql.org/docs/current/runtime-config-client.html#GUC-TIMEZONE
 timezone = 'UTC'
+{%- if content is defined %}
+
+{{ content }}
+{%- endif %}

salt/postgres/init.sls

@@ -59,16 +59,11 @@ postgresql-reload:
       - module: postgresql-reload
 
 {% if pillar.postgres.configuration %}
-# Although we can add `shared.include` as a separate file (e.g. looping over configurations, and using `loop.index0`
-# to prefix the files), this makes changes harder to deploy, since re-ordering or removing a configuration will rename
-# the new files, but not remove the old files. Instead, a developer needs to `include` it in the configuration file.
-#
-# (Unfortunately, `file.managed` doesn't have a `sources` option like `file.append` in order to create a target file
-# from many source files, and `file.accumulated` doesn't have a `source` option.)
-/etc/postgresql/{{ pillar.postgres.version }}/main/conf.d/030_{{ pillar.postgres.configuration }}.conf:
+/etc/postgresql/{{ pillar.postgres.version }}/main/conf.d/030_{{ pillar.postgres.configuration.name }}.conf:
   file.managed:
-    - source: salt://postgres/files/conf/{{ pillar.postgres.configuration }}.conf
+    - source: salt://postgres/files/conf/{{ pillar.postgres.configuration.source }}.conf
     - template: jinja
+    - context: {{ pillar.postgres.configuration.context|yaml }}
     - user: postgres
     - group: postgres
     - mode: 640

script/setup

@@ -16,7 +16,6 @@ echo "salt-ssh:
   # https://github.com/saltstack/salt/issues/21370
   extra_filerefs:
     - $REPO_DIR/salt/apache/includes/cove.include.jinja
-    - $REPO_DIR/salt/postgres/files/conf/shared.include
 " > Saltfile
 
 echo "# https://docs.saltproject.io/en/latest/ref/configuration/master.html#std-conf_master-file_roots