From 0ca21d80d710613e01bd278a56cb3898a13bfa4c Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 8 Jul 2019 09:38:24 +0300 Subject: [PATCH] Add Synapse Maintenance docs and synapse-janitor integration --- CHANGELOG.md | 9 ++ docs/README.md | 2 + docs/maintenance-postgres.md | 15 +++ docs/maintenance-synapse.md | 72 ++++++++++++ roles/matrix-base/defaults/main.yml | 2 + roles/matrix-postgres/defaults/main.yml | 2 + roles/matrix-postgres/tasks/main.yml | 10 ++ .../tasks/run_synapse_janitor.yml | 110 ++++++++++++++++++ roles/matrix-postgres/tasks/run_vacuum.yml | 90 ++++++++++++++ 9 files changed, 312 insertions(+) create mode 100644 docs/maintenance-synapse.md create mode 100644 roles/matrix-postgres/tasks/run_synapse_janitor.yml create mode 100644 roles/matrix-postgres/tasks/run_vacuum.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d255358c..d2f763c10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # 2019-07-08 +## Synapse Maintenance docs and synapse-janitor support are available + +The playbook can now help you with Synapse's maintenance. + +There's a new documentation page about [Synapse maintenance](./docs/maintenance-synapse.md) and another section on [Postgres vacuuming](./docs/maintenance-postgres.md#vacuuming-postgresql). + +Among other things, if your Postgres database has grown significantly over time, you may wish to [ask the playbook to purge unused data with synapse-janitor](./docs/maintenance-synapse.md#purging-unused-data-with-synapse-janitor) for you. + + ## (BC Break) Rename run control variables Some internal playbook control variables have been renamed. diff --git a/docs/README.md b/docs/README.md index f623085bb..88354c0f6 100644 --- a/docs/README.md +++ b/docs/README.md @@ -20,6 +20,8 @@ - [Maintenance / upgrading services](maintenance-upgrading-services.md) +- [Maintenance / Synapse](maintenance-synapse.md) + - [Maintenance / PostgreSQL](maintenance-postgres.md) - [Maintenance and Troubleshooting](maintenance-and-troubleshooting.md) diff --git a/docs/maintenance-postgres.md b/docs/maintenance-postgres.md index 4be26a805..959da6586 100644 --- a/docs/maintenance-postgres.md +++ b/docs/maintenance-postgres.md @@ -6,6 +6,8 @@ Table of contents: - [Getting a database terminal](#getting-a-database-terminal), for when you wish to execute SQL queries +- [Vacuuming PostgreSQL](#vacuuming-postgresql), for when you wish to run a Postgres [VACUUM](https://www.postgresql.org/docs/current/sql-vacuum.html) (optimizing disk space) + - [Backing up PostgreSQL](#backing-up-postgresql), for when you wish to make a backup - [Upgrading PostgreSQL](#upgrading-postgresql), for upgrading to new major versions of PostgreSQL. Such **manual upgrades are sometimes required**. @@ -18,6 +20,19 @@ You can use the `/usr/local/bin/matrix-postgres-cli` tool to get interactive ter If you are using an [external Postgres server](configuring-playbook-external-postgres.md), the above tool will not be available. +## Vacuuming PostgreSQL + +To perform a `FULL` Postgres [VACUUM](https://www.postgresql.org/docs/current/sql-vacuum.html), run the playbook with `--tags=run-postgres-vacuum`. + +Example: + +```bash +ansible-playbook -i inventory/hosts setup.yml --tags=run-postgres-vacuum +``` + +**Note**: this will automatically stop Synapse temporarily and restart it later. You'll also need plenty of available disk space in your Postgres data directory (usually `/matrix/postgres/data`). + + ## Backing up PostgreSQL To make a back up of the current PostgreSQL database, make sure it's running and then execute a command like this on the server: diff --git a/docs/maintenance-synapse.md b/docs/maintenance-synapse.md new file mode 100644 index 000000000..012b76990 --- /dev/null +++ b/docs/maintenance-synapse.md @@ -0,0 +1,72 @@ +# Synapse maintenance + +This document shows you how to perform various maintenance tasks related to the Synapse chat server. + +Table of contents: + +- [Purging unused data with synapse-janitor](#purging-unused-data-with-synapse-janitor), for when you wish to delete unused data from the Synapse database + +- [Purging old data with the Purge History API](#purging-old-data-with-the-purge-history-api), for when you wish to delete in-use (but old) data from the Synapse database + +- [Compressing state with rust-synapse-compress-state](#compressing-state-with-rust-synapse-compress-state), for when you wish to compress some Synapse state tables using the [rust-synapse-compress-state](https://github.com/matrix-org/rust-synapse-compress-state) tool + + +## Purging unused data with synapse-janitor + +When you **leave** and **forget** a room, Synapse can clean up its data, but currently doesn't. +This **unused and unreachable data** remains in your database forever. + +There are external tools (like [synapse-janitor](https://github.com/xwiki-labs/synapse_scripts)), which are meant to solve this problem. + +To ask the playbook to run synapse-janitor, execute: + +```bash +ansible-playbook -i inventory/hosts setup.yml --tags=run-postgres-synapse-janitor +``` + +**Note**: this will automatically stop Synapse temporarily and restart it later. + + +### Vacuuming Postgres + +Running synapse-janitor potentially deletes a lot of data from the Postgres database. +However, disk space only ever gets released after a [`FULL` Postgres `VACUUM`](./maintenance-postgres.md#vacuuming-postgresql). + +It's easiest if you ask the playbook to run both synapse-janitor and a `VACUUM FULL` in one call: + +```bash +ansible-playbook -i inventory/hosts setup.yml --tags=run-postgres-synapse-janitor,run-postgres-vacuum +``` + +**Note**: this will automatically stop Synapse temporarily and restart it later. You'll also need plenty of available disk space in your Postgres data directory (usually `/matrix/postgres/data`). + + +## Purging old data with the Purge History API + +If [purging unused and unreachable data](#purging-unused-data-with-synapse-janitor) is not enough for you, you can start deleting in-use (but old) data. + +**This is destructive** (especially for non-federated rooms), because it means **people will no longer have access to history past a certain point**. + +Synapse provides a [Purge History API](https://github.com/matrix-org/synapse/blob/master/docs/admin_api/purge_history_api.rst) that you can use to purge on a per-room basis. + +To make use of this API, **you'll need an admin access token** first. You can find your access token in the setting of some clients (like riot-web). +Alternatively, you can log in and obtain a new access token like this: + +``` +curl \ +--data '{"identifier": {"type": "m.id.user", "user": "YOUR_MATRIX_USERNAME" }, "password": "YOUR_MATRIX_PASSWORD", "type": "m.login.password", "device_id": "Synapse-Purge-History-API"}' \ +https://matrix.DOMAIN/_matrix/client/r0/login +``` + +Follow the [Purge History API](https://github.com/matrix-org/synapse/blob/master/docs/admin_api/purge_history_api.rst) documentation page for the actual purging instructions. + +Don't forget that disk space only ever gets released after a [`FULL` Postgres `VACUUM`](./maintenance-postgres.md#vacuuming-postgresql) - something the playbook can help you with. + + +## Compressing state with rust-synapse-compress-state + +[rust-synapse-compress-state](https://github.com/matrix-org/rust-synapse-compress-state) can be used to optimize some `_state` tables used by Synapse. + +Unfortunately, at this time the playbook can't help you run this **experimental tool**. + +Since it's also experimental, you may wish to stay away from it, or at least [make Postgres backups](./docs/maintenance-postgres.md#backing-up-postgresql) first. diff --git a/roles/matrix-base/defaults/main.yml b/roles/matrix-base/defaults/main.yml index 983b0cf3f..fc119a44e 100644 --- a/roles/matrix-base/defaults/main.yml +++ b/roles/matrix-base/defaults/main.yml @@ -46,6 +46,8 @@ matrix_well_known_matrix_server_enabled: true run_postgres_import: true run_postgres_upgrade: true run_postgres_import_sqlite_db: true +run_postgres_synapse_janitor: true +run_postgres_vacuum: true run_synapse_register_user: true run_synapse_update_user_password: true run_synapse_import_media_store: true diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml index 9a2ca703c..d656ab168 100644 --- a/roles/matrix-postgres/defaults/main.yml +++ b/roles/matrix-postgres/defaults/main.yml @@ -28,3 +28,5 @@ matrix_postgres_container_extra_arguments: [] # # Takes an ":" or "" value (e.g. "127.0.0.1:5432"), or empty string to not expose. matrix_postgres_container_postgres_bind_port: "" + +matrix_postgres_tool_synapse_janitor: "https://raw.githubusercontent.com/xwiki-labs/synapse_scripts/0b3f035951932ceb396631de3fc701043b9723bc/synapse_janitor.sql" diff --git a/roles/matrix-postgres/tasks/main.yml b/roles/matrix-postgres/tasks/main.yml index 9527d1ff7..41b9c8613 100644 --- a/roles/matrix-postgres/tasks/main.yml +++ b/roles/matrix-postgres/tasks/main.yml @@ -28,3 +28,13 @@ when: run_postgres_upgrade|bool tags: - upgrade-postgres + +- import_tasks: "{{ role_path }}/tasks/run_synapse_janitor.yml" + when: run_postgres_synapse_janitor|bool + tags: + - run-postgres-synapse-janitor + +- import_tasks: "{{ role_path }}/tasks/run_vacuum.yml" + when: run_postgres_vacuum|bool + tags: + - run-postgres-vacuum diff --git a/roles/matrix-postgres/tasks/run_synapse_janitor.yml b/roles/matrix-postgres/tasks/run_synapse_janitor.yml new file mode 100644 index 000000000..5591b75a3 --- /dev/null +++ b/roles/matrix-postgres/tasks/run_synapse_janitor.yml @@ -0,0 +1,110 @@ +--- + +# Pre-checks + +- name: Fail if Postgres not enabled + fail: + msg: "Postgres via the matrix-postgres role is not enabled (`matrix_postgres_enabled`). Cannot run synapse-janitor." + when: "not matrix_postgres_enabled|bool" + + +# Defaults + +- name: Set postgres_start_wait_time, if not provided + set_fact: + postgres_start_wait_time: 15 + when: "postgres_start_wait_time|default('') == ''" + +- name: Set postgres_synapse_janitor_wait_time, if not provided + set_fact: + postgres_synapse_janitor_wait_time: "{{ 7 * 86400 }}" + when: "postgres_synapse_janitor_wait_time|default('') == ''" + +- name: Set postgres_synapse_janitor_tool_path, if not provided + set_fact: + postgres_synapse_janitor_tool_path: "{{ matrix_postgres_base_path }}/synapse_janitor.sql" + when: "postgres_synapse_janitor_tool_path|default('') == ''" + + +# Actual janitor work + +- name: Download synapse-janitor tool + get_url: + url: "{{ matrix_postgres_tool_synapse_janitor }}" + dest: "{{ postgres_synapse_janitor_tool_path }}" + force: true + mode: 0550 + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_username }}" + +- name: Ensure matrix-postgres is started + service: + name: matrix-postgres + state: started + daemon_reload: yes + +- name: Wait a bit, so that Postgres can start + wait_for: + timeout: "{{ postgres_start_wait_time }}" + delegate_to: 127.0.0.1 + become: false + +- import_tasks: tasks/util/detect_existing_postgres_version.yml + +- name: Abort, if no existing Postgres version detected + fail: + msg: "Could not find existing Postgres installation" + when: "not matrix_postgres_detected_existing|bool" + +- name: Generate Postgres database synapse-janitor command + set_fact: + matrix_postgres_synapse_janitor_command: >- + /usr/bin/docker run --rm --name matrix-postgres-synapse-janitor + --user={{ matrix_user_uid }}:{{ matrix_user_gid }} + --cap-drop=ALL + --network={{ matrix_docker_network }} + --env-file={{ matrix_postgres_base_path }}/env-postgres-psql + --mount type=bind,src={{ postgres_synapse_janitor_tool_path }},dst=/synapse_janitor.sql,ro=true + {{ matrix_postgres_docker_image_latest }} + psql -v ON_ERROR_STOP=1 -h matrix-postgres {{ matrix_synapse_database_database }} -f /synapse_janitor.sql + +- name: Note about Postgres purging alternative + debug: + msg: >- + Running synapse-janitor with the following Postgres command: `{{ matrix_postgres_synapse_janitor_command }}`. + If this crashes, you can stop all processes (`systemctl stop matrix-*`), + start Postgres only (`systemctl start matrix-postgres`) + and manually run the above command directly on the server. + +- name: Populate service facts + service_facts: + +- set_fact: + matrix_postgres_synapse_was_running: "{{ ansible_facts.services['matrix-synapse.service']|default(none) is not none and ansible_facts.services['matrix-synapse.service'].state == 'running' }}" + +- name: Ensure matrix-synapse is stopped + service: + name: matrix-synapse + state: stopped + daemon_reload: yes + +- name: Run synapse-janitor + command: "{{ matrix_postgres_synapse_janitor_command }}" + async: "{{ postgres_synapse_janitor_wait_time }}" + poll: 10 + register: matrix_postgres_synapse_janitor_result + +# Intentionally show the results +- debug: var="matrix_postgres_synapse_janitor_result" + +- name: Ensure matrix-synapse is started, if it previously was + service: + name: matrix-synapse + state: started + daemon_reload: yes + when: "matrix_postgres_synapse_was_running|bool" + +- name: Delete synapse-janitor tool + file: + path: "{{ postgres_synapse_janitor_tool_path }}" + state: absent diff --git a/roles/matrix-postgres/tasks/run_vacuum.yml b/roles/matrix-postgres/tasks/run_vacuum.yml new file mode 100644 index 000000000..3c3292fff --- /dev/null +++ b/roles/matrix-postgres/tasks/run_vacuum.yml @@ -0,0 +1,90 @@ +--- + +# Pre-checks + +- name: Fail if Postgres not enabled + fail: + msg: "Postgres via the matrix-postgres role is not enabled (`matrix_postgres_enabled`). Cannot run vacuum." + when: "not matrix_postgres_enabled|bool" + + +# Defaults + +- name: Set postgres_start_wait_time, if not provided + set_fact: + postgres_start_wait_time: 15 + when: "postgres_start_wait_time|default('') == ''" + +- name: Set postgres_vacuum_wait_time, if not provided + set_fact: + postgres_vacuum_wait_time: "{{ 7 * 86400 }}" + when: "postgres_vacuum_wait_time|default('') == ''" + + +# Actual vacuuming work + +- name: Ensure matrix-postgres is started + service: + name: matrix-postgres + state: started + daemon_reload: yes + +- name: Wait a bit, so that Postgres can start + wait_for: + timeout: "{{ postgres_start_wait_time }}" + delegate_to: 127.0.0.1 + become: false + +- import_tasks: tasks/util/detect_existing_postgres_version.yml + +- name: Abort, if no existing Postgres version detected + fail: + msg: "Could not find existing Postgres installation" + when: "not matrix_postgres_detected_existing|bool" + +- name: Generate Postgres database vacuum command + set_fact: + matrix_postgres_vacuum_command: >- + /usr/bin/docker run --rm --name matrix-postgres-synapse-vacuum + --user={{ matrix_user_uid }}:{{ matrix_user_gid }} + --cap-drop=ALL + --network={{ matrix_docker_network }} + --env-file={{ matrix_postgres_base_path }}/env-postgres-psql + {{ matrix_postgres_docker_image_latest }} + psql -v ON_ERROR_STOP=1 -h matrix-postgres {{ matrix_synapse_database_database }} -c 'VACUUM FULL VERBOSE' + +- name: Note about Postgres vacuum alternative + debug: + msg: >- + Running vacuum with the following Postgres command: `{{ matrix_postgres_vacuum_command }}`. + If this crashes, you can stop all processes (`systemctl stop matrix-*`), + start Postgres only (`systemctl start matrix-postgres`) + and manually run the above command directly on the server. + +- name: Populate service facts + service_facts: + +- set_fact: + matrix_postgres_synapse_was_running: "{{ ansible_facts.services['matrix-synapse.service']|default(none) is not none and ansible_facts.services['matrix-synapse.service'].state == 'running' }}" + +- name: Ensure matrix-synapse is stopped + service: + name: matrix-synapse + state: stopped + daemon_reload: yes + +- name: Run Postgres vacuum command + command: "{{ matrix_postgres_vacuum_command }}" + async: "{{ postgres_vacuum_wait_time }}" + poll: 10 + register: matrix_postgres_synapse_vacuum_result + +# Intentionally show the results +- debug: var="matrix_postgres_synapse_vacuum_result" + +- name: Ensure matrix-synapse is started, if it previously was + service: + name: matrix-synapse + state: started + daemon_reload: yes + when: "matrix_postgres_synapse_was_running|bool"