From 9c52a8b3a11ba51e0fd3de90e28994f1373bce21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Darveau?= Date: Mon, 10 Oct 2022 16:05:40 -0400 Subject: [PATCH 001/118] update Jitsi on LAN configuration doc --- docs/configuring-playbook-jitsi.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/configuring-playbook-jitsi.md b/docs/configuring-playbook-jitsi.md index f7e8a949c..51ad29b79 100644 --- a/docs/configuring-playbook-jitsi.md +++ b/docs/configuring-playbook-jitsi.md @@ -87,7 +87,7 @@ For more information refer to the [docker-jitsi-meet](https://github.com/jitsi/d By default the Jitsi Meet instance does not work with a client in LAN (Local Area Network), even if others are connected from WAN. There are no video and audio. In the case of WAN to WAN everything is ok. -The reason is the Jitsi VideoBridge git to LAN client the IP address of the docker image instead of the host. The [documentation](https://github.com/jitsi/docker-jitsi-meet#running-behind-nat-or-on-a-lan-environment) of Jitsi in docker suggest to add `DOCKER_HOST_ADDRESS` in enviornment variable to make it work. +The reason is the Jitsi VideoBridge git to LAN client the IP address of the docker image instead of the host. The [documentation](https://jitsi.github.io/handbook/docs/devops-guide/devops-guide-docker/#running-behind-nat-or-on-a-lan-environment) of Jitsi in docker suggest to add `JVB_ADVERTISE_IPS` in enviornment variable to make it work. Here is how to do it in the playbook. @@ -95,7 +95,7 @@ Add these two lines to your `inventory/host_vars/matrix.DOMAIN/vars.yml` configu ```yaml matrix_jitsi_jvb_container_extra_arguments: - - '--env "DOCKER_HOST_ADDRESS="' + - '--env "JVB_ADVERTISE_IPS="' ``` ## (Optional) Fine tune Jitsi From 3a1847e193f6d99bdf5e4551856c302a7a60de55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Darveau?= Date: Mon, 10 Oct 2022 16:25:17 -0400 Subject: [PATCH 002/118] moved bind port variables documentation for webserver on different docker network or host in a more generic section --- docs/configuring-playbook-own-webserver.md | 30 ++++++++++++---------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/docs/configuring-playbook-own-webserver.md b/docs/configuring-playbook-own-webserver.md index 7e5d60013..ecae105bc 100644 --- a/docs/configuring-playbook-own-webserver.md +++ b/docs/configuring-playbook-own-webserver.md @@ -27,11 +27,23 @@ No matter which external webserver you decide to go with, you'll need to: 1) Make sure your web server user (something like `http`, `apache`, `www-data`, `nginx`) is part of the `matrix` group. You should run something like this: `usermod -a -G matrix nginx`. This allows your webserver user to access files owned by the `matrix` group. When using an external nginx webserver, this allows it to read configuration files from `/matrix/nginx-proxy/conf.d`. When using another server, it would make other files, such as `/matrix/static-files/.well-known`, accessible to it. -2) Edit your configuration file (`inventory/host_vars/matrix./vars.yml`) to disable the integrated nginx server: +2) Edit your configuration file (`inventory/host_vars/matrix./vars.yml`) + - to disable the integrated nginx server: -```yaml -matrix_nginx_proxy_enabled: false -``` + ```yaml + matrix_nginx_proxy_enabled: false + ``` + - if using an external server on another docker network or host, add the `_http_host_bind_port` or `_http_bind_port` variables for the services that will be exposed by the external server on the other docker network or host. The actual name of the variable is listed in the `roles//defaults/vars.yml` file for each service. Most variables follow the `_http_host_bind_port` format. + + These variables will make Docker expose the ports to the local network instead of localhost only. + [Keep in mind that there are some security concerns if you simply proxy everything.](https://github.com/matrix-org/synapse/blob/master/docs/reverse_proxy.md#synapse-administration-endpoints) + + Here are the variables required for the default configuration (Synapse and Element) + ``` + matrix_synapse_container_client_api_host_bind_port: '8008' + matrix_synapse_container_federation_api_plain_host_bind_port: '8048' + matrix_client_element_container_http_host_bind_port: "8765" + ``` 3) **If you'll manage SSL certificates by yourself**, edit your configuration file (`inventory/host_vars/matrix./vars.yml`) to disable SSL certificate retrieval: @@ -41,7 +53,6 @@ matrix_ssl_retrieval_method: none **Note**: During [installation](installing.md), unless you've disabled SSL certificate management (`matrix_ssl_retrieval_method: none`), the playbook would need 80 to be available, in order to retrieve SSL certificates. **Please manually stop your other webserver while installing**. You can start it back up afterwards. - ### Using your own external nginx webserver Once you've followed the [Preparation](#preparation) guide above, it's time to set up your external nginx server. @@ -60,15 +71,6 @@ matrix_nginx_proxy_ssl_protocols: "TLSv1.2" If you are experiencing issues, try updating to a newer version of Nginx. As a data point in May 2021 a user reported that Nginx 1.14.2 was not working for them. They were getting errors about socket leaks. Updating to Nginx 1.19 fixed their issue. -If you are not going to be running your webserver on the same docker network, or the same machine as matrix, these variables can be set to bind synapse to an exposed port. [Keep in mind that there are some security concerns if you simply proxy everything to it](https://github.com/matrix-org/synapse/blob/master/docs/reverse_proxy.md#synapse-administration-endpoints) -```yaml -# Takes an ":" or "" value (e.g. "127.0.0.1:8048" or "192.168.1.3:80"), or empty string to not expose. -matrix_synapse_container_client_api_host_bind_port: '' -matrix_synapse_container_federation_api_plain_host_bind_port: '' -``` - - - ### Using your own external Apache webserver Once you've followed the [Preparation](#preparation) guide above, you can take a look at the [examples/apache](../examples/apache) directory for a sample configuration. From 527184097131b22ef913cc031355d3e5ace8fd0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Darveau?= Date: Mon, 10 Oct 2022 16:43:42 -0400 Subject: [PATCH 003/118] caddy2 Caddyfile : add CORS handling, update click-jacking protection to handle different subdomains and add example for cases with other well-knowns --- examples/caddy2/Caddyfile | 45 ++++++++++++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/examples/caddy2/Caddyfile b/examples/caddy2/Caddyfile index 162e539e5..a19ce6dd7 100644 --- a/examples/caddy2/Caddyfile +++ b/examples/caddy2/Caddyfile @@ -1,3 +1,15 @@ +(cors) { + @cors_preflight method OPTIONS + + handle @cors_preflight { + header Access-Control-Allow-Origin "{args.0}" + header Access-Control-Allow-Methods "HEAD, GET, POST, PUT, PATCH, DELETE" + header Access-Control-Allow-Headers "Content-Type, Authorization" + header Access-Control-Max-Age "3600" + } +} + + matrix.DOMAIN.tld { # creates letsencrypt certificate @@ -81,6 +93,13 @@ matrix.DOMAIN.tld { header Access-Control-Allow-Origin * file_server } + + # If you have other well-knowns already handled by your base domain, you can replace the above block by this one, along with the replacement suggested in the base domain + #handle @wellknown { + # # .well-known is handled by base domain + # reverse_proxy https://DOMAIN.tld { + # header_up Host {http.reverse_proxy.upstream.hostport} + #} handle { encode zstd gzip @@ -114,6 +133,8 @@ element.DOMAIN.tld { # creates letsencrypt certificate # tls your@email.com + import cors https://*.DOMAIN.tld + header { # Enable HTTP Strict Transport Security (HSTS) to force clients to always connect via HTTPS Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" @@ -123,6 +144,8 @@ element.DOMAIN.tld { X-Content-Type-Options "nosniff" # Disallow the site to be rendered within a frame (clickjacking protection) X-Frame-Options "DENY" + # If using integrations that add frames to Element, such as Dimension and its integrations running on the same domain, it can be a good idea to limit sources allowed to be rendered + # Content-Security-Policy frame-src https://*.DOMAIN.tld # X-Robots-Tag X-Robots-Tag "noindex, noarchive, nofollow" } @@ -144,6 +167,8 @@ element.DOMAIN.tld { # # creates letsencrypt certificate # # tls your@email.com # +# import cors https://*.DOMAIN.tld +# # header { # # Enable HTTP Strict Transport Security (HSTS) to force clients to always connect via HTTPS # Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" @@ -151,8 +176,8 @@ element.DOMAIN.tld { # X-XSS-Protection "1; mode=block" # # Prevent some browsers from MIME-sniffing a response away from the declared Content-Type # X-Content-Type-Options "nosniff" -# # Disallow the site to be rendered within a frame (clickjacking protection) -# X-Frame-Options "DENY" +# # Only allow same base domain to render this website in a frame; Can be removed if the client (Element for example) is hosted on another domain (clickjacking protection) +# # Content-Security-Policy frame-ancestors https://*.DOMAIN.tld # # X-Robots-Tag # X-Robots-Tag "noindex, noarchive, nofollow" # } @@ -176,6 +201,8 @@ element.DOMAIN.tld { # creates letsencrypt certificate # tls your@email.com # +# import cors https://*.DOMAIN.tld +# # header { # # Enable HTTP Strict Transport Security (HSTS) to force clients to always connect via HTTPS # Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" @@ -185,9 +212,9 @@ element.DOMAIN.tld { # # # Prevent some browsers from MIME-sniffing a response away from the declared Content-Type # X-Content-Type-Options "nosniff" -# -# # Disallow the site to be rendered within a frame (clickjacking protection) -# X-Frame-Options "SAMEORIGIN" + +# # Only allow same base domain to render this website in a frame; Can be removed if the client (Element for example) is hosted on another domain +# # Content-Security-Policy frame-ancestors https://*.DOMAIN.tld # # # Disable some features # Feature-Policy "accelerometer 'none';ambient-light-sensor 'none'; autoplay 'none';camera 'none';encrypted-media 'none';focus-without-user-activation 'none'; geolocation 'none';gyroscope #'none';magnetometer 'none';microphone 'none';midi 'none';payment 'none';picture-in-picture 'none'; speaker 'none';sync-xhr 'none';usb 'none';vr 'none'" @@ -225,6 +252,14 @@ element.DOMAIN.tld { # header_up Host {http.reverse_proxy.upstream.hostport} # } # } +# # If you have other well-knowns already handled by your base domain, you can replace the above block by this one, along with the replacement suggested in the matrix subdomain +# # handle /.well-known/* { +# # encode zstd gzip +# # header Cache-Control max-age=14400 +# # header Content-Type application/json +# # header Access-Control-Allow-Origin * +# #} +# # # Configration for the base domain goes here # # handle { # # header -Server From a47ce70cd26a83148785d112e8b647377fe50300 Mon Sep 17 00:00:00 2001 From: Jim Myhrberg Date: Tue, 11 Oct 2022 00:43:22 +0100 Subject: [PATCH 004/118] fix(goofys): fix synapse systemd unit file to correctly require goofys On Debian 10 (buster) at least, while the Synapse systemd service unit was referring to Goofys as "matrix-goofys" without a ".service" suffix, systemd was ignoring the goofys dependency, starting Synapse before Goofys. All other dependant units which work are using the ".service" suffix. This generally leads to the mount path goofys using having been populated by Synapse before Goofys starts, causing it to fail due to the mount target not being empty. The fix seems to be to ensure that the Synapse service unit refers to Goofys as "matrix-goofys.service". This change causes the following two lines in "/etc/systemd/system/matrix-synapse.service": Requires=matrix-goofys After=matrix-goofys To be changed to: Requires=matrix-goofys.service After=matrix-goofys.service --- group_vars/matrix_servers | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index cfccd3876..b055245bc 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -2364,7 +2364,7 @@ matrix_synapse_systemd_required_services_list: | + (['matrix-postgres.service'] if matrix_postgres_enabled else []) + - (['matrix-goofys'] if matrix_s3_media_store_enabled else []) + (['matrix-goofys.service'] if matrix_s3_media_store_enabled else []) }} matrix_synapse_systemd_wanted_services_list: | @@ -2650,7 +2650,7 @@ matrix_dendrite_systemd_required_services_list: | + (['matrix-postgres.service'] if matrix_postgres_enabled else []) + - (['matrix-goofys'] if matrix_s3_media_store_enabled else []) + (['matrix-goofys.service'] if matrix_s3_media_store_enabled else []) }} matrix_dendrite_systemd_wanted_services_list: | From 9412140477902e9aff7177c9a039527ba40c4c8d Mon Sep 17 00:00:00 2001 From: Aine <97398200+etkecc@users.noreply.github.com> Date: Tue, 11 Oct 2022 06:00:49 +0000 Subject: [PATCH 005/118] Enable hookshot feeds by default as per documentation: "Services that need no further configuration are enabled by default" and feeds service doesn't require it --- roles/matrix-bridge-hookshot/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bridge-hookshot/defaults/main.yml b/roles/matrix-bridge-hookshot/defaults/main.yml index b9b308660..0854edbf5 100644 --- a/roles/matrix-bridge-hookshot/defaults/main.yml +++ b/roles/matrix-bridge-hookshot/defaults/main.yml @@ -128,7 +128,7 @@ matrix_hookshot_generic_allow_js_transformation_functions: false matrix_hookshot_generic_user_id_prefix: '_webhooks_' -matrix_hookshot_feeds_enabled: false +matrix_hookshot_feeds_enabled: true # polling interval in seconds matrix_hookshot_feeds_interval: 600 From 5ff03c0b15eb16e5e3a7f016925f6df78f36e1c6 Mon Sep 17 00:00:00 2001 From: Aine <97398200+etkecc@users.noreply.github.com> Date: Tue, 11 Oct 2022 06:04:18 +0000 Subject: [PATCH 006/118] Change hookshot logging level to warn all other bridges use that level --- roles/matrix-bridge-hookshot/templates/config.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bridge-hookshot/templates/config.yml.j2 b/roles/matrix-bridge-hookshot/templates/config.yml.j2 index 6fbce7709..5cd11a874 100644 --- a/roles/matrix-bridge-hookshot/templates/config.yml.j2 +++ b/roles/matrix-bridge-hookshot/templates/config.yml.j2 @@ -108,7 +108,7 @@ metrics: logging: # (Optional) Logging settings. You can have a severity debug,info,warn,error # - level: info + level: warn {% if matrix_hookshot_widgets_enabled %} widgets: # (Optional) EXPERIMENTAL support for complimentary widgets From da041bc60c67f674ac9f748e528ee3d42ebafb28 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 11 Oct 2022 09:38:11 +0300 Subject: [PATCH 007/118] Upgrade mautrix-instagram (0.2.1 -> latest) We don't like updating to untagged releases, but.. 0.2.1 has some regression and upstream is not releasing 0.2.2 or 0.3.0 just yet, so we either need to downgrade to 0.2.0 or go `latest`. We can hopefully switch back to a tagged release soon. Related to https://github.com/mautrix/instagram/issues/56 --- roles/matrix-bridge-mautrix-instagram/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bridge-mautrix-instagram/defaults/main.yml b/roles/matrix-bridge-mautrix-instagram/defaults/main.yml index b739a4a37..7305d0fbf 100644 --- a/roles/matrix-bridge-mautrix-instagram/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-instagram/defaults/main.yml @@ -8,7 +8,7 @@ matrix_mautrix_instagram_container_image_self_build: false matrix_mautrix_instagram_container_image_self_build_repo: "https://github.com/mautrix/instagram.git" matrix_mautrix_instagram_container_image_self_build_repo_version: "{{ 'master' if matrix_mautrix_instagram_version == 'latest' else matrix_mautrix_instagram_version }}" -matrix_mautrix_instagram_version: v0.2.1 +matrix_mautrix_instagram_version: latest # See: https://mau.dev/tulir/mautrix-instagram/container_registry matrix_mautrix_instagram_docker_image: "{{ matrix_mautrix_instagram_docker_image_name_prefix }}mautrix/instagram:{{ matrix_mautrix_instagram_version }}" matrix_mautrix_instagram_docker_image_name_prefix: "{{ 'localhost/' if matrix_mautrix_instagram_container_image_self_build else 'dock.mau.dev/' }}" From 8311c59a8fa3a9b13b9076d0d6fbaefff66e74be Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 11 Oct 2022 10:43:01 +0300 Subject: [PATCH 008/118] Add Usage section to Hookshot docs --- docs/configuring-playbook-bridge-hookshot.md | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/docs/configuring-playbook-bridge-hookshot.md b/docs/configuring-playbook-bridge-hookshot.md index 3e8a54a20..51d7a335f 100644 --- a/docs/configuring-playbook-bridge-hookshot.md +++ b/docs/configuring-playbook-bridge-hookshot.md @@ -8,6 +8,7 @@ See the project's [documentation](https://matrix-org.github.io/matrix-hookshot/l Note: the playbook also supports [matrix-appservice-webhooks](configuring-playbook-bridge-appservice-webhooks.md), which however is soon to be archived by its author and to be replaced by hookshot. + ## Setup Instructions Refer to the [official instructions](https://matrix-org.github.io/matrix-hookshot/latest/setup.html) to learn what the individual options do. @@ -16,10 +17,25 @@ Refer to the [official instructions](https://matrix-org.github.io/matrix-hooksho 2. Take special note of the `matrix_hookshot_*_enabled` variables. Services that need no further configuration are enabled by default (GitLab, Generic), while you must first add the required configuration and enable the others (GitHub, Jira, Figma). 3. If you're setting up the GitHub bridge, you'll need to generate and download a private key file after you created your GitHub app. Copy the contents of that file to the variable `matrix_hookshot_github_private_key` so the playbook can install it for you, or use one of the [other methods](#manage-github-private-key-with-matrix-aux-role) explained below. 4. If you've already installed Matrix services using the playbook before, you'll need to re-run it (`--tags=setup-all,start`). If not, proceed with [configuring other playbook services](configuring-playbook.md) and then with [Installing](installing.md). Get back to this guide once ready. Hookshot can be set up individually using the tag `setup-hookshot`. -5. Refer to [Hookshot's official instructions](https://matrix-org.github.io/matrix-hookshot/latest/usage.html) to start using the bridge. **Important:** Note that the different listeners are bound to certain paths which might differ from those assumed by the hookshot documentation, see [URLs for bridges setup](urls-for-bridges-setup) below. Other configuration options are available via the `matrix_hookshot_configuration_extension_yaml` and `matrix_hookshot_registration_extension_yaml` variables, see the comments in [main.yml](/roles/matrix-bridge-hookshot/defaults/main.yml) for how to use them. + +## Usage + +Create a room and invite the Hookshot bot (`@hookshot:DOMAIN`) to it. + +Make sure the bot is able to send state events (usually the Moderator power level in clients). + +Send a `!hookshot help` message to see a list of help commands. + +Refer to [Hookshot's documentation](https://matrix-org.github.io/matrix-hookshot/latest/usage.html) for more details about using the brige's various features. + +**Important:** Note that the different listeners are bound to certain paths which might differ from those assumed by the hookshot documentation, see [URLs for bridges setup](#urls-for-bridges-setup) below. + + +## More setup documentation + ### URLs for bridges setup Unless indicated otherwise, the following endpoints are reachable on your `matrix.` subdomain (if the feature is enabled). From 21bed90e13c46b6ed5fb3f0945bf9435e76ede90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Darveau?= Date: Tue, 11 Oct 2022 07:26:29 -0400 Subject: [PATCH 009/118] remove mention of other docker host in external webserver documentation (binding ports section) --- docs/configuring-playbook-own-webserver.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/configuring-playbook-own-webserver.md b/docs/configuring-playbook-own-webserver.md index ecae105bc..50d9d1da7 100644 --- a/docs/configuring-playbook-own-webserver.md +++ b/docs/configuring-playbook-own-webserver.md @@ -33,16 +33,16 @@ No matter which external webserver you decide to go with, you'll need to: ```yaml matrix_nginx_proxy_enabled: false ``` - - if using an external server on another docker network or host, add the `_http_host_bind_port` or `_http_bind_port` variables for the services that will be exposed by the external server on the other docker network or host. The actual name of the variable is listed in the `roles//defaults/vars.yml` file for each service. Most variables follow the `_http_host_bind_port` format. + - if using an external server on another host, add the `_http_host_bind_port` or `_http_bind_port` variables for the services that will be exposed by the external server on the other host. The actual name of the variable is listed in the `roles//defaults/vars.yml` file for each service. Most variables follow the `_http_host_bind_port` format. These variables will make Docker expose the ports to the local network instead of localhost only. [Keep in mind that there are some security concerns if you simply proxy everything.](https://github.com/matrix-org/synapse/blob/master/docs/reverse_proxy.md#synapse-administration-endpoints) Here are the variables required for the default configuration (Synapse and Element) ``` - matrix_synapse_container_client_api_host_bind_port: '8008' - matrix_synapse_container_federation_api_plain_host_bind_port: '8048' - matrix_client_element_container_http_host_bind_port: "8765" + matrix_synapse_container_client_api_host_bind_port: '0.0.0.0:8008' + matrix_synapse_container_federation_api_plain_host_bind_port: '0.0.0.0:8048' + matrix_client_element_container_http_host_bind_port: "0.0.0.0:8765" ``` 3) **If you'll manage SSL certificates by yourself**, edit your configuration file (`inventory/host_vars/matrix./vars.yml`) to disable SSL certificate retrieval: From a061ea54b3e302e1582daa442690ad8d71d6ddc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Darveau?= Date: Tue, 11 Oct 2022 07:32:02 -0400 Subject: [PATCH 010/118] Caddyfile example : enable Content-Security-Policy by default instead of having the line commented --- examples/caddy2/Caddyfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/caddy2/Caddyfile b/examples/caddy2/Caddyfile index a19ce6dd7..43005ca41 100644 --- a/examples/caddy2/Caddyfile +++ b/examples/caddy2/Caddyfile @@ -177,7 +177,7 @@ element.DOMAIN.tld { # # Prevent some browsers from MIME-sniffing a response away from the declared Content-Type # X-Content-Type-Options "nosniff" # # Only allow same base domain to render this website in a frame; Can be removed if the client (Element for example) is hosted on another domain (clickjacking protection) -# # Content-Security-Policy frame-ancestors https://*.DOMAIN.tld +# Content-Security-Policy frame-ancestors https://*.DOMAIN.tld # # X-Robots-Tag # X-Robots-Tag "noindex, noarchive, nofollow" # } @@ -214,7 +214,7 @@ element.DOMAIN.tld { # X-Content-Type-Options "nosniff" # # Only allow same base domain to render this website in a frame; Can be removed if the client (Element for example) is hosted on another domain -# # Content-Security-Policy frame-ancestors https://*.DOMAIN.tld +# Content-Security-Policy frame-ancestors https://*.DOMAIN.tld # # # Disable some features # Feature-Policy "accelerometer 'none';ambient-light-sensor 'none'; autoplay 'none';camera 'none';encrypted-media 'none';focus-without-user-activation 'none'; geolocation 'none';gyroscope #'none';magnetometer 'none';microphone 'none';midi 'none';payment 'none';picture-in-picture 'none'; speaker 'none';sync-xhr 'none';usb 'none';vr 'none'" From 19f1c8ea5ee3ffbd520cc6c54fc45957bea8b0e5 Mon Sep 17 00:00:00 2001 From: Aine <97398200+etkecc@users.noreply.github.com> Date: Tue, 11 Oct 2022 14:29:52 +0000 Subject: [PATCH 011/118] Update element 1.11.8 -> 1.11.9 --- roles/matrix-client-element/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-client-element/defaults/main.yml b/roles/matrix-client-element/defaults/main.yml index e4b6315b0..acdaec83c 100644 --- a/roles/matrix-client-element/defaults/main.yml +++ b/roles/matrix-client-element/defaults/main.yml @@ -10,7 +10,7 @@ matrix_client_element_container_image_self_build_repo: "https://github.com/vecto # - https://github.com/vector-im/element-web/issues/19544 matrix_client_element_container_image_self_build_low_memory_system_patch_enabled: "{{ ansible_memtotal_mb < 4096 }}" -matrix_client_element_version: v1.11.8 +matrix_client_element_version: v1.11.9 matrix_client_element_docker_image: "{{ matrix_client_element_docker_image_name_prefix }}vectorim/element-web:{{ matrix_client_element_version }}" matrix_client_element_docker_image_name_prefix: "{{ 'localhost/' if matrix_client_element_container_image_self_build else matrix_container_global_registry_prefix }}" matrix_client_element_docker_image_force_pull: "{{ matrix_client_element_docker_image.endswith(':latest') }}" From fe8f91a9dac67e6b48a8704e7010768575eed1ab Mon Sep 17 00:00:00 2001 From: Aine <97398200+etkecc@users.noreply.github.com> Date: Tue, 11 Oct 2022 14:30:55 +0000 Subject: [PATCH 012/118] Update postmoogle 0.9.5 -> 0.9.6 * show recipient's email (header `TO`) * add `norecipient` option * add anti-spam options (`spamcheck:` and `spamlist:`) * fix bugs --- roles/matrix-bot-postmoogle/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bot-postmoogle/defaults/main.yml b/roles/matrix-bot-postmoogle/defaults/main.yml index 949f0913c..8e76998eb 100644 --- a/roles/matrix-bot-postmoogle/defaults/main.yml +++ b/roles/matrix-bot-postmoogle/defaults/main.yml @@ -9,7 +9,7 @@ matrix_bot_postmoogle_docker_repo: "https://gitlab.com/etke.cc/postmoogle.git" matrix_bot_postmoogle_docker_repo_version: "{{ 'main' if matrix_bot_postmoogle_version == 'latest' else matrix_bot_postmoogle_version }}" matrix_bot_postmoogle_docker_src_files_path: "{{ matrix_base_data_path }}/postmoogle/docker-src" -matrix_bot_postmoogle_version: v0.9.5 +matrix_bot_postmoogle_version: v0.9.6 matrix_bot_postmoogle_docker_image: "{{ matrix_bot_postmoogle_docker_image_name_prefix }}postmoogle:{{ matrix_bot_postmoogle_version }}" matrix_bot_postmoogle_docker_image_name_prefix: "{{ 'localhost/' if matrix_bot_postmoogle_container_image_self_build else 'registry.gitlab.com/etke.cc/' }}" matrix_bot_postmoogle_docker_image_force_pull: "{{ matrix_bot_postmoogle_docker_image.endswith(':latest') }}" From 3c669761cec4ffd6e7811776934c8927890d1180 Mon Sep 17 00:00:00 2001 From: Aine <97398200+etkecc@users.noreply.github.com> Date: Tue, 11 Oct 2022 18:31:46 +0000 Subject: [PATCH 013/118] Update element 1.11.9 -> 1.11.10 --- roles/matrix-client-element/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-client-element/defaults/main.yml b/roles/matrix-client-element/defaults/main.yml index acdaec83c..c8e476a19 100644 --- a/roles/matrix-client-element/defaults/main.yml +++ b/roles/matrix-client-element/defaults/main.yml @@ -10,7 +10,7 @@ matrix_client_element_container_image_self_build_repo: "https://github.com/vecto # - https://github.com/vector-im/element-web/issues/19544 matrix_client_element_container_image_self_build_low_memory_system_patch_enabled: "{{ ansible_memtotal_mb < 4096 }}" -matrix_client_element_version: v1.11.9 +matrix_client_element_version: v1.11.10 matrix_client_element_docker_image: "{{ matrix_client_element_docker_image_name_prefix }}vectorim/element-web:{{ matrix_client_element_version }}" matrix_client_element_docker_image_name_prefix: "{{ 'localhost/' if matrix_client_element_container_image_self_build else matrix_container_global_registry_prefix }}" matrix_client_element_docker_image_force_pull: "{{ matrix_client_element_docker_image.endswith(':latest') }}" From 43b1be9a3c121764c9c48eb7a3542aaee7e0f469 Mon Sep 17 00:00:00 2001 From: Aine <97398200+etkecc@users.noreply.github.com> Date: Tue, 11 Oct 2022 18:57:52 +0000 Subject: [PATCH 014/118] Update grafana 9.1.7 -> 9.2.0 --- roles/matrix-grafana/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-grafana/defaults/main.yml b/roles/matrix-grafana/defaults/main.yml index 2b139cd47..677435e82 100644 --- a/roles/matrix-grafana/defaults/main.yml +++ b/roles/matrix-grafana/defaults/main.yml @@ -5,7 +5,7 @@ matrix_grafana_enabled: true -matrix_grafana_version: 9.1.7 +matrix_grafana_version: 9.2.0 matrix_grafana_docker_image: "{{ matrix_container_global_registry_prefix }}grafana/grafana:{{ matrix_grafana_version }}" matrix_grafana_docker_image_force_pull: "{{ matrix_grafana_docker_image.endswith(':latest') }}" From 7c7b9d119f73a42222f089548e4a97be959ca5fe Mon Sep 17 00:00:00 2001 From: Aine <97398200+etkecc@users.noreply.github.com> Date: Wed, 12 Oct 2022 10:57:25 +0000 Subject: [PATCH 015/118] Update postmoogle 0.9.6 -> 0.9.7 * fix room account data retrieving --- roles/matrix-bot-postmoogle/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bot-postmoogle/defaults/main.yml b/roles/matrix-bot-postmoogle/defaults/main.yml index 8e76998eb..1bcd570a5 100644 --- a/roles/matrix-bot-postmoogle/defaults/main.yml +++ b/roles/matrix-bot-postmoogle/defaults/main.yml @@ -9,7 +9,7 @@ matrix_bot_postmoogle_docker_repo: "https://gitlab.com/etke.cc/postmoogle.git" matrix_bot_postmoogle_docker_repo_version: "{{ 'main' if matrix_bot_postmoogle_version == 'latest' else matrix_bot_postmoogle_version }}" matrix_bot_postmoogle_docker_src_files_path: "{{ matrix_base_data_path }}/postmoogle/docker-src" -matrix_bot_postmoogle_version: v0.9.6 +matrix_bot_postmoogle_version: v0.9.7 matrix_bot_postmoogle_docker_image: "{{ matrix_bot_postmoogle_docker_image_name_prefix }}postmoogle:{{ matrix_bot_postmoogle_version }}" matrix_bot_postmoogle_docker_image_name_prefix: "{{ 'localhost/' if matrix_bot_postmoogle_container_image_self_build else 'registry.gitlab.com/etke.cc/' }}" matrix_bot_postmoogle_docker_image_force_pull: "{{ matrix_bot_postmoogle_docker_image.endswith(':latest') }}" From 35e80b708781e7db6c04a778ce0892c7db225605 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Darveau?= Date: Wed, 12 Oct 2022 09:15:03 -0400 Subject: [PATCH 016/118] Update docs/configuring-playbook-own-webserver.md Co-authored-by: Slavi Pantaleev --- docs/configuring-playbook-own-webserver.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuring-playbook-own-webserver.md b/docs/configuring-playbook-own-webserver.md index 50d9d1da7..87201e3e9 100644 --- a/docs/configuring-playbook-own-webserver.md +++ b/docs/configuring-playbook-own-webserver.md @@ -35,7 +35,7 @@ No matter which external webserver you decide to go with, you'll need to: ``` - if using an external server on another host, add the `_http_host_bind_port` or `_http_bind_port` variables for the services that will be exposed by the external server on the other host. The actual name of the variable is listed in the `roles//defaults/vars.yml` file for each service. Most variables follow the `_http_host_bind_port` format. - These variables will make Docker expose the ports to the local network instead of localhost only. + These variables will make Docker expose the ports on all network interfaces instead of localhost only. [Keep in mind that there are some security concerns if you simply proxy everything.](https://github.com/matrix-org/synapse/blob/master/docs/reverse_proxy.md#synapse-administration-endpoints) Here are the variables required for the default configuration (Synapse and Element) From 337016da28e2cb18c4cf3624a47c2faec4fed272 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 14 Oct 2022 09:27:46 +0300 Subject: [PATCH 017/118] Improve S3 docs a bit Fixes https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/1869 --- docs/configuring-playbook-s3.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/docs/configuring-playbook-s3.md b/docs/configuring-playbook-s3.md index 9132ff718..43aaa8792 100644 --- a/docs/configuring-playbook-s3.md +++ b/docs/configuring-playbook-s3.md @@ -13,7 +13,7 @@ If you'd like to move your locally-stored media store data to Amazon S3 (or anot ## Amazon S3 -You'll need an Amazon S3 bucket and some IAM user credentials (access key + secret key) with full write access to the bucket. Example security policy: +You'll need an Amazon S3 bucket and some IAM user credentials (access key + secret key) with full write access to the bucket. Example IAM security policy: ```json { @@ -34,6 +34,8 @@ You'll need an Amazon S3 bucket and some IAM user credentials (access key + secr } ``` +**NOTE**: This policy needs to be attached to an IAM user creted from the **Security Credentials** menu. This is not a **Bucket Policy**. + You then need to enable S3 support in your configuration file (`inventory/host_vars/matrix./vars.yml`). It would be something like this: @@ -91,8 +93,13 @@ It's a good idea to [make a complete server backup](faq.md#how-do-i-backup-the-d Follow one of the guides below for a migration path from a locally-stored media store to one stored on S3-compatible storage: -- [Migrating to any S3-compatible storage (universal, but likely slow)](#migrating-to-any-s3-compatible-storage-universal-but-likely-slow) -- [Migrating to Backblaze B2](#migrating-to-backblaze-b2) +- [Storing Matrix media files on Amazon S3 (optional)](#storing-matrix-media-files-on-amazon-s3-optional) + - [Amazon S3](#amazon-s3) + - [Using other S3-compatible object stores](#using-other-s3-compatible-object-stores) + - [Backblaze B2](#backblaze-b2) + - [Migrating from local filesystem storage to S3](#migrating-from-local-filesystem-storage-to-s3) + - [Migrating to any S3-compatible storage (universal, but likely slow)](#migrating-to-any-s3-compatible-storage-universal-but-likely-slow) + - [Migrating to Backblaze B2](#migrating-to-backblaze-b2) ### Migrating to any S3-compatible storage (universal, but likely slow) From 62215a87fed6cbeffdb124261209df00f82ba1a5 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 14 Oct 2022 16:25:57 +0300 Subject: [PATCH 018/118] Upgrade Postgres minor versions (14.4 -> 14.5, 13.7 -> 13.8, etc.) --- roles/matrix-postgres/defaults/main.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml index 117611a34..de28d7ada 100644 --- a/roles/matrix-postgres/defaults/main.yml +++ b/roles/matrix-postgres/defaults/main.yml @@ -24,11 +24,11 @@ matrix_postgres_architecture: amd64 matrix_postgres_docker_image_suffix: "{{ '-alpine' if matrix_postgres_architecture in ['amd64', 'arm64'] else '' }}" matrix_postgres_docker_image_v9: "{{ matrix_container_global_registry_prefix }}postgres:9.6.24{{ matrix_postgres_docker_image_suffix }}" -matrix_postgres_docker_image_v10: "{{ matrix_container_global_registry_prefix }}postgres:10.21{{ matrix_postgres_docker_image_suffix }}" -matrix_postgres_docker_image_v11: "{{ matrix_container_global_registry_prefix }}postgres:11.16{{ matrix_postgres_docker_image_suffix }}" -matrix_postgres_docker_image_v12: "{{ matrix_container_global_registry_prefix }}postgres:12.11{{ matrix_postgres_docker_image_suffix }}" -matrix_postgres_docker_image_v13: "{{ matrix_container_global_registry_prefix }}postgres:13.7{{ matrix_postgres_docker_image_suffix }}" -matrix_postgres_docker_image_v14: "{{ matrix_container_global_registry_prefix }}postgres:14.4{{ matrix_postgres_docker_image_suffix }}" +matrix_postgres_docker_image_v10: "{{ matrix_container_global_registry_prefix }}postgres:10.22{{ matrix_postgres_docker_image_suffix }}" +matrix_postgres_docker_image_v11: "{{ matrix_container_global_registry_prefix }}postgres:11.17{{ matrix_postgres_docker_image_suffix }}" +matrix_postgres_docker_image_v12: "{{ matrix_container_global_registry_prefix }}postgres:12.12{{ matrix_postgres_docker_image_suffix }}" +matrix_postgres_docker_image_v13: "{{ matrix_container_global_registry_prefix }}postgres:13.8{{ matrix_postgres_docker_image_suffix }}" +matrix_postgres_docker_image_v14: "{{ matrix_container_global_registry_prefix }}postgres:14.5{{ matrix_postgres_docker_image_suffix }}" matrix_postgres_docker_image_latest: "{{ matrix_postgres_docker_image_v14 }}" # This variable is assigned at runtime. Overriding its value has no effect. From 0da068c7e61a35c2fdb226fe95df4f06b6527953 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 14 Oct 2022 16:28:47 +0300 Subject: [PATCH 019/118] Add support for configuring Synapse's media_storage_providers In the future, we'll also inject media storage providers that we support into the `matrix_synapse_media_storage_providers_auto` list. --- roles/matrix-synapse/defaults/main.yml | 20 +++++++++++++++++++ .../templates/synapse/homeserver.yaml.j2 | 1 + 2 files changed, 21 insertions(+) diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index 4c5fc09e8..d117eaafa 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -798,6 +798,26 @@ matrix_synapse_spam_checker: [] # Certain Synapse extensions that you can enable below auto-inject themselves into `matrix_synapse_modules` at runtime. matrix_synapse_modules: [] +# matrix_synapse_media_storage_providers contains the Synapse `media_storage_providers` configuration setting. +# To add your own custom `media_storage_providers`, use `matrix_synapse_media_storage_providers_custom`. +matrix_synapse_media_storage_providers: "{{ matrix_synapse_media_storage_providers_auto + matrix_synapse_media_storage_providers_custom }}" + +# matrix_synapse_media_storage_providers_auto contains a list of storage providers that are added by the playbook based on other configuration +matrix_synapse_media_storage_providers_auto: | + {{ + [] + }} + +# matrix_synapse_media_storage_providers_custom contains your own custom list of storage providers. +# You're meant to define each custom module as valid keys and values, not as a YAML string that needs to be parsed. +# +# Example: +# matrix_synapse_media_storage_providers_custom: +# - module: module.SomeModule +# store_local: True +# # ... +matrix_synapse_media_storage_providers_custom: [] + matrix_synapse_encryption_enabled_by_default_for_room_type: "off" matrix_synapse_trusted_key_servers: diff --git a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 index ae71b7aec..9b02346cf 100644 --- a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 +++ b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 @@ -1029,6 +1029,7 @@ media_store_path: "/matrix-media-store-parent/{{ matrix_synapse_media_store_dire # store_synchronous: false # config: # directory: /mnt/some/other/directory +media_storage_providers: {{ matrix_synapse_media_storage_providers | to_json }} # The largest allowed upload size in bytes # From 881fdd28f0bf2e10635ac7a6c3941fa8636cdbff Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 14 Oct 2022 16:33:19 +0300 Subject: [PATCH 020/118] Add support for Synapse container image customization This allows people to augment the Synapse image with custom tools and addons without having to rebuild it from scratch. If customizations are enabled, the playbook will build a new `localhost/matrixdotorg/synapse:VERSION-customized` image on top of the default one (`FROM matrixdotorg/synapse:VERSION`) and with custom Dockerfile build steps. For servers that self-build the Synapse image, the Synapse image will be built first, before proceding to extend it the same way. In the future, we'll also have easy to enable Dockerfile build steps for modules that the playbook supports. --- .../tasks/import_synapse_sqlite_db.yml | 2 +- roles/matrix-synapse/defaults/main.yml | 33 +++++++++++++++++++ roles/matrix-synapse/tasks/setup_synapse.yml | 1 + .../tasks/synapse/setup_install.yml | 19 +++++++++++ .../tasks/synapse/setup_uninstall.yml | 5 ++- .../synapse/customizations/Dockerfile.j2 | 3 ++ .../systemd/matrix-synapse-worker.service.j2 | 2 +- .../synapse/systemd/matrix-synapse.service.j2 | 2 +- 8 files changed, 63 insertions(+), 4 deletions(-) create mode 100644 roles/matrix-synapse/templates/synapse/customizations/Dockerfile.j2 diff --git a/roles/matrix-postgres/tasks/import_synapse_sqlite_db.yml b/roles/matrix-postgres/tasks/import_synapse_sqlite_db.yml index a459b6e1d..b885ea975 100644 --- a/roles/matrix-postgres/tasks/import_synapse_sqlite_db.yml +++ b/roles/matrix-postgres/tasks/import_synapse_sqlite_db.yml @@ -83,7 +83,7 @@ --mount type=bind,src={{ matrix_synapse_config_dir_path }},dst=/data --mount type=bind,src={{ matrix_synapse_config_dir_path }},dst=/matrix-media-store-parent/media-store --mount type=bind,src={{ server_path_homeserver_db }},dst=/{{ server_path_homeserver_db | basename }} - {{ matrix_synapse_docker_image }} + {{ matrix_synapse_docker_image_final }} /usr/local/bin/synapse_port_db --sqlite-database /{{ server_path_homeserver_db | basename }} --postgres-config /data/homeserver.yaml register: matrix_postgres_import_synapse_sqlite_db_result changed_when: matrix_postgres_import_synapse_sqlite_db_result.rc == 0 diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index d117eaafa..40e05be72 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -7,14 +7,47 @@ matrix_synapse_enabled: true matrix_synapse_container_image_self_build: false matrix_synapse_container_image_self_build_repo: "https://github.com/matrix-org/synapse.git" +# matrix_synapse_container_image_customizations_enabled controls whether a customized Synapse image will be built. +# +# We toggle this variable to `true` when certain features which require a custom build are enabled. +# Feel free to toggle this to `true` yourself and specify build steps in `matrix_synapse_container_image_customizations_dockerfile_body_custom`. +# +# See: +# - `roles/matrix-synapse/templates/synapse/customizations/Dockerfile.j2` +# - `matrix_synapse_container_image_customizations_dockerfile_body_custom` +# - `matrix_synapse_docker_image_customized` +# - `matrix_synapse_docker_image_final` +matrix_synapse_container_image_customizations_enabled: "{{ matrix_synapse_ext_synapse_s3_storage_provider_enabled }}" + +# matrix_synapse_container_image_customizations_dockerfile_body contains your custom Dockerfile steps +# for building your customized Synapse image based on the original (upstream) image (`matrix_synapse_docker_image`). +# A `FROM ...` clause is included automatically so you don't have to. +# +# Example: +# matrix_synapse_container_image_customizations_dockerfile_body_custom: | +# RUN echo 'This is a custom step for building the customized Docker image for Synapse.' +# RUN echo 'You can override matrix_synapse_container_image_customizations_dockerfile_body_custom to add your own steps.' +# RUN echo 'Note that matrix_synapse_container_image_customizations_dockerfile_body_auto injects steps before this.' +# RUN echo 'You do NOT need to include a FROM clause yourself.' +matrix_synapse_container_image_customizations_dockerfile_body_custom: '' + matrix_synapse_docker_image: "{{ matrix_synapse_docker_image_name_prefix }}matrixdotorg/synapse:{{ matrix_synapse_docker_image_tag }}" matrix_synapse_docker_image_name_prefix: "{{ 'localhost/' if matrix_synapse_container_image_self_build else matrix_container_global_registry_prefix }}" matrix_synapse_version: v1.68.0 matrix_synapse_docker_image_tag: "{{ matrix_synapse_version }}" matrix_synapse_docker_image_force_pull: "{{ matrix_synapse_docker_image.endswith(':latest') }}" +# matrix_synapse_docker_image_customized is the name of the locally built Synapse image +# which adds various customizations on top of the original (upstream) Synapse image. +# This image will be based on the upstream `matrix_synapse_docker_image` image, only if `matrix_synapse_container_image_customizations_enabled: true`. +matrix_synapse_docker_image_customized: "localhost/matrixdotorg/synapse:{{ matrix_synapse_docker_image_tag }}-customized" + +# matrix_synapse_docker_image_final holds the name of the Synapse image to run depending on whether or not customizations are enabled. +matrix_synapse_docker_image_final: "{{ matrix_synapse_docker_image_customized if matrix_synapse_container_image_customizations_enabled else matrix_synapse_docker_image }} " + matrix_synapse_base_path: "{{ matrix_base_data_path }}/synapse" matrix_synapse_docker_src_files_path: "{{ matrix_synapse_base_path }}/docker-src" +matrix_synapse_customized_docker_src_files_path: "{{ matrix_synapse_base_path }}/customized-docker-src" matrix_synapse_config_dir_path: "{{ matrix_synapse_base_path }}/config" matrix_synapse_storage_path: "{{ matrix_synapse_base_path }}/storage" matrix_synapse_media_store_path: "{{ matrix_synapse_storage_path }}/media-store" diff --git a/roles/matrix-synapse/tasks/setup_synapse.yml b/roles/matrix-synapse/tasks/setup_synapse.yml index d4e6ae95a..7b887f30f 100644 --- a/roles/matrix-synapse/tasks/setup_synapse.yml +++ b/roles/matrix-synapse/tasks/setup_synapse.yml @@ -11,6 +11,7 @@ - {path: "{{ matrix_synapse_config_dir_path }}", when: true} - {path: "{{ matrix_synapse_ext_path }}", when: true} - {path: "{{ matrix_synapse_docker_src_files_path }}", when: "{{ matrix_synapse_container_image_self_build }}"} + - {path: "{{ matrix_synapse_customized_docker_src_files_path }}", when: "{{ matrix_synapse_container_image_customizations_enabled }}"} # We handle matrix_synapse_media_store_path elsewhere (in ./synapse/setup_install.yml), # because if it's using Goofys and it's already mounted (from before), # trying to chown/chmod it here will cause trouble. diff --git a/roles/matrix-synapse/tasks/synapse/setup_install.yml b/roles/matrix-synapse/tasks/synapse/setup_install.yml index aea03f059..4d0e749f9 100644 --- a/roles/matrix-synapse/tasks/synapse/setup_install.yml +++ b/roles/matrix-synapse/tasks/synapse/setup_install.yml @@ -62,6 +62,25 @@ delay: "{{ matrix_container_retries_delay }}" until: result is not failed +- when: "matrix_synapse_container_image_customizations_enabled | bool" + block: + - name: Ensure customizations Dockerfile is created + ansible.builtin.template: + src: "{{ role_path }}/templates/synapse/customizations/Dockerfile.j2" + dest: "{{ matrix_synapse_customized_docker_src_files_path }}/Dockerfile" + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_groupname }}" + mode: 0640 + + - name: Ensure customized Docker image for Synapse is built + docker_image: + name: "{{ matrix_synapse_docker_image_customized }}" + source: build + build: + dockerfile: Dockerfile + path: "{{ matrix_synapse_customized_docker_src_files_path }}" + pull: true + - name: Check if a Synapse signing key exists ansible.builtin.stat: path: "{{ matrix_synapse_config_dir_path }}/{{ matrix_server_fqn_matrix }}.signing.key" diff --git a/roles/matrix-synapse/tasks/synapse/setup_uninstall.yml b/roles/matrix-synapse/tasks/synapse/setup_uninstall.yml index 17b1b8c45..06e550145 100644 --- a/roles/matrix-synapse/tasks/synapse/setup_uninstall.yml +++ b/roles/matrix-synapse/tasks/synapse/setup_uninstall.yml @@ -27,8 +27,11 @@ - name: Ensure Synapse Docker image doesn't exist docker_image: - name: "{{ matrix_synapse_docker_image }}" + name: "{{ item }}" state: absent + with_items: + - "{{ matrix_synapse_docker_image_final }}" + - "{{ matrix_synapse_docker_image }}" - name: Ensure sample prometheus.yml for external scraping is deleted ansible.builtin.file: diff --git a/roles/matrix-synapse/templates/synapse/customizations/Dockerfile.j2 b/roles/matrix-synapse/templates/synapse/customizations/Dockerfile.j2 new file mode 100644 index 000000000..7cce2086d --- /dev/null +++ b/roles/matrix-synapse/templates/synapse/customizations/Dockerfile.j2 @@ -0,0 +1,3 @@ +FROM {{ matrix_synapse_docker_image }} + +{{ matrix_synapse_container_image_customizations_dockerfile_body_custom }} diff --git a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker.service.j2 b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker.service.j2 index 96b65a0ae..3855b850c 100644 --- a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker.service.j2 +++ b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker.service.j2 @@ -42,7 +42,7 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name {{ matrix_synapse_wor {% for arg in matrix_synapse_container_arguments %} {{ arg }} \ {% endfor %} - {{ matrix_synapse_docker_image }} \ + {{ matrix_synapse_docker_image_final }} \ run -m synapse.app.{{ matrix_synapse_worker_details.app }} -c /data/homeserver.yaml -c /data/{{ matrix_synapse_worker_config_file_name }} diff --git a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 index eed50ad1e..f41cc54c8 100644 --- a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 +++ b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 @@ -60,7 +60,7 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-synapse \ {% for arg in matrix_synapse_container_arguments %} {{ arg }} \ {% endfor %} - {{ matrix_synapse_docker_image }} \ + {{ matrix_synapse_docker_image_final }} \ run -m synapse.app.homeserver -c /data/homeserver.yaml ExecStop=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} kill matrix-synapse 2>/dev/null || true' From ac7cb3619dfe522af280875566e859007f7a0374 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 14 Oct 2022 17:41:41 +0300 Subject: [PATCH 021/118] Add support for synapse-s3-storage-provider --- docs/configuring-playbook-s3-goofys.md | 137 ++++++++++++ docs/configuring-playbook-s3.md | 195 +++++------------- ...ng-playbook-synapse-s3-storage-provider.md | 104 ++++++++++ roles/matrix-synapse/defaults/main.yml | 35 ++++ .../tasks/ext/s3-storage-provider/init.yml | 5 + .../tasks/ext/s3-storage-provider/setup.yml | 10 + .../ext/s3-storage-provider/setup_install.yml | 54 +++++ .../s3-storage-provider/setup_uninstall.yml | 24 +++ .../s3-storage-provider/validate_config.yml | 18 ++ roles/matrix-synapse/tasks/ext/setup.yml | 2 + roles/matrix-synapse/tasks/init.yml | 3 + roles/matrix-synapse/tasks/setup_synapse.yml | 1 + .../synapse/customizations/Dockerfile.j2 | 4 + .../ext/s3-storage-provider/database.yaml.j2 | 5 + .../synapse/ext/s3-storage-provider/env.j2 | 16 ++ .../media_storage_provider.yaml.j2 | 14 ++ ...pse-s3-storage-provider-migrate.service.j2 | 7 + ...napse-s3-storage-provider-migrate.timer.j2 | 10 + ...rix-synapse-s3-storage-provider-migrate.j2 | 13 ++ ...atrix-synapse-s3-storage-provider-shell.j2 | 13 ++ 20 files changed, 529 insertions(+), 141 deletions(-) create mode 100644 docs/configuring-playbook-s3-goofys.md create mode 100644 docs/configuring-playbook-synapse-s3-storage-provider.md create mode 100644 roles/matrix-synapse/tasks/ext/s3-storage-provider/init.yml create mode 100644 roles/matrix-synapse/tasks/ext/s3-storage-provider/setup.yml create mode 100644 roles/matrix-synapse/tasks/ext/s3-storage-provider/setup_install.yml create mode 100644 roles/matrix-synapse/tasks/ext/s3-storage-provider/setup_uninstall.yml create mode 100644 roles/matrix-synapse/tasks/ext/s3-storage-provider/validate_config.yml create mode 100644 roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/database.yaml.j2 create mode 100644 roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 create mode 100644 roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/media_storage_provider.yaml.j2 create mode 100644 roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/systemd/matrix-synapse-s3-storage-provider-migrate.service.j2 create mode 100644 roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/systemd/matrix-synapse-s3-storage-provider-migrate.timer.j2 create mode 100644 roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/usr-local-bin/matrix-synapse-s3-storage-provider-migrate.j2 create mode 100644 roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/usr-local-bin/matrix-synapse-s3-storage-provider-shell.j2 diff --git a/docs/configuring-playbook-s3-goofys.md b/docs/configuring-playbook-s3-goofys.md new file mode 100644 index 000000000..2efacddcc --- /dev/null +++ b/docs/configuring-playbook-s3-goofys.md @@ -0,0 +1,137 @@ +# Storing Matrix media files on Amazon S3 with Goofys (optional) + +If you'd like to store Synapse's content repository (`media_store`) files on Amazon S3 (or other S3-compatible service), +you can let this playbook configure [Goofys](https://github.com/kahing/goofys) for you. + +Another (and better performing) way to use S3 storage with Synapse is [synapse-s3-storage-provider](configuring-playbook-synapse-s3-storage-provider.md). + +Using a Goofys-backed media store works, but performance may not be ideal. If possible, try to use a region which is close to your Matrix server. + +If you'd like to move your locally-stored media store data to Amazon S3 (or another S3-compatible object store), we also provide some migration instructions below. + + +## Usage + +After [creating the S3 bucket and configuring it](configuring-playbook-s3.md#bucket-creation-and-security-configuration), you can proceed to configure Goofys in your configuration file (`inventory/host_vars/matrix./vars.yml`): + +```yaml +matrix_s3_media_store_enabled: true +matrix_s3_media_store_bucket_name: "your-bucket-name" +matrix_s3_media_store_aws_access_key: "access-key-goes-here" +matrix_s3_media_store_aws_secret_key: "secret-key-goes-here" +matrix_s3_media_store_region: "eu-central-1" +``` + +You can use any S3-compatible object store by **additionally** configuring these variables: + +```yaml +matrix_s3_media_store_custom_endpoint_enabled: true +matrix_s3_media_store_custom_endpoint: "https://your-custom-endpoint" +``` + +If you have local media store files and wish to migrate to Backblaze B2 subsequently, follow our [migration guide to Backblaze B2](#migrating-to-backblaze-b2) below instead of applying this configuration as-is. + + +## Migrating from local filesystem storage to S3 + +It's a good idea to [make a complete server backup](faq.md#how-do-i-backup-the-data-on-my-server) before migrating your local media store to an S3-backed one. + +Follow one of the guides below for a migration path from a locally-stored media store to one stored on S3-compatible storage: + +- [Storing Matrix media files on Amazon S3 with Goofys (optional)](#storing-matrix-media-files-on-amazon-s3-with-goofys-optional) + - [Usage](#usage) + - [Migrating from local filesystem storage to S3](#migrating-from-local-filesystem-storage-to-s3) + - [Migrating to any S3-compatible storage (universal, but likely slow)](#migrating-to-any-s3-compatible-storage-universal-but-likely-slow) + - [Migrating to Backblaze B2](#migrating-to-backblaze-b2) + +### Migrating to any S3-compatible storage (universal, but likely slow) + +It's a good idea to [make a complete server backup](faq.md#how-do-i-backup-the-data-on-my-server) before doing this. + +1. Proceed with the steps below without stopping Matrix services + +2. Start by adding the base S3 configuration in your `vars.yml` file (seen above, may be different depending on the S3 provider of your choice) + +3. In addition to the base configuration you see above, add this to your `vars.yml` file: + +```yaml +matrix_s3_media_store_path: /matrix/s3-media-store +``` + +This enables S3 support, but mounts the S3 storage bucket to `/matrix/s3-media-store` without hooking it to your homeserver yet. Your homeserver will still continue using your local filesystem for its media store. + +5. Run the playbook to apply the changes: `ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start` + +6. Do an **initial sync of your files** by running this **on the server** (it may take a very long time): + +```sh +sudo -u matrix -- rsync --size-only --ignore-existing -avr /matrix/synapse/storage/media-store/. /matrix/s3-media-store/. +``` + +You may need to install `rsync` manually. + +7. Stop all Matrix services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`) + +8. Start the S3 service by running this **on the server**: `systemctl start matrix-goofys` + +9. Sync the files again by re-running the `rsync` command you see in step #6 + +10. Stop the S3 service by running this **on the server**: `systemctl stop matrix-goofys` + +11. Get the old media store out of the way by running this command on the server: + +```sh +mv /matrix/synapse/storage/media-store /matrix/synapse/storage/media-store-local-backup +``` + +12. Remove the `matrix_s3_media_store_path` configuration from your `vars.yml` file (undoing step #3 above) + +13. Run the playbook: `ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start` + +14. You're done! Verify that loading existing (old) media files works and that you can upload new ones. + +15. When confident that it all works, get rid of the local media store directory: `rm -rf /matrix/synapse/storage/media-store-local-backup` + + +### Migrating to Backblaze B2 + +It's a good idea to [make a complete server backup](faq.md#how-do-i-backup-the-data-on-my-server) before doing this. + +1. While all Matrix services are running, run the following command on the server: + +(you need to adjust the 3 `--env` line below with your own data) + +```sh +docker run -it --rm -w /work \ +--env='B2_KEY_ID=YOUR_KEY_GOES_HERE' \ +--env='B2_KEY_SECRET=YOUR_SECRET_GOES_HERE' \ +--env='B2_BUCKET_NAME=YOUR_BUCKET_NAME_GOES_HERE' \ +-v /matrix/synapse/storage/media-store/:/work \ +--entrypoint=/bin/sh \ +docker.io/tianon/backblaze-b2:2.1.0 \ +-c 'b2 authorize-account $B2_KEY_ID $B2_KEY_SECRET > /dev/null && b2 sync /work/ b2://$B2_BUCKET_NAME' +``` + +This is some initial file sync, which may take a very long time. + +2. Stop all Matrix services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`) + +3. Run the command from step #1 again. + +Doing this will sync any new files that may have been created locally in the meantime. + +Now that Matrix services aren't running, we're sure to get Backblaze B2 and your local media store fully in sync. + +4. Get the old media store out of the way by running this command on the server: + +```sh +mv /matrix/synapse/storage/media-store /matrix/synapse/storage/media-store-local-backup +``` + +5. Put the [Backblaze B2 settings seen above](#backblaze-b2) in your `vars.yml` file + +6. Run the playbook: `ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start` + +7. You're done! Verify that loading existing (old) media files works and that you can upload new ones. + +8. When confident that it all works, get rid of the local media store directory: `rm -rf /matrix/synapse/storage/media-store-local-backup` diff --git a/docs/configuring-playbook-s3.md b/docs/configuring-playbook-s3.md index 43aaa8792..539f96d32 100644 --- a/docs/configuring-playbook-s3.md +++ b/docs/configuring-playbook-s3.md @@ -1,15 +1,44 @@ -# Storing Matrix media files on Amazon S3 (optional) +# Storing Synapse media files on Amazon S3 or another compatible Object Storage (optional) By default, this playbook configures your server to store Synapse's content repository (`media_store`) files on the local filesystem. If that's alright, you can skip this. -If you'd like to store Synapse's content repository (`media_store`) files on Amazon S3 (or other S3-compatible service), -you can let this playbook configure [Goofys](https://github.com/kahing/goofys) for you. +As an alternative to storing media files on the local filesystem, you can store them on [Amazon S3](https://aws.amazon.com/s3/) or another S3-compatible object store. -Using a Goofys-backed media store works, but performance may not be ideal. If possible, try to use a region which is close to your Matrix server. +First, [choose an Object Storage provider](#choosing-an-object-storage-provider). -If you'd like to move your locally-stored media store data to Amazon S3 (or another S3-compatible object store), we also provide some migration instructions below. +Then, [create the S3 bucket](#bucket-creation-and-security-configuration). +Finally, [set up S3 storage for Synapse](#setting-up) (with [Goofys](configuring-playbook-s3-goofys.md) or [synapse-s3-storage-provider](configuring-playbook-synapse-s3-storage-provider.md)). + + +## Choosing an Object Storage provider + +You can create [Amazon S3](https://aws.amazon.com/s3/) or another S3-compatible object store like [Backblaze B2](https://www.backblaze.com/b2/cloud-storage.html), [Wasabi](https://wasabi.com), [Digital Ocean Spaces](https://www.digitalocean.com/products/spaces), etc. + +Amazon S3 and Backblaze S3 are pay-as-you with no minimum charges for storing too little data. + +All these providers have different prices, with Backblaze B2 appearing to be the cheapest. + +Wasabi has a minimum charge of 1TB if you're storing less than 1TB, which becomes expensive if you need to store less data than that. + +Digital Ocean Spaces has a minimum charge of 250GB ($5/month as of 2022-10), which is also expensive if you're storing less data than that. + +Important aspects of choosing the right provider are: + +- a provider by a company you like and trust (or dislike less than the others) +- a provider which has a data region close to your Matrix server (if it's farther away, high latency may cause slowdowns) +- a provider which is OK pricewise +- a provider with free or cheap egress (if you need to get the data out often, for some reason) - likely not too important for the common use-case + + +## Bucket creation and Security Configuration + +Now that you've [chosen an Object Storage provider](#choosing-an-object-storage-provider), you need to create a storage bucket. + +How you do this varies from provider to provider, with Amazon S3 being the most complicated due to its vast number of services and complicated security policies. + +Below, we provider some guides for common providers. If you don't see yours, look at the others for inspiration or read some guides online about how to create a bucket. Feel free to contribute to this documentation with an update! ## Amazon S3 @@ -34,161 +63,45 @@ You'll need an Amazon S3 bucket and some IAM user credentials (access key + secr } ``` -**NOTE**: This policy needs to be attached to an IAM user creted from the **Security Credentials** menu. This is not a **Bucket Policy**. - -You then need to enable S3 support in your configuration file (`inventory/host_vars/matrix./vars.yml`). -It would be something like this: - -```yaml -matrix_s3_media_store_enabled: true -matrix_s3_media_store_bucket_name: "your-bucket-name" -matrix_s3_media_store_aws_access_key: "access-key-goes-here" -matrix_s3_media_store_aws_secret_key: "secret-key-goes-here" -matrix_s3_media_store_region: "eu-central-1" -``` +**NOTE**: This policy needs to be attached to an IAM user created from the **Security Credentials** menu. This is not a **Bucket Policy**. -## Using other S3-compatible object stores +## Backblaze B2 -You can use any S3-compatible object store by **additionally** configuring these variables: +To use [Backblaze B2](https://www.backblaze.com/b2/cloud-storage.html) you first need to sign up. -```yaml -matrix_s3_media_store_custom_endpoint_enabled: true -# Example: "https://storage.googleapis.com" -matrix_s3_media_store_custom_endpoint: "your-custom-endpoint" -``` +You [can't easily change which region (US, Europe) your Backblaze account stores files in](https://old.reddit.com/r/backblaze/comments/hi1v90/make_the_choice_for_the_b2_data_center_region/), so make sure to carefully choose the region when signing up (hint: it's a hard to see dropdown below the username/password fields in the signup form). -### Backblaze B2 - -To use [Backblaze B2](https://www.backblaze.com/b2/cloud-storage.html): +After logging in to Backblaze: - create a new **private** bucket through its user interface (you can call it something like `matrix-DOMAIN-media-store`) -- note the **Endpoint** for your bucket (something like `s3.us-west-002.backblazeb2.com`) -- adjust its lifecycle rules to use the following **custom** rules: - - File Path: *empty value* - - Days Till Hide: *empty value* - - Days Till Delete: `1` +- note the **Endpoint** for your bucket (something like `s3.us-west-002.backblazeb2.com`). +- adjust its Lifecycle Rules to: Keep only the last version of the file - go to [App Keys](https://secure.backblaze.com/app_keys.htm) and use the **Add a New Application Key** to create a new one - restrict it to the previously created bucket (e.g. `matrix-DOMAIN-media-store`) - give it *Read & Write* access -Copy the `keyID` and `applicationKey`. +The `keyID` value is your **Access Key** and `applicationKey` is your **Secret Key**. -You need the following *additional* playbook configuration (on top of what you see above): +For configuring [Goofys](configuring-playbook-s3-goofys.md) or [s3-synapse-storage-provider](configuring-playbook-synapse-s3-storage-provider.md) you will need: -```yaml -matrix_s3_media_store_bucket_name: "YOUR_BUCKET_NAME_GOES_HERE" -matrix_s3_media_store_aws_access_key: "YOUR_keyID_GOES_HERE" -matrix_s3_media_store_aws_secret_key: "YOUR_applicationKey_GOES_HERE" -matrix_s3_media_store_custom_endpoint_enabled: true -matrix_s3_media_store_custom_endpoint: "https://s3.us-west-002.backblazeb2.com" # this may be different for your bucket -``` +- **Endpoint URL** - this is the **Endpoint** value you saw above, but prefixed with `https://` -If you have local media store files and wish to migrate to Backblaze B2 subsequently, follow our [migration guide to Backblaze B2](#migrating-to-backblaze-b2) below instead of applying this configuration as-is. +- **Region** - use the value you see in the Endpoint (e.g. `us-west-002`) + +- **Storage Class** - use `STANDARD`. Backblaze B2 does not have different storage classes, so it doesn't make sense to use any other value. -## Migrating from local filesystem storage to S3 +## Other providers -It's a good idea to [make a complete server backup](faq.md#how-do-i-backup-the-data-on-my-server) before migrating your local media store to an S3-backed one. +For other S3-compatible providers, you may not need to configure security policies, etc. (just like for [Backblaze B2](#backblaze-b2)). -Follow one of the guides below for a migration path from a locally-stored media store to one stored on S3-compatible storage: - -- [Storing Matrix media files on Amazon S3 (optional)](#storing-matrix-media-files-on-amazon-s3-optional) - - [Amazon S3](#amazon-s3) - - [Using other S3-compatible object stores](#using-other-s3-compatible-object-stores) - - [Backblaze B2](#backblaze-b2) - - [Migrating from local filesystem storage to S3](#migrating-from-local-filesystem-storage-to-s3) - - [Migrating to any S3-compatible storage (universal, but likely slow)](#migrating-to-any-s3-compatible-storage-universal-but-likely-slow) - - [Migrating to Backblaze B2](#migrating-to-backblaze-b2) - -### Migrating to any S3-compatible storage (universal, but likely slow) - -It's a good idea to [make a complete server backup](faq.md#how-do-i-backup-the-data-on-my-server) before doing this. - -1. Proceed with the steps below without stopping Matrix services - -2. Start by adding the base S3 configuration in your `vars.yml` file (seen above, may be different depending on the S3 provider of your choice) - -3. In addition to the base configuration you see above, add this to your `vars.yml` file: - -```yaml -matrix_s3_media_store_path: /matrix/s3-media-store -``` - -This enables S3 support, but mounts the S3 storage bucket to `/matrix/s3-media-store` without hooking it to your homeserver yet. Your homeserver will still continue using your local filesystem for its media store. - -5. Run the playbook to apply the changes: `ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start` - -6. Do an **initial sync of your files** by running this **on the server** (it may take a very long time): - -```sh -sudo -u matrix -- rsync --size-only --ignore-existing -avr /matrix/synapse/storage/media-store/. /matrix/s3-media-store/. -``` - -You may need to install `rsync` manually. - -7. Stop all Matrix services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`) - -8. Start the S3 service by running this **on the server**: `systemctl start matrix-goofys` - -9. Sync the files again by re-running the `rsync` command you see in step #6 - -10. Stop the S3 service by running this **on the server**: `systemctl stop matrix-goofys` - -11. Get the old media store out of the way by running this command on the server: - -```sh -mv /matrix/synapse/storage/media-store /matrix/synapse/storage/media-store-local-backup -``` - -12. Remove the `matrix_s3_media_store_path` configuration from your `vars.yml` file (undoing step #3 above) - -13. Run the playbook: `ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start` - -14. You're done! Verify that loading existing (old) media files works and that you can upload new ones. - -15. When confident that it all works, get rid of the local media store directory: `rm -rf /matrix/synapse/storage/media-store-local-backup` +You most likely just need to create an S3 bucket and get some credentials (access key and secret key) for accessing the bucket in a read/write manner. -### Migrating to Backblaze B2 +## Setting up -It's a good idea to [make a complete server backup](faq.md#how-do-i-backup-the-data-on-my-server) before doing this. +To set up Synapse to store files in S3, follow the instructions for the method of your choice: -1. While all Matrix services are running, run the following command on the server: - -(you need to adjust the 3 `--env` line below with your own data) - -```sh -docker run -it --rm -w /work \ ---env='B2_KEY_ID=YOUR_KEY_GOES_HERE' \ ---env='B2_KEY_SECRET=YOUR_SECRET_GOES_HERE' \ ---env='B2_BUCKET_NAME=YOUR_BUCKET_NAME_GOES_HERE' \ --v /matrix/synapse/storage/media-store/:/work \ ---entrypoint=/bin/sh \ -docker.io/tianon/backblaze-b2:2.1.0 \ --c 'b2 authorize-account $B2_KEY_ID $B2_KEY_SECRET > /dev/null && b2 sync /work/ b2://$B2_BUCKET_NAME' -``` - -This is some initial file sync, which may take a very long time. - -2. Stop all Matrix services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`) - -3. Run the command from step #1 again. - -Doing this will sync any new files that may have been created locally in the meantime. - -Now that Matrix services aren't running, we're sure to get Backblaze B2 and your local media store fully in sync. - -4. Get the old media store out of the way by running this command on the server: - -```sh -mv /matrix/synapse/storage/media-store /matrix/synapse/storage/media-store-local-backup -``` - -5. Put the [Backblaze B2 settings seen above](#backblaze-b2) in your `vars.yml` file - -6. Run the playbook: `ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start` - -7. You're done! Verify that loading existing (old) media files works and that you can upload new ones. - -8. When confident that it all works, get rid of the local media store directory: `rm -rf /matrix/synapse/storage/media-store-local-backup` +- using [synapse-s3-storage-provider](configuring-playbook-synapse-s3-storage-provider.md) (recommended) +- using [Goofys to mount the S3 store to the local filesystem](configuring-playbook-s3-goofys.md) diff --git a/docs/configuring-playbook-synapse-s3-storage-provider.md b/docs/configuring-playbook-synapse-s3-storage-provider.md new file mode 100644 index 000000000..bc0250f11 --- /dev/null +++ b/docs/configuring-playbook-synapse-s3-storage-provider.md @@ -0,0 +1,104 @@ +# Storing Synapse media files on Amazon S3 with synapse-s3-storage-provider (optional) + +If you'd like to store Synapse's content repository (`media_store`) files on Amazon S3 (or other S3-compatible service), +you can use the [synapse-s3-storage-provider](https://github.com/matrix-org/synapse-s3-storage-provider) media provider module for Synapse. + +An alternative (which has worse performance) is to use [Goofys to mount the S3 store to the local filesystem](configuring-playbook-s3-goofys.md). + + +## How it works? + +Summarized writings here are inspired by [this article](https://quentin.dufour.io/blog/2021-09-14/matrix-synapse-s3-storage/). + +The way media storage providers in Synapse work has some caveats: + +- Synapse still continues to use locally-stored files (for creating thumbnails, serving files, etc) +- the media storage provider is just an extra storage mechanism (in addition to the local filesystem) +- all files are stored locally at first, and then copied to the media storage provider (either synchronously or asynchronously) +- if a file is not available on the local filesystem, it's pulled from a media storage provider + +You may be thinking **if all files are stored locally as well, what's the point**? + +You can run some scripts to delete the local files once in a while, thus freeing up local disk space. If these files are needed in the future (for serving them to users, etc.), Synapse will pull them from the media storage provider on demand. + +While you will need some local disk space around, it's only to accommodate usage, etc., and won't grow as large as your S3 store. + + +## Installing + +After [creating the S3 bucket and configuring it](configuring-playbook-s3.md#bucket-creation-and-security-configuration), you can proceed to configure Goofys in your configuration file (`inventory/host_vars/matrix./vars.yml`): + +```yaml +matrix_synapse_ext_synapse_s3_storage_provider_enabled: true +matrix_synapse_ext_synapse_s3_storage_provider_config_bucket: your-bucket-name +matrix_synapse_ext_synapse_s3_storage_provider_config_region_name: some-region-name # e.g. eu-central-1 +matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url: https://.. # delete this whole line for Amazon S3 +matrix_synapse_ext_synapse_s3_storage_provider_config_access_key_id: access-key-goes-here +matrix_synapse_ext_synapse_s3_storage_provider_config_secret_access_key: secret-key-goes-here +matrix_synapse_ext_synapse_s3_storage_provider_config_storage_class: STANDARD # or STANDARD_IA, etc. + +# For additional advanced settings, take a look at `roles/matrix-synapse/defaults/main.yml` +``` + +If you have existing files in Synapse's media repository (`/matrix/synapse/media-store/..`): + +- new files will start being stored both locally and on the S3 store +- the existing files will remain on the local filesystem only until [migrating them to the S3 store](#migrating-your-existing-media-files-to-the-s3-store) +- at some point (and periodically in the future), you can delete local files which have been uploaded to the S3 store already + + +## Migrating your existing media files to the S3 store + +Migrating your existing data can happen in multiple ways: + +- [using the `s3_media_upload` script from `synapse-s3-storage-provider`](#using-the-s3_media_upload-script-from-synapse-s3-storage-provider) (very slow when dealing with lots of data) +- [using another tool in combination with `s3_media_upload`](#using-another-tool-in-combination-with-s3_media_upload) (quicker when dealing with lots of data) + +### Using the `s3_media_upload` script from `synapse-s3-storage-provider` + +Instead of using `s3_media_upload` directly, which is very slow and painful for an initial data migration, we recommend [using another tool in combination with `s3_media_upload`](#using-another-tool-in-combination-with-s3_media_upload). + +To copy your existing files, SSH into the server and run `/usr/local/bin/matrix-synapse-s3-storage-provider-shell`. + +This launches a Synapse container, which has access to the local media store, Postgres database, S3 store and has some convenient environment variables configured for you to use (`MEDIA_PATH`, `BUCKET`, `ENDPOINT`, `UPDATE_DB_DAYS`, etc). + +Then use the following commands (`$` values come from environment variables - they're **not placeholders** that you need to substitute): + +- `s3_media_upload update-db $UPDATE_DB_DURATION` - create a local SQLite database (`cache.db`) with a list of media repository files (from the `synapse` Postgres database) eligible for operating on + - `$UPDATE_DB_DURATION` is influenced by the `matrix_synapse_ext_synapse_s3_storage_provider_update_db_day_count` variable (defaults to `0`) + - `$UPDATE_DB_DURATION` defaults to `0d` (0 days), which means **include files which haven't been accessed for more than 0 days** (that is, **all files will be included**). +- `s3_media_upload check-deleted $MEDIA_PATH` - check whether files in the local cache still exist in the local media repository directory +- `s3_media_upload upload $MEDIA_PATH $BUCKET --delete --endpoint-url $ENDPOINT` - uploads locally-stored files to S3 and deletes them from the local media repository directory + +The `upload` command may take a lot of time to complete. + + +### Using another tool in combination with `s3_media_upload` + +To migrate your existing local data to S3, we recommend to: + +- **first** use another tool ([`aws s3`](#copying-data-to-amazon-s3) or [`b2 sync`](#copying-data-to-backblaze-b2), etc.) to copy the local files to the S3 bucket + +- **only then** [use the `s3_media_upload` tool to finish the migration](#using-the-s3_media_upload-script-from-synapse-s3-storage-provider) (this checks to ensure all files are uploaded and then deletes the local files) + +#### Copying data to Amazon S3 + +Generally, you need to use the `aws s3` tool. + +This documentation section could use an improvement. Ideally, we'd come up with a guide like the one used in [Copying data to Backblaze B2](#copying-data-to-backblaze-b2) - running `aws s3` in a container, etc. + +#### Copying data to Backblaze B2 + +To copy to Backblaze B2, start a container like this: + +```sh +docker run -it --rm \ +-w /work \ +--env='B2_KEY_ID=YOUR_KEY_GOES_HERE' \ +--env='B2_KEY_SECRET=YOUR_SECRET_GOES_HERE' \ +--env='B2_BUCKET_NAME=YOUR_BUCKET_NAME_GOES_HERE' \ +--mount type=bind,src=/matrix/synapse/storage/media-store,dst=/work,ro \ +--entrypoint=/bin/sh \ +tianon/backblaze-b2:3.6.0 \ +-c 'b2 authorize-account $B2_KEY_ID $B2_KEY_SECRET > /dev/null && b2 sync /work b2://$B2_BUCKET_NAME --skipNewer' +``` diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index 40e05be72..383e67ab8 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -19,6 +19,10 @@ matrix_synapse_container_image_self_build_repo: "https://github.com/matrix-org/s # - `matrix_synapse_docker_image_final` matrix_synapse_container_image_customizations_enabled: "{{ matrix_synapse_ext_synapse_s3_storage_provider_enabled }}" +# Controls whether custom build steps will be added to the Dockerfile for installing s3-storage-provider. +# The version that will be installed is specified in `matrix_synapse_ext_synapse_s3_storage_provider_version`. +matrix_synapse_container_image_customizations_s3_storage_provider_installation_enabled: "{{ matrix_synapse_ext_synapse_s3_storage_provider_enabled }}" + # matrix_synapse_container_image_customizations_dockerfile_body contains your custom Dockerfile steps # for building your customized Synapse image based on the original (upstream) image (`matrix_synapse_docker_image`). # A `FROM ...` clause is included automatically so you don't have to. @@ -52,6 +56,7 @@ matrix_synapse_config_dir_path: "{{ matrix_synapse_base_path }}/config" matrix_synapse_storage_path: "{{ matrix_synapse_base_path }}/storage" matrix_synapse_media_store_path: "{{ matrix_synapse_storage_path }}/media-store" matrix_synapse_ext_path: "{{ matrix_synapse_base_path }}/ext" +matrix_synapse_ext_s3_storage_provider_path: "{{ matrix_synapse_base_path }}/ext/s3-storage-provider" matrix_synapse_container_client_api_port: 8008 @@ -787,6 +792,32 @@ matrix_synapse_ext_encryption_config_yaml: | patch_power_levels: {{ matrix_synapse_ext_encryption_disabler_patch_power_levels | to_json }} +# matrix_synapse_ext_synapse_s3_storage_provider_enabled controls whether to enable https://github.com/matrix-org/synapse-s3-storage-provider +# Installing it requires building a customized Docker image for Synapse (see `matrix_synapse_container_image_customizations_enabled`). +# Enabling this will enable customizations and inject the appropriate Dockerfile clauses for installing synapse-s3-storage-provider. +matrix_synapse_ext_synapse_s3_storage_provider_enabled: false +matrix_synapse_ext_synapse_s3_storage_provider_version: 1.1.2 +# Controls whether media from this (local) server is stored in s3-storage-provider +matrix_synapse_ext_synapse_s3_storage_provider_store_local: true +# Controls whether media from remote servers is stored in s3-storage-provider +matrix_synapse_ext_synapse_s3_storage_provider_store_remote: true +# Controls whether files are stored to S3 at the same time they are stored on the local filesystem. +# For slightly improved reliability, consider setting this to `true`. +# Even with asynchronous uploading to S3 (`false` value), data loss shouldn't be possible, +# because the local filesystem is a reliable data store anyway. +matrix_synapse_ext_synapse_s3_storage_provider_store_synchronous: false +matrix_synapse_ext_synapse_s3_storage_provider_config_bucket: '' +matrix_synapse_ext_synapse_s3_storage_provider_config_region_name: '' +matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url: '' +matrix_synapse_ext_synapse_s3_storage_provider_config_access_key_id: '' +matrix_synapse_ext_synapse_s3_storage_provider_config_secret_access_key: '' +matrix_synapse_ext_synapse_s3_storage_provider_config_storage_class: STANDARD +matrix_synapse_ext_synapse_s3_storage_provider_config_threadpool_size: 40 +# matrix_synapse_ext_synapse_s3_storage_provider_update_db_day_count is a day value (number) for the `s3_media_upload update-db` command. +# It specifies how old files need to have been inactive to be eligible for migration from the local filesystem to the S3 data store. +# By default, we use `0` which says "all files are eligible for migration". +matrix_synapse_ext_synapse_s3_storage_provider_update_db_day_count: 0 + matrix_s3_media_store_enabled: false matrix_s3_media_store_custom_endpoint_enabled: false matrix_s3_goofys_docker_image: "ewoutp/goofys:latest" @@ -839,6 +870,10 @@ matrix_synapse_media_storage_providers: "{{ matrix_synapse_media_storage_provide matrix_synapse_media_storage_providers_auto: | {{ [] + + + [ + lookup('ansible.builtin.template', role_path + '/templates/synapse/ext/s3-storage-provider/media_storage_provider.yaml.j2') | from_yaml + ] if matrix_synapse_ext_synapse_s3_storage_provider_enabled else [] }} # matrix_synapse_media_storage_providers_custom contains your own custom list of storage providers. diff --git a/roles/matrix-synapse/tasks/ext/s3-storage-provider/init.yml b/roles/matrix-synapse/tasks/ext/s3-storage-provider/init.yml new file mode 100644 index 000000000..008161cb1 --- /dev/null +++ b/roles/matrix-synapse/tasks/ext/s3-storage-provider/init.yml @@ -0,0 +1,5 @@ +--- + +- ansible.builtin.set_fact: + matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-synapse-s3-storage-provider-migrate.timer'] }}" + when: matrix_synapse_ext_synapse_s3_storage_provider_enabled | bool diff --git a/roles/matrix-synapse/tasks/ext/s3-storage-provider/setup.yml b/roles/matrix-synapse/tasks/ext/s3-storage-provider/setup.yml new file mode 100644 index 000000000..aefa49fe4 --- /dev/null +++ b/roles/matrix-synapse/tasks/ext/s3-storage-provider/setup.yml @@ -0,0 +1,10 @@ +--- + +- ansible.builtin.import_tasks: "{{ role_path }}/tasks/ext/s3-storage-provider/validate_config.yml" + when: matrix_synapse_ext_synapse_s3_storage_provider_enabled | bool + +- ansible.builtin.import_tasks: "{{ role_path }}/tasks/ext/s3-storage-provider/setup_install.yml" + when: matrix_synapse_ext_synapse_s3_storage_provider_enabled | bool + +- ansible.builtin.import_tasks: "{{ role_path }}/tasks/ext/s3-storage-provider/setup_uninstall.yml" + when: not matrix_synapse_ext_synapse_s3_storage_provider_enabled | bool diff --git a/roles/matrix-synapse/tasks/ext/s3-storage-provider/setup_install.yml b/roles/matrix-synapse/tasks/ext/s3-storage-provider/setup_install.yml new file mode 100644 index 000000000..31f721819 --- /dev/null +++ b/roles/matrix-synapse/tasks/ext/s3-storage-provider/setup_install.yml @@ -0,0 +1,54 @@ +--- + +# We install this into Synapse by making `matrix_synapse_ext_synapse_s3_storage_provider_enabled` influence other variables: +# - `matrix_synapse_media_storage_providers` (via `matrix_synapse_media_storage_providers_auto`) +# - `matrix_synapse_container_image_customizations_enabled` +# - `matrix_synapse_container_image_customizations_s3_storage_provider_installation_enabled` +# +# Below are additional tasks for setting up various helper scripts, etc. + +- name: Ensure s3-storage-provider env file installed + ansible.builtin.template: + src: "{{ role_path }}/templates/synapse/ext/s3-storage-provider/env.j2" + dest: "{{ matrix_synapse_ext_s3_storage_provider_path }}/env" + mode: 0640 + +- name: Ensure s3-storage-provider data path exists + ansible.builtin.file: + path: "{{ matrix_synapse_ext_s3_storage_provider_path }}/data" + state: directory + mode: 0750 + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_groupname }}" + +- name: Ensure s3-storage-provider database.yaml file installed + ansible.builtin.template: + src: "{{ role_path }}/templates/synapse/ext/s3-storage-provider/database.yaml.j2" + dest: "{{ matrix_synapse_ext_s3_storage_provider_path }}/data/database.yaml" + mode: 0640 + +- name: Ensure s3-storage-provider scripts installed + ansible.builtin.template: + src: "{{ role_path }}/templates/synapse/ext/s3-storage-provider/usr-local-bin/{{ item }}.j2" + dest: "{{ matrix_local_bin_path }}/{{ item }}" + mode: 0750 + with_items: + - matrix-synapse-s3-storage-provider-shell + - matrix-synapse-s3-storage-provider-migrate + +- name: Ensure matrix-synapse-s3-storage-provider-migrate.service and timer are installed + ansible.builtin.template: + src: "{{ role_path }}/templates/systemd/.j2" + src: "{{ role_path }}/templates/synapse/ext/s3-storage-provider/systemd/{{ item }}.j2" + dest: "{{ matrix_systemd_path }}/{{ item }}" + mode: 0640 + with_items: + - matrix-synapse-s3-storage-provider-migrate.service + - matrix-synapse-s3-storage-provider-migrate.timer + register: matrix_synapse_s3_storage_provider_systemd_service_result + +- name: Ensure systemd reloaded after matrix-synapse-s3-storage-provider-migrate.service installation + ansible.builtin.service: + daemon_reload: true + when: matrix_synapse_s3_storage_provider_systemd_service_result.changed | bool + diff --git a/roles/matrix-synapse/tasks/ext/s3-storage-provider/setup_uninstall.yml b/roles/matrix-synapse/tasks/ext/s3-storage-provider/setup_uninstall.yml new file mode 100644 index 000000000..205a55417 --- /dev/null +++ b/roles/matrix-synapse/tasks/ext/s3-storage-provider/setup_uninstall.yml @@ -0,0 +1,24 @@ +--- + +- name: Ensure matrix-synapse-s3-storage-provider-migrate.service and timer don't exist + ansible.builtin.file: + path: "{{ matrix_systemd_path }}/{{ item }}" + state: absent + with_items: + - matrix-synapse-s3-storage-provider-migrate.timer + - matrix-synapse-s3-storage-provider-migrate.service + register: matrix_synapse_s3_storage_provider_migrate_sevice_removal + +- name: Ensure systemd reloaded after matrix-synapse-s3-storage-provider-migrate.service removal + ansible.builtin.service: + daemon_reload: true + when: matrix_synapse_s3_storage_provider_migrate_sevice_removal.changed | bool + +- name: Ensure s3-storage-provider files don't exist + ansible.builtin.file: + path: "{{ item }}" + state: absent + with_items: + - "{{ matrix_local_bin_path }}/matrix-synapse-s3-storage-provider-shell" + - "{{ matrix_local_bin_path }}/matrix-synapse-s3-storage-provider-migrate" + - "{{ matrix_synapse_ext_s3_storage_provider_path }}" diff --git a/roles/matrix-synapse/tasks/ext/s3-storage-provider/validate_config.yml b/roles/matrix-synapse/tasks/ext/s3-storage-provider/validate_config.yml new file mode 100644 index 000000000..d71809fe5 --- /dev/null +++ b/roles/matrix-synapse/tasks/ext/s3-storage-provider/validate_config.yml @@ -0,0 +1,18 @@ +--- + +- name: Fail if required s3-storage-provider settings not defined + ansible.builtin.fail: + msg: >- + You need to define a required configuration setting (`{{ item }}`) for using s3-storage-provider. + when: "vars[item] == ''" + with_items: + - "matrix_synapse_ext_synapse_s3_storage_provider_config_bucket" + - "matrix_synapse_ext_synapse_s3_storage_provider_config_region_name" + - "matrix_synapse_ext_synapse_s3_storage_provider_config_access_key_id" + - "matrix_synapse_ext_synapse_s3_storage_provider_config_secret_access_key" + +- name: Fail if required matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url looks invalid + ansible.builtin.fail: + msg: >- + `matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url` needs to look like a URL (`http://` or `https://` prefix). + when: "matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url != '' and not matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url.startswith('http')" diff --git a/roles/matrix-synapse/tasks/ext/setup.yml b/roles/matrix-synapse/tasks/ext/setup.yml index d944f2574..6cf1afaa4 100644 --- a/roles/matrix-synapse/tasks/ext/setup.yml +++ b/roles/matrix-synapse/tasks/ext/setup.yml @@ -11,3 +11,5 @@ - ansible.builtin.import_tasks: "{{ role_path }}/tasks/ext/synapse-simple-antispam/setup.yml" - ansible.builtin.import_tasks: "{{ role_path }}/tasks/ext/mjolnir-antispam/setup.yml" + +- ansible.builtin.import_tasks: "{{ role_path }}/tasks/ext/s3-storage-provider/setup.yml" diff --git a/roles/matrix-synapse/tasks/init.yml b/roles/matrix-synapse/tasks/init.yml index a77320c22..9146936a0 100644 --- a/roles/matrix-synapse/tasks/init.yml +++ b/roles/matrix-synapse/tasks/init.yml @@ -26,6 +26,9 @@ matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-goofys.service'] }}" when: matrix_s3_media_store_enabled | bool +- ansible.builtin.include_tasks: "{{ role_path }}/tasks/ext/s3-storage-provider/init.yml" + when: matrix_synapse_ext_synapse_s3_storage_provider_enabled | bool + - when: matrix_synapse_enabled | bool and matrix_synapse_metrics_proxying_enabled | bool block: - name: Fail if matrix-nginx-proxy role already executed diff --git a/roles/matrix-synapse/tasks/setup_synapse.yml b/roles/matrix-synapse/tasks/setup_synapse.yml index 7b887f30f..13a5819e1 100644 --- a/roles/matrix-synapse/tasks/setup_synapse.yml +++ b/roles/matrix-synapse/tasks/setup_synapse.yml @@ -12,6 +12,7 @@ - {path: "{{ matrix_synapse_ext_path }}", when: true} - {path: "{{ matrix_synapse_docker_src_files_path }}", when: "{{ matrix_synapse_container_image_self_build }}"} - {path: "{{ matrix_synapse_customized_docker_src_files_path }}", when: "{{ matrix_synapse_container_image_customizations_enabled }}"} + - {path: "{{ matrix_synapse_ext_s3_storage_provider_path }}", when: "{{ matrix_synapse_ext_synapse_s3_storage_provider_enabled }}"} # We handle matrix_synapse_media_store_path elsewhere (in ./synapse/setup_install.yml), # because if it's using Goofys and it's already mounted (from before), # trying to chown/chmod it here will cause trouble. diff --git a/roles/matrix-synapse/templates/synapse/customizations/Dockerfile.j2 b/roles/matrix-synapse/templates/synapse/customizations/Dockerfile.j2 index 7cce2086d..3919e9557 100644 --- a/roles/matrix-synapse/templates/synapse/customizations/Dockerfile.j2 +++ b/roles/matrix-synapse/templates/synapse/customizations/Dockerfile.j2 @@ -1,3 +1,7 @@ FROM {{ matrix_synapse_docker_image }} +{% if matrix_synapse_container_image_customizations_s3_storage_provider_installation_enabled %} +RUN pip install synapse-s3-storage-provider=={{ matrix_synapse_ext_synapse_s3_storage_provider_version }} +{% endif %} + {{ matrix_synapse_container_image_customizations_dockerfile_body_custom }} diff --git a/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/database.yaml.j2 b/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/database.yaml.j2 new file mode 100644 index 000000000..ed11645eb --- /dev/null +++ b/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/database.yaml.j2 @@ -0,0 +1,5 @@ +user: {{ matrix_synapse_database_user | to_json }} +password: {{ matrix_synapse_database_password | to_json }} +database: {{ matrix_synapse_database_database | to_json }} +host: {{ matrix_synapse_database_host | to_json }} +port: {{ matrix_synapse_database_port | to_json }} diff --git a/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 b/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 new file mode 100644 index 000000000..4b09688ba --- /dev/null +++ b/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 @@ -0,0 +1,16 @@ +AWS_ACCESS_KEY_ID={{ matrix_synapse_ext_synapse_s3_storage_provider_config_access_key_id }} +AWS_SECRET_ACCESS_KEY={{ matrix_synapse_ext_synapse_s3_storage_provider_config_secret_access_key }} +AWS_DEFAULT_REGION={{ matrix_synapse_ext_synapse_s3_storage_provider_config_region_name }} + +ENDPOINT={{ matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url }} +BUCKET={{ matrix_synapse_ext_synapse_s3_storage_provider_config_bucket }} + +PG_USER={{ matrix_synapse_database_user }} +PG_PASS={{ matrix_synapse_database_password }} +PG_DB={{ matrix_synapse_database_database }} +PG_HOST={{ matrix_synapse_database_host }} +PG_PORT={{ matrix_synapse_database_port }} + +MEDIA_PATH=/matrix-media-store-parent/{{ matrix_synapse_media_store_directory_name }} + +UPDATE_DB_DURATION={{ matrix_synapse_ext_synapse_s3_storage_provider_update_db_day_count }}d diff --git a/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/media_storage_provider.yaml.j2 b/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/media_storage_provider.yaml.j2 new file mode 100644 index 000000000..97b0f5f2b --- /dev/null +++ b/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/media_storage_provider.yaml.j2 @@ -0,0 +1,14 @@ +module: s3_storage_provider.S3StorageProviderBackend +store_local: {{ matrix_synapse_ext_synapse_s3_storage_provider_store_local | to_json }} +store_remote: {{ matrix_synapse_ext_synapse_s3_storage_provider_store_remote | to_json }} +store_synchronous: {{ matrix_synapse_ext_synapse_s3_storage_provider_store_synchronous | to_json }} +config: + bucket: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_bucket | to_json }} + region_name: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_region_name | to_json }} + endpoint_url: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url | to_json }} + access_key_id: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_access_key_id | to_json }} + secret_access_key: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_secret_access_key | to_json }} + + storage_class: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_storage_class | to_json }} + + threadpool_size: {{ matrix_synapse_ext_synapse_s3_storage_provider_config_threadpool_size | to_json }} diff --git a/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/systemd/matrix-synapse-s3-storage-provider-migrate.service.j2 b/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/systemd/matrix-synapse-s3-storage-provider-migrate.service.j2 new file mode 100644 index 000000000..ea8f0c8cb --- /dev/null +++ b/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/systemd/matrix-synapse-s3-storage-provider-migrate.service.j2 @@ -0,0 +1,7 @@ +[Unit] +Description=Migrates locally-stored Synapse media store files to S3 + +[Service] +Type=oneshot +Environment="HOME={{ matrix_systemd_unit_home_path }}" +ExecStart={{ matrix_local_bin_path }}/matrix-synapse-s3-storage-provider-migrate diff --git a/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/systemd/matrix-synapse-s3-storage-provider-migrate.timer.j2 b/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/systemd/matrix-synapse-s3-storage-provider-migrate.timer.j2 new file mode 100644 index 000000000..61526ac12 --- /dev/null +++ b/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/systemd/matrix-synapse-s3-storage-provider-migrate.timer.j2 @@ -0,0 +1,10 @@ +[Unit] +Description=Migrates locally-stored Synapse media store files to S3 + +[Timer] +Unit=matrix-synapse-s3-storage-provider-migrate.service +OnCalendar=*-*-* 05:00:00 +RandomizedDelaySec=2h + +[Install] +WantedBy=timers.target diff --git a/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/usr-local-bin/matrix-synapse-s3-storage-provider-migrate.j2 b/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/usr-local-bin/matrix-synapse-s3-storage-provider-migrate.j2 new file mode 100644 index 000000000..0893f5d66 --- /dev/null +++ b/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/usr-local-bin/matrix-synapse-s3-storage-provider-migrate.j2 @@ -0,0 +1,13 @@ +#jinja2: lstrip_blocks: "True" +#!/bin/bash + +{{ matrix_host_command_docker }} run \ + --rm \ + --env-file={{ matrix_synapse_ext_s3_storage_provider_path }}/env \ + --mount type=bind,src={{ matrix_synapse_storage_path }},dst=/matrix-media-store-parent,bind-propagation=slave \ + --mount type=bind,src={{ matrix_synapse_ext_s3_storage_provider_path }}/data,dst=/data \ + --workdir=/data \ + --network={{ matrix_docker_network }} \ + --entrypoint=/bin/bash \ + {{ matrix_synapse_docker_image_final }} \ + -c 's3_media_upload update-db $UPDATE_DB_DURATION && s3_media_upload --no-progress check-deleted $MEDIA_PATH && s3_media_upload --no-progress upload $MEDIA_PATH $BUCKET --delete --endpoint-url $ENDPOINT' diff --git a/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/usr-local-bin/matrix-synapse-s3-storage-provider-shell.j2 b/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/usr-local-bin/matrix-synapse-s3-storage-provider-shell.j2 new file mode 100644 index 000000000..c67a6dda0 --- /dev/null +++ b/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/usr-local-bin/matrix-synapse-s3-storage-provider-shell.j2 @@ -0,0 +1,13 @@ +#jinja2: lstrip_blocks: "True" +#!/bin/bash + +{{ matrix_host_command_docker }} run \ + -it \ + --rm \ + --env-file={{ matrix_synapse_ext_s3_storage_provider_path }}/env \ + --mount type=bind,src={{ matrix_synapse_storage_path }},dst=/matrix-media-store-parent,bind-propagation=slave \ + --mount type=bind,src={{ matrix_synapse_ext_s3_storage_provider_path }}/data,dst=/data \ + --workdir=/data \ + --network={{ matrix_docker_network }} \ + --entrypoint=/bin/bash \ + {{ matrix_synapse_docker_image_final }} From a3759b0466fac32d482942177d7e690990d799e9 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 14 Oct 2022 17:50:48 +0300 Subject: [PATCH 022/118] Announce Synapse customization and synapse-s3-storage-provider support --- CHANGELOG.md | 33 ++++++++++++++++++++++++++ roles/matrix-synapse/defaults/main.yml | 1 - 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 06664634d..e99cd360a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,36 @@ +# 2022-10-14 + +## synapse-s3-storage-provider support + +You can now store your Synapse media repository files on Amazon S3 (or another S3-compatible object store) using [synapse-s3-storage-provider](https://github.com/matrix-org/synapse-s3-storage-provider) - a media provider for Synapse (Python module), which should work faster and more reliably than our previous [Goofys](docs/configuring-playbook-s3-goofys.md) implementation (Goofys will continue to work). + +This is not just for initial installations. Users with existing files (stored in the local filesystem) can also migrate their files to `synapse-s3-storage-provider`. + +To get started, see our [Storing Synapse media files on Amazon S3 with synapse-s3-storage-provider](docs/configuring-playbook-synapse-s3-storage-provider.md) documentation. + + +## Synapse container image customization support + +We now support customizing the Synapse container image by adding additional build steps to its [`Dockerfile`](https://docs.docker.com/engine/reference/builder/). + +Our [synapse-s3-storage-provider support](#synapse-s3-storage-provider-support) is actually built on this. When `s3-storage-provider` is enabled, we automatically add additional build steps to install its Python module into the Synapse image. + +Besides this kind of auto-added build steps (for components supported by the playbook), we also let you inject your own custom build steps using configuration like this: + +```yaml +matrix_synapse_container_image_customizations_enabled: true + +matrix_synapse_container_image_customizations_dockerfile_body_custom: | + RUN echo 'This is a custom step for building the customized Docker image for Synapse.' + RUN echo 'You can override matrix_synapse_container_image_customizations_dockerfile_body_custom to add your own steps.' + RUN echo 'You do NOT need to include a FROM clause yourself.' +``` + +People who have needed to customize Synapse previously had to fork the git repository, make their changes to the `Dockerfile` there, point the playbook to the new repository (`matrix_synapse_container_image_self_build_repo`) and enable self-building from scratch (`matrix_synapse_container_image_self_build: true`). This is harder and slower. + +With the new Synapse-customization feature in the playbook, we use the original upstream (pre-built, if available) Synapse image and only build on top of it, right on the Matrix server. This is much faster than building all of Synapse from scratch. + + # 2022-10-02 ## matrix-ldap-registration-proxy support diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index 383e67ab8..4b9791594 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -31,7 +31,6 @@ matrix_synapse_container_image_customizations_s3_storage_provider_installation_e # matrix_synapse_container_image_customizations_dockerfile_body_custom: | # RUN echo 'This is a custom step for building the customized Docker image for Synapse.' # RUN echo 'You can override matrix_synapse_container_image_customizations_dockerfile_body_custom to add your own steps.' -# RUN echo 'Note that matrix_synapse_container_image_customizations_dockerfile_body_auto injects steps before this.' # RUN echo 'You do NOT need to include a FROM clause yourself.' matrix_synapse_container_image_customizations_dockerfile_body_custom: '' From 63a0e5c4f6b1b3384a8178ac6414712094736bc6 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 14 Oct 2022 17:58:47 +0300 Subject: [PATCH 023/118] Add warnings to synapse-s3-storage-provider support feature --- CHANGELOG.md | 2 ++ docs/configuring-playbook-synapse-s3-storage-provider.md | 2 ++ 2 files changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e99cd360a..3ee9afac2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## synapse-s3-storage-provider support +**`synapse-s3-storage-provider` support is very new and still relatively untested. Using it may cause data loss.** + You can now store your Synapse media repository files on Amazon S3 (or another S3-compatible object store) using [synapse-s3-storage-provider](https://github.com/matrix-org/synapse-s3-storage-provider) - a media provider for Synapse (Python module), which should work faster and more reliably than our previous [Goofys](docs/configuring-playbook-s3-goofys.md) implementation (Goofys will continue to work). This is not just for initial installations. Users with existing files (stored in the local filesystem) can also migrate their files to `synapse-s3-storage-provider`. diff --git a/docs/configuring-playbook-synapse-s3-storage-provider.md b/docs/configuring-playbook-synapse-s3-storage-provider.md index bc0250f11..3490705ae 100644 --- a/docs/configuring-playbook-synapse-s3-storage-provider.md +++ b/docs/configuring-playbook-synapse-s3-storage-provider.md @@ -3,6 +3,8 @@ If you'd like to store Synapse's content repository (`media_store`) files on Amazon S3 (or other S3-compatible service), you can use the [synapse-s3-storage-provider](https://github.com/matrix-org/synapse-s3-storage-provider) media provider module for Synapse. +**`synapse-s3-storage-provider` support is very new and still relatively untested. Using it may cause data loss.** + An alternative (which has worse performance) is to use [Goofys to mount the S3 store to the local filesystem](configuring-playbook-s3-goofys.md). From fe9647559538756ca2f0bb547c621de638506c14 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 14 Oct 2022 18:01:51 +0300 Subject: [PATCH 024/118] Remove some useless PG_ variables fron the s3-storage-provider env-file These Postgres configuration variables are part of the `database.yaml` file and don't need to be here. --- .../templates/synapse/ext/s3-storage-provider/env.j2 | 6 ------ 1 file changed, 6 deletions(-) diff --git a/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 b/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 index 4b09688ba..3788499b3 100644 --- a/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 +++ b/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 @@ -5,12 +5,6 @@ AWS_DEFAULT_REGION={{ matrix_synapse_ext_synapse_s3_storage_provider_config_regi ENDPOINT={{ matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url }} BUCKET={{ matrix_synapse_ext_synapse_s3_storage_provider_config_bucket }} -PG_USER={{ matrix_synapse_database_user }} -PG_PASS={{ matrix_synapse_database_password }} -PG_DB={{ matrix_synapse_database_database }} -PG_HOST={{ matrix_synapse_database_host }} -PG_PORT={{ matrix_synapse_database_port }} - MEDIA_PATH=/matrix-media-store-parent/{{ matrix_synapse_media_store_directory_name }} UPDATE_DB_DURATION={{ matrix_synapse_ext_synapse_s3_storage_provider_update_db_day_count }}d From 2e0fc5c11c9d7dd165b09c8dc7bcbf8fcf6c2020 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 14 Oct 2022 18:04:49 +0300 Subject: [PATCH 025/118] Fix some ansible-lint-reported warnings --- .../tasks/ext/s3-storage-provider/setup_install.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/roles/matrix-synapse/tasks/ext/s3-storage-provider/setup_install.yml b/roles/matrix-synapse/tasks/ext/s3-storage-provider/setup_install.yml index 31f721819..796842704 100644 --- a/roles/matrix-synapse/tasks/ext/s3-storage-provider/setup_install.yml +++ b/roles/matrix-synapse/tasks/ext/s3-storage-provider/setup_install.yml @@ -38,7 +38,6 @@ - name: Ensure matrix-synapse-s3-storage-provider-migrate.service and timer are installed ansible.builtin.template: - src: "{{ role_path }}/templates/systemd/.j2" src: "{{ role_path }}/templates/synapse/ext/s3-storage-provider/systemd/{{ item }}.j2" dest: "{{ matrix_systemd_path }}/{{ item }}" mode: 0640 @@ -51,4 +50,3 @@ ansible.builtin.service: daemon_reload: true when: matrix_synapse_s3_storage_provider_systemd_service_result.changed | bool - From 3cf0dcda6d39506cc71c249d3001ec30d826b554 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 14 Oct 2022 18:07:10 +0300 Subject: [PATCH 026/118] Update "b2 sync" commands --- docs/configuring-playbook-s3-goofys.md | 6 +++--- docs/configuring-playbook-synapse-s3-storage-provider.md | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/configuring-playbook-s3-goofys.md b/docs/configuring-playbook-s3-goofys.md index 2efacddcc..ef8f20c2c 100644 --- a/docs/configuring-playbook-s3-goofys.md +++ b/docs/configuring-playbook-s3-goofys.md @@ -106,10 +106,10 @@ docker run -it --rm -w /work \ --env='B2_KEY_ID=YOUR_KEY_GOES_HERE' \ --env='B2_KEY_SECRET=YOUR_SECRET_GOES_HERE' \ --env='B2_BUCKET_NAME=YOUR_BUCKET_NAME_GOES_HERE' \ --v /matrix/synapse/storage/media-store/:/work \ +--mount type=bind,src=/matrix/synapse/storage/media-store,dst=/work,ro \ --entrypoint=/bin/sh \ -docker.io/tianon/backblaze-b2:2.1.0 \ --c 'b2 authorize-account $B2_KEY_ID $B2_KEY_SECRET > /dev/null && b2 sync /work/ b2://$B2_BUCKET_NAME' +docker.io/tianon/backblaze-b2:3.6.0 \ +-c 'b2 authorize-account $B2_KEY_ID $B2_KEY_SECRET && b2 sync /work b2://$B2_BUCKET_NAME --skipNewer' ``` This is some initial file sync, which may take a very long time. diff --git a/docs/configuring-playbook-synapse-s3-storage-provider.md b/docs/configuring-playbook-synapse-s3-storage-provider.md index 3490705ae..7f2af49c5 100644 --- a/docs/configuring-playbook-synapse-s3-storage-provider.md +++ b/docs/configuring-playbook-synapse-s3-storage-provider.md @@ -102,5 +102,5 @@ docker run -it --rm \ --mount type=bind,src=/matrix/synapse/storage/media-store,dst=/work,ro \ --entrypoint=/bin/sh \ tianon/backblaze-b2:3.6.0 \ --c 'b2 authorize-account $B2_KEY_ID $B2_KEY_SECRET > /dev/null && b2 sync /work b2://$B2_BUCKET_NAME --skipNewer' +-c 'b2 authorize-account $B2_KEY_ID $B2_KEY_SECRET && b2 sync /work b2://$B2_BUCKET_NAME --skipNewer' ``` From de8fd519c56e5905b729ce18c558cc478dc05de7 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 14 Oct 2022 18:14:34 +0300 Subject: [PATCH 027/118] Document matrix-synapse-s3-storage-provider-migrate.{service,timer} --- ...iguring-playbook-synapse-s3-storage-provider.md | 14 +++++++++++++- ...ix-synapse-s3-storage-provider-migrate.timer.j2 | 1 - 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/docs/configuring-playbook-synapse-s3-storage-provider.md b/docs/configuring-playbook-synapse-s3-storage-provider.md index 7f2af49c5..6ab2820a4 100644 --- a/docs/configuring-playbook-synapse-s3-storage-provider.md +++ b/docs/configuring-playbook-synapse-s3-storage-provider.md @@ -21,7 +21,7 @@ The way media storage providers in Synapse work has some caveats: You may be thinking **if all files are stored locally as well, what's the point**? -You can run some scripts to delete the local files once in a while, thus freeing up local disk space. If these files are needed in the future (for serving them to users, etc.), Synapse will pull them from the media storage provider on demand. +You can run some scripts to delete the local files once in a while (which we do automatically by default - see [Periodically cleaning up the local filesystem](#periodically-cleaning-up-the-local-filesystem)), thus freeing up local disk space. If these files are needed in the future (for serving them to users, etc.), Synapse will pull them from the media storage provider on demand. While you will need some local disk space around, it's only to accommodate usage, etc., and won't grow as large as your S3 store. @@ -104,3 +104,15 @@ docker run -it --rm \ tianon/backblaze-b2:3.6.0 \ -c 'b2 authorize-account $B2_KEY_ID $B2_KEY_SECRET && b2 sync /work b2://$B2_BUCKET_NAME --skipNewer' ``` + +## Periodically cleaning up the local filesystem + +As described in [How it works?](#how-it-works) above, when new media is uploaded to the Synapse homeserver, it's first stored locally and then also stored on the remote S3 storage. + +By default, we periodically ensure that all local files are uploaded to S3 and are then removed from the local filesystem. This is done automatically using: + +- the `/usr/local/bin/matrix-synapse-s3-storage-provider-migrate` script +- .. invoked via the `matrix-synapse-s3-storage-provider-migrate.service` service +- .. triggered by the `matrix-synapse-s3-storage-provider-migrate.timer` timer, every day at 05:00 + +So.. you don't need to perform any maintenance yourself. diff --git a/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/systemd/matrix-synapse-s3-storage-provider-migrate.timer.j2 b/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/systemd/matrix-synapse-s3-storage-provider-migrate.timer.j2 index 61526ac12..5013c7a81 100644 --- a/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/systemd/matrix-synapse-s3-storage-provider-migrate.timer.j2 +++ b/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/systemd/matrix-synapse-s3-storage-provider-migrate.timer.j2 @@ -4,7 +4,6 @@ Description=Migrates locally-stored Synapse media store files to S3 [Timer] Unit=matrix-synapse-s3-storage-provider-migrate.service OnCalendar=*-*-* 05:00:00 -RandomizedDelaySec=2h [Install] WantedBy=timers.target From 6f56b9fb38fbc5216054067a1658e8530a98f4fc Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 14 Oct 2022 18:17:08 +0300 Subject: [PATCH 028/118] Upgrade Dendrite (0.10.2 -> 0.10.3) --- roles/matrix-dendrite/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-dendrite/defaults/main.yml b/roles/matrix-dendrite/defaults/main.yml index 92a9ff9f7..2adf13daa 100644 --- a/roles/matrix-dendrite/defaults/main.yml +++ b/roles/matrix-dendrite/defaults/main.yml @@ -6,7 +6,7 @@ matrix_dendrite_enabled: true matrix_dendrite_docker_image: "{{ matrix_dendrite_docker_image_name_prefix }}matrixdotorg/dendrite-monolith:{{ matrix_dendrite_docker_image_tag }}" matrix_dendrite_docker_image_name_prefix: "docker.io/" -matrix_dendrite_docker_image_tag: "v0.10.2" +matrix_dendrite_docker_image_tag: "v0.10.3" matrix_dendrite_docker_image_force_pull: "{{ matrix_dendrite_docker_image.endswith(':latest') }}" matrix_dendrite_base_path: "{{ matrix_base_data_path }}/dendrite" From 30745db6d5f098fa68ac228b900f6efa82fd5ed1 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 14 Oct 2022 18:29:30 +0300 Subject: [PATCH 029/118] Add storage-class support to the s3_media_upload command and mention matrix-synapse-s3-storage-provider-migrate script --- ...configuring-playbook-synapse-s3-storage-provider.md | 10 ++++++++-- .../templates/synapse/ext/s3-storage-provider/env.j2 | 1 + .../matrix-synapse-s3-storage-provider-migrate.j2 | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/docs/configuring-playbook-synapse-s3-storage-provider.md b/docs/configuring-playbook-synapse-s3-storage-provider.md index 6ab2820a4..fce5b4edd 100644 --- a/docs/configuring-playbook-synapse-s3-storage-provider.md +++ b/docs/configuring-playbook-synapse-s3-storage-provider.md @@ -70,10 +70,16 @@ Then use the following commands (`$` values come from environment variables - th - `$UPDATE_DB_DURATION` is influenced by the `matrix_synapse_ext_synapse_s3_storage_provider_update_db_day_count` variable (defaults to `0`) - `$UPDATE_DB_DURATION` defaults to `0d` (0 days), which means **include files which haven't been accessed for more than 0 days** (that is, **all files will be included**). - `s3_media_upload check-deleted $MEDIA_PATH` - check whether files in the local cache still exist in the local media repository directory -- `s3_media_upload upload $MEDIA_PATH $BUCKET --delete --endpoint-url $ENDPOINT` - uploads locally-stored files to S3 and deletes them from the local media repository directory +- `s3_media_upload upload $MEDIA_PATH $BUCKET --delete --storage-class $STORAGE_CLASS --endpoint-url $ENDPOINT` - uploads locally-stored files to S3 and deletes them from the local media repository directory -The `upload` command may take a lot of time to complete. +The `s3_media_upload upload` command may take a lot of time to complete. +Instead of running the above commands manually in the shell, you can also run the `/usr/local/bin/matrix-synapse-s3-storage-provider-migrate` script which will run the same commands automatically. We demonstrate how to do it manually, because: + +- it's what the upstream project demonstrates and it teaches you how to use the `s3_media_upload` tool +- allows you to check and verify the output of each command, to catch mistakes +- includes progress bars and detailed output for each command +- allows you to easily interrupt slow-running commands, etc. (the `/usr/local/bin/matrix-synapse-s3-storage-provider-migrate` starts a container without interactive TTY support, so `Ctrl+C` may not work and you and require killing via `docker kill ..`) ### Using another tool in combination with `s3_media_upload` diff --git a/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 b/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 index 3788499b3..6dfcbe418 100644 --- a/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 +++ b/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/env.j2 @@ -4,6 +4,7 @@ AWS_DEFAULT_REGION={{ matrix_synapse_ext_synapse_s3_storage_provider_config_regi ENDPOINT={{ matrix_synapse_ext_synapse_s3_storage_provider_config_endpoint_url }} BUCKET={{ matrix_synapse_ext_synapse_s3_storage_provider_config_bucket }} +STORAGE_CLASS={{ matrix_synapse_ext_synapse_s3_storage_provider_config_storage_class }} MEDIA_PATH=/matrix-media-store-parent/{{ matrix_synapse_media_store_directory_name }} diff --git a/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/usr-local-bin/matrix-synapse-s3-storage-provider-migrate.j2 b/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/usr-local-bin/matrix-synapse-s3-storage-provider-migrate.j2 index 0893f5d66..e6684e696 100644 --- a/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/usr-local-bin/matrix-synapse-s3-storage-provider-migrate.j2 +++ b/roles/matrix-synapse/templates/synapse/ext/s3-storage-provider/usr-local-bin/matrix-synapse-s3-storage-provider-migrate.j2 @@ -10,4 +10,4 @@ --network={{ matrix_docker_network }} \ --entrypoint=/bin/bash \ {{ matrix_synapse_docker_image_final }} \ - -c 's3_media_upload update-db $UPDATE_DB_DURATION && s3_media_upload --no-progress check-deleted $MEDIA_PATH && s3_media_upload --no-progress upload $MEDIA_PATH $BUCKET --delete --endpoint-url $ENDPOINT' + -c 's3_media_upload update-db $UPDATE_DB_DURATION && s3_media_upload --no-progress check-deleted $MEDIA_PATH && s3_media_upload --no-progress upload $MEDIA_PATH $BUCKET --delete --storage-class $STORAGE_CLASS --endpoint-url $ENDPOINT' From 5a7b80d9e48326f8d6bedf1e31a234fbe735c8d6 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 14 Oct 2022 18:31:17 +0300 Subject: [PATCH 030/118] Update configuring-playbook-synapse-s3-storage-provider.md --- docs/configuring-playbook-synapse-s3-storage-provider.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/configuring-playbook-synapse-s3-storage-provider.md b/docs/configuring-playbook-synapse-s3-storage-provider.md index fce5b4edd..a71e5a0bf 100644 --- a/docs/configuring-playbook-synapse-s3-storage-provider.md +++ b/docs/configuring-playbook-synapse-s3-storage-provider.md @@ -48,6 +48,8 @@ If you have existing files in Synapse's media repository (`/matrix/synapse/media - the existing files will remain on the local filesystem only until [migrating them to the S3 store](#migrating-your-existing-media-files-to-the-s3-store) - at some point (and periodically in the future), you can delete local files which have been uploaded to the S3 store already +Regardless of whether you need to [Migrate your existing files to the S3 store](#migrating-your-existing-media-files-to-the-s3-store) or not, make sure you've familiarized yourself with [How it works?](#how-it-works) above and [Periodically cleaning up the local filesystem](#periodically-cleaning-up-the-local-filesystem) below. + ## Migrating your existing media files to the S3 store From 1e0d10586d4bd04dde30adb1e860ada432084865 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 14 Oct 2022 18:49:27 +0300 Subject: [PATCH 031/118] Update README to mention synapse-s3-storage-provider --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b93fdd5da..569dbb3f8 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Using this playbook, you can get the following services configured on your serve - (optional) a [Dendrite](https://github.com/matrix-org/dendrite) homeserver - storing your data and managing your presence in the [Matrix](http://matrix.org/) network. Dendrite is a second-generation Matrix homeserver written in Go, an alternative to Synapse. -- (optional) [Amazon S3](https://aws.amazon.com/s3/) storage for Synapse's content repository (`media_store`) files using [Goofys](https://github.com/kahing/goofys) +- (optional) [Amazon S3](https://aws.amazon.com/s3/) (or other S3-compatible object store) storage for Synapse's content repository (`media_store`) files using [Goofys](https://github.com/kahing/goofys) or [`synapse-s3-storage-provider`](https://github.com/matrix-org/synapse-s3-storage-provider) - (optional, default) [PostgreSQL](https://www.postgresql.org/) database for Synapse. [Using an external PostgreSQL server](docs/configuring-playbook-external-postgres.md) is also possible. From 8025bd19b0b93ac52230f756e86155b6b7bf17a7 Mon Sep 17 00:00:00 2001 From: smargold476 <105579587+smargold476@users.noreply.github.com> Date: Sat, 15 Oct 2022 14:33:28 +0200 Subject: [PATCH 032/118] update Docs regarding Filter-Mode-Flag --- docs/configuring-playbook-bridge-mautrix-telegram.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/configuring-playbook-bridge-mautrix-telegram.md b/docs/configuring-playbook-bridge-mautrix-telegram.md index 08ee83ccd..91596afc4 100644 --- a/docs/configuring-playbook-bridge-mautrix-telegram.md +++ b/docs/configuring-playbook-bridge-mautrix-telegram.md @@ -59,3 +59,8 @@ matrix_mautrix_telegram_configuration_extension_yaml: | More details about permissions in this example: https://github.com/mautrix/telegram/blob/master/mautrix_telegram/example-config.yaml#L410 + +If you like to exclude all groups from syncing and use the Telgeram-Bridge only for direct chats, you can add the following additional playbook configuration: +```yaml +matrix_mautrix_telegram_filter_mode: whitelist +``` From 01078fea8f0c32e09e20a81070ce0cc5f6712435 Mon Sep 17 00:00:00 2001 From: smargold476 <105579587+smargold476@users.noreply.github.com> Date: Sat, 15 Oct 2022 14:37:09 +0200 Subject: [PATCH 033/118] add default for TG filter_mode --- roles/matrix-bridge-mautrix-telegram/defaults/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml index 9fdfd4302..0f52cc222 100644 --- a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml @@ -38,6 +38,9 @@ matrix_mautrix_telegram_api_id: '' matrix_mautrix_telegram_api_hash: '' matrix_mautrix_telegram_bot_token: disabled +# Define the filter-mode +matrix_mautrix_telegram_filter_mode: "blacklist" + # Whether or not the public-facing endpoints should be enabled (web-based login) matrix_mautrix_telegram_appservice_public_enabled: true From a6e167cf91932eac58fdcb117062f2a31d62ed64 Mon Sep 17 00:00:00 2001 From: smargold476 <105579587+smargold476@users.noreply.github.com> Date: Sat, 15 Oct 2022 14:38:51 +0200 Subject: [PATCH 034/118] add option to change filter_mode --- roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2 b/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2 index d50be47da..2e6aae51b 100644 --- a/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2 @@ -273,7 +273,7 @@ bridge: # Filter mode to use. Either "blacklist" or "whitelist". # If the mode is "blacklist", the listed chats will never be bridged. # If the mode is "whitelist", only the listed chats can be bridged. - mode: blacklist + mode: "{{ matrix_mautrix_telegram_filter_mode }}" # The list of group/channel IDs to filter. list: [] From 7b3b22ce072ce02d7ebb8665f899147422a84841 Mon Sep 17 00:00:00 2001 From: smargold476 <105579587+smargold476@users.noreply.github.com> Date: Sat, 15 Oct 2022 14:43:25 +0200 Subject: [PATCH 035/118] update docs to exclude grou-sync --- docs/configuring-playbook-bridge-mautrix-telegram.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/configuring-playbook-bridge-mautrix-telegram.md b/docs/configuring-playbook-bridge-mautrix-telegram.md index 08ee83ccd..91596afc4 100644 --- a/docs/configuring-playbook-bridge-mautrix-telegram.md +++ b/docs/configuring-playbook-bridge-mautrix-telegram.md @@ -59,3 +59,8 @@ matrix_mautrix_telegram_configuration_extension_yaml: | More details about permissions in this example: https://github.com/mautrix/telegram/blob/master/mautrix_telegram/example-config.yaml#L410 + +If you like to exclude all groups from syncing and use the Telgeram-Bridge only for direct chats, you can add the following additional playbook configuration: +```yaml +matrix_mautrix_telegram_filter_mode: whitelist +``` From a7dc9a406de907ab21804613a13c335f335c4352 Mon Sep 17 00:00:00 2001 From: Felix Stupp Date: Sat, 15 Oct 2022 20:17:58 +0000 Subject: [PATCH 036/118] docs/configuring-playbook: Add link to etherpad configuration --- docs/configuring-playbook.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/configuring-playbook.md b/docs/configuring-playbook.md index 43b68ee8f..efb0f7e3c 100644 --- a/docs/configuring-playbook.md +++ b/docs/configuring-playbook.md @@ -34,6 +34,8 @@ When you're done with all the configuration you'd like to do, continue with [Ins - [Setting up the Jitsi video-conferencing platform](configuring-playbook-jitsi.md) (optional) +- [Setting up Etherpad](configuring-playbook-etherpad.md) (optional) + - [Setting up Dynamic DNS](configuring-playbook-dynamic-dns.md) (optional) - [Enabling metrics and graphs (Prometheus, Grafana) for your Matrix server](configuring-playbook-prometheus-grafana.md) (optional) From 4d267dad4084859ca9c69b23e241c4ae2c1d2817 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sun, 16 Oct 2022 18:24:55 +0300 Subject: [PATCH 037/118] Upgrade mautrix-whatsapp (0.7.0 -> 0.7.1) and sync bridge config with upstream --- .../matrix-bridge-mautrix-whatsapp/defaults/main.yml | 2 +- .../templates/config.yaml.j2 | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml b/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml index cb4900eec..55b7387f1 100644 --- a/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml @@ -8,7 +8,7 @@ matrix_mautrix_whatsapp_container_image_self_build: false matrix_mautrix_whatsapp_container_image_self_build_repo: "https://mau.dev/mautrix/whatsapp.git" matrix_mautrix_whatsapp_container_image_self_build_branch: "{{ 'master' if matrix_mautrix_whatsapp_version == 'latest' else matrix_mautrix_whatsapp_version }}" -matrix_mautrix_whatsapp_version: v0.7.0 +matrix_mautrix_whatsapp_version: v0.7.1 # See: https://mau.dev/mautrix/whatsapp/container_registry matrix_mautrix_whatsapp_docker_image: "{{ matrix_mautrix_whatsapp_docker_image_name_prefix }}mautrix/whatsapp:{{ matrix_mautrix_whatsapp_version }}" matrix_mautrix_whatsapp_docker_image_name_prefix: "{{ 'localhost/' if matrix_mautrix_whatsapp_container_image_self_build else 'dock.mau.dev/' }}" diff --git a/roles/matrix-bridge-mautrix-whatsapp/templates/config.yaml.j2 b/roles/matrix-bridge-mautrix-whatsapp/templates/config.yaml.j2 index 28971447e..9c0b95e8d 100644 --- a/roles/matrix-bridge-mautrix-whatsapp/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mautrix-whatsapp/templates/config.yaml.j2 @@ -147,6 +147,12 @@ bridge: # provisioning endpoint is used or when a message comes in from that # chat. max_initial_conversations: -1 + # If this value is greater than 0, then if the conversation's last + # message was more than this number of hours ago, then the conversation + # will automatically be marked it as read. + # Conversations that have a last message that is less than this number + # of hours ago will have their unread status synced from WhatsApp. + unread_hours_threshold: 0 # Settings for immediate backfills. These backfills should generally be # small and their main purpose is to populate each of the initial chats # (as configured by max_initial_conversations) with a few messages so @@ -228,7 +234,10 @@ bridge: # manually. login_shared_secret_map: {{ matrix_mautrix_whatsapp_bridge_login_shared_secret_map|to_json }} # Should the bridge explicitly set the avatar and room name for private chat portal rooms? + # This is implicitly enabled in encrypted rooms. private_chat_portal_meta: false + # Should group members be synced in parallel? This makes member sync faster + parallel_member_sync: false # Should Matrix m.notice-type messages be bridged? bridge_notices: true # Set this to true to tell the bridge to re-send m.bridge events to all rooms on the next run. @@ -281,6 +290,9 @@ bridge: # Send captions in the same message as images. This will send data compatible with both MSC2530 and MSC3552. # This is currently not supported in most clients. caption_in_message: false + # Should Matrix edits be bridged to WhatsApp edits? + # Official WhatsApp clients don't render edits yet, but once they do, the bridge should work with them right away. + send_whatsapp_edits: false # Maximum time for handling Matrix events. Duration strings formatted for https://pkg.go.dev/time#ParseDuration # Null means there's no enforced timeout. message_handling_timeout: From 07a7234ba0c642f910e3381776edf5b40735c9de Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sun, 16 Oct 2022 18:27:35 +0300 Subject: [PATCH 038/118] Use | to_json --- .../matrix-bridge-mautrix-telegram/templates/config.yaml.j2 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2 b/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2 index 2e6aae51b..b7af83f57 100644 --- a/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2 @@ -273,12 +273,12 @@ bridge: # Filter mode to use. Either "blacklist" or "whitelist". # If the mode is "blacklist", the listed chats will never be bridged. # If the mode is "whitelist", only the listed chats can be bridged. - mode: "{{ matrix_mautrix_telegram_filter_mode }}" + mode: {{ matrix_mautrix_telegram_filter_mode | to_json }} # The list of group/channel IDs to filter. list: [] # The prefix for commands. Only required in non-management rooms. - command_prefix: "{{ matrix_mautrix_telegram_command_prefix }}" + command_prefix: {{ matrix_mautrix_telegram_command_prefix | to_json }} # Permissions for using the bridge. # Permitted values: @@ -291,7 +291,7 @@ bridge: # * - All Matrix users # domain - All users on that homeserver # mxid - Specific user - permissions: {{ matrix_mautrix_telegram_bridge_permissions|to_json }} + permissions: {{ matrix_mautrix_telegram_bridge_permissions | to_json }} # Options related to the message relay Telegram bot. relaybot: From 6480cc36293da31fa58b3aa1f7fe90fffbae87bf Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 17 Oct 2022 08:03:10 +0300 Subject: [PATCH 039/118] Fix `'something' is undefined` error in matrix-bridge-appservice-slack .. when `matrix_nginx_proxy_enabled: false` Fixes https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/2177 --- roles/matrix-bridge-appservice-slack/tasks/init.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bridge-appservice-slack/tasks/init.yml b/roles/matrix-bridge-appservice-slack/tasks/init.yml index d06e5aaf1..5d03b24b9 100644 --- a/roles/matrix-bridge-appservice-slack/tasks/init.yml +++ b/roles/matrix-bridge-appservice-slack/tasks/init.yml @@ -85,7 +85,7 @@ msg: >- NOTE: You've enabled the Matrix Slack bridge but are not using the matrix-nginx-proxy reverse proxy. - Please make sure that you're proxying the `{{ something }}` + Please make sure that you're proxying the `{{ matrix_appservice_slack_public_endpoint }}` URL endpoint to the matrix-appservice-slack container. You can expose the container's port using the `matrix_appservice_slack_container_http_host_bind_port` variable. when: "matrix_appservice_slack_enabled | bool and not matrix_nginx_proxy_enabled | default(False) | bool" From c781bdea992435fd9768d5a7458058528b712300 Mon Sep 17 00:00:00 2001 From: Aine <97398200+etkecc@users.noreply.github.com> Date: Mon, 17 Oct 2022 14:24:57 +0000 Subject: [PATCH 040/118] Update Synapse 1.68.0 -> 1.69.0 --- roles/matrix-synapse/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index 4b9791594..5e5867eb8 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -36,7 +36,7 @@ matrix_synapse_container_image_customizations_dockerfile_body_custom: '' matrix_synapse_docker_image: "{{ matrix_synapse_docker_image_name_prefix }}matrixdotorg/synapse:{{ matrix_synapse_docker_image_tag }}" matrix_synapse_docker_image_name_prefix: "{{ 'localhost/' if matrix_synapse_container_image_self_build else matrix_container_global_registry_prefix }}" -matrix_synapse_version: v1.68.0 +matrix_synapse_version: v1.69.0 matrix_synapse_docker_image_tag: "{{ matrix_synapse_version }}" matrix_synapse_docker_image_force_pull: "{{ matrix_synapse_docker_image.endswith(':latest') }}" From bcba84389d683f9143dbb7c2ccb516e4b68773d6 Mon Sep 17 00:00:00 2001 From: Aine <97398200+etkecc@users.noreply.github.com> Date: Tue, 18 Oct 2022 12:43:13 +0000 Subject: [PATCH 041/118] Update grafana 9.2.0 -> 9.2.1 --- roles/matrix-grafana/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-grafana/defaults/main.yml b/roles/matrix-grafana/defaults/main.yml index 677435e82..767a749bc 100644 --- a/roles/matrix-grafana/defaults/main.yml +++ b/roles/matrix-grafana/defaults/main.yml @@ -5,7 +5,7 @@ matrix_grafana_enabled: true -matrix_grafana_version: 9.2.0 +matrix_grafana_version: 9.2.1 matrix_grafana_docker_image: "{{ matrix_container_global_registry_prefix }}grafana/grafana:{{ matrix_grafana_version }}" matrix_grafana_docker_image_force_pull: "{{ matrix_grafana_docker_image.endswith(':latest') }}" From 2eef6af23e6a8c4e5a46c9ccf49a12c0e4f24c91 Mon Sep 17 00:00:00 2001 From: David Napier Date: Wed, 19 Oct 2022 10:45:11 -0400 Subject: [PATCH 042/118] Bump postgres version to newly released 15 --- roles/matrix-postgres/defaults/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml index de28d7ada..93de2612b 100644 --- a/roles/matrix-postgres/defaults/main.yml +++ b/roles/matrix-postgres/defaults/main.yml @@ -29,7 +29,8 @@ matrix_postgres_docker_image_v11: "{{ matrix_container_global_registry_prefix }} matrix_postgres_docker_image_v12: "{{ matrix_container_global_registry_prefix }}postgres:12.12{{ matrix_postgres_docker_image_suffix }}" matrix_postgres_docker_image_v13: "{{ matrix_container_global_registry_prefix }}postgres:13.8{{ matrix_postgres_docker_image_suffix }}" matrix_postgres_docker_image_v14: "{{ matrix_container_global_registry_prefix }}postgres:14.5{{ matrix_postgres_docker_image_suffix }}" -matrix_postgres_docker_image_latest: "{{ matrix_postgres_docker_image_v14 }}" +matrix_postgres_docker_image_v15: "{{ matrix_container_global_registry_prefix }}postgres:15{{ matrix_postgres_docker_image_suffix }}" +matrix_postgres_docker_image_latest: "{{ matrix_postgres_docker_image_v15 }}" # This variable is assigned at runtime. Overriding its value has no effect. matrix_postgres_docker_image_to_use: '{{ matrix_postgres_docker_image_latest }}' From c4a57428a6bedac73f1f8253ab0c5e4010aa00dc Mon Sep 17 00:00:00 2001 From: array-in-a-matrix Date: Thu, 20 Oct 2022 00:34:30 -0400 Subject: [PATCH 043/118] add if hostname is not availble for SRV record --- docs/configuring-dns.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/configuring-dns.md b/docs/configuring-dns.md index 8d31ab3ff..ea5fea88f 100644 --- a/docs/configuring-dns.md +++ b/docs/configuring-dns.md @@ -45,6 +45,8 @@ If you are using Cloudflare DNS, make sure to disable the proxy and set all reco | [Postmoogle](configuring-playbook-bot-postmoogle.md) email bridge | TXT | `_dmarc.matrix` | - | - | - | `v=DMARC1; p=quarantine;` | | [Postmoogle](configuring-playbook-bot-postmoogle.md) email bridge | TXT | `postmoogle._domainkey.matrix` | - | - | - | get it from `!pm dkim` | +When setting up a SRV record, if you are asked for a service and protocol instead of a hostname split the host value from the table where the period is. For example use service as `_matrix-identity` and protocol as `_tcp`. + ## Subdomains setup As the table above illustrates, you need to create 2 subdomains (`matrix.` and `element.`) and point both of them to your new server's IP address (DNS `A` record or `CNAME` record is fine). From 20db57d288e037bd651d718e945fff96faa33d77 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 17 Oct 2022 17:35:52 +0300 Subject: [PATCH 044/118] Upgrade certbot (v1.30.0 -> v1.31.0) --- roles/matrix-nginx-proxy/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-nginx-proxy/defaults/main.yml b/roles/matrix-nginx-proxy/defaults/main.yml index 7cdc0c921..84ec4f7e1 100644 --- a/roles/matrix-nginx-proxy/defaults/main.yml +++ b/roles/matrix-nginx-proxy/defaults/main.yml @@ -547,7 +547,7 @@ matrix_ssl_lets_encrypt_staging: false # Learn more here: https://eff-certbot.readthedocs.io/en/stable/using.html#changing-the-acme-server matrix_ssl_lets_encrypt_server: '' -matrix_ssl_lets_encrypt_certbot_docker_image: "{{ matrix_container_global_registry_prefix }}certbot/certbot:{{ matrix_ssl_architecture }}-v1.30.0" +matrix_ssl_lets_encrypt_certbot_docker_image: "{{ matrix_container_global_registry_prefix }}certbot/certbot:{{ matrix_ssl_architecture }}-v1.31.0" matrix_ssl_lets_encrypt_certbot_docker_image_force_pull: "{{ matrix_ssl_lets_encrypt_certbot_docker_image.endswith(':latest') }}" matrix_ssl_lets_encrypt_certbot_standalone_http_port: 2402 matrix_ssl_lets_encrypt_support_email: ~ From a32eea41fe56bdee721f8062b1199217704b0daa Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 20 Oct 2022 07:43:01 +0300 Subject: [PATCH 045/118] Make roles/matrix-postgres/tasks/detect_existing_postgres_version.yml detect Postgres v14 --- .../tasks/detect_existing_postgres_version.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/roles/matrix-postgres/tasks/detect_existing_postgres_version.yml b/roles/matrix-postgres/tasks/detect_existing_postgres_version.yml index 687d5e3a5..1be8291b6 100644 --- a/roles/matrix-postgres/tasks/detect_existing_postgres_version.yml +++ b/roles/matrix-postgres/tasks/detect_existing_postgres_version.yml @@ -68,3 +68,8 @@ ansible.builtin.set_fact: matrix_postgres_detected_version_corresponding_docker_image: "{{ matrix_postgres_docker_image_v13 }}" when: "matrix_postgres_detected_version == '13' or matrix_postgres_detected_version.startswith('13.')" + +- name: Determine corresponding Docker image to detected version (use 14.x, if detected) + ansible.builtin.set_fact: + matrix_postgres_detected_version_corresponding_docker_image: "{{ matrix_postgres_docker_image_v14 }}" + when: "matrix_postgres_detected_version == '14' or matrix_postgres_detected_version.startswith('14.')" From 80da7dfb0bc4dbff1b6982d213ca5cbb98ff7f94 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 20 Oct 2022 11:48:56 +0300 Subject: [PATCH 046/118] Update docs/prerequisites.md --- docs/prerequisites.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/prerequisites.md b/docs/prerequisites.md index 1ed4befea..74954648d 100644 --- a/docs/prerequisites.md +++ b/docs/prerequisites.md @@ -26,7 +26,7 @@ If your distro runs within an [LXC container](https://linuxcontainers.org/), you - Properly configured DNS records for `` (details in [Configuring DNS](configuring-dns.md)). -- Some TCP/UDP ports open. This playbook configures the server's internal firewall for you. In most cases, you don't need to do anything special. But **if your server is running behind another firewall**, you'd need to open these ports: +- Some TCP/UDP ports open. This playbook (actually [Docker itself](https://docs.docker.com/network/iptables/)) configures the server's internal firewall for you. In most cases, you don't need to do anything special. But **if your server is running behind another firewall**, you'd need to open these ports: - `80/tcp`: HTTP webserver - `443/tcp`: HTTPS webserver From af3a32cf6a7f7c6f73ee6bd85b3fede721b5e70c Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 20 Oct 2022 15:46:02 +0300 Subject: [PATCH 047/118] Fix Jinja2 interpolation for some default variables We're overriding these in the correct way in `group_vars/matrix_servers` so this wasn't causing any problem in practice. --- roles/matrix-postgres/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml index 93de2612b..1b11bad8c 100644 --- a/roles/matrix-postgres/defaults/main.yml +++ b/roles/matrix-postgres/defaults/main.yml @@ -68,7 +68,7 @@ matrix_postgres_additional_databases: [] # If a dump file contains the roles and they've also been created beforehand (see `matrix_postgres_additional_databases`), # importing would fail. # We either need to not create them or to ignore the `CREATE ROLE` statements in the dump. -matrix_postgres_import_roles_to_ignore: [matrix_postgres_connection_username] +matrix_postgres_import_roles_to_ignore: "{{ [matrix_postgres_connection_username] }}" # When importing an existing Postgres database (when restoring a backup) or when doing a Postgres upgrade (which dumps & restores), we'd like to avoid: # - creating users (`CREATE ROLE ..`) @@ -85,7 +85,7 @@ matrix_postgres_import_roles_ignore_regex: "^(CREATE|ALTER) ROLE ({{ matrix_post # If a dump file contains the databases and they've also been created beforehand (see `matrix_postgres_additional_databases`), # importing would fail. # We either need to not create them or to ignore the `CREATE DATABASE` statements in the dump. -matrix_postgres_import_databases_to_ignore: [matrix_postgres_db_name] +matrix_postgres_import_databases_to_ignore: "{{ [matrix_postgres_db_name] }}" matrix_postgres_import_databases_ignore_regex: "^CREATE DATABASE ({{ matrix_postgres_import_databases_to_ignore | join('|') }})\\s" # noqa jinja[spacing] From b8097b0bd6f79f9efb66c4df7676129f86e07bd2 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 20 Oct 2022 16:05:55 +0300 Subject: [PATCH 048/118] Add support for binary content to matrix-aux --- roles/matrix-aux/defaults/main.yml | 9 +++++++++ roles/matrix-aux/tasks/setup.yml | 3 ++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/roles/matrix-aux/defaults/main.yml b/roles/matrix-aux/defaults/main.yml index e4a4e8277..4c1f88796 100644 --- a/roles/matrix-aux/defaults/main.yml +++ b/roles/matrix-aux/defaults/main.yml @@ -50,6 +50,9 @@ matrix_aux_file_default_mode: '0640' # then you likely need to add `/matrix/some/path` to `matrix_aux_directory_definitions` as well. # You don't need to do this for directories that the playbook already creates for you. # +# Use a `content` key for text content and `src` with a location to a file for binary content. +# The `content` key does not support binary content (see https://github.com/ansible/ansible/issues/11594). +# # Example: # # matrix_aux_file_definitions: @@ -69,4 +72,10 @@ matrix_aux_file_default_mode: '0640' # mode: '0600' # owner: 'some-user' # group: 'some-group' +# +# - dest: /matrix/aux/binary-file.dat +# src: "/path/to/binary.dat" +# mode: '0600' +# owner: 'some-user' +# group: 'some-group' matrix_aux_file_definitions: [] diff --git a/roles/matrix-aux/tasks/setup.yml b/roles/matrix-aux/tasks/setup.yml index ccb0bdcb4..eb0adad49 100644 --- a/roles/matrix-aux/tasks/setup.yml +++ b/roles/matrix-aux/tasks/setup.yml @@ -11,8 +11,9 @@ - name: Ensure AUX files are created ansible.builtin.copy: + src: "{{ item.src if 'src' in item else omit }}" + content: "{{ item.content if 'content' in item else omit }}" dest: "{{ item.dest }}" - content: "{{ item.content }}" owner: "{{ item.owner | default(matrix_user_username) }}" group: "{{ item.group | default(matrix_user_groupname) }}" mode: "{{ item.mode | default(matrix_aux_file_default_mode) }}" From bed9c18ab04156e6e96521a380acc3be0c36e047 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 21 Oct 2022 05:31:52 +0300 Subject: [PATCH 049/118] Pin Postgres version to 15.0 Related to https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/2180 Just specifying `15` means we won't automatically re-pull `15.1` when it comes out. --- roles/matrix-postgres/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml index 1b11bad8c..6ddbdf9fd 100644 --- a/roles/matrix-postgres/defaults/main.yml +++ b/roles/matrix-postgres/defaults/main.yml @@ -29,7 +29,7 @@ matrix_postgres_docker_image_v11: "{{ matrix_container_global_registry_prefix }} matrix_postgres_docker_image_v12: "{{ matrix_container_global_registry_prefix }}postgres:12.12{{ matrix_postgres_docker_image_suffix }}" matrix_postgres_docker_image_v13: "{{ matrix_container_global_registry_prefix }}postgres:13.8{{ matrix_postgres_docker_image_suffix }}" matrix_postgres_docker_image_v14: "{{ matrix_container_global_registry_prefix }}postgres:14.5{{ matrix_postgres_docker_image_suffix }}" -matrix_postgres_docker_image_v15: "{{ matrix_container_global_registry_prefix }}postgres:15{{ matrix_postgres_docker_image_suffix }}" +matrix_postgres_docker_image_v15: "{{ matrix_container_global_registry_prefix }}postgres:15.0{{ matrix_postgres_docker_image_suffix }}" matrix_postgres_docker_image_latest: "{{ matrix_postgres_docker_image_v15 }}" # This variable is assigned at runtime. Overriding its value has no effect. From e37e86eb1cfdabf0bbe7464037060cbb0cacf2dc Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 21 Oct 2022 07:33:10 +0300 Subject: [PATCH 050/118] Fix 'could not save history to file "//.psql_history"' errors --- roles/matrix-postgres/templates/env-postgres-psql.j2 | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/roles/matrix-postgres/templates/env-postgres-psql.j2 b/roles/matrix-postgres/templates/env-postgres-psql.j2 index c61927a3e..220589874 100644 --- a/roles/matrix-postgres/templates/env-postgres-psql.j2 +++ b/roles/matrix-postgres/templates/env-postgres-psql.j2 @@ -1,4 +1,8 @@ #jinja2: lstrip_blocks: "True" PGUSER={{ matrix_postgres_connection_username }} PGPASSWORD={{ matrix_postgres_connection_password }} -PGDATABASE={{ matrix_postgres_db_name }} \ No newline at end of file +PGDATABASE={{ matrix_postgres_db_name }} +# Prevent errors like this: +# > could not save history to file "//.psql_history": Permission denied +# .. due to Postgres not being able to write to the filesystem. +PSQL_HISTORY=/dev/null From 0a022db256688a9924d199828f4f555bbb94d168 Mon Sep 17 00:00:00 2001 From: Array in a Matrix Date: Fri, 21 Oct 2022 00:52:48 -0400 Subject: [PATCH 051/118] grammar correction the word "both" would refer to only 2 urls in this context --- docs/configuring-well-known.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuring-well-known.md b/docs/configuring-well-known.md index 81caf04cb..fd548aa63 100644 --- a/docs/configuring-well-known.md +++ b/docs/configuring-well-known.md @@ -192,7 +192,7 @@ Make sure to: ## Confirming it works -No matter which method you've used to set up the well-known files, if you've done it correctly you should be able to see a JSON file at both of these URLs: +No matter which method you've used to set up the well-known files, if you've done it correctly you should be able to see a JSON file at these URLs: - `https:///.well-known/matrix/server` - `https:///.well-known/matrix/client` From 0b44ec19b4e1d39420710f229808f0780c5820d2 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 21 Oct 2022 10:00:49 +0300 Subject: [PATCH 052/118] Do not override matrix_postgres_import_roles_to_ignore/matrix_postgres_import_databases_to_ignore in group_vars These values that we were setting also make sense in the context of the `matrix-postgres` role even when not used within the playbook. --- group_vars/matrix_servers | 14 -------------- roles/matrix-postgres/defaults/main.yml | 14 ++++++++++++-- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index b055245bc..6728eaaae 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -2112,20 +2112,6 @@ matrix_postgres_additional_databases: | }} -matrix_postgres_import_roles_to_ignore: | - {{ - [matrix_postgres_connection_username] - + - matrix_postgres_additional_databases|map(attribute='username') | list - }} - -matrix_postgres_import_databases_to_ignore: | - {{ - [matrix_postgres_db_name] - + - matrix_postgres_additional_databases|map(attribute='name') | list - }} - ###################################################################### # # /matrix-postgres diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml index 6ddbdf9fd..e34b3b606 100644 --- a/roles/matrix-postgres/defaults/main.yml +++ b/roles/matrix-postgres/defaults/main.yml @@ -68,7 +68,12 @@ matrix_postgres_additional_databases: [] # If a dump file contains the roles and they've also been created beforehand (see `matrix_postgres_additional_databases`), # importing would fail. # We either need to not create them or to ignore the `CREATE ROLE` statements in the dump. -matrix_postgres_import_roles_to_ignore: "{{ [matrix_postgres_connection_username] }}" +matrix_postgres_import_roles_to_ignore: | + {{ + [matrix_postgres_connection_username] + + + matrix_postgres_additional_databases|map(attribute='username') | list + }} # When importing an existing Postgres database (when restoring a backup) or when doing a Postgres upgrade (which dumps & restores), we'd like to avoid: # - creating users (`CREATE ROLE ..`) @@ -85,7 +90,12 @@ matrix_postgres_import_roles_ignore_regex: "^(CREATE|ALTER) ROLE ({{ matrix_post # If a dump file contains the databases and they've also been created beforehand (see `matrix_postgres_additional_databases`), # importing would fail. # We either need to not create them or to ignore the `CREATE DATABASE` statements in the dump. -matrix_postgres_import_databases_to_ignore: "{{ [matrix_postgres_db_name] }}" +matrix_postgres_import_databases_to_ignore: | + {{ + [matrix_postgres_db_name] + + + matrix_postgres_additional_databases|map(attribute='name') | list + }} matrix_postgres_import_databases_ignore_regex: "^CREATE DATABASE ({{ matrix_postgres_import_databases_to_ignore | join('|') }})\\s" # noqa jinja[spacing] From ff0fc88faeac6444dcee93241dca23a24356cde1 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 21 Oct 2022 11:25:30 +0300 Subject: [PATCH 053/118] Make Postgres import not break for databases with special names We haven't encountered such a problem yet, but it doesn't hurt to make things more robust. --- roles/matrix-postgres/defaults/main.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml index e34b3b606..7b0660bbd 100644 --- a/roles/matrix-postgres/defaults/main.yml +++ b/roles/matrix-postgres/defaults/main.yml @@ -84,7 +84,9 @@ matrix_postgres_import_roles_to_ignore: | # which is unsupported by default by newer Postgres versions (v14+). # When users are created and passwords are set by the playbook, they end up hashed as `scram-sha-256` on Postgres v14+. # If an md5-hashed password is restored on top, Postgres v14+ will refuse to authenticate users with it by default. -matrix_postgres_import_roles_ignore_regex: "^(CREATE|ALTER) ROLE ({{ matrix_postgres_import_roles_to_ignore | join('|') }})(;| WITH)" # noqa jinja[spacing] +# +# We also allow for the role name to be quoted, which is rare, but might happen for role names which are special keywords (e.g. `default`). +matrix_postgres_import_roles_ignore_regex: "^(CREATE|ALTER) ROLE \\\"?({{ matrix_postgres_import_roles_to_ignore | join('|') }})\\\"?(;| WITH)" # noqa jinja[spacing] # A list of databases to avoid creating when importing (or upgrading) the database. # If a dump file contains the databases and they've also been created beforehand (see `matrix_postgres_additional_databases`), @@ -97,7 +99,8 @@ matrix_postgres_import_databases_to_ignore: | matrix_postgres_additional_databases|map(attribute='name') | list }} -matrix_postgres_import_databases_ignore_regex: "^CREATE DATABASE ({{ matrix_postgres_import_databases_to_ignore | join('|') }})\\s" # noqa jinja[spacing] +# We also allow for the database name to be quoted, which is rare, but might happen for database names which are special keywords (e.g. `default`). +matrix_postgres_import_databases_ignore_regex: "^CREATE DATABASE \\\"?({{ matrix_postgres_import_databases_to_ignore | join('|') }})\\\"?\\s" # noqa jinja[spacing] # The number of seconds to wait after starting `matrix-postgres.service` # and before trying to run queries for creating additional databases/users against it. From 7e8184e210798da376b2fd16baa7c730811dcbb4 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 21 Oct 2022 14:58:21 +0300 Subject: [PATCH 054/118] Do not break the guided installation flow --- docs/configuring-dns.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/configuring-dns.md b/docs/configuring-dns.md index ea5fea88f..3803ba8f6 100644 --- a/docs/configuring-dns.md +++ b/docs/configuring-dns.md @@ -26,6 +26,8 @@ Be mindful as to how long it will take for the DNS records to propagate. If you are using Cloudflare DNS, make sure to disable the proxy and set all records to `DNS only`. Otherwise, fetching certificates will fail. +When you're done configuring DNS, proceed to [Configuring the playbook](configuring-playbook.md). + ## DNS settings for optional services/features | Used by component | Type | Host | Priority | Weight | Port | Target | From 4cf85605a911e998409d9e2a1cb7a94d83854491 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 21 Oct 2022 17:36:24 +0300 Subject: [PATCH 055/118] Upgrade Dendrite (0.10.3 -> 0.10.4) --- roles/matrix-dendrite/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-dendrite/defaults/main.yml b/roles/matrix-dendrite/defaults/main.yml index 2adf13daa..dc0af2dc3 100644 --- a/roles/matrix-dendrite/defaults/main.yml +++ b/roles/matrix-dendrite/defaults/main.yml @@ -6,7 +6,7 @@ matrix_dendrite_enabled: true matrix_dendrite_docker_image: "{{ matrix_dendrite_docker_image_name_prefix }}matrixdotorg/dendrite-monolith:{{ matrix_dendrite_docker_image_tag }}" matrix_dendrite_docker_image_name_prefix: "docker.io/" -matrix_dendrite_docker_image_tag: "v0.10.3" +matrix_dendrite_docker_image_tag: "v0.10.4" matrix_dendrite_docker_image_force_pull: "{{ matrix_dendrite_docker_image.endswith(':latest') }}" matrix_dendrite_base_path: "{{ matrix_base_data_path }}/dendrite" From 463596884028dc38099d40155cc4117976bca8d6 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 21 Oct 2022 20:07:29 +0300 Subject: [PATCH 056/118] Upgrade Hookshot (2.3.0 -> 2.4.0) --- roles/matrix-bridge-hookshot/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bridge-hookshot/defaults/main.yml b/roles/matrix-bridge-hookshot/defaults/main.yml index 0854edbf5..1c390f340 100644 --- a/roles/matrix-bridge-hookshot/defaults/main.yml +++ b/roles/matrix-bridge-hookshot/defaults/main.yml @@ -10,7 +10,7 @@ matrix_hookshot_container_image_self_build: false matrix_hookshot_container_image_self_build_repo: "https://github.com/matrix-org/matrix-hookshot.git" matrix_hookshot_container_image_self_build_branch: "{{ 'main' if matrix_hookshot_version == 'latest' else matrix_hookshot_version }}" -matrix_hookshot_version: 2.3.0 +matrix_hookshot_version: 2.4.0 matrix_hookshot_docker_image: "{{ matrix_hookshot_docker_image_name_prefix }}halfshot/matrix-hookshot:{{ matrix_hookshot_version }}" matrix_hookshot_docker_image_name_prefix: "{{ 'localhost/' if matrix_hookshot_container_image_self_build else matrix_container_global_registry_prefix }}" From 7d043489bdc5533b188fe891692d1d85ec13ed93 Mon Sep 17 00:00:00 2001 From: smargold476 <105579587+smargold476@users.noreply.github.com> Date: Fri, 21 Oct 2022 21:51:54 +0200 Subject: [PATCH 057/118] update remote-repo-infos with link to doc was a litte confusing for me, so i added the link to the syntax-descr. --- docs/configuring-playbook-backup-borg.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuring-playbook-backup-borg.md b/docs/configuring-playbook-backup-borg.md index 41ca0156c..72ec9e678 100644 --- a/docs/configuring-playbook-backup-borg.md +++ b/docs/configuring-playbook-backup-borg.md @@ -56,7 +56,7 @@ where: * USER - SSH user of a provider/server * HOST - SSH host of a provider/server -* REPO - borg repository name, it will be initialized on backup start, eg: `matrix` +* REPO - borg repository name, it will be initialized on backup start, eg: `matrix`, regarding Syntax see [Remote repositories](https://borgbackup.readthedocs.io/en/stable/usage/general.html#repository-urls) * PASSPHRASE - passphrase used for encrypting backups, you may generate it with `pwgen -s 64 1` or use any password manager * PRIVATE KEY - the content of the **private** part of the SSH key you created before. The whole key (all of its belonging lines) under `matrix_backup_borg_ssh_key_private` needs to be indented with 2 spaces From ac8312d95f58d573488ef49907830cbdf876a62b Mon Sep 17 00:00:00 2001 From: smargold476 <105579587+smargold476@users.noreply.github.com> Date: Fri, 21 Oct 2022 22:19:03 +0200 Subject: [PATCH 058/118] update link to example in my environment i see the matrix. conf in that file: /matrix/nginx-proxy/conf.d/matrix-domain.conf --- docs/configuring-playbook-ssl-certificates.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuring-playbook-ssl-certificates.md b/docs/configuring-playbook-ssl-certificates.md index 30a8f0b87..606160da4 100644 --- a/docs/configuring-playbook-ssl-certificates.md +++ b/docs/configuring-playbook-ssl-certificates.md @@ -99,7 +99,7 @@ The certificate files would be made available in `/matrix/ssl/config/live/ Date: Sat, 22 Oct 2022 08:45:27 +0300 Subject: [PATCH 059/118] Make ntfy not try to listen on a privileged port We're starting it with `--user` and dropped capabilities, after all. Hopefully fixes https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/2188 --- roles/matrix-ntfy/templates/ntfy/server.yml.j2 | 1 + roles/matrix-ntfy/templates/systemd/matrix-ntfy.service.j2 | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/matrix-ntfy/templates/ntfy/server.yml.j2 b/roles/matrix-ntfy/templates/ntfy/server.yml.j2 index 4cafcd62a..096991a70 100644 --- a/roles/matrix-ntfy/templates/ntfy/server.yml.j2 +++ b/roles/matrix-ntfy/templates/ntfy/server.yml.j2 @@ -1,3 +1,4 @@ base_url: {{ matrix_ntfy_base_url }} behind_proxy: true cache_file: /data/cache.db +listen-http: :8080 diff --git a/roles/matrix-ntfy/templates/systemd/matrix-ntfy.service.j2 b/roles/matrix-ntfy/templates/systemd/matrix-ntfy.service.j2 index da292e5ce..f41598560 100644 --- a/roles/matrix-ntfy/templates/systemd/matrix-ntfy.service.j2 +++ b/roles/matrix-ntfy/templates/systemd/matrix-ntfy.service.j2 @@ -21,7 +21,7 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-ntfy \ {% endfor %} --network={{ matrix_docker_network }} \ {% if matrix_ntfy_container_http_host_bind_port %} - -p {{ matrix_ntfy_container_http_host_bind_port }}:80 \ + -p {{ matrix_ntfy_container_http_host_bind_port }}:8080 \ {% endif %} --mount type=bind,src={{ matrix_ntfy_config_dir_path }},dst=/etc/ntfy,ro \ --mount type=bind,src={{ matrix_ntfy_data_path }},dst=/data \ From 3bd0af76ddec6004e9e794fd821e0809123cc4a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian-Samuel=20Geb=C3=BChr?= Date: Sat, 22 Oct 2022 07:47:00 +0200 Subject: [PATCH 060/118] Add information on how to manually run the backup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Julian-Samuel Gebühr --- docs/configuring-playbook-backup-borg.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/configuring-playbook-backup-borg.md b/docs/configuring-playbook-backup-borg.md index 41ca0156c..0fda66da9 100644 --- a/docs/configuring-playbook-backup-borg.md +++ b/docs/configuring-playbook-backup-borg.md @@ -73,3 +73,9 @@ After configuring the playbook, run the [installation](installing.md) command ag ``` ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start ``` + +## Manually start a backup + +For testing your setup it can be helpful to not wait until 4am. If you want to run the backup immediately, log onto the server +and run `systemctl start matrix-backup-borg`. This will not return until the backup is done, so possibly a long time. +Consider using [tmux](https://en.wikipedia.org/wiki/Tmux) if your SSH connection is unstable. From 9fd3e00b71fdd450bf0643b30179f880e63d175c Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sun, 23 Oct 2022 07:45:54 +0300 Subject: [PATCH 061/118] Upgrade nginx (1.23.1 -> 1.23.2) --- roles/matrix-nginx-proxy/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-nginx-proxy/defaults/main.yml b/roles/matrix-nginx-proxy/defaults/main.yml index 84ec4f7e1..efef89af3 100644 --- a/roles/matrix-nginx-proxy/defaults/main.yml +++ b/roles/matrix-nginx-proxy/defaults/main.yml @@ -1,7 +1,7 @@ --- # Project source code URL: https://github.com/nginx/nginx matrix_nginx_proxy_enabled: true -matrix_nginx_proxy_version: 1.23.1-alpine +matrix_nginx_proxy_version: 1.23.2-alpine # We use an official nginx image, which we fix-up to run unprivileged. # An alternative would be an `nginxinc/nginx-unprivileged` image, but From 716efe7ad5923cdbdc7721fc58f0db38f588a391 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sun, 23 Oct 2022 09:52:02 +0300 Subject: [PATCH 062/118] Upgrade ddclient (v3.9.1-ls100 -> v3.10.0-ls102) --- roles/matrix-dynamic-dns/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-dynamic-dns/defaults/main.yml b/roles/matrix-dynamic-dns/defaults/main.yml index c465fef96..af4d95927 100644 --- a/roles/matrix-dynamic-dns/defaults/main.yml +++ b/roles/matrix-dynamic-dns/defaults/main.yml @@ -7,7 +7,7 @@ matrix_dynamic_dns_enabled: true # The dynamic dns daemon interval matrix_dynamic_dns_daemon_interval: '300' -matrix_dynamic_dns_version: v3.9.1-ls100 +matrix_dynamic_dns_version: v3.10.0-ls102 # The docker container to use when in mode matrix_dynamic_dns_docker_image: "{{ matrix_dynamic_dns_docker_image_name_prefix }}linuxserver/ddclient:{{ matrix_dynamic_dns_version }}" From d44b0378505ac5df4b9f6cafb93f46248234a5c4 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sun, 23 Oct 2022 18:45:22 +0300 Subject: [PATCH 063/118] fgrep -> grep -F --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f0aeb3971..576dcbf56 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ .PHONY: lint help: ## Show this help. - @fgrep -h "##" $(MAKEFILE_LIST) | fgrep -v fgrep | sed -e 's/\\$$//' | sed -e 's/##//' + @grep -F -h "##" $(MAKEFILE_LIST) | grep -v grep | sed -e 's/\\$$//' | sed -e 's/##//' lint: ## Runs ansible-lint against all roles in the playbook ansible-lint From 24409766eb5cfeadc1f668e077532f0b8b78091d Mon Sep 17 00:00:00 2001 From: Aine <97398200+etkecc@users.noreply.github.com> Date: Mon, 24 Oct 2022 05:42:18 +0000 Subject: [PATCH 064/118] Fix 502 ref: 57bb340343c692c9439c863cd1d13c54cecb87b8 --- .../templates/nginx/conf.d/matrix-ntfy.conf.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-ntfy.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-ntfy.conf.j2 index 988b3b355..ae100edac 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-ntfy.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-ntfy.conf.j2 @@ -21,7 +21,7 @@ {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} resolver 127.0.0.11 valid=5s; - set $backend "matrix-ntfy:80"; + set $backend "matrix-ntfy:8080"; proxy_pass http://$backend; {% else %} {# Generic configuration for use outside of our container setup #} From 9c549a185f6b035cb2979d0c41f9d7e2858037fc Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 25 Oct 2022 06:28:24 +0300 Subject: [PATCH 065/118] Auto-purge orphaned Let's Encrypt renewal configuration files --- roles/matrix-nginx-proxy/defaults/main.yml | 14 ++++++++++ ...urge_ssl_lets_encrypt_orphaned_configs.yml | 27 +++++++++++++++++++ .../tasks/ssl/setup_ssl_lets_encrypt.yml | 3 +++ 3 files changed, 44 insertions(+) create mode 100644 roles/matrix-nginx-proxy/tasks/ssl/purge_ssl_lets_encrypt_orphaned_configs.yml diff --git a/roles/matrix-nginx-proxy/defaults/main.yml b/roles/matrix-nginx-proxy/defaults/main.yml index efef89af3..c233dc43c 100644 --- a/roles/matrix-nginx-proxy/defaults/main.yml +++ b/roles/matrix-nginx-proxy/defaults/main.yml @@ -572,6 +572,20 @@ matrix_ssl_log_dir_path: "{{ matrix_ssl_base_path }}/log" matrix_ssl_pre_obtaining_required_service_name: ~ matrix_ssl_pre_obtaining_required_service_start_wait_time_seconds: 60 +# matrix_ssl_orphaned_renewal_configs_purging_enabled controls whether the playbook will delete Let's Encryption renewal configuration files (`/matrix/ssl/config/renewal/*.conf) +# for domains that are not part of the `matrix_ssl_domains_to_obtain_certificates_for` list. +# +# As the `matrix_ssl_domains_to_obtain_certificates_for` list changes over time, the playbook obtains certificates for various domains +# and sets up "renewal" configuration files to keep these certificates fresh. +# When a domain disappears from the `matrix_ssl_domains_to_obtain_certificates_for` list (because its associated service had gotten disabled), +# the certificate files and renewal configuration still remain in the filesystem and certbot may try to renewal the certificate for this domain. +# If there's no DNS record for this domain or it doesn't point to this server anymore, the `matrix-ssl-lets-encrypt-certificates-renew.service` systemd service +# won't be able to renew the certificate and will generate an error. +# +# With `matrix_ssl_orphaned_renewal_configs_purging_enabled` enabled, orphaned renewal configurations will be purged on each playbook run. +# Some other leftover files will still remain, but we don't bother purging them because they don't cause troubles. +matrix_ssl_orphaned_renewal_configs_purging_enabled: true + # Nginx Optimize SSL Session # # ssl_session_cache: diff --git a/roles/matrix-nginx-proxy/tasks/ssl/purge_ssl_lets_encrypt_orphaned_configs.yml b/roles/matrix-nginx-proxy/tasks/ssl/purge_ssl_lets_encrypt_orphaned_configs.yml new file mode 100644 index 000000000..51fd1f314 --- /dev/null +++ b/roles/matrix-nginx-proxy/tasks/ssl/purge_ssl_lets_encrypt_orphaned_configs.yml @@ -0,0 +1,27 @@ +--- + +- name: Check if a Let's Encrypt renewal configuration directory exists + ansible.builtin.stat: + path: "{{ matrix_ssl_config_dir_path }}/renewal" + register: matrix_ssl_config_renewal_directory_stat_result + +- when: matrix_ssl_config_renewal_directory_stat_result.stat.exists | bool + block: + - name: Determine current Let's Encrypt renewal configs + ansible.builtin.find: + path: "{{ matrix_ssl_config_dir_path }}/renewal" + patterns: ".*.conf$" + use_regex: true + register: matrix_ssl_current_renewal_config_files + + - name: Determine unnecessary Let's Encrypt renewal configs + ansible.builtin.set_fact: + matrix_ssl_current_renewal_config_files_to_purge: "{{ matrix_ssl_current_renewal_config_files_to_purge | default([]) + [item.path] }}" + with_items: "{{ matrix_ssl_current_renewal_config_files.files }}" + when: "item.path | basename | replace('.conf', '') not in matrix_ssl_domains_to_obtain_certificates_for" + + - name: Purge unneceessary Let's Encrypt renewal config files + ansible.builtin.file: + path: "{{ item }}" + state: absent + with_items: "{{ matrix_ssl_current_renewal_config_files_to_purge | default([]) }}" diff --git a/roles/matrix-nginx-proxy/tasks/ssl/setup_ssl_lets_encrypt.yml b/roles/matrix-nginx-proxy/tasks/ssl/setup_ssl_lets_encrypt.yml index 029ef860d..62430f4bb 100644 --- a/roles/matrix-nginx-proxy/tasks/ssl/setup_ssl_lets_encrypt.yml +++ b/roles/matrix-nginx-proxy/tasks/ssl/setup_ssl_lets_encrypt.yml @@ -18,6 +18,9 @@ - when: "matrix_ssl_retrieval_method == 'lets-encrypt'" block: + - when: matrix_ssl_orphaned_renewal_configs_purging_enabled | bool + ansible.builtin.import_tasks: "{{ role_path }}/tasks/ssl/purge_ssl_lets_encrypt_orphaned_configs.yml" + - name: Ensure certbot Docker image is pulled docker_image: name: "{{ matrix_ssl_lets_encrypt_certbot_docker_image }}" From cb2fecbea96ead3e9003c22bcabda00e20dd3b01 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 25 Oct 2022 06:43:57 +0300 Subject: [PATCH 066/118] Fix some ansible-lint-reported warnings --- roles/matrix-base/tasks/server_base/setup.yml | 3 ++- .../tasks/validate_config.yml | 3 ++- roles/matrix-ldap-registration-proxy/tasks/init.yml | 9 ++++----- roles/matrix-postgres/tasks/import_generic_sqlite_db.yml | 3 ++- roles/matrix-postgres/tasks/run_vacuum.yml | 3 ++- .../tasks/rust-synapse-compress-state/compress_room.yml | 3 ++- .../tasks/rust-synapse-compress-state/main.yml | 3 ++- 7 files changed, 16 insertions(+), 11 deletions(-) diff --git a/roles/matrix-base/tasks/server_base/setup.yml b/roles/matrix-base/tasks/server_base/setup.yml index 8cc4dff4c..9e3319f5f 100644 --- a/roles/matrix-base/tasks/server_base/setup.yml +++ b/roles/matrix-base/tasks/server_base/setup.yml @@ -21,7 +21,8 @@ register: lsb_release_installation_result - name: Reread ansible_lsb facts if lsb-release got installed - ansible.builtin.setup: filter=ansible_lsb* + ansible.builtin.setup: + filter: ansible_lsb* when: lsb_release_installation_result.changed - ansible.builtin.include_tasks: "{{ role_path }}/tasks/server_base/setup_debian.yml" diff --git a/roles/matrix-bridge-appservice-discord/tasks/validate_config.yml b/roles/matrix-bridge-appservice-discord/tasks/validate_config.yml index 901b760bf..e005f162b 100644 --- a/roles/matrix-bridge-appservice-discord/tasks/validate_config.yml +++ b/roles/matrix-bridge-appservice-discord/tasks/validate_config.yml @@ -22,5 +22,6 @@ - {'old': 'matrix_appservice_discord_container_expose_client_server_api_port', 'new': ''} - name: Require a valid database engine - ansible.builtin.fail: msg="`matrix_appservice_discord_database_engine` needs to be either 'sqlite' or 'postgres'" + ansible.builtin.fail: + msg: "`matrix_appservice_discord_database_engine` needs to be either 'sqlite' or 'postgres'" when: "matrix_appservice_discord_database_engine not in ['sqlite', 'postgres']" diff --git a/roles/matrix-ldap-registration-proxy/tasks/init.yml b/roles/matrix-ldap-registration-proxy/tasks/init.yml index 0b2051c3d..406236095 100644 --- a/roles/matrix-ldap-registration-proxy/tasks/init.yml +++ b/roles/matrix-ldap-registration-proxy/tasks/init.yml @@ -10,7 +10,10 @@ matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-ldap-registration-proxy.service'] }}" when: matrix_ldap_registration_proxy_enabled | bool -- block: +- when: matrix_ldap_registration_proxy_enabled | bool + tags: + - always + block: - name: Fail if matrix-nginx-proxy role already executed ansible.builtin.fail: msg: >- @@ -52,7 +55,3 @@ URL endpoint to the matrix-ldap-proxy container. You can expose the container's port using the `matrix_ldap_registration_proxy_container_http_host_bind_port` variable. when: "not matrix_nginx_proxy_enabled | default(False) | bool" - - tags: - - always - when: matrix_ldap_registration_proxy_enabled | bool diff --git a/roles/matrix-postgres/tasks/import_generic_sqlite_db.yml b/roles/matrix-postgres/tasks/import_generic_sqlite_db.yml index 897ebc2e9..4a515c27d 100644 --- a/roles/matrix-postgres/tasks/import_generic_sqlite_db.yml +++ b/roles/matrix-postgres/tasks/import_generic_sqlite_db.yml @@ -28,7 +28,8 @@ - when: 'postgres_connection_string_variable_name is defined' block: - name: Fail if postgres_connection_string_variable_name points to an undefined variable - ansible.builtin.fail: msg="postgres_connection_string_variable_name is defined, but there is no variable with the name `{{ postgres_connection_string_variable_name }}`" + ansible.builtin.fail: + msg: "postgres_connection_string_variable_name is defined, but there is no variable with the name `{{ postgres_connection_string_variable_name }}`" when: "postgres_connection_string_variable_name not in vars" - name: Get Postgres connection string from variable diff --git a/roles/matrix-postgres/tasks/run_vacuum.yml b/roles/matrix-postgres/tasks/run_vacuum.yml index aafa761db..439599828 100644 --- a/roles/matrix-postgres/tasks/run_vacuum.yml +++ b/roles/matrix-postgres/tasks/run_vacuum.yml @@ -82,7 +82,8 @@ changed_when: matrix_postgres_synapse_vacuum_result.finished and matrix_postgres_synapse_vacuum_result.rc == 0 # Intentionally show the results -- ansible.builtin.debug: var="matrix_postgres_synapse_vacuum_result" +- ansible.builtin.debug: + var: "matrix_postgres_synapse_vacuum_result" - name: Ensure matrix-synapse is started, if it previously was ansible.builtin.service: diff --git a/roles/matrix-synapse/tasks/rust-synapse-compress-state/compress_room.yml b/roles/matrix-synapse/tasks/rust-synapse-compress-state/compress_room.yml index 6ae016fcc..9834e256a 100644 --- a/roles/matrix-synapse/tasks/rust-synapse-compress-state/compress_room.yml +++ b/roles/matrix-synapse/tasks/rust-synapse-compress-state/compress_room.yml @@ -24,7 +24,8 @@ failed_when: not matrix_synapse_rust_synapse_compress_state_compress_room_command_result.finished changed_when: matrix_synapse_rust_synapse_compress_state_compress_room_command_result.finished and matrix_synapse_rust_synapse_compress_state_compress_room_command_result.rc == 0 -- ansible.builtin.debug: var="matrix_synapse_rust_synapse_compress_state_compress_room_command_result" +- ansible.builtin.debug: + var: "matrix_synapse_rust_synapse_compress_state_compress_room_command_result" - name: Generate Postgres compression SQL import command ansible.builtin.set_fact: diff --git a/roles/matrix-synapse/tasks/rust-synapse-compress-state/main.yml b/roles/matrix-synapse/tasks/rust-synapse-compress-state/main.yml index b48e60775..80c25f7fa 100644 --- a/roles/matrix-synapse/tasks/rust-synapse-compress-state/main.yml +++ b/roles/matrix-synapse/tasks/rust-synapse-compress-state/main.yml @@ -88,7 +88,8 @@ - when: "matrix_synapse_rust_synapse_compress_state_find_rooms_command_result.failed or matrix_synapse_rust_synapse_compress_state_find_rooms_command_result.stdout_lines | length != 4" block: - - ansible.builtin.debug: var="matrix_synapse_rust_synapse_compress_state_find_rooms_command_result" + - ansible.builtin.debug: + var: "matrix_synapse_rust_synapse_compress_state_find_rooms_command_result" - name: Fail if room find result is not what we expect ansible.builtin.fail: From fd7e1604a5880242b67d92fa1d8df4180fc53c6f Mon Sep 17 00:00:00 2001 From: Aine <97398200+etkecc@users.noreply.github.com> Date: Tue, 25 Oct 2022 09:16:24 +0000 Subject: [PATCH 067/118] Update appservice-irc 0.35.1 -> 0.36.0 --- roles/matrix-bridge-appservice-irc/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bridge-appservice-irc/defaults/main.yml b/roles/matrix-bridge-appservice-irc/defaults/main.yml index c4fa75fe4..d54a7685d 100644 --- a/roles/matrix-bridge-appservice-irc/defaults/main.yml +++ b/roles/matrix-bridge-appservice-irc/defaults/main.yml @@ -11,7 +11,7 @@ matrix_appservice_irc_docker_src_files_path: "{{ matrix_base_data_path }}/appser # matrix_appservice_irc_version used to contain the full Docker image tag (e.g. `release-X.X.X`). # It's a bare version number now. We try to somewhat retain compatibility below. -matrix_appservice_irc_version: 0.35.1 +matrix_appservice_irc_version: 0.36.0 matrix_appservice_irc_docker_image: "{{ matrix_container_global_registry_prefix }}matrixdotorg/matrix-appservice-irc:{{ matrix_appservice_irc_docker_image_tag }}" matrix_appservice_irc_docker_image_tag: "{{ 'latest' if matrix_appservice_irc_version == 'latest' else ('release-' + matrix_appservice_irc_version) }}" matrix_appservice_irc_docker_image_force_pull: "{{ matrix_appservice_irc_docker_image.endswith(':latest') }}" From 81d8785811bd3aca8a6334910e2143edda8a3dda Mon Sep 17 00:00:00 2001 From: Aine <97398200+etkecc@users.noreply.github.com> Date: Tue, 25 Oct 2022 13:03:50 +0000 Subject: [PATCH 068/118] Update grafana 9.2.1 -> 9.2.2 --- roles/matrix-grafana/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-grafana/defaults/main.yml b/roles/matrix-grafana/defaults/main.yml index 767a749bc..5bccb60d2 100644 --- a/roles/matrix-grafana/defaults/main.yml +++ b/roles/matrix-grafana/defaults/main.yml @@ -5,7 +5,7 @@ matrix_grafana_enabled: true -matrix_grafana_version: 9.2.1 +matrix_grafana_version: 9.2.2 matrix_grafana_docker_image: "{{ matrix_container_global_registry_prefix }}grafana/grafana:{{ matrix_grafana_version }}" matrix_grafana_docker_image_force_pull: "{{ matrix_grafana_docker_image.endswith(':latest') }}" From 145a57a7b8ad1d01a3a05c28abe6586078611418 Mon Sep 17 00:00:00 2001 From: Aine Date: Tue, 25 Oct 2022 18:25:07 +0300 Subject: [PATCH 069/118] update honoroit 0.9.15 -> 0.9.16 --- group_vars/matrix_servers | 2 +- roles/matrix-bot-honoroit/defaults/main.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 6728eaaae..e31e0dba3 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -1163,7 +1163,7 @@ matrix_bot_honoroit_systemd_required_services_list: | # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_bot_honoroit_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" matrix_bot_honoroit_database_password: "{{ '%s' | format(matrix_homeserver_generic_secret_key) | password_hash('sha512', 'honoroit.bot.db') | to_uuid }}" -matrix_bot_honoroit_container_image_self_build: "{{ matrix_architecture not in ['amd64', 'arm32', 'arm64'] }}" +matrix_bot_honoroit_container_image_self_build: "{{ matrix_architecture not in ['amd64', 'arm64'] }}" ###################################################################### # diff --git a/roles/matrix-bot-honoroit/defaults/main.yml b/roles/matrix-bot-honoroit/defaults/main.yml index 68fb8c177..3510f4732 100644 --- a/roles/matrix-bot-honoroit/defaults/main.yml +++ b/roles/matrix-bot-honoroit/defaults/main.yml @@ -9,7 +9,7 @@ matrix_bot_honoroit_docker_repo: "https://gitlab.com/etke.cc/honoroit.git" matrix_bot_honoroit_docker_repo_version: "{{ matrix_bot_honoroit_version }}" matrix_bot_honoroit_docker_src_files_path: "{{ matrix_base_data_path }}/honoroit/docker-src" -matrix_bot_honoroit_version: v0.9.15 +matrix_bot_honoroit_version: v0.9.16 matrix_bot_honoroit_docker_image: "{{ matrix_bot_honoroit_docker_image_name_prefix }}honoroit:{{ matrix_bot_honoroit_version }}" matrix_bot_honoroit_docker_image_name_prefix: "{{ 'localhost/' if matrix_bot_honoroit_container_image_self_build else 'registry.gitlab.com/etke.cc/' }}" matrix_bot_honoroit_docker_image_force_pull: "{{ matrix_bot_honoroit_docker_image.endswith(':latest') }}" From a22d5b1726a207f6349e47890684759a14e1e697 Mon Sep 17 00:00:00 2001 From: Aine Date: Tue, 25 Oct 2022 18:26:57 +0300 Subject: [PATCH 070/118] update postmoogle 0.9.7 -> 0.9.8 --- group_vars/matrix_servers | 2 +- roles/matrix-bot-postmoogle/defaults/main.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 6728eaaae..354b66913 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -1227,7 +1227,7 @@ matrix_bot_postmoogle_systemd_required_services_list: | matrix_bot_postmoogle_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" matrix_bot_postmoogle_database_password: "{{ '%s' | format(matrix_homeserver_generic_secret_key) | password_hash('sha512', 'postmoogle.db') | to_uuid }}" -matrix_bot_postmoogle_container_image_self_build: "{{ matrix_architecture not in ['amd64', 'arm32', 'arm64'] }}" +matrix_bot_postmoogle_container_image_self_build: "{{ matrix_architecture not in ['amd64', 'arm64'] }}" ###################################################################### # diff --git a/roles/matrix-bot-postmoogle/defaults/main.yml b/roles/matrix-bot-postmoogle/defaults/main.yml index 1bcd570a5..af6c23ac0 100644 --- a/roles/matrix-bot-postmoogle/defaults/main.yml +++ b/roles/matrix-bot-postmoogle/defaults/main.yml @@ -9,7 +9,7 @@ matrix_bot_postmoogle_docker_repo: "https://gitlab.com/etke.cc/postmoogle.git" matrix_bot_postmoogle_docker_repo_version: "{{ 'main' if matrix_bot_postmoogle_version == 'latest' else matrix_bot_postmoogle_version }}" matrix_bot_postmoogle_docker_src_files_path: "{{ matrix_base_data_path }}/postmoogle/docker-src" -matrix_bot_postmoogle_version: v0.9.7 +matrix_bot_postmoogle_version: v0.9.8 matrix_bot_postmoogle_docker_image: "{{ matrix_bot_postmoogle_docker_image_name_prefix }}postmoogle:{{ matrix_bot_postmoogle_version }}" matrix_bot_postmoogle_docker_image_name_prefix: "{{ 'localhost/' if matrix_bot_postmoogle_container_image_self_build else 'registry.gitlab.com/etke.cc/' }}" matrix_bot_postmoogle_docker_image_force_pull: "{{ matrix_bot_postmoogle_docker_image.endswith(':latest') }}" From 12fe1f417c2816939755bfc5e1e61a376b59ea90 Mon Sep 17 00:00:00 2001 From: Aine Date: Tue, 25 Oct 2022 18:39:39 +0300 Subject: [PATCH 071/118] update buscarron 1.2.1 -> 1.3.0 --- docs/configuring-playbook-bot-buscarron.md | 4 +-- group_vars/matrix_servers | 2 +- roles/matrix-bot-buscarron/defaults/main.yml | 32 ++++++++++++++------ roles/matrix-bot-buscarron/templates/env.j2 | 6 +++- 4 files changed, 30 insertions(+), 14 deletions(-) diff --git a/docs/configuring-playbook-bot-buscarron.md b/docs/configuring-playbook-bot-buscarron.md index 3a5822abd..b38f25c67 100644 --- a/docs/configuring-playbook-bot-buscarron.md +++ b/docs/configuring-playbook-bot-buscarron.md @@ -36,10 +36,10 @@ matrix_bot_buscarron_forms: room: "!yourRoomID:DOMAIN" # (mandatory) Room ID where form submission will be posted redirect: https://DOMAIN # (mandatory) To what page user will be redirected after the form submission ratelimit: 1r/m # (optional) rate limit of the form, format: r/, eg: 1r/s or 54r/m + hasemail: 1 # (optional) form has "email" field that should be validated extensions: [] # (optional) list of form extensions (not used yet) -matrix_bot_buscarron_spam_hosts: [] # (optional) list of email domains/hosts that should be rejected automatically -matrix_bot_buscarron_spam_emails: [] # (optional) list of email addresses that should be rejected automatically +matrix_bot_buscarron_spamlist: [] # (optional) list of emails/domains/hosts (with wildcards support) that should be rejected automatically ``` You will also need to add a DNS record so that buscarron can be accessed. diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 6728eaaae..5d463da2b 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -1194,7 +1194,7 @@ matrix_bot_buscarron_systemd_required_services_list: | # Postgres is the default, except if not using `matrix_postgres` (internal postgres) matrix_bot_buscarron_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" matrix_bot_buscarron_database_password: "{{ '%s' | format(matrix_homeserver_generic_secret_key) | password_hash('sha512', 'buscarron.bot.db') | to_uuid }}" -matrix_bot_buscarron_container_image_self_build: "{{ matrix_architecture not in ['amd64', 'arm32', 'arm64'] }}" +matrix_bot_buscarron_container_image_self_build: "{{ matrix_architecture not in ['amd64', 'arm64'] }}" ###################################################################### # diff --git a/roles/matrix-bot-buscarron/defaults/main.yml b/roles/matrix-bot-buscarron/defaults/main.yml index 648d53447..21d9a4a39 100644 --- a/roles/matrix-bot-buscarron/defaults/main.yml +++ b/roles/matrix-bot-buscarron/defaults/main.yml @@ -9,7 +9,7 @@ matrix_bot_buscarron_docker_repo: "https://gitlab.com/etke.cc/buscarron.git" matrix_bot_buscarron_docker_repo_version: "{{ matrix_bot_buscarron_version }}" matrix_bot_buscarron_docker_src_files_path: "{{ matrix_base_data_path }}/buscarron/docker-src" -matrix_bot_buscarron_version: v1.2.1 +matrix_bot_buscarron_version: v1.3.0 matrix_bot_buscarron_docker_image: "{{ matrix_bot_buscarron_docker_image_name_prefix }}buscarron:{{ matrix_bot_buscarron_version }}" matrix_bot_buscarron_docker_image_name_prefix: "{{ 'localhost/' if matrix_bot_buscarron_container_image_self_build else 'registry.gitlab.com/etke.cc/' }}" matrix_bot_buscarron_docker_image_force_pull: "{{ matrix_bot_buscarron_docker_image.endswith(':latest') }}" @@ -78,37 +78,49 @@ matrix_bot_buscarron_homeserver: "{{ matrix_homeserver_container_url }}" matrix_bot_buscarron_forms: [] # Disable encryption -matrix_bot_buscarron_noencryption: +matrix_bot_buscarron_noencryption: false # Sentry DSN -matrix_bot_buscarron_sentry: +matrix_bot_buscarron_sentry: '' # Log level matrix_bot_buscarron_loglevel: INFO -# spam hosts/domains +# list of spammers with wildcards support, eg: *@spam.com spam@*, spam@spam.com +matrix_bot_buscarron_spamlist: [] + +# spam hosts/domains. +# deprecated, use matrix_bot_buscarron_spamlist matrix_bot_buscarron_spam_hosts: [] # spam email addresses +# deprecated, use matrix_bot_buscarron_spamlist matrix_bot_buscarron_spam_emails: [] # spam email localparts +# deprecated, use matrix_bot_buscarron_spamlist matrix_bot_buscarron_spam_localparts: [] -# Ban duration in hours -matrix_bot_buscarron_ban_duration: 24 - # Banlist size matrix_bot_buscarron_ban_size: 10000 +# Permanent banlist +matrix_bot_buscarron_ban_list: [] + # Postmark token (confirmation emails) -matrix_bot_buscarron_pm_token: +matrix_bot_buscarron_pm_token: '' # Postmark sender signature -matrix_bot_buscarron_pm_from: +matrix_bot_buscarron_pm_from: '' # Postmark confirmation email's reply-to -matrix_bot_buscarron_pm_replyto: +matrix_bot_buscarron_pm_replyto: '' + +# email address (from) for SMTP validation. Must be valid email on valid SMTP server, otherwise it will be rejected by other servers +matrix_bot_buscarron_smtp_from: '' + +# enforce SMTP validation +matrix_bot_buscarron_smtp_validation: false # Additional environment variables to pass to the buscarron container # diff --git a/roles/matrix-bot-buscarron/templates/env.j2 b/roles/matrix-bot-buscarron/templates/env.j2 index 42da0d349..90ae5f7e8 100644 --- a/roles/matrix-bot-buscarron/templates/env.j2 +++ b/roles/matrix-bot-buscarron/templates/env.j2 @@ -3,22 +3,26 @@ BUSCARRON_PASSWORD={{ matrix_bot_buscarron_password }} BUSCARRON_HOMESERVER={{ matrix_bot_buscarron_homeserver }} BUSCARRON_DB_DSN={{ matrix_bot_buscarron_database_connection_string }} BUSCARRON_DB_DIALECT={{ matrix_bot_buscarron_database_dialect }} +BUSCARRON_SPAMLIST={{ matrix_bot_buscarron_spamlist|join(" ") }} BUSCARRON_SPAM_HOSTS={{ matrix_bot_buscarron_spam_hosts|join(" ") }} BUSCARRON_SPAM_EMAILS={{ matrix_bot_buscarron_spam_emails|join(" ") }} BUSCARRON_SPAM_LOCALPARTS={{ matrix_bot_buscarron_spam_localparts|join(" ") }} BUSCARRON_SENTRY={{ matrix_bot_buscarron_sentry }} BUSCARRON_LOGLEVEL={{ matrix_bot_buscarron_loglevel }} -BUSCARRON_BAN_DURATION={{ matrix_bot_buscarron_ban_duration }} BUSCARRON_BAN_SIZE={{ matrix_bot_buscarron_ban_size }} +BUSCARRON_BAN_LIST={{ matrix_bot_buscarron_ban_list }} BUSCARRON_PM_TOKEN={{ matrix_bot_buscarron_pm_token }} BUSCARRON_PM_FROM={{ matrix_bot_buscarron_pm_from }} BUSCARRON_PM_REPLYTO={{ matrix_bot_buscarron_pm_replyto }} +BUSCARRON_SMTP_FROM={{ matrix_bot_buscarron_smtp_from }} +BUSCARRON_SMTP_VALIDATION={{ matrix_bot_buscarron_smtp_validation }} BUSCARRON_NOENCRYPTION={{ matrix_bot_buscarron_noencryption }} {% set forms = [] %} {% for form in matrix_bot_buscarron_forms -%}{{- forms.append(form.name) -}} BUSCARRON_{{ form.name|upper }}_ROOM={{ form.room|default('') }} BUSCARRON_{{ form.name|upper }}_REDIRECT={{ form.redirect|default('') }} BUSCARRON_{{ form.name|upper }}_HASDOMAIN={{ form.hasdomain|default('') }} +BUSCARRON_{{ form.name|upper }}_HASEMAIL={{ form.hasemail|default('') }} BUSCARRON_{{ form.name|upper }}_RATELIMIT={{ form.ratelimit|default('') }} BUSCARRON_{{ form.name|upper }}_EXTENSIONS={{ form.extensions|default('')|join(' ') }} BUSCARRON_{{ form.name|upper }}_CONFIRMATION_SUBJECT={{ form.confirmation_subject|default('') }} From 437d177d3133827379d303bba452480bd30d3ab9 Mon Sep 17 00:00:00 2001 From: Aine <97398200+etkecc@users.noreply.github.com> Date: Tue, 25 Oct 2022 18:17:47 +0000 Subject: [PATCH 072/118] update element v1.11.10 -> v1.11.11 --- roles/matrix-client-element/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-client-element/defaults/main.yml b/roles/matrix-client-element/defaults/main.yml index c8e476a19..7204aa107 100644 --- a/roles/matrix-client-element/defaults/main.yml +++ b/roles/matrix-client-element/defaults/main.yml @@ -10,7 +10,7 @@ matrix_client_element_container_image_self_build_repo: "https://github.com/vecto # - https://github.com/vector-im/element-web/issues/19544 matrix_client_element_container_image_self_build_low_memory_system_patch_enabled: "{{ ansible_memtotal_mb < 4096 }}" -matrix_client_element_version: v1.11.10 +matrix_client_element_version: v1.11.11 matrix_client_element_docker_image: "{{ matrix_client_element_docker_image_name_prefix }}vectorim/element-web:{{ matrix_client_element_version }}" matrix_client_element_docker_image_name_prefix: "{{ 'localhost/' if matrix_client_element_container_image_self_build else matrix_container_global_registry_prefix }}" matrix_client_element_docker_image_force_pull: "{{ matrix_client_element_docker_image.endswith(':latest') }}" From 9a439f91ae5b412756c51bb3dbf55643cd8127a9 Mon Sep 17 00:00:00 2001 From: Array in a Matrix Date: Wed, 26 Oct 2022 12:58:09 -0400 Subject: [PATCH 073/118] update way to login to discord --- docs/configuring-playbook-bridge-mautrix-discord.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/configuring-playbook-bridge-mautrix-discord.md b/docs/configuring-playbook-bridge-mautrix-discord.md index 9fbf14240..065609b25 100644 --- a/docs/configuring-playbook-bridge-mautrix-discord.md +++ b/docs/configuring-playbook-bridge-mautrix-discord.md @@ -12,7 +12,7 @@ See the project's [documentation](https://docs.mau.fi/bridges/go/discord/index.h ## Prerequisites -For using this bridge, you would **need to authenticate by scanning a QR code with the Discord app on your phone**. +For using this bridge, you would need to authenticate by **scanning a QR code** with the Discord app on your phone **or** by using **discord account token**. You can delete the Discord app after the authentication process. @@ -70,7 +70,7 @@ When using this method, **each user** that wishes to enable Double Puppeting nee ## Usage 1. Start a chat with `@discordbot:YOUR_DOMAIN` (where `YOUR_DOMAIN` is your base domain, not the `matrix.` domain). -2. Send a `login` command +2. If you would like to login to Discord using a token, send `login-token` command, otherwise, send `login-qr` command. 3. You'll see a QR code which you need to scan with the Discord app on your phone. You can scan it with the camera app too, which will open Discord, which will then instruct you to scan it a 2nd time in the Discord app. 4. After confirming (in the Discord app) that you'd like to allow this login, the bot should respond with "Succcessfully authenticated as ..." 5. Now that you're logged in, you can send a `help` command to the bot again, to see additional commands you have access to From 170960be1aafcdf2b094322f8c9f268b744a29bf Mon Sep 17 00:00:00 2001 From: Joe Kappus Date: Wed, 26 Oct 2022 18:47:09 -0400 Subject: [PATCH 074/118] update element v1.11.11 -> v1.11.12 fixes jitsi issue --- roles/matrix-client-element/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-client-element/defaults/main.yml b/roles/matrix-client-element/defaults/main.yml index 7204aa107..2d5c99d2b 100644 --- a/roles/matrix-client-element/defaults/main.yml +++ b/roles/matrix-client-element/defaults/main.yml @@ -10,7 +10,7 @@ matrix_client_element_container_image_self_build_repo: "https://github.com/vecto # - https://github.com/vector-im/element-web/issues/19544 matrix_client_element_container_image_self_build_low_memory_system_patch_enabled: "{{ ansible_memtotal_mb < 4096 }}" -matrix_client_element_version: v1.11.11 +matrix_client_element_version: v1.11.12 matrix_client_element_docker_image: "{{ matrix_client_element_docker_image_name_prefix }}vectorim/element-web:{{ matrix_client_element_version }}" matrix_client_element_docker_image_name_prefix: "{{ 'localhost/' if matrix_client_element_container_image_self_build else matrix_container_global_registry_prefix }}" matrix_client_element_docker_image_force_pull: "{{ matrix_client_element_docker_image.endswith(':latest') }}" From e5e5ee4f7237c8047abfe85fc2aa0e25eb38edb2 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 27 Oct 2022 06:45:37 +0300 Subject: [PATCH 075/118] Use fixed image tags for prodrigestivill/postgres-backup-local At some point, we can drop v9.6 support and use a single variable that holds this value. Fixes https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/2191 --- roles/matrix-postgres-backup/defaults/main.yml | 16 ++++++++-------- .../tasks/validate_config.yml | 11 +++++++++++ 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/roles/matrix-postgres-backup/defaults/main.yml b/roles/matrix-postgres-backup/defaults/main.yml index ed42266f2..d59c84e95 100644 --- a/roles/matrix-postgres-backup/defaults/main.yml +++ b/roles/matrix-postgres-backup/defaults/main.yml @@ -24,18 +24,18 @@ matrix_postgres_backup_postgres_data_path: "" matrix_postgres_backup_architecture: amd64 -# matrix_postgres_docker_image_suffix controls whether we use Alpine-based images (`-alpine`) or the normal Debian-based images. +# matrix_postgres_backup_docker_image_distro controls whether we use Alpine-based images (`-alpine`) or the normal Debian-based images. # Alpine-based Postgres images are smaller and we usually prefer them, but they don't work on ARM32 (tested on a Raspberry Pi 3 running Raspbian 10.7). # On ARM32, `-alpine` images fail with the following error: # > LOG: startup process (PID 37) was terminated by signal 11: Segmentation fault -matrix_postgres_backup_docker_image_suffix: "{{ '-alpine' if matrix_postgres_backup_architecture in ['amd64', 'arm64'] else '' }}" +matrix_postgres_backup_docker_image_distro: "{{ 'alpine' if matrix_postgres_backup_architecture in ['amd64', 'arm64'] else 'debian' }}" -matrix_postgres_backup_docker_image_v9: "{{ matrix_container_global_registry_prefix }}prodrigestivill/postgres-backup-local:9.6{{ matrix_postgres_backup_docker_image_suffix }}" -matrix_postgres_backup_docker_image_v10: "{{ matrix_container_global_registry_prefix }}prodrigestivill/postgres-backup-local:10{{ matrix_postgres_backup_docker_image_suffix }}" -matrix_postgres_backup_docker_image_v11: "{{ matrix_container_global_registry_prefix }}prodrigestivill/postgres-backup-local:11{{ matrix_postgres_backup_docker_image_suffix }}" -matrix_postgres_backup_docker_image_v12: "{{ matrix_container_global_registry_prefix }}prodrigestivill/postgres-backup-local:12{{ matrix_postgres_backup_docker_image_suffix }}" -matrix_postgres_backup_docker_image_v13: "{{ matrix_container_global_registry_prefix }}prodrigestivill/postgres-backup-local:13{{ matrix_postgres_backup_docker_image_suffix }}" -matrix_postgres_backup_docker_image_v14: "{{ matrix_container_global_registry_prefix }}prodrigestivill/postgres-backup-local:14{{ matrix_postgres_backup_docker_image_suffix }}" +matrix_postgres_backup_docker_image_v9: "{{ matrix_container_global_registry_prefix }}prodrigestivill/postgres-backup-local:9.6-{{ matrix_postgres_backup_docker_image_distro }}-2aa03d1" +matrix_postgres_backup_docker_image_v10: "{{ matrix_container_global_registry_prefix }}prodrigestivill/postgres-backup-local:10-{{ matrix_postgres_backup_docker_image_distro }}-2cf00a5" +matrix_postgres_backup_docker_image_v11: "{{ matrix_container_global_registry_prefix }}prodrigestivill/postgres-backup-local:11-{{ matrix_postgres_backup_docker_image_distro }}-2cf00a5" +matrix_postgres_backup_docker_image_v12: "{{ matrix_container_global_registry_prefix }}prodrigestivill/postgres-backup-local:12-{{ matrix_postgres_backup_docker_image_distro }}-2cf00a5" +matrix_postgres_backup_docker_image_v13: "{{ matrix_container_global_registry_prefix }}prodrigestivill/postgres-backup-local:13-{{ matrix_postgres_backup_docker_image_distro }}-2cf00a5" +matrix_postgres_backup_docker_image_v14: "{{ matrix_container_global_registry_prefix }}prodrigestivill/postgres-backup-local:14-{{ matrix_postgres_backup_docker_image_distro }}-2cf00a5" matrix_postgres_backup_docker_image_latest: "{{ matrix_postgres_backup_docker_image_v14 }}" # This variable is assigned at runtime. Overriding its value has no effect. diff --git a/roles/matrix-postgres-backup/tasks/validate_config.yml b/roles/matrix-postgres-backup/tasks/validate_config.yml index 8a2ddb5ad..2ba643748 100644 --- a/roles/matrix-postgres-backup/tasks/validate_config.yml +++ b/roles/matrix-postgres-backup/tasks/validate_config.yml @@ -16,3 +16,14 @@ - "matrix_postgres_backup_keep_months" - "matrix_postgres_backup_path" - "matrix_postgres_backup_databases" + +- name: (Deprecation) Catch and report renamed settings + ansible.builtin.fail: + msg: >- + Your configuration contains a variable, which now has a different name. + Please change your configuration to rename the variable (`{{ item.old }}` -> `{{ item.new }}`). + when: "item.old in vars" + with_items: + - {'old': 'matrix_postgres_backup_docker_image_suffix', 'new': 'matrix_postgres_backup_docker_image_distro'} + + From 6cacf7297d7dc88a37537569fd9d9fc81db62176 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 27 Oct 2022 06:50:04 +0300 Subject: [PATCH 076/118] Add support for postgres-backup-local v15 Fixes https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/2200 --- roles/matrix-postgres-backup/defaults/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/matrix-postgres-backup/defaults/main.yml b/roles/matrix-postgres-backup/defaults/main.yml index d59c84e95..abdfa74c5 100644 --- a/roles/matrix-postgres-backup/defaults/main.yml +++ b/roles/matrix-postgres-backup/defaults/main.yml @@ -36,7 +36,8 @@ matrix_postgres_backup_docker_image_v11: "{{ matrix_container_global_registry_pr matrix_postgres_backup_docker_image_v12: "{{ matrix_container_global_registry_prefix }}prodrigestivill/postgres-backup-local:12-{{ matrix_postgres_backup_docker_image_distro }}-2cf00a5" matrix_postgres_backup_docker_image_v13: "{{ matrix_container_global_registry_prefix }}prodrigestivill/postgres-backup-local:13-{{ matrix_postgres_backup_docker_image_distro }}-2cf00a5" matrix_postgres_backup_docker_image_v14: "{{ matrix_container_global_registry_prefix }}prodrigestivill/postgres-backup-local:14-{{ matrix_postgres_backup_docker_image_distro }}-2cf00a5" -matrix_postgres_backup_docker_image_latest: "{{ matrix_postgres_backup_docker_image_v14 }}" +matrix_postgres_backup_docker_image_v15: "{{ matrix_container_global_registry_prefix }}prodrigestivill/postgres-backup-local:15-{{ matrix_postgres_backup_docker_image_distro }}-2cf00a5" +matrix_postgres_backup_docker_image_latest: "{{ matrix_postgres_backup_docker_image_v15 }}" # This variable is assigned at runtime. Overriding its value has no effect. matrix_postgres_backup_docker_image_to_use: '{{ matrix_postgres_backup_docker_image_latest }}' From 35c5c23a3e4c0f0d9981f18895a0da300a7ea8ac Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 27 Oct 2022 06:50:41 +0300 Subject: [PATCH 077/118] Remove useless tasks from matrix-postgres-backup role `matrix-postgres-backup` reuses the `matrix-postgres` role for these tasks, not its own definitions. --- .../util/detect_existing_postgres_version.yml | 61 ------------------- 1 file changed, 61 deletions(-) delete mode 100644 roles/matrix-postgres-backup/tasks/util/detect_existing_postgres_version.yml diff --git a/roles/matrix-postgres-backup/tasks/util/detect_existing_postgres_version.yml b/roles/matrix-postgres-backup/tasks/util/detect_existing_postgres_version.yml deleted file mode 100644 index 877e5934d..000000000 --- a/roles/matrix-postgres-backup/tasks/util/detect_existing_postgres_version.yml +++ /dev/null @@ -1,61 +0,0 @@ ---- - -# This utility aims to determine if there is some existing Postgres version in use or not. -# If there is, it also tries to detect the Docker image that corresponds to that version. - -- name: Initialize Postgres version determination variables (default to empty) - ansible.builtin.set_fact: - matrix_postgres_backup_detection_pg_version_path: "{{ matrix_postgres_data_path }}/PG_VERSION" - matrix_postgres_backup_detected_existing: false - matrix_postgres_backup_detected_version: "" - matrix_postgres_backup_detected_version_corresponding_docker_image: "" - -- name: Determine existing Postgres version (check PG_VERSION file) - ansible.builtin.stat: - path: "{{ matrix_postgres_backup_detection_pg_version_path }}" - register: result_pg_version_stat - -- ansible.builtin.set_fact: - matrix_postgres_backup_detected_existing: true - when: "result_pg_version_stat.stat.exists" - -- name: Determine existing Postgres version (read PG_VERSION file) - ansible.builtin.slurp: - src: "{{ matrix_postgres_backup_detection_pg_version_path }}" - register: result_pg_version - when: matrix_postgres_backup_detected_existing | bool - -- name: Determine existing Postgres version (make sense of PG_VERSION file) - ansible.builtin.set_fact: - matrix_postgres_backup_detected_version: "{{ result_pg_version['content'] | b64decode | replace('\n', '') }}" - when: matrix_postgres_backup_detected_existing | bool - -- name: Determine corresponding Docker image to detected version (assume default of latest) - ansible.builtin.set_fact: - matrix_postgres_backup_detected_version_corresponding_docker_image: "{{ matrix_postgres_backup_docker_image_latest }}" - when: "matrix_postgres_backup_detected_version != ''" - -- name: Determine corresponding Docker image to detected version (use 9.x, if detected) - ansible.builtin.set_fact: - matrix_postgres_backup_detected_version_corresponding_docker_image: "{{ matrix_postgres_backup_docker_image_v9 }}" - when: "matrix_postgres_backup_detected_version.startswith('9.')" - -- name: Determine corresponding Docker image to detected version (use 10.x, if detected) - ansible.builtin.set_fact: - matrix_postgres_backup_detected_version_corresponding_docker_image: "{{ matrix_postgres_backup_docker_image_v10 }}" - when: "matrix_postgres_backup_detected_version == '10' or matrix_postgres_backup_detected_version.startswith('10.')" - -- name: Determine corresponding Docker image to detected version (use 11.x, if detected) - ansible.builtin.set_fact: - matrix_postgres_backup_detected_version_corresponding_docker_image: "{{ matrix_postgres_backup_docker_image_v11 }}" - when: "matrix_postgres_backup_detected_version == '11' or matrix_postgres_backup_detected_version.startswith('11.')" - -- name: Determine corresponding Docker image to detected version (use 12.x, if detected) - ansible.builtin.set_fact: - matrix_postgres_backup_detected_version_corresponding_docker_image: "{{ matrix_postgres_backup_docker_image_v12 }}" - when: "matrix_postgres_backup_detected_version == '12' or matrix_postgres_backup_detected_version.startswith('12.')" - -- name: Determine corresponding Docker image to detected version (use 13.x, if detected) - ansible.builtin.set_fact: - matrix_postgres_backup_detected_version_corresponding_docker_image: "{{ matrix_postgres_backup_docker_image_v13 }}" - when: "matrix_postgres_backup_detected_version == '13' or matrix_postgres_backup_detected_version.startswith('13.')" From ce8e9be6a857c2541a00b22a3ec14ca01cd41562 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 27 Oct 2022 09:48:49 +0300 Subject: [PATCH 078/118] Document Conduit installation Fixes https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/2076 --- docs/configuring-playbook-conduit.md | 58 ++++++++++++++++++++++++++++ docs/configuring-playbook.md | 5 ++- examples/vars.yml | 4 +- 3 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 docs/configuring-playbook-conduit.md diff --git a/docs/configuring-playbook-conduit.md b/docs/configuring-playbook-conduit.md new file mode 100644 index 000000000..8739a5674 --- /dev/null +++ b/docs/configuring-playbook-conduit.md @@ -0,0 +1,58 @@ +# Configuring Conduit (optional) + +By default, this playbook configures the [Synapse](https://github.com/matrix-org/synapse) Matrix server, but you can also use [Conduit](https://conduit.rs). + +**NOTES**: + +- **You can't switch an existing Matrix server's implementation** (e.g. Synapse -> Conduit). Proceed below only if you're OK with losing data or you're dealing with a server on a new domain name, which hasn't participated in the Matrix federation yet. + +- **homeserver implementations other than Synapse may not be fully functional**. The playbook may also not assist you in an optimal way (like it does with Synapse). Make yourself familiar with the downsides before proceeding + + +## Installation + +To use Conduit, you **generally** need the following additional `vars.yml` configuration: + +```yaml +matrix_homeserver_implementation: conduit +``` + +However, since Conduit is difficult (see [famedly/conduit#276](https://gitlab.com/famedly/conduit/-/issues/276) and [famedly/conduit#354](https://gitlab.com/famedly/conduit/-/merge_requests/354)) when it comes to creating the first user account and does not support [registering users](registering-users.md) (via the command line or via the playbook) like Synapse and Dendrite do, we recommend the following flow: + +1. Add `matrix_conduit_allow_registration: true` to your `vars.yml` the first time around, temporarily +2. Run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start` - see [Installing](installing.md)) +3. Create your first user via Element or any other client which supports creating users +4. Get rid of `matrix_conduit_allow_registration: true` from your `vars.yml` +5. Run the playbook again (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-conduit,start` would be enough this time) +6. You can now use your server safely. Additional users can be created by messaging the internal Conduit bot + + +## Configuring bridges / appservices + +Automatic appservice setup is currently unsupported when using conduit. After setting up the service as usual you may notice that it is unable to start. + +You will have to manually register appservices using the the [register-appservice](https://gitlab.com/famedly/conduit/-/blob/next/APPSERVICES.md) command. + +Find the `registration.yaml` in the `/matrix` directory, for example `/matrix/mautrix-signal/bridge/registration.yaml`, then pass the content to conduit: + + + @conduit:your.server.name: register-appservice + ``` + as_token: + de.sorunome.msc2409.push_ephemeral: true + hs_token: + id: signal + namespaces: + aliases: + - exclusive: true + regex: ^#signal_.+:example\.org$ + users: + - exclusive: true + regex: ^@signal_.+:example\.org$ + - exclusive: true + regex: ^@signalbot:example\.org$ + rate_limited: false + sender_localpart: _bot_signalbot + url: http://matrix-mautrix-signal:29328 + ``` + diff --git a/docs/configuring-playbook.md b/docs/configuring-playbook.md index efb0f7e3c..127ab47da 100644 --- a/docs/configuring-playbook.md +++ b/docs/configuring-playbook.md @@ -42,7 +42,10 @@ When you're done with all the configuration you'd like to do, continue with [Ins ### Core service adjustments -- [Configuring Synapse](configuring-playbook-synapse.md) (optional) +- Homeserver configuration: + - [Configuring Synapse](configuring-playbook-synapse.md), if you're going with the default/recommended homeserver implementation (optional) + + - [Configuring Conduit](configuring-playbook-conduit.md), if you've switched to the [Conduit](https://conduit.rs) homeserver implementation (optional) - [Configuring Element](configuring-playbook-client-element.md) (optional) diff --git a/examples/vars.yml b/examples/vars.yml index 3ca8f4601..248f906ba 100644 --- a/examples/vars.yml +++ b/examples/vars.yml @@ -12,7 +12,9 @@ matrix_domain: YOUR_BARE_DOMAIN_NAME_HERE # The Matrix homeserver software to install. -# See `roles/matrix-base/defaults/main.yml` for valid options. +# See: +# - `roles/matrix-base/defaults/main.yml` for valid options +# - the `docs/configuring-playbook-IMPLEMENTATION_NAME.md` documentation page, if one is available for your implementation choice matrix_homeserver_implementation: synapse # A secret used as a base, for generating various other secrets. From 7303f9241c583ce17a9bc224092c89c73d0233e1 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 27 Oct 2022 09:59:54 +0300 Subject: [PATCH 079/118] Fix yamllint error --- roles/matrix-postgres-backup/tasks/validate_config.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/roles/matrix-postgres-backup/tasks/validate_config.yml b/roles/matrix-postgres-backup/tasks/validate_config.yml index 2ba643748..aab68eafa 100644 --- a/roles/matrix-postgres-backup/tasks/validate_config.yml +++ b/roles/matrix-postgres-backup/tasks/validate_config.yml @@ -25,5 +25,3 @@ when: "item.old in vars" with_items: - {'old': 'matrix_postgres_backup_docker_image_suffix', 'new': 'matrix_postgres_backup_docker_image_distro'} - - From 45226f31410e89ab9a116aa061c293dda4aa2985 Mon Sep 17 00:00:00 2001 From: Aine <97398200+etkecc@users.noreply.github.com> Date: Thu, 27 Oct 2022 08:14:31 +0000 Subject: [PATCH 080/118] Update signald 0.22.2 -> 0.23.0 ref: https://gitlab.com/signald/signald/-/issues/343 --- roles/matrix-bridge-mautrix-signal/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bridge-mautrix-signal/defaults/main.yml b/roles/matrix-bridge-mautrix-signal/defaults/main.yml index 9a06e423a..83b0a5ce0 100644 --- a/roles/matrix-bridge-mautrix-signal/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-signal/defaults/main.yml @@ -10,7 +10,7 @@ matrix_mautrix_signal_docker_repo_version: "{{ 'master' if matrix_mautrix_signal matrix_mautrix_signal_docker_src_files_path: "{{ matrix_base_data_path }}/mautrix-signal/docker-src" matrix_mautrix_signal_version: v0.4.0 -matrix_mautrix_signal_daemon_version: 0.22.2 +matrix_mautrix_signal_daemon_version: 0.23.0 # See: https://mau.dev/mautrix/signal/container_registry matrix_mautrix_signal_docker_image: "dock.mau.dev/mautrix/signal:{{ matrix_mautrix_signal_version }}" matrix_mautrix_signal_docker_image_force_pull: "{{ matrix_mautrix_signal_docker_image.endswith(':latest') }}" From 3c31638de969e5b1a7fd7fe3e6ced64125afc156 Mon Sep 17 00:00:00 2001 From: Aine <97398200+etkecc@users.noreply.github.com> Date: Thu, 27 Oct 2022 10:45:08 +0000 Subject: [PATCH 081/118] fix buscarron ban list --- roles/matrix-bot-buscarron/templates/env.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bot-buscarron/templates/env.j2 b/roles/matrix-bot-buscarron/templates/env.j2 index 90ae5f7e8..80ddd38cc 100644 --- a/roles/matrix-bot-buscarron/templates/env.j2 +++ b/roles/matrix-bot-buscarron/templates/env.j2 @@ -10,7 +10,7 @@ BUSCARRON_SPAM_LOCALPARTS={{ matrix_bot_buscarron_spam_localparts|join(" ") }} BUSCARRON_SENTRY={{ matrix_bot_buscarron_sentry }} BUSCARRON_LOGLEVEL={{ matrix_bot_buscarron_loglevel }} BUSCARRON_BAN_SIZE={{ matrix_bot_buscarron_ban_size }} -BUSCARRON_BAN_LIST={{ matrix_bot_buscarron_ban_list }} +BUSCARRON_BAN_LIST={{ matrix_bot_buscarron_ban_list|default('')|join(' ') }} BUSCARRON_PM_TOKEN={{ matrix_bot_buscarron_pm_token }} BUSCARRON_PM_FROM={{ matrix_bot_buscarron_pm_from }} BUSCARRON_PM_REPLYTO={{ matrix_bot_buscarron_pm_replyto }} From 48388a3d96218fe61a7501c131f67ccf61718254 Mon Sep 17 00:00:00 2001 From: Sebastian Gumprich Date: Fri, 28 Oct 2022 13:20:17 +0200 Subject: [PATCH 082/118] use fqcns for some task Signed-off-by: Sebastian Gumprich --- roles/matrix-backup-borg/tasks/setup_install.yml | 4 ++-- roles/matrix-backup-borg/tasks/setup_uninstall.yml | 2 +- roles/matrix-base/tasks/server_base/setup_archlinux.yml | 4 ++-- roles/matrix-base/tasks/setup_matrix_base.yml | 2 +- roles/matrix-base/tasks/util/ensure_fuse_installed.yml | 2 +- roles/matrix-base/tasks/util/ensure_openssl_installed.yml | 2 +- roles/matrix-bot-buscarron/tasks/setup_install.yml | 4 ++-- roles/matrix-bot-buscarron/tasks/setup_uninstall.yml | 2 +- roles/matrix-bot-go-neb/tasks/setup_install.yml | 2 +- roles/matrix-bot-go-neb/tasks/setup_uninstall.yml | 2 +- roles/matrix-bot-honoroit/tasks/setup_install.yml | 4 ++-- roles/matrix-bot-honoroit/tasks/setup_uninstall.yml | 2 +- .../tasks/setup_install.yml | 4 ++-- .../tasks/setup_uninstall.yml | 2 +- .../tasks/setup_install.yml | 4 ++-- .../tasks/setup_uninstall.yml | 2 +- roles/matrix-bot-maubot/tasks/setup_install.yml | 4 ++-- roles/matrix-bot-maubot/tasks/setup_uninstall.yml | 2 +- roles/matrix-bot-mjolnir/tasks/setup_install.yml | 4 ++-- roles/matrix-bot-mjolnir/tasks/setup_uninstall.yml | 2 +- roles/matrix-bot-postmoogle/tasks/setup_install.yml | 4 ++-- roles/matrix-bot-postmoogle/tasks/setup_uninstall.yml | 2 +- .../tasks/setup_install.yml | 2 +- .../matrix-bridge-appservice-irc/tasks/setup_install.yml | 4 ++-- .../tasks/setup_install.yml | 8 ++++---- .../tasks/setup_install.yml | 4 ++-- .../tasks/setup_install.yml | 4 ++-- .../matrix-bridge-beeper-linkedin/tasks/setup_install.yml | 4 ++-- .../matrix-bridge-go-skype-bridge/tasks/setup_install.yml | 4 ++-- roles/matrix-bridge-heisenbridge/tasks/setup_install.yml | 2 +- roles/matrix-bridge-hookshot/tasks/setup_install.yml | 4 ++-- .../matrix-bridge-mautrix-discord/tasks/setup_install.yml | 4 ++-- .../tasks/setup_install.yml | 4 ++-- .../tasks/setup_install.yml | 4 ++-- .../tasks/setup_install.yml | 4 ++-- .../tasks/setup_install.yml | 4 ++-- .../matrix-bridge-mautrix-signal/tasks/setup_install.yml | 8 ++++---- .../tasks/setup_install.yml | 6 +++--- .../matrix-bridge-mautrix-twitter/tasks/setup_install.yml | 4 ++-- .../tasks/setup_install.yml | 4 ++-- .../tasks/setup_install.yml | 4 ++-- .../tasks/setup_install.yml | 4 ++-- .../tasks/setup_install.yml | 4 ++-- .../matrix-bridge-mx-puppet-slack/tasks/setup_install.yml | 4 ++-- .../matrix-bridge-mx-puppet-steam/tasks/setup_install.yml | 4 ++-- .../tasks/setup_install.yml | 4 ++-- roles/matrix-bridge-sms/tasks/setup_install.yml | 2 +- roles/matrix-cactus-comments/tasks/setup_install.yml | 4 ++-- roles/matrix-cactus-comments/tasks/setup_uninstall.yml | 2 +- roles/matrix-client-cinny/tasks/setup_install.yml | 4 ++-- roles/matrix-client-cinny/tasks/setup_uninstall.yml | 2 +- roles/matrix-client-element/tasks/setup_install.yml | 4 ++-- roles/matrix-client-element/tasks/setup_uninstall.yml | 2 +- roles/matrix-client-hydrogen/tasks/setup_install.yml | 4 ++-- roles/matrix-client-hydrogen/tasks/setup_uninstall.yml | 2 +- roles/matrix-conduit/tasks/conduit/setup_install.yml | 2 +- roles/matrix-conduit/tasks/conduit/setup_uninstall.yml | 2 +- roles/matrix-corporal/tasks/setup_corporal.yml | 6 +++--- roles/matrix-coturn/tasks/setup_install.yml | 6 +++--- roles/matrix-dendrite/tasks/dendrite/setup_install.yml | 2 +- roles/matrix-dendrite/tasks/dendrite/setup_uninstall.yml | 2 +- roles/matrix-dimension/tasks/setup_install.yml | 4 ++-- roles/matrix-dimension/tasks/setup_uninstall.yml | 2 +- roles/matrix-dynamic-dns/tasks/install.yml | 4 ++-- roles/matrix-email2matrix/tasks/setup_install.yml | 4 ++-- roles/matrix-email2matrix/tasks/setup_uninstall.yml | 2 +- roles/matrix-etherpad/tasks/setup_install.yml | 2 +- roles/matrix-etherpad/tasks/setup_uninstall.yml | 2 +- roles/matrix-grafana/tasks/setup.yml | 2 +- roles/matrix-jitsi/tasks/setup_jitsi_jicofo.yml | 2 +- roles/matrix-jitsi/tasks/setup_jitsi_jvb.yml | 2 +- roles/matrix-jitsi/tasks/setup_jitsi_prosody.yml | 2 +- roles/matrix-jitsi/tasks/setup_jitsi_web.yml | 2 +- .../tasks/setup_install.yml | 2 +- .../tasks/setup_uninstall.yml | 2 +- roles/matrix-ma1sd/tasks/setup_install.yml | 6 +++--- roles/matrix-ma1sd/tasks/setup_uninstall.yml | 2 +- roles/matrix-mailer/tasks/setup_mailer.yml | 6 +++--- .../tasks/nginx-proxy/setup_metrics_auth.yml | 2 +- roles/matrix-nginx-proxy/tasks/setup_nginx_proxy.yml | 2 +- .../tasks/ssl/setup_ssl_lets_encrypt.yml | 2 +- roles/matrix-ntfy/tasks/setup_install.yml | 2 +- roles/matrix-ntfy/tasks/setup_uninstall.yml | 2 +- .../tasks/setup_postgres_backup.yml | 2 +- roles/matrix-postgres/tasks/migrate_db_to_postgres.yml | 4 ++-- roles/matrix-postgres/tasks/setup_postgres.yml | 2 +- roles/matrix-prometheus-node-exporter/tasks/setup.yml | 2 +- roles/matrix-prometheus-postgres-exporter/tasks/setup.yml | 2 +- roles/matrix-prometheus/tasks/setup_install.yml | 2 +- roles/matrix-redis/tasks/setup_redis.yml | 2 +- roles/matrix-registration/tasks/setup_install.yml | 4 ++-- roles/matrix-registration/tasks/setup_uninstall.yml | 2 +- roles/matrix-sygnal/tasks/setup_install.yml | 2 +- roles/matrix-sygnal/tasks/setup_uninstall.yml | 2 +- roles/matrix-synapse-admin/tasks/setup.yml | 6 +++--- .../tasks/ext/mjolnir-antispam/setup_install.yml | 2 +- .../tasks/ext/synapse-simple-antispam/setup_install.yml | 2 +- roles/matrix-synapse/tasks/goofys/setup_install.yml | 2 +- roles/matrix-synapse/tasks/goofys/setup_uninstall.yml | 2 +- roles/matrix-synapse/tasks/import_media_store.yml | 2 +- .../tasks/rust-synapse-compress-state/main.yml | 2 +- roles/matrix-synapse/tasks/synapse/setup_install.yml | 4 ++-- roles/matrix-synapse/tasks/synapse/setup_uninstall.yml | 2 +- 103 files changed, 159 insertions(+), 159 deletions(-) diff --git a/roles/matrix-backup-borg/tasks/setup_install.yml b/roles/matrix-backup-borg/tasks/setup_install.yml index b44a8fa1b..f99051e35 100644 --- a/roles/matrix-backup-borg/tasks/setup_install.yml +++ b/roles/matrix-backup-borg/tasks/setup_install.yml @@ -59,7 +59,7 @@ mode: 0600 - name: Ensure borg image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_backup_borg_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_backup_borg_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -82,7 +82,7 @@ when: "matrix_backup_borg_container_image_self_build | bool" - name: Ensure borg image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_backup_borg_docker_image }}" source: build force_source: "{{ matrix_backup_borg_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-backup-borg/tasks/setup_uninstall.yml b/roles/matrix-backup-borg/tasks/setup_uninstall.yml index fb583f57a..37832b2b1 100644 --- a/roles/matrix-backup-borg/tasks/setup_uninstall.yml +++ b/roles/matrix-backup-borg/tasks/setup_uninstall.yml @@ -36,6 +36,6 @@ state: absent - name: Ensure borg Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_backup_borg_docker_image }}" state: absent diff --git a/roles/matrix-base/tasks/server_base/setup_archlinux.yml b/roles/matrix-base/tasks/server_base/setup_archlinux.yml index c912e58f3..a93136148 100644 --- a/roles/matrix-base/tasks/server_base/setup_archlinux.yml +++ b/roles/matrix-base/tasks/server_base/setup_archlinux.yml @@ -1,7 +1,7 @@ --- - name: Install host dependencies - pacman: + community.general.pacman: name: - python-docker - python-dnspython @@ -9,7 +9,7 @@ update_cache: true - name: Ensure Docker is installed - pacman: + community.general.pacman: name: - docker state: present diff --git a/roles/matrix-base/tasks/setup_matrix_base.yml b/roles/matrix-base/tasks/setup_matrix_base.yml index 2e8609873..6eebe3c0e 100644 --- a/roles/matrix-base/tasks/setup_matrix_base.yml +++ b/roles/matrix-base/tasks/setup_matrix_base.yml @@ -20,7 +20,7 @@ when: "matrix_vars_yml_snapshotting_enabled | bool" - name: Ensure Matrix network is created in Docker - docker_network: + community.docker.docker_network: name: "{{ matrix_docker_network }}" driver: bridge diff --git a/roles/matrix-base/tasks/util/ensure_fuse_installed.yml b/roles/matrix-base/tasks/util/ensure_fuse_installed.yml index 47d2d9e85..240a5c625 100644 --- a/roles/matrix-base/tasks/util/ensure_fuse_installed.yml +++ b/roles/matrix-base/tasks/util/ensure_fuse_installed.yml @@ -16,7 +16,7 @@ when: ansible_os_family == 'Debian' - name: Ensure fuse installed (Archlinux) - pacman: + community.general.pacman: name: - fuse3 state: present diff --git a/roles/matrix-base/tasks/util/ensure_openssl_installed.yml b/roles/matrix-base/tasks/util/ensure_openssl_installed.yml index ae22fb495..a5bdf21a5 100644 --- a/roles/matrix-base/tasks/util/ensure_openssl_installed.yml +++ b/roles/matrix-base/tasks/util/ensure_openssl_installed.yml @@ -16,7 +16,7 @@ when: ansible_os_family == 'Debian' - name: Ensure openssl installed (Archlinux) - pacman: + community.general.pacman: name: - openssl state: present diff --git a/roles/matrix-bot-buscarron/tasks/setup_install.yml b/roles/matrix-bot-buscarron/tasks/setup_install.yml index 564ff7e0f..4a23d7e72 100644 --- a/roles/matrix-bot-buscarron/tasks/setup_install.yml +++ b/roles/matrix-bot-buscarron/tasks/setup_install.yml @@ -50,7 +50,7 @@ mode: 0640 - name: Ensure buscarron image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_bot_buscarron_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_bot_buscarron_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -73,7 +73,7 @@ when: "matrix_bot_buscarron_container_image_self_build | bool" - name: Ensure buscarron image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_bot_buscarron_docker_image }}" source: build force_source: "{{ matrix_bot_buscarron_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bot-buscarron/tasks/setup_uninstall.yml b/roles/matrix-bot-buscarron/tasks/setup_uninstall.yml index ad9e78cdd..848d24d2b 100644 --- a/roles/matrix-bot-buscarron/tasks/setup_uninstall.yml +++ b/roles/matrix-bot-buscarron/tasks/setup_uninstall.yml @@ -31,6 +31,6 @@ state: absent - name: Ensure buscarron Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_bot_buscarron_docker_image }}" state: absent diff --git a/roles/matrix-bot-go-neb/tasks/setup_install.yml b/roles/matrix-bot-go-neb/tasks/setup_install.yml index a651c1608..522155972 100644 --- a/roles/matrix-bot-go-neb/tasks/setup_install.yml +++ b/roles/matrix-bot-go-neb/tasks/setup_install.yml @@ -17,7 +17,7 @@ when: "item.when | bool" - name: Ensure go-neb image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_bot_go_neb_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_bot_go_neb_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bot-go-neb/tasks/setup_uninstall.yml b/roles/matrix-bot-go-neb/tasks/setup_uninstall.yml index 9794a90ad..833910942 100644 --- a/roles/matrix-bot-go-neb/tasks/setup_uninstall.yml +++ b/roles/matrix-bot-go-neb/tasks/setup_uninstall.yml @@ -31,6 +31,6 @@ state: absent - name: Ensure go-neb Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_bot_go_neb_docker_image }}" state: absent diff --git a/roles/matrix-bot-honoroit/tasks/setup_install.yml b/roles/matrix-bot-honoroit/tasks/setup_install.yml index 8a440484e..3c9745576 100644 --- a/roles/matrix-bot-honoroit/tasks/setup_install.yml +++ b/roles/matrix-bot-honoroit/tasks/setup_install.yml @@ -50,7 +50,7 @@ mode: 0640 - name: Ensure honoroit image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_bot_honoroit_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_bot_honoroit_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -73,7 +73,7 @@ when: "matrix_bot_honoroit_container_image_self_build | bool" - name: Ensure honoroit image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_bot_honoroit_docker_image }}" source: build force_source: "{{ matrix_bot_honoroit_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bot-honoroit/tasks/setup_uninstall.yml b/roles/matrix-bot-honoroit/tasks/setup_uninstall.yml index 6ede0d1d2..54869e312 100644 --- a/roles/matrix-bot-honoroit/tasks/setup_uninstall.yml +++ b/roles/matrix-bot-honoroit/tasks/setup_uninstall.yml @@ -31,6 +31,6 @@ state: absent - name: Ensure honoroit Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_bot_honoroit_docker_image }}" state: absent diff --git a/roles/matrix-bot-matrix-registration-bot/tasks/setup_install.yml b/roles/matrix-bot-matrix-registration-bot/tasks/setup_install.yml index d4522321e..5896ac620 100644 --- a/roles/matrix-bot-matrix-registration-bot/tasks/setup_install.yml +++ b/roles/matrix-bot-matrix-registration-bot/tasks/setup_install.yml @@ -22,7 +22,7 @@ mode: 0640 - name: Ensure matrix-registration-bot image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_bot_matrix_registration_bot_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_bot_matrix_registration_bot_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -45,7 +45,7 @@ when: "matrix_bot_matrix_registration_bot_container_image_self_build | bool" - name: Ensure matrix-registration-bot image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_bot_matrix_registration_bot_docker_image }}" source: build force_source: "{{ matrix_bot_matrix_registration_bot_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bot-matrix-registration-bot/tasks/setup_uninstall.yml b/roles/matrix-bot-matrix-registration-bot/tasks/setup_uninstall.yml index 426eefc10..63bc53ad0 100644 --- a/roles/matrix-bot-matrix-registration-bot/tasks/setup_uninstall.yml +++ b/roles/matrix-bot-matrix-registration-bot/tasks/setup_uninstall.yml @@ -31,6 +31,6 @@ state: absent - name: Ensure matrix-registration-bot Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_bot_matrix_registration_bot_docker_image }}" state: absent diff --git a/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml b/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml index 93285bf60..5d784ef34 100644 --- a/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml +++ b/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml @@ -43,7 +43,7 @@ when: "item.when | bool" - name: Ensure matrix-reminder-bot image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_bot_matrix_reminder_bot_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_bot_matrix_reminder_bot_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -66,7 +66,7 @@ when: "matrix_bot_matrix_reminder_bot_container_image_self_build | bool" - name: Ensure matrix-reminder-bot image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_bot_matrix_reminder_bot_docker_image }}" source: build force_source: "{{ matrix_bot_matrix_reminder_bot_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bot-matrix-reminder-bot/tasks/setup_uninstall.yml b/roles/matrix-bot-matrix-reminder-bot/tasks/setup_uninstall.yml index d8926df75..de9e04271 100644 --- a/roles/matrix-bot-matrix-reminder-bot/tasks/setup_uninstall.yml +++ b/roles/matrix-bot-matrix-reminder-bot/tasks/setup_uninstall.yml @@ -31,6 +31,6 @@ state: absent - name: Ensure matrix-reminder-bot Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_bot_matrix_reminder_bot_docker_image }}" state: absent diff --git a/roles/matrix-bot-maubot/tasks/setup_install.yml b/roles/matrix-bot-maubot/tasks/setup_install.yml index 185a29889..50e48254c 100644 --- a/roles/matrix-bot-maubot/tasks/setup_install.yml +++ b/roles/matrix-bot-maubot/tasks/setup_install.yml @@ -26,7 +26,7 @@ mode: "u=rwx" - name: Ensure maubot image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_bot_maubot_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_bot_maubot_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -49,7 +49,7 @@ when: "matrix_bot_maubot_container_image_self_build|bool" - name: Ensure maubot image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_bot_maubot_docker_image }}" source: build force_source: "{{ matrix_bot_maubot_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bot-maubot/tasks/setup_uninstall.yml b/roles/matrix-bot-maubot/tasks/setup_uninstall.yml index dd0fc1f6a..6a5e7fdc4 100644 --- a/roles/matrix-bot-maubot/tasks/setup_uninstall.yml +++ b/roles/matrix-bot-maubot/tasks/setup_uninstall.yml @@ -31,6 +31,6 @@ state: absent - name: Ensure maubot Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_bot_maubot_docker_image }}" state: absent diff --git a/roles/matrix-bot-mjolnir/tasks/setup_install.yml b/roles/matrix-bot-mjolnir/tasks/setup_install.yml index 08ac9d034..b9bcf37d8 100644 --- a/roles/matrix-bot-mjolnir/tasks/setup_install.yml +++ b/roles/matrix-bot-mjolnir/tasks/setup_install.yml @@ -18,7 +18,7 @@ when: "item.when | bool" - name: Ensure mjolnir Docker image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_bot_mjolnir_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_bot_mjolnir_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -41,7 +41,7 @@ when: "matrix_bot_mjolnir_container_image_self_build | bool" - name: Ensure mjolnir Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_bot_mjolnir_docker_image }}" source: build force_source: "{{ matrix_bot_mjolnir_git_pull_results.changed }}" diff --git a/roles/matrix-bot-mjolnir/tasks/setup_uninstall.yml b/roles/matrix-bot-mjolnir/tasks/setup_uninstall.yml index 5c7f4c896..afefcc489 100644 --- a/roles/matrix-bot-mjolnir/tasks/setup_uninstall.yml +++ b/roles/matrix-bot-mjolnir/tasks/setup_uninstall.yml @@ -31,6 +31,6 @@ state: absent - name: Ensure mjolnir Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_bot_mjolnir_docker_image }}" state: absent diff --git a/roles/matrix-bot-postmoogle/tasks/setup_install.yml b/roles/matrix-bot-postmoogle/tasks/setup_install.yml index 9c9e59ee5..5edfd4a9b 100644 --- a/roles/matrix-bot-postmoogle/tasks/setup_install.yml +++ b/roles/matrix-bot-postmoogle/tasks/setup_install.yml @@ -46,7 +46,7 @@ mode: 0640 - name: Ensure postmoogle image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_bot_postmoogle_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_bot_postmoogle_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -69,7 +69,7 @@ when: "matrix_bot_postmoogle_container_image_self_build | bool" - name: Ensure postmoogle image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_bot_postmoogle_docker_image }}" source: build force_source: "{{ matrix_bot_postmoogle_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bot-postmoogle/tasks/setup_uninstall.yml b/roles/matrix-bot-postmoogle/tasks/setup_uninstall.yml index 64164a860..5502298cd 100644 --- a/roles/matrix-bot-postmoogle/tasks/setup_uninstall.yml +++ b/roles/matrix-bot-postmoogle/tasks/setup_uninstall.yml @@ -31,6 +31,6 @@ state: absent - name: Ensure postmoogle Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_bot_postmoogle_docker_image }}" state: absent diff --git a/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml b/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml index e12c15727..3ef48c3df 100644 --- a/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml +++ b/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml @@ -29,7 +29,7 @@ matrix_appservice_discord_requires_restart: true - name: Ensure Appservice Discord image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_appservice_discord_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_appservice_discord_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bridge-appservice-irc/tasks/setup_install.yml b/roles/matrix-bridge-appservice-irc/tasks/setup_install.yml index 6794e8148..4b4614b17 100644 --- a/roles/matrix-bridge-appservice-irc/tasks/setup_install.yml +++ b/roles/matrix-bridge-appservice-irc/tasks/setup_install.yml @@ -64,7 +64,7 @@ matrix_appservice_irc_requires_restart: true - name: Ensure Appservice IRC image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_appservice_irc_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_appservice_irc_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -87,7 +87,7 @@ when: "matrix_appservice_irc_enabled | bool and matrix_appservice_irc_container_image_self_build | bool" - name: Ensure matrix-appservice-irc Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_appservice_irc_docker_image }}" source: build force_source: "{{ matrix_appservice_irc_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bridge-appservice-kakaotalk/tasks/setup_install.yml b/roles/matrix-bridge-appservice-kakaotalk/tasks/setup_install.yml index def73c595..2dd334cb7 100644 --- a/roles/matrix-bridge-appservice-kakaotalk/tasks/setup_install.yml +++ b/roles/matrix-bridge-appservice-kakaotalk/tasks/setup_install.yml @@ -9,7 +9,7 @@ when: "matrix_synapse_role_executed | default(False)" - name: Ensure matrix-appservice-kakaotalk image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_appservice_kakaotalk_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_appservice_kakaotalk_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -21,7 +21,7 @@ until: result is not failed - name: Ensure matrix-appservice-kakaotalk-node image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_appservice_kakaotalk_node_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_appservice_kakaotalk_node_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -58,7 +58,7 @@ when: "matrix_appservice_kakaotalk_container_image_self_build | bool" - name: Ensure matrix-appservice-kakaotalk-node Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_appservice_kakaotalk_node_docker_image }}" source: build force_source: "{{ matrix_appservice_kakaotalk_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -70,7 +70,7 @@ when: "matrix_appservice_kakaotalk_container_image_self_build | bool" - name: Ensure matrix-appservice-kakaotalk Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_appservice_kakaotalk_docker_image }}" source: build force_source: "{{ matrix_appservice_kakaotalk_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bridge-appservice-slack/tasks/setup_install.yml b/roles/matrix-bridge-appservice-slack/tasks/setup_install.yml index 9b741d69f..d882d27d1 100644 --- a/roles/matrix-bridge-appservice-slack/tasks/setup_install.yml +++ b/roles/matrix-bridge-appservice-slack/tasks/setup_install.yml @@ -32,7 +32,7 @@ matrix_appservice_slack_requires_restart: true - name: Ensure Appservice Slack image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_appservice_slack_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_appservice_slack_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -55,7 +55,7 @@ when: "matrix_appservice_slack_container_image_self_build | bool" - name: Ensure matrix-appservice-slack Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_appservice_slack_docker_image }}" source: build force_source: "{{ matrix_appservice_slack_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bridge-appservice-webhooks/tasks/setup_install.yml b/roles/matrix-bridge-appservice-webhooks/tasks/setup_install.yml index 5cd8da882..824b5b783 100644 --- a/roles/matrix-bridge-appservice-webhooks/tasks/setup_install.yml +++ b/roles/matrix-bridge-appservice-webhooks/tasks/setup_install.yml @@ -15,7 +15,7 @@ when: "item.when | bool" - name: Ensure Appservice webhooks image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_appservice_webhooks_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_appservice_webhooks_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -39,7 +39,7 @@ register: matrix_appservice_webhooks_git_pull_results - name: Ensure Appservice webhooks Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_appservice_webhooks_docker_image }}" source: build force_source: "{{ matrix_appservice_webhooks_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bridge-beeper-linkedin/tasks/setup_install.yml b/roles/matrix-bridge-beeper-linkedin/tasks/setup_install.yml index c1b19df99..97464adcf 100644 --- a/roles/matrix-bridge-beeper-linkedin/tasks/setup_install.yml +++ b/roles/matrix-bridge-beeper-linkedin/tasks/setup_install.yml @@ -23,7 +23,7 @@ - name: Ensure Beeper LinkedIn image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_beeper_linkedin_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_beeper_linkedin_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -62,7 +62,7 @@ changed_when: matrix_beeper_linkedin_generate_docker_requirements_result.rc == 0 - name: Ensure Beeper LinkedIn Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_beeper_linkedin_docker_image }}" source: build force_source: "{{ matrix_beeper_linkedin_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bridge-go-skype-bridge/tasks/setup_install.yml b/roles/matrix-bridge-go-skype-bridge/tasks/setup_install.yml index 320196860..82ccc72c3 100644 --- a/roles/matrix-bridge-go-skype-bridge/tasks/setup_install.yml +++ b/roles/matrix-bridge-go-skype-bridge/tasks/setup_install.yml @@ -52,7 +52,7 @@ when: item.when | bool - name: Ensure Go Skype Bridge image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_go_skype_bridge_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_go_skype_bridge_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -75,7 +75,7 @@ when: "matrix_go_skype_bridge_container_image_self_build | bool" - name: Ensure Go Skype Bridge Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_go_skype_bridge_docker_image }}" source: build force_source: "{{ matrix_go_skype_bridge_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bridge-heisenbridge/tasks/setup_install.yml b/roles/matrix-bridge-heisenbridge/tasks/setup_install.yml index ffcc1c8b9..f8e1259bc 100644 --- a/roles/matrix-bridge-heisenbridge/tasks/setup_install.yml +++ b/roles/matrix-bridge-heisenbridge/tasks/setup_install.yml @@ -1,7 +1,7 @@ --- - name: Ensure heisenbridge image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_heisenbridge_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_heisenbridge_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bridge-hookshot/tasks/setup_install.yml b/roles/matrix-bridge-hookshot/tasks/setup_install.yml index 0c6bfc34b..7c1cdf950 100644 --- a/roles/matrix-bridge-hookshot/tasks/setup_install.yml +++ b/roles/matrix-bridge-hookshot/tasks/setup_install.yml @@ -15,7 +15,7 @@ when: item.when | bool - name: Ensure hookshot image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_hookshot_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_hookshot_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -38,7 +38,7 @@ when: "matrix_hookshot_container_image_self_build | bool" - name: Ensure hookshot Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_hookshot_docker_image }}" source: build force_source: "{{ matrix_hookshot_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bridge-mautrix-discord/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-discord/tasks/setup_install.yml index 4b05765ac..ae3862d47 100644 --- a/roles/matrix-bridge-mautrix-discord/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-discord/tasks/setup_install.yml @@ -52,7 +52,7 @@ when: item.when | bool - name: Ensure Mautrix Discord image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_mautrix_discord_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_mautrix_discord_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -75,7 +75,7 @@ when: "matrix_mautrix_discord_container_image_self_build | bool" - name: Ensure Mautrix discord Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_mautrix_discord_docker_image }}" source: build force_source: "{{ matrix_mautrix_discord_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml index ca882fb05..651e77942 100644 --- a/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml @@ -37,7 +37,7 @@ matrix_mautrix_facebook_requires_restart: true - name: Ensure Mautrix Facebook image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_mautrix_facebook_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_mautrix_facebook_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -74,7 +74,7 @@ when: "matrix_mautrix_facebook_container_image_self_build | bool" - name: Ensure Mautrix Facebook Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_mautrix_facebook_docker_image }}" source: build force_source: "{{ matrix_mautrix_facebook_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bridge-mautrix-googlechat/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-googlechat/tasks/setup_install.yml index f2fccb293..2e043def2 100644 --- a/roles/matrix-bridge-mautrix-googlechat/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-googlechat/tasks/setup_install.yml @@ -37,7 +37,7 @@ matrix_mautrix_googlechat_requires_restart: true - name: Ensure Mautrix googlechat image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_mautrix_googlechat_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_mautrix_googlechat_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -74,7 +74,7 @@ when: "matrix_mautrix_googlechat_container_image_self_build | bool" - name: Ensure Mautrix googlechat Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_mautrix_googlechat_docker_image }}" source: build force_source: "{{ matrix_mautrix_googlechat_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml index 48c835846..e62ef21c0 100644 --- a/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml @@ -37,7 +37,7 @@ matrix_mautrix_hangouts_requires_restart: true - name: Ensure Mautrix Hangouts image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_mautrix_hangouts_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_mautrix_hangouts_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -74,7 +74,7 @@ when: "matrix_mautrix_hangouts_container_image_self_build | bool" - name: Ensure Mautrix Hangouts Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_mautrix_hangouts_docker_image }}" source: build force_source: "{{ matrix_mautrix_hangouts_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bridge-mautrix-instagram/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-instagram/tasks/setup_install.yml index 88b0286e7..47076eb7e 100644 --- a/roles/matrix-bridge-mautrix-instagram/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-instagram/tasks/setup_install.yml @@ -8,7 +8,7 @@ when: "matrix_synapse_role_executed | default(False)" - name: Ensure Mautrix instagram image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_mautrix_instagram_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_mautrix_instagram_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -45,7 +45,7 @@ when: "matrix_mautrix_instagram_container_image_self_build | bool" - name: Ensure Mautrix instagram Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_mautrix_instagram_docker_image }}" source: build force_source: "{{ matrix_mautrix_instagram_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bridge-mautrix-signal/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-signal/tasks/setup_install.yml index cfc704a82..577e80a90 100644 --- a/roles/matrix-bridge-mautrix-signal/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-signal/tasks/setup_install.yml @@ -9,7 +9,7 @@ when: "matrix_synapse_role_executed | default(False)" - name: Ensure Mautrix Signal image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_mautrix_signal_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_mautrix_signal_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -33,7 +33,7 @@ when: "matrix_mautrix_signal_container_image_self_build | bool" - name: Ensure Mautrix Signal image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_mautrix_signal_docker_image }}" source: build force_source: "{{ matrix_mautrix_signal_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -46,7 +46,7 @@ - name: Ensure Mautrix Signal Daemon image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_mautrix_signal_daemon_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_mautrix_signal_daemon_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -66,7 +66,7 @@ when: "matrix_mautrix_signal_daemon_container_image_self_build | bool" - name: Ensure Mautrix Signal Daemon image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_mautrix_signal_daemon_docker_image }}" source: build force_source: "{{ matrix_mautrix_signal_daemon_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml index 7a50b709c..3d6e66f67 100644 --- a/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml @@ -51,7 +51,7 @@ when: item.when | bool - name: Ensure Mautrix Telegram image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_mautrix_telegram_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_mautrix_telegram_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -74,7 +74,7 @@ when: "matrix_telegram_lottieconverter_container_image_self_build | bool and matrix_mautrix_telegram_container_image_self_build | bool" - name: Ensure lottieconverter Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_telegram_lottieconverter_docker_image }}" source: build force_source: "{{ matrix_telegram_lottieconverter_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -97,7 +97,7 @@ when: "matrix_mautrix_telegram_container_image_self_build | bool" - name: Ensure matrix-mautrix-telegram Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_mautrix_telegram_docker_image }}" source: build force_source: "{{ matrix_mautrix_telegram_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bridge-mautrix-twitter/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-twitter/tasks/setup_install.yml index 05887c6d7..485e8be4b 100644 --- a/roles/matrix-bridge-mautrix-twitter/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-twitter/tasks/setup_install.yml @@ -12,7 +12,7 @@ matrix_mautrix_twitter_requires_restart: false - name: Ensure Mautrix Twitter image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_mautrix_twitter_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_mautrix_twitter_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -49,7 +49,7 @@ when: "matrix_mautrix_twitter_enabled | bool and matrix_mautrix_twitter_container_image_self_build" - name: Ensure Mautrix Twitter Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_mautrix_twitter_docker_image }}" source: build force_source: "{{ matrix_mautrix_twitter_git_pull_results.changed }}" diff --git a/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml index b58542f72..20dd2cc10 100644 --- a/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml @@ -52,7 +52,7 @@ when: item.when | bool - name: Ensure Mautrix Whatsapp image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_mautrix_whatsapp_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_mautrix_whatsapp_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -75,7 +75,7 @@ when: "matrix_mautrix_whatsapp_container_image_self_build | bool" - name: Ensure Mautrix Whatsapp Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_mautrix_whatsapp_docker_image }}" source: build force_source: "{{ matrix_mautrix_whatsapp_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml index b863b4446..715c9e422 100644 --- a/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml +++ b/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml @@ -71,7 +71,7 @@ matrix_mx_puppet_discord_requires_restart: true - name: Ensure MX Puppet Discord image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_mx_puppet_discord_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_mx_puppet_discord_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -94,7 +94,7 @@ when: "matrix_mx_puppet_discord_enabled | bool and matrix_mx_puppet_discord_container_image_self_build" - name: Ensure MX Puppet Discord Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_mx_puppet_discord_docker_image }}" source: build force_source: "{{ matrix_mx_puppet_discord_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bridge-mx-puppet-groupme/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-groupme/tasks/setup_install.yml index f81ae4a01..4e33961b6 100644 --- a/roles/matrix-bridge-mx-puppet-groupme/tasks/setup_install.yml +++ b/roles/matrix-bridge-mx-puppet-groupme/tasks/setup_install.yml @@ -69,7 +69,7 @@ matrix_mx_puppet_groupme_requires_restart: true - name: Ensure MX Puppet Groupme image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_mx_puppet_groupme_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_mx_puppet_groupme_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -92,7 +92,7 @@ when: "matrix_mx_puppet_groupme_enabled | bool and matrix_mx_puppet_groupme_container_image_self_build" - name: Ensure MX Puppet Groupme Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_mx_puppet_groupme_docker_image }}" source: build force_source: "{{ matrix_mx_puppet_groupme_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bridge-mx-puppet-instagram/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-instagram/tasks/setup_install.yml index 600ed63c6..7b1a26eed 100644 --- a/roles/matrix-bridge-mx-puppet-instagram/tasks/setup_install.yml +++ b/roles/matrix-bridge-mx-puppet-instagram/tasks/setup_install.yml @@ -38,7 +38,7 @@ matrix_mx_puppet_instagram_requires_restart: true - name: Ensure mx-puppet-instagram image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_mx_puppet_instagram_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_mx_puppet_instagram_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -75,7 +75,7 @@ when: "matrix_mx_puppet_instagram_enabled | bool and matrix_mx_puppet_instagram_container_image_self_build | bool" - name: Ensure mx-puppet-instagram Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_mx_puppet_instagram_docker_image }}" source: build force_source: "{{ matrix_mx_puppet_instagram_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bridge-mx-puppet-slack/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-slack/tasks/setup_install.yml index 9c5ae4fc8..db5841240 100644 --- a/roles/matrix-bridge-mx-puppet-slack/tasks/setup_install.yml +++ b/roles/matrix-bridge-mx-puppet-slack/tasks/setup_install.yml @@ -65,7 +65,7 @@ matrix_mx_puppet_slack_requires_restart: true - name: Ensure MX Puppet Slack image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_mx_puppet_slack_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_mx_puppet_slack_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -88,7 +88,7 @@ when: "matrix_mx_puppet_slack_enabled | bool and matrix_mx_puppet_slack_container_image_self_build" - name: Ensure MX Puppet Slack Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_mx_puppet_slack_docker_image }}" source: build force_source: "{{ matrix_mx_puppet_slack_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bridge-mx-puppet-steam/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-steam/tasks/setup_install.yml index c75566f30..9875dcce4 100644 --- a/roles/matrix-bridge-mx-puppet-steam/tasks/setup_install.yml +++ b/roles/matrix-bridge-mx-puppet-steam/tasks/setup_install.yml @@ -69,7 +69,7 @@ matrix_mx_puppet_steam_requires_restart: true - name: Ensure MX Puppet Steam image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_mx_puppet_steam_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_mx_puppet_steam_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -92,7 +92,7 @@ when: "matrix_mx_puppet_steam_enabled | bool and matrix_mx_puppet_steam_container_image_self_build" - name: Ensure MX Puppet Steam Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_mx_puppet_steam_docker_image }}" source: build force_source: "{{ matrix_mx_puppet_steam_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bridge-mx-puppet-twitter/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-twitter/tasks/setup_install.yml index ed94eae59..87da97b9f 100644 --- a/roles/matrix-bridge-mx-puppet-twitter/tasks/setup_install.yml +++ b/roles/matrix-bridge-mx-puppet-twitter/tasks/setup_install.yml @@ -69,7 +69,7 @@ matrix_mx_puppet_twitter_requires_restart: true - name: Ensure MX Puppet Twitter image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_mx_puppet_twitter_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_mx_puppet_twitter_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -92,7 +92,7 @@ when: "matrix_mx_puppet_twitter_enabled | bool and matrix_mx_puppet_twitter_container_image_self_build" - name: Ensure MX Puppet Twitter Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_mx_puppet_twitter_docker_image }}" source: build force_source: "{{ matrix_mx_puppet_twitter_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-bridge-sms/tasks/setup_install.yml b/roles/matrix-bridge-sms/tasks/setup_install.yml index dcc317ba2..b4125e511 100644 --- a/roles/matrix-bridge-sms/tasks/setup_install.yml +++ b/roles/matrix-bridge-sms/tasks/setup_install.yml @@ -1,7 +1,7 @@ --- - name: Ensure matrix-sms-bridge image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_sms_bridge_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" register: result diff --git a/roles/matrix-cactus-comments/tasks/setup_install.yml b/roles/matrix-cactus-comments/tasks/setup_install.yml index ec5311e82..7085290f9 100644 --- a/roles/matrix-cactus-comments/tasks/setup_install.yml +++ b/roles/matrix-cactus-comments/tasks/setup_install.yml @@ -31,7 +31,7 @@ mode: 0640 - name: Ensure cactus comments image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_cactus_comments_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_cactus_comments_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -54,7 +54,7 @@ when: "matrix_cactus_comments_container_image_self_build | bool" - name: Ensure cactus comments image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_cactus_comments_docker_image }}" source: build force_source: "{{ matrix_cactus_comments_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-cactus-comments/tasks/setup_uninstall.yml b/roles/matrix-cactus-comments/tasks/setup_uninstall.yml index 011c04b85..3491d9125 100644 --- a/roles/matrix-cactus-comments/tasks/setup_uninstall.yml +++ b/roles/matrix-cactus-comments/tasks/setup_uninstall.yml @@ -31,6 +31,6 @@ state: absent - name: Ensure cactus comments Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_cactus_comments_docker_image }}" state: absent diff --git a/roles/matrix-client-cinny/tasks/setup_install.yml b/roles/matrix-client-cinny/tasks/setup_install.yml index 755b872fd..a39c7f9dd 100644 --- a/roles/matrix-client-cinny/tasks/setup_install.yml +++ b/roles/matrix-client-cinny/tasks/setup_install.yml @@ -12,7 +12,7 @@ when: "item.when | bool" - name: Ensure Cinny Docker image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_client_cinny_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_client_cinny_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -54,7 +54,7 @@ when: "item.src is not none" - name: Ensure Cinny Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_client_cinny_docker_image }}" source: build force_source: "{{ matrix_client_cinny_git_pull_results.changed }}" diff --git a/roles/matrix-client-cinny/tasks/setup_uninstall.yml b/roles/matrix-client-cinny/tasks/setup_uninstall.yml index e6f71b0bc..6cc93e1be 100644 --- a/roles/matrix-client-cinny/tasks/setup_uninstall.yml +++ b/roles/matrix-client-cinny/tasks/setup_uninstall.yml @@ -30,6 +30,6 @@ state: absent - name: Ensure Cinny Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_client_cinny_docker_image }}" state: absent diff --git a/roles/matrix-client-element/tasks/setup_install.yml b/roles/matrix-client-element/tasks/setup_install.yml index b21da064d..044ed6113 100644 --- a/roles/matrix-client-element/tasks/setup_install.yml +++ b/roles/matrix-client-element/tasks/setup_install.yml @@ -13,7 +13,7 @@ when: "item.when | bool" - name: Ensure Element Docker image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_client_element_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_client_element_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -50,7 +50,7 @@ when: "matrix_client_element_container_image_self_build | bool and matrix_client_element_container_image_self_build_low_memory_system_patch_enabled | bool" - name: Ensure Element Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_client_element_docker_image }}" source: build force_source: "{{ matrix_client_element_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-client-element/tasks/setup_uninstall.yml b/roles/matrix-client-element/tasks/setup_uninstall.yml index b3cdd05e5..c40a4fc66 100644 --- a/roles/matrix-client-element/tasks/setup_uninstall.yml +++ b/roles/matrix-client-element/tasks/setup_uninstall.yml @@ -31,6 +31,6 @@ state: absent - name: Ensure Element Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_client_element_docker_image }}" state: absent diff --git a/roles/matrix-client-hydrogen/tasks/setup_install.yml b/roles/matrix-client-hydrogen/tasks/setup_install.yml index 4cd445d01..dfd0607bf 100644 --- a/roles/matrix-client-hydrogen/tasks/setup_install.yml +++ b/roles/matrix-client-hydrogen/tasks/setup_install.yml @@ -13,7 +13,7 @@ when: "item.when | bool" - name: Ensure Hydrogen Docker image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_client_hydrogen_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_client_hydrogen_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -58,7 +58,7 @@ # This step MUST come after the steps to install the configuration files because the config files # are currently only read at build time, not at run time like most other components in the playbook - name: Ensure Hydrogen Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_client_hydrogen_docker_image }}" source: build force_source: "{{ matrix_client_hydrogen_git_pull_results.changed }}" diff --git a/roles/matrix-client-hydrogen/tasks/setup_uninstall.yml b/roles/matrix-client-hydrogen/tasks/setup_uninstall.yml index d543cbb32..49d2f4cac 100644 --- a/roles/matrix-client-hydrogen/tasks/setup_uninstall.yml +++ b/roles/matrix-client-hydrogen/tasks/setup_uninstall.yml @@ -31,6 +31,6 @@ state: absent - name: Ensure Hydrogen Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_client_hydrogen_docker_image }}" state: absent diff --git a/roles/matrix-conduit/tasks/conduit/setup_install.yml b/roles/matrix-conduit/tasks/conduit/setup_install.yml index ac5be14dd..cf8c66577 100644 --- a/roles/matrix-conduit/tasks/conduit/setup_install.yml +++ b/roles/matrix-conduit/tasks/conduit/setup_install.yml @@ -1,6 +1,6 @@ --- - name: Ensure Conduit Docker image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_conduit_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_conduit_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-conduit/tasks/conduit/setup_uninstall.yml b/roles/matrix-conduit/tasks/conduit/setup_uninstall.yml index 3bbbc3a78..1bba9a9e0 100644 --- a/roles/matrix-conduit/tasks/conduit/setup_uninstall.yml +++ b/roles/matrix-conduit/tasks/conduit/setup_uninstall.yml @@ -25,6 +25,6 @@ when: "matrix_conduit_service_stat.stat.exists" - name: Ensure Conduit Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_conduit_docker_image }}" state: absent diff --git a/roles/matrix-corporal/tasks/setup_corporal.yml b/roles/matrix-corporal/tasks/setup_corporal.yml index 583c27eba..e4fce8976 100644 --- a/roles/matrix-corporal/tasks/setup_corporal.yml +++ b/roles/matrix-corporal/tasks/setup_corporal.yml @@ -29,7 +29,7 @@ when: "matrix_corporal_enabled | bool and matrix_corporal_container_image_self_build | bool" - name: Ensure Matrix Corporal Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_corporal_docker_image }}" source: build force_source: "{{ matrix_corporal_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -41,7 +41,7 @@ when: "matrix_corporal_enabled | bool and matrix_corporal_container_image_self_build | bool" - name: Ensure Matrix Corporal Docker image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_corporal_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_corporal_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -115,7 +115,7 @@ when: "not matrix_corporal_enabled | bool" - name: Ensure Matrix Corporal Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_corporal_docker_image }}" state: absent when: "not matrix_corporal_enabled | bool" diff --git a/roles/matrix-coturn/tasks/setup_install.yml b/roles/matrix-coturn/tasks/setup_install.yml index be9d8574f..ef44c073f 100644 --- a/roles/matrix-coturn/tasks/setup_install.yml +++ b/roles/matrix-coturn/tasks/setup_install.yml @@ -18,7 +18,7 @@ when: "item.when | bool" - name: Ensure Coturn image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_coturn_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_coturn_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -42,7 +42,7 @@ register: matrix_coturn_git_pull_results - name: Ensure Coturn Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_coturn_docker_image }}" source: build force_source: "{{ matrix_coturn_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -69,7 +69,7 @@ group: "{{ matrix_user_groupname }}" - name: Ensure Coturn network is created in Docker - docker_network: + community.docker.docker_network: name: "{{ matrix_coturn_docker_network }}" driver: bridge diff --git a/roles/matrix-dendrite/tasks/dendrite/setup_install.yml b/roles/matrix-dendrite/tasks/dendrite/setup_install.yml index 3052e1010..98090e156 100644 --- a/roles/matrix-dendrite/tasks/dendrite/setup_install.yml +++ b/roles/matrix-dendrite/tasks/dendrite/setup_install.yml @@ -18,7 +18,7 @@ when: "not local_path_media_store_stat.failed and not local_path_media_store_stat.stat.exists" - name: Ensure Dendrite Docker image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_dendrite_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_dendrite_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-dendrite/tasks/dendrite/setup_uninstall.yml b/roles/matrix-dendrite/tasks/dendrite/setup_uninstall.yml index b6d8cfaca..6a2ea5b1d 100644 --- a/roles/matrix-dendrite/tasks/dendrite/setup_uninstall.yml +++ b/roles/matrix-dendrite/tasks/dendrite/setup_uninstall.yml @@ -25,6 +25,6 @@ when: "matrix_dendrite_service_stat.stat.exists" - name: Ensure Dendrite Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_dendrite_docker_image }}" state: absent diff --git a/roles/matrix-dimension/tasks/setup_install.yml b/roles/matrix-dimension/tasks/setup_install.yml index c55708365..fc4763973 100644 --- a/roles/matrix-dimension/tasks/setup_install.yml +++ b/roles/matrix-dimension/tasks/setup_install.yml @@ -87,7 +87,7 @@ group: "{{ matrix_dimension_user_gid }}" - name: Ensure Dimension image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_dimension_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_dimension_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -110,7 +110,7 @@ register: matrix_dimension_git_pull_results - name: Ensure Dimension Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_dimension_docker_image }}" source: build force_source: "{{ matrix_dimension_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-dimension/tasks/setup_uninstall.yml b/roles/matrix-dimension/tasks/setup_uninstall.yml index 3e2026a11..c939e66ae 100644 --- a/roles/matrix-dimension/tasks/setup_uninstall.yml +++ b/roles/matrix-dimension/tasks/setup_uninstall.yml @@ -31,6 +31,6 @@ state: absent - name: Ensure Dimension Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_dimension_docker_image }}" state: absent diff --git a/roles/matrix-dynamic-dns/tasks/install.yml b/roles/matrix-dynamic-dns/tasks/install.yml index e83637bfc..4be6d9f07 100644 --- a/roles/matrix-dynamic-dns/tasks/install.yml +++ b/roles/matrix-dynamic-dns/tasks/install.yml @@ -1,7 +1,7 @@ --- - name: Ensure Dynamic DNS image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_dynamic_dns_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_dynamic_dns_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -37,7 +37,7 @@ when: "matrix_dynamic_dns_enabled | bool and matrix_dynamic_dns_container_image_self_build | bool" - name: Ensure Dynamic DNS Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_dynamic_dns_docker_image }}" source: build force_source: "{{ matrix_dynamic_dns_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-email2matrix/tasks/setup_install.yml b/roles/matrix-email2matrix/tasks/setup_install.yml index 2a782a1a8..a6399a4ef 100644 --- a/roles/matrix-email2matrix/tasks/setup_install.yml +++ b/roles/matrix-email2matrix/tasks/setup_install.yml @@ -22,7 +22,7 @@ mode: 0640 - name: Ensure Email2Matrix image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_email2matrix_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_email2matrix_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -45,7 +45,7 @@ when: "matrix_email2matrix_container_image_self_build | bool" - name: Ensure Email2Matrix Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_email2matrix_docker_image }}" source: build force_source: "{{ matrix_email2matrix_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-email2matrix/tasks/setup_uninstall.yml b/roles/matrix-email2matrix/tasks/setup_uninstall.yml index a713a65ad..6aec40d2e 100644 --- a/roles/matrix-email2matrix/tasks/setup_uninstall.yml +++ b/roles/matrix-email2matrix/tasks/setup_uninstall.yml @@ -31,6 +31,6 @@ state: absent - name: Ensure Email2Matrix Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_email2matrix_docker_image }}" state: absent diff --git a/roles/matrix-etherpad/tasks/setup_install.yml b/roles/matrix-etherpad/tasks/setup_install.yml index 0243e9d7e..4974bd47b 100644 --- a/roles/matrix-etherpad/tasks/setup_install.yml +++ b/roles/matrix-etherpad/tasks/setup_install.yml @@ -17,7 +17,7 @@ group: "{{ matrix_etherpad_user_gid }}" - name: Ensure Etherpad image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_etherpad_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_etherpad_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-etherpad/tasks/setup_uninstall.yml b/roles/matrix-etherpad/tasks/setup_uninstall.yml index 38697366c..1a5d003e1 100644 --- a/roles/matrix-etherpad/tasks/setup_uninstall.yml +++ b/roles/matrix-etherpad/tasks/setup_uninstall.yml @@ -31,6 +31,6 @@ state: absent - name: Ensure Etherpad Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_etherpad_docker_image }}" state: absent diff --git a/roles/matrix-grafana/tasks/setup.yml b/roles/matrix-grafana/tasks/setup.yml index 25378ce7e..9198ffd8c 100644 --- a/roles/matrix-grafana/tasks/setup.yml +++ b/roles/matrix-grafana/tasks/setup.yml @@ -5,7 +5,7 @@ # - name: Ensure matrix-grafana image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_grafana_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_grafana_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-jitsi/tasks/setup_jitsi_jicofo.yml b/roles/matrix-jitsi/tasks/setup_jitsi_jicofo.yml index 8b2ec6a7a..5654fe3d5 100644 --- a/roles/matrix-jitsi/tasks/setup_jitsi_jicofo.yml +++ b/roles/matrix-jitsi/tasks/setup_jitsi_jicofo.yml @@ -17,7 +17,7 @@ when: matrix_jitsi_enabled | bool and item.when - name: Ensure jitsi-jicofo Docker image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_jitsi_jicofo_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_jitsi_jicofo_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-jitsi/tasks/setup_jitsi_jvb.yml b/roles/matrix-jitsi/tasks/setup_jitsi_jvb.yml index cdb94ebdd..9a50f8c62 100644 --- a/roles/matrix-jitsi/tasks/setup_jitsi_jvb.yml +++ b/roles/matrix-jitsi/tasks/setup_jitsi_jvb.yml @@ -17,7 +17,7 @@ when: matrix_jitsi_enabled | bool and item.when - name: Ensure jitsi-jvb Docker image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_jitsi_jvb_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_jitsi_jvb_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-jitsi/tasks/setup_jitsi_prosody.yml b/roles/matrix-jitsi/tasks/setup_jitsi_prosody.yml index 9383b48b6..8ba99f91d 100644 --- a/roles/matrix-jitsi/tasks/setup_jitsi_prosody.yml +++ b/roles/matrix-jitsi/tasks/setup_jitsi_prosody.yml @@ -18,7 +18,7 @@ when: matrix_jitsi_enabled | bool and item.when - name: Ensure jitsi-prosody Docker image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_jitsi_prosody_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_jitsi_prosody_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-jitsi/tasks/setup_jitsi_web.yml b/roles/matrix-jitsi/tasks/setup_jitsi_web.yml index 1c7daa4b9..9326ee610 100644 --- a/roles/matrix-jitsi/tasks/setup_jitsi_web.yml +++ b/roles/matrix-jitsi/tasks/setup_jitsi_web.yml @@ -19,7 +19,7 @@ when: matrix_jitsi_enabled | bool and item.when - name: Ensure jitsi-web Docker image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_jitsi_web_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_jitsi_web_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-ldap-registration-proxy/tasks/setup_install.yml b/roles/matrix-ldap-registration-proxy/tasks/setup_install.yml index 870373370..97b7e8eb0 100644 --- a/roles/matrix-ldap-registration-proxy/tasks/setup_install.yml +++ b/roles/matrix-ldap-registration-proxy/tasks/setup_install.yml @@ -26,7 +26,7 @@ register: matrix_ldap_registration_proxy_git_pull_results - name: Ensure matrix_ldap_registration_proxy Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_ldap_registration_proxy_docker_image }}" source: build force_source: "{{ matrix_ldap_registration_proxy_git_pull_results.changed }}" diff --git a/roles/matrix-ldap-registration-proxy/tasks/setup_uninstall.yml b/roles/matrix-ldap-registration-proxy/tasks/setup_uninstall.yml index 3225a3ae5..96ab0b67a 100644 --- a/roles/matrix-ldap-registration-proxy/tasks/setup_uninstall.yml +++ b/roles/matrix-ldap-registration-proxy/tasks/setup_uninstall.yml @@ -31,6 +31,6 @@ state: absent - name: Ensure matrix_ldap_registration_proxy Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_ldap_registration_proxy_docker_image }}" state: absent diff --git a/roles/matrix-ma1sd/tasks/setup_install.yml b/roles/matrix-ma1sd/tasks/setup_install.yml index 5f4b29574..38ff3036c 100644 --- a/roles/matrix-ma1sd/tasks/setup_install.yml +++ b/roles/matrix-ma1sd/tasks/setup_install.yml @@ -48,7 +48,7 @@ matrix_ma1sd_requires_restart: true - name: Ensure ma1sd image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_ma1sd_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_ma1sd_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -75,7 +75,7 @@ when: ansible_os_family == 'RedHat' - name: Ensure gradle is installed for self-building (Archlinux) - pacman: + community.general.pacman: name: - gradle state: present @@ -101,7 +101,7 @@ when: matrix_ma1sd_git_pull_results.changed - name: Ensure ma1sd Docker image is tagged correctly - docker_image: + community.docker.docker_image: # The build script always tags the image with 2 tags: # - based on the branch/version: e.g. `ma1uta/ma1sd:2.4.0` (when on `2.4.0`) # or `ma1uta/ma1sd:2.4.0-19-ga71d32b` (when on a given commit for a pre-release) diff --git a/roles/matrix-ma1sd/tasks/setup_uninstall.yml b/roles/matrix-ma1sd/tasks/setup_uninstall.yml index 2bc505b0f..0349ec32e 100644 --- a/roles/matrix-ma1sd/tasks/setup_uninstall.yml +++ b/roles/matrix-ma1sd/tasks/setup_uninstall.yml @@ -31,6 +31,6 @@ state: absent - name: Ensure ma1sd Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_ma1sd_docker_image }}" state: absent diff --git a/roles/matrix-mailer/tasks/setup_mailer.yml b/roles/matrix-mailer/tasks/setup_mailer.yml index 2ab39df5d..36ec80164 100644 --- a/roles/matrix-mailer/tasks/setup_mailer.yml +++ b/roles/matrix-mailer/tasks/setup_mailer.yml @@ -35,7 +35,7 @@ when: "matrix_mailer_enabled | bool and matrix_mailer_container_image_self_build | bool" - name: Ensure exim-relay Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_mailer_docker_image }}" source: build force_source: "{{ matrix_mailer_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -47,7 +47,7 @@ when: "matrix_mailer_enabled | bool and matrix_mailer_container_image_self_build | bool" - name: Ensure exim-relay image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_mailer_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_mailer_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -108,7 +108,7 @@ when: "not matrix_mailer_enabled | bool" - name: Ensure mailer Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_mailer_docker_image }}" state: absent when: "not matrix_mailer_enabled | bool" diff --git a/roles/matrix-nginx-proxy/tasks/nginx-proxy/setup_metrics_auth.yml b/roles/matrix-nginx-proxy/tasks/nginx-proxy/setup_metrics_auth.yml index c2215ebad..6129a49fa 100644 --- a/roles/matrix-nginx-proxy/tasks/nginx-proxy/setup_metrics_auth.yml +++ b/roles/matrix-nginx-proxy/tasks/nginx-proxy/setup_metrics_auth.yml @@ -18,7 +18,7 @@ - when: matrix_nginx_proxy_proxy_matrix_metrics_basic_auth_username != '' block: - name: Ensure Apache Docker image is pulled for generating matrix-metrics-htpasswd from username/password (protecting /metrics/* URIs) - docker_image: + community.docker.docker_image: name: "{{ matrix_nginx_proxy_proxy_matrix_metrics_basic_auth_apache_container_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_nginx_proxy_proxy_matrix_metrics_basic_auth_apache_container_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-nginx-proxy/tasks/setup_nginx_proxy.yml b/roles/matrix-nginx-proxy/tasks/setup_nginx_proxy.yml index 4d93e769c..11a1cc069 100644 --- a/roles/matrix-nginx-proxy/tasks/setup_nginx_proxy.yml +++ b/roles/matrix-nginx-proxy/tasks/setup_nginx_proxy.yml @@ -193,7 +193,7 @@ # Tasks related to setting up matrix-nginx-proxy # - name: Ensure nginx Docker image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_nginx_proxy_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_nginx_proxy_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-nginx-proxy/tasks/ssl/setup_ssl_lets_encrypt.yml b/roles/matrix-nginx-proxy/tasks/ssl/setup_ssl_lets_encrypt.yml index 62430f4bb..b212752c9 100644 --- a/roles/matrix-nginx-proxy/tasks/ssl/setup_ssl_lets_encrypt.yml +++ b/roles/matrix-nginx-proxy/tasks/ssl/setup_ssl_lets_encrypt.yml @@ -22,7 +22,7 @@ ansible.builtin.import_tasks: "{{ role_path }}/tasks/ssl/purge_ssl_lets_encrypt_orphaned_configs.yml" - name: Ensure certbot Docker image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_ssl_lets_encrypt_certbot_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_ssl_lets_encrypt_certbot_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-ntfy/tasks/setup_install.yml b/roles/matrix-ntfy/tasks/setup_install.yml index 9afabc4cd..ef50c42a1 100644 --- a/roles/matrix-ntfy/tasks/setup_install.yml +++ b/roles/matrix-ntfy/tasks/setup_install.yml @@ -1,7 +1,7 @@ --- - name: Ensure matrix-ntfy image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_ntfy_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_ntfy_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-ntfy/tasks/setup_uninstall.yml b/roles/matrix-ntfy/tasks/setup_uninstall.yml index f6d9cc8aa..d5da1d8e6 100644 --- a/roles/matrix-ntfy/tasks/setup_uninstall.yml +++ b/roles/matrix-ntfy/tasks/setup_uninstall.yml @@ -31,6 +31,6 @@ state: absent - name: Ensure ntfy Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_ntfy_docker_image }}" state: absent diff --git a/roles/matrix-postgres-backup/tasks/setup_postgres_backup.yml b/roles/matrix-postgres-backup/tasks/setup_postgres_backup.yml index a62467572..2518326a3 100644 --- a/roles/matrix-postgres-backup/tasks/setup_postgres_backup.yml +++ b/roles/matrix-postgres-backup/tasks/setup_postgres_backup.yml @@ -17,7 +17,7 @@ when: matrix_postgres_backup_enabled | bool - name: Ensure postgres backup Docker image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_postgres_backup_docker_image_to_use }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_postgres_backup_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-postgres/tasks/migrate_db_to_postgres.yml b/roles/matrix-postgres/tasks/migrate_db_to_postgres.yml index dd75fb17c..215d36c10 100644 --- a/roles/matrix-postgres/tasks/migrate_db_to_postgres.yml +++ b/roles/matrix-postgres/tasks/migrate_db_to_postgres.yml @@ -61,7 +61,7 @@ replace: 'FROM debian:bullseye-slim' - name: Ensure pgloader Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_postgres_pgloader_docker_image }}" source: build force_source: "{{ matrix_postgres_pgloader_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -72,7 +72,7 @@ pull: true - name: Ensure pgloader Docker image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_postgres_pgloader_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_postgres_pgloader_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-postgres/tasks/setup_postgres.yml b/roles/matrix-postgres/tasks/setup_postgres.yml index 49eb3249b..9efc73a66 100644 --- a/roles/matrix-postgres/tasks/setup_postgres.yml +++ b/roles/matrix-postgres/tasks/setup_postgres.yml @@ -37,7 +37,7 @@ # Even if we don't run the internal server, we still need this for running the CLI - name: Ensure postgres Docker image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_postgres_docker_image_to_use }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_postgres_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-prometheus-node-exporter/tasks/setup.yml b/roles/matrix-prometheus-node-exporter/tasks/setup.yml index 0c6e77fcb..370b23d13 100644 --- a/roles/matrix-prometheus-node-exporter/tasks/setup.yml +++ b/roles/matrix-prometheus-node-exporter/tasks/setup.yml @@ -5,7 +5,7 @@ # - name: Ensure matrix-prometheus-node-exporter image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_prometheus_node_exporter_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_prometheus_node_exporter_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-prometheus-postgres-exporter/tasks/setup.yml b/roles/matrix-prometheus-postgres-exporter/tasks/setup.yml index 00a61df62..dda614da8 100644 --- a/roles/matrix-prometheus-postgres-exporter/tasks/setup.yml +++ b/roles/matrix-prometheus-postgres-exporter/tasks/setup.yml @@ -5,7 +5,7 @@ # - name: Ensure matrix-prometheus-postgres-exporter image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_prometheus_postgres_exporter_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_prometheus_postgres_exporter_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-prometheus/tasks/setup_install.yml b/roles/matrix-prometheus/tasks/setup_install.yml index c3aeaa7a6..928b1ab5f 100644 --- a/roles/matrix-prometheus/tasks/setup_install.yml +++ b/roles/matrix-prometheus/tasks/setup_install.yml @@ -1,7 +1,7 @@ --- - name: Ensure matrix-prometheus image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_prometheus_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_prometheus_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-redis/tasks/setup_redis.yml b/roles/matrix-redis/tasks/setup_redis.yml index 7dd7ea9f2..f3b047c09 100644 --- a/roles/matrix-redis/tasks/setup_redis.yml +++ b/roles/matrix-redis/tasks/setup_redis.yml @@ -5,7 +5,7 @@ # - name: Ensure redis Docker image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_redis_docker_image_to_use }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_redis_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-registration/tasks/setup_install.yml b/roles/matrix-registration/tasks/setup_install.yml index 6b895d694..2f630ef00 100644 --- a/roles/matrix-registration/tasks/setup_install.yml +++ b/roles/matrix-registration/tasks/setup_install.yml @@ -48,7 +48,7 @@ when: "item.when | bool" - name: Ensure matrix-registration image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_registration_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_registration_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -79,7 +79,7 @@ when: "matrix_registration_container_image_self_build | bool and matrix_registration_container_image_self_build_python_dependencies_patch_enabled | bool" - name: Ensure matrix-registration Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_registration_docker_image }}" source: build force_source: "{{ matrix_registration_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-registration/tasks/setup_uninstall.yml b/roles/matrix-registration/tasks/setup_uninstall.yml index e3d713dc5..70e4fe97e 100644 --- a/roles/matrix-registration/tasks/setup_uninstall.yml +++ b/roles/matrix-registration/tasks/setup_uninstall.yml @@ -26,6 +26,6 @@ when: "matrix_registration_service_stat.stat.exists | bool" - name: Ensure matrix-registration Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_registration_docker_image }}" state: absent diff --git a/roles/matrix-sygnal/tasks/setup_install.yml b/roles/matrix-sygnal/tasks/setup_install.yml index 8f5f69377..e5c41bf00 100644 --- a/roles/matrix-sygnal/tasks/setup_install.yml +++ b/roles/matrix-sygnal/tasks/setup_install.yml @@ -1,7 +1,7 @@ --- - name: Ensure Sygnal image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_sygnal_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_sygnal_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-sygnal/tasks/setup_uninstall.yml b/roles/matrix-sygnal/tasks/setup_uninstall.yml index eff4a74b8..dc752def3 100644 --- a/roles/matrix-sygnal/tasks/setup_uninstall.yml +++ b/roles/matrix-sygnal/tasks/setup_uninstall.yml @@ -31,6 +31,6 @@ state: absent - name: Ensure Sygnal Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_sygnal_docker_image }}" state: absent diff --git a/roles/matrix-synapse-admin/tasks/setup.yml b/roles/matrix-synapse-admin/tasks/setup.yml index 660212724..1dfa68a24 100644 --- a/roles/matrix-synapse-admin/tasks/setup.yml +++ b/roles/matrix-synapse-admin/tasks/setup.yml @@ -5,7 +5,7 @@ # - name: Ensure matrix-synapse-admin image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_synapse_admin_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_synapse_admin_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -28,7 +28,7 @@ when: "matrix_synapse_admin_enabled | bool and matrix_synapse_admin_container_image_self_build | bool" - name: Ensure matrix-synapse-admin Docker image is built - docker_image: + community.docker.docker_image: name: "{{ matrix_synapse_admin_docker_image }}" source: build force_source: "{{ matrix_synapse_admin_git_pull_results.changed if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -82,7 +82,7 @@ when: "not matrix_synapse_admin_enabled | bool and matrix_synapse_admin_service_stat.stat.exists" - name: Ensure matrix-synapse-admin Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_synapse_admin_docker_image }}" state: absent when: "not matrix_synapse_admin_enabled | bool" diff --git a/roles/matrix-synapse/tasks/ext/mjolnir-antispam/setup_install.yml b/roles/matrix-synapse/tasks/ext/mjolnir-antispam/setup_install.yml index 3869f1aa7..117fb7576 100644 --- a/roles/matrix-synapse/tasks/ext/mjolnir-antispam/setup_install.yml +++ b/roles/matrix-synapse/tasks/ext/mjolnir-antispam/setup_install.yml @@ -17,7 +17,7 @@ when: "ansible_os_family == 'Debian'" - name: Ensure git installed (Archlinux) - pacman: + community.general.pacman: name: - git state: present diff --git a/roles/matrix-synapse/tasks/ext/synapse-simple-antispam/setup_install.yml b/roles/matrix-synapse/tasks/ext/synapse-simple-antispam/setup_install.yml index 34a7a7e06..dd3ff2d3f 100644 --- a/roles/matrix-synapse/tasks/ext/synapse-simple-antispam/setup_install.yml +++ b/roles/matrix-synapse/tasks/ext/synapse-simple-antispam/setup_install.yml @@ -22,7 +22,7 @@ when: "ansible_os_family == 'Debian'" - name: Ensure git installed (Archlinux) - pacman: + community.general.pacman: name: - git state: present diff --git a/roles/matrix-synapse/tasks/goofys/setup_install.yml b/roles/matrix-synapse/tasks/goofys/setup_install.yml index e3c341502..01eee8b87 100644 --- a/roles/matrix-synapse/tasks/goofys/setup_install.yml +++ b/roles/matrix-synapse/tasks/goofys/setup_install.yml @@ -3,7 +3,7 @@ - ansible.builtin.import_tasks: "{{ role_path }}/../matrix-base/tasks/util/ensure_fuse_installed.yml" - name: Ensure Goofys Docker image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_s3_goofys_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_s3_goofys_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-synapse/tasks/goofys/setup_uninstall.yml b/roles/matrix-synapse/tasks/goofys/setup_uninstall.yml index da78003f5..1e9f166dc 100644 --- a/roles/matrix-synapse/tasks/goofys/setup_uninstall.yml +++ b/roles/matrix-synapse/tasks/goofys/setup_uninstall.yml @@ -31,6 +31,6 @@ state: absent - name: Ensure Goofys Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ matrix_s3_goofys_docker_image }}" state: absent diff --git a/roles/matrix-synapse/tasks/import_media_store.yml b/roles/matrix-synapse/tasks/import_media_store.yml index 36ab9779b..740eb5102 100644 --- a/roles/matrix-synapse/tasks/import_media_store.yml +++ b/roles/matrix-synapse/tasks/import_media_store.yml @@ -51,7 +51,7 @@ # This can only work with local files, not if the media store is on Amazon S3, # as it won't be accessible in such a case. - name: Ensure provided media store directory is synchronized - synchronize: + ansible.builtin.synchronize: src: "{{ server_path_media_store }}/" dest: "{{ matrix_synapse_media_store_path }}" delete: true diff --git a/roles/matrix-synapse/tasks/rust-synapse-compress-state/main.yml b/roles/matrix-synapse/tasks/rust-synapse-compress-state/main.yml index 80c25f7fa..c8138c0bd 100644 --- a/roles/matrix-synapse/tasks/rust-synapse-compress-state/main.yml +++ b/roles/matrix-synapse/tasks/rust-synapse-compress-state/main.yml @@ -43,7 +43,7 @@ group: "{{ matrix_user_groupname }}" - name: Ensure rust-synapse-compress-state image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_synapse_rust_synapse_compress_state_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_synapse_rust_synapse_compress_state_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" diff --git a/roles/matrix-synapse/tasks/synapse/setup_install.yml b/roles/matrix-synapse/tasks/synapse/setup_install.yml index 4d0e749f9..86395db2d 100644 --- a/roles/matrix-synapse/tasks/synapse/setup_install.yml +++ b/roles/matrix-synapse/tasks/synapse/setup_install.yml @@ -51,7 +51,7 @@ when: "matrix_synapse_git_pull_results.changed | bool or matrix_synapse_docker_image_check_result.stdout == ''" - name: Ensure Synapse Docker image is pulled - docker_image: + community.docker.docker_image: name: "{{ matrix_synapse_docker_image }}" source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_synapse_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" @@ -73,7 +73,7 @@ mode: 0640 - name: Ensure customized Docker image for Synapse is built - docker_image: + community.docker.docker_image: name: "{{ matrix_synapse_docker_image_customized }}" source: build build: diff --git a/roles/matrix-synapse/tasks/synapse/setup_uninstall.yml b/roles/matrix-synapse/tasks/synapse/setup_uninstall.yml index 06e550145..ac79f3709 100644 --- a/roles/matrix-synapse/tasks/synapse/setup_uninstall.yml +++ b/roles/matrix-synapse/tasks/synapse/setup_uninstall.yml @@ -26,7 +26,7 @@ when: "matrix_synapse_service_stat.stat.exists" - name: Ensure Synapse Docker image doesn't exist - docker_image: + community.docker.docker_image: name: "{{ item }}" state: absent with_items: From 7fb45a507d2abf0d34670dd72a342f3fd9bcfa75 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 28 Oct 2022 17:39:53 +0300 Subject: [PATCH 083/118] Make --tags=run-postgres-vacuum and --tags=upgrade-postgres not assume Synapse Fixes https://github.com/spantaleev/matrix-docker-ansible-deploy/issues/2211 --- group_vars/matrix_servers | 5 +++++ roles/matrix-postgres/defaults/main.yml | 4 ++++ roles/matrix-postgres/tasks/run_vacuum.yml | 11 ++++++----- roles/matrix-postgres/tasks/upgrade_postgres.yml | 11 +++++++---- 4 files changed, 22 insertions(+), 9 deletions(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index d4464df9f..18aa553c0 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -2112,6 +2112,11 @@ matrix_postgres_additional_databases: | }} +matrix_postgres_systemd_services_to_stop_for_maintenance_list: | + {{ + ['matrix-' + matrix_homeserver_implementation + '.service'] + }} + ###################################################################### # # /matrix-postgres diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml index 7b0660bbd..fe469f16c 100644 --- a/roles/matrix-postgres/defaults/main.yml +++ b/roles/matrix-postgres/defaults/main.yml @@ -15,6 +15,10 @@ matrix_postgres_db_name: "matrix" matrix_postgres_base_path: "{{ matrix_base_data_path }}/postgres" matrix_postgres_data_path: "{{ matrix_postgres_base_path }}/data" +# matrix_postgres_systemd_services_to_stop_for_maintenance_list specifies the list of systemd services to stop before vacuuming or upgrading. +# These services will be restarted after the operation completes. +matrix_postgres_systemd_services_to_stop_for_maintenance_list: [] + matrix_postgres_architecture: amd64 # matrix_postgres_docker_image_suffix controls whether we use Alpine-based images (`-alpine`) or the normal Debian-based images. diff --git a/roles/matrix-postgres/tasks/run_vacuum.yml b/roles/matrix-postgres/tasks/run_vacuum.yml index 439599828..97dec022a 100644 --- a/roles/matrix-postgres/tasks/run_vacuum.yml +++ b/roles/matrix-postgres/tasks/run_vacuum.yml @@ -67,11 +67,12 @@ - ansible.builtin.set_fact: matrix_postgres_synapse_was_running: "{{ ansible_facts.services['matrix-synapse.service'] | default(none) is not none and ansible_facts.services['matrix-synapse.service'].state == 'running' }}" -- name: Ensure matrix-synapse is stopped +- name: Ensure services are stopped ansible.builtin.service: - name: matrix-synapse + name: "{{ item }}" state: stopped daemon_reload: true + with_items: "{{ matrix_postgres_systemd_services_to_stop_for_maintenance_list }}" - name: Run Postgres vacuum command ansible.builtin.command: "{{ matrix_postgres_vacuum_command }}" @@ -85,9 +86,9 @@ - ansible.builtin.debug: var: "matrix_postgres_synapse_vacuum_result" -- name: Ensure matrix-synapse is started, if it previously was +- name: Ensure services are started ansible.builtin.service: - name: matrix-synapse + name: "{{ item }}" state: started daemon_reload: true - when: "matrix_postgres_synapse_was_running | bool" + with_items: "{{ matrix_postgres_systemd_services_to_stop_for_maintenance_list }}" diff --git a/roles/matrix-postgres/tasks/upgrade_postgres.yml b/roles/matrix-postgres/tasks/upgrade_postgres.yml index 2f228a4c2..3d22407ca 100644 --- a/roles/matrix-postgres/tasks/upgrade_postgres.yml +++ b/roles/matrix-postgres/tasks/upgrade_postgres.yml @@ -55,10 +55,12 @@ - ansible.builtin.debug: msg: "Upgrading database from {{ matrix_postgres_detected_version_corresponding_docker_image }} to {{ matrix_postgres_docker_image_latest }}" -- name: Ensure matrix-synapse is stopped +- name: Ensure services are stopped ansible.builtin.service: - name: matrix-synapse + name: "{{ item }}" state: stopped + daemon_reload: true + with_items: "{{ matrix_postgres_systemd_services_to_stop_for_maintenance_list }}" - name: Ensure matrix-postgres is started ansible.builtin.service: @@ -175,11 +177,12 @@ path: "{{ postgres_dump_dir }}/{{ postgres_dump_name }}" state: absent -- name: Ensure matrix-synapse is started +- name: Ensure services are started ansible.builtin.service: - name: matrix-synapse + name: "{{ item }}" state: started daemon_reload: true + with_items: "{{ matrix_postgres_systemd_services_to_stop_for_maintenance_list }}" - ansible.builtin.debug: msg: "NOTE: Your old Postgres data directory is preserved at `{{ postgres_auto_upgrade_backup_data_path }}`. You might want to get rid of it once you've confirmed that all is well." From 30180048c3b6f797617ddf7f12c20b00db14e5b1 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Fri, 28 Oct 2022 19:43:44 +0300 Subject: [PATCH 084/118] Upgrade Synapse (v1.69.0 -> v1.70.1) --- roles/matrix-synapse/defaults/main.yml | 2 +- roles/matrix-synapse/vars/workers.yml | 38 +++++++++++++++++--------- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index 5e5867eb8..2028d8b77 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -36,7 +36,7 @@ matrix_synapse_container_image_customizations_dockerfile_body_custom: '' matrix_synapse_docker_image: "{{ matrix_synapse_docker_image_name_prefix }}matrixdotorg/synapse:{{ matrix_synapse_docker_image_tag }}" matrix_synapse_docker_image_name_prefix: "{{ 'localhost/' if matrix_synapse_container_image_self_build else matrix_container_global_registry_prefix }}" -matrix_synapse_version: v1.69.0 +matrix_synapse_version: v1.70.1 matrix_synapse_docker_image_tag: "{{ matrix_synapse_version }}" matrix_synapse_docker_image_force_pull: "{{ matrix_synapse_docker_image.endswith(':latest') }}" diff --git a/roles/matrix-synapse/vars/workers.yml b/roles/matrix-synapse/vars/workers.yml index bc49e8354..2598caa68 100644 --- a/roles/matrix-synapse/vars/workers.yml +++ b/roles/matrix-synapse/vars/workers.yml @@ -43,6 +43,8 @@ matrix_synapse_workers_generic_worker_endpoints: - ^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/members$ - ^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/state$ - ^/_matrix/client/v1/rooms/.*/hierarchy$ + - ^/_matrix/client/(v1|unstable)/rooms/.*/relations/ + - ^/_matrix/client/v1/rooms/.*/threads$ - ^/_matrix/client/unstable/org.matrix.msc2716/rooms/.*/batch_send$ - ^/_matrix/client/unstable/im.nheko.summary/rooms/.*/summary$ - ^/_matrix/client/(r0|v3|unstable)/account/3pid$ @@ -136,8 +138,9 @@ matrix_synapse_workers_generic_worker_endpoints: # [#7530](https://github.com/matrix-org/synapse/issues/7530) and # [#9427](https://github.com/matrix-org/synapse/issues/9427). - # Note that a HTTP listener with `client` and `federation` resources must be - # configured in the `worker_listeners` option in the worker config. + # Note that a [HTTP listener](usage/configuration/config_documentation.md#listeners) + # with `client` and `federation` `resources` must be configured in the `worker_listeners` + # option in the worker config. # #### Load balancing @@ -177,7 +180,8 @@ matrix_synapse_workers_generic_worker_endpoints: # Additionally, the writing of specific streams (such as events) can be moved off # of the main process to a particular worker. - # To enable this, the worker must have a HTTP replication listener configured, + # To enable this, the worker must have a + # [HTTP `replication` listener](usage/configuration/config_documentation.md#listeners) configured, # have a `worker_name` and be listed in the `instance_map` config. The same worker # can handle multiple streams, but unless otherwise documented, each stream can only # have a single writer. @@ -266,7 +270,7 @@ matrix_synapse_workers_generic_worker_endpoints: # There is also support for moving background tasks to a separate # worker. Background tasks are run periodically or started via replication. Exactly # which tasks are configured to run depends on your Synapse configuration (e.g. if - # stats is enabled). + # stats is enabled). This worker doesn't handle any REST endpoints itself. # To enable this, the worker must have a `worker_name` and can be configured to run # background tasks. For example, to move background tasks to a dedicated worker, @@ -314,8 +318,8 @@ matrix_synapse_workers_generic_worker_endpoints: # #### Notifying Application Services # You can designate one generic worker to send output traffic to Application Services. - - # Specify its name in the shared configuration as follows: + # Doesn't handle any REST endpoints itself, but you should specify its name in the + # shared configuration as follows: # ```yaml # notify_appservices_from_worker: worker_name @@ -342,6 +346,12 @@ matrix_synapse_workers_generic_worker_endpoints: # - pusher_worker2 # ``` + # An example for a pusher instance: + + # ```yaml + # {{#include systemd-with-workers/workers/pusher_worker.yaml}} + # ``` + # ] # appservice worker (no API endpoints) [ @@ -371,6 +381,12 @@ matrix_synapse_workers_generic_worker_endpoints: # - federation_sender1 # - federation_sender2 # ``` + + # An example for a federation sender instance: + + # ```yaml + # {{#include systemd-with-workers/workers/federation_sender.yaml}} + # ``` # ] matrix_synapse_workers_media_repository_endpoints: @@ -392,16 +408,12 @@ matrix_synapse_workers_media_repository_endpoints: # media repository. Note that doing so will prevent the main process from being # able to handle the above endpoints. - # In the `media_repository` worker configuration file, configure the http listener to + # In the `media_repository` worker configuration file, configure the + # [HTTP listener](usage/configuration/config_documentation.md#listeners) to # expose the `media` resource. For example: # ```yaml - # worker_listeners: - # - type: http - # port: 8085 - # resources: - # - names: - # - media + # {{#include systemd-with-workers/workers/media_worker.yaml}} # ``` # Note that if running multiple media repositories they must be on the same server From e25464409dbc59e53d181ebcb41c37e3bf473e4e Mon Sep 17 00:00:00 2001 From: Aine <97398200+etkecc@users.noreply.github.com> Date: Fri, 28 Oct 2022 20:53:48 +0000 Subject: [PATCH 085/118] Update mautrix-signal 0.4.0 -> 0.4.1 ref: https://github.com/mautrix/signal/releases/tag/v0.4.1 --- roles/matrix-bridge-mautrix-signal/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bridge-mautrix-signal/defaults/main.yml b/roles/matrix-bridge-mautrix-signal/defaults/main.yml index 83b0a5ce0..24ba9b39d 100644 --- a/roles/matrix-bridge-mautrix-signal/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-signal/defaults/main.yml @@ -9,7 +9,7 @@ matrix_mautrix_signal_docker_repo: "https://mau.dev/mautrix/signal.git" matrix_mautrix_signal_docker_repo_version: "{{ 'master' if matrix_mautrix_signal_version == 'latest' else matrix_mautrix_signal_version }}" matrix_mautrix_signal_docker_src_files_path: "{{ matrix_base_data_path }}/mautrix-signal/docker-src" -matrix_mautrix_signal_version: v0.4.0 +matrix_mautrix_signal_version: v0.4.1 matrix_mautrix_signal_daemon_version: 0.23.0 # See: https://mau.dev/mautrix/signal/container_registry matrix_mautrix_signal_docker_image: "dock.mau.dev/mautrix/signal:{{ matrix_mautrix_signal_version }}" From 80e57417f497c3c37d53daa04f8ef6b010b96e19 Mon Sep 17 00:00:00 2001 From: ikkemaniac Date: Sun, 19 Jun 2022 11:30:28 +0200 Subject: [PATCH 086/118] To recover from a failure or allow to quickly reset to known state we need to know with what playbook-commit the sever is installed. This commit saves the GIT hash/status when we run the playbook in a file called git_hash.yml. It also backs up that file by copying it to the target machine. --- roles/matrix-base/defaults/main.yml | 4 ++ roles/matrix-base/tasks/setup_matrix_base.yml | 53 +++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/roles/matrix-base/defaults/main.yml b/roles/matrix-base/defaults/main.yml index eccda626c..9213c661c 100644 --- a/roles/matrix-base/defaults/main.yml +++ b/roles/matrix-base/defaults/main.yml @@ -298,3 +298,7 @@ run_setup: true run_self_check: true run_start: true run_stop: true + +# Saves the git hash in a file called git_hash.yml +# Set this to false if GIT is not installed on the local system (the system where the ansible command is run on) +git_save_hash: true \ No newline at end of file diff --git a/roles/matrix-base/tasks/setup_matrix_base.yml b/roles/matrix-base/tasks/setup_matrix_base.yml index 6eebe3c0e..5d3c5820f 100644 --- a/roles/matrix-base/tasks/setup_matrix_base.yml +++ b/roles/matrix-base/tasks/setup_matrix_base.yml @@ -19,6 +19,59 @@ mode: '0660' when: "matrix_vars_yml_snapshotting_enabled | bool" +- name: Save current git-repo status on the target to aid with restoring in case of problems + block: + - name: Get local git hash + delegate_to: 127.0.0.1 + become: false + register: git_describe + shell: + git describe + --always + --tags + --dirty + --long + --all + + - set_fact: + git_hash: "{{ git_describe.stdout }}" + + - name: Git hash + debug: + msg: "Git hash: {{ git_hash }}" + + - name: Save git hash in git_hash.yml + become: false + local_action: + copy + content="git_hash_last_run{{ ":" }} {{ git_hash }}\n" + dest="{{ matrix_vars_yml_snapshotting_src }}/git_hash.yml" + + - name: Copy git_hash.yml file to target + copy: + src: "{{ matrix_vars_yml_snapshotting_src }}/git_hash.yml" + dest: "{{ matrix_base_data_path }}/git_hash.yml" + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_groupname }}" + mode: '0660' + + rescue: + - name: GIT not found error + ansible.builtin.debug: + msg: >- + Couldn't find GIT on the local machine. Continuing without saving the GIT hash. + You can disable saving the GIT hash by setting 'git_save_hash: false' in vars.yml + when: "git_describe.stderr.find('not found') != -1" + + - name: GIT hash error + ansible.builtin.fail: + msg: >- + Error when trying to get the GIT hash. + You can disable saving the GIT hash by setting 'git_save_hash: false' in vars.yml + when: "git_describe.stderr.find('not found') == -1" + + when: "matrix_vars_yml_snapshotting_enabled|bool and git_save_hash|bool" + - name: Ensure Matrix network is created in Docker community.docker.docker_network: name: "{{ matrix_docker_network }}" From ab398276af1d1b82c685a2cbd3f017f4cac2683a Mon Sep 17 00:00:00 2001 From: Stefan Warnat Date: Sun, 30 Oct 2022 00:55:36 +0200 Subject: [PATCH 087/118] Always use resolver variable in nginx conf.d files --- .../nginx/conf.d/matrix-base-domain.conf.j2 | 2 +- .../nginx/conf.d/matrix-bot-buscarron.conf.j2 | 4 ++-- .../nginx/conf.d/matrix-bot-go-neb.conf.j2 | 4 ++-- .../nginx/conf.d/matrix-client-cinny.conf.j2 | 4 ++-- .../nginx/conf.d/matrix-client-element.conf.j2 | 4 ++-- .../nginx/conf.d/matrix-client-hydrogen.conf.j2 | 4 ++-- .../templates/nginx/conf.d/matrix-conduit.conf.j2 | 4 ++-- .../templates/nginx/conf.d/matrix-dendrite.conf.j2 | 4 ++-- .../nginx/conf.d/matrix-dimension.conf.j2 | 4 ++-- .../templates/nginx/conf.d/matrix-domain.conf.j2 | 14 +++++++------- .../templates/nginx/conf.d/matrix-grafana.conf.j2 | 4 ++-- .../templates/nginx/conf.d/matrix-jitsi.conf.j2 | 8 ++++---- .../templates/nginx/conf.d/matrix-ntfy.conf.j2 | 4 ++-- .../templates/nginx/conf.d/matrix-riot-web.conf.j2 | 2 +- .../templates/nginx/conf.d/matrix-sygnal.conf.j2 | 4 ++-- .../templates/nginx/conf.d/matrix-synapse.conf.j2 | 4 ++-- 16 files changed, 37 insertions(+), 37 deletions(-) diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-base-domain.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-base-domain.conf.j2 index 3aff997db..44978dc40 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-base-domain.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-base-domain.conf.j2 @@ -45,7 +45,7 @@ server { location /.well-known/acme-challenge { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "matrix-certbot:8080"; proxy_pass http://$backend; {% else %} diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-bot-buscarron.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-bot-buscarron.conf.j2 index 0ce1473be..4f0fd4a8f 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-bot-buscarron.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-bot-buscarron.conf.j2 @@ -24,7 +24,7 @@ location / { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "matrix-bot-buscarron:8080"; proxy_pass http://$backend; {% else %} @@ -51,7 +51,7 @@ server { location /.well-known/acme-challenge { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "matrix-certbot:8080"; proxy_pass http://$backend; {% else %} diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-bot-go-neb.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-bot-go-neb.conf.j2 index e5589f555..a62ddfc81 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-bot-go-neb.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-bot-go-neb.conf.j2 @@ -18,7 +18,7 @@ location / { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "matrix-bot-go-neb:4050"; proxy_pass http://$backend; {% else %} @@ -44,7 +44,7 @@ server { location /.well-known/acme-challenge { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "matrix-certbot:8080"; proxy_pass http://$backend; {% else %} diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-client-cinny.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-client-cinny.conf.j2 index df66349fe..2ec6eb1ba 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-client-cinny.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-client-cinny.conf.j2 @@ -24,7 +24,7 @@ location / { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "matrix-client-cinny:8080"; proxy_pass http://$backend; {% else %} @@ -51,7 +51,7 @@ server { location /.well-known/acme-challenge { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "matrix-certbot:8080"; proxy_pass http://$backend; {% else %} diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-client-element.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-client-element.conf.j2 index dea91b212..0beeae52b 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-client-element.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-client-element.conf.j2 @@ -26,7 +26,7 @@ location / { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "matrix-client-element:8080"; proxy_pass http://$backend; {% else %} @@ -53,7 +53,7 @@ server { location /.well-known/acme-challenge { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "matrix-certbot:8080"; proxy_pass http://$backend; {% else %} diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-client-hydrogen.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-client-hydrogen.conf.j2 index e9428c557..7a2e9dfac 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-client-hydrogen.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-client-hydrogen.conf.j2 @@ -24,7 +24,7 @@ location / { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "matrix-client-hydrogen:8080"; proxy_pass http://$backend; {% else %} @@ -51,7 +51,7 @@ server { location /.well-known/acme-challenge { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "matrix-certbot:8080"; proxy_pass http://$backend; {% else %} diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-conduit.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-conduit.conf.j2 index 2106acc48..6e7aca793 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-conduit.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-conduit.conf.j2 @@ -28,7 +28,7 @@ server { location / { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "{{ matrix_nginx_proxy_proxy_conduit_client_api_addr_with_container }}"; proxy_pass http://$backend; {% else %} @@ -59,7 +59,7 @@ server { location / { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "{{ matrix_nginx_proxy_proxy_conduit_federation_api_addr_with_container }}"; proxy_pass http://$backend; {% else %} diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-dendrite.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-dendrite.conf.j2 index 939156a31..10eccf373 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-dendrite.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-dendrite.conf.j2 @@ -28,7 +28,7 @@ server { location / { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "{{ matrix_nginx_proxy_proxy_dendrite_client_api_addr_with_container }}"; proxy_pass http://$backend; {% else %} @@ -59,7 +59,7 @@ server { location / { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "{{ matrix_nginx_proxy_proxy_dendrite_federation_api_addr_with_container }}"; proxy_pass http://$backend; {% else %} diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-dimension.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-dimension.conf.j2 index 07347be62..730fc4c1d 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-dimension.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-dimension.conf.j2 @@ -21,7 +21,7 @@ location / { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "matrix-dimension:8184"; proxy_pass http://$backend; {% else %} @@ -47,7 +47,7 @@ server { location /.well-known/acme-challenge { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "matrix-certbot:8080"; proxy_pass http://$backend; {% else %} diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-domain.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-domain.conf.j2 index 2895ba14a..63d45bc63 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-domain.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-domain.conf.j2 @@ -62,7 +62,7 @@ location ^~ /_matrix/corporal { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "{{ matrix_nginx_proxy_proxy_matrix_corporal_api_addr_with_container }}"; proxy_pass http://$backend; {% else %} @@ -80,7 +80,7 @@ location ^~ /_matrix/identity { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "{{ matrix_nginx_proxy_proxy_matrix_identity_api_addr_with_container }}"; proxy_pass http://$backend; {% else %} @@ -98,7 +98,7 @@ location ^~ /_matrix/client/r0/user_directory/search { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "{{ matrix_nginx_proxy_proxy_matrix_user_directory_search_addr_with_container }}"; proxy_pass http://$backend; {% else %} @@ -115,7 +115,7 @@ location ~ ^/_matrix/client/r0/register/(email|msisdn)/requestToken$ { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "{{ matrix_nginx_proxy_proxy_matrix_3pid_registration_addr_with_container }}"; proxy_pass http://$backend; {% else %} @@ -140,7 +140,7 @@ location ~* ^({{ matrix_nginx_proxy_proxy_matrix_client_api_forwarded_location_prefix_regexes|join('|') }}) { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "{{ matrix_nginx_proxy_proxy_matrix_client_api_addr_with_container }}"; proxy_pass http://$backend; {% else %} @@ -185,7 +185,7 @@ server { location /.well-known/acme-challenge { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "matrix-certbot:8080"; proxy_pass http://$backend; {% else %} @@ -288,7 +288,7 @@ server { location / { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "{{ matrix_nginx_proxy_proxy_matrix_federation_api_addr_with_container }}"; proxy_pass http://$backend; {% else %} diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-grafana.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-grafana.conf.j2 index def67f666..094180448 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-grafana.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-grafana.conf.j2 @@ -28,7 +28,7 @@ location / { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "matrix-grafana:3000"; proxy_pass http://$backend; {% else %} @@ -55,7 +55,7 @@ server { location /.well-known/acme-challenge { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "matrix-certbot:8080"; proxy_pass http://$backend; {% else %} diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-jitsi.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-jitsi.conf.j2 index 54b8ea436..aa4b6b446 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-jitsi.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-jitsi.conf.j2 @@ -21,7 +21,7 @@ location / { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "matrix-jitsi-web:80"; proxy_pass http://$backend; {% else %} @@ -36,7 +36,7 @@ # colibri (JVB) websockets location ~ ^/colibri-ws/([a-zA-Z0-9-\.]+)/(.*) { {% if matrix_nginx_proxy_enabled %} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "matrix-jitsi-jvb:9090"; proxy_pass http://$backend; {% else %} @@ -57,7 +57,7 @@ # XMPP websocket location = /xmpp-websocket { {% if matrix_nginx_proxy_enabled %} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend {{ matrix_jitsi_xmpp_bosh_url_base }}; proxy_pass $backend/xmpp-websocket; {% else %} @@ -89,7 +89,7 @@ server { location /.well-known/acme-challenge { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "matrix-certbot:8080"; proxy_pass http://$backend; {% else %} diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-ntfy.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-ntfy.conf.j2 index ae100edac..fbae47e17 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-ntfy.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-ntfy.conf.j2 @@ -20,7 +20,7 @@ location / { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "matrix-ntfy:8080"; proxy_pass http://$backend; {% else %} @@ -49,7 +49,7 @@ server { location /.well-known/acme-challenge { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "matrix-certbot:8080"; proxy_pass http://$backend; {% else %} diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-riot-web.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-riot-web.conf.j2 index 5bcbeba5b..990269134 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-riot-web.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-riot-web.conf.j2 @@ -36,7 +36,7 @@ server { location /.well-known/acme-challenge { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "matrix-certbot:8080"; proxy_pass http://$backend; {% else %} diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-sygnal.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-sygnal.conf.j2 index 0f33c0a77..e3c6a461a 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-sygnal.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-sygnal.conf.j2 @@ -19,7 +19,7 @@ location / { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "matrix-sygnal:6000"; proxy_pass http://$backend; {% else %} @@ -46,7 +46,7 @@ server { location /.well-known/acme-challenge { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "matrix-certbot:8080"; proxy_pass http://$backend; {% else %} diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 index 1d6f2106a..da1893299 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 @@ -150,7 +150,7 @@ server { location / { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "{{ matrix_nginx_proxy_proxy_synapse_client_api_addr_with_container }}"; proxy_pass http://$backend; {% else %} @@ -211,7 +211,7 @@ server { location / { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; + resolver {{ matrix_nginx_proxy_http_level_resolver }} valid=5s; set $backend "{{ matrix_nginx_proxy_proxy_synapse_federation_api_addr_with_container }}"; proxy_pass http://$backend; {% else %} From 678dfc6dc3afe990fc71decf65260115458371bc Mon Sep 17 00:00:00 2001 From: ikkemaniac Date: Sun, 30 Oct 2022 14:00:04 +0100 Subject: [PATCH 088/118] Rename var --- roles/matrix-base/defaults/main.yml | 2 +- roles/matrix-base/tasks/setup_matrix_base.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/roles/matrix-base/defaults/main.yml b/roles/matrix-base/defaults/main.yml index 9213c661c..22d89c231 100644 --- a/roles/matrix-base/defaults/main.yml +++ b/roles/matrix-base/defaults/main.yml @@ -301,4 +301,4 @@ run_stop: true # Saves the git hash in a file called git_hash.yml # Set this to false if GIT is not installed on the local system (the system where the ansible command is run on) -git_save_hash: true \ No newline at end of file +matrix_playbook_commit_hash_preservation_enabled: true \ No newline at end of file diff --git a/roles/matrix-base/tasks/setup_matrix_base.yml b/roles/matrix-base/tasks/setup_matrix_base.yml index 5d3c5820f..d443fee8d 100644 --- a/roles/matrix-base/tasks/setup_matrix_base.yml +++ b/roles/matrix-base/tasks/setup_matrix_base.yml @@ -60,17 +60,17 @@ ansible.builtin.debug: msg: >- Couldn't find GIT on the local machine. Continuing without saving the GIT hash. - You can disable saving the GIT hash by setting 'git_save_hash: false' in vars.yml + You can disable saving the GIT hash by setting 'matrix_playbook_commit_hash_preservation_enabled: false' in vars.yml when: "git_describe.stderr.find('not found') != -1" - name: GIT hash error ansible.builtin.fail: msg: >- Error when trying to get the GIT hash. - You can disable saving the GIT hash by setting 'git_save_hash: false' in vars.yml + You can disable saving the GIT hash by setting 'matrix_playbook_commit_hash_preservation_enabled: false' in vars.yml when: "git_describe.stderr.find('not found') == -1" - when: "matrix_vars_yml_snapshotting_enabled|bool and git_save_hash|bool" + when: "matrix_vars_yml_snapshotting_enabled|bool and matrix_playbook_commit_hash_preservation_enabled|bool" - name: Ensure Matrix network is created in Docker community.docker.docker_network: From 230eea678471e9b46b3a9069d5505c4499dc6f63 Mon Sep 17 00:00:00 2001 From: ikkemaniac Date: Sun, 30 Oct 2022 14:04:33 +0100 Subject: [PATCH 089/118] Save git_hash.yml only on the target --- roles/matrix-base/tasks/setup_matrix_base.yml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/roles/matrix-base/tasks/setup_matrix_base.yml b/roles/matrix-base/tasks/setup_matrix_base.yml index d443fee8d..a97738551 100644 --- a/roles/matrix-base/tasks/setup_matrix_base.yml +++ b/roles/matrix-base/tasks/setup_matrix_base.yml @@ -40,16 +40,9 @@ debug: msg: "Git hash: {{ git_hash }}" - - name: Save git hash in git_hash.yml - become: false - local_action: - copy - content="git_hash_last_run{{ ":" }} {{ git_hash }}\n" - dest="{{ matrix_vars_yml_snapshotting_src }}/git_hash.yml" - - - name: Copy git_hash.yml file to target + - name: Save git_hash.yml on target copy: - src: "{{ matrix_vars_yml_snapshotting_src }}/git_hash.yml" + content: "{{ git_hash }}" dest: "{{ matrix_base_data_path }}/git_hash.yml" owner: "{{ matrix_user_username }}" group: "{{ matrix_user_groupname }}" From 930867d50d7bf399c843c13211ed78473a430943 Mon Sep 17 00:00:00 2001 From: ikkemaniac Date: Sun, 30 Oct 2022 14:07:39 +0100 Subject: [PATCH 090/118] Remove matrix_vars_yml_snapshotting_enabled condition --- roles/matrix-base/tasks/setup_matrix_base.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-base/tasks/setup_matrix_base.yml b/roles/matrix-base/tasks/setup_matrix_base.yml index a97738551..6fe7e47d2 100644 --- a/roles/matrix-base/tasks/setup_matrix_base.yml +++ b/roles/matrix-base/tasks/setup_matrix_base.yml @@ -63,7 +63,7 @@ You can disable saving the GIT hash by setting 'matrix_playbook_commit_hash_preservation_enabled: false' in vars.yml when: "git_describe.stderr.find('not found') == -1" - when: "matrix_vars_yml_snapshotting_enabled|bool and matrix_playbook_commit_hash_preservation_enabled|bool" + when: "matrix_playbook_commit_hash_preservation_enabled|bool" - name: Ensure Matrix network is created in Docker community.docker.docker_network: From 3d902b7fe733957e966593775979ec22e349899e Mon Sep 17 00:00:00 2001 From: ikkemaniac Date: Sun, 30 Oct 2022 14:33:09 +0100 Subject: [PATCH 091/118] be more specific on GIT not found error --- roles/matrix-base/tasks/setup_matrix_base.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/matrix-base/tasks/setup_matrix_base.yml b/roles/matrix-base/tasks/setup_matrix_base.yml index 6fe7e47d2..59dbe087d 100644 --- a/roles/matrix-base/tasks/setup_matrix_base.yml +++ b/roles/matrix-base/tasks/setup_matrix_base.yml @@ -54,14 +54,14 @@ msg: >- Couldn't find GIT on the local machine. Continuing without saving the GIT hash. You can disable saving the GIT hash by setting 'matrix_playbook_commit_hash_preservation_enabled: false' in vars.yml - when: "git_describe.stderr.find('not found') != -1" + when: "git_describe.stderr.find('git: not found') != -1" - - name: GIT hash error + - name: Get GIT hash error ansible.builtin.fail: msg: >- Error when trying to get the GIT hash. You can disable saving the GIT hash by setting 'matrix_playbook_commit_hash_preservation_enabled: false' in vars.yml - when: "git_describe.stderr.find('not found') == -1" + when: "git_describe.stderr.find('git: not found') == -1" when: "matrix_playbook_commit_hash_preservation_enabled|bool" From fd25cf8f54e9341b005004f1dd991b9458c0be2e Mon Sep 17 00:00:00 2001 From: ikkemaniac Date: Sun, 30 Oct 2022 14:37:45 +0100 Subject: [PATCH 092/118] Update wording --- roles/matrix-base/defaults/main.yml | 11 ++++++----- roles/matrix-base/tasks/setup_matrix_base.yml | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/roles/matrix-base/defaults/main.yml b/roles/matrix-base/defaults/main.yml index 22d89c231..4ad5fe556 100644 --- a/roles/matrix-base/defaults/main.yml +++ b/roles/matrix-base/defaults/main.yml @@ -284,6 +284,11 @@ matrix_docker_installation_enabled: true # Possible values are "docker-ce" (default) and "docker.io" (Debian). matrix_docker_package_name: docker-ce +# Controls whether the current playbook's commit hash is saved in `git_hash.yml` on the target +# Set this to false if GIT is not installed on the local system (the system where the ansible command is run on) +# to suppres the warning message. +matrix_playbook_commit_hash_preservation_enabled: true + # Variables to Control which parts of our roles run. run_postgres_import: true run_postgres_upgrade: true @@ -297,8 +302,4 @@ run_dendrite_register_user: true run_setup: true run_self_check: true run_start: true -run_stop: true - -# Saves the git hash in a file called git_hash.yml -# Set this to false if GIT is not installed on the local system (the system where the ansible command is run on) -matrix_playbook_commit_hash_preservation_enabled: true \ No newline at end of file +run_stop: true \ No newline at end of file diff --git a/roles/matrix-base/tasks/setup_matrix_base.yml b/roles/matrix-base/tasks/setup_matrix_base.yml index 59dbe087d..44e4137c9 100644 --- a/roles/matrix-base/tasks/setup_matrix_base.yml +++ b/roles/matrix-base/tasks/setup_matrix_base.yml @@ -59,7 +59,7 @@ - name: Get GIT hash error ansible.builtin.fail: msg: >- - Error when trying to get the GIT hash. + Error when trying to get the GIT hash. Please consult the error message above. You can disable saving the GIT hash by setting 'matrix_playbook_commit_hash_preservation_enabled: false' in vars.yml when: "git_describe.stderr.find('git: not found') == -1" From 15fbc525cf4f70e8e1f9a0814e4b9fb671d18dd3 Mon Sep 17 00:00:00 2001 From: ikkemaniac Date: Sun, 30 Oct 2022 14:53:27 +0100 Subject: [PATCH 093/118] Typo --- roles/matrix-base/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-base/defaults/main.yml b/roles/matrix-base/defaults/main.yml index 4ad5fe556..bcc821a45 100644 --- a/roles/matrix-base/defaults/main.yml +++ b/roles/matrix-base/defaults/main.yml @@ -286,7 +286,7 @@ matrix_docker_package_name: docker-ce # Controls whether the current playbook's commit hash is saved in `git_hash.yml` on the target # Set this to false if GIT is not installed on the local system (the system where the ansible command is run on) -# to suppres the warning message. +# to suppress the warning message. matrix_playbook_commit_hash_preservation_enabled: true # Variables to Control which parts of our roles run. From 54e71f2012afa055f6a4f119b00ffd76d756cc7a Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 31 Oct 2022 12:38:43 +0200 Subject: [PATCH 094/118] Mention "matrix_hookshot_enabled: true" on the Hookshot documentation page --- docs/configuring-playbook-bridge-hookshot.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/configuring-playbook-bridge-hookshot.md b/docs/configuring-playbook-bridge-hookshot.md index 51d7a335f..5505d0ad1 100644 --- a/docs/configuring-playbook-bridge-hookshot.md +++ b/docs/configuring-playbook-bridge-hookshot.md @@ -13,13 +13,16 @@ Note: the playbook also supports [matrix-appservice-webhooks](configuring-playbo Refer to the [official instructions](https://matrix-org.github.io/matrix-hookshot/latest/setup.html) to learn what the individual options do. -1. For each of the services (GitHub, GitLab, Jira, Figma, generic webhooks) fill in the respective variables `matrix_hookshot_service_*` listed in [main.yml](/roles/matrix-bridge-hookshot/defaults/main.yml) as required. -2. Take special note of the `matrix_hookshot_*_enabled` variables. Services that need no further configuration are enabled by default (GitLab, Generic), while you must first add the required configuration and enable the others (GitHub, Jira, Figma). -3. If you're setting up the GitHub bridge, you'll need to generate and download a private key file after you created your GitHub app. Copy the contents of that file to the variable `matrix_hookshot_github_private_key` so the playbook can install it for you, or use one of the [other methods](#manage-github-private-key-with-matrix-aux-role) explained below. -4. If you've already installed Matrix services using the playbook before, you'll need to re-run it (`--tags=setup-all,start`). If not, proceed with [configuring other playbook services](configuring-playbook.md) and then with [Installing](installing.md). Get back to this guide once ready. Hookshot can be set up individually using the tag `setup-hookshot`. +1. Enable the bridge by adding `matrix_hookshot_enabled: true` to your `vars.yml` file +2. For each of the services (GitHub, GitLab, Jira, Figma, generic webhooks) fill in the respective variables `matrix_hookshot_service_*` listed in [main.yml](/roles/matrix-bridge-hookshot/defaults/main.yml) as required. +3. Take special note of the `matrix_hookshot_*_enabled` variables. Services that need no further configuration are enabled by default (GitLab, Generic), while you must first add the required configuration and enable the others (GitHub, Jira, Figma). +4. If you're setting up the GitHub bridge, you'll need to generate and download a private key file after you created your GitHub app. Copy the contents of that file to the variable `matrix_hookshot_github_private_key` so the playbook can install it for you, or use one of the [other methods](#manage-github-private-key-with-matrix-aux-role) explained below. +5. If you've already installed Matrix services using the playbook before, you'll need to re-run it (`--tags=setup-all,start`). If not, proceed with [configuring other playbook services](configuring-playbook.md) and then with [Installing](installing.md). Get back to this guide once ready. Hookshot can be set up individually using the tag `setup-hookshot`. Other configuration options are available via the `matrix_hookshot_configuration_extension_yaml` and `matrix_hookshot_registration_extension_yaml` variables, see the comments in [main.yml](/roles/matrix-bridge-hookshot/defaults/main.yml) for how to use them. +Finally, run the playbook (see [installing](installing.md)). + ## Usage From cada3ef48b05b35da11b714d8505eeb785173a19 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 31 Oct 2022 12:47:10 +0200 Subject: [PATCH 095/118] Use pre-built image for Hookshot on arm64 --- group_vars/matrix_servers | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 18aa553c0..e5de094fb 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -765,7 +765,7 @@ matrix_heisenbridge_systemd_wanted_services_list: | # We don't enable bridges by default. matrix_hookshot_enabled: false -matrix_hookshot_container_image_self_build: "{{ matrix_architecture not in ['amd64'] }}" +matrix_hookshot_container_image_self_build: "{{ matrix_architecture not in ['arm64', 'amd64'] }}" matrix_hookshot_appservice_token: "{{ '%s' | format(matrix_homeserver_generic_secret_key) | password_hash('sha512', 'hookshot.as.tok') | to_uuid }}" From 975dda54c9db16f7158f33d1263318626c9cf735 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 31 Oct 2022 21:34:14 +0200 Subject: [PATCH 096/118] Upgrade Dendrite (0.10.4 -> 0.10.5) --- roles/matrix-dendrite/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-dendrite/defaults/main.yml b/roles/matrix-dendrite/defaults/main.yml index dc0af2dc3..945c16197 100644 --- a/roles/matrix-dendrite/defaults/main.yml +++ b/roles/matrix-dendrite/defaults/main.yml @@ -6,7 +6,7 @@ matrix_dendrite_enabled: true matrix_dendrite_docker_image: "{{ matrix_dendrite_docker_image_name_prefix }}matrixdotorg/dendrite-monolith:{{ matrix_dendrite_docker_image_tag }}" matrix_dendrite_docker_image_name_prefix: "docker.io/" -matrix_dendrite_docker_image_tag: "v0.10.4" +matrix_dendrite_docker_image_tag: "v0.10.5" matrix_dendrite_docker_image_force_pull: "{{ matrix_dendrite_docker_image.endswith(':latest') }}" matrix_dendrite_base_path: "{{ matrix_base_data_path }}/dendrite" From ec45b873762826044c558e91764b8abfe3651368 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 31 Oct 2022 21:50:39 +0200 Subject: [PATCH 097/118] Add matrix_hookshot_feeds_pollTimeoutSeconds --- roles/matrix-bridge-hookshot/defaults/main.yml | 1 + roles/matrix-bridge-hookshot/templates/config.yml.j2 | 1 + 2 files changed, 2 insertions(+) diff --git a/roles/matrix-bridge-hookshot/defaults/main.yml b/roles/matrix-bridge-hookshot/defaults/main.yml index 1c390f340..775ef087b 100644 --- a/roles/matrix-bridge-hookshot/defaults/main.yml +++ b/roles/matrix-bridge-hookshot/defaults/main.yml @@ -131,6 +131,7 @@ matrix_hookshot_generic_user_id_prefix: '_webhooks_' matrix_hookshot_feeds_enabled: true # polling interval in seconds matrix_hookshot_feeds_interval: 600 +matrix_hookshot_feeds_pollTimeoutSeconds: 10 # There is no need to edit ports. use matrix_hookshot_container_http_host_bind_ports below to expose ports instead. diff --git a/roles/matrix-bridge-hookshot/templates/config.yml.j2 b/roles/matrix-bridge-hookshot/templates/config.yml.j2 index 5cd11a874..fb029efd6 100644 --- a/roles/matrix-bridge-hookshot/templates/config.yml.j2 +++ b/roles/matrix-bridge-hookshot/templates/config.yml.j2 @@ -84,6 +84,7 @@ feeds: # enabled: {{ matrix_hookshot_feeds_enabled }} pollIntervalSeconds: {{ matrix_hookshot_feeds_interval }} + pollTimeoutSeconds: {{ matrix_hookshot_feeds_pollTimeoutSeconds | to_json }} {% endif %} {% if matrix_hookshot_provisioning_enabled %} provisioning: From e1274a6e56ea3e06284b2867a76d951276c160c8 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Mon, 31 Oct 2022 21:52:01 +0200 Subject: [PATCH 098/118] Rename variable (matrix_hookshot_feeds_interval -> matrix_hookshot_feeds_pollIntervalSeconds) This is more consistent with how we name variables. It's also less confusing, especially given that we have `matrix_hookshot_feeds_pollTimeoutSeconds` as well. --- roles/matrix-bridge-hookshot/defaults/main.yml | 3 +-- roles/matrix-bridge-hookshot/tasks/validate_config.yml | 9 +++++++++ roles/matrix-bridge-hookshot/templates/config.yml.j2 | 4 ++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/roles/matrix-bridge-hookshot/defaults/main.yml b/roles/matrix-bridge-hookshot/defaults/main.yml index 775ef087b..96781f9aa 100644 --- a/roles/matrix-bridge-hookshot/defaults/main.yml +++ b/roles/matrix-bridge-hookshot/defaults/main.yml @@ -129,8 +129,7 @@ matrix_hookshot_generic_user_id_prefix: '_webhooks_' matrix_hookshot_feeds_enabled: true -# polling interval in seconds -matrix_hookshot_feeds_interval: 600 +matrix_hookshot_feeds_pollIntervalSeconds: 600 matrix_hookshot_feeds_pollTimeoutSeconds: 10 diff --git a/roles/matrix-bridge-hookshot/tasks/validate_config.yml b/roles/matrix-bridge-hookshot/tasks/validate_config.yml index 0fbcf53c2..3392f1b64 100644 --- a/roles/matrix-bridge-hookshot/tasks/validate_config.yml +++ b/roles/matrix-bridge-hookshot/tasks/validate_config.yml @@ -58,6 +58,15 @@ with_items: - "matrix_hookshot_provisioning_secret" +- name: (Deprecation) Catch and report renamed Hookshot variables + ansible.builtin.fail: + msg: >- + Your configuration contains a variable, which now has a different name. + Please change your configuration to rename the variable (`{{ item.old }}` -> `{{ item.new }}`). + when: "item.old in vars" + with_items: + - {'old': 'matrix_hookshot_feeds_interval', 'new': 'matrix_hookshot_feeds_pollIntervalSeconds'} + - name: (Deprecation) Catch and report old metrics usage ansible.builtin.fail: msg: >- diff --git a/roles/matrix-bridge-hookshot/templates/config.yml.j2 b/roles/matrix-bridge-hookshot/templates/config.yml.j2 index fb029efd6..527afafaf 100644 --- a/roles/matrix-bridge-hookshot/templates/config.yml.j2 +++ b/roles/matrix-bridge-hookshot/templates/config.yml.j2 @@ -82,8 +82,8 @@ generic: feeds: # (Optional) Configure this to enable RSS/Atom feed support # - enabled: {{ matrix_hookshot_feeds_enabled }} - pollIntervalSeconds: {{ matrix_hookshot_feeds_interval }} + enabled: {{ matrix_hookshot_feeds_enabled | to_json }} + pollIntervalSeconds: {{ matrix_hookshot_feeds_pollIntervalSeconds | to_json }} pollTimeoutSeconds: {{ matrix_hookshot_feeds_pollTimeoutSeconds | to_json }} {% endif %} {% if matrix_hookshot_provisioning_enabled %} From 39e6484956671080c1d7e2fbd94c6d7427669b66 Mon Sep 17 00:00:00 2001 From: ikkemaniac Date: Mon, 31 Oct 2022 22:57:16 +0100 Subject: [PATCH 099/118] fix lints --- roles/matrix-base/tasks/setup_matrix_base.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/roles/matrix-base/tasks/setup_matrix_base.yml b/roles/matrix-base/tasks/setup_matrix_base.yml index 44e4137c9..5c45323e5 100644 --- a/roles/matrix-base/tasks/setup_matrix_base.yml +++ b/roles/matrix-base/tasks/setup_matrix_base.yml @@ -21,11 +21,11 @@ - name: Save current git-repo status on the target to aid with restoring in case of problems block: - - name: Get local git hash + - name: Get local git hash delegate_to: 127.0.0.1 become: false register: git_describe - shell: + ansible.builtin.shell: git describe --always --tags @@ -33,15 +33,15 @@ --long --all - - set_fact: + - ansible.builtin.set_fact: git_hash: "{{ git_describe.stdout }}" - name: Git hash - debug: + ansible.builtin.debug: msg: "Git hash: {{ git_hash }}" - + - name: Save git_hash.yml on target - copy: + ansible.builtin.copy: content: "{{ git_hash }}" dest: "{{ matrix_base_data_path }}/git_hash.yml" owner: "{{ matrix_user_username }}" From 2b049da963ddab978e6be9fdc014d23f819ed272 Mon Sep 17 00:00:00 2001 From: ikkemaniac Date: Mon, 31 Oct 2022 23:00:04 +0100 Subject: [PATCH 100/118] move when clause per linting suggestion --- roles/matrix-base/tasks/setup_matrix_base.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/roles/matrix-base/tasks/setup_matrix_base.yml b/roles/matrix-base/tasks/setup_matrix_base.yml index 5c45323e5..cdde09d28 100644 --- a/roles/matrix-base/tasks/setup_matrix_base.yml +++ b/roles/matrix-base/tasks/setup_matrix_base.yml @@ -20,6 +20,7 @@ when: "matrix_vars_yml_snapshotting_enabled | bool" - name: Save current git-repo status on the target to aid with restoring in case of problems + when: "matrix_playbook_commit_hash_preservation_enabled|bool" block: - name: Get local git hash delegate_to: 127.0.0.1 @@ -63,8 +64,6 @@ You can disable saving the GIT hash by setting 'matrix_playbook_commit_hash_preservation_enabled: false' in vars.yml when: "git_describe.stderr.find('git: not found') == -1" - when: "matrix_playbook_commit_hash_preservation_enabled|bool" - - name: Ensure Matrix network is created in Docker community.docker.docker_network: name: "{{ matrix_docker_network }}" From 45c9c2b17a6e60f828b0d4a1723c4eb8a5aae1cb Mon Sep 17 00:00:00 2001 From: ikkemaniac Date: Mon, 31 Oct 2022 23:08:45 +0100 Subject: [PATCH 101/118] convert shell to command usage --- roles/matrix-base/tasks/setup_matrix_base.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/roles/matrix-base/tasks/setup_matrix_base.yml b/roles/matrix-base/tasks/setup_matrix_base.yml index cdde09d28..0aafe1834 100644 --- a/roles/matrix-base/tasks/setup_matrix_base.yml +++ b/roles/matrix-base/tasks/setup_matrix_base.yml @@ -26,13 +26,14 @@ delegate_to: 127.0.0.1 become: false register: git_describe - ansible.builtin.shell: - git describe - --always - --tags - --dirty - --long - --all + ansible.builtin.command: + cmd: >- + git describe + --always + --tags + --dirty + --long + --all - ansible.builtin.set_fact: git_hash: "{{ git_describe.stdout }}" From 93615b6ad988edb4a7972e32f0595d70fb2fd1e0 Mon Sep 17 00:00:00 2001 From: ikkemaniac Date: Mon, 31 Oct 2022 23:20:56 +0100 Subject: [PATCH 102/118] Revert "convert shell to command usage" to allow for a catching the specific "git: not found" error. This reverts commit 45c9c2b17a6e60f828b0d4a1723c4eb8a5aae1cb. --- roles/matrix-base/tasks/setup_matrix_base.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/roles/matrix-base/tasks/setup_matrix_base.yml b/roles/matrix-base/tasks/setup_matrix_base.yml index 0aafe1834..cdde09d28 100644 --- a/roles/matrix-base/tasks/setup_matrix_base.yml +++ b/roles/matrix-base/tasks/setup_matrix_base.yml @@ -26,14 +26,13 @@ delegate_to: 127.0.0.1 become: false register: git_describe - ansible.builtin.command: - cmd: >- - git describe - --always - --tags - --dirty - --long - --all + ansible.builtin.shell: + git describe + --always + --tags + --dirty + --long + --all - ansible.builtin.set_fact: git_hash: "{{ git_describe.stdout }}" From 8b425b995ec446109b6b13f711c85d11b57e8136 Mon Sep 17 00:00:00 2001 From: ikkemaniac Date: Mon, 31 Oct 2022 23:44:47 +0100 Subject: [PATCH 103/118] add newline char --- roles/matrix-base/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-base/defaults/main.yml b/roles/matrix-base/defaults/main.yml index bcc821a45..52049ed51 100644 --- a/roles/matrix-base/defaults/main.yml +++ b/roles/matrix-base/defaults/main.yml @@ -302,4 +302,4 @@ run_dendrite_register_user: true run_setup: true run_self_check: true run_start: true -run_stop: true \ No newline at end of file +run_stop: true From 3fc19295e2373a2149b0ebe67b48d63915c12faf Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 1 Nov 2022 07:05:26 +0200 Subject: [PATCH 104/118] Fix ansible-lint error (ansible.builtin.synchronize -> ansible.posix.synchronize) --- roles/matrix-synapse/tasks/import_media_store.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-synapse/tasks/import_media_store.yml b/roles/matrix-synapse/tasks/import_media_store.yml index 740eb5102..bdd99ca4d 100644 --- a/roles/matrix-synapse/tasks/import_media_store.yml +++ b/roles/matrix-synapse/tasks/import_media_store.yml @@ -51,7 +51,7 @@ # This can only work with local files, not if the media store is on Amazon S3, # as it won't be accessible in such a case. - name: Ensure provided media store directory is synchronized - ansible.builtin.synchronize: + ansible.posix.synchronize: src: "{{ server_path_media_store }}/" dest: "{{ matrix_synapse_media_store_path }}" delete: true From 3a49b0077c8b8704c6f7b1cf00b8c19d2e24270b Mon Sep 17 00:00:00 2001 From: ikkemaniac Date: Tue, 1 Nov 2022 09:23:55 +0100 Subject: [PATCH 105/118] fix ansible-lint errors --- roles/matrix-base/tasks/setup_matrix_base.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/matrix-base/tasks/setup_matrix_base.yml b/roles/matrix-base/tasks/setup_matrix_base.yml index cdde09d28..0bf197b66 100644 --- a/roles/matrix-base/tasks/setup_matrix_base.yml +++ b/roles/matrix-base/tasks/setup_matrix_base.yml @@ -22,10 +22,11 @@ - name: Save current git-repo status on the target to aid with restoring in case of problems when: "matrix_playbook_commit_hash_preservation_enabled|bool" block: - - name: Get local git hash + - name: Get local git hash # noqa command-instead-of-module delegate_to: 127.0.0.1 become: false register: git_describe + changed_when: false ansible.builtin.shell: git describe --always From f7b45fd4402a25db979da3086d053fd61816dcab Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 1 Nov 2022 12:58:22 +0200 Subject: [PATCH 106/118] Upgrade ddclient (v3.10.0-ls102 -> v3.10.0-ls103) --- roles/matrix-dynamic-dns/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-dynamic-dns/defaults/main.yml b/roles/matrix-dynamic-dns/defaults/main.yml index af4d95927..056eecba3 100644 --- a/roles/matrix-dynamic-dns/defaults/main.yml +++ b/roles/matrix-dynamic-dns/defaults/main.yml @@ -7,7 +7,7 @@ matrix_dynamic_dns_enabled: true # The dynamic dns daemon interval matrix_dynamic_dns_daemon_interval: '300' -matrix_dynamic_dns_version: v3.10.0-ls102 +matrix_dynamic_dns_version: v3.10.0-ls103 # The docker container to use when in mode matrix_dynamic_dns_docker_image: "{{ matrix_dynamic_dns_docker_image_name_prefix }}linuxserver/ddclient:{{ matrix_dynamic_dns_version }}" From 17caa13a71a36365343ba417c7d42d5d44d68dda Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 1 Nov 2022 12:59:15 +0200 Subject: [PATCH 107/118] Upgrade Grafana (9.2.2 -> 9.2.3) --- roles/matrix-grafana/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-grafana/defaults/main.yml b/roles/matrix-grafana/defaults/main.yml index 5bccb60d2..43b29c94a 100644 --- a/roles/matrix-grafana/defaults/main.yml +++ b/roles/matrix-grafana/defaults/main.yml @@ -5,7 +5,7 @@ matrix_grafana_enabled: true -matrix_grafana_version: 9.2.2 +matrix_grafana_version: 9.2.3 matrix_grafana_docker_image: "{{ matrix_container_global_registry_prefix }}grafana/grafana:{{ matrix_grafana_version }}" matrix_grafana_docker_image_force_pull: "{{ matrix_grafana_docker_image.endswith(':latest') }}" From 9fb21b89a8a7be51d014ad40fb6bb4cccbbb8dec Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 1 Nov 2022 14:57:29 +0200 Subject: [PATCH 108/118] Fix import-postgres/run-postgres-vacuum failures being suppressed We were only reporting failures for when the async task didn't finish. We also need to report a failure for when the task finished, but returned a non-zero exit code. --- roles/matrix-postgres/tasks/import_postgres.yml | 2 +- roles/matrix-postgres/tasks/run_vacuum.yml | 2 +- .../tasks/rust-synapse-compress-state/compress_room.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/matrix-postgres/tasks/import_postgres.yml b/roles/matrix-postgres/tasks/import_postgres.yml index 40877e0b5..302a8f411 100644 --- a/roles/matrix-postgres/tasks/import_postgres.yml +++ b/roles/matrix-postgres/tasks/import_postgres.yml @@ -108,5 +108,5 @@ async: "{{ postgres_import_wait_time }}" poll: 10 register: matrix_postgres_import_postgres_command_result - failed_when: not matrix_postgres_import_postgres_command_result.finished + failed_when: not matrix_postgres_import_postgres_command_result.finished or matrix_postgres_import_postgres_command_result.rc != 0 changed_when: matrix_postgres_import_postgres_command_result.finished and matrix_postgres_import_postgres_command_result.rc == 0 diff --git a/roles/matrix-postgres/tasks/run_vacuum.yml b/roles/matrix-postgres/tasks/run_vacuum.yml index 97dec022a..9801ab2c7 100644 --- a/roles/matrix-postgres/tasks/run_vacuum.yml +++ b/roles/matrix-postgres/tasks/run_vacuum.yml @@ -79,7 +79,7 @@ async: "{{ postgres_vacuum_wait_time }}" poll: 10 register: matrix_postgres_synapse_vacuum_result - failed_when: not matrix_postgres_synapse_vacuum_result.finished + failed_when: not matrix_postgres_synapse_vacuum_result.finished or matrix_postgres_synapse_vacuum_result.rc != 0 changed_when: matrix_postgres_synapse_vacuum_result.finished and matrix_postgres_synapse_vacuum_result.rc == 0 # Intentionally show the results diff --git a/roles/matrix-synapse/tasks/rust-synapse-compress-state/compress_room.yml b/roles/matrix-synapse/tasks/rust-synapse-compress-state/compress_room.yml index 9834e256a..88db265d3 100644 --- a/roles/matrix-synapse/tasks/rust-synapse-compress-state/compress_room.yml +++ b/roles/matrix-synapse/tasks/rust-synapse-compress-state/compress_room.yml @@ -21,7 +21,7 @@ async: "{{ matrix_synapse_rust_synapse_compress_state_compress_room_time }}" poll: 10 register: matrix_synapse_rust_synapse_compress_state_compress_room_command_result - failed_when: not matrix_synapse_rust_synapse_compress_state_compress_room_command_result.finished + failed_when: not matrix_synapse_rust_synapse_compress_state_compress_room_command_result.finished or matrix_synapse_rust_synapse_compress_state_compress_room_command_result.rc != 0 changed_when: matrix_synapse_rust_synapse_compress_state_compress_room_command_result.finished and matrix_synapse_rust_synapse_compress_state_compress_room_command_result.rc == 0 - ansible.builtin.debug: From 74b186a4543cd7c59ffcdf6f13c20da4590ea2dc Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 1 Nov 2022 15:05:41 +0200 Subject: [PATCH 109/118] Fix "too many spaces after colon" ansible-lint error --- roles/matrix-dynamic-dns/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-dynamic-dns/defaults/main.yml b/roles/matrix-dynamic-dns/defaults/main.yml index 056eecba3..77e01d0ee 100644 --- a/roles/matrix-dynamic-dns/defaults/main.yml +++ b/roles/matrix-dynamic-dns/defaults/main.yml @@ -7,7 +7,7 @@ matrix_dynamic_dns_enabled: true # The dynamic dns daemon interval matrix_dynamic_dns_daemon_interval: '300' -matrix_dynamic_dns_version: v3.10.0-ls103 +matrix_dynamic_dns_version: v3.10.0-ls103 # The docker container to use when in mode matrix_dynamic_dns_docker_image: "{{ matrix_dynamic_dns_docker_image_name_prefix }}linuxserver/ddclient:{{ matrix_dynamic_dns_version }}" From a4662660d2ca70cdadf496e1047fc50204937faf Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 1 Nov 2022 15:06:10 +0200 Subject: [PATCH 110/118] Do not wait needlessly when vacuuming or importing Postgres --- roles/matrix-postgres/tasks/import_postgres.yml | 4 +++- roles/matrix-postgres/tasks/run_vacuum.yml | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/roles/matrix-postgres/tasks/import_postgres.yml b/roles/matrix-postgres/tasks/import_postgres.yml index 302a8f411..107a1f887 100644 --- a/roles/matrix-postgres/tasks/import_postgres.yml +++ b/roles/matrix-postgres/tasks/import_postgres.yml @@ -49,8 +49,10 @@ name: matrix-postgres state: started daemon_reload: true + register: matrix_postgres_import_start_result -- name: Wait a bit, so that Postgres can start +- when: matrix_postgres_import_start_result.changed | bool + name: Wait a bit, so that Postgres can start ansible.builtin.wait_for: timeout: "{{ postgres_start_wait_time }}" delegate_to: 127.0.0.1 diff --git a/roles/matrix-postgres/tasks/run_vacuum.yml b/roles/matrix-postgres/tasks/run_vacuum.yml index 9801ab2c7..1b769f01f 100644 --- a/roles/matrix-postgres/tasks/run_vacuum.yml +++ b/roles/matrix-postgres/tasks/run_vacuum.yml @@ -28,8 +28,10 @@ name: matrix-postgres state: started daemon_reload: true + register: matrix_postgres_vacuum_start_result -- name: Wait a bit, so that Postgres can start +- when: matrix_postgres_vacuum_start_result.changed | bool + name: Wait a bit, so that Postgres can start ansible.builtin.wait_for: timeout: "{{ postgres_start_wait_time }}" delegate_to: 127.0.0.1 From 125ca5569d4cc81d8b0d9f07a9b299dd3852436c Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 1 Nov 2022 15:47:00 +0200 Subject: [PATCH 111/118] Use unique filter for matrix_postgres_import_roles_to_ignore and matrix_postgres_import_databases_to_ignore Dendrite uses a lot of databases, but a single (`dendrite`) role, which leads to `matrix_postgres_import_roles_to_ignore` being something like `['dendrite', 'dendrite', 'dendrite', ...]` needlessly. This leads to weird regexes being generated for `matrix_postgres_import_roles_ignore_regex`. It's not that it hurts, but it just looks odd. --- roles/matrix-postgres/defaults/main.yml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml index fe469f16c..5a0cdb6c6 100644 --- a/roles/matrix-postgres/defaults/main.yml +++ b/roles/matrix-postgres/defaults/main.yml @@ -74,9 +74,11 @@ matrix_postgres_additional_databases: [] # We either need to not create them or to ignore the `CREATE ROLE` statements in the dump. matrix_postgres_import_roles_to_ignore: | {{ - [matrix_postgres_connection_username] - + - matrix_postgres_additional_databases|map(attribute='username') | list + ( + [matrix_postgres_connection_username] + + + matrix_postgres_additional_databases|map(attribute='username') | list + ) | unique }} # When importing an existing Postgres database (when restoring a backup) or when doing a Postgres upgrade (which dumps & restores), we'd like to avoid: @@ -98,9 +100,11 @@ matrix_postgres_import_roles_ignore_regex: "^(CREATE|ALTER) ROLE \\\"?({{ matrix # We either need to not create them or to ignore the `CREATE DATABASE` statements in the dump. matrix_postgres_import_databases_to_ignore: | {{ - [matrix_postgres_db_name] - + - matrix_postgres_additional_databases|map(attribute='name') | list + ( + [matrix_postgres_db_name] + + + matrix_postgres_additional_databases|map(attribute='name') | list + ) | unique }} # We also allow for the database name to be quoted, which is rare, but might happen for database names which are special keywords (e.g. `default`). From 501420f3cc09d2fc071ff620fd949c10b3fa2109 Mon Sep 17 00:00:00 2001 From: Aine <97398200+etkecc@users.noreply.github.com> Date: Tue, 1 Nov 2022 14:18:41 +0000 Subject: [PATCH 112/118] Update element 1.11.12 -> 1.11.13 --- roles/matrix-client-element/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-client-element/defaults/main.yml b/roles/matrix-client-element/defaults/main.yml index 2d5c99d2b..b2ffd05da 100644 --- a/roles/matrix-client-element/defaults/main.yml +++ b/roles/matrix-client-element/defaults/main.yml @@ -10,7 +10,7 @@ matrix_client_element_container_image_self_build_repo: "https://github.com/vecto # - https://github.com/vector-im/element-web/issues/19544 matrix_client_element_container_image_self_build_low_memory_system_patch_enabled: "{{ ansible_memtotal_mb < 4096 }}" -matrix_client_element_version: v1.11.12 +matrix_client_element_version: v1.11.13 matrix_client_element_docker_image: "{{ matrix_client_element_docker_image_name_prefix }}vectorim/element-web:{{ matrix_client_element_version }}" matrix_client_element_docker_image_name_prefix: "{{ 'localhost/' if matrix_client_element_container_image_self_build else matrix_container_global_registry_prefix }}" matrix_client_element_docker_image_force_pull: "{{ matrix_client_element_docker_image.endswith(':latest') }}" From c3dc64b1d5e06fb356fce37dcab27664ef335ef7 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 1 Nov 2022 16:22:58 +0200 Subject: [PATCH 113/118] Add matrix-user-creator role - automatic user account creation support We no longer ask users to create Matrix user accounts for these bots: - Postmoogle - Honoroit - Reminder Bot Other bots and services (matrix-registration-bot, maubot, mjolnir, Dimension, etc.) require an Access Token to run (not a password), so this new role doesn't help for them. It does help for the above bots though, and for defining your own "initial user accounts" in the `matrix_user_creator_users_additional` variable. --- docs/configuring-playbook-bot-honoroit.md | 30 +++---- ...ng-playbook-bot-matrix-registration-bot.md | 2 +- ...guring-playbook-bot-matrix-reminder-bot.md | 30 +++---- docs/configuring-playbook-bot-postmoogle.md | 30 +++---- docs/installing.md | 84 ++++++++++++++----- group_vars/matrix_servers | 34 ++++++++ roles/matrix-user-creator/defaults/main.yml | 22 +++++ roles/matrix-user-creator/tasks/main.yml | 9 ++ roles/matrix-user-creator/tasks/setup.yml | 33 ++++++++ .../util/ensure_user_registered_conduit.yml | 5 ++ .../util/ensure_user_registered_dendrite.yml | 17 ++++ .../util/ensure_user_registered_synapse.yml | 22 +++++ .../tasks/util/validate_user.yml | 16 ++++ roles/matrix-user-creator/vars/main.yml | 34 ++++++++ setup.yml | 1 + 15 files changed, 294 insertions(+), 75 deletions(-) create mode 100644 roles/matrix-user-creator/defaults/main.yml create mode 100644 roles/matrix-user-creator/tasks/main.yml create mode 100644 roles/matrix-user-creator/tasks/setup.yml create mode 100644 roles/matrix-user-creator/tasks/util/ensure_user_registered_conduit.yml create mode 100644 roles/matrix-user-creator/tasks/util/ensure_user_registered_dendrite.yml create mode 100644 roles/matrix-user-creator/tasks/util/ensure_user_registered_synapse.yml create mode 100644 roles/matrix-user-creator/tasks/util/validate_user.yml create mode 100644 roles/matrix-user-creator/vars/main.yml diff --git a/docs/configuring-playbook-bot-honoroit.md b/docs/configuring-playbook-bot-honoroit.md index 45fc033da..42f31d49a 100644 --- a/docs/configuring-playbook-bot-honoroit.md +++ b/docs/configuring-playbook-bot-honoroit.md @@ -7,21 +7,6 @@ It's a bot you can use to setup **your own helpdesk on matrix** See the project's [documentation](https://gitlab.com/etke.cc/honoroit#how-it-looks-like) to learn what it does with screenshots and why it might be useful to you. -## Registering the bot user - -By default, the playbook will set up the bot with a username like this: `@honoroit:DOMAIN`. - -(to use a different username, adjust the `matrix_bot_honoroit_login` variable). - -You **need to register the bot user manually** before setting up the bot. You can use the playbook to [register a new user](registering-users.md): - -``` -ansible-playbook -i inventory/hosts setup.yml --extra-vars='username=honoroit password=PASSWORD_FOR_THE_BOT admin=no' --tags=register-user -``` - -Choose a strong password for the bot. You can generate a good password with a command like this: `pwgen -s 64 1`. - - ## Adjusting the playbook configuration Add the following configuration to your `inventory/host_vars/matrix.DOMAIN/vars.yml` file: @@ -29,7 +14,10 @@ Add the following configuration to your `inventory/host_vars/matrix.DOMAIN/vars. ```yaml matrix_bot_honoroit_enabled: true -# Adjust this to whatever password you chose when registering the bot user +# Uncomment and adjust this part if you'd like to use a username different than the default +# matrix_bot_honoroit_login: honoroit + +# Generate a strong password here. Consider generating it with `pwgen -s 64 1` matrix_bot_honoroit_password: PASSWORD_FOR_THE_BOT # Adjust this to your room ID @@ -41,9 +29,15 @@ matrix_bot_honoroit_roomid: "!yourRoomID:DOMAIN" After configuring the playbook, run the [installation](installing.md) command again: +```sh +ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,ensure-matrix-users-created,start ``` -ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start -``` + +**Notes**: + +- the `ensure-matrix-users-created` playbook tag makes the playbook automatically create the bot's user account + +- if you change the bot password (`matrix_bot_honoroit_password` in your `vars.yml` file) subsequently, the bot user's credentials on the homeserver won't be updated automatically. If you'd like to change the bot user's password, use a tool like [synapse-admin](configuring-playbook-synapse-admin.md) to change it, and then update `matrix_bot_honoroit_password` to let the bot know its new password ## Usage diff --git a/docs/configuring-playbook-bot-matrix-registration-bot.md b/docs/configuring-playbook-bot-matrix-registration-bot.md index a3e4bbeb1..b1e3fdc6c 100644 --- a/docs/configuring-playbook-bot-matrix-registration-bot.md +++ b/docs/configuring-playbook-bot-matrix-registration-bot.md @@ -16,7 +16,7 @@ By default, the playbook will set use the bot with a username like this: `@bot.m (to use a different username, adjust the `matrix_bot_matrix_registration_bot_matrix_user_id_localpart` variable). -You **need to register the bot user manually** before setting up the bot. You can use the playbook to [register a new user](registering-users.md): +For [other bots supported by the playbook](configuring-playbook.md#bots), Matrix bot user accounts are created and put to use automatically. For `matrix-registration-bot`, however, this is not the case - you **need to register the bot user manually** before setting up the bot. You can use the playbook to [register a new user](registering-users.md): ``` ansible-playbook -i inventory/hosts setup.yml --extra-vars='username=bot.matrix-registration-bot password=PASSWORD_FOR_THE_BOT admin=yes' --tags=register-user diff --git a/docs/configuring-playbook-bot-matrix-reminder-bot.md b/docs/configuring-playbook-bot-matrix-reminder-bot.md index aaf5670c1..da73e5dbe 100644 --- a/docs/configuring-playbook-bot-matrix-reminder-bot.md +++ b/docs/configuring-playbook-bot-matrix-reminder-bot.md @@ -7,21 +7,6 @@ It's a bot you can use to **schedule one-off & recurring reminders and alarms**. See the project's [documentation](https://github.com/anoadragon453/matrix-reminder-bot#usage) to learn what it does and why it might be useful to you. -## Registering the bot user - -By default, the playbook will set up the bot with a username like this: `@bot.matrix-reminder-bot:DOMAIN`. - -(to use a different username, adjust the `matrix_bot_matrix_reminder_bot_matrix_user_id_localpart` variable). - -You **need to register the bot user manually** before setting up the bot. You can use the playbook to [register a new user](registering-users.md): - -``` -ansible-playbook -i inventory/hosts setup.yml --extra-vars='username=bot.matrix-reminder-bot password=PASSWORD_FOR_THE_BOT admin=no' --tags=register-user -``` - -Choose a strong password for the bot. You can generate a good password with a command like this: `pwgen -s 64 1`. - - ## Adjusting the playbook configuration Add the following configuration to your `inventory/host_vars/matrix.DOMAIN/vars.yml` file: @@ -29,7 +14,10 @@ Add the following configuration to your `inventory/host_vars/matrix.DOMAIN/vars. ```yaml matrix_bot_matrix_reminder_bot_enabled: true -# Adjust this to whatever password you chose when registering the bot user +# Uncomment and adjust this part if you'd like to use a username different than the default +# matrix_bot_matrix_reminder_bot_matrix_user_id_localpart: bot.matrix-reminder-bot + +# Generate a strong password here. Consider generating it with `pwgen -s 64 1` matrix_bot_matrix_reminder_bot_matrix_user_password: PASSWORD_FOR_THE_BOT # Adjust this to your timezone @@ -41,9 +29,15 @@ matrix_bot_matrix_reminder_bot_reminders_timezone: Europe/London After configuring the playbook, run the [installation](installing.md) command again: +```sh +ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,ensure-matrix-users-created,start ``` -ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start -``` + +**Notes**: + +- the `ensure-matrix-users-created` playbook tag makes the playbook automatically create the bot's user account + +- if you change the bot password (`matrix_bot_matrix_reminder_bot_matrix_user_password` in your `vars.yml` file) subsequently, the bot user's credentials on the homeserver won't be updated automatically. If you'd like to change the bot user's password, use a tool like [synapse-admin](configuring-playbook-synapse-admin.md) to change it, and then update `matrix_bot_matrix_reminder_bot_matrix_user_password` to let the bot know its new password ## Usage diff --git a/docs/configuring-playbook-bot-postmoogle.md b/docs/configuring-playbook-bot-postmoogle.md index 31566da9f..0fa366935 100644 --- a/docs/configuring-playbook-bot-postmoogle.md +++ b/docs/configuring-playbook-bot-postmoogle.md @@ -9,21 +9,6 @@ It's a bot/bridge you can use to forward emails to Matrix rooms See the project's [documentation](https://gitlab.com/etke.cc/postmoogle) to learn what it does and why it might be useful to you. -## Registering the bot user - -By default, the playbook will set up the bot with a username like this: `@postmoogle:DOMAIN`. - -(to use a different username, adjust the `matrix_bot_postmoogle_login` variable). - -You **need to register the bot user manually** before setting up the bot. You can use the playbook to [register a new user](registering-users.md): - -``` -ansible-playbook -i inventory/hosts setup.yml --extra-vars='username=postmoogle password=PASSWORD_FOR_THE_BOT admin=no' --tags=register-user -``` - -Choose a strong password for the bot. You can generate a good password with a command like this: `pwgen -s 64 1`. - - ## Adjusting the playbook configuration Add the following configuration to your `inventory/host_vars/matrix.DOMAIN/vars.yml` file: @@ -31,7 +16,10 @@ Add the following configuration to your `inventory/host_vars/matrix.DOMAIN/vars. ```yaml matrix_bot_postmoogle_enabled: true -# Adjust this to whatever password you chose when registering the bot user +# Uncomment and adjust this part if you'd like to use a username different than the default +# matrix_bot_postmoogle_login: postmoogle + +# Generate a strong password here. Consider generating it with `pwgen -s 64 1` matrix_bot_postmoogle_password: PASSWORD_FOR_THE_BOT ``` @@ -43,9 +31,15 @@ See [Configuring DNS](configuring-dns.md). After configuring the playbook, run the [installation](installing.md) command again: +```sh +ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,ensure-matrix-users-created,start ``` -ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start -``` + +**Notes**: + +- the `ensure-matrix-users-created` playbook tag makes the playbook automatically create the bot's user account + +- if you change the bot password (`matrix_bot_postmoogle_password` in your `vars.yml` file) subsequently, the bot user's credentials on the homeserver won't be updated automatically. If you'd like to change the bot user's password, use a tool like [synapse-admin](configuring-playbook-synapse-admin.md) to change it, and then update `matrix_bot_postmoogle_password` to let the bot know its new password ## Usage diff --git a/docs/installing.md b/docs/installing.md index 8b1b51aa5..308e6fe66 100644 --- a/docs/installing.md +++ b/docs/installing.md @@ -1,25 +1,64 @@ # Installing -## 1. Installing the Matrix services - If you've [configured your DNS](configuring-dns.md) and have [configured the playbook](configuring-playbook.md), you can start the installation procedure. -Run this command to install the Matrix services: +## Playbook tags introduction -```bash +The Ansible playbook's tasks are tagged, so that certain parts of the Ansible playbook can be run without running all other tasks. + +The general command syntax is: `ansible-playbook -i inventory/hosts setup.yml --tags=COMMA_SEPARATED_TAGS_GO_HERE` + +Here are some playbook tags that you should be familiar with: + +- `setup-all` - runs all setup tasks for all components, but does not start/restart services + +- `setup-SERVICE` (e.g. `setup-bot-postmoogle`) - runs the setup tasks only for a given role, but does not start/restart services. You can discover these additional tags in each role (`roles/*/main.yml`). Running per-component setup tasks is **not recommended**, as components sometimes depend on each other and running just the setup tasks for a given component may not be enough. For example, setting up the [mautrix-telegram bridge](configuring-playbook-bridge-mautrix-telegram.md), in addition to the `setup-mautrix-telegram` tag, requires database changes (the `setup-postgres` tag) as well as reverse-proxy changes (the `setup-nginx-proxy` tag). + +- `start` - starts all systemd services and makes them start automatically in the future + +- `stop` - stops all systemd services + +- `ensure-matrix-users-created` - a special tag which ensures that all special users needed by the playbook (for bots, etc.) are created + +`setup-*` tags **do not start services** automatically, because you may wish to do things before starting services, such as importing a database dump, restoring data from another server, etc. + + +## 1. Installing Matrix + +If you **don't** use SSH keys for authentication, but rather a regular password, you may need to add `--ask-pass` to the all Ansible commands + +If you **do** use SSH keys for authentication, **and** use a non-root user to *become* root (sudo), you may need to add `-K` (`--ask-become-pass`) to all Ansible commands + +There 2 ways to start the installation process - depending on whether you're [Installing a brand new server (without importing data)](#installing-a-brand-new-server-without-importing-data) or [Installing a server into which you'll import old data](#installing-a-server-into-which-youll-import-old-data). + + +### Installing a brand new server (without importing data) + +If this is **a brand new** Matrix server and you **won't be importing old data into it**, run all these tags: + +```sh +ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,ensure-matrix-users-created,start +``` + +This will do a full installation and start all Matrix services. + +Proceed to [Maintaining your setup in the future](#2-maintaining-your-setup-in-the-future) and [Finalize the installation](#3-finalize-the-installation) + + +### Installing a server into which you'll import old data + +If you will be importing data into your newly created Matrix server, install it, but **do not** start its services just yet. +Starting its services or messing with its database now will affect your data import later on. + +To do the installation **without** starting services, run only the `setup-all` tag: + +```sh ansible-playbook -i inventory/hosts setup.yml --tags=setup-all ``` -The above command **doesn't start any services just yet** (another step does this later - below). Feel free to **re-run this setup command any time** you think something is off with the server configuration. +When this command completes, services won't be running yet. -**Notes**: -- if you **don't** use SSH keys for authentication, but rather a regular password, you may need to add `--ask-pass` to the above (and all other) Ansible commands. -- if you **do** use SSH keys for authentication, **and** use a non-root user to *become* root (sudo), you may need to add `-K` (`--ask-become-pass`) to the above (and all other) Ansible commands. - - -## 2. Things you might want to do after installing - -**Before starting the services**, you may want to do additional things like: +You can now: - [Importing an existing SQLite database (from another Synapse installation)](importing-synapse-sqlite.md) (optional) @@ -27,21 +66,26 @@ The above command **doesn't start any services just yet** (another step does thi - [Importing `media_store` data files from an existing Synapse installation](importing-synapse-media-store.md) (optional) +.. and then proceed to starting all services: -## 3. Starting the services - -When you're ready to start the Matrix services (and set them up to auto-start in the future), run this command: - -```bash +```sh ansible-playbook -i inventory/hosts setup.yml --tags=start ``` -## 4. Finalize the installation +Proceed to [Maintaining your setup in the future](#2-maintaining-your-setup-in-the-future) and [Finalize the installation](#3-finalize-the-installation) + + +## 2. Maintaining your setup in the future + +Feel free to **re-run the setup command any time** you think something is off with the server configuration. Ansible will take your configuration and update your server to match. + + +## 3. Finalize the installation Now that services are running, you need to **finalize the installation process** (required for federation to work!) by [Configuring Service Discovery via .well-known](configuring-well-known.md). -## 5. Things to do next +## 4. Things to do next After you have started the services and **finalized the installation process** (required for federation to work!) by [Configuring Service Discovery via .well-known](configuring-well-known.md), you can: diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index e5de094fb..872e778bc 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -2677,3 +2677,37 @@ matrix_conduit_systemd_required_services_list: | # /matrix-conduit # ###################################################################### + + +###################################################################### +# +# matrix-user-creator +# +###################################################################### + +matrix_user_creator_users_auto: | + {{ + [{ + 'username': matrix_bot_matrix_reminder_bot_matrix_user_id_localpart, + 'initial_password': matrix_bot_matrix_reminder_bot_matrix_user_password, + 'initial_type': 'bot', + }] if matrix_bot_matrix_reminder_bot_enabled else [] + + + [{ + 'username': matrix_bot_honoroit_login, + 'initial_password': matrix_bot_honoroit_password, + 'initial_type': 'bot', + }] if matrix_bot_honoroit_enabled else [] + + + [{ + 'username': matrix_bot_postmoogle_login, + 'initial_password': matrix_bot_postmoogle_password, + 'initial_type': 'bot', + }] if matrix_bot_postmoogle_enabled else [] + }} + +###################################################################### +# +# /matrix-user-creator +# +###################################################################### diff --git a/roles/matrix-user-creator/defaults/main.yml b/roles/matrix-user-creator/defaults/main.yml new file mode 100644 index 000000000..5c90a1e0f --- /dev/null +++ b/roles/matrix-user-creator/defaults/main.yml @@ -0,0 +1,22 @@ +--- + +# matrix-user-creator is a role that aims to automate initial Matrix user account creation. +# +# This role only supports initial user account creation and will not manage subsequent user-type changes +# or password changes. +# +# The playbook registers various bot user accounts automatically using this role by injecting +# user creation definitions into the `matrix_user_creator_users_auto` variable. +# +# To get started creating your own Matrix user accounts, use the `matrix_user_creator_users_additional` variable. + +# matrix_user_creator_users_auto holds a list of users that should be created on the Matrix homeserver. +# The playbook adds some user definitions here to have them end up in `matrix_user_creator_users` (see `vars/main.yml`) and get automatically created. +# This value is influenced by the playbook and will be overwritten elsewhere (`group_vars/`, etc.) +# To add your own user definitions, use the `matrix_user_creator_users_additional` variable. +matrix_user_creator_users_auto: [] + +# matrix_user_creator_users_additional holds a list of users that should be created on the Matrix homeserver. +# Add your own users here to have them end up in `matrix_user_creator_users` (see `vars/main.yml`) and get automatically created. +# For example syntax for this variable, see the documentation for `matrix_user_creator_users` in `vars/main.yml`. +matrix_user_creator_users_additional: [] diff --git a/roles/matrix-user-creator/tasks/main.yml b/roles/matrix-user-creator/tasks/main.yml new file mode 100644 index 000000000..ba20504f7 --- /dev/null +++ b/roles/matrix-user-creator/tasks/main.yml @@ -0,0 +1,9 @@ +--- + +- when: matrix_user_creator_users | length > 0 + ansible.builtin.import_tasks: "{{ role_path }}/tasks/setup.yml" + tags: + # This role intentionally doesn't do work on a `setup-all` tag. + # If it did, the initial installation (`--tags=setup-all`) would also potentially polute the database with data, + # which would make importing a database dump problematic. + - ensure-matrix-users-created diff --git a/roles/matrix-user-creator/tasks/setup.yml b/roles/matrix-user-creator/tasks/setup.yml new file mode 100644 index 000000000..73dfd4548 --- /dev/null +++ b/roles/matrix-user-creator/tasks/setup.yml @@ -0,0 +1,33 @@ +--- + +- name: Validate Matrix users to create + ansible.builtin.include_tasks: "{{ role_path }}/tasks/util/validate_user.yml" + with_items: "{{ matrix_user_creator_users }}" + loop_control: + loop_var: user + # Suppress logging to avoid dumping the credentials to the shell + no_log: true + +- name: Ensure systemd is reloaded before starting the homeserver + ansible.builtin.service: + daemon_reload: true + +- name: Ensure homeserver is started before creating Matrix users + ansible.builtin.service: + name: "matrix-{{ matrix_homeserver_implementation }}.service" + state: started + daemon_reload: true + register: matrix_user_registrator_homeserver_start_result + +- name: Wait a while, so that the homeserver can manage to start before creating Matrix users + ansible.builtin.pause: + seconds: 7 + when: matrix_user_registrator_homeserver_start_result.changed | bool + +- name: Ensure Matrix users are created + ansible.builtin.include_tasks: "{{ role_path }}/tasks/util/ensure_user_registered_{{ matrix_homeserver_implementation }}.yml" + with_items: "{{ matrix_user_creator_users }}" + loop_control: + loop_var: user + # Suppress logging to avoid dumping the credentials to the shell + no_log: true diff --git a/roles/matrix-user-creator/tasks/util/ensure_user_registered_conduit.yml b/roles/matrix-user-creator/tasks/util/ensure_user_registered_conduit.yml new file mode 100644 index 000000000..8bbd147b5 --- /dev/null +++ b/roles/matrix-user-creator/tasks/util/ensure_user_registered_conduit.yml @@ -0,0 +1,5 @@ +--- + +- name: Ensure Conduit user registered - {{ user.username | quote }} + ansible.builtin.debug: + msg: "Not registering user. To register Conduit users, message the Conduit bot" diff --git a/roles/matrix-user-creator/tasks/util/ensure_user_registered_dendrite.yml b/roles/matrix-user-creator/tasks/util/ensure_user_registered_dendrite.yml new file mode 100644 index 000000000..5fb1636fc --- /dev/null +++ b/roles/matrix-user-creator/tasks/util/ensure_user_registered_dendrite.yml @@ -0,0 +1,17 @@ +--- + +- name: Ensure Dendrite user registered - {{ user.username | quote }} + ansible.builtin.command: + cmd: |- + {{ matrix_host_command_docker }} exec matrix-dendrite + create-account + -config /data/dendrite.yaml + -username {{ user.username | quote }} + -password {{ user.initial_password | quote }} + {% if user.initial_type == 'admin' %} + -admin + {% endif %} + -url http://localhost:{{ matrix_dendrite_http_bind_port }} + register: matrix_dendrite_register_user_result + changed_when: matrix_dendrite_register_user_result.rc == 0 and 'Desired user ID is already taken' not in matrix_dendrite_register_user_result.stderr + failed_when: matrix_dendrite_register_user_result.rc != 0 and 'Desired user ID is already taken' not in matrix_dendrite_register_user_result.stderr diff --git a/roles/matrix-user-creator/tasks/util/ensure_user_registered_synapse.yml b/roles/matrix-user-creator/tasks/util/ensure_user_registered_synapse.yml new file mode 100644 index 000000000..fde82096c --- /dev/null +++ b/roles/matrix-user-creator/tasks/util/ensure_user_registered_synapse.yml @@ -0,0 +1,22 @@ +--- + +- name: Ensure Synapse user registered - {{ user.username | quote }} + ansible.builtin.command: + cmd: |- + {{ matrix_host_command_docker }} exec matrix-synapse + register_new_matrix_user + -u {{ user.username | quote }} + -p {{ user.initial_password | quote }} + -c /data/homeserver.yaml + {% if user.initial_type == 'admin' %} + --admin + {% else %} + --no-admin + {% if user.initial_type != 'user' %} + --user_type={{ user.initial_type | quote }} + {% endif %} + {% endif %} + http://localhost:{{ matrix_synapse_container_client_api_port }} + register: matrix_synapse_register_user_result + changed_when: matrix_synapse_register_user_result.rc == 0 and 'User ID already taken' not in matrix_synapse_register_user_result.stdout + failed_when: matrix_synapse_register_user_result.rc != 0 and 'User ID already taken' not in matrix_synapse_register_user_result.stdout diff --git a/roles/matrix-user-creator/tasks/util/validate_user.yml b/roles/matrix-user-creator/tasks/util/validate_user.yml new file mode 100644 index 000000000..e35475d4e --- /dev/null +++ b/roles/matrix-user-creator/tasks/util/validate_user.yml @@ -0,0 +1,16 @@ +--- + +- name: Fail if invalid username + ansible.builtin.fail: + msg: "Empty usernames values are not allowed ({{ user }})" + when: not (user.username | default('')) + +- name: Fail if invalid initial_password for user - {{ user.username }} + ansible.builtin.fail: + msg: "Empty initial_password values are not allowed" + when: not (user.initial_password | default('')) + +- name: Fail if invalid initial_type for user - {{ user.username }} + ansible.builtin.fail: + msg: "User initial_type `{{ user.initial_type | default('undefined') }}` is not supported" + when: user.initial_type | default('undefined') not in ['admin', 'user', 'bot', 'support'] diff --git a/roles/matrix-user-creator/vars/main.yml b/roles/matrix-user-creator/vars/main.yml new file mode 100644 index 000000000..7b65bb8aa --- /dev/null +++ b/roles/matrix-user-creator/vars/main.yml @@ -0,0 +1,34 @@ +--- + +# matrix_user_creator_users holds a list of users that should be created on the Matrix homeserver. +# +# Removing a user from this list will not automatically delete/disable the Matrix user on the homeserver. +# +# As the `initial_password` / `initial_type` field names indicate, these are just initial values. +# Changing the password or type values subsequently will not update the already existing user's details. +# +# The known user types are: 'admin', 'user', 'bot', 'support'. +# These are inspired by Synapse's user types. +# 'admin' and 'user' types are generally recognized across homeservers. +# Other homeservers may not support 'bot' and 'support'. Such homeservers will fall back to whatever types they do support. +# +# Example: +# matrix_user_creator_users: +# - username: root +# initial_password: some-password +# initial_type: admin +# +# - username: john +# initial_password: some-password +# initial_type: user +# +# - username: bot.matrix-reminder-bot +# initial_password: some-password +# initial_type: bot +# +# - username: bot.matrix-reminder-bot +# initial_password: some-password +# initial_type: support +# +# To create you own users, use the `matrix_user_creator_users_additional` variable. +matrix_user_creator_users: "{{ matrix_user_creator_users_auto + matrix_user_creator_users_additional }}" diff --git a/setup.yml b/setup.yml index 723f87d9c..b41793540 100755 --- a/setup.yml +++ b/setup.yml @@ -72,4 +72,5 @@ - matrix-aux - matrix-postgres-backup - matrix-backup-borg + - matrix-user-creator - matrix-common-after From eaa9b7cfc4a05bce65cbc0424508bec827c44fb1 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Tue, 1 Nov 2022 17:03:53 +0200 Subject: [PATCH 114/118] Add automatic user account creation support to Buscarron Continuation of c3dc64b1d5e06f --- docs/configuring-playbook-bot-buscarron.md | 29 +++++++++------------- group_vars/matrix_servers | 6 +++++ 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/docs/configuring-playbook-bot-buscarron.md b/docs/configuring-playbook-bot-buscarron.md index b38f25c67..0b80ba403 100644 --- a/docs/configuring-playbook-bot-buscarron.md +++ b/docs/configuring-playbook-bot-buscarron.md @@ -5,20 +5,6 @@ The playbook can install and configure [buscarron](https://gitlab.com/etke.cc/bu It's a bot you can use to setup **your own helpdesk on matrix** It's a bot you can use to send any form (HTTP POST, HTML) to a (encrypted) matrix room -## Registering the bot user - -By default, the playbook will set up the bot with a username like this: `@bot.buscarron:DOMAIN`. - -(to use a different username, adjust the `matrix_bot_buscarron_login` variable). - -You **need to register the bot user manually** before setting up the bot. You can use the playbook to [register a new user](registering-users.md): - -``` -ansible-playbook -i inventory/hosts setup.yml --extra-vars='username=bot.buscarron password=PASSWORD_FOR_THE_BOT admin=no' --tags=register-user -``` - -Choose a strong password for the bot. You can generate a good password with a command like this: `pwgen -s 64 1`. - ## Adjusting the playbook configuration @@ -27,7 +13,10 @@ Add the following configuration to your `inventory/host_vars/matrix.DOMAIN/vars. ```yaml matrix_bot_buscarron_enabled: true -# Adjust this to whatever password you chose when registering the bot user +# Uncomment and adjust this part if you'd like to use a username different than the default +# matrix_bot_buscarron_login: bot.buscarron + +# Generate a strong password here. Consider generating it with `pwgen -s 64 1` matrix_bot_buscarron_password: PASSWORD_FOR_THE_BOT # Adjust accepted forms @@ -57,9 +46,15 @@ matrix_server_fqn_buscarron: "form.{{ matrix_domain }}" After configuring the playbook, run the [installation](installing.md) command again: +```sh +ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,ensure-matrix-users-created,start ``` -ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start -``` + +**Notes**: + +- the `ensure-matrix-users-created` playbook tag makes the playbook automatically create the bot's user account + +- if you change the bot password (`matrix_bot_buscarron_password` in your `vars.yml` file) subsequently, the bot user's credentials on the homeserver won't be updated automatically. If you'd like to change the bot user's password, use a tool like [synapse-admin](configuring-playbook-synapse-admin.md) to change it, and then update `matrix_bot_buscarron_password` to let the bot know its new password ## Usage diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 872e778bc..bdf5dccec 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -2704,6 +2704,12 @@ matrix_user_creator_users_auto: | 'initial_password': matrix_bot_postmoogle_password, 'initial_type': 'bot', }] if matrix_bot_postmoogle_enabled else [] + + + [{ + 'username': matrix_bot_buscarron_login, + 'initial_password': matrix_bot_buscarron_password, + 'initial_type': 'bot', + }] if matrix_bot_buscarron_enabled else [] }} ###################################################################### From b7bd68d3cb58e7e48e239ba66867a8b4c4f1631e Mon Sep 17 00:00:00 2001 From: Michael Hollister Date: Tue, 1 Nov 2022 13:11:11 -0500 Subject: [PATCH 115/118] Fixed filter for Synapse stream_writer HS config --- roles/matrix-synapse/tasks/synapse/workers/init.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-synapse/tasks/synapse/workers/init.yml b/roles/matrix-synapse/tasks/synapse/workers/init.yml index 0fc4e79c3..d88884d09 100644 --- a/roles/matrix-synapse/tasks/synapse/workers/init.yml +++ b/roles/matrix-synapse/tasks/synapse/workers/init.yml @@ -37,7 +37,7 @@ - name: Populate matrix_synapse_stream_writers from enabled stream writer workers list ansible.builtin.set_fact: - matrix_synapse_stream_writers: "{{ matrix_synapse_stream_writers | combine({item.ansible_facts.worker.stream_writer_stream: [item.ansible_facts.worker.name]}) }}" + matrix_synapse_stream_writers: "{{ matrix_synapse_stream_writers | combine({item.ansible_facts.worker.stream_writer_stream: [item.ansible_facts.worker.name]}, list_merge='append') }}" with_items: "{{ matrix_synapse_workers_list_results_stream_writer_workers.results }}" - name: Build federation sender workers From 28d9ee857895e257bef651df9d91e45b5f28335c Mon Sep 17 00:00:00 2001 From: Darren Rambaud Date: Tue, 1 Nov 2022 14:12:52 -0500 Subject: [PATCH 116/118] dendrite: update tag to v0.10.6 - update tag to latest dendrite release (https://github.com/matrix-org/dendrite/releases/tag/v0.10.6) --- roles/matrix-dendrite/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-dendrite/defaults/main.yml b/roles/matrix-dendrite/defaults/main.yml index 945c16197..a1d03960c 100644 --- a/roles/matrix-dendrite/defaults/main.yml +++ b/roles/matrix-dendrite/defaults/main.yml @@ -6,7 +6,7 @@ matrix_dendrite_enabled: true matrix_dendrite_docker_image: "{{ matrix_dendrite_docker_image_name_prefix }}matrixdotorg/dendrite-monolith:{{ matrix_dendrite_docker_image_tag }}" matrix_dendrite_docker_image_name_prefix: "docker.io/" -matrix_dendrite_docker_image_tag: "v0.10.5" +matrix_dendrite_docker_image_tag: "v0.10.6" matrix_dendrite_docker_image_force_pull: "{{ matrix_dendrite_docker_image.endswith(':latest') }}" matrix_dendrite_base_path: "{{ matrix_base_data_path }}/dendrite" From 8a609e5cf2b0bf59fdbf36cdcaafdfc9cf750d1d Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Wed, 2 Nov 2022 07:00:45 +0200 Subject: [PATCH 117/118] Fix some ansible-lint-reported errors --- roles/matrix-base/tasks/setup_matrix_base.yml | 2 +- roles/matrix-bridge-hookshot/defaults/main.yml | 4 ++-- roles/matrix-postgres/tasks/import_postgres.yml | 4 ++-- roles/matrix-postgres/tasks/run_vacuum.yml | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/roles/matrix-base/tasks/setup_matrix_base.yml b/roles/matrix-base/tasks/setup_matrix_base.yml index 0bf197b66..2439fdea3 100644 --- a/roles/matrix-base/tasks/setup_matrix_base.yml +++ b/roles/matrix-base/tasks/setup_matrix_base.yml @@ -22,7 +22,7 @@ - name: Save current git-repo status on the target to aid with restoring in case of problems when: "matrix_playbook_commit_hash_preservation_enabled|bool" block: - - name: Get local git hash # noqa command-instead-of-module + - name: Get local git hash # noqa command-instead-of-module delegate_to: 127.0.0.1 become: false register: git_describe diff --git a/roles/matrix-bridge-hookshot/defaults/main.yml b/roles/matrix-bridge-hookshot/defaults/main.yml index 96781f9aa..4c39876a2 100644 --- a/roles/matrix-bridge-hookshot/defaults/main.yml +++ b/roles/matrix-bridge-hookshot/defaults/main.yml @@ -129,8 +129,8 @@ matrix_hookshot_generic_user_id_prefix: '_webhooks_' matrix_hookshot_feeds_enabled: true -matrix_hookshot_feeds_pollIntervalSeconds: 600 -matrix_hookshot_feeds_pollTimeoutSeconds: 10 +matrix_hookshot_feeds_pollIntervalSeconds: 600 # no-qa var-naming +matrix_hookshot_feeds_pollTimeoutSeconds: 10 # no-qa var-naming # There is no need to edit ports. use matrix_hookshot_container_http_host_bind_ports below to expose ports instead. diff --git a/roles/matrix-postgres/tasks/import_postgres.yml b/roles/matrix-postgres/tasks/import_postgres.yml index 107a1f887..d5bfaa86c 100644 --- a/roles/matrix-postgres/tasks/import_postgres.yml +++ b/roles/matrix-postgres/tasks/import_postgres.yml @@ -51,8 +51,8 @@ daemon_reload: true register: matrix_postgres_import_start_result -- when: matrix_postgres_import_start_result.changed | bool - name: Wait a bit, so that Postgres can start +- name: Wait a bit, so that Postgres can start + when: matrix_postgres_import_start_result.changed | bool ansible.builtin.wait_for: timeout: "{{ postgres_start_wait_time }}" delegate_to: 127.0.0.1 diff --git a/roles/matrix-postgres/tasks/run_vacuum.yml b/roles/matrix-postgres/tasks/run_vacuum.yml index 1b769f01f..c1bf8d762 100644 --- a/roles/matrix-postgres/tasks/run_vacuum.yml +++ b/roles/matrix-postgres/tasks/run_vacuum.yml @@ -30,8 +30,8 @@ daemon_reload: true register: matrix_postgres_vacuum_start_result -- when: matrix_postgres_vacuum_start_result.changed | bool - name: Wait a bit, so that Postgres can start +- name: Wait a bit, so that Postgres can start + when: matrix_postgres_vacuum_start_result.changed | bool ansible.builtin.wait_for: timeout: "{{ postgres_start_wait_time }}" delegate_to: 127.0.0.1 From 0fd9f3d4f99076fab4bc85a8b16dfda324aa4091 Mon Sep 17 00:00:00 2001 From: Aine <97398200+etkecc@users.noreply.github.com> Date: Wed, 2 Nov 2022 07:21:09 +0000 Subject: [PATCH 118/118] Pin mautrix-instagram to v0.2.2 --- roles/matrix-bridge-mautrix-instagram/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-bridge-mautrix-instagram/defaults/main.yml b/roles/matrix-bridge-mautrix-instagram/defaults/main.yml index 7305d0fbf..dcdf6723d 100644 --- a/roles/matrix-bridge-mautrix-instagram/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-instagram/defaults/main.yml @@ -8,7 +8,7 @@ matrix_mautrix_instagram_container_image_self_build: false matrix_mautrix_instagram_container_image_self_build_repo: "https://github.com/mautrix/instagram.git" matrix_mautrix_instagram_container_image_self_build_repo_version: "{{ 'master' if matrix_mautrix_instagram_version == 'latest' else matrix_mautrix_instagram_version }}" -matrix_mautrix_instagram_version: latest +matrix_mautrix_instagram_version: v0.2.2 # See: https://mau.dev/tulir/mautrix-instagram/container_registry matrix_mautrix_instagram_docker_image: "{{ matrix_mautrix_instagram_docker_image_name_prefix }}mautrix/instagram:{{ matrix_mautrix_instagram_version }}" matrix_mautrix_instagram_docker_image_name_prefix: "{{ 'localhost/' if matrix_mautrix_instagram_container_image_self_build else 'dock.mau.dev/' }}"