From 3c72971705042bcfce7d057a87f9ef4615209c75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mois=C3=A9s=20Calzado?= Date: Tue, 2 Aug 2022 18:23:48 +0200 Subject: [PATCH 1/2] Terminate locking queries when swapping tables during a synchronization --- NEWS.md | 1 + app/models/synchronization/adapter.rb | 13 +++++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/NEWS.md b/NEWS.md index 4b3c655365..309793a2ac 100644 --- a/NEWS.md +++ b/NEWS.md @@ -98,6 +98,7 @@ Development - Add retry if a timeout is thrown when swapping the tables related with a sync process [#16430](https://github.com/CartoDB/cartodb/pull/16430) - Add AUTODETECT_SIZE_LIMIT to ogr2ogr process when guessing CSV file column types [#16431](https://github.com/CartoDB/cartodb/pull/16431) - Log pg locks if there is any problem during a sync table import process [#16432](https://github.com/CartoDB/cartodb/pull/16432) +- Check pg locks during sync table swap and terminate locking queries [#16433](https://github.com/CartoDB/cartodb/pull/16433) 4.45.0 (2021-04-14) ------------------- diff --git a/app/models/synchronization/adapter.rb b/app/models/synchronization/adapter.rb index ccc523d3ad..cb65af0d2c 100644 --- a/app/models/synchronization/adapter.rb +++ b/app/models/synchronization/adapter.rb @@ -394,8 +394,8 @@ module CartoDB end rescue Exception => exception if exception.message.include?('canceling statement due to statement timeout') - # Check if the table has any lock - lock = user.in_database(as: :superuser).fetch(%Q{ + # Check if the table has any lock and cancel locking queries + locks = user.in_database(as: :superuser).fetch(%Q{ SELECT pid, state, usename, query, query_start FROM pg_stat_activity WHERE pid in ( @@ -404,8 +404,13 @@ module CartoDB AND t.relkind = 'r' WHERE t.relname IN ('#{table_name}') ); - }).first - @logger.append_and_store "Transaction timed out as the table is blocked by query #{lock[:query]}. Retrying in 60 seconds..." if @logger && lock.present? + }).all + @logger.append_and_store "Transaction timed out as the table is blocked by other queries. Terminating locking queries and retrying in 60 seconds..." if @logger && locks.present? + locks.each do |lock| + user.in_database(as: :superuser).execute %Q{ + SELECT pg_terminate_backend(#{lock[:pid]}); + } + end sleep(60) # wait 60 seconds and retry the swap database.transaction do rename(table_name, temporary_name) if exists?(table_name) From 2a8c5cac41e840ae5d08d7a771cf9d3c996f25a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mois=C3=A9s=20Calzado?= Date: Tue, 2 Aug 2022 19:12:55 +0200 Subject: [PATCH 2/2] Log locking queries --- app/models/synchronization/adapter.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/models/synchronization/adapter.rb b/app/models/synchronization/adapter.rb index cb65af0d2c..d9b67fd8bc 100644 --- a/app/models/synchronization/adapter.rb +++ b/app/models/synchronization/adapter.rb @@ -396,7 +396,7 @@ module CartoDB if exception.message.include?('canceling statement due to statement timeout') # Check if the table has any lock and cancel locking queries locks = user.in_database(as: :superuser).fetch(%Q{ - SELECT pid, state, usename, query, query_start + SELECT pid, query FROM pg_stat_activity WHERE pid in ( SELECT pid FROM pg_locks l @@ -407,6 +407,7 @@ module CartoDB }).all @logger.append_and_store "Transaction timed out as the table is blocked by other queries. Terminating locking queries and retrying in 60 seconds..." if @logger && locks.present? locks.each do |lock| + @logger.append_and_store "Terminating query: #{lock[:query]}" if @logger user.in_database(as: :superuser).execute %Q{ SELECT pg_terminate_backend(#{lock[:pid]}); }