From 9a298009818d7903153d28cc76631a701cc40c8b Mon Sep 17 00:00:00 2001 From: bdiniscia Date: Thu, 7 Jul 2022 17:00:43 +0200 Subject: [PATCH 01/10] Removing the full path on the canonical tag for urls with filter parameters for SEO. --- lib/assets/javascripts/do-catalog/router.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/assets/javascripts/do-catalog/router.js b/lib/assets/javascripts/do-catalog/router.js index 713a163191..b1cb6994fa 100644 --- a/lib/assets/javascripts/do-catalog/router.js +++ b/lib/assets/javascripts/do-catalog/router.js @@ -94,7 +94,7 @@ router.afterEach(to => { if (!to.meta.titleInComponent) { document.title = to.meta.title(to); } - addCanonical(`https://carto.com/spatial-data-catalog/browser${to.fullPath}`); + addCanonical(`https://carto.com/spatial-data-catalog/browser${to.path}`); }); }); From 67aeb9bea5265ea58b7e8d2f1aeb09e332291216 Mon Sep 17 00:00:00 2001 From: bdiniscia Date: Thu, 7 Jul 2022 17:37:24 +0200 Subject: [PATCH 02/10] Updated News and package.json --- NEWS.md | 1 + package-lock.json | 2 +- package.json | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/NEWS.md b/NEWS.md index 73f56c11f4..c74ac7abb5 100644 --- a/NEWS.md +++ b/NEWS.md @@ -23,6 +23,7 @@ Development * Update banner to notify about data migrations to CARTO 3 [#16420](https://github.com/CartoDB/cartodb/pull/16420) ### Bug fixes / enhancements +- Removing the full path from urls with filter parameters in the Spatial Data Catalog [#16426](https://github.com/CartoDB/cartodb/pull/16426) - Fix rubocop integration [#16382](https://github.com/CartoDB/cartodb/pull/16382) - Add marginTop to Page when notification is displayed [#16355](https://github.com/CartoDB/cartodb/pull/16355) - Add "element" param to DO-Catalog entry function [#16343](https://github.com/CartoDB/cartodb/pull/16343) diff --git a/package-lock.json b/package-lock.json index 102902d7e6..0f61ea8eba 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "cartodb-ui", - "version": "1.0.0-assets.289", + "version": "1.0.0-assets.290", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 175395872f..fe30437076 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cartodb-ui", - "version": "1.0.0-assets.289", + "version": "1.0.0-assets.290", "description": "CARTO UI frontend", "repository": { "type": "git", From 21d1dd337fba6bccae9b7932cde7587ec6203a16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mois=C3=A9s=20Calzado?= Date: Tue, 26 Jul 2022 14:25:34 +0200 Subject: [PATCH 03/10] Handle timeout swaping tables during sync process --- app/models/synchronization/adapter.rb | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/app/models/synchronization/adapter.rb b/app/models/synchronization/adapter.rb index c8d94ec4af..b71305d580 100644 --- a/app/models/synchronization/adapter.rb +++ b/app/models/synchronization/adapter.rb @@ -140,11 +140,7 @@ module CartoDB table_statements = @table_setup.generate_table_statements(schema, table_name) temporary_name = temporary_name_for(result.table_name) - database.transaction do - rename(table_name, temporary_name) if exists?(table_name) - drop(temporary_name) if exists?(temporary_name) - rename(result.table_name, table_name) - end + swap_tables(table_name, temporary_name, result) @table_setup.fix_oid(table_name) @table_setup.update_cdb_tablemetadata(table_name) @table_setup.run_table_statements(table_statements, @database) @@ -389,6 +385,25 @@ module CartoDB private + def swap_tables(table_name, temporary_name, result) + database.transaction do + rename(table_name, temporary_name) if exists?(table_name) + drop(temporary_name) if exists?(temporary_name) + rename(result.table_name, table_name) + end + rescue Exception => exception + if exception.message.include?('canceling statement due to statement timeout') + sleep(60) # wait 60 seconds and retry the swap + database.transaction do + rename(table_name, temporary_name) if exists?(table_name) + drop(temporary_name) if exists?(temporary_name) + rename(result.table_name, table_name) + end + else + raise exception + end + end + def valid_cartodb_id_candidate?(user, table_name, qualified_table_name, col_name) return false unless column_names(user, table_name).include?(col_name) user.transaction_with_timeout(statement_timeout: STATEMENT_TIMEOUT, as: :superuser) do |db| From 5e06df730d5bfcddd5e76864f36b559a721f718d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mois=C3=A9s=20Calzado?= Date: Tue, 26 Jul 2022 14:28:35 +0200 Subject: [PATCH 04/10] Update NEWS.md --- NEWS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.md b/NEWS.md index c74ac7abb5..ccc0e9cf8c 100644 --- a/NEWS.md +++ b/NEWS.md @@ -95,6 +95,7 @@ Development - Fix duplicated attributions in datasets [#16384](https://github.com/CartoDB/cartodb/pull/16384) - Moving assets cdn domain from global.ssl.fastly.net to libs.cartocdn.com [#16399](https://github.com/CartoDB/cartodb/pull/16399) - Fix error while rolling back a user migration from one cloud to another [#16421](https://github.com/CartoDB/cartodb/pull/16421) +- Add retry if a timeout is thrown when swapping the tables related with a sync process [#16430](https://github.com/CartoDB/cartodb/pull/16430) 4.45.0 (2021-04-14) ------------------- From 081212ca45964e2549a76266185fbabe43a8161a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mois=C3=A9s=20Calzado?= Date: Fri, 29 Jul 2022 17:01:04 +0200 Subject: [PATCH 05/10] Add AUTODETECT_SIZE_LIMIT option when importing CSVs --- services/importer/lib/importer/ogr2ogr.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/services/importer/lib/importer/ogr2ogr.rb b/services/importer/lib/importer/ogr2ogr.rb index b459518ac1..b94369c4df 100644 --- a/services/importer/lib/importer/ogr2ogr.rb +++ b/services/importer/lib/importer/ogr2ogr.rb @@ -180,6 +180,7 @@ module CartoDB # Inverse of the selection: if I want guessing I must NOT leave quoted fields as string [ '-oo', 'AUTODETECT_TYPE=YES', + '-oo', 'AUTODETECT_SIZE_LIMIT=20000000', '-oo', "QUOTED_FIELDS_AS_STRING=#{quoted_fields_guessing ? 'NO' : 'YES'}" ] + x_y_possible_names_option + ['-s_srs', 'EPSG:4326', '-t_srs', 'EPSG:4326'] end From 980cc0f36b41666270884bc64d04c28161290de0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mois=C3=A9s=20Calzado?= Date: Fri, 29 Jul 2022 17:03:43 +0200 Subject: [PATCH 06/10] Update NEWS.md --- NEWS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.md b/NEWS.md index ccc0e9cf8c..17709b83d9 100644 --- a/NEWS.md +++ b/NEWS.md @@ -96,6 +96,7 @@ Development - Moving assets cdn domain from global.ssl.fastly.net to libs.cartocdn.com [#16399](https://github.com/CartoDB/cartodb/pull/16399) - Fix error while rolling back a user migration from one cloud to another [#16421](https://github.com/CartoDB/cartodb/pull/16421) - Add retry if a timeout is thrown when swapping the tables related with a sync process [#16430](https://github.com/CartoDB/cartodb/pull/16430) +- Add AUTODETECT_SIZE_LIMIT to ogr2ogr process when guessing CSV file column types [#16431](https://github.com/CartoDB/cartodb/pull/16431) 4.45.0 (2021-04-14) ------------------- From b60ecc3d54c69724ad46fb085958dc06bf2b8f1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mois=C3=A9s=20Calzado?= Date: Mon, 1 Aug 2022 14:21:08 +0200 Subject: [PATCH 07/10] Log pg_locks when there is a timeout during a sync table importation --- NEWS.md | 1 + app/models/synchronization/adapter.rb | 15 ++++++++++++++- app/models/synchronization/member.rb | 2 +- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/NEWS.md b/NEWS.md index 17709b83d9..4b3c655365 100644 --- a/NEWS.md +++ b/NEWS.md @@ -97,6 +97,7 @@ Development - Fix error while rolling back a user migration from one cloud to another [#16421](https://github.com/CartoDB/cartodb/pull/16421) - Add retry if a timeout is thrown when swapping the tables related with a sync process [#16430](https://github.com/CartoDB/cartodb/pull/16430) - Add AUTODETECT_SIZE_LIMIT to ogr2ogr process when guessing CSV file column types [#16431](https://github.com/CartoDB/cartodb/pull/16431) +- Log pg locks if there is any problem during a sync table import process [#16432](https://github.com/CartoDB/cartodb/pull/16432) 4.45.0 (2021-04-14) ------------------- diff --git a/app/models/synchronization/adapter.rb b/app/models/synchronization/adapter.rb index b71305d580..ccc523d3ad 100644 --- a/app/models/synchronization/adapter.rb +++ b/app/models/synchronization/adapter.rb @@ -11,7 +11,7 @@ module CartoDB THE_GEOM = 'the_geom'.freeze OVERWRITE_ERROR = 2013 - def initialize(table_name, runner, database, user, overviews_creator, synchronization_id) + def initialize(table_name, runner, database, user, overviews_creator, synchronization_id, logger = nil) @table_name = table_name @runner = runner @database = database @@ -25,6 +25,7 @@ module CartoDB ) @error_code = nil @synchronization_id = synchronization_id + @logger = logger end def run(&tracker) @@ -393,6 +394,18 @@ module CartoDB end rescue Exception => exception if exception.message.include?('canceling statement due to statement timeout') + # Check if the table has any lock + lock = user.in_database(as: :superuser).fetch(%Q{ + SELECT pid, state, usename, query, query_start + FROM pg_stat_activity + WHERE pid in ( + SELECT pid FROM pg_locks l + JOIN pg_class t ON l.relation = t.oid + AND t.relkind = 'r' + WHERE t.relname IN ('#{table_name}') + ); + }).first + @logger.append_and_store "Transaction timed out as the table is blocked by query #{lock[:query]}. Retrying in 60 seconds..." if @logger && lock.present? sleep(60) # wait 60 seconds and retry the swap database.transaction do rename(table_name, temporary_name) if exists?(table_name) diff --git a/app/models/synchronization/member.rb b/app/models/synchronization/member.rb index a5cd0a8d10..e0a8bc7e89 100644 --- a/app/models/synchronization/member.rb +++ b/app/models/synchronization/member.rb @@ -192,7 +192,7 @@ module CartoDB database = user.in_database overviews_creator = CartoDB::Importer2::Overviews.new(runner, user) - importer = CartoDB::Synchronization::Adapter.new(name, runner, database, user, overviews_creator, id) + importer = CartoDB::Synchronization::Adapter.new(name, runner, database, user, overviews_creator, id, @log) importer.run self.ran_at = Time.now From 3c72971705042bcfce7d057a87f9ef4615209c75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mois=C3=A9s=20Calzado?= Date: Tue, 2 Aug 2022 18:23:48 +0200 Subject: [PATCH 08/10] Terminate locking queries when swapping tables during a synchronization --- NEWS.md | 1 + app/models/synchronization/adapter.rb | 13 +++++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/NEWS.md b/NEWS.md index 4b3c655365..309793a2ac 100644 --- a/NEWS.md +++ b/NEWS.md @@ -98,6 +98,7 @@ Development - Add retry if a timeout is thrown when swapping the tables related with a sync process [#16430](https://github.com/CartoDB/cartodb/pull/16430) - Add AUTODETECT_SIZE_LIMIT to ogr2ogr process when guessing CSV file column types [#16431](https://github.com/CartoDB/cartodb/pull/16431) - Log pg locks if there is any problem during a sync table import process [#16432](https://github.com/CartoDB/cartodb/pull/16432) +- Check pg locks during sync table swap and terminate locking queries [#16433](https://github.com/CartoDB/cartodb/pull/16433) 4.45.0 (2021-04-14) ------------------- diff --git a/app/models/synchronization/adapter.rb b/app/models/synchronization/adapter.rb index ccc523d3ad..cb65af0d2c 100644 --- a/app/models/synchronization/adapter.rb +++ b/app/models/synchronization/adapter.rb @@ -394,8 +394,8 @@ module CartoDB end rescue Exception => exception if exception.message.include?('canceling statement due to statement timeout') - # Check if the table has any lock - lock = user.in_database(as: :superuser).fetch(%Q{ + # Check if the table has any lock and cancel locking queries + locks = user.in_database(as: :superuser).fetch(%Q{ SELECT pid, state, usename, query, query_start FROM pg_stat_activity WHERE pid in ( @@ -404,8 +404,13 @@ module CartoDB AND t.relkind = 'r' WHERE t.relname IN ('#{table_name}') ); - }).first - @logger.append_and_store "Transaction timed out as the table is blocked by query #{lock[:query]}. Retrying in 60 seconds..." if @logger && lock.present? + }).all + @logger.append_and_store "Transaction timed out as the table is blocked by other queries. Terminating locking queries and retrying in 60 seconds..." if @logger && locks.present? + locks.each do |lock| + user.in_database(as: :superuser).execute %Q{ + SELECT pg_terminate_backend(#{lock[:pid]}); + } + end sleep(60) # wait 60 seconds and retry the swap database.transaction do rename(table_name, temporary_name) if exists?(table_name) From 2a8c5cac41e840ae5d08d7a771cf9d3c996f25a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mois=C3=A9s=20Calzado?= Date: Tue, 2 Aug 2022 19:12:55 +0200 Subject: [PATCH 09/10] Log locking queries --- app/models/synchronization/adapter.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/models/synchronization/adapter.rb b/app/models/synchronization/adapter.rb index cb65af0d2c..d9b67fd8bc 100644 --- a/app/models/synchronization/adapter.rb +++ b/app/models/synchronization/adapter.rb @@ -396,7 +396,7 @@ module CartoDB if exception.message.include?('canceling statement due to statement timeout') # Check if the table has any lock and cancel locking queries locks = user.in_database(as: :superuser).fetch(%Q{ - SELECT pid, state, usename, query, query_start + SELECT pid, query FROM pg_stat_activity WHERE pid in ( SELECT pid FROM pg_locks l @@ -407,6 +407,7 @@ module CartoDB }).all @logger.append_and_store "Transaction timed out as the table is blocked by other queries. Terminating locking queries and retrying in 60 seconds..." if @logger && locks.present? locks.each do |lock| + @logger.append_and_store "Terminating query: #{lock[:query]}" if @logger user.in_database(as: :superuser).execute %Q{ SELECT pg_terminate_backend(#{lock[:pid]}); } From d9ef0bcdb478a4c1cd36fcced63aca84169662bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alberto=20Hern=C3=A1ndez?= Date: Mon, 12 Sep 2022 10:17:19 +0200 Subject: [PATCH 10/10] increase cloud build timeout (#16437) * increase cloud build timeout * increase timeout * downgrade rubocop version * Update rubocop.yml * rubocop 1.17.0 * Update rubocop.yml * revert version --- script/ci/cloudbuild-build-master.yaml | 2 +- script/ci/cloudbuild-build-pr-branch.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/script/ci/cloudbuild-build-master.yaml b/script/ci/cloudbuild-build-master.yaml index 31522031be..6cc859327e 100644 --- a/script/ci/cloudbuild-build-master.yaml +++ b/script/ci/cloudbuild-build-master.yaml @@ -208,7 +208,7 @@ steps: fi waitFor: ['build-cartodb-onprem', 'build-cartodb', 'build-cartodb-onprem-cron'] -timeout: 1800s +timeout: 3600s substitutions: _BRANCH_TAG: ${BRANCH_NAME//\//-} _DOCKER_IMAGE_NAME: gcr.io/cartodb-on-gcp-main-artifacts/builder diff --git a/script/ci/cloudbuild-build-pr-branch.yaml b/script/ci/cloudbuild-build-pr-branch.yaml index 4bc4e29e26..8f012d88e8 100644 --- a/script/ci/cloudbuild-build-pr-branch.yaml +++ b/script/ci/cloudbuild-build-pr-branch.yaml @@ -155,7 +155,7 @@ steps: docker build --label="org.opencontainers.image.created=$$(date --rfc-3339=seconds)" --label=org.opencontainers.image.revision=${COMMIT_SHA} -t ${_DOCKER_IMAGE_NAME}-onprem-cron:latest -t ${_DOCKER_IMAGE_NAME}-onprem-cron:${_BRANCH_TAG} -t ${_DOCKER_IMAGE_NAME}-onprem-cron:${SHORT_SHA} -t ${_DOCKER_IMAGE_NAME}-onprem-cron:${_BRANCH_TAG}--${SHORT_SHA} -f Dockerfile.cron . waitFor: ['build-cartodb-onprem'] -timeout: 2700s +timeout: 3600s images: - ${_DOCKER_IMAGE_NAME}:${_BRANCH_TAG}--${SHORT_SHA} - ${_DOCKER_IMAGE_NAME}:${_BRANCH_TAG}