diff --git a/NEWS.md b/NEWS.md index db8a57670f..56a91659b8 100644 --- a/NEWS.md +++ b/NEWS.md @@ -23,6 +23,7 @@ Development * Update banner to notify about data migrations to CARTO 3 [#16420](https://github.com/CartoDB/cartodb/pull/16420) ### Bug fixes / enhancements +- Removing the full path from urls with filter parameters in the Spatial Data Catalog [#16426](https://github.com/CartoDB/cartodb/pull/16426) - Fix rubocop integration [#16382](https://github.com/CartoDB/cartodb/pull/16382) - Add marginTop to Page when notification is displayed [#16355](https://github.com/CartoDB/cartodb/pull/16355) - Add "element" param to DO-Catalog entry function [#16343](https://github.com/CartoDB/cartodb/pull/16343) @@ -95,6 +96,10 @@ Development - Fix duplicated attributions in datasets [#16384](https://github.com/CartoDB/cartodb/pull/16384) - Moving assets cdn domain from global.ssl.fastly.net to libs.cartocdn.com [#16399](https://github.com/CartoDB/cartodb/pull/16399) - Fix error while rolling back a user migration from one cloud to another [#16421](https://github.com/CartoDB/cartodb/pull/16421) +- Add retry if a timeout is thrown when swapping the tables related with a sync process [#16430](https://github.com/CartoDB/cartodb/pull/16430) +- Add AUTODETECT_SIZE_LIMIT to ogr2ogr process when guessing CSV file column types [#16431](https://github.com/CartoDB/cartodb/pull/16431) +- Log pg locks if there is any problem during a sync table import process [#16432](https://github.com/CartoDB/cartodb/pull/16432) +- Check pg locks during sync table swap and terminate locking queries [#16433](https://github.com/CartoDB/cartodb/pull/16433) 4.45.0 (2021-04-14) ------------------- diff --git a/app/models/synchronization/adapter.rb b/app/models/synchronization/adapter.rb index c8d94ec4af..d9b67fd8bc 100644 --- a/app/models/synchronization/adapter.rb +++ b/app/models/synchronization/adapter.rb @@ -11,7 +11,7 @@ module CartoDB THE_GEOM = 'the_geom'.freeze OVERWRITE_ERROR = 2013 - def initialize(table_name, runner, database, user, overviews_creator, synchronization_id) + def initialize(table_name, runner, database, user, overviews_creator, synchronization_id, logger = nil) @table_name = table_name @runner = runner @database = database @@ -25,6 +25,7 @@ module CartoDB ) @error_code = nil @synchronization_id = synchronization_id + @logger = logger end def run(&tracker) @@ -140,11 +141,7 @@ module CartoDB table_statements = @table_setup.generate_table_statements(schema, table_name) temporary_name = temporary_name_for(result.table_name) - database.transaction do - rename(table_name, temporary_name) if exists?(table_name) - drop(temporary_name) if exists?(temporary_name) - rename(result.table_name, table_name) - end + swap_tables(table_name, temporary_name, result) @table_setup.fix_oid(table_name) @table_setup.update_cdb_tablemetadata(table_name) @table_setup.run_table_statements(table_statements, @database) @@ -389,6 +386,43 @@ module CartoDB private + def swap_tables(table_name, temporary_name, result) + database.transaction do + rename(table_name, temporary_name) if exists?(table_name) + drop(temporary_name) if exists?(temporary_name) + rename(result.table_name, table_name) + end + rescue Exception => exception + if exception.message.include?('canceling statement due to statement timeout') + # Check if the table has any lock and cancel locking queries + locks = user.in_database(as: :superuser).fetch(%Q{ + SELECT pid, query + FROM pg_stat_activity + WHERE pid in ( + SELECT pid FROM pg_locks l + JOIN pg_class t ON l.relation = t.oid + AND t.relkind = 'r' + WHERE t.relname IN ('#{table_name}') + ); + }).all + @logger.append_and_store "Transaction timed out as the table is blocked by other queries. Terminating locking queries and retrying in 60 seconds..." if @logger && locks.present? + locks.each do |lock| + @logger.append_and_store "Terminating query: #{lock[:query]}" if @logger + user.in_database(as: :superuser).execute %Q{ + SELECT pg_terminate_backend(#{lock[:pid]}); + } + end + sleep(60) # wait 60 seconds and retry the swap + database.transaction do + rename(table_name, temporary_name) if exists?(table_name) + drop(temporary_name) if exists?(temporary_name) + rename(result.table_name, table_name) + end + else + raise exception + end + end + def valid_cartodb_id_candidate?(user, table_name, qualified_table_name, col_name) return false unless column_names(user, table_name).include?(col_name) user.transaction_with_timeout(statement_timeout: STATEMENT_TIMEOUT, as: :superuser) do |db| diff --git a/app/models/synchronization/member.rb b/app/models/synchronization/member.rb index a5cd0a8d10..e0a8bc7e89 100644 --- a/app/models/synchronization/member.rb +++ b/app/models/synchronization/member.rb @@ -192,7 +192,7 @@ module CartoDB database = user.in_database overviews_creator = CartoDB::Importer2::Overviews.new(runner, user) - importer = CartoDB::Synchronization::Adapter.new(name, runner, database, user, overviews_creator, id) + importer = CartoDB::Synchronization::Adapter.new(name, runner, database, user, overviews_creator, id, @log) importer.run self.ran_at = Time.now diff --git a/lib/assets/javascripts/do-catalog/router.js b/lib/assets/javascripts/do-catalog/router.js index 713a163191..b1cb6994fa 100644 --- a/lib/assets/javascripts/do-catalog/router.js +++ b/lib/assets/javascripts/do-catalog/router.js @@ -94,7 +94,7 @@ router.afterEach(to => { if (!to.meta.titleInComponent) { document.title = to.meta.title(to); } - addCanonical(`https://carto.com/spatial-data-catalog/browser${to.fullPath}`); + addCanonical(`https://carto.com/spatial-data-catalog/browser${to.path}`); }); }); diff --git a/package-lock.json b/package-lock.json index 102902d7e6..0f61ea8eba 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "cartodb-ui", - "version": "1.0.0-assets.289", + "version": "1.0.0-assets.290", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 175395872f..fe30437076 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cartodb-ui", - "version": "1.0.0-assets.289", + "version": "1.0.0-assets.290", "description": "CARTO UI frontend", "repository": { "type": "git", diff --git a/script/ci/cloudbuild-build-master.yaml b/script/ci/cloudbuild-build-master.yaml index 31522031be..6cc859327e 100644 --- a/script/ci/cloudbuild-build-master.yaml +++ b/script/ci/cloudbuild-build-master.yaml @@ -208,7 +208,7 @@ steps: fi waitFor: ['build-cartodb-onprem', 'build-cartodb', 'build-cartodb-onprem-cron'] -timeout: 1800s +timeout: 3600s substitutions: _BRANCH_TAG: ${BRANCH_NAME//\//-} _DOCKER_IMAGE_NAME: gcr.io/cartodb-on-gcp-main-artifacts/builder diff --git a/script/ci/cloudbuild-build-pr-branch.yaml b/script/ci/cloudbuild-build-pr-branch.yaml index 4bc4e29e26..8f012d88e8 100644 --- a/script/ci/cloudbuild-build-pr-branch.yaml +++ b/script/ci/cloudbuild-build-pr-branch.yaml @@ -155,7 +155,7 @@ steps: docker build --label="org.opencontainers.image.created=$$(date --rfc-3339=seconds)" --label=org.opencontainers.image.revision=${COMMIT_SHA} -t ${_DOCKER_IMAGE_NAME}-onprem-cron:latest -t ${_DOCKER_IMAGE_NAME}-onprem-cron:${_BRANCH_TAG} -t ${_DOCKER_IMAGE_NAME}-onprem-cron:${SHORT_SHA} -t ${_DOCKER_IMAGE_NAME}-onprem-cron:${_BRANCH_TAG}--${SHORT_SHA} -f Dockerfile.cron . waitFor: ['build-cartodb-onprem'] -timeout: 2700s +timeout: 3600s images: - ${_DOCKER_IMAGE_NAME}:${_BRANCH_TAG}--${SHORT_SHA} - ${_DOCKER_IMAGE_NAME}:${_BRANCH_TAG} diff --git a/services/importer/lib/importer/ogr2ogr.rb b/services/importer/lib/importer/ogr2ogr.rb index b459518ac1..b94369c4df 100644 --- a/services/importer/lib/importer/ogr2ogr.rb +++ b/services/importer/lib/importer/ogr2ogr.rb @@ -180,6 +180,7 @@ module CartoDB # Inverse of the selection: if I want guessing I must NOT leave quoted fields as string [ '-oo', 'AUTODETECT_TYPE=YES', + '-oo', 'AUTODETECT_SIZE_LIMIT=20000000', '-oo', "QUOTED_FIELDS_AS_STRING=#{quoted_fields_guessing ? 'NO' : 'YES'}" ] + x_y_possible_names_option + ['-s_srs', 'EPSG:4326', '-t_srs', 'EPSG:4326'] end