更新 'package.json'

[sc230232] Avoid deleting a user if it has shared entities

.gitmodules

@@ -1,12 +1,9 @@
 [submodule "lib/assets/javascripts/cdb"]
 	path = lib/assets/javascripts/cdb
-        url = git@github.com:CartoDB/carto.js.git
+        url = https://github.com/CartoDB/carto.js.git
 [submodule "app/assets/stylesheets/old_common"]
 	path = app/assets/stylesheets/old_common
-	url = git@github.com:CartoDB/cartodb.css.git
+	url = https://github.com/CartoDB/cartodb.css.git
 [submodule "lib/sql"]
 	path = lib/sql
-	url = git@github.com:CartoDB/cartodb-postgresql.git
-[submodule "private"]
-	path = private
-	url = git@github.com:CartoDB/cartodb-private.git
+	url = https://github.com/CartoDB/cartodb-postgresql.git

.readthedocs.yaml

@@ -0,0 +1,4 @@
+version: 2
+
+submodules:
+  exclude: all

NEWS.md

@@ -2,9 +2,10 @@ Development
 -----------
 
 ### NOTICES
-- None yet
+- None yet 
 
 ### Features
+* Display GeoParquet link in Catalog [16453](https://github.com/CartoDB/cartodb/pull/16453)
 * Display notifications about the new CARTO platform release [16352](https://github.com/CartoDB/cartodb/pull/16352)
 * Upgrade to deck.gl 8.5.6 [16338](https://github.com/CartoDB/cartodb/pull/16338)
 * Update DO Catalog dependencies and some changes to use bundle on CARTO Workspace [#16325](https://github.com/CartoDB/cartodb/pull/16325)
@@ -37,6 +38,7 @@ Development
 - Update browser version checker to allow Firefox/100.0 [#16415](https://github.com/CartoDB/cartodb/pull/16415)
 - Update analysis schemas after giving required permissions on user promotion [#16390](https://github.com/CartoDB/cartodb/pull/16390)
 - Add timeout for SQL API exports [#16377](https://github.com/CartoDB/cartodb/pull/16377)
+- Avoid deleting a user if it has shared entities [#16424](https://github.com/CartoDB/cartodb/pull/16424)
 - Remove all references to Spatial Data Catalog and Kepler GL maps in on-premises [#16293](https://github.com/CartoDB/cartodb/pull/16293)
 - Increase hard-limit of MAX_TABLES_PER_IMPORT [#16374](https://github.com/CartoDB/cartodb/pull/16374)
 - Guard code for vizjson users [#16267](https://github.com/CartoDB/cartodb/pull/16267)
@@ -95,6 +97,11 @@ Development
 - Fix duplicated attributions in datasets [#16384](https://github.com/CartoDB/cartodb/pull/16384)
 - Moving assets cdn domain from global.ssl.fastly.net to libs.cartocdn.com [#16399](https://github.com/CartoDB/cartodb/pull/16399)
 - Fix error while rolling back a user migration from one cloud to another [#16421](https://github.com/CartoDB/cartodb/pull/16421)
+- Add retry if a timeout is thrown when swapping the tables related with a sync process [#16430](https://github.com/CartoDB/cartodb/pull/16430)
+- Add AUTODETECT_SIZE_LIMIT to ogr2ogr process when guessing CSV file column types [#16431](https://github.com/CartoDB/cartodb/pull/16431)
+- Log pg locks if there is any problem during a sync table import process [#16432](https://github.com/CartoDB/cartodb/pull/16432)
+- Check pg locks during sync table swap and terminate locking queries [#16433](https://github.com/CartoDB/cartodb/pull/16433)
+- Add deprecation notice in docs [#16446](https://github.com/CartoDB/cartodb/pull/16446)
 
 4.45.0 (2021-04-14)
 -------------------

README.md

@@ -1,3 +1,10 @@
+# [DEPRECATED]
+
+Hey! This content applies only to previous CARTO products.
+
+Please check if it's relevant to your use case. On October 2021 we released the current version of our platform.
+You can learn more and read the latest documentation at docs.carto.com
+
 # What is CARTO?
 
 [![Code Climate](https://codeclimate.com/github/CartoDB/cartodb20.png)](https://codeclimate.com/github/CartoDB/cartodb20)

app/controllers/carto/api/organization_users_controller.rb

@@ -121,8 +121,8 @@ module Carto
         force_destroy = params[:force].present?
 
         if !force_destroy && @user.has_shared_entities?
-          error_message = "Can't delete @user. 'Has shared entities"
-          render_jsonp(error_message, 410 ) and return
+          error_message = "Can't delete user. Has shared entities"
+          render_jsonp(error_message, 401) and return
         end
 
         @user.set_force_destroy if force_destroy

app/controllers/carto/api/users_controller.rb

@@ -111,7 +111,13 @@ module Carto
         deletion_password_confirmation = params[:deletion_password_confirmation]
 
         if user.needs_password_confirmation? && !user.validate_old_password(deletion_password_confirmation)
-          render_jsonp({ message: "Error deleting user: #{PASSWORD_DOES_NOT_MATCH_MESSAGE}" }, 400) and return
+          render_jsonp({ message: "Error deleting user: #{PASSWORD_DOES_NOT_MATCH_MESSAGE}" }, 400)
+          return
+        end
+
+        if user.has_shared_entities?
+          render_jsonp({ message: "User can't be deleted because there are shared entities. Please, unshare or delete them and try again." }, 401)
+          return
         end
 
         user.destroy_account

app/models/synchronization/adapter.rb

@@ -11,7 +11,7 @@ module CartoDB
       THE_GEOM = 'the_geom'.freeze
       OVERWRITE_ERROR = 2013
 
-      def initialize(table_name, runner, database, user, overviews_creator, synchronization_id)
+      def initialize(table_name, runner, database, user, overviews_creator, synchronization_id, logger = nil)
         @table_name   = table_name
         @runner       = runner
         @database     = database
@@ -25,6 +25,7 @@ module CartoDB
         )
         @error_code = nil
         @synchronization_id = synchronization_id
+        @logger = logger
       end
 
       def run(&tracker)
@@ -140,11 +141,7 @@ module CartoDB
         table_statements = @table_setup.generate_table_statements(schema, table_name)
 
         temporary_name = temporary_name_for(result.table_name)
-        database.transaction do
-          rename(table_name, temporary_name) if exists?(table_name)
-          drop(temporary_name) if exists?(temporary_name)
-          rename(result.table_name, table_name)
-        end
+        swap_tables(table_name, temporary_name, result)
         @table_setup.fix_oid(table_name)
         @table_setup.update_cdb_tablemetadata(table_name)
         @table_setup.run_table_statements(table_statements, @database)
@@ -389,6 +386,43 @@ module CartoDB
 
       private
 
+      def swap_tables(table_name, temporary_name, result)
+        database.transaction do
+          rename(table_name, temporary_name) if exists?(table_name)
+          drop(temporary_name) if exists?(temporary_name)
+          rename(result.table_name, table_name)
+        end
+      rescue Exception => exception
+        if exception.message.include?('canceling statement due to statement timeout')
+          # Check if the table has any lock and cancel locking queries
+          locks = user.in_database(as: :superuser).fetch(%Q{
+            SELECT pid, query 
+            FROM pg_stat_activity 
+            WHERE pid in (
+              SELECT pid FROM pg_locks l 
+              JOIN pg_class t ON l.relation = t.oid 
+              AND t.relkind = 'r' 
+              WHERE t.relname IN ('#{table_name}')
+            );
+          }).all
+          @logger.append_and_store "Transaction timed out as the table is blocked by other queries. Terminating locking queries and retrying in 60 seconds..." if @logger && locks.present?
+          locks.each do |lock|
+              @logger.append_and_store "Terminating query: #{lock[:query]}" if @logger
+              user.in_database(as: :superuser).execute %Q{
+              SELECT pg_terminate_backend(#{lock[:pid]});
+            }
+          end
+          sleep(60) # wait 60 seconds and retry the swap
+          database.transaction do
+            rename(table_name, temporary_name) if exists?(table_name)
+            drop(temporary_name) if exists?(temporary_name)
+            rename(result.table_name, table_name)
+          end
+        else
+          raise exception
+        end
+      end
+
       def valid_cartodb_id_candidate?(user, table_name, qualified_table_name, col_name)
         return false unless column_names(user, table_name).include?(col_name)
         user.transaction_with_timeout(statement_timeout: STATEMENT_TIMEOUT, as: :superuser) do |db|

app/models/synchronization/member.rb

@@ -192,7 +192,7 @@ module CartoDB
 
         database = user.in_database
         overviews_creator = CartoDB::Importer2::Overviews.new(runner, user)
-        importer = CartoDB::Synchronization::Adapter.new(name, runner, database, user, overviews_creator, id)
+        importer = CartoDB::Synchronization::Adapter.new(name, runner, database, user, overviews_creator, id, @log)
 
         importer.run
         self.ran_at   = Time.now

doc/manual/source/_static/deprecation_popup.js

@@ -0,0 +1,19 @@
+function ready() {
+  var div = document.createElement("div");
+  div.style.position = 'absolute',
+  div.style.bottom = '40px'
+  div.style.left = '0'
+  div.style.right = '0'
+  div.style.with = '100%'
+  div.style.padding = '10px'
+  div.innerHTML = `
+  <h4>Hey! This content applies only to previous CARTO products</h4>
+  <p>Please check if it's relevant to your use case. On October 2021 we released a new version of our platform.</p>
+  <p>You can learn more and read the latest documentation at <a target="_blank" href="https://docs.carto.com">docs.carto.com</a></p>
+  `;
+  
+  document.getElementsByTagName("nav")[0].appendChild(div);
+}
+
+
+document.addEventListener("DOMContentLoaded", ready);

doc/manual/source/conf.py

@@ -267,3 +267,7 @@ texinfo_documents = [
 
 # Example configuration for intersphinx: refer to the Python standard library.
 intersphinx_mapping = {'http://docs.python.org/': None}
+
+html_js_files = [
+  'deprecation_popup.js'
+]

lib/assets/javascripts/new-dashboard/pages/Data/CatalogDatasetSummary.vue

@@ -101,6 +101,17 @@
             </router-link>
           </p>
         </li>
+        <li
+          class="u-mb--32 u-mb--12--tablet"
+          v-if="hasGeoparquetAvailable"
+        >
+          <h4 class="text is-small is-txtSoftGrey u-mb--10">
+            GeoParquet
+          </h4>
+          <p class="text is-caption">
+            <a class="underline" :href="geoparquetUrl" target="_blank">Download</a>
+          </p>
+        </li>
       </ul>
     </div>
   </div>
@@ -112,6 +123,7 @@ import { temporalAggregationName } from 'new-dashboard/utils/catalog/temporal-ag
 import { geometryTypeName } from 'new-dashboard/utils/catalog/geometry-type-name';
 import { updateFrequencyName } from 'new-dashboard/utils/catalog/update-frequency-name';
 import { sendCustomDimensions } from 'new-dashboard/utils/catalog/custom-dimensions-ga';
+import { checkGeoparquetBucket } from 'new-dashboard/utils/catalog/geoparquet';
 import CatalogMapPreview from 'new-dashboard/components/Catalog/CatalogMapPreview';
 
 export default {
@@ -119,6 +131,12 @@ export default {
   components: {
     CatalogMapPreview
   },
+  data () {
+    return {
+      hasGeoparquetAvailable: false,
+      geoparquetUrl: undefined
+    };
+  },
   watch: {
     dataset: {
       handler (value) {
@@ -169,10 +187,16 @@ export default {
         id: this.$route.params.entity_id,
         type: this.$route.params.entity_type
       });
+    },
+    async checkIfGeoparquetBucketExists () {
+      const { ok, url } = await checkGeoparquetBucket(this.$route.params.entity_id);
+      this.hasGeoparquetAvailable = ok;
+      this.geoparquetUrl = url;
     }
   },
   mounted () {
     this.fetchKeyVariables();
+    this.checkIfGeoparquetBucketExists();
   }
 };
 </script>

lib/assets/javascripts/new-dashboard/utils/catalog/geoparquet.js

@@ -0,0 +1,9 @@
+export async function checkGeoparquetBucket (dataset) {
+  try {
+    const url = `https://storage.googleapis.com/geoparquet/carto/${dataset}.parquet`;
+    const response = await fetch(url, { method: 'HEAD' });
+    return { ok: response.ok, url };
+  } catch (e) {
+    return { ok: false };
+  }
+}

package.json

@@ -1,6 +1,6 @@
 {
   "name": "cartodb-ui",
-  "version": "1.0.0-assets.290",
+  "version": "1.0.0-assets.291",
   "description": "CARTO UI frontend",
   "repository": {
     "type": "git",
@@ -23,7 +23,7 @@
     "@carto/carto.js": "^4.2.1",
     "@carto/toolkit-core": "0.0.1-rc.18",
     "@carto/toolkit-custom-storage": "0.0.1-rc.18",
-    "@carto/viewer": "github:CartoDB/viewer#v1.0.8",
+    "@carto/viewer": "https://github.com/CartoDB/viewer#v1.0.8",
     "@carto/zera": "1.0.7",
     "@deck.gl/carto": "8.5.6",
     "@deck.gl/core": "8.5.6",
@@ -65,7 +65,7 @@
     "moment": "2.18.1",
     "moment-timezone": "^0.5.13",
     "node-polyglot": "1.0.0",
-    "perfect-scrollbar": "github:CartoDB/perfect-scrollbar.git#master",
+    "perfect-scrollbar": "https://github.com/CartoDB/perfect-scrollbar.git#master",
     "postcss": "5.0.19",
     "postcss-scss": "0.4.0",
     "postcss-strip-inline-comments": "0.1.5",

script/ci/cloudbuild-build-master.yaml

@@ -208,7 +208,7 @@ steps:
       fi
   waitFor: ['build-cartodb-onprem', 'build-cartodb', 'build-cartodb-onprem-cron']
 
-timeout: 1800s
+timeout: 3600s
 substitutions:
   _BRANCH_TAG: ${BRANCH_NAME//\//-}
   _DOCKER_IMAGE_NAME: gcr.io/cartodb-on-gcp-main-artifacts/builder

script/ci/cloudbuild-build-pr-branch.yaml

@@ -155,7 +155,7 @@ steps:
       docker build --label="org.opencontainers.image.created=$$(date --rfc-3339=seconds)" --label=org.opencontainers.image.revision=${COMMIT_SHA} -t ${_DOCKER_IMAGE_NAME}-onprem-cron:latest -t ${_DOCKER_IMAGE_NAME}-onprem-cron:${_BRANCH_TAG} -t ${_DOCKER_IMAGE_NAME}-onprem-cron:${SHORT_SHA} -t ${_DOCKER_IMAGE_NAME}-onprem-cron:${_BRANCH_TAG}--${SHORT_SHA}  -f Dockerfile.cron .        
   waitFor: ['build-cartodb-onprem']
 
-timeout: 2700s
+timeout: 3600s
 images:
     - ${_DOCKER_IMAGE_NAME}:${_BRANCH_TAG}--${SHORT_SHA} 
     - ${_DOCKER_IMAGE_NAME}:${_BRANCH_TAG} 

script/ci/known_hosts

@@ -1 +1,3 @@
-github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==
+|1|ueDZd5nj+jew6gJuR6J043fBwRc=|+HaH18kuYx21mjgs1Uz/SWRhQEY= ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg=
+|1|0RrLFzimw+0ZVQNhDVHeUHbG1xk=|MdpZry4/IdRdz3jpEKPAPiDEWmA= ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg=
+|1|IU85mbarZxbwrbxBhbXKiI7hSuw=|FXKLY4UBpBGMLAB0WDDeijtQ7u0= ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg=

services/importer/lib/importer/ogr2ogr.rb

@@ -180,6 +180,7 @@ module CartoDB
           # Inverse of the selection: if I want guessing I must NOT leave quoted fields as string
           [
             '-oo', 'AUTODETECT_TYPE=YES',
+            '-oo', 'AUTODETECT_SIZE_LIMIT=20000000',
             '-oo', "QUOTED_FIELDS_AS_STRING=#{quoted_fields_guessing ? 'NO' : 'YES'}"
           ] + x_y_possible_names_option + ['-s_srs', 'EPSG:4326', '-t_srs', 'EPSG:4326']
         end