Merge branch 'master' into feature/ch91595/set-up-cartodb-to-start-using-the-new-carto

.gitignore

@@ -84,3 +84,5 @@ docker-compose.yml
 Dockerfile*
 app_config.yml
 database.yml
+build_resources/
+.dockerignore

NEWS.md

@@ -8,11 +8,14 @@ Development
 - None yet
 
 ### Bug fixes / enhancements
+- WMTS compatibility: Replace the var `tile_matrix_set` by a supported SRS of the WMTS provided.
 - ArcGIS imports: raise http timeout and max retry attempts for arcgis import service
 - ArcGIS imports: improve log traces to better diagnose json non-conformance errors
 - Downgrade bundler to 1.17.3 to avoid problems with Rails version
 - Fix to prevent removing datasets from api_keys when it is replaced using overwrite as collision_strategy ([80981](https://app.clubhouse.io/cartoteam/story/80981/joinzoe-change-on-custom-api-key-after-import-collision-strategy-overwrite))
+- Fix imports from query that contain `(sql_expression)::cast` ([#15765](https://github.com/CartoDB/cartodb/pull/15765))
 - Fix wrong popup position, via new internal carto.js version 4.2.2-1 ([CARTO.js#2254](https://github.com/CartoDB/carto.js/pull/2254))
+- Modify .gitignore
 - Install Carto::Common::Logger with JSON support ([#15762](https://github.com/CartoDB/cartodb/pull/15762))
 
 4.39.0 (2020-07-20)

app/models/data_import.rb

@@ -292,14 +292,19 @@ class DataImport < Sequel::Model
     if data_source.nil?
       values[:data_type] = TYPE_DATASOURCE
       values[:data_source] = ''
-    else
-      path = uploaded_file_path(data_source)
-      if File.exist?(path) && !File.directory?(path)
-        values[:data_type] = TYPE_FILE
-        values[:data_source] = path
-      elsif Addressable::URI.parse(data_source).host.present?
-        values[:data_type] = TYPE_URL
-        values[:data_source] = data_source
+    elsif self.data_type != TYPE_QUERY
+      begin
+        path = uploaded_file_path(data_source)
+        if File.exist?(path) && !File.directory?(path)
+          values[:data_type] = TYPE_FILE
+          values[:data_source] = path
+        elsif Addressable::URI.parse(data_source).host.present?
+          values[:data_type] = TYPE_URL
+          values[:data_source] = data_source
+        end
+      rescue Addressable::URI::InvalidURIError
+        # this should only happen in testing, but just in case capture and log
+        CartoDB::Logger.warning(message: 'InvalidURIError when processing data_source', data_source: data_source)
       end
     end
 

lib/assets/javascripts/builder/components/modals/add-basemap/wms/wms-layer-model.js

@@ -1,6 +1,7 @@
 var _ = require('underscore');
 var Backbone = require('backbone');
 var CustomBaselayerModel = require('builder/data/custom-baselayer-model');
+const DEFAULT_MATRIX_SET = '3857';
 
 /**
  * Model for an individual WMS/WMTS layer.
@@ -149,8 +150,24 @@ module.exports = Backbone.Model.extend({
 
   _xyzURLTemplate: function () {
     var urlTemplate = this.get('url_template') || '';
-    // Convert the proxy template variables to XYZ format, http://foo.com/bar/%%(z)s/%%(x)s/%%(y)s.png"
-    return urlTemplate.replace(/%%\((\w)\)s/g, '{$1}');
+    const matrixSets = this.get('matrix_sets') || [];
+
+    if (matrixSets.length === 0) {
+      throw new Error('The service does not support any SRS.');
+    }
+
+    let matrixSet = DEFAULT_MATRIX_SET;
+
+    if (matrixSets.indexOf(DEFAULT_MATRIX_SET) === -1) {
+      matrixSet = matrixSets[0];
+    }
+
+    // Convert the proxy template variables to XYZ format:
+    // http://foo.com/bar/%(tile_matrix_set)s/%%(z)s/%%(x)s/%%(y)s.png"
+    urlTemplate = urlTemplate.replace(/%%\((\w)\)s/g, '{$1}');
+    urlTemplate = urlTemplate.replace(/%\(tile_matrix_set\)s/g, matrixSet);
+
+    return urlTemplate;
   },
 
   _setCustomBaselayerModel: function (customBaselayerModel) {

lib/assets/test/spec/builder/components/modals/add-basemap/wms/wms-layer-model.spec.js

@@ -104,6 +104,56 @@ describe('editor/components/modals/add-basemap/wms/wms-layer-model', function ()
         expect(this.model.get('state')).toEqual('saveDone');
       });
     });
+
+    describe('when is a WMTS with the tile_matrix_set variable and two matrix sets one of them is the default 3857', function () {
+      beforeEach(function () {
+        this.model.set({
+          type: 'wmts',
+          url_template: 'http://foo.com/bar/%(tile_matrix_set)s/%%(z)s/%%(x)s/%%(y)s.png',
+          matrix_sets: ['4326', '3857']
+        });
+        spyOn(this.model, '_byCustomURL').and.callThrough();
+        this.model.createProxiedLayerOrCustomBaselayerModel();
+      });
+
+      it('should create the tile layer with a proper XYZ URL', function () {
+        expect(this.model._byCustomURL).toHaveBeenCalled();
+        expect(this.model._byCustomURL.calls.argsFor(0)[0]).toEqual('http://foo.com/bar/3857/{z}/{x}/{y}.png');
+      });
+
+      it('should return a tile layer directly instead', function () {
+        expect(this.model.get('customBaselayerModel') instanceof CustomBaselayerModel).toBeTruthy();
+      });
+
+      it('should set saveDone', function () {
+        expect(this.model.get('state')).toEqual('saveDone');
+      });
+    });
+
+    describe('when is a WMTS with the tile_matrix_set variable and two matrix sets', function () {
+      beforeEach(function () {
+        this.model.set({
+          type: 'wmts',
+          url_template: 'http://foo.com/bar/%(tile_matrix_set)s/%%(z)s/%%(x)s/%%(y)s.png',
+          matrix_sets: ['25830', '4326']
+        });
+        spyOn(this.model, '_byCustomURL').and.callThrough();
+        this.model.createProxiedLayerOrCustomBaselayerModel();
+      });
+
+      it('should create the tile layer with a proper XYZ URL', function () {
+        expect(this.model._byCustomURL).toHaveBeenCalled();
+        expect(this.model._byCustomURL.calls.argsFor(0)[0]).toEqual('http://foo.com/bar/25830/{z}/{x}/{y}.png');
+      });
+
+      it('should return a tile layer directly instead', function () {
+        expect(this.model.get('customBaselayerModel') instanceof CustomBaselayerModel).toBeTruthy();
+      });
+
+      it('should set saveDone', function () {
+        expect(this.model.get('state')).toEqual('saveDone');
+      });
+    });
   });
 
   describe('._newProxiedBaselayerModel', function () {

spec/models/data_import_spec.rb

@@ -155,6 +155,15 @@ describe DataImport do
     data_import.log.entries.should match(/relation.*does not exist/)
   end
 
+  it 'should work with a query that contains a "non-valid URI scheme"' do
+    data_import = create_import_from_query(
+      overwrite: false,
+      from_query: 'SELECT (1)::numeric'
+    )
+    data_import.run_import!
+    data_import.state.should eq 'complete'
+  end
+
   it 'should raise an exception if overwriting with missing data' do
     carto_user = Carto::User.find(@user.id)
     carto_user.visualizations.count.should eq 1