Adding `location` parameter to the import API

If the user uses Storage API and the dataset is located in different
location than "US" then they have to provide the `location` parameter.
With this parameter the connector creates a temp dataset in that
location if it does not exist in order to allow ODBC driver to create
temp tables.

lib/carto/connector/providers/fdw/odbc/bigquery.rb

@@ -100,7 +100,7 @@ module Carto
       user_attributes %I(RefreshToken)
 
       required_parameters %I(billing_project)
-      optional_parameters %I(project import_as dataset table sql_query storage_api)
+      optional_parameters %I(project location import_as dataset table sql_query storage_api)
 
       # Class constants
       DATASOURCE_NAME              = id
@@ -148,9 +148,11 @@ module Carto
       end
 
       def fixed_odbc_attributes
+        return @server_conf if @server_conf.present?
+
         proxy_conf = create_proxy_conf
 
-        conf = {
+        @server_conf = {
           Driver:         DRIVER_NAME,
           SQLDialect:     SQL_DIALECT,
           OAuthMechanism: OAUTH_MECHANISM,
@@ -165,11 +167,41 @@ module Carto
           LargeResultsTempTableExpirationTime: HTAPI_TEMP_TABLE_EXP
         }
 
+        if @params[:storage_api] == true
+          @server_conf = @server_conf.merge({
+            UseDefaultLargeResultsDataset: 1
+          })
+          if @params[:location].present?
+            @params[:location].upcase!
+            @server_conf = @server_conf.merge({
+              UseDefaultLargeResultsDataset: 0,
+              LargeResultsDataSetId: create_temp_dataset(@params[:billing_project], @params[:location])
+          })
+          end
+        end
+
         if !proxy_conf.nil?
-          conf = conf.merge(proxy_conf)
+          @server_conf = @server_conf.merge(proxy_conf)
         end
 
-        return conf
+        return @server_conf
+      end
+
+      def create_temp_dataset(project_id, location)
+        temp_dataset_id = %{#{HTAPI_TEMP_DATASET}_#{location.downcase}}
+        oauth_client = @sync_oauth&.get_service_datasource
+        if oauth_client
+          begin
+            oauth_client.create_dataset(project_id, temp_dataset_id, {
+              :default_table_expiration_ms => HTAPI_TEMP_TABLE_EXP,
+              :location => location
+            })
+          rescue Google::Apis::ClientError => error
+            # if the dataset exists (409 conflict) do it nothing
+            raise error unless error.status_code == 409
+          end
+        end
+        temp_dataset_id
       end
 
       def remote_schema_name

services/datasources/lib/datasources/url/bigquery.rb

@@ -2,6 +2,7 @@
 
 require 'signet/oauth_2/client'
 require_relative '../../../../../lib/carto/http/client'
+require 'google/apis/bigquery_v2'
 
 module CartoDB
   module Datasources
@@ -41,6 +42,8 @@ module CartoDB
             access_type: :offline
           )
           @revoke_uri = config.fetch('revoke_auth_uri')
+          @bigquery_api = Google::Apis::BigqueryV2::BigqueryService.new
+          @bigquery_api.authorization = @client
         end
 
         # Factory method
@@ -201,6 +204,16 @@ module CartoDB
           raise AuthError.new("revoke_token: #{ex.message}", DATASOURCE_NAME)
         end
 
+        def create_dataset(project_id, dataset_id, options)
+          dataset = Google::Apis::BigqueryV2::Dataset.new(options.merge({
+            :dataset_reference => Google::Apis::BigqueryV2::DatasetReference.new({
+              :project_id => project_id,
+              :dataset_id => dataset_id,
+            })
+          }))
+          @bigquery_api.insert_dataset(project_id, dataset)
+        end
+
       end
     end
   end