zhongjin
/
cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Changes from CODE REVIEW

Browse Source
pull/15479/head
manmorjim 5 years ago
parent 968778e1af
commit 6c297459e6
8 changed files with 78 additions and 122 deletions
Show Stats Download Patch File Download Diff File

26
app/models/user/db_service.rb
Unescape View File

@ -908,20 +908,19 @@ module CartoDB
# PG12_DEPRECATED in postgis 3+
def set_raster_privileges(role_name = nil)
database = @user.in_database(as: :superuser)
if database.table_exists?('raster_overviews')
# Postgis lives at public schema, so raster catalogs too
catalogs_schema = SCHEMA_PUBLIC
queries = [
"GRANT SELECT ON TABLE \"#{catalogs_schema}\".\"raster_overviews\" TO \"#{CartoDB::PUBLIC_DB_USER}\"",
"GRANT SELECT ON TABLE \"#{catalogs_schema}\".\"raster_columns\" TO \"#{CartoDB::PUBLIC_DB_USER}\""
]
target_user = role_name.nil? ? @user.database_public_username : role_name
unless @user.organization.nil?
queries << "GRANT SELECT ON TABLE \"#{catalogs_schema}\".\"raster_overviews\" TO \"#{target_user}\""
queries << "GRANT SELECT ON TABLE \"#{catalogs_schema}\".\"raster_columns\" TO \"#{target_user}\""
end
@queries.run_in_transaction(queries, true)
return unless database.table_exists?('raster_overviews')
# Postgis lives at public schema, so raster catalogs too
catalogs_schema = SCHEMA_PUBLIC
queries = [
"GRANT SELECT ON TABLE \"#{catalogs_schema}\".\"raster_overviews\" TO \"#{CartoDB::PUBLIC_DB_USER}\"",
"GRANT SELECT ON TABLE \"#{catalogs_schema}\".\"raster_columns\" TO \"#{CartoDB::PUBLIC_DB_USER}\""
]
target_user = role_name || @user.database_public_username
unless @user.organization.nil?
queries << "GRANT SELECT ON TABLE \"#{catalogs_schema}\".\"raster_overviews\" TO \"#{target_user}\""
queries << "GRANT SELECT ON TABLE \"#{catalogs_schema}\".\"raster_columns\" TO \"#{target_user}\""
end
@queries.run_in_transaction(queries, true)
end
def setup_organization_role_permissions
@ -1517,6 +1516,7 @@ module CartoDB
varnish_retry = Cartodb.config[:varnish_management].try(:[], 'retry') || 5
varnish_trigger_verbose = Cartodb.config[:varnish_management].fetch('trigger_verbose', true) == true ? 1 : 0
# PG12_DEPRECATED remove support for python 2
@user.in_database(as: :superuser).run(
<<-TRIGGER
BEGIN;

5
app/models/visualization/support_tables.rb
Unescape View File

@ -19,7 +19,8 @@ module CartoDB
# Only intended to be used if from the Visualization Relator (who will set the parent)
# PG12_DEPRECATED in postgis 3+
def load_actual_list(parent_name=nil)
return [] if @parent_id.nil? || @parent_kind != Visualization::Member::KIND_RASTER
return [] if @parent_id.nil? || @parent_kind != Visualization::Member::KIND_RASTER ||
!@database.table_exists?('raster_overviews')
parent = Visualization::Member.new(id:@parent_id).fetch
table_data = @database.fetch(%Q{
SELECT o_table_catalog AS catalog, o_table_schema AS schema, o_table_name AS name
@ -27,7 +28,7 @@ module CartoDB
WHERE r_table_catalog = '#{parent.user.database_name}'
AND r_table_schema = '#{parent.user.database_schema}'
AND r_table_name = '#{parent_name.nil? ? parent.name : parent_name}'
}).all if @database.table_exists?('raster_overviews')
}).all
table_data.nil? ? [] : table_data
end

2
lib/sql

@ -1 +1 @@
Subproject commit 076cacc6f1b30c84d949d8041b7649ece585456a
Subproject commit 998402e531c903352f16003efc2f25ef5e094019

42
services/importer/spec/acceptance/raster2pgsql_spec.rb
Unescape View File

@ -111,46 +111,4 @@ describe 'raster2pgsql acceptance tests' do
AND table_name LIKE 'o_%_#{job.table_name}'})
raster_tables.should be 0
end
it 'keeps the original table unaltered regardless of overviews' do
pending "Fix for multiple CI configs #7645"
TOLERANCE = 1e-6
filepath = path_to('raster_simple.tif')
downloader = CartoDB::Importer2::Downloader.new(@user.id, filepath)
log = CartoDB::Importer2::Doubles::Log.new(@user)
job = Job.new({ logger: log, pg_options: @user.db_service.db_configuration_for.with_indifferent_access })
runner = CartoDB::Importer2::Runner.new({
pg: @user.db_service.db_configuration_for,
downloader: downloader,
log: CartoDB::Importer2::Doubles::Log.new(@user),
user: @user,
job: job
})
runner.run
# Values taken from `gdalinfo -stats services/importer/spec/fixtures/raster_simple.tif`
# PG12_DEPRECATED
if @user.in_database.table_exists?('raster_columns')
metadata = @user.in_database.fetch(%{
SELECT * FROM raster_columns
WHERE r_table_schema = '#{job.schema}' AND r_table_name = '#{job.table_name}'
}).first
metadata[:srid].should eq 4326
metadata[:scale_x].should be_within(TOLERANCE).of +0.148148148148133
metadata[:scale_y].should be_within(TOLERANCE).of -0.148148148148133
end
stats = @user.in_database.fetch(%{
WITH foo AS (
SELECT the_raster_webmercator rast FROM #{job.schema}.#{job.table_name}
) SELECT (stats).* FROM (SELECT ST_summarystatsagg(rast, TRUE, 1) stats FROM foo) bar;
}).first
stats[:count].should eq 2430 * 1215
stats[:mean].should be_within(TOLERANCE).of 204.51453640197
stats[:stddev].should be_within(TOLERANCE).of 11.11767348697
stats[:min].should be_within(TOLERANCE).of 58
stats[:max].should be_within(TOLERANCE).of 253
end
end

18
services/user-mover/export_user.rb
Unescape View File

@ -79,15 +79,15 @@ module CartoDB
FROM pg_views
WHERE viewname = 'raster_overviews';
}).count > 0
if raster_available
raster_tables = user_pg_conn.exec("SELECT DISTINCT r_table_schema, r_table_name FROM raster_columns").map {
|r| "#{r['r_table_schema']}.#{r['r_table_name']}"
}
overview_re = Regexp.new('([^\.]+)\.o_\d+_(.+)$')
@orphan_overviews = raster_tables.select do |table|
match = overview_re.match(table)
match && !raster_tables.include?("#{match.captures.first}.#{match.captures.last}")
end
return @orphan_overviews ||= [] unless raster_available
raster_tables = user_pg_conn.exec("SELECT DISTINCT r_table_schema, r_table_name FROM raster_columns").map {
|r| "#{r['r_table_schema']}.#{r['r_table_name']}"
}
overview_re = Regexp.new('([^\.]+)\.o_\d+_(.+)$')
@orphan_overviews = raster_tables.select do |table|
match = overview_re.match(table)
match && !raster_tables.include?("#{match.captures.first}.#{match.captures.last}")
end
@orphan_overviews ||= []
end

45
spec/lib/carto/ghost_tables_manager_spec.rb
Unescape View File

@ -229,35 +229,34 @@ module Carto
end
it 'should link raster tables' do
if ::User[@user.id].in_database.table_exists?('raster_overviews')
run_in_user_database(%{
CREATE TABLE manolo_raster ("cartodb_id" uuid, "the_raster_webmercator" raster);
CREATE TRIGGER test_quota_per_row
BEFORE INSERT OR UPDATE
ON manolo_raster
FOR EACH ROW
EXECUTE PROCEDURE cdb_checkquota('0.001', '-1', 'public');
})
next unless ::User[@user.id].in_database.table_exists?('raster_overviews')
run_in_user_database(%{
CREATE TABLE manolo_raster ("cartodb_id" uuid, "the_raster_webmercator" raster);
CREATE TRIGGER test_quota_per_row
BEFORE INSERT OR UPDATE
ON manolo_raster
FOR EACH ROW
EXECUTE PROCEDURE cdb_checkquota('0.001', '-1', 'public');
})
@user.tables.count.should eq 0
@ghost_tables_manager.instance_eval { user_tables_synced_with_db? }.should be_false
@user.tables.count.should eq 0
@ghost_tables_manager.instance_eval { user_tables_synced_with_db? }.should be_false
@ghost_tables_manager.link_ghost_tables_synchronously
@ghost_tables_manager.link_ghost_tables_synchronously
@user.tables.count.should eq 1
@user.tables.first.name.should == 'manolo_raster'
@user.tables.count.should eq 1
@user.tables.first.name.should == 'manolo_raster'
run_in_user_database(%{
DROP TABLE manolo_raster;
})
run_in_user_database(%{
DROP TABLE manolo_raster;
})
@user.tables.count.should eq 1
@ghost_tables_manager.instance_eval { user_tables_synced_with_db? }.should be_false
@ghost_tables_manager.link_ghost_tables_synchronously
@user.tables.count.should eq 1
@ghost_tables_manager.instance_eval { user_tables_synced_with_db? }.should be_false
@ghost_tables_manager.link_ghost_tables_synchronously
@user.tables.count.should eq 0
@ghost_tables_manager.instance_eval { user_tables_synced_with_db? }.should be_true
end
@user.tables.count.should eq 0
@ghost_tables_manager.instance_eval { user_tables_synced_with_db? }.should be_true
end
it 'should regenerate user tables with bad table_ids' do

43
spec/models/carto/user_migration_api_key_spec.rb
Unescape View File

@ -16,32 +16,31 @@ describe 'UserMigration' do
it 'exports and imports a user with raster overviews because exporting skips them' do
CartoDB::UserModule::DBService.any_instance.stubs(:enable_remote_db_user).returns(true)
user = FactoryGirl.build(:valid_user).save
next unless user.in_database.table_exists?('raster_overviews')
carto_user = Carto::User.find(user.id)
user_attributes = carto_user.attributes
if user.in_database.table_exists?('raster_overviews')
user.in_database.execute('CREATE TABLE i_hate_raster(rast raster)')
user.in_database.execute('INSERT INTO i_hate_raster VALUES(ST_MakeEmptyRaster(100, 100, 0, 0, 100, 100, 0, 0, 2274))')
user.in_database.execute("UPDATE i_hate_raster SET rast = ST_AddBand(rast, 1, '32BF'::text, 0)")
user.in_database.execute("UPDATE i_hate_raster SET rast = ST_AddBand(rast, 1, '32BF'::text, 0)")
user.in_database.execute("SELECT AddRasterConstraints('i_hate_raster', 'rast')")
user.in_database.execute("SELECT ST_CreateOverview('i_hate_raster'::regclass, 'rast', 2)")
user.in_database.execute('DROP TABLE i_hate_raster')
export = Carto::UserMigrationExport.create(user: carto_user, export_metadata: true)
export.run_export
user.destroy
user.in_database.execute('CREATE TABLE i_hate_raster(rast raster)')
user.in_database.execute('INSERT INTO i_hate_raster VALUES(ST_MakeEmptyRaster(100, 100, 0, 0, 100, 100, 0, 0, 2274))')
user.in_database.execute("UPDATE i_hate_raster SET rast = ST_AddBand(rast, 1, '32BF'::text, 0)")
user.in_database.execute("UPDATE i_hate_raster SET rast = ST_AddBand(rast, 1, '32BF'::text, 0)")
user.in_database.execute("SELECT AddRasterConstraints('i_hate_raster', 'rast')")
user.in_database.execute("SELECT ST_CreateOverview('i_hate_raster'::regclass, 'rast', 2)")
user.in_database.execute('DROP TABLE i_hate_raster')
export = Carto::UserMigrationExport.create(user: carto_user, export_metadata: true)
export.run_export
user.destroy
import = Carto::UserMigrationImport.create(
exported_file: export.exported_file,
database_host: user_attributes['database_host'],
org_import: false,
json_file: export.json_file,
import_metadata: true,
dry: false
)
import.run_import
import = Carto::UserMigrationImport.create(
exported_file: export.exported_file,
database_host: user_attributes['database_host'],
org_import: false,
json_file: export.json_file,
import_metadata: true,
dry: false
)
import.run_import
expect(import.state).to eq(Carto::UserMigrationImport::STATE_COMPLETE)
end
expect(import.state).to eq(Carto::UserMigrationImport::STATE_COMPLETE)
end
describe 'legacy functions' do

19
spec/models/table_spec.rb
Unescape View File

@ -1362,18 +1362,17 @@ describe Table do
end
it "should not fail when the analyze is executed in update_table_geom_pg_stats and raises a PG::UndefinedColumn" do
if @user.in_database.table_exists?('raster_overviews')
delete_user_data @user
data_import = DataImport.create(user_id: @user.id,
data_source: fake_data_path('import_raster.tif.zip'))
data_import.run_import!
next unless @user.in_database.table_exists?('raster_overviews')
delete_user_data @user
data_import = DataImport.create(user_id: @user.id,
data_source: fake_data_path('import_raster.tif.zip'))
data_import.run_import!
table = Table.new(user_table: UserTable[data_import.table_id])
table.should_not be_nil, "Import failure: #{data_import.log}"
table.name.should match(/^import_raster/)
table = Table.new(user_table: UserTable[data_import.table_id])
table.should_not be_nil, "Import failure: #{data_import.log}"
table.name.should match(/^import_raster/)
table.update_table_geom_pg_stats
end
table.update_table_geom_pg_stats
end
it "should not drop a table that exists when upload fails" do

Write Preview
Loading…
Cancel
Save