require 'rollbar' require_relative '../../models/carto/visualization' require_relative '../../models/carto/external_source' require_relative '../../models/common_data/singleton' module CartoDB module Visualization class CommonDataService def initialize(datasets = nil) @datasets = datasets end def self.load_common_data(user, controller) if self.configured? common_data_url = CartoDB::Visualization::CommonDataService.build_url(controller) user.load_common_data(common_data_url) end end def self.configured? Cartodb.get_config(:common_data, 'base_url').present? end def self.build_url(controller) common_data_config = Cartodb.config[:common_data] return nil unless common_data_config common_data_base_url = common_data_config['base_url'] common_data_username = common_data_config['username'] common_data_user = Carto::User.where(username: common_data_username).first if !common_data_base_url.nil? # We set user_domain to nil to avoid duplication in the url for subdomainfull urls. Ie. user.carto.com/u/cartodb/... common_data_base_url + CartoDB.path(controller, 'api_v1_visualizations_index', {type: 'table', privacy: 'public', user_domain: nil}) elsif !common_data_user.nil? CartoDB.url(controller, 'api_v1_visualizations_index', params: { type: 'table', privacy: 'public' }, user: common_data_user) else CartoDB.notify_error( 'cant create common-data url. User doesn\'t exist and base_url is nil', username: common_data_username ) end end def load_common_data_for_user(user, visualizations_api_url) update_user_date_flag(user) datasets = begin get_datasets(visualizations_api_url) rescue StandardError => e CartoDB::Logger.error(message: "Loading common data failed", exception: e, user: user) nil end # As deletion would delete all user syncs, if the endpoint fails or return nothing, just do nothing. # The user date flag is updated to avoid DDoS-ing. return nil unless datasets.present? added = 0 updated = 0 not_modified = 0 deleted = 0 failed = 0 carto_user = Carto::User.find(user.id) common_data_config = Cartodb.config[:common_data] common_data_username = common_data_config['username'] common_data_remotes_by_name = Carto::Visualization.remotes .where(user_id: user.id) .joins(:external_source) .readonly(false) # joins causes readonly .where(external_sources: { username: common_data_username }) .map { |v| [v.name, v] }.to_h ActiveRecord::Base.transaction do datasets.each do |dataset| begin visualization = common_data_remotes_by_name.delete(dataset['name']) if visualization visualization.attributes = dataset_visualization_attributes(dataset) if visualization.changed? visualization.save! updated += 1 else not_modified += 1 end else visualization = Carto::Visualization.new( dataset_visualization_attributes(dataset).merge( name: dataset['name'], user: carto_user, type: Carto::Visualization::TYPE_REMOTE ) ) visualization.save! added += 1 end external_source = Carto::ExternalSource.where(visualization_id: visualization.id).first if external_source if external_source.update_data( dataset['url'], dataset['geometry_types'], dataset['rows'], dataset['size'], 'common-data' ).changed? external_source.save! end else external_source = Carto::ExternalSource.create( visualization_id: visualization.id, import_url: dataset['url'], rows_counted: dataset['rows'], size: dataset['size'], geometry_types: dataset['geometry_types'], username: 'common-data' ) end rescue => e CartoDB.notify_exception(e, { name: dataset.fetch('name', 'ERR: name'), source: dataset.fetch('source', 'ERR: source'), rows: dataset.fetch('rows', 'ERR: rows'), updated_at: dataset.fetch('updated_at', 'ERR: updated_at'), url: dataset.fetch('url', 'ERR: url') }) failed += 1 end end end common_data_remotes_by_name.each_value do |remote| deleted += 1 if delete_remote_visualization(remote) end return added, updated, not_modified, deleted, failed end def update_user_date_flag(user) begin user.last_common_data_update_date = Time.now if user.valid? user.save(raise_on_failure: true) elsif user.errors[:quota_in_bytes] # This happens for the organization quota validation in the user model so we bypass this user.save(:validate => false, raise_on_failure: true) end rescue => e CartoDB.notify_exception(e, {user: user}) end end def delete_common_data_for_user(user) deleted = 0 vqb = Carto::VisualizationQueryBuilder.new vqb.with_type(Carto::Visualization::TYPE_REMOTE).with_user_id(user.id).build.each do |v| delete_remote_visualization(v) deleted += 1 end deleted end private def get_datasets(visualizations_api_url) @datasets ||= CommonDataSingleton.instance.datasets(visualizations_api_url) end def delete_remote_visualization(visualization) begin visualization.destroy true rescue => e match = e.message =~ /violates foreign key constraint "external_data_imports_external_source_id_fkey"/ if match.present? && match >= 0 # After #13667 this should no longer happen: deleting remote visualizations is propagated, and external # sources, external data imports and syncs are deleted CartoDB::Logger.error(message: "Couldn't delete, already imported", visualization_id: visualization.id) false else CartoDB.notify_error( "Couldn't delete remote visualization", visualization: visualization.id, error: e.inspect ) raise e end end end def dataset_visualization_attributes(dataset) { privacy: Carto::Visualization::PRIVACY_PUBLIC, description: dataset['description'], tags: dataset['tags'], license: dataset['license'], source: dataset['source'], attributions: dataset['attributions'], display_name: dataset['display_name'] } end end end end