cartodb/app/services/visualization/common_data_service.rb

require 'rollbar'
require_relative '../../models/carto/visualization'
require_relative '../../models/carto/external_source'
require_relative '../../models/common_data/singleton'

module CartoDB

  module Visualization

    class CommonDataService

      def initialize(datasets = nil)
        @datasets = datasets
      end

      def self.load_common_data(user, controller)
        if self.configured?
          common_data_url = CartoDB::Visualization::CommonDataService.build_url(controller)
          user.load_common_data(common_data_url)
        end
      end

      def self.configured?
        Cartodb.get_config(:common_data, 'base_url').present?
      end

      def self.build_url(controller)
        common_data_config = Cartodb.config[:common_data]
        return nil unless common_data_config

        common_data_base_url = common_data_config['base_url']
        common_data_username = common_data_config['username']
        common_data_user = Carto::User.where(username: common_data_username).first
        if !common_data_base_url.nil?
          # We set user_domain to nil to avoid duplication in the url for subdomainfull urls. Ie. user.carto.com/u/cartodb/...
          common_data_base_url + CartoDB.path(controller, 'api_v1_visualizations_index', {type: 'table', privacy: 'public', user_domain: nil})
        elsif !common_data_user.nil?
          CartoDB.url(controller, 'api_v1_visualizations_index', params: { type: 'table', privacy: 'public' },
                                                                 user: common_data_user)
        else
          CartoDB.notify_error(
            'cant create common-data url. User doesn\'t exist and base_url is nil',
            username: common_data_username
          )
        end
      end

      def load_common_data_for_user(user, visualizations_api_url)
        update_user_date_flag(user)

        datasets = begin
                     get_datasets(visualizations_api_url)
                   rescue StandardError => e
                     CartoDB::Logger.error(message: "Loading common data failed", exception: e, user: user)
                     nil
                   end
        # As deletion would delete all user syncs, if the endpoint fails or return nothing, just do nothing.
        # The user date flag is updated to avoid DDoS-ing.
        return nil unless datasets.present?

        added = 0
        updated = 0
        not_modified = 0
        deleted = 0
        failed = 0

        carto_user = Carto::User.find(user.id)

        common_data_config = Cartodb.config[:common_data]
        common_data_username = common_data_config['username']
        common_data_remotes_by_name = Carto::Visualization.remotes
                                                          .where(user_id: user.id)
                                                          .joins(:external_source)
                                                          .readonly(false) # joins causes readonly
                                                          .where(external_sources: { username: common_data_username })
                                                          .map { |v| [v.name, v] }.to_h
        ActiveRecord::Base.transaction do
          datasets.each do |dataset|
            begin
              visualization = common_data_remotes_by_name.delete(dataset['name'])
              if visualization
                visualization.attributes = dataset_visualization_attributes(dataset)
                if visualization.changed?
                  visualization.save!
                  updated += 1
                else
                  not_modified += 1
                end
              else
                visualization = Carto::Visualization.new(
                  dataset_visualization_attributes(dataset).merge(
                    name: dataset['name'],
                    user: carto_user,
                    type: Carto::Visualization::TYPE_REMOTE
                  )
                )
                visualization.save!
                added += 1
              end

              external_source = Carto::ExternalSource.where(visualization_id: visualization.id).first
              if external_source
                if external_source.update_data(
                  dataset['url'],
                  dataset['geometry_types'],
                  dataset['rows'],
                  dataset['size'],
                  'common-data'
                ).changed?
                  external_source.save!
                end
              else
                external_source = Carto::ExternalSource.create(
                  visualization_id: visualization.id,
                  import_url: dataset['url'],
                  rows_counted: dataset['rows'],
                  size: dataset['size'],
                  geometry_types: dataset['geometry_types'],
                  username: 'common-data'
                )
              end
            rescue => e
              CartoDB.notify_exception(e, {
                name: dataset.fetch('name', 'ERR: name'),
                source: dataset.fetch('source', 'ERR: source'),
                rows: dataset.fetch('rows', 'ERR: rows'),
                updated_at: dataset.fetch('updated_at', 'ERR: updated_at'),
                url: dataset.fetch('url', 'ERR: url')
              })
              failed += 1
            end
          end
        end

        common_data_remotes_by_name.each_value do |remote|
          deleted += 1 if delete_remote_visualization(remote)
        end

        return added, updated, not_modified, deleted, failed
      end

      def update_user_date_flag(user)
        begin
          user.last_common_data_update_date = Time.now
          if user.valid?
            user.save(raise_on_failure: true)
          elsif user.errors[:quota_in_bytes]
            # This happens for the organization quota validation in the user model so we bypass this
            user.save(:validate => false, raise_on_failure: true)
          end
        rescue => e
          CartoDB.notify_exception(e, {user: user})
        end
      end

      def delete_common_data_for_user(user)
        deleted = 0
        vqb = Carto::VisualizationQueryBuilder.new
        vqb.with_type(Carto::Visualization::TYPE_REMOTE).with_user_id(user.id).build.each do |v|
          delete_remote_visualization(v)
          deleted += 1
        end

        deleted
      end

      private

      def get_datasets(visualizations_api_url)
        @datasets ||= CommonDataSingleton.instance.datasets(visualizations_api_url)
      end

      def delete_remote_visualization(visualization)
        begin
          visualization.destroy
          true
        rescue => e
          match = e.message =~ /violates foreign key constraint "external_data_imports_external_source_id_fkey"/
          if match.present? && match >= 0
            # After #13667 this should no longer happen: deleting remote visualizations is propagated, and external
            # sources, external data imports and syncs are deleted
            CartoDB::Logger.error(message: "Couldn't delete, already imported", visualization_id: visualization.id)
            false
          else
            CartoDB.notify_error(
              "Couldn't delete remote visualization",
              visualization: visualization.id,
              error: e.inspect
            )
            raise e
          end
        end
      end

      def dataset_visualization_attributes(dataset)
        {
          privacy: Carto::Visualization::PRIVACY_PUBLIC,
          description: dataset['description'],
          tags: dataset['tags'],
          license: dataset['license'],
          source: dataset['source'],
          attributions: dataset['attributions'],
          display_name: dataset['display_name']
        }
      end

    end

  end

end