require_relative '../../lib/carto/http/client' require_relative '../../services/sql-api/sql_api' require_relative '../helpers/common_data_redis_cache' require_relative '../../lib/carto_api/json_client' class CommonData def initialize(visualizations_api_url) @datasets = nil @http_client = Carto::Http::Client.get('common_data', log_requests: true) @visualizations_api_url = visualizations_api_url end def datasets return @datasets unless @datasets.nil? @datasets = [] if is_enabled? is_https_request = (@visualizations_api_url =~ /^https:\/\//) cached_data = redis_cache.get(is_https_request) if cached_data.nil? client = CartoAPI::JsonClient.new(http_client_tag: 'common_data') begin response = client.get_visualizations_v1_from_url(@visualizations_api_url) if response.code == 200 @datasets = get_datasets(response.response_body) redis_cache.set(is_https_request, response.headers, response.response_body) else CartoDB::Logger.warning(message: "Error retrieving common data datasets", response: response.to_s) end rescue StandardError => e CartoDB::Logger.error(exception: e) end else @datasets = get_datasets(cached_data[:body]) end CartoDB::Logger.error(message: 'common-data empty', url: @visualizations_api_url) if @datasets.empty? end @datasets end def is_enabled? !config('username').nil? end private def get_datasets(json) begin rows = JSON.parse(json).fetch('visualizations', []) rescue StandardError => e CartoDB::Logger.error(exception: e) rows = [] end datasets = [] rows.each do |row| # Common-data's meta tables starts with meta_ and we want to avoid them next if row["name"] =~ /meta_/ datasets << get_common_data_from_visualization(row) end datasets end def get_common_data_from_visualization(row) { "name" => row["name"], "display_name" => row["display_name"].nil? ? row["name"] : row["display_name"], "tabname" => row["name"], "attributions" => row["attributions"], "description" => row["description"], "source" => row["source"], "license" => row["license"], "tags" => row["tags"], "geometry_types" => %Q[{#{row["table"]["geometry_types"].join(',')}}], "rows" => row["table"]["row_count"], "size" => row["table"]["size"], "url" => export_url(row["name"]), "created_at" => row["created_at"], "updated_at" => row["updated_at"] } end def export_url(table_name) query = %Q[select * from "#{table_name}"] sql_api_url(query, table_name, config('format', 'gpkg')) end def sql_api_url(query, filename, format) common_data_base_url = config('base_url') CartoDB::SQLApi.new({ base_url: common_data_base_url, protocol: 'https', username: config('username'), }).url(query, format, filename) end def config(key, default=nil) if Cartodb.config[:common_data].present? Cartodb.config[:common_data][key].present? ? Cartodb.config[:common_data][key] : default else default end end def redis_cache @redis_cache ||= CommonDataRedisCache.new end end