From 2f6d067b4e5e7e05c35276d3941c18dce77535a4 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Mon, 5 Nov 2018 17:06:52 +0100 Subject: [PATCH] Parse JSON parameters to avoid problems If we pass JSON as string, with scape characters and so on, we are going to have problems parsing that metrics in ES --- ...db_dataservices_server--0.34.0--0.34.1.sql | 36 ++++++++++++++++++- ...db_dataservices_server--0.34.1--0.34.0.sql | 36 ++++++++++++++++++- .../extension/sql/21_bulk_geocode_street.sql | 3 +- 3 files changed, 72 insertions(+), 3 deletions(-) diff --git a/server/extension/cdb_dataservices_server--0.34.0--0.34.1.sql b/server/extension/cdb_dataservices_server--0.34.0--0.34.1.sql index 642f0da..5fd13f5 100644 --- a/server/extension/cdb_dataservices_server--0.34.0--0.34.1.sql +++ b/server/extension/cdb_dataservices_server--0.34.0--0.34.1.sql @@ -2,4 +2,38 @@ -- Complain if script is sourced in psql, rather than via CREATE EXTENSION \echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.34.1'" to load this file. \quit --- HERE goes your code to upgrade/downgrade \ No newline at end of file +-- HERE goes your code to upgrade/downgrade +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + params = {'username': username, 'orgname': orgname, 'searches': json.loads(searches)} + + with metrics('cdb_bulk_geocode_street_point', user_geocoder_config, logger, params): + if user_geocoder_config.google_geocoder: + provider_function = "_cdb_bulk_google_geocode_street_point"; + elif user_geocoder_config.heremaps_geocoder: + provider_function = "_cdb_bulk_heremaps_geocode_street_point"; + elif user_geocoder_config.tomtom_geocoder: + provider_function = "_cdb_bulk_tomtom_geocode_street_point"; + elif user_geocoder_config.mapbox_geocoder: + provider_function = "_cdb_bulk_mapbox_geocode_street_point"; + else: + raise Exception('Requested geocoder is not available') + + plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.{}($1, $2, $3); ".format(provider_function), ["text", "text", "jsonb"]) + return plpy.execute(plan, [username, orgname, searches]) + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; diff --git a/server/extension/cdb_dataservices_server--0.34.1--0.34.0.sql b/server/extension/cdb_dataservices_server--0.34.1--0.34.0.sql index 7ad4422..8fd4d74 100644 --- a/server/extension/cdb_dataservices_server--0.34.1--0.34.0.sql +++ b/server/extension/cdb_dataservices_server--0.34.1--0.34.0.sql @@ -2,4 +2,38 @@ -- Complain if script is sourced in psql, rather than via CREATE EXTENSION \echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.34.0'" to load this file. \quit --- HERE goes your code to upgrade/downgrade \ No newline at end of file +-- HERE goes your code to upgrade/downgrade +CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocode_street_point(username TEXT, orgname TEXT, searches jsonb) +RETURNS SETOF cdb_dataservices_server.geocoding AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger + import json + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + + plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + + params = {'username': username, 'orgname': orgname, 'searches': searches} + + with metrics('cdb_bulk_geocode_street_point', user_geocoder_config, logger, params): + if user_geocoder_config.google_geocoder: + provider_function = "_cdb_bulk_google_geocode_street_point"; + elif user_geocoder_config.heremaps_geocoder: + provider_function = "_cdb_bulk_heremaps_geocode_street_point"; + elif user_geocoder_config.tomtom_geocoder: + provider_function = "_cdb_bulk_tomtom_geocode_street_point"; + elif user_geocoder_config.mapbox_geocoder: + provider_function = "_cdb_bulk_mapbox_geocode_street_point"; + else: + raise Exception('Requested geocoder is not available') + + plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.{}($1, $2, $3); ".format(provider_function), ["text", "text", "jsonb"]) + return plpy.execute(plan, [username, orgname, searches]) + +$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED; diff --git a/server/extension/sql/21_bulk_geocode_street.sql b/server/extension/sql/21_bulk_geocode_street.sql index 173e684..aa8a875 100644 --- a/server/extension/sql/21_bulk_geocode_street.sql +++ b/server/extension/sql/21_bulk_geocode_street.sql @@ -9,6 +9,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocode_street_poin RETURNS SETOF cdb_dataservices_server.geocoding AS $$ from cartodb_services.metrics import metrics from cartodb_services.tools import Logger + import json plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] @@ -20,7 +21,7 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) - params = {'username': username, 'orgname': orgname, 'searches': searches} + params = {'username': username, 'orgname': orgname, 'searches': json.loads(searches)} with metrics('cdb_bulk_geocode_street_point', user_geocoder_config, logger, params): if user_geocoder_config.google_geocoder: