diff --git a/server/extension/cdb_dataservices_server--0.16.0--0.17.0.sql b/server/extension/cdb_dataservices_server--0.16.0--0.17.0.sql new file mode 100644 index 0000000..e69de29 diff --git a/server/extension/cdb_dataservices_server--0.17.0--0.16.0.sql b/server/extension/cdb_dataservices_server--0.17.0--0.16.0.sql new file mode 100644 index 0000000..e69de29 diff --git a/server/extension/cdb_dataservices_server.control b/server/extension/cdb_dataservices_server.control index 08a06f3..a84fc1c 100644 --- a/server/extension/cdb_dataservices_server.control +++ b/server/extension/cdb_dataservices_server.control @@ -1,5 +1,5 @@ comment = 'CartoDB dataservices server extension' -default_version = '0.16.0' +default_version = '0.17.0' requires = 'plpythonu, plproxy, postgis, cdb_geocoder' superuser = true schema = cdb_dataservices_server diff --git a/server/extension/cdb_dataservices_server--0.15.1--0.16.0.sql b/server/extension/old_versions/cdb_dataservices_server--0.15.1--0.16.0.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.15.1--0.16.0.sql rename to server/extension/old_versions/cdb_dataservices_server--0.15.1--0.16.0.sql diff --git a/server/extension/cdb_dataservices_server--0.16.0--0.15.1.sql b/server/extension/old_versions/cdb_dataservices_server--0.16.0--0.15.1.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.16.0--0.15.1.sql rename to server/extension/old_versions/cdb_dataservices_server--0.16.0--0.15.1.sql diff --git a/server/extension/cdb_dataservices_server--0.16.0.sql b/server/extension/old_versions/cdb_dataservices_server--0.16.0.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.16.0.sql rename to server/extension/old_versions/cdb_dataservices_server--0.16.0.sql diff --git a/server/extension/sql/105_route_between_points.sql b/server/extension/sql/105_route_between_points.sql index cc17933..217fc06 100644 --- a/server/extension/sql/105_route_between_points.sql +++ b/server/extension/sql/105_route_between_points.sql @@ -7,15 +7,21 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point( options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers') RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) - waypoints = [origin, destination] - mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) - result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units]) - return [result[0]['shape'],result[0]['length'], result[0]['duration']] + with metrics('cdb_route_with_point', user_routing_config, logger): + waypoints = [origin, destination] + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] $$ LANGUAGE plpythonu; @@ -27,12 +33,18 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints( options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers') RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) - mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) - result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units]) - return [result[0]['shape'],result[0]['length'], result[0]['duration']] + with metrics('cdb_route_with_waypoints', user_routing_config, logger): + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] $$ LANGUAGE plpythonu; diff --git a/server/extension/sql/110_data_observatory_augmentation.sql b/server/extension/sql/110_data_observatory_augmentation.sql index d75bcb1..117ee9c 100644 --- a/server/extension/sql/110_data_observatory_augmentation.sql +++ b/server/extension/sql/110_data_observatory_augmentation.sql @@ -11,9 +11,9 @@ RETURNS text AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] + user_obs_config = GD["user_obs_config_{0}".format(username)] - return user_obs_snapshot_config.connection_str + return user_obs_config.connection_str $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshotJSON( @@ -34,6 +34,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot( time_span TEXT DEFAULT NULL, geometry_level TEXT DEFAULT NULL) RETURNS json AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.tools import Logger,LoggerConfig import json @@ -41,31 +42,32 @@ RETURNS json AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] + user_obs_config = GD["user_obs_config_{0}".format(username)] plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") logger_config = GD["logger_config"] logger = Logger(logger_config) - quota_service = QuotaService(user_obs_snapshot_config, redis_conn) + quota_service = QuotaService(user_obs_config, redis_conn) if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - try: - obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshotJSON($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) - if result: - quota_service.increment_success_service_use() - return result[0]['snapshot'] - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to obs_get_demographic_snapshot') - finally: - quota_service.increment_total_service_use() + with metrics('obs_getdemographicsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshotJSON($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_demographic_snapshot') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshot( @@ -86,41 +88,43 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetDemographicSnapshot( time_span TEXT DEFAULT NULL, geometry_level TEXT DEFAULT NULL) RETURNS SETOF JSON AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.tools import Logger,LoggerConfig plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] + user_obs_config = GD["user_obs_config_{0}".format(username)] plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") logger_config = GD["logger_config"] logger = Logger(logger_config) - quota_service = QuotaService(user_obs_snapshot_config, redis_conn) + quota_service = QuotaService(user_obs_config, redis_conn) if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - try: - obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) - if result: - resp = [] - for element in result: - value = element['snapshot'] - resp.append(value) - quota_service.increment_success_service_use() - return resp - else: - quota_service.increment_empty_service_use() - return [] - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to obs_get_demographic_snapshot') - finally: - quota_service.increment_total_service_use() + with metrics('obs_getdemographicsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) + if result: + resp = [] + for element in result: + value = element['snapshot'] + resp.append(value) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_demographic_snapshot') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshotJSON( @@ -139,6 +143,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot( geom geometry(Geometry, 4326), geometry_level TEXT DEFAULT NULL) RETURNS json AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.tools import Logger,LoggerConfig import json @@ -146,31 +151,32 @@ RETURNS json AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] + user_obs_config = GD["user_obs_config_{0}".format(username)] plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") logger_config = GD["logger_config"] logger = Logger(logger_config) - quota_service = QuotaService(user_obs_snapshot_config, redis_conn) + quota_service = QuotaService(user_obs_config, redis_conn) if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - try: - obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshotJSON($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) - if result: - quota_service.increment_success_service_use() - return result[0]['snapshot'] - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to obs_get_segment_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to obs_get_segment_snapshot') - finally: - quota_service.increment_total_service_use() + with metrics('obs_getsegmentsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshotJSON($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_segment_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_segment_snapshot') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshot( @@ -189,41 +195,43 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetSegmentSnapshot( geom geometry(Geometry, 4326), geometry_level TEXT DEFAULT NULL) RETURNS SETOF JSON AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.tools import Logger,LoggerConfig plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] + user_obs_config = GD["user_obs_config_{0}".format(username)] plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") logger_config = GD["logger_config"] logger = Logger(logger_config) - quota_service = QuotaService(user_obs_snapshot_config, redis_conn) + quota_service = QuotaService(user_obs_config, redis_conn) if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - try: - obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) - if result: - resp = [] - for element in result: - value = element['snapshot'] - resp.append(value) - quota_service.increment_success_service_use() - return resp - else: - quota_service.increment_empty_service_use() - return [] - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetSegmentSnapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_GetSegmentSnapshot') - finally: - quota_service.increment_total_service_use() + with metrics('obs_getsegmentsnapshot', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) + if result: + resp = [] + for element in result: + value = element['snapshot'] + resp.append(value) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetSegmentSnapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetSegmentSnapshot') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasure( @@ -248,6 +256,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasure( boundary_id TEXT DEFAULT NULL, time_span TEXT DEFAULT NULL) RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.tools import Logger,LoggerConfig @@ -263,22 +272,23 @@ RETURNS NUMERIC AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - try: - obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasure($1, $2, $3, $4, $5, $6, $7) as measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, measure_id, normalize, boundary_id, time_span]) - if result: - quota_service.increment_success_service_use() - return result[0]['measure'] - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetMeasure', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_GetMeasure') - finally: - quota_service.increment_total_service_use() + with metrics('obs_getmeasure', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasure($1, $2, $3, $4, $5, $6, $7) as measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, measure_id, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasure', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasure') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetCategory( @@ -301,6 +311,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetCategory( boundary_id TEXT DEFAULT NULL, time_span TEXT DEFAULT NULL) RETURNS TEXT AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.tools import Logger,LoggerConfig @@ -316,22 +327,23 @@ RETURNS TEXT AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - try: - obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetCategory($1, $2, $3, $4, $5, $6) as category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, category_id, boundary_id, time_span]) - if result: - quota_service.increment_success_service_use() - return result[0]['category'] - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetCategory', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_GetCategory') - finally: - quota_service.increment_total_service_use() + with metrics('obs_getcategory', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetCategory($1, $2, $3, $4, $5, $6) as category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, category_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['category'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetCategory', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetCategory') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusMeasure( @@ -356,6 +368,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusMeasure( boundary_id TEXT DEFAULT NULL, time_span TEXT DEFAULT NULL) RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.tools import Logger,LoggerConfig @@ -371,22 +384,23 @@ RETURNS NUMERIC AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - try: - obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusMeasure($1, $2, $3, $4, $5, $6, $7) as census_measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, name, normalize, boundary_id, time_span]) - if result: - quota_service.increment_success_service_use() - return result[0]['census_measure'] - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetUSCensusMeasure', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_GetUSCensusMeasure') - finally: - quota_service.increment_total_service_use() + with metrics('obs_getuscensusmeasure', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusMeasure($1, $2, $3, $4, $5, $6, $7) as census_measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, name, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['census_measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetUSCensusMeasure', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetUSCensusMeasure') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusCategory( @@ -409,6 +423,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusCategory( boundary_id TEXT DEFAULT NULL, time_span TEXT DEFAULT NULL) RETURNS TEXT AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.tools import Logger,LoggerConfig @@ -424,22 +439,23 @@ RETURNS TEXT AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - try: - obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusCategory($1, $2, $3, $4, $5, $6) as census_category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, name, boundary_id, time_span]) - if result: - quota_service.increment_success_service_use() - return result[0]['census_category'] - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetUSCensusCategory', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_GetUSCensusCategory') - finally: - quota_service.increment_total_service_use() + with metrics('obs_getuscensuscategory', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusCategory($1, $2, $3, $4, $5, $6) as census_category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, name, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['census_category'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetUSCensusCategory', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetUSCensusCategory') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPopulation( @@ -462,6 +478,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPopulation( boundary_id TEXT DEFAULT NULL, time_span TEXT DEFAULT NULL) RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.tools import Logger,LoggerConfig @@ -477,22 +494,23 @@ RETURNS NUMERIC AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - try: - obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetPopulation($1, $2, $3, $4, $5, $6) as population;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, normalize, boundary_id, time_span]) - if result: - quota_service.increment_success_service_use() - return result[0]['population'] - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetPopulation', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_GetPopulation') - finally: - quota_service.increment_total_service_use() + with metrics('obs_getpopulation', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetPopulation($1, $2, $3, $4, $5, $6) as population;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['population'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPopulation', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPopulation') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasureById( @@ -515,6 +533,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasureById( boundary_id TEXT, time_span TEXT DEFAULT NULL) RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.tools import Logger,LoggerConfig @@ -530,20 +549,21 @@ RETURNS NUMERIC AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - try: - obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasureById($1, $2, $3, $4, $5, $6) as measure;", ["text", "text", "text", "text", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom_ref, measure_id, boundary_id, time_span]) - if result: - quota_service.increment_success_service_use() - return result[0]['measure'] - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_GetMeasureById') - finally: - quota_service.increment_total_service_use() + with metrics('obs_getmeasurebyid', user_obs_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasureById($1, $2, $3, $4, $5, $6) as measure;", ["text", "text", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom_ref, measure_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasureById') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; diff --git a/server/extension/sql/115_data_observatory_exploration.sql b/server/extension/sql/115_data_observatory_exploration.sql index 301e0bb..9abe2b2 100644 --- a/server/extension/sql/115_data_observatory_exploration.sql +++ b/server/extension/sql/115_data_observatory_exploration.sql @@ -14,6 +14,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_Search( search_term TEXT, relevant_boundary TEXT DEFAULT NULL) RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.tools import Logger,LoggerConfig @@ -29,30 +30,31 @@ RETURNS TABLE(id text, description text, name text, aggregate text, source text) if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - try: - obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_Search($1, $2, $3, $4);", ["text", "text", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, search_term, relevant_boundary]) - if result: - resp = [] - for element in result: - id = element['id'] - description = element['description'] - name = element['name'] - aggregate = element['aggregate'] - source = element['source'] - resp.append([id, description, name, aggregate, source]) - quota_service.increment_success_service_use() - return resp - else: - quota_service.increment_empty_service_use() - return [None, None, None, None, None] - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_Search', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_Search') - finally: - quota_service.increment_total_service_use() + with metrics('obs_search', user_obs_snapshot_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_Search($1, $2, $3, $4);", ["text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, search_term, relevant_boundary]) + if result: + resp = [] + for element in result: + id = element['id'] + description = element['description'] + name = element['name'] + aggregate = element['aggregate'] + source = element['source'] + resp.append([id, description, name, aggregate, source]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [None, None, None, None, None] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_Search', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_Search') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetAvailableBoundaries( @@ -71,6 +73,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableBoundaries( geom geometry(Geometry, 4326), time_span TEXT DEFAULT NULL) RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.tools import Logger,LoggerConfig @@ -86,27 +89,28 @@ RETURNS TABLE(boundary_id text, description text, time_span text, tablename text if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - try: - obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetAvailableBoundaries($1, $2, $3, $4) as available_boundaries;", ["text", "text", "geometry(Geometry, 4326)", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, time_span]) - if result: - resp = [] - for element in result: - id = element['boundary_id'] - description = element['description'] - tspan = element['time_span'] - tablename = element['tablename'] - resp.append([id, description, tspan, tablename]) - quota_service.increment_success_service_use() - return resp - else: - quota_service.increment_empty_service_use() - return [] - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_GetMeasureById') - finally: - quota_service.increment_total_service_use() + with metrics('obs_getavailableboundaries', user_obs_snapshot_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetAvailableBoundaries($1, $2, $3, $4) as available_boundaries;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span]) + if result: + resp = [] + for element in result: + id = element['boundary_id'] + description = element['description'] + tspan = element['time_span'] + tablename = element['tablename'] + resp.append([id, description, tspan, tablename]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasureById') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; diff --git a/server/extension/sql/120_data_observatory_geometries.sql b/server/extension/sql/120_data_observatory_geometries.sql index f7b815f..b69f318 100644 --- a/server/extension/sql/120_data_observatory_geometries.sql +++ b/server/extension/sql/120_data_observatory_geometries.sql @@ -16,6 +16,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundary( boundary_id TEXT, time_span TEXT DEFAULT NULL) RETURNS geometry(Geometry, 4326) AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.tools import Logger,LoggerConfig @@ -31,22 +32,23 @@ RETURNS geometry(Geometry, 4326) AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - try: - obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundary($1, $2, $3, $4) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) - if result: - quota_service.increment_success_service_use() - return result[0]['boundary'] - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetBoundary', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_GetBoundary') - finally: - quota_service.increment_total_service_use() + with metrics('obs_getboundary', user_obs_snapshot_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundary($1, $2, $3, $4) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundary', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundary') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryId( @@ -67,6 +69,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryId( boundary_id TEXT, time_span TEXT DEFAULT NULL) RETURNS TEXT AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.tools import Logger,LoggerConfig @@ -82,22 +85,23 @@ RETURNS TEXT AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - try: - obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryId($1, $2, $3, $4, $5) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) - if result: - quota_service.increment_success_service_use() - return result[0]['boundary'] - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetBoundaryId', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_GetBoundaryId') - finally: - quota_service.increment_total_service_use() + with metrics('obs_getboundaryid', user_obs_snapshot_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryId($1, $2, $3, $4, $5) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundaryId', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundaryId') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryById( @@ -118,6 +122,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryById( boundary_id TEXT, time_span TEXT DEFAULT NULL) RETURNS geometry(Geometry, 4326) AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.tools import Logger,LoggerConfig @@ -133,22 +138,23 @@ RETURNS geometry(Geometry, 4326) AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - try: - obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryById($1, $2, $3, $4, $5) as boundary;", ["text", "text", "text", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geometry_id, boundary_id, time_span]) - if result: - quota_service.increment_success_service_use() - return result[0]['boundary'] - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetBoundaryById', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_GetBoundaryById') - finally: - quota_service.increment_total_service_use() + with metrics('obs_getboundarybyid', user_obs_snapshot_config, logger): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryById($1, $2, $3, $4, $5) as boundary;", ["text", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geometry_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundaryById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundaryById') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByGeometry( @@ -171,6 +177,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByGeometry( time_span TEXT DEFAULT NULL, overlap_type TEXT DEFAULT NULL) RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.tools import Logger,LoggerConfig @@ -186,27 +193,28 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - try: - obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) - if result: - resp = [] - for element in result: - the_geom = element['the_geom'] - geom_refs = element['geom_refs'] - resp.append([the_geom, geom_refs]) - quota_service.increment_success_service_use() - return resp - else: - quota_service.increment_empty_service_use() - return [] - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetBoundariesByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_GetBoundariesByGeometry') - finally: - quota_service.increment_total_service_use() + with metrics('obs_getboundariesbygeometry', user_obs_snapshot_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundariesByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundariesByGeometry') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius( @@ -231,6 +239,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByPointAndRa time_span TEXT DEFAULT NULL, overlap_type TEXT DEFAULT NULL) RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.tools import Logger,LoggerConfig @@ -246,27 +255,28 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - try: - obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) - if result: - resp = [] - for element in result: - the_geom = element['the_geom'] - geom_refs = element['geom_refs'] - resp.append([the_geom, geom_refs]) - quota_service.increment_success_service_use() - return resp - else: - quota_service.increment_empty_service_use() - return [] - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetBoundariesByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_GetBoundariesByPointAndRadius') - finally: - quota_service.increment_total_service_use() + with metrics('obs_getboundariesbypointandradius', user_obs_snapshot_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundariesByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundariesByPointAndRadius') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByGeometry( @@ -289,6 +299,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByGeometry( time_span TEXT DEFAULT NULL, overlap_type TEXT DEFAULT NULL) RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.tools import Logger,LoggerConfig @@ -304,27 +315,28 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - try: - obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) - if result: - resp = [] - for element in result: - the_geom = element['the_geom'] - geom_refs = element['geom_refs'] - resp.append([the_geom, geom_refs]) - quota_service.increment_success_service_use() - return resp - else: - quota_service.increment_empty_service_use() - return [] - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetPointsByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_GetPointsByGeometry') - finally: - quota_service.increment_total_service_use() + with metrics('obs_getpointsbygeometry', user_obs_snapshot_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPointsByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPointsByGeometry') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByPointAndRadius( @@ -349,6 +361,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByPointAndRadius time_span TEXT DEFAULT NULL, overlap_type TEXT DEFAULT NULL) RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.tools import Logger,LoggerConfig @@ -364,25 +377,26 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - try: - obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) - if result: - resp = [] - for element in result: - the_geom = element['the_geom'] - geom_refs = element['geom_refs'] - resp.append([the_geom, geom_refs]) - quota_service.increment_success_service_use() - return resp - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetPointsByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_GetPointsByPointAndRadius') - finally: - quota_service.increment_total_service_use() + with metrics('obs_getpointsbypointandradius', user_obs_snapshot_config, logger): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPointsByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPointsByPointAndRadius') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; diff --git a/server/extension/sql/20_geocode_street.sql b/server/extension/sql/20_geocode_street.sql index ef1fc9b..b4fbcd1 100644 --- a/server/extension/sql/20_geocode_street.sql +++ b/server/extension/sql/20_geocode_street.sql @@ -1,22 +1,28 @@ -- Geocodes a street address given a searchtext and a state and/or country CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) - if user_geocoder_config.heremaps_geocoder: - here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) - return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] - elif user_geocoder_config.google_geocoder: - google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) - return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] - elif user_geocoder_config.mapzen_geocoder: - mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) - return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] - else: - raise Exception('Requested geocoder is not available') + with metrics('cdb_geocode_street_point', user_geocoder_config, logger): + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.mapzen_geocoder: + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Requested geocoder is not available') $$ LANGUAGE plpythonu; diff --git a/server/extension/sql/30_admin0.sql b/server/extension/sql/30_admin0.sql index 10e8873..593063a 100644 --- a/server/extension/sql/30_admin0.sql +++ b/server/extension/sql/30_admin0.sql @@ -2,6 +2,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(us RETURNS Geometry AS $$ from cartodb_services.metrics import QuotaService from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.metrics import metrics from cartodb_services.tools import Logger,LoggerConfig plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) @@ -13,23 +14,24 @@ RETURNS Geometry AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) - try: - plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"]) - rv = plpy.execute(plan, [country_name], 1) - result = rv[0]["mypolygon"] - if result: - quota_service.increment_success_service_use() - return result - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to geocode admin0 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to geocode admin0 polygon') - finally: - quota_service.increment_total_service_use() + with metrics('cdb_geocode_admin0_polygon', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin0 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin0 polygon') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; diff --git a/server/extension/sql/40_admin1.sql b/server/extension/sql/40_admin1.sql index 9a17394..d883e22 100644 --- a/server/extension/sql/40_admin1.sql +++ b/server/extension/sql/40_admin1.sql @@ -1,6 +1,7 @@ ---- cdb_geocode_admin1_polygon(admin1_name text) CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.metrics import InternalGeocoderConfig from cartodb_services.tools import Logger,LoggerConfig @@ -14,44 +15,11 @@ RETURNS Geometry AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) - try: - plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1)) AS mypolygon", ["text"]) - rv = plpy.execute(plan, [admin1_name], 1) - result = rv[0]["mypolygon"] - if result: - quota_service.increment_success_service_use() - return result - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to geocode admin1 polygon') - finally: - quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu; ----- cdb_geocode_admin1_polygon(admin1_name text, country_name text) -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) -RETURNS Geometry AS $$ - from cartodb_services.metrics import QuotaService - from cartodb_services.metrics import InternalGeocoderConfig - from cartodb_services.tools import Logger,LoggerConfig - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] - - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger): try: - plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1), trim($2)) AS mypolygon", ["text", "text"]) - rv = plpy.execute(plan, [admin1_name, country_name], 1) + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [admin1_name], 1) result = rv[0]["mypolygon"] if result: quota_service.increment_success_service_use() @@ -68,6 +36,44 @@ RETURNS Geometry AS $$ quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1), trim($2)) AS mypolygon", ["text", "text"]) + rv = plpy.execute(plan, [admin1_name, country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin1 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + -------------------------------------------------------------------------------- -- Implementation of the server extension diff --git a/server/extension/sql/50_namedplaces.sql b/server/extension/sql/50_namedplaces.sql index c586f3f..338591d 100644 --- a/server/extension/sql/50_namedplaces.sql +++ b/server/extension/sql/50_namedplaces.sql @@ -35,7 +35,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_namedplac RETURNS Geometry AS $$ from cartodb_services.mapzen import MapzenGeocoder from cartodb_services.mapzen.types import country_to_iso3 - from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import QuotaService, metrics from cartodb_services.tools import Logger,LoggerConfig plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) @@ -50,35 +50,36 @@ RETURNS Geometry AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - try: - geocoder = MapzenGeocoder(user_geocoder_config.mapzen_api_key, logger) - country_iso3 = None - if country_name: - country_iso3 = country_to_iso3(country_name) - coordinates = geocoder.geocode(searchtext=city_name, city=None, - state_province=admin1_name, - country=country_iso3, search_type='locality') - if coordinates: - quota_service.increment_success_service_use() - plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) - point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] - return point['st_setsrid'] - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to geocode city point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to geocode city point using mapzen') - finally: - quota_service.increment_total_service_use() + with metrics('cdb_geocode_namedplace_point', user_geocoder_config, logger): + try: + geocoder = MapzenGeocoder(user_geocoder_config.mapzen_api_key, logger) + country_iso3 = None + if country_name: + country_iso3 = country_to_iso3(country_name) + coordinates = geocoder.geocode(searchtext=city_name, city=None, + state_province=admin1_name, + country=country_iso3, search_type='locality') + if coordinates: + quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode city point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode city point using mapzen') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_internal_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL) RETURNS Geometry AS $$ from cartodb_services.metrics import QuotaService - from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.metrics import InternalGeocoderConfig, metrics from cartodb_services.tools import Logger,LoggerConfig plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) @@ -90,30 +91,32 @@ RETURNS Geometry AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) - try: - if admin1_name and country_name: - plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"]) - rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1) - elif country_name: - plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"]) - rv = plpy.execute(plan, [city_name, country_name], 1) - else: - plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"]) - rv = plpy.execute(plan, [city_name], 1) - result = rv[0]["mypoint"] - if result: - quota_service.increment_success_service_use() - return result - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to geocode namedplace point', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to geocode namedplace point') - finally: - quota_service.increment_total_service_use() + + with metrics('cdb_geocode_namedplace_point', user_geocoder_config, logger): + try: + if admin1_name and country_name: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"]) + rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1) + elif country_name: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"]) + rv = plpy.execute(plan, [city_name, country_name], 1) + else: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [city_name], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode namedplace point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode namedplace point') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; -------------------------------------------------------------------------------- diff --git a/server/extension/sql/60_postalcodes.sql b/server/extension/sql/60_postalcodes.sql index 9d7243a..d7d610e 100644 --- a/server/extension/sql/60_postalcodes.sql +++ b/server/extension/sql/60_postalcodes.sql @@ -1,5 +1,6 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text) RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.metrics import InternalGeocoderConfig from cartodb_services.tools import Logger,LoggerConfig @@ -13,27 +14,29 @@ RETURNS Geometry AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) - try: - plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1)) AS mypoint", ["text"]) - rv = plpy.execute(plan, [code], 1) - result = rv[0]["mypoint"] - if result: - quota_service.increment_success_service_use() - return result - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to geocode postal code point') - finally: - quota_service.increment_total_service_use() + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code point') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text) RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.metrics import InternalGeocoderConfig from cartodb_services.tools import Logger,LoggerConfig @@ -47,27 +50,29 @@ RETURNS Geometry AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) - try: - plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1), trim($2)) AS mypoint", ["TEXT", "TEXT"]) - rv = plpy.execute(plan, [code, country], 1) - result = rv[0]["mypoint"] - if result: - quota_service.increment_success_service_use() - return result - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to geocode postal code point') - finally: - quota_service.increment_total_service_use() + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1), trim($2)) AS mypoint", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code point') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text) RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.metrics import InternalGeocoderConfig from cartodb_services.tools import Logger,LoggerConfig @@ -81,27 +86,29 @@ RETURNS Geometry AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) - try: - plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1)) AS mypolygon", ["text"]) - rv = plpy.execute(plan, [code], 1) - result = rv[0]["mypolygon"] - if result: - quota_service.increment_success_service_use() - return result - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to geocode postal code polygon') - finally: - quota_service.increment_total_service_use() + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text, country text) RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.metrics import InternalGeocoderConfig from cartodb_services.tools import Logger,LoggerConfig @@ -115,23 +122,24 @@ RETURNS Geometry AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) - try: - plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1), trim($2)) AS mypolygon", ["TEXT", "TEXT"]) - rv = plpy.execute(plan, [code, country], 1) - result = rv[0]["mypolygon"] - if result: - quota_service.increment_success_service_use() - return result - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to geocode postal code polygon') - finally: - quota_service.increment_total_service_use() + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1), trim($2)) AS mypolygon", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; -------------------------------------------------------------------------------- diff --git a/server/extension/sql/70_ips.sql b/server/extension/sql/70_ips.sql index 2212d09..2b452aa 100644 --- a/server/extension/sql/70_ips.sql +++ b/server/extension/sql/70_ips.sql @@ -1,5 +1,6 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text) RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.metrics import InternalGeocoderConfig from cartodb_services.tools import Logger,LoggerConfig @@ -13,23 +14,24 @@ RETURNS Geometry AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) - try: - plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_ipaddress_point(trim($1)) AS mypoint", ["TEXT"]) - rv = plpy.execute(plan, [ip], 1) - result = rv[0]["mypoint"] - if result: - quota_service.increment_success_service_use() - return result - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to geocode postal code polygon') - finally: - quota_service.increment_total_service_use() + with metrics('cdb_geocode_ipaddress_point', user_geocoder_config, logger): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_ipaddress_point(trim($1)) AS mypoint", ["TEXT"]) + rv = plpy.execute(plan, [ip], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; -------------------------------------------------------------------------------- diff --git a/server/extension/sql/90_isochrone.sql b/server/extension/sql/90_isochrone.sql index 24d336e..6f4186c 100644 --- a/server/extension/sql/90_isochrone.sql +++ b/server/extension/sql/90_isochrone.sql @@ -1,21 +1,27 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.metrics import metrics + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) if user_isolines_config.google_services_user: raise Exception('This service is not available for google service users.') - if user_isolines_config.heremaps_provider: - here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) - elif user_isolines_config.mapzen_provider: - mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) - else: - raise Exception('Requested isolines provider is not available') + with metrics('cb_isochrone', user_isolines_config, logger): + if user_isolines_config.heremaps_provider: + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + else: + raise Exception('Requested isolines provider is not available') $$ LANGUAGE plpythonu; -- heremaps isochrone diff --git a/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py index a221db9..d176a7d 100644 --- a/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py @@ -5,9 +5,10 @@ import json import requests from exceptions import * +from cartodb_services.metrics import Traceable -class HereMapsGeocoder: +class HereMapsGeocoder(Traceable): 'A Here Maps Geocoder wrapper for python' PRODUCTION_GEOCODE_JSON_URL = 'https://geocoder.api.here.com/6.2/geocode.json' @@ -89,6 +90,7 @@ class HereMapsGeocoder: request_params.update(params) response = requests.get(self.host, params=request_params, timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT)) + self.add_response_data(response, self._logger) if response.status_code == requests.codes.ok: return json.loads(response.text) elif response.status_code == requests.codes.bad_request: diff --git a/server/lib/python/cartodb_services/cartodb_services/here/routing.py b/server/lib/python/cartodb_services/cartodb_services/here/routing.py index 11856be..221a2ea 100644 --- a/server/lib/python/cartodb_services/cartodb_services/here/routing.py +++ b/server/lib/python/cartodb_services/cartodb_services/here/routing.py @@ -2,9 +2,10 @@ import requests import json from exceptions import WrongParams +from cartodb_services.metrics import Traceable -class HereMapsRoutingIsoline: +class HereMapsRoutingIsoline(Traceable): 'A Here Maps Routing wrapper for python' PRODUCTION_ROUTING_BASE_URL = 'https://isoline.route.api.here.com' @@ -54,6 +55,7 @@ class HereMapsRoutingIsoline: parsed_options) response = requests.get(self._url, params=request_params, timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT)) + self.add_response_data(response, self._logger) if response.status_code == requests.codes.ok: return self.__parse_isolines_response(response.text) elif response.status_code == requests.codes.bad_request: diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py index 4cc974d..3ba3dff 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py @@ -5,9 +5,10 @@ import re from exceptions import WrongParams, MalformedResult, ServiceException from qps import qps_retry from cartodb_services.tools import Coordinate, PolyLine +from cartodb_services.metrics import Traceable -class MapzenGeocoder: +class MapzenGeocoder(Traceable): 'A Mapzen Geocoder wrapper for python' BASE_URL = 'https://search.mapzen.com/v1/search' @@ -28,6 +29,7 @@ class MapzenGeocoder: try: response = requests.get(self._url, params=request_params, timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT)) + self.add_response_data(response, self._logger) if response.status_code == requests.codes.ok: return self.__parse_response(response.text) elif response.status_code == requests.codes.bad_request: diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py index 98639e3..c10a912 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py @@ -2,9 +2,10 @@ import requests import json from qps import qps_retry from exceptions import ServiceException +from cartodb_services.metrics import Traceable -class MatrixClient: +class MatrixClient(Traceable): """ A minimal client for Mapzen Time-Distance Matrix Service @@ -45,6 +46,7 @@ class MatrixClient: } response = requests.get(self.ONE_TO_MANY_URL, params=request_params, timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT)) + self.add_response_data(response, self._logger) if response.status_code != requests.codes.ok: self._logger.error('Error trying to get matrix distance from mapzen', diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/routing.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/routing.py index e0dca62..d970d69 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/routing.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/routing.py @@ -5,9 +5,10 @@ import re from exceptions import WrongParams, MalformedResult, ServiceException from qps import qps_retry from cartodb_services.tools import Coordinate, PolyLine +from cartodb_services.metrics import MetricsDataGatherer, Traceable -class MapzenRouting: +class MapzenRouting(Traceable): 'A Mapzen Routing wrapper for python' PRODUCTION_ROUTING_BASE_URL = 'https://valhalla.mapzen.com/route' @@ -47,6 +48,7 @@ class MapzenRouting: request_params = self.__parse_request_parameters(json_request_params) response = requests.get(self._url, params=request_params, timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT)) + self.add_response_data(response, self._logger) if response.status_code == requests.codes.ok: return self.__parse_routing_response(response.text) elif response.status_code == requests.codes.bad_request: diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py b/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py index 078bbbd..b0a4181 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py @@ -1,3 +1,4 @@ from config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatorySnapshotConfig, ObservatoryConfig from quota import QuotaService from user import UserMetricsService +from log import metrics, MetricsDataGatherer, Traceable diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index 8c739fa..897b179 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -15,6 +15,7 @@ class ServiceConfig(object): self._username = username self._orgname = orgname self._db_config = ServicesDBConfig(db_conn, username, orgname) + self._metrics_log_path = self.__get_metrics_log_path() self._environment = self._db_config._server_environment if redis_connection: self._redis_config = ServicesRedisConfig(redis_connection).build( @@ -38,9 +39,20 @@ class ServiceConfig(object): def environment(self): return self._environment + @property + def metrics_log_path(self): + return self._metrics_log_path + + def __get_metrics_log_path(self): + if self.METRICS_LOG_KEY: + return self._db_config.logger_config.get(self.METRICS_LOG_KEY, None) + else: + return None class DataObservatoryConfig(ServiceConfig): + METRICS_LOG_KEY = 'do_log_path' + def __init__(self, redis_connection, db_conn, username, orgname=None): super(DataObservatoryConfig, self).__init__(redis_connection, db_conn, username, orgname) @@ -61,6 +73,10 @@ class DataObservatoryConfig(ServiceConfig): def connection_str(self): return self._connection_str + @property + def provider(self): + return 'data observatory' + class ObservatorySnapshotConfig(DataObservatoryConfig): @@ -118,6 +134,7 @@ class RoutingConfig(ServiceConfig): DEFAULT_PROVIDER = 'mapzen' QUOTA_KEY = 'mapzen_routing_quota' SOFT_LIMIT_KEY = 'soft_mapzen_routing_limit' + METRICS_LOG_KEY = 'routing_log_path' def __init__(self, redis_connection, db_conn, username, orgname=None): super(RoutingConfig, self).__init__(redis_connection, db_conn, @@ -135,6 +152,10 @@ class RoutingConfig(ServiceConfig): if self._routing_provider == self.MAPZEN_PROVIDER: return 'routing_mapzen' + @property + def provider(self): + return self._routing_provider + @property def mapzen_api_key(self): return self._mapzen_api_key @@ -151,7 +172,6 @@ class RoutingConfig(ServiceConfig): def soft_limit(self): return self._soft_limit - def _set_monthly_quota(self): self._monthly_quota = self._get_effective_monthly_quota() @@ -169,7 +189,6 @@ class RoutingConfig(ServiceConfig): self._soft_limit = False - class IsolinesRoutingConfig(ServiceConfig): ISOLINES_CONFIG_KEYS = ['here_isolines_quota', 'soft_here_isolines_limit', @@ -184,6 +203,7 @@ class IsolinesRoutingConfig(ServiceConfig): MAPZEN_PROVIDER = 'mapzen' HEREMAPS_PROVIDER = 'heremaps' DEFAULT_PROVIDER = 'heremaps' + METRICS_LOG_KEY = 'isolines_log_path' def __init__(self, redis_connection, db_conn, username, orgname=None): super(IsolinesRoutingConfig, self).__init__(redis_connection, db_conn, @@ -260,11 +280,12 @@ class IsolinesRoutingConfig(ServiceConfig): class InternalGeocoderConfig(ServiceConfig): + METRICS_LOG_KEY = 'geocoder_log_path' + def __init__(self, redis_connection, db_conn, username, orgname=None): # For now, internal geocoder doesn't use the redis config super(InternalGeocoderConfig, self).__init__(None, db_conn, username, orgname) - self._log_path = self._db_config.geocoder_log_path @property def service_type(self): @@ -283,8 +304,8 @@ class InternalGeocoderConfig(ServiceConfig): return None @property - def log_path(self): - return self._log_path + def provider(self): + return 'internal' class GeocoderConfig(ServiceConfig): @@ -310,6 +331,7 @@ class GeocoderConfig(ServiceConfig): ORGNAME_KEY = 'orgname' PERIOD_END_DATE = 'period_end_date' DEFAULT_PROVIDER = 'mapzen' + METRICS_LOG_KEY = 'geocoder_log_path' def __init__(self, redis_connection, db_conn, username, orgname=None, forced_provider=None): super(GeocoderConfig, self).__init__(redis_connection, db_conn, @@ -341,7 +363,6 @@ class GeocoderConfig(ServiceConfig): self._geocoder_provider = self.DEFAULT_PROVIDER self._geocoding_quota = float(filtered_config[self.QUOTA_KEY]) self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE]) - self._log_path = db_config.geocoder_log_path if filtered_config[self.SOFT_LIMIT_KEY].lower() == 'true': self._soft_geocoding_limit = True else: @@ -424,8 +445,8 @@ class GeocoderConfig(ServiceConfig): return self._cost_per_hit @property - def log_path(self): - return self._log_path + def provider(self): + return self._geocoder_provider class ServicesDBConfig: @@ -440,7 +461,6 @@ class ServicesDBConfig: self._get_server_config() self._get_here_config() self._get_mapzen_config() - self._get_logger_config() self._get_data_observatory_config() def _get_server_config(self): @@ -493,13 +513,6 @@ class ServicesDBConfig: else: self._data_observatory_connection_str = do_conf['connection']['production'] - def _get_logger_config(self): - logger_conf_json = self._get_conf('logger_conf') - if not logger_conf_json: - raise ConfigException('Logger configuration missing') - else: - logger_conf = json.loads(logger_conf_json) - self._geocoder_log_path = logger_conf['geocoder_log_path'] def _get_conf(self, key): try: @@ -507,7 +520,7 @@ class ServicesDBConfig: conf = self._db_conn.execute(sql, 1) return conf[0]['conf'] except Exception as e: - raise ConfigException("Malformed config for {0}: {1}".format(key, e)) + raise ConfigException("Error trying to get config for {0}: {1}".format(key, e)) @property def server_environment(self): @@ -557,14 +570,18 @@ class ServicesDBConfig: def mapzen_geocoder_monthly_quota(self): return self._mapzen_geocoder_quota - @property - def geocoder_log_path(self): - return self._geocoder_log_path - @property def data_observatory_connection_str(self): return self._data_observatory_connection_str + @property + def logger_config(self): + logger_conf_json = self._get_conf('logger_conf') + if not logger_conf_json: + raise ConfigException('Logger configuration missing') + else: + return json.loads(logger_conf_json) + class ServicesRedisConfig: diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/log.py b/server/lib/python/cartodb_services/cartodb_services/metrics/log.py index ab4739e..dd25ce1 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/log.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/log.py @@ -1,15 +1,130 @@ -from datetime import datetime import abc import json import re +import time +import uuid +import plpy +from datetime import datetime +from contextlib import contextmanager +from urlparse import urlparse -class MetricsLoggerFactory: +@contextmanager +def metrics(function, service_config, logger=None): + try: + start_time = time.time() + yield + finally: + end_time = time.time() + MetricsDataGatherer.add('uuid', str(uuid.uuid1())) + MetricsDataGatherer.add('function_name', function) + MetricsDataGatherer.add('function_execution_time', (end_time - start_time)) + metrics_logger = MetricsServiceLoggerFactory.build(service_config, + logger) + if metrics_logger: + data = MetricsDataGatherer.get() + metrics_logger.log(data) + MetricsDataGatherer.clean() + + +class Traceable: + """ + Module to add metrics traceability, for example to get response object + in order to add to the metrics dump + """ + def add_response_data(self, response, logger=None): + try: + response_data = {} + response_data['type'] = "request" + response_data['date'] = datetime.now().isoformat() + response_data['elapsed_time'] = response.elapsed.total_seconds() + response_data['code'] = response.status_code + response_data['message'] = response.reason + response_data['url'] = self._parse_response_url(response.url) + stored_data = MetricsDataGatherer.get_element('response') + if stored_data: + stored_data.append(response_data) + else: + MetricsDataGatherer.add('response', [response_data]) + except BaseException as e: + # We don't want to stop the job for some error processing response + if logger: + logger.error("Error trying to process response metricd data", + exception=e) + + def _parse_response_url(self, url): + u = urlparse(url) + return "{0}://{1}{2}".format(u.scheme, u.netloc, u.path) + + +class MetricsDataGatherer: + """ + Metrics gatherer used as a singleton. The intend is to use it as a global + storage for the metrics along the function request. + """ + + class __MetricsDataGatherer: + def __init__(self): + self.data = {} + + def add(self, key, value): + self.data[key] = value + + def get(self): + return self.data + + def get_element(self, key): + return self.data.get(key, None) + + def clean(self): + self.data = {} + + + # We use pgbouncer so we need to have multiples instances per request id + __instance = {} @classmethod - def build(self, service_config): - if re.match('geocoder_*', service_config.service_type): - return MetricsGeocoderLogger(service_config) + def add(self, key, value): + MetricsDataGatherer.instance().add(key, value) + + @classmethod + def get(self): + return MetricsDataGatherer.instance().get() + + @classmethod + def get_element(self, key): + return MetricsDataGatherer.instance().get_element(key) + + @classmethod + def clean(self): + MetricsDataGatherer.instance().clean() + + @classmethod + def instance(self): + txid = MetricsDataGatherer._get_txid() + if txid not in MetricsDataGatherer.__instance: + MetricsDataGatherer.__instance[txid] = MetricsDataGatherer.__MetricsDataGatherer() + + return MetricsDataGatherer.__instance[txid] + + @classmethod + def _get_txid(self): + result = plpy.execute('select txid_current() as txid') + return result[0]['txid'] + + +class MetricsServiceLoggerFactory: + + @classmethod + def build(self, service_config, logger=None): + if re.search('^geocoder_*', service_config.service_type): + return MetricsGeocoderLogger(service_config, logger) + elif re.search('^routing_*', service_config.service_type): + return MetricsGenericLogger(service_config, logger) + elif re.search('_isolines$', service_config.service_type): + return MetricsIsolinesLogger(service_config, logger) + elif re.search('^obs_*', service_config.service_type): + return MetricsGenericLogger(service_config, logger) else: return None @@ -17,58 +132,120 @@ class MetricsLoggerFactory: class MetricsLogger(object): __metaclass__ = abc.ABCMeta - def __init__(self, file_path): - self._file_path = file_path + def __init__(self, service_config, logger): + self._service_config = service_config + self._logger = logger def dump_to_file(self, data): - with open(self._file_path, 'a') as logfile: - json.dump(data, logfile) - logfile.write('\n') + try: + log_path = self.service_config.metrics_log_path + response_data = data.pop('response', []) + uuid = data.get('uuid') + if log_path: + with open(log_path, 'a') as logfile: + self._dump_response_to_file(uuid, response_data, logfile) + json.dump(data, logfile) + logfile.write('\n') + except BaseException as e: + self._logger("Error dumping metrics to file {0}".format(log_path), + exception=e) + + def collect_data(self, data): + return { + "uuid": data.get('uuid', uuid.uuid1()), + "type": 'function', + "function_name": data.get('function_name', None), + "function_execution_time": data.get('function_execution_time', + None), + "service": self._service_config.service_type, + "processable_rows": 1, + "success": data.get('success', False), + "successful_rows": data.get('successful_rows', 0), + "failed_rows": data.get('failed_rows', 0), + "empty_rows": data.get('empty_rows', 0), + "created_at": datetime.now().isoformat(), + "provider": self._service_config.provider, + "username": self._service_config.username, + "organization": self._service_config.organization, + "response": data.get('response', []) + } + + def _dump_response_to_file(self, uuid, response_data, log_file): + for r in response_data: + r['uuid'] = uuid + self._logger.info(r) + json.dump(r, log_file) + log_file.write('\n') + + @property + def service_config(self): + return self._service_config @abc.abstractproperty - def log(self, **data): + def log(self, data): raise NotImplementedError('log method must be defined') class MetricsGeocoderLogger(MetricsLogger): - def __init__(self, service_config): - super(MetricsGeocoderLogger, self).__init__(service_config.log_path) - self._service_config = service_config + def __init__(self, service_config, logger): + super(MetricsGeocoderLogger, self).__init__(service_config, logger) - def log(self, **data): - dump_data = self._dump_data(**data) + def log(self, data): + dump_data = self.collect_data(data) self.dump_to_file(dump_data) - def _dump_data(self, **data): - if data['success']: - cost = self._service_config.cost_per_hit - failed_rows = 0 - successful_rows = 1 + def collect_data(self, data): + dump_data = super(MetricsGeocoderLogger, self).collect_data(data) + if data.get('success', False): + cost = self.service_config.cost_per_hit else: cost = 0 - failed_rows = 1 - successful_rows = 0 - if self._service_config.is_high_resolution: + if self.service_config.is_high_resolution: kind = 'high-resolution' else: kind = 'internal' - return { + dump_data.update({ "batched": False, "cache_hits": 0, # Always 0 because no cache involved # https://github.com/CartoDB/cartodb/blob/master/app/models/geocoding.rb#L208-L211 "cost": cost, - "created_at": datetime.now().isoformat(), - "failed_rows": failed_rows, - "geocoder_type": self._service_config.service_type, + "geocoder_type": self.service_config.service_type, "kind": kind, - "processable_rows": 1, - "processed_rows": successful_rows, - "real_rows": successful_rows, - "success": data['success'], - "successful_rows": successful_rows, - "username": self._service_config.username, - "organization": self._service_config.organization - } + "processed_rows": data.get('successful_rows', 0), + "real_rows": data.get('successful_rows', 0), + }) + + return dump_data + + +class MetricsGenericLogger(MetricsLogger): + + def __init__(self, service_config, logger): + super(MetricsGenericLogger, self).__init__(service_config, logger) + + def log(self, data): + dump_data = self.collect_data(data) + self.dump_to_file(dump_data) + + def collect_data(self, data): + return super(MetricsGenericLogger, self).collect_data(data) + + +class MetricsIsolinesLogger(MetricsLogger): + + def __init__(self, service_config, logger): + super(MetricsIsolinesLogger, self).__init__(service_config, logger) + + def log(self, data): + dump_data = self.collect_data(data) + self.dump_to_file(dump_data) + + def collect_data(self, data): + dump_data = super(MetricsIsolinesLogger, self).collect_data(data) + dump_data.update({ + "isolines_generated": data.get('isolines_generated', 0) + }) + return dump_data diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py index ea54508..b372ae9 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py @@ -1,5 +1,5 @@ from user import UserMetricsService -from log import MetricsLoggerFactory +from log import MetricsDataGatherer from datetime import date import re @@ -14,7 +14,6 @@ class QuotaService: redis_connection) self._user_service = UserMetricsService(self._user_service_config, redis_connection) - self._metrics_logger = MetricsLoggerFactory.build(user_service_config) def check_user_quota(self): return self._quota_checker.check() @@ -46,13 +45,19 @@ class QuotaService: self._user_service.increment_service_use( self._user_service_config.service_type, "isolines_generated", amount=amount) + MetricsDataGatherer.add('isolines_generated', amount) def _log_service_process(self, event): - if self._metrics_logger: - if event is 'success' or event is 'empty': - self._metrics_logger.log(success=True) - elif event is 'empty': - self._metrics_logger.log(success=False) + if event is 'success': + MetricsDataGatherer.add('success', True) + MetricsDataGatherer.add('successful_rows', 1) + elif event is 'empty': + MetricsDataGatherer.add('success', True) + MetricsDataGatherer.add('successful_rows', 1) + MetricsDataGatherer.add('empty_rows', 1) + elif event is 'fail': + MetricsDataGatherer.add('success', False) + MetricsDataGatherer.add('failed_rows', 1) class QuotaChecker: diff --git a/server/lib/python/cartodb_services/setup.py b/server/lib/python/cartodb_services/setup.py index dc075bc..bcdef1d 100644 --- a/server/lib/python/cartodb_services/setup.py +++ b/server/lib/python/cartodb_services/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages setup( name='cartodb_services', - version='0.9.4', + version='0.10.0', description='CartoDB Services API Python Library', diff --git a/server/lib/python/cartodb_services/test/__init__.py b/server/lib/python/cartodb_services/test/__init__.py index e69de29..79a7695 100644 --- a/server/lib/python/cartodb_services/test/__init__.py +++ b/server/lib/python/cartodb_services/test/__init__.py @@ -0,0 +1,5 @@ +from test_helper import plpy_mock_config + + +def setup(): + plpy_mock_config() diff --git a/server/lib/python/cartodb_services/test/metrics/test_config.py b/server/lib/python/cartodb_services/test/metrics/test_config.py index a9142f0..8d0ceb5 100644 --- a/server/lib/python/cartodb_services/test/metrics/test_config.py +++ b/server/lib/python/cartodb_services/test/metrics/test_config.py @@ -1,13 +1,14 @@ from unittest import TestCase from mockredis import MockRedis +from ..test_helper import * from cartodb_services.metrics.config import RoutingConfig, ServicesRedisConfig -from ..test_helper import build_plpy_mock + class TestRoutingConfig(TestCase): def setUp(self): self._redis_conn = MockRedis() - self._db_conn = build_plpy_mock() + self._db_conn = plpy_mock self._username = 'my_test_user' self._user_key = "rails:users:{0}".format(self._username) self._redis_conn.hset(self._user_key, 'period_end_date', '2016-10-10') diff --git a/server/lib/python/cartodb_services/test/metrics/test_quota.py b/server/lib/python/cartodb_services/test/metrics/test_quota.py index c56756c..18f2105 100644 --- a/server/lib/python/cartodb_services/test/metrics/test_quota.py +++ b/server/lib/python/cartodb_services/test/metrics/test_quota.py @@ -1,6 +1,6 @@ from unittest import TestCase from mockredis import MockRedis -from ..test_helper import build_plpy_mock +from ..test_helper import * from cartodb_services.metrics.quota import QuotaChecker from cartodb_services.metrics import RoutingConfig from datetime import datetime diff --git a/server/lib/python/cartodb_services/test/mock_plpy.py b/server/lib/python/cartodb_services/test/mock_plpy.py new file mode 100644 index 0000000..5298a98 --- /dev/null +++ b/server/lib/python/cartodb_services/test/mock_plpy.py @@ -0,0 +1,53 @@ +import re + + +class MockCursor: + def __init__(self, data): + self.cursor_pos = 0 + self.data = data + + def fetch(self, batch_size): + batch = self.data[self.cursor_pos: self.cursor_pos + batch_size] + self.cursor_pos += batch_size + return batch + + +class MockPlPy: + def __init__(self): + self._reset() + + def _reset(self): + self.infos = [] + self.notices = [] + self.debugs = [] + self.logs = [] + self.warnings = [] + self.errors = [] + self.fatals = [] + self.executes = [] + self.results = [] + self.prepares = [] + self.results = {} + + def _define_result(self, query, result): + pattern = re.compile(query, re.IGNORECASE | re.MULTILINE) + self.results[pattern] = result + + def notice(self, msg): + self.notices.append(msg) + + def debug(self, msg): + self.notices.append(msg) + + def info(self, msg): + self.infos.append(msg) + + def cursor(self, query): + data = self.execute(query) + return MockCursor(data) + + def execute(self, query, rows=1): + for pattern, result in self.results.iteritems(): + if pattern.search(query): + return result + return [] diff --git a/server/lib/python/cartodb_services/test/test_config.py b/server/lib/python/cartodb_services/test/test_config.py index 4a639ec..b55b991 100644 --- a/server/lib/python/cartodb_services/test/test_config.py +++ b/server/lib/python/cartodb_services/test/test_config.py @@ -1,20 +1,20 @@ -import test_helper -from cartodb_services.metrics import GeocoderConfig, ObservatorySnapshotConfig, ConfigException +from test_helper import * from unittest import TestCase from nose.tools import assert_raises from mockredis import MockRedis from datetime import datetime, timedelta +from cartodb_services.metrics import GeocoderConfig, ObservatorySnapshotConfig, ConfigException class TestConfig(TestCase): def setUp(self): self.redis_conn = MockRedis() - self.plpy_mock = test_helper.build_plpy_mock() + plpy_mock_config() def test_should_return_list_of_nokia_geocoder_config_if_its_ok(self): - test_helper.build_redis_user_config(self.redis_conn, 'test_user') - geocoder_config = GeocoderConfig(self.redis_conn, self.plpy_mock, + build_redis_user_config(self.redis_conn, 'test_user') + geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock, 'test_user', None) assert geocoder_config.heremaps_geocoder is True assert geocoder_config.geocoding_quota == 100 @@ -22,10 +22,10 @@ class TestConfig(TestCase): def test_should_return_list_of_nokia_geocoder_config_ok_for_org(self): yesterday = datetime.today() - timedelta(days=1) - test_helper.build_redis_user_config(self.redis_conn, 'test_user') - test_helper.build_redis_org_config(self.redis_conn, 'test_org', - quota=200, end_date=yesterday) - geocoder_config = GeocoderConfig(self.redis_conn, self.plpy_mock, + build_redis_user_config(self.redis_conn, 'test_user') + build_redis_org_config(self.redis_conn, 'test_org', + quota=200, end_date=yesterday) + geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock, 'test_user', 'test_org') assert geocoder_config.heremaps_geocoder is True assert geocoder_config.geocoding_quota == 200 @@ -34,10 +34,10 @@ class TestConfig(TestCase): def test_should_return_config_for_obs_snapshot(self): yesterday = datetime.today() - timedelta(days=1) - test_helper.build_redis_user_config(self.redis_conn, 'test_user', - do_quota=100, soft_do_limit=True, - end_date=yesterday) - do_config = ObservatorySnapshotConfig(self.redis_conn, self.plpy_mock, + build_redis_user_config(self.redis_conn, 'test_user', + do_quota=100, soft_do_limit=True, + end_date=yesterday) + do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock, 'test_user') assert do_config.monthly_quota == 100 assert do_config.soft_limit is True @@ -45,16 +45,16 @@ class TestConfig(TestCase): def test_should_return_db_quota_if_not_redis_quota_config_obs_snapshot(self): yesterday = datetime.today() - timedelta(days=1) - test_helper.build_redis_user_config(self.redis_conn, 'test_user', - end_date=yesterday) - do_config = ObservatorySnapshotConfig(self.redis_conn, self.plpy_mock, + build_redis_user_config(self.redis_conn, 'test_user', + end_date=yesterday) + do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock, 'test_user') assert do_config.monthly_quota == 0 assert do_config.soft_limit is False assert do_config.period_end_date.date() == yesterday.date() def test_should_raise_exception_when_missing_parameters(self): - plpy_mock = test_helper.build_plpy_mock(empty=True) - test_helper.build_redis_user_config(self.redis_conn, 'test_user') + plpy_mock._reset() + build_redis_user_config(self.redis_conn, 'test_user') assert_raises(ConfigException, GeocoderConfig, self.redis_conn, plpy_mock, 'test_user', None) diff --git a/server/lib/python/cartodb_services/test/test_helper.py b/server/lib/python/cartodb_services/test/test_helper.py index d6e9d80..2272278 100644 --- a/server/lib/python/cartodb_services/test/test_helper.py +++ b/server/lib/python/cartodb_services/test/test_helper.py @@ -1,7 +1,11 @@ from datetime import datetime, date -from mock import Mock +from mock import Mock, MagicMock +import random import sys -sys.modules['plpy'] = Mock() +from mock_plpy import MockPlPy + +plpy_mock = MockPlPy() +sys.modules['plpy'] = plpy_mock def build_redis_user_config(redis_conn, username, quota=100, soft_limit=False, @@ -57,22 +61,10 @@ def increment_service_uses(redis_conn, username, orgname=None, redis_conn.zincrby(redis_name, date.day, amount) -def build_plpy_mock(empty=False): - plpy_mock = Mock() - if not empty: - plpy_mock.execute.side_effect = _plpy_execute_side_effect - - return plpy_mock - - -def _plpy_execute_side_effect(*args, **kwargs): - if args[0] == "SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as conf": - return [{'conf': '{"geocoder": {"app_id": "app_id", "app_code": "code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "app_id", "app_code": "code"}}'}] - elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('mapzen_conf') as conf": - return [{'conf': '{"routing": {"api_key": "api_key_rou", "monthly_quota": 1500000}, "geocoder": {"api_key": "api_key_geo", "monthly_quota": 1500000}, "matrix": {"api_key": "api_key_mat", "monthly_quota": 1500000}}'}] - elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('logger_conf') as conf": - return [{'conf': '{"geocoder_log_path": "/dev/null"}'}] - elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('data_observatory_conf') as conf": - return [{'conf': '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}}'}] - elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('server_conf') as conf": - return [{'conf': '{"environment": "testing"}'}] +def plpy_mock_config(): + plpy_mock._define_result("CDB_Conf_GetConf\('heremaps_conf'\)", [{'conf': '{"geocoder": {"app_id": "app_id", "app_code": "code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "app_id", "app_code": "code"}}'}]) + plpy_mock._define_result("CDB_Conf_GetConf\('mapzen_conf'\)", [{'conf': '{"routing": {"api_key": "api_key_rou", "monthly_quota": 1500000}, "geocoder": {"api_key": "api_key_geo", "monthly_quota": 1500000}, "matrix": {"api_key": "api_key_mat", "monthly_quota": 1500000}}'}]) + plpy_mock._define_result("CDB_Conf_GetConf\('logger_conf'\)", [{'conf': '{"geocoder_log_path": "/dev/null"}'}]) + plpy_mock._define_result("CDB_Conf_GetConf\('data_observatory_conf'\)", [{'conf': '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}}'}]) + plpy_mock._define_result("CDB_Conf_GetConf\('server_conf'\)", [{'conf': '{"environment": "testing"}'}]) + plpy_mock._define_result("select txid_current", [{'txid': random.randint(0, 1000)}]) diff --git a/server/lib/python/cartodb_services/test/test_metrics_log.py b/server/lib/python/cartodb_services/test/test_metrics_log.py new file mode 100644 index 0000000..963a34a --- /dev/null +++ b/server/lib/python/cartodb_services/test/test_metrics_log.py @@ -0,0 +1,23 @@ +from test_helper import * +from cartodb_services.metrics import MetricsDataGatherer +from unittest import TestCase +from mock import Mock, MagicMock +from nose.tools import assert_raises +from datetime import datetime, date + + +class TestMetricsDataGatherer(TestCase): + + def setUp(self): + plpy_mock_config() + + + def test_should_use_multiple_instances_for_multiples_requests(self): + plpy_mock._define_result("select txid_current", [{'txid': 100}]) + MetricsDataGatherer.add('test', 1) + plpy_mock._define_result("select txid_current", [{'txid': 101}]) + MetricsDataGatherer.add('test', 2) + plpy_mock._define_result("select txid_current", [{'txid': 100}]) + assert MetricsDataGatherer.get_element('test') is 1 + plpy_mock._define_result("select txid_current", [{'txid': 101}]) + assert MetricsDataGatherer.get_element('test') is 2 diff --git a/server/lib/python/cartodb_services/test/test_qps.py b/server/lib/python/cartodb_services/test/test_qps.py index 014c510..bf396ca 100644 --- a/server/lib/python/cartodb_services/test/test_qps.py +++ b/server/lib/python/cartodb_services/test/test_qps.py @@ -1,4 +1,3 @@ -import test_helper import requests from unittest import TestCase from nose.tools import assert_raises diff --git a/server/lib/python/cartodb_services/test/test_quota_service.py b/server/lib/python/cartodb_services/test/test_quota_service.py index 7cc49dd..c104235 100644 --- a/server/lib/python/cartodb_services/test/test_quota_service.py +++ b/server/lib/python/cartodb_services/test/test_quota_service.py @@ -1,4 +1,4 @@ -import test_helper +from test_helper import * from mockredis import MockRedis from cartodb_services.metrics import QuotaService from cartodb_services.metrics import GeocoderConfig, RoutingConfig, ObservatorySnapshotConfig, IsolinesRoutingConfig @@ -28,32 +28,32 @@ class TestQuotaService(TestCase): def test_should_return_true_if_user_quota_is_not_completely_used(self): qs = self.__build_geocoder_quota_service('test_user') - test_helper.increment_service_uses(self.redis_conn, 'test_user') + increment_service_uses(self.redis_conn, 'test_user') assert qs.check_user_quota() is True def test_should_return_true_if_org_quota_is_not_completely_used(self): qs = self.__build_geocoder_quota_service('test_user', orgname='test_org') - test_helper.increment_service_uses(self.redis_conn, 'test_user', + increment_service_uses(self.redis_conn, 'test_user', orgname='test_org') assert qs.check_user_quota() is True def test_should_return_false_if_user_quota_is_surpassed(self): qs = self.__build_geocoder_quota_service('test_user') - test_helper.increment_service_uses(self.redis_conn, 'test_user', + increment_service_uses(self.redis_conn, 'test_user', amount=300) assert qs.check_user_quota() is False def test_should_return_false_if_org_quota_is_surpassed(self): qs = self.__build_geocoder_quota_service('test_user', orgname='test_org') - test_helper.increment_service_uses(self.redis_conn, 'test_user', + increment_service_uses(self.redis_conn, 'test_user', orgname='test_org', amount=400) assert qs.check_user_quota() is False def test_should_return_true_if_user_quota_is_surpassed_but_soft_limit_is_enabled(self): qs = self.__build_geocoder_quota_service('test_user', soft_limit=True) - test_helper.increment_service_uses(self.redis_conn, 'test_user', + increment_service_uses(self.redis_conn, 'test_user', amount=300) assert qs.check_user_quota() is True @@ -61,7 +61,7 @@ class TestQuotaService(TestCase): qs = self.__build_geocoder_quota_service('test_user', orgname='test_org', soft_limit=True) - test_helper.increment_service_uses(self.redis_conn, 'test_user', + increment_service_uses(self.redis_conn, 'test_user', orgname='test_org', amount=400) assert qs.check_user_quota() is True @@ -140,18 +140,17 @@ class TestQuotaService(TestCase): def __prepare_quota_service(self, username, quota, service, orgname, soft_limit, do_quota, soft_do_limit, end_date): - test_helper.build_redis_user_config(self.redis_conn, username, + build_redis_user_config(self.redis_conn, username, quota=quota, service=service, soft_limit=soft_limit, soft_do_limit=soft_do_limit, do_quota=do_quota, end_date=end_date) if orgname: - test_helper.build_redis_org_config(self.redis_conn, orgname, + build_redis_org_config(self.redis_conn, orgname, quota=quota, service=service, do_quota=do_quota, end_date=end_date) - self._plpy_mock = test_helper.build_plpy_mock() def __build_geocoder_quota_service(self, username, quota=100, service='heremaps', orgname=None, @@ -159,7 +158,7 @@ class TestQuotaService(TestCase): end_date=datetime.today()): self.__prepare_quota_service(username, quota, service, orgname, soft_limit, 0, False, end_date) - geocoder_config = GeocoderConfig(self.redis_conn, self._plpy_mock, + geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock, username, orgname) return QuotaService(geocoder_config, redis_connection=self.redis_conn) @@ -168,7 +167,7 @@ class TestQuotaService(TestCase): quota=100, end_date=datetime.today()): self.__prepare_quota_service(username, quota, service, orgname, soft_limit, 0, False, end_date) - routing_config = RoutingConfig(self.redis_conn, self._plpy_mock, + routing_config = RoutingConfig(self.redis_conn, plpy_mock, username, orgname) return QuotaService(routing_config, redis_connection=self.redis_conn) @@ -177,7 +176,7 @@ class TestQuotaService(TestCase): quota=100, end_date=datetime.today()): self.__prepare_quota_service(username, quota, service, orgname, soft_limit, 0, False, end_date) - isolines_config = IsolinesRoutingConfig(self.redis_conn, self._plpy_mock, + isolines_config = IsolinesRoutingConfig(self.redis_conn, plpy_mock, username, orgname) return QuotaService(isolines_config, redis_connection=self.redis_conn) @@ -188,6 +187,6 @@ class TestQuotaService(TestCase): end_date=datetime.today()): self.__prepare_quota_service(username, 0, service, orgname, False, quota, soft_limit, end_date) - do_config = ObservatorySnapshotConfig(self.redis_conn, self._plpy_mock, + do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock, username, orgname) return QuotaService(do_config, redis_connection=self.redis_conn) diff --git a/server/lib/python/cartodb_services/test/test_user_service.py b/server/lib/python/cartodb_services/test/test_user_service.py index a73e00b..2905f8b 100644 --- a/server/lib/python/cartodb_services/test/test_user_service.py +++ b/server/lib/python/cartodb_services/test/test_user_service.py @@ -1,4 +1,4 @@ -import test_helper +from test_helper import * from mockredis import MockRedis from cartodb_services.metrics import UserMetricsService from cartodb_services.metrics import GeocoderConfig @@ -19,13 +19,13 @@ class TestUserService(TestCase): def test_user_used_quota_for_a_day(self): us = self.__build_user_service('test_user') - test_helper.increment_service_uses(self.redis_conn, 'test_user', + increment_service_uses(self.redis_conn, 'test_user', amount=400) assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 400 def test_org_used_quota_for_a_day(self): us = self.__build_user_service('test_user', orgname='test_org') - test_helper.increment_service_uses(self.redis_conn, 'test_user', + increment_service_uses(self.redis_conn, 'test_user', orgname='test_org', amount=400) assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 400 @@ -135,14 +135,13 @@ class TestUserService(TestCase): def __build_user_service(self, username, quota=100, service='heremaps', orgname=None, soft_limit=False, end_date=date.today()): - test_helper.build_redis_user_config(self.redis_conn, username, + build_redis_user_config(self.redis_conn, username, quota=quota, service=service, soft_limit=soft_limit, end_date=end_date) if orgname: - test_helper.build_redis_org_config(self.redis_conn, orgname, + build_redis_org_config(self.redis_conn, orgname, quota=quota, end_date=end_date) - plpy_mock = test_helper.build_plpy_mock() geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock, username, orgname) return UserMetricsService(geocoder_config, self.redis_conn) diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index 263aaa6..7edda67 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -14,8 +14,8 @@ class TestStreetFunctions(TestCase): self.env_variables['api_key'] ) - def test_if_select_with_street_point_is_ok(self): - query = "SELECT cdb_geocode_street_point(street) " \ + def test_if_select_with_here_street_point_is_ok(self): + query = "SELECT cdb_here_geocode_street_point(street) " \ "as geometry FROM {0} LIMIT 1&api_key={1}".format( self.env_variables['table_name'], self.env_variables['api_key'])