cartodb4/dataservices-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #301 from CartoDB/development

Browse Source 
Release 0.17.0
This commit is contained in:
Mario de Frutos 2016-11-07 10:48:15 +01:00 committed by GitHub
commit 2c49f09aad
50 changed files with 6156 additions and 732 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1341
server/extension/cdb_dataservices_server--0.16.0--0.17.0.sql Normal file
View File

File diff suppressed because it is too large Load Diff

1208
server/extension/cdb_dataservices_server--0.17.0--0.16.0.sql Normal file
View File

File diff suppressed because it is too large Load Diff

2491
server/extension/cdb_dataservices_server--0.17.0.sql Normal file
View File

File diff suppressed because it is too large Load Diff

2
server/extension/cdb_dataservices_server.control
View File

@ -1,5 +1,5 @@
comment = 'CartoDB dataservices server extension' comment = 'CartoDB dataservices server extension'
default_version = '0.16.0' default_version = '0.17.0'
requires = 'plpythonu, plproxy, postgis, cdb_geocoder' requires = 'plpythonu, plproxy, postgis, cdb_geocoder'
superuser = true superuser = true
schema = cdb_dataservices_server schema = cdb_dataservices_server

0
server/extension/cdb_dataservices_server--0.15.1--0.16.0.sql → server/extension/old_versions/cdb_dataservices_server--0.15.1--0.16.0.sql
View File

0
server/extension/cdb_dataservices_server--0.16.0--0.15.1.sql → server/extension/old_versions/cdb_dataservices_server--0.16.0--0.15.1.sql
View File

0
server/extension/cdb_dataservices_server--0.16.0.sql → server/extension/old_versions/cdb_dataservices_server--0.16.0.sql
View File

26
server/extension/sql/105_route_between_points.sql
View File

@ -7,15 +7,21 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point(
options text[] DEFAULT ARRAY[]::text[], options text[] DEFAULT ARRAY[]::text[],
units text DEFAULT 'kilometers') units text DEFAULT 'kilometers')
RETURNS cdb_dataservices_server.simple_route AS $$ RETURNS cdb_dataservices_server.simple_route AS $$
from cartodb_services.metrics import metrics
from cartodb_services.tools import Logger
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_routing_config = GD["user_routing_config_{0}".format(username)] user_routing_config = GD["user_routing_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
waypoints = [origin, destination] with metrics('cdb_route_with_point', user_routing_config, logger):
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) waypoints = [origin, destination]
result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units]) mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"])
return [result[0]['shape'],result[0]['length'], result[0]['duration']] result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units])
return [result[0]['shape'],result[0]['length'], result[0]['duration']]
$$ LANGUAGE plpythonu; $$ LANGUAGE plpythonu;
@ -27,12 +33,18 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints(
options text[] DEFAULT ARRAY[]::text[], options text[] DEFAULT ARRAY[]::text[],
units text DEFAULT 'kilometers') units text DEFAULT 'kilometers')
RETURNS cdb_dataservices_server.simple_route AS $$ RETURNS cdb_dataservices_server.simple_route AS $$
from cartodb_services.metrics import metrics
from cartodb_services.tools import Logger
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_routing_config = GD["user_routing_config_{0}".format(username)] user_routing_config = GD["user_routing_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) with metrics('cdb_route_with_waypoints', user_routing_config, logger):
result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units]) mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"])
return [result[0]['shape'],result[0]['length'], result[0]['duration']] result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units])
return [result[0]['shape'],result[0]['length'], result[0]['duration']]
$$ LANGUAGE plpythonu; $$ LANGUAGE plpythonu;

376
server/extension/sql/110_data_observatory_augmentation.sql
View File

@ -11,9 +11,9 @@ RETURNS text AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)]
return user_obs_snapshot_config.connection_str return user_obs_config.connection_str
$$ LANGUAGE plpythonu; $$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshotJSON( CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshotJSON(
@ -34,6 +34,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot(
time_span TEXT DEFAULT NULL, time_span TEXT DEFAULT NULL,
geometry_level TEXT DEFAULT NULL) geometry_level TEXT DEFAULT NULL)
RETURNS json AS $$ RETURNS json AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig from cartodb_services.tools import Logger,LoggerConfig
import json import json
@ -41,31 +42,32 @@ RETURNS json AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"] logger_config = GD["logger_config"]
logger = Logger(logger_config) logger = Logger(logger_config)
quota_service = QuotaService(user_obs_snapshot_config, redis_conn) quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota(): if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota') raise Exception('You have reached the limit of your quota')
try: with metrics('obs_getdemographicsnapshot', user_obs_config, logger):
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshotJSON($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) try:
result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshotJSON($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"])
if result: result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level])
quota_service.increment_success_service_use() if result:
return result[0]['snapshot'] quota_service.increment_success_service_use()
else: return result[0]['snapshot']
quota_service.increment_empty_service_use() else:
return None quota_service.increment_empty_service_use()
except BaseException as e: return None
import sys except BaseException as e:
quota_service.increment_failed_service_use() import sys
logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) quota_service.increment_failed_service_use()
raise Exception('Error trying to obs_get_demographic_snapshot') logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname})
finally: raise Exception('Error trying to obs_get_demographic_snapshot')
quota_service.increment_total_service_use() finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu; $$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshot( CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshot(
@ -86,41 +88,43 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetDemographicSnapshot(
time_span TEXT DEFAULT NULL, time_span TEXT DEFAULT NULL,
geometry_level TEXT DEFAULT NULL) geometry_level TEXT DEFAULT NULL)
RETURNS SETOF JSON AS $$ RETURNS SETOF JSON AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"] logger_config = GD["logger_config"]
logger = Logger(logger_config) logger = Logger(logger_config)
quota_service = QuotaService(user_obs_snapshot_config, redis_conn) quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota(): if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota') raise Exception('You have reached the limit of your quota')
try: with metrics('obs_getdemographicsnapshot', user_obs_config, logger):
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) try:
result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"])
if result: result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level])
resp = [] if result:
for element in result: resp = []
value = element['snapshot'] for element in result:
resp.append(value) value = element['snapshot']
quota_service.increment_success_service_use() resp.append(value)
return resp quota_service.increment_success_service_use()
else: return resp
quota_service.increment_empty_service_use() else:
return [] quota_service.increment_empty_service_use()
except BaseException as e: return []
import sys except BaseException as e:
quota_service.increment_failed_service_use() import sys
logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) quota_service.increment_failed_service_use()
raise Exception('Error trying to obs_get_demographic_snapshot') logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname})
finally: raise Exception('Error trying to obs_get_demographic_snapshot')
quota_service.increment_total_service_use() finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu; $$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshotJSON( CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshotJSON(
@ -139,6 +143,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot(
geom geometry(Geometry, 4326), geom geometry(Geometry, 4326),
geometry_level TEXT DEFAULT NULL) geometry_level TEXT DEFAULT NULL)
RETURNS json AS $$ RETURNS json AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig from cartodb_services.tools import Logger,LoggerConfig
import json import json
@ -146,31 +151,32 @@ RETURNS json AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"] logger_config = GD["logger_config"]
logger = Logger(logger_config) logger = Logger(logger_config)
quota_service = QuotaService(user_obs_snapshot_config, redis_conn) quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota(): if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota') raise Exception('You have reached the limit of your quota')
try: with metrics('obs_getsegmentsnapshot', user_obs_config, logger):
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshotJSON($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) try:
result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshotJSON($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"])
if result: result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level])
quota_service.increment_success_service_use() if result:
return result[0]['snapshot'] quota_service.increment_success_service_use()
else: return result[0]['snapshot']
quota_service.increment_empty_service_use() else:
return None quota_service.increment_empty_service_use()
except BaseException as e: return None
import sys except BaseException as e:
quota_service.increment_failed_service_use() import sys
logger.error('Error trying to obs_get_segment_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) quota_service.increment_failed_service_use()
raise Exception('Error trying to obs_get_segment_snapshot') logger.error('Error trying to obs_get_segment_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname})
finally: raise Exception('Error trying to obs_get_segment_snapshot')
quota_service.increment_total_service_use() finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu; $$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshot( CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshot(
@ -189,41 +195,43 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetSegmentSnapshot(
geom geometry(Geometry, 4326), geom geometry(Geometry, 4326),
geometry_level TEXT DEFAULT NULL) geometry_level TEXT DEFAULT NULL)
RETURNS SETOF JSON AS $$ RETURNS SETOF JSON AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] user_obs_config = GD["user_obs_snapshot_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"] logger_config = GD["logger_config"]
logger = Logger(logger_config) logger = Logger(logger_config)
quota_service = QuotaService(user_obs_snapshot_config, redis_conn) quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota(): if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota') raise Exception('You have reached the limit of your quota')
try: with metrics('obs_getsegmentsnapshot', user_obs_config, logger):
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) try:
result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"])
if result: result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level])
resp = [] if result:
for element in result: resp = []
value = element['snapshot'] for element in result:
resp.append(value) value = element['snapshot']
quota_service.increment_success_service_use() resp.append(value)
return resp quota_service.increment_success_service_use()
else: return resp
quota_service.increment_empty_service_use() else:
return [] quota_service.increment_empty_service_use()
except BaseException as e: return []
import sys except BaseException as e:
quota_service.increment_failed_service_use() import sys
logger.error('Error trying to OBS_GetSegmentSnapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) quota_service.increment_failed_service_use()
raise Exception('Error trying to OBS_GetSegmentSnapshot') logger.error('Error trying to OBS_GetSegmentSnapshot', sys.exc_info(), data={"username": username, "orgname": orgname})
finally: raise Exception('Error trying to OBS_GetSegmentSnapshot')
quota_service.increment_total_service_use() finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu; $$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasure( CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasure(
@ -248,6 +256,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasure(
boundary_id TEXT DEFAULT NULL, boundary_id TEXT DEFAULT NULL,
time_span TEXT DEFAULT NULL) time_span TEXT DEFAULT NULL)
RETURNS NUMERIC AS $$ RETURNS NUMERIC AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig from cartodb_services.tools import Logger,LoggerConfig
@ -263,22 +272,23 @@ RETURNS NUMERIC AS $$
if not quota_service.check_user_quota(): if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota') raise Exception('You have reached the limit of your quota')
try: with metrics('obs_getmeasure', user_obs_config, logger):
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasure($1, $2, $3, $4, $5, $6, $7) as measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) try:
result = plpy.execute(obs_plan, [username, orgname, geom, measure_id, normalize, boundary_id, time_span]) obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasure($1, $2, $3, $4, $5, $6, $7) as measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"])
if result: result = plpy.execute(obs_plan, [username, orgname, geom, measure_id, normalize, boundary_id, time_span])
quota_service.increment_success_service_use() if result:
return result[0]['measure'] quota_service.increment_success_service_use()
else: return result[0]['measure']
quota_service.increment_empty_service_use() else:
return None quota_service.increment_empty_service_use()
except BaseException as e: return None
import sys except BaseException as e:
quota_service.increment_failed_service_use() import sys
logger.error('Error trying to OBS_GetMeasure', sys.exc_info(), data={"username": username, "orgname": orgname}) quota_service.increment_failed_service_use()
raise Exception('Error trying to OBS_GetMeasure') logger.error('Error trying to OBS_GetMeasure', sys.exc_info(), data={"username": username, "orgname": orgname})
finally: raise Exception('Error trying to OBS_GetMeasure')
quota_service.increment_total_service_use() finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu; $$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetCategory( CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetCategory(
@ -301,6 +311,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetCategory(
boundary_id TEXT DEFAULT NULL, boundary_id TEXT DEFAULT NULL,
time_span TEXT DEFAULT NULL) time_span TEXT DEFAULT NULL)
RETURNS TEXT AS $$ RETURNS TEXT AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig from cartodb_services.tools import Logger,LoggerConfig
@ -316,22 +327,23 @@ RETURNS TEXT AS $$
if not quota_service.check_user_quota(): if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota') raise Exception('You have reached the limit of your quota')
try: with metrics('obs_getcategory', user_obs_config, logger):
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetCategory($1, $2, $3, $4, $5, $6) as category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) try:
result = plpy.execute(obs_plan, [username, orgname, geom, category_id, boundary_id, time_span]) obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetCategory($1, $2, $3, $4, $5, $6) as category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"])
if result: result = plpy.execute(obs_plan, [username, orgname, geom, category_id, boundary_id, time_span])
quota_service.increment_success_service_use() if result:
return result[0]['category'] quota_service.increment_success_service_use()
else: return result[0]['category']
quota_service.increment_empty_service_use() else:
return None quota_service.increment_empty_service_use()
except BaseException as e: return None
import sys except BaseException as e:
quota_service.increment_failed_service_use() import sys
logger.error('Error trying to OBS_GetCategory', sys.exc_info(), data={"username": username, "orgname": orgname}) quota_service.increment_failed_service_use()
raise Exception('Error trying to OBS_GetCategory') logger.error('Error trying to OBS_GetCategory', sys.exc_info(), data={"username": username, "orgname": orgname})
finally: raise Exception('Error trying to OBS_GetCategory')
quota_service.increment_total_service_use() finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu; $$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusMeasure( CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusMeasure(
@ -356,6 +368,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusMeasure(
boundary_id TEXT DEFAULT NULL, boundary_id TEXT DEFAULT NULL,
time_span TEXT DEFAULT NULL) time_span TEXT DEFAULT NULL)
RETURNS NUMERIC AS $$ RETURNS NUMERIC AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig from cartodb_services.tools import Logger,LoggerConfig
@ -371,22 +384,23 @@ RETURNS NUMERIC AS $$
if not quota_service.check_user_quota(): if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota') raise Exception('You have reached the limit of your quota')
try: with metrics('obs_getuscensusmeasure', user_obs_config, logger):
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusMeasure($1, $2, $3, $4, $5, $6, $7) as census_measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) try:
result = plpy.execute(obs_plan, [username, orgname, geom, name, normalize, boundary_id, time_span]) obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusMeasure($1, $2, $3, $4, $5, $6, $7) as census_measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"])
if result: result = plpy.execute(obs_plan, [username, orgname, geom, name, normalize, boundary_id, time_span])
quota_service.increment_success_service_use() if result:
return result[0]['census_measure'] quota_service.increment_success_service_use()
else: return result[0]['census_measure']
quota_service.increment_empty_service_use() else:
return None quota_service.increment_empty_service_use()
except BaseException as e: return None
import sys except BaseException as e:
quota_service.increment_failed_service_use() import sys
logger.error('Error trying to OBS_GetUSCensusMeasure', sys.exc_info(), data={"username": username, "orgname": orgname}) quota_service.increment_failed_service_use()
raise Exception('Error trying to OBS_GetUSCensusMeasure') logger.error('Error trying to OBS_GetUSCensusMeasure', sys.exc_info(), data={"username": username, "orgname": orgname})
finally: raise Exception('Error trying to OBS_GetUSCensusMeasure')
quota_service.increment_total_service_use() finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu; $$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusCategory( CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusCategory(
@ -409,6 +423,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusCategory(
boundary_id TEXT DEFAULT NULL, boundary_id TEXT DEFAULT NULL,
time_span TEXT DEFAULT NULL) time_span TEXT DEFAULT NULL)
RETURNS TEXT AS $$ RETURNS TEXT AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig from cartodb_services.tools import Logger,LoggerConfig
@ -424,22 +439,23 @@ RETURNS TEXT AS $$
if not quota_service.check_user_quota(): if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota') raise Exception('You have reached the limit of your quota')
try: with metrics('obs_getuscensuscategory', user_obs_config, logger):
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusCategory($1, $2, $3, $4, $5, $6) as census_category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) try:
result = plpy.execute(obs_plan, [username, orgname, geom, name, boundary_id, time_span]) obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusCategory($1, $2, $3, $4, $5, $6) as census_category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"])
if result: result = plpy.execute(obs_plan, [username, orgname, geom, name, boundary_id, time_span])
quota_service.increment_success_service_use() if result:
return result[0]['census_category'] quota_service.increment_success_service_use()
else: return result[0]['census_category']
quota_service.increment_empty_service_use() else:
return None quota_service.increment_empty_service_use()
except BaseException as e: return None
import sys except BaseException as e:
quota_service.increment_failed_service_use() import sys
logger.error('Error trying to OBS_GetUSCensusCategory', sys.exc_info(), data={"username": username, "orgname": orgname}) quota_service.increment_failed_service_use()
raise Exception('Error trying to OBS_GetUSCensusCategory') logger.error('Error trying to OBS_GetUSCensusCategory', sys.exc_info(), data={"username": username, "orgname": orgname})
finally: raise Exception('Error trying to OBS_GetUSCensusCategory')
quota_service.increment_total_service_use() finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu; $$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPopulation( CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPopulation(
@ -462,6 +478,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPopulation(
boundary_id TEXT DEFAULT NULL, boundary_id TEXT DEFAULT NULL,
time_span TEXT DEFAULT NULL) time_span TEXT DEFAULT NULL)
RETURNS NUMERIC AS $$ RETURNS NUMERIC AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig from cartodb_services.tools import Logger,LoggerConfig
@ -477,22 +494,23 @@ RETURNS NUMERIC AS $$
if not quota_service.check_user_quota(): if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota') raise Exception('You have reached the limit of your quota')
try: with metrics('obs_getpopulation', user_obs_config, logger):
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetPopulation($1, $2, $3, $4, $5, $6) as population;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) try:
result = plpy.execute(obs_plan, [username, orgname, geom, normalize, boundary_id, time_span]) obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetPopulation($1, $2, $3, $4, $5, $6) as population;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"])
if result: result = plpy.execute(obs_plan, [username, orgname, geom, normalize, boundary_id, time_span])
quota_service.increment_success_service_use() if result:
return result[0]['population'] quota_service.increment_success_service_use()
else: return result[0]['population']
quota_service.increment_empty_service_use() else:
return None quota_service.increment_empty_service_use()
except BaseException as e: return None
import sys except BaseException as e:
quota_service.increment_failed_service_use() import sys
logger.error('Error trying to OBS_GetPopulation', sys.exc_info(), data={"username": username, "orgname": orgname}) quota_service.increment_failed_service_use()
raise Exception('Error trying to OBS_GetPopulation') logger.error('Error trying to OBS_GetPopulation', sys.exc_info(), data={"username": username, "orgname": orgname})
finally: raise Exception('Error trying to OBS_GetPopulation')
quota_service.increment_total_service_use() finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu; $$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasureById( CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasureById(
@ -515,6 +533,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasureById(
boundary_id TEXT, boundary_id TEXT,
time_span TEXT DEFAULT NULL) time_span TEXT DEFAULT NULL)
RETURNS NUMERIC AS $$ RETURNS NUMERIC AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig from cartodb_services.tools import Logger,LoggerConfig
@ -530,20 +549,21 @@ RETURNS NUMERIC AS $$
if not quota_service.check_user_quota(): if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota') raise Exception('You have reached the limit of your quota')
try: with metrics('obs_getmeasurebyid', user_obs_config, logger):
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasureById($1, $2, $3, $4, $5, $6) as measure;", ["text", "text", "text", "text", "text", "text"]) try:
result = plpy.execute(obs_plan, [username, orgname, geom_ref, measure_id, boundary_id, time_span]) obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasureById($1, $2, $3, $4, $5, $6) as measure;", ["text", "text", "text", "text", "text", "text"])
if result: result = plpy.execute(obs_plan, [username, orgname, geom_ref, measure_id, boundary_id, time_span])
quota_service.increment_success_service_use() if result:
return result[0]['measure'] quota_service.increment_success_service_use()
else: return result[0]['measure']
quota_service.increment_empty_service_use() else:
return None quota_service.increment_empty_service_use()
except BaseException as e: return None
import sys except BaseException as e:
quota_service.increment_failed_service_use() import sys
logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname}) quota_service.increment_failed_service_use()
raise Exception('Error trying to OBS_GetMeasureById') logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname})
finally: raise Exception('Error trying to OBS_GetMeasureById')
quota_service.increment_total_service_use() finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu; $$ LANGUAGE plpythonu;

98
server/extension/sql/115_data_observatory_exploration.sql
View File

@ -14,6 +14,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_Search(
search_term TEXT, search_term TEXT,
relevant_boundary TEXT DEFAULT NULL) relevant_boundary TEXT DEFAULT NULL)
RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig from cartodb_services.tools import Logger,LoggerConfig
@ -29,30 +30,31 @@ RETURNS TABLE(id text, description text, name text, aggregate text, source text)
if not quota_service.check_user_quota(): if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota') raise Exception('You have reached the limit of your quota')
try: with metrics('obs_search', user_obs_snapshot_config, logger):
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_Search($1, $2, $3, $4);", ["text", "text", "text", "text"]) try:
result = plpy.execute(obs_plan, [username, orgname, search_term, relevant_boundary]) obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_Search($1, $2, $3, $4);", ["text", "text", "text", "text"])
if result: result = plpy.execute(obs_plan, [username, orgname, search_term, relevant_boundary])
resp = [] if result:
for element in result: resp = []
id = element['id'] for element in result:
description = element['description'] id = element['id']
name = element['name'] description = element['description']
aggregate = element['aggregate'] name = element['name']
source = element['source'] aggregate = element['aggregate']
resp.append([id, description, name, aggregate, source]) source = element['source']
quota_service.increment_success_service_use() resp.append([id, description, name, aggregate, source])
return resp quota_service.increment_success_service_use()
else: return resp
quota_service.increment_empty_service_use() else:
return [None, None, None, None, None] quota_service.increment_empty_service_use()
except BaseException as e: return [None, None, None, None, None]
import sys except BaseException as e:
quota_service.increment_failed_service_use() import sys
logger.error('Error trying to OBS_Search', sys.exc_info(), data={"username": username, "orgname": orgname}) quota_service.increment_failed_service_use()
raise Exception('Error trying to OBS_Search') logger.error('Error trying to OBS_Search', sys.exc_info(), data={"username": username, "orgname": orgname})
finally: raise Exception('Error trying to OBS_Search')
quota_service.increment_total_service_use() finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu; $$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetAvailableBoundaries( CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetAvailableBoundaries(
@ -71,6 +73,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableBoundaries(
geom geometry(Geometry, 4326), geom geometry(Geometry, 4326),
time_span TEXT DEFAULT NULL) time_span TEXT DEFAULT NULL)
RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig from cartodb_services.tools import Logger,LoggerConfig
@ -86,27 +89,28 @@ RETURNS TABLE(boundary_id text, description text, time_span text, tablename text
if not quota_service.check_user_quota(): if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota') raise Exception('You have reached the limit of your quota')
try: with metrics('obs_getavailableboundaries', user_obs_snapshot_config, logger):
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetAvailableBoundaries($1, $2, $3, $4) as available_boundaries;", ["text", "text", "geometry(Geometry, 4326)", "text"]) try:
result = plpy.execute(obs_plan, [username, orgname, geom, time_span]) obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetAvailableBoundaries($1, $2, $3, $4) as available_boundaries;", ["text", "text", "geometry(Geometry, 4326)", "text"])
if result: result = plpy.execute(obs_plan, [username, orgname, geom, time_span])
resp = [] if result:
for element in result: resp = []
id = element['boundary_id'] for element in result:
description = element['description'] id = element['boundary_id']
tspan = element['time_span'] description = element['description']
tablename = element['tablename'] tspan = element['time_span']
resp.append([id, description, tspan, tablename]) tablename = element['tablename']
quota_service.increment_success_service_use() resp.append([id, description, tspan, tablename])
return resp quota_service.increment_success_service_use()
else: return resp
quota_service.increment_empty_service_use() else:
return [] quota_service.increment_empty_service_use()
except BaseException as e: return []
import sys except BaseException as e:
quota_service.increment_failed_service_use() import sys
logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname}) quota_service.increment_failed_service_use()
raise Exception('Error trying to OBS_GetMeasureById') logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname})
finally: raise Exception('Error trying to OBS_GetMeasureById')
quota_service.increment_total_service_use() finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu; $$ LANGUAGE plpythonu;

278
server/extension/sql/120_data_observatory_geometries.sql
View File

@ -16,6 +16,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundary(
boundary_id TEXT, boundary_id TEXT,
time_span TEXT DEFAULT NULL) time_span TEXT DEFAULT NULL)
RETURNS geometry(Geometry, 4326) AS $$ RETURNS geometry(Geometry, 4326) AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig from cartodb_services.tools import Logger,LoggerConfig
@ -31,22 +32,23 @@ RETURNS geometry(Geometry, 4326) AS $$
if not quota_service.check_user_quota(): if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota') raise Exception('You have reached the limit of your quota')
try: with metrics('obs_getboundary', user_obs_snapshot_config, logger):
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundary($1, $2, $3, $4) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) try:
result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundary($1, $2, $3, $4) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"])
if result: result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span])
quota_service.increment_success_service_use() if result:
return result[0]['boundary'] quota_service.increment_success_service_use()
else: return result[0]['boundary']
quota_service.increment_empty_service_use() else:
return None quota_service.increment_empty_service_use()
except BaseException as e: return None
import sys except BaseException as e:
quota_service.increment_failed_service_use() import sys
logger.error('Error trying to OBS_GetBoundary', sys.exc_info(), data={"username": username, "orgname": orgname}) quota_service.increment_failed_service_use()
raise Exception('Error trying to OBS_GetBoundary') logger.error('Error trying to OBS_GetBoundary', sys.exc_info(), data={"username": username, "orgname": orgname})
finally: raise Exception('Error trying to OBS_GetBoundary')
quota_service.increment_total_service_use() finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu; $$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryId( CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryId(
@ -67,6 +69,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryId(
boundary_id TEXT, boundary_id TEXT,
time_span TEXT DEFAULT NULL) time_span TEXT DEFAULT NULL)
RETURNS TEXT AS $$ RETURNS TEXT AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig from cartodb_services.tools import Logger,LoggerConfig
@ -82,22 +85,23 @@ RETURNS TEXT AS $$
if not quota_service.check_user_quota(): if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota') raise Exception('You have reached the limit of your quota')
try: with metrics('obs_getboundaryid', user_obs_snapshot_config, logger):
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryId($1, $2, $3, $4, $5) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) try:
result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryId($1, $2, $3, $4, $5) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"])
if result: result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span])
quota_service.increment_success_service_use() if result:
return result[0]['boundary'] quota_service.increment_success_service_use()
else: return result[0]['boundary']
quota_service.increment_empty_service_use() else:
return None quota_service.increment_empty_service_use()
except BaseException as e: return None
import sys except BaseException as e:
quota_service.increment_failed_service_use() import sys
logger.error('Error trying to OBS_GetBoundaryId', sys.exc_info(), data={"username": username, "orgname": orgname}) quota_service.increment_failed_service_use()
raise Exception('Error trying to OBS_GetBoundaryId') logger.error('Error trying to OBS_GetBoundaryId', sys.exc_info(), data={"username": username, "orgname": orgname})
finally: raise Exception('Error trying to OBS_GetBoundaryId')
quota_service.increment_total_service_use() finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu; $$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryById( CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryById(
@ -118,6 +122,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryById(
boundary_id TEXT, boundary_id TEXT,
time_span TEXT DEFAULT NULL) time_span TEXT DEFAULT NULL)
RETURNS geometry(Geometry, 4326) AS $$ RETURNS geometry(Geometry, 4326) AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig from cartodb_services.tools import Logger,LoggerConfig
@ -133,22 +138,23 @@ RETURNS geometry(Geometry, 4326) AS $$
if not quota_service.check_user_quota(): if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota') raise Exception('You have reached the limit of your quota')
try: with metrics('obs_getboundarybyid', user_obs_snapshot_config, logger):
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryById($1, $2, $3, $4, $5) as boundary;", ["text", "text", "text", "text", "text"]) try:
result = plpy.execute(obs_plan, [username, orgname, geometry_id, boundary_id, time_span]) obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryById($1, $2, $3, $4, $5) as boundary;", ["text", "text", "text", "text", "text"])
if result: result = plpy.execute(obs_plan, [username, orgname, geometry_id, boundary_id, time_span])
quota_service.increment_success_service_use() if result:
return result[0]['boundary'] quota_service.increment_success_service_use()
else: return result[0]['boundary']
quota_service.increment_empty_service_use() else:
return None quota_service.increment_empty_service_use()
except BaseException as e: return None
import sys except BaseException as e:
quota_service.increment_failed_service_use() import sys
logger.error('Error trying to OBS_GetBoundaryById', sys.exc_info(), data={"username": username, "orgname": orgname}) quota_service.increment_failed_service_use()
raise Exception('Error trying to OBS_GetBoundaryById') logger.error('Error trying to OBS_GetBoundaryById', sys.exc_info(), data={"username": username, "orgname": orgname})
finally: raise Exception('Error trying to OBS_GetBoundaryById')
quota_service.increment_total_service_use() finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu; $$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByGeometry( CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByGeometry(
@ -171,6 +177,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByGeometry(
time_span TEXT DEFAULT NULL, time_span TEXT DEFAULT NULL,
overlap_type TEXT DEFAULT NULL) overlap_type TEXT DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig from cartodb_services.tools import Logger,LoggerConfig
@ -186,27 +193,28 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
if not quota_service.check_user_quota(): if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota') raise Exception('You have reached the limit of your quota')
try: with metrics('obs_getboundariesbygeometry', user_obs_snapshot_config, logger):
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) try:
result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"])
if result: result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type])
resp = [] if result:
for element in result: resp = []
the_geom = element['the_geom'] for element in result:
geom_refs = element['geom_refs'] the_geom = element['the_geom']
resp.append([the_geom, geom_refs]) geom_refs = element['geom_refs']
quota_service.increment_success_service_use() resp.append([the_geom, geom_refs])
return resp quota_service.increment_success_service_use()
else: return resp
quota_service.increment_empty_service_use() else:
return [] quota_service.increment_empty_service_use()
except BaseException as e: return []
import sys except BaseException as e:
quota_service.increment_failed_service_use() import sys
logger.error('Error trying to OBS_GetBoundariesByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname}) quota_service.increment_failed_service_use()
raise Exception('Error trying to OBS_GetBoundariesByGeometry') logger.error('Error trying to OBS_GetBoundariesByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname})
finally: raise Exception('Error trying to OBS_GetBoundariesByGeometry')
quota_service.increment_total_service_use() finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu; $$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius( CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius(
@ -231,6 +239,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByPointAndRa
time_span TEXT DEFAULT NULL, time_span TEXT DEFAULT NULL,
overlap_type TEXT DEFAULT NULL) overlap_type TEXT DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig from cartodb_services.tools import Logger,LoggerConfig
@ -246,27 +255,28 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
if not quota_service.check_user_quota(): if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota') raise Exception('You have reached the limit of your quota')
try: with metrics('obs_getboundariesbypointandradius', user_obs_snapshot_config, logger):
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) try:
result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"])
if result: result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type])
resp = [] if result:
for element in result: resp = []
the_geom = element['the_geom'] for element in result:
geom_refs = element['geom_refs'] the_geom = element['the_geom']
resp.append([the_geom, geom_refs]) geom_refs = element['geom_refs']
quota_service.increment_success_service_use() resp.append([the_geom, geom_refs])
return resp quota_service.increment_success_service_use()
else: return resp
quota_service.increment_empty_service_use() else:
return [] quota_service.increment_empty_service_use()
except BaseException as e: return []
import sys except BaseException as e:
quota_service.increment_failed_service_use() import sys
logger.error('Error trying to OBS_GetBoundariesByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname}) quota_service.increment_failed_service_use()
raise Exception('Error trying to OBS_GetBoundariesByPointAndRadius') logger.error('Error trying to OBS_GetBoundariesByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname})
finally: raise Exception('Error trying to OBS_GetBoundariesByPointAndRadius')
quota_service.increment_total_service_use() finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu; $$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByGeometry( CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByGeometry(
@ -289,6 +299,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByGeometry(
time_span TEXT DEFAULT NULL, time_span TEXT DEFAULT NULL,
overlap_type TEXT DEFAULT NULL) overlap_type TEXT DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig from cartodb_services.tools import Logger,LoggerConfig
@ -304,27 +315,28 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
if not quota_service.check_user_quota(): if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota') raise Exception('You have reached the limit of your quota')
try: with metrics('obs_getpointsbygeometry', user_obs_snapshot_config, logger):
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) try:
result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"])
if result: result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type])
resp = [] if result:
for element in result: resp = []
the_geom = element['the_geom'] for element in result:
geom_refs = element['geom_refs'] the_geom = element['the_geom']
resp.append([the_geom, geom_refs]) geom_refs = element['geom_refs']
quota_service.increment_success_service_use() resp.append([the_geom, geom_refs])
return resp quota_service.increment_success_service_use()
else: return resp
quota_service.increment_empty_service_use() else:
return [] quota_service.increment_empty_service_use()
except BaseException as e: return []
import sys except BaseException as e:
quota_service.increment_failed_service_use() import sys
logger.error('Error trying to OBS_GetPointsByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname}) quota_service.increment_failed_service_use()
raise Exception('Error trying to OBS_GetPointsByGeometry') logger.error('Error trying to OBS_GetPointsByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname})
finally: raise Exception('Error trying to OBS_GetPointsByGeometry')
quota_service.increment_total_service_use() finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu; $$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByPointAndRadius( CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByPointAndRadius(
@ -349,6 +361,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByPointAndRadius
time_span TEXT DEFAULT NULL, time_span TEXT DEFAULT NULL,
overlap_type TEXT DEFAULT NULL) overlap_type TEXT DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig from cartodb_services.tools import Logger,LoggerConfig
@ -364,25 +377,26 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
if not quota_service.check_user_quota(): if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota') raise Exception('You have reached the limit of your quota')
try: with metrics('obs_getpointsbypointandradius', user_obs_snapshot_config, logger):
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) try:
result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"])
if result: result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type])
resp = [] if result:
for element in result: resp = []
the_geom = element['the_geom'] for element in result:
geom_refs = element['geom_refs'] the_geom = element['the_geom']
resp.append([the_geom, geom_refs]) geom_refs = element['geom_refs']
quota_service.increment_success_service_use() resp.append([the_geom, geom_refs])
return resp quota_service.increment_success_service_use()
else: return resp
quota_service.increment_empty_service_use() else:
return None quota_service.increment_empty_service_use()
except BaseException as e: return None
import sys except BaseException as e:
quota_service.increment_failed_service_use() import sys
logger.error('Error trying to OBS_GetPointsByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname}) quota_service.increment_failed_service_use()
raise Exception('Error trying to OBS_GetPointsByPointAndRadius') logger.error('Error trying to OBS_GetPointsByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname})
finally: raise Exception('Error trying to OBS_GetPointsByPointAndRadius')
quota_service.increment_total_service_use() finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu; $$ LANGUAGE plpythonu;

28
server/extension/sql/20_geocode_street.sql
View File

@ -1,22 +1,28 @@
-- Geocodes a street address given a searchtext and a state and/or country -- Geocodes a street address given a searchtext and a state and/or country
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$ RETURNS Geometry AS $$
from cartodb_services.metrics import metrics
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
if user_geocoder_config.heremaps_geocoder: with metrics('cdb_geocode_street_point', user_geocoder_config, logger):
here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) if user_geocoder_config.heremaps_geocoder:
return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
elif user_geocoder_config.google_geocoder: return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) elif user_geocoder_config.google_geocoder:
return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
elif user_geocoder_config.mapzen_geocoder: return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) elif user_geocoder_config.mapzen_geocoder:
return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
else: return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
raise Exception('Requested geocoder is not available') else:
raise Exception('Requested geocoder is not available')
$$ LANGUAGE plpythonu; $$ LANGUAGE plpythonu;

36
server/extension/sql/30_admin0.sql
View File

@ -2,6 +2,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(us
RETURNS Geometry AS $$ RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.metrics import metrics
from cartodb_services.tools import Logger,LoggerConfig from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
@ -13,23 +14,24 @@ RETURNS Geometry AS $$
logger_config = GD["logger_config"] logger_config = GD["logger_config"]
logger = Logger(logger_config) logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn) quota_service = QuotaService(user_geocoder_config, redis_conn)
try: with metrics('cdb_geocode_admin0_polygon', user_geocoder_config, logger):
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"]) try:
rv = plpy.execute(plan, [country_name], 1) plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"])
result = rv[0]["mypolygon"] rv = plpy.execute(plan, [country_name], 1)
if result: result = rv[0]["mypolygon"]
quota_service.increment_success_service_use() if result:
return result quota_service.increment_success_service_use()
else: return result
quota_service.increment_empty_service_use() else:
return None quota_service.increment_empty_service_use()
except BaseException as e: return None
import sys except BaseException as e:
quota_service.increment_failed_service_use() import sys
logger.error('Error trying to geocode admin0 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) quota_service.increment_failed_service_use()
raise Exception('Error trying to geocode admin0 polygon') logger.error('Error trying to geocode admin0 polygon', sys.exc_info(), data={"username": username, "orgname": orgname})
finally: raise Exception('Error trying to geocode admin0 polygon')
quota_service.increment_total_service_use() finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu; $$ LANGUAGE plpythonu;

78
server/extension/sql/40_admin1.sql
View File

@ -1,6 +1,7 @@
---- cdb_geocode_admin1_polygon(admin1_name text) ---- cdb_geocode_admin1_polygon(admin1_name text)
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text)
RETURNS Geometry AS $$ RETURNS Geometry AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger,LoggerConfig from cartodb_services.tools import Logger,LoggerConfig
@ -14,44 +15,11 @@ RETURNS Geometry AS $$
logger_config = GD["logger_config"] logger_config = GD["logger_config"]
logger = Logger(logger_config) logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn) quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1)) AS mypolygon", ["text"])
rv = plpy.execute(plan, [admin1_name], 1)
result = rv[0]["mypolygon"]
if result:
quota_service.increment_success_service_use()
return result
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode admin1 polygon')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger):
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text)
RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try: try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1), trim($2)) AS mypolygon", ["text", "text"]) plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1)) AS mypolygon", ["text"])
rv = plpy.execute(plan, [admin1_name, country_name], 1) rv = plpy.execute(plan, [admin1_name], 1)
result = rv[0]["mypolygon"] result = rv[0]["mypolygon"]
if result: if result:
quota_service.increment_success_service_use() quota_service.increment_success_service_use()
@ -68,6 +36,44 @@ RETURNS Geometry AS $$
quota_service.increment_total_service_use() quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu; $$ LANGUAGE plpythonu;
---- cdb_geocode_admin1_polygon(admin1_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text)
RETURNS Geometry AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger):
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1), trim($2)) AS mypolygon", ["text", "text"])
rv = plpy.execute(plan, [admin1_name, country_name], 1)
result = rv[0]["mypolygon"]
if result:
quota_service.increment_success_service_use()
return result
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode admin1 polygon')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
-------------------------------------------------------------------------------- --------------------------------------------------------------------------------
-- Implementation of the server extension -- Implementation of the server extension

101
server/extension/sql/50_namedplaces.sql
View File

@ -35,7 +35,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_namedplac
RETURNS Geometry AS $$ RETURNS Geometry AS $$
from cartodb_services.mapzen import MapzenGeocoder from cartodb_services.mapzen import MapzenGeocoder
from cartodb_services.mapzen.types import country_to_iso3 from cartodb_services.mapzen.types import country_to_iso3
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService, metrics
from cartodb_services.tools import Logger,LoggerConfig from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
@ -50,35 +50,36 @@ RETURNS Geometry AS $$
if not quota_service.check_user_quota(): if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota') raise Exception('You have reached the limit of your quota')
try: with metrics('cdb_geocode_namedplace_point', user_geocoder_config, logger):
geocoder = MapzenGeocoder(user_geocoder_config.mapzen_api_key, logger) try:
country_iso3 = None geocoder = MapzenGeocoder(user_geocoder_config.mapzen_api_key, logger)
if country_name: country_iso3 = None
country_iso3 = country_to_iso3(country_name) if country_name:
coordinates = geocoder.geocode(searchtext=city_name, city=None, country_iso3 = country_to_iso3(country_name)
state_province=admin1_name, coordinates = geocoder.geocode(searchtext=city_name, city=None,
country=country_iso3, search_type='locality') state_province=admin1_name,
if coordinates: country=country_iso3, search_type='locality')
quota_service.increment_success_service_use() if coordinates:
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) quota_service.increment_success_service_use()
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
return point['st_setsrid'] point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
else: return point['st_setsrid']
quota_service.increment_empty_service_use() else:
return None quota_service.increment_empty_service_use()
except BaseException as e: return None
import sys except BaseException as e:
quota_service.increment_failed_service_use() import sys
logger.error('Error trying to geocode city point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname}) quota_service.increment_failed_service_use()
raise Exception('Error trying to geocode city point using mapzen') logger.error('Error trying to geocode city point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname})
finally: raise Exception('Error trying to geocode city point using mapzen')
quota_service.increment_total_service_use() finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu; $$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_internal_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL) CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_internal_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL)
RETURNS Geometry AS $$ RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig from cartodb_services.metrics import InternalGeocoderConfig, metrics
from cartodb_services.tools import Logger,LoggerConfig from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
@ -90,30 +91,32 @@ RETURNS Geometry AS $$
logger_config = GD["logger_config"] logger_config = GD["logger_config"]
logger = Logger(logger_config) logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn) quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
if admin1_name and country_name: with metrics('cdb_geocode_namedplace_point', user_geocoder_config, logger):
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"]) try:
rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1) if admin1_name and country_name:
elif country_name: plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"])
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"]) rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1)
rv = plpy.execute(plan, [city_name, country_name], 1) elif country_name:
else: plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"])
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"]) rv = plpy.execute(plan, [city_name, country_name], 1)
rv = plpy.execute(plan, [city_name], 1) else:
result = rv[0]["mypoint"] plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"])
if result: rv = plpy.execute(plan, [city_name], 1)
quota_service.increment_success_service_use() result = rv[0]["mypoint"]
return result if result:
else: quota_service.increment_success_service_use()
quota_service.increment_empty_service_use() return result
return None else:
except BaseException as e: quota_service.increment_empty_service_use()
import sys return None
quota_service.increment_failed_service_use() except BaseException as e:
logger.error('Error trying to geocode namedplace point', sys.exc_info(), data={"username": username, "orgname": orgname}) import sys
raise Exception('Error trying to geocode namedplace point') quota_service.increment_failed_service_use()
finally: logger.error('Error trying to geocode namedplace point', sys.exc_info(), data={"username": username, "orgname": orgname})
quota_service.increment_total_service_use() raise Exception('Error trying to geocode namedplace point')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu; $$ LANGUAGE plpythonu;
-------------------------------------------------------------------------------- --------------------------------------------------------------------------------

144
server/extension/sql/60_postalcodes.sql
View File

@ -1,5 +1,6 @@
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text) CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text)
RETURNS Geometry AS $$ RETURNS Geometry AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger,LoggerConfig from cartodb_services.tools import Logger,LoggerConfig
@ -13,27 +14,29 @@ RETURNS Geometry AS $$
logger_config = GD["logger_config"] logger_config = GD["logger_config"]
logger = Logger(logger_config) logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn) quota_service = QuotaService(user_geocoder_config, redis_conn)
try: with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger):
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1)) AS mypoint", ["text"]) try:
rv = plpy.execute(plan, [code], 1) plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1)) AS mypoint", ["text"])
result = rv[0]["mypoint"] rv = plpy.execute(plan, [code], 1)
if result: result = rv[0]["mypoint"]
quota_service.increment_success_service_use() if result:
return result quota_service.increment_success_service_use()
else: return result
quota_service.increment_empty_service_use() else:
return None quota_service.increment_empty_service_use()
except BaseException as e: return None
import sys except BaseException as e:
quota_service.increment_failed_service_use() import sys
logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname}) quota_service.increment_failed_service_use()
raise Exception('Error trying to geocode postal code point') logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname})
finally: raise Exception('Error trying to geocode postal code point')
quota_service.increment_total_service_use() finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu; $$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text) CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text)
RETURNS Geometry AS $$ RETURNS Geometry AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger,LoggerConfig from cartodb_services.tools import Logger,LoggerConfig
@ -47,27 +50,29 @@ RETURNS Geometry AS $$
logger_config = GD["logger_config"] logger_config = GD["logger_config"]
logger = Logger(logger_config) logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn) quota_service = QuotaService(user_geocoder_config, redis_conn)
try: with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger):
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1), trim($2)) AS mypoint", ["TEXT", "TEXT"]) try:
rv = plpy.execute(plan, [code, country], 1) plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1), trim($2)) AS mypoint", ["TEXT", "TEXT"])
result = rv[0]["mypoint"] rv = plpy.execute(plan, [code, country], 1)
if result: result = rv[0]["mypoint"]
quota_service.increment_success_service_use() if result:
return result quota_service.increment_success_service_use()
else: return result
quota_service.increment_empty_service_use() else:
return None quota_service.increment_empty_service_use()
except BaseException as e: return None
import sys except BaseException as e:
quota_service.increment_failed_service_use() import sys
logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname}) quota_service.increment_failed_service_use()
raise Exception('Error trying to geocode postal code point') logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname})
finally: raise Exception('Error trying to geocode postal code point')
quota_service.increment_total_service_use() finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu; $$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text) CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text)
RETURNS Geometry AS $$ RETURNS Geometry AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger,LoggerConfig from cartodb_services.tools import Logger,LoggerConfig
@ -81,27 +86,29 @@ RETURNS Geometry AS $$
logger_config = GD["logger_config"] logger_config = GD["logger_config"]
logger = Logger(logger_config) logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn) quota_service = QuotaService(user_geocoder_config, redis_conn)
try: with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger):
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1)) AS mypolygon", ["text"]) try:
rv = plpy.execute(plan, [code], 1) plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1)) AS mypolygon", ["text"])
result = rv[0]["mypolygon"] rv = plpy.execute(plan, [code], 1)
if result: result = rv[0]["mypolygon"]
quota_service.increment_success_service_use() if result:
return result quota_service.increment_success_service_use()
else: return result
quota_service.increment_empty_service_use() else:
return None quota_service.increment_empty_service_use()
except BaseException as e: return None
import sys except BaseException as e:
quota_service.increment_failed_service_use() import sys
logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) quota_service.increment_failed_service_use()
raise Exception('Error trying to geocode postal code polygon') logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname})
finally: raise Exception('Error trying to geocode postal code polygon')
quota_service.increment_total_service_use() finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu; $$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text, country text) CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text, country text)
RETURNS Geometry AS $$ RETURNS Geometry AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger,LoggerConfig from cartodb_services.tools import Logger,LoggerConfig
@ -115,23 +122,24 @@ RETURNS Geometry AS $$
logger_config = GD["logger_config"] logger_config = GD["logger_config"]
logger = Logger(logger_config) logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn) quota_service = QuotaService(user_geocoder_config, redis_conn)
try: with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger):
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1), trim($2)) AS mypolygon", ["TEXT", "TEXT"]) try:
rv = plpy.execute(plan, [code, country], 1) plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1), trim($2)) AS mypolygon", ["TEXT", "TEXT"])
result = rv[0]["mypolygon"] rv = plpy.execute(plan, [code, country], 1)
if result: result = rv[0]["mypolygon"]
quota_service.increment_success_service_use() if result:
return result quota_service.increment_success_service_use()
else: return result
quota_service.increment_empty_service_use() else:
return None quota_service.increment_empty_service_use()
except BaseException as e: return None
import sys except BaseException as e:
quota_service.increment_failed_service_use() import sys
logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) quota_service.increment_failed_service_use()
raise Exception('Error trying to geocode postal code polygon') logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname})
finally: raise Exception('Error trying to geocode postal code polygon')
quota_service.increment_total_service_use() finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu; $$ LANGUAGE plpythonu;
-------------------------------------------------------------------------------- --------------------------------------------------------------------------------

36
server/extension/sql/70_ips.sql
View File

@ -1,5 +1,6 @@
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text) CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text)
RETURNS Geometry AS $$ RETURNS Geometry AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger,LoggerConfig from cartodb_services.tools import Logger,LoggerConfig
@ -13,23 +14,24 @@ RETURNS Geometry AS $$
logger_config = GD["logger_config"] logger_config = GD["logger_config"]
logger = Logger(logger_config) logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn) quota_service = QuotaService(user_geocoder_config, redis_conn)
try: with metrics('cdb_geocode_ipaddress_point', user_geocoder_config, logger):
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_ipaddress_point(trim($1)) AS mypoint", ["TEXT"]) try:
rv = plpy.execute(plan, [ip], 1) plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_ipaddress_point(trim($1)) AS mypoint", ["TEXT"])
result = rv[0]["mypoint"] rv = plpy.execute(plan, [ip], 1)
if result: result = rv[0]["mypoint"]
quota_service.increment_success_service_use() if result:
return result quota_service.increment_success_service_use()
else: return result
quota_service.increment_empty_service_use() else:
return None quota_service.increment_empty_service_use()
except BaseException as e: return None
import sys except BaseException as e:
quota_service.increment_failed_service_use() import sys
logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) quota_service.increment_failed_service_use()
raise Exception('Error trying to geocode postal code polygon') logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname})
finally: raise Exception('Error trying to geocode postal code polygon')
quota_service.increment_total_service_use() finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu; $$ LANGUAGE plpythonu;
-------------------------------------------------------------------------------- --------------------------------------------------------------------------------

22
server/extension/sql/90_isochrone.sql
View File

@ -1,21 +1,27 @@
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$ RETURNS SETOF cdb_dataservices_server.isoline AS $$
from cartodb_services.metrics import metrics
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
if user_isolines_config.google_services_user: if user_isolines_config.google_services_user:
raise Exception('This service is not available for google service users.') raise Exception('This service is not available for google service users.')
if user_isolines_config.heremaps_provider: with metrics('cb_isochrone', user_isolines_config, logger):
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) if user_isolines_config.heremaps_provider:
return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
elif user_isolines_config.mapzen_provider: return plpy.execute(here_plan, [username, orgname, source, mode, range, options])
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) elif user_isolines_config.mapzen_provider:
return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
else: return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options])
raise Exception('Requested isolines provider is not available') else:
raise Exception('Requested isolines provider is not available')
$$ LANGUAGE plpythonu; $$ LANGUAGE plpythonu;
-- heremaps isochrone -- heremaps isochrone

4
server/lib/python/cartodb_services/cartodb_services/here/geocoder.py
View File

@ -5,9 +5,10 @@ import json
import requests import requests
from exceptions import * from exceptions import *
from cartodb_services.metrics import Traceable
class HereMapsGeocoder: class HereMapsGeocoder(Traceable):
'A Here Maps Geocoder wrapper for python' 'A Here Maps Geocoder wrapper for python'
PRODUCTION_GEOCODE_JSON_URL = 'https://geocoder.api.here.com/6.2/geocode.json' PRODUCTION_GEOCODE_JSON_URL = 'https://geocoder.api.here.com/6.2/geocode.json'
@ -89,6 +90,7 @@ class HereMapsGeocoder:
request_params.update(params) request_params.update(params)
response = requests.get(self.host, params=request_params, response = requests.get(self.host, params=request_params,
timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT)) timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
self.add_response_data(response, self._logger)
if response.status_code == requests.codes.ok: if response.status_code == requests.codes.ok:
return json.loads(response.text) return json.loads(response.text)
elif response.status_code == requests.codes.bad_request: elif response.status_code == requests.codes.bad_request:

4
server/lib/python/cartodb_services/cartodb_services/here/routing.py
View File

@ -2,9 +2,10 @@ import requests
import json import json
from exceptions import WrongParams from exceptions import WrongParams
from cartodb_services.metrics import Traceable
class HereMapsRoutingIsoline: class HereMapsRoutingIsoline(Traceable):
'A Here Maps Routing wrapper for python' 'A Here Maps Routing wrapper for python'
PRODUCTION_ROUTING_BASE_URL = 'https://isoline.route.api.here.com' PRODUCTION_ROUTING_BASE_URL = 'https://isoline.route.api.here.com'
@ -54,6 +55,7 @@ class HereMapsRoutingIsoline:
parsed_options) parsed_options)
response = requests.get(self._url, params=request_params, response = requests.get(self._url, params=request_params,
timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT)) timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
self.add_response_data(response, self._logger)
if response.status_code == requests.codes.ok: if response.status_code == requests.codes.ok:
return self.__parse_isolines_response(response.text) return self.__parse_isolines_response(response.text)
elif response.status_code == requests.codes.bad_request: elif response.status_code == requests.codes.bad_request:

4
server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py
View File

@ -5,9 +5,10 @@ import re
from exceptions import WrongParams, MalformedResult, ServiceException from exceptions import WrongParams, MalformedResult, ServiceException
from qps import qps_retry from qps import qps_retry
from cartodb_services.tools import Coordinate, PolyLine from cartodb_services.tools import Coordinate, PolyLine
from cartodb_services.metrics import Traceable
class MapzenGeocoder: class MapzenGeocoder(Traceable):
'A Mapzen Geocoder wrapper for python' 'A Mapzen Geocoder wrapper for python'
BASE_URL = 'https://search.mapzen.com/v1/search' BASE_URL = 'https://search.mapzen.com/v1/search'
@ -28,6 +29,7 @@ class MapzenGeocoder:
try: try:
response = requests.get(self._url, params=request_params, response = requests.get(self._url, params=request_params,
timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT)) timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
self.add_response_data(response, self._logger)
if response.status_code == requests.codes.ok: if response.status_code == requests.codes.ok:
return self.__parse_response(response.text) return self.__parse_response(response.text)
elif response.status_code == requests.codes.bad_request: elif response.status_code == requests.codes.bad_request:

4
server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py
View File

@ -2,9 +2,10 @@ import requests
import json import json
from qps import qps_retry from qps import qps_retry
from exceptions import ServiceException from exceptions import ServiceException
from cartodb_services.metrics import Traceable
class MatrixClient: class MatrixClient(Traceable):
""" """
A minimal client for Mapzen Time-Distance Matrix Service A minimal client for Mapzen Time-Distance Matrix Service
@ -45,6 +46,7 @@ class MatrixClient:
} }
response = requests.get(self.ONE_TO_MANY_URL, params=request_params, response = requests.get(self.ONE_TO_MANY_URL, params=request_params,
timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT)) timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
self.add_response_data(response, self._logger)
if response.status_code != requests.codes.ok: if response.status_code != requests.codes.ok:
self._logger.error('Error trying to get matrix distance from mapzen', self._logger.error('Error trying to get matrix distance from mapzen',

4
server/lib/python/cartodb_services/cartodb_services/mapzen/routing.py
View File

@ -5,9 +5,10 @@ import re
from exceptions import WrongParams, MalformedResult, ServiceException from exceptions import WrongParams, MalformedResult, ServiceException
from qps import qps_retry from qps import qps_retry
from cartodb_services.tools import Coordinate, PolyLine from cartodb_services.tools import Coordinate, PolyLine
from cartodb_services.metrics import MetricsDataGatherer, Traceable
class MapzenRouting: class MapzenRouting(Traceable):
'A Mapzen Routing wrapper for python' 'A Mapzen Routing wrapper for python'
PRODUCTION_ROUTING_BASE_URL = 'https://valhalla.mapzen.com/route' PRODUCTION_ROUTING_BASE_URL = 'https://valhalla.mapzen.com/route'
@ -47,6 +48,7 @@ class MapzenRouting:
request_params = self.__parse_request_parameters(json_request_params) request_params = self.__parse_request_parameters(json_request_params)
response = requests.get(self._url, params=request_params, response = requests.get(self._url, params=request_params,
timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT)) timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
self.add_response_data(response, self._logger)
if response.status_code == requests.codes.ok: if response.status_code == requests.codes.ok:
return self.__parse_routing_response(response.text) return self.__parse_routing_response(response.text)
elif response.status_code == requests.codes.bad_request: elif response.status_code == requests.codes.bad_request:

1
server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py
View File

@ -1,3 +1,4 @@
from config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatorySnapshotConfig, ObservatoryConfig from config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatorySnapshotConfig, ObservatoryConfig
from quota import QuotaService from quota import QuotaService
from user import UserMetricsService from user import UserMetricsService
from log import metrics, MetricsDataGatherer, Traceable

59
server/lib/python/cartodb_services/cartodb_services/metrics/config.py
View File

@ -15,6 +15,7 @@ class ServiceConfig(object):
self._username = username self._username = username
self._orgname = orgname self._orgname = orgname
self._db_config = ServicesDBConfig(db_conn, username, orgname) self._db_config = ServicesDBConfig(db_conn, username, orgname)
self._metrics_log_path = self.__get_metrics_log_path()
self._environment = self._db_config._server_environment self._environment = self._db_config._server_environment
if redis_connection: if redis_connection:
self._redis_config = ServicesRedisConfig(redis_connection).build( self._redis_config = ServicesRedisConfig(redis_connection).build(
@ -38,9 +39,20 @@ class ServiceConfig(object):
def environment(self): def environment(self):
return self._environment return self._environment
@property
def metrics_log_path(self):
return self._metrics_log_path
def __get_metrics_log_path(self):
if self.METRICS_LOG_KEY:
return self._db_config.logger_config.get(self.METRICS_LOG_KEY, None)
else:
return None
class DataObservatoryConfig(ServiceConfig): class DataObservatoryConfig(ServiceConfig):
METRICS_LOG_KEY = 'do_log_path'
def __init__(self, redis_connection, db_conn, username, orgname=None): def __init__(self, redis_connection, db_conn, username, orgname=None):
super(DataObservatoryConfig, self).__init__(redis_connection, db_conn, super(DataObservatoryConfig, self).__init__(redis_connection, db_conn,
username, orgname) username, orgname)
@ -61,6 +73,10 @@ class DataObservatoryConfig(ServiceConfig):
def connection_str(self): def connection_str(self):
return self._connection_str return self._connection_str
@property
def provider(self):
return 'data observatory'
class ObservatorySnapshotConfig(DataObservatoryConfig): class ObservatorySnapshotConfig(DataObservatoryConfig):
@ -118,6 +134,7 @@ class RoutingConfig(ServiceConfig):
DEFAULT_PROVIDER = 'mapzen' DEFAULT_PROVIDER = 'mapzen'
QUOTA_KEY = 'mapzen_routing_quota' QUOTA_KEY = 'mapzen_routing_quota'
SOFT_LIMIT_KEY = 'soft_mapzen_routing_limit' SOFT_LIMIT_KEY = 'soft_mapzen_routing_limit'
METRICS_LOG_KEY = 'routing_log_path'
def __init__(self, redis_connection, db_conn, username, orgname=None): def __init__(self, redis_connection, db_conn, username, orgname=None):
super(RoutingConfig, self).__init__(redis_connection, db_conn, super(RoutingConfig, self).__init__(redis_connection, db_conn,
@ -135,6 +152,10 @@ class RoutingConfig(ServiceConfig):
if self._routing_provider == self.MAPZEN_PROVIDER: if self._routing_provider == self.MAPZEN_PROVIDER:
return 'routing_mapzen' return 'routing_mapzen'
@property
def provider(self):
return self._routing_provider
@property @property
def mapzen_api_key(self): def mapzen_api_key(self):
return self._mapzen_api_key return self._mapzen_api_key
@ -151,7 +172,6 @@ class RoutingConfig(ServiceConfig):
def soft_limit(self): def soft_limit(self):
return self._soft_limit return self._soft_limit
def _set_monthly_quota(self): def _set_monthly_quota(self):
self._monthly_quota = self._get_effective_monthly_quota() self._monthly_quota = self._get_effective_monthly_quota()
@ -169,7 +189,6 @@ class RoutingConfig(ServiceConfig):
self._soft_limit = False self._soft_limit = False
class IsolinesRoutingConfig(ServiceConfig): class IsolinesRoutingConfig(ServiceConfig):
ISOLINES_CONFIG_KEYS = ['here_isolines_quota', 'soft_here_isolines_limit', ISOLINES_CONFIG_KEYS = ['here_isolines_quota', 'soft_here_isolines_limit',
@ -184,6 +203,7 @@ class IsolinesRoutingConfig(ServiceConfig):
MAPZEN_PROVIDER = 'mapzen' MAPZEN_PROVIDER = 'mapzen'
HEREMAPS_PROVIDER = 'heremaps' HEREMAPS_PROVIDER = 'heremaps'
DEFAULT_PROVIDER = 'heremaps' DEFAULT_PROVIDER = 'heremaps'
METRICS_LOG_KEY = 'isolines_log_path'
def __init__(self, redis_connection, db_conn, username, orgname=None): def __init__(self, redis_connection, db_conn, username, orgname=None):
super(IsolinesRoutingConfig, self).__init__(redis_connection, db_conn, super(IsolinesRoutingConfig, self).__init__(redis_connection, db_conn,
@ -260,11 +280,12 @@ class IsolinesRoutingConfig(ServiceConfig):
class InternalGeocoderConfig(ServiceConfig): class InternalGeocoderConfig(ServiceConfig):
METRICS_LOG_KEY = 'geocoder_log_path'
def __init__(self, redis_connection, db_conn, username, orgname=None): def __init__(self, redis_connection, db_conn, username, orgname=None):
# For now, internal geocoder doesn't use the redis config # For now, internal geocoder doesn't use the redis config
super(InternalGeocoderConfig, self).__init__(None, db_conn, super(InternalGeocoderConfig, self).__init__(None, db_conn,
username, orgname) username, orgname)
self._log_path = self._db_config.geocoder_log_path
@property @property
def service_type(self): def service_type(self):
@ -283,8 +304,8 @@ class InternalGeocoderConfig(ServiceConfig):
return None return None
@property @property
def log_path(self): def provider(self):
return self._log_path return 'internal'
class GeocoderConfig(ServiceConfig): class GeocoderConfig(ServiceConfig):
@ -310,6 +331,7 @@ class GeocoderConfig(ServiceConfig):
ORGNAME_KEY = 'orgname' ORGNAME_KEY = 'orgname'
PERIOD_END_DATE = 'period_end_date' PERIOD_END_DATE = 'period_end_date'
DEFAULT_PROVIDER = 'mapzen' DEFAULT_PROVIDER = 'mapzen'
METRICS_LOG_KEY = 'geocoder_log_path'
def __init__(self, redis_connection, db_conn, username, orgname=None, forced_provider=None): def __init__(self, redis_connection, db_conn, username, orgname=None, forced_provider=None):
super(GeocoderConfig, self).__init__(redis_connection, db_conn, super(GeocoderConfig, self).__init__(redis_connection, db_conn,
@ -341,7 +363,6 @@ class GeocoderConfig(ServiceConfig):
self._geocoder_provider = self.DEFAULT_PROVIDER self._geocoder_provider = self.DEFAULT_PROVIDER
self._geocoding_quota = float(filtered_config[self.QUOTA_KEY]) self._geocoding_quota = float(filtered_config[self.QUOTA_KEY])
self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE]) self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE])
self._log_path = db_config.geocoder_log_path
if filtered_config[self.SOFT_LIMIT_KEY].lower() == 'true': if filtered_config[self.SOFT_LIMIT_KEY].lower() == 'true':
self._soft_geocoding_limit = True self._soft_geocoding_limit = True
else: else:
@ -424,8 +445,8 @@ class GeocoderConfig(ServiceConfig):
return self._cost_per_hit return self._cost_per_hit
@property @property
def log_path(self): def provider(self):
return self._log_path return self._geocoder_provider
class ServicesDBConfig: class ServicesDBConfig:
@ -440,7 +461,6 @@ class ServicesDBConfig:
self._get_server_config() self._get_server_config()
self._get_here_config() self._get_here_config()
self._get_mapzen_config() self._get_mapzen_config()
self._get_logger_config()
self._get_data_observatory_config() self._get_data_observatory_config()
def _get_server_config(self): def _get_server_config(self):
@ -493,13 +513,6 @@ class ServicesDBConfig:
else: else:
self._data_observatory_connection_str = do_conf['connection']['production'] self._data_observatory_connection_str = do_conf['connection']['production']
def _get_logger_config(self):
logger_conf_json = self._get_conf('logger_conf')
if not logger_conf_json:
raise ConfigException('Logger configuration missing')
else:
logger_conf = json.loads(logger_conf_json)
self._geocoder_log_path = logger_conf['geocoder_log_path']
def _get_conf(self, key): def _get_conf(self, key):
try: try:
@ -507,7 +520,7 @@ class ServicesDBConfig:
conf = self._db_conn.execute(sql, 1) conf = self._db_conn.execute(sql, 1)
return conf[0]['conf'] return conf[0]['conf']
except Exception as e: except Exception as e:
raise ConfigException("Malformed config for {0}: {1}".format(key, e)) raise ConfigException("Error trying to get config for {0}: {1}".format(key, e))
@property @property
def server_environment(self): def server_environment(self):
@ -557,14 +570,18 @@ class ServicesDBConfig:
def mapzen_geocoder_monthly_quota(self): def mapzen_geocoder_monthly_quota(self):
return self._mapzen_geocoder_quota return self._mapzen_geocoder_quota
@property
def geocoder_log_path(self):
return self._geocoder_log_path
@property @property
def data_observatory_connection_str(self): def data_observatory_connection_str(self):
return self._data_observatory_connection_str return self._data_observatory_connection_str
@property
def logger_config(self):
logger_conf_json = self._get_conf('logger_conf')
if not logger_conf_json:
raise ConfigException('Logger configuration missing')
else:
return json.loads(logger_conf_json)
class ServicesRedisConfig: class ServicesRedisConfig:

249
server/lib/python/cartodb_services/cartodb_services/metrics/log.py
View File

@ -1,15 +1,130 @@
from datetime import datetime
import abc import abc
import json import json
import re import re
import time
import uuid
import plpy
from datetime import datetime
from contextlib import contextmanager
from urlparse import urlparse
class MetricsLoggerFactory: @contextmanager
def metrics(function, service_config, logger=None):
try:
start_time = time.time()
yield
finally:
end_time = time.time()
MetricsDataGatherer.add('uuid', str(uuid.uuid1()))
MetricsDataGatherer.add('function_name', function)
MetricsDataGatherer.add('function_execution_time', (end_time - start_time))
metrics_logger = MetricsServiceLoggerFactory.build(service_config,
logger)
if metrics_logger:
data = MetricsDataGatherer.get()
metrics_logger.log(data)
MetricsDataGatherer.clean()
class Traceable:
"""
Module to add metrics traceability, for example to get response object
in order to add to the metrics dump
"""
def add_response_data(self, response, logger=None):
try:
response_data = {}
response_data['type'] = "request"
response_data['date'] = datetime.now().isoformat()
response_data['elapsed_time'] = response.elapsed.total_seconds()
response_data['code'] = response.status_code
response_data['message'] = response.reason
response_data['url'] = self._parse_response_url(response.url)
stored_data = MetricsDataGatherer.get_element('response')
if stored_data:
stored_data.append(response_data)
else:
MetricsDataGatherer.add('response', [response_data])
except BaseException as e:
# We don't want to stop the job for some error processing response
if logger:
logger.error("Error trying to process response metricd data",
exception=e)
def _parse_response_url(self, url):
u = urlparse(url)
return "{0}://{1}{2}".format(u.scheme, u.netloc, u.path)
class MetricsDataGatherer:
"""
Metrics gatherer used as a singleton. The intend is to use it as a global
storage for the metrics along the function request.
"""
class __MetricsDataGatherer:
def __init__(self):
self.data = {}
def add(self, key, value):
self.data[key] = value
def get(self):
return self.data
def get_element(self, key):
return self.data.get(key, None)
def clean(self):
self.data = {}
# We use pgbouncer so we need to have multiples instances per request id
__instance = {}
@classmethod @classmethod
def build(self, service_config): def add(self, key, value):
if re.match('geocoder_*', service_config.service_type): MetricsDataGatherer.instance().add(key, value)
return MetricsGeocoderLogger(service_config)
@classmethod
def get(self):
return MetricsDataGatherer.instance().get()
@classmethod
def get_element(self, key):
return MetricsDataGatherer.instance().get_element(key)
@classmethod
def clean(self):
MetricsDataGatherer.instance().clean()
@classmethod
def instance(self):
txid = MetricsDataGatherer._get_txid()
if txid not in MetricsDataGatherer.__instance:
MetricsDataGatherer.__instance[txid] = MetricsDataGatherer.__MetricsDataGatherer()
return MetricsDataGatherer.__instance[txid]
@classmethod
def _get_txid(self):
result = plpy.execute('select txid_current() as txid')
return result[0]['txid']
class MetricsServiceLoggerFactory:
@classmethod
def build(self, service_config, logger=None):
if re.search('^geocoder_*', service_config.service_type):
return MetricsGeocoderLogger(service_config, logger)
elif re.search('^routing_*', service_config.service_type):
return MetricsGenericLogger(service_config, logger)
elif re.search('_isolines$', service_config.service_type):
return MetricsIsolinesLogger(service_config, logger)
elif re.search('^obs_*', service_config.service_type):
return MetricsGenericLogger(service_config, logger)
else: else:
return None return None
@ -17,58 +132,120 @@ class MetricsLoggerFactory:
class MetricsLogger(object): class MetricsLogger(object):
__metaclass__ = abc.ABCMeta __metaclass__ = abc.ABCMeta
def __init__(self, file_path): def __init__(self, service_config, logger):
self._file_path = file_path self._service_config = service_config
self._logger = logger
def dump_to_file(self, data): def dump_to_file(self, data):
with open(self._file_path, 'a') as logfile: try:
json.dump(data, logfile) log_path = self.service_config.metrics_log_path
logfile.write('\n') response_data = data.pop('response', [])
uuid = data.get('uuid')
if log_path:
with open(log_path, 'a') as logfile:
self._dump_response_to_file(uuid, response_data, logfile)
json.dump(data, logfile)
logfile.write('\n')
except BaseException as e:
self._logger("Error dumping metrics to file {0}".format(log_path),
exception=e)
def collect_data(self, data):
return {
"uuid": data.get('uuid', uuid.uuid1()),
"type": 'function',
"function_name": data.get('function_name', None),
"function_execution_time": data.get('function_execution_time',
None),
"service": self._service_config.service_type,
"processable_rows": 1,
"success": data.get('success', False),
"successful_rows": data.get('successful_rows', 0),
"failed_rows": data.get('failed_rows', 0),
"empty_rows": data.get('empty_rows', 0),
"created_at": datetime.now().isoformat(),
"provider": self._service_config.provider,
"username": self._service_config.username,
"organization": self._service_config.organization,
"response": data.get('response', [])
}
def _dump_response_to_file(self, uuid, response_data, log_file):
for r in response_data:
r['uuid'] = uuid
self._logger.info(r)
json.dump(r, log_file)
log_file.write('\n')
@property
def service_config(self):
return self._service_config
@abc.abstractproperty @abc.abstractproperty
def log(self, **data): def log(self, data):
raise NotImplementedError('log method must be defined') raise NotImplementedError('log method must be defined')
class MetricsGeocoderLogger(MetricsLogger): class MetricsGeocoderLogger(MetricsLogger):
def __init__(self, service_config): def __init__(self, service_config, logger):
super(MetricsGeocoderLogger, self).__init__(service_config.log_path) super(MetricsGeocoderLogger, self).__init__(service_config, logger)
self._service_config = service_config
def log(self, **data): def log(self, data):
dump_data = self._dump_data(**data) dump_data = self.collect_data(data)
self.dump_to_file(dump_data) self.dump_to_file(dump_data)
def _dump_data(self, **data): def collect_data(self, data):
if data['success']: dump_data = super(MetricsGeocoderLogger, self).collect_data(data)
cost = self._service_config.cost_per_hit if data.get('success', False):
failed_rows = 0 cost = self.service_config.cost_per_hit
successful_rows = 1
else: else:
cost = 0 cost = 0
failed_rows = 1
successful_rows = 0
if self._service_config.is_high_resolution: if self.service_config.is_high_resolution:
kind = 'high-resolution' kind = 'high-resolution'
else: else:
kind = 'internal' kind = 'internal'
return { dump_data.update({
"batched": False, "batched": False,
"cache_hits": 0, # Always 0 because no cache involved "cache_hits": 0, # Always 0 because no cache involved
# https://github.com/CartoDB/cartodb/blob/master/app/models/geocoding.rb#L208-L211 # https://github.com/CartoDB/cartodb/blob/master/app/models/geocoding.rb#L208-L211
"cost": cost, "cost": cost,
"created_at": datetime.now().isoformat(), "geocoder_type": self.service_config.service_type,
"failed_rows": failed_rows,
"geocoder_type": self._service_config.service_type,
"kind": kind, "kind": kind,
"processable_rows": 1, "processed_rows": data.get('successful_rows', 0),
"processed_rows": successful_rows, "real_rows": data.get('successful_rows', 0),
"real_rows": successful_rows, })
"success": data['success'],
"successful_rows": successful_rows, return dump_data
"username": self._service_config.username,
"organization": self._service_config.organization
} class MetricsGenericLogger(MetricsLogger):
def __init__(self, service_config, logger):
super(MetricsGenericLogger, self).__init__(service_config, logger)
def log(self, data):
dump_data = self.collect_data(data)
self.dump_to_file(dump_data)
def collect_data(self, data):
return super(MetricsGenericLogger, self).collect_data(data)
class MetricsIsolinesLogger(MetricsLogger):
def __init__(self, service_config, logger):
super(MetricsIsolinesLogger, self).__init__(service_config, logger)
def log(self, data):
dump_data = self.collect_data(data)
self.dump_to_file(dump_data)
def collect_data(self, data):
dump_data = super(MetricsIsolinesLogger, self).collect_data(data)
dump_data.update({
"isolines_generated": data.get('isolines_generated', 0)
})
return dump_data

19
server/lib/python/cartodb_services/cartodb_services/metrics/quota.py
View File

@ -1,5 +1,5 @@
from user import UserMetricsService from user import UserMetricsService
from log import MetricsLoggerFactory from log import MetricsDataGatherer
from datetime import date from datetime import date
import re import re
@ -14,7 +14,6 @@ class QuotaService:
redis_connection) redis_connection)
self._user_service = UserMetricsService(self._user_service_config, self._user_service = UserMetricsService(self._user_service_config,
redis_connection) redis_connection)
self._metrics_logger = MetricsLoggerFactory.build(user_service_config)
def check_user_quota(self): def check_user_quota(self):
return self._quota_checker.check() return self._quota_checker.check()
@ -46,13 +45,19 @@ class QuotaService:
self._user_service.increment_service_use( self._user_service.increment_service_use(
self._user_service_config.service_type, "isolines_generated", self._user_service_config.service_type, "isolines_generated",
amount=amount) amount=amount)
MetricsDataGatherer.add('isolines_generated', amount)
def _log_service_process(self, event): def _log_service_process(self, event):
if self._metrics_logger: if event is 'success':
if event is 'success' or event is 'empty': MetricsDataGatherer.add('success', True)
self._metrics_logger.log(success=True) MetricsDataGatherer.add('successful_rows', 1)
elif event is 'empty': elif event is 'empty':
self._metrics_logger.log(success=False) MetricsDataGatherer.add('success', True)
MetricsDataGatherer.add('successful_rows', 1)
MetricsDataGatherer.add('empty_rows', 1)
elif event is 'fail':
MetricsDataGatherer.add('success', False)
MetricsDataGatherer.add('failed_rows', 1)
class QuotaChecker: class QuotaChecker:

2
server/lib/python/cartodb_services/setup.py
View File

@ -10,7 +10,7 @@ from setuptools import setup, find_packages
setup( setup(
name='cartodb_services', name='cartodb_services',
version='0.9.4', version='0.10.0',
description='CartoDB Services API Python Library', description='CartoDB Services API Python Library',

5
server/lib/python/cartodb_services/test/__init__.py
View File

@ -0,0 +1,5 @@
from test_helper import plpy_mock_config
def setup():
plpy_mock_config()

5
server/lib/python/cartodb_services/test/metrics/test_config.py
View File

@ -1,13 +1,14 @@
from unittest import TestCase from unittest import TestCase
from mockredis import MockRedis from mockredis import MockRedis
from ..test_helper import *
from cartodb_services.metrics.config import RoutingConfig, ServicesRedisConfig from cartodb_services.metrics.config import RoutingConfig, ServicesRedisConfig
from ..test_helper import build_plpy_mock
class TestRoutingConfig(TestCase): class TestRoutingConfig(TestCase):
def setUp(self): def setUp(self):
self._redis_conn = MockRedis() self._redis_conn = MockRedis()
self._db_conn = build_plpy_mock() self._db_conn = plpy_mock
self._username = 'my_test_user' self._username = 'my_test_user'
self._user_key = "rails:users:{0}".format(self._username) self._user_key = "rails:users:{0}".format(self._username)
self._redis_conn.hset(self._user_key, 'period_end_date', '2016-10-10') self._redis_conn.hset(self._user_key, 'period_end_date', '2016-10-10')

2
server/lib/python/cartodb_services/test/metrics/test_quota.py
View File

@ -1,6 +1,6 @@
from unittest import TestCase from unittest import TestCase
from mockredis import MockRedis from mockredis import MockRedis
from ..test_helper import build_plpy_mock from ..test_helper import *
from cartodb_services.metrics.quota import QuotaChecker from cartodb_services.metrics.quota import QuotaChecker
from cartodb_services.metrics import RoutingConfig from cartodb_services.metrics import RoutingConfig
from datetime import datetime from datetime import datetime

53
server/lib/python/cartodb_services/test/mock_plpy.py Normal file
View File

@ -0,0 +1,53 @@
import re
class MockCursor:
def __init__(self, data):
self.cursor_pos = 0
self.data = data
def fetch(self, batch_size):
batch = self.data[self.cursor_pos: self.cursor_pos + batch_size]
self.cursor_pos += batch_size
return batch
class MockPlPy:
def __init__(self):
self._reset()
def _reset(self):
self.infos = []
self.notices = []
self.debugs = []
self.logs = []
self.warnings = []
self.errors = []
self.fatals = []
self.executes = []
self.results = []
self.prepares = []
self.results = {}
def _define_result(self, query, result):
pattern = re.compile(query, re.IGNORECASE | re.MULTILINE)
self.results[pattern] = result
def notice(self, msg):
self.notices.append(msg)
def debug(self, msg):
self.notices.append(msg)
def info(self, msg):
self.infos.append(msg)
def cursor(self, query):
data = self.execute(query)
return MockCursor(data)
def execute(self, query, rows=1):
for pattern, result in self.results.iteritems():
if pattern.search(query):
return result
return []

36
server/lib/python/cartodb_services/test/test_config.py
View File

@ -1,20 +1,20 @@
import test_helper from test_helper import *
from cartodb_services.metrics import GeocoderConfig, ObservatorySnapshotConfig, ConfigException
from unittest import TestCase from unittest import TestCase
from nose.tools import assert_raises from nose.tools import assert_raises
from mockredis import MockRedis from mockredis import MockRedis
from datetime import datetime, timedelta from datetime import datetime, timedelta
from cartodb_services.metrics import GeocoderConfig, ObservatorySnapshotConfig, ConfigException
class TestConfig(TestCase): class TestConfig(TestCase):
def setUp(self): def setUp(self):
self.redis_conn = MockRedis() self.redis_conn = MockRedis()
self.plpy_mock = test_helper.build_plpy_mock() plpy_mock_config()
def test_should_return_list_of_nokia_geocoder_config_if_its_ok(self): def test_should_return_list_of_nokia_geocoder_config_if_its_ok(self):
test_helper.build_redis_user_config(self.redis_conn, 'test_user') build_redis_user_config(self.redis_conn, 'test_user')
geocoder_config = GeocoderConfig(self.redis_conn, self.plpy_mock, geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock,
'test_user', None) 'test_user', None)
assert geocoder_config.heremaps_geocoder is True assert geocoder_config.heremaps_geocoder is True
assert geocoder_config.geocoding_quota == 100 assert geocoder_config.geocoding_quota == 100
@ -22,10 +22,10 @@ class TestConfig(TestCase):
def test_should_return_list_of_nokia_geocoder_config_ok_for_org(self): def test_should_return_list_of_nokia_geocoder_config_ok_for_org(self):
yesterday = datetime.today() - timedelta(days=1) yesterday = datetime.today() - timedelta(days=1)
test_helper.build_redis_user_config(self.redis_conn, 'test_user') build_redis_user_config(self.redis_conn, 'test_user')
test_helper.build_redis_org_config(self.redis_conn, 'test_org', build_redis_org_config(self.redis_conn, 'test_org',
quota=200, end_date=yesterday) quota=200, end_date=yesterday)
geocoder_config = GeocoderConfig(self.redis_conn, self.plpy_mock, geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock,
'test_user', 'test_org') 'test_user', 'test_org')
assert geocoder_config.heremaps_geocoder is True assert geocoder_config.heremaps_geocoder is True
assert geocoder_config.geocoding_quota == 200 assert geocoder_config.geocoding_quota == 200
@ -34,10 +34,10 @@ class TestConfig(TestCase):
def test_should_return_config_for_obs_snapshot(self): def test_should_return_config_for_obs_snapshot(self):
yesterday = datetime.today() - timedelta(days=1) yesterday = datetime.today() - timedelta(days=1)
test_helper.build_redis_user_config(self.redis_conn, 'test_user', build_redis_user_config(self.redis_conn, 'test_user',
do_quota=100, soft_do_limit=True, do_quota=100, soft_do_limit=True,
end_date=yesterday) end_date=yesterday)
do_config = ObservatorySnapshotConfig(self.redis_conn, self.plpy_mock, do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock,
'test_user') 'test_user')
assert do_config.monthly_quota == 100 assert do_config.monthly_quota == 100
assert do_config.soft_limit is True assert do_config.soft_limit is True
@ -45,16 +45,16 @@ class TestConfig(TestCase):
def test_should_return_db_quota_if_not_redis_quota_config_obs_snapshot(self): def test_should_return_db_quota_if_not_redis_quota_config_obs_snapshot(self):
yesterday = datetime.today() - timedelta(days=1) yesterday = datetime.today() - timedelta(days=1)
test_helper.build_redis_user_config(self.redis_conn, 'test_user', build_redis_user_config(self.redis_conn, 'test_user',
end_date=yesterday) end_date=yesterday)
do_config = ObservatorySnapshotConfig(self.redis_conn, self.plpy_mock, do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock,
'test_user') 'test_user')
assert do_config.monthly_quota == 0 assert do_config.monthly_quota == 0
assert do_config.soft_limit is False assert do_config.soft_limit is False
assert do_config.period_end_date.date() == yesterday.date() assert do_config.period_end_date.date() == yesterday.date()
def test_should_raise_exception_when_missing_parameters(self): def test_should_raise_exception_when_missing_parameters(self):
plpy_mock = test_helper.build_plpy_mock(empty=True) plpy_mock._reset()
test_helper.build_redis_user_config(self.redis_conn, 'test_user') build_redis_user_config(self.redis_conn, 'test_user')
assert_raises(ConfigException, GeocoderConfig, self.redis_conn, assert_raises(ConfigException, GeocoderConfig, self.redis_conn,
plpy_mock, 'test_user', None) plpy_mock, 'test_user', None)

34
server/lib/python/cartodb_services/test/test_helper.py
View File

@ -1,7 +1,11 @@
from datetime import datetime, date from datetime import datetime, date
from mock import Mock from mock import Mock, MagicMock
import random
import sys import sys
sys.modules['plpy'] = Mock() from mock_plpy import MockPlPy
plpy_mock = MockPlPy()
sys.modules['plpy'] = plpy_mock
def build_redis_user_config(redis_conn, username, quota=100, soft_limit=False, def build_redis_user_config(redis_conn, username, quota=100, soft_limit=False,
@ -57,22 +61,10 @@ def increment_service_uses(redis_conn, username, orgname=None,
redis_conn.zincrby(redis_name, date.day, amount) redis_conn.zincrby(redis_name, date.day, amount)
def build_plpy_mock(empty=False): def plpy_mock_config():
plpy_mock = Mock() plpy_mock._define_result("CDB_Conf_GetConf\('heremaps_conf'\)", [{'conf': '{"geocoder": {"app_id": "app_id", "app_code": "code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "app_id", "app_code": "code"}}'}])
if not empty: plpy_mock._define_result("CDB_Conf_GetConf\('mapzen_conf'\)", [{'conf': '{"routing": {"api_key": "api_key_rou", "monthly_quota": 1500000}, "geocoder": {"api_key": "api_key_geo", "monthly_quota": 1500000}, "matrix": {"api_key": "api_key_mat", "monthly_quota": 1500000}}'}])
plpy_mock.execute.side_effect = _plpy_execute_side_effect plpy_mock._define_result("CDB_Conf_GetConf\('logger_conf'\)", [{'conf': '{"geocoder_log_path": "/dev/null"}'}])
plpy_mock._define_result("CDB_Conf_GetConf\('data_observatory_conf'\)", [{'conf': '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}}'}])
return plpy_mock plpy_mock._define_result("CDB_Conf_GetConf\('server_conf'\)", [{'conf': '{"environment": "testing"}'}])
plpy_mock._define_result("select txid_current", [{'txid': random.randint(0, 1000)}])
def _plpy_execute_side_effect(*args, **kwargs):
if args[0] == "SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as conf":
return [{'conf': '{"geocoder": {"app_id": "app_id", "app_code": "code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "app_id", "app_code": "code"}}'}]
elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('mapzen_conf') as conf":
return [{'conf': '{"routing": {"api_key": "api_key_rou", "monthly_quota": 1500000}, "geocoder": {"api_key": "api_key_geo", "monthly_quota": 1500000}, "matrix": {"api_key": "api_key_mat", "monthly_quota": 1500000}}'}]
elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('logger_conf') as conf":
return [{'conf': '{"geocoder_log_path": "/dev/null"}'}]
elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('data_observatory_conf') as conf":
return [{'conf': '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}}'}]
elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('server_conf') as conf":
return [{'conf': '{"environment": "testing"}'}]

23
server/lib/python/cartodb_services/test/test_metrics_log.py Normal file
View File

@ -0,0 +1,23 @@
from test_helper import *
from cartodb_services.metrics import MetricsDataGatherer
from unittest import TestCase
from mock import Mock, MagicMock
from nose.tools import assert_raises
from datetime import datetime, date
class TestMetricsDataGatherer(TestCase):
def setUp(self):
plpy_mock_config()
def test_should_use_multiple_instances_for_multiples_requests(self):
plpy_mock._define_result("select txid_current", [{'txid': 100}])
MetricsDataGatherer.add('test', 1)
plpy_mock._define_result("select txid_current", [{'txid': 101}])
MetricsDataGatherer.add('test', 2)
plpy_mock._define_result("select txid_current", [{'txid': 100}])
assert MetricsDataGatherer.get_element('test') is 1
plpy_mock._define_result("select txid_current", [{'txid': 101}])
assert MetricsDataGatherer.get_element('test') is 2

1
server/lib/python/cartodb_services/test/test_qps.py
View File

@ -1,4 +1,3 @@
import test_helper
import requests import requests
from unittest import TestCase from unittest import TestCase
from nose.tools import assert_raises from nose.tools import assert_raises

27
server/lib/python/cartodb_services/test/test_quota_service.py
View File

@ -1,4 +1,4 @@
import test_helper from test_helper import *
from mockredis import MockRedis from mockredis import MockRedis
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import GeocoderConfig, RoutingConfig, ObservatorySnapshotConfig, IsolinesRoutingConfig from cartodb_services.metrics import GeocoderConfig, RoutingConfig, ObservatorySnapshotConfig, IsolinesRoutingConfig
@ -28,32 +28,32 @@ class TestQuotaService(TestCase):
def test_should_return_true_if_user_quota_is_not_completely_used(self): def test_should_return_true_if_user_quota_is_not_completely_used(self):
qs = self.__build_geocoder_quota_service('test_user') qs = self.__build_geocoder_quota_service('test_user')
test_helper.increment_service_uses(self.redis_conn, 'test_user') increment_service_uses(self.redis_conn, 'test_user')
assert qs.check_user_quota() is True assert qs.check_user_quota() is True
def test_should_return_true_if_org_quota_is_not_completely_used(self): def test_should_return_true_if_org_quota_is_not_completely_used(self):
qs = self.__build_geocoder_quota_service('test_user', qs = self.__build_geocoder_quota_service('test_user',
orgname='test_org') orgname='test_org')
test_helper.increment_service_uses(self.redis_conn, 'test_user', increment_service_uses(self.redis_conn, 'test_user',
orgname='test_org') orgname='test_org')
assert qs.check_user_quota() is True assert qs.check_user_quota() is True
def test_should_return_false_if_user_quota_is_surpassed(self): def test_should_return_false_if_user_quota_is_surpassed(self):
qs = self.__build_geocoder_quota_service('test_user') qs = self.__build_geocoder_quota_service('test_user')
test_helper.increment_service_uses(self.redis_conn, 'test_user', increment_service_uses(self.redis_conn, 'test_user',
amount=300) amount=300)
assert qs.check_user_quota() is False assert qs.check_user_quota() is False
def test_should_return_false_if_org_quota_is_surpassed(self): def test_should_return_false_if_org_quota_is_surpassed(self):
qs = self.__build_geocoder_quota_service('test_user', qs = self.__build_geocoder_quota_service('test_user',
orgname='test_org') orgname='test_org')
test_helper.increment_service_uses(self.redis_conn, 'test_user', increment_service_uses(self.redis_conn, 'test_user',
orgname='test_org', amount=400) orgname='test_org', amount=400)
assert qs.check_user_quota() is False assert qs.check_user_quota() is False
def test_should_return_true_if_user_quota_is_surpassed_but_soft_limit_is_enabled(self): def test_should_return_true_if_user_quota_is_surpassed_but_soft_limit_is_enabled(self):
qs = self.__build_geocoder_quota_service('test_user', soft_limit=True) qs = self.__build_geocoder_quota_service('test_user', soft_limit=True)
test_helper.increment_service_uses(self.redis_conn, 'test_user', increment_service_uses(self.redis_conn, 'test_user',
amount=300) amount=300)
assert qs.check_user_quota() is True assert qs.check_user_quota() is True
@ -61,7 +61,7 @@ class TestQuotaService(TestCase):
qs = self.__build_geocoder_quota_service('test_user', qs = self.__build_geocoder_quota_service('test_user',
orgname='test_org', orgname='test_org',
soft_limit=True) soft_limit=True)
test_helper.increment_service_uses(self.redis_conn, 'test_user', increment_service_uses(self.redis_conn, 'test_user',
orgname='test_org', amount=400) orgname='test_org', amount=400)
assert qs.check_user_quota() is True assert qs.check_user_quota() is True
@ -140,18 +140,17 @@ class TestQuotaService(TestCase):
def __prepare_quota_service(self, username, quota, service, orgname, def __prepare_quota_service(self, username, quota, service, orgname,
soft_limit, do_quota, soft_do_limit, end_date): soft_limit, do_quota, soft_do_limit, end_date):
test_helper.build_redis_user_config(self.redis_conn, username, build_redis_user_config(self.redis_conn, username,
quota=quota, service=service, quota=quota, service=service,
soft_limit=soft_limit, soft_limit=soft_limit,
soft_do_limit=soft_do_limit, soft_do_limit=soft_do_limit,
do_quota=do_quota, do_quota=do_quota,
end_date=end_date) end_date=end_date)
if orgname: if orgname:
test_helper.build_redis_org_config(self.redis_conn, orgname, build_redis_org_config(self.redis_conn, orgname,
quota=quota, service=service, quota=quota, service=service,
do_quota=do_quota, do_quota=do_quota,
end_date=end_date) end_date=end_date)
self._plpy_mock = test_helper.build_plpy_mock()
def __build_geocoder_quota_service(self, username, quota=100, def __build_geocoder_quota_service(self, username, quota=100,
service='heremaps', orgname=None, service='heremaps', orgname=None,
@ -159,7 +158,7 @@ class TestQuotaService(TestCase):
end_date=datetime.today()): end_date=datetime.today()):
self.__prepare_quota_service(username, quota, service, orgname, self.__prepare_quota_service(username, quota, service, orgname,
soft_limit, 0, False, end_date) soft_limit, 0, False, end_date)
geocoder_config = GeocoderConfig(self.redis_conn, self._plpy_mock, geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock,
username, orgname) username, orgname)
return QuotaService(geocoder_config, redis_connection=self.redis_conn) return QuotaService(geocoder_config, redis_connection=self.redis_conn)
@ -168,7 +167,7 @@ class TestQuotaService(TestCase):
quota=100, end_date=datetime.today()): quota=100, end_date=datetime.today()):
self.__prepare_quota_service(username, quota, service, orgname, self.__prepare_quota_service(username, quota, service, orgname,
soft_limit, 0, False, end_date) soft_limit, 0, False, end_date)
routing_config = RoutingConfig(self.redis_conn, self._plpy_mock, routing_config = RoutingConfig(self.redis_conn, plpy_mock,
username, orgname) username, orgname)
return QuotaService(routing_config, redis_connection=self.redis_conn) return QuotaService(routing_config, redis_connection=self.redis_conn)
@ -177,7 +176,7 @@ class TestQuotaService(TestCase):
quota=100, end_date=datetime.today()): quota=100, end_date=datetime.today()):
self.__prepare_quota_service(username, quota, service, orgname, self.__prepare_quota_service(username, quota, service, orgname,
soft_limit, 0, False, end_date) soft_limit, 0, False, end_date)
isolines_config = IsolinesRoutingConfig(self.redis_conn, self._plpy_mock, isolines_config = IsolinesRoutingConfig(self.redis_conn, plpy_mock,
username, orgname) username, orgname)
return QuotaService(isolines_config, redis_connection=self.redis_conn) return QuotaService(isolines_config, redis_connection=self.redis_conn)
@ -188,6 +187,6 @@ class TestQuotaService(TestCase):
end_date=datetime.today()): end_date=datetime.today()):
self.__prepare_quota_service(username, 0, service, orgname, False, self.__prepare_quota_service(username, 0, service, orgname, False,
quota, soft_limit, end_date) quota, soft_limit, end_date)
do_config = ObservatorySnapshotConfig(self.redis_conn, self._plpy_mock, do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock,
username, orgname) username, orgname)
return QuotaService(do_config, redis_connection=self.redis_conn) return QuotaService(do_config, redis_connection=self.redis_conn)

11
server/lib/python/cartodb_services/test/test_user_service.py
View File

@ -1,4 +1,4 @@
import test_helper from test_helper import *
from mockredis import MockRedis from mockredis import MockRedis
from cartodb_services.metrics import UserMetricsService from cartodb_services.metrics import UserMetricsService
from cartodb_services.metrics import GeocoderConfig from cartodb_services.metrics import GeocoderConfig
@ -19,13 +19,13 @@ class TestUserService(TestCase):
def test_user_used_quota_for_a_day(self): def test_user_used_quota_for_a_day(self):
us = self.__build_user_service('test_user') us = self.__build_user_service('test_user')
test_helper.increment_service_uses(self.redis_conn, 'test_user', increment_service_uses(self.redis_conn, 'test_user',
amount=400) amount=400)
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 400 assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 400
def test_org_used_quota_for_a_day(self): def test_org_used_quota_for_a_day(self):
us = self.__build_user_service('test_user', orgname='test_org') us = self.__build_user_service('test_user', orgname='test_org')
test_helper.increment_service_uses(self.redis_conn, 'test_user', increment_service_uses(self.redis_conn, 'test_user',
orgname='test_org', orgname='test_org',
amount=400) amount=400)
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 400 assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 400
@ -135,14 +135,13 @@ class TestUserService(TestCase):
def __build_user_service(self, username, quota=100, service='heremaps', def __build_user_service(self, username, quota=100, service='heremaps',
orgname=None, soft_limit=False, orgname=None, soft_limit=False,
end_date=date.today()): end_date=date.today()):
test_helper.build_redis_user_config(self.redis_conn, username, build_redis_user_config(self.redis_conn, username,
quota=quota, service=service, quota=quota, service=service,
soft_limit=soft_limit, soft_limit=soft_limit,
end_date=end_date) end_date=end_date)
if orgname: if orgname:
test_helper.build_redis_org_config(self.redis_conn, orgname, build_redis_org_config(self.redis_conn, orgname,
quota=quota, end_date=end_date) quota=quota, end_date=end_date)
plpy_mock = test_helper.build_plpy_mock()
geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock, geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock,
username, orgname) username, orgname)
return UserMetricsService(geocoder_config, self.redis_conn) return UserMetricsService(geocoder_config, self.redis_conn)

23
test/helpers/import_helper.py
View File

@ -9,11 +9,11 @@ import time
class ImportHelper: class ImportHelper:
@classmethod @classmethod
def import_test_dataset(cls, username, api_key, host): def import_test_dataset(cls, username, api_key, host, schema):
requests.packages.urllib3.disable_warnings() requests.packages.urllib3.disable_warnings()
url = "https://{0}.{1}/api/v1/imports/"\ url = "{0}://{1}.{2}/api/v1/imports/"\
"?type_guessing=false&privacy=public&api_key={2}".format( "?type_guessing=false&privacy=public&api_key={3}".format(
username, host, api_key) schema, username, host, api_key)
dataset = { dataset = {
'file': open('fixtures/geocoder_api_test_dataset.csv', 'rb')} 'file': open('fixtures/geocoder_api_test_dataset.csv', 'rb')}
response = requests.post(url, files=dataset) response = requests.post(url, files=dataset)
@ -27,7 +27,8 @@ class ImportHelper:
username, username,
host, host,
api_key, api_key,
response_json['item_queue_id'] response_json['item_queue_id'],
schema
) )
if table_name: if table_name:
return table_name return table_name
@ -35,10 +36,10 @@ class ImportHelper:
time.sleep(5) time.sleep(5)
@classmethod @classmethod
def get_imported_table_name(cls, username, host, api_key, import_id): def get_imported_table_name(cls, username, host, api_key, import_id, schema):
requests.packages.urllib3.disable_warnings() requests.packages.urllib3.disable_warnings()
import_url = "https://{0}.{1}/api/v1/imports/{2}?api_key={3}".format( import_url = "{0}://{1}.{2}/api/v1/imports/{3}?api_key={4}".format(
username, host, import_id, api_key) schema, username, host, import_id, api_key)
import_data_response = requests.get(import_url) import_data_response = requests.get(import_url)
if import_data_response.status_code != 200: if import_data_response.status_code != 200:
print "Error getting the table name from " \ print "Error getting the table name from " \
@ -49,10 +50,10 @@ class ImportHelper:
return import_data_json['table_name'] return import_data_json['table_name']
@classmethod @classmethod
def clean_test_dataset(cls, username, api_key, table_name, host): def clean_test_dataset(cls, username, api_key, table_name, host, schema):
requests.packages.urllib3.disable_warnings() requests.packages.urllib3.disable_warnings()
url = "https://{0}.{1}/api/v2/sql?q=drop table {2}&api_key={3}".format( url = "{0}://{1}.{2}/api/v2/sql?q=drop table {3}&api_key={4}".format(
username, host, table_name, api_key schema, username, host, table_name, api_key
) )
response = requests.get(url) response = requests.get(url)
if response.status_code != 200: if response.status_code != 200:

2
test/helpers/integration_test_helper.py
View File

@ -10,11 +10,13 @@ class IntegrationTestHelper:
username = os.environ["GEOCODER_API_TEST_USERNAME"] username = os.environ["GEOCODER_API_TEST_USERNAME"]
api_key = os.environ["GEOCODER_API_TEST_API_KEY"] api_key = os.environ["GEOCODER_API_TEST_API_KEY"]
host = os.environ["GEOCODER_API_TEST_HOST"] host = os.environ["GEOCODER_API_TEST_HOST"]
schema = os.environ["GEOCODER_API_TEST_SCHEMA"]
table_name = os.environ["GEOCODER_API_TEST_TABLE_NAME"] table_name = os.environ["GEOCODER_API_TEST_TABLE_NAME"]
return { return {
"username": username, "username": username,
"api_key": api_key, "api_key": api_key,
"schema": schema,
"host": host, "host": host,
"table_name": table_name "table_name": table_name
} }

3
test/integration/test_admin0_functions.py
View File

@ -8,7 +8,8 @@ class TestAdmin0Functions(TestCase):
def setUp(self): def setUp(self):
self.env_variables = IntegrationTestHelper.get_environment_variables() self.env_variables = IntegrationTestHelper.get_environment_variables()
self.sql_api_url = "https://{0}.{1}/api/v1/sql".format( self.sql_api_url = "{0}://{1}.{2}/api/v1/sql".format(
self.env_variables['schema'],
self.env_variables['username'], self.env_variables['username'],
self.env_variables['host'], self.env_variables['host'],
self.env_variables['api_key'] self.env_variables['api_key']

3
test/integration/test_admin1_functions.py
View File

@ -8,7 +8,8 @@ class TestAdmin1Functions(TestCase):
def setUp(self): def setUp(self):
self.env_variables = IntegrationTestHelper.get_environment_variables() self.env_variables = IntegrationTestHelper.get_environment_variables()
self.sql_api_url = "https://{0}.{1}/api/v1/sql".format( self.sql_api_url = "{0}://{1}.{2}/api/v1/sql".format(
self.env_variables['schema'],
self.env_variables['username'], self.env_variables['username'],
self.env_variables['host'], self.env_variables['host'],
self.env_variables['api_key'] self.env_variables['api_key']

3
test/integration/test_data_observatory_functions.py
View File

@ -8,7 +8,8 @@ class TestDataObservatoryFunctions(TestCase):
def setUp(self): def setUp(self):
self.env_variables = IntegrationTestHelper.get_environment_variables() self.env_variables = IntegrationTestHelper.get_environment_variables()
self.sql_api_url = "https://{0}.{1}/api/v1/sql".format( self.sql_api_url = "{0}://{1}.{2}/api/v1/sql".format(
self.env_variables['schema'],
self.env_variables['username'], self.env_variables['username'],
self.env_variables['host'], self.env_variables['host'],
self.env_variables['api_key'] self.env_variables['api_key']

3
test/integration/test_ipaddress_functions.py
View File

@ -8,7 +8,8 @@ class TestPostalcodeFunctions(TestCase):
def setUp(self): def setUp(self):
self.env_variables = IntegrationTestHelper.get_environment_variables() self.env_variables = IntegrationTestHelper.get_environment_variables()
self.sql_api_url = "https://{0}.{1}/api/v1/sql".format( self.sql_api_url = "{0}://{1}.{2}/api/v1/sql".format(
self.env_variables['schema'],
self.env_variables['username'], self.env_variables['username'],
self.env_variables['host'], self.env_variables['host'],
self.env_variables['api_key'] self.env_variables['api_key']

3
test/integration/test_isolines_functions.py
View File

@ -8,7 +8,8 @@ class TestIsolinesFunctions(TestCase):
def setUp(self): def setUp(self):
self.env_variables = IntegrationTestHelper.get_environment_variables() self.env_variables = IntegrationTestHelper.get_environment_variables()
self.sql_api_url = "https://{0}.{1}/api/v1/sql".format( self.sql_api_url = "{0}://{1}.{2}/api/v1/sql".format(
self.env_variables['schema'],
self.env_variables['username'], self.env_variables['username'],
self.env_variables['host'], self.env_variables['host'],
self.env_variables['api_key'] self.env_variables['api_key']

3
test/integration/test_namedplace_functions.py
View File

@ -8,7 +8,8 @@ class TestNameplaceFunctions(TestCase):
def setUp(self): def setUp(self):
self.env_variables = IntegrationTestHelper.get_environment_variables() self.env_variables = IntegrationTestHelper.get_environment_variables()
self.sql_api_url = "https://{0}.{1}/api/v1/sql".format( self.sql_api_url = "{0}://{1}.{2}/api/v1/sql".format(
self.env_variables['schema'],
self.env_variables['username'], self.env_variables['username'],
self.env_variables['host'], self.env_variables['host'],
self.env_variables['api_key'] self.env_variables['api_key']

3
test/integration/test_postalcode_functions.py
View File

@ -8,7 +8,8 @@ class TestPostalcodeFunctions(TestCase):
def setUp(self): def setUp(self):
self.env_variables = IntegrationTestHelper.get_environment_variables() self.env_variables = IntegrationTestHelper.get_environment_variables()
self.sql_api_url = "https://{0}.{1}/api/v1/sql".format( self.sql_api_url = "{0}://{1}.{2}/api/v1/sql".format(
self.env_variables['schema'],
self.env_variables['username'], self.env_variables['username'],
self.env_variables['host'], self.env_variables['host'],
self.env_variables['api_key'] self.env_variables['api_key']

3
test/integration/test_routing_functions.py
View File

@ -8,7 +8,8 @@ class TestRoutingFunctions(TestCase):
def setUp(self): def setUp(self):
self.env_variables = IntegrationTestHelper.get_environment_variables() self.env_variables = IntegrationTestHelper.get_environment_variables()
self.sql_api_url = "https://{0}.{1}/api/v1/sql".format( self.sql_api_url = "{0}://{1}.{2}/api/v1/sql".format(
self.env_variables['schema'],
self.env_variables['username'], self.env_variables['username'],
self.env_variables['host'], self.env_variables['host'],
self.env_variables['api_key'] self.env_variables['api_key']

7
test/integration/test_street_functions.py
View File

@ -8,14 +8,15 @@ class TestStreetFunctions(TestCase):
def setUp(self): def setUp(self):
self.env_variables = IntegrationTestHelper.get_environment_variables() self.env_variables = IntegrationTestHelper.get_environment_variables()
self.sql_api_url = "https://{0}.{1}/api/v1/sql".format( self.sql_api_url = "{0}://{1}.{2}/api/v1/sql".format(
self.env_variables['schema'],
self.env_variables['username'], self.env_variables['username'],
self.env_variables['host'], self.env_variables['host'],
self.env_variables['api_key'] self.env_variables['api_key']
) )
def test_if_select_with_street_point_is_ok(self): def test_if_select_with_here_street_point_is_ok(self):
query = "SELECT cdb_geocode_street_point(street) " \ query = "SELECT cdb_here_geocode_street_point(street) " \
"as geometry FROM {0} LIMIT 1&api_key={1}".format( "as geometry FROM {0} LIMIT 1&api_key={1}".format(
self.env_variables['table_name'], self.env_variables['table_name'],
self.env_variables['api_key']) self.env_variables['api_key'])

20
test/run_tests.py
View File

@ -8,12 +8,13 @@ from helpers.import_helper import ImportHelper
def main(): def main():
opts, args = getopt.getopt(sys.argv[1:], "h", ["help", "host="]) opts, args = getopt.getopt(sys.argv[1:], "h", ["help", "host=", "schema="])
if len(args) < 2: if len(args) < 2:
usage() usage()
sys.exit() sys.exit()
schema = "https"
host = "cartodb.com" host = "cartodb.com"
username = args[0] username = args[0]
api_key = args[1] api_key = args[1]
@ -24,26 +25,31 @@ def main():
sys.exit() sys.exit()
elif o in ("--host"): elif o in ("--host"):
host = opts[0][1] host = opts[0][1]
elif o in ("--schema"):
schema = opts[1][1]
else: else:
assert False, "unhandled option" assert False, "unhandled option"
try: try:
table_name = ImportHelper.import_test_dataset(username, api_key, host) table_name = ImportHelper.import_test_dataset(username, api_key, host,
set_environment_variables(username, api_key, table_name, host) schema)
set_environment_variables(username, api_key, table_name, host, schema)
execute_tests() execute_tests()
except Exception as e: except Exception as e:
print e.message print e.message
sys.exit(1) sys.exit(1)
finally: finally:
clean_environment_variables() clean_environment_variables()
ImportHelper.clean_test_dataset(username, api_key, table_name, host) ImportHelper.clean_test_dataset(username, api_key, table_name, host,
schema)
def usage(): def usage():
print """Usage: run_tests.py [options] username api_key print """Usage: run_tests.py [options] username api_key
Options: Options:
-h: Show this help -h: Show this help
--host: take that host as base (by default is cartodb.com)""" --host: take that host as base (by default is cartodb.com)
--schema: define the url schema [http/https] (by default https)"""
def execute_tests(): def execute_tests():
@ -59,11 +65,12 @@ def execute_tests():
sys.exit(1) sys.exit(1)
def set_environment_variables(username, api_key, table_name, host): def set_environment_variables(username, api_key, table_name, host, schema):
os.environ["GEOCODER_API_TEST_USERNAME"] = username os.environ["GEOCODER_API_TEST_USERNAME"] = username
os.environ["GEOCODER_API_TEST_API_KEY"] = api_key os.environ["GEOCODER_API_TEST_API_KEY"] = api_key
os.environ["GEOCODER_API_TEST_TABLE_NAME"] = table_name os.environ["GEOCODER_API_TEST_TABLE_NAME"] = table_name
os.environ["GEOCODER_API_TEST_HOST"] = host os.environ["GEOCODER_API_TEST_HOST"] = host
os.environ["GEOCODER_API_TEST_SCHEMA"] = schema
def clean_environment_variables(): def clean_environment_variables():
@ -71,6 +78,7 @@ def clean_environment_variables():
del os.environ["GEOCODER_API_TEST_API_KEY"] del os.environ["GEOCODER_API_TEST_API_KEY"]
del os.environ["GEOCODER_API_TEST_TABLE_NAME"] del os.environ["GEOCODER_API_TEST_TABLE_NAME"]
del os.environ["GEOCODER_API_TEST_HOST"] del os.environ["GEOCODER_API_TEST_HOST"]
del os.environ["GEOCODER_API_TEST_SCHEMA"]
if __name__ == "__main__": if __name__ == "__main__":
main() main()