cartodb4/dataservices-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #299 from CartoDB/add_metrics

Browse Source 
Data services metrics enhanced
This commit is contained in:
Mario de Frutos 2016-11-04 13:40:31 +01:00 committed by GitHub
commit a4858bbf3c
38 changed files with 1070 additions and 706 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
server/extension/cdb_dataservices_server--0.16.0--0.17.0.sql Normal file
View File

0
server/extension/cdb_dataservices_server--0.17.0--0.16.0.sql Normal file
View File

2
server/extension/cdb_dataservices_server.control
View File

@ -1,5 +1,5 @@
comment = 'CartoDB dataservices server extension'
default_version = '0.16.0'
default_version = '0.17.0'
requires = 'plpythonu, plproxy, postgis, cdb_geocoder'
superuser = true
schema = cdb_dataservices_server

0
server/extension/cdb_dataservices_server--0.15.1--0.16.0.sql → server/extension/old_versions/cdb_dataservices_server--0.15.1--0.16.0.sql
View File

0
server/extension/cdb_dataservices_server--0.16.0--0.15.1.sql → server/extension/old_versions/cdb_dataservices_server--0.16.0--0.15.1.sql
View File

0
server/extension/cdb_dataservices_server--0.16.0.sql → server/extension/old_versions/cdb_dataservices_server--0.16.0.sql
View File

26
server/extension/sql/105_route_between_points.sql
View File

@ -7,15 +7,21 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point(
options text[] DEFAULT ARRAY[]::text[],
units text DEFAULT 'kilometers')
RETURNS cdb_dataservices_server.simple_route AS $$
from cartodb_services.metrics import metrics
from cartodb_services.tools import Logger
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_routing_config = GD["user_routing_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
waypoints = [origin, destination]
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"])
result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units])
return [result[0]['shape'],result[0]['length'], result[0]['duration']]
with metrics('cdb_route_with_point', user_routing_config, logger):
waypoints = [origin, destination]
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"])
result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units])
return [result[0]['shape'],result[0]['length'], result[0]['duration']]
$$ LANGUAGE plpythonu;
@ -27,12 +33,18 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints(
options text[] DEFAULT ARRAY[]::text[],
units text DEFAULT 'kilometers')
RETURNS cdb_dataservices_server.simple_route AS $$
from cartodb_services.metrics import metrics
from cartodb_services.tools import Logger
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_routing_config = GD["user_routing_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"])
result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units])
return [result[0]['shape'],result[0]['length'], result[0]['duration']]
with metrics('cdb_route_with_waypoints', user_routing_config, logger):
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"])
result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units])
return [result[0]['shape'],result[0]['length'], result[0]['duration']]
$$ LANGUAGE plpythonu;

376
server/extension/sql/110_data_observatory_augmentation.sql
View File

@ -11,9 +11,9 @@ RETURNS text AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]
user_obs_config = GD["user_obs_config_{0}".format(username)]
return user_obs_snapshot_config.connection_str
return user_obs_config.connection_str
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshotJSON(
@ -34,6 +34,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot(
time_span TEXT DEFAULT NULL,
geometry_level TEXT DEFAULT NULL)
RETURNS json AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
import json
@ -41,31 +42,32 @@ RETURNS json AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_snapshot_config, redis_conn)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshotJSON($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level])
if result:
quota_service.increment_success_service_use()
return result[0]['snapshot']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to obs_get_demographic_snapshot')
finally:
quota_service.increment_total_service_use()
with metrics('obs_getdemographicsnapshot', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshotJSON($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level])
if result:
quota_service.increment_success_service_use()
return result[0]['snapshot']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to obs_get_demographic_snapshot')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshot(
@ -86,41 +88,43 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetDemographicSnapshot(
time_span TEXT DEFAULT NULL,
geometry_level TEXT DEFAULT NULL)
RETURNS SETOF JSON AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_snapshot_config, redis_conn)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level])
if result:
resp = []
for element in result:
value = element['snapshot']
resp.append(value)
quota_service.increment_success_service_use()
return resp
else:
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to obs_get_demographic_snapshot')
finally:
quota_service.increment_total_service_use()
with metrics('obs_getdemographicsnapshot', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level])
if result:
resp = []
for element in result:
value = element['snapshot']
resp.append(value)
quota_service.increment_success_service_use()
return resp
else:
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to obs_get_demographic_snapshot')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshotJSON(
@ -139,6 +143,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot(
geom geometry(Geometry, 4326),
geometry_level TEXT DEFAULT NULL)
RETURNS json AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
import json
@ -146,31 +151,32 @@ RETURNS json AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_snapshot_config, redis_conn)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshotJSON($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level])
if result:
quota_service.increment_success_service_use()
return result[0]['snapshot']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to obs_get_segment_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to obs_get_segment_snapshot')
finally:
quota_service.increment_total_service_use()
with metrics('obs_getsegmentsnapshot', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshotJSON($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level])
if result:
quota_service.increment_success_service_use()
return result[0]['snapshot']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to obs_get_segment_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to obs_get_segment_snapshot')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshot(
@ -189,41 +195,43 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetSegmentSnapshot(
geom geometry(Geometry, 4326),
geometry_level TEXT DEFAULT NULL)
RETURNS SETOF JSON AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]
user_obs_config = GD["user_obs_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_snapshot_config, redis_conn)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level])
if result:
resp = []
for element in result:
value = element['snapshot']
resp.append(value)
quota_service.increment_success_service_use()
return resp
else:
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetSegmentSnapshot', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetSegmentSnapshot')
finally:
quota_service.increment_total_service_use()
with metrics('obs_getsegmentsnapshot', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level])
if result:
resp = []
for element in result:
value = element['snapshot']
resp.append(value)
quota_service.increment_success_service_use()
return resp
else:
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetSegmentSnapshot', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetSegmentSnapshot')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasure(
@ -248,6 +256,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasure(
boundary_id TEXT DEFAULT NULL,
time_span TEXT DEFAULT NULL)
RETURNS NUMERIC AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
@ -263,22 +272,23 @@ RETURNS NUMERIC AS $$
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasure($1, $2, $3, $4, $5, $6, $7) as measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, measure_id, normalize, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['measure']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetMeasure', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetMeasure')
finally:
quota_service.increment_total_service_use()
with metrics('obs_getmeasure', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasure($1, $2, $3, $4, $5, $6, $7) as measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, measure_id, normalize, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['measure']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetMeasure', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetMeasure')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetCategory(
@ -301,6 +311,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetCategory(
boundary_id TEXT DEFAULT NULL,
time_span TEXT DEFAULT NULL)
RETURNS TEXT AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
@ -316,22 +327,23 @@ RETURNS TEXT AS $$
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetCategory($1, $2, $3, $4, $5, $6) as category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, category_id, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['category']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetCategory', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetCategory')
finally:
quota_service.increment_total_service_use()
with metrics('obs_getcategory', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetCategory($1, $2, $3, $4, $5, $6) as category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, category_id, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['category']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetCategory', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetCategory')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusMeasure(
@ -356,6 +368,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusMeasure(
boundary_id TEXT DEFAULT NULL,
time_span TEXT DEFAULT NULL)
RETURNS NUMERIC AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
@ -371,22 +384,23 @@ RETURNS NUMERIC AS $$
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusMeasure($1, $2, $3, $4, $5, $6, $7) as census_measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, name, normalize, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['census_measure']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetUSCensusMeasure', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetUSCensusMeasure')
finally:
quota_service.increment_total_service_use()
with metrics('obs_getuscensusmeasure', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusMeasure($1, $2, $3, $4, $5, $6, $7) as census_measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, name, normalize, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['census_measure']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetUSCensusMeasure', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetUSCensusMeasure')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusCategory(
@ -409,6 +423,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusCategory(
boundary_id TEXT DEFAULT NULL,
time_span TEXT DEFAULT NULL)
RETURNS TEXT AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
@ -424,22 +439,23 @@ RETURNS TEXT AS $$
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusCategory($1, $2, $3, $4, $5, $6) as census_category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, name, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['census_category']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetUSCensusCategory', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetUSCensusCategory')
finally:
quota_service.increment_total_service_use()
with metrics('obs_getuscensuscategory', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusCategory($1, $2, $3, $4, $5, $6) as census_category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, name, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['census_category']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetUSCensusCategory', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetUSCensusCategory')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPopulation(
@ -462,6 +478,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPopulation(
boundary_id TEXT DEFAULT NULL,
time_span TEXT DEFAULT NULL)
RETURNS NUMERIC AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
@ -477,22 +494,23 @@ RETURNS NUMERIC AS $$
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetPopulation($1, $2, $3, $4, $5, $6) as population;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, normalize, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['population']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetPopulation', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetPopulation')
finally:
quota_service.increment_total_service_use()
with metrics('obs_getpopulation', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetPopulation($1, $2, $3, $4, $5, $6) as population;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, normalize, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['population']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetPopulation', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetPopulation')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasureById(
@ -515,6 +533,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasureById(
boundary_id TEXT,
time_span TEXT DEFAULT NULL)
RETURNS NUMERIC AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
@ -530,20 +549,21 @@ RETURNS NUMERIC AS $$
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasureById($1, $2, $3, $4, $5, $6) as measure;", ["text", "text", "text", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom_ref, measure_id, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['measure']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetMeasureById')
finally:
quota_service.increment_total_service_use()
with metrics('obs_getmeasurebyid', user_obs_config, logger):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasureById($1, $2, $3, $4, $5, $6) as measure;", ["text", "text", "text", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom_ref, measure_id, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['measure']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetMeasureById')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;

98
server/extension/sql/115_data_observatory_exploration.sql
View File

@ -14,6 +14,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_Search(
search_term TEXT,
relevant_boundary TEXT DEFAULT NULL)
RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
@ -29,30 +30,31 @@ RETURNS TABLE(id text, description text, name text, aggregate text, source text)
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_Search($1, $2, $3, $4);", ["text", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, search_term, relevant_boundary])
if result:
resp = []
for element in result:
id = element['id']
description = element['description']
name = element['name']
aggregate = element['aggregate']
source = element['source']
resp.append([id, description, name, aggregate, source])
quota_service.increment_success_service_use()
return resp
else:
quota_service.increment_empty_service_use()
return [None, None, None, None, None]
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_Search', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_Search')
finally:
quota_service.increment_total_service_use()
with metrics('obs_search', user_obs_snapshot_config, logger):
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_Search($1, $2, $3, $4);", ["text", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, search_term, relevant_boundary])
if result:
resp = []
for element in result:
id = element['id']
description = element['description']
name = element['name']
aggregate = element['aggregate']
source = element['source']
resp.append([id, description, name, aggregate, source])
quota_service.increment_success_service_use()
return resp
else:
quota_service.increment_empty_service_use()
return [None, None, None, None, None]
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_Search', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_Search')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetAvailableBoundaries(
@ -71,6 +73,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableBoundaries(
geom geometry(Geometry, 4326),
time_span TEXT DEFAULT NULL)
RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
@ -86,27 +89,28 @@ RETURNS TABLE(boundary_id text, description text, time_span text, tablename text
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetAvailableBoundaries($1, $2, $3, $4) as available_boundaries;", ["text", "text", "geometry(Geometry, 4326)", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, time_span])
if result:
resp = []
for element in result:
id = element['boundary_id']
description = element['description']
tspan = element['time_span']
tablename = element['tablename']
resp.append([id, description, tspan, tablename])
quota_service.increment_success_service_use()
return resp
else:
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetMeasureById')
finally:
quota_service.increment_total_service_use()
with metrics('obs_getavailableboundaries', user_obs_snapshot_config, logger):
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetAvailableBoundaries($1, $2, $3, $4) as available_boundaries;", ["text", "text", "geometry(Geometry, 4326)", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, time_span])
if result:
resp = []
for element in result:
id = element['boundary_id']
description = element['description']
tspan = element['time_span']
tablename = element['tablename']
resp.append([id, description, tspan, tablename])
quota_service.increment_success_service_use()
return resp
else:
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetMeasureById')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;

278
server/extension/sql/120_data_observatory_geometries.sql
View File

@ -16,6 +16,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundary(
boundary_id TEXT,
time_span TEXT DEFAULT NULL)
RETURNS geometry(Geometry, 4326) AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
@ -31,22 +32,23 @@ RETURNS geometry(Geometry, 4326) AS $$
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundary($1, $2, $3, $4) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['boundary']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetBoundary', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetBoundary')
finally:
quota_service.increment_total_service_use()
with metrics('obs_getboundary', user_obs_snapshot_config, logger):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundary($1, $2, $3, $4) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['boundary']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetBoundary', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetBoundary')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryId(
@ -67,6 +69,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryId(
boundary_id TEXT,
time_span TEXT DEFAULT NULL)
RETURNS TEXT AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
@ -82,22 +85,23 @@ RETURNS TEXT AS $$
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryId($1, $2, $3, $4, $5) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['boundary']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetBoundaryId', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetBoundaryId')
finally:
quota_service.increment_total_service_use()
with metrics('obs_getboundaryid', user_obs_snapshot_config, logger):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryId($1, $2, $3, $4, $5) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['boundary']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetBoundaryId', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetBoundaryId')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryById(
@ -118,6 +122,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryById(
boundary_id TEXT,
time_span TEXT DEFAULT NULL)
RETURNS geometry(Geometry, 4326) AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
@ -133,22 +138,23 @@ RETURNS geometry(Geometry, 4326) AS $$
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryById($1, $2, $3, $4, $5) as boundary;", ["text", "text", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geometry_id, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['boundary']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetBoundaryById', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetBoundaryById')
finally:
quota_service.increment_total_service_use()
with metrics('obs_getboundarybyid', user_obs_snapshot_config, logger):
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryById($1, $2, $3, $4, $5) as boundary;", ["text", "text", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geometry_id, boundary_id, time_span])
if result:
quota_service.increment_success_service_use()
return result[0]['boundary']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetBoundaryById', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetBoundaryById')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByGeometry(
@ -171,6 +177,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByGeometry(
time_span TEXT DEFAULT NULL,
overlap_type TEXT DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
@ -186,27 +193,28 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type])
if result:
resp = []
for element in result:
the_geom = element['the_geom']
geom_refs = element['geom_refs']
resp.append([the_geom, geom_refs])
quota_service.increment_success_service_use()
return resp
else:
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetBoundariesByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetBoundariesByGeometry')
finally:
quota_service.increment_total_service_use()
with metrics('obs_getboundariesbygeometry', user_obs_snapshot_config, logger):
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type])
if result:
resp = []
for element in result:
the_geom = element['the_geom']
geom_refs = element['geom_refs']
resp.append([the_geom, geom_refs])
quota_service.increment_success_service_use()
return resp
else:
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetBoundariesByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetBoundariesByGeometry')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius(
@ -231,6 +239,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByPointAndRa
time_span TEXT DEFAULT NULL,
overlap_type TEXT DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
@ -246,27 +255,28 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type])
if result:
resp = []
for element in result:
the_geom = element['the_geom']
geom_refs = element['geom_refs']
resp.append([the_geom, geom_refs])
quota_service.increment_success_service_use()
return resp
else:
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetBoundariesByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetBoundariesByPointAndRadius')
finally:
quota_service.increment_total_service_use()
with metrics('obs_getboundariesbypointandradius', user_obs_snapshot_config, logger):
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type])
if result:
resp = []
for element in result:
the_geom = element['the_geom']
geom_refs = element['geom_refs']
resp.append([the_geom, geom_refs])
quota_service.increment_success_service_use()
return resp
else:
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetBoundariesByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetBoundariesByPointAndRadius')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByGeometry(
@ -289,6 +299,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByGeometry(
time_span TEXT DEFAULT NULL,
overlap_type TEXT DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
@ -304,27 +315,28 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type])
if result:
resp = []
for element in result:
the_geom = element['the_geom']
geom_refs = element['geom_refs']
resp.append([the_geom, geom_refs])
quota_service.increment_success_service_use()
return resp
else:
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetPointsByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetPointsByGeometry')
finally:
quota_service.increment_total_service_use()
with metrics('obs_getpointsbygeometry', user_obs_snapshot_config, logger):
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type])
if result:
resp = []
for element in result:
the_geom = element['the_geom']
geom_refs = element['geom_refs']
resp.append([the_geom, geom_refs])
quota_service.increment_success_service_use()
return resp
else:
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetPointsByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetPointsByGeometry')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByPointAndRadius(
@ -349,6 +361,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByPointAndRadius
time_span TEXT DEFAULT NULL,
overlap_type TEXT DEFAULT NULL)
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
@ -364,25 +377,26 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type])
if result:
resp = []
for element in result:
the_geom = element['the_geom']
geom_refs = element['geom_refs']
resp.append([the_geom, geom_refs])
quota_service.increment_success_service_use()
return resp
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetPointsByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetPointsByPointAndRadius')
finally:
quota_service.increment_total_service_use()
with metrics('obs_getpointsbypointandradius', user_obs_snapshot_config, logger):
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"])
result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type])
if result:
resp = []
for element in result:
the_geom = element['the_geom']
geom_refs = element['geom_refs']
resp.append([the_geom, geom_refs])
quota_service.increment_success_service_use()
return resp
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to OBS_GetPointsByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to OBS_GetPointsByPointAndRadius')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;

28
server/extension/sql/20_geocode_street.sql
View File

@ -1,22 +1,28 @@
-- Geocodes a street address given a searchtext and a state and/or country
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
from cartodb_services.metrics import metrics
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
if user_geocoder_config.heremaps_geocoder:
here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
elif user_geocoder_config.google_geocoder:
google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
elif user_geocoder_config.mapzen_geocoder:
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
else:
raise Exception('Requested geocoder is not available')
with metrics('cdb_geocode_street_point', user_geocoder_config, logger):
if user_geocoder_config.heremaps_geocoder:
here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
elif user_geocoder_config.google_geocoder:
google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
elif user_geocoder_config.mapzen_geocoder:
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
else:
raise Exception('Requested geocoder is not available')
$$ LANGUAGE plpythonu;

36
server/extension/sql/30_admin0.sql
View File

@ -2,6 +2,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(us
RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.metrics import metrics
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
@ -13,23 +14,24 @@ RETURNS Geometry AS $$
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"])
rv = plpy.execute(plan, [country_name], 1)
result = rv[0]["mypolygon"]
if result:
quota_service.increment_success_service_use()
return result
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to geocode admin0 polygon', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode admin0 polygon')
finally:
quota_service.increment_total_service_use()
with metrics('cdb_geocode_admin0_polygon', user_geocoder_config, logger):
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"])
rv = plpy.execute(plan, [country_name], 1)
result = rv[0]["mypolygon"]
if result:
quota_service.increment_success_service_use()
return result
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to geocode admin0 polygon', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode admin0 polygon')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;

78
server/extension/sql/40_admin1.sql
View File

@ -1,6 +1,7 @@
---- cdb_geocode_admin1_polygon(admin1_name text)
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text)
RETURNS Geometry AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger,LoggerConfig
@ -14,44 +15,11 @@ RETURNS Geometry AS $$
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1)) AS mypolygon", ["text"])
rv = plpy.execute(plan, [admin1_name], 1)
result = rv[0]["mypolygon"]
if result:
quota_service.increment_success_service_use()
return result
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode admin1 polygon')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
---- cdb_geocode_admin1_polygon(admin1_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text)
RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger):
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1), trim($2)) AS mypolygon", ["text", "text"])
rv = plpy.execute(plan, [admin1_name, country_name], 1)
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1)) AS mypolygon", ["text"])
rv = plpy.execute(plan, [admin1_name], 1)
result = rv[0]["mypolygon"]
if result:
quota_service.increment_success_service_use()
@ -68,6 +36,44 @@ RETURNS Geometry AS $$
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
---- cdb_geocode_admin1_polygon(admin1_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text)
RETURNS Geometry AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger):
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1), trim($2)) AS mypolygon", ["text", "text"])
rv = plpy.execute(plan, [admin1_name, country_name], 1)
result = rv[0]["mypolygon"]
if result:
quota_service.increment_success_service_use()
return result
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode admin1 polygon')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
--------------------------------------------------------------------------------
-- Implementation of the server extension

101
server/extension/sql/50_namedplaces.sql
View File

@ -35,7 +35,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_namedplac
RETURNS Geometry AS $$
from cartodb_services.mapzen import MapzenGeocoder
from cartodb_services.mapzen.types import country_to_iso3
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import QuotaService, metrics
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
@ -50,35 +50,36 @@ RETURNS Geometry AS $$
if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota')
try:
geocoder = MapzenGeocoder(user_geocoder_config.mapzen_api_key, logger)
country_iso3 = None
if country_name:
country_iso3 = country_to_iso3(country_name)
coordinates = geocoder.geocode(searchtext=city_name, city=None,
state_province=admin1_name,
country=country_iso3, search_type='locality')
if coordinates:
quota_service.increment_success_service_use()
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
return point['st_setsrid']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to geocode city point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode city point using mapzen')
finally:
quota_service.increment_total_service_use()
with metrics('cdb_geocode_namedplace_point', user_geocoder_config, logger):
try:
geocoder = MapzenGeocoder(user_geocoder_config.mapzen_api_key, logger)
country_iso3 = None
if country_name:
country_iso3 = country_to_iso3(country_name)
coordinates = geocoder.geocode(searchtext=city_name, city=None,
state_province=admin1_name,
country=country_iso3, search_type='locality')
if coordinates:
quota_service.increment_success_service_use()
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
return point['st_setsrid']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to geocode city point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode city point using mapzen')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_internal_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL)
RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.metrics import InternalGeocoderConfig, metrics
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
@ -90,30 +91,32 @@ RETURNS Geometry AS $$
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
if admin1_name and country_name:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"])
rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1)
elif country_name:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"])
rv = plpy.execute(plan, [city_name, country_name], 1)
else:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"])
rv = plpy.execute(plan, [city_name], 1)
result = rv[0]["mypoint"]
if result:
quota_service.increment_success_service_use()
return result
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to geocode namedplace point', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode namedplace point')
finally:
quota_service.increment_total_service_use()
with metrics('cdb_geocode_namedplace_point', user_geocoder_config, logger):
try:
if admin1_name and country_name:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"])
rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1)
elif country_name:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"])
rv = plpy.execute(plan, [city_name, country_name], 1)
else:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"])
rv = plpy.execute(plan, [city_name], 1)
result = rv[0]["mypoint"]
if result:
quota_service.increment_success_service_use()
return result
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to geocode namedplace point', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode namedplace point')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
--------------------------------------------------------------------------------

144
server/extension/sql/60_postalcodes.sql
View File

@ -1,5 +1,6 @@
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text)
RETURNS Geometry AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger,LoggerConfig
@ -13,27 +14,29 @@ RETURNS Geometry AS $$
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1)) AS mypoint", ["text"])
rv = plpy.execute(plan, [code], 1)
result = rv[0]["mypoint"]
if result:
quota_service.increment_success_service_use()
return result
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode postal code point')
finally:
quota_service.increment_total_service_use()
with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger):
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1)) AS mypoint", ["text"])
rv = plpy.execute(plan, [code], 1)
result = rv[0]["mypoint"]
if result:
quota_service.increment_success_service_use()
return result
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode postal code point')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text)
RETURNS Geometry AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger,LoggerConfig
@ -47,27 +50,29 @@ RETURNS Geometry AS $$
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1), trim($2)) AS mypoint", ["TEXT", "TEXT"])
rv = plpy.execute(plan, [code, country], 1)
result = rv[0]["mypoint"]
if result:
quota_service.increment_success_service_use()
return result
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode postal code point')
finally:
quota_service.increment_total_service_use()
with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger):
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1), trim($2)) AS mypoint", ["TEXT", "TEXT"])
rv = plpy.execute(plan, [code, country], 1)
result = rv[0]["mypoint"]
if result:
quota_service.increment_success_service_use()
return result
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode postal code point')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text)
RETURNS Geometry AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger,LoggerConfig
@ -81,27 +86,29 @@ RETURNS Geometry AS $$
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1)) AS mypolygon", ["text"])
rv = plpy.execute(plan, [code], 1)
result = rv[0]["mypolygon"]
if result:
quota_service.increment_success_service_use()
return result
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode postal code polygon')
finally:
quota_service.increment_total_service_use()
with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger):
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1)) AS mypolygon", ["text"])
rv = plpy.execute(plan, [code], 1)
result = rv[0]["mypolygon"]
if result:
quota_service.increment_success_service_use()
return result
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode postal code polygon')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text, country text)
RETURNS Geometry AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger,LoggerConfig
@ -115,23 +122,24 @@ RETURNS Geometry AS $$
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1), trim($2)) AS mypolygon", ["TEXT", "TEXT"])
rv = plpy.execute(plan, [code, country], 1)
result = rv[0]["mypolygon"]
if result:
quota_service.increment_success_service_use()
return result
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode postal code polygon')
finally:
quota_service.increment_total_service_use()
with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger):
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1), trim($2)) AS mypolygon", ["TEXT", "TEXT"])
rv = plpy.execute(plan, [code, country], 1)
result = rv[0]["mypolygon"]
if result:
quota_service.increment_success_service_use()
return result
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode postal code polygon')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
--------------------------------------------------------------------------------

36
server/extension/sql/70_ips.sql
View File

@ -1,5 +1,6 @@
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text)
RETURNS Geometry AS $$
from cartodb_services.metrics import metrics
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger,LoggerConfig
@ -13,23 +14,24 @@ RETURNS Geometry AS $$
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_ipaddress_point(trim($1)) AS mypoint", ["TEXT"])
rv = plpy.execute(plan, [ip], 1)
result = rv[0]["mypoint"]
if result:
quota_service.increment_success_service_use()
return result
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode postal code polygon')
finally:
quota_service.increment_total_service_use()
with metrics('cdb_geocode_ipaddress_point', user_geocoder_config, logger):
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_ipaddress_point(trim($1)) AS mypoint", ["TEXT"])
rv = plpy.execute(plan, [ip], 1)
result = rv[0]["mypoint"]
if result:
quota_service.increment_success_service_use()
return result
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode postal code polygon')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
--------------------------------------------------------------------------------

22
server/extension/sql/90_isochrone.sql
View File

@ -1,21 +1,27 @@
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
from cartodb_services.metrics import metrics
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
if user_isolines_config.google_services_user:
raise Exception('This service is not available for google service users.')
if user_isolines_config.heremaps_provider:
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(here_plan, [username, orgname, source, mode, range, options])
elif user_isolines_config.mapzen_provider:
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options])
else:
raise Exception('Requested isolines provider is not available')
with metrics('cb_isochrone', user_isolines_config, logger):
if user_isolines_config.heremaps_provider:
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(here_plan, [username, orgname, source, mode, range, options])
elif user_isolines_config.mapzen_provider:
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options])
else:
raise Exception('Requested isolines provider is not available')
$$ LANGUAGE plpythonu;
-- heremaps isochrone

4
server/lib/python/cartodb_services/cartodb_services/here/geocoder.py
View File

@ -5,9 +5,10 @@ import json
import requests
from exceptions import *
from cartodb_services.metrics import Traceable
class HereMapsGeocoder:
class HereMapsGeocoder(Traceable):
'A Here Maps Geocoder wrapper for python'
PRODUCTION_GEOCODE_JSON_URL = 'https://geocoder.api.here.com/6.2/geocode.json'
@ -89,6 +90,7 @@ class HereMapsGeocoder:
request_params.update(params)
response = requests.get(self.host, params=request_params,
timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
self.add_response_data(response, self._logger)
if response.status_code == requests.codes.ok:
return json.loads(response.text)
elif response.status_code == requests.codes.bad_request:

4
server/lib/python/cartodb_services/cartodb_services/here/routing.py
View File

@ -2,9 +2,10 @@ import requests
import json
from exceptions import WrongParams
from cartodb_services.metrics import Traceable
class HereMapsRoutingIsoline:
class HereMapsRoutingIsoline(Traceable):
'A Here Maps Routing wrapper for python'
PRODUCTION_ROUTING_BASE_URL = 'https://isoline.route.api.here.com'
@ -54,6 +55,7 @@ class HereMapsRoutingIsoline:
parsed_options)
response = requests.get(self._url, params=request_params,
timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
self.add_response_data(response, self._logger)
if response.status_code == requests.codes.ok:
return self.__parse_isolines_response(response.text)
elif response.status_code == requests.codes.bad_request:

4
server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py
View File

@ -5,9 +5,10 @@ import re
from exceptions import WrongParams, MalformedResult, ServiceException
from qps import qps_retry
from cartodb_services.tools import Coordinate, PolyLine
from cartodb_services.metrics import Traceable
class MapzenGeocoder:
class MapzenGeocoder(Traceable):
'A Mapzen Geocoder wrapper for python'
BASE_URL = 'https://search.mapzen.com/v1/search'
@ -28,6 +29,7 @@ class MapzenGeocoder:
try:
response = requests.get(self._url, params=request_params,
timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
self.add_response_data(response, self._logger)
if response.status_code == requests.codes.ok:
return self.__parse_response(response.text)
elif response.status_code == requests.codes.bad_request:

4
server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py
View File

@ -2,9 +2,10 @@ import requests
import json
from qps import qps_retry
from exceptions import ServiceException
from cartodb_services.metrics import Traceable
class MatrixClient:
class MatrixClient(Traceable):
"""
A minimal client for Mapzen Time-Distance Matrix Service
@ -45,6 +46,7 @@ class MatrixClient:
}
response = requests.get(self.ONE_TO_MANY_URL, params=request_params,
timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
self.add_response_data(response, self._logger)
if response.status_code != requests.codes.ok:
self._logger.error('Error trying to get matrix distance from mapzen',

4
server/lib/python/cartodb_services/cartodb_services/mapzen/routing.py
View File

@ -5,9 +5,10 @@ import re
from exceptions import WrongParams, MalformedResult, ServiceException
from qps import qps_retry
from cartodb_services.tools import Coordinate, PolyLine
from cartodb_services.metrics import MetricsDataGatherer, Traceable
class MapzenRouting:
class MapzenRouting(Traceable):
'A Mapzen Routing wrapper for python'
PRODUCTION_ROUTING_BASE_URL = 'https://valhalla.mapzen.com/route'
@ -47,6 +48,7 @@ class MapzenRouting:
request_params = self.__parse_request_parameters(json_request_params)
response = requests.get(self._url, params=request_params,
timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
self.add_response_data(response, self._logger)
if response.status_code == requests.codes.ok:
return self.__parse_routing_response(response.text)
elif response.status_code == requests.codes.bad_request:

1
server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py
View File

@ -1,3 +1,4 @@
from config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatorySnapshotConfig, ObservatoryConfig
from quota import QuotaService
from user import UserMetricsService
from log import metrics, MetricsDataGatherer, Traceable

59
server/lib/python/cartodb_services/cartodb_services/metrics/config.py
View File

@ -15,6 +15,7 @@ class ServiceConfig(object):
self._username = username
self._orgname = orgname
self._db_config = ServicesDBConfig(db_conn, username, orgname)
self._metrics_log_path = self.__get_metrics_log_path()
self._environment = self._db_config._server_environment
if redis_connection:
self._redis_config = ServicesRedisConfig(redis_connection).build(
@ -38,9 +39,20 @@ class ServiceConfig(object):
def environment(self):
return self._environment
@property
def metrics_log_path(self):
return self._metrics_log_path
def __get_metrics_log_path(self):
if self.METRICS_LOG_KEY:
return self._db_config.logger_config.get(self.METRICS_LOG_KEY, None)
else:
return None
class DataObservatoryConfig(ServiceConfig):
METRICS_LOG_KEY = 'do_log_path'
def __init__(self, redis_connection, db_conn, username, orgname=None):
super(DataObservatoryConfig, self).__init__(redis_connection, db_conn,
username, orgname)
@ -61,6 +73,10 @@ class DataObservatoryConfig(ServiceConfig):
def connection_str(self):
return self._connection_str
@property
def provider(self):
return 'data observatory'
class ObservatorySnapshotConfig(DataObservatoryConfig):
@ -118,6 +134,7 @@ class RoutingConfig(ServiceConfig):
DEFAULT_PROVIDER = 'mapzen'
QUOTA_KEY = 'mapzen_routing_quota'
SOFT_LIMIT_KEY = 'soft_mapzen_routing_limit'
METRICS_LOG_KEY = 'routing_log_path'
def __init__(self, redis_connection, db_conn, username, orgname=None):
super(RoutingConfig, self).__init__(redis_connection, db_conn,
@ -135,6 +152,10 @@ class RoutingConfig(ServiceConfig):
if self._routing_provider == self.MAPZEN_PROVIDER:
return 'routing_mapzen'
@property
def provider(self):
return self._routing_provider
@property
def mapzen_api_key(self):
return self._mapzen_api_key
@ -151,7 +172,6 @@ class RoutingConfig(ServiceConfig):
def soft_limit(self):
return self._soft_limit
def _set_monthly_quota(self):
self._monthly_quota = self._get_effective_monthly_quota()
@ -169,7 +189,6 @@ class RoutingConfig(ServiceConfig):
self._soft_limit = False
class IsolinesRoutingConfig(ServiceConfig):
ISOLINES_CONFIG_KEYS = ['here_isolines_quota', 'soft_here_isolines_limit',
@ -184,6 +203,7 @@ class IsolinesRoutingConfig(ServiceConfig):
MAPZEN_PROVIDER = 'mapzen'
HEREMAPS_PROVIDER = 'heremaps'
DEFAULT_PROVIDER = 'heremaps'
METRICS_LOG_KEY = 'isolines_log_path'
def __init__(self, redis_connection, db_conn, username, orgname=None):
super(IsolinesRoutingConfig, self).__init__(redis_connection, db_conn,
@ -260,11 +280,12 @@ class IsolinesRoutingConfig(ServiceConfig):
class InternalGeocoderConfig(ServiceConfig):
METRICS_LOG_KEY = 'geocoder_log_path'
def __init__(self, redis_connection, db_conn, username, orgname=None):
# For now, internal geocoder doesn't use the redis config
super(InternalGeocoderConfig, self).__init__(None, db_conn,
username, orgname)
self._log_path = self._db_config.geocoder_log_path
@property
def service_type(self):
@ -283,8 +304,8 @@ class InternalGeocoderConfig(ServiceConfig):
return None
@property
def log_path(self):
return self._log_path
def provider(self):
return 'internal'
class GeocoderConfig(ServiceConfig):
@ -310,6 +331,7 @@ class GeocoderConfig(ServiceConfig):
ORGNAME_KEY = 'orgname'
PERIOD_END_DATE = 'period_end_date'
DEFAULT_PROVIDER = 'mapzen'
METRICS_LOG_KEY = 'geocoder_log_path'
def __init__(self, redis_connection, db_conn, username, orgname=None, forced_provider=None):
super(GeocoderConfig, self).__init__(redis_connection, db_conn,
@ -341,7 +363,6 @@ class GeocoderConfig(ServiceConfig):
self._geocoder_provider = self.DEFAULT_PROVIDER
self._geocoding_quota = float(filtered_config[self.QUOTA_KEY])
self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE])
self._log_path = db_config.geocoder_log_path
if filtered_config[self.SOFT_LIMIT_KEY].lower() == 'true':
self._soft_geocoding_limit = True
else:
@ -424,8 +445,8 @@ class GeocoderConfig(ServiceConfig):
return self._cost_per_hit
@property
def log_path(self):
return self._log_path
def provider(self):
return self._geocoder_provider
class ServicesDBConfig:
@ -440,7 +461,6 @@ class ServicesDBConfig:
self._get_server_config()
self._get_here_config()
self._get_mapzen_config()
self._get_logger_config()
self._get_data_observatory_config()
def _get_server_config(self):
@ -493,13 +513,6 @@ class ServicesDBConfig:
else:
self._data_observatory_connection_str = do_conf['connection']['production']
def _get_logger_config(self):
logger_conf_json = self._get_conf('logger_conf')
if not logger_conf_json:
raise ConfigException('Logger configuration missing')
else:
logger_conf = json.loads(logger_conf_json)
self._geocoder_log_path = logger_conf['geocoder_log_path']
def _get_conf(self, key):
try:
@ -507,7 +520,7 @@ class ServicesDBConfig:
conf = self._db_conn.execute(sql, 1)
return conf[0]['conf']
except Exception as e:
raise ConfigException("Malformed config for {0}: {1}".format(key, e))
raise ConfigException("Error trying to get config for {0}: {1}".format(key, e))
@property
def server_environment(self):
@ -557,14 +570,18 @@ class ServicesDBConfig:
def mapzen_geocoder_monthly_quota(self):
return self._mapzen_geocoder_quota
@property
def geocoder_log_path(self):
return self._geocoder_log_path
@property
def data_observatory_connection_str(self):
return self._data_observatory_connection_str
@property
def logger_config(self):
logger_conf_json = self._get_conf('logger_conf')
if not logger_conf_json:
raise ConfigException('Logger configuration missing')
else:
return json.loads(logger_conf_json)
class ServicesRedisConfig:

249
server/lib/python/cartodb_services/cartodb_services/metrics/log.py
View File

@ -1,15 +1,130 @@
from datetime import datetime
import abc
import json
import re
import time
import uuid
import plpy
from datetime import datetime
from contextlib import contextmanager
from urlparse import urlparse
class MetricsLoggerFactory:
@contextmanager
def metrics(function, service_config, logger=None):
try:
start_time = time.time()
yield
finally:
end_time = time.time()
MetricsDataGatherer.add('uuid', str(uuid.uuid1()))
MetricsDataGatherer.add('function_name', function)
MetricsDataGatherer.add('function_execution_time', (end_time - start_time))
metrics_logger = MetricsServiceLoggerFactory.build(service_config,
logger)
if metrics_logger:
data = MetricsDataGatherer.get()
metrics_logger.log(data)
MetricsDataGatherer.clean()
class Traceable:
"""
Module to add metrics traceability, for example to get response object
in order to add to the metrics dump
"""
def add_response_data(self, response, logger=None):
try:
response_data = {}
response_data['type'] = "request"
response_data['date'] = datetime.now().isoformat()
response_data['elapsed_time'] = response.elapsed.total_seconds()
response_data['code'] = response.status_code
response_data['message'] = response.reason
response_data['url'] = self._parse_response_url(response.url)
stored_data = MetricsDataGatherer.get_element('response')
if stored_data:
stored_data.append(response_data)
else:
MetricsDataGatherer.add('response', [response_data])
except BaseException as e:
# We don't want to stop the job for some error processing response
if logger:
logger.error("Error trying to process response metricd data",
exception=e)
def _parse_response_url(self, url):
u = urlparse(url)
return "{0}://{1}{2}".format(u.scheme, u.netloc, u.path)
class MetricsDataGatherer:
"""
Metrics gatherer used as a singleton. The intend is to use it as a global
storage for the metrics along the function request.
"""
class __MetricsDataGatherer:
def __init__(self):
self.data = {}
def add(self, key, value):
self.data[key] = value
def get(self):
return self.data
def get_element(self, key):
return self.data.get(key, None)
def clean(self):
self.data = {}
# We use pgbouncer so we need to have multiples instances per request id
__instance = {}
@classmethod
def build(self, service_config):
if re.match('geocoder_*', service_config.service_type):
return MetricsGeocoderLogger(service_config)
def add(self, key, value):
MetricsDataGatherer.instance().add(key, value)
@classmethod
def get(self):
return MetricsDataGatherer.instance().get()
@classmethod
def get_element(self, key):
return MetricsDataGatherer.instance().get_element(key)
@classmethod
def clean(self):
MetricsDataGatherer.instance().clean()
@classmethod
def instance(self):
txid = MetricsDataGatherer._get_txid()
if txid not in MetricsDataGatherer.__instance:
MetricsDataGatherer.__instance[txid] = MetricsDataGatherer.__MetricsDataGatherer()
return MetricsDataGatherer.__instance[txid]
@classmethod
def _get_txid(self):
result = plpy.execute('select txid_current() as txid')
return result[0]['txid']
class MetricsServiceLoggerFactory:
@classmethod
def build(self, service_config, logger=None):
if re.search('^geocoder_*', service_config.service_type):
return MetricsGeocoderLogger(service_config, logger)
elif re.search('^routing_*', service_config.service_type):
return MetricsGenericLogger(service_config, logger)
elif re.search('_isolines$', service_config.service_type):
return MetricsIsolinesLogger(service_config, logger)
elif re.search('^obs_*', service_config.service_type):
return MetricsGenericLogger(service_config, logger)
else:
return None
@ -17,58 +132,120 @@ class MetricsLoggerFactory:
class MetricsLogger(object):
__metaclass__ = abc.ABCMeta
def __init__(self, file_path):
self._file_path = file_path
def __init__(self, service_config, logger):
self._service_config = service_config
self._logger = logger
def dump_to_file(self, data):
with open(self._file_path, 'a') as logfile:
json.dump(data, logfile)
logfile.write('\n')
try:
log_path = self.service_config.metrics_log_path
response_data = data.pop('response', [])
uuid = data.get('uuid')
if log_path:
with open(log_path, 'a') as logfile:
self._dump_response_to_file(uuid, response_data, logfile)
json.dump(data, logfile)
logfile.write('\n')
except BaseException as e:
self._logger("Error dumping metrics to file {0}".format(log_path),
exception=e)
def collect_data(self, data):
return {
"uuid": data.get('uuid', uuid.uuid1()),
"type": 'function',
"function_name": data.get('function_name', None),
"function_execution_time": data.get('function_execution_time',
None),
"service": self._service_config.service_type,
"processable_rows": 1,
"success": data.get('success', False),
"successful_rows": data.get('successful_rows', 0),
"failed_rows": data.get('failed_rows', 0),
"empty_rows": data.get('empty_rows', 0),
"created_at": datetime.now().isoformat(),
"provider": self._service_config.provider,
"username": self._service_config.username,
"organization": self._service_config.organization,
"response": data.get('response', [])
}
def _dump_response_to_file(self, uuid, response_data, log_file):
for r in response_data:
r['uuid'] = uuid
self._logger.info(r)
json.dump(r, log_file)
log_file.write('\n')
@property
def service_config(self):
return self._service_config
@abc.abstractproperty
def log(self, **data):
def log(self, data):
raise NotImplementedError('log method must be defined')
class MetricsGeocoderLogger(MetricsLogger):
def __init__(self, service_config):
super(MetricsGeocoderLogger, self).__init__(service_config.log_path)
self._service_config = service_config
def __init__(self, service_config, logger):
super(MetricsGeocoderLogger, self).__init__(service_config, logger)
def log(self, **data):
dump_data = self._dump_data(**data)
def log(self, data):
dump_data = self.collect_data(data)
self.dump_to_file(dump_data)
def _dump_data(self, **data):
if data['success']:
cost = self._service_config.cost_per_hit
failed_rows = 0
successful_rows = 1
def collect_data(self, data):
dump_data = super(MetricsGeocoderLogger, self).collect_data(data)
if data.get('success', False):
cost = self.service_config.cost_per_hit
else:
cost = 0
failed_rows = 1
successful_rows = 0
if self._service_config.is_high_resolution:
if self.service_config.is_high_resolution:
kind = 'high-resolution'
else:
kind = 'internal'
return {
dump_data.update({
"batched": False,
"cache_hits": 0, # Always 0 because no cache involved
# https://github.com/CartoDB/cartodb/blob/master/app/models/geocoding.rb#L208-L211
"cost": cost,
"created_at": datetime.now().isoformat(),
"failed_rows": failed_rows,
"geocoder_type": self._service_config.service_type,
"geocoder_type": self.service_config.service_type,
"kind": kind,
"processable_rows": 1,
"processed_rows": successful_rows,
"real_rows": successful_rows,
"success": data['success'],
"successful_rows": successful_rows,
"username": self._service_config.username,
"organization": self._service_config.organization
}
"processed_rows": data.get('successful_rows', 0),
"real_rows": data.get('successful_rows', 0),
})
return dump_data
class MetricsGenericLogger(MetricsLogger):
def __init__(self, service_config, logger):
super(MetricsGenericLogger, self).__init__(service_config, logger)
def log(self, data):
dump_data = self.collect_data(data)
self.dump_to_file(dump_data)
def collect_data(self, data):
return super(MetricsGenericLogger, self).collect_data(data)
class MetricsIsolinesLogger(MetricsLogger):
def __init__(self, service_config, logger):
super(MetricsIsolinesLogger, self).__init__(service_config, logger)
def log(self, data):
dump_data = self.collect_data(data)
self.dump_to_file(dump_data)
def collect_data(self, data):
dump_data = super(MetricsIsolinesLogger, self).collect_data(data)
dump_data.update({
"isolines_generated": data.get('isolines_generated', 0)
})
return dump_data

19
server/lib/python/cartodb_services/cartodb_services/metrics/quota.py
View File

@ -1,5 +1,5 @@
from user import UserMetricsService
from log import MetricsLoggerFactory
from log import MetricsDataGatherer
from datetime import date
import re
@ -14,7 +14,6 @@ class QuotaService:
redis_connection)
self._user_service = UserMetricsService(self._user_service_config,
redis_connection)
self._metrics_logger = MetricsLoggerFactory.build(user_service_config)
def check_user_quota(self):
return self._quota_checker.check()
@ -46,13 +45,19 @@ class QuotaService:
self._user_service.increment_service_use(
self._user_service_config.service_type, "isolines_generated",
amount=amount)
MetricsDataGatherer.add('isolines_generated', amount)
def _log_service_process(self, event):
if self._metrics_logger:
if event is 'success' or event is 'empty':
self._metrics_logger.log(success=True)
elif event is 'empty':
self._metrics_logger.log(success=False)
if event is 'success':
MetricsDataGatherer.add('success', True)
MetricsDataGatherer.add('successful_rows', 1)
elif event is 'empty':
MetricsDataGatherer.add('success', True)
MetricsDataGatherer.add('successful_rows', 1)
MetricsDataGatherer.add('empty_rows', 1)
elif event is 'fail':
MetricsDataGatherer.add('success', False)
MetricsDataGatherer.add('failed_rows', 1)
class QuotaChecker:

2
server/lib/python/cartodb_services/setup.py
View File

@ -10,7 +10,7 @@ from setuptools import setup, find_packages
setup(
name='cartodb_services',
version='0.9.4',
version='0.10.0',
description='CartoDB Services API Python Library',

5
server/lib/python/cartodb_services/test/__init__.py
View File

@ -0,0 +1,5 @@
from test_helper import plpy_mock_config
def setup():
plpy_mock_config()

5
server/lib/python/cartodb_services/test/metrics/test_config.py
View File

@ -1,13 +1,14 @@
from unittest import TestCase
from mockredis import MockRedis
from ..test_helper import *
from cartodb_services.metrics.config import RoutingConfig, ServicesRedisConfig
from ..test_helper import build_plpy_mock
class TestRoutingConfig(TestCase):
def setUp(self):
self._redis_conn = MockRedis()
self._db_conn = build_plpy_mock()
self._db_conn = plpy_mock
self._username = 'my_test_user'
self._user_key = "rails:users:{0}".format(self._username)
self._redis_conn.hset(self._user_key, 'period_end_date', '2016-10-10')

2
server/lib/python/cartodb_services/test/metrics/test_quota.py
View File

@ -1,6 +1,6 @@
from unittest import TestCase
from mockredis import MockRedis
from ..test_helper import build_plpy_mock
from ..test_helper import *
from cartodb_services.metrics.quota import QuotaChecker
from cartodb_services.metrics import RoutingConfig
from datetime import datetime

53
server/lib/python/cartodb_services/test/mock_plpy.py Normal file
View File

@ -0,0 +1,53 @@
import re
class MockCursor:
def __init__(self, data):
self.cursor_pos = 0
self.data = data
def fetch(self, batch_size):
batch = self.data[self.cursor_pos: self.cursor_pos + batch_size]
self.cursor_pos += batch_size
return batch
class MockPlPy:
def __init__(self):
self._reset()
def _reset(self):
self.infos = []
self.notices = []
self.debugs = []
self.logs = []
self.warnings = []
self.errors = []
self.fatals = []
self.executes = []
self.results = []
self.prepares = []
self.results = {}
def _define_result(self, query, result):
pattern = re.compile(query, re.IGNORECASE | re.MULTILINE)
self.results[pattern] = result
def notice(self, msg):
self.notices.append(msg)
def debug(self, msg):
self.notices.append(msg)
def info(self, msg):
self.infos.append(msg)
def cursor(self, query):
data = self.execute(query)
return MockCursor(data)
def execute(self, query, rows=1):
for pattern, result in self.results.iteritems():
if pattern.search(query):
return result
return []

36
server/lib/python/cartodb_services/test/test_config.py
View File

@ -1,20 +1,20 @@
import test_helper
from cartodb_services.metrics import GeocoderConfig, ObservatorySnapshotConfig, ConfigException
from test_helper import *
from unittest import TestCase
from nose.tools import assert_raises
from mockredis import MockRedis
from datetime import datetime, timedelta
from cartodb_services.metrics import GeocoderConfig, ObservatorySnapshotConfig, ConfigException
class TestConfig(TestCase):
def setUp(self):
self.redis_conn = MockRedis()
self.plpy_mock = test_helper.build_plpy_mock()
plpy_mock_config()
def test_should_return_list_of_nokia_geocoder_config_if_its_ok(self):
test_helper.build_redis_user_config(self.redis_conn, 'test_user')
geocoder_config = GeocoderConfig(self.redis_conn, self.plpy_mock,
build_redis_user_config(self.redis_conn, 'test_user')
geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock,
'test_user', None)
assert geocoder_config.heremaps_geocoder is True
assert geocoder_config.geocoding_quota == 100
@ -22,10 +22,10 @@ class TestConfig(TestCase):
def test_should_return_list_of_nokia_geocoder_config_ok_for_org(self):
yesterday = datetime.today() - timedelta(days=1)
test_helper.build_redis_user_config(self.redis_conn, 'test_user')
test_helper.build_redis_org_config(self.redis_conn, 'test_org',
quota=200, end_date=yesterday)
geocoder_config = GeocoderConfig(self.redis_conn, self.plpy_mock,
build_redis_user_config(self.redis_conn, 'test_user')
build_redis_org_config(self.redis_conn, 'test_org',
quota=200, end_date=yesterday)
geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock,
'test_user', 'test_org')
assert geocoder_config.heremaps_geocoder is True
assert geocoder_config.geocoding_quota == 200
@ -34,10 +34,10 @@ class TestConfig(TestCase):
def test_should_return_config_for_obs_snapshot(self):
yesterday = datetime.today() - timedelta(days=1)
test_helper.build_redis_user_config(self.redis_conn, 'test_user',
do_quota=100, soft_do_limit=True,
end_date=yesterday)
do_config = ObservatorySnapshotConfig(self.redis_conn, self.plpy_mock,
build_redis_user_config(self.redis_conn, 'test_user',
do_quota=100, soft_do_limit=True,
end_date=yesterday)
do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock,
'test_user')
assert do_config.monthly_quota == 100
assert do_config.soft_limit is True
@ -45,16 +45,16 @@ class TestConfig(TestCase):
def test_should_return_db_quota_if_not_redis_quota_config_obs_snapshot(self):
yesterday = datetime.today() - timedelta(days=1)
test_helper.build_redis_user_config(self.redis_conn, 'test_user',
end_date=yesterday)
do_config = ObservatorySnapshotConfig(self.redis_conn, self.plpy_mock,
build_redis_user_config(self.redis_conn, 'test_user',
end_date=yesterday)
do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock,
'test_user')
assert do_config.monthly_quota == 0
assert do_config.soft_limit is False
assert do_config.period_end_date.date() == yesterday.date()
def test_should_raise_exception_when_missing_parameters(self):
plpy_mock = test_helper.build_plpy_mock(empty=True)
test_helper.build_redis_user_config(self.redis_conn, 'test_user')
plpy_mock._reset()
build_redis_user_config(self.redis_conn, 'test_user')
assert_raises(ConfigException, GeocoderConfig, self.redis_conn,
plpy_mock, 'test_user', None)

34
server/lib/python/cartodb_services/test/test_helper.py
View File

@ -1,7 +1,11 @@
from datetime import datetime, date
from mock import Mock
from mock import Mock, MagicMock
import random
import sys
sys.modules['plpy'] = Mock()
from mock_plpy import MockPlPy
plpy_mock = MockPlPy()
sys.modules['plpy'] = plpy_mock
def build_redis_user_config(redis_conn, username, quota=100, soft_limit=False,
@ -57,22 +61,10 @@ def increment_service_uses(redis_conn, username, orgname=None,
redis_conn.zincrby(redis_name, date.day, amount)
def build_plpy_mock(empty=False):
plpy_mock = Mock()
if not empty:
plpy_mock.execute.side_effect = _plpy_execute_side_effect
return plpy_mock
def _plpy_execute_side_effect(*args, **kwargs):
if args[0] == "SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as conf":
return [{'conf': '{"geocoder": {"app_id": "app_id", "app_code": "code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "app_id", "app_code": "code"}}'}]
elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('mapzen_conf') as conf":
return [{'conf': '{"routing": {"api_key": "api_key_rou", "monthly_quota": 1500000}, "geocoder": {"api_key": "api_key_geo", "monthly_quota": 1500000}, "matrix": {"api_key": "api_key_mat", "monthly_quota": 1500000}}'}]
elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('logger_conf') as conf":
return [{'conf': '{"geocoder_log_path": "/dev/null"}'}]
elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('data_observatory_conf') as conf":
return [{'conf': '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}}'}]
elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('server_conf') as conf":
return [{'conf': '{"environment": "testing"}'}]
def plpy_mock_config():
plpy_mock._define_result("CDB_Conf_GetConf\('heremaps_conf'\)", [{'conf': '{"geocoder": {"app_id": "app_id", "app_code": "code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "app_id", "app_code": "code"}}'}])
plpy_mock._define_result("CDB_Conf_GetConf\('mapzen_conf'\)", [{'conf': '{"routing": {"api_key": "api_key_rou", "monthly_quota": 1500000}, "geocoder": {"api_key": "api_key_geo", "monthly_quota": 1500000}, "matrix": {"api_key": "api_key_mat", "monthly_quota": 1500000}}'}])
plpy_mock._define_result("CDB_Conf_GetConf\('logger_conf'\)", [{'conf': '{"geocoder_log_path": "/dev/null"}'}])
plpy_mock._define_result("CDB_Conf_GetConf\('data_observatory_conf'\)", [{'conf': '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}}'}])
plpy_mock._define_result("CDB_Conf_GetConf\('server_conf'\)", [{'conf': '{"environment": "testing"}'}])
plpy_mock._define_result("select txid_current", [{'txid': random.randint(0, 1000)}])

23
server/lib/python/cartodb_services/test/test_metrics_log.py Normal file
View File

@ -0,0 +1,23 @@
from test_helper import *
from cartodb_services.metrics import MetricsDataGatherer
from unittest import TestCase
from mock import Mock, MagicMock
from nose.tools import assert_raises
from datetime import datetime, date
class TestMetricsDataGatherer(TestCase):
def setUp(self):
plpy_mock_config()
def test_should_use_multiple_instances_for_multiples_requests(self):
plpy_mock._define_result("select txid_current", [{'txid': 100}])
MetricsDataGatherer.add('test', 1)
plpy_mock._define_result("select txid_current", [{'txid': 101}])
MetricsDataGatherer.add('test', 2)
plpy_mock._define_result("select txid_current", [{'txid': 100}])
assert MetricsDataGatherer.get_element('test') is 1
plpy_mock._define_result("select txid_current", [{'txid': 101}])
assert MetricsDataGatherer.get_element('test') is 2

1
server/lib/python/cartodb_services/test/test_qps.py
View File

@ -1,4 +1,3 @@
import test_helper
import requests
from unittest import TestCase
from nose.tools import assert_raises

27
server/lib/python/cartodb_services/test/test_quota_service.py
View File

@ -1,4 +1,4 @@
import test_helper
from test_helper import *
from mockredis import MockRedis
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import GeocoderConfig, RoutingConfig, ObservatorySnapshotConfig, IsolinesRoutingConfig
@ -28,32 +28,32 @@ class TestQuotaService(TestCase):
def test_should_return_true_if_user_quota_is_not_completely_used(self):
qs = self.__build_geocoder_quota_service('test_user')
test_helper.increment_service_uses(self.redis_conn, 'test_user')
increment_service_uses(self.redis_conn, 'test_user')
assert qs.check_user_quota() is True
def test_should_return_true_if_org_quota_is_not_completely_used(self):
qs = self.__build_geocoder_quota_service('test_user',
orgname='test_org')
test_helper.increment_service_uses(self.redis_conn, 'test_user',
increment_service_uses(self.redis_conn, 'test_user',
orgname='test_org')
assert qs.check_user_quota() is True
def test_should_return_false_if_user_quota_is_surpassed(self):
qs = self.__build_geocoder_quota_service('test_user')
test_helper.increment_service_uses(self.redis_conn, 'test_user',
increment_service_uses(self.redis_conn, 'test_user',
amount=300)
assert qs.check_user_quota() is False
def test_should_return_false_if_org_quota_is_surpassed(self):
qs = self.__build_geocoder_quota_service('test_user',
orgname='test_org')
test_helper.increment_service_uses(self.redis_conn, 'test_user',
increment_service_uses(self.redis_conn, 'test_user',
orgname='test_org', amount=400)
assert qs.check_user_quota() is False
def test_should_return_true_if_user_quota_is_surpassed_but_soft_limit_is_enabled(self):
qs = self.__build_geocoder_quota_service('test_user', soft_limit=True)
test_helper.increment_service_uses(self.redis_conn, 'test_user',
increment_service_uses(self.redis_conn, 'test_user',
amount=300)
assert qs.check_user_quota() is True
@ -61,7 +61,7 @@ class TestQuotaService(TestCase):
qs = self.__build_geocoder_quota_service('test_user',
orgname='test_org',
soft_limit=True)
test_helper.increment_service_uses(self.redis_conn, 'test_user',
increment_service_uses(self.redis_conn, 'test_user',
orgname='test_org', amount=400)
assert qs.check_user_quota() is True
@ -140,18 +140,17 @@ class TestQuotaService(TestCase):
def __prepare_quota_service(self, username, quota, service, orgname,
soft_limit, do_quota, soft_do_limit, end_date):
test_helper.build_redis_user_config(self.redis_conn, username,
build_redis_user_config(self.redis_conn, username,
quota=quota, service=service,
soft_limit=soft_limit,
soft_do_limit=soft_do_limit,
do_quota=do_quota,
end_date=end_date)
if orgname:
test_helper.build_redis_org_config(self.redis_conn, orgname,
build_redis_org_config(self.redis_conn, orgname,
quota=quota, service=service,
do_quota=do_quota,
end_date=end_date)
self._plpy_mock = test_helper.build_plpy_mock()
def __build_geocoder_quota_service(self, username, quota=100,
service='heremaps', orgname=None,
@ -159,7 +158,7 @@ class TestQuotaService(TestCase):
end_date=datetime.today()):
self.__prepare_quota_service(username, quota, service, orgname,
soft_limit, 0, False, end_date)
geocoder_config = GeocoderConfig(self.redis_conn, self._plpy_mock,
geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock,
username, orgname)
return QuotaService(geocoder_config, redis_connection=self.redis_conn)
@ -168,7 +167,7 @@ class TestQuotaService(TestCase):
quota=100, end_date=datetime.today()):
self.__prepare_quota_service(username, quota, service, orgname,
soft_limit, 0, False, end_date)
routing_config = RoutingConfig(self.redis_conn, self._plpy_mock,
routing_config = RoutingConfig(self.redis_conn, plpy_mock,
username, orgname)
return QuotaService(routing_config, redis_connection=self.redis_conn)
@ -177,7 +176,7 @@ class TestQuotaService(TestCase):
quota=100, end_date=datetime.today()):
self.__prepare_quota_service(username, quota, service, orgname,
soft_limit, 0, False, end_date)
isolines_config = IsolinesRoutingConfig(self.redis_conn, self._plpy_mock,
isolines_config = IsolinesRoutingConfig(self.redis_conn, plpy_mock,
username, orgname)
return QuotaService(isolines_config, redis_connection=self.redis_conn)
@ -188,6 +187,6 @@ class TestQuotaService(TestCase):
end_date=datetime.today()):
self.__prepare_quota_service(username, 0, service, orgname, False,
quota, soft_limit, end_date)
do_config = ObservatorySnapshotConfig(self.redis_conn, self._plpy_mock,
do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock,
username, orgname)
return QuotaService(do_config, redis_connection=self.redis_conn)

11
server/lib/python/cartodb_services/test/test_user_service.py
View File

@ -1,4 +1,4 @@
import test_helper
from test_helper import *
from mockredis import MockRedis
from cartodb_services.metrics import UserMetricsService
from cartodb_services.metrics import GeocoderConfig
@ -19,13 +19,13 @@ class TestUserService(TestCase):
def test_user_used_quota_for_a_day(self):
us = self.__build_user_service('test_user')
test_helper.increment_service_uses(self.redis_conn, 'test_user',
increment_service_uses(self.redis_conn, 'test_user',
amount=400)
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 400
def test_org_used_quota_for_a_day(self):
us = self.__build_user_service('test_user', orgname='test_org')
test_helper.increment_service_uses(self.redis_conn, 'test_user',
increment_service_uses(self.redis_conn, 'test_user',
orgname='test_org',
amount=400)
assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 400
@ -135,14 +135,13 @@ class TestUserService(TestCase):
def __build_user_service(self, username, quota=100, service='heremaps',
orgname=None, soft_limit=False,
end_date=date.today()):
test_helper.build_redis_user_config(self.redis_conn, username,
build_redis_user_config(self.redis_conn, username,
quota=quota, service=service,
soft_limit=soft_limit,
end_date=end_date)
if orgname:
test_helper.build_redis_org_config(self.redis_conn, orgname,
build_redis_org_config(self.redis_conn, orgname,
quota=quota, end_date=end_date)
plpy_mock = test_helper.build_plpy_mock()
geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock,
username, orgname)
return UserMetricsService(geocoder_config, self.redis_conn)

4
test/integration/test_street_functions.py
View File

@ -14,8 +14,8 @@ class TestStreetFunctions(TestCase):
self.env_variables['api_key']
)
def test_if_select_with_street_point_is_ok(self):
query = "SELECT cdb_geocode_street_point(street) " \
def test_if_select_with_here_street_point_is_ok(self):
query = "SELECT cdb_here_geocode_street_point(street) " \
"as geometry FROM {0} LIMIT 1&api_key={1}".format(
self.env_variables['table_name'],
self.env_variables['api_key'])