cartodb4/dataservices-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added new logger to all the functions

Browse Source
This commit is contained in:
Mario de Frutos 2016-08-01 19:04:56 +02:00
parent 5d5e8d6f9a
commit ed5cf25e9c
37 changed files with 384 additions and 257 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
README.md
View File

@ -77,8 +77,11 @@ Steps to deploy a new Data Services API version :
SELECT CDB_Conf_SetConf('heremaps_conf', '{"geocoder": {"app_id": "here_geocoder_app_id", "app_code": "here_geocoder_app_code", "geocoder_cost_per_hit": "1"}, "isolines" : {"app_id": "here_isolines_app_id", "app_code": "here_geocoder_app_code"}}');
SELECT CDB_Conf_SetConf('user_config', '{"is_organization": false, "entity_name": "<YOUR_USERNAME>"}')
SELECT CDB_Conf_SetConf('mapzen_conf', '{"routing": {"api_key": "valhalla_app_key", "monthly_quota": 999999}, "geocoder": {"api_key": "search_app_key", "monthly_quota": 999999}, "matrix": {"api_key": "[your_matrix_key]", "monthly_quota": 1500000}}');
SELECT CDB_Conf_SetConf('logger_conf', '{"geocoder_log_path": "/tmp/geocodings.log"}')
SELECT CDB_Conf_SetConf('logger_conf', '{"geocoder_log_path": "/tmp/geocodings.log", [ "min_log_level": "[debug|info|warning|error]", "rollbar_api_key": "SERVER_SIDE_API_KEY", "log_file_path": "LOG_FILE_PATH"]}')
SELECT CDB_Conf_SetConf('data_observatory_conf', '{"connection": {"whitelist": [], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}}')
# Environment to decide: rollbar message, which servers for third party use, etc. If not setted uses production by default (current behavior)
SELECT CDB_Conf_SetConf('server_conf', '{"environment": "[development|staging|production]"}')
```
- configure the user DB:

2
server/extension/Makefile
View File

@ -10,7 +10,7 @@ NEW_EXTENSION_ARTIFACT = $(EXTENSION)--$(EXTVERSION).sql
REGRESS = $(notdir $(basename $(sort $(wildcard test/sql/*test.sql))))
TEST_DIR = test/
REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)'
REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)' --user='postgres'
# DATA is a special variable used by postgres build infrastructure
# These are the files to be installed in the server shared dir,

0
server/extension/cdb_dataservices_server--0.13.3.1--0.14.sql Normal file
View File

0
server/extension/cdb_dataservices_server--0.14--0.13.3.1.sql Normal file
View File

2
server/extension/cdb_dataservices_server.control
View File

@ -1,5 +1,5 @@
comment = 'CartoDB dataservices server extension'
default_version = '0.13.3.1'
default_version = '0.14'
requires = 'plpythonu, plproxy, postgis, cdb_geocoder'
superuser = true
schema = cdb_dataservices_server

0
server/extension/cdb_dataservices_server--0.13.3--0.13.3.1.sql → server/extension/old_versions/cdb_dataservices_server--0.13.3--0.13.3.1.sql
View File

0
server/extension/cdb_dataservices_server--0.13.3.1--0.13.3.sql → server/extension/old_versions/cdb_dataservices_server--0.13.3.1--0.13.3.sql
View File

0
server/extension/cdb_dataservices_server--0.13.3.1.sql → server/extension/old_versions/cdb_dataservices_server--0.13.3.1.sql
View File

17
server/extension/sql/100_routing_helper.sql
View File

@ -17,19 +17,22 @@ RETURNS cdb_dataservices_server.simple_route AS $$
from cartodb_services.mapzen.types import polyline_to_linestring
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Coordinate
from cartodb_services.tools import Logger
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_routing_config = GD["user_routing_config_{0}".format(username)]
logger = Logger(user_routing_config)
quota_service = QuotaService(user_routing_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
client = MapzenRouting(user_routing_config.mapzen_api_key)
client = MapzenRouting(user_routing_config.mapzen_api_key, logger)
if not waypoints or len(waypoints) < 2:
plpy.notice("Empty origin or destination")
logger.notice("Empty origin or destination")
quota_service.increment_empty_service_use()
return [None, None, None]
@ -52,12 +55,10 @@ RETURNS cdb_dataservices_server.simple_route AS $$
quota_service.increment_empty_service_use()
return [None, None, None]
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to obtain route using mapzen provider: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to get mapzen routing', sys.exc_info())
raise e
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu SECURITY DEFINER;

120
server/extension/sql/110_data_observatory_augmentation.sql
View File

@ -35,6 +35,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot(
geometry_level TEXT DEFAULT NULL)
RETURNS json AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger
import json
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
@ -42,9 +43,10 @@ RETURNS json AS $$
plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]
logger = Logger(user_obs_config)
quota_service = QuotaService(user_obs_snapshot_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshotJSON($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"])
@ -56,12 +58,10 @@ RETURNS json AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use get_geographic_snapshot: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to obst_get_demographic_snapshot', sys.exc_info())
raise e
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@ -85,15 +85,17 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetDemographicSnapshot(
geometry_level TEXT DEFAULT NULL)
RETURNS SETOF JSON AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]
logger = Logger(user_obs_config)
quota_service = QuotaService(user_obs_snapshot_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"])
@ -109,12 +111,10 @@ RETURNS SETOF JSON AS $$
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use get_geographic_snapshot: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info())
raise e
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@ -136,6 +136,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot(
geometry_level TEXT DEFAULT NULL)
RETURNS json AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger
import json
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
@ -143,9 +144,10 @@ RETURNS json AS $$
plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]
logger = Logger(user_obs_config)
quota_service = QuotaService(user_obs_snapshot_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshotJSON($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"])
@ -157,12 +159,10 @@ RETURNS json AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use get_segment_snapshot: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to obs_get_segment_snapshot', sys.exc_info())
raise e
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@ -184,15 +184,17 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetSegmentSnapshot(
geometry_level TEXT DEFAULT NULL)
RETURNS SETOF JSON AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]
logger = Logger(user_obs_config)
quota_service = QuotaService(user_obs_snapshot_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"])
@ -208,12 +210,10 @@ RETURNS SETOF JSON AS $$
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use get_segment_snapshot: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to OBS_GetSegmentSnapshot', sys.exc_info())
raise e
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@ -241,15 +241,17 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasure(
time_span TEXT DEFAULT NULL)
RETURNS NUMERIC AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
logger = Logger(user_obs_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasure($1, $2, $3, $4, $5, $6, $7) as measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"])
@ -261,12 +263,10 @@ RETURNS NUMERIC AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use OBS_GetMeasure: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to OBS_GetMeasure', sys.exc_info())
raise e
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@ -292,15 +292,17 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetCategory(
time_span TEXT DEFAULT NULL)
RETURNS TEXT AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
logger = Logger(user_obs_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetCategory($1, $2, $3, $4, $5, $6) as category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"])
@ -312,12 +314,10 @@ RETURNS TEXT AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use OBS_GetCategory: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to OBS_GetCategory', sys.exc_info())
raise e
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@ -345,15 +345,17 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusMeasure(
time_span TEXT DEFAULT NULL)
RETURNS NUMERIC AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
logger = Logger(user_obs_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusMeasure($1, $2, $3, $4, $5, $6, $7) as census_measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"])
@ -365,12 +367,10 @@ RETURNS NUMERIC AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use OBS_GetUSCensusMeasure: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to OBS_GetUSCensusMeasure', sys.exc_info())
raise e
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@ -396,15 +396,17 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusCategory(
time_span TEXT DEFAULT NULL)
RETURNS TEXT AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
logger = Logger(user_obs_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusCategory($1, $2, $3, $4, $5, $6) as census_category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"])
@ -416,12 +418,10 @@ RETURNS TEXT AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use OBS_GetUSCensusCategory: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to OBS_GetUSCensusCategory', sys.exc_info())
raise e
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@ -447,15 +447,17 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPopulation(
time_span TEXT DEFAULT NULL)
RETURNS NUMERIC AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
logger = Logger(user_obs_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetPopulation($1, $2, $3, $4, $5, $6) as population;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"])
@ -467,12 +469,10 @@ RETURNS NUMERIC AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use OBS_GetPopulation: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to OBS_GetPopulation', sys.exc_info())
raise e
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@ -498,15 +498,17 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasureById(
time_span TEXT DEFAULT NULL)
RETURNS NUMERIC AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
logger = Logger(user_obs_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasureById($1, $2, $3, $4, $5, $6) as measure;", ["text", "text", "text", "text", "text", "text"])
@ -518,12 +520,10 @@ RETURNS NUMERIC AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use OBS_GetMeasureById: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to OBS_GetMeasureById', sys.exc_info())
raise e
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;

24
server/extension/sql/115_data_observatory_exploration.sql
View File

@ -15,15 +15,17 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_Search(
relevant_boundary TEXT DEFAULT NULL)
RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
logger = Logger(user_obs_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_Search($1, $2, $3, $4);", ["text", "text", "text", "text"])
@ -43,12 +45,10 @@ RETURNS TABLE(id text, description text, name text, aggregate text, source text)
quota_service.increment_empty_service_use()
return [None, None, None, None, None]
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use OBS_Search: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to OBS_Search', sys.exc_info())
raise e
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@ -70,15 +70,17 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableBoundaries(
time_span TEXT DEFAULT NULL)
RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
logger = Logger(user_obs_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetAvailableBoundaries($1, $2, $3, $4) as available_boundaries;", ["text", "text", "geometry(Geometry, 4326)", "text"])
@ -97,12 +99,10 @@ RETURNS TABLE(boundary_id text, description text, time_span text, tablename text
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use OBS_GetAvailableBoundaries: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to OBS_GetMeasureById', sys.exc_info())
raise e
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;

84
server/extension/sql/120_data_observatory_geometries.sql
View File

@ -17,15 +17,17 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundary(
time_span TEXT DEFAULT NULL)
RETURNS geometry(Geometry, 4326) AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
logger = Logger(user_obs_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundary($1, $2, $3, $4) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"])
@ -37,12 +39,10 @@ RETURNS geometry(Geometry, 4326) AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use OBS_GetBoundary: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to OBS_GetBoundary', sys.exc_info())
raise e
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@ -66,15 +66,17 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryId(
time_span TEXT DEFAULT NULL)
RETURNS TEXT AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
logger = Logger(user_obs_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryId($1, $2, $3, $4, $5) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"])
@ -86,12 +88,10 @@ RETURNS TEXT AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use obs_search: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to OBS_GetBoundaryId', sys.exc_info())
raise e
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@ -115,15 +115,17 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryById(
time_span TEXT DEFAULT NULL)
RETURNS geometry(Geometry, 4326) AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
logger = Logger(user_obs_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryById($1, $2, $3, $4, $5) as boundary;", ["text", "text", "text", "text", "text"])
@ -135,12 +137,10 @@ RETURNS geometry(Geometry, 4326) AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use OBS_GetBoundaryById: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to OBS_GetBoundaryById', sys.exc_info())
raise e
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@ -166,15 +166,17 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByGeometry(
overlap_type TEXT DEFAULT 'intersects')
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
logger = Logger(user_obs_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"])
@ -191,12 +193,10 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use OBS_GetBoundariesByGeometry: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to OBS_GetBoundariesByGeometry', sys.exc_info())
raise e
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@ -224,15 +224,17 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByPointAndRa
overlap_type TEXT DEFAULT 'intersects')
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
logger = Logger(user_obs_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"])
@ -249,12 +251,10 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use OBS_GetBoundariesByPointAndRadius: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to OBS_GetBoundariesByPointAndRadius', sys.exc_info())
raise e
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@ -280,15 +280,17 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByGeometry(
overlap_type TEXT DEFAULT 'intersects')
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
logger = Logger(user_obs_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"])
@ -305,12 +307,10 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use OBS_GetPointsByGeometry: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to OBS_GetPointsByGeometry', sys.exc_info())
raise e
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@ -338,15 +338,17 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByPointAndRadius
overlap_type TEXT DEFAULT 'intersects')
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)]
logger = Logger(user_obs_config)
quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"])
@ -363,12 +365,10 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to use OBS_GetPointsByPointAndRadius: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to OBS_GetPointsByPointAndRadius', sys.exc_info())
raise e
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;

51
server/extension/sql/20_geocode_street.sql
View File

@ -7,19 +7,16 @@ RETURNS Geometry AS $$
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
if user_geocoder_config.heremaps_geocoder:
plpy.debug('Requested geocoder is heremaps')
here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
elif user_geocoder_config.google_geocoder:
plpy.debug('Requested geocoder is google')
google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
elif user_geocoder_config.mapzen_geocoder:
plpy.debug('Requested geocoder is mapzen')
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
else:
plpy.error('Requested geocoder is not available')
raise Exception('Requested geocoder is not available')
$$ LANGUAGE plpythonu;
@ -35,7 +32,7 @@ RETURNS Geometry AS $$
here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
else:
plpy.error('Here geocoder is not available for your account.')
raise Exception('Here geocoder is not available for your account.')
$$ LANGUAGE plpythonu;
@ -50,7 +47,7 @@ RETURNS Geometry AS $$
google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
else:
plpy.error('Google geocoder is not available for your account.')
raise Exception('Google geocoder is not available for your account.')
$$ LANGUAGE plpythonu;
@ -71,17 +68,19 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_poin
RETURNS Geometry AS $$
from cartodb_services.here import HereMapsGeocoder
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
logger = Logger(user_geocoder_config)
# -- Check the quota
quota_service = QuotaService(user_geocoder_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
geocoder = HereMapsGeocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code)
geocoder = HereMapsGeocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code, logger)
coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country)
if coordinates:
quota_service.increment_success_service_use()
@ -92,12 +91,10 @@ RETURNS Geometry AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using here maps geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to geocode street point using here maps', sys.exc_info())
raise e
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@ -106,13 +103,16 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_google_geocode_street_po
RETURNS Geometry AS $$
from cartodb_services.google import GoogleMapsGeocoder
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
logger = Logger(user_geocoder_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
geocoder = GoogleMapsGeocoder(user_geocoder_config.google_client_id, user_geocoder_config.google_api_key)
geocoder = GoogleMapsGeocoder(user_geocoder_config.google_client_id, user_geocoder_config.google_api_key, logger)
coordinates = geocoder.geocode(searchtext=searchtext, city=city, state=state_province, country=country)
if coordinates:
quota_service.increment_success_service_use()
@ -123,12 +123,10 @@ RETURNS Geometry AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using google maps geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to geocode street point using google maps', sys.exc_info())
raise e
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@ -138,15 +136,18 @@ RETURNS Geometry AS $$
from cartodb_services.mapzen import MapzenGeocoder
from cartodb_services.mapzen.types import country_to_iso3
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
logger = Logger(user_geocoder_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
geocoder = MapzenGeocoder(user_geocoder_config.mapzen_api_key)
geocoder = MapzenGeocoder(user_geocoder_config.mapzen_api_key, logger)
country_iso3 = None
if country:
country_iso3 = country_to_iso3(country)
@ -162,12 +163,10 @@ RETURNS Geometry AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using mapzen geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to geocode street point using mapzen', sys.exc_info())
raise e
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;

10
server/extension/sql/30_admin0.sql
View File

@ -2,12 +2,14 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(us
RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
logger = Logger(user_geocoder_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"])
@ -20,12 +22,10 @@ RETURNS Geometry AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to geocode admin0 polygon', sys.exc_info())
raise e
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;

20
server/extension/sql/40_admin1.sql
View File

@ -3,12 +3,14 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(us
RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
logger = Logger(user_geocoder_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1)) AS mypolygon", ["text"])
@ -21,12 +23,10 @@ RETURNS Geometry AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to geocode admin1 polygon', sys.exc_info())
raise e
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@ -36,12 +36,14 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(us
RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
logger = Logger(user_geocoder_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1), trim($2)) AS mypolygon", ["text", "text"])
@ -54,12 +56,10 @@ RETURNS Geometry AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to geocode admin1 polygon', sys.exc_info())
raise e
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;

31
server/extension/sql/50_namedplaces.sql
View File

@ -3,12 +3,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(
RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
logger = Logger(user_geocoder_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"])
@ -21,12 +24,10 @@ RETURNS Geometry AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to geocode namedplace point', sys.exc_info())
raise e
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@ -36,12 +37,14 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(
RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
logger = Logger(user_geocoder_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"])
@ -54,12 +57,10 @@ RETURNS Geometry AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to geocode namedplace point', sys.exc_info())
raise e
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@ -69,12 +70,14 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(
RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
logger = Logger(user_geocoder_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"])
@ -87,12 +90,10 @@ RETURNS Geometry AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to geocode namedplace point', sys.exc_info())
raise e
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;

40
server/extension/sql/60_postalcodes.sql
View File

@ -2,12 +2,14 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(
RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
logger = Logger(user_geocoder_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1)) AS mypoint", ["text"])
@ -20,12 +22,10 @@ RETURNS Geometry AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to geocode postal code point', sys.exc_info())
raise e
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@ -34,12 +34,14 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(
RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
logger = Logger(user_geocoder_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1), trim($2)) AS mypoint", ["TEXT", "TEXT"])
@ -52,12 +54,10 @@ RETURNS Geometry AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to geocode postal code point', sys.exc_info())
raise e
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@ -66,12 +66,14 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygo
RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
logger = Logger(user_geocoder_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1)) AS mypolygon", ["text"])
@ -84,12 +86,10 @@ RETURNS Geometry AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to geocode postal code polygon', sys.exc_info())
raise e
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
@ -98,12 +98,14 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygo
RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
logger = Logger(user_geocoder_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1), trim($2)) AS mypolygon", ["TEXT", "TEXT"])
@ -116,12 +118,10 @@ RETURNS Geometry AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to geocode postal code polygon', sys.exc_info())
raise e
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;

10
server/extension/sql/70_ips.sql
View File

@ -2,12 +2,14 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(u
RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
logger = Logger(user_geocoder_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_ipaddress_point(trim($1)) AS mypoint", ["TEXT"])
@ -20,12 +22,10 @@ RETURNS Geometry AS $$
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using admin0 geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to geocode postal code polygon', sys.exc_info())
raise e
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;

33
server/extension/sql/80_isolines_helper.sql
View File

@ -6,17 +6,20 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
from cartodb_services.here import HereMapsRoutingIsoline
from cartodb_services.metrics import QuotaService
from cartodb_services.here.types import geo_polyline_to_multipolygon
from cartodb_services.tools import Logger
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)]
logger = Logger(user_isolines_routing_config)
# -- Check the quota
quota_service = QuotaService(user_isolines_routing_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
client = HereMapsRoutingIsoline(user_isolines_routing_config.heremaps_app_id, user_isolines_routing_config.heremaps_app_code, base_url = HereMapsRoutingIsoline.PRODUCTION_ROUTING_BASE_URL)
client = HereMapsRoutingIsoline(user_isolines_routing_config.heremaps_app_id,
user_isolines_routing_config.heremaps_app_code, logger)
if source:
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
@ -44,12 +47,10 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
quota_service.increment_empty_service_use()
return []
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to obtain isodistances using here maps geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
logger.error('Error trying to get mapzen isolines', sys.exc_info())
raise e
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu SECURITY DEFINER;
@ -65,21 +66,22 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isolines(
options text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
import json
from cartodb_services.mapzen import MatrixClient
from cartodb_services.mapzen import MapzenIsolines
from cartodb_services.mapzen import MatrixClient, MapzenIsolines
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)]
logger = Logger(user_isolines_routing_config)
# -- Check the quota
quota_service = QuotaService(user_isolines_routing_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
raise Exception('You have reached the limit of your quota')
try:
client = MatrixClient(user_isolines_routing_config.mapzen_matrix_api_key)
mapzen_isolines = MapzenIsolines(client)
client = MatrixClient(user_isolines_routing_config.mapzen_matrix_api_key, logger)
mapzen_isolines = MapzenIsolines(client, logger)
if source:
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
@ -117,13 +119,10 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
quota_service.increment_isolines_service_use(len(isolines))
return result
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
import sys
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to obtain isolines using mapzen: {0}'.format(e)
plpy.debug(traceback.format_tb(traceback_))
logger.error('Error trying to get mapzen isolines', sys.exc_info())
raise e
#plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu SECURITY DEFINER;

6
server/extension/sql/85_isodistance.sql
View File

@ -6,18 +6,16 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
if user_isolines_config.google_services_user:
plpy.error('This service is not available for google service users.')
raise Exception('This service is not available for google service users.')
if user_isolines_config.heremaps_provider:
plpy.debug('Requested isolines provider is heremaps')
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(here_plan, [username, orgname, source, mode, range, options])
elif user_isolines_config.mapzen_provider:
plpy.debug('Requested isolines provider is mapzen')
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options])
else:
plpy.error('Requested isolines provider is not available')
raise Exception('Requested isolines provider is not available')
$$ LANGUAGE plpythonu;
-- heremaps isodistance

6
server/extension/sql/90_isochrone.sql
View File

@ -6,18 +6,16 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)]
if user_isolines_config.google_services_user:
plpy.error('This service is not available for google service users.')
raise Exception('This service is not available for google service users.')
if user_isolines_config.heremaps_provider:
plpy.debug('Requested isolines provider is heremaps')
here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(here_plan, [username, orgname, source, mode, range, options])
elif user_isolines_config.mapzen_provider:
plpy.debug('Requested isolines provider is mapzen')
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"])
return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options])
else:
plpy.error('Requested isolines provider is not available')
raise Exception('Requested isolines provider is not available')
$$ LANGUAGE plpythonu;
-- heremaps isochrone

3
server/lib/python/cartodb_services/cartodb_services/google/geocoder.py
View File

@ -9,11 +9,12 @@ from exceptions import MalformedResult
class GoogleMapsGeocoder:
"""A Google Maps Geocoder wrapper for python"""
def __init__(self, client_id, client_secret):
def __init__(self, client_id, client_secret, logger):
self.client_id = self._clean_client_id(client_id)
self.client_secret = client_secret
self.geocoder = googlemaps.Client(
client_id=self.client_id, client_secret=self.client_secret)
self._logger = logger
def geocode(self, searchtext, city=None, state=None,
country=None):

11
server/lib/python/cartodb_services/cartodb_services/here/geocoder.py
View File

@ -47,10 +47,11 @@ class HereMapsGeocoder:
'strictlanguagemode'
] + ADDRESS_PARAMS
def __init__(self, app_id, app_code, maxresults=DEFAULT_MAXRESULTS,
def __init__(self, app_id, app_code, logger, maxresults=DEFAULT_MAXRESULTS,
gen=DEFAULT_GEN, host=PRODUCTION_GEOCODE_JSON_URL):
self.app_id = app_id
self.app_code = app_code
self._logger = logger
self.maxresults = maxresults
self.gen = gen
self.host = host
@ -88,9 +89,15 @@ class HereMapsGeocoder:
if response.status_code == requests.codes.ok:
return json.loads(response.text)
elif response.status_code == requests.codes.bad_request:
self._logger.warning('Error 4xx trying to geocode street using HERE',
data={"response": response.json(), "params":
params})
return []
else:
response.raise_for_status()
self._logger.error('Error trying to geocode street using HERE',
data={"response": response.json(), "params":
params})
raise Exception('Error trying to geocode street using Here')
def _extract_lng_lat_from_result(self, result):
location = result['Location']

11
server/lib/python/cartodb_services/cartodb_services/here/routing.py
View File

@ -25,9 +25,11 @@ class HereMapsRoutingIsoline:
'quality'
]
def __init__(self, app_id, app_code, base_url=PRODUCTION_ROUTING_BASE_URL):
def __init__(self, app_id, app_code, logger,
base_url=PRODUCTION_ROUTING_BASE_URL):
self._app_id = app_id
self._app_code = app_code
self._logger = logger
self._url = "{0}{1}".format(base_url, self.ISOLINE_PATH)
def calculate_isodistance(self, source, mode, data_range, options=[]):
@ -54,7 +56,12 @@ class HereMapsRoutingIsoline:
elif response.status_code == requests.codes.bad_request:
return []
else:
response.raise_for_status()
self._logger.error('Error trying to calculate HERE isolines',
data={"response": response.json(), "source":
source, "mode": mode, "data_range":
data_range, "range_type": range_type,
"options": options})
raise Exception('Error trying to calculate HERE isolines')
def __parse_options(self, options):
return dict(option.split('=') for option in options)

5
server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py
View File

@ -12,9 +12,10 @@ class MapzenGeocoder:
BASE_URL = 'https://search.mapzen.com/v1/search'
def __init__(self, app_key, base_url=BASE_URL):
def __init__(self, app_key, logger, base_url=BASE_URL):
self._app_key = app_key
self._url = base_url
self._logger = logger
@qps_retry
def geocode(self, searchtext, city=None, state_province=None, country=None):
@ -25,7 +26,7 @@ class MapzenGeocoder:
elif response.status_code == requests.codes.bad_request:
return []
else:
response.raise_for_status()
raise Exception('Error trying to geocode {0} using mapzen'.format(searchtext))
def _build_requests_parameters(self, searchtext, city=None,
state_province=None, country=None):

13
server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py
View File

@ -2,6 +2,7 @@ import requests
import json
from qps import qps_retry
class MatrixClient:
"""
@ -42,8 +43,14 @@ class MatrixClient:
response = requests.get(self.ONE_TO_MANY_URL, params=request_params)
if not requests.codes.ok:
self._logger.warning('Error trying to get matrix distance from mapzen', data={"response": response, "locations": locations, "costing": costing})
response.raise_for_status() # raise exception if not 200 OK
self._logger.error('Error trying to get matrix distance from mapzen',
data={"response": response.json(), "locations":
locations, "costing": costing})
raise Exception('Error trying to get matrix distance from mapzen')
else:
self._logger.debug('Done get matrix distance from mapzen',
data={"response": response.json(),
"locations": locations,
"costing": costing})
return response.json()

9
server/lib/python/cartodb_services/cartodb_services/mapzen/routing.py
View File

@ -28,9 +28,10 @@ class MapzenRouting:
METRICS_UNITS = 'kilometers'
IMPERIAL_UNITS = 'miles'
def __init__(self, app_key, base_url=PRODUCTION_ROUTING_BASE_URL):
def __init__(self, app_key, logger, base_url=PRODUCTION_ROUTING_BASE_URL):
self._app_key = app_key
self._url = base_url
self._logger = logger
@qps_retry
def calculate_route_point_to_point(self, waypoints, mode,
@ -48,7 +49,11 @@ class MapzenRouting:
elif response.status_code == requests.codes.bad_request:
return MapzenRoutingResponse(None, None, None)
else:
response.raise_for_status()
self._logger.error('Error trying to calculate route using HERE',
data={"response": response.json(), "waypoints":
waypoints, "mode": mode, "options":
options})
raise Exception('Error trying to calculate route using HERE')
def __parse_options(self, options):
return dict(option.split('=') for option in options)

29
server/lib/python/cartodb_services/cartodb_services/metrics/config.py
View File

@ -17,6 +17,8 @@ class ServiceConfig(object):
self._db_config = ServicesDBConfig(db_conn, username, orgname)
self._environment = self._db_config._server_environment
self._rollbar_api_key = self._db_config._rollbar_api_key
self._log_file_path = self._db_config._log_file_path
self._min_log_level = self._db_config._min_log_level
if redis_connection:
self._redis_config = ServicesRedisConfig(redis_connection).build(
username, orgname)
@ -39,10 +41,19 @@ class ServiceConfig(object):
def rollbar_api_key(self):
return self._rollbar_api_key
@property
def log_file_path(self):
return self._log_file_path
@property
def min_log_level(self):
return self._min_log_level
@property
def environment(self):
return self._environment
class DataObservatoryConfig(ServiceConfig):
def __init__(self, redis_connection, db_conn, username, orgname=None):
@ -65,6 +76,7 @@ class DataObservatoryConfig(ServiceConfig):
def connection_str(self):
return self._connection_str
class ObservatorySnapshotConfig(DataObservatoryConfig):
SOFT_LIMIT_KEY = 'soft_obs_snapshot_limit'
@ -88,6 +100,7 @@ class ObservatorySnapshotConfig(DataObservatoryConfig):
def service_type(self):
return 'obs_snapshot'
class ObservatoryConfig(DataObservatoryConfig):
SOFT_LIMIT_KEY = 'soft_obs_general_limit'
@ -111,6 +124,7 @@ class ObservatoryConfig(DataObservatoryConfig):
def service_type(self):
return 'obs_general'
class RoutingConfig(ServiceConfig):
PERIOD_END_DATE = 'period_end_date'
@ -172,6 +186,7 @@ class IsolinesRoutingConfig(ServiceConfig):
if not self._isolines_provider:
self._isolines_provider = self.DEFAULT_PROVIDER
self._geocoder_provider = filtered_config[self.GEOCODER_PROVIDER_KEY].lower()
self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE])
if self._isolines_provider == self.HEREMAPS_PROVIDER:
self._isolines_quota = float(filtered_config[self.QUOTA_KEY])
self._heremaps_app_id = db_config.heremaps_isolines_app_id
@ -184,7 +199,6 @@ class IsolinesRoutingConfig(ServiceConfig):
self._mapzen_matrix_api_key = self._db_config.mapzen_matrix_api_key
self._isolines_quota = self._db_config.mapzen_matrix_monthly_quota
self._soft_isolines_limit = False
self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE])
@property
def service_type(self):
@ -473,10 +487,15 @@ class ServicesDBConfig:
else:
logger_conf = json.loads(logger_conf_json)
self._geocoder_log_path = logger_conf['geocoder_log_path']
self._rollbar_api_key = None
self._min_log_level = 'warning'
self._log_file_path = None
if 'min_log_level' in logger_conf:
self._min_log_level = logger_conf['min_log_level']
if 'rollbar_api_key' in logger_conf:
self._rollbar_api_key = logger_conf['rollbar_api_key']
else:
self._rollbar_api_key = None
if 'log_file_path' in logger_conf:
self._log_file_path = logger_conf['log_file_path']
def _get_conf(self, key):
try:
@ -542,6 +561,10 @@ class ServicesDBConfig:
def rollbar_api_key(self):
return self._rollbar_api_key
@property
def log_file_path(self):
return self._log_file_path
@property
def data_observatory_connection_str(self):
return self._data_observatory_connection_str

2
server/lib/python/cartodb_services/cartodb_services/tools/__init__.py
View File

@ -1,4 +1,4 @@
from redis_tools import RedisConnection, RedisDBConfig
from coordinates import Coordinate
from polyline import PolyLine
from log import Logger
from log import Logger

67
server/lib/python/cartodb_services/cartodb_services/tools/log.py
View File

@ -1,5 +1,7 @@
import plpy
import rollbar
import logging
import traceback
import sys
# Monkey path because plpython sys module doesn't have argv and rollbar
# package use it
@ -8,28 +10,51 @@ sys.__dict__['argv'] = []
class Logger:
LEVELS = {'debug': 1, 'info': 2, 'warning': 3, 'error': 4}
def __init__(self, config):
self._config = config
self._min_level = self.LEVELS[self._config.min_log_level]
# We need to set the handler blocking (synchronous) because
# spawn a thread from plpython interpreter don't work
rollbar.init(self._config.rollbar_api_key,
self._config.environment, handler='blocking')
if self._rollbar_activated():
rollbar.init(self._config.rollbar_api_key,
self._config.environment, handler='blocking')
if self._log_file_activated():
self._setup_log_file_config(self._config.log_file_path)
def debug(self, text, exception=None, data={}):
if not self._check_min_level('debug'):
return
self._send_to_rollbar('debug', text, exception, data)
self._send_to_log_file('debug', text, exception, data)
plpy.debug(text)
def info(self, text, exception=None, data={}):
if not self._check_min_level('info'):
return
self._send_to_rollbar('info', text, exception, data)
self._send_to_log_file('info', text, exception, data)
plpy.info(text)
def warning(self, text, exception=None, data={}):
if not self._check_min_level('warning'):
return
self._send_to_rollbar('warning', text, exception, data)
self._send_to_log_file('warning', text, exception, data)
plpy.warning(text)
def error(self, text, exception=None, data={}):
if not self._check_min_level('error'):
return
self._send_to_rollbar('error', text, exception, data)
plpy.error(text)
self._send_to_log_file('error', text, exception, data)
# Plpy.error and fatal raises exceptions and we only want to log an
# error, exceptions should be raise explicitly
plpy.warning(text)
def _check_min_level(self, level):
return True if self.LEVELS[level] >= self._min_level else False
def _send_to_rollbar(self, level, text, exception, data):
if self._rollbar_activated():
@ -43,5 +68,41 @@ class Logger:
plpy.warning('Error sending message/exception to rollbar: {0}'.
format(e))
def _send_to_log_file(self, level, text, exception, data):
extra_data = self._parse_log_extra_data(exception, data)
if level == 'debug':
logging.debug(text, extra=extra_data)
elif level == 'info':
logging.info(text, extra=extra_data)
elif level == 'warning':
logging.warning(text, extra=extra_data)
elif level == 'error':
logging.error(text, extra=extra_data)
def _parse_log_extra_data(self, exception, data):
if exception:
type_, value_, traceback_ = exception
exception_traceback = traceback.format_tb(traceback_)
extra_data = {"exception_type": type_, "exception_message": value_,
"exception_traceback": exception_traceback,
"log_data": data}
else:
extra_data = {"exception_type": '', "exception_message": '',
"exception_traceback": ''}
if data:
extra_data['data'] = data
else:
extra_data['data'] = ''
return extra_data
def _setup_log_file_config(self, log_file_path):
format_str = "%(asctime)s %(name)-12s %(levelname)-8s %(message)s %(data)s %(exception_type)s %(exception_message)s %(exception_traceback)s"
logging.basicConfig(filename=log_file_path, format=format_str, level=self._config.min_log_level.upper())
def _rollbar_activated(self):
return True if self._config.rollbar_api_key else False
def _log_file_activated(self):
return True if self._config.log_file_path else False

4
server/lib/python/cartodb_services/test/test_googlegeocoder.py
View File

@ -3,6 +3,7 @@
import unittest
import requests_mock
from mock import Mock
from cartodb_services.google import GoogleMapsGeocoder
from cartodb_services.google.exceptions import BadGeocodingParams
@ -89,8 +90,9 @@ class GoogleGeocoderTestCase(unittest.TestCase):
MALFORMED_RESPONSE = """{"manolo": "escobar"}"""
def setUp(self):
logger = Mock()
self.geocoder = GoogleMapsGeocoder('dummy_client_id',
'MgxyOFxjZXIyOGO52jJlMzEzY1Oqy4hsO49E')
'MgxyOFxjZXIyOGO52jJlMzEzY1Oqy4hsO49E', logger)
def test_geocode_address_with_valid_params(self, req_mock):
req_mock.register_uri('GET', self.GOOGLE_MAPS_GEOCODER_URL,

4
server/lib/python/cartodb_services/test/test_helper.py
View File

@ -1,5 +1,7 @@
from datetime import datetime, date
from mock import Mock
import sys
sys.modules['plpy'] = Mock()
def build_redis_user_config(redis_conn, username, quota=100, soft_limit=False,
@ -72,3 +74,5 @@ def _plpy_execute_side_effect(*args, **kwargs):
return [{'conf': '{"geocoder_log_path": "/dev/null"}'}]
elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('data_observatory_conf') as conf":
return [{'conf': '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}}'}]
elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('server_conf') as conf":
return [{'conf': '{"environment": "testing"}'}]

4
server/lib/python/cartodb_services/test/test_heremapsgeocoder.py
View File

@ -3,6 +3,7 @@
import unittest
import requests_mock
from mock import Mock
from cartodb_services.here import HereMapsGeocoder
from cartodb_services.here.exceptions import BadGeocodingParams
@ -102,7 +103,8 @@ class HereMapsGeocoderTestCase(unittest.TestCase):
MALFORMED_RESPONSE = """{"manolo": "escobar"}"""
def setUp(self):
self.geocoder = HereMapsGeocoder(None, None)
logger = Mock()
self.geocoder = HereMapsGeocoder(None, None, logger)
def test_geocode_address_with_valid_params(self, req_mock):
req_mock.register_uri('GET', HereMapsGeocoder.PRODUCTION_GEOCODE_JSON_URL,

4
server/lib/python/cartodb_services/test/test_heremapsrouting.py
View File

@ -3,6 +3,7 @@
import unittest
import requests_mock
from mock import Mock
from urlparse import urlparse, parse_qs
from cartodb_services.here import HereMapsRoutingIsoline
@ -127,7 +128,8 @@ class HereMapsRoutingIsolineTestCase(unittest.TestCase):
MALFORMED_RESPONSE = """{"manolo": "escobar"}"""
def setUp(self):
self.routing = HereMapsRoutingIsoline(None, None)
logger = Mock()
self.routing = HereMapsRoutingIsoline(None, None, logger)
self.isoline_url = "{0}{1}".format(HereMapsRoutingIsoline.PRODUCTION_ROUTING_BASE_URL,
HereMapsRoutingIsoline.ISOLINE_PATH)

6
server/lib/python/cartodb_services/test/test_mapzengeocoder.py
View File

@ -1,8 +1,9 @@
#!/usr/local/bin/python
# -*- coding: utf-8 -*-
import mock
import unittest
import requests_mock
from mock import Mock
from cartodb_services.mapzen import MapzenGeocoder
from cartodb_services.mapzen.exceptions import MalformedResult
@ -88,7 +89,8 @@ class MapzenGeocoderTestCase(unittest.TestCase):
MALFORMED_RESPONSE = """{"manolo": "escobar"}"""
def setUp(self):
self.geocoder = MapzenGeocoder('search-XXXXXXX')
logger = Mock()
self.geocoder = MapzenGeocoder('search-XXXXXXX', logger)
def test_geocode_address_with_valid_params(self, req_mock):
req_mock.register_uri('GET', self.MAPZEN_GEOCODER_URL,

4
server/lib/python/cartodb_services/test/test_mapzenisolines.py
View File

@ -1,4 +1,5 @@
import unittest
from mock import Mock
from cartodb_services.mapzen import MapzenIsolines
from math import radians, cos, sin, asin, sqrt
@ -10,6 +11,7 @@ It uses a mocked client, which returns the cost based on a very simple model:
just proportional to the distance from origin to the target point.
"""
class MatrixClientMock():
def __init__(self, speed):
@ -67,7 +69,7 @@ class MapzenIsolinesTestCase(unittest.TestCase):
def setUp(self):
speed = 4 # in km/h
matrix_client = MatrixClientMock(speed)
self.mapzen_isolines = MapzenIsolines(matrix_client)
self.mapzen_isolines = MapzenIsolines(matrix_client, Mock())
def test_calculate_isochrone(self):
origin = {"lat":40.744014,"lon":-73.990508}

4
server/lib/python/cartodb_services/test/test_mapzenrouting.py
View File

@ -6,6 +6,7 @@ import requests_mock
import re
from nose.tools import assert_raises
from urlparse import urlparse, parse_qs
from mock import Mock
from cartodb_services.mapzen import MapzenRouting, MapzenRoutingResponse
from cartodb_services.mapzen.exceptions import WrongParams
@ -99,7 +100,8 @@ class MapzenRoutingTestCase(unittest.TestCase):
MALFORMED_RESPONSE = """{"manolo": "escobar"}"""
def setUp(self):
self.routing = MapzenRouting('api_key')
logger = Mock()
self.routing = MapzenRouting('api_key', logger)
self.url = MapzenRouting.PRODUCTION_ROUTING_BASE_URL
def test_calculate_simple_routing_with_valid_params(self, req_mock):