cartodb4/dataservices-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Log part decoupled from the main config class

Browse Source
This commit is contained in:
Mario de Frutos 2016-08-02 17:28:48 +02:00
parent ed5cf25e9c
commit 9186eb5d5b
14 changed files with 196 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
server/extension/sql/100_routing_helper.sql
View File

@ -17,12 +17,13 @@ RETURNS cdb_dataservices_server.simple_route AS $$
from cartodb_services.mapzen.types import polyline_to_linestring from cartodb_services.mapzen.types import polyline_to_linestring
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Coordinate from cartodb_services.tools import Coordinate
from cartodb_services.tools import Logger from cartodb_services.tools import Logger,LoggerConfig
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_routing_config = GD["user_routing_config_{0}".format(username)] user_routing_config = GD["user_routing_config_{0}".format(username)]
logger = Logger(user_routing_config) logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_routing_config, redis_conn) quota_service = QuotaService(user_routing_config, redis_conn)
if not quota_service.check_user_quota(): if not quota_service.check_user_quota():

50
server/extension/sql/110_data_observatory_augmentation.sql
View File

@ -35,7 +35,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot(
geometry_level TEXT DEFAULT NULL) geometry_level TEXT DEFAULT NULL)
RETURNS json AS $$ RETURNS json AS $$
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger from cartodb_services.tools import Logger,LoggerConfig
import json import json
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
@ -43,7 +43,8 @@ RETURNS json AS $$
plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]
logger = Logger(user_obs_config) logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_snapshot_config, redis_conn) quota_service = QuotaService(user_obs_snapshot_config, redis_conn)
if not quota_service.check_user_quota(): if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota') raise Exception('You have reached the limit of your quota')
@ -85,14 +86,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetDemographicSnapshot(
geometry_level TEXT DEFAULT NULL) geometry_level TEXT DEFAULT NULL)
RETURNS SETOF JSON AS $$ RETURNS SETOF JSON AS $$
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]
logger = Logger(user_obs_config) logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_snapshot_config, redis_conn) quota_service = QuotaService(user_obs_snapshot_config, redis_conn)
if not quota_service.check_user_quota(): if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota') raise Exception('You have reached the limit of your quota')
@ -136,7 +138,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot(
geometry_level TEXT DEFAULT NULL) geometry_level TEXT DEFAULT NULL)
RETURNS json AS $$ RETURNS json AS $$
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger from cartodb_services.tools import Logger,LoggerConfig
import json import json
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
@ -144,7 +146,8 @@ RETURNS json AS $$
plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]
logger = Logger(user_obs_config) logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_snapshot_config, redis_conn) quota_service = QuotaService(user_obs_snapshot_config, redis_conn)
if not quota_service.check_user_quota(): if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota') raise Exception('You have reached the limit of your quota')
@ -184,14 +187,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetSegmentSnapshot(
geometry_level TEXT DEFAULT NULL) geometry_level TEXT DEFAULT NULL)
RETURNS SETOF JSON AS $$ RETURNS SETOF JSON AS $$
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)]
logger = Logger(user_obs_config) logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_snapshot_config, redis_conn) quota_service = QuotaService(user_obs_snapshot_config, redis_conn)
if not quota_service.check_user_quota(): if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota') raise Exception('You have reached the limit of your quota')
@ -241,14 +245,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasure(
time_span TEXT DEFAULT NULL) time_span TEXT DEFAULT NULL)
RETURNS NUMERIC AS $$ RETURNS NUMERIC AS $$
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)] user_obs_config = GD["user_obs_config_{0}".format(username)]
logger = Logger(user_obs_config) logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn) quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota(): if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota') raise Exception('You have reached the limit of your quota')
@ -292,14 +297,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetCategory(
time_span TEXT DEFAULT NULL) time_span TEXT DEFAULT NULL)
RETURNS TEXT AS $$ RETURNS TEXT AS $$
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)] user_obs_config = GD["user_obs_config_{0}".format(username)]
logger = Logger(user_obs_config) logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn) quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota(): if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota') raise Exception('You have reached the limit of your quota')
@ -345,14 +351,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusMeasure(
time_span TEXT DEFAULT NULL) time_span TEXT DEFAULT NULL)
RETURNS NUMERIC AS $$ RETURNS NUMERIC AS $$
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)] user_obs_config = GD["user_obs_config_{0}".format(username)]
logger = Logger(user_obs_config) logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn) quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota(): if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota') raise Exception('You have reached the limit of your quota')
@ -396,14 +403,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusCategory(
time_span TEXT DEFAULT NULL) time_span TEXT DEFAULT NULL)
RETURNS TEXT AS $$ RETURNS TEXT AS $$
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)] user_obs_config = GD["user_obs_config_{0}".format(username)]
logger = Logger(user_obs_config) logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn) quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota(): if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota') raise Exception('You have reached the limit of your quota')
@ -447,14 +455,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPopulation(
time_span TEXT DEFAULT NULL) time_span TEXT DEFAULT NULL)
RETURNS NUMERIC AS $$ RETURNS NUMERIC AS $$
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)] user_obs_config = GD["user_obs_config_{0}".format(username)]
logger = Logger(user_obs_config) logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn) quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota(): if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota') raise Exception('You have reached the limit of your quota')
@ -498,14 +507,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasureById(
time_span TEXT DEFAULT NULL) time_span TEXT DEFAULT NULL)
RETURNS NUMERIC AS $$ RETURNS NUMERIC AS $$
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)] user_obs_config = GD["user_obs_config_{0}".format(username)]
logger = Logger(user_obs_config) logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn) quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota(): if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota') raise Exception('You have reached the limit of your quota')

10
server/extension/sql/115_data_observatory_exploration.sql
View File

@ -15,14 +15,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_Search(
relevant_boundary TEXT DEFAULT NULL) relevant_boundary TEXT DEFAULT NULL)
RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)] user_obs_config = GD["user_obs_config_{0}".format(username)]
logger = Logger(user_obs_config) logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn) quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota(): if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota') raise Exception('You have reached the limit of your quota')
@ -70,14 +71,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableBoundaries(
time_span TEXT DEFAULT NULL) time_span TEXT DEFAULT NULL)
RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)] user_obs_config = GD["user_obs_config_{0}".format(username)]
logger = Logger(user_obs_config) logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn) quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota(): if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota') raise Exception('You have reached the limit of your quota')

35
server/extension/sql/120_data_observatory_geometries.sql
View File

@ -17,14 +17,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundary(
time_span TEXT DEFAULT NULL) time_span TEXT DEFAULT NULL)
RETURNS geometry(Geometry, 4326) AS $$ RETURNS geometry(Geometry, 4326) AS $$
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)] user_obs_config = GD["user_obs_config_{0}".format(username)]
logger = Logger(user_obs_config) logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn) quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota(): if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota') raise Exception('You have reached the limit of your quota')
@ -66,14 +67,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryId(
time_span TEXT DEFAULT NULL) time_span TEXT DEFAULT NULL)
RETURNS TEXT AS $$ RETURNS TEXT AS $$
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)] user_obs_config = GD["user_obs_config_{0}".format(username)]
logger = Logger(user_obs_config) logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn) quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota(): if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota') raise Exception('You have reached the limit of your quota')
@ -115,14 +117,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryById(
time_span TEXT DEFAULT NULL) time_span TEXT DEFAULT NULL)
RETURNS geometry(Geometry, 4326) AS $$ RETURNS geometry(Geometry, 4326) AS $$
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)] user_obs_config = GD["user_obs_config_{0}".format(username)]
logger = Logger(user_obs_config) logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn) quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota(): if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota') raise Exception('You have reached the limit of your quota')
@ -166,14 +169,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByGeometry(
overlap_type TEXT DEFAULT 'intersects') overlap_type TEXT DEFAULT 'intersects')
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)] user_obs_config = GD["user_obs_config_{0}".format(username)]
logger = Logger(user_obs_config) logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn) quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota(): if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota') raise Exception('You have reached the limit of your quota')
@ -224,14 +228,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByPointAndRa
overlap_type TEXT DEFAULT 'intersects') overlap_type TEXT DEFAULT 'intersects')
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)] user_obs_config = GD["user_obs_config_{0}".format(username)]
logger = Logger(user_obs_config) logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn) quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota(): if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota') raise Exception('You have reached the limit of your quota')
@ -280,14 +285,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByGeometry(
overlap_type TEXT DEFAULT 'intersects') overlap_type TEXT DEFAULT 'intersects')
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)] user_obs_config = GD["user_obs_config_{0}".format(username)]
logger = Logger(user_obs_config) logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn) quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota(): if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota') raise Exception('You have reached the limit of your quota')
@ -338,14 +344,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByPointAndRadius
overlap_type TEXT DEFAULT 'intersects') overlap_type TEXT DEFAULT 'intersects')
RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_obs_config = GD["user_obs_config_{0}".format(username)] user_obs_config = GD["user_obs_config_{0}".format(username)]
logger = Logger(user_obs_config) logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_obs_config, redis_conn) quota_service = QuotaService(user_obs_config, redis_conn)
if not quota_service.check_user_quota(): if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota') raise Exception('You have reached the limit of your quota')

15
server/extension/sql/20_geocode_street.sql
View File

@ -68,12 +68,13 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_poin
RETURNS Geometry AS $$ RETURNS Geometry AS $$
from cartodb_services.here import HereMapsGeocoder from cartodb_services.here import HereMapsGeocoder
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger from cartodb_services.tools import Logger,LoggerConfig
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
logger = Logger(user_geocoder_config) logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
# -- Check the quota # -- Check the quota
quota_service = QuotaService(user_geocoder_config, redis_conn) quota_service = QuotaService(user_geocoder_config, redis_conn)
if not quota_service.check_user_quota(): if not quota_service.check_user_quota():
@ -103,12 +104,13 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_google_geocode_street_po
RETURNS Geometry AS $$ RETURNS Geometry AS $$
from cartodb_services.google import GoogleMapsGeocoder from cartodb_services.google import GoogleMapsGeocoder
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger from cartodb_services.tools import Logger,LoggerConfig
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
logger = Logger(user_geocoder_config) logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn) quota_service = QuotaService(user_geocoder_config, redis_conn)
try: try:
@ -136,12 +138,13 @@ RETURNS Geometry AS $$
from cartodb_services.mapzen import MapzenGeocoder from cartodb_services.mapzen import MapzenGeocoder
from cartodb_services.mapzen.types import country_to_iso3 from cartodb_services.mapzen.types import country_to_iso3
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger from cartodb_services.tools import Logger,LoggerConfig
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
logger = Logger(user_geocoder_config) logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn) quota_service = QuotaService(user_geocoder_config, redis_conn)
if not quota_service.check_user_quota(): if not quota_service.check_user_quota():
raise Exception('You have reached the limit of your quota') raise Exception('You have reached the limit of your quota')

5
server/extension/sql/30_admin0.sql
View File

@ -2,14 +2,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(us
RETURNS Geometry AS $$ RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
logger = Logger(user_geocoder_config) logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn) quota_service = QuotaService(user_geocoder_config, redis_conn)
try: try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"]) plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"])

10
server/extension/sql/40_admin1.sql
View File

@ -3,14 +3,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(us
RETURNS Geometry AS $$ RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
logger = Logger(user_geocoder_config) logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn) quota_service = QuotaService(user_geocoder_config, redis_conn)
try: try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1)) AS mypolygon", ["text"]) plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1)) AS mypolygon", ["text"])
@ -36,14 +37,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(us
RETURNS Geometry AS $$ RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
logger = Logger(user_geocoder_config) logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn) quota_service = QuotaService(user_geocoder_config, redis_conn)
try: try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1), trim($2)) AS mypolygon", ["text", "text"]) plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1), trim($2)) AS mypolygon", ["text", "text"])

15
server/extension/sql/50_namedplaces.sql
View File

@ -3,7 +3,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(
RETURNS Geometry AS $$ RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
@ -11,7 +11,8 @@ RETURNS Geometry AS $$
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
logger = Logger(user_geocoder_config) logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn) quota_service = QuotaService(user_geocoder_config, redis_conn)
try: try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"]) plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"])
@ -37,14 +38,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(
RETURNS Geometry AS $$ RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
logger = Logger(user_geocoder_config) logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn) quota_service = QuotaService(user_geocoder_config, redis_conn)
try: try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"]) plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"])
@ -70,14 +72,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(
RETURNS Geometry AS $$ RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
logger = Logger(user_geocoder_config) logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn) quota_service = QuotaService(user_geocoder_config, redis_conn)
try: try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"]) plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"])

20
server/extension/sql/60_postalcodes.sql
View File

@ -2,14 +2,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(
RETURNS Geometry AS $$ RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
logger = Logger(user_geocoder_config) logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn) quota_service = QuotaService(user_geocoder_config, redis_conn)
try: try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1)) AS mypoint", ["text"]) plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1)) AS mypoint", ["text"])
@ -34,14 +35,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(
RETURNS Geometry AS $$ RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
logger = Logger(user_geocoder_config) logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn) quota_service = QuotaService(user_geocoder_config, redis_conn)
try: try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1), trim($2)) AS mypoint", ["TEXT", "TEXT"]) plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1), trim($2)) AS mypoint", ["TEXT", "TEXT"])
@ -66,14 +68,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygo
RETURNS Geometry AS $$ RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
logger = Logger(user_geocoder_config) logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn) quota_service = QuotaService(user_geocoder_config, redis_conn)
try: try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1)) AS mypolygon", ["text"]) plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1)) AS mypolygon", ["text"])
@ -98,14 +101,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygo
RETURNS Geometry AS $$ RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
logger = Logger(user_geocoder_config) logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn) quota_service = QuotaService(user_geocoder_config, redis_conn)
try: try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1), trim($2)) AS mypolygon", ["TEXT", "TEXT"]) plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1), trim($2)) AS mypolygon", ["TEXT", "TEXT"])

5
server/extension/sql/70_ips.sql
View File

@ -2,14 +2,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(u
RETURNS Geometry AS $$ RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.tools import Logger from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
logger = Logger(user_geocoder_config) logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn) quota_service = QuotaService(user_geocoder_config, redis_conn)
try: try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_ipaddress_point(trim($1)) AS mypoint", ["TEXT"]) plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_ipaddress_point(trim($1)) AS mypoint", ["TEXT"])

10
server/extension/sql/80_isolines_helper.sql
View File

@ -6,12 +6,13 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
from cartodb_services.here import HereMapsRoutingIsoline from cartodb_services.here import HereMapsRoutingIsoline
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.here.types import geo_polyline_to_multipolygon from cartodb_services.here.types import geo_polyline_to_multipolygon
from cartodb_services.tools import Logger from cartodb_services.tools import Logger,LoggerConfig
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)]
logger = Logger(user_isolines_routing_config) logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
# -- Check the quota # -- Check the quota
quota_service = QuotaService(user_isolines_routing_config, redis_conn) quota_service = QuotaService(user_isolines_routing_config, redis_conn)
if not quota_service.check_user_quota(): if not quota_service.check_user_quota():
@ -68,12 +69,13 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
import json import json
from cartodb_services.mapzen import MatrixClient, MapzenIsolines from cartodb_services.mapzen import MatrixClient, MapzenIsolines
from cartodb_services.metrics import QuotaService from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger from cartodb_services.tools import Logger,LoggerConfig
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)]
logger = Logger(user_isolines_routing_config) logger_config = LoggerConfig(plpy)
logger = Logger(logger_config)
# -- Check the quota # -- Check the quota
quota_service = QuotaService(user_isolines_routing_config, redis_conn) quota_service = QuotaService(user_isolines_routing_config, redis_conn)
if not quota_service.check_user_quota(): if not quota_service.check_user_quota():

32
server/lib/python/cartodb_services/cartodb_services/metrics/config.py
View File

@ -16,9 +16,6 @@ class ServiceConfig(object):
self._orgname = orgname self._orgname = orgname
self._db_config = ServicesDBConfig(db_conn, username, orgname) self._db_config = ServicesDBConfig(db_conn, username, orgname)
self._environment = self._db_config._server_environment self._environment = self._db_config._server_environment
self._rollbar_api_key = self._db_config._rollbar_api_key
self._log_file_path = self._db_config._log_file_path
self._min_log_level = self._db_config._min_log_level
if redis_connection: if redis_connection:
self._redis_config = ServicesRedisConfig(redis_connection).build( self._redis_config = ServicesRedisConfig(redis_connection).build(
username, orgname) username, orgname)
@ -37,18 +34,6 @@ class ServiceConfig(object):
def organization(self): def organization(self):
return self._orgname return self._orgname
@property
def rollbar_api_key(self):
return self._rollbar_api_key
@property
def log_file_path(self):
return self._log_file_path
@property
def min_log_level(self):
return self._min_log_level
@property @property
def environment(self): def environment(self):
return self._environment return self._environment
@ -487,15 +472,6 @@ class ServicesDBConfig:
else: else:
logger_conf = json.loads(logger_conf_json) logger_conf = json.loads(logger_conf_json)
self._geocoder_log_path = logger_conf['geocoder_log_path'] self._geocoder_log_path = logger_conf['geocoder_log_path']
self._rollbar_api_key = None
self._min_log_level = 'warning'
self._log_file_path = None
if 'min_log_level' in logger_conf:
self._min_log_level = logger_conf['min_log_level']
if 'rollbar_api_key' in logger_conf:
self._rollbar_api_key = logger_conf['rollbar_api_key']
if 'log_file_path' in logger_conf:
self._log_file_path = logger_conf['log_file_path']
def _get_conf(self, key): def _get_conf(self, key):
try: try:
@ -557,14 +533,6 @@ class ServicesDBConfig:
def geocoder_log_path(self): def geocoder_log_path(self):
return self._geocoder_log_path return self._geocoder_log_path
@property
def rollbar_api_key(self):
return self._rollbar_api_key
@property
def log_file_path(self):
return self._log_file_path
@property @property
def data_observatory_connection_str(self): def data_observatory_connection_str(self):
return self._data_observatory_connection_str return self._data_observatory_connection_str

2
server/lib/python/cartodb_services/cartodb_services/tools/__init__.py
View File

@ -1,4 +1,4 @@
from redis_tools import RedisConnection, RedisDBConfig from redis_tools import RedisConnection, RedisDBConfig
from coordinates import Coordinate from coordinates import Coordinate
from polyline import PolyLine from polyline import PolyLine
from log import Logger from log import Logger, LoggerConfig

90
server/lib/python/cartodb_services/cartodb_services/tools/log.py
View File

@ -1,9 +1,10 @@
import plpy import plpy
import rollbar import rollbar
import logging import logging
import json
import traceback import traceback
import sys import sys
# Monkey path because plpython sys module doesn't have argv and rollbar # Monkey patch because plpython sys module doesn't have argv and rollbar
# package use it # package use it
sys.__dict__['argv'] = [] sys.__dict__['argv'] = []
@ -21,7 +22,7 @@ class Logger:
rollbar.init(self._config.rollbar_api_key, rollbar.init(self._config.rollbar_api_key,
self._config.environment, handler='blocking') self._config.environment, handler='blocking')
if self._log_file_activated(): if self._log_file_activated():
self._setup_log_file_config(self._config.log_file_path) self._file_logger = self._setup_file_logger(self._config.log_file_path)
def debug(self, text, exception=None, data={}): def debug(self, text, exception=None, data={}):
if not self._check_min_level('debug'): if not self._check_min_level('debug'):
@ -69,15 +70,16 @@ class Logger:
format(e)) format(e))
def _send_to_log_file(self, level, text, exception, data): def _send_to_log_file(self, level, text, exception, data):
if self._log_file_activated():
extra_data = self._parse_log_extra_data(exception, data) extra_data = self._parse_log_extra_data(exception, data)
if level == 'debug': if level == 'debug':
logging.debug(text, extra=extra_data) self._file_logger.debug(text, extra=extra_data)
elif level == 'info': elif level == 'info':
logging.info(text, extra=extra_data) self._file_logger.info(text, extra=extra_data)
elif level == 'warning': elif level == 'warning':
logging.warning(text, extra=extra_data) self._file_logger.warning(text, extra=extra_data)
elif level == 'error': elif level == 'error':
logging.error(text, extra=extra_data) self._file_logger.error(text, extra=extra_data)
def _parse_log_extra_data(self, exception, data): def _parse_log_extra_data(self, exception, data):
if exception: if exception:
@ -97,12 +99,84 @@ class Logger:
return extra_data return extra_data
def _setup_log_file_config(self, log_file_path): def _setup_file_logger(self, log_file_path):
format_str = "%(asctime)s %(name)-12s %(levelname)-8s %(message)s %(data)s %(exception_type)s %(exception_message)s %(exception_traceback)s" format_str = "%(asctime)s %(name)-12s %(levelname)-8s %(message)s %(data)s %(exception_type)s %(exception_message)s %(exception_traceback)s"
logging.basicConfig(filename=log_file_path, format=format_str, level=self._config.min_log_level.upper()) logger = logging.getLogger('dataservices_file_logger')
handler = logging.FileHandler(log_file_path)
handler.setFormatter(format_str)
handler.setLevel(self._config.min_log_level.upper())
logger.addHandler(handler)
return logger
def _rollbar_activated(self): def _rollbar_activated(self):
return True if self._config.rollbar_api_key else False return True if self._config.rollbar_api_key else False
def _log_file_activated(self): def _log_file_activated(self):
return True if self._config.log_file_path else False return True if self._config.log_file_path else False
class ConfigException(Exception):
pass
class LoggerConfig:
def __init__(self, db_conn):
self._db_conn = db_conn
return self._build()
def _build(self):
self._get_server_config()
self._get_logger_config()
def _get_server_config(self):
server_config_json = self._get_conf('server_conf')
if not server_config_json:
self._server_environment = 'development'
else:
server_config_json = json.loads(server_config_json)
if 'environment' in server_config_json:
self._server_environment = server_config_json['environment']
else:
self._server_environment = 'development'
def _get_logger_config(self):
logger_conf_json = self._get_conf('logger_conf')
if not logger_conf_json:
raise ConfigException('Logger configuration missing')
else:
logger_conf = json.loads(logger_conf_json)
self._rollbar_api_key = None
self._min_log_level = 'warning'
self._log_file_path = None
if 'min_log_level' in logger_conf:
self._min_log_level = logger_conf['min_log_level']
if 'rollbar_api_key' in logger_conf:
self._rollbar_api_key = logger_conf['rollbar_api_key']
if 'log_file_path' in logger_conf:
self._log_file_path = logger_conf['log_file_path']
def _get_conf(self, key):
try:
sql = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(key)
conf = self._db_conn.execute(sql, 1)
return conf[0]['conf']
except Exception as e:
raise ConfigException("Malformed config for {0}: {1}".format(key, e))
@property
def environment(self):
return self._server_environment
@property
def rollbar_api_key(self):
return self._rollbar_api_key
@property
def log_file_path(self):
return self._log_file_path
@property
def min_log_level(self):
return self._min_log_level