From 9186eb5d5bbdef28fa783af73014ca7169cbfb0b Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 2 Aug 2016 17:28:48 +0200 Subject: [PATCH] Log part decoupled from the main config class --- server/extension/sql/100_routing_helper.sql | 5 +- .../sql/110_data_observatory_augmentation.sql | 50 +++++---- .../sql/115_data_observatory_exploration.sql | 10 +- .../sql/120_data_observatory_geometries.sql | 35 +++--- server/extension/sql/20_geocode_street.sql | 15 +-- server/extension/sql/30_admin0.sql | 5 +- server/extension/sql/40_admin1.sql | 10 +- server/extension/sql/50_namedplaces.sql | 15 +-- server/extension/sql/60_postalcodes.sql | 20 ++-- server/extension/sql/70_ips.sql | 5 +- server/extension/sql/80_isolines_helper.sql | 10 +- .../cartodb_services/metrics/config.py | 32 ------ .../cartodb_services/tools/__init__.py | 2 +- .../cartodb_services/tools/log.py | 100 +++++++++++++++--- 14 files changed, 196 insertions(+), 118 deletions(-) diff --git a/server/extension/sql/100_routing_helper.sql b/server/extension/sql/100_routing_helper.sql index cf72a90..96786a7 100644 --- a/server/extension/sql/100_routing_helper.sql +++ b/server/extension/sql/100_routing_helper.sql @@ -17,12 +17,13 @@ RETURNS cdb_dataservices_server.simple_route AS $$ from cartodb_services.mapzen.types import polyline_to_linestring from cartodb_services.metrics import QuotaService from cartodb_services.tools import Coordinate - from cartodb_services.tools import Logger + from cartodb_services.tools import Logger,LoggerConfig redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] user_routing_config = GD["user_routing_config_{0}".format(username)] - logger = Logger(user_routing_config) + logger_config = LoggerConfig(plpy) + logger = Logger(logger_config) quota_service = QuotaService(user_routing_config, redis_conn) if not quota_service.check_user_quota(): diff --git a/server/extension/sql/110_data_observatory_augmentation.sql b/server/extension/sql/110_data_observatory_augmentation.sql index f00b6f0..f3d19cc 100644 --- a/server/extension/sql/110_data_observatory_augmentation.sql +++ b/server/extension/sql/110_data_observatory_augmentation.sql @@ -35,7 +35,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot( geometry_level TEXT DEFAULT NULL) RETURNS json AS $$ from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger + from cartodb_services.tools import Logger,LoggerConfig import json plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) @@ -43,7 +43,8 @@ RETURNS json AS $$ plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] - logger = Logger(user_obs_config) + logger_config = LoggerConfig(plpy) + logger = Logger(logger_config) quota_service = QuotaService(user_obs_snapshot_config, redis_conn) if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') @@ -85,14 +86,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetDemographicSnapshot( geometry_level TEXT DEFAULT NULL) RETURNS SETOF JSON AS $$ from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger + from cartodb_services.tools import Logger,LoggerConfig plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] - logger = Logger(user_obs_config) + logger_config = LoggerConfig(plpy) + logger = Logger(logger_config) quota_service = QuotaService(user_obs_snapshot_config, redis_conn) if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') @@ -136,7 +138,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot( geometry_level TEXT DEFAULT NULL) RETURNS json AS $$ from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger + from cartodb_services.tools import Logger,LoggerConfig import json plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) @@ -144,7 +146,8 @@ RETURNS json AS $$ plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] - logger = Logger(user_obs_config) + logger_config = LoggerConfig(plpy) + logger = Logger(logger_config) quota_service = QuotaService(user_obs_snapshot_config, redis_conn) if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') @@ -184,14 +187,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetSegmentSnapshot( geometry_level TEXT DEFAULT NULL) RETURNS SETOF JSON AS $$ from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger + from cartodb_services.tools import Logger,LoggerConfig plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] - logger = Logger(user_obs_config) + logger_config = LoggerConfig(plpy) + logger = Logger(logger_config) quota_service = QuotaService(user_obs_snapshot_config, redis_conn) if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') @@ -241,14 +245,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasure( time_span TEXT DEFAULT NULL) RETURNS NUMERIC AS $$ from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger + from cartodb_services.tools import Logger,LoggerConfig plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_obs_config = GD["user_obs_config_{0}".format(username)] - logger = Logger(user_obs_config) + logger_config = LoggerConfig(plpy) + logger = Logger(logger_config) quota_service = QuotaService(user_obs_config, redis_conn) if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') @@ -292,14 +297,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetCategory( time_span TEXT DEFAULT NULL) RETURNS TEXT AS $$ from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger + from cartodb_services.tools import Logger,LoggerConfig plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_obs_config = GD["user_obs_config_{0}".format(username)] - logger = Logger(user_obs_config) + logger_config = LoggerConfig(plpy) + logger = Logger(logger_config) quota_service = QuotaService(user_obs_config, redis_conn) if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') @@ -345,14 +351,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusMeasure( time_span TEXT DEFAULT NULL) RETURNS NUMERIC AS $$ from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger + from cartodb_services.tools import Logger,LoggerConfig plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_obs_config = GD["user_obs_config_{0}".format(username)] - logger = Logger(user_obs_config) + logger_config = LoggerConfig(plpy) + logger = Logger(logger_config) quota_service = QuotaService(user_obs_config, redis_conn) if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') @@ -396,14 +403,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusCategory( time_span TEXT DEFAULT NULL) RETURNS TEXT AS $$ from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger + from cartodb_services.tools import Logger,LoggerConfig plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_obs_config = GD["user_obs_config_{0}".format(username)] - logger = Logger(user_obs_config) + logger_config = LoggerConfig(plpy) + logger = Logger(logger_config) quota_service = QuotaService(user_obs_config, redis_conn) if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') @@ -447,14 +455,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPopulation( time_span TEXT DEFAULT NULL) RETURNS NUMERIC AS $$ from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger + from cartodb_services.tools import Logger,LoggerConfig plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_obs_config = GD["user_obs_config_{0}".format(username)] - logger = Logger(user_obs_config) + logger_config = LoggerConfig(plpy) + logger = Logger(logger_config) quota_service = QuotaService(user_obs_config, redis_conn) if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') @@ -498,14 +507,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasureById( time_span TEXT DEFAULT NULL) RETURNS NUMERIC AS $$ from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger + from cartodb_services.tools import Logger,LoggerConfig plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_obs_config = GD["user_obs_config_{0}".format(username)] - logger = Logger(user_obs_config) + logger_config = LoggerConfig(plpy) + logger = Logger(logger_config) quota_service = QuotaService(user_obs_config, redis_conn) if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') diff --git a/server/extension/sql/115_data_observatory_exploration.sql b/server/extension/sql/115_data_observatory_exploration.sql index 42be50f..c6d0863 100644 --- a/server/extension/sql/115_data_observatory_exploration.sql +++ b/server/extension/sql/115_data_observatory_exploration.sql @@ -15,14 +15,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_Search( relevant_boundary TEXT DEFAULT NULL) RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger + from cartodb_services.tools import Logger,LoggerConfig plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_obs_config = GD["user_obs_config_{0}".format(username)] - logger = Logger(user_obs_config) + logger_config = LoggerConfig(plpy) + logger = Logger(logger_config) quota_service = QuotaService(user_obs_config, redis_conn) if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') @@ -70,14 +71,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableBoundaries( time_span TEXT DEFAULT NULL) RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger + from cartodb_services.tools import Logger,LoggerConfig plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_obs_config = GD["user_obs_config_{0}".format(username)] - logger = Logger(user_obs_config) + logger_config = LoggerConfig(plpy) + logger = Logger(logger_config) quota_service = QuotaService(user_obs_config, redis_conn) if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') diff --git a/server/extension/sql/120_data_observatory_geometries.sql b/server/extension/sql/120_data_observatory_geometries.sql index 110b481..f37d9a8 100644 --- a/server/extension/sql/120_data_observatory_geometries.sql +++ b/server/extension/sql/120_data_observatory_geometries.sql @@ -17,14 +17,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundary( time_span TEXT DEFAULT NULL) RETURNS geometry(Geometry, 4326) AS $$ from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger + from cartodb_services.tools import Logger,LoggerConfig plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_obs_config = GD["user_obs_config_{0}".format(username)] - logger = Logger(user_obs_config) + logger_config = LoggerConfig(plpy) + logger = Logger(logger_config) quota_service = QuotaService(user_obs_config, redis_conn) if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') @@ -66,14 +67,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryId( time_span TEXT DEFAULT NULL) RETURNS TEXT AS $$ from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger + from cartodb_services.tools import Logger,LoggerConfig plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_obs_config = GD["user_obs_config_{0}".format(username)] - logger = Logger(user_obs_config) + logger_config = LoggerConfig(plpy) + logger = Logger(logger_config) quota_service = QuotaService(user_obs_config, redis_conn) if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') @@ -115,14 +117,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryById( time_span TEXT DEFAULT NULL) RETURNS geometry(Geometry, 4326) AS $$ from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger + from cartodb_services.tools import Logger,LoggerConfig plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_obs_config = GD["user_obs_config_{0}".format(username)] - logger = Logger(user_obs_config) + logger_config = LoggerConfig(plpy) + logger = Logger(logger_config) quota_service = QuotaService(user_obs_config, redis_conn) if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') @@ -166,14 +169,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByGeometry( overlap_type TEXT DEFAULT 'intersects') RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger + from cartodb_services.tools import Logger,LoggerConfig plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_obs_config = GD["user_obs_config_{0}".format(username)] - logger = Logger(user_obs_config) + logger_config = LoggerConfig(plpy) + logger = Logger(logger_config) quota_service = QuotaService(user_obs_config, redis_conn) if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') @@ -224,14 +228,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByPointAndRa overlap_type TEXT DEFAULT 'intersects') RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger + from cartodb_services.tools import Logger,LoggerConfig plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_obs_config = GD["user_obs_config_{0}".format(username)] - logger = Logger(user_obs_config) + logger_config = LoggerConfig(plpy) + logger = Logger(logger_config) quota_service = QuotaService(user_obs_config, redis_conn) if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') @@ -280,14 +285,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByGeometry( overlap_type TEXT DEFAULT 'intersects') RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger + from cartodb_services.tools import Logger,LoggerConfig plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_obs_config = GD["user_obs_config_{0}".format(username)] - logger = Logger(user_obs_config) + logger_config = LoggerConfig(plpy) + logger = Logger(logger_config) quota_service = QuotaService(user_obs_config, redis_conn) if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') @@ -338,14 +344,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByPointAndRadius overlap_type TEXT DEFAULT 'intersects') RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger + from cartodb_services.tools import Logger,LoggerConfig plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_obs_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_obs_config = GD["user_obs_config_{0}".format(username)] - logger = Logger(user_obs_config) + logger_config = LoggerConfig(plpy) + logger = Logger(logger_config) quota_service = QuotaService(user_obs_config, redis_conn) if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') diff --git a/server/extension/sql/20_geocode_street.sql b/server/extension/sql/20_geocode_street.sql index edd6056..bda170e 100644 --- a/server/extension/sql/20_geocode_street.sql +++ b/server/extension/sql/20_geocode_street.sql @@ -68,12 +68,13 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_poin RETURNS Geometry AS $$ from cartodb_services.here import HereMapsGeocoder from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger + from cartodb_services.tools import Logger,LoggerConfig redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] - logger = Logger(user_geocoder_config) + logger_config = LoggerConfig(plpy) + logger = Logger(logger_config) # -- Check the quota quota_service = QuotaService(user_geocoder_config, redis_conn) if not quota_service.check_user_quota(): @@ -103,12 +104,13 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_google_geocode_street_po RETURNS Geometry AS $$ from cartodb_services.google import GoogleMapsGeocoder from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger + from cartodb_services.tools import Logger,LoggerConfig redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] - logger = Logger(user_geocoder_config) + logger_config = LoggerConfig(plpy) + logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) try: @@ -136,12 +138,13 @@ RETURNS Geometry AS $$ from cartodb_services.mapzen import MapzenGeocoder from cartodb_services.mapzen.types import country_to_iso3 from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger + from cartodb_services.tools import Logger,LoggerConfig redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] - logger = Logger(user_geocoder_config) + logger_config = LoggerConfig(plpy) + logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') diff --git a/server/extension/sql/30_admin0.sql b/server/extension/sql/30_admin0.sql index 93b9bea..1849344 100644 --- a/server/extension/sql/30_admin0.sql +++ b/server/extension/sql/30_admin0.sql @@ -2,14 +2,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(us RETURNS Geometry AS $$ from cartodb_services.metrics import QuotaService from cartodb_services.metrics import InternalGeocoderConfig - from cartodb_services.tools import Logger + from cartodb_services.tools import Logger,LoggerConfig plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] - logger = Logger(user_geocoder_config) + logger_config = LoggerConfig(plpy) + logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) try: plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"]) diff --git a/server/extension/sql/40_admin1.sql b/server/extension/sql/40_admin1.sql index 208875a..d061fbd 100644 --- a/server/extension/sql/40_admin1.sql +++ b/server/extension/sql/40_admin1.sql @@ -3,14 +3,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(us RETURNS Geometry AS $$ from cartodb_services.metrics import QuotaService from cartodb_services.metrics import InternalGeocoderConfig - from cartodb_services.tools import Logger + from cartodb_services.tools import Logger,LoggerConfig plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] - logger = Logger(user_geocoder_config) + logger_config = LoggerConfig(plpy) + logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) try: plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1)) AS mypolygon", ["text"]) @@ -36,14 +37,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(us RETURNS Geometry AS $$ from cartodb_services.metrics import QuotaService from cartodb_services.metrics import InternalGeocoderConfig - from cartodb_services.tools import Logger + from cartodb_services.tools import Logger,LoggerConfig plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] - logger = Logger(user_geocoder_config) + logger_config = LoggerConfig(plpy) + logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) try: plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1), trim($2)) AS mypolygon", ["text", "text"]) diff --git a/server/extension/sql/50_namedplaces.sql b/server/extension/sql/50_namedplaces.sql index 070b2fd..240f52e 100644 --- a/server/extension/sql/50_namedplaces.sql +++ b/server/extension/sql/50_namedplaces.sql @@ -3,7 +3,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point( RETURNS Geometry AS $$ from cartodb_services.metrics import QuotaService from cartodb_services.metrics import InternalGeocoderConfig - from cartodb_services.tools import Logger + from cartodb_services.tools import Logger,LoggerConfig plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) @@ -11,7 +11,8 @@ RETURNS Geometry AS $$ plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] - logger = Logger(user_geocoder_config) + logger_config = LoggerConfig(plpy) + logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) try: plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"]) @@ -37,14 +38,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point( RETURNS Geometry AS $$ from cartodb_services.metrics import QuotaService from cartodb_services.metrics import InternalGeocoderConfig - from cartodb_services.tools import Logger + from cartodb_services.tools import Logger,LoggerConfig plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] - logger = Logger(user_geocoder_config) + logger_config = LoggerConfig(plpy) + logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) try: plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"]) @@ -70,14 +72,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point( RETURNS Geometry AS $$ from cartodb_services.metrics import QuotaService from cartodb_services.metrics import InternalGeocoderConfig - from cartodb_services.tools import Logger + from cartodb_services.tools import Logger,LoggerConfig plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] - logger = Logger(user_geocoder_config) + logger_config = LoggerConfig(plpy) + logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) try: plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"]) diff --git a/server/extension/sql/60_postalcodes.sql b/server/extension/sql/60_postalcodes.sql index 6dd4ffd..328d27c 100644 --- a/server/extension/sql/60_postalcodes.sql +++ b/server/extension/sql/60_postalcodes.sql @@ -2,14 +2,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point( RETURNS Geometry AS $$ from cartodb_services.metrics import QuotaService from cartodb_services.metrics import InternalGeocoderConfig - from cartodb_services.tools import Logger + from cartodb_services.tools import Logger,LoggerConfig plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] - logger = Logger(user_geocoder_config) + logger_config = LoggerConfig(plpy) + logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) try: plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1)) AS mypoint", ["text"]) @@ -34,14 +35,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point( RETURNS Geometry AS $$ from cartodb_services.metrics import QuotaService from cartodb_services.metrics import InternalGeocoderConfig - from cartodb_services.tools import Logger + from cartodb_services.tools import Logger,LoggerConfig plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] - logger = Logger(user_geocoder_config) + logger_config = LoggerConfig(plpy) + logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) try: plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1), trim($2)) AS mypoint", ["TEXT", "TEXT"]) @@ -66,14 +68,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygo RETURNS Geometry AS $$ from cartodb_services.metrics import QuotaService from cartodb_services.metrics import InternalGeocoderConfig - from cartodb_services.tools import Logger + from cartodb_services.tools import Logger,LoggerConfig plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] - logger = Logger(user_geocoder_config) + logger_config = LoggerConfig(plpy) + logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) try: plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1)) AS mypolygon", ["text"]) @@ -98,14 +101,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygo RETURNS Geometry AS $$ from cartodb_services.metrics import QuotaService from cartodb_services.metrics import InternalGeocoderConfig - from cartodb_services.tools import Logger + from cartodb_services.tools import Logger,LoggerConfig plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] - logger = Logger(user_geocoder_config) + logger_config = LoggerConfig(plpy) + logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) try: plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1), trim($2)) AS mypolygon", ["TEXT", "TEXT"]) diff --git a/server/extension/sql/70_ips.sql b/server/extension/sql/70_ips.sql index 2cc9259..4d519b5 100644 --- a/server/extension/sql/70_ips.sql +++ b/server/extension/sql/70_ips.sql @@ -2,14 +2,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(u RETURNS Geometry AS $$ from cartodb_services.metrics import QuotaService from cartodb_services.metrics import InternalGeocoderConfig - from cartodb_services.tools import Logger + from cartodb_services.tools import Logger,LoggerConfig plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] - logger = Logger(user_geocoder_config) + logger_config = LoggerConfig(plpy) + logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) try: plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_ipaddress_point(trim($1)) AS mypoint", ["TEXT"]) diff --git a/server/extension/sql/80_isolines_helper.sql b/server/extension/sql/80_isolines_helper.sql index a3deb7b..17acbdf 100644 --- a/server/extension/sql/80_isolines_helper.sql +++ b/server/extension/sql/80_isolines_helper.sql @@ -6,12 +6,13 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ from cartodb_services.here import HereMapsRoutingIsoline from cartodb_services.metrics import QuotaService from cartodb_services.here.types import geo_polyline_to_multipolygon - from cartodb_services.tools import Logger + from cartodb_services.tools import Logger,LoggerConfig redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] - logger = Logger(user_isolines_routing_config) + logger_config = LoggerConfig(plpy) + logger = Logger(logger_config) # -- Check the quota quota_service = QuotaService(user_isolines_routing_config, redis_conn) if not quota_service.check_user_quota(): @@ -68,12 +69,13 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ import json from cartodb_services.mapzen import MatrixClient, MapzenIsolines from cartodb_services.metrics import QuotaService - from cartodb_services.tools import Logger + from cartodb_services.tools import Logger,LoggerConfig redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] user_isolines_routing_config = GD["user_isolines_routing_config_{0}".format(username)] - logger = Logger(user_isolines_routing_config) + logger_config = LoggerConfig(plpy) + logger = Logger(logger_config) # -- Check the quota quota_service = QuotaService(user_isolines_routing_config, redis_conn) if not quota_service.check_user_quota(): diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index 33dbb37..3c41cfe 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -16,9 +16,6 @@ class ServiceConfig(object): self._orgname = orgname self._db_config = ServicesDBConfig(db_conn, username, orgname) self._environment = self._db_config._server_environment - self._rollbar_api_key = self._db_config._rollbar_api_key - self._log_file_path = self._db_config._log_file_path - self._min_log_level = self._db_config._min_log_level if redis_connection: self._redis_config = ServicesRedisConfig(redis_connection).build( username, orgname) @@ -37,18 +34,6 @@ class ServiceConfig(object): def organization(self): return self._orgname - @property - def rollbar_api_key(self): - return self._rollbar_api_key - - @property - def log_file_path(self): - return self._log_file_path - - @property - def min_log_level(self): - return self._min_log_level - @property def environment(self): return self._environment @@ -487,15 +472,6 @@ class ServicesDBConfig: else: logger_conf = json.loads(logger_conf_json) self._geocoder_log_path = logger_conf['geocoder_log_path'] - self._rollbar_api_key = None - self._min_log_level = 'warning' - self._log_file_path = None - if 'min_log_level' in logger_conf: - self._min_log_level = logger_conf['min_log_level'] - if 'rollbar_api_key' in logger_conf: - self._rollbar_api_key = logger_conf['rollbar_api_key'] - if 'log_file_path' in logger_conf: - self._log_file_path = logger_conf['log_file_path'] def _get_conf(self, key): try: @@ -557,14 +533,6 @@ class ServicesDBConfig: def geocoder_log_path(self): return self._geocoder_log_path - @property - def rollbar_api_key(self): - return self._rollbar_api_key - - @property - def log_file_path(self): - return self._log_file_path - @property def data_observatory_connection_str(self): return self._data_observatory_connection_str diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/__init__.py b/server/lib/python/cartodb_services/cartodb_services/tools/__init__.py index 55a007e..7bea3e2 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tools/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/tools/__init__.py @@ -1,4 +1,4 @@ from redis_tools import RedisConnection, RedisDBConfig from coordinates import Coordinate from polyline import PolyLine -from log import Logger \ No newline at end of file +from log import Logger, LoggerConfig diff --git a/server/lib/python/cartodb_services/cartodb_services/tools/log.py b/server/lib/python/cartodb_services/cartodb_services/tools/log.py index dbd49ac..822a8d1 100644 --- a/server/lib/python/cartodb_services/cartodb_services/tools/log.py +++ b/server/lib/python/cartodb_services/cartodb_services/tools/log.py @@ -1,9 +1,10 @@ import plpy import rollbar import logging +import json import traceback import sys -# Monkey path because plpython sys module doesn't have argv and rollbar +# Monkey patch because plpython sys module doesn't have argv and rollbar # package use it sys.__dict__['argv'] = [] @@ -21,7 +22,7 @@ class Logger: rollbar.init(self._config.rollbar_api_key, self._config.environment, handler='blocking') if self._log_file_activated(): - self._setup_log_file_config(self._config.log_file_path) + self._file_logger = self._setup_file_logger(self._config.log_file_path) def debug(self, text, exception=None, data={}): if not self._check_min_level('debug'): @@ -69,15 +70,16 @@ class Logger: format(e)) def _send_to_log_file(self, level, text, exception, data): - extra_data = self._parse_log_extra_data(exception, data) - if level == 'debug': - logging.debug(text, extra=extra_data) - elif level == 'info': - logging.info(text, extra=extra_data) - elif level == 'warning': - logging.warning(text, extra=extra_data) - elif level == 'error': - logging.error(text, extra=extra_data) + if self._log_file_activated(): + extra_data = self._parse_log_extra_data(exception, data) + if level == 'debug': + self._file_logger.debug(text, extra=extra_data) + elif level == 'info': + self._file_logger.info(text, extra=extra_data) + elif level == 'warning': + self._file_logger.warning(text, extra=extra_data) + elif level == 'error': + self._file_logger.error(text, extra=extra_data) def _parse_log_extra_data(self, exception, data): if exception: @@ -97,12 +99,84 @@ class Logger: return extra_data - def _setup_log_file_config(self, log_file_path): + def _setup_file_logger(self, log_file_path): format_str = "%(asctime)s %(name)-12s %(levelname)-8s %(message)s %(data)s %(exception_type)s %(exception_message)s %(exception_traceback)s" - logging.basicConfig(filename=log_file_path, format=format_str, level=self._config.min_log_level.upper()) + logger = logging.getLogger('dataservices_file_logger') + handler = logging.FileHandler(log_file_path) + handler.setFormatter(format_str) + handler.setLevel(self._config.min_log_level.upper()) + logger.addHandler(handler) + + return logger def _rollbar_activated(self): return True if self._config.rollbar_api_key else False def _log_file_activated(self): return True if self._config.log_file_path else False + + +class ConfigException(Exception): + pass + + +class LoggerConfig: + + def __init__(self, db_conn): + self._db_conn = db_conn + return self._build() + + def _build(self): + self._get_server_config() + self._get_logger_config() + + def _get_server_config(self): + server_config_json = self._get_conf('server_conf') + if not server_config_json: + self._server_environment = 'development' + else: + server_config_json = json.loads(server_config_json) + if 'environment' in server_config_json: + self._server_environment = server_config_json['environment'] + else: + self._server_environment = 'development' + + def _get_logger_config(self): + logger_conf_json = self._get_conf('logger_conf') + if not logger_conf_json: + raise ConfigException('Logger configuration missing') + else: + logger_conf = json.loads(logger_conf_json) + self._rollbar_api_key = None + self._min_log_level = 'warning' + self._log_file_path = None + if 'min_log_level' in logger_conf: + self._min_log_level = logger_conf['min_log_level'] + if 'rollbar_api_key' in logger_conf: + self._rollbar_api_key = logger_conf['rollbar_api_key'] + if 'log_file_path' in logger_conf: + self._log_file_path = logger_conf['log_file_path'] + + def _get_conf(self, key): + try: + sql = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(key) + conf = self._db_conn.execute(sql, 1) + return conf[0]['conf'] + except Exception as e: + raise ConfigException("Malformed config for {0}: {1}".format(key, e)) + + @property + def environment(self): + return self._server_environment + + @property + def rollbar_api_key(self): + return self._rollbar_api_key + + @property + def log_file_path(self): + return self._log_file_path + + @property + def min_log_level(self): + return self._min_log_level