From 93579532e30b5b487ffa04a4d966720c64ac281b Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 28 Oct 2016 10:42:25 +0200 Subject: [PATCH 1/9] Move old versions --- .../cdb_dataservices_server--0.15.1--0.16.0.sql | 0 .../cdb_dataservices_server--0.16.0--0.15.1.sql | 0 .../{ => old_versions}/cdb_dataservices_server--0.16.0.sql | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.15.1--0.16.0.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.16.0--0.15.1.sql (100%) rename server/extension/{ => old_versions}/cdb_dataservices_server--0.16.0.sql (100%) diff --git a/server/extension/cdb_dataservices_server--0.15.1--0.16.0.sql b/server/extension/old_versions/cdb_dataservices_server--0.15.1--0.16.0.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.15.1--0.16.0.sql rename to server/extension/old_versions/cdb_dataservices_server--0.15.1--0.16.0.sql diff --git a/server/extension/cdb_dataservices_server--0.16.0--0.15.1.sql b/server/extension/old_versions/cdb_dataservices_server--0.16.0--0.15.1.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.16.0--0.15.1.sql rename to server/extension/old_versions/cdb_dataservices_server--0.16.0--0.15.1.sql diff --git a/server/extension/cdb_dataservices_server--0.16.0.sql b/server/extension/old_versions/cdb_dataservices_server--0.16.0.sql similarity index 100% rename from server/extension/cdb_dataservices_server--0.16.0.sql rename to server/extension/old_versions/cdb_dataservices_server--0.16.0.sql From cf3c6f2ce5afc8e6d42616bb3256776e5d6b50cd Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 28 Oct 2016 10:45:04 +0200 Subject: [PATCH 2/9] New metrics logger system - It's not mandatory to have the log_path defined for all the services - Now we add metrics to a MetricsGatherer using the add method in order to gather data from all the function scope, not just in one defined point - There is a `metrics` context manager to wrap the function block. This context manager get the execution time for the block among other things --- server/extension/sql/30_admin0.sql | 36 ++-- .../cartodb_services/mapzen/geocoder.py | 4 +- .../cartodb_services/mapzen/matrix_client.py | 4 +- .../cartodb_services/mapzen/routing.py | 4 +- .../cartodb_services/metrics/__init__.py | 1 + .../cartodb_services/metrics/config.py | 59 ++++-- .../cartodb_services/metrics/log.py | 195 +++++++++++++++--- .../cartodb_services/metrics/quota.py | 19 +- server/lib/python/cartodb_services/setup.py | 2 +- 9 files changed, 243 insertions(+), 81 deletions(-) diff --git a/server/extension/sql/30_admin0.sql b/server/extension/sql/30_admin0.sql index 10e8873..7b06f97 100644 --- a/server/extension/sql/30_admin0.sql +++ b/server/extension/sql/30_admin0.sql @@ -2,6 +2,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(us RETURNS Geometry AS $$ from cartodb_services.metrics import QuotaService from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.metrics import metrics from cartodb_services.tools import Logger,LoggerConfig plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) @@ -13,23 +14,24 @@ RETURNS Geometry AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) - try: - plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"]) - rv = plpy.execute(plan, [country_name], 1) - result = rv[0]["mypolygon"] - if result: - quota_service.increment_success_service_use() - return result - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to geocode admin0 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to geocode admin0 polygon') - finally: - quota_service.increment_total_service_use() + with metrics('cdb_geocode_admin0_polygon', user_geocoder_config): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin0 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin0 polygon') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py index 4cc974d..3ba3dff 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py @@ -5,9 +5,10 @@ import re from exceptions import WrongParams, MalformedResult, ServiceException from qps import qps_retry from cartodb_services.tools import Coordinate, PolyLine +from cartodb_services.metrics import Traceable -class MapzenGeocoder: +class MapzenGeocoder(Traceable): 'A Mapzen Geocoder wrapper for python' BASE_URL = 'https://search.mapzen.com/v1/search' @@ -28,6 +29,7 @@ class MapzenGeocoder: try: response = requests.get(self._url, params=request_params, timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT)) + self.add_response_data(response, self._logger) if response.status_code == requests.codes.ok: return self.__parse_response(response.text) elif response.status_code == requests.codes.bad_request: diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py index 98639e3..8147be1 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py @@ -2,9 +2,10 @@ import requests import json from qps import qps_retry from exceptions import ServiceException +from cartodb_services.metrics import Traceable -class MatrixClient: +class MatrixClient(Traceable): """ A minimal client for Mapzen Time-Distance Matrix Service @@ -45,6 +46,7 @@ class MatrixClient: } response = requests.get(self.ONE_TO_MANY_URL, params=request_params, timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT)) + self.add_response_data(response) if response.status_code != requests.codes.ok: self._logger.error('Error trying to get matrix distance from mapzen', diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/routing.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/routing.py index e0dca62..6c7c71b 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/routing.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/routing.py @@ -5,9 +5,10 @@ import re from exceptions import WrongParams, MalformedResult, ServiceException from qps import qps_retry from cartodb_services.tools import Coordinate, PolyLine +from cartodb_services.metrics import MetricsDataGatherer, Traceable -class MapzenRouting: +class MapzenRouting(Traceable): 'A Mapzen Routing wrapper for python' PRODUCTION_ROUTING_BASE_URL = 'https://valhalla.mapzen.com/route' @@ -47,6 +48,7 @@ class MapzenRouting: request_params = self.__parse_request_parameters(json_request_params) response = requests.get(self._url, params=request_params, timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT)) + self.add_response_data(response) if response.status_code == requests.codes.ok: return self.__parse_routing_response(response.text) elif response.status_code == requests.codes.bad_request: diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py b/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py index 078bbbd..b0a4181 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py @@ -1,3 +1,4 @@ from config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatorySnapshotConfig, ObservatoryConfig from quota import QuotaService from user import UserMetricsService +from log import metrics, MetricsDataGatherer, Traceable diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index 8c739fa..897b179 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -15,6 +15,7 @@ class ServiceConfig(object): self._username = username self._orgname = orgname self._db_config = ServicesDBConfig(db_conn, username, orgname) + self._metrics_log_path = self.__get_metrics_log_path() self._environment = self._db_config._server_environment if redis_connection: self._redis_config = ServicesRedisConfig(redis_connection).build( @@ -38,9 +39,20 @@ class ServiceConfig(object): def environment(self): return self._environment + @property + def metrics_log_path(self): + return self._metrics_log_path + + def __get_metrics_log_path(self): + if self.METRICS_LOG_KEY: + return self._db_config.logger_config.get(self.METRICS_LOG_KEY, None) + else: + return None class DataObservatoryConfig(ServiceConfig): + METRICS_LOG_KEY = 'do_log_path' + def __init__(self, redis_connection, db_conn, username, orgname=None): super(DataObservatoryConfig, self).__init__(redis_connection, db_conn, username, orgname) @@ -61,6 +73,10 @@ class DataObservatoryConfig(ServiceConfig): def connection_str(self): return self._connection_str + @property + def provider(self): + return 'data observatory' + class ObservatorySnapshotConfig(DataObservatoryConfig): @@ -118,6 +134,7 @@ class RoutingConfig(ServiceConfig): DEFAULT_PROVIDER = 'mapzen' QUOTA_KEY = 'mapzen_routing_quota' SOFT_LIMIT_KEY = 'soft_mapzen_routing_limit' + METRICS_LOG_KEY = 'routing_log_path' def __init__(self, redis_connection, db_conn, username, orgname=None): super(RoutingConfig, self).__init__(redis_connection, db_conn, @@ -135,6 +152,10 @@ class RoutingConfig(ServiceConfig): if self._routing_provider == self.MAPZEN_PROVIDER: return 'routing_mapzen' + @property + def provider(self): + return self._routing_provider + @property def mapzen_api_key(self): return self._mapzen_api_key @@ -151,7 +172,6 @@ class RoutingConfig(ServiceConfig): def soft_limit(self): return self._soft_limit - def _set_monthly_quota(self): self._monthly_quota = self._get_effective_monthly_quota() @@ -169,7 +189,6 @@ class RoutingConfig(ServiceConfig): self._soft_limit = False - class IsolinesRoutingConfig(ServiceConfig): ISOLINES_CONFIG_KEYS = ['here_isolines_quota', 'soft_here_isolines_limit', @@ -184,6 +203,7 @@ class IsolinesRoutingConfig(ServiceConfig): MAPZEN_PROVIDER = 'mapzen' HEREMAPS_PROVIDER = 'heremaps' DEFAULT_PROVIDER = 'heremaps' + METRICS_LOG_KEY = 'isolines_log_path' def __init__(self, redis_connection, db_conn, username, orgname=None): super(IsolinesRoutingConfig, self).__init__(redis_connection, db_conn, @@ -260,11 +280,12 @@ class IsolinesRoutingConfig(ServiceConfig): class InternalGeocoderConfig(ServiceConfig): + METRICS_LOG_KEY = 'geocoder_log_path' + def __init__(self, redis_connection, db_conn, username, orgname=None): # For now, internal geocoder doesn't use the redis config super(InternalGeocoderConfig, self).__init__(None, db_conn, username, orgname) - self._log_path = self._db_config.geocoder_log_path @property def service_type(self): @@ -283,8 +304,8 @@ class InternalGeocoderConfig(ServiceConfig): return None @property - def log_path(self): - return self._log_path + def provider(self): + return 'internal' class GeocoderConfig(ServiceConfig): @@ -310,6 +331,7 @@ class GeocoderConfig(ServiceConfig): ORGNAME_KEY = 'orgname' PERIOD_END_DATE = 'period_end_date' DEFAULT_PROVIDER = 'mapzen' + METRICS_LOG_KEY = 'geocoder_log_path' def __init__(self, redis_connection, db_conn, username, orgname=None, forced_provider=None): super(GeocoderConfig, self).__init__(redis_connection, db_conn, @@ -341,7 +363,6 @@ class GeocoderConfig(ServiceConfig): self._geocoder_provider = self.DEFAULT_PROVIDER self._geocoding_quota = float(filtered_config[self.QUOTA_KEY]) self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE]) - self._log_path = db_config.geocoder_log_path if filtered_config[self.SOFT_LIMIT_KEY].lower() == 'true': self._soft_geocoding_limit = True else: @@ -424,8 +445,8 @@ class GeocoderConfig(ServiceConfig): return self._cost_per_hit @property - def log_path(self): - return self._log_path + def provider(self): + return self._geocoder_provider class ServicesDBConfig: @@ -440,7 +461,6 @@ class ServicesDBConfig: self._get_server_config() self._get_here_config() self._get_mapzen_config() - self._get_logger_config() self._get_data_observatory_config() def _get_server_config(self): @@ -493,13 +513,6 @@ class ServicesDBConfig: else: self._data_observatory_connection_str = do_conf['connection']['production'] - def _get_logger_config(self): - logger_conf_json = self._get_conf('logger_conf') - if not logger_conf_json: - raise ConfigException('Logger configuration missing') - else: - logger_conf = json.loads(logger_conf_json) - self._geocoder_log_path = logger_conf['geocoder_log_path'] def _get_conf(self, key): try: @@ -507,7 +520,7 @@ class ServicesDBConfig: conf = self._db_conn.execute(sql, 1) return conf[0]['conf'] except Exception as e: - raise ConfigException("Malformed config for {0}: {1}".format(key, e)) + raise ConfigException("Error trying to get config for {0}: {1}".format(key, e)) @property def server_environment(self): @@ -557,14 +570,18 @@ class ServicesDBConfig: def mapzen_geocoder_monthly_quota(self): return self._mapzen_geocoder_quota - @property - def geocoder_log_path(self): - return self._geocoder_log_path - @property def data_observatory_connection_str(self): return self._data_observatory_connection_str + @property + def logger_config(self): + logger_conf_json = self._get_conf('logger_conf') + if not logger_conf_json: + raise ConfigException('Logger configuration missing') + else: + return json.loads(logger_conf_json) + class ServicesRedisConfig: diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/log.py b/server/lib/python/cartodb_services/cartodb_services/metrics/log.py index ab4739e..3b94c19 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/log.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/log.py @@ -2,14 +2,100 @@ from datetime import datetime import abc import json import re +import time +from contextlib import contextmanager -class MetricsLoggerFactory: +@contextmanager +def metrics(function, service_config): + try: + start_time = time.time() + yield + finally: + end_time = time.time() + MetricsDataGatherer.add('function_name', function) + MetricsDataGatherer.add('function_execution_time', (end_time - start_time)) + logger = MetricsServiceLoggerFactory.build(service_config) + if logger: + data = MetricsDataGatherer.get() + logger.log(data) + MetricsDataGatherer.clean() + + +class Traceable: + + def add_response_data(self, response): + try: + response_data = {} + response_data['time'] = response.elapsed.total_seconds() + response_data['code'] = response.status_code + response_data['message'] = response.reason + stored_data = MetricsDataGatherer.get_element('response') + if stored_data: + stored_data.append(response_data) + else: + MetricsDataGatherer.add('response', [response_data]) + except BaseException as e: + # We don't want to stop the job for some error here + plpy.warning(e) + + +class MetricsDataGatherer: + + class __MetricsDataGatherer: + def __init__(self): + self.data = {} + + def add(self, key, value): + self.data[key] = value + + def get(self): + return self.data + + def get_element(self, key): + return self.data.get(key, None) + + def clean(self): + self.data = {} + + __instance = None + + @classmethod + def add(self, key, value): + MetricsDataGatherer.instance().add(key, value) + + @classmethod + def get(self): + return MetricsDataGatherer.instance().get() + + @classmethod + def get_element(self, key): + return MetricsDataGatherer.instance().get_element(key) + + @classmethod + def clean(self): + MetricsDataGatherer.instance().clean() + + @classmethod + def instance(self): + if not MetricsDataGatherer.__instance: + MetricsDataGatherer.__instance = MetricsDataGatherer.__MetricsDataGatherer() + + return MetricsDataGatherer.__instance + + +class MetricsServiceLoggerFactory: @classmethod def build(self, service_config): - if re.match('geocoder_*', service_config.service_type): + if re.search('^geocoder_*', service_config.service_type): return MetricsGeocoderLogger(service_config) + elif re.search('^routing_*', service_config.service_type): + return MetricsGenericLogger(service_config) + elif re.search('_isolines$', service_config.service_type): + return MetricsIsolinesLogger(service_config) + elif re.search('^obs_*', service_config.service_type): + return MetricsGenericLogger(service_config) else: return None @@ -17,58 +103,103 @@ class MetricsLoggerFactory: class MetricsLogger(object): __metaclass__ = abc.ABCMeta - def __init__(self, file_path): - self._file_path = file_path + def __init__(self, service_config): + self._service_config = service_config def dump_to_file(self, data): - with open(self._file_path, 'a') as logfile: - json.dump(data, logfile) - logfile.write('\n') + log_path = self.service_config.metrics_log_path + if log_path: + with open(log_path, 'a') as logfile: + json.dump(data, logfile) + logfile.write('\n') + + def collect_data(self, data): + return { + "function_name": data.get('function_name', None), + "function_execution_time": data.get('function_execution_time', None), + "service": self._service_config.service_type, + "processable_rows": 1, + "success": data.get('success', False), + "successful_rows": data.get('successful_rows', 0), + "failed_rows": data.get('failed_rows', 0), + "empty_rows": data.get('empty_rows', 0), + "created_at": datetime.now().isoformat(), + "provider": self._service_config.provider, + "username": self._service_config.username, + "organization": self._service_config.organization, + "response": data.get('response', None) + } + + @property + def service_config(self): + return self._service_config @abc.abstractproperty - def log(self, **data): + def log(self, data): raise NotImplementedError('log method must be defined') class MetricsGeocoderLogger(MetricsLogger): def __init__(self, service_config): - super(MetricsGeocoderLogger, self).__init__(service_config.log_path) - self._service_config = service_config + super(MetricsGeocoderLogger, self).__init__(service_config) - def log(self, **data): - dump_data = self._dump_data(**data) + def log(self, data): + dump_data = self.collect_data(data) self.dump_to_file(dump_data) - def _dump_data(self, **data): - if data['success']: - cost = self._service_config.cost_per_hit - failed_rows = 0 - successful_rows = 1 + def collect_data(self, data): + dump_data = super(MetricsGeocoderLogger, self).collect_data(data) + if data.get('success', False): + cost = self.service_config.cost_per_hit else: cost = 0 - failed_rows = 1 - successful_rows = 0 - if self._service_config.is_high_resolution: + if self.service_config.is_high_resolution: kind = 'high-resolution' else: kind = 'internal' - return { + dump_data.update({ "batched": False, "cache_hits": 0, # Always 0 because no cache involved # https://github.com/CartoDB/cartodb/blob/master/app/models/geocoding.rb#L208-L211 "cost": cost, - "created_at": datetime.now().isoformat(), - "failed_rows": failed_rows, - "geocoder_type": self._service_config.service_type, + "geocoder_type": self.service_config.service_type, "kind": kind, - "processable_rows": 1, - "processed_rows": successful_rows, - "real_rows": successful_rows, - "success": data['success'], - "successful_rows": successful_rows, - "username": self._service_config.username, - "organization": self._service_config.organization - } + "processed_rows": data.get('successful_rows', 0), + "real_rows": data.get('successful_rows', 0), + }) + + return dump_data + + +class MetricsGenericLogger(MetricsLogger): + + def __init__(self, service_config): + super(MetricsGenericLogger, self).__init__(service_config) + + def log(self, data): + dump_data = self.collect_data(data) + self.dump_to_file(dump_data) + + def collect_data(self, data): + return super(MetricsGenericLogger, self).collect_data(data) + +class MetricsIsolinesLogger(MetricsLogger): + + def __init__(self, service_config): + super(MetricsIsolinesLogger, self).__init__(service_config) + + def log(self, data): + dump_data = self.collect_data(data) + self.dump_to_file(dump_data) + + def collect_data(self, data): + dump_data = super(MetricsIsolinesLogger, self).collect_data(data) + + dump_data.update({ + "isolines_generated": data.get('isolines_generated', 0) + }) + + return dump_data diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py index ea54508..b372ae9 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py @@ -1,5 +1,5 @@ from user import UserMetricsService -from log import MetricsLoggerFactory +from log import MetricsDataGatherer from datetime import date import re @@ -14,7 +14,6 @@ class QuotaService: redis_connection) self._user_service = UserMetricsService(self._user_service_config, redis_connection) - self._metrics_logger = MetricsLoggerFactory.build(user_service_config) def check_user_quota(self): return self._quota_checker.check() @@ -46,13 +45,19 @@ class QuotaService: self._user_service.increment_service_use( self._user_service_config.service_type, "isolines_generated", amount=amount) + MetricsDataGatherer.add('isolines_generated', amount) def _log_service_process(self, event): - if self._metrics_logger: - if event is 'success' or event is 'empty': - self._metrics_logger.log(success=True) - elif event is 'empty': - self._metrics_logger.log(success=False) + if event is 'success': + MetricsDataGatherer.add('success', True) + MetricsDataGatherer.add('successful_rows', 1) + elif event is 'empty': + MetricsDataGatherer.add('success', True) + MetricsDataGatherer.add('successful_rows', 1) + MetricsDataGatherer.add('empty_rows', 1) + elif event is 'fail': + MetricsDataGatherer.add('success', False) + MetricsDataGatherer.add('failed_rows', 1) class QuotaChecker: diff --git a/server/lib/python/cartodb_services/setup.py b/server/lib/python/cartodb_services/setup.py index dc075bc..bcdef1d 100644 --- a/server/lib/python/cartodb_services/setup.py +++ b/server/lib/python/cartodb_services/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages setup( name='cartodb_services', - version='0.9.4', + version='0.10.0', description='CartoDB Services API Python Library', From 708bee2912eb938f91658b8799bfc203564dad29 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 28 Oct 2016 18:12:04 +0200 Subject: [PATCH 3/9] Added metrics context manager to public functions --- ...db_dataservices_server--0.16.0--0.17.0.sql | 0 ...db_dataservices_server--0.17.0--0.16.0.sql | 0 .../extension/cdb_dataservices_server.control | 2 +- .../sql/105_route_between_points.sql | 20 +- .../sql/110_data_observatory_augmentation.sql | 356 +++++++++--------- .../sql/115_data_observatory_exploration.sql | 98 ++--- .../sql/120_data_observatory_geometries.sql | 278 +++++++------- server/extension/sql/20_geocode_street.sql | 24 +- server/extension/sql/40_admin1.sql | 78 ++-- server/extension/sql/50_namedplaces.sql | 101 ++--- server/extension/sql/60_postalcodes.sql | 144 +++---- server/extension/sql/70_ips.sql | 36 +- server/extension/sql/90_isochrone.sql | 19 +- 13 files changed, 612 insertions(+), 544 deletions(-) create mode 100644 server/extension/cdb_dataservices_server--0.16.0--0.17.0.sql create mode 100644 server/extension/cdb_dataservices_server--0.17.0--0.16.0.sql diff --git a/server/extension/cdb_dataservices_server--0.16.0--0.17.0.sql b/server/extension/cdb_dataservices_server--0.16.0--0.17.0.sql new file mode 100644 index 0000000..e69de29 diff --git a/server/extension/cdb_dataservices_server--0.17.0--0.16.0.sql b/server/extension/cdb_dataservices_server--0.17.0--0.16.0.sql new file mode 100644 index 0000000..e69de29 diff --git a/server/extension/cdb_dataservices_server.control b/server/extension/cdb_dataservices_server.control index 08a06f3..a84fc1c 100644 --- a/server/extension/cdb_dataservices_server.control +++ b/server/extension/cdb_dataservices_server.control @@ -1,5 +1,5 @@ comment = 'CartoDB dataservices server extension' -default_version = '0.16.0' +default_version = '0.17.0' requires = 'plpythonu, plproxy, postgis, cdb_geocoder' superuser = true schema = cdb_dataservices_server diff --git a/server/extension/sql/105_route_between_points.sql b/server/extension/sql/105_route_between_points.sql index cc17933..2762c5f 100644 --- a/server/extension/sql/105_route_between_points.sql +++ b/server/extension/sql/105_route_between_points.sql @@ -7,15 +7,18 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_point_to_point( options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers') RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_routing_config = GD["user_routing_config_{0}".format(username)] - waypoints = [origin, destination] - mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) - result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units]) - return [result[0]['shape'],result[0]['length'], result[0]['duration']] + with metrics('cdb_route_with_point', user_routing_config): + waypoints = [origin, destination] + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] $$ LANGUAGE plpythonu; @@ -27,12 +30,15 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_route_with_waypoints( options text[] DEFAULT ARRAY[]::text[], units text DEFAULT 'kilometers') RETURNS cdb_dataservices_server.simple_route AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_routing_config = GD["user_routing_config_{0}".format(username)] - mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) - result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units]) - return [result[0]['shape'],result[0]['length'], result[0]['duration']] + with metrics('cdb_route_with_waypoints', user_routing_config): + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) + result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units]) + return [result[0]['shape'],result[0]['length'], result[0]['duration']] $$ LANGUAGE plpythonu; diff --git a/server/extension/sql/110_data_observatory_augmentation.sql b/server/extension/sql/110_data_observatory_augmentation.sql index d75bcb1..4ccadfa 100644 --- a/server/extension/sql/110_data_observatory_augmentation.sql +++ b/server/extension/sql/110_data_observatory_augmentation.sql @@ -34,6 +34,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_demographic_snapshot( time_span TEXT DEFAULT NULL, geometry_level TEXT DEFAULT NULL) RETURNS json AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.tools import Logger,LoggerConfig import json @@ -50,22 +51,23 @@ RETURNS json AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - try: - obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshotJSON($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) - if result: - quota_service.increment_success_service_use() - return result[0]['snapshot'] - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to obs_get_demographic_snapshot') - finally: - quota_service.increment_total_service_use() + with metrics('obs_getdemographicsnapshot', user_obs_snapshot_config): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshotJSON($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_demographic_snapshot') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshot( @@ -86,6 +88,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetDemographicSnapshot( time_span TEXT DEFAULT NULL, geometry_level TEXT DEFAULT NULL) RETURNS SETOF JSON AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.tools import Logger,LoggerConfig @@ -101,26 +104,27 @@ RETURNS SETOF JSON AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - try: - obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) - if result: - resp = [] - for element in result: - value = element['snapshot'] - resp.append(value) - quota_service.increment_success_service_use() - return resp - else: - quota_service.increment_empty_service_use() - return [] - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to obs_get_demographic_snapshot') - finally: - quota_service.increment_total_service_use() + with metrics('obs_getdemographicsnapshot', user_obs_snapshot_config): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) + if result: + resp = [] + for element in result: + value = element['snapshot'] + resp.append(value) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_demographic_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_demographic_snapshot') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshotJSON( @@ -139,6 +143,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_get_segment_snapshot( geom geometry(Geometry, 4326), geometry_level TEXT DEFAULT NULL) RETURNS json AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.tools import Logger,LoggerConfig import json @@ -155,22 +160,23 @@ RETURNS json AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - try: - obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshotJSON($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) - if result: - quota_service.increment_success_service_use() - return result[0]['snapshot'] - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to obs_get_segment_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to obs_get_segment_snapshot') - finally: - quota_service.increment_total_service_use() + with metrics('obs_getsegmentsnapshot', user_obs_snapshot_config): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshotJSON($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) + if result: + quota_service.increment_success_service_use() + return result[0]['snapshot'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to obs_get_segment_snapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to obs_get_segment_snapshot') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetSegmentSnapshot( @@ -189,6 +195,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetSegmentSnapshot( geom geometry(Geometry, 4326), geometry_level TEXT DEFAULT NULL) RETURNS SETOF JSON AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.tools import Logger,LoggerConfig @@ -204,26 +211,27 @@ RETURNS SETOF JSON AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - try: - obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) - if result: - resp = [] - for element in result: - value = element['snapshot'] - resp.append(value) - quota_service.increment_success_service_use() - return resp - else: - quota_service.increment_empty_service_use() - return [] - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetSegmentSnapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_GetSegmentSnapshot') - finally: - quota_service.increment_total_service_use() + with metrics('obs_getsegmentsnapshot', user_obs_snapshot_config): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) + if result: + resp = [] + for element in result: + value = element['snapshot'] + resp.append(value) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetSegmentSnapshot', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetSegmentSnapshot') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasure( @@ -248,6 +256,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasure( boundary_id TEXT DEFAULT NULL, time_span TEXT DEFAULT NULL) RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.tools import Logger,LoggerConfig @@ -263,22 +272,23 @@ RETURNS NUMERIC AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - try: - obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasure($1, $2, $3, $4, $5, $6, $7) as measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, measure_id, normalize, boundary_id, time_span]) - if result: - quota_service.increment_success_service_use() - return result[0]['measure'] - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetMeasure', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_GetMeasure') - finally: - quota_service.increment_total_service_use() + with metrics('obs_getmeasure', user_obs_snapshot_config): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasure($1, $2, $3, $4, $5, $6, $7) as measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, measure_id, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasure', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasure') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetCategory( @@ -301,6 +311,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetCategory( boundary_id TEXT DEFAULT NULL, time_span TEXT DEFAULT NULL) RETURNS TEXT AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.tools import Logger,LoggerConfig @@ -316,22 +327,23 @@ RETURNS TEXT AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - try: - obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetCategory($1, $2, $3, $4, $5, $6) as category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, category_id, boundary_id, time_span]) - if result: - quota_service.increment_success_service_use() - return result[0]['category'] - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetCategory', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_GetCategory') - finally: - quota_service.increment_total_service_use() + with metrics('obs_getcategory', user_obs_snapshot_config): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetCategory($1, $2, $3, $4, $5, $6) as category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, category_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['category'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetCategory', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetCategory') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusMeasure( @@ -356,6 +368,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusMeasure( boundary_id TEXT DEFAULT NULL, time_span TEXT DEFAULT NULL) RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.tools import Logger,LoggerConfig @@ -371,22 +384,23 @@ RETURNS NUMERIC AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - try: - obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusMeasure($1, $2, $3, $4, $5, $6, $7) as census_measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, name, normalize, boundary_id, time_span]) - if result: - quota_service.increment_success_service_use() - return result[0]['census_measure'] - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetUSCensusMeasure', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_GetUSCensusMeasure') - finally: - quota_service.increment_total_service_use() + with metrics('obs_getuscensusmeasure', user_obs_snapshot_config): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusMeasure($1, $2, $3, $4, $5, $6, $7) as census_measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, name, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['census_measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetUSCensusMeasure', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetUSCensusMeasure') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetUSCensusCategory( @@ -409,6 +423,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetUSCensusCategory( boundary_id TEXT DEFAULT NULL, time_span TEXT DEFAULT NULL) RETURNS TEXT AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.tools import Logger,LoggerConfig @@ -424,22 +439,23 @@ RETURNS TEXT AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - try: - obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusCategory($1, $2, $3, $4, $5, $6) as census_category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, name, boundary_id, time_span]) - if result: - quota_service.increment_success_service_use() - return result[0]['census_category'] - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetUSCensusCategory', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_GetUSCensusCategory') - finally: - quota_service.increment_total_service_use() + with metrics('obs_getuscensuscategory', user_obs_snapshot_config): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusCategory($1, $2, $3, $4, $5, $6) as census_category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, name, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['census_category'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetUSCensusCategory', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetUSCensusCategory') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPopulation( @@ -462,6 +478,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPopulation( boundary_id TEXT DEFAULT NULL, time_span TEXT DEFAULT NULL) RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.tools import Logger,LoggerConfig @@ -477,22 +494,23 @@ RETURNS NUMERIC AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - try: - obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetPopulation($1, $2, $3, $4, $5, $6) as population;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, normalize, boundary_id, time_span]) - if result: - quota_service.increment_success_service_use() - return result[0]['population'] - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetPopulation', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_GetPopulation') - finally: - quota_service.increment_total_service_use() + with metrics('obs_getpopulation', user_obs_snapshot_config): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetPopulation($1, $2, $3, $4, $5, $6) as population;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, normalize, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['population'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPopulation', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPopulation') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetMeasureById( @@ -515,6 +533,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetMeasureById( boundary_id TEXT, time_span TEXT DEFAULT NULL) RETURNS NUMERIC AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.tools import Logger,LoggerConfig @@ -530,20 +549,21 @@ RETURNS NUMERIC AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - try: - obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasureById($1, $2, $3, $4, $5, $6) as measure;", ["text", "text", "text", "text", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom_ref, measure_id, boundary_id, time_span]) - if result: - quota_service.increment_success_service_use() - return result[0]['measure'] - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_GetMeasureById') - finally: - quota_service.increment_total_service_use() + with metrics('obs_getmeasurebyid', user_obs_snapshot_config): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasureById($1, $2, $3, $4, $5, $6) as measure;", ["text", "text", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom_ref, measure_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['measure'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasureById') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; diff --git a/server/extension/sql/115_data_observatory_exploration.sql b/server/extension/sql/115_data_observatory_exploration.sql index 301e0bb..05da6d8 100644 --- a/server/extension/sql/115_data_observatory_exploration.sql +++ b/server/extension/sql/115_data_observatory_exploration.sql @@ -14,6 +14,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_Search( search_term TEXT, relevant_boundary TEXT DEFAULT NULL) RETURNS TABLE(id text, description text, name text, aggregate text, source text) AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.tools import Logger,LoggerConfig @@ -29,30 +30,31 @@ RETURNS TABLE(id text, description text, name text, aggregate text, source text) if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - try: - obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_Search($1, $2, $3, $4);", ["text", "text", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, search_term, relevant_boundary]) - if result: - resp = [] - for element in result: - id = element['id'] - description = element['description'] - name = element['name'] - aggregate = element['aggregate'] - source = element['source'] - resp.append([id, description, name, aggregate, source]) - quota_service.increment_success_service_use() - return resp - else: - quota_service.increment_empty_service_use() - return [None, None, None, None, None] - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_Search', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_Search') - finally: - quota_service.increment_total_service_use() + with metrics('obs_search', user_obs_snapshot_config): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_Search($1, $2, $3, $4);", ["text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, search_term, relevant_boundary]) + if result: + resp = [] + for element in result: + id = element['id'] + description = element['description'] + name = element['name'] + aggregate = element['aggregate'] + source = element['source'] + resp.append([id, description, name, aggregate, source]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [None, None, None, None, None] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_Search', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_Search') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetAvailableBoundaries( @@ -71,6 +73,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableBoundaries( geom geometry(Geometry, 4326), time_span TEXT DEFAULT NULL) RETURNS TABLE(boundary_id text, description text, time_span text, tablename text) AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.tools import Logger,LoggerConfig @@ -86,27 +89,28 @@ RETURNS TABLE(boundary_id text, description text, time_span text, tablename text if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - try: - obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetAvailableBoundaries($1, $2, $3, $4) as available_boundaries;", ["text", "text", "geometry(Geometry, 4326)", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, time_span]) - if result: - resp = [] - for element in result: - id = element['boundary_id'] - description = element['description'] - tspan = element['time_span'] - tablename = element['tablename'] - resp.append([id, description, tspan, tablename]) - quota_service.increment_success_service_use() - return resp - else: - quota_service.increment_empty_service_use() - return [] - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_GetMeasureById') - finally: - quota_service.increment_total_service_use() + with metrics('obs_getavailableboundaries', user_obs_snapshot_config): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetAvailableBoundaries($1, $2, $3, $4) as available_boundaries;", ["text", "text", "geometry(Geometry, 4326)", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, time_span]) + if result: + resp = [] + for element in result: + id = element['boundary_id'] + description = element['description'] + tspan = element['time_span'] + tablename = element['tablename'] + resp.append([id, description, tspan, tablename]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetMeasureById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetMeasureById') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; diff --git a/server/extension/sql/120_data_observatory_geometries.sql b/server/extension/sql/120_data_observatory_geometries.sql index f7b815f..bfc09fc 100644 --- a/server/extension/sql/120_data_observatory_geometries.sql +++ b/server/extension/sql/120_data_observatory_geometries.sql @@ -16,6 +16,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundary( boundary_id TEXT, time_span TEXT DEFAULT NULL) RETURNS geometry(Geometry, 4326) AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.tools import Logger,LoggerConfig @@ -31,22 +32,23 @@ RETURNS geometry(Geometry, 4326) AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - try: - obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundary($1, $2, $3, $4) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) - if result: - quota_service.increment_success_service_use() - return result[0]['boundary'] - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetBoundary', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_GetBoundary') - finally: - quota_service.increment_total_service_use() + with metrics('obs_getboundary', user_obs_snapshot_config): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundary($1, $2, $3, $4) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundary', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundary') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryId( @@ -67,6 +69,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryId( boundary_id TEXT, time_span TEXT DEFAULT NULL) RETURNS TEXT AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.tools import Logger,LoggerConfig @@ -82,22 +85,23 @@ RETURNS TEXT AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - try: - obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryId($1, $2, $3, $4, $5) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) - if result: - quota_service.increment_success_service_use() - return result[0]['boundary'] - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetBoundaryId', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_GetBoundaryId') - finally: - quota_service.increment_total_service_use() + with metrics('obs_getboundaryid', user_obs_snapshot_config): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryId($1, $2, $3, $4, $5) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundaryId', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundaryId') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundaryById( @@ -118,6 +122,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundaryById( boundary_id TEXT, time_span TEXT DEFAULT NULL) RETURNS geometry(Geometry, 4326) AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.tools import Logger,LoggerConfig @@ -133,22 +138,23 @@ RETURNS geometry(Geometry, 4326) AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - try: - obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryById($1, $2, $3, $4, $5) as boundary;", ["text", "text", "text", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geometry_id, boundary_id, time_span]) - if result: - quota_service.increment_success_service_use() - return result[0]['boundary'] - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetBoundaryById', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_GetBoundaryById') - finally: - quota_service.increment_total_service_use() + with metrics('obs_getboundarybyid', user_obs_snapshot_config): + try: + obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryById($1, $2, $3, $4, $5) as boundary;", ["text", "text", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geometry_id, boundary_id, time_span]) + if result: + quota_service.increment_success_service_use() + return result[0]['boundary'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundaryById', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundaryById') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByGeometry( @@ -171,6 +177,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByGeometry( time_span TEXT DEFAULT NULL, overlap_type TEXT DEFAULT NULL) RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.tools import Logger,LoggerConfig @@ -186,27 +193,28 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - try: - obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) - if result: - resp = [] - for element in result: - the_geom = element['the_geom'] - geom_refs = element['geom_refs'] - resp.append([the_geom, geom_refs]) - quota_service.increment_success_service_use() - return resp - else: - quota_service.increment_empty_service_use() - return [] - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetBoundariesByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_GetBoundariesByGeometry') - finally: - quota_service.increment_total_service_use() + with metrics('obs_getboundariesbygeometry', user_obs_snapshot_config): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundariesByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundariesByGeometry') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius( @@ -231,6 +239,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetBoundariesByPointAndRa time_span TEXT DEFAULT NULL, overlap_type TEXT DEFAULT NULL) RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.tools import Logger,LoggerConfig @@ -246,27 +255,28 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - try: - obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) - if result: - resp = [] - for element in result: - the_geom = element['the_geom'] - geom_refs = element['geom_refs'] - resp.append([the_geom, geom_refs]) - quota_service.increment_success_service_use() - return resp - else: - quota_service.increment_empty_service_use() - return [] - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetBoundariesByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_GetBoundariesByPointAndRadius') - finally: - quota_service.increment_total_service_use() + with metrics('obs_getboundariesbypointandradius', user_obs_snapshot_config): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetBoundariesByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetBoundariesByPointAndRadius') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByGeometry( @@ -289,6 +299,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByGeometry( time_span TEXT DEFAULT NULL, overlap_type TEXT DEFAULT NULL) RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.tools import Logger,LoggerConfig @@ -304,27 +315,28 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - try: - obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) - if result: - resp = [] - for element in result: - the_geom = element['the_geom'] - geom_refs = element['geom_refs'] - resp.append([the_geom, geom_refs]) - quota_service.increment_success_service_use() - return resp - else: - quota_service.increment_empty_service_use() - return [] - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetPointsByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_GetPointsByGeometry') - finally: - quota_service.increment_total_service_use() + with metrics('obs_getpointsbygeometry', user_obs_snapshot_config): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return [] + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPointsByGeometry', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPointsByGeometry') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetPointsByPointAndRadius( @@ -349,6 +361,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetPointsByPointAndRadius time_span TEXT DEFAULT NULL, overlap_type TEXT DEFAULT NULL) RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.tools import Logger,LoggerConfig @@ -364,25 +377,26 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - try: - obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) - result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) - if result: - resp = [] - for element in result: - the_geom = element['the_geom'] - geom_refs = element['geom_refs'] - resp.append([the_geom, geom_refs]) - quota_service.increment_success_service_use() - return resp - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to OBS_GetPointsByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to OBS_GetPointsByPointAndRadius') - finally: - quota_service.increment_total_service_use() + with metrics('obs_getpointsbypointandradius', user_obs_snapshot_config): + try: + obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) + result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) + if result: + resp = [] + for element in result: + the_geom = element['the_geom'] + geom_refs = element['geom_refs'] + resp.append([the_geom, geom_refs]) + quota_service.increment_success_service_use() + return resp + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to OBS_GetPointsByPointAndRadius', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to OBS_GetPointsByPointAndRadius') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; diff --git a/server/extension/sql/20_geocode_street.sql b/server/extension/sql/20_geocode_street.sql index ef1fc9b..d1a4560 100644 --- a/server/extension/sql/20_geocode_street.sql +++ b/server/extension/sql/20_geocode_street.sql @@ -1,22 +1,24 @@ -- Geocodes a street address given a searchtext and a state and/or country CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] - if user_geocoder_config.heremaps_geocoder: - here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) - return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] - elif user_geocoder_config.google_geocoder: - google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) - return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] - elif user_geocoder_config.mapzen_geocoder: - mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) - return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] - else: - raise Exception('Requested geocoder is not available') + with metrics('cdb_geocode_street_point', user_geocoder_config): + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.mapzen_geocoder: + mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + raise Exception('Requested geocoder is not available') $$ LANGUAGE plpythonu; diff --git a/server/extension/sql/40_admin1.sql b/server/extension/sql/40_admin1.sql index 9a17394..29dbc02 100644 --- a/server/extension/sql/40_admin1.sql +++ b/server/extension/sql/40_admin1.sql @@ -1,6 +1,7 @@ ---- cdb_geocode_admin1_polygon(admin1_name text) CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.metrics import InternalGeocoderConfig from cartodb_services.tools import Logger,LoggerConfig @@ -14,44 +15,11 @@ RETURNS Geometry AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) - try: - plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1)) AS mypolygon", ["text"]) - rv = plpy.execute(plan, [admin1_name], 1) - result = rv[0]["mypolygon"] - if result: - quota_service.increment_success_service_use() - return result - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to geocode admin1 polygon') - finally: - quota_service.increment_total_service_use() -$$ LANGUAGE plpythonu; ----- cdb_geocode_admin1_polygon(admin1_name text, country_name text) -CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) -RETURNS Geometry AS $$ - from cartodb_services.metrics import QuotaService - from cartodb_services.metrics import InternalGeocoderConfig - from cartodb_services.tools import Logger,LoggerConfig - - plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) - redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] - plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] - - plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") - logger_config = GD["logger_config"] - logger = Logger(logger_config) - quota_service = QuotaService(user_geocoder_config, redis_conn) + with metrics('cdb_geocode_admin1_polygon', user_geocoder_config): try: - plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1), trim($2)) AS mypolygon", ["text", "text"]) - rv = plpy.execute(plan, [admin1_name, country_name], 1) + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [admin1_name], 1) result = rv[0]["mypolygon"] if result: quota_service.increment_success_service_use() @@ -68,6 +36,44 @@ RETURNS Geometry AS $$ quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics + from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.tools import Logger,LoggerConfig + + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)] + + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) + quota_service = QuotaService(user_geocoder_config, redis_conn) + + with metrics('cdb_geocode_admin1_polygon', user_geocoder_config): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1), trim($2)) AS mypolygon", ["text", "text"]) + rv = plpy.execute(plan, [admin1_name, country_name], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode admin1 polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode admin1 polygon') + finally: + quota_service.increment_total_service_use() +$$ LANGUAGE plpythonu; + -------------------------------------------------------------------------------- -- Implementation of the server extension diff --git a/server/extension/sql/50_namedplaces.sql b/server/extension/sql/50_namedplaces.sql index c586f3f..338591d 100644 --- a/server/extension/sql/50_namedplaces.sql +++ b/server/extension/sql/50_namedplaces.sql @@ -35,7 +35,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_namedplac RETURNS Geometry AS $$ from cartodb_services.mapzen import MapzenGeocoder from cartodb_services.mapzen.types import country_to_iso3 - from cartodb_services.metrics import QuotaService + from cartodb_services.metrics import QuotaService, metrics from cartodb_services.tools import Logger,LoggerConfig plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) @@ -50,35 +50,36 @@ RETURNS Geometry AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - try: - geocoder = MapzenGeocoder(user_geocoder_config.mapzen_api_key, logger) - country_iso3 = None - if country_name: - country_iso3 = country_to_iso3(country_name) - coordinates = geocoder.geocode(searchtext=city_name, city=None, - state_province=admin1_name, - country=country_iso3, search_type='locality') - if coordinates: - quota_service.increment_success_service_use() - plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) - point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] - return point['st_setsrid'] - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to geocode city point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to geocode city point using mapzen') - finally: - quota_service.increment_total_service_use() + with metrics('cdb_geocode_namedplace_point', user_geocoder_config, logger): + try: + geocoder = MapzenGeocoder(user_geocoder_config.mapzen_api_key, logger) + country_iso3 = None + if country_name: + country_iso3 = country_to_iso3(country_name) + coordinates = geocoder.geocode(searchtext=city_name, city=None, + state_province=admin1_name, + country=country_iso3, search_type='locality') + if coordinates: + quota_service.increment_success_service_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode city point using mapzen', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode city point using mapzen') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_internal_geocode_namedplace(username text, orgname text, city_name text, admin1_name text DEFAULT NULL, country_name text DEFAULT NULL) RETURNS Geometry AS $$ from cartodb_services.metrics import QuotaService - from cartodb_services.metrics import InternalGeocoderConfig + from cartodb_services.metrics import InternalGeocoderConfig, metrics from cartodb_services.tools import Logger,LoggerConfig plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) @@ -90,30 +91,32 @@ RETURNS Geometry AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) - try: - if admin1_name and country_name: - plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"]) - rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1) - elif country_name: - plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"]) - rv = plpy.execute(plan, [city_name, country_name], 1) - else: - plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"]) - rv = plpy.execute(plan, [city_name], 1) - result = rv[0]["mypoint"] - if result: - quota_service.increment_success_service_use() - return result - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to geocode namedplace point', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to geocode namedplace point') - finally: - quota_service.increment_total_service_use() + + with metrics('cdb_geocode_namedplace_point', user_geocoder_config, logger): + try: + if admin1_name and country_name: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2), trim($3)) AS mypoint", ["text", "text", "text"]) + rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1) + elif country_name: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1), trim($2)) AS mypoint", ["text", "text"]) + rv = plpy.execute(plan, [city_name, country_name], 1) + else: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_namedplace_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [city_name], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode namedplace point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode namedplace point') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; -------------------------------------------------------------------------------- diff --git a/server/extension/sql/60_postalcodes.sql b/server/extension/sql/60_postalcodes.sql index 9d7243a..8b9cb27 100644 --- a/server/extension/sql/60_postalcodes.sql +++ b/server/extension/sql/60_postalcodes.sql @@ -1,5 +1,6 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text) RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.metrics import InternalGeocoderConfig from cartodb_services.tools import Logger,LoggerConfig @@ -13,27 +14,29 @@ RETURNS Geometry AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) - try: - plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1)) AS mypoint", ["text"]) - rv = plpy.execute(plan, [code], 1) - result = rv[0]["mypoint"] - if result: - quota_service.increment_success_service_use() - return result - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to geocode postal code point') - finally: - quota_service.increment_total_service_use() + with metrics('cdb_geocode_postalcode_point', user_geocoder_config): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1)) AS mypoint", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code point') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text) RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.metrics import InternalGeocoderConfig from cartodb_services.tools import Logger,LoggerConfig @@ -47,27 +50,29 @@ RETURNS Geometry AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) - try: - plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1), trim($2)) AS mypoint", ["TEXT", "TEXT"]) - rv = plpy.execute(plan, [code, country], 1) - result = rv[0]["mypoint"] - if result: - quota_service.increment_success_service_use() - return result - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to geocode postal code point') - finally: - quota_service.increment_total_service_use() + with metrics('cdb_geocode_postalcode_point', user_geocoder_config): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1), trim($2)) AS mypoint", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code point', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code point') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text) RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.metrics import InternalGeocoderConfig from cartodb_services.tools import Logger,LoggerConfig @@ -81,27 +86,29 @@ RETURNS Geometry AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) - try: - plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1)) AS mypolygon", ["text"]) - rv = plpy.execute(plan, [code], 1) - result = rv[0]["mypolygon"] - if result: - quota_service.increment_success_service_use() - return result - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to geocode postal code polygon') - finally: - quota_service.increment_total_service_use() + with metrics('cdb_geocode_postalcode_point', user_geocoder_config): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1)) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [code], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text, country text) RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.metrics import InternalGeocoderConfig from cartodb_services.tools import Logger,LoggerConfig @@ -115,23 +122,24 @@ RETURNS Geometry AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) - try: - plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1), trim($2)) AS mypolygon", ["TEXT", "TEXT"]) - rv = plpy.execute(plan, [code, country], 1) - result = rv[0]["mypolygon"] - if result: - quota_service.increment_success_service_use() - return result - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to geocode postal code polygon') - finally: - quota_service.increment_total_service_use() + with metrics('cdb_geocode_postalcode_point', user_geocoder_config): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1), trim($2)) AS mypolygon", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + result = rv[0]["mypolygon"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; -------------------------------------------------------------------------------- diff --git a/server/extension/sql/70_ips.sql b/server/extension/sql/70_ips.sql index 2212d09..08d4559 100644 --- a/server/extension/sql/70_ips.sql +++ b/server/extension/sql/70_ips.sql @@ -1,5 +1,6 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text) RETURNS Geometry AS $$ + from cartodb_services.metrics import metrics from cartodb_services.metrics import QuotaService from cartodb_services.metrics import InternalGeocoderConfig from cartodb_services.tools import Logger,LoggerConfig @@ -13,23 +14,24 @@ RETURNS Geometry AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) - try: - plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_ipaddress_point(trim($1)) AS mypoint", ["TEXT"]) - rv = plpy.execute(plan, [ip], 1) - result = rv[0]["mypoint"] - if result: - quota_service.increment_success_service_use() - return result - else: - quota_service.increment_empty_service_use() - return None - except BaseException as e: - import sys - quota_service.increment_failed_service_use() - logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) - raise Exception('Error trying to geocode postal code polygon') - finally: - quota_service.increment_total_service_use() + with metrics('cdb_geocode_ipaddress_point', user_geocoder_config): + try: + plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_ipaddress_point(trim($1)) AS mypoint", ["TEXT"]) + rv = plpy.execute(plan, [ip], 1) + result = rv[0]["mypoint"] + if result: + quota_service.increment_success_service_use() + return result + else: + quota_service.increment_empty_service_use() + return None + except BaseException as e: + import sys + quota_service.increment_failed_service_use() + logger.error('Error trying to geocode postal code polygon', sys.exc_info(), data={"username": username, "orgname": orgname}) + raise Exception('Error trying to geocode postal code polygon') + finally: + quota_service.increment_total_service_use() $$ LANGUAGE plpythonu; -------------------------------------------------------------------------------- diff --git a/server/extension/sql/90_isochrone.sql b/server/extension/sql/90_isochrone.sql index 24d336e..0464bc1 100644 --- a/server/extension/sql/90_isochrone.sql +++ b/server/extension/sql/90_isochrone.sql @@ -1,5 +1,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[]) RETURNS SETOF cdb_dataservices_server.isoline AS $$ + from cartodb_services.metrics import metrics + plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) @@ -8,14 +10,15 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ if user_isolines_config.google_services_user: raise Exception('This service is not available for google service users.') - if user_isolines_config.heremaps_provider: - here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) - elif user_isolines_config.mapzen_provider: - mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) - return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) - else: - raise Exception('Requested isolines provider is not available') + with metrics('cb_isochrone', user_isolines_config): + if user_isolines_config.heremaps_provider: + here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) + elif user_isolines_config.mapzen_provider: + mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) + return plpy.execute(mapzen_plan, [username, orgname, source, mode, range, options]) + else: + raise Exception('Requested isolines provider is not available') $$ LANGUAGE plpythonu; -- heremaps isochrone From 49fd51f6da97ef5b6c1c97bd0a1f2f8d79ed5bde Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 2 Nov 2016 09:58:35 +0100 Subject: [PATCH 4/9] Added logger to metrics context manager --- .../sql/105_route_between_points.sql | 10 +++- .../sql/110_data_observatory_augmentation.sql | 40 +++++++------- .../sql/115_data_observatory_exploration.sql | 4 +- .../sql/120_data_observatory_geometries.sql | 14 ++--- server/extension/sql/20_geocode_street.sql | 6 ++- server/extension/sql/30_admin0.sql | 2 +- server/extension/sql/40_admin1.sql | 4 +- server/extension/sql/60_postalcodes.sql | 8 +-- server/extension/sql/70_ips.sql | 2 +- server/extension/sql/90_isochrone.sql | 5 +- .../cartodb_services/here/geocoder.py | 4 +- .../cartodb_services/here/routing.py | 4 +- .../cartodb_services/mapzen/matrix_client.py | 2 +- .../cartodb_services/mapzen/routing.py | 2 +- .../cartodb_services/metrics/log.py | 53 +++++++++++-------- 15 files changed, 92 insertions(+), 68 deletions(-) diff --git a/server/extension/sql/105_route_between_points.sql b/server/extension/sql/105_route_between_points.sql index 2762c5f..217fc06 100644 --- a/server/extension/sql/105_route_between_points.sql +++ b/server/extension/sql/105_route_between_points.sql @@ -13,8 +13,11 @@ RETURNS cdb_dataservices_server.simple_route AS $$ redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) - with metrics('cdb_route_with_point', user_routing_config): + with metrics('cdb_route_with_point', user_routing_config, logger): waypoints = [origin, destination] mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units]) @@ -36,8 +39,11 @@ RETURNS cdb_dataservices_server.simple_route AS $$ redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_routing_config = GD["user_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) - with metrics('cdb_route_with_waypoints', user_routing_config): + with metrics('cdb_route_with_waypoints', user_routing_config, logger): mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_route_with_waypoints($1, $2, $3, $4, $5, $6) as route;", ["text", "text", "geometry(Point, 4326)[]", "text", "text[]", "text"]) result = plpy.execute(mapzen_plan, [username, orgname, waypoints, mode, options, units]) return [result[0]['shape'],result[0]['length'], result[0]['duration']] diff --git a/server/extension/sql/110_data_observatory_augmentation.sql b/server/extension/sql/110_data_observatory_augmentation.sql index 4ccadfa..117ee9c 100644 --- a/server/extension/sql/110_data_observatory_augmentation.sql +++ b/server/extension/sql/110_data_observatory_augmentation.sql @@ -11,9 +11,9 @@ RETURNS text AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] + user_obs_config = GD["user_obs_config_{0}".format(username)] - return user_obs_snapshot_config.connection_str + return user_obs_config.connection_str $$ LANGUAGE plpythonu; CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetDemographicSnapshotJSON( @@ -42,16 +42,16 @@ RETURNS json AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] + user_obs_config = GD["user_obs_config_{0}".format(username)] plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") logger_config = GD["logger_config"] logger = Logger(logger_config) - quota_service = QuotaService(user_obs_snapshot_config, redis_conn) + quota_service = QuotaService(user_obs_config, redis_conn) if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - with metrics('obs_getdemographicsnapshot', user_obs_snapshot_config): + with metrics('obs_getdemographicsnapshot', user_obs_config, logger): try: obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshotJSON($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) @@ -95,16 +95,16 @@ RETURNS SETOF JSON AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] + user_obs_config = GD["user_obs_config_{0}".format(username)] plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") logger_config = GD["logger_config"] logger = Logger(logger_config) - quota_service = QuotaService(user_obs_snapshot_config, redis_conn) + quota_service = QuotaService(user_obs_config, redis_conn) if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - with metrics('obs_getdemographicsnapshot', user_obs_snapshot_config): + with metrics('obs_getdemographicsnapshot', user_obs_config, logger): try: obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetDemographicSnapshot($1, $2, $3, $4, $5) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text", "text"]) result = plpy.execute(obs_plan, [username, orgname, geom, time_span, geometry_level]) @@ -151,16 +151,16 @@ RETURNS json AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] + user_obs_config = GD["user_obs_config_{0}".format(username)] plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") logger_config = GD["logger_config"] logger = Logger(logger_config) - quota_service = QuotaService(user_obs_snapshot_config, redis_conn) + quota_service = QuotaService(user_obs_config, redis_conn) if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - with metrics('obs_getsegmentsnapshot', user_obs_snapshot_config): + with metrics('obs_getsegmentsnapshot', user_obs_config, logger): try: obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetSegmentSnapshotJSON($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) @@ -202,16 +202,16 @@ RETURNS SETOF JSON AS $$ plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_obs_snapshot_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) - user_obs_snapshot_config = GD["user_obs_snapshot_config_{0}".format(username)] + user_obs_config = GD["user_obs_config_{0}".format(username)] plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") logger_config = GD["logger_config"] logger = Logger(logger_config) - quota_service = QuotaService(user_obs_snapshot_config, redis_conn) + quota_service = QuotaService(user_obs_config, redis_conn) if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - with metrics('obs_getsegmentsnapshot', user_obs_snapshot_config): + with metrics('obs_getsegmentsnapshot', user_obs_config, logger): try: obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetSegmentSnapshot($1, $2, $3, $4) as snapshot;", ["text", "text", "geometry(Geometry, 4326)", "text"]) result = plpy.execute(obs_plan, [username, orgname, geom, geometry_level]) @@ -272,7 +272,7 @@ RETURNS NUMERIC AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - with metrics('obs_getmeasure', user_obs_snapshot_config): + with metrics('obs_getmeasure', user_obs_config, logger): try: obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasure($1, $2, $3, $4, $5, $6, $7) as measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) result = plpy.execute(obs_plan, [username, orgname, geom, measure_id, normalize, boundary_id, time_span]) @@ -327,7 +327,7 @@ RETURNS TEXT AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - with metrics('obs_getcategory', user_obs_snapshot_config): + with metrics('obs_getcategory', user_obs_config, logger): try: obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetCategory($1, $2, $3, $4, $5, $6) as category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) result = plpy.execute(obs_plan, [username, orgname, geom, category_id, boundary_id, time_span]) @@ -384,7 +384,7 @@ RETURNS NUMERIC AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - with metrics('obs_getuscensusmeasure', user_obs_snapshot_config): + with metrics('obs_getuscensusmeasure', user_obs_config, logger): try: obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusMeasure($1, $2, $3, $4, $5, $6, $7) as census_measure;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text", "text"]) result = plpy.execute(obs_plan, [username, orgname, geom, name, normalize, boundary_id, time_span]) @@ -439,7 +439,7 @@ RETURNS TEXT AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - with metrics('obs_getuscensuscategory', user_obs_snapshot_config): + with metrics('obs_getuscensuscategory', user_obs_config, logger): try: obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetUSCensusCategory($1, $2, $3, $4, $5, $6) as census_category;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) result = plpy.execute(obs_plan, [username, orgname, geom, name, boundary_id, time_span]) @@ -494,7 +494,7 @@ RETURNS NUMERIC AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - with metrics('obs_getpopulation', user_obs_snapshot_config): + with metrics('obs_getpopulation', user_obs_config, logger): try: obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetPopulation($1, $2, $3, $4, $5, $6) as population;", ["text", "text", "geometry(Geometry, 4326)", "text", "text", "text"]) result = plpy.execute(obs_plan, [username, orgname, geom, normalize, boundary_id, time_span]) @@ -549,7 +549,7 @@ RETURNS NUMERIC AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - with metrics('obs_getmeasurebyid', user_obs_snapshot_config): + with metrics('obs_getmeasurebyid', user_obs_config, logger): try: obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetMeasureById($1, $2, $3, $4, $5, $6) as measure;", ["text", "text", "text", "text", "text", "text"]) result = plpy.execute(obs_plan, [username, orgname, geom_ref, measure_id, boundary_id, time_span]) diff --git a/server/extension/sql/115_data_observatory_exploration.sql b/server/extension/sql/115_data_observatory_exploration.sql index 05da6d8..9abe2b2 100644 --- a/server/extension/sql/115_data_observatory_exploration.sql +++ b/server/extension/sql/115_data_observatory_exploration.sql @@ -30,7 +30,7 @@ RETURNS TABLE(id text, description text, name text, aggregate text, source text) if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - with metrics('obs_search', user_obs_snapshot_config): + with metrics('obs_search', user_obs_snapshot_config, logger): try: obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_Search($1, $2, $3, $4);", ["text", "text", "text", "text"]) result = plpy.execute(obs_plan, [username, orgname, search_term, relevant_boundary]) @@ -89,7 +89,7 @@ RETURNS TABLE(boundary_id text, description text, time_span text, tablename text if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - with metrics('obs_getavailableboundaries', user_obs_snapshot_config): + with metrics('obs_getavailableboundaries', user_obs_snapshot_config, logger): try: obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetAvailableBoundaries($1, $2, $3, $4) as available_boundaries;", ["text", "text", "geometry(Geometry, 4326)", "text"]) result = plpy.execute(obs_plan, [username, orgname, geom, time_span]) diff --git a/server/extension/sql/120_data_observatory_geometries.sql b/server/extension/sql/120_data_observatory_geometries.sql index bfc09fc..b69f318 100644 --- a/server/extension/sql/120_data_observatory_geometries.sql +++ b/server/extension/sql/120_data_observatory_geometries.sql @@ -32,7 +32,7 @@ RETURNS geometry(Geometry, 4326) AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - with metrics('obs_getboundary', user_obs_snapshot_config): + with metrics('obs_getboundary', user_obs_snapshot_config, logger): try: obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundary($1, $2, $3, $4) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) @@ -85,7 +85,7 @@ RETURNS TEXT AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - with metrics('obs_getboundaryid', user_obs_snapshot_config): + with metrics('obs_getboundaryid', user_obs_snapshot_config, logger): try: obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryId($1, $2, $3, $4, $5) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text"]) result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span]) @@ -138,7 +138,7 @@ RETURNS geometry(Geometry, 4326) AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - with metrics('obs_getboundarybyid', user_obs_snapshot_config): + with metrics('obs_getboundarybyid', user_obs_snapshot_config, logger): try: obs_plan = plpy.prepare("SELECT cdb_dataservices_server._OBS_GetBoundaryById($1, $2, $3, $4, $5) as boundary;", ["text", "text", "text", "text", "text"]) result = plpy.execute(obs_plan, [username, orgname, geometry_id, boundary_id, time_span]) @@ -193,7 +193,7 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - with metrics('obs_getboundariesbygeometry', user_obs_snapshot_config): + with metrics('obs_getboundariesbygeometry', user_obs_snapshot_config, logger): try: obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) @@ -255,7 +255,7 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - with metrics('obs_getboundariesbypointandradius', user_obs_snapshot_config): + with metrics('obs_getboundariesbypointandradius', user_obs_snapshot_config, logger): try: obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetBoundariesByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) @@ -315,7 +315,7 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - with metrics('obs_getpointsbygeometry', user_obs_snapshot_config): + with metrics('obs_getpointsbygeometry', user_obs_snapshot_config, logger): try: obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByGeometry($1, $2, $3, $4, $5, $6) as boundary;", ["text", "text", "geometry(Point, 4326)", "text", "text", "text"]) result = plpy.execute(obs_plan, [username, orgname, geom, boundary_id, time_span, overlap_type]) @@ -377,7 +377,7 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$ if not quota_service.check_user_quota(): raise Exception('You have reached the limit of your quota') - with metrics('obs_getpointsbypointandradius', user_obs_snapshot_config): + with metrics('obs_getpointsbypointandradius', user_obs_snapshot_config, logger): try: obs_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._OBS_GetPointsByPointAndRadius($1, $2, $3, $4, $5, $6, $7) as boundary;", ["text", "text", "geometry(Point, 4326)", "numeric", "text", "text", "text"]) result = plpy.execute(obs_plan, [username, orgname, geom, radius, boundary_id, time_span, overlap_type]) diff --git a/server/extension/sql/20_geocode_street.sql b/server/extension/sql/20_geocode_street.sql index d1a4560..b4fbcd1 100644 --- a/server/extension/sql/20_geocode_street.sql +++ b/server/extension/sql/20_geocode_street.sql @@ -2,12 +2,16 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) RETURNS Geometry AS $$ from cartodb_services.metrics import metrics + from cartodb_services.tools import Logger,LoggerConfig plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username)) redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) - with metrics('cdb_geocode_street_point', user_geocoder_config): + with metrics('cdb_geocode_street_point', user_geocoder_config, logger): if user_geocoder_config.heremaps_geocoder: here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] diff --git a/server/extension/sql/30_admin0.sql b/server/extension/sql/30_admin0.sql index 7b06f97..593063a 100644 --- a/server/extension/sql/30_admin0.sql +++ b/server/extension/sql/30_admin0.sql @@ -14,7 +14,7 @@ RETURNS Geometry AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) - with metrics('cdb_geocode_admin0_polygon', user_geocoder_config): + with metrics('cdb_geocode_admin0_polygon', user_geocoder_config, logger): try: plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"]) rv = plpy.execute(plan, [country_name], 1) diff --git a/server/extension/sql/40_admin1.sql b/server/extension/sql/40_admin1.sql index 29dbc02..d883e22 100644 --- a/server/extension/sql/40_admin1.sql +++ b/server/extension/sql/40_admin1.sql @@ -16,7 +16,7 @@ RETURNS Geometry AS $$ logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) - with metrics('cdb_geocode_admin1_polygon', user_geocoder_config): + with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger): try: plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1)) AS mypolygon", ["text"]) rv = plpy.execute(plan, [admin1_name], 1) @@ -54,7 +54,7 @@ RETURNS Geometry AS $$ logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) - with metrics('cdb_geocode_admin1_polygon', user_geocoder_config): + with metrics('cdb_geocode_admin1_polygon', user_geocoder_config, logger): try: plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin1_polygon(trim($1), trim($2)) AS mypolygon", ["text", "text"]) rv = plpy.execute(plan, [admin1_name, country_name], 1) diff --git a/server/extension/sql/60_postalcodes.sql b/server/extension/sql/60_postalcodes.sql index 8b9cb27..d7d610e 100644 --- a/server/extension/sql/60_postalcodes.sql +++ b/server/extension/sql/60_postalcodes.sql @@ -14,7 +14,7 @@ RETURNS Geometry AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) - with metrics('cdb_geocode_postalcode_point', user_geocoder_config): + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): try: plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1)) AS mypoint", ["text"]) rv = plpy.execute(plan, [code], 1) @@ -50,7 +50,7 @@ RETURNS Geometry AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) - with metrics('cdb_geocode_postalcode_point', user_geocoder_config): + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): try: plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_point(trim($1), trim($2)) AS mypoint", ["TEXT", "TEXT"]) rv = plpy.execute(plan, [code, country], 1) @@ -86,7 +86,7 @@ RETURNS Geometry AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) - with metrics('cdb_geocode_postalcode_point', user_geocoder_config): + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): try: plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1)) AS mypolygon", ["text"]) rv = plpy.execute(plan, [code], 1) @@ -122,7 +122,7 @@ RETURNS Geometry AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) - with metrics('cdb_geocode_postalcode_point', user_geocoder_config): + with metrics('cdb_geocode_postalcode_point', user_geocoder_config, logger): try: plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_postalcode_polygon(trim($1), trim($2)) AS mypolygon", ["TEXT", "TEXT"]) rv = plpy.execute(plan, [code, country], 1) diff --git a/server/extension/sql/70_ips.sql b/server/extension/sql/70_ips.sql index 08d4559..2b452aa 100644 --- a/server/extension/sql/70_ips.sql +++ b/server/extension/sql/70_ips.sql @@ -14,7 +14,7 @@ RETURNS Geometry AS $$ logger_config = GD["logger_config"] logger = Logger(logger_config) quota_service = QuotaService(user_geocoder_config, redis_conn) - with metrics('cdb_geocode_ipaddress_point', user_geocoder_config): + with metrics('cdb_geocode_ipaddress_point', user_geocoder_config, logger): try: plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_ipaddress_point(trim($1)) AS mypoint", ["TEXT"]) rv = plpy.execute(plan, [ip], 1) diff --git a/server/extension/sql/90_isochrone.sql b/server/extension/sql/90_isochrone.sql index 0464bc1..6f4186c 100644 --- a/server/extension/sql/90_isochrone.sql +++ b/server/extension/sql/90_isochrone.sql @@ -6,11 +6,14 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$ redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] plpy.execute("SELECT cdb_dataservices_server._get_isolines_routing_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) user_isolines_config = GD["user_isolines_routing_config_{0}".format(username)] + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") + logger_config = GD["logger_config"] + logger = Logger(logger_config) if user_isolines_config.google_services_user: raise Exception('This service is not available for google service users.') - with metrics('cb_isochrone', user_isolines_config): + with metrics('cb_isochrone', user_isolines_config, logger): if user_isolines_config.heremaps_provider: here_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.cdb_here_isochrone($1, $2, $3, $4, $5, $6) as isoline; ", ["text", "text", "geometry(geometry, 4326)", "text", "integer[]", "text[]"]) return plpy.execute(here_plan, [username, orgname, source, mode, range, options]) diff --git a/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py index a221db9..d176a7d 100644 --- a/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/here/geocoder.py @@ -5,9 +5,10 @@ import json import requests from exceptions import * +from cartodb_services.metrics import Traceable -class HereMapsGeocoder: +class HereMapsGeocoder(Traceable): 'A Here Maps Geocoder wrapper for python' PRODUCTION_GEOCODE_JSON_URL = 'https://geocoder.api.here.com/6.2/geocode.json' @@ -89,6 +90,7 @@ class HereMapsGeocoder: request_params.update(params) response = requests.get(self.host, params=request_params, timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT)) + self.add_response_data(response, self._logger) if response.status_code == requests.codes.ok: return json.loads(response.text) elif response.status_code == requests.codes.bad_request: diff --git a/server/lib/python/cartodb_services/cartodb_services/here/routing.py b/server/lib/python/cartodb_services/cartodb_services/here/routing.py index 11856be..221a2ea 100644 --- a/server/lib/python/cartodb_services/cartodb_services/here/routing.py +++ b/server/lib/python/cartodb_services/cartodb_services/here/routing.py @@ -2,9 +2,10 @@ import requests import json from exceptions import WrongParams +from cartodb_services.metrics import Traceable -class HereMapsRoutingIsoline: +class HereMapsRoutingIsoline(Traceable): 'A Here Maps Routing wrapper for python' PRODUCTION_ROUTING_BASE_URL = 'https://isoline.route.api.here.com' @@ -54,6 +55,7 @@ class HereMapsRoutingIsoline: parsed_options) response = requests.get(self._url, params=request_params, timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT)) + self.add_response_data(response, self._logger) if response.status_code == requests.codes.ok: return self.__parse_isolines_response(response.text) elif response.status_code == requests.codes.bad_request: diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py index 8147be1..c10a912 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py @@ -46,7 +46,7 @@ class MatrixClient(Traceable): } response = requests.get(self.ONE_TO_MANY_URL, params=request_params, timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT)) - self.add_response_data(response) + self.add_response_data(response, self._logger) if response.status_code != requests.codes.ok: self._logger.error('Error trying to get matrix distance from mapzen', diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/routing.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/routing.py index 6c7c71b..d970d69 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/routing.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/routing.py @@ -48,7 +48,7 @@ class MapzenRouting(Traceable): request_params = self.__parse_request_parameters(json_request_params) response = requests.get(self._url, params=request_params, timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT)) - self.add_response_data(response) + self.add_response_data(response, self._logger) if response.status_code == requests.codes.ok: return self.__parse_routing_response(response.text) elif response.status_code == requests.codes.bad_request: diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/log.py b/server/lib/python/cartodb_services/cartodb_services/metrics/log.py index 3b94c19..67d02cf 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/log.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/log.py @@ -7,7 +7,7 @@ from contextlib import contextmanager @contextmanager -def metrics(function, service_config): +def metrics(function, service_config, logger=None): try: start_time = time.time() yield @@ -15,16 +15,16 @@ def metrics(function, service_config): end_time = time.time() MetricsDataGatherer.add('function_name', function) MetricsDataGatherer.add('function_execution_time', (end_time - start_time)) - logger = MetricsServiceLoggerFactory.build(service_config) - if logger: + metrics_logger = MetricsServiceLoggerFactory.build(service_config, logger) + if metrics_logger: data = MetricsDataGatherer.get() - logger.log(data) + metrics_logger.log(data) MetricsDataGatherer.clean() class Traceable: - def add_response_data(self, response): + def add_response_data(self, response, logger=None): try: response_data = {} response_data['time'] = response.elapsed.total_seconds() @@ -37,7 +37,9 @@ class Traceable: MetricsDataGatherer.add('response', [response_data]) except BaseException as e: # We don't want to stop the job for some error here - plpy.warning(e) + if logger: + logger.error("Error trying to process response data for metrics", + exception=e) class MetricsDataGatherer: @@ -87,15 +89,15 @@ class MetricsDataGatherer: class MetricsServiceLoggerFactory: @classmethod - def build(self, service_config): + def build(self, service_config, logger=None): if re.search('^geocoder_*', service_config.service_type): - return MetricsGeocoderLogger(service_config) + return MetricsGeocoderLogger(service_config, logger) elif re.search('^routing_*', service_config.service_type): - return MetricsGenericLogger(service_config) + return MetricsGenericLogger(service_config, logger) elif re.search('_isolines$', service_config.service_type): - return MetricsIsolinesLogger(service_config) + return MetricsIsolinesLogger(service_config, logger) elif re.search('^obs_*', service_config.service_type): - return MetricsGenericLogger(service_config) + return MetricsGenericLogger(service_config, logger) else: return None @@ -103,15 +105,20 @@ class MetricsServiceLoggerFactory: class MetricsLogger(object): __metaclass__ = abc.ABCMeta - def __init__(self, service_config): + def __init__(self, service_config, logger): self._service_config = service_config + self._logger = logger def dump_to_file(self, data): - log_path = self.service_config.metrics_log_path - if log_path: - with open(log_path, 'a') as logfile: - json.dump(data, logfile) - logfile.write('\n') + try: + log_path = self.service_config.metrics_log_path + if log_path: + with open(log_path, 'a') as logfile: + json.dump(data, logfile) + logfile.write('\n') + except BaseException as e: + self._logger("Error dumping metrics to file {0}".format(log_path), + exception=e) def collect_data(self, data): return { @@ -141,8 +148,8 @@ class MetricsLogger(object): class MetricsGeocoderLogger(MetricsLogger): - def __init__(self, service_config): - super(MetricsGeocoderLogger, self).__init__(service_config) + def __init__(self, service_config, logger): + super(MetricsGeocoderLogger, self).__init__(service_config, logger) def log(self, data): dump_data = self.collect_data(data) @@ -176,8 +183,8 @@ class MetricsGeocoderLogger(MetricsLogger): class MetricsGenericLogger(MetricsLogger): - def __init__(self, service_config): - super(MetricsGenericLogger, self).__init__(service_config) + def __init__(self, service_config, logger): + super(MetricsGenericLogger, self).__init__(service_config, logger) def log(self, data): dump_data = self.collect_data(data) @@ -188,8 +195,8 @@ class MetricsGenericLogger(MetricsLogger): class MetricsIsolinesLogger(MetricsLogger): - def __init__(self, service_config): - super(MetricsIsolinesLogger, self).__init__(service_config) + def __init__(self, service_config, logger): + super(MetricsIsolinesLogger, self).__init__(service_config, logger) def log(self, data): dump_data = self.collect_data(data) From e878031744fb17b3193720b6c52325b5225fa68a Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 2 Nov 2016 10:43:17 +0100 Subject: [PATCH 5/9] Added url to the response data --- .../cartodb_services/cartodb_services/metrics/log.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/log.py b/server/lib/python/cartodb_services/cartodb_services/metrics/log.py index 67d02cf..26f79d9 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/log.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/log.py @@ -4,6 +4,7 @@ import json import re import time from contextlib import contextmanager +from urlparse import urlparse @contextmanager @@ -23,24 +24,28 @@ def metrics(function, service_config, logger=None): class Traceable: - def add_response_data(self, response, logger=None): try: response_data = {} response_data['time'] = response.elapsed.total_seconds() response_data['code'] = response.status_code response_data['message'] = response.reason + response_data['url'] = self._parse_response_url(response.url) stored_data = MetricsDataGatherer.get_element('response') if stored_data: stored_data.append(response_data) else: MetricsDataGatherer.add('response', [response_data]) except BaseException as e: - # We don't want to stop the job for some error here + # We don't want to stop the job for some error processing response if logger: logger.error("Error trying to process response data for metrics", exception=e) + def _parse_response_url(self, url): + u = urlparse(url) + return "{0}://{1}{2}".format(u.scheme, u.netloc, u.path) + class MetricsDataGatherer: From c571edb21eb13cb2e347b1f86ee4a86962c7abfc Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 2 Nov 2016 18:15:37 +0100 Subject: [PATCH 6/9] Split response and function data, added uuid and type of element --- .../cartodb_services/metrics/log.py | 40 ++++++++++++++++--- 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/log.py b/server/lib/python/cartodb_services/cartodb_services/metrics/log.py index 26f79d9..51bdd7c 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/log.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/log.py @@ -3,6 +3,7 @@ import abc import json import re import time +import uuid from contextlib import contextmanager from urlparse import urlparse @@ -14,9 +15,11 @@ def metrics(function, service_config, logger=None): yield finally: end_time = time.time() + MetricsDataGatherer.add('uuid', str(uuid.uuid1())) MetricsDataGatherer.add('function_name', function) MetricsDataGatherer.add('function_execution_time', (end_time - start_time)) - metrics_logger = MetricsServiceLoggerFactory.build(service_config, logger) + metrics_logger = MetricsServiceLoggerFactory.build(service_config, + logger) if metrics_logger: data = MetricsDataGatherer.get() metrics_logger.log(data) @@ -24,10 +27,16 @@ def metrics(function, service_config, logger=None): class Traceable: + """ + Module to add metrics traceability, for example to get response object + in order to add to the metrics dump + """ def add_response_data(self, response, logger=None): try: response_data = {} - response_data['time'] = response.elapsed.total_seconds() + response_data['type'] = "request" + response_data['date'] = datetime.now().isoformat() + response_data['elapsed_time'] = response.elapsed.total_seconds() response_data['code'] = response.status_code response_data['message'] = response.reason response_data['url'] = self._parse_response_url(response.url) @@ -39,7 +48,7 @@ class Traceable: except BaseException as e: # We don't want to stop the job for some error processing response if logger: - logger.error("Error trying to process response data for metrics", + logger.error("Error trying to process response metricd data", exception=e) def _parse_response_url(self, url): @@ -48,6 +57,10 @@ class Traceable: class MetricsDataGatherer: + """ + Metrics gatherer used as a singleton. The intend is to use it as a global + storage for the metrics along the function request. + """ class __MetricsDataGatherer: def __init__(self): @@ -117,8 +130,11 @@ class MetricsLogger(object): def dump_to_file(self, data): try: log_path = self.service_config.metrics_log_path + response_data = data.pop('response', []) + uuid = data.get('uuid') if log_path: with open(log_path, 'a') as logfile: + self._dump_response_to_file(uuid, response_data, logfile) json.dump(data, logfile) logfile.write('\n') except BaseException as e: @@ -127,8 +143,11 @@ class MetricsLogger(object): def collect_data(self, data): return { + "uuid": data.get('uuid', uuid.uuid1()), + "type": 'function', "function_name": data.get('function_name', None), - "function_execution_time": data.get('function_execution_time', None), + "function_execution_time": data.get('function_execution_time', + None), "service": self._service_config.service_type, "processable_rows": 1, "success": data.get('success', False), @@ -139,9 +158,17 @@ class MetricsLogger(object): "provider": self._service_config.provider, "username": self._service_config.username, "organization": self._service_config.organization, - "response": data.get('response', None) + "response": data.get('response', []) } + def _dump_response_to_file(self, uuid, response_data, log_file): + for r in response_data: + r['uuid'] = uuid + self._logger.info(r) + json.dump(r, log_file) + log_file.write('\n') + + @property def service_config(self): return self._service_config @@ -198,6 +225,7 @@ class MetricsGenericLogger(MetricsLogger): def collect_data(self, data): return super(MetricsGenericLogger, self).collect_data(data) + class MetricsIsolinesLogger(MetricsLogger): def __init__(self, service_config, logger): @@ -208,7 +236,7 @@ class MetricsIsolinesLogger(MetricsLogger): self.dump_to_file(dump_data) def collect_data(self, data): - dump_data = super(MetricsIsolinesLogger, self).collect_data(data) + dump_data = super(MetricsIsolinesLogger, self).collect_data(data) dump_data.update({ "isolines_generated": data.get('isolines_generated', 0) From 73aa26590b6ea4c9624350576c8e7504ef097ca3 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 3 Nov 2016 10:48:37 +0100 Subject: [PATCH 7/9] Make the data gather singleton support multiple instances Due to we use pgbouncer, we could share the same instance of the metrics gatherer singleton in multiples requests. Added support to have multiple instances identified by transaction id --- .../cartodb_services/metrics/log.py | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/log.py b/server/lib/python/cartodb_services/cartodb_services/metrics/log.py index 51bdd7c..dd25ce1 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/log.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/log.py @@ -1,9 +1,10 @@ -from datetime import datetime import abc import json import re import time import uuid +import plpy +from datetime import datetime from contextlib import contextmanager from urlparse import urlparse @@ -78,7 +79,9 @@ class MetricsDataGatherer: def clean(self): self.data = {} - __instance = None + + # We use pgbouncer so we need to have multiples instances per request id + __instance = {} @classmethod def add(self, key, value): @@ -98,10 +101,16 @@ class MetricsDataGatherer: @classmethod def instance(self): - if not MetricsDataGatherer.__instance: - MetricsDataGatherer.__instance = MetricsDataGatherer.__MetricsDataGatherer() + txid = MetricsDataGatherer._get_txid() + if txid not in MetricsDataGatherer.__instance: + MetricsDataGatherer.__instance[txid] = MetricsDataGatherer.__MetricsDataGatherer() - return MetricsDataGatherer.__instance + return MetricsDataGatherer.__instance[txid] + + @classmethod + def _get_txid(self): + result = plpy.execute('select txid_current() as txid') + return result[0]['txid'] class MetricsServiceLoggerFactory: @@ -168,7 +177,6 @@ class MetricsLogger(object): json.dump(r, log_file) log_file.write('\n') - @property def service_config(self): return self._service_config @@ -237,9 +245,7 @@ class MetricsIsolinesLogger(MetricsLogger): def collect_data(self, data): dump_data = super(MetricsIsolinesLogger, self).collect_data(data) - dump_data.update({ "isolines_generated": data.get('isolines_generated', 0) }) - return dump_data From 6f0db2016308bd35d6b7a284552b6a3b3fcb621a Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 3 Nov 2016 15:14:11 +0100 Subject: [PATCH 8/9] Added here integration test for geocoder street --- test/integration/test_street_functions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py index 263aaa6..7edda67 100644 --- a/test/integration/test_street_functions.py +++ b/test/integration/test_street_functions.py @@ -14,8 +14,8 @@ class TestStreetFunctions(TestCase): self.env_variables['api_key'] ) - def test_if_select_with_street_point_is_ok(self): - query = "SELECT cdb_geocode_street_point(street) " \ + def test_if_select_with_here_street_point_is_ok(self): + query = "SELECT cdb_here_geocode_street_point(street) " \ "as geometry FROM {0} LIMIT 1&api_key={1}".format( self.env_variables['table_name'], self.env_variables['api_key']) From 88ed7777857a7a09348f156ec48e35fb7ee97fe4 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 4 Nov 2016 12:14:48 +0100 Subject: [PATCH 9/9] Added plpy_mock and check multiple instances for metricsdatagatherer --- .../python/cartodb_services/test/__init__.py | 5 ++ .../test/metrics/test_config.py | 5 +- .../test/metrics/test_quota.py | 2 +- .../python/cartodb_services/test/mock_plpy.py | 53 +++++++++++++++++++ .../cartodb_services/test/test_config.py | 36 ++++++------- .../cartodb_services/test/test_helper.py | 34 +++++------- .../cartodb_services/test/test_metrics_log.py | 23 ++++++++ .../python/cartodb_services/test/test_qps.py | 1 - .../test/test_quota_service.py | 27 +++++----- .../test/test_user_service.py | 11 ++-- 10 files changed, 134 insertions(+), 63 deletions(-) create mode 100644 server/lib/python/cartodb_services/test/mock_plpy.py create mode 100644 server/lib/python/cartodb_services/test/test_metrics_log.py diff --git a/server/lib/python/cartodb_services/test/__init__.py b/server/lib/python/cartodb_services/test/__init__.py index e69de29..79a7695 100644 --- a/server/lib/python/cartodb_services/test/__init__.py +++ b/server/lib/python/cartodb_services/test/__init__.py @@ -0,0 +1,5 @@ +from test_helper import plpy_mock_config + + +def setup(): + plpy_mock_config() diff --git a/server/lib/python/cartodb_services/test/metrics/test_config.py b/server/lib/python/cartodb_services/test/metrics/test_config.py index a9142f0..8d0ceb5 100644 --- a/server/lib/python/cartodb_services/test/metrics/test_config.py +++ b/server/lib/python/cartodb_services/test/metrics/test_config.py @@ -1,13 +1,14 @@ from unittest import TestCase from mockredis import MockRedis +from ..test_helper import * from cartodb_services.metrics.config import RoutingConfig, ServicesRedisConfig -from ..test_helper import build_plpy_mock + class TestRoutingConfig(TestCase): def setUp(self): self._redis_conn = MockRedis() - self._db_conn = build_plpy_mock() + self._db_conn = plpy_mock self._username = 'my_test_user' self._user_key = "rails:users:{0}".format(self._username) self._redis_conn.hset(self._user_key, 'period_end_date', '2016-10-10') diff --git a/server/lib/python/cartodb_services/test/metrics/test_quota.py b/server/lib/python/cartodb_services/test/metrics/test_quota.py index c56756c..18f2105 100644 --- a/server/lib/python/cartodb_services/test/metrics/test_quota.py +++ b/server/lib/python/cartodb_services/test/metrics/test_quota.py @@ -1,6 +1,6 @@ from unittest import TestCase from mockredis import MockRedis -from ..test_helper import build_plpy_mock +from ..test_helper import * from cartodb_services.metrics.quota import QuotaChecker from cartodb_services.metrics import RoutingConfig from datetime import datetime diff --git a/server/lib/python/cartodb_services/test/mock_plpy.py b/server/lib/python/cartodb_services/test/mock_plpy.py new file mode 100644 index 0000000..5298a98 --- /dev/null +++ b/server/lib/python/cartodb_services/test/mock_plpy.py @@ -0,0 +1,53 @@ +import re + + +class MockCursor: + def __init__(self, data): + self.cursor_pos = 0 + self.data = data + + def fetch(self, batch_size): + batch = self.data[self.cursor_pos: self.cursor_pos + batch_size] + self.cursor_pos += batch_size + return batch + + +class MockPlPy: + def __init__(self): + self._reset() + + def _reset(self): + self.infos = [] + self.notices = [] + self.debugs = [] + self.logs = [] + self.warnings = [] + self.errors = [] + self.fatals = [] + self.executes = [] + self.results = [] + self.prepares = [] + self.results = {} + + def _define_result(self, query, result): + pattern = re.compile(query, re.IGNORECASE | re.MULTILINE) + self.results[pattern] = result + + def notice(self, msg): + self.notices.append(msg) + + def debug(self, msg): + self.notices.append(msg) + + def info(self, msg): + self.infos.append(msg) + + def cursor(self, query): + data = self.execute(query) + return MockCursor(data) + + def execute(self, query, rows=1): + for pattern, result in self.results.iteritems(): + if pattern.search(query): + return result + return [] diff --git a/server/lib/python/cartodb_services/test/test_config.py b/server/lib/python/cartodb_services/test/test_config.py index 4a639ec..b55b991 100644 --- a/server/lib/python/cartodb_services/test/test_config.py +++ b/server/lib/python/cartodb_services/test/test_config.py @@ -1,20 +1,20 @@ -import test_helper -from cartodb_services.metrics import GeocoderConfig, ObservatorySnapshotConfig, ConfigException +from test_helper import * from unittest import TestCase from nose.tools import assert_raises from mockredis import MockRedis from datetime import datetime, timedelta +from cartodb_services.metrics import GeocoderConfig, ObservatorySnapshotConfig, ConfigException class TestConfig(TestCase): def setUp(self): self.redis_conn = MockRedis() - self.plpy_mock = test_helper.build_plpy_mock() + plpy_mock_config() def test_should_return_list_of_nokia_geocoder_config_if_its_ok(self): - test_helper.build_redis_user_config(self.redis_conn, 'test_user') - geocoder_config = GeocoderConfig(self.redis_conn, self.plpy_mock, + build_redis_user_config(self.redis_conn, 'test_user') + geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock, 'test_user', None) assert geocoder_config.heremaps_geocoder is True assert geocoder_config.geocoding_quota == 100 @@ -22,10 +22,10 @@ class TestConfig(TestCase): def test_should_return_list_of_nokia_geocoder_config_ok_for_org(self): yesterday = datetime.today() - timedelta(days=1) - test_helper.build_redis_user_config(self.redis_conn, 'test_user') - test_helper.build_redis_org_config(self.redis_conn, 'test_org', - quota=200, end_date=yesterday) - geocoder_config = GeocoderConfig(self.redis_conn, self.plpy_mock, + build_redis_user_config(self.redis_conn, 'test_user') + build_redis_org_config(self.redis_conn, 'test_org', + quota=200, end_date=yesterday) + geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock, 'test_user', 'test_org') assert geocoder_config.heremaps_geocoder is True assert geocoder_config.geocoding_quota == 200 @@ -34,10 +34,10 @@ class TestConfig(TestCase): def test_should_return_config_for_obs_snapshot(self): yesterday = datetime.today() - timedelta(days=1) - test_helper.build_redis_user_config(self.redis_conn, 'test_user', - do_quota=100, soft_do_limit=True, - end_date=yesterday) - do_config = ObservatorySnapshotConfig(self.redis_conn, self.plpy_mock, + build_redis_user_config(self.redis_conn, 'test_user', + do_quota=100, soft_do_limit=True, + end_date=yesterday) + do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock, 'test_user') assert do_config.monthly_quota == 100 assert do_config.soft_limit is True @@ -45,16 +45,16 @@ class TestConfig(TestCase): def test_should_return_db_quota_if_not_redis_quota_config_obs_snapshot(self): yesterday = datetime.today() - timedelta(days=1) - test_helper.build_redis_user_config(self.redis_conn, 'test_user', - end_date=yesterday) - do_config = ObservatorySnapshotConfig(self.redis_conn, self.plpy_mock, + build_redis_user_config(self.redis_conn, 'test_user', + end_date=yesterday) + do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock, 'test_user') assert do_config.monthly_quota == 0 assert do_config.soft_limit is False assert do_config.period_end_date.date() == yesterday.date() def test_should_raise_exception_when_missing_parameters(self): - plpy_mock = test_helper.build_plpy_mock(empty=True) - test_helper.build_redis_user_config(self.redis_conn, 'test_user') + plpy_mock._reset() + build_redis_user_config(self.redis_conn, 'test_user') assert_raises(ConfigException, GeocoderConfig, self.redis_conn, plpy_mock, 'test_user', None) diff --git a/server/lib/python/cartodb_services/test/test_helper.py b/server/lib/python/cartodb_services/test/test_helper.py index d6e9d80..2272278 100644 --- a/server/lib/python/cartodb_services/test/test_helper.py +++ b/server/lib/python/cartodb_services/test/test_helper.py @@ -1,7 +1,11 @@ from datetime import datetime, date -from mock import Mock +from mock import Mock, MagicMock +import random import sys -sys.modules['plpy'] = Mock() +from mock_plpy import MockPlPy + +plpy_mock = MockPlPy() +sys.modules['plpy'] = plpy_mock def build_redis_user_config(redis_conn, username, quota=100, soft_limit=False, @@ -57,22 +61,10 @@ def increment_service_uses(redis_conn, username, orgname=None, redis_conn.zincrby(redis_name, date.day, amount) -def build_plpy_mock(empty=False): - plpy_mock = Mock() - if not empty: - plpy_mock.execute.side_effect = _plpy_execute_side_effect - - return plpy_mock - - -def _plpy_execute_side_effect(*args, **kwargs): - if args[0] == "SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as conf": - return [{'conf': '{"geocoder": {"app_id": "app_id", "app_code": "code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "app_id", "app_code": "code"}}'}] - elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('mapzen_conf') as conf": - return [{'conf': '{"routing": {"api_key": "api_key_rou", "monthly_quota": 1500000}, "geocoder": {"api_key": "api_key_geo", "monthly_quota": 1500000}, "matrix": {"api_key": "api_key_mat", "monthly_quota": 1500000}}'}] - elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('logger_conf') as conf": - return [{'conf': '{"geocoder_log_path": "/dev/null"}'}] - elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('data_observatory_conf') as conf": - return [{'conf': '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}}'}] - elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('server_conf') as conf": - return [{'conf': '{"environment": "testing"}'}] +def plpy_mock_config(): + plpy_mock._define_result("CDB_Conf_GetConf\('heremaps_conf'\)", [{'conf': '{"geocoder": {"app_id": "app_id", "app_code": "code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "app_id", "app_code": "code"}}'}]) + plpy_mock._define_result("CDB_Conf_GetConf\('mapzen_conf'\)", [{'conf': '{"routing": {"api_key": "api_key_rou", "monthly_quota": 1500000}, "geocoder": {"api_key": "api_key_geo", "monthly_quota": 1500000}, "matrix": {"api_key": "api_key_mat", "monthly_quota": 1500000}}'}]) + plpy_mock._define_result("CDB_Conf_GetConf\('logger_conf'\)", [{'conf': '{"geocoder_log_path": "/dev/null"}'}]) + plpy_mock._define_result("CDB_Conf_GetConf\('data_observatory_conf'\)", [{'conf': '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}}'}]) + plpy_mock._define_result("CDB_Conf_GetConf\('server_conf'\)", [{'conf': '{"environment": "testing"}'}]) + plpy_mock._define_result("select txid_current", [{'txid': random.randint(0, 1000)}]) diff --git a/server/lib/python/cartodb_services/test/test_metrics_log.py b/server/lib/python/cartodb_services/test/test_metrics_log.py new file mode 100644 index 0000000..963a34a --- /dev/null +++ b/server/lib/python/cartodb_services/test/test_metrics_log.py @@ -0,0 +1,23 @@ +from test_helper import * +from cartodb_services.metrics import MetricsDataGatherer +from unittest import TestCase +from mock import Mock, MagicMock +from nose.tools import assert_raises +from datetime import datetime, date + + +class TestMetricsDataGatherer(TestCase): + + def setUp(self): + plpy_mock_config() + + + def test_should_use_multiple_instances_for_multiples_requests(self): + plpy_mock._define_result("select txid_current", [{'txid': 100}]) + MetricsDataGatherer.add('test', 1) + plpy_mock._define_result("select txid_current", [{'txid': 101}]) + MetricsDataGatherer.add('test', 2) + plpy_mock._define_result("select txid_current", [{'txid': 100}]) + assert MetricsDataGatherer.get_element('test') is 1 + plpy_mock._define_result("select txid_current", [{'txid': 101}]) + assert MetricsDataGatherer.get_element('test') is 2 diff --git a/server/lib/python/cartodb_services/test/test_qps.py b/server/lib/python/cartodb_services/test/test_qps.py index 014c510..bf396ca 100644 --- a/server/lib/python/cartodb_services/test/test_qps.py +++ b/server/lib/python/cartodb_services/test/test_qps.py @@ -1,4 +1,3 @@ -import test_helper import requests from unittest import TestCase from nose.tools import assert_raises diff --git a/server/lib/python/cartodb_services/test/test_quota_service.py b/server/lib/python/cartodb_services/test/test_quota_service.py index 7cc49dd..c104235 100644 --- a/server/lib/python/cartodb_services/test/test_quota_service.py +++ b/server/lib/python/cartodb_services/test/test_quota_service.py @@ -1,4 +1,4 @@ -import test_helper +from test_helper import * from mockredis import MockRedis from cartodb_services.metrics import QuotaService from cartodb_services.metrics import GeocoderConfig, RoutingConfig, ObservatorySnapshotConfig, IsolinesRoutingConfig @@ -28,32 +28,32 @@ class TestQuotaService(TestCase): def test_should_return_true_if_user_quota_is_not_completely_used(self): qs = self.__build_geocoder_quota_service('test_user') - test_helper.increment_service_uses(self.redis_conn, 'test_user') + increment_service_uses(self.redis_conn, 'test_user') assert qs.check_user_quota() is True def test_should_return_true_if_org_quota_is_not_completely_used(self): qs = self.__build_geocoder_quota_service('test_user', orgname='test_org') - test_helper.increment_service_uses(self.redis_conn, 'test_user', + increment_service_uses(self.redis_conn, 'test_user', orgname='test_org') assert qs.check_user_quota() is True def test_should_return_false_if_user_quota_is_surpassed(self): qs = self.__build_geocoder_quota_service('test_user') - test_helper.increment_service_uses(self.redis_conn, 'test_user', + increment_service_uses(self.redis_conn, 'test_user', amount=300) assert qs.check_user_quota() is False def test_should_return_false_if_org_quota_is_surpassed(self): qs = self.__build_geocoder_quota_service('test_user', orgname='test_org') - test_helper.increment_service_uses(self.redis_conn, 'test_user', + increment_service_uses(self.redis_conn, 'test_user', orgname='test_org', amount=400) assert qs.check_user_quota() is False def test_should_return_true_if_user_quota_is_surpassed_but_soft_limit_is_enabled(self): qs = self.__build_geocoder_quota_service('test_user', soft_limit=True) - test_helper.increment_service_uses(self.redis_conn, 'test_user', + increment_service_uses(self.redis_conn, 'test_user', amount=300) assert qs.check_user_quota() is True @@ -61,7 +61,7 @@ class TestQuotaService(TestCase): qs = self.__build_geocoder_quota_service('test_user', orgname='test_org', soft_limit=True) - test_helper.increment_service_uses(self.redis_conn, 'test_user', + increment_service_uses(self.redis_conn, 'test_user', orgname='test_org', amount=400) assert qs.check_user_quota() is True @@ -140,18 +140,17 @@ class TestQuotaService(TestCase): def __prepare_quota_service(self, username, quota, service, orgname, soft_limit, do_quota, soft_do_limit, end_date): - test_helper.build_redis_user_config(self.redis_conn, username, + build_redis_user_config(self.redis_conn, username, quota=quota, service=service, soft_limit=soft_limit, soft_do_limit=soft_do_limit, do_quota=do_quota, end_date=end_date) if orgname: - test_helper.build_redis_org_config(self.redis_conn, orgname, + build_redis_org_config(self.redis_conn, orgname, quota=quota, service=service, do_quota=do_quota, end_date=end_date) - self._plpy_mock = test_helper.build_plpy_mock() def __build_geocoder_quota_service(self, username, quota=100, service='heremaps', orgname=None, @@ -159,7 +158,7 @@ class TestQuotaService(TestCase): end_date=datetime.today()): self.__prepare_quota_service(username, quota, service, orgname, soft_limit, 0, False, end_date) - geocoder_config = GeocoderConfig(self.redis_conn, self._plpy_mock, + geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock, username, orgname) return QuotaService(geocoder_config, redis_connection=self.redis_conn) @@ -168,7 +167,7 @@ class TestQuotaService(TestCase): quota=100, end_date=datetime.today()): self.__prepare_quota_service(username, quota, service, orgname, soft_limit, 0, False, end_date) - routing_config = RoutingConfig(self.redis_conn, self._plpy_mock, + routing_config = RoutingConfig(self.redis_conn, plpy_mock, username, orgname) return QuotaService(routing_config, redis_connection=self.redis_conn) @@ -177,7 +176,7 @@ class TestQuotaService(TestCase): quota=100, end_date=datetime.today()): self.__prepare_quota_service(username, quota, service, orgname, soft_limit, 0, False, end_date) - isolines_config = IsolinesRoutingConfig(self.redis_conn, self._plpy_mock, + isolines_config = IsolinesRoutingConfig(self.redis_conn, plpy_mock, username, orgname) return QuotaService(isolines_config, redis_connection=self.redis_conn) @@ -188,6 +187,6 @@ class TestQuotaService(TestCase): end_date=datetime.today()): self.__prepare_quota_service(username, 0, service, orgname, False, quota, soft_limit, end_date) - do_config = ObservatorySnapshotConfig(self.redis_conn, self._plpy_mock, + do_config = ObservatorySnapshotConfig(self.redis_conn, plpy_mock, username, orgname) return QuotaService(do_config, redis_connection=self.redis_conn) diff --git a/server/lib/python/cartodb_services/test/test_user_service.py b/server/lib/python/cartodb_services/test/test_user_service.py index a73e00b..2905f8b 100644 --- a/server/lib/python/cartodb_services/test/test_user_service.py +++ b/server/lib/python/cartodb_services/test/test_user_service.py @@ -1,4 +1,4 @@ -import test_helper +from test_helper import * from mockredis import MockRedis from cartodb_services.metrics import UserMetricsService from cartodb_services.metrics import GeocoderConfig @@ -19,13 +19,13 @@ class TestUserService(TestCase): def test_user_used_quota_for_a_day(self): us = self.__build_user_service('test_user') - test_helper.increment_service_uses(self.redis_conn, 'test_user', + increment_service_uses(self.redis_conn, 'test_user', amount=400) assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 400 def test_org_used_quota_for_a_day(self): us = self.__build_user_service('test_user', orgname='test_org') - test_helper.increment_service_uses(self.redis_conn, 'test_user', + increment_service_uses(self.redis_conn, 'test_user', orgname='test_org', amount=400) assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 400 @@ -135,14 +135,13 @@ class TestUserService(TestCase): def __build_user_service(self, username, quota=100, service='heremaps', orgname=None, soft_limit=False, end_date=date.today()): - test_helper.build_redis_user_config(self.redis_conn, username, + build_redis_user_config(self.redis_conn, username, quota=quota, service=service, soft_limit=soft_limit, end_date=end_date) if orgname: - test_helper.build_redis_org_config(self.redis_conn, orgname, + build_redis_org_config(self.redis_conn, orgname, quota=quota, end_date=end_date) - plpy_mock = test_helper.build_plpy_mock() geocoder_config = GeocoderConfig(self.redis_conn, plpy_mock, username, orgname) return UserMetricsService(geocoder_config, self.redis_conn)