diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py index d2b3b80..0f318a0 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/config.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/config.py @@ -146,6 +146,18 @@ class InternalGeocoderConfig(ServiceConfig): def service_type(self): return 'geocoder_internal' + @property + def is_high_resolution(self): + return False + + @property + def cost_per_hit(self): + return 0 + + @property + def geocoding_quota(self): + return None + class GeocoderConfig(ServiceConfig): @@ -168,6 +180,7 @@ class GeocoderConfig(ServiceConfig): USERNAME_KEY = 'username' ORGNAME_KEY = 'orgname' PERIOD_END_DATE = 'period_end_date' + LOG_PATH = '/var/log/postgresql/geocodings.log' def __init__(self, redis_connection, username, orgname=None, heremaps_app_id=None, heremaps_app_code=None): @@ -256,7 +269,10 @@ class GeocoderConfig(ServiceConfig): @property def geocoding_quota(self): - return self._geocoding_quota + if self.heremaps_geocoder: + return self._geocoding_quota + else: + return None @property def soft_geocoding_limit(self): @@ -273,3 +289,18 @@ class GeocoderConfig(ServiceConfig): @property def heremaps_app_code(self): return self._heremaps_app_code + + @property + def is_high_resolution(self): + return True + + @property + def cost_per_hit(self): + if self.heremaps_geocoder: + return 1 + else: + return 0 + + @property + def log_path(self): + return self.LOG_PATH diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/log.py b/server/lib/python/cartodb_services/cartodb_services/metrics/log.py new file mode 100644 index 0000000..82dacb2 --- /dev/null +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/log.py @@ -0,0 +1,74 @@ + +import abc +import json +import re + + +class LoggerFactory: + + @classmethod + def build(self, service_config): + if re.match('geocoder_*', service_config.service_type): + return GeocoderLogger(service_config) + else: + return None + + +class Logger(object): + __metaclass__ = abc.ABCMeta + + def __init__(self, file_path): + self._file_path = file_path + + def dump_to_file(self, data): + with open(self._file_path, 'a') as logfile: + json.dump(data, logfile) + + @abc.abstractproperty + def log(self, **data): + raise NotImplementedError('log method must be defined') + + +class GeocoderLogger(Logger): + + def __init__(self, service_config): + super(GeocoderLogger, self).__init__(service_config.log_path) + self._service_config = service_config + + def log(self, **data): + dump_data = self._dump_data(**data) + self.dump_to_file(dump_data) + + def _dump_data(self, **data): + if data['success']: + cost = self._service_config.cost_per_hit + failed_rows = 0 + successful_rows = 1 + else: + cost = 0 + failed_rows = 1 + successful_rows = 0 + + if self._service_config.is_high_resolution: + kind = 'high-resolution' + else: + kind = 'internal' + + return { + "batched": False, + "cache_hits": 0, # Always 0 because no cache involved + # https://github.com/CartoDB/cartodb/blob/master/app/models/geocoding.rb#L208-L211 + "cost": cost, + "created_at": datetime.now().isoformat(), + "failed_rows": failed_rows, + "geocoder_type": self._service_config.service_type, + "kind": kind, + "processable_rows": 1, + "processed_rows": successful_rows, + "real_rows": successful_rows, + "success": data['success'], + "successful_rows": successful_rows, + "used_credits": 0, + "username": self._service_config.username, + "organization": self._service_config.organization + } diff --git a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py index 33550cc..7eefd11 100644 --- a/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py +++ b/server/lib/python/cartodb_services/cartodb_services/metrics/quota.py @@ -1,19 +1,20 @@ from user import UserMetricsService +from log import LoggerFactory from datetime import date import re class QuotaService: """ Class to manage all the quota operation for - the Geocoder SQL API Extension """ + the Dataservices SQL API Extension """ def __init__(self, user_service_config, redis_connection): self._user_service_config = user_service_config - # TODO First step to extract to a factory if needed in the future self._quota_checker = QuotaChecker(user_service_config, redis_connection) - self._user_service = UserMetricsService( - self._user_service_config, redis_connection) + self._user_service = UserMetricsService(self._user_service_config, + redis_connection) + self._logger = LoggerFactory.build(user_service_config) def check_user_quota(self): return self._quota_checker.check() @@ -22,16 +23,19 @@ class QuotaService: self._user_service.increment_service_use( self._user_service_config.service_type, "success_responses", amount=amount) + self._log_service_process("success") def increment_empty_service_use(self, amount=1): self._user_service.increment_service_use( self._user_service_config.service_type, "empty_responses", amount=amount) + self._log_service_process("empty") def increment_failed_service_use(self, amount=1): self._user_service.increment_service_use( self._user_service_config.service_type, "fail_responses", amount=amount) + self._log_service_process("fail") def increment_total_service_use(self, amount=1): self._user_service.increment_service_use( @@ -43,6 +47,13 @@ class QuotaService: self._user_service_config.service_type, "isolines_generated", amount=amount) + def _log_service_process(self, event): + if self._logger: + if event is 'success' or event is 'empty': + self._logger.log(success=True) + elif event is 'empty': + self._logger.log(success=False) + class QuotaChecker: