From 29cdd8099a83724a440fdd65bde81be50b108ff7 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Mon, 6 Mar 2017 10:20:20 +0100 Subject: [PATCH] Add service configuration (URL, etc) for Mapzen Geocoding --- server/extension/sql/20_geocode_street.sql | 4 ++-- .../cartodb_services/mapzen/geocoder.py | 12 +++++++---- .../service/mapzen_geocoder_config.py | 12 +++++++++-- .../service/test_mapzen_geocoder_config.py | 20 +++++++++++++++++-- .../test/test_mapzengeocoder.py | 11 ++++++++++ 5 files changed, 49 insertions(+), 10 deletions(-) diff --git a/server/extension/sql/20_geocode_street.sql b/server/extension/sql/20_geocode_street.sql index ebe3207..a8c9067 100644 --- a/server/extension/sql/20_geocode_street.sql +++ b/server/extension/sql/20_geocode_street.sql @@ -115,7 +115,7 @@ RETURNS Geometry AS $$ redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] - + plpy.execute("SELECT cdb_dataservices_server._get_logger_config()") logger_config = GD["logger_config"] logger = Logger(logger_config) @@ -174,7 +174,7 @@ RETURNS Geometry AS $$ raise Exception('You have reached the limit of your quota') try: - geocoder = MapzenGeocoder(mapzen_geocoder_config.mapzen_api_key, logger) + geocoder = MapzenGeocoder(mapzen_geocoder_config.mapzen_api_key, logger, mapzen_geocoder_config.service_params) country_iso3 = None if country: country_iso3 = country_to_iso3(country) diff --git a/server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py b/server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py index 1209388..f6767c5 100644 --- a/server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py +++ b/server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py @@ -17,9 +17,13 @@ class MapzenGeocoder(Traceable): CONNECT_TIMEOUT = 10 MAX_RETRIES = 1 - def __init__(self, app_key, logger, base_url=BASE_URL): + def __init__(self, app_key, logger, service_params={}): + service_params = service_params or {} self._app_key = app_key - self._url = base_url + self._url = service_params.get('base_url', self.BASE_URL) + self._connect_timeout = service_params.get('connect_timeout', self.CONNECT_TIMEOUT) + self._read_timeout = service_params.get('read_timeout', self.READ_TIMEOUT) + self._max_retries = service_params.get('max_retries', self.MAX_RETRIES) self._logger = logger @qps_retry(qps=20) @@ -31,9 +35,9 @@ class MapzenGeocoder(Traceable): try: # TODO Extract HTTP client wrapper session = requests.Session() - session.mount(self._url, HTTPAdapter(max_retries=self.MAX_RETRIES)) + session.mount(self._url, HTTPAdapter(max_retries=self._max_retries)) response = session.get(self._url, params=request_params, - timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT)) + timeout=(self._connect_timeout, self._read_timeout)) self.add_response_data(response, self._logger) if response.status_code == requests.codes.ok: return self.__parse_response(response.text) diff --git a/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapzen_geocoder_config.py b/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapzen_geocoder_config.py index 4c5884a..af74e00 100644 --- a/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapzen_geocoder_config.py +++ b/server/lib/python/cartodb_services/cartodb_services/refactor/service/mapzen_geocoder_config.py @@ -13,7 +13,8 @@ class MapzenGeocoderConfig(object): log_path, mapzen_api_key, username, - organization): + organization, + service_params): self._geocoding_quota = geocoding_quota self._soft_geocoding_limit = soft_geocoding_limit self._period_end_date = period_end_date @@ -22,6 +23,7 @@ class MapzenGeocoderConfig(object): self._mapzen_api_key = mapzen_api_key self._username = username self._organization = organization + self._service_params = service_params # Kind of generic properties. Note which ones are for actually running the # service and which ones are needed for quota stuff. @@ -72,6 +74,10 @@ class MapzenGeocoderConfig(object): def organization(self): return self._organization + @property + def service_params(self): + return self._service_params + # TODO: for BW compat, remove @property def google_geocoder(self): @@ -90,6 +96,7 @@ class MapzenGeocoderConfigBuilder(object): def get(self): mapzen_server_conf = self._server_conf.get('mapzen_conf') mapzen_api_key = mapzen_server_conf['geocoder']['api_key'] + mapzen_service_params = mapzen_server_conf['geocoder'].get('service', {}) geocoding_quota = self._get_quota(mapzen_server_conf) soft_geocoding_limit = self._user_conf.get('soft_geocoding_limit').lower() == 'true' @@ -107,7 +114,8 @@ class MapzenGeocoderConfigBuilder(object): log_path, mapzen_api_key, self._username, - self._orgname) + self._orgname, + mapzen_service_params) def _get_quota(self, mapzen_server_conf): geocoding_quota = self._org_conf.get('geocoding_quota') or self._user_conf.get('geocoding_quota') diff --git a/server/lib/python/cartodb_services/test/refactor/service/test_mapzen_geocoder_config.py b/server/lib/python/cartodb_services/test/refactor/service/test_mapzen_geocoder_config.py index d67a2bf..b61f4e6 100644 --- a/server/lib/python/cartodb_services/test/refactor/service/test_mapzen_geocoder_config.py +++ b/server/lib/python/cartodb_services/test/refactor/service/test_mapzen_geocoder_config.py @@ -13,7 +13,7 @@ class TestMapzenGeocoderUserConfig(TestCase): self._server_config = InMemoryConfigStorage({"server_conf": {"environment": "testing"}, "mapzen_conf": {"geocoder": - {"api_key": "search-xxxxxxx", "monthly_quota": 1500000} + {"api_key": "search-xxxxxxx", "monthly_quota": 1500000, "service":{"base_url":"http://base"}} }, "logger_conf": {}}) self._username = 'test_user' self._user_key = "rails:users:{0}".format(self._username) @@ -81,6 +81,14 @@ class TestMapzenGeocoderUserConfig(TestCase): self._redis_connection.hset(self._user_key, 'soft_geocoding_limit', 'false') self._redis_connection.hset(self._user_key, 'period_end_date', '2016-12-31 00:00:00') + def test_config_service_values(self): + config = MapzenGeocoderConfigBuilder(self._server_config, + self._user_config, + self._org_config, + self._username, + None).get() + assert config.service_params == {"base_url":"http://base"} + class TestMapzenGeocoderOrgConfig(TestCase): def setUp(self): @@ -151,4 +159,12 @@ class TestMapzenGeocoderOrgConfig(TestCase): self._redis_connection.hset(self._user_key, 'soft_geocoding_limit', 'false') self._redis_connection.hset(self._user_key, 'period_end_date', '2016-12-15 00:00:00') self._redis_connection.hset(self._org_key, 'geocoding_quota', '200') - self._redis_connection.hset(self._org_key, 'period_end_date', '2016-12-31 00:00:00') \ No newline at end of file + self._redis_connection.hset(self._org_key, 'period_end_date', '2016-12-31 00:00:00') + + def test_config_default_service_values(self): + config = MapzenGeocoderConfigBuilder(self._server_config, + self._user_config, + self._org_config, + self._username, + self._organization).get() + assert config.service_params == {} diff --git a/server/lib/python/cartodb_services/test/test_mapzengeocoder.py b/server/lib/python/cartodb_services/test/test_mapzengeocoder.py index 19d8f42..3a3b1e6 100644 --- a/server/lib/python/cartodb_services/test/test_mapzengeocoder.py +++ b/server/lib/python/cartodb_services/test/test_mapzengeocoder.py @@ -109,3 +109,14 @@ class MapzenGeocoderTestCase(unittest.TestCase): self.geocoder.geocode( searchtext='Calle Siempreviva 3, Valladolid', country='ESP') + + def test_geocode_address_with_nonstandard_url(self, req_mock): + nonstandard_url = 'http://nonstandardmapzen' + req_mock.register_uri('GET', nonstandard_url, text=self.GOOD_RESPONSE) + geocoder = MapzenGeocoder('search-XXXXXXX', Mock(), { 'base_url': nonstandard_url }) + response = geocoder.geocode( + searchtext='Calle Siempreviva 3, Valldolid', + country='ESP') + + self.assertEqual(response[0], -4.730928) + self.assertEqual(response[1], 41.669034)