cartodb4/dataservices-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add service configuration (URL, etc) for Mapzen Geocoding

Browse Source
This commit is contained in:
Javier Goizueta 2017-03-06 10:20:20 +01:00
parent c0374f5800
commit 29cdd8099a
5 changed files with 49 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
server/extension/sql/20_geocode_street.sql
View File

@ -115,7 +115,7 @@ RETURNS Geometry AS $$
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
@ -174,7 +174,7 @@ RETURNS Geometry AS $$
raise Exception('You have reached the limit of your quota')
try:
geocoder = MapzenGeocoder(mapzen_geocoder_config.mapzen_api_key, logger)
geocoder = MapzenGeocoder(mapzen_geocoder_config.mapzen_api_key, logger, mapzen_geocoder_config.service_params)
country_iso3 = None
if country:
country_iso3 = country_to_iso3(country)

12
server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py
View File

@ -17,9 +17,13 @@ class MapzenGeocoder(Traceable):
CONNECT_TIMEOUT = 10
MAX_RETRIES = 1
def __init__(self, app_key, logger, base_url=BASE_URL):
def __init__(self, app_key, logger, service_params={}):
service_params = service_params or {}
self._app_key = app_key
self._url = base_url
self._url = service_params.get('base_url', self.BASE_URL)
self._connect_timeout = service_params.get('connect_timeout', self.CONNECT_TIMEOUT)
self._read_timeout = service_params.get('read_timeout', self.READ_TIMEOUT)
self._max_retries = service_params.get('max_retries', self.MAX_RETRIES)
self._logger = logger
@qps_retry(qps=20)
@ -31,9 +35,9 @@ class MapzenGeocoder(Traceable):
try:
# TODO Extract HTTP client wrapper
session = requests.Session()
session.mount(self._url, HTTPAdapter(max_retries=self.MAX_RETRIES))
session.mount(self._url, HTTPAdapter(max_retries=self._max_retries))
response = session.get(self._url, params=request_params,
timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
timeout=(self._connect_timeout, self._read_timeout))
self.add_response_data(response, self._logger)
if response.status_code == requests.codes.ok:
return self.__parse_response(response.text)

12
server/lib/python/cartodb_services/cartodb_services/refactor/service/mapzen_geocoder_config.py
View File

@ -13,7 +13,8 @@ class MapzenGeocoderConfig(object):
log_path,
mapzen_api_key,
username,
organization):
organization,
service_params):
self._geocoding_quota = geocoding_quota
self._soft_geocoding_limit = soft_geocoding_limit
self._period_end_date = period_end_date
@ -22,6 +23,7 @@ class MapzenGeocoderConfig(object):
self._mapzen_api_key = mapzen_api_key
self._username = username
self._organization = organization
self._service_params = service_params
# Kind of generic properties. Note which ones are for actually running the
# service and which ones are needed for quota stuff.
@ -72,6 +74,10 @@ class MapzenGeocoderConfig(object):
def organization(self):
return self._organization
@property
def service_params(self):
return self._service_params
# TODO: for BW compat, remove
@property
def google_geocoder(self):
@ -90,6 +96,7 @@ class MapzenGeocoderConfigBuilder(object):
def get(self):
mapzen_server_conf = self._server_conf.get('mapzen_conf')
mapzen_api_key = mapzen_server_conf['geocoder']['api_key']
mapzen_service_params = mapzen_server_conf['geocoder'].get('service', {})
geocoding_quota = self._get_quota(mapzen_server_conf)
soft_geocoding_limit = self._user_conf.get('soft_geocoding_limit').lower() == 'true'
@ -107,7 +114,8 @@ class MapzenGeocoderConfigBuilder(object):
log_path,
mapzen_api_key,
self._username,
self._orgname)
self._orgname,
mapzen_service_params)
def _get_quota(self, mapzen_server_conf):
geocoding_quota = self._org_conf.get('geocoding_quota') or self._user_conf.get('geocoding_quota')

20
server/lib/python/cartodb_services/test/refactor/service/test_mapzen_geocoder_config.py
View File

@ -13,7 +13,7 @@ class TestMapzenGeocoderUserConfig(TestCase):
self._server_config = InMemoryConfigStorage({"server_conf": {"environment": "testing"},
"mapzen_conf":
{"geocoder":
{"api_key": "search-xxxxxxx", "monthly_quota": 1500000}
{"api_key": "search-xxxxxxx", "monthly_quota": 1500000, "service":{"base_url":"http://base"}}
}, "logger_conf": {}})
self._username = 'test_user'
self._user_key = "rails:users:{0}".format(self._username)
@ -81,6 +81,14 @@ class TestMapzenGeocoderUserConfig(TestCase):
self._redis_connection.hset(self._user_key, 'soft_geocoding_limit', 'false')
self._redis_connection.hset(self._user_key, 'period_end_date', '2016-12-31 00:00:00')
def test_config_service_values(self):
config = MapzenGeocoderConfigBuilder(self._server_config,
self._user_config,
self._org_config,
self._username,
None).get()
assert config.service_params == {"base_url":"http://base"}
class TestMapzenGeocoderOrgConfig(TestCase):
def setUp(self):
@ -151,4 +159,12 @@ class TestMapzenGeocoderOrgConfig(TestCase):
self._redis_connection.hset(self._user_key, 'soft_geocoding_limit', 'false')
self._redis_connection.hset(self._user_key, 'period_end_date', '2016-12-15 00:00:00')
self._redis_connection.hset(self._org_key, 'geocoding_quota', '200')
self._redis_connection.hset(self._org_key, 'period_end_date', '2016-12-31 00:00:00')
self._redis_connection.hset(self._org_key, 'period_end_date', '2016-12-31 00:00:00')
def test_config_default_service_values(self):
config = MapzenGeocoderConfigBuilder(self._server_config,
self._user_config,
self._org_config,
self._username,
self._organization).get()
assert config.service_params == {}

11
server/lib/python/cartodb_services/test/test_mapzengeocoder.py
View File

@ -109,3 +109,14 @@ class MapzenGeocoderTestCase(unittest.TestCase):
self.geocoder.geocode(
searchtext='Calle Siempreviva 3, Valladolid',
country='ESP')
def test_geocode_address_with_nonstandard_url(self, req_mock):
nonstandard_url = 'http://nonstandardmapzen'
req_mock.register_uri('GET', nonstandard_url, text=self.GOOD_RESPONSE)
geocoder = MapzenGeocoder('search-XXXXXXX', Mock(), { 'base_url': nonstandard_url })
response = geocoder.geocode(
searchtext='Calle Siempreviva 3, Valldolid',
country='ESP')
self.assertEqual(response[0], -4.730928)
self.assertEqual(response[1], 41.669034)