Merge pull request #571 from CartoDB/antoniocarlon/568/Add_geocodio_geocoder
Add Geocodio geocoder
This commit is contained in:
commit
89dff1244f
@ -180,6 +180,15 @@ SELECT CDB_Conf_SetConf(
|
||||
);
|
||||
```
|
||||
|
||||
#### Geocod.io configuration
|
||||
|
||||
```sql
|
||||
SELECT CDB_Conf_SetConf(
|
||||
'geocodio_conf',
|
||||
'{"geocoder": {"api_keys": ["your_api_key"], "monthly_quota": 999999}}'
|
||||
);
|
||||
```
|
||||
|
||||
#### Data Observatory
|
||||
|
||||
```sql
|
||||
|
83
client/cdb_dataservices_client--0.27.0--0.28.0.sql
Normal file
83
client/cdb_dataservices_client--0.27.0--0.28.0.sql
Normal file
@ -0,0 +1,83 @@
|
||||
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
|
||||
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
|
||||
\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.28.0'" to load this file. \quit
|
||||
|
||||
-- Make sure we have a sane search path to create/update the extension
|
||||
SET search_path = "$user",cartodb,public,cdb_dataservices_client;
|
||||
|
||||
-- HERE goes your code to upgrade/downgrade
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocodio_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL)
|
||||
RETURNS public.Geometry AS $$
|
||||
DECLARE
|
||||
ret public.Geometry;
|
||||
username text;
|
||||
orgname text;
|
||||
_returned_sqlstate TEXT;
|
||||
_message_text TEXT;
|
||||
_pg_exception_context TEXT;
|
||||
apikey_permissions json;
|
||||
BEGIN
|
||||
|
||||
SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json);
|
||||
IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN
|
||||
RAISE EXCEPTION 'Geocoding permission denied';
|
||||
END IF;
|
||||
|
||||
-- JSON value stored "" is taken as literal
|
||||
IF username IS NULL OR username = '' OR username = '""' THEN
|
||||
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
|
||||
END IF;
|
||||
|
||||
|
||||
BEGIN
|
||||
SELECT cdb_dataservices_client._cdb_geocodio_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret;
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE,
|
||||
_message_text = MESSAGE_TEXT,
|
||||
_pg_exception_context = PG_EXCEPTION_CONTEXT;
|
||||
RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context;
|
||||
RETURN ret;
|
||||
END;
|
||||
END;
|
||||
$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE
|
||||
SET search_path = pg_temp;
|
||||
|
||||
DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocodio_geocode_street_point (username text, orgname text, searchtext text, city text, state_province text, country text);
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocodio_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL)
|
||||
RETURNS Geometry AS $$
|
||||
CONNECT cdb_dataservices_client._server_conn_str();
|
||||
|
||||
SELECT cdb_dataservices_server.cdb_geocodio_geocode_street_point (username, orgname, searchtext, city, state_province, country);
|
||||
|
||||
$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
|
||||
|
||||
DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_geocodio_geocode_street_point (searchtext text ,city text ,state_province text ,country text);
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocodio_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL)
|
||||
RETURNS public.Geometry AS $$
|
||||
DECLARE
|
||||
ret public.Geometry;
|
||||
username text;
|
||||
orgname text;
|
||||
apikey_permissions json;
|
||||
BEGIN
|
||||
|
||||
SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json);
|
||||
IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN
|
||||
RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007';
|
||||
END IF;
|
||||
|
||||
-- JSON value stored "" is taken as literal
|
||||
IF username IS NULL OR username = '' OR username = '""' THEN
|
||||
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
|
||||
END IF;
|
||||
|
||||
SELECT cdb_dataservices_client._cdb_geocodio_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret;
|
||||
END;
|
||||
$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE
|
||||
SET search_path = pg_temp;
|
||||
|
||||
GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocodio_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser;
|
||||
GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocodio_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text ) TO publicuser;
|
||||
|
14
client/cdb_dataservices_client--0.28.0--0.27.0.sql
Normal file
14
client/cdb_dataservices_client--0.28.0--0.27.0.sql
Normal file
@ -0,0 +1,14 @@
|
||||
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
|
||||
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
|
||||
\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.27.0'" to load this file. \quit
|
||||
|
||||
-- Make sure we have a sane search path to create/update the extension
|
||||
SET search_path = "$user",cartodb,public,cdb_dataservices_client;
|
||||
|
||||
-- HERE goes your code to upgrade/downgrade
|
||||
|
||||
DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocodio_geocode_street_point_exception_safe;
|
||||
|
||||
DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocodio_geocode_street_point;
|
||||
|
||||
DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_geocodio_geocode_street_point;
|
5823
client/cdb_dataservices_client--0.28.0.sql
Normal file
5823
client/cdb_dataservices_client--0.28.0.sql
Normal file
File diff suppressed because it is too large
Load Diff
@ -1,5 +1,5 @@
|
||||
comment = 'CartoDB dataservices client API extension'
|
||||
default_version = '0.27.0'
|
||||
default_version = '0.28.0'
|
||||
requires = 'plproxy, cartodb'
|
||||
superuser = true
|
||||
schema = cdb_dataservices_client
|
||||
|
@ -160,6 +160,17 @@
|
||||
- { name: state_province, type: text, default: 'NULL'}
|
||||
- { name: country, type: text, default: 'NULL'}
|
||||
|
||||
- name: cdb_geocodio_geocode_street_point
|
||||
return_type: public.Geometry
|
||||
requires_permission: true
|
||||
permission_name: geocoding
|
||||
permission_error: Geocoding permission denied
|
||||
params:
|
||||
- { name: searchtext, type: text}
|
||||
- { name: city, type: text, default: 'NULL'}
|
||||
- { name: state_province, type: text, default: 'NULL'}
|
||||
- { name: country, type: text, default: 'NULL'}
|
||||
|
||||
- name: cdb_mapzen_geocode_street_point
|
||||
return_type: public.Geometry
|
||||
requires_permission: true
|
||||
|
197
server/extension/cdb_dataservices_server--0.35.1--0.36.0.sql
Normal file
197
server/extension/cdb_dataservices_server--0.35.1--0.36.0.sql
Normal file
@ -0,0 +1,197 @@
|
||||
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
|
||||
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
|
||||
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.36.0'" to load this file. \quit
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
|
||||
RETURNS Geometry AS $$
|
||||
from cartodb_services.metrics import metrics
|
||||
from cartodb_services.tools import Logger,LoggerConfig
|
||||
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
|
||||
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
|
||||
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
|
||||
logger_config = GD["logger_config"]
|
||||
logger = Logger(logger_config)
|
||||
|
||||
params = {'username': username, 'orgname': orgname, 'searchtext': searchtext, 'city': city, 'state_province': state_province, 'country': country}
|
||||
|
||||
with metrics('cdb_geocode_street_point', user_geocoder_config, logger, params):
|
||||
if user_geocoder_config.heremaps_geocoder:
|
||||
here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
|
||||
return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
|
||||
elif user_geocoder_config.google_geocoder:
|
||||
google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
|
||||
return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
|
||||
elif user_geocoder_config.mapzen_geocoder:
|
||||
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
|
||||
return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
|
||||
elif user_geocoder_config.mapbox_geocoder:
|
||||
mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
|
||||
return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
|
||||
elif user_geocoder_config.tomtom_geocoder:
|
||||
tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
|
||||
return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
|
||||
elif user_geocoder_config.geocodio_geocoder:
|
||||
geocodio_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocodio_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
|
||||
return plpy.execute(geocodio_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
|
||||
else:
|
||||
raise Exception('Requested geocoder is not available')
|
||||
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocodio_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
|
||||
RETURNS Geometry AS $$
|
||||
# The configuration is retrieved but no checks are performed on it
|
||||
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
|
||||
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
|
||||
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
|
||||
|
||||
geocodio_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocodio_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
|
||||
return plpy.execute(geocodio_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
|
||||
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocodio_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
|
||||
RETURNS Geometry AS $$
|
||||
from iso3166 import countries
|
||||
from cartodb_services.tools import ServiceManager, QuotaExceededException
|
||||
from cartodb_services.geocodio import GeocodioGeocoder
|
||||
from cartodb_services.tools.country import country_to_iso3
|
||||
from cartodb_services.refactor.service.geocodio_geocoder_config import GeocodioGeocoderConfigBuilder
|
||||
|
||||
import cartodb_services
|
||||
cartodb_services.init(plpy, GD)
|
||||
|
||||
service_manager = ServiceManager('geocoder', GeocodioGeocoderConfigBuilder, username, orgname, GD)
|
||||
|
||||
try:
|
||||
service_manager.assert_within_limits()
|
||||
geocoder = GeocodioGeocoder(service_manager.config.geocodio_api_key, service_manager.logger, service_manager.config.service_params)
|
||||
|
||||
country_iso3166 = None
|
||||
if country:
|
||||
country_iso3 = country_to_iso3(country)
|
||||
if country_iso3:
|
||||
country_iso3166 = countries.get(country_iso3).alpha2.lower()
|
||||
|
||||
coordinates = geocoder.geocode(searchtext=searchtext, city=city,
|
||||
state_province=state_province,
|
||||
country=country_iso3166)
|
||||
|
||||
if coordinates:
|
||||
service_manager.quota_service.increment_success_service_use()
|
||||
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
|
||||
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
|
||||
return point['st_setsrid']
|
||||
else:
|
||||
service_manager.quota_service.increment_empty_service_use()
|
||||
return None
|
||||
except QuotaExceededException as qe:
|
||||
service_manager.quota_service.increment_failed_service_use()
|
||||
return None
|
||||
except BaseException as e:
|
||||
import sys
|
||||
service_manager.quota_service.increment_failed_service_use()
|
||||
service_manager.logger.error('Error trying to geocode street point using Geocodio', sys.exc_info(), data={"username": username, "orgname": orgname})
|
||||
raise Exception('Error trying to geocode street point using Geocodio')
|
||||
finally:
|
||||
service_manager.quota_service.increment_total_service_use()
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocode_street_point(username TEXT, orgname TEXT, searches jsonb)
|
||||
RETURNS SETOF cdb_dataservices_server.geocoding AS $$
|
||||
from cartodb_services.metrics import metrics
|
||||
from cartodb_services.tools import Logger
|
||||
import json
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
|
||||
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
|
||||
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
|
||||
logger_config = GD["logger_config"]
|
||||
logger = Logger(logger_config)
|
||||
|
||||
params = {'username': username, 'orgname': orgname, 'searches': json.loads(searches)}
|
||||
|
||||
with metrics('cdb_bulk_geocode_street_point', user_geocoder_config, logger, params):
|
||||
if user_geocoder_config.google_geocoder:
|
||||
provider_function = "_cdb_bulk_google_geocode_street_point";
|
||||
elif user_geocoder_config.heremaps_geocoder:
|
||||
provider_function = "_cdb_bulk_heremaps_geocode_street_point";
|
||||
elif user_geocoder_config.tomtom_geocoder:
|
||||
provider_function = "_cdb_bulk_tomtom_geocode_street_point";
|
||||
elif user_geocoder_config.mapbox_geocoder:
|
||||
provider_function = "_cdb_bulk_mapbox_geocode_street_point";
|
||||
elif user_geocoder_config.geocodio_geocoder:
|
||||
provider_function = "_cdb_bulk_geocodio_geocode_street_point";
|
||||
else:
|
||||
raise Exception('Requested geocoder is not available')
|
||||
|
||||
plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.{}($1, $2, $3); ".format(provider_function), ["text", "text", "jsonb"])
|
||||
return plpy.execute(plan, [username, orgname, searches])
|
||||
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocodio_geocode_street_point(username TEXT, orgname TEXT, searches jsonb)
|
||||
RETURNS SETOF cdb_dataservices_server.geocoding AS $$
|
||||
from cartodb_services import run_street_point_geocoder
|
||||
from cartodb_services.tools import ServiceManager
|
||||
from cartodb_services.refactor.service.geocodio_geocoder_config import GeocodioGeocoderConfigBuilder
|
||||
from cartodb_services.geocodio import GeocodioBulkGeocoder
|
||||
from cartodb_services.tools import Logger
|
||||
import cartodb_services
|
||||
cartodb_services.init(plpy, GD)
|
||||
|
||||
logger_config = GD["logger_config"]
|
||||
logger = Logger(logger_config)
|
||||
service_manager = ServiceManager('geocoder', GeocodioGeocoderConfigBuilder, username, orgname, GD)
|
||||
geocoder = GeocodioBulkGeocoder(service_manager.config.geocodio_api_key, service_manager.logger, service_manager.config.service_params)
|
||||
return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches)
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text)
|
||||
RETURNS Geometry AS $$
|
||||
from cartodb_services.metrics import QuotaService
|
||||
from cartodb_services.metrics import InternalGeocoderConfig
|
||||
from cartodb_services.metrics import metrics
|
||||
from cartodb_services.tools import Logger,LoggerConfig
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
|
||||
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
|
||||
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
|
||||
logger_config = GD["logger_config"]
|
||||
logger = Logger(logger_config)
|
||||
quota_service = QuotaService(user_geocoder_config, redis_conn)
|
||||
|
||||
params = {'username': username, 'orgname': orgname, 'country_name': country_name}
|
||||
|
||||
with metrics('cdb_geocode_admin0_polygon', user_geocoder_config, logger, params):
|
||||
try:
|
||||
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"])
|
||||
rv = plpy.execute(plan, [country_name], 1)
|
||||
result = rv[0]["mypolygon"]
|
||||
if result:
|
||||
quota_service.increment_success_service_use()
|
||||
return result
|
||||
else:
|
||||
quota_service.increment_empty_service_use()
|
||||
return None
|
||||
except BaseException as e:
|
||||
import sys
|
||||
quota_service.increment_failed_service_use()
|
||||
logger.error('Error trying to geocode admin0 polygon', sys.exc_info(), data={"username": username, "orgname": orgname})
|
||||
raise Exception('Error trying to geocode admin0 polygon')
|
||||
finally:
|
||||
quota_service.increment_total_service_use()
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
79
server/extension/cdb_dataservices_server--0.36.0--0.35.1.sql
Normal file
79
server/extension/cdb_dataservices_server--0.36.0--0.35.1.sql
Normal file
@ -0,0 +1,79 @@
|
||||
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
|
||||
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
|
||||
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.35.1'" to load this file. \quit
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
|
||||
RETURNS Geometry AS $$
|
||||
from cartodb_services.metrics import metrics
|
||||
from cartodb_services.tools import Logger,LoggerConfig
|
||||
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
|
||||
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
|
||||
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
|
||||
logger_config = GD["logger_config"]
|
||||
logger = Logger(logger_config)
|
||||
|
||||
params = {'username': username, 'orgname': orgname, 'searchtext': searchtext, 'city': city, 'state_province': state_province, 'country': country}
|
||||
|
||||
with metrics('cdb_geocode_street_point', user_geocoder_config, logger, params):
|
||||
if user_geocoder_config.heremaps_geocoder:
|
||||
here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
|
||||
return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
|
||||
elif user_geocoder_config.google_geocoder:
|
||||
google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
|
||||
return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
|
||||
elif user_geocoder_config.mapzen_geocoder:
|
||||
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
|
||||
return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
|
||||
elif user_geocoder_config.mapbox_geocoder:
|
||||
mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
|
||||
return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
|
||||
elif user_geocoder_config.tomtom_geocoder:
|
||||
tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
|
||||
return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
|
||||
else:
|
||||
raise Exception('Requested geocoder is not available')
|
||||
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
||||
DROP FUNCTION IF EXISTS cdb_dataservices_server.cdb_geocodio_geocode_street_point;
|
||||
|
||||
DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_geocodio_geocode_street_point;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocode_street_point(username TEXT, orgname TEXT, searches jsonb)
|
||||
RETURNS SETOF cdb_dataservices_server.geocoding AS $$
|
||||
from cartodb_services.metrics import metrics
|
||||
from cartodb_services.tools import Logger
|
||||
import json
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
|
||||
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
|
||||
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
|
||||
logger_config = GD["logger_config"]
|
||||
logger = Logger(logger_config)
|
||||
|
||||
params = {'username': username, 'orgname': orgname, 'searches': json.loads(searches)}
|
||||
|
||||
with metrics('cdb_bulk_geocode_street_point', user_geocoder_config, logger, params):
|
||||
if user_geocoder_config.google_geocoder:
|
||||
provider_function = "_cdb_bulk_google_geocode_street_point";
|
||||
elif user_geocoder_config.heremaps_geocoder:
|
||||
provider_function = "_cdb_bulk_heremaps_geocode_street_point";
|
||||
elif user_geocoder_config.tomtom_geocoder:
|
||||
provider_function = "_cdb_bulk_tomtom_geocode_street_point";
|
||||
elif user_geocoder_config.mapbox_geocoder:
|
||||
provider_function = "_cdb_bulk_mapbox_geocode_street_point";
|
||||
else:
|
||||
raise Exception('Requested geocoder is not available')
|
||||
|
||||
plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.{}($1, $2, $3); ".format(provider_function), ["text", "text", "jsonb"])
|
||||
return plpy.execute(plan, [username, orgname, searches])
|
||||
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
||||
DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_bulk_geocodio_geocode_street_point;
|
3888
server/extension/cdb_dataservices_server--0.36.0.sql
Normal file
3888
server/extension/cdb_dataservices_server--0.36.0.sql
Normal file
File diff suppressed because it is too large
Load Diff
@ -1,5 +1,5 @@
|
||||
comment = 'CartoDB dataservices server extension'
|
||||
default_version = '0.35.1'
|
||||
default_version = '0.36.0'
|
||||
requires = 'plpythonu, plproxy, postgis, cdb_geocoder'
|
||||
superuser = true
|
||||
schema = cdb_dataservices_server
|
||||
|
@ -30,6 +30,9 @@ RETURNS Geometry AS $$
|
||||
elif user_geocoder_config.tomtom_geocoder:
|
||||
tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
|
||||
return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
|
||||
elif user_geocoder_config.geocodio_geocoder:
|
||||
geocodio_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocodio_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
|
||||
return plpy.execute(geocodio_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
|
||||
else:
|
||||
raise Exception('Requested geocoder is not available')
|
||||
|
||||
@ -105,6 +108,19 @@ RETURNS Geometry AS $$
|
||||
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocodio_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
|
||||
RETURNS Geometry AS $$
|
||||
# The configuration is retrieved but no checks are performed on it
|
||||
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
|
||||
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
|
||||
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
|
||||
|
||||
geocodio_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocodio_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
|
||||
return plpy.execute(geocodio_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
|
||||
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
|
||||
RETURNS Geometry AS $$
|
||||
from cartodb_services.tools import LegacyServiceManager
|
||||
@ -302,3 +318,49 @@ RETURNS Geometry AS $$
|
||||
finally:
|
||||
service_manager.quota_service.increment_total_service_use()
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocodio_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
|
||||
RETURNS Geometry AS $$
|
||||
from iso3166 import countries
|
||||
from cartodb_services.tools import ServiceManager, QuotaExceededException
|
||||
from cartodb_services.geocodio import GeocodioGeocoder
|
||||
from cartodb_services.tools.country import country_to_iso3
|
||||
from cartodb_services.refactor.service.geocodio_geocoder_config import GeocodioGeocoderConfigBuilder
|
||||
|
||||
import cartodb_services
|
||||
cartodb_services.init(plpy, GD)
|
||||
|
||||
service_manager = ServiceManager('geocoder', GeocodioGeocoderConfigBuilder, username, orgname, GD)
|
||||
|
||||
try:
|
||||
service_manager.assert_within_limits()
|
||||
geocoder = GeocodioGeocoder(service_manager.config.geocodio_api_key, service_manager.logger, service_manager.config.service_params)
|
||||
|
||||
country_iso3166 = None
|
||||
if country:
|
||||
country_iso3 = country_to_iso3(country)
|
||||
if country_iso3:
|
||||
country_iso3166 = countries.get(country_iso3).alpha2.lower()
|
||||
|
||||
coordinates = geocoder.geocode(searchtext=searchtext, city=city,
|
||||
state_province=state_province,
|
||||
country=country_iso3166)
|
||||
if coordinates:
|
||||
service_manager.quota_service.increment_success_service_use()
|
||||
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
|
||||
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
|
||||
return point['st_setsrid']
|
||||
else:
|
||||
service_manager.quota_service.increment_empty_service_use()
|
||||
return None
|
||||
except QuotaExceededException as qe:
|
||||
service_manager.quota_service.increment_failed_service_use()
|
||||
return None
|
||||
except BaseException as e:
|
||||
import sys
|
||||
service_manager.quota_service.increment_failed_service_use()
|
||||
service_manager.logger.error('Error trying to geocode street point using Geocodio', sys.exc_info(), data={"username": username, "orgname": orgname})
|
||||
raise Exception('Error trying to geocode street point using Geocodio')
|
||||
finally:
|
||||
service_manager.quota_service.increment_total_service_use()
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
@ -32,6 +32,8 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$
|
||||
provider_function = "_cdb_bulk_tomtom_geocode_street_point";
|
||||
elif user_geocoder_config.mapbox_geocoder:
|
||||
provider_function = "_cdb_bulk_mapbox_geocode_street_point";
|
||||
elif user_geocoder_config.geocodio_geocoder:
|
||||
provider_function = "_cdb_bulk_geocodio_geocode_street_point";
|
||||
else:
|
||||
raise Exception('Requested geocoder is not available')
|
||||
|
||||
@ -96,3 +98,19 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$
|
||||
return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches)
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocodio_geocode_street_point(username TEXT, orgname TEXT, searches jsonb)
|
||||
RETURNS SETOF cdb_dataservices_server.geocoding AS $$
|
||||
from cartodb_services import run_street_point_geocoder
|
||||
from cartodb_services.tools import ServiceManager
|
||||
from cartodb_services.refactor.service.geocodio_geocoder_config import GeocodioGeocoderConfigBuilder
|
||||
from cartodb_services.geocodio import GeocodioBulkGeocoder
|
||||
from cartodb_services.tools import Logger
|
||||
import cartodb_services
|
||||
cartodb_services.init(plpy, GD)
|
||||
|
||||
logger_config = GD["logger_config"]
|
||||
logger = Logger(logger_config)
|
||||
service_manager = ServiceManager('geocoder', GeocodioGeocoderConfigBuilder, username, orgname, GD)
|
||||
geocoder = GeocodioBulkGeocoder(service_manager.config.geocodio_api_key, service_manager.logger, service_manager.config.service_params)
|
||||
return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches)
|
||||
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
|
||||
|
@ -45,6 +45,12 @@ SELECT cartodb.cdb_conf_setconf('tomtom_conf', '{"routing": {"api_keys": ["routi
|
||||
|
||||
(1 row)
|
||||
|
||||
SELECT cartodb.cdb_conf_setconf('geocodio_conf', '{"geocoder": {"api_keys": ["geocoder_dummy_api_key"], "monthly_quota": 1500000}}');
|
||||
cdb_conf_setconf
|
||||
------------------
|
||||
|
||||
(1 row)
|
||||
|
||||
SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/dev/null"}');
|
||||
cdb_conf_setconf
|
||||
------------------
|
||||
|
@ -17,6 +17,7 @@ SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"geocoder": {"app_id": "dummy
|
||||
SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing": {"api_key": "routing_dummy_api_key", "monthly_quota": 1500000}, "geocoder": {"api_key": "geocoder_dummy_api_key", "monthly_quota": 1500000}, "matrix": {"api_key": "matrix_dummy_api_key", "monthly_quota": 1500000}}');
|
||||
SELECT cartodb.cdb_conf_setconf('mapbox_conf', '{"routing": {"api_keys": ["routing_dummy_api_key"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["geocoder_dummy_api_key"], "monthly_quota": 1500000}, "matrix": {"api_keys": ["matrix_dummy_api_key"], "monthly_quota": 1500000}}');
|
||||
SELECT cartodb.cdb_conf_setconf('tomtom_conf', '{"routing": {"api_keys": ["routing_dummy_api_key"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["geocoder_dummy_api_key"], "monthly_quota": 1500000}, "isolines": {"api_keys": ["matrix_dummy_api_key"], "monthly_quota": 1500000}}');
|
||||
SELECT cartodb.cdb_conf_setconf('geocodio_conf', '{"geocoder": {"api_keys": ["geocoder_dummy_api_key"], "monthly_quota": 1500000}}');
|
||||
SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/dev/null"}');
|
||||
SELECT cartodb.cdb_conf_setconf('data_observatory_conf', '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=contrib_regression user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}, "monthly_quota": 100000}');
|
||||
|
||||
|
@ -2,10 +2,12 @@ from google import GoogleMapsBulkGeocoder
|
||||
from here import HereMapsBulkGeocoder
|
||||
from tomtom import TomTomBulkGeocoder
|
||||
from mapbox import MapboxBulkGeocoder
|
||||
from geocodio import GeocodioBulkGeocoder
|
||||
|
||||
BATCH_GEOCODER_CLASS_BY_PROVIDER = {
|
||||
'google': GoogleMapsBulkGeocoder,
|
||||
'heremaps': HereMapsBulkGeocoder,
|
||||
'tomtom': TomTomBulkGeocoder,
|
||||
'mapbox': MapboxBulkGeocoder
|
||||
'mapbox': MapboxBulkGeocoder,
|
||||
'geocodio': GeocodioBulkGeocoder,
|
||||
}
|
||||
|
@ -0,0 +1,2 @@
|
||||
from geocoder import GeocodioGeocoder
|
||||
from bulk_geocoder import GeocodioBulkGeocoder
|
@ -0,0 +1,76 @@
|
||||
import requests
|
||||
from cartodb_services import StreetPointBulkGeocoder
|
||||
from cartodb_services.geocodio import GeocodioGeocoder
|
||||
from iso3166 import countries
|
||||
from cartodb_services.tools.country import country_to_iso3
|
||||
|
||||
|
||||
class GeocodioBulkGeocoder(GeocodioGeocoder, StreetPointBulkGeocoder):
|
||||
MAX_BATCH_SIZE = 100 # Setting an upper limit (not stated in the documentation)
|
||||
MIN_BATCHED_SEARCH = 0
|
||||
READ_TIMEOUT = 60
|
||||
CONNECT_TIMEOUT = 10
|
||||
MAX_RETRIES = 1
|
||||
|
||||
def __init__(self, token, logger, service_params=None):
|
||||
GeocodioGeocoder.__init__(self, token, logger, service_params)
|
||||
|
||||
self.connect_timeout = self.CONNECT_TIMEOUT
|
||||
self.read_timeout = self.READ_TIMEOUT
|
||||
self.max_retries = self.MAX_RETRIES
|
||||
|
||||
if service_params is not None:
|
||||
self.connect_timeout = service_params.get('connect_timeout', self.CONNECT_TIMEOUT)
|
||||
self.read_timeout = service_params.get('read_timeout', self.READ_TIMEOUT)
|
||||
self.max_retries = service_params.get('max_retries', self.MAX_RETRIES)
|
||||
|
||||
self.session = requests.Session()
|
||||
|
||||
def _should_use_batch(self, searches):
|
||||
return len(searches) >= self.MIN_BATCHED_SEARCH
|
||||
|
||||
def _serial_geocode(self, searches):
|
||||
results = []
|
||||
for search in searches:
|
||||
elements = self._encoded_elements(search)
|
||||
result = self.geocode_meta(*elements)
|
||||
|
||||
if result:
|
||||
results.append((search[0], result[0], result[1]))
|
||||
else:
|
||||
results.append((search[0], None, None))
|
||||
|
||||
return results
|
||||
|
||||
def _encoded_elements(self, search):
|
||||
(search_id, address, city, state, country) = search
|
||||
address = address.encode('utf-8') if address else None
|
||||
city = city.encode('utf-8') if city else None
|
||||
state = state.encode('utf-8') if state else None
|
||||
country = self._country_code(country) if country else None
|
||||
return address, city, state, country
|
||||
|
||||
def _batch_geocode(self, searches):
|
||||
if len(searches) == 1:
|
||||
return self._serial_geocode(searches)
|
||||
else:
|
||||
frees = []
|
||||
for search in searches:
|
||||
elements = self._encoded_elements(search)
|
||||
free = ', '.join([elem for elem in elements if elem])
|
||||
frees.append(free)
|
||||
|
||||
full_results = self.geocode_free_text_meta(frees)
|
||||
results = []
|
||||
for s, r in zip(searches, full_results):
|
||||
results.append((s[0], r[0], r[1]))
|
||||
return results
|
||||
|
||||
def _country_code(self, country):
|
||||
country_iso3166 = country
|
||||
country_iso3 = country_to_iso3(country)
|
||||
if country_iso3:
|
||||
country_iso3166 = countries.get(country_iso3).alpha2.lower()
|
||||
|
||||
return country_iso3166
|
||||
|
@ -0,0 +1,126 @@
|
||||
from geocodio import GeocodioClient
|
||||
from geocodio.exceptions import GeocodioAuthError, GeocodioServerError, GeocodioDataError, GeocodioError
|
||||
|
||||
from cartodb_services.tools.qps import qps_retry
|
||||
from cartodb_services.metrics import Traceable
|
||||
from cartodb_services.geocoder import EMPTY_RESPONSE, geocoder_metadata
|
||||
from cartodb_services.tools.exceptions import ServiceException
|
||||
|
||||
|
||||
RELEVANCE_BY_LOCATION_TYPE = {
|
||||
'rooftop': 1,
|
||||
'point': 0.9,
|
||||
'range_interpolation': 0.8,
|
||||
'nearest_rooftop_match': 0.7,
|
||||
'intersection': 0.6,
|
||||
'street_center': 0.5,
|
||||
'place': 0.4,
|
||||
'state': 0.1,
|
||||
}
|
||||
|
||||
|
||||
class GeocodioGeocoder(Traceable):
|
||||
'''
|
||||
Python wrapper for the Geocodio Geocoder service.
|
||||
'''
|
||||
|
||||
def __init__(self, token, logger, service_params=None):
|
||||
service_params = service_params or {}
|
||||
self._token = token
|
||||
self._logger = logger
|
||||
|
||||
self._geocoder = GeocodioClient(self._token)
|
||||
|
||||
def _validate_input(self, searchtext, city=None, state_province=None,
|
||||
country=None):
|
||||
if searchtext and searchtext.strip():
|
||||
return True
|
||||
elif city:
|
||||
return True
|
||||
elif state_province:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
@qps_retry(qps=15, provider='geocodio')
|
||||
def geocode(self, searchtext, city=None, state_province=None,
|
||||
country=None):
|
||||
return self._geocode_meta(searchtext, city, state_province, country)[0]
|
||||
|
||||
def geocode_meta(self, searchtext, city=None, state_province=None,
|
||||
country=None):
|
||||
return self._geocode_meta(searchtext, city, state_province, country)[0]
|
||||
|
||||
@qps_retry(qps=15, provider='geocodio')
|
||||
def _geocode_meta(self, searchtext, city=None, state_province=None,
|
||||
country=None):
|
||||
if not self._validate_input(searchtext, city, state_province, country):
|
||||
return EMPTY_RESPONSE
|
||||
|
||||
try:
|
||||
free_text_components = [searchtext, city, state_province, country]
|
||||
req = '; '.join([c for c in free_text_components if c is not None and c.strip()])
|
||||
response = self._geocoder.geocode(req)
|
||||
|
||||
return self._parse_geocoder_response(response)
|
||||
except GeocodioDataError as gde:
|
||||
return EMPTY_RESPONSE
|
||||
except GeocodioAuthError as gae:
|
||||
raise ServiceException('Geocodio authorization error: ' + str(gae), None)
|
||||
except GeocodioServerError as gse:
|
||||
raise ServiceException('geocodio server error: ' + str(gse), None)
|
||||
except GeocodioError as ge:
|
||||
raise ServiceException('Unknown Geocodio error: ' + str(ge), None)
|
||||
|
||||
@qps_retry(qps=15)
|
||||
def geocode_free_text_meta(self, free_searches, country=None):
|
||||
"""
|
||||
:param free_searches: Free text searches
|
||||
:return: list of [x, y] on success, [] on error
|
||||
"""
|
||||
output = []
|
||||
|
||||
try:
|
||||
if country:
|
||||
free_searches = ['{s}, {country}'.format(s, country) for s in free_searches]
|
||||
|
||||
responses = self._geocoder.geocode(free_searches)
|
||||
|
||||
for response in responses:
|
||||
output.append(self._parse_geocoder_response(response))
|
||||
except GeocodioDataError as gde:
|
||||
return EMPTY_RESPONSE
|
||||
except GeocodioAuthError as gae:
|
||||
raise ServiceException('Geocodio authorization error: ' + str(gae), None)
|
||||
except GeocodioServerError as gse:
|
||||
raise ServiceException('geocodio server error: ' + str(gse), None)
|
||||
except GeocodioError as ge:
|
||||
raise ServiceException('Unknown Geocodio error: ' + str(ge), None)
|
||||
|
||||
return output
|
||||
|
||||
def _parse_geocoder_response(self, response):
|
||||
if response is None or not response:
|
||||
return EMPTY_RESPONSE
|
||||
|
||||
if response.get('results') is None or not response.get('results'):
|
||||
return EMPTY_RESPONSE
|
||||
|
||||
if response.coords is None or not response.coords:
|
||||
return EMPTY_RESPONSE
|
||||
|
||||
coords = [None, None]
|
||||
accuracy = None
|
||||
accuracy_type = None
|
||||
|
||||
accuracy = response.accuracy
|
||||
|
||||
if response.coords is not None and response.coords:
|
||||
coords = [response.coords[1], response.coords[0]]
|
||||
|
||||
if response.get('results'):
|
||||
accuracy_type = response.get('results')[0].get('accuracy_type')
|
||||
|
||||
metadata = geocoder_metadata(RELEVANCE_BY_LOCATION_TYPE.get(accuracy_type), response.accuracy, accuracy_type)
|
||||
|
||||
return [coords, metadata]
|
@ -0,0 +1 @@
|
||||
GEOCODIO_GEOCODER_APIKEY_ROUNDROBIN = 'geocodio_geocoder_apikey_roundrobin'
|
@ -405,6 +405,8 @@ class GeocoderConfig(ServiceConfig):
|
||||
MAPBOX_GEOCODER_API_KEYS = 'mapbox_geocoder_api_keys'
|
||||
TOMTOM_GEOCODER = 'tomtom'
|
||||
TOMTOM_GEOCODER_API_KEYS = 'tomtom_geocoder_api_keys'
|
||||
GEOCODIO_GEOCODER = 'geocodio'
|
||||
GEOCODIO_GEOCODER_API_KEYS = 'geocodio_geocoder_api_keys'
|
||||
QUOTA_KEY = 'geocoding_quota'
|
||||
SOFT_LIMIT_KEY = 'soft_geocoding_limit'
|
||||
USERNAME_KEY = 'username'
|
||||
@ -437,6 +439,9 @@ class GeocoderConfig(ServiceConfig):
|
||||
elif self._geocoder_provider == self.TOMTOM_GEOCODER:
|
||||
if not self.tomtom_api_keys:
|
||||
raise ConfigException("""TomTom config is not set up""")
|
||||
elif self._geocoder_provider == self.GEOCODIO_GEOCODER:
|
||||
if not self.geocodio_api_keys:
|
||||
raise ConfigException("""Geocodio config is not set up""")
|
||||
|
||||
return True
|
||||
|
||||
@ -476,6 +481,10 @@ class GeocoderConfig(ServiceConfig):
|
||||
self._tomtom_api_keys = db_config.tomtom_geocoder_api_keys
|
||||
self._cost_per_hit = 0
|
||||
self._tomtom_service_params = db_config.tomtom_geocoder_service_params
|
||||
elif self._geocoder_provider == self.GEOCODIO_GEOCODER:
|
||||
self._geocodio_api_keys = db_config.geocodio_geocoder_api_keys
|
||||
self._cost_per_hit = 0
|
||||
self._geocodio_service_params = db_config.geocodio_geocoder_service_params
|
||||
|
||||
@property
|
||||
def service_type(self):
|
||||
@ -489,6 +498,8 @@ class GeocoderConfig(ServiceConfig):
|
||||
return 'geocoder_tomtom'
|
||||
elif self._geocoder_provider == self.NOKIA_GEOCODER:
|
||||
return 'geocoder_here'
|
||||
elif self._geocoder_provider == self.GEOCODIO_GEOCODER:
|
||||
return 'geocoder_geocodio'
|
||||
|
||||
@property
|
||||
def heremaps_geocoder(self):
|
||||
@ -510,6 +521,10 @@ class GeocoderConfig(ServiceConfig):
|
||||
def tomtom_geocoder(self):
|
||||
return self._geocoder_provider == self.TOMTOM_GEOCODER
|
||||
|
||||
@property
|
||||
def geocodio_geocoder(self):
|
||||
return self._geocoder_provider == self.GEOCODIO_GEOCODER
|
||||
|
||||
@property
|
||||
def google_client_id(self):
|
||||
return self._google_maps_client_id
|
||||
@ -569,6 +584,14 @@ class GeocoderConfig(ServiceConfig):
|
||||
def tomtom_service_params(self):
|
||||
return self._tomtom_service_params
|
||||
|
||||
@property
|
||||
def geocodio_api_keys(self):
|
||||
return self._geocodio_api_keys
|
||||
|
||||
@property
|
||||
def geocodio_service_params(self):
|
||||
return self._geocodio_service_params
|
||||
|
||||
@property
|
||||
def is_high_resolution(self):
|
||||
return True
|
||||
@ -600,6 +623,7 @@ class ServicesDBConfig:
|
||||
self._get_mapzen_config()
|
||||
self._get_mapbox_config()
|
||||
self._get_tomtom_config()
|
||||
self._get_geocodio_config()
|
||||
self._get_data_observatory_config()
|
||||
|
||||
def _get_server_config(self):
|
||||
@ -679,6 +703,16 @@ class ServicesDBConfig:
|
||||
self._tomtom_geocoder_quota = tomtom_conf['geocoder']['monthly_quota']
|
||||
self._tomtom_geocoder_service_params = tomtom_conf['geocoder'].get('service', {})
|
||||
|
||||
def _get_geocodio_config(self):
|
||||
geocodio_conf_json = self._get_conf('geocodio_conf')
|
||||
if not geocodio_conf_json:
|
||||
raise ConfigException('Geocodio configuration missing')
|
||||
else:
|
||||
geocodio_conf = json.loads(geocodio_conf_json)
|
||||
self._geocodio_geocoder_api_keys = geocodio_conf['geocoder']['api_keys']
|
||||
self._geocodio_geocoder_quota = geocodio_conf['geocoder']['monthly_quota']
|
||||
self._geocodio_geocoder_service_params = geocodio_conf['geocoder'].get('service', {})
|
||||
|
||||
def _get_data_observatory_config(self):
|
||||
do_conf_json = self._get_conf('data_observatory_conf')
|
||||
if not do_conf_json:
|
||||
@ -848,6 +882,18 @@ class ServicesDBConfig:
|
||||
def tomtom_geocoder_service_params(self):
|
||||
return self._tomtom_geocoder_service_params
|
||||
|
||||
@property
|
||||
def geocodio_geocoder_api_keys(self):
|
||||
return self._geocodio_geocoder_api_keys
|
||||
|
||||
@property
|
||||
def geocodio_geocoder_monthly_quota(self):
|
||||
return self.geocodio_geocoder_quota
|
||||
|
||||
@property
|
||||
def geocodio_geocoder_service_params(self):
|
||||
return self._geocodio_geocoder_service_params
|
||||
|
||||
@property
|
||||
def data_observatory_connection_str(self):
|
||||
return self._data_observatory_connection_str
|
||||
|
@ -0,0 +1,128 @@
|
||||
from dateutil.parser import parse as date_parse
|
||||
from cartodb_services.refactor.service.utils import round_robin
|
||||
from cartodb_services.geocodio.types import GEOCODIO_GEOCODER_APIKEY_ROUNDROBIN
|
||||
|
||||
|
||||
class GeocodioGeocoderConfig(object):
|
||||
"""
|
||||
Configuration needed to operate the Geocodio geocoder service.
|
||||
"""
|
||||
|
||||
def __init__(self,
|
||||
geocoding_quota,
|
||||
soft_geocoding_limit,
|
||||
period_end_date,
|
||||
cost_per_hit,
|
||||
log_path,
|
||||
geocodio_api_keys,
|
||||
username,
|
||||
organization,
|
||||
service_params,
|
||||
GD):
|
||||
self._geocoding_quota = geocoding_quota
|
||||
self._soft_geocoding_limit = soft_geocoding_limit
|
||||
self._period_end_date = period_end_date
|
||||
self._cost_per_hit = cost_per_hit
|
||||
self._log_path = log_path
|
||||
self._geocodio_api_keys = geocodio_api_keys
|
||||
self._username = username
|
||||
self._organization = organization
|
||||
self._service_params = service_params
|
||||
self._GD = GD
|
||||
|
||||
@property
|
||||
def service_type(self):
|
||||
return 'geocoder_geocodio'
|
||||
|
||||
@property
|
||||
def provider(self):
|
||||
return 'geocodio'
|
||||
|
||||
@property
|
||||
def is_high_resolution(self):
|
||||
return True
|
||||
|
||||
@property
|
||||
def geocoding_quota(self):
|
||||
return self._geocoding_quota
|
||||
|
||||
@property
|
||||
def soft_geocoding_limit(self):
|
||||
return self._soft_geocoding_limit
|
||||
|
||||
@property
|
||||
def period_end_date(self):
|
||||
return self._period_end_date
|
||||
|
||||
@property
|
||||
def cost_per_hit(self):
|
||||
return self._cost_per_hit
|
||||
|
||||
@property
|
||||
def log_path(self):
|
||||
return self._log_path
|
||||
|
||||
@property
|
||||
def geocodio_api_key(self):
|
||||
return round_robin(self._geocodio_api_keys, self._GD,
|
||||
GEOCODIO_GEOCODER_APIKEY_ROUNDROBIN)
|
||||
|
||||
@property
|
||||
def username(self):
|
||||
return self._username
|
||||
|
||||
@property
|
||||
def organization(self):
|
||||
return self._organization
|
||||
|
||||
@property
|
||||
def service_params(self):
|
||||
return self._service_params
|
||||
|
||||
# TODO: for BW compat, remove
|
||||
@property
|
||||
def google_geocoder(self):
|
||||
return False
|
||||
|
||||
|
||||
class GeocodioGeocoderConfigBuilder(object):
|
||||
|
||||
def __init__(self, server_conf, user_conf, org_conf, username, orgname, GD):
|
||||
self._server_conf = server_conf
|
||||
self._user_conf = user_conf
|
||||
self._org_conf = org_conf
|
||||
self._username = username
|
||||
self._orgname = orgname
|
||||
self._GD = GD
|
||||
|
||||
def get(self):
|
||||
geocodio_server_conf = self._server_conf.get('geocodio_conf')
|
||||
geocodio_api_keys = geocodio_server_conf['geocoder']['api_keys']
|
||||
geocodio_service_params = geocodio_server_conf['geocoder'].get('service', {})
|
||||
|
||||
geocoding_quota = self._get_quota()
|
||||
soft_geocoding_limit = self._user_conf.get('soft_geocoding_limit').lower() == 'true'
|
||||
cost_per_hit = 0
|
||||
period_end_date_str = self._org_conf.get('period_end_date') or self._user_conf.get('period_end_date')
|
||||
period_end_date = date_parse(period_end_date_str)
|
||||
|
||||
logger_conf = self._server_conf.get('logger_conf')
|
||||
log_path = logger_conf.get('geocoder_log_path', None)
|
||||
|
||||
return GeocodioGeocoderConfig(geocoding_quota,
|
||||
soft_geocoding_limit,
|
||||
period_end_date,
|
||||
cost_per_hit,
|
||||
log_path,
|
||||
geocodio_api_keys,
|
||||
self._username,
|
||||
self._orgname,
|
||||
geocodio_service_params,
|
||||
self._GD)
|
||||
|
||||
def _get_quota(self):
|
||||
geocoding_quota = self._org_conf.get('geocoding_quota') or self._user_conf.get('geocoding_quota')
|
||||
if geocoding_quota is '':
|
||||
return 0
|
||||
|
||||
return int(geocoding_quota)
|
@ -7,6 +7,7 @@ rollbar==0.13.2
|
||||
requests==2.9.1
|
||||
rratelimit==0.0.4
|
||||
mapbox==0.14.0
|
||||
pygeocodio==0.11.1
|
||||
|
||||
# Test
|
||||
mock==1.3.0
|
||||
|
@ -10,7 +10,7 @@ from setuptools import setup, find_packages
|
||||
setup(
|
||||
name='cartodb_services',
|
||||
|
||||
version='0.21.4',
|
||||
version='0.22.0',
|
||||
|
||||
description='CartoDB Services API Python Library',
|
||||
|
||||
|
@ -9,3 +9,8 @@ def mapbox_api_key():
|
||||
def tomtom_api_key():
|
||||
"""Returns TomTom API key. Requires setting TOMTOM_API_KEY environment variable."""
|
||||
return os.environ['TOMTOM_API_KEY']
|
||||
|
||||
|
||||
def geocodio_api_key():
|
||||
"""Returns Geocodio API key. Requires setting GEOCODIO_API_KEY environment variable."""
|
||||
return os.environ['GEOCODIO_API_KEY']
|
||||
|
@ -7,7 +7,7 @@ from cartodb_services.metrics.config import *
|
||||
|
||||
class TestGeocoderUserConfig(TestCase):
|
||||
|
||||
GEOCODER_PROVIDERS = ['heremaps', 'mapzen', 'mapbox', 'tomtom', 'google']
|
||||
GEOCODER_PROVIDERS = ['heremaps', 'mapzen', 'mapbox', 'tomtom', 'geocodio', 'google']
|
||||
|
||||
def setUp(self):
|
||||
self.redis_conn = MockRedis()
|
||||
@ -31,6 +31,9 @@ class TestGeocoderUserConfig(TestCase):
|
||||
elif geocoder_provider == 'tomtom':
|
||||
assert geocoder_config.tomtom_geocoder is True
|
||||
assert geocoder_config.geocoding_quota == 100
|
||||
elif geocoder_provider == 'geocodio':
|
||||
assert geocoder_config.geocodio_geocoder is True
|
||||
assert geocoder_config.geocoding_quota == 100
|
||||
elif geocoder_provider == 'google':
|
||||
assert geocoder_config.google_geocoder is True
|
||||
assert geocoder_config.geocoding_quota is None
|
||||
@ -84,7 +87,7 @@ class TestGeocoderUserConfig(TestCase):
|
||||
|
||||
class TestGeocoderOrgConfig(TestCase):
|
||||
|
||||
GEOCODER_PROVIDERS = ['heremaps', 'mapzen', 'mapbox', 'tomtom', 'google']
|
||||
GEOCODER_PROVIDERS = ['heremaps', 'mapzen', 'mapbox', 'tomtom', 'geocodio', 'google']
|
||||
|
||||
def setUp(self):
|
||||
self.redis_conn = MockRedis()
|
||||
@ -113,6 +116,9 @@ class TestGeocoderOrgConfig(TestCase):
|
||||
elif geocoder_provider == 'tomtom':
|
||||
assert geocoder_config.tomtom_geocoder is True
|
||||
assert geocoder_config.geocoding_quota == 200
|
||||
elif geocoder_provider == 'geocodio':
|
||||
assert geocoder_config.geocodio_geocoder is True
|
||||
assert geocoder_config.geocoding_quota == 200
|
||||
elif geocoder_provider == 'google':
|
||||
assert geocoder_config.google_geocoder is True
|
||||
assert geocoder_config.geocoding_quota is None
|
||||
|
268
server/lib/python/cartodb_services/test/test_geocodiogeocoder.py
Normal file
268
server/lib/python/cartodb_services/test/test_geocodiogeocoder.py
Normal file
@ -0,0 +1,268 @@
|
||||
import unittest
|
||||
from mock import Mock
|
||||
from cartodb_services.geocodio import GeocodioGeocoder
|
||||
from cartodb_services.geocodio import GeocodioBulkGeocoder
|
||||
from cartodb_services.tools.exceptions import ServiceException
|
||||
from credentials import geocodio_api_key
|
||||
|
||||
INVALID_TOKEN = 'invalid_token'
|
||||
|
||||
VALID_ADDRESS_1 = 'Lexington Ave; New York; US'
|
||||
VALID_ADDRESS_2 = 'E 14th St; New York; US'
|
||||
VALID_ADDRESS_3 = '652 Lombard Street; San Francisco; California; United States'
|
||||
|
||||
VALID_SEARCH_TEXT_1='Lexington Ave'
|
||||
VALID_CITY_1='New York'
|
||||
VALID_STATE_PROVINCE_1='New York'
|
||||
VALID_COUNTRY_1='US'
|
||||
|
||||
VALID_SEARCH_TEXT_2='E 14th St'
|
||||
VALID_CITY_2='New York'
|
||||
VALID_STATE_PROVINCE_2='New York'
|
||||
VALID_COUNTRY_2='US'
|
||||
|
||||
VALID_SEARCH_TEXT_3='652 Lombard Street'
|
||||
VALID_CITY_3='San Francisco'
|
||||
VALID_STATE_PROVINCE_3='California'
|
||||
VALID_COUNTRY_3='United States'
|
||||
|
||||
WELL_KNOWN_LONGITUDE_1 = -73.960
|
||||
WELL_KNOWN_LATITUDE_1 = 40.774
|
||||
WELL_KNOWN_LONGITUDE_2 = -73.983
|
||||
WELL_KNOWN_LATITUDE_2 = 40.731
|
||||
WELL_KNOWN_LONGITUDE_3 = -122.412
|
||||
WELL_KNOWN_LATITUDE_3 = 37.803207
|
||||
|
||||
SEARCH_ID_1 = 1
|
||||
SEARCH_ID_2 = 2
|
||||
|
||||
|
||||
class GeocodioGeocoderTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.geocoder = GeocodioGeocoder(token=geocodio_api_key(), logger=Mock())
|
||||
self.bulk_geocoder = GeocodioBulkGeocoder(token=geocodio_api_key(), logger=Mock())
|
||||
|
||||
### NON BULK
|
||||
|
||||
def test_invalid_token(self):
|
||||
invalid_geocoder = GeocodioGeocoder(token=INVALID_TOKEN, logger=Mock())
|
||||
with self.assertRaises(ServiceException):
|
||||
invalid_geocoder.geocode(VALID_ADDRESS_1)
|
||||
|
||||
def test_valid_requests(self):
|
||||
place = self.geocoder.geocode(VALID_ADDRESS_1)
|
||||
|
||||
self.assertEqual('%.3f' % place[0], '%.3f' % WELL_KNOWN_LONGITUDE_1)
|
||||
self.assertEqual('%.3f' % place[1], '%.3f' % WELL_KNOWN_LATITUDE_1)
|
||||
|
||||
place = self.geocoder.geocode(VALID_ADDRESS_2)
|
||||
|
||||
self.assertEqual('%.3f' % place[0], '%.3f' % WELL_KNOWN_LONGITUDE_2)
|
||||
self.assertEqual('%.3f' % place[1], '%.3f' % WELL_KNOWN_LATITUDE_2)
|
||||
|
||||
place = self.geocoder.geocode(VALID_ADDRESS_3)
|
||||
|
||||
self.assertEqual('%.3f' % place[0], '%.3f' % WELL_KNOWN_LONGITUDE_3)
|
||||
self.assertEqual('%.3f' % place[1], '%.3f' % WELL_KNOWN_LATITUDE_3)
|
||||
|
||||
def test_valid_request_components(self):
|
||||
place = self.geocoder.geocode(searchtext=VALID_SEARCH_TEXT_1,
|
||||
city=VALID_CITY_1,
|
||||
state_province=VALID_STATE_PROVINCE_1,
|
||||
country=VALID_COUNTRY_1)
|
||||
|
||||
self.assertEqual('%.3f' % place[0], '%.3f' % WELL_KNOWN_LONGITUDE_1)
|
||||
self.assertEqual('%.3f' % place[1], '%.3f' % WELL_KNOWN_LATITUDE_1)
|
||||
|
||||
place = self.geocoder.geocode(searchtext=VALID_SEARCH_TEXT_2,
|
||||
city=VALID_CITY_2,
|
||||
state_province=VALID_STATE_PROVINCE_2,
|
||||
country=VALID_COUNTRY_2)
|
||||
|
||||
self.assertEqual('%.3f' % place[0], '%.3f' % WELL_KNOWN_LONGITUDE_2)
|
||||
self.assertEqual('%.3f' % place[1], '%.3f' % WELL_KNOWN_LATITUDE_2)
|
||||
|
||||
place = self.geocoder.geocode(searchtext=VALID_SEARCH_TEXT_3,
|
||||
city=VALID_CITY_3,
|
||||
state_province=VALID_STATE_PROVINCE_3,
|
||||
country=VALID_COUNTRY_3)
|
||||
|
||||
self.assertEqual('%.3f' % place[0], '%.3f' % WELL_KNOWN_LONGITUDE_3)
|
||||
self.assertEqual('%.3f' % place[1], '%.3f' % WELL_KNOWN_LATITUDE_3)
|
||||
|
||||
def test_valid_request_namedplace(self):
|
||||
place = self.geocoder.geocode(searchtext='New York')
|
||||
|
||||
assert place
|
||||
|
||||
def test_valid_request_namedplace2(self):
|
||||
place = self.geocoder.geocode(searchtext='New York', country='us')
|
||||
|
||||
assert place
|
||||
|
||||
def test_odd_characters(self):
|
||||
place = self.geocoder.geocode(searchtext='New York; "USA"')
|
||||
|
||||
assert place
|
||||
|
||||
def test_empty_request(self):
|
||||
place = self.geocoder.geocode(searchtext='', country=None, city=None, state_province=None)
|
||||
|
||||
assert place == []
|
||||
|
||||
def test_empty_search_text_request(self):
|
||||
place = self.geocoder.geocode(searchtext=' ', country='us', city=None, state_province="")
|
||||
|
||||
assert place == []
|
||||
|
||||
def test_unknown_place_request(self):
|
||||
place = self.geocoder.geocode(searchtext='[unknown]', country='ch', state_province=None, city=None)
|
||||
|
||||
assert place == []
|
||||
|
||||
### BULK ONE
|
||||
|
||||
def test_invalid_token_bulk_one(self):
|
||||
invalid_geocoder = GeocodioBulkGeocoder(token=INVALID_TOKEN, logger=Mock())
|
||||
with self.assertRaises(ServiceException):
|
||||
invalid_geocoder._batch_geocode([(SEARCH_ID_1, VALID_ADDRESS_1, None, None, None)])
|
||||
|
||||
def test_valid_request_bulk_one(self):
|
||||
place = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, VALID_ADDRESS_1, None, None, None)])
|
||||
|
||||
self.assertEqual(place[0][0], SEARCH_ID_1)
|
||||
self.assertEqual('%.3f' % place[0][1], '%.3f' % WELL_KNOWN_LONGITUDE_1)
|
||||
self.assertEqual('%.3f' % place[0][2], '%.3f' % WELL_KNOWN_LATITUDE_1)
|
||||
|
||||
place = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, VALID_ADDRESS_2, None, None, None)])
|
||||
|
||||
self.assertEqual(place[0][0], SEARCH_ID_1)
|
||||
self.assertEqual('%.3f' % place[0][1], '%.3f' % WELL_KNOWN_LONGITUDE_2)
|
||||
self.assertEqual('%.3f' % place[0][2], '%.3f' % WELL_KNOWN_LATITUDE_2)
|
||||
|
||||
place = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, VALID_ADDRESS_3, None, None, None)])
|
||||
|
||||
self.assertEqual(place[0][0], SEARCH_ID_1)
|
||||
self.assertEqual('%.3f' % place[0][1], '%.3f' % WELL_KNOWN_LONGITUDE_3)
|
||||
self.assertEqual('%.3f' % place[0][2], '%.3f' % WELL_KNOWN_LATITUDE_3)
|
||||
|
||||
def test_valid_request_components_bulk_one(self):
|
||||
place = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, VALID_SEARCH_TEXT_1, VALID_CITY_1, VALID_STATE_PROVINCE_1, VALID_COUNTRY_1)])
|
||||
|
||||
self.assertEqual(place[0][0], SEARCH_ID_1)
|
||||
self.assertEqual('%.3f' % place[0][1], '%.3f' % WELL_KNOWN_LONGITUDE_1)
|
||||
self.assertEqual('%.3f' % place[0][2], '%.3f' % WELL_KNOWN_LATITUDE_1)
|
||||
|
||||
place = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, VALID_SEARCH_TEXT_2, VALID_CITY_2, VALID_STATE_PROVINCE_2, VALID_COUNTRY_2)])
|
||||
|
||||
self.assertEqual(place[0][0], SEARCH_ID_1)
|
||||
self.assertEqual('%.3f' % place[0][1], '%.3f' % WELL_KNOWN_LONGITUDE_2)
|
||||
self.assertEqual('%.3f' % place[0][2], '%.3f' % WELL_KNOWN_LATITUDE_2)
|
||||
|
||||
place = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, VALID_SEARCH_TEXT_3, VALID_CITY_3, VALID_STATE_PROVINCE_3, VALID_COUNTRY_3)])
|
||||
|
||||
self.assertEqual(place[0][0], SEARCH_ID_1)
|
||||
self.assertEqual('%.3f' % place[0][1], '%.3f' % WELL_KNOWN_LONGITUDE_3)
|
||||
self.assertEqual('%.3f' % place[0][2], '%.3f' % WELL_KNOWN_LATITUDE_3)
|
||||
|
||||
def test_valid_request_namedplace_bulk_one(self):
|
||||
place = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, 'New York', None, None, None)])
|
||||
|
||||
assert place
|
||||
|
||||
def test_valid_request_namedplace2_bulk_one(self):
|
||||
place = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, 'New York', 'us', None, None)])
|
||||
|
||||
assert place
|
||||
|
||||
def test_odd_characters_bulk_one(self):
|
||||
place = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, 'New York; "USA"', None, None, None)])
|
||||
|
||||
assert place
|
||||
|
||||
def test_empty_request_bulk_one(self):
|
||||
place = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, '', None, None, None)])
|
||||
|
||||
assert place == [(SEARCH_ID_1, None, None)]
|
||||
|
||||
def test_empty_search_text_request_bulk_one(self):
|
||||
place = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, ' ', 'us', None, "")])
|
||||
|
||||
assert place == [(SEARCH_ID_1, None, None)]
|
||||
|
||||
def test_unknown_place_request_bulk_one(self):
|
||||
place = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, '[unknown]', 'ch', None, None)])
|
||||
|
||||
assert place == [(SEARCH_ID_1, None, None)]
|
||||
|
||||
### BULK MANY
|
||||
|
||||
def test_invalid_token_bulk_many(self):
|
||||
invalid_geocoder = GeocodioBulkGeocoder(token=INVALID_TOKEN, logger=Mock())
|
||||
with self.assertRaises(ServiceException):
|
||||
invalid_geocoder._batch_geocode([(SEARCH_ID_1, VALID_ADDRESS_1, None, None, None),
|
||||
(SEARCH_ID_2, VALID_ADDRESS_2, None, None, None)])
|
||||
|
||||
def test_valid_request_bulk_many(self):
|
||||
places = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, VALID_ADDRESS_1, None, None, None),
|
||||
(SEARCH_ID_2, VALID_ADDRESS_2, None, None, None)])
|
||||
|
||||
self.assertEqual(places[0][0], SEARCH_ID_1)
|
||||
self.assertEqual('%.3f' % places[0][1][0], '%.3f' % WELL_KNOWN_LONGITUDE_1)
|
||||
self.assertEqual('%.3f' % places[0][1][1], '%.3f' % WELL_KNOWN_LATITUDE_1)
|
||||
|
||||
self.assertEqual(places[1][0], SEARCH_ID_2)
|
||||
self.assertEqual('%.3f' % places[1][1][0], '%.3f' % WELL_KNOWN_LONGITUDE_2)
|
||||
self.assertEqual('%.3f' % places[1][1][1], '%.3f' % WELL_KNOWN_LATITUDE_2)
|
||||
|
||||
def test_valid_request_components_bulk_many(self):
|
||||
places = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, VALID_SEARCH_TEXT_1, VALID_CITY_1, VALID_STATE_PROVINCE_1, VALID_COUNTRY_1),
|
||||
(SEARCH_ID_2, VALID_SEARCH_TEXT_2, VALID_CITY_2, VALID_STATE_PROVINCE_2, VALID_COUNTRY_2)])
|
||||
|
||||
self.assertEqual(places[0][0], SEARCH_ID_1)
|
||||
self.assertEqual(places[1][0], SEARCH_ID_2)
|
||||
|
||||
def test_valid_request_namedplace_bulk_many(self):
|
||||
places = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, 'New York', None, None, None),
|
||||
(SEARCH_ID_2, 'Los Angeles', None, None, None)])
|
||||
|
||||
assert places
|
||||
|
||||
self.assertEqual(places[0][0], SEARCH_ID_1)
|
||||
self.assertEqual(places[1][0], SEARCH_ID_2)
|
||||
|
||||
def test_valid_request_namedplace2_bulk_many(self):
|
||||
places = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, 'New York', 'us', None, None),
|
||||
(SEARCH_ID_2, 'Los Angeles', None, None, None)])
|
||||
|
||||
assert places
|
||||
|
||||
self.assertEqual(places[0][0], SEARCH_ID_1)
|
||||
self.assertEqual(places[1][0], SEARCH_ID_2)
|
||||
|
||||
def test_odd_characters_bulk_many(self):
|
||||
places = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, 'New York; "USA"', None, None, None),
|
||||
(SEARCH_ID_2, 'Los Angeles', None, None, None)])
|
||||
|
||||
assert places
|
||||
|
||||
self.assertEqual(places[0][0], SEARCH_ID_1)
|
||||
self.assertEqual(places[1][0], SEARCH_ID_2)
|
||||
|
||||
def test_empty_request_bulk_many(self):
|
||||
places = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, '', None, None, None),
|
||||
(SEARCH_ID_2, '', None, None, None)])
|
||||
|
||||
assert places == [(SEARCH_ID_1, [], {}), (SEARCH_ID_2, [], {})]
|
||||
|
||||
def test_empty_search_text_request_bulk_many(self):
|
||||
places = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, ' ', 'us', None, ""),
|
||||
(SEARCH_ID_2, ' ', 'us', None, "")])
|
||||
|
||||
assert places == [(SEARCH_ID_1, [], {}), (SEARCH_ID_2, [], {})]
|
||||
|
||||
def test_unknown_place_request_bulk_many(self):
|
||||
places = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, '[unknown]', 'ch', None, None),
|
||||
(SEARCH_ID_2, '[unknown]', 'ch', None, None)])
|
||||
|
||||
assert places == [(SEARCH_ID_1, [], {}), (SEARCH_ID_2, [], {})]
|
@ -78,6 +78,7 @@ def plpy_mock_config():
|
||||
plpy_mock._define_result("CDB_Conf_GetConf\('mapzen_conf'\)", [{'conf': '{"routing": {"api_key": "api_key_rou", "monthly_quota": 1500000}, "geocoder": {"api_key": "api_key_geo", "monthly_quota": 1500000}, "matrix": {"api_key": "api_key_mat", "monthly_quota": 1500000}}'}])
|
||||
plpy_mock._define_result("CDB_Conf_GetConf\('mapbox_conf'\)", [{'conf': '{"routing": {"api_keys": ["api_key_rou"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["api_key_geo"], "monthly_quota": 1500000}, "matrix": {"api_keys": ["api_key_mat"], "monthly_quota": 1500000}}'}])
|
||||
plpy_mock._define_result("CDB_Conf_GetConf\('tomtom_conf'\)", [{'conf': '{"routing": {"api_keys": ["api_key_rou"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["api_key_geo"], "monthly_quota": 1500000}, "isolines": {"api_keys": ["api_key_mat"], "monthly_quota": 1500000}}'}])
|
||||
plpy_mock._define_result("CDB_Conf_GetConf\('geocodio_conf'\)", [{'conf': '{"geocoder": {"api_keys": ["api_key_geo"], "monthly_quota": 1500000}}'}])
|
||||
plpy_mock._define_result("CDB_Conf_GetConf\('logger_conf'\)", [{'conf': '{"geocoder_log_path": "/dev/null"}'}])
|
||||
plpy_mock._define_result("CDB_Conf_GetConf\('data_observatory_conf'\)", [{'conf': '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}}'}])
|
||||
plpy_mock._define_result("CDB_Conf_GetConf\('server_conf'\)", [{'conf': '{"environment": "testing"}'}])
|
||||
|
Loading…
Reference in New Issue
Block a user