cartodb4/dataservices-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge development

Browse Source
This commit is contained in:
antoniocarlon 2020-02-07 14:05:41 +01:00
commit a171d91204
33 changed files with 10867 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
NEWS.md
View File

@ -1,3 +1,12 @@
Jan 20th, 2020
==============
* Version `0.22.0` of the Python library
* Added geocodio geocoder
* Version `0.36.0` of the server extension
* Added geocodio geocoder
* Version `0.28.0` of the client extension
* Added geocodio geocoder
Oct 9th, 2019
==============
* Version `0.27.0` of the client extension

9
README.md
View File

@ -180,6 +180,15 @@ SELECT CDB_Conf_SetConf(
);
```
#### Geocod.io configuration
```sql
SELECT CDB_Conf_SetConf(
'geocodio_conf',
'{"geocoder": {"api_keys": ["your_api_key"], "monthly_quota": 999999}}'
);
```
#### Data Observatory
```sql

83
client/cdb_dataservices_client--0.27.0--0.28.0.sql Normal file
View File

@ -0,0 +1,83 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.28.0'" to load this file. \quit
-- Make sure we have a sane search path to create/update the extension
SET search_path = "$user",cartodb,public,cdb_dataservices_client;
-- HERE goes your code to upgrade/downgrade
CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocodio_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL)
RETURNS public.Geometry AS $$
DECLARE
ret public.Geometry;
username text;
orgname text;
_returned_sqlstate TEXT;
_message_text TEXT;
_pg_exception_context TEXT;
apikey_permissions json;
BEGIN
SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json);
IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN
RAISE EXCEPTION 'Geocoding permission denied';
END IF;
-- JSON value stored "" is taken as literal
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
END IF;
BEGIN
SELECT cdb_dataservices_client._cdb_geocodio_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret;
EXCEPTION
WHEN OTHERS THEN
GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE,
_message_text = MESSAGE_TEXT,
_pg_exception_context = PG_EXCEPTION_CONTEXT;
RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context;
RETURN ret;
END;
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE
SET search_path = pg_temp;
DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocodio_geocode_street_point (username text, orgname text, searchtext text, city text, state_province text, country text);
CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocodio_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL)
RETURNS Geometry AS $$
CONNECT cdb_dataservices_client._server_conn_str();
SELECT cdb_dataservices_server.cdb_geocodio_geocode_street_point (username, orgname, searchtext, city, state_province, country);
$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_geocodio_geocode_street_point (searchtext text ,city text ,state_province text ,country text);
CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocodio_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL)
RETURNS public.Geometry AS $$
DECLARE
ret public.Geometry;
username text;
orgname text;
apikey_permissions json;
BEGIN
SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json);
IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN
RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007';
END IF;
-- JSON value stored "" is taken as literal
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
END IF;
SELECT cdb_dataservices_client._cdb_geocodio_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret;
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE
SET search_path = pg_temp;
GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocodio_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser;
GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocodio_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text ) TO publicuser;

14
client/cdb_dataservices_client--0.28.0--0.27.0.sql Normal file
View File

@ -0,0 +1,14 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.27.0'" to load this file. \quit
-- Make sure we have a sane search path to create/update the extension
SET search_path = "$user",cartodb,public,cdb_dataservices_client;
-- HERE goes your code to upgrade/downgrade
DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocodio_geocode_street_point_exception_safe;
DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocodio_geocode_street_point;
DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_geocodio_geocode_street_point;

5823
client/cdb_dataservices_client--0.28.0.sql Normal file
View File

File diff suppressed because it is too large Load Diff

2
client/cdb_dataservices_client.control
View File

@ -1,5 +1,5 @@
comment = 'CartoDB dataservices client API extension'
default_version = '0.27.0'
default_version = '0.28.0'
requires = 'plproxy, cartodb'
superuser = true
schema = cdb_dataservices_client

0
client/cdb_dataservices_client--0.26.2--0.27.0.sql → client/old_versions/cdb_dataservices_client--0.26.2--0.27.0.sql
View File

0
client/cdb_dataservices_client--0.27.0--0.26.2.sql → client/old_versions/cdb_dataservices_client--0.27.0--0.26.2.sql
View File

0
client/cdb_dataservices_client--0.27.0.sql → client/old_versions/cdb_dataservices_client--0.27.0.sql
View File

11
client/renderer/interface.yaml
View File

@ -160,6 +160,17 @@
- { name: state_province, type: text, default: 'NULL'}
- { name: country, type: text, default: 'NULL'}
- name: cdb_geocodio_geocode_street_point
return_type: public.Geometry
requires_permission: true
permission_name: geocoding
permission_error: Geocoding permission denied
params:
- { name: searchtext, type: text}
- { name: city, type: text, default: 'NULL'}
- { name: state_province, type: text, default: 'NULL'}
- { name: country, type: text, default: 'NULL'}
- name: cdb_mapzen_geocode_street_point
return_type: public.Geometry
requires_permission: true

197
server/extension/cdb_dataservices_server--0.35.1--0.36.0.sql Normal file
View File

@ -0,0 +1,197 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.36.0'" to load this file. \quit
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
from cartodb_services.metrics import metrics
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
params = {'username': username, 'orgname': orgname, 'searchtext': searchtext, 'city': city, 'state_province': state_province, 'country': country}
with metrics('cdb_geocode_street_point', user_geocoder_config, logger, params):
if user_geocoder_config.heremaps_geocoder:
here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
elif user_geocoder_config.google_geocoder:
google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
elif user_geocoder_config.mapzen_geocoder:
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
elif user_geocoder_config.mapbox_geocoder:
mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
elif user_geocoder_config.tomtom_geocoder:
tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
elif user_geocoder_config.geocodio_geocoder:
geocodio_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocodio_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(geocodio_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
else:
raise Exception('Requested geocoder is not available')
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocodio_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
# The configuration is retrieved but no checks are performed on it
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
geocodio_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocodio_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(geocodio_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocodio_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
from iso3166 import countries
from cartodb_services.tools import ServiceManager, QuotaExceededException
from cartodb_services.geocodio import GeocodioGeocoder
from cartodb_services.tools.country import country_to_iso3
from cartodb_services.refactor.service.geocodio_geocoder_config import GeocodioGeocoderConfigBuilder
import cartodb_services
cartodb_services.init(plpy, GD)
service_manager = ServiceManager('geocoder', GeocodioGeocoderConfigBuilder, username, orgname, GD)
try:
service_manager.assert_within_limits()
geocoder = GeocodioGeocoder(service_manager.config.geocodio_api_key, service_manager.logger, service_manager.config.service_params)
country_iso3166 = None
if country:
country_iso3 = country_to_iso3(country)
if country_iso3:
country_iso3166 = countries.get(country_iso3).alpha2.lower()
coordinates = geocoder.geocode(searchtext=searchtext, city=city,
state_province=state_province,
country=country_iso3166)
if coordinates:
service_manager.quota_service.increment_success_service_use()
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
return point['st_setsrid']
else:
service_manager.quota_service.increment_empty_service_use()
return None
except QuotaExceededException as qe:
service_manager.quota_service.increment_failed_service_use()
return None
except BaseException as e:
import sys
service_manager.quota_service.increment_failed_service_use()
service_manager.logger.error('Error trying to geocode street point using Geocodio', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode street point using Geocodio')
finally:
service_manager.quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocode_street_point(username TEXT, orgname TEXT, searches jsonb)
RETURNS SETOF cdb_dataservices_server.geocoding AS $$
from cartodb_services.metrics import metrics
from cartodb_services.tools import Logger
import json
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
params = {'username': username, 'orgname': orgname, 'searches': json.loads(searches)}
with metrics('cdb_bulk_geocode_street_point', user_geocoder_config, logger, params):
if user_geocoder_config.google_geocoder:
provider_function = "_cdb_bulk_google_geocode_street_point";
elif user_geocoder_config.heremaps_geocoder:
provider_function = "_cdb_bulk_heremaps_geocode_street_point";
elif user_geocoder_config.tomtom_geocoder:
provider_function = "_cdb_bulk_tomtom_geocode_street_point";
elif user_geocoder_config.mapbox_geocoder:
provider_function = "_cdb_bulk_mapbox_geocode_street_point";
elif user_geocoder_config.geocodio_geocoder:
provider_function = "_cdb_bulk_geocodio_geocode_street_point";
else:
raise Exception('Requested geocoder is not available')
plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.{}($1, $2, $3); ".format(provider_function), ["text", "text", "jsonb"])
return plpy.execute(plan, [username, orgname, searches])
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocodio_geocode_street_point(username TEXT, orgname TEXT, searches jsonb)
RETURNS SETOF cdb_dataservices_server.geocoding AS $$
from cartodb_services import run_street_point_geocoder
from cartodb_services.tools import ServiceManager
from cartodb_services.refactor.service.geocodio_geocoder_config import GeocodioGeocoderConfigBuilder
from cartodb_services.geocodio import GeocodioBulkGeocoder
from cartodb_services.tools import Logger
import cartodb_services
cartodb_services.init(plpy, GD)
logger_config = GD["logger_config"]
logger = Logger(logger_config)
service_manager = ServiceManager('geocoder', GeocodioGeocoderConfigBuilder, username, orgname, GD)
geocoder = GeocodioBulkGeocoder(service_manager.config.geocodio_api_key, service_manager.logger, service_manager.config.service_params)
return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches)
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text)
RETURNS Geometry AS $$
from cartodb_services.metrics import QuotaService
from cartodb_services.metrics import InternalGeocoderConfig
from cartodb_services.metrics import metrics
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
quota_service = QuotaService(user_geocoder_config, redis_conn)
params = {'username': username, 'orgname': orgname, 'country_name': country_name}
with metrics('cdb_geocode_admin0_polygon', user_geocoder_config, logger, params):
try:
plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"])
rv = plpy.execute(plan, [country_name], 1)
result = rv[0]["mypolygon"]
if result:
quota_service.increment_success_service_use()
return result
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys
quota_service.increment_failed_service_use()
logger.error('Error trying to geocode admin0 polygon', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode admin0 polygon')
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;

79
server/extension/cdb_dataservices_server--0.36.0--0.35.1.sql Normal file
View File

@ -0,0 +1,79 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.35.1'" to load this file. \quit
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
from cartodb_services.metrics import metrics
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
params = {'username': username, 'orgname': orgname, 'searchtext': searchtext, 'city': city, 'state_province': state_province, 'country': country}
with metrics('cdb_geocode_street_point', user_geocoder_config, logger, params):
if user_geocoder_config.heremaps_geocoder:
here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
elif user_geocoder_config.google_geocoder:
google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
elif user_geocoder_config.mapzen_geocoder:
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
elif user_geocoder_config.mapbox_geocoder:
mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
elif user_geocoder_config.tomtom_geocoder:
tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
else:
raise Exception('Requested geocoder is not available')
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
DROP FUNCTION IF EXISTS cdb_dataservices_server.cdb_geocodio_geocode_street_point;
DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_geocodio_geocode_street_point;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocode_street_point(username TEXT, orgname TEXT, searches jsonb)
RETURNS SETOF cdb_dataservices_server.geocoding AS $$
from cartodb_services.metrics import metrics
from cartodb_services.tools import Logger
import json
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
logger_config = GD["logger_config"]
logger = Logger(logger_config)
params = {'username': username, 'orgname': orgname, 'searches': json.loads(searches)}
with metrics('cdb_bulk_geocode_street_point', user_geocoder_config, logger, params):
if user_geocoder_config.google_geocoder:
provider_function = "_cdb_bulk_google_geocode_street_point";
elif user_geocoder_config.heremaps_geocoder:
provider_function = "_cdb_bulk_heremaps_geocode_street_point";
elif user_geocoder_config.tomtom_geocoder:
provider_function = "_cdb_bulk_tomtom_geocode_street_point";
elif user_geocoder_config.mapbox_geocoder:
provider_function = "_cdb_bulk_mapbox_geocode_street_point";
else:
raise Exception('Requested geocoder is not available')
plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.{}($1, $2, $3); ".format(provider_function), ["text", "text", "jsonb"])
return plpy.execute(plan, [username, orgname, searches])
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_bulk_geocodio_geocode_street_point;

3888
server/extension/cdb_dataservices_server--0.36.0.sql Normal file
View File

File diff suppressed because it is too large Load Diff

2
server/extension/cdb_dataservices_server.control
View File

@ -1,5 +1,5 @@
comment = 'CartoDB dataservices server extension'
default_version = '0.35.1'
default_version = '0.36.0'
requires = 'plpythonu, plproxy, postgis, cdb_geocoder'
superuser = true
schema = cdb_dataservices_server

0
server/extension/cdb_dataservices_server--0.35.0--0.35.1.sql → server/extension/old_versions/cdb_dataservices_server--0.35.0--0.35.1.sql
View File

0
server/extension/cdb_dataservices_server--0.35.1--0.35.0.sql → server/extension/old_versions/cdb_dataservices_server--0.35.1--0.35.0.sql
View File

0
server/extension/cdb_dataservices_server--0.35.1.sql → server/extension/old_versions/cdb_dataservices_server--0.35.1.sql
View File

62
server/extension/sql/20_geocode_street.sql
View File

@ -30,6 +30,9 @@ RETURNS Geometry AS $$
elif user_geocoder_config.tomtom_geocoder:
tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
elif user_geocoder_config.geocodio_geocoder:
geocodio_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocodio_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(geocodio_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
else:
raise Exception('Requested geocoder is not available')
@ -105,6 +108,19 @@ RETURNS Geometry AS $$
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocodio_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
# The configuration is retrieved but no checks are performed on it
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
geocodio_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocodio_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(geocodio_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
from cartodb_services.tools import LegacyServiceManager
@ -302,3 +318,49 @@ RETURNS Geometry AS $$
finally:
service_manager.quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocodio_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
from iso3166 import countries
from cartodb_services.tools import ServiceManager, QuotaExceededException
from cartodb_services.geocodio import GeocodioGeocoder
from cartodb_services.tools.country import country_to_iso3
from cartodb_services.refactor.service.geocodio_geocoder_config import GeocodioGeocoderConfigBuilder
import cartodb_services
cartodb_services.init(plpy, GD)
service_manager = ServiceManager('geocoder', GeocodioGeocoderConfigBuilder, username, orgname, GD)
try:
service_manager.assert_within_limits()
geocoder = GeocodioGeocoder(service_manager.config.geocodio_api_key, service_manager.logger, service_manager.config.service_params)
country_iso3166 = None
if country:
country_iso3 = country_to_iso3(country)
if country_iso3:
country_iso3166 = countries.get(country_iso3).alpha2.lower()
coordinates = geocoder.geocode(searchtext=searchtext, city=city,
state_province=state_province,
country=country_iso3166)
if coordinates:
service_manager.quota_service.increment_success_service_use()
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
return point['st_setsrid']
else:
service_manager.quota_service.increment_empty_service_use()
return None
except QuotaExceededException as qe:
service_manager.quota_service.increment_failed_service_use()
return None
except BaseException as e:
import sys
service_manager.quota_service.increment_failed_service_use()
service_manager.logger.error('Error trying to geocode street point using Geocodio', sys.exc_info(), data={"username": username, "orgname": orgname})
raise Exception('Error trying to geocode street point using Geocodio')
finally:
service_manager.quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;

18
server/extension/sql/21_bulk_geocode_street.sql
View File

@ -32,6 +32,8 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$
provider_function = "_cdb_bulk_tomtom_geocode_street_point";
elif user_geocoder_config.mapbox_geocoder:
provider_function = "_cdb_bulk_mapbox_geocode_street_point";
elif user_geocoder_config.geocodio_geocoder:
provider_function = "_cdb_bulk_geocodio_geocode_street_point";
else:
raise Exception('Requested geocoder is not available')
@ -96,3 +98,19 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$
return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches)
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocodio_geocode_street_point(username TEXT, orgname TEXT, searches jsonb)
RETURNS SETOF cdb_dataservices_server.geocoding AS $$
from cartodb_services import run_street_point_geocoder
from cartodb_services.tools import ServiceManager
from cartodb_services.refactor.service.geocodio_geocoder_config import GeocodioGeocoderConfigBuilder
from cartodb_services.geocodio import GeocodioBulkGeocoder
from cartodb_services.tools import Logger
import cartodb_services
cartodb_services.init(plpy, GD)
logger_config = GD["logger_config"]
logger = Logger(logger_config)
service_manager = ServiceManager('geocoder', GeocodioGeocoderConfigBuilder, username, orgname, GD)
geocoder = GeocodioBulkGeocoder(service_manager.config.geocodio_api_key, service_manager.logger, service_manager.config.service_params)
return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches)
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;

6
server/extension/test/expected/00_install_test.out
View File

@ -45,6 +45,12 @@ SELECT cartodb.cdb_conf_setconf('tomtom_conf', '{"routing": {"api_keys": ["routi
(1 row)
SELECT cartodb.cdb_conf_setconf('geocodio_conf', '{"geocoder": {"api_keys": ["geocoder_dummy_api_key"], "monthly_quota": 1500000}}');
cdb_conf_setconf
------------------
(1 row)
SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/dev/null"}');
cdb_conf_setconf
------------------

1
server/extension/test/sql/00_install_test.sql
View File

@ -17,6 +17,7 @@ SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"geocoder": {"app_id": "dummy
SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing": {"api_key": "routing_dummy_api_key", "monthly_quota": 1500000}, "geocoder": {"api_key": "geocoder_dummy_api_key", "monthly_quota": 1500000}, "matrix": {"api_key": "matrix_dummy_api_key", "monthly_quota": 1500000}}');
SELECT cartodb.cdb_conf_setconf('mapbox_conf', '{"routing": {"api_keys": ["routing_dummy_api_key"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["geocoder_dummy_api_key"], "monthly_quota": 1500000}, "matrix": {"api_keys": ["matrix_dummy_api_key"], "monthly_quota": 1500000}}');
SELECT cartodb.cdb_conf_setconf('tomtom_conf', '{"routing": {"api_keys": ["routing_dummy_api_key"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["geocoder_dummy_api_key"], "monthly_quota": 1500000}, "isolines": {"api_keys": ["matrix_dummy_api_key"], "monthly_quota": 1500000}}');
SELECT cartodb.cdb_conf_setconf('geocodio_conf', '{"geocoder": {"api_keys": ["geocoder_dummy_api_key"], "monthly_quota": 1500000}}');
SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/dev/null"}');
SELECT cartodb.cdb_conf_setconf('data_observatory_conf', '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=contrib_regression user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}, "monthly_quota": 100000}');

4
server/lib/python/cartodb_services/cartodb_services/bulk_geocoders.py
View File

@ -2,10 +2,12 @@ from google import GoogleMapsBulkGeocoder
from here import HereMapsBulkGeocoder
from tomtom import TomTomBulkGeocoder
from mapbox import MapboxBulkGeocoder
from geocodio import GeocodioBulkGeocoder
BATCH_GEOCODER_CLASS_BY_PROVIDER = {
'google': GoogleMapsBulkGeocoder,
'heremaps': HereMapsBulkGeocoder,
'tomtom': TomTomBulkGeocoder,
'mapbox': MapboxBulkGeocoder
'mapbox': MapboxBulkGeocoder,
'geocodio': GeocodioBulkGeocoder,
}

2
server/lib/python/cartodb_services/cartodb_services/geocodio/__init__.py Normal file
View File

@ -0,0 +1,2 @@
from geocoder import GeocodioGeocoder
from bulk_geocoder import GeocodioBulkGeocoder

76
server/lib/python/cartodb_services/cartodb_services/geocodio/bulk_geocoder.py Normal file
View File

@ -0,0 +1,76 @@
import requests
from cartodb_services import StreetPointBulkGeocoder
from cartodb_services.geocodio import GeocodioGeocoder
from iso3166 import countries
from cartodb_services.tools.country import country_to_iso3
class GeocodioBulkGeocoder(GeocodioGeocoder, StreetPointBulkGeocoder):
MAX_BATCH_SIZE = 100 # Setting an upper limit (not stated in the documentation)
MIN_BATCHED_SEARCH = 0
READ_TIMEOUT = 60
CONNECT_TIMEOUT = 10
MAX_RETRIES = 1
def __init__(self, token, logger, service_params=None):
GeocodioGeocoder.__init__(self, token, logger, service_params)
self.connect_timeout = self.CONNECT_TIMEOUT
self.read_timeout = self.READ_TIMEOUT
self.max_retries = self.MAX_RETRIES
if service_params is not None:
self.connect_timeout = service_params.get('connect_timeout', self.CONNECT_TIMEOUT)
self.read_timeout = service_params.get('read_timeout', self.READ_TIMEOUT)
self.max_retries = service_params.get('max_retries', self.MAX_RETRIES)
self.session = requests.Session()
def _should_use_batch(self, searches):
return len(searches) >= self.MIN_BATCHED_SEARCH
def _serial_geocode(self, searches):
results = []
for search in searches:
elements = self._encoded_elements(search)
result = self.geocode_meta(*elements)
if result:
results.append((search[0], result[0], result[1]))
else:
results.append((search[0], None, None))
return results
def _encoded_elements(self, search):
(search_id, address, city, state, country) = search
address = address.encode('utf-8') if address else None
city = city.encode('utf-8') if city else None
state = state.encode('utf-8') if state else None
country = self._country_code(country) if country else None
return address, city, state, country
def _batch_geocode(self, searches):
if len(searches) == 1:
return self._serial_geocode(searches)
else:
frees = []
for search in searches:
elements = self._encoded_elements(search)
free = ', '.join([elem for elem in elements if elem])
frees.append(free)
full_results = self.geocode_free_text_meta(frees)
results = []
for s, r in zip(searches, full_results):
results.append((s[0], r[0], r[1]))
return results
def _country_code(self, country):
country_iso3166 = country
country_iso3 = country_to_iso3(country)
if country_iso3:
country_iso3166 = countries.get(country_iso3).alpha2.lower()
return country_iso3166

126
server/lib/python/cartodb_services/cartodb_services/geocodio/geocoder.py Normal file
View File

@ -0,0 +1,126 @@
from geocodio import GeocodioClient
from geocodio.exceptions import GeocodioAuthError, GeocodioServerError, GeocodioDataError, GeocodioError
from cartodb_services.tools.qps import qps_retry
from cartodb_services.metrics import Traceable
from cartodb_services.geocoder import EMPTY_RESPONSE, geocoder_metadata
from cartodb_services.tools.exceptions import ServiceException
RELEVANCE_BY_LOCATION_TYPE = {
'rooftop': 1,
'point': 0.9,
'range_interpolation': 0.8,
'nearest_rooftop_match': 0.7,
'intersection': 0.6,
'street_center': 0.5,
'place': 0.4,
'state': 0.1,
}
class GeocodioGeocoder(Traceable):
'''
Python wrapper for the Geocodio Geocoder service.
'''
def __init__(self, token, logger, service_params=None):
service_params = service_params or {}
self._token = token
self._logger = logger
self._geocoder = GeocodioClient(self._token)
def _validate_input(self, searchtext, city=None, state_province=None,
country=None):
if searchtext and searchtext.strip():
return True
elif city:
return True
elif state_province:
return True
return False
@qps_retry(qps=15, provider='geocodio')
def geocode(self, searchtext, city=None, state_province=None,
country=None):
return self._geocode_meta(searchtext, city, state_province, country)[0]
def geocode_meta(self, searchtext, city=None, state_province=None,
country=None):
return self._geocode_meta(searchtext, city, state_province, country)[0]
@qps_retry(qps=15, provider='geocodio')
def _geocode_meta(self, searchtext, city=None, state_province=None,
country=None):
if not self._validate_input(searchtext, city, state_province, country):
return EMPTY_RESPONSE
try:
free_text_components = [searchtext, city, state_province, country]
req = '; '.join([c for c in free_text_components if c is not None and c.strip()])
response = self._geocoder.geocode(req)
return self._parse_geocoder_response(response)
except GeocodioDataError as gde:
return EMPTY_RESPONSE
except GeocodioAuthError as gae:
raise ServiceException('Geocodio authorization error: ' + str(gae), None)
except GeocodioServerError as gse:
raise ServiceException('geocodio server error: ' + str(gse), None)
except GeocodioError as ge:
raise ServiceException('Unknown Geocodio error: ' + str(ge), None)
@qps_retry(qps=15)
def geocode_free_text_meta(self, free_searches, country=None):
"""
:param free_searches: Free text searches
:return: list of [x, y] on success, [] on error
"""
output = []
try:
if country:
free_searches = ['{s}, {country}'.format(s, country) for s in free_searches]
responses = self._geocoder.geocode(free_searches)
for response in responses:
output.append(self._parse_geocoder_response(response))
except GeocodioDataError as gde:
return EMPTY_RESPONSE
except GeocodioAuthError as gae:
raise ServiceException('Geocodio authorization error: ' + str(gae), None)
except GeocodioServerError as gse:
raise ServiceException('geocodio server error: ' + str(gse), None)
except GeocodioError as ge:
raise ServiceException('Unknown Geocodio error: ' + str(ge), None)
return output
def _parse_geocoder_response(self, response):
if response is None or not response:
return EMPTY_RESPONSE
if response.get('results') is None or not response.get('results'):
return EMPTY_RESPONSE
if response.coords is None or not response.coords:
return EMPTY_RESPONSE
coords = [None, None]
accuracy = None
accuracy_type = None
accuracy = response.accuracy
if response.coords is not None and response.coords:
coords = [response.coords[1], response.coords[0]]
if response.get('results'):
accuracy_type = response.get('results')[0].get('accuracy_type')
metadata = geocoder_metadata(RELEVANCE_BY_LOCATION_TYPE.get(accuracy_type), response.accuracy, accuracy_type)
return [coords, metadata]

1
server/lib/python/cartodb_services/cartodb_services/geocodio/types.py Normal file
View File

@ -0,0 +1 @@
GEOCODIO_GEOCODER_APIKEY_ROUNDROBIN = 'geocodio_geocoder_apikey_roundrobin'

46
server/lib/python/cartodb_services/cartodb_services/metrics/config.py
View File

@ -423,6 +423,8 @@ class GeocoderConfig(ServiceConfig):
MAPBOX_GEOCODER_API_KEYS = 'mapbox_geocoder_api_keys'
TOMTOM_GEOCODER = 'tomtom'
TOMTOM_GEOCODER_API_KEYS = 'tomtom_geocoder_api_keys'
GEOCODIO_GEOCODER = 'geocodio'
GEOCODIO_GEOCODER_API_KEYS = 'geocodio_geocoder_api_keys'
QUOTA_KEY = 'geocoding_quota'
SOFT_LIMIT_KEY = 'soft_geocoding_limit'
USERNAME_KEY = 'username'
@ -455,6 +457,9 @@ class GeocoderConfig(ServiceConfig):
elif self._geocoder_provider == self.TOMTOM_GEOCODER:
if not self.tomtom_api_keys:
raise ConfigException("""TomTom config is not set up""")
elif self._geocoder_provider == self.GEOCODIO_GEOCODER:
if not self.geocodio_api_keys:
raise ConfigException("""Geocodio config is not set up""")
return True
@ -494,6 +499,10 @@ class GeocoderConfig(ServiceConfig):
self._tomtom_api_keys = db_config.tomtom_geocoder_api_keys
self._cost_per_hit = 0
self._tomtom_service_params = db_config.tomtom_geocoder_service_params
elif self._geocoder_provider == self.GEOCODIO_GEOCODER:
self._geocodio_api_keys = db_config.geocodio_geocoder_api_keys
self._cost_per_hit = 0
self._geocodio_service_params = db_config.geocodio_geocoder_service_params
@property
def service_type(self):
@ -507,6 +516,8 @@ class GeocoderConfig(ServiceConfig):
return 'geocoder_tomtom'
elif self._geocoder_provider == self.NOKIA_GEOCODER:
return 'geocoder_here'
elif self._geocoder_provider == self.GEOCODIO_GEOCODER:
return 'geocoder_geocodio'
@property
def heremaps_geocoder(self):
@ -528,6 +539,10 @@ class GeocoderConfig(ServiceConfig):
def tomtom_geocoder(self):
return self._geocoder_provider == self.TOMTOM_GEOCODER
@property
def geocodio_geocoder(self):
return self._geocoder_provider == self.GEOCODIO_GEOCODER
@property
def google_client_id(self):
return self._google_maps_client_id
@ -587,6 +602,14 @@ class GeocoderConfig(ServiceConfig):
def tomtom_service_params(self):
return self._tomtom_service_params
@property
def geocodio_api_keys(self):
return self._geocodio_api_keys
@property
def geocodio_service_params(self):
return self._geocodio_service_params
@property
def is_high_resolution(self):
return True
@ -619,6 +642,7 @@ class ServicesDBConfig:
self._get_mapbox_config()
self._get_mapbox_iso_config()
self._get_tomtom_config()
self._get_geocodio_config()
self._get_data_observatory_config()
def _get_server_config(self):
@ -708,6 +732,16 @@ class ServicesDBConfig:
self._tomtom_geocoder_quota = tomtom_conf['geocoder']['monthly_quota']
self._tomtom_geocoder_service_params = tomtom_conf['geocoder'].get('service', {})
def _get_geocodio_config(self):
geocodio_conf_json = self._get_conf('geocodio_conf')
if not geocodio_conf_json:
raise ConfigException('Geocodio configuration missing')
else:
geocodio_conf = json.loads(geocodio_conf_json)
self._geocodio_geocoder_api_keys = geocodio_conf['geocoder']['api_keys']
self._geocodio_geocoder_quota = geocodio_conf['geocoder']['monthly_quota']
self._geocodio_geocoder_service_params = geocodio_conf['geocoder'].get('service', {})
def _get_data_observatory_config(self):
do_conf_json = self._get_conf('data_observatory_conf')
if not do_conf_json:
@ -889,6 +923,18 @@ class ServicesDBConfig:
def tomtom_geocoder_service_params(self):
return self._tomtom_geocoder_service_params
@property
def geocodio_geocoder_api_keys(self):
return self._geocodio_geocoder_api_keys
@property
def geocodio_geocoder_monthly_quota(self):
return self.geocodio_geocoder_quota
@property
def geocodio_geocoder_service_params(self):
return self._geocodio_geocoder_service_params
@property
def data_observatory_connection_str(self):
return self._data_observatory_connection_str

128
server/lib/python/cartodb_services/cartodb_services/refactor/service/geocodio_geocoder_config.py Normal file
View File

@ -0,0 +1,128 @@
from dateutil.parser import parse as date_parse
from cartodb_services.refactor.service.utils import round_robin
from cartodb_services.geocodio.types import GEOCODIO_GEOCODER_APIKEY_ROUNDROBIN
class GeocodioGeocoderConfig(object):
"""
Configuration needed to operate the Geocodio geocoder service.
"""
def __init__(self,
geocoding_quota,
soft_geocoding_limit,
period_end_date,
cost_per_hit,
log_path,
geocodio_api_keys,
username,
organization,
service_params,
GD):
self._geocoding_quota = geocoding_quota
self._soft_geocoding_limit = soft_geocoding_limit
self._period_end_date = period_end_date
self._cost_per_hit = cost_per_hit
self._log_path = log_path
self._geocodio_api_keys = geocodio_api_keys
self._username = username
self._organization = organization
self._service_params = service_params
self._GD = GD
@property
def service_type(self):
return 'geocoder_geocodio'
@property
def provider(self):
return 'geocodio'
@property
def is_high_resolution(self):
return True
@property
def geocoding_quota(self):
return self._geocoding_quota
@property
def soft_geocoding_limit(self):
return self._soft_geocoding_limit
@property
def period_end_date(self):
return self._period_end_date
@property
def cost_per_hit(self):
return self._cost_per_hit
@property
def log_path(self):
return self._log_path
@property
def geocodio_api_key(self):
return round_robin(self._geocodio_api_keys, self._GD,
GEOCODIO_GEOCODER_APIKEY_ROUNDROBIN)
@property
def username(self):
return self._username
@property
def organization(self):
return self._organization
@property
def service_params(self):
return self._service_params
# TODO: for BW compat, remove
@property
def google_geocoder(self):
return False
class GeocodioGeocoderConfigBuilder(object):
def __init__(self, server_conf, user_conf, org_conf, username, orgname, GD):
self._server_conf = server_conf
self._user_conf = user_conf
self._org_conf = org_conf
self._username = username
self._orgname = orgname
self._GD = GD
def get(self):
geocodio_server_conf = self._server_conf.get('geocodio_conf')
geocodio_api_keys = geocodio_server_conf['geocoder']['api_keys']
geocodio_service_params = geocodio_server_conf['geocoder'].get('service', {})
geocoding_quota = self._get_quota()
soft_geocoding_limit = self._user_conf.get('soft_geocoding_limit').lower() == 'true'
cost_per_hit = 0
period_end_date_str = self._org_conf.get('period_end_date') or self._user_conf.get('period_end_date')
period_end_date = date_parse(period_end_date_str)
logger_conf = self._server_conf.get('logger_conf')
log_path = logger_conf.get('geocoder_log_path', None)
return GeocodioGeocoderConfig(geocoding_quota,
soft_geocoding_limit,
period_end_date,
cost_per_hit,
log_path,
geocodio_api_keys,
self._username,
self._orgname,
geocodio_service_params,
self._GD)
def _get_quota(self):
geocoding_quota = self._org_conf.get('geocoding_quota') or self._user_conf.get('geocoding_quota')
if geocoding_quota is '':
return 0
return int(geocoding_quota)

1
server/lib/python/cartodb_services/requirements.txt
View File

@ -7,6 +7,7 @@ rollbar==0.13.2
requests==2.9.1
rratelimit==0.0.4
mapbox==0.14.0
pygeocodio==0.11.1
# Test
mock==1.3.0

5
server/lib/python/cartodb_services/test/credentials.py
View File

@ -9,3 +9,8 @@ def mapbox_api_key():
def tomtom_api_key():
"""Returns TomTom API key. Requires setting TOMTOM_API_KEY environment variable."""
return os.environ['TOMTOM_API_KEY']
def geocodio_api_key():
"""Returns Geocodio API key. Requires setting GEOCODIO_API_KEY environment variable."""
return os.environ['GEOCODIO_API_KEY']

10
server/lib/python/cartodb_services/test/metrics/test_config.py
View File

@ -7,7 +7,7 @@ from cartodb_services.metrics.config import *
class TestGeocoderUserConfig(TestCase):
GEOCODER_PROVIDERS = ['heremaps', 'mapzen', 'mapbox', 'tomtom', 'google']
GEOCODER_PROVIDERS = ['heremaps', 'mapzen', 'mapbox', 'tomtom', 'geocodio', 'google']
def setUp(self):
self.redis_conn = MockRedis()
@ -31,6 +31,9 @@ class TestGeocoderUserConfig(TestCase):
elif geocoder_provider == 'tomtom':
assert geocoder_config.tomtom_geocoder is True
assert geocoder_config.geocoding_quota == 100
elif geocoder_provider == 'geocodio':
assert geocoder_config.geocodio_geocoder is True
assert geocoder_config.geocoding_quota == 100
elif geocoder_provider == 'google':
assert geocoder_config.google_geocoder is True
assert geocoder_config.geocoding_quota is None
@ -84,7 +87,7 @@ class TestGeocoderUserConfig(TestCase):
class TestGeocoderOrgConfig(TestCase):
GEOCODER_PROVIDERS = ['heremaps', 'mapzen', 'mapbox', 'tomtom', 'google']
GEOCODER_PROVIDERS = ['heremaps', 'mapzen', 'mapbox', 'tomtom', 'geocodio', 'google']
def setUp(self):
self.redis_conn = MockRedis()
@ -113,6 +116,9 @@ class TestGeocoderOrgConfig(TestCase):
elif geocoder_provider == 'tomtom':
assert geocoder_config.tomtom_geocoder is True
assert geocoder_config.geocoding_quota == 200
elif geocoder_provider == 'geocodio':
assert geocoder_config.geocodio_geocoder is True
assert geocoder_config.geocoding_quota == 200
elif geocoder_provider == 'google':
assert geocoder_config.google_geocoder is True
assert geocoder_config.geocoding_quota is None

268
server/lib/python/cartodb_services/test/test_geocodiogeocoder.py Normal file
View File

@ -0,0 +1,268 @@
import unittest
from mock import Mock
from cartodb_services.geocodio import GeocodioGeocoder
from cartodb_services.geocodio import GeocodioBulkGeocoder
from cartodb_services.tools.exceptions import ServiceException
from credentials import geocodio_api_key
INVALID_TOKEN = 'invalid_token'
VALID_ADDRESS_1 = 'Lexington Ave; New York; US'
VALID_ADDRESS_2 = 'E 14th St; New York; US'
VALID_ADDRESS_3 = '652 Lombard Street; San Francisco; California; United States'
VALID_SEARCH_TEXT_1='Lexington Ave'
VALID_CITY_1='New York'
VALID_STATE_PROVINCE_1='New York'
VALID_COUNTRY_1='US'
VALID_SEARCH_TEXT_2='E 14th St'
VALID_CITY_2='New York'
VALID_STATE_PROVINCE_2='New York'
VALID_COUNTRY_2='US'
VALID_SEARCH_TEXT_3='652 Lombard Street'
VALID_CITY_3='San Francisco'
VALID_STATE_PROVINCE_3='California'
VALID_COUNTRY_3='United States'
WELL_KNOWN_LONGITUDE_1 = -73.960
WELL_KNOWN_LATITUDE_1 = 40.774
WELL_KNOWN_LONGITUDE_2 = -73.983
WELL_KNOWN_LATITUDE_2 = 40.731
WELL_KNOWN_LONGITUDE_3 = -122.412
WELL_KNOWN_LATITUDE_3 = 37.803207
SEARCH_ID_1 = 1
SEARCH_ID_2 = 2
class GeocodioGeocoderTestCase(unittest.TestCase):
def setUp(self):
self.geocoder = GeocodioGeocoder(token=geocodio_api_key(), logger=Mock())
self.bulk_geocoder = GeocodioBulkGeocoder(token=geocodio_api_key(), logger=Mock())
### NON BULK
def test_invalid_token(self):
invalid_geocoder = GeocodioGeocoder(token=INVALID_TOKEN, logger=Mock())
with self.assertRaises(ServiceException):
invalid_geocoder.geocode(VALID_ADDRESS_1)
def test_valid_requests(self):
place = self.geocoder.geocode(VALID_ADDRESS_1)
self.assertEqual('%.3f' % place[0], '%.3f' % WELL_KNOWN_LONGITUDE_1)
self.assertEqual('%.3f' % place[1], '%.3f' % WELL_KNOWN_LATITUDE_1)
place = self.geocoder.geocode(VALID_ADDRESS_2)
self.assertEqual('%.3f' % place[0], '%.3f' % WELL_KNOWN_LONGITUDE_2)
self.assertEqual('%.3f' % place[1], '%.3f' % WELL_KNOWN_LATITUDE_2)
place = self.geocoder.geocode(VALID_ADDRESS_3)
self.assertEqual('%.3f' % place[0], '%.3f' % WELL_KNOWN_LONGITUDE_3)
self.assertEqual('%.3f' % place[1], '%.3f' % WELL_KNOWN_LATITUDE_3)
def test_valid_request_components(self):
place = self.geocoder.geocode(searchtext=VALID_SEARCH_TEXT_1,
city=VALID_CITY_1,
state_province=VALID_STATE_PROVINCE_1,
country=VALID_COUNTRY_1)
self.assertEqual('%.3f' % place[0], '%.3f' % WELL_KNOWN_LONGITUDE_1)
self.assertEqual('%.3f' % place[1], '%.3f' % WELL_KNOWN_LATITUDE_1)
place = self.geocoder.geocode(searchtext=VALID_SEARCH_TEXT_2,
city=VALID_CITY_2,
state_province=VALID_STATE_PROVINCE_2,
country=VALID_COUNTRY_2)
self.assertEqual('%.3f' % place[0], '%.3f' % WELL_KNOWN_LONGITUDE_2)
self.assertEqual('%.3f' % place[1], '%.3f' % WELL_KNOWN_LATITUDE_2)
place = self.geocoder.geocode(searchtext=VALID_SEARCH_TEXT_3,
city=VALID_CITY_3,
state_province=VALID_STATE_PROVINCE_3,
country=VALID_COUNTRY_3)
self.assertEqual('%.3f' % place[0], '%.3f' % WELL_KNOWN_LONGITUDE_3)
self.assertEqual('%.3f' % place[1], '%.3f' % WELL_KNOWN_LATITUDE_3)
def test_valid_request_namedplace(self):
place = self.geocoder.geocode(searchtext='New York')
assert place
def test_valid_request_namedplace2(self):
place = self.geocoder.geocode(searchtext='New York', country='us')
assert place
def test_odd_characters(self):
place = self.geocoder.geocode(searchtext='New York; "USA"')
assert place
def test_empty_request(self):
place = self.geocoder.geocode(searchtext='', country=None, city=None, state_province=None)
assert place == []
def test_empty_search_text_request(self):
place = self.geocoder.geocode(searchtext=' ', country='us', city=None, state_province="")
assert place == []
def test_unknown_place_request(self):
place = self.geocoder.geocode(searchtext='[unknown]', country='ch', state_province=None, city=None)
assert place == []
### BULK ONE
def test_invalid_token_bulk_one(self):
invalid_geocoder = GeocodioBulkGeocoder(token=INVALID_TOKEN, logger=Mock())
with self.assertRaises(ServiceException):
invalid_geocoder._batch_geocode([(SEARCH_ID_1, VALID_ADDRESS_1, None, None, None)])
def test_valid_request_bulk_one(self):
place = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, VALID_ADDRESS_1, None, None, None)])
self.assertEqual(place[0][0], SEARCH_ID_1)
self.assertEqual('%.3f' % place[0][1], '%.3f' % WELL_KNOWN_LONGITUDE_1)
self.assertEqual('%.3f' % place[0][2], '%.3f' % WELL_KNOWN_LATITUDE_1)
place = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, VALID_ADDRESS_2, None, None, None)])
self.assertEqual(place[0][0], SEARCH_ID_1)
self.assertEqual('%.3f' % place[0][1], '%.3f' % WELL_KNOWN_LONGITUDE_2)
self.assertEqual('%.3f' % place[0][2], '%.3f' % WELL_KNOWN_LATITUDE_2)
place = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, VALID_ADDRESS_3, None, None, None)])
self.assertEqual(place[0][0], SEARCH_ID_1)
self.assertEqual('%.3f' % place[0][1], '%.3f' % WELL_KNOWN_LONGITUDE_3)
self.assertEqual('%.3f' % place[0][2], '%.3f' % WELL_KNOWN_LATITUDE_3)
def test_valid_request_components_bulk_one(self):
place = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, VALID_SEARCH_TEXT_1, VALID_CITY_1, VALID_STATE_PROVINCE_1, VALID_COUNTRY_1)])
self.assertEqual(place[0][0], SEARCH_ID_1)
self.assertEqual('%.3f' % place[0][1], '%.3f' % WELL_KNOWN_LONGITUDE_1)
self.assertEqual('%.3f' % place[0][2], '%.3f' % WELL_KNOWN_LATITUDE_1)
place = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, VALID_SEARCH_TEXT_2, VALID_CITY_2, VALID_STATE_PROVINCE_2, VALID_COUNTRY_2)])
self.assertEqual(place[0][0], SEARCH_ID_1)
self.assertEqual('%.3f' % place[0][1], '%.3f' % WELL_KNOWN_LONGITUDE_2)
self.assertEqual('%.3f' % place[0][2], '%.3f' % WELL_KNOWN_LATITUDE_2)
place = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, VALID_SEARCH_TEXT_3, VALID_CITY_3, VALID_STATE_PROVINCE_3, VALID_COUNTRY_3)])
self.assertEqual(place[0][0], SEARCH_ID_1)
self.assertEqual('%.3f' % place[0][1], '%.3f' % WELL_KNOWN_LONGITUDE_3)
self.assertEqual('%.3f' % place[0][2], '%.3f' % WELL_KNOWN_LATITUDE_3)
def test_valid_request_namedplace_bulk_one(self):
place = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, 'New York', None, None, None)])
assert place
def test_valid_request_namedplace2_bulk_one(self):
place = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, 'New York', 'us', None, None)])
assert place
def test_odd_characters_bulk_one(self):
place = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, 'New York; "USA"', None, None, None)])
assert place
def test_empty_request_bulk_one(self):
place = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, '', None, None, None)])
assert place == [(SEARCH_ID_1, None, None)]
def test_empty_search_text_request_bulk_one(self):
place = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, ' ', 'us', None, "")])
assert place == [(SEARCH_ID_1, None, None)]
def test_unknown_place_request_bulk_one(self):
place = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, '[unknown]', 'ch', None, None)])
assert place == [(SEARCH_ID_1, None, None)]
### BULK MANY
def test_invalid_token_bulk_many(self):
invalid_geocoder = GeocodioBulkGeocoder(token=INVALID_TOKEN, logger=Mock())
with self.assertRaises(ServiceException):
invalid_geocoder._batch_geocode([(SEARCH_ID_1, VALID_ADDRESS_1, None, None, None),
(SEARCH_ID_2, VALID_ADDRESS_2, None, None, None)])
def test_valid_request_bulk_many(self):
places = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, VALID_ADDRESS_1, None, None, None),
(SEARCH_ID_2, VALID_ADDRESS_2, None, None, None)])
self.assertEqual(places[0][0], SEARCH_ID_1)
self.assertEqual('%.3f' % places[0][1][0], '%.3f' % WELL_KNOWN_LONGITUDE_1)
self.assertEqual('%.3f' % places[0][1][1], '%.3f' % WELL_KNOWN_LATITUDE_1)
self.assertEqual(places[1][0], SEARCH_ID_2)
self.assertEqual('%.3f' % places[1][1][0], '%.3f' % WELL_KNOWN_LONGITUDE_2)
self.assertEqual('%.3f' % places[1][1][1], '%.3f' % WELL_KNOWN_LATITUDE_2)
def test_valid_request_components_bulk_many(self):
places = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, VALID_SEARCH_TEXT_1, VALID_CITY_1, VALID_STATE_PROVINCE_1, VALID_COUNTRY_1),
(SEARCH_ID_2, VALID_SEARCH_TEXT_2, VALID_CITY_2, VALID_STATE_PROVINCE_2, VALID_COUNTRY_2)])
self.assertEqual(places[0][0], SEARCH_ID_1)
self.assertEqual(places[1][0], SEARCH_ID_2)
def test_valid_request_namedplace_bulk_many(self):
places = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, 'New York', None, None, None),
(SEARCH_ID_2, 'Los Angeles', None, None, None)])
assert places
self.assertEqual(places[0][0], SEARCH_ID_1)
self.assertEqual(places[1][0], SEARCH_ID_2)
def test_valid_request_namedplace2_bulk_many(self):
places = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, 'New York', 'us', None, None),
(SEARCH_ID_2, 'Los Angeles', None, None, None)])
assert places
self.assertEqual(places[0][0], SEARCH_ID_1)
self.assertEqual(places[1][0], SEARCH_ID_2)
def test_odd_characters_bulk_many(self):
places = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, 'New York; "USA"', None, None, None),
(SEARCH_ID_2, 'Los Angeles', None, None, None)])
assert places
self.assertEqual(places[0][0], SEARCH_ID_1)
self.assertEqual(places[1][0], SEARCH_ID_2)
def test_empty_request_bulk_many(self):
places = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, '', None, None, None),
(SEARCH_ID_2, '', None, None, None)])
assert places == [(SEARCH_ID_1, [], {}), (SEARCH_ID_2, [], {})]
def test_empty_search_text_request_bulk_many(self):
places = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, ' ', 'us', None, ""),
(SEARCH_ID_2, ' ', 'us', None, "")])
assert places == [(SEARCH_ID_1, [], {}), (SEARCH_ID_2, [], {})]
def test_unknown_place_request_bulk_many(self):
places = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, '[unknown]', 'ch', None, None),
(SEARCH_ID_2, '[unknown]', 'ch', None, None)])
assert places == [(SEARCH_ID_1, [], {}), (SEARCH_ID_2, [], {})]

1
server/lib/python/cartodb_services/test/test_helper.py
View File

@ -79,6 +79,7 @@ def plpy_mock_config():
plpy_mock._define_result("CDB_Conf_GetConf\('mapbox_conf'\)", [{'conf': '{"routing": {"api_keys": ["api_key_rou"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["api_key_geo"], "monthly_quota": 1500000}, "matrix": {"api_keys": ["api_key_mat"], "monthly_quota": 1500000}}'}])
plpy_mock._define_result("CDB_Conf_GetConf\('mapbox_iso_conf'\)", [{'conf': '{"isolines": {"api_keys": ["api_key_mat"], "monthly_quota": 1500000}}'}])
plpy_mock._define_result("CDB_Conf_GetConf\('tomtom_conf'\)", [{'conf': '{"routing": {"api_keys": ["api_key_rou"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["api_key_geo"], "monthly_quota": 1500000}, "isolines": {"api_keys": ["api_key_mat"], "monthly_quota": 1500000}}'}])
plpy_mock._define_result("CDB_Conf_GetConf\('geocodio_conf'\)", [{'conf': '{"geocoder": {"api_keys": ["api_key_geo"], "monthly_quota": 1500000}}'}])
plpy_mock._define_result("CDB_Conf_GetConf\('logger_conf'\)", [{'conf': '{"geocoder_log_path": "/dev/null"}'}])
plpy_mock._define_result("CDB_Conf_GetConf\('data_observatory_conf'\)", [{'conf': '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}}'}])
plpy_mock._define_result("CDB_Conf_GetConf\('server_conf'\)", [{'conf': '{"environment": "testing"}'}])