Merge development

NEWS.md

@@ -1,3 +1,12 @@
+Jan 20th, 2020
+==============
+* Version `0.22.0` of the Python library
+    * Added geocodio geocoder
+* Version `0.36.0` of the server extension
+    * Added geocodio geocoder
+* Version `0.28.0` of the client extension
+    * Added geocodio geocoder
+
 Oct 9th, 2019
 ==============
 * Version `0.27.0` of the client extension

README.md

@@ -180,6 +180,15 @@ SELECT CDB_Conf_SetConf(
 );
 ```
 
+#### Geocod.io configuration
+
+```sql
+SELECT CDB_Conf_SetConf(
+    'geocodio_conf',
+    '{"geocoder": {"api_keys": ["your_api_key"], "monthly_quota": 999999}}'
+);
+```
+
 #### Data Observatory
 
 ```sql

client/cdb_dataservices_client--0.27.0--0.28.0.sql

@@ -0,0 +1,83 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.28.0'" to load this file. \quit
+
+-- Make sure we have a sane search path to create/update the extension
+SET search_path = "$user",cartodb,public,cdb_dataservices_client;
+
+-- HERE goes your code to upgrade/downgrade
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocodio_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL)
+RETURNS public.Geometry AS $$
+DECLARE
+  ret public.Geometry;
+  username text;
+  orgname text;
+  _returned_sqlstate TEXT;
+  _message_text TEXT;
+  _pg_exception_context TEXT;
+  apikey_permissions json;
+BEGIN
+  
+  SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json);
+  IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN
+    RAISE EXCEPTION 'Geocoding permission denied';
+  END IF;
+  
+  -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+
+
+  BEGIN
+    SELECT cdb_dataservices_client._cdb_geocodio_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret;
+  EXCEPTION
+    WHEN OTHERS THEN
+        GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE,
+                                _message_text = MESSAGE_TEXT,
+                                _pg_exception_context = PG_EXCEPTION_CONTEXT;
+        RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context;
+         RETURN ret;
+  END;
+END;
+$$  LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE
+    SET search_path = pg_temp;
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocodio_geocode_street_point (username text, orgname text, searchtext text, city text, state_province text, country text);
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocodio_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL)
+RETURNS Geometry AS $$
+  CONNECT cdb_dataservices_client._server_conn_str();
+  
+  SELECT cdb_dataservices_server.cdb_geocodio_geocode_street_point (username, orgname, searchtext, city, state_province, country);
+
+$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_geocodio_geocode_street_point (searchtext text ,city text ,state_province text ,country text);
+CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocodio_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL)
+RETURNS public.Geometry AS $$
+DECLARE
+  ret public.Geometry;
+  username text;
+  orgname text;
+  apikey_permissions json;
+BEGIN
+  
+  SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json);
+  IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN
+    RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007';
+  END IF;
+  
+  -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+
+  SELECT cdb_dataservices_client._cdb_geocodio_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret;
+END;
+$$  LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE
+    SET search_path = pg_temp;
+
+GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocodio_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser;
+GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocodio_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text )  TO publicuser;
+

client/cdb_dataservices_client--0.28.0--0.27.0.sql

@@ -0,0 +1,14 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.27.0'" to load this file. \quit
+
+-- Make sure we have a sane search path to create/update the extension
+SET search_path = "$user",cartodb,public,cdb_dataservices_client;
+
+-- HERE goes your code to upgrade/downgrade
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocodio_geocode_street_point_exception_safe;
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocodio_geocode_street_point;
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_geocodio_geocode_street_point;

client/cdb_dataservices_client.control

@@ -1,5 +1,5 @@
 comment = 'CartoDB dataservices client API extension'
-default_version = '0.27.0'
+default_version = '0.28.0'
 requires = 'plproxy, cartodb'
 superuser = true
 schema = cdb_dataservices_client

client/renderer/interface.yaml

@@ -160,6 +160,17 @@
     - { name: state_province,  type: text, default: 'NULL'}
     - { name: country,  type: text, default: 'NULL'}
 
+- name: cdb_geocodio_geocode_street_point
+  return_type: public.Geometry
+  requires_permission: true
+  permission_name: geocoding
+  permission_error: Geocoding permission denied
+  params:
+    - { name: searchtext,  type: text}
+    - { name: city,  type: text, default: 'NULL'}
+    - { name: state_province,  type: text, default: 'NULL'}
+    - { name: country,  type: text, default: 'NULL'}
+
 - name: cdb_mapzen_geocode_street_point
   return_type: public.Geometry
   requires_permission: true

server/extension/cdb_dataservices_server--0.35.1--0.36.0.sql

@@ -0,0 +1,197 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.36.0'" to load this file. \quit
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
+RETURNS Geometry AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.tools import Logger,LoggerConfig
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  params = {'username': username, 'orgname': orgname, 'searchtext': searchtext, 'city': city, 'state_province': state_province, 'country': country}
+
+  with metrics('cdb_geocode_street_point', user_geocoder_config, logger, params):
+    if user_geocoder_config.heremaps_geocoder:
+      here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+      return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+    elif user_geocoder_config.google_geocoder:
+      google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+      return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+    elif user_geocoder_config.mapzen_geocoder:
+      mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+      return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+    elif user_geocoder_config.mapbox_geocoder:
+      mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+      return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+    elif user_geocoder_config.tomtom_geocoder:
+      tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+      return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+    elif user_geocoder_config.geocodio_geocoder:
+      geocodio_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocodio_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+      return plpy.execute(geocodio_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+    else:
+      raise Exception('Requested geocoder is not available')
+
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocodio_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
+RETURNS Geometry AS $$
+  # The configuration is retrieved but no checks are performed on it
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
+
+  geocodio_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocodio_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+  return plpy.execute(geocodio_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocodio_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
+RETURNS Geometry AS $$
+  from iso3166 import countries
+  from cartodb_services.tools import ServiceManager, QuotaExceededException
+  from cartodb_services.geocodio import GeocodioGeocoder
+  from cartodb_services.tools.country import country_to_iso3
+  from cartodb_services.refactor.service.geocodio_geocoder_config import GeocodioGeocoderConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_manager = ServiceManager('geocoder', GeocodioGeocoderConfigBuilder, username, orgname, GD)
+
+  try:
+    service_manager.assert_within_limits()
+    geocoder = GeocodioGeocoder(service_manager.config.geocodio_api_key, service_manager.logger, service_manager.config.service_params)
+
+    country_iso3166 = None
+    if country:
+      country_iso3 = country_to_iso3(country)
+      if country_iso3:
+        country_iso3166 = countries.get(country_iso3).alpha2.lower()
+
+    coordinates = geocoder.geocode(searchtext=searchtext, city=city,
+                                   state_province=state_province,
+                                   country=country_iso3166)
+
+    if coordinates:
+      service_manager.quota_service.increment_success_service_use()
+      plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
+      point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
+      return point['st_setsrid']
+    else:
+      service_manager.quota_service.increment_empty_service_use()
+      return None
+  except QuotaExceededException as qe:
+    service_manager.quota_service.increment_failed_service_use()
+    return None
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to geocode street point using Geocodio', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to geocode street point using Geocodio')
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocode_street_point(username TEXT, orgname TEXT, searches jsonb)
+RETURNS SETOF cdb_dataservices_server.geocoding AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.tools import Logger
+  import json
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+
+  plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  params = {'username': username, 'orgname': orgname, 'searches': json.loads(searches)}
+
+  with metrics('cdb_bulk_geocode_street_point', user_geocoder_config, logger, params):
+    if user_geocoder_config.google_geocoder:
+      provider_function = "_cdb_bulk_google_geocode_street_point";
+    elif user_geocoder_config.heremaps_geocoder:
+      provider_function = "_cdb_bulk_heremaps_geocode_street_point";
+    elif user_geocoder_config.tomtom_geocoder:
+      provider_function = "_cdb_bulk_tomtom_geocode_street_point";
+    elif user_geocoder_config.mapbox_geocoder:
+      provider_function = "_cdb_bulk_mapbox_geocode_street_point";
+    elif user_geocoder_config.geocodio_geocoder:
+      provider_function = "_cdb_bulk_geocodio_geocode_street_point";
+    else:
+      raise Exception('Requested geocoder is not available')
+
+    plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.{}($1, $2, $3); ".format(provider_function), ["text", "text", "jsonb"])
+    return plpy.execute(plan, [username, orgname, searches])
+
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocodio_geocode_street_point(username TEXT, orgname TEXT, searches jsonb)
+RETURNS SETOF cdb_dataservices_server.geocoding AS $$
+  from cartodb_services import run_street_point_geocoder
+  from cartodb_services.tools import ServiceManager
+  from cartodb_services.refactor.service.geocodio_geocoder_config import GeocodioGeocoderConfigBuilder
+  from cartodb_services.geocodio import GeocodioBulkGeocoder
+  from cartodb_services.tools import Logger
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+  service_manager = ServiceManager('geocoder', GeocodioGeocoderConfigBuilder, username, orgname, GD)
+  geocoder = GeocodioBulkGeocoder(service_manager.config.geocodio_api_key, service_manager.logger, service_manager.config.service_params)
+  return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches)
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text)
+RETURNS Geometry AS $$
+  from cartodb_services.metrics import QuotaService
+  from cartodb_services.metrics import InternalGeocoderConfig
+  from cartodb_services.metrics import metrics
+  from cartodb_services.tools import Logger,LoggerConfig
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_internal_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_geocoder_config = GD["user_internal_geocoder_config_{0}".format(username)]
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+  quota_service = QuotaService(user_geocoder_config, redis_conn)
+
+  params = {'username': username, 'orgname': orgname, 'country_name': country_name}
+
+  with metrics('cdb_geocode_admin0_polygon', user_geocoder_config, logger, params):
+    try:
+      plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocode_admin0_polygon(trim($1)) AS mypolygon", ["text"])
+      rv = plpy.execute(plan, [country_name], 1)
+      result = rv[0]["mypolygon"]
+      if result:
+        quota_service.increment_success_service_use()
+        return result
+      else:
+        quota_service.increment_empty_service_use()
+        return None
+    except BaseException as e:
+      import sys
+      quota_service.increment_failed_service_use()
+      logger.error('Error trying to geocode admin0 polygon', sys.exc_info(), data={"username": username, "orgname": orgname})
+      raise Exception('Error trying to geocode admin0 polygon')
+    finally:
+      quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;

server/extension/cdb_dataservices_server--0.36.0--0.35.1.sql

@@ -0,0 +1,79 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.35.1'" to load this file. \quit
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
+RETURNS Geometry AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.tools import Logger,LoggerConfig
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  params = {'username': username, 'orgname': orgname, 'searchtext': searchtext, 'city': city, 'state_province': state_province, 'country': country}
+
+  with metrics('cdb_geocode_street_point', user_geocoder_config, logger, params):
+    if user_geocoder_config.heremaps_geocoder:
+      here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+      return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+    elif user_geocoder_config.google_geocoder:
+      google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+      return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+    elif user_geocoder_config.mapzen_geocoder:
+      mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+      return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+    elif user_geocoder_config.mapbox_geocoder:
+      mapbox_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapbox_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+      return plpy.execute(mapbox_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+    elif user_geocoder_config.tomtom_geocoder:
+      tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+      return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+    else:
+      raise Exception('Requested geocoder is not available')
+
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+DROP FUNCTION IF EXISTS cdb_dataservices_server.cdb_geocodio_geocode_street_point;
+
+DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_geocodio_geocode_street_point;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocode_street_point(username TEXT, orgname TEXT, searches jsonb)
+RETURNS SETOF cdb_dataservices_server.geocoding AS $$
+  from cartodb_services.metrics import metrics
+  from cartodb_services.tools import Logger
+  import json
+
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+
+  plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  params = {'username': username, 'orgname': orgname, 'searches': json.loads(searches)}
+
+  with metrics('cdb_bulk_geocode_street_point', user_geocoder_config, logger, params):
+    if user_geocoder_config.google_geocoder:
+      provider_function = "_cdb_bulk_google_geocode_street_point";
+    elif user_geocoder_config.heremaps_geocoder:
+      provider_function = "_cdb_bulk_heremaps_geocode_street_point";
+    elif user_geocoder_config.tomtom_geocoder:
+      provider_function = "_cdb_bulk_tomtom_geocode_street_point";
+    elif user_geocoder_config.mapbox_geocoder:
+      provider_function = "_cdb_bulk_mapbox_geocode_street_point";
+    else:
+      raise Exception('Requested geocoder is not available')
+
+    plan = plpy.prepare("SELECT * FROM cdb_dataservices_server.{}($1, $2, $3); ".format(provider_function), ["text", "text", "jsonb"])
+    return plpy.execute(plan, [username, orgname, searches])
+
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_bulk_geocodio_geocode_street_point;

server/extension/cdb_dataservices_server.control

@@ -1,5 +1,5 @@
 comment = 'CartoDB dataservices server extension'
-default_version = '0.35.1'
+default_version = '0.36.0'
 requires = 'plpythonu, plproxy, postgis, cdb_geocoder'
 superuser = true
 schema = cdb_dataservices_server

server/extension/sql/20_geocode_street.sql

@@ -30,6 +30,9 @@ RETURNS Geometry AS $$
     elif user_geocoder_config.tomtom_geocoder:
       tomtom_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_tomtom_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
       return plpy.execute(tomtom_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+    elif user_geocoder_config.geocodio_geocoder:
+      geocodio_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocodio_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+      return plpy.execute(geocodio_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
     else:
       raise Exception('Requested geocoder is not available')
 
@@ -105,6 +108,19 @@ RETURNS Geometry AS $$
 
 $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
 
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocodio_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
+RETURNS Geometry AS $$
+  # The configuration is retrieved but no checks are performed on it
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
+
+  geocodio_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_geocodio_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
+  return plpy.execute(geocodio_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
+
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
 RETURNS Geometry AS $$
   from cartodb_services.tools import LegacyServiceManager
@@ -302,3 +318,49 @@ RETURNS Geometry AS $$
   finally:
     service_manager.quota_service.increment_total_service_use()
 $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_geocodio_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
+RETURNS Geometry AS $$
+  from iso3166 import countries
+  from cartodb_services.tools import ServiceManager, QuotaExceededException
+  from cartodb_services.geocodio import GeocodioGeocoder
+  from cartodb_services.tools.country import country_to_iso3
+  from cartodb_services.refactor.service.geocodio_geocoder_config import GeocodioGeocoderConfigBuilder
+
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  service_manager = ServiceManager('geocoder', GeocodioGeocoderConfigBuilder, username, orgname, GD)
+
+  try:
+    service_manager.assert_within_limits()
+    geocoder = GeocodioGeocoder(service_manager.config.geocodio_api_key, service_manager.logger, service_manager.config.service_params)
+
+    country_iso3166 = None
+    if country:
+      country_iso3 = country_to_iso3(country)
+      if country_iso3:
+        country_iso3166 = countries.get(country_iso3).alpha2.lower()
+
+    coordinates = geocoder.geocode(searchtext=searchtext, city=city,
+                                   state_province=state_province,
+                                   country=country_iso3166)
+    if coordinates:
+      service_manager.quota_service.increment_success_service_use()
+      plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
+      point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
+      return point['st_setsrid']
+    else:
+      service_manager.quota_service.increment_empty_service_use()
+      return None
+  except QuotaExceededException as qe:
+    service_manager.quota_service.increment_failed_service_use()
+    return None
+  except BaseException as e:
+    import sys
+    service_manager.quota_service.increment_failed_service_use()
+    service_manager.logger.error('Error trying to geocode street point using Geocodio', sys.exc_info(), data={"username": username, "orgname": orgname})
+    raise Exception('Error trying to geocode street point using Geocodio')
+  finally:
+    service_manager.quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;

server/extension/sql/21_bulk_geocode_street.sql

@@ -32,6 +32,8 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$
       provider_function = "_cdb_bulk_tomtom_geocode_street_point";
     elif user_geocoder_config.mapbox_geocoder:
       provider_function = "_cdb_bulk_mapbox_geocode_street_point";
+    elif user_geocoder_config.geocodio_geocoder:
+      provider_function = "_cdb_bulk_geocodio_geocode_street_point";
     else:
       raise Exception('Requested geocoder is not available')
 
@@ -96,3 +98,19 @@ RETURNS SETOF cdb_dataservices_server.geocoding AS $$
   return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches)
 $$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
 
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_bulk_geocodio_geocode_street_point(username TEXT, orgname TEXT, searches jsonb)
+RETURNS SETOF cdb_dataservices_server.geocoding AS $$
+  from cartodb_services import run_street_point_geocoder
+  from cartodb_services.tools import ServiceManager
+  from cartodb_services.refactor.service.geocodio_geocoder_config import GeocodioGeocoderConfigBuilder
+  from cartodb_services.geocodio import GeocodioBulkGeocoder
+  from cartodb_services.tools import Logger
+  import cartodb_services
+  cartodb_services.init(plpy, GD)
+
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+  service_manager = ServiceManager('geocoder', GeocodioGeocoderConfigBuilder, username, orgname, GD)
+  geocoder = GeocodioBulkGeocoder(service_manager.config.geocodio_api_key, service_manager.logger, service_manager.config.service_params)
+  return run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, orgname, searches)
+$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;

server/extension/test/expected/00_install_test.out

@@ -45,6 +45,12 @@ SELECT cartodb.cdb_conf_setconf('tomtom_conf', '{"routing": {"api_keys": ["routi
  
 (1 row)
 
+SELECT cartodb.cdb_conf_setconf('geocodio_conf', '{"geocoder": {"api_keys": ["geocoder_dummy_api_key"], "monthly_quota": 1500000}}');
+ cdb_conf_setconf 
+------------------
+ 
+(1 row)
+
 SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/dev/null"}');
  cdb_conf_setconf 
 ------------------

server/extension/test/sql/00_install_test.sql

@@ -17,6 +17,7 @@ SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"geocoder": {"app_id": "dummy
 SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing": {"api_key": "routing_dummy_api_key", "monthly_quota": 1500000}, "geocoder": {"api_key": "geocoder_dummy_api_key", "monthly_quota": 1500000}, "matrix": {"api_key": "matrix_dummy_api_key", "monthly_quota": 1500000}}');
 SELECT cartodb.cdb_conf_setconf('mapbox_conf', '{"routing": {"api_keys": ["routing_dummy_api_key"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["geocoder_dummy_api_key"], "monthly_quota": 1500000}, "matrix": {"api_keys": ["matrix_dummy_api_key"], "monthly_quota": 1500000}}');
 SELECT cartodb.cdb_conf_setconf('tomtom_conf', '{"routing": {"api_keys": ["routing_dummy_api_key"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["geocoder_dummy_api_key"], "monthly_quota": 1500000}, "isolines": {"api_keys": ["matrix_dummy_api_key"], "monthly_quota": 1500000}}');
+SELECT cartodb.cdb_conf_setconf('geocodio_conf', '{"geocoder": {"api_keys": ["geocoder_dummy_api_key"], "monthly_quota": 1500000}}');
 SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/dev/null"}');
 SELECT cartodb.cdb_conf_setconf('data_observatory_conf', '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=contrib_regression user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}, "monthly_quota": 100000}');
 

server/lib/python/cartodb_services/cartodb_services/bulk_geocoders.py

@@ -2,10 +2,12 @@ from google import GoogleMapsBulkGeocoder
 from here import HereMapsBulkGeocoder
 from tomtom import TomTomBulkGeocoder
 from mapbox import MapboxBulkGeocoder
+from geocodio import GeocodioBulkGeocoder
 
 BATCH_GEOCODER_CLASS_BY_PROVIDER = {
     'google': GoogleMapsBulkGeocoder,
     'heremaps': HereMapsBulkGeocoder,
     'tomtom': TomTomBulkGeocoder,
-    'mapbox': MapboxBulkGeocoder
+    'mapbox': MapboxBulkGeocoder,
+    'geocodio': GeocodioBulkGeocoder,
 }

server/lib/python/cartodb_services/cartodb_services/geocodio/__init__.py

@@ -0,0 +1,2 @@
+from geocoder import GeocodioGeocoder
+from bulk_geocoder import GeocodioBulkGeocoder

server/lib/python/cartodb_services/cartodb_services/geocodio/bulk_geocoder.py

@@ -0,0 +1,76 @@
+import requests
+from cartodb_services import StreetPointBulkGeocoder
+from cartodb_services.geocodio import GeocodioGeocoder
+from iso3166 import countries
+from cartodb_services.tools.country import country_to_iso3
+
+
+class GeocodioBulkGeocoder(GeocodioGeocoder, StreetPointBulkGeocoder):
+    MAX_BATCH_SIZE = 100  # Setting an upper limit (not stated in the documentation)
+    MIN_BATCHED_SEARCH = 0
+    READ_TIMEOUT = 60
+    CONNECT_TIMEOUT = 10
+    MAX_RETRIES = 1
+
+    def __init__(self, token, logger, service_params=None):
+        GeocodioGeocoder.__init__(self, token, logger, service_params)
+
+        self.connect_timeout = self.CONNECT_TIMEOUT
+        self.read_timeout = self.READ_TIMEOUT
+        self.max_retries = self.MAX_RETRIES
+
+        if service_params is not None:
+            self.connect_timeout = service_params.get('connect_timeout', self.CONNECT_TIMEOUT)
+            self.read_timeout = service_params.get('read_timeout', self.READ_TIMEOUT)
+            self.max_retries = service_params.get('max_retries', self.MAX_RETRIES)
+
+        self.session = requests.Session()
+
+    def _should_use_batch(self, searches):
+        return len(searches) >= self.MIN_BATCHED_SEARCH
+
+    def _serial_geocode(self, searches):
+        results = []
+        for search in searches:
+            elements = self._encoded_elements(search)
+            result = self.geocode_meta(*elements)
+
+            if result:
+                results.append((search[0], result[0], result[1]))
+            else:
+                results.append((search[0], None, None))
+
+        return results
+
+    def _encoded_elements(self, search):
+        (search_id, address, city, state, country) = search
+        address = address.encode('utf-8') if address else None
+        city = city.encode('utf-8') if city else None
+        state = state.encode('utf-8') if state else None
+        country = self._country_code(country) if country else None
+        return address, city, state, country
+
+    def _batch_geocode(self, searches):
+        if len(searches) == 1:
+            return self._serial_geocode(searches)
+        else:
+            frees = []
+            for search in searches:
+                elements = self._encoded_elements(search)
+                free = ', '.join([elem for elem in elements if elem])
+                frees.append(free)
+
+            full_results = self.geocode_free_text_meta(frees)
+            results = []
+            for s, r in zip(searches, full_results):
+                results.append((s[0], r[0], r[1]))
+            return results
+
+    def _country_code(self, country):
+        country_iso3166 = country
+        country_iso3 = country_to_iso3(country)
+        if country_iso3:
+            country_iso3166 = countries.get(country_iso3).alpha2.lower()
+
+        return country_iso3166
+

server/lib/python/cartodb_services/cartodb_services/geocodio/geocoder.py

@@ -0,0 +1,126 @@
+from geocodio import GeocodioClient
+from geocodio.exceptions import GeocodioAuthError, GeocodioServerError, GeocodioDataError, GeocodioError
+
+from cartodb_services.tools.qps import qps_retry
+from cartodb_services.metrics import Traceable
+from cartodb_services.geocoder import EMPTY_RESPONSE, geocoder_metadata
+from cartodb_services.tools.exceptions import ServiceException
+
+
+RELEVANCE_BY_LOCATION_TYPE = {
+    'rooftop': 1,
+    'point': 0.9,
+    'range_interpolation': 0.8,
+    'nearest_rooftop_match': 0.7,
+    'intersection': 0.6,
+    'street_center': 0.5,
+    'place': 0.4,
+    'state': 0.1,
+}
+
+
+class GeocodioGeocoder(Traceable):
+    '''
+    Python wrapper for the Geocodio Geocoder service.
+    '''
+
+    def __init__(self, token, logger, service_params=None):
+        service_params = service_params or {}
+        self._token = token
+        self._logger = logger
+
+        self._geocoder = GeocodioClient(self._token)
+
+    def _validate_input(self, searchtext, city=None, state_province=None,
+                        country=None):
+        if searchtext and searchtext.strip():
+            return True
+        elif city:
+            return True
+        elif state_province:
+            return True
+
+        return False
+
+    @qps_retry(qps=15, provider='geocodio')
+    def geocode(self, searchtext, city=None, state_province=None,
+                country=None):
+        return self._geocode_meta(searchtext, city, state_province, country)[0]
+
+    def geocode_meta(self, searchtext, city=None, state_province=None,
+                     country=None):
+        return self._geocode_meta(searchtext, city, state_province, country)[0]
+
+    @qps_retry(qps=15, provider='geocodio')
+    def _geocode_meta(self, searchtext, city=None, state_province=None,
+                      country=None):
+        if not self._validate_input(searchtext, city, state_province, country):
+            return EMPTY_RESPONSE
+
+        try:
+            free_text_components = [searchtext, city, state_province, country]
+            req = '; '.join([c for c in free_text_components if c is not None and c.strip()])
+            response = self._geocoder.geocode(req)
+
+            return self._parse_geocoder_response(response)
+        except GeocodioDataError as gde:
+            return EMPTY_RESPONSE
+        except GeocodioAuthError as gae:
+            raise ServiceException('Geocodio authorization error: ' + str(gae), None)
+        except GeocodioServerError as gse:
+            raise ServiceException('geocodio server error: ' + str(gse), None)
+        except GeocodioError as ge:
+            raise ServiceException('Unknown Geocodio error: ' + str(ge), None)
+
+    @qps_retry(qps=15)
+    def geocode_free_text_meta(self, free_searches, country=None):
+        """
+        :param free_searches: Free text searches
+        :return: list of [x, y] on success, [] on error
+        """
+        output = []
+
+        try:
+            if country:
+                free_searches = ['{s}, {country}'.format(s, country) for s in free_searches]
+
+            responses = self._geocoder.geocode(free_searches)
+
+            for response in responses:
+                output.append(self._parse_geocoder_response(response))
+        except GeocodioDataError as gde:
+            return EMPTY_RESPONSE
+        except GeocodioAuthError as gae:
+            raise ServiceException('Geocodio authorization error: ' + str(gae), None)
+        except GeocodioServerError as gse:
+            raise ServiceException('geocodio server error: ' + str(gse), None)
+        except GeocodioError as ge:
+            raise ServiceException('Unknown Geocodio error: ' + str(ge), None)
+
+        return output
+
+    def _parse_geocoder_response(self, response):
+        if response is None or not response:
+            return EMPTY_RESPONSE
+
+        if response.get('results') is None or not response.get('results'):
+            return EMPTY_RESPONSE
+
+        if response.coords is None or not response.coords:
+            return EMPTY_RESPONSE
+
+        coords = [None, None]
+        accuracy = None
+        accuracy_type = None
+
+        accuracy = response.accuracy
+
+        if response.coords is not None and response.coords:
+            coords = [response.coords[1], response.coords[0]]
+
+        if response.get('results'):
+            accuracy_type = response.get('results')[0].get('accuracy_type')
+
+        metadata = geocoder_metadata(RELEVANCE_BY_LOCATION_TYPE.get(accuracy_type), response.accuracy, accuracy_type)
+
+        return [coords, metadata]

server/lib/python/cartodb_services/cartodb_services/geocodio/types.py

@@ -0,0 +1 @@
+GEOCODIO_GEOCODER_APIKEY_ROUNDROBIN = 'geocodio_geocoder_apikey_roundrobin'

server/lib/python/cartodb_services/cartodb_services/metrics/config.py

@@ -423,6 +423,8 @@ class GeocoderConfig(ServiceConfig):
     MAPBOX_GEOCODER_API_KEYS = 'mapbox_geocoder_api_keys'
     TOMTOM_GEOCODER = 'tomtom'
     TOMTOM_GEOCODER_API_KEYS = 'tomtom_geocoder_api_keys'
+    GEOCODIO_GEOCODER = 'geocodio'
+    GEOCODIO_GEOCODER_API_KEYS = 'geocodio_geocoder_api_keys'
     QUOTA_KEY = 'geocoding_quota'
     SOFT_LIMIT_KEY = 'soft_geocoding_limit'
     USERNAME_KEY = 'username'
@@ -455,6 +457,9 @@ class GeocoderConfig(ServiceConfig):
         elif self._geocoder_provider == self.TOMTOM_GEOCODER:
             if not self.tomtom_api_keys:
                 raise ConfigException("""TomTom config is not set up""")
+        elif self._geocoder_provider == self.GEOCODIO_GEOCODER:
+            if not self.geocodio_api_keys:
+                raise ConfigException("""Geocodio config is not set up""")
 
         return True
 
@@ -494,6 +499,10 @@ class GeocoderConfig(ServiceConfig):
             self._tomtom_api_keys = db_config.tomtom_geocoder_api_keys
             self._cost_per_hit = 0
             self._tomtom_service_params = db_config.tomtom_geocoder_service_params
+        elif self._geocoder_provider == self.GEOCODIO_GEOCODER:
+            self._geocodio_api_keys = db_config.geocodio_geocoder_api_keys
+            self._cost_per_hit = 0
+            self._geocodio_service_params = db_config.geocodio_geocoder_service_params
 
     @property
     def service_type(self):
@@ -507,6 +516,8 @@ class GeocoderConfig(ServiceConfig):
             return 'geocoder_tomtom'
         elif self._geocoder_provider == self.NOKIA_GEOCODER:
             return 'geocoder_here'
+        elif self._geocoder_provider == self.GEOCODIO_GEOCODER:
+            return 'geocoder_geocodio'
 
     @property
     def heremaps_geocoder(self):
@@ -528,6 +539,10 @@ class GeocoderConfig(ServiceConfig):
     def tomtom_geocoder(self):
         return self._geocoder_provider == self.TOMTOM_GEOCODER
 
+    @property
+    def geocodio_geocoder(self):
+        return self._geocoder_provider == self.GEOCODIO_GEOCODER
+
     @property
     def google_client_id(self):
         return self._google_maps_client_id
@@ -587,6 +602,14 @@ class GeocoderConfig(ServiceConfig):
     def tomtom_service_params(self):
         return self._tomtom_service_params
 
+    @property
+    def geocodio_api_keys(self):
+        return self._geocodio_api_keys
+
+    @property
+    def geocodio_service_params(self):
+        return self._geocodio_service_params
+
     @property
     def is_high_resolution(self):
         return True
@@ -619,6 +642,7 @@ class ServicesDBConfig:
         self._get_mapbox_config()
         self._get_mapbox_iso_config()
         self._get_tomtom_config()
+        self._get_geocodio_config()
         self._get_data_observatory_config()
 
     def _get_server_config(self):
@@ -708,6 +732,16 @@ class ServicesDBConfig:
             self._tomtom_geocoder_quota = tomtom_conf['geocoder']['monthly_quota']
             self._tomtom_geocoder_service_params = tomtom_conf['geocoder'].get('service', {})
 
+    def _get_geocodio_config(self):
+        geocodio_conf_json = self._get_conf('geocodio_conf')
+        if not geocodio_conf_json:
+            raise ConfigException('Geocodio configuration missing')
+        else:
+            geocodio_conf = json.loads(geocodio_conf_json)
+            self._geocodio_geocoder_api_keys = geocodio_conf['geocoder']['api_keys']
+            self._geocodio_geocoder_quota = geocodio_conf['geocoder']['monthly_quota']
+            self._geocodio_geocoder_service_params = geocodio_conf['geocoder'].get('service', {})
+
     def _get_data_observatory_config(self):
         do_conf_json = self._get_conf('data_observatory_conf')
         if not do_conf_json:
@@ -889,6 +923,18 @@ class ServicesDBConfig:
     def tomtom_geocoder_service_params(self):
         return self._tomtom_geocoder_service_params
 
+    @property
+    def geocodio_geocoder_api_keys(self):
+        return self._geocodio_geocoder_api_keys
+
+    @property
+    def geocodio_geocoder_monthly_quota(self):
+        return self.geocodio_geocoder_quota
+
+    @property
+    def geocodio_geocoder_service_params(self):
+        return self._geocodio_geocoder_service_params
+
     @property
     def data_observatory_connection_str(self):
         return self._data_observatory_connection_str

server/lib/python/cartodb_services/cartodb_services/refactor/service/geocodio_geocoder_config.py

@@ -0,0 +1,128 @@
+from dateutil.parser import parse as date_parse
+from cartodb_services.refactor.service.utils import round_robin
+from cartodb_services.geocodio.types import GEOCODIO_GEOCODER_APIKEY_ROUNDROBIN
+
+
+class GeocodioGeocoderConfig(object):
+    """
+    Configuration needed to operate the Geocodio geocoder service.
+    """
+
+    def __init__(self,
+                 geocoding_quota,
+                 soft_geocoding_limit,
+                 period_end_date,
+                 cost_per_hit,
+                 log_path,
+                 geocodio_api_keys,
+                 username,
+                 organization,
+                 service_params,
+                 GD):
+        self._geocoding_quota = geocoding_quota
+        self._soft_geocoding_limit = soft_geocoding_limit
+        self._period_end_date = period_end_date
+        self._cost_per_hit = cost_per_hit
+        self._log_path = log_path
+        self._geocodio_api_keys = geocodio_api_keys
+        self._username = username
+        self._organization = organization
+        self._service_params = service_params
+        self._GD = GD
+
+    @property
+    def service_type(self):
+        return 'geocoder_geocodio'
+
+    @property
+    def provider(self):
+        return 'geocodio'
+
+    @property
+    def is_high_resolution(self):
+        return True
+
+    @property
+    def geocoding_quota(self):
+        return self._geocoding_quota
+
+    @property
+    def soft_geocoding_limit(self):
+        return self._soft_geocoding_limit
+
+    @property
+    def period_end_date(self):
+        return self._period_end_date
+
+    @property
+    def cost_per_hit(self):
+        return self._cost_per_hit
+
+    @property
+    def log_path(self):
+        return self._log_path
+
+    @property
+    def geocodio_api_key(self):
+        return round_robin(self._geocodio_api_keys, self._GD,
+                           GEOCODIO_GEOCODER_APIKEY_ROUNDROBIN)
+
+    @property
+    def username(self):
+        return self._username
+
+    @property
+    def organization(self):
+        return self._organization
+
+    @property
+    def service_params(self):
+        return self._service_params
+
+    # TODO: for BW compat, remove
+    @property
+    def google_geocoder(self):
+        return False
+
+
+class GeocodioGeocoderConfigBuilder(object):
+
+    def __init__(self, server_conf, user_conf, org_conf, username, orgname, GD):
+        self._server_conf = server_conf
+        self._user_conf = user_conf
+        self._org_conf = org_conf
+        self._username = username
+        self._orgname = orgname
+        self._GD = GD
+
+    def get(self):
+        geocodio_server_conf = self._server_conf.get('geocodio_conf')
+        geocodio_api_keys = geocodio_server_conf['geocoder']['api_keys']
+        geocodio_service_params = geocodio_server_conf['geocoder'].get('service', {})
+
+        geocoding_quota = self._get_quota()
+        soft_geocoding_limit = self._user_conf.get('soft_geocoding_limit').lower() == 'true'
+        cost_per_hit = 0
+        period_end_date_str = self._org_conf.get('period_end_date') or self._user_conf.get('period_end_date')
+        period_end_date = date_parse(period_end_date_str)
+
+        logger_conf = self._server_conf.get('logger_conf')
+        log_path = logger_conf.get('geocoder_log_path', None)
+
+        return GeocodioGeocoderConfig(geocoding_quota,
+                                    soft_geocoding_limit,
+                                    period_end_date,
+                                    cost_per_hit,
+                                    log_path,
+                                    geocodio_api_keys,
+                                    self._username,
+                                    self._orgname,
+                                    geocodio_service_params,
+                                    self._GD)
+
+    def _get_quota(self):
+        geocoding_quota = self._org_conf.get('geocoding_quota') or self._user_conf.get('geocoding_quota')
+        if geocoding_quota is '':
+            return 0
+
+        return int(geocoding_quota)

server/lib/python/cartodb_services/requirements.txt

@@ -7,6 +7,7 @@ rollbar==0.13.2
 requests==2.9.1
 rratelimit==0.0.4
 mapbox==0.14.0
+pygeocodio==0.11.1
 
 # Test
 mock==1.3.0

server/lib/python/cartodb_services/test/credentials.py

@@ -9,3 +9,8 @@ def mapbox_api_key():
 def tomtom_api_key():
     """Returns TomTom API key. Requires setting TOMTOM_API_KEY environment variable."""
     return os.environ['TOMTOM_API_KEY']
+
+
+def geocodio_api_key():
+    """Returns Geocodio API key. Requires setting GEOCODIO_API_KEY environment variable."""
+    return os.environ['GEOCODIO_API_KEY']

server/lib/python/cartodb_services/test/metrics/test_config.py

@@ -7,7 +7,7 @@ from cartodb_services.metrics.config import *
 
 class TestGeocoderUserConfig(TestCase):
 
-    GEOCODER_PROVIDERS = ['heremaps', 'mapzen', 'mapbox', 'tomtom', 'google']
+    GEOCODER_PROVIDERS = ['heremaps', 'mapzen', 'mapbox', 'tomtom', 'geocodio', 'google']
 
     def setUp(self):
         self.redis_conn = MockRedis()
@@ -31,6 +31,9 @@ class TestGeocoderUserConfig(TestCase):
             elif geocoder_provider == 'tomtom':
                 assert geocoder_config.tomtom_geocoder is True
                 assert geocoder_config.geocoding_quota == 100
+            elif geocoder_provider == 'geocodio':
+                assert geocoder_config.geocodio_geocoder is True
+                assert geocoder_config.geocoding_quota == 100
             elif geocoder_provider == 'google':
                 assert geocoder_config.google_geocoder is True
                 assert geocoder_config.geocoding_quota is None
@@ -84,7 +87,7 @@ class TestGeocoderUserConfig(TestCase):
 
 class TestGeocoderOrgConfig(TestCase):
 
-    GEOCODER_PROVIDERS = ['heremaps', 'mapzen', 'mapbox', 'tomtom', 'google']
+    GEOCODER_PROVIDERS = ['heremaps', 'mapzen', 'mapbox', 'tomtom', 'geocodio', 'google']
 
     def setUp(self):
         self.redis_conn = MockRedis()
@@ -113,6 +116,9 @@ class TestGeocoderOrgConfig(TestCase):
             elif geocoder_provider == 'tomtom':
                 assert geocoder_config.tomtom_geocoder is True
                 assert geocoder_config.geocoding_quota == 200
+            elif geocoder_provider == 'geocodio':
+                assert geocoder_config.geocodio_geocoder is True
+                assert geocoder_config.geocoding_quota == 200
             elif geocoder_provider == 'google':
                 assert geocoder_config.google_geocoder is True
                 assert geocoder_config.geocoding_quota is None

server/lib/python/cartodb_services/test/test_geocodiogeocoder.py

@@ -0,0 +1,268 @@
+import unittest
+from mock import Mock
+from cartodb_services.geocodio import GeocodioGeocoder
+from cartodb_services.geocodio import GeocodioBulkGeocoder
+from cartodb_services.tools.exceptions import ServiceException
+from credentials import geocodio_api_key
+
+INVALID_TOKEN = 'invalid_token'
+
+VALID_ADDRESS_1 = 'Lexington Ave; New York; US'
+VALID_ADDRESS_2 = 'E 14th St; New York; US'
+VALID_ADDRESS_3 = '652 Lombard Street; San Francisco; California; United States'
+
+VALID_SEARCH_TEXT_1='Lexington Ave'
+VALID_CITY_1='New York'
+VALID_STATE_PROVINCE_1='New York'
+VALID_COUNTRY_1='US'
+
+VALID_SEARCH_TEXT_2='E 14th St'
+VALID_CITY_2='New York'
+VALID_STATE_PROVINCE_2='New York'
+VALID_COUNTRY_2='US'
+
+VALID_SEARCH_TEXT_3='652 Lombard Street'
+VALID_CITY_3='San Francisco'
+VALID_STATE_PROVINCE_3='California'
+VALID_COUNTRY_3='United States'
+
+WELL_KNOWN_LONGITUDE_1 = -73.960
+WELL_KNOWN_LATITUDE_1 = 40.774
+WELL_KNOWN_LONGITUDE_2 = -73.983
+WELL_KNOWN_LATITUDE_2 = 40.731
+WELL_KNOWN_LONGITUDE_3 = -122.412
+WELL_KNOWN_LATITUDE_3 = 37.803207
+
+SEARCH_ID_1 = 1
+SEARCH_ID_2 = 2
+
+
+class GeocodioGeocoderTestCase(unittest.TestCase):
+    def setUp(self):
+        self.geocoder = GeocodioGeocoder(token=geocodio_api_key(), logger=Mock())
+        self.bulk_geocoder = GeocodioBulkGeocoder(token=geocodio_api_key(), logger=Mock())
+
+    ### NON BULK
+
+    def test_invalid_token(self):
+        invalid_geocoder = GeocodioGeocoder(token=INVALID_TOKEN, logger=Mock())
+        with self.assertRaises(ServiceException):
+            invalid_geocoder.geocode(VALID_ADDRESS_1)
+
+    def test_valid_requests(self):
+        place = self.geocoder.geocode(VALID_ADDRESS_1)
+
+        self.assertEqual('%.3f' % place[0], '%.3f' % WELL_KNOWN_LONGITUDE_1)
+        self.assertEqual('%.3f' % place[1], '%.3f' % WELL_KNOWN_LATITUDE_1)
+
+        place = self.geocoder.geocode(VALID_ADDRESS_2)
+
+        self.assertEqual('%.3f' % place[0], '%.3f' % WELL_KNOWN_LONGITUDE_2)
+        self.assertEqual('%.3f' % place[1], '%.3f' % WELL_KNOWN_LATITUDE_2)
+
+        place = self.geocoder.geocode(VALID_ADDRESS_3)
+
+        self.assertEqual('%.3f' % place[0], '%.3f' % WELL_KNOWN_LONGITUDE_3)
+        self.assertEqual('%.3f' % place[1], '%.3f' % WELL_KNOWN_LATITUDE_3)
+
+    def test_valid_request_components(self):
+        place = self.geocoder.geocode(searchtext=VALID_SEARCH_TEXT_1,
+                                      city=VALID_CITY_1,
+                                      state_province=VALID_STATE_PROVINCE_1,
+                                      country=VALID_COUNTRY_1)
+
+        self.assertEqual('%.3f' % place[0], '%.3f' % WELL_KNOWN_LONGITUDE_1)
+        self.assertEqual('%.3f' % place[1], '%.3f' % WELL_KNOWN_LATITUDE_1)
+
+        place = self.geocoder.geocode(searchtext=VALID_SEARCH_TEXT_2,
+                                      city=VALID_CITY_2,
+                                      state_province=VALID_STATE_PROVINCE_2,
+                                      country=VALID_COUNTRY_2)
+
+        self.assertEqual('%.3f' % place[0], '%.3f' % WELL_KNOWN_LONGITUDE_2)
+        self.assertEqual('%.3f' % place[1], '%.3f' % WELL_KNOWN_LATITUDE_2)
+
+        place = self.geocoder.geocode(searchtext=VALID_SEARCH_TEXT_3,
+                                      city=VALID_CITY_3,
+                                      state_province=VALID_STATE_PROVINCE_3,
+                                      country=VALID_COUNTRY_3)
+
+        self.assertEqual('%.3f' % place[0], '%.3f' % WELL_KNOWN_LONGITUDE_3)
+        self.assertEqual('%.3f' % place[1], '%.3f' % WELL_KNOWN_LATITUDE_3)
+
+    def test_valid_request_namedplace(self):
+        place = self.geocoder.geocode(searchtext='New York')
+
+        assert place
+
+    def test_valid_request_namedplace2(self):
+        place = self.geocoder.geocode(searchtext='New York', country='us')
+
+        assert place
+
+    def test_odd_characters(self):
+        place = self.geocoder.geocode(searchtext='New York; "USA"')
+
+        assert place
+
+    def test_empty_request(self):
+        place = self.geocoder.geocode(searchtext='', country=None, city=None, state_province=None)
+
+        assert place == []
+
+    def test_empty_search_text_request(self):
+        place = self.geocoder.geocode(searchtext='     ', country='us', city=None, state_province="")
+
+        assert place == []
+
+    def test_unknown_place_request(self):
+        place = self.geocoder.geocode(searchtext='[unknown]', country='ch', state_province=None, city=None)
+
+        assert place == []
+
+    ### BULK ONE
+
+    def test_invalid_token_bulk_one(self):
+        invalid_geocoder = GeocodioBulkGeocoder(token=INVALID_TOKEN, logger=Mock())
+        with self.assertRaises(ServiceException):
+            invalid_geocoder._batch_geocode([(SEARCH_ID_1, VALID_ADDRESS_1, None, None, None)])
+
+    def test_valid_request_bulk_one(self):
+        place = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, VALID_ADDRESS_1, None, None, None)])
+
+        self.assertEqual(place[0][0], SEARCH_ID_1)
+        self.assertEqual('%.3f' % place[0][1], '%.3f' % WELL_KNOWN_LONGITUDE_1)
+        self.assertEqual('%.3f' % place[0][2], '%.3f' % WELL_KNOWN_LATITUDE_1)
+
+        place = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, VALID_ADDRESS_2, None, None, None)])
+
+        self.assertEqual(place[0][0], SEARCH_ID_1)
+        self.assertEqual('%.3f' % place[0][1], '%.3f' % WELL_KNOWN_LONGITUDE_2)
+        self.assertEqual('%.3f' % place[0][2], '%.3f' % WELL_KNOWN_LATITUDE_2)
+
+        place = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, VALID_ADDRESS_3, None, None, None)])
+
+        self.assertEqual(place[0][0], SEARCH_ID_1)
+        self.assertEqual('%.3f' % place[0][1], '%.3f' % WELL_KNOWN_LONGITUDE_3)
+        self.assertEqual('%.3f' % place[0][2], '%.3f' % WELL_KNOWN_LATITUDE_3)
+
+    def test_valid_request_components_bulk_one(self):
+        place = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, VALID_SEARCH_TEXT_1, VALID_CITY_1, VALID_STATE_PROVINCE_1, VALID_COUNTRY_1)])
+
+        self.assertEqual(place[0][0], SEARCH_ID_1)
+        self.assertEqual('%.3f' % place[0][1], '%.3f' % WELL_KNOWN_LONGITUDE_1)
+        self.assertEqual('%.3f' % place[0][2], '%.3f' % WELL_KNOWN_LATITUDE_1)
+
+        place = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, VALID_SEARCH_TEXT_2, VALID_CITY_2, VALID_STATE_PROVINCE_2, VALID_COUNTRY_2)])
+
+        self.assertEqual(place[0][0], SEARCH_ID_1)
+        self.assertEqual('%.3f' % place[0][1], '%.3f' % WELL_KNOWN_LONGITUDE_2)
+        self.assertEqual('%.3f' % place[0][2], '%.3f' % WELL_KNOWN_LATITUDE_2)
+
+        place = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, VALID_SEARCH_TEXT_3, VALID_CITY_3, VALID_STATE_PROVINCE_3, VALID_COUNTRY_3)])
+
+        self.assertEqual(place[0][0], SEARCH_ID_1)
+        self.assertEqual('%.3f' % place[0][1], '%.3f' % WELL_KNOWN_LONGITUDE_3)
+        self.assertEqual('%.3f' % place[0][2], '%.3f' % WELL_KNOWN_LATITUDE_3)
+
+    def test_valid_request_namedplace_bulk_one(self):
+        place = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, 'New York', None, None, None)])
+
+        assert place
+
+    def test_valid_request_namedplace2_bulk_one(self):
+        place = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, 'New York', 'us', None, None)])
+
+        assert place
+
+    def test_odd_characters_bulk_one(self):
+        place = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, 'New York; "USA"', None, None, None)])
+
+        assert place
+
+    def test_empty_request_bulk_one(self):
+        place = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, '', None, None, None)])
+
+        assert place == [(SEARCH_ID_1, None, None)]
+
+    def test_empty_search_text_request_bulk_one(self):
+        place = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, '     ', 'us', None, "")])
+
+        assert place == [(SEARCH_ID_1, None, None)]
+
+    def test_unknown_place_request_bulk_one(self):
+        place = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, '[unknown]', 'ch', None, None)])
+
+        assert place == [(SEARCH_ID_1, None, None)]
+
+    ### BULK MANY
+
+    def test_invalid_token_bulk_many(self):
+        invalid_geocoder = GeocodioBulkGeocoder(token=INVALID_TOKEN, logger=Mock())
+        with self.assertRaises(ServiceException):
+            invalid_geocoder._batch_geocode([(SEARCH_ID_1, VALID_ADDRESS_1, None, None, None),
+                                             (SEARCH_ID_2, VALID_ADDRESS_2, None, None, None)])
+
+    def test_valid_request_bulk_many(self):
+        places = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, VALID_ADDRESS_1, None, None, None),
+                                                   (SEARCH_ID_2, VALID_ADDRESS_2, None, None, None)])
+
+        self.assertEqual(places[0][0], SEARCH_ID_1)
+        self.assertEqual('%.3f' % places[0][1][0], '%.3f' % WELL_KNOWN_LONGITUDE_1)
+        self.assertEqual('%.3f' % places[0][1][1], '%.3f' % WELL_KNOWN_LATITUDE_1)
+
+        self.assertEqual(places[1][0], SEARCH_ID_2)
+        self.assertEqual('%.3f' % places[1][1][0], '%.3f' % WELL_KNOWN_LONGITUDE_2)
+        self.assertEqual('%.3f' % places[1][1][1], '%.3f' % WELL_KNOWN_LATITUDE_2)
+
+    def test_valid_request_components_bulk_many(self):
+        places = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, VALID_SEARCH_TEXT_1, VALID_CITY_1, VALID_STATE_PROVINCE_1, VALID_COUNTRY_1),
+                                                    (SEARCH_ID_2, VALID_SEARCH_TEXT_2, VALID_CITY_2, VALID_STATE_PROVINCE_2, VALID_COUNTRY_2)])
+
+        self.assertEqual(places[0][0], SEARCH_ID_1)
+        self.assertEqual(places[1][0], SEARCH_ID_2)
+
+    def test_valid_request_namedplace_bulk_many(self):
+        places = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, 'New York', None, None, None),
+                                                    (SEARCH_ID_2, 'Los Angeles', None, None, None)])
+
+        assert places
+
+        self.assertEqual(places[0][0], SEARCH_ID_1)
+        self.assertEqual(places[1][0], SEARCH_ID_2)
+
+    def test_valid_request_namedplace2_bulk_many(self):
+        places = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, 'New York', 'us', None, None),
+                                                    (SEARCH_ID_2, 'Los Angeles', None, None, None)])
+
+        assert places
+
+        self.assertEqual(places[0][0], SEARCH_ID_1)
+        self.assertEqual(places[1][0], SEARCH_ID_2)
+
+    def test_odd_characters_bulk_many(self):
+        places = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, 'New York; "USA"', None, None, None),
+                                                    (SEARCH_ID_2, 'Los Angeles', None, None, None)])
+
+        assert places
+
+        self.assertEqual(places[0][0], SEARCH_ID_1)
+        self.assertEqual(places[1][0], SEARCH_ID_2)
+
+    def test_empty_request_bulk_many(self):
+        places = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, '', None, None, None),
+                                                    (SEARCH_ID_2, '', None, None, None)])
+
+        assert places == [(SEARCH_ID_1, [], {}), (SEARCH_ID_2, [], {})]
+
+    def test_empty_search_text_request_bulk_many(self):
+        places = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, '     ', 'us', None, ""),
+                                                    (SEARCH_ID_2, '     ', 'us', None, "")])
+
+        assert places == [(SEARCH_ID_1, [], {}), (SEARCH_ID_2, [], {})]
+
+    def test_unknown_place_request_bulk_many(self):
+        places = self.bulk_geocoder._batch_geocode([(SEARCH_ID_1, '[unknown]', 'ch', None, None),
+                                                    (SEARCH_ID_2, '[unknown]', 'ch', None, None)])
+
+        assert places == [(SEARCH_ID_1, [], {}), (SEARCH_ID_2, [], {})]

server/lib/python/cartodb_services/test/test_helper.py

@@ -79,6 +79,7 @@ def plpy_mock_config():
     plpy_mock._define_result("CDB_Conf_GetConf\('mapbox_conf'\)", [{'conf': '{"routing": {"api_keys": ["api_key_rou"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["api_key_geo"], "monthly_quota": 1500000}, "matrix": {"api_keys": ["api_key_mat"], "monthly_quota": 1500000}}'}])
     plpy_mock._define_result("CDB_Conf_GetConf\('mapbox_iso_conf'\)", [{'conf': '{"isolines": {"api_keys": ["api_key_mat"], "monthly_quota": 1500000}}'}])
     plpy_mock._define_result("CDB_Conf_GetConf\('tomtom_conf'\)", [{'conf': '{"routing": {"api_keys": ["api_key_rou"], "monthly_quota": 1500000}, "geocoder": {"api_keys": ["api_key_geo"], "monthly_quota": 1500000}, "isolines": {"api_keys": ["api_key_mat"], "monthly_quota": 1500000}}'}])
+    plpy_mock._define_result("CDB_Conf_GetConf\('geocodio_conf'\)", [{'conf': '{"geocoder": {"api_keys": ["api_key_geo"], "monthly_quota": 1500000}}'}])
     plpy_mock._define_result("CDB_Conf_GetConf\('logger_conf'\)", [{'conf': '{"geocoder_log_path": "/dev/null"}'}])
     plpy_mock._define_result("CDB_Conf_GetConf\('data_observatory_conf'\)", [{'conf': '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}}'}])
     plpy_mock._define_result("CDB_Conf_GetConf\('server_conf'\)", [{'conf': '{"environment": "testing"}'}])