Merge pull request #375 from CartoDB/development

Release server 0.24.2 and python library 0.15.1

NEWS.md

@@ -1,3 +1,12 @@
+May 26th, 2017
+=============
+* Version `0.24.2` of the server
+    * Fixed fallback logic for namedplaces geocoding functions
+
+* Version `0.15.1` of the python library
+    * Fixed some typos and improve exception messages
+    * Added a check for the google client credentials in order to improve the error message
+
 May 16th, 2017
 =============
 * Version `0.24.1` of the server
@@ -159,7 +168,7 @@ July 25, 2016
 ===========
 * Release server 0.13.3
 * Add provider per service
-* Default provider in case the provider is not setted
+* Default provider in case the provider is not set
 * Refactor and improvements in the multiprovider services functions
 
 https://github.com/CartoDB/dataservices-api/releases/tag/0.13.3-server
@@ -475,7 +484,7 @@ https://github.com/CartoDB/dataservices-api/releases/tag/0.3.0-server
 Feb 4, 2016:
 ===========
 * Release server 0.2.0
-* Logic for the google geocoder so the users with this geocoder setted up can use street level geocoding too
+* Logic for the google geocoder so the users with this geocoder set up can use street level geocoding too
 * Refactor of the python library in order to reflect the change to a services extension more than only geocoder
 
 https://github.com/CartoDB/dataservices-api/releases/tag/0.2.0-server

doc/internal/exception_safe.md

@@ -18,7 +18,7 @@ These functions are useful in cases when it is undesirable to rollback a transac
 Fo example if a table is geocoded with:
 
 ```sql
-UPDATE table SET the_geom=cdb_geocode_street_point(user,NULL,address,city,NULL,country);
+UPDATE table SET the_geom=cdb_geocode_street_point(address,city,NULL,country);
 ```
 
 In case of the user geocoding quota being exhausted mid-process, the user could
@@ -28,7 +28,7 @@ transaction rollback.
 We can avoid the problem using the corresponding exception-safe function:
 
 ```sql
-UPDATE table SET the_geom=_cdb_geocode_street_point_exception_safe(user,NULL,address,city,NULL,country);
+UPDATE table SET the_geom=_cdb_geocode_street_point_exception_safe(address,city,NULL,country);
 ```
 
 # Addition Information

server/extension/cdb_dataservices_server--0.24.1--0.24.2.sql

@@ -0,0 +1,40 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.24.2'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+---- cdb_geocode_namedplace_point(city_name text)
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text)
+RETURNS Geometry AS $$
+  import spiexceptions
+  try:
+    mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"])
+    return plpy.execute(mapzen_plan, [username, orgname, city_name])[0]['point']
+  except spiexceptions.ExternalRoutineException as e:
+    internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"])
+    return plpy.execute(internal_plan, [username, orgname, city_name])[0]['point']
+$$ LANGUAGE plpythonu;
+
+---- cdb_geocode_namedplace_point(city_name text, country_name text)
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text)
+RETURNS Geometry AS $$
+  import spiexceptions
+  try:
+    mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"])
+    return plpy.execute(mapzen_plan, [username, orgname, city_name, country_name])[0]['point']
+  except spiexceptions.ExternalRoutineException as e:
+    internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"])
+    return plpy.execute(internal_plan, [username, orgname, city_name, country_name])[0]['point']
+$$ LANGUAGE plpythonu;
+
+---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text)
+RETURNS Geometry AS $$
+  import spiexceptions
+  try:
+    mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"])
+    return plpy.execute(mapzen_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point']
+  except spiexceptions.ExternalRoutineException as e:
+    internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"])
+    return plpy.execute(internal_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point']
+$$ LANGUAGE plpythonu;

server/extension/cdb_dataservices_server--0.24.2--0.24.1.sql

@@ -0,0 +1,36 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.24.1'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text)
+RETURNS Geometry AS $$
+  try:
+    mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"])
+    return plpy.execute(mapzen_plan, [username, orgname, city_name])[0]['point']
+  except BaseException as e:
+    internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"])
+    return plpy.execute(internal_plan, [username, orgname, city_name])[0]['point']
+$$ LANGUAGE plpythonu;
+
+---- cdb_geocode_namedplace_point(city_name text, country_name text)
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text)
+RETURNS Geometry AS $$
+  try:
+    mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"])
+    return plpy.execute(mapzen_plan, [username, orgname, city_name, country_name])[0]['point']
+  except BaseException as e:
+    internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"])
+    return plpy.execute(internal_plan, [username, orgname, city_name, country_name])[0]['point']
+$$ LANGUAGE plpythonu;
+
+---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text)
+RETURNS Geometry AS $$
+  try:
+    mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"])
+    return plpy.execute(mapzen_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point']
+  except BaseException as e:
+    internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"])
+    return plpy.execute(internal_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point']
+$$ LANGUAGE plpythonu;

server/extension/cdb_dataservices_server.control

@@ -1,5 +1,5 @@
 comment = 'CartoDB dataservices server extension'
-default_version = '0.24.1'
+default_version = '0.24.2'
 requires = 'plpythonu, plproxy, postgis, cdb_geocoder'
 superuser = true
 schema = cdb_dataservices_server

server/extension/sql/50_namedplaces.sql

@@ -1,10 +1,11 @@
 ---- cdb_geocode_namedplace_point(city_name text)
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text)
 RETURNS Geometry AS $$
+  import spiexceptions
   try:
     mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"])
     return plpy.execute(mapzen_plan, [username, orgname, city_name])[0]['point']
-  except BaseException as e:
+  except spiexceptions.ExternalRoutineException as e:
     internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"])
     return plpy.execute(internal_plan, [username, orgname, city_name])[0]['point']
 $$ LANGUAGE plpythonu;
@@ -12,10 +13,11 @@ $$ LANGUAGE plpythonu;
 ---- cdb_geocode_namedplace_point(city_name text, country_name text)
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text)
 RETURNS Geometry AS $$
+  import spiexceptions
   try:
     mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"])
     return plpy.execute(mapzen_plan, [username, orgname, city_name, country_name])[0]['point']
-  except BaseException as e:
+  except spiexceptions.ExternalRoutineException as e:
     internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"])
     return plpy.execute(internal_plan, [username, orgname, city_name, country_name])[0]['point']
 $$ LANGUAGE plpythonu;
@@ -23,10 +25,11 @@ $$ LANGUAGE plpythonu;
 ---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text)
 RETURNS Geometry AS $$
+  import spiexceptions
   try:
     mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"])
     return plpy.execute(mapzen_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point']
-  except BaseException as e:
+  except spiexceptions.ExternalRoutineException as e:
     internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"])
     return plpy.execute(internal_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point']
 $$ LANGUAGE plpythonu;

server/lib/python/cartodb_services/cartodb_services/google/exceptions.py

@@ -2,6 +2,8 @@
 # -*- coding: utf-8 -*-
 import json
 
+class InvalidGoogleCredentials(Exception):
+    pass
 
 class BadGeocodingParams(Exception):
     def __init__(self, value):

server/lib/python/cartodb_services/cartodb_services/google/geocoder.py

@@ -1,15 +1,18 @@
 #!/usr/local/bin/python
 # -*- coding: utf-8 -*-
 
+import base64
 import googlemaps
 
-from exceptions import MalformedResult
+from exceptions import MalformedResult, InvalidGoogleCredentials
 
 
 class GoogleMapsGeocoder:
     """A Google Maps Geocoder wrapper for python"""
 
     def __init__(self, client_id, client_secret, logger):
+        if not self._valid_credentials(client_secret):
+            raise InvalidGoogleCredentials('Invalid google secret key')
         self.client_id = self._clean_client_id(client_id)
         self.client_secret = client_secret
         self.geocoder = googlemaps.Client(
@@ -36,7 +39,7 @@ class GoogleMapsGeocoder:
         return [longitude, latitude]
 
     def _build_optional_parameters(self, city=None, state=None,
-                       country=None):
+                                   country=None):
         optional_params = {}
         if city:
             optional_params['locality'] = city
@@ -49,3 +52,13 @@ class GoogleMapsGeocoder:
     def _clean_client_id(self, client_id):
         # Consistency with how the client_id is saved in metadata
         return client_id.replace('client=', '')
+
+    def _valid_credentials(self, private_key):
+        try:
+            # Only fails if the string dont have a correct padding for b64
+            # but this way we could provide a more clear error than
+            # TypeError: Incorrect padding
+            base64.b64decode(private_key)
+            return True
+        except TypeError:
+            return False

server/lib/python/cartodb_services/cartodb_services/metrics/config.py

@@ -359,13 +359,13 @@ class GeocoderConfig(ServiceConfig):
         if self._geocoder_provider == self.NOKIA_GEOCODER:
             if not set(self.NOKIA_GEOCODER_REDIS_MANDATORY_KEYS).issubset(set(filtered_config.keys())) or \
             not self.heremaps_app_id or not self.heremaps_app_code:
-                raise ConfigException("""Some mandatory parameter/s for Nokia geocoder are missing. Check it please""")
+                raise ConfigException("""Heremaps app id or app code is not set up""")
         elif self._geocoder_provider == self.GOOGLE_GEOCODER:
             if self.GOOGLE_GEOCODER_API_KEY not in filtered_config.keys():
-                raise ConfigException("""Google geocoder need the mandatory parameter 'google_maps_private_key'""")
+                raise ConfigException("""Google geocoder private key is not set up""")
         elif self._geocoder_provider == self.MAPZEN_GEOCODER:
             if not self.mapzen_api_key:
-                raise ConfigException("""Mapzen config is not setted up""")
+                raise ConfigException("""Mapzen config is not set up""")
 
         return True
 

server/lib/python/cartodb_services/setup.py

@@ -10,7 +10,7 @@ from setuptools import setup, find_packages
 setup(
     name='cartodb_services',
 
-    version='0.15.0',
+    version='0.15.1',
 
     description='CartoDB Services API Python Library',
 

server/lib/python/cartodb_services/test/test_googlegeocoder.py

@@ -6,9 +6,7 @@ import requests_mock
 from mock import Mock
 
 from cartodb_services.google import GoogleMapsGeocoder
-from cartodb_services.google.exceptions import BadGeocodingParams
+from cartodb_services.google.exceptions import MalformedResult, InvalidGoogleCredentials
-from cartodb_services.google.exceptions import NoGeocodingParams
-from cartodb_services.google.exceptions import MalformedResult
 
 requests_mock.Mocker.TEST_PREFIX = 'test_'
 
@@ -92,7 +90,8 @@ class GoogleGeocoderTestCase(unittest.TestCase):
     def setUp(self):
         logger = Mock()
         self.geocoder = GoogleMapsGeocoder('dummy_client_id',
-                                 'MgxyOFxjZXIyOGO52jJlMzEzY1Oqy4hsO49E', logger)
+                                           'MgxyOFxjZXIyOGO52jJlMzEzY1Oqy4hsO49E',
+                                           logger)
 
     def test_geocode_address_with_valid_params(self, req_mock):
         req_mock.register_uri('GET', self.GOOGLE_MAPS_GEOCODER_URL,
@@ -119,3 +118,9 @@ class GoogleGeocoderTestCase(unittest.TestCase):
                 searchtext='Calle Eloy Gonzalo 27',
                 city='Madrid',
                 country='España')
+
+    def test_invalid_credentials(self, req_mock):
+        with self.assertRaises(InvalidGoogleCredentials):
+            GoogleMapsGeocoder('dummy_client_id',
+                               'lalala',
+                               None)