Relevance metadata for HERE

server/lib/python/cartodb_services/cartodb_services/geocoder.py

@@ -22,12 +22,18 @@ def run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, org
         if geocode_results:
             results = []
             for result in geocode_results:
+                if len(result) > 2:
+                    metadata = json.dumps(result[2])
+                else:
+                    logger.warning('Geocoding for {} without metadata'.format(username))
+                    metadata = '{}'
+
                 if result[1] and len(result[1]) == 2:
                     plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326) as the_geom; ", ["double precision", "double precision"])
                     point = plpy.execute(plan, result[1], 1)[0]
-                    results.append([result[0], point['the_geom'], None])
+                    results.append([result[0], point['the_geom'], metadata])
                 else:
-                    results.append([result[0], None, None])
+                    results.append([result[0], None, metadata])
             service_manager.quota_service.increment_success_service_use(len(results))
             return results
         else:

server/lib/python/cartodb_services/cartodb_services/here/bulk_geocoder.py

@@ -16,7 +16,7 @@ HereJobStatus = namedtuple('HereJobStatus', 'total_count processed_count status'
 
 class HereMapsBulkGeocoder(HereMapsGeocoder, StreetPointBulkGeocoder):
     MAX_BATCH_SIZE = 1000000  # From the docs
-    MIN_BATCHED_SEARCH = 1000  # Under this, serial will be used
+    MIN_BATCHED_SEARCH = 100  # Under this, serial will be used
     BATCH_URL = 'https://batch.geocoder.cit.api.here.com/6.2/jobs'
     # https://developer.here.com/documentation/batch-geocoder/topics/read-batch-request-output.html
     META_COLS = ['relevance', 'matchType', 'matchCode', 'matchLevel', 'matchQualityStreet']
@@ -41,8 +41,8 @@ class HereMapsBulkGeocoder(HereMapsGeocoder, StreetPointBulkGeocoder):
         results = []
         for search in searches:
             (search_id, address, city, state, country) = search
-            coordinates = self.geocode(searchtext=address, city=city, state=state, country=country)
-            results.append((search_id, coordinates, []))
+            result = self.geocode_meta(searchtext=address, city=city, state=state, country=country)
+            results.append((search_id, result[0], result[1]))
         return results
 
     def _batch_geocode(self, searches):
@@ -140,7 +140,11 @@ class HereMapsBulkGeocoder(HereMapsGeocoder, StreetPointBulkGeocoder):
                 reader = csv.DictReader(root_zip.open(name), delimiter='|')
                 for row in reader:
                     if row['SeqNumber'] == '1':  # First per requested data
-                        results.append((row['recId'], [row['displayLongitude'], row['displayLatitude']]))
+                        results.append((row['recId'],
+                                        [row['displayLongitude'], row['displayLatitude']],
+                                        {
+                                            'relevance': float(row['relevance'])
+                                        }))
 
         return results
 

server/lib/python/cartodb_services/cartodb_services/here/geocoder.py

@@ -65,12 +65,15 @@ class HereMapsGeocoder(Traceable):
         self.max_retries = service_params.get('max_retries', self.MAX_RETRIES)
 
     def geocode(self, **kwargs):
+        return self.geocode_meta(**kwargs)[0]
+
+    def geocode_meta(self, **kwargs):
         params = {}
         for key, value in kwargs.iteritems():
             if value and value.strip():
                 params[key] = value
         if not params:
-            return []
+            return [[], {}]
         return self._execute_geocode(params)
 
     def _execute_geocode(self, params):
@@ -78,10 +81,11 @@ class HereMapsGeocoder(Traceable):
             raise BadGeocodingParams(params)
         try:
             response = self._perform_request(params)
-            results = response['Response']['View'][0]['Result'][0]
-            return self._extract_lng_lat_from_result(results)
+            result = response['Response']['View'][0]['Result'][0]
+            return [self._extract_lng_lat_from_result(result),
+                    self._extract_metadata_from_result(result)]
         except IndexError:
-            return []
+            return [[], {}]
         except KeyError:
             raise MalformedResult()
 
@@ -118,3 +122,8 @@ class HereMapsGeocoder(Traceable):
         latitude = location['DisplayPosition']['Latitude']
 
         return [longitude, latitude]
+
+    def _extract_metadata_from_result(self, result):
+        return {
+            'relevance': result['Relevance']
+        }

test/integration/test_street_functions.py

@@ -2,10 +2,9 @@
 # -*- coding: utf-8 -*-
 
 from unittest import TestCase
-from nose.tools import assert_raises
-from nose.tools import assert_not_equal, assert_equal
+from nose.tools import assert_not_equal, assert_equal, assert_true
 from ..helpers.integration_test_helper import IntegrationTestHelper
-from ..helpers.integration_test_helper import assert_close_enough
+from ..helpers.integration_test_helper import assert_close_enough, isclose
 
 class TestStreetFunctionsSetUp(TestCase):
     provider = None
@@ -321,6 +320,16 @@ class TestBulkStreetFunctions(TestStreetFunctionsSetUp):
         assert_equal(len(response['rows']), count)
         assert_not_equal(response['rows'][0]['st_x'], None)
 
+    def test_relevance(self):
+        query = "select metadata " \
+                "FROM cdb_dataservices_client.cdb_bulk_geocode_street_point(" \
+                "'select 1 as cartodb_id, ''Spain'' as country, " \
+                "''Barcelona'' as city, " \
+                "''Plaza España 1'' as street' " \
+                ", 'street', 'city', NULL, 'country')"
+        response = self._run_authenticated(query)
+
+        assert_true(isclose(response['rows'][0]['metadata']['relevance'], 1))
 
     def _run_authenticated(self, query):
         authenticated_query = "{}&api_key={}".format(query,