Adapt dataservices-api server to python3

Related to https://github.com/CartoDB/cartodb-platform/issues/6237

server/extension/sql/50_namedplaces.sql

@@ -1,7 +1,7 @@
 ---- cdb_geocode_namedplace_point(city_name text)
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text)
 RETURNS Geometry AS $$
-  import spiexceptions
+  from plpy import spiexceptions
   from cartodb_services.tools import Logger,LoggerConfig
 
   plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
@@ -21,7 +21,7 @@ $$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 ---- cdb_geocode_namedplace_point(city_name text, country_name text)
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text)
 RETURNS Geometry AS $$
-  import spiexceptions
+  from plpy import spiexceptions
   from cartodb_services.tools import Logger,LoggerConfig
 
   plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
@@ -41,7 +41,7 @@ $$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
 ---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text)
 RETURNS Geometry AS $$
-  import spiexceptions
+  from plpy import spiexceptions
   from cartodb_services.tools import Logger,LoggerConfig
 
   plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")

server/lib/python/cartodb_services/cartodb_services/__init__.py

@@ -34,4 +34,4 @@ def _reset():
     plpy = None
     GD = None
 
-from geocoder import run_street_point_geocoder, StreetPointBulkGeocoder
+from cartodb_services.geocoder import run_street_point_geocoder, StreetPointBulkGeocoder

server/lib/python/cartodb_services/cartodb_services/bulk_geocoders.py

@@ -1,8 +1,8 @@
-from google import GoogleMapsBulkGeocoder
+from cartodb_services.google import GoogleMapsBulkGeocoder
-from here import HereMapsBulkGeocoder
+from cartodb_services.here import HereMapsBulkGeocoder
-from tomtom import TomTomBulkGeocoder
+from cartodb_services.tomtom import TomTomBulkGeocoder
-from mapbox import MapboxBulkGeocoder
+from cartodb_services.mapbox import MapboxBulkGeocoder
-from geocodio import GeocodioBulkGeocoder
+from cartodb_services.geocodio import GeocodioBulkGeocoder
 
 BATCH_GEOCODER_CLASS_BY_PROVIDER = {
     'google': GoogleMapsBulkGeocoder,

server/lib/python/cartodb_services/cartodb_services/config/__init__.py

@@ -1,3 +1,3 @@
-from service_configuration import ServiceConfiguration
+from .service_configuration import ServiceConfiguration
-from rate_limits import RateLimitsConfig, RateLimitsConfigBuilder, RateLimitsConfigSetter
+from .rate_limits import RateLimitsConfig, RateLimitsConfigBuilder, RateLimitsConfigSetter
-from legacy_rate_limits import RateLimitsConfigLegacyBuilder
+from .legacy_rate_limits import RateLimitsConfigLegacyBuilder

server/lib/python/cartodb_services/cartodb_services/config/legacy_rate_limits.py

@@ -1,5 +1,5 @@
 import json
-from rate_limits import RateLimitsConfig
+from cartodb_services.config.rate_limits import RateLimitsConfig
 
 class RateLimitsConfigLegacyBuilder(object):
     """

server/lib/python/cartodb_services/cartodb_services/config/rate_limits.py

@@ -1,6 +1,6 @@
 import json
 
-from service_configuration import ServiceConfiguration
+from cartodb_services.config.service_configuration import ServiceConfiguration
 
 class RateLimitsConfig(object):
     """

server/lib/python/cartodb_services/cartodb_services/geocoder.py

@@ -1,7 +1,7 @@
 #!/usr/local/bin/python
 # -*- coding: utf-8 -*-
 
-from tools import QuotaExceededException, Logger
+from cartodb_services.tools import QuotaExceededException, Logger
 from collections import namedtuple
 import json
 
@@ -71,7 +71,7 @@ def run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, org
                 except Exception as e:
                     import sys
                     logger.error("Error processing geocode", sys.exc_info(), data={"username": username, "orgname": orgname})
-                    metadata['processing_error'] = 'Error: {}'.format(e.message)
+                    metadata['processing_error'] = 'Error: {}'.format(e)
                     results.append([result[0], None, json.dumps(metadata)])
                     failed_count += 1
 

server/lib/python/cartodb_services/cartodb_services/geocodio/__init__.py

@@ -1,2 +1,2 @@
-from geocoder import GeocodioGeocoder
+from cartodb_services.geocodio.geocoder import GeocodioGeocoder
-from bulk_geocoder import GeocodioBulkGeocoder
+from cartodb_services.geocodio.bulk_geocoder import GeocodioBulkGeocoder

server/lib/python/cartodb_services/cartodb_services/google/__init__.py

@@ -1,2 +1,2 @@
-from geocoder import GoogleMapsGeocoder
+from cartodb_services.google.geocoder import GoogleMapsGeocoder
-from bulk_geocoder import GoogleMapsBulkGeocoder
+from cartodb_services.google.bulk_geocoder import GoogleMapsBulkGeocoder

server/lib/python/cartodb_services/cartodb_services/google/bulk_geocoder.py

@@ -1,5 +1,5 @@
 from multiprocessing import Pool
-from exceptions import MalformedResult
+from cartodb_services.google.exceptions import MalformedResult
 from cartodb_services import StreetPointBulkGeocoder
 from cartodb_services.geocoder import compose_address, geocoder_error_response
 from cartodb_services.google import GoogleMapsGeocoder

server/lib/python/cartodb_services/cartodb_services/google/client_factory.py

@@ -3,7 +3,7 @@
 
 import googlemaps
 import base64
-from exceptions import InvalidGoogleCredentials
+from cartodb_services.google.exceptions import InvalidGoogleCredentials
 
 
 class GoogleMapsClientFactory():

server/lib/python/cartodb_services/cartodb_services/google/geocoder.py

@@ -1,9 +1,11 @@
 #!/usr/local/bin/python
 # -*- coding: utf-8 -*-
-
+try:
     from urlparse import parse_qs
+except:
+    from urllib.parse import parse_qs
 
-from exceptions import MalformedResult
+from cartodb_services.google.exceptions import MalformedResult
 from cartodb_services.geocoder import compose_address, geocoder_metadata, PRECISION_PRECISE, PRECISION_INTERPOLATED, EMPTY_RESPONSE
 from cartodb_services.google.exceptions import InvalidGoogleCredentials
 from client_factory import GoogleMapsClientFactory

server/lib/python/cartodb_services/cartodb_services/here/__init__.py

@@ -1,3 +1,3 @@
-from geocoder import HereMapsGeocoder
+from cartodb_services.here.geocoder import HereMapsGeocoder
-from bulk_geocoder import HereMapsBulkGeocoder
+from cartodb_services.here.bulk_geocoder import HereMapsBulkGeocoder
-from routing import HereMapsRoutingIsoline
+from cartodb_services.here.routing import HereMapsRoutingIsoline

server/lib/python/cartodb_services/cartodb_services/here/routing.py

@@ -1,7 +1,7 @@
 import requests
 import json
 
-from exceptions import WrongParams
+from cartodb_services.here.exceptions import WrongParams
 from requests.adapters import HTTPAdapter
 from cartodb_services.metrics import Traceable
 

server/lib/python/cartodb_services/cartodb_services/mapbox/__init__.py

@@ -1,4 +1,4 @@
-from routing import MapboxRouting, MapboxRoutingResponse
+from cartodb_services.mapbox.routing import MapboxRouting, MapboxRoutingResponse
-from geocoder import MapboxGeocoder
+from cartodb_services.mapbox.geocoder import MapboxGeocoder
-from bulk_geocoder import MapboxBulkGeocoder
+from cartodb_services.mapbox.bulk_geocoder import MapboxBulkGeocoder
-from isolines import MapboxIsolines, MapboxIsochronesResponse
+from cartodb_services.mapbox.isolines import MapboxIsolines, MapboxIsochronesResponse

server/lib/python/cartodb_services/cartodb_services/mapzen/__init__.py

@@ -1,5 +1,5 @@
-from routing import MapzenRouting, MapzenRoutingResponse
+from cartodb_services.mapzen.routing import MapzenRouting, MapzenRoutingResponse
-from isolines import MapzenIsolines
+from cartodb_services.mapzen.isolines import MapzenIsolines
-from geocoder import MapzenGeocoder
+from cartodb_services.mapzen.geocoder import MapzenGeocoder
-from matrix_client import MatrixClient
+from cartodb_services.mapzen.matrix_client import MatrixClient
-from isochrones import MapzenIsochrones
+from cartodb_services.mapzen.isochrones import MapzenIsochrones

server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py

@@ -135,7 +135,7 @@ class MapzenIsolines:
         # delete points that got None
         location_estimates_filtered = []
         for i, c in enumerate(costs):
-            if c <> isorange:
+            if c != isorange:
                 location_estimates_filtered.append(location_estimates[i])
 
         return location_estimates_filtered

server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py

@@ -1,4 +1,4 @@
-from config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatoryConfig
+from cartodb_services.metrics.config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatoryConfig
-from quota import QuotaService
+from cartodb_services.metrics.quota import QuotaService
-from user import UserMetricsService
+from cartodb_services.metrics.user import UserMetricsService
-from log import metrics, MetricsDataGatherer, Traceable
+from cartodb_services.metrics.log import metrics, MetricsDataGatherer, Traceable

server/lib/python/cartodb_services/cartodb_services/metrics/config.py

@@ -45,7 +45,7 @@ class ServiceConfig(object):
 
     def _get_effective_monthly_quota(self, quota_key, default=0):
         quota_from_redis = self._redis_config.get(quota_key, None)
-        if quota_from_redis and quota_from_redis <> '':
+        if quota_from_redis and quota_from_redis != '':
             return int(quota_from_redis)
         else:
             return default

server/lib/python/cartodb_services/cartodb_services/metrics/log.py

@@ -6,7 +6,10 @@ import uuid
 import plpy
 from datetime import datetime
 from contextlib import contextmanager
+try:
     from urlparse import urlparse
+except:
+    from urllib.parse import urlparse
 
 
 @contextmanager

server/lib/python/cartodb_services/cartodb_services/metrics/quota.py

@@ -1,5 +1,5 @@
-from user import UserMetricsService
+from cartodb_services.metrics.user import UserMetricsService
-from log import MetricsDataGatherer
+from cartodb_services.metrics.log import MetricsDataGatherer
 from datetime import date
 import re
 

server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py

@@ -1,5 +1,5 @@
 from ..core.interfaces import ConfigBackendInterface
-from null_config import NullConfigStorage
+from cartodb_services.refactor.storage.null_config import NullConfigStorage
 
 
 class RedisConfigStorage(ConfigBackendInterface):

server/lib/python/cartodb_services/cartodb_services/tomtom/__init__.py

@@ -1,4 +1,4 @@
-from geocoder import TomTomGeocoder
+from cartodb_services.tomtom.geocoder import TomTomGeocoder
-from bulk_geocoder import TomTomBulkGeocoder
+from cartodb_services.tomtom.bulk_geocoder import TomTomBulkGeocoder
-from routing import TomTomRouting, TomTomRoutingResponse
+from cartodb_services.tomtom.routing import TomTomRouting, TomTomRoutingResponse
-from isolines import TomTomIsolines, TomTomIsochronesResponse
+from cartodb_services.tomtom.isolines import TomTomIsolines, TomTomIsochronesResponse

server/lib/python/cartodb_services/cartodb_services/tools/__init__.py

@@ -1,9 +1,9 @@
-from redis_tools import RedisConnection, RedisDBConfig
+from .redis_tools import RedisConnection, RedisDBConfig
-from coordinates import Coordinate
+from .coordinates import Coordinate
-from polyline import PolyLine
+from .polyline import PolyLine
-from log import Logger, LoggerConfig
+from .log import Logger, LoggerConfig
-from rate_limiter import RateLimiter
+from .rate_limiter import RateLimiter
-from service_manager import ServiceManager
+from .service_manager import ServiceManager
-from legacy_service_manager import LegacyServiceManager
+from .legacy_service_manager import LegacyServiceManager
-from exceptions import QuotaExceededException, RateLimitExceeded
+from .exceptions import QuotaExceededException, RateLimitExceeded
-from country import country_to_iso3
+from .country import country_to_iso3

server/lib/python/cartodb_services/cartodb_services/tools/qps.py

@@ -1,7 +1,7 @@
 import time
 import random
 from datetime import datetime
-from exceptions import TimeoutException
+from cartodb_services.tools.exceptions import TimeoutException
 import re
 
 DEFAULT_RETRY_TIMEOUT = 60

test/helpers/import_helper.py

@@ -20,7 +20,7 @@ class ImportHelper:
         response.raise_for_status()
         response_json = json.loads(response.text)
         if not response_json['success']:
-            print "Error importing the test dataset: {0}".format(response.text)
+            print("Error importing the test dataset: {0}".format(response.text))
             sys.exit(1)
         while(True):
             table_name = ImportHelper.get_imported_table_name(
@@ -42,8 +42,8 @@ class ImportHelper:
             schema, username, host, import_id, api_key)
         import_data_response = requests.get(import_url)
         if import_data_response.status_code != 200:
-            print "Error getting the table name from " \
+            print("Error getting the table name from " \
-                "the import data: {0}".format(import_data_response.text)
+                "the import data: {0}".format(import_data_response.text))
             sys.exit(1)
         import_data_json = json.loads(import_data_response.text)
 
@@ -57,5 +57,5 @@ class ImportHelper:
         )
         response = requests.get(url)
         if response.status_code != 200:
-            print "Error cleaning the test dataset: {0}".format(response.text)
+            print("Error cleaning the test dataset: {0}".format(response.text))
             sys.exit(1)

test/helpers/integration_test_helper.py

@@ -48,7 +48,7 @@ class IntegrationTestHelper:
     def execute_query_raw(cls, sql_api_url, query):
         requests.packages.urllib3.disable_warnings()
         query_url = "{0}?q={1}".format(sql_api_url, query)
-        print "Executing query: {0}".format(query_url)
+        print("Executing query: {0}".format(query_url))
         query_response = requests.get(query_url)
         if query_response.status_code != 200:
             raise Exception(json.loads(query_response.text)['error'])

test/run_tests.py

@@ -36,7 +36,7 @@ def main():
         set_environment_variables(username, api_key, table_name, host, schema)
         execute_tests()
     except Exception as e:
-        print e.message
+        print(e)
         sys.exit(1)
     finally:
         clean_environment_variables()
@@ -45,11 +45,11 @@ def main():
 
 
 def usage():
-    print """Usage: run_tests.py [options] username api_key
+    print("""Usage: run_tests.py [options] username api_key
         Options:
         -h: Show this help
         --host: take that host as base (by default is cartodb.com)
-        --schema: define the url schema [http/https] (by default https)"""
+        --schema: define the url schema [http/https] (by default https)""")
 
 
 def execute_tests():
@@ -59,7 +59,7 @@ def execute_tests():
         stderr=subprocess.PIPE
     )
     out, err = process.communicate()
-    print err
+    print(err)
     regexp = re.compile(r'FAILED \(.*\)')
     if regexp.search(err) is not None:
         sys.exit(1)