Adapt dataservices-api server to python3
Related to https://github.com/CartoDB/cartodb-platform/issues/6237
This commit is contained in:
parent
b2a06c6ed6
commit
3c916bdfc9
@ -1,7 +1,7 @@
|
||||
---- cdb_geocode_namedplace_point(city_name text)
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text)
|
||||
RETURNS Geometry AS $$
|
||||
import spiexceptions
|
||||
from plpy import spiexceptions
|
||||
from cartodb_services.tools import Logger,LoggerConfig
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
|
||||
@ -21,7 +21,7 @@ $$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
|
||||
---- cdb_geocode_namedplace_point(city_name text, country_name text)
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text)
|
||||
RETURNS Geometry AS $$
|
||||
import spiexceptions
|
||||
from plpy import spiexceptions
|
||||
from cartodb_services.tools import Logger,LoggerConfig
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
|
||||
@ -41,7 +41,7 @@ $$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
|
||||
---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text)
|
||||
RETURNS Geometry AS $$
|
||||
import spiexceptions
|
||||
from plpy import spiexceptions
|
||||
from cartodb_services.tools import Logger,LoggerConfig
|
||||
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
|
||||
|
@ -34,4 +34,4 @@ def _reset():
|
||||
plpy = None
|
||||
GD = None
|
||||
|
||||
from geocoder import run_street_point_geocoder, StreetPointBulkGeocoder
|
||||
from cartodb_services.geocoder import run_street_point_geocoder, StreetPointBulkGeocoder
|
||||
|
@ -1,8 +1,8 @@
|
||||
from google import GoogleMapsBulkGeocoder
|
||||
from here import HereMapsBulkGeocoder
|
||||
from tomtom import TomTomBulkGeocoder
|
||||
from mapbox import MapboxBulkGeocoder
|
||||
from geocodio import GeocodioBulkGeocoder
|
||||
from cartodb_services.google import GoogleMapsBulkGeocoder
|
||||
from cartodb_services.here import HereMapsBulkGeocoder
|
||||
from cartodb_services.tomtom import TomTomBulkGeocoder
|
||||
from cartodb_services.mapbox import MapboxBulkGeocoder
|
||||
from cartodb_services.geocodio import GeocodioBulkGeocoder
|
||||
|
||||
BATCH_GEOCODER_CLASS_BY_PROVIDER = {
|
||||
'google': GoogleMapsBulkGeocoder,
|
||||
|
@ -1,3 +1,3 @@
|
||||
from service_configuration import ServiceConfiguration
|
||||
from rate_limits import RateLimitsConfig, RateLimitsConfigBuilder, RateLimitsConfigSetter
|
||||
from legacy_rate_limits import RateLimitsConfigLegacyBuilder
|
||||
from .service_configuration import ServiceConfiguration
|
||||
from .rate_limits import RateLimitsConfig, RateLimitsConfigBuilder, RateLimitsConfigSetter
|
||||
from .legacy_rate_limits import RateLimitsConfigLegacyBuilder
|
||||
|
@ -1,5 +1,5 @@
|
||||
import json
|
||||
from rate_limits import RateLimitsConfig
|
||||
from cartodb_services.config.rate_limits import RateLimitsConfig
|
||||
|
||||
class RateLimitsConfigLegacyBuilder(object):
|
||||
"""
|
||||
|
@ -1,6 +1,6 @@
|
||||
import json
|
||||
|
||||
from service_configuration import ServiceConfiguration
|
||||
from cartodb_services.config.service_configuration import ServiceConfiguration
|
||||
|
||||
class RateLimitsConfig(object):
|
||||
"""
|
||||
|
@ -1,7 +1,7 @@
|
||||
#!/usr/local/bin/python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from tools import QuotaExceededException, Logger
|
||||
from cartodb_services.tools import QuotaExceededException, Logger
|
||||
from collections import namedtuple
|
||||
import json
|
||||
|
||||
@ -71,7 +71,7 @@ def run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, org
|
||||
except Exception as e:
|
||||
import sys
|
||||
logger.error("Error processing geocode", sys.exc_info(), data={"username": username, "orgname": orgname})
|
||||
metadata['processing_error'] = 'Error: {}'.format(e.message)
|
||||
metadata['processing_error'] = 'Error: {}'.format(e)
|
||||
results.append([result[0], None, json.dumps(metadata)])
|
||||
failed_count += 1
|
||||
|
||||
|
@ -1,2 +1,2 @@
|
||||
from geocoder import GeocodioGeocoder
|
||||
from bulk_geocoder import GeocodioBulkGeocoder
|
||||
from cartodb_services.geocodio.geocoder import GeocodioGeocoder
|
||||
from cartodb_services.geocodio.bulk_geocoder import GeocodioBulkGeocoder
|
||||
|
@ -1,2 +1,2 @@
|
||||
from geocoder import GoogleMapsGeocoder
|
||||
from bulk_geocoder import GoogleMapsBulkGeocoder
|
||||
from cartodb_services.google.geocoder import GoogleMapsGeocoder
|
||||
from cartodb_services.google.bulk_geocoder import GoogleMapsBulkGeocoder
|
||||
|
@ -1,5 +1,5 @@
|
||||
from multiprocessing import Pool
|
||||
from exceptions import MalformedResult
|
||||
from cartodb_services.google.exceptions import MalformedResult
|
||||
from cartodb_services import StreetPointBulkGeocoder
|
||||
from cartodb_services.geocoder import compose_address, geocoder_error_response
|
||||
from cartodb_services.google import GoogleMapsGeocoder
|
||||
|
@ -3,7 +3,7 @@
|
||||
|
||||
import googlemaps
|
||||
import base64
|
||||
from exceptions import InvalidGoogleCredentials
|
||||
from cartodb_services.google.exceptions import InvalidGoogleCredentials
|
||||
|
||||
|
||||
class GoogleMapsClientFactory():
|
||||
|
@ -1,9 +1,11 @@
|
||||
#!/usr/local/bin/python
|
||||
# -*- coding: utf-8 -*-
|
||||
try:
|
||||
from urlparse import parse_qs
|
||||
except:
|
||||
from urllib.parse import parse_qs
|
||||
|
||||
from urlparse import parse_qs
|
||||
|
||||
from exceptions import MalformedResult
|
||||
from cartodb_services.google.exceptions import MalformedResult
|
||||
from cartodb_services.geocoder import compose_address, geocoder_metadata, PRECISION_PRECISE, PRECISION_INTERPOLATED, EMPTY_RESPONSE
|
||||
from cartodb_services.google.exceptions import InvalidGoogleCredentials
|
||||
from client_factory import GoogleMapsClientFactory
|
||||
|
@ -1,3 +1,3 @@
|
||||
from geocoder import HereMapsGeocoder
|
||||
from bulk_geocoder import HereMapsBulkGeocoder
|
||||
from routing import HereMapsRoutingIsoline
|
||||
from cartodb_services.here.geocoder import HereMapsGeocoder
|
||||
from cartodb_services.here.bulk_geocoder import HereMapsBulkGeocoder
|
||||
from cartodb_services.here.routing import HereMapsRoutingIsoline
|
||||
|
@ -1,7 +1,7 @@
|
||||
import requests
|
||||
import json
|
||||
|
||||
from exceptions import WrongParams
|
||||
from cartodb_services.here.exceptions import WrongParams
|
||||
from requests.adapters import HTTPAdapter
|
||||
from cartodb_services.metrics import Traceable
|
||||
|
||||
|
@ -1,4 +1,4 @@
|
||||
from routing import MapboxRouting, MapboxRoutingResponse
|
||||
from geocoder import MapboxGeocoder
|
||||
from bulk_geocoder import MapboxBulkGeocoder
|
||||
from isolines import MapboxIsolines, MapboxIsochronesResponse
|
||||
from cartodb_services.mapbox.routing import MapboxRouting, MapboxRoutingResponse
|
||||
from cartodb_services.mapbox.geocoder import MapboxGeocoder
|
||||
from cartodb_services.mapbox.bulk_geocoder import MapboxBulkGeocoder
|
||||
from cartodb_services.mapbox.isolines import MapboxIsolines, MapboxIsochronesResponse
|
||||
|
@ -1,5 +1,5 @@
|
||||
from routing import MapzenRouting, MapzenRoutingResponse
|
||||
from isolines import MapzenIsolines
|
||||
from geocoder import MapzenGeocoder
|
||||
from matrix_client import MatrixClient
|
||||
from isochrones import MapzenIsochrones
|
||||
from cartodb_services.mapzen.routing import MapzenRouting, MapzenRoutingResponse
|
||||
from cartodb_services.mapzen.isolines import MapzenIsolines
|
||||
from cartodb_services.mapzen.geocoder import MapzenGeocoder
|
||||
from cartodb_services.mapzen.matrix_client import MatrixClient
|
||||
from cartodb_services.mapzen.isochrones import MapzenIsochrones
|
||||
|
@ -135,7 +135,7 @@ class MapzenIsolines:
|
||||
# delete points that got None
|
||||
location_estimates_filtered = []
|
||||
for i, c in enumerate(costs):
|
||||
if c <> isorange:
|
||||
if c != isorange:
|
||||
location_estimates_filtered.append(location_estimates[i])
|
||||
|
||||
return location_estimates_filtered
|
||||
|
@ -1,4 +1,4 @@
|
||||
from config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatoryConfig
|
||||
from quota import QuotaService
|
||||
from user import UserMetricsService
|
||||
from log import metrics, MetricsDataGatherer, Traceable
|
||||
from cartodb_services.metrics.config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatoryConfig
|
||||
from cartodb_services.metrics.quota import QuotaService
|
||||
from cartodb_services.metrics.user import UserMetricsService
|
||||
from cartodb_services.metrics.log import metrics, MetricsDataGatherer, Traceable
|
||||
|
@ -45,7 +45,7 @@ class ServiceConfig(object):
|
||||
|
||||
def _get_effective_monthly_quota(self, quota_key, default=0):
|
||||
quota_from_redis = self._redis_config.get(quota_key, None)
|
||||
if quota_from_redis and quota_from_redis <> '':
|
||||
if quota_from_redis and quota_from_redis != '':
|
||||
return int(quota_from_redis)
|
||||
else:
|
||||
return default
|
||||
|
@ -6,7 +6,10 @@ import uuid
|
||||
import plpy
|
||||
from datetime import datetime
|
||||
from contextlib import contextmanager
|
||||
from urlparse import urlparse
|
||||
try:
|
||||
from urlparse import urlparse
|
||||
except:
|
||||
from urllib.parse import urlparse
|
||||
|
||||
|
||||
@contextmanager
|
||||
|
@ -1,5 +1,5 @@
|
||||
from user import UserMetricsService
|
||||
from log import MetricsDataGatherer
|
||||
from cartodb_services.metrics.user import UserMetricsService
|
||||
from cartodb_services.metrics.log import MetricsDataGatherer
|
||||
from datetime import date
|
||||
import re
|
||||
|
||||
|
@ -1,5 +1,5 @@
|
||||
from ..core.interfaces import ConfigBackendInterface
|
||||
from null_config import NullConfigStorage
|
||||
from cartodb_services.refactor.storage.null_config import NullConfigStorage
|
||||
|
||||
|
||||
class RedisConfigStorage(ConfigBackendInterface):
|
||||
|
@ -1,4 +1,4 @@
|
||||
from geocoder import TomTomGeocoder
|
||||
from bulk_geocoder import TomTomBulkGeocoder
|
||||
from routing import TomTomRouting, TomTomRoutingResponse
|
||||
from isolines import TomTomIsolines, TomTomIsochronesResponse
|
||||
from cartodb_services.tomtom.geocoder import TomTomGeocoder
|
||||
from cartodb_services.tomtom.bulk_geocoder import TomTomBulkGeocoder
|
||||
from cartodb_services.tomtom.routing import TomTomRouting, TomTomRoutingResponse
|
||||
from cartodb_services.tomtom.isolines import TomTomIsolines, TomTomIsochronesResponse
|
||||
|
@ -1,9 +1,9 @@
|
||||
from redis_tools import RedisConnection, RedisDBConfig
|
||||
from coordinates import Coordinate
|
||||
from polyline import PolyLine
|
||||
from log import Logger, LoggerConfig
|
||||
from rate_limiter import RateLimiter
|
||||
from service_manager import ServiceManager
|
||||
from legacy_service_manager import LegacyServiceManager
|
||||
from exceptions import QuotaExceededException, RateLimitExceeded
|
||||
from country import country_to_iso3
|
||||
from .redis_tools import RedisConnection, RedisDBConfig
|
||||
from .coordinates import Coordinate
|
||||
from .polyline import PolyLine
|
||||
from .log import Logger, LoggerConfig
|
||||
from .rate_limiter import RateLimiter
|
||||
from .service_manager import ServiceManager
|
||||
from .legacy_service_manager import LegacyServiceManager
|
||||
from .exceptions import QuotaExceededException, RateLimitExceeded
|
||||
from .country import country_to_iso3
|
||||
|
@ -1,7 +1,7 @@
|
||||
import time
|
||||
import random
|
||||
from datetime import datetime
|
||||
from exceptions import TimeoutException
|
||||
from cartodb_services.tools.exceptions import TimeoutException
|
||||
import re
|
||||
|
||||
DEFAULT_RETRY_TIMEOUT = 60
|
||||
|
@ -20,7 +20,7 @@ class ImportHelper:
|
||||
response.raise_for_status()
|
||||
response_json = json.loads(response.text)
|
||||
if not response_json['success']:
|
||||
print "Error importing the test dataset: {0}".format(response.text)
|
||||
print("Error importing the test dataset: {0}".format(response.text))
|
||||
sys.exit(1)
|
||||
while(True):
|
||||
table_name = ImportHelper.get_imported_table_name(
|
||||
@ -42,8 +42,8 @@ class ImportHelper:
|
||||
schema, username, host, import_id, api_key)
|
||||
import_data_response = requests.get(import_url)
|
||||
if import_data_response.status_code != 200:
|
||||
print "Error getting the table name from " \
|
||||
"the import data: {0}".format(import_data_response.text)
|
||||
print("Error getting the table name from " \
|
||||
"the import data: {0}".format(import_data_response.text))
|
||||
sys.exit(1)
|
||||
import_data_json = json.loads(import_data_response.text)
|
||||
|
||||
@ -57,5 +57,5 @@ class ImportHelper:
|
||||
)
|
||||
response = requests.get(url)
|
||||
if response.status_code != 200:
|
||||
print "Error cleaning the test dataset: {0}".format(response.text)
|
||||
print("Error cleaning the test dataset: {0}".format(response.text))
|
||||
sys.exit(1)
|
||||
|
@ -48,7 +48,7 @@ class IntegrationTestHelper:
|
||||
def execute_query_raw(cls, sql_api_url, query):
|
||||
requests.packages.urllib3.disable_warnings()
|
||||
query_url = "{0}?q={1}".format(sql_api_url, query)
|
||||
print "Executing query: {0}".format(query_url)
|
||||
print("Executing query: {0}".format(query_url))
|
||||
query_response = requests.get(query_url)
|
||||
if query_response.status_code != 200:
|
||||
raise Exception(json.loads(query_response.text)['error'])
|
||||
|
@ -36,7 +36,7 @@ def main():
|
||||
set_environment_variables(username, api_key, table_name, host, schema)
|
||||
execute_tests()
|
||||
except Exception as e:
|
||||
print e.message
|
||||
print(e)
|
||||
sys.exit(1)
|
||||
finally:
|
||||
clean_environment_variables()
|
||||
@ -45,11 +45,11 @@ def main():
|
||||
|
||||
|
||||
def usage():
|
||||
print """Usage: run_tests.py [options] username api_key
|
||||
print("""Usage: run_tests.py [options] username api_key
|
||||
Options:
|
||||
-h: Show this help
|
||||
--host: take that host as base (by default is cartodb.com)
|
||||
--schema: define the url schema [http/https] (by default https)"""
|
||||
--schema: define the url schema [http/https] (by default https)""")
|
||||
|
||||
|
||||
def execute_tests():
|
||||
@ -59,7 +59,7 @@ def execute_tests():
|
||||
stderr=subprocess.PIPE
|
||||
)
|
||||
out, err = process.communicate()
|
||||
print err
|
||||
print(err)
|
||||
regexp = re.compile(r'FAILED \(.*\)')
|
||||
if regexp.search(err) is not None:
|
||||
sys.exit(1)
|
||||
|
Loading…
Reference in New Issue
Block a user