cartodb4/dataservices-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Adapt dataservices-api server to python3

Browse Source 
Related to https://github.com/CartoDB/cartodb-platform/issues/6237
This commit is contained in:
manmorjim 2020-02-26 15:49:15 +01:00
parent b2a06c6ed6
commit 3c916bdfc9
28 changed files with 76 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
server/extension/sql/50_namedplaces.sql
View File

@ -1,7 +1,7 @@
---- cdb_geocode_namedplace_point(city_name text)
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text)
RETURNS Geometry AS $$
import spiexceptions
from plpy import spiexceptions
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
@ -21,7 +21,7 @@ $$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
---- cdb_geocode_namedplace_point(city_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text)
RETURNS Geometry AS $$
import spiexceptions
from plpy import spiexceptions
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
@ -41,7 +41,7 @@ $$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text)
RETURNS Geometry AS $$
import spiexceptions
from plpy import spiexceptions
from cartodb_services.tools import Logger,LoggerConfig
plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")

2
server/lib/python/cartodb_services/cartodb_services/__init__.py
View File

@ -34,4 +34,4 @@ def _reset():
plpy = None
GD = None
from geocoder import run_street_point_geocoder, StreetPointBulkGeocoder
from cartodb_services.geocoder import run_street_point_geocoder, StreetPointBulkGeocoder

10
server/lib/python/cartodb_services/cartodb_services/bulk_geocoders.py
View File

@ -1,8 +1,8 @@
from google import GoogleMapsBulkGeocoder
from here import HereMapsBulkGeocoder
from tomtom import TomTomBulkGeocoder
from mapbox import MapboxBulkGeocoder
from geocodio import GeocodioBulkGeocoder
from cartodb_services.google import GoogleMapsBulkGeocoder
from cartodb_services.here import HereMapsBulkGeocoder
from cartodb_services.tomtom import TomTomBulkGeocoder
from cartodb_services.mapbox import MapboxBulkGeocoder
from cartodb_services.geocodio import GeocodioBulkGeocoder
BATCH_GEOCODER_CLASS_BY_PROVIDER = {
'google': GoogleMapsBulkGeocoder,

6
server/lib/python/cartodb_services/cartodb_services/config/__init__.py
View File

@ -1,3 +1,3 @@
from service_configuration import ServiceConfiguration
from rate_limits import RateLimitsConfig, RateLimitsConfigBuilder, RateLimitsConfigSetter
from legacy_rate_limits import RateLimitsConfigLegacyBuilder
from .service_configuration import ServiceConfiguration
from .rate_limits import RateLimitsConfig, RateLimitsConfigBuilder, RateLimitsConfigSetter
from .legacy_rate_limits import RateLimitsConfigLegacyBuilder

2
server/lib/python/cartodb_services/cartodb_services/config/legacy_rate_limits.py
View File

@ -1,5 +1,5 @@
import json
from rate_limits import RateLimitsConfig
from cartodb_services.config.rate_limits import RateLimitsConfig
class RateLimitsConfigLegacyBuilder(object):
"""

2
server/lib/python/cartodb_services/cartodb_services/config/rate_limits.py
View File

@ -1,6 +1,6 @@
import json
from service_configuration import ServiceConfiguration
from cartodb_services.config.service_configuration import ServiceConfiguration
class RateLimitsConfig(object):
"""

4
server/lib/python/cartodb_services/cartodb_services/geocoder.py
View File

@ -1,7 +1,7 @@
#!/usr/local/bin/python
# -*- coding: utf-8 -*-
from tools import QuotaExceededException, Logger
from cartodb_services.tools import QuotaExceededException, Logger
from collections import namedtuple
import json
@ -71,7 +71,7 @@ def run_street_point_geocoder(plpy, GD, geocoder, service_manager, username, org
except Exception as e:
import sys
logger.error("Error processing geocode", sys.exc_info(), data={"username": username, "orgname": orgname})
metadata['processing_error'] = 'Error: {}'.format(e.message)
metadata['processing_error'] = 'Error: {}'.format(e)
results.append([result[0], None, json.dumps(metadata)])
failed_count += 1

4
server/lib/python/cartodb_services/cartodb_services/geocodio/__init__.py
View File

@ -1,2 +1,2 @@
from geocoder import GeocodioGeocoder
from bulk_geocoder import GeocodioBulkGeocoder
from cartodb_services.geocodio.geocoder import GeocodioGeocoder
from cartodb_services.geocodio.bulk_geocoder import GeocodioBulkGeocoder

4
server/lib/python/cartodb_services/cartodb_services/google/__init__.py
View File

@ -1,2 +1,2 @@
from geocoder import GoogleMapsGeocoder
from bulk_geocoder import GoogleMapsBulkGeocoder
from cartodb_services.google.geocoder import GoogleMapsGeocoder
from cartodb_services.google.bulk_geocoder import GoogleMapsBulkGeocoder

2
server/lib/python/cartodb_services/cartodb_services/google/bulk_geocoder.py
View File

@ -1,5 +1,5 @@
from multiprocessing import Pool
from exceptions import MalformedResult
from cartodb_services.google.exceptions import MalformedResult
from cartodb_services import StreetPointBulkGeocoder
from cartodb_services.geocoder import compose_address, geocoder_error_response
from cartodb_services.google import GoogleMapsGeocoder

2
server/lib/python/cartodb_services/cartodb_services/google/client_factory.py
View File

@ -3,7 +3,7 @@
import googlemaps
import base64
from exceptions import InvalidGoogleCredentials
from cartodb_services.google.exceptions import InvalidGoogleCredentials
class GoogleMapsClientFactory():

6
server/lib/python/cartodb_services/cartodb_services/google/geocoder.py
View File

@ -1,9 +1,11 @@
#!/usr/local/bin/python
# -*- coding: utf-8 -*-
try:
from urlparse import parse_qs
except:
from urllib.parse import parse_qs
from exceptions import MalformedResult
from cartodb_services.google.exceptions import MalformedResult
from cartodb_services.geocoder import compose_address, geocoder_metadata, PRECISION_PRECISE, PRECISION_INTERPOLATED, EMPTY_RESPONSE
from cartodb_services.google.exceptions import InvalidGoogleCredentials
from client_factory import GoogleMapsClientFactory

6
server/lib/python/cartodb_services/cartodb_services/here/__init__.py
View File

@ -1,3 +1,3 @@
from geocoder import HereMapsGeocoder
from bulk_geocoder import HereMapsBulkGeocoder
from routing import HereMapsRoutingIsoline
from cartodb_services.here.geocoder import HereMapsGeocoder
from cartodb_services.here.bulk_geocoder import HereMapsBulkGeocoder
from cartodb_services.here.routing import HereMapsRoutingIsoline

2
server/lib/python/cartodb_services/cartodb_services/here/routing.py
View File

@ -1,7 +1,7 @@
import requests
import json
from exceptions import WrongParams
from cartodb_services.here.exceptions import WrongParams
from requests.adapters import HTTPAdapter
from cartodb_services.metrics import Traceable

8
server/lib/python/cartodb_services/cartodb_services/mapbox/__init__.py
View File

@ -1,4 +1,4 @@
from routing import MapboxRouting, MapboxRoutingResponse
from geocoder import MapboxGeocoder
from bulk_geocoder import MapboxBulkGeocoder
from isolines import MapboxIsolines, MapboxIsochronesResponse
from cartodb_services.mapbox.routing import MapboxRouting, MapboxRoutingResponse
from cartodb_services.mapbox.geocoder import MapboxGeocoder
from cartodb_services.mapbox.bulk_geocoder import MapboxBulkGeocoder
from cartodb_services.mapbox.isolines import MapboxIsolines, MapboxIsochronesResponse

10
server/lib/python/cartodb_services/cartodb_services/mapzen/__init__.py
View File

@ -1,5 +1,5 @@
from routing import MapzenRouting, MapzenRoutingResponse
from isolines import MapzenIsolines
from geocoder import MapzenGeocoder
from matrix_client import MatrixClient
from isochrones import MapzenIsochrones
from cartodb_services.mapzen.routing import MapzenRouting, MapzenRoutingResponse
from cartodb_services.mapzen.isolines import MapzenIsolines
from cartodb_services.mapzen.geocoder import MapzenGeocoder
from cartodb_services.mapzen.matrix_client import MatrixClient
from cartodb_services.mapzen.isochrones import MapzenIsochrones

2
server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py
View File

@ -135,7 +135,7 @@ class MapzenIsolines:
# delete points that got None
location_estimates_filtered = []
for i, c in enumerate(costs):
if c <> isorange:
if c != isorange:
location_estimates_filtered.append(location_estimates[i])
return location_estimates_filtered

8
server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py
View File

@ -1,4 +1,4 @@
from config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatoryConfig
from quota import QuotaService
from user import UserMetricsService
from log import metrics, MetricsDataGatherer, Traceable
from cartodb_services.metrics.config import GeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatoryConfig
from cartodb_services.metrics.quota import QuotaService
from cartodb_services.metrics.user import UserMetricsService
from cartodb_services.metrics.log import metrics, MetricsDataGatherer, Traceable

2
server/lib/python/cartodb_services/cartodb_services/metrics/config.py
View File

@ -45,7 +45,7 @@ class ServiceConfig(object):
def _get_effective_monthly_quota(self, quota_key, default=0):
quota_from_redis = self._redis_config.get(quota_key, None)
if quota_from_redis and quota_from_redis <> '':
if quota_from_redis and quota_from_redis != '':
return int(quota_from_redis)
else:
return default

3
server/lib/python/cartodb_services/cartodb_services/metrics/log.py
View File

@ -6,7 +6,10 @@ import uuid
import plpy
from datetime import datetime
from contextlib import contextmanager
try:
from urlparse import urlparse
except:
from urllib.parse import urlparse
@contextmanager

4
server/lib/python/cartodb_services/cartodb_services/metrics/quota.py
View File

@ -1,5 +1,5 @@
from user import UserMetricsService
from log import MetricsDataGatherer
from cartodb_services.metrics.user import UserMetricsService
from cartodb_services.metrics.log import MetricsDataGatherer
from datetime import date
import re

2
server/lib/python/cartodb_services/cartodb_services/refactor/storage/redis_config.py
View File

@ -1,5 +1,5 @@
from ..core.interfaces import ConfigBackendInterface
from null_config import NullConfigStorage
from cartodb_services.refactor.storage.null_config import NullConfigStorage
class RedisConfigStorage(ConfigBackendInterface):

8
server/lib/python/cartodb_services/cartodb_services/tomtom/__init__.py
View File

@ -1,4 +1,4 @@
from geocoder import TomTomGeocoder
from bulk_geocoder import TomTomBulkGeocoder
from routing import TomTomRouting, TomTomRoutingResponse
from isolines import TomTomIsolines, TomTomIsochronesResponse
from cartodb_services.tomtom.geocoder import TomTomGeocoder
from cartodb_services.tomtom.bulk_geocoder import TomTomBulkGeocoder
from cartodb_services.tomtom.routing import TomTomRouting, TomTomRoutingResponse
from cartodb_services.tomtom.isolines import TomTomIsolines, TomTomIsochronesResponse

18
server/lib/python/cartodb_services/cartodb_services/tools/__init__.py
View File

@ -1,9 +1,9 @@
from redis_tools import RedisConnection, RedisDBConfig
from coordinates import Coordinate
from polyline import PolyLine
from log import Logger, LoggerConfig
from rate_limiter import RateLimiter
from service_manager import ServiceManager
from legacy_service_manager import LegacyServiceManager
from exceptions import QuotaExceededException, RateLimitExceeded
from country import country_to_iso3
from .redis_tools import RedisConnection, RedisDBConfig
from .coordinates import Coordinate
from .polyline import PolyLine
from .log import Logger, LoggerConfig
from .rate_limiter import RateLimiter
from .service_manager import ServiceManager
from .legacy_service_manager import LegacyServiceManager
from .exceptions import QuotaExceededException, RateLimitExceeded
from .country import country_to_iso3

2
server/lib/python/cartodb_services/cartodb_services/tools/qps.py
View File

@ -1,7 +1,7 @@
import time
import random
from datetime import datetime
from exceptions import TimeoutException
from cartodb_services.tools.exceptions import TimeoutException
import re
DEFAULT_RETRY_TIMEOUT = 60

8
test/helpers/import_helper.py
View File

@ -20,7 +20,7 @@ class ImportHelper:
response.raise_for_status()
response_json = json.loads(response.text)
if not response_json['success']:
print "Error importing the test dataset: {0}".format(response.text)
print("Error importing the test dataset: {0}".format(response.text))
sys.exit(1)
while(True):
table_name = ImportHelper.get_imported_table_name(
@ -42,8 +42,8 @@ class ImportHelper:
schema, username, host, import_id, api_key)
import_data_response = requests.get(import_url)
if import_data_response.status_code != 200:
print "Error getting the table name from " \
"the import data: {0}".format(import_data_response.text)
print("Error getting the table name from " \
"the import data: {0}".format(import_data_response.text))
sys.exit(1)
import_data_json = json.loads(import_data_response.text)
@ -57,5 +57,5 @@ class ImportHelper:
)
response = requests.get(url)
if response.status_code != 200:
print "Error cleaning the test dataset: {0}".format(response.text)
print("Error cleaning the test dataset: {0}".format(response.text))
sys.exit(1)

2
test/helpers/integration_test_helper.py
View File

@ -48,7 +48,7 @@ class IntegrationTestHelper:
def execute_query_raw(cls, sql_api_url, query):
requests.packages.urllib3.disable_warnings()
query_url = "{0}?q={1}".format(sql_api_url, query)
print "Executing query: {0}".format(query_url)
print("Executing query: {0}".format(query_url))
query_response = requests.get(query_url)
if query_response.status_code != 200:
raise Exception(json.loads(query_response.text)['error'])

8
test/run_tests.py
View File

@ -36,7 +36,7 @@ def main():
set_environment_variables(username, api_key, table_name, host, schema)
execute_tests()
except Exception as e:
print e.message
print(e)
sys.exit(1)
finally:
clean_environment_variables()
@ -45,11 +45,11 @@ def main():
def usage():
print """Usage: run_tests.py [options] username api_key
print("""Usage: run_tests.py [options] username api_key
Options:
-h: Show this help
--host: take that host as base (by default is cartodb.com)
--schema: define the url schema [http/https] (by default https)"""
--schema: define the url schema [http/https] (by default https)""")
def execute_tests():
@ -59,7 +59,7 @@ def execute_tests():
stderr=subprocess.PIPE
)
out, err = process.communicate()
print err
print(err)
regexp = re.compile(r'FAILED \(.*\)')
if regexp.search(err) is not None:
sys.exit(1)