cartodb4/dataservices-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #315 from CartoDB/add_retries_to_requests

Browse Source 
Make the HTTP requests retry on timeout
This commit is contained in:
Rafa de la Torre 2016-12-07 12:51:02 +01:00 committed by GitHub
commit a977bc97ab
6 changed files with 30 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
server/lib/python/cartodb_services/cartodb_services/here/geocoder.py
View File

@ -4,6 +4,7 @@
import json import json
import requests import requests
from requests.adapters import HTTPAdapter
from exceptions import * from exceptions import *
from cartodb_services.metrics import Traceable from cartodb_services.metrics import Traceable
@ -17,6 +18,7 @@ class HereMapsGeocoder(Traceable):
DEFAULT_GEN = 9 DEFAULT_GEN = 9
READ_TIMEOUT = 60 READ_TIMEOUT = 60
CONNECT_TIMEOUT = 10 CONNECT_TIMEOUT = 10
MAX_RETRIES=3
ADDRESS_PARAMS = [ ADDRESS_PARAMS = [
'city', 'city',
@ -88,7 +90,10 @@ class HereMapsGeocoder(Traceable):
'gen': self.gen 'gen': self.gen
} }
request_params.update(params) request_params.update(params)
response = requests.get(self.host, params=request_params, # TODO Extract HTTP client wrapper
session = requests.Session()
session.mount(self.host, HTTPAdapter(self.MAX_RETRIES))
response = session.get(self.host, params=request_params,
timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT)) timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
self.add_response_data(response, self._logger) self.add_response_data(response, self._logger)
if response.status_code == requests.codes.ok: if response.status_code == requests.codes.ok:

5
server/lib/python/cartodb_services/cartodb_services/here/routing.py
View File

@ -2,6 +2,7 @@ import requests
import json import json
from exceptions import WrongParams from exceptions import WrongParams
from requests.adapters import HTTPAdapter
from cartodb_services.metrics import Traceable from cartodb_services.metrics import Traceable
@ -13,6 +14,7 @@ class HereMapsRoutingIsoline(Traceable):
ISOLINE_PATH = '/routing/7.2/calculateisoline.json' ISOLINE_PATH = '/routing/7.2/calculateisoline.json'
READ_TIMEOUT = 60 READ_TIMEOUT = 60
CONNECT_TIMEOUT = 10 CONNECT_TIMEOUT = 10
MAX_RETRIES = 3
ACCEPTED_MODES = { ACCEPTED_MODES = {
"walk": "pedestrian", "walk": "pedestrian",
@ -53,6 +55,9 @@ class HereMapsRoutingIsoline(Traceable):
data_range, data_range,
range_type, range_type,
parsed_options) parsed_options)
# TODO Extract HTTP client wrapper
session = requests.Session()
session.mount(self._url, HTTPAdapter(self.MAX_RETRIES))
response = requests.get(self._url, params=request_params, response = requests.get(self._url, params=request_params,
timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT)) timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
self.add_response_data(response, self._logger) self.add_response_data(response, self._logger)

7
server/lib/python/cartodb_services/cartodb_services/mapzen/geocoder.py
View File

@ -2,6 +2,7 @@ import requests
import json import json
import re import re
from requests.adapters import HTTPAdapter
from exceptions import WrongParams, MalformedResult, ServiceException from exceptions import WrongParams, MalformedResult, ServiceException
from qps import qps_retry from qps import qps_retry
from cartodb_services.tools import Coordinate, PolyLine from cartodb_services.tools import Coordinate, PolyLine
@ -14,6 +15,7 @@ class MapzenGeocoder(Traceable):
BASE_URL = 'https://search.mapzen.com/v1/search' BASE_URL = 'https://search.mapzen.com/v1/search'
READ_TIMEOUT = 60 READ_TIMEOUT = 60
CONNECT_TIMEOUT = 10 CONNECT_TIMEOUT = 10
MAX_RETRIES = 3
def __init__(self, app_key, logger, base_url=BASE_URL): def __init__(self, app_key, logger, base_url=BASE_URL):
self._app_key = app_key self._app_key = app_key
@ -27,7 +29,10 @@ class MapzenGeocoder(Traceable):
state_province, state_province,
country, search_type) country, search_type)
try: try:
response = requests.get(self._url, params=request_params, # TODO Extract HTTP client wrapper
session = requests.Session()
session.mount(self._url, HTTPAdapter(self.MAX_RETRIES))
response = session.get(self._url, params=request_params,
timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT)) timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
self.add_response_data(response, self._logger) self.add_response_data(response, self._logger)
if response.status_code == requests.codes.ok: if response.status_code == requests.codes.ok:

7
server/lib/python/cartodb_services/cartodb_services/mapzen/isochrones.py
View File

@ -2,6 +2,7 @@ import requests
import json import json
import re import re
from requests.adapters import HTTPAdapter
from exceptions import WrongParams, MalformedResult, ServiceException from exceptions import WrongParams, MalformedResult, ServiceException
from qps import qps_retry from qps import qps_retry
@ -12,6 +13,7 @@ class MapzenIsochrones:
BASE_URL = 'https://matrix.mapzen.com/isochrone' BASE_URL = 'https://matrix.mapzen.com/isochrone'
READ_TIMEOUT = 60 READ_TIMEOUT = 60
CONNECT_TIMEOUT = 10 CONNECT_TIMEOUT = 10
MAX_RETRIES = 3
ACCEPTED_MODES = { ACCEPTED_MODES = {
"walk": "pedestrian", "walk": "pedestrian",
@ -28,7 +30,10 @@ class MapzenIsochrones:
request_params = self._parse_request_params(locations, costing, request_params = self._parse_request_params(locations, costing,
ranges) ranges)
try: try:
response = requests.get(self._url, params=request_params, # TODO Extract HTTP client wrapper
session = requests.Session()
session.mount(self._url, HTTPAdapter(self.MAX_RETRIES))
response = session.get(self._url, params=request_params,
timeout=(self.CONNECT_TIMEOUT, timeout=(self.CONNECT_TIMEOUT,
self.READ_TIMEOUT)) self.READ_TIMEOUT))

7
server/lib/python/cartodb_services/cartodb_services/mapzen/routing.py
View File

@ -2,6 +2,7 @@ import requests
import json import json
import re import re
from requests.adapters import HTTPAdapter
from exceptions import WrongParams, MalformedResult, ServiceException from exceptions import WrongParams, MalformedResult, ServiceException
from qps import qps_retry from qps import qps_retry
from cartodb_services.tools import Coordinate, PolyLine from cartodb_services.tools import Coordinate, PolyLine
@ -14,6 +15,7 @@ class MapzenRouting(Traceable):
PRODUCTION_ROUTING_BASE_URL = 'https://valhalla.mapzen.com/route' PRODUCTION_ROUTING_BASE_URL = 'https://valhalla.mapzen.com/route'
READ_TIMEOUT = 60 READ_TIMEOUT = 60
CONNECT_TIMEOUT = 10 CONNECT_TIMEOUT = 10
MAX_RETRIES=3
ACCEPTED_MODES = { ACCEPTED_MODES = {
"walk": "pedestrian", "walk": "pedestrian",
@ -46,7 +48,10 @@ class MapzenRouting(Traceable):
mode_param, mode_param,
units) units)
request_params = self.__parse_request_parameters(json_request_params) request_params = self.__parse_request_parameters(json_request_params)
response = requests.get(self._url, params=request_params, # TODO Extract HTTP client wrapper
session = requests.Session()
session.mount(self._url, HTTPAdapter(self.MAX_RETRIES))
response = session.get(self._url, params=request_params,
timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT)) timeout=(self.CONNECT_TIMEOUT, self.READ_TIMEOUT))
self.add_response_data(response, self._logger) self.add_response_data(response, self._logger)
if response.status_code == requests.codes.ok: if response.status_code == requests.codes.ok:

2
server/lib/python/cartodb_services/setup.py
View File

@ -10,7 +10,7 @@ from setuptools import setup, find_packages
setup( setup(
name='cartodb_services', name='cartodb_services',
version='0.12.0', version='0.12.1',
description='CartoDB Services API Python Library', description='CartoDB Services API Python Library',