cartodb4/dataservices-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Solved data conversion issues

Browse Source
This commit is contained in:
Antonio 2018-01-03 14:14:25 +01:00
parent 941af522ae
commit 801c5a3cee
7 changed files with 85 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
client/cdb_dataservices_client--0.23.0.sql Executable file → Normal file
View File

33
server/extension/cdb_dataservices_server--0.29.0--0.30.0.sql
View File

@ -12,6 +12,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_route_with_waypoi
RETURNS cdb_dataservices_server.simple_route AS $$
import json
from cartodb_services.mapbox import MapboxRouting, MapboxRoutingResponse
from cartodb_services.mapbox.types import MODE_TO_MAPBOX_PROFILE
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Coordinate
from cartodb_services.tools import Logger,LoggerConfig
@ -47,7 +48,9 @@ RETURNS cdb_dataservices_server.simple_route AS $$
lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon']
waypoint_coords.append(Coordinate(lon,lat))
resp = client.directions(waypoint_coords, mode)
profile = MODE_TO_MAPBOX_PROFILE.get(mode)
resp = client.directions(waypoint_coords, profile)
if resp and resp.shape:
shape_linestring = polyline_to_linestring(resp.shape)
if shape_linestring:
@ -2033,6 +2036,7 @@ RETURNS Geometry AS $$
from cartodb_services.mapbox import MapboxGeocoder
from cartodb_services.metrics import QuotaService, metrics
from cartodb_services.tools import Logger,LoggerConfig
from cartodb_services.tools.country import country_to_iso3
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
@ -2049,9 +2053,14 @@ RETURNS Geometry AS $$
with metrics('cdb_geocode_namedplace_point', user_geocoder_config, logger):
try:
geocoder = MapboxGeocoder(user_geocoder_config.mapbox_api_key, logger, user_geocoder_config.mapbox_service_params)
country_iso3 = None
if country:
country_iso3 = country_to_iso3(country)
coordinates = geocoder.geocode(searchtext=searchtext, city=city,
state_province=state_province,
country=country)
state_province=state_province,
country=country_iso3)
if coordinates:
quota_service.increment_success_service_use()
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
@ -2384,6 +2393,7 @@ RETURNS Geometry AS $$
from cartodb_services.mapbox import MapboxGeocoder
from cartodb_services.metrics import QuotaService, metrics
from cartodb_services.tools import Logger,LoggerConfig
from cartodb_services.mapzen.types import country_to_iso3
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
@ -2400,9 +2410,14 @@ RETURNS Geometry AS $$
with metrics('cdb_geocode_namedplace_point', user_geocoder_config, logger):
try:
geocoder = MapboxGeocoder(user_geocoder_config.mapbox_api_key, logger)
country_iso3 = None
if country_name:
country_iso3 = country_to_iso3(country_name)
coordinates = geocoder.geocode(searchtext=city_name, city=None,
state_province=admin1_name,
country=country)
country=country_iso3)
if coordinates:
quota_service.increment_success_service_use()
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
@ -2973,6 +2988,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance(
RETURNS SETOF cdb_dataservices_server.isoline AS $$
import json
from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines
from cartodb_services.mapbox.types import MODE_TO_MAPBOX_PROFILE
from cartodb_services.tools import Coordinate
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
@ -2998,10 +3014,12 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
else:
raise Exception('source is NULL')
profile = MODE_TO_MAPBOX_PROFILE.get(mode)
# -- TODO Support options properly
isolines = {}
for r in data_range:
isoline = mapbox_isolines.calculate_isodistance(origin, r, mode)
isoline = mapbox_isolines.calculate_isodistance(origin, r, profile)
isolines[r] = isoline
result = []
@ -3106,6 +3124,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isochrones(
RETURNS SETOF cdb_dataservices_server.isoline AS $$
import json
from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines
from cartodb_services.mapbox.types import MODE_TO_MAPBOX_PROFILE
from cartodb_services.tools import Coordinate
from cartodb_services.tools.coordinates import coordinates_to_polygon
from cartodb_services.metrics import QuotaService
@ -3133,7 +3152,9 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
else:
raise Exception('source is NULL')
resp = mapbox_isolines.calculate_isochrone(origin, data_range, mode)
profile = MODE_TO_MAPBOX_PROFILE.get(mode)
resp = mapbox_isolines.calculate_isochrone(origin, data_range, profile)
if resp:
result = []

35
server/extension/cdb_dataservices_server--0.30.0.sql
View File

@ -15,6 +15,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_route_with_waypoi
RETURNS cdb_dataservices_server.simple_route AS $$
import json
from cartodb_services.mapbox import MapboxRouting, MapboxRoutingResponse
from cartodb_services.mapbox.types import MODE_TO_MAPBOX_PROFILE
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Coordinate
from cartodb_services.tools import Logger,LoggerConfig
@ -50,7 +51,9 @@ RETURNS cdb_dataservices_server.simple_route AS $$
lon = plpy.execute("SELECT ST_X('%s') AS lon" % waypoint)[0]['lon']
waypoint_coords.append(Coordinate(lon,lat))
resp = client.directions(waypoint_coords, mode)
profile = MODE_TO_MAPBOX_PROFILE.get(mode)
resp = client.directions(waypoint_coords, profile)
if resp and resp.shape:
shape_linestring = polyline_to_linestring(resp.shape)
if shape_linestring:
@ -1506,6 +1509,7 @@ RETURNS TABLE(the_geom geometry, geom_refs text) AS $$
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu STABLE PARALLEL RESTRICTED;
CREATE TYPE cdb_dataservices_server.ds_fdw_metadata as (schemaname text, tabname text, servername text);
CREATE TYPE cdb_dataservices_server.ds_return_metadata as (colnames text[], coltypes text[]);
@ -2048,6 +2052,7 @@ RETURNS Geometry AS $$
from cartodb_services.mapbox import MapboxGeocoder
from cartodb_services.metrics import QuotaService, metrics
from cartodb_services.tools import Logger,LoggerConfig
from cartodb_services.tools.country import country_to_iso3
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
@ -2064,9 +2069,14 @@ RETURNS Geometry AS $$
with metrics('cdb_geocode_namedplace_point', user_geocoder_config, logger):
try:
geocoder = MapboxGeocoder(user_geocoder_config.mapbox_api_key, logger, user_geocoder_config.mapbox_service_params)
country_iso3 = None
if country:
country_iso3 = country_to_iso3(country)
coordinates = geocoder.geocode(searchtext=searchtext, city=city,
state_province=state_province,
country=country)
state_province=state_province,
country=country_iso3)
if coordinates:
quota_service.increment_success_service_use()
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
@ -2399,6 +2409,7 @@ RETURNS Geometry AS $$
from cartodb_services.mapbox import MapboxGeocoder
from cartodb_services.metrics import QuotaService, metrics
from cartodb_services.tools import Logger,LoggerConfig
from cartodb_services.mapzen.types import country_to_iso3
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
@ -2415,9 +2426,14 @@ RETURNS Geometry AS $$
with metrics('cdb_geocode_namedplace_point', user_geocoder_config, logger):
try:
geocoder = MapboxGeocoder(user_geocoder_config.mapbox_api_key, logger)
country_iso3 = None
if country_name:
country_iso3 = country_to_iso3(country_name)
coordinates = geocoder.geocode(searchtext=city_name, city=None,
state_province=admin1_name,
country=country)
country=country_iso3)
if coordinates:
quota_service.increment_success_service_use()
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
@ -2920,6 +2936,7 @@ RETURNS Geometry AS $$
RETURN ret;
END
$$ LANGUAGE plpgsql STABLE PARALLEL SAFE;
CREATE TYPE cdb_dataservices_server.isoline AS (center geometry(Geometry,4326), data_range integer, the_geom geometry(Multipolygon,4326));
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_here_routing_isolines(username TEXT, orgname TEXT, type TEXT, source geometry(Geometry, 4326), mode TEXT, data_range integer[], options text[])
@ -2989,6 +3006,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isodistance(
RETURNS SETOF cdb_dataservices_server.isoline AS $$
import json
from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines
from cartodb_services.mapbox.types import MODE_TO_MAPBOX_PROFILE
from cartodb_services.tools import Coordinate
from cartodb_services.metrics import QuotaService
from cartodb_services.tools import Logger,LoggerConfig
@ -3014,10 +3032,12 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
else:
raise Exception('source is NULL')
profile = MODE_TO_MAPBOX_PROFILE.get(mode)
# -- TODO Support options properly
isolines = {}
for r in data_range:
isoline = mapbox_isolines.calculate_isodistance(origin, r, mode)
isoline = mapbox_isolines.calculate_isodistance(origin, r, profile)
isolines[r] = isoline
result = []
@ -3122,6 +3142,7 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapbox_isochrones(
RETURNS SETOF cdb_dataservices_server.isoline AS $$
import json
from cartodb_services.mapbox import MapboxMatrixClient, MapboxIsolines
from cartodb_services.mapbox.types import MODE_TO_MAPBOX_PROFILE
from cartodb_services.tools import Coordinate
from cartodb_services.tools.coordinates import coordinates_to_polygon
from cartodb_services.metrics import QuotaService
@ -3149,7 +3170,9 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
else:
raise Exception('source is NULL')
resp = mapbox_isolines.calculate_isochrone(origin, data_range, mode)
profile = MODE_TO_MAPBOX_PROFILE.get(mode)
resp = mapbox_isolines.calculate_isochrone(origin, data_range, profile)
if resp:
result = []

2
server/lib/python/cartodb_services/cartodb_services/mapbox/isolines.py
View File

@ -92,7 +92,7 @@ class MapboxIsolines():
return self.calculate_isochrone(origin=origin,
time_ranges=[time_range],
profile=profile)
profile=profile)[0].coordinates
def calculate_isoline(self, origin, isorange, upper_rmax,
cost_method=_calculate_matrix_cost,

4
server/lib/python/cartodb_services/cartodb_services/mapbox/types.py Normal file
View File

@ -0,0 +1,4 @@
MODE_TO_MAPBOX_PROFILE = {
'car': 'driving',
'walk': 'walking',
}

19
server/lib/python/cartodb_services/cartodb_services/tools/country.py Normal file
View File

@ -0,0 +1,19 @@
import plpy
def country_to_iso3(country):
""" Convert country to its iso3 code """
if not country:
return None
try:
country_plan = plpy.prepare("SELECT adm0_a3 as iso3 FROM admin0_synonyms WHERE lower(regexp_replace($1, " \
"'[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text = name_; ", ['text'])
country_result = plpy.execute(country_plan, [country], 1)
if country_result:
return country_result[0]['iso3']
else:
return None
except BaseException as e:
plpy.warning("Can't get the iso3 code from {0}: {1}".format(country, e))
return None

5
server/lib/python/cartodb_services/test/test_mapboxgeocoder.py
View File

@ -24,3 +24,8 @@ class MapboxGeocoderTestCase(unittest.TestCase):
self.assertEqual(place[0], WELL_KNOWN_LONGITUDE)
self.assertEqual(place[1], WELL_KNOWN_LATITUDE)
def test_valid_request_namedplace(self):
place = self.geocoder.geocode(searchtext='Barcelona')
assert place