Fix conflicts take 4

This commit is contained in:
Carla Iriberri 2016-07-11 12:18:07 +02:00
commit 7b1132b4d2
42 changed files with 4774 additions and 55 deletions

View File

@ -1,5 +1,5 @@
# Data Services API
The CartoDB Data Services SQL API (server and client FTM)
# CARTO Data Services API
The CARTO Data Services SQL API
### Deploy instructions
Steps to deploy a new Data Services API version :

View File

@ -13,8 +13,8 @@ OLD_VERSIONS = $(wildcard old_versions/*.sql)
# @see http://www.postgresql.org/docs/current/static/extend-pgxs.html
DATA = $(NEW_EXTENSION_ARTIFACT) \
$(OLD_VERSIONS) \
cdb_dataservices_client--0.7.0--0.8.0.sql \
cdb_dataservices_client--0.8.0--0.7.0.sql
cdb_dataservices_client--0.8.0--0.9.0.sql \
cdb_dataservices_client--0.9.0--0.8.0.sql
REGRESS = $(notdir $(basename $(wildcard test/sql/*test.sql)))

View File

@ -1,5 +1,5 @@
# CartoDB dataservices API client extension
Postgres extension for the CartoDB dataservices API, client side.
# CARTO Data Services API client extension
Postgres extension for the CARTO Data Services API, client side.
## Dependencies
This extension is thought to be used on top of CartoDB geocoder extension, for the multiples available geocoders (internal, nokia, etc).
@ -9,7 +9,7 @@ The following is a non-comprehensive list of dependencies:
- Postgres 9.3+
- Postgis extension
- Schema triggers extension
- CartoDB extension
- cartodb-postgresql CARTO extension
## Installation into the db cluster
This requires root privileges
@ -28,7 +28,7 @@ One-liner:
sudo PGUSER=postgres make all install installcheck
```
## Install onto a cartodb user's database
## Install onto a CARTO user's database
```
psql -U postgres cartodb_dev_user_fe3b850a-01c0-48f9-8a26-a82f09e9b53f_db

View File

@ -0,0 +1,90 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.9.0'" to load this file. \quit
--
-- Public dataservices API function
--
-- These are the only ones with permissions to publicuser role
-- and should also be the only ones with SECURITY DEFINER
CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapzen_isochrone (source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[])
RETURNS SETOF cdb_dataservices_client.isoline AS $$
DECLARE
username text;
orgname text;
BEGIN
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
RAISE EXCEPTION 'The api_key must be provided';
END IF;
SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
-- JSON value stored "" is taken as literal
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
END IF;
RETURN QUERY
SELECT * FROM cdb_dataservices_client._cdb_mapzen_isochrone(username, orgname, source, mode, range, options);
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
--
-- Public dataservices API function
--
-- These are the only ones with permissions to publicuser role
-- and should also be the only ones with SECURITY DEFINER
CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapzen_isodistance (source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[])
RETURNS SETOF cdb_dataservices_client.isoline AS $$
DECLARE
username text;
orgname text;
BEGIN
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
RAISE EXCEPTION 'The api_key must be provided';
END IF;
SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
-- JSON value stored "" is taken as literal
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
END IF;
RETURN QUERY
SELECT * FROM cdb_dataservices_client._cdb_mapzen_isodistance(username, orgname, source, mode, range, options);
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapzen_isochrone (username text, organization_name text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[])
RETURNS SETOF cdb_dataservices_client.isoline AS $$
CONNECT cdb_dataservices_client._server_conn_str();
SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone (username, organization_name, source, mode, range, options);
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapzen_isodistance (username text, organization_name text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[])
RETURNS SETOF cdb_dataservices_client.isoline AS $$
CONNECT cdb_dataservices_client._server_conn_str();
SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance (username, organization_name, source, mode, range, options);
$$ LANGUAGE plproxy;
GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_mapzen_isochrone(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser;
GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_mapzen_isodistance(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser;
-- Grants missing in previous version
GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_here_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser;
GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_google_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser;
GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_mapzen_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser;

View File

@ -0,0 +1,7 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.8.0'" to load this file. \quit
DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_mapzen_isochrone(geometry(Geometry, 4326), text, integer[], text[]);
DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_mapzen_isodistance(geometry(Geometry, 4326), text, integer[], text[]);
DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapzen_isochrone(text, text, geometry(Geometry, 4326), text, integer[], text[]);
DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapzen_isodistance(text, text, geometry(Geometry, 4326), text, integer[], text[]);

File diff suppressed because it is too large Load Diff

View File

@ -1,5 +1,5 @@
comment = 'CartoDB dataservices client API extension'
default_version = '0.8.0'
default_version = '0.9.0'
requires = 'plproxy, cartodb'
superuser = true
schema = cdb_dataservices_client

View File

@ -0,0 +1,113 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.8.0'" to load this file. \quit
--
-- Public dataservices API function
--
-- These are the only ones with permissions to publicuser role
-- and should also be the only ones with SECURITY DEFINER
CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_here_geocode_street_point (searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
username text;
orgname text;
BEGIN
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
RAISE EXCEPTION 'The api_key must be provided';
END IF;
SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
-- JSON value stored "" is taken as literal
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
END IF;
SELECT cdb_dataservices_client._cdb_here_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret;
RETURN ret;
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
--
-- Public dataservices API function
--
-- These are the only ones with permissions to publicuser role
-- and should also be the only ones with SECURITY DEFINER
CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_google_geocode_street_point (searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
username text;
orgname text;
BEGIN
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
RAISE EXCEPTION 'The api_key must be provided';
END IF;
SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
-- JSON value stored "" is taken as literal
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
END IF;
SELECT cdb_dataservices_client._cdb_google_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret;
RETURN ret;
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
--
-- Public dataservices API function
--
-- These are the only ones with permissions to publicuser role
-- and should also be the only ones with SECURITY DEFINER
CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapzen_geocode_street_point (searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
username text;
orgname text;
BEGIN
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
RAISE EXCEPTION 'The api_key must be provided';
END IF;
SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
-- JSON value stored "" is taken as literal
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
END IF;
SELECT cdb_dataservices_client._cdb_mapzen_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret;
RETURN ret;
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_here_geocode_street_point (username text, organization_name text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL)
RETURNS Geometry AS $$
CONNECT cdb_dataservices_client._server_conn_str();
SELECT cdb_dataservices_server.cdb_here_geocode_street_point (username, organization_name, searchtext, city, state_province, country);
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_google_geocode_street_point (username text, organization_name text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL)
RETURNS Geometry AS $$
CONNECT cdb_dataservices_client._server_conn_str();
SELECT cdb_dataservices_server.cdb_google_geocode_street_point (username, organization_name, searchtext, city, state_province, country);
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapzen_geocode_street_point (username text, organization_name text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL)
RETURNS Geometry AS $$
CONNECT cdb_dataservices_client._server_conn_str();
SELECT cdb_dataservices_server.cdb_mapzen_geocode_street_point (username, organization_name, searchtext, city, state_province, country);
$$ LANGUAGE plproxy;

View File

@ -0,0 +1,9 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.7.0'" to load this file. \quit
DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_here_geocode_street_point (text, text, text, text);
DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_google_geocode_street_point (text, text, text, text);
DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_mapzen_geocode_street_point (text, text, text, text);
DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_here_geocode_street_point (text, text, text, text, text, text);
DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_google_geocode_street_point (text, text, text, text, text, text);
DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapzen_geocode_street_point (text, text, text, text, text, text);

View File

@ -103,6 +103,26 @@
- { name: range, type: "integer[]" }
- { name: options, type: "text[]", default: 'ARRAY[]::text[]' }
- name: cdb_mapzen_isochrone
return_type: SETOF cdb_dataservices_client.isoline
multi_row: true
multi_field: true
params:
- { name: source, type: "geometry(Geometry, 4326)" }
- { name: mode, type: text }
- { name: range, type: "integer[]" }
- { name: options, type: "text[]", default: 'ARRAY[]::text[]' }
- name: cdb_mapzen_isodistance
return_type: SETOF cdb_dataservices_client.isoline
multi_row: true
multi_field: true
params:
- { name: source, type: "geometry(Geometry, 4326)" }
- { name: mode, type: text }
- { name: range, type: "integer[]" }
- { name: options, type: "text[]", default: 'ARRAY[]::text[]' }
- name: cdb_route_point_to_point
return_type: cdb_dataservices_client.simple_route
multi_field: true

View File

@ -1,6 +1,6 @@
# Data Services API
The CartoDB Data Services API offers a set of location based services that can be used programatically to empower your geospatial applications.
The CARTO Data Services API offers a set of location based services that can be used programatically to empower your geospatial applications.
## Documentation

View File

@ -1,6 +1,6 @@
# Demographic Functions
The Demographic Snapshot enables you to collect demographic reports around a point location. For example, you can take the coordinates of a coffee shop and find the average population characteristics, such as total population, educational attainment, housing and income information around that location. You can use raw street addresses by combining the Demographic Snapshot with CartoDB's geocoding features. If you need help creating coordinates from addresses, see the [Geocoding Functions](/cartodb-platform/dataservices-api/geocoding-functions/) documentation.
The Demographic Snapshot enables you to collect demographic reports around a point location. For example, you can take the coordinates of a coffee shop and find the average population characteristics, such as total population, educational attainment, housing and income information around that location. You can use raw street addresses by combining the Demographic Snapshot with CARTO's geocoding features. If you need help creating coordinates from addresses, see the [Geocoding Functions](/carto-engine/dataservices-api/geocoding-functions/) documentation.
_**Note:** The Demographic Snapshot functions are only available for the United States._
@ -40,7 +40,7 @@ obs_getdemographicsnapshot: {
### Examples
```bash
https://{username}.cartodb.com/api/v2/sql?q=SELECT * FROM
https://{username}.carto.com/api/v2/sql?q=SELECT * FROM
OBS_GetDemographicSnapshot({{point geometry}})
```
@ -49,14 +49,14 @@ OBS_GetDemographicSnapshot({{point geometry}})
__Get the Demographic Snapshot at Camp David__
```bash
https://{username}.cartodb.com/api/v2/sql?q=SELECT * FROM
https://{username}.carto.com/api/v2/sql?q=SELECT * FROM
OBS_GetDemographicSnapshot(CDB_LatLng(39.648333, -77.465))
```
__Get the Demographic Snapshot in the Upper West Side__
```bash
https://{username}.cartodb.com/api/v2/sql?q=SELECT * FROM
https://{username}.carto.com/api/v2/sql?q=SELECT * FROM
OBS_GetDemographicSnapshot(CDB_LatLng(40.80, -73.960))
```

View File

@ -1,25 +1,25 @@
# Geocoding Functions
The [geocoder](https://cartodb.com/data/geocoder-api/) functions allow you to match your data with geometries on your map. This geocoding service can be used programatically to geocode datasets via the CartoDB SQL API. It is fed from _Open Data_ and it serves geometries for countries, provinces, states, cities, postal codes, IP addresses and street addresses. CartoDB provides functions for several different categories of geocoding through the Data Services API.
The [geocoder](https://carto.com/data/geocoder-api/) functions allow you to match your data with geometries on your map. This geocoding service can be used programatically to geocode datasets via the CARTO SQL API. It is fed from _Open Data_ and it serves geometries for countries, provinces, states, cities, postal codes, IP addresses and street addresses. CARTO provides functions for several different categories of geocoding through the Data Services API.
_**This service is subject to quota limitations and extra fees may apply**. View the [Quota Information](http://docs.cartodb.com/cartodb-platform/dataservices-api/quota-information/) section for details and recommendations about to quota consumption._
_**This service is subject to quota limitations and extra fees may apply**. View the [Quota Information](https://carto.com/docs/carto-engine/dataservices-api/quota-information/) section for details and recommendations about to quota consumption._
Here is an example of how to geocode a single country:
```bash
https://{username}.cartodb.com/api/v2/sql?q=SELECT cdb_geocode_admin0_polygon('USA')&api_key={api_key}
https://{username}.carto.com/api/v2/sql?q=SELECT cdb_geocode_admin0_polygon('USA')&api_key={api_key}
```
In order to geocode an existent CartoDB dataset, an SQL UPDATE statement must be used to populate the geometry column in the dataset with the results of the Data Services API. For example, if the column where you are storing the country names for each one of our rows is called `country_column`, run the following statement in order to geocode the dataset:
In order to geocode an existent CARTO dataset, an SQL UPDATE statement must be used to populate the geometry column in the dataset with the results of the Data Services API. For example, if the column where you are storing the country names for each one of our rows is called `country_column`, run the following statement in order to geocode the dataset:
```bash
https://{username}.cartodb.com/api/v2/sql?q=UPDATE {tablename} SET the_geom = cdb_geocode_admin0_polygon('USA')&api_key={api_key}
https://{username}.carto.com/api/v2/sql?q=UPDATE {tablename} SET the_geom = cdb_geocode_admin0_polygon('USA')&api_key={api_key}
```
Notice that you can make use of Postgres or PostGIS functions in your Data Services API requests, as the result is a geometry that can be handled by the system. For example, suppose you need to retrieve the centroid of a specific country, you can wrap the resulting geometry from the geocoder functions inside the PostGIS `ST_Centroid` function:
```bash
https://{username}.cartodb.com/api/v2/sql?q=UPDATE {tablename} SET the_geom = ST_Centroid(cdb_geocode_admin0_polygon('USA'))&api_key={api_key}
https://{username}.carto.com/api/v2/sql?q=UPDATE {tablename} SET the_geom = ST_Centroid(cdb_geocode_admin0_polygon('USA'))&api_key={api_key}
```
@ -217,7 +217,7 @@ INSERT INTO {tablename} (the_geom) SELECT cdb_geocode_namedplace_point('New York
This function geocodes your data into point, or polygon, geometries for postal codes. The postal code polygon geocoder covers the United States, France, Australia and Canada; a request for a different country will return an empty response.
**Note:** For the USA, US Census [Zip Code Tabulation Areas](https://www.census.gov/geo/reference/zctas.html) (ZCTA) are used to reference geocodes for USPS postal codes service areas. See the [FAQs](http://docs.cartodb.com/faqs/datasets-and-data/#why-does-cartodb-use-census-bureau-zctas-and-not-usps-zip-codes-for-postal-codes) about datasets and data for details.
**Note:** For the USA, US Census [Zip Code Tabulation Areas](https://www.census.gov/geo/reference/zctas.html) (ZCTA) are used to reference geocodes for USPS postal codes service areas. See the [FAQs](https://carto.com/docs/faqs/datasets-and-data/#why-does-carto-use-census-bureau-zctas-and-not-usps-zip-codes-for-postal-codes) about datasets and data for details.
### cdb_geocode_postalcode_polygon(_postal_code text, country_name text_)
@ -313,9 +313,9 @@ INSERT INTO {tablename} (the_geom) SELECT cdb_geocode_ipaddress_point('102.23.34
## Street-Level Geocoder
This function geocodes your data into a point geometry for a street address. CartoDB uses several different service providers for street-level geocoding, depending on your platform. If you access CartoDB on a Google Cloud Platform, [Google Maps geocoding](https://developers.google.com/maps/documentation/geocoding/intro) is applied. All other platform users are provided with [HERE geocoding services](https://developer.here.com/rest-apis/documentation/geocoder/topics/quick-start.html). Additional service providers will be implemented in the future.
This function geocodes your data into a point geometry for a street address. CARTO uses several different service providers for street-level geocoding, depending on your platform. If you access CARTO on a Google Cloud Platform, [Google Maps geocoding](https://developers.google.com/maps/documentation/geocoding/intro) is applied. All other platform users are provided with [HERE geocoding services](https://developer.here.com/rest-apis/documentation/geocoder/topics/quick-start.html). Additional service providers will be implemented in the future.
**This service is subject to quota limitations, and extra fees may apply**. View the [Quota information](http://docs.cartodb.com/cartodb-platform/dataservices-api/quota-information/) for details and recommendations about quota consumption.
**This service is subject to quota limitations, and extra fees may apply**. View the [Quota information](https://carto.com/docs/carto-engine/dataservices-api/quota-information/) for details and recommendations about quota consumption.
### cdb_geocode_street_point(_search_text text, [city text], [state text], [country text]_)

View File

@ -1,13 +1,13 @@
# Isoline Functions
[Isolines](https://cartodb.com/data/isolines/) are contoured lines that display equally calculated levels over a given surface area. This enables you to view polygon dimensions by forward or reverse measurements. Isoline functions are calculated as the intersection of areas from the origin point, measured by distance (isodistance) or time (isochrone). For example, the distance of a road from a sidewalk. Isoline services through CartoDB are available by requesting a single function in the Data Services API.
[Isolines](https://carto.com/data/isolines/) are contoured lines that display equally calculated levels over a given surface area. This enables you to view polygon dimensions by forward or reverse measurements. Isoline functions are calculated as the intersection of areas from the origin point, measured by distance (isodistance) or time (isochrone). For example, the distance of a road from a sidewalk. Isoline services through CARTO are available by requesting a single function in the Data Services API.
_**This service is subject to quota limitations and extra fees may apply**. View the [Quota Information](http://docs.cartodb.com/cartodb-platform/dataservices-api/quota-information/) section for details and recommendations about to quota consumption._
_**This service is subject to quota limitations and extra fees may apply**. View the [Quota Information](https://carto.com/docs/carto-engine/dataservices-api/quota-information/) section for details and recommendations about to quota consumption._
You can use the isoline functions to retrieve, for example, isochrone lines from a certain location, specifying the mode and the ranges that will define each of the isolines. The following query calculates isolines for areas that are 5, 10 and 15 minutes (300, 600 and 900 seconds, respectively) away from the location by following a path defined by car routing and inserts them into a table.
```bash
https://{username}.cartodb.com/api/v2/sql?q=INSERT INTO {table} (the_geom) SELECT the_geom FROM cdb_isodistance('POINT(-3.70568 40.42028)'::geometry, 'car', ARRAY[300, 600, 900]::integer[])&api_key={api_key}
https://{username}.carto.com/api/v2/sql?q=INSERT INTO {table} (the_geom) SELECT the_geom FROM cdb_isodistance('POINT(-3.70568 40.42028)'::geometry, 'car', ARRAY[300, 600, 900]::integer[])&api_key={api_key}
```
The following functions provide an isoline generator service, based on time or distance. This service uses the isolines service defined for your account. The default service limits the usage of displayed polygons represented on top of [HERE](https://developer.here.com/coverage-info) maps.

View File

@ -1,32 +1,32 @@
# Overview
By using CartoDB libraries and the SQL API, you can apply location data services to your maps with unique data services functions. These functions are integrated with a number of internal and external services, enabling you to programatically customize subsets of data for your visualizations. These features are useful for geospatial analysis and the results can be saved, and stored, for additional location data service operations.
By using CARTO libraries and the SQL API, you can apply location data services to your maps with unique data services functions. These functions are integrated with a number of internal and external services, enabling you to programatically customize subsets of data for your visualizations. These features are useful for geospatial analysis and the results can be saved, and stored, for additional location data service operations.
**Note:** Based on your account plan, some of these data services are subject to different [quota limitations](http://docs.cartodb.com/cartodb-platform/dataservices-api/quota-information/#quota-information).
**Note:** Based on your account plan, some of these data services are subject to different [quota limitations](https://carto.com/docs/carto-engine/dataservices-api/quota-information/#quota-information).
_The Data Services API is collaborating with [Mapzen](https://mapzen.com/), and several other geospatial service providers, in order to supply the best location data services from within our CartoDB Platform._
_The Data Services API is collaborating with [Mapzen](https://mapzen.com/), and several other geospatial service providers, in order to supply the best location data services from within our CARTO Engine._
## Data Services Integration
By using the SQL API to query the Data Services API functions, you can manage specific operations and the corresponding geometries (a `polygon` or a `point`), according to the input information.
The Data Services API decouples the geocoding and isoline services from the CartoDB Editor. The API allows you to geocode data (from single rows, complete datasets, or simple inputs) and to perform trade areas analysis (computing isodistances or isochrones) programatically, through authenticated requests.
The Data Services API decouples the geocoding and isoline services from the CARTO Editor. The API allows you to geocode data (from single rows, complete datasets, or simple inputs) and to perform trade areas analysis (computing isodistances or isochrones) programatically, through authenticated requests.
The geometries provided by this API are projected in the projection [WGS 84 SRID 4326](http://spatialreference.org/ref/epsg/wgs-84/).
**Note:** The Data Services API [geocoding functions](http://docs.cartodb.com/cartodb-platform/dataservices-api/geocoding-functions/#geocoding-functions) return different types of geometries (points or polygons) as result of different geocoding processes. The CartoDB Platform does not support multi-geometry layers or datasets, therefore you must confirm that you are using consistent geometry types inside a table, to avoid future conflicts in your map visualization.
**Note:** The Data Services API [geocoding functions](https://carto.com/docs/carto-engine/dataservices-api/geocoding-functions/#geocoding-functions) return different types of geometries (points or polygons) as result of different geocoding processes. The CARTO Engine does not support multi-geometry layers or datasets, therefore you must confirm that you are using consistent geometry types inside a table, to avoid future conflicts in your map visualization.
### Best Practices
_Be mindful of the following usage notes when using the Data Services functions with the SQL API:_
It is discouraged to use the SELECT operation with the Data Services API functions in your map layers, as these type of queries consume quota when rendering tiles for your live map views. It may also result in sync performance issues, due to executing multiple requests to the API each time your map is viewed. See details about [Quota Consumption](http://docs.cartodb.com/cartodb-platform/dataservices-api/quota-information/#quota-consumption).
It is discouraged to use the SELECT operation with the Data Services API functions in your map layers, as these type of queries consume quota when rendering tiles for your live map views. It may also result in sync performance issues, due to executing multiple requests to the API each time your map is viewed. See details about [Quota Consumption](https://carto.com/docs/carto-engine/dataservices-api/quota-information/#quota-consumption).
The Data Services API is **recommended** to be used with INSERT or UPDATE operations, for applying location data to your tables. While SELECT (retrieve) is standard for SQL API requests, be mindful of quota consumption and use INSERT (to insert a new record) or UPDATE (to update an existing record), for best practices.
## Authentication
All requests performed to the CartoDB Data Services API must be authenticated with the user API Key. For more information about where to find your API Key, and how to authenticate your SQL API requests, view the [SQL API authentication](/cartodb-platform/sql-api/authentication/) documentation.
All requests performed to the CARTO Data Services API must be authenticated with the user API Key. For more information about where to find your API Key, and how to authenticate your SQL API requests, view the [SQL API authentication](/carto-engine/sql-api/authentication/) documentation.
## Errors
@ -40,10 +40,10 @@ Errors are described in the response of the request. An example is as follows:
}
```
Since the Data Services API is used on top of the CartoDB SQL API, you can refer to the [Making calls to the SQL API](/cartodb-platform/sql-api/making-calls/) documentation for help debugging your SQL errors.
Since the Data Services API is used on top of the CARTO SQL API, you can refer to the [Making calls to the SQL API](https://carto.com/docs/carto-engine/sql-api/making-calls/) documentation for help debugging your SQL errors.
If the requested information is not in the CartoDB geocoding database, or if CartoDB is unable to recognize your input and match it with a result, the geocoding function returns `null` as a result.
If the requested information is not in the CARTO geocoding database, or if CARTO is unable to recognize your input and match it with a result, the geocoding function returns `null` as a result.
## Limits
Usage of the Data Services API is subject to the CartoDB SQL API limits, stated in our [Terms of Service](https://cartodb.com/terms/#excessive).
Usage of the Data Services API is subject to the CARTO SQL API limits, stated in our [Terms of Service](https://carto.com/terms/#excessive).

View File

@ -1,6 +1,6 @@
# Quota Information
**Based on your account plan, some of the Data Services API functions are subject to quota limitations and extra fees may apply.** View our [terms and conditions](https://cartodb.com/terms/), or [contact us](mailto:sales@cartodb.com) for details about which functions require service credits to your account.
**Based on your account plan, some of the Data Services API functions are subject to quota limitations and extra fees may apply.** View our [terms and conditions](https://carto.com/terms/), or [contact us](mailto:sales@carto.com) for details about which functions require service credits to your account.
## Quota Consumption

View File

@ -1,6 +1,6 @@
# Routing Functions
Routing is the navigation from a defined start location to a defined end location. The calculated results are displayed as turn-by-turn directions on your map, based on the transportation mode that you specified. Routing services through CartoDB are available by using the available functions in the Data Services API.
Routing is the navigation from a defined start location to a defined end location. The calculated results are displayed as turn-by-turn directions on your map, based on the transportation mode that you specified. Routing services through CARTO are available by using the available functions in the Data Services API.
## cdb_route_point_to_point(_origin geometry(Point), destination geometry(Point), mode text, [options text[], units text]_)

View File

@ -1,6 +1,6 @@
# Segmentation Functions
The Segmentation Snapshot functions enable you to determine the pre-calculated population segment for a location. Segmentation is a method that divides a populations into subclassifications based on common traits. For example, you can take the a store location and determine what classification of population exists around that location. If you need help creating coordinates from addresses, see the [Geocoding Functions](/cartodb-platform/dataservices-api/geocoding-functions/) documentation.
The Segmentation Snapshot functions enable you to determine the pre-calculated population segment for a location. Segmentation is a method that divides a populations into subclassifications based on common traits. For example, you can take the a store location and determine what classification of population exists around that location. If you need help creating coordinates from addresses, see the [Geocoding Functions](https://carto.com/docs/carto-engine/dataservices-api/geocoding-functions/) documentation.
_**Note:** The Segmentation Snapshot functions are only available for the United States. Our first release (May 18, 2016) is derived from Census 2010 variables. Our next release will be based on Census 2014 data. For the latest information, see the [Open Segments](https://github.com/CartoDB/open-segments) project repository._
@ -158,7 +158,7 @@ The possible segments are:
### Examples
```bash
https://{username}.cartodb.com/api/v2/sql?q=SELECT * FROM
https://{username}.carto.com/api/v2/sql?q=SELECT * FROM
OBS_GetSegmentSnapshot({{point geometry}})
```
@ -168,14 +168,14 @@ __Get the Segmentation Snapshot around the MGM Grand__
```bash
https://{username}.cartodb.com/api/v2/sql?q=SELECT * FROM
https://{username}.carto.com/api/v2/sql?q=SELECT * FROM
OBS_GetSegmentSnapshot(CDB_LatLng(36.10222, -115.169516))
```
__Get the Segmentation Snapshot at CartoDB's NYC HQ__
__Get the Segmentation Snapshot at CARTO's NYC HQ__
```bash
https://{username}.cartodb.com/api/v2/sql?q=SELECT * FROM
https://{username}.carto.com/api/v2/sql?q=SELECT * FROM
OBS_GetSegmentSnapshot(CDB_LatLng(40.704512, -73.936669))
```

View File

@ -1,15 +1,15 @@
# CartoDB dataservices API server extension
Postgres extension for the CartoDB dataservices API, server side.
# CARTO Data Services API server extension
Postgres extension for the CARTO Data Services API, server side.
## Dependencies
This extension is thought to be used on top of CartoDB geocoder extension, for the internal geocoder.
This extension is thought to be used on top of CARTO geocoder extension, for the internal geocoder.
The following is a non-comprehensive list of dependencies:
- Postgres 9.3+
- Postgis extension
- Schema triggers extension
- CartoDB extension
- cartodb-postgresql CARTO extension
## Installation into the db cluster
This requires root privileges
@ -28,7 +28,7 @@ One-liner:
sudo PGUSER=postgres make all install installcheck
```
## Install onto a cartodb user's database
## Install onto a CARTO user's database
Remember that **is mandatory to install it on top of cdb_geocoder**

View File

@ -0,0 +1,133 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.12.0'" to load this file. \quit
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isolines(
username TEXT,
orgname TEXT,
isotype TEXT,
source geometry(Geometry, 4326),
mode TEXT,
data_range integer[],
options text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
import json
from cartodb_services.mapzen import MatrixClient
from cartodb_services.mapzen import MapzenIsolines
from cartodb_services.metrics import QuotaService
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_mapzen_isolines_routing_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)]
# -- Check the quota
quota_service = QuotaService(user_mapzen_isolines_routing_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
try:
client = MatrixClient(user_mapzen_isolines_routing_config.mapzen_matrix_api_key)
mapzen_isolines = MapzenIsolines(client)
if source:
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
origin = {'lat': lat, 'lon': lon}
else:
raise Exception('source is NULL')
# -- TODO Support options properly
isolines = {}
if isotype == 'isodistance':
for r in data_range:
isoline = mapzen_isolines.calculate_isodistance(origin, mode, r)
isolines[r] = (isoline)
elif isotype == 'isochrone':
for r in data_range:
isoline = mapzen_isolines.calculate_isochrone(origin, mode, r)
isolines[r] = (isoline)
result = []
for r in data_range:
# -- TODO encapsulate this block into a func/method
locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
wkt_coordinates = ','.join(["%f %f" % (l['lon'], l['lat']) for l in locations])
sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates)
multipolygon = plpy.execute(sql, 1)[0]['geom']
result.append([source, r, multipolygon])
quota_service.increment_success_service_use()
quota_service.increment_isolines_service_use(len(isolines))
return result
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to obtain isolines using mapzen: {0}'.format(e)
plpy.debug(traceback.format_tb(traceback_))
raise e
#plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu SECURITY DEFINER;
-- mapzen isodistance
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_mapzen_isolines_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)]
type = 'isodistance'
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options])
isolines = []
for element in result:
isoline = element['isoline']
isoline = isoline.translate(None, "()").split(',')
isolines.append(isoline)
return isolines
$$ LANGUAGE plpythonu;
-- mapzen isochrones
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_mapzen_isolines_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)]
type = 'isochrone'
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options])
isolines = []
for element in result:
isoline = element['isoline']
isoline = isoline.translate(None, "()").split(',') #--TODO what is this for?
isolines.append(isoline)
return isolines
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_mapzen_isolines_config(username text, orgname text)
RETURNS boolean AS $$
cache_key = "user_mapzen_isolines_routing_config_{0}".format(username)
if cache_key in GD:
return False
else:
from cartodb_services.metrics import MapzenIsolinesRoutingConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
mapzen_isolines_config = MapzenIsolinesRoutingConfig(redis_conn, plpy, username, orgname)
GD[cache_key] = mapzen_isolines_config
return True
$$ LANGUAGE plpythonu SECURITY DEFINER;

View File

@ -0,0 +1,8 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.11.0'" to load this file. \quit
DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_mapzen_isolines(text, text, text, geometry(Geometry, 4326), text, integer[], text[]);
DROP FUNCTION IF EXISTS cdb_dataservices_server.cdb_mapzen_isodistance(TEXT, TEXT, geometry(Geometry, 4326), TEXT, integer[], text[]);
DROP FUNCTION IF EXISTS cdb_dataservices_server.cdb_mapzen_isochrone(TEXT, TEXT, geometry(Geometry, 4326), TEXT, integer[], text[]);
DROP FUNCTION IF EXISTS cdb_dataservices_server._get_mapzen_isolines_config(text, text);

File diff suppressed because it is too large Load Diff

View File

@ -1,5 +1,5 @@
comment = 'CartoDB dataservices server extension'
default_version = '0.11.0'
default_version = '0.12.0'
requires = 'plpythonu, plproxy, postgis, cdb_geocoder'
superuser = true
schema = cdb_dataservices_server

View File

@ -0,0 +1,101 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.11.0'" to load this file. \quit
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
if user_geocoder_config.heremaps_geocoder:
here_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
else:
plpy.error('Here geocoder is not available for your account.')
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
if user_geocoder_config.google_geocoder:
google_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
else:
plpy.error('Google geocoder is not available for your account.')
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
# The configuration is retrieved but no checks are performed on it
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_mapzen_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_mapzen_geocoder_config_{0}".format(username)]
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(mapzen_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
from cartodb_services.mapzen import MapzenGeocoder
from cartodb_services.mapzen.types import country_to_iso3
from cartodb_services.metrics import QuotaService
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_mapzen_geocoder_config = GD["user_mapzen_geocoder_config_{0}".format(username)]
quota_service = QuotaService(user_mapzen_geocoder_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
try:
geocoder = MapzenGeocoder(user_mapzen_geocoder_config.mapzen_api_key)
country_iso3 = None
if country:
country_iso3 = country_to_iso3(country)
coordinates = geocoder.geocode(searchtext=searchtext, city=city,
state_province=state_province,
country=country_iso3)
if coordinates:
quota_service.increment_success_service_use()
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
return point['st_setsrid']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using mapzen geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_mapzen_geocoder_config(username text, orgname text)
RETURNS boolean AS $$
cache_key = "user_mapzen_geocoder_config_{0}".format(username)
if cache_key in GD:
return False
else:
from cartodb_services.metrics import MapzenGeocoderConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
mapzen_geocoder_config = MapzenGeocoderConfig(redis_conn, plpy, username, orgname)
GD[cache_key] = mapzen_geocoder_config
return True
$$ LANGUAGE plpythonu SECURITY DEFINER;

View File

@ -0,0 +1,46 @@
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.10.0'" to load this file. \quit
DROP FUNCTION IF EXISTS cdb_dataservices_server.cdb_here_geocode_street_point(TEXT, TEXT, TEXT, TEXT, TEXT, TEXT);
DROP FUNCTION IF EXISTS cdb_dataservices_server.cdb_google_geocode_street_point(TEXT, TEXT, TEXT, TEXT, TEXT, TEXT);
DROP FUNCTION IF EXISTS cdb_dataservices_server.cdb_mapzen_geocode_street_point(TEXT, TEXT, TEXT, TEXT, TEXT, TEXT);
DROP FUNCTION IF EXISTS cdb_dataservices_server._get_mapzen_geocoder_config(text, text);
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
from cartodb_services.mapzen import MapzenGeocoder
from cartodb_services.mapzen.types import country_to_iso3
from cartodb_services.metrics import QuotaService
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
quota_service = QuotaService(user_geocoder_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
try:
geocoder = MapzenGeocoder(user_geocoder_config.mapzen_api_key)
country_iso3 = None
if country:
country_iso3 = country_to_iso3(country)
coordinates = geocoder.geocode(searchtext=searchtext, city=city,
state_province=state_province,
country=country_iso3)
if coordinates:
quota_service.increment_success_service_use()
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
return point['st_setsrid']
else:
quota_service.increment_empty_service_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to geocode using mapzen geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu;

View File

@ -26,6 +26,20 @@ RETURNS boolean AS $$
return True
$$ LANGUAGE plpythonu SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_mapzen_isolines_config(username text, orgname text)
RETURNS boolean AS $$
cache_key = "user_mapzen_isolines_routing_config_{0}".format(username)
if cache_key in GD:
return False
else:
from cartodb_services.metrics import MapzenIsolinesRoutingConfig
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
mapzen_isolines_config = MapzenIsolinesRoutingConfig(redis_conn, plpy, username, orgname)
GD[cache_key] = mapzen_isolines_config
return True
$$ LANGUAGE plpythonu SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_internal_geocoder_config(username text, orgname text)
RETURNS boolean AS $$
cache_key = "user_internal_geocoder_config_{0}".format(username)

View File

@ -53,3 +53,74 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isolines(
username TEXT,
orgname TEXT,
isotype TEXT,
source geometry(Geometry, 4326),
mode TEXT,
data_range integer[],
options text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
import json
from cartodb_services.mapzen import MatrixClient
from cartodb_services.mapzen import MapzenIsolines
from cartodb_services.metrics import QuotaService
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_mapzen_isolines_routing_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)]
# -- Check the quota
quota_service = QuotaService(user_mapzen_isolines_routing_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reached the limit of your quota')
try:
client = MatrixClient(user_mapzen_isolines_routing_config.mapzen_matrix_api_key)
mapzen_isolines = MapzenIsolines(client)
if source:
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
origin = {'lat': lat, 'lon': lon}
else:
raise Exception('source is NULL')
# -- TODO Support options properly
isolines = {}
if isotype == 'isodistance':
for r in data_range:
isoline = mapzen_isolines.calculate_isodistance(origin, mode, r)
isolines[r] = (isoline)
elif isotype == 'isochrone':
for r in data_range:
isoline = mapzen_isolines.calculate_isochrone(origin, mode, r)
isolines[r] = (isoline)
result = []
for r in data_range:
# -- TODO encapsulate this block into a func/method
locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
wkt_coordinates = ','.join(["%f %f" % (l['lon'], l['lat']) for l in locations])
sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates)
multipolygon = plpy.execute(sql, 1)[0]['geom']
result.append([source, r, multipolygon])
quota_service.increment_success_service_use()
quota_service.increment_isolines_service_use(len(isolines))
return result
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_service_use()
error_msg = 'There was an error trying to obtain isolines using mapzen: {0}'.format(e)
plpy.debug(traceback.format_tb(traceback_))
raise e
#plpy.error(error_msg)
finally:
quota_service.increment_total_service_use()
$$ LANGUAGE plpythonu SECURITY DEFINER;

View File

@ -19,3 +19,23 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
return isolines
$$ LANGUAGE plpythonu;
-- mapzen isodistance
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_mapzen_isolines_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)]
type = 'isodistance'
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options])
isolines = []
for element in result:
isoline = element['isoline']
isoline = isoline.translate(None, "()").split(',')
isolines.append(isoline)
return isolines
$$ LANGUAGE plpythonu;

View File

@ -19,3 +19,24 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
return isolines
$$ LANGUAGE plpythonu;
-- mapzen isochrones
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
RETURNS SETOF cdb_dataservices_server.isoline AS $$
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_dataservices_server._get_mapzen_isolines_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_isolines_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)]
type = 'isochrone'
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options])
isolines = []
for element in result:
isoline = element['isoline']
isoline = isoline.translate(None, "()").split(',') #--TODO what is this for?
isolines.append(isoline)
return isolines
$$ LANGUAGE plpythonu;

View File

@ -1,6 +1,6 @@
# CartoDB dataservices API python module
This directory contains the python library used by the server side of CartoDB LDS (Location Data Services).
This directory contains the python library used by the server side of CARTO LDS (Location Data Services).
It is used from pl/python functions contained in the `cdb_dataservices_server` extension. It goes hand in hand with the extension so please consider running the integration tests.

View File

@ -1,2 +1,4 @@
from routing import MapzenRouting, MapzenRoutingResponse
from isolines import MapzenIsolines
from geocoder import MapzenGeocoder
from matrix_client import MatrixClient

View File

@ -0,0 +1,160 @@
from math import cos, sin, tan, sqrt, pi, radians, degrees, asin, atan2
import logging
class MapzenIsolines:
NUMBER_OF_ANGLES = 24
MAX_ITERS = 5
TOLERANCE = 0.1
EARTH_RADIUS_METERS = 6367444
def __init__(self, matrix_client):
self._matrix_client = matrix_client
"""Get an isochrone using mapzen API.
The implementation tries to sick close to the SQL API:
cdb_isochrone(source geometry, mode text, range integer[], [options text[]]) -> SETOF isoline
But this calculates just one isoline.
Args:
origin dict containing {lat: y, lon: x}
transport_mode string, for the moment just "car" or "walk"
isorange int range of the isoline in seconds
Returns:
Array of {lon: x, lat: y} as a representation of the isoline
"""
def calculate_isochrone(self, origin, transport_mode, time_range):
if transport_mode == 'walk':
max_speed = 3.3333333 # In m/s, assuming 12km/h walking speed
costing_model = 'pedestrian'
elif transport_mode == 'car':
max_speed = 41.67 # In m/s, assuming 140km/h max speed
costing_model = 'auto'
else:
raise NotImplementedError('car and walk are the only supported modes for the moment')
upper_rmax = max_speed * time_range # an upper bound for the radius
return self.calculate_isoline(origin, costing_model, time_range, upper_rmax, 'time')
"""Get an isodistance using mapzen API.
Args:
origin dict containing {lat: y, lon: x}
transport_mode string, for the moment just "car" or "walk"
isorange int range of the isoline in seconds
Returns:
Array of {lon: x, lat: y} as a representation of the isoline
"""
def calculate_isodistance(self, origin, transport_mode, distance_range):
if transport_mode == 'walk':
costing_model = 'pedestrian'
elif transport_mode == 'car':
costing_model = 'auto'
else:
raise NotImplementedError('car and walk are the only supported modes for the moment')
upper_rmax = distance_range # an upper bound for the radius, going in a straight line
return self.calculate_isoline(origin, costing_model, distance_range, upper_rmax, 'distance', 1000.0)
"""Get an isoline using mapzen API.
The implementation tries to sick close to the SQL API:
cdb_isochrone(source geometry, mode text, range integer[], [options text[]]) -> SETOF isoline
But this calculates just one isoline.
Args:
origin dict containing {lat: y, lon: x}
costing_model string "auto" or "pedestrian"
isorange int Range of the isoline in seconds
upper_rmax float An upper bound for the binary search
cost_variable string Variable to optimize "time" or "distance"
unit_factor float A factor to adapt units of isorange (meters) and units of distance (km)
Returns:
Array of {lon: x, lat: y} as a representation of the isoline
"""
def calculate_isoline(self, origin, costing_model, isorange, upper_rmax, cost_variable, unit_factor=1.0):
# NOTE: not for production
#logging.basicConfig(level=logging.DEBUG, filename='/tmp/isolines.log')
#logging.basicConfig(level=logging.DEBUG)
logging.debug('origin = %s' % origin)
logging.debug('costing_model = %s' % costing_model)
logging.debug('isorange = %d' % isorange)
# Formally, a solution is an array of {angle, radius, lat, lon, cost} with cardinality NUMBER_OF_ANGLES
# we're looking for a solution in which abs(cost - isorange) / isorange <= TOLERANCE
# Initial setup
angles = self._get_angles(self.NUMBER_OF_ANGLES) # array of angles
rmax = [upper_rmax] * self.NUMBER_OF_ANGLES
rmin = [0.0] * self.NUMBER_OF_ANGLES
location_estimates = [self._calculate_dest_location(origin, a, upper_rmax / 2.0) for a in angles]
# Iterate to refine the first solution
for i in xrange(0, self.MAX_ITERS):
# Calculate the "actual" cost for each location estimate.
# NOTE: sometimes it cannot calculate the cost and returns None.
# Just assume isorange and stop the calculations there
response = self._matrix_client.one_to_many([origin] + location_estimates, costing_model)
costs = [None] * self.NUMBER_OF_ANGLES
for idx, c in enumerate(response['one_to_many'][0][1:]):
if c[cost_variable]:
costs[idx] = c[cost_variable]*unit_factor
else:
costs[idx] = isorange
logging.debug('i = %d, costs = %s' % (i, costs))
errors = [(cost - isorange) / float(isorange) for cost in costs]
max_abs_error = max([abs(e) for e in errors])
if max_abs_error <= self.TOLERANCE:
# good enough, stop there
break
# let's refine the solution, binary search
for j in xrange(0, self.NUMBER_OF_ANGLES):
if abs(errors[j]) > self.TOLERANCE:
if errors[j] > 0:
rmax[j] = (rmax[j] + rmin[j]) / 2.0
else:
rmin[j] = (rmax[j] + rmin[j]) / 2.0
location_estimates[j] = self._calculate_dest_location(origin, angles[j], (rmax[j]+rmin[j])/2.0)
# delete points that got None
location_estimates_filtered = []
for i, c in enumerate(costs):
if c <> isorange:
location_estimates_filtered.append(location_estimates[i])
return location_estimates_filtered
# NOTE: all angles in calculations are in radians
def _get_angles(self, number_of_angles):
step = (2.0 * pi) / number_of_angles
return [(x * step) for x in xrange(0, number_of_angles)]
def _calculate_dest_location(self, origin, angle, radius):
origin_lat_radians = radians(origin['lat'])
origin_long_radians = radians(origin['lon'])
dest_lat_radians = asin(sin(origin_lat_radians) * cos(radius / self.EARTH_RADIUS_METERS) + cos(origin_lat_radians) * sin(radius / self.EARTH_RADIUS_METERS) * cos(angle))
dest_lng_radians = origin_long_radians + atan2(sin(angle) * sin(radius / self.EARTH_RADIUS_METERS) * cos(origin_lat_radians), cos(radius / self.EARTH_RADIUS_METERS) - sin(origin_lat_radians) * sin(dest_lat_radians))
return {
'lon': degrees(dest_lng_radians),
'lat': degrees(dest_lat_radians)
}

View File

@ -0,0 +1,43 @@
import requests
import json
class MatrixClient:
"""
A minimal client for Mapzen Time-Distance Matrix Service
Example:
client = MatrixClient('your_api_key')
locations = [{"lat":40.744014,"lon":-73.990508},{"lat":40.739735,"lon":-73.979713},{"lat":40.752522,"lon":-73.985015},{"lat":40.750117,"lon":-73.983704},{"lat":40.750552,"lon":-73.993519}]
costing = 'pedestrian'
print client.one_to_many(locations, costing)
"""
ONE_TO_MANY_URL = 'https://matrix.mapzen.com/one_to_many'
def __init__(self, matrix_key):
self._matrix_key = matrix_key
"""Get distances and times to a set of locations.
See https://mapzen.com/documentation/matrix/api-reference/
Args:
locations Array of {lat: y, lon: x}
costing Costing model to use
Returns:
A dict with one_to_many, units and locations
"""
def one_to_many(self, locations, costing):
request_params = {
'json': json.dumps({'locations': locations}),
'costing': costing,
'api_key': self._matrix_key
}
response = requests.get(self.ONE_TO_MANY_URL, params=request_params)
response.raise_for_status() # raise exception if not 200 OK
return response.json()

View File

@ -1,3 +1,3 @@
from config import GeocoderConfig, MapzenGeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatorySnapshotConfig, ObservatoryConfig
from config import GeocoderConfig, MapzenGeocoderConfig, IsolinesRoutingConfig, MapzenIsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatorySnapshotConfig, ObservatoryConfig
from quota import QuotaService
from user import UserMetricsService

View File

@ -144,7 +144,7 @@ class MapzenGeocoderConfig(ServiceConfig):
self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE])
self._cost_per_hit = 0
except Exception as e:
raise ConfigException("Malformed config for Mapzen geocoder: {1}".format(key, e))
raise ConfigException("Malformed config for Mapzen geocoder: {0}".format(e))
@property
def service_type(self):
@ -241,6 +241,40 @@ class IsolinesRoutingConfig(ServiceConfig):
return self._geocoder_type == self.GOOGLE_GEOCODER
class MapzenIsolinesRoutingConfig(ServiceConfig):
PERIOD_END_DATE = 'period_end_date'
def __init__(self, redis_connection, db_conn, username, orgname=None):
super(MapzenIsolinesRoutingConfig, self).__init__(redis_connection, db_conn,
username, orgname)
try:
self._mapzen_matrix_api_key = self._db_config.mapzen_matrix_api_key
self._isolines_quota = self._db_config.mapzen_matrix_monthly_quota
self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE])
except Exception as e:
raise ConfigException("Malformed config for Mapzen isolines: {0}".format(e))
@property
def service_type(self):
return 'mapzen_isolines'
@property
def isolines_quota(self):
return self._isolines_quota
@property
def soft_isolines_limit(self):
return False
@property
def period_end_date(self):
return self._period_end_date
@property
def mapzen_matrix_api_key(self):
return self._mapzen_matrix_api_key
class InternalGeocoderConfig(ServiceConfig):
def __init__(self, redis_connection, db_conn, username, orgname=None):
@ -438,6 +472,8 @@ class ServicesDBConfig:
raise ConfigException('Mapzen configuration missing')
else:
mapzen_conf = json.loads(mapzen_conf_json)
self._mapzen_matrix_api_key = mapzen_conf['matrix']['api_key']
self._mapzen_matrix_quota = mapzen_conf['matrix']['monthly_quota']
self._mapzen_routing_api_key = mapzen_conf['routing']['api_key']
self._mapzen_routing_quota = mapzen_conf['routing']['monthly_quota']
self._mapzen_geocoder_api_key = mapzen_conf['geocoder']['api_key']
@ -492,6 +528,14 @@ class ServicesDBConfig:
def heremaps_geocoder_cost_per_hit(self):
return self._heremaps_geocoder_cost_per_hit
@property
def mapzen_matrix_api_key(self):
return self._mapzen_matrix_api_key
@property
def mapzen_matrix_monthly_quota(self):
return self._mapzen_matrix_quota
@property
def mapzen_routing_api_key(self):
return self._mapzen_routing_api_key

View File

@ -70,6 +70,9 @@ class QuotaChecker:
elif re.match('here_isolines',
self._user_service_config.service_type) is not None:
return self.__check_isolines_quota()
elif re.match('mapzen_isolines',
self._user_service_config.service_type) is not None:
return self.__check_isolines_quota()
elif re.match('routing_mapzen',
self._user_service_config.service_type) is not None:
return self.__check_routing_quota()

View File

@ -10,7 +10,7 @@ from setuptools import setup, find_packages
setup(
name='cartodb_services',
version='0.6.5',
version='0.7.0',
description='CartoDB Services API Python Library',

View File

@ -65,7 +65,7 @@ def _plpy_execute_side_effect(*args, **kwargs):
if args[0] == "SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as conf":
return [{'conf': '{"geocoder": {"app_id": "app_id", "app_code": "code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "app_id", "app_code": "code"}}'}]
elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('mapzen_conf') as conf":
return [{'conf': '{"routing": {"api_key": "api_key_rou", "monthly_quota": 1500000}, "geocoder": {"api_key": "api_key_geo", "monthly_quota": 1500000}}'}]
return [{'conf': '{"routing": {"api_key": "api_key_rou", "monthly_quota": 1500000}, "geocoder": {"api_key": "api_key_geo", "monthly_quota": 1500000}, "matrix": {"api_key": "api_key_mat", "monthly_quota": 1500000}}'}]
elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('logger_conf') as conf":
return [{'conf': '{"geocoder_log_path": "/dev/null"}'}]
elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('data_observatory_conf') as conf":

View File

@ -11,7 +11,7 @@ requests_mock.Mocker.TEST_PREFIX = 'test_'
@requests_mock.Mocker()
class GoogleGeocoderTestCase(unittest.TestCase):
class MapzenGeocoderTestCase(unittest.TestCase):
MAPZEN_GEOCODER_URL = 'https://search.mapzen.com/v1/search'
EMPTY_RESPONSE = """{

View File

@ -0,0 +1,76 @@
import unittest
from cartodb_services.mapzen import MapzenIsolines
from math import radians, cos, sin, asin, sqrt
"""
This file is basically a sanity test on the algorithm.
It uses a mocked client, which returns the cost based on a very simple model:
just proportional to the distance from origin to the target point.
"""
class MatrixClientMock():
def __init__(self, speed):
"""
Sets up the mock with a speed in km/h
"""
self._speed = speed
def one_to_many(self, locations, costing):
origin = locations[0]
distances = [self._distance(origin, l) for l in locations]
response = {
'one_to_many': [
[
{
'distance': distances[i] * self._speed,
'time': distances[i] / self._speed * 3600,
'to_index': i,
'from_index': 0
}
for i in xrange(0, len(distances))
]
],
'units': 'km',
'locations': [
locations
]
}
return response
def _distance(self, a, b):
"""
Calculate the great circle distance between two points
on the earth (specified in decimal degrees)
http://stackoverflow.com/questions/4913349/haversine-formula-in-python-bearing-and-distance-between-two-gps-points
Returns:
distance in meters
"""
# convert decimal degrees to radians
lon1, lat1, lon2, lat2 = map(radians, [a['lon'], a['lat'], b['lon'], b['lat']])
# haversine formula
dlon = lon2 - lon1
dlat = lat2 - lat1
a = sin(dlat/2)**2 + cos(lat1) * cos(lat2) * sin(dlon/2)**2
c = 2 * asin(sqrt(a))
r = 6371 # Radius of earth in kilometers. Use 3956 for miles
return c * r
class MapzenIsolinesTestCase(unittest.TestCase):
def setUp(self):
speed = 4 # in km/h
matrix_client = MatrixClientMock(speed)
self.mapzen_isolines = MapzenIsolines(matrix_client)
def test_calculate_isochrone(self):
origin = {"lat":40.744014,"lon":-73.990508}
transport_mode = 'walk'
isorange = 10 * 60 # 10 minutes
solution = self.mapzen_isolines.calculate_isochrone(origin, transport_mode, isorange)