cartodb4/dataservices-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #62 from CartoDB/redis_quota_usage_and_metrics

Browse Source 
Geocoder API v2: Nokia geocoder
This commit is contained in:
Mario de Frutos 2016-01-25 18:08:11 +01:00
commit e4f327f890
110 changed files with 3996 additions and 410 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
README.md
View File

@ -1,2 +1,21 @@
# geocoder-api
The CartoDB Geocoder SQL API (server and client FTM)
### Deploy instructions
Steps to deploy a new Geocoder API version :
- Deploy new version of geocoder API to all servers
- Update the server user using: ALTER EXTENSION cdb_geocoder_server UPDATE TO '<CURRENT_VERSION>';
- Update the python dependencies if needed: **cartodb_geocoder** and **heremaps**
- Add the needed config in the `cdb_conf` table:
- `redis_metadata_config` and `redis_metrics_conf`
- `{"sentinel_host": "localhost", "sentinel_port": 26739, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}`
- `heremaps_conf`
- `{"app_id": "APP_ID", "app_code": "APP_CODE"}`
- Deploy the client to all the servers with the new version
- Deploy the editor with the new geocoder api version changed (https://github.com/CartoDB/cartodb/blob/master/app/models/user/db_service.rb#L18)
- Execute the rails task to update first the CartoDB team organizaton to test in production
- `RAILS_ENV=production bundle exec rake cartodb:db:configure_geocoder_extension_for_organizations['team']`
- Check if all works perfectly for our team. If so, execute the rake tasks to update all the users and organizations:
- `RAILS_ENV=production bundle exec rake cartodb:db:configure_geocoder_extension_for_organizations['', true]`
- `RAILS_ENV=production bundle exec rake cartodb:db:configure_geocoder_extension_for_non_org_users['', true]`

4
client/.gitignore vendored
View File

@ -1,4 +1,8 @@
results/
regression.diffs
regression.out
20_public_functions.sql
30_plproxy_functions.sql
90_grant_execute.sql
cdb_geocoder_client--0.0.1.sql
cdb_geocoder_client--0.1.0.sql

23
client/Makefile
View File

@ -3,9 +3,22 @@
EXTENSION = cdb_geocoder_client
EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/")
DATA = $(EXTENSION)--$(EXTVERSION).sql
# The new version to be generated from templates
NEW_EXTENSION_ARTIFACT = $(EXTENSION)--$(EXTVERSION).sql
REGRESS = $(notdir $(basename $(wildcard sql/*test.sql)))
# DATA is a special variable used by postgres build infrastructure
# These are the files to be installed in the server shared dir,
# for installation from scratch, upgrades and downgrades.
# @see http://www.postgresql.org/docs/current/static/extend-pgxs.html
DATA = $(NEW_EXTENSION_ARTIFACT) \
cdb_geocoder_client--0.0.1.sql \
cdb_geocoder_client--0.1.0--0.0.1.sql \
cdb_geocoder_client--0.0.1--0.1.0.sql
REGRESS = $(notdir $(basename $(wildcard test/$(EXTVERSION)/sql/*test.sql)))
TEST_DIR = test/$(EXTVERSION)
REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)'
# postgres build stuff
PG_CONFIG = pg_config
@ -16,7 +29,7 @@ SOURCES_DATA_DIR = sql/$(EXTVERSION)
# The interface definition is used along with some templates to automatically generate code
RENDERER = ../sql-template-renderer
INTERFACE_FILE = ../interface.yaml
INTERFACE_FILE = ../interface_$(EXTVERSION).yaml
TEMPLATE_DIR = templates
TEMPLATE_FILES = $(wildcard $(TEMPLATE_DIR)/*.erb)
GENERATED_SQL_FILES = $(patsubst $(TEMPLATE_DIR)/%.erb, $(SOURCES_DATA_DIR)/%.sql, $(TEMPLATE_FILES))
@ -26,7 +39,7 @@ $(GENERATED_SQL_FILES): $(SOURCES_DATA_DIR)/%.sql: $(TEMPLATE_DIR)/%.erb $(INTER
SOURCES_DATA = $(wildcard $(SOURCES_DATA_DIR)/*.sql) $(GENERATED_SQL_FILES)
$(DATA): $(SOURCES_DATA)
$(NEW_EXTENSION_ARTIFACT): $(SOURCES_DATA)
rm -f $@
cat $(SOURCES_DATA_DIR)/*.sql >> $@
@ -34,5 +47,5 @@ all: $(DATA)
# Only meant for development time, do not use once a version is released
devclean:
rm -f $(DATA)
rm -f $(NEW_EXTENSION_ARTIFACT)
rm -f $(GENERATED_SQL_FILES)

60
client/cdb_geocoder_client--0.0.1--0.1.0.sql Normal file
View File

@ -0,0 +1,60 @@
--
-- Get entity config function
--
-- The purpose of this function is to retrieve the username and organization name from
-- a) schema where he/her is the owner in case is an organization user
-- b) entity_name from the cdb_conf database in case is a non organization user
CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_entity_config()
RETURNS record AS $$
DECLARE
result cdb_geocoder_client._entity_config;
is_organization boolean;
username text;
organization_name text;
BEGIN
SELECT cartodb.cdb_conf_getconf('user_config')->'is_organization' INTO is_organization;
IF is_organization IS NULL THEN
RAISE EXCEPTION 'User must have user configuration in the config table';
ELSIF is_organization = TRUE THEN
SELECT nspname
FROM pg_namespace s
LEFT JOIN pg_roles r ON s.nspowner = r.oid
WHERE r.rolname = session_user INTO username;
SELECT cartodb.cdb_conf_getconf('user_config')->>'entity_name' INTO organization_name;
ELSE
SELECT cartodb.cdb_conf_getconf('user_config')->>'entity_name' INTO username;
organization_name = NULL;
END IF;
result.username = username;
result.organization_name = organization_name;
RETURN result;
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_street_point_v2 (searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
username text;
orgname text;
BEGIN
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
RAISE EXCEPTION 'The api_key must be provided';
END IF;
SELECT u, o INTO username, orgname FROM cdb_geocoder_client._cdb_entity_config() AS (u text, o text);
-- JSON value stored "" is taken as literal
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
END IF;
SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country) INTO ret;
RETURN ret;
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_street_point_v2 (username text, organization_name text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL)
RETURNS Geometry AS $$
CONNECT cdb_geocoder_client._server_conn_str();
SELECT cdb_geocoder_server.cdb_geocode_street_point_v2 (username, organization_name, searchtext, city, state_province, country);
$$ LANGUAGE plproxy;
GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_street_point_v2(searchtext text, city text, state_province text, country text) TO publicuser;

359
client/cdb_geocoder_client--0.0.1.sql Normal file
View File

@ -0,0 +1,359 @@
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "CREATE EXTENSION cdb_geocoder_client" to load this file. \quit
--
-- Geocoder server connection config
--
-- The purpose of this function is provide to the PL/Proxy functions
-- the connection string needed to connect with the server
CREATE OR REPLACE FUNCTION cdb_geocoder_client._server_conn_str()
RETURNS text AS $$
DECLARE
db_connection_str text;
BEGIN
SELECT cartodb.cdb_conf_getconf('geocoder_server_config')->'connection_str' INTO db_connection_str;
SELECT trim(both '"' FROM db_connection_str) INTO db_connection_str;
RETURN db_connection_str;
END;
$$ LANGUAGE 'plpgsql';CREATE TYPE cdb_geocoder_client._entity_config AS (
username text,
organization_name text
);
--
-- Get entity config function
--
-- The purpose of this function is to retrieve the username and organization name from
-- a) schema where he/her is the owner in case is an organization user
-- b) entity_name from the cdb_conf database in case is a non organization user
CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_entity_config()
RETURNS record AS $$
DECLARE
result cdb_geocoder_client._entity_config;
is_organization boolean;
username text;
organization_name text;
BEGIN
SELECT cartodb.cdb_conf_getconf('user_config')->'is_organization' INTO is_organization;
IF is_organization IS NULL THEN
RAISE EXCEPTION 'User must have user configuration in the config table';
ELSIF is_organization = TRUE THEN
SELECT nspname
FROM pg_namespace s
LEFT JOIN pg_roles r ON s.nspowner = r.oid
WHERE r.rolname = session_user INTO username;
SELECT cartodb.cdb_conf_getconf('user_config')->'entity_name' INTO organization_name;
ELSE
SELECT cartodb.cdb_conf_getconf('user_config')->'entity_name' INTO username;
organization_name = NULL;
END IF;
result.username = username;
result.organization_name = organization_name;
RETURN result;
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;--
-- Public geocoder API function
--
-- These are the only ones with permissions to publicuser role
-- and should also be the only ones with SECURITY DEFINER
CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_admin0_polygon (country_name text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
username text;
orgname text;
BEGIN
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
RAISE EXCEPTION 'The api_key must be provided';
END IF;
SELECT u, o INTO username, orgname FROM cdb_geocoder_client._cdb_entity_config() AS (u text, o text);
-- JSON value stored "" is taken as literal
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
END IF;
SELECT cdb_geocoder_client._cdb_geocode_admin0_polygon(username, orgname, country_name) INTO ret;
RETURN ret;
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
--
-- Public geocoder API function
--
-- These are the only ones with permissions to publicuser role
-- and should also be the only ones with SECURITY DEFINER
CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_admin1_polygon (admin1_name text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
username text;
orgname text;
BEGIN
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
RAISE EXCEPTION 'The api_key must be provided';
END IF;
SELECT u, o INTO username, orgname FROM cdb_geocoder_client._cdb_entity_config() AS (u text, o text);
-- JSON value stored "" is taken as literal
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
END IF;
SELECT cdb_geocoder_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name) INTO ret;
RETURN ret;
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
--
-- Public geocoder API function
--
-- These are the only ones with permissions to publicuser role
-- and should also be the only ones with SECURITY DEFINER
CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_admin1_polygon (admin1_name text, country_name text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
username text;
orgname text;
BEGIN
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
RAISE EXCEPTION 'The api_key must be provided';
END IF;
SELECT u, o INTO username, orgname FROM cdb_geocoder_client._cdb_entity_config() AS (u text, o text);
-- JSON value stored "" is taken as literal
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
END IF;
SELECT cdb_geocoder_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name, country_name) INTO ret;
RETURN ret;
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
--
-- Public geocoder API function
--
-- These are the only ones with permissions to publicuser role
-- and should also be the only ones with SECURITY DEFINER
CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_namedplace_point (city_name text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
username text;
orgname text;
BEGIN
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
RAISE EXCEPTION 'The api_key must be provided';
END IF;
SELECT u, o INTO username, orgname FROM cdb_geocoder_client._cdb_entity_config() AS (u text, o text);
-- JSON value stored "" is taken as literal
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
END IF;
SELECT cdb_geocoder_client._cdb_geocode_namedplace_point(username, orgname, city_name) INTO ret;
RETURN ret;
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
--
-- Public geocoder API function
--
-- These are the only ones with permissions to publicuser role
-- and should also be the only ones with SECURITY DEFINER
CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_namedplace_point (city_name text, country_name text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
username text;
orgname text;
BEGIN
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
RAISE EXCEPTION 'The api_key must be provided';
END IF;
SELECT u, o INTO username, orgname FROM cdb_geocoder_client._cdb_entity_config() AS (u text, o text);
-- JSON value stored "" is taken as literal
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
END IF;
SELECT cdb_geocoder_client._cdb_geocode_namedplace_point(username, orgname, city_name, country_name) INTO ret;
RETURN ret;
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
--
-- Public geocoder API function
--
-- These are the only ones with permissions to publicuser role
-- and should also be the only ones with SECURITY DEFINER
CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_namedplace_point (city_name text, admin1_name text, country_name text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
username text;
orgname text;
BEGIN
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
RAISE EXCEPTION 'The api_key must be provided';
END IF;
SELECT u, o INTO username, orgname FROM cdb_geocoder_client._cdb_entity_config() AS (u text, o text);
-- JSON value stored "" is taken as literal
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
END IF;
SELECT cdb_geocoder_client._cdb_geocode_namedplace_point(username, orgname, city_name, admin1_name, country_name) INTO ret;
RETURN ret;
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
--
-- Public geocoder API function
--
-- These are the only ones with permissions to publicuser role
-- and should also be the only ones with SECURITY DEFINER
CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_postalcode_polygon (postal_code text, country_name text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
username text;
orgname text;
BEGIN
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
RAISE EXCEPTION 'The api_key must be provided';
END IF;
SELECT u, o INTO username, orgname FROM cdb_geocoder_client._cdb_entity_config() AS (u text, o text);
-- JSON value stored "" is taken as literal
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
END IF;
SELECT cdb_geocoder_client._cdb_geocode_postalcode_polygon(username, orgname, postal_code, country_name) INTO ret;
RETURN ret;
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
--
-- Public geocoder API function
--
-- These are the only ones with permissions to publicuser role
-- and should also be the only ones with SECURITY DEFINER
CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_postalcode_point (postal_code text, country_name text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
username text;
orgname text;
BEGIN
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
RAISE EXCEPTION 'The api_key must be provided';
END IF;
SELECT u, o INTO username, orgname FROM cdb_geocoder_client._cdb_entity_config() AS (u text, o text);
-- JSON value stored "" is taken as literal
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
END IF;
SELECT cdb_geocoder_client._cdb_geocode_postalcode_point(username, orgname, postal_code, country_name) INTO ret;
RETURN ret;
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
--
-- Public geocoder API function
--
-- These are the only ones with permissions to publicuser role
-- and should also be the only ones with SECURITY DEFINER
CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_ipaddress_point (ip_address text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
username text;
orgname text;
BEGIN
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
RAISE EXCEPTION 'The api_key must be provided';
END IF;
SELECT u, o INTO username, orgname FROM cdb_geocoder_client._cdb_entity_config() AS (u text, o text);
-- JSON value stored "" is taken as literal
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
END IF;
SELECT cdb_geocoder_client._cdb_geocode_ipaddress_point(username, orgname, ip_address) INTO ret;
RETURN ret;
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_admin0_polygon (username text, organization_name text, country_name text)
RETURNS Geometry AS $$
CONNECT cdb_geocoder_client._server_conn_str();
SELECT cdb_geocoder_server.cdb_geocode_admin0_polygon (username, organization_name, country_name);
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_admin1_polygon (username text, organization_name text, admin1_name text)
RETURNS Geometry AS $$
CONNECT cdb_geocoder_client._server_conn_str();
SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon (username, organization_name, admin1_name);
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_admin1_polygon (username text, organization_name text, admin1_name text, country_name text)
RETURNS Geometry AS $$
CONNECT cdb_geocoder_client._server_conn_str();
SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon (username, organization_name, admin1_name, country_name);
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_namedplace_point (username text, organization_name text, city_name text)
RETURNS Geometry AS $$
CONNECT cdb_geocoder_client._server_conn_str();
SELECT cdb_geocoder_server.cdb_geocode_namedplace_point (username, organization_name, city_name);
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_namedplace_point (username text, organization_name text, city_name text, country_name text)
RETURNS Geometry AS $$
CONNECT cdb_geocoder_client._server_conn_str();
SELECT cdb_geocoder_server.cdb_geocode_namedplace_point (username, organization_name, city_name, country_name);
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_namedplace_point (username text, organization_name text, city_name text, admin1_name text, country_name text)
RETURNS Geometry AS $$
CONNECT cdb_geocoder_client._server_conn_str();
SELECT cdb_geocoder_server.cdb_geocode_namedplace_point (username, organization_name, city_name, admin1_name, country_name);
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_postalcode_polygon (username text, organization_name text, postal_code text, country_name text)
RETURNS Geometry AS $$
CONNECT cdb_geocoder_client._server_conn_str();
SELECT cdb_geocoder_server.cdb_geocode_postalcode_polygon (username, organization_name, postal_code, country_name);
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_postalcode_point (username text, organization_name text, postal_code text, country_name text)
RETURNS Geometry AS $$
CONNECT cdb_geocoder_client._server_conn_str();
SELECT cdb_geocoder_server.cdb_geocode_postalcode_point (username, organization_name, postal_code, country_name);
$$ LANGUAGE plproxy;
CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_ipaddress_point (username text, organization_name text, ip_address text)
RETURNS Geometry AS $$
CONNECT cdb_geocoder_client._server_conn_str();
SELECT cdb_geocoder_server.cdb_geocode_ipaddress_point (username, organization_name, ip_address);
$$ LANGUAGE plproxy;
-- Make sure by default there are no permissions for publicuser
-- NOTE: this happens at extension creation time, as part of an implicit transaction.
REVOKE ALL PRIVILEGES ON SCHEMA cdb_geocoder_client FROM PUBLIC, publicuser CASCADE;
-- Grant permissions on the schema to publicuser (but just the schema)
GRANT USAGE ON SCHEMA cdb_geocoder_client TO publicuser;
-- Revoke execute permissions on all functions in the schema by default
REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_geocoder_client FROM PUBLIC, publicuser;
GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_admin0_polygon(country_name text) TO publicuser;
GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_admin1_polygon(admin1_name text) TO publicuser;
GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_admin1_polygon(admin1_name text, country_name text) TO publicuser;
GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_namedplace_point(city_name text) TO publicuser;
GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_namedplace_point(city_name text, country_name text) TO publicuser;
GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) TO publicuser;
GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_postalcode_polygon(postal_code text, country_name text) TO publicuser;
GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_postalcode_point(postal_code text, country_name text) TO publicuser;
GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_ipaddress_point(ip_address text) TO publicuser;

35
client/cdb_geocoder_client--0.1.0--0.0.1.sql Normal file
View File

@ -0,0 +1,35 @@
DROP FUNCTION IF EXISTS cdb_geocoder_client.cdb_geocode_street_point_v2 (text, text, text, text);
DROP FUNCTION IF EXISTS cdb_geocoder_client._cdb_geocode_street_point_v2 (text, text, text, text);
--
-- Get entity config function
--
-- The purpose of this function is to retrieve the username and organization name from
-- a) schema where he/her is the owner in case is an organization user
-- b) entity_name from the cdb_conf database in case is a non organization user
CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_entity_config()
RETURNS record AS $$
DECLARE
result cdb_geocoder_client._entity_config;
is_organization boolean;
username text;
organization_name text;
BEGIN
SELECT cartodb.cdb_conf_getconf('user_config')->'is_organization' INTO is_organization;
IF is_organization IS NULL THEN
RAISE EXCEPTION 'User must have user configuration in the config table';
ELSIF is_organization = TRUE THEN
SELECT nspname
FROM pg_namespace s
LEFT JOIN pg_roles r ON s.nspowner = r.oid
WHERE r.rolname = session_user INTO username;
SELECT cartodb.cdb_conf_getconf('user_config')->'entity_name' INTO organization_name;
ELSE
SELECT cartodb.cdb_conf_getconf('user_config')->'entity_name' INTO username;
organization_name = NULL;
END IF;
result.username = username;
result.organization_name = organization_name;
RETURN result;
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;

2
client/cdb_geocoder_client.control
View File

@ -1,6 +1,6 @@
# CartoDB geocoder client API extension
comment = 'CartoDB geocoder client API extension'
default_version = '0.0.1'
default_version = '0.1.0'
requires = 'plproxy, cartodb'
superuser = true
schema = cdb_geocoder_client

3
client/sql/0.0.1/.gitignore vendored
View File

@ -1,3 +0,0 @@
20_public_functions.sql
30_plproxy_functions.sql
90_grant_execute.sql

2
client/sql/0.1.0/00_header.sql Normal file
View File

@ -0,0 +1,2 @@
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "CREATE EXTENSION cdb_geocoder_client" to load this file. \quit

16
client/sql/0.1.0/10_geocoder_server_conn.sql Normal file
View File

@ -0,0 +1,16 @@
--
-- Geocoder server connection config
--
-- The purpose of this function is provide to the PL/Proxy functions
-- the connection string needed to connect with the server
CREATE OR REPLACE FUNCTION cdb_geocoder_client._server_conn_str()
RETURNS text AS $$
DECLARE
db_connection_str text;
BEGIN
SELECT cartodb.cdb_conf_getconf('geocoder_server_config')->'connection_str' INTO db_connection_str;
SELECT trim(both '"' FROM db_connection_str) INTO db_connection_str;
RETURN db_connection_str;
END;
$$ LANGUAGE 'plpgsql';

37
client/sql/0.1.0/15_config_management.sql Normal file
View File

@ -0,0 +1,37 @@
CREATE TYPE cdb_geocoder_client._entity_config AS (
username text,
organization_name text
);
--
-- Get entity config function
--
-- The purpose of this function is to retrieve the username and organization name from
-- a) schema where he/her is the owner in case is an organization user
-- b) entity_name from the cdb_conf database in case is a non organization user
CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_entity_config()
RETURNS record AS $$
DECLARE
result cdb_geocoder_client._entity_config;
is_organization boolean;
username text;
organization_name text;
BEGIN
SELECT cartodb.cdb_conf_getconf('user_config')->'is_organization' INTO is_organization;
IF is_organization IS NULL THEN
RAISE EXCEPTION 'User must have user configuration in the config table';
ELSIF is_organization = TRUE THEN
SELECT nspname
FROM pg_namespace s
LEFT JOIN pg_roles r ON s.nspowner = r.oid
WHERE r.rolname = session_user INTO username;
SELECT cartodb.cdb_conf_getconf('user_config')->>'entity_name' INTO organization_name;
ELSE
SELECT cartodb.cdb_conf_getconf('user_config')->>'entity_name' INTO username;
organization_name = NULL;
END IF;
result.username = username;
result.organization_name = organization_name;
RETURN result;
END;
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;

9
client/sql/0.1.0/80_permissions.sql Normal file
View File

@ -0,0 +1,9 @@
-- Make sure by default there are no permissions for publicuser
-- NOTE: this happens at extension creation time, as part of an implicit transaction.
REVOKE ALL PRIVILEGES ON SCHEMA cdb_geocoder_client FROM PUBLIC, publicuser CASCADE;
-- Grant permissions on the schema to publicuser (but just the schema)
GRANT USAGE ON SCHEMA cdb_geocoder_client TO publicuser;
-- Revoke execute permissions on all functions in the schema by default
REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_geocoder_client FROM PUBLIC, publicuser;

4
client/templates/20_public_functions.erb
View File

@ -4,7 +4,7 @@
-- These are the only ones with permissions to publicuser role
-- and should also be the only ones with SECURITY DEFINER
CREATE OR REPLACE FUNCTION <%= GEOCODER_CLIENT_SCHEMA %>.<%= name %> (<%= params_with_type %>)
CREATE OR REPLACE FUNCTION <%= GEOCODER_CLIENT_SCHEMA %>.<%= name %> (<%= params_with_type_and_default %>)
RETURNS <%= return_type %> AS $$
DECLARE
ret <%= return_type %>;
@ -14,7 +14,7 @@ BEGIN
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
RAISE EXCEPTION 'The api_key must be provided';
END IF;
SELECT u, o INTO username, orgname FROM _cdb_entity_config() AS (u text, o text);
SELECT u, o INTO username, orgname FROM <%= GEOCODER_CLIENT_SCHEMA %>._cdb_entity_config() AS (u text, o text);
-- JSON value stored "" is taken as literal
IF username IS NULL OR username = '' OR username = '""' THEN
RAISE EXCEPTION 'Username is a mandatory argument, check it out';

2
client/templates/30_plproxy_functions.erb
View File

@ -1,4 +1,4 @@
CREATE OR REPLACE FUNCTION <%= GEOCODER_CLIENT_SCHEMA %>._<%= name %> (username text, organization_name text, <%= params_with_type %>)
CREATE OR REPLACE FUNCTION <%= GEOCODER_CLIENT_SCHEMA %>._<%= name %> (username text, organization_name text, <%= params_with_type_and_default %>)
RETURNS <%= return_type %> AS $$
CONNECT <%= GEOCODER_CLIENT_SCHEMA %>._server_conn_str();
SELECT cdb_geocoder_server.<%= name %> (username, organization_name, <%= params %>);

0
client/expected/00_installation_test.out → client/test/0.0.1/expected/00_installation_test.out
View File

0
client/expected/10_admin0_test.out → client/test/0.0.1/expected/10_admin0_test.out
View File

0
client/expected/20_admin1_test.out → client/test/0.0.1/expected/20_admin1_test.out
View File

0
client/expected/30_namedplaces_test.out → client/test/0.0.1/expected/30_namedplaces_test.out
View File

0
client/expected/40_postalcodes_test.out → client/test/0.0.1/expected/40_postalcodes_test.out
View File

0
client/expected/50_ipaddresses_test.out → client/test/0.0.1/expected/50_ipaddresses_test.out
View File

0
client/expected/90_permissions_test.out → client/test/0.0.1/expected/90_permissions_test.out
View File

0
client/sql/00_installation_test.sql → client/test/0.0.1/sql/00_installation_test.sql
View File

0
client/sql/10_admin0_test.sql → client/test/0.0.1/sql/10_admin0_test.sql
View File

0
client/sql/20_admin1_test.sql → client/test/0.0.1/sql/20_admin1_test.sql
View File

0
client/sql/30_namedplaces_test.sql → client/test/0.0.1/sql/30_namedplaces_test.sql
View File

0
client/sql/40_postalcodes_test.sql → client/test/0.0.1/sql/40_postalcodes_test.sql
View File

0
client/sql/50_ipaddresses_test.sql → client/test/0.0.1/sql/50_ipaddresses_test.sql
View File

0
client/sql/90_permissions_test.sql → client/test/0.0.1/sql/90_permissions_test.sql
View File

29
client/test/0.1.0/expected/00_installation_test.out Normal file
View File

@ -0,0 +1,29 @@
-- Install dependencies
CREATE EXTENSION postgis;
CREATE EXTENSION schema_triggers;
CREATE EXTENSION plpythonu;
CREATE EXTENSION cartodb;
CREATE EXTENSION plproxy;
-- Install the extension
CREATE EXTENSION cdb_geocoder_client;
-- Mock the server connection to point to this very test db
SELECT cartodb.cdb_conf_setconf('geocoder_server_config', '{"connection_str": "dbname=contrib_regression host=127.0.0.1 user=postgres"}');
cdb_conf_setconf
------------------
(1 row)
-- Mock the user configuration
SELECT cartodb.cdb_conf_setconf('user_config', '{"is_organization": false, "entity_name": "test_user"}');
cdb_conf_setconf
------------------
(1 row)
-- Mock the server schema
CREATE SCHEMA cdb_geocoder_server;
-- Create a test user to check permissions
DROP ROLE IF EXISTS test_regular_user;
CREATE ROLE test_regular_user;
GRANT publicuser TO test_regular_user;
ALTER ROLE test_regular_user SET search_path TO public,cartodb,cdb_geocoder_client;

20
client/test/0.1.0/expected/10_admin0_test.out Normal file
View File

@ -0,0 +1,20 @@
-- Add to the search path the schema
SET search_path TO public,cartodb,cdb_geocoder_client;
-- Mock the server function
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text)
RETURNS Geometry AS $$
BEGIN
RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_admin0_polygon invoked with params (%, %, %)', username, orgname, country_name;
RETURN NULL;
END;
$$ LANGUAGE 'plpgsql';
-- Exercise the public and the proxied function
SELECT cdb_geocode_admin0_polygon('Spain');
NOTICE: cdb_geocoder_client._cdb_geocode_admin0_polygon(3): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_admin0_polygon invoked with params (test_user, <NULL>, Spain)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_admin0_polygon(username, orgname, country_name)"
PL/pgSQL function cdb_geocode_admin0_polygon(text) line 15 at SQL statement
cdb_geocode_admin0_polygon
----------------------------
(1 row)

36
client/test/0.1.0/expected/20_admin1_test.out Normal file
View File

@ -0,0 +1,36 @@
-- Add to the search path the schema
SET search_path TO public,cartodb,cdb_geocoder_client;
-- Mock the server functions
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text)
RETURNS Geometry AS $$
BEGIN
RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_admin1_polygon invoked with params (%, %, %)', username, orgname, admin1_name;
RETURN NULL;
END;
$$ LANGUAGE 'plpgsql';
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text)
RETURNS Geometry AS $$
BEGIN
RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_admin1_polygon invoked with params (%, %, %, %)', username, orgname, admin1_name, country_name;
RETURN NULL;
END;
$$ LANGUAGE 'plpgsql';
-- Exercise the public and the proxied function
SELECT cdb_geocode_admin1_polygon('California');
NOTICE: cdb_geocoder_client._cdb_geocode_admin1_polygon(3): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_admin1_polygon invoked with params (test_user, <NULL>, California)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name)"
PL/pgSQL function cdb_geocode_admin1_polygon(text) line 15 at SQL statement
cdb_geocode_admin1_polygon
----------------------------
(1 row)
SELECT cdb_geocode_admin1_polygon('California', 'United States');
NOTICE: cdb_geocoder_client._cdb_geocode_admin1_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_admin1_polygon invoked with params (test_user, <NULL>, California, United States)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name, country_name)"
PL/pgSQL function cdb_geocode_admin1_polygon(text,text) line 15 at SQL statement
cdb_geocode_admin1_polygon
----------------------------
(1 row)

52
client/test/0.1.0/expected/30_namedplaces_test.out Normal file
View File

@ -0,0 +1,52 @@
-- Add to the search path the schema
SET search_path TO public,cartodb,cdb_geocoder_client;
-- Mock the server functions
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text)
RETURNS Geometry AS $$
BEGIN
RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_namedplace_point invoked with params (%, %, %)', username, orgname, city_name;
RETURN NULL;
END;
$$ LANGUAGE 'plpgsql';
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text)
RETURNS Geometry AS $$
BEGIN
RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_namedplace_point invoked with params (%, %, %, %)', username, orgname, city_name, country_name;
RETURN NULL;
END;
$$ LANGUAGE 'plpgsql';
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text)
RETURNS Geometry AS $$
BEGIN
RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_namedplace_point invoked with params (%, %, %, %, %)', username, orgname, city_name, admin1_name, country_name;
RETURN NULL;
END;
$$ LANGUAGE 'plpgsql';
-- Exercise the public and the proxied function
SELECT cdb_geocode_namedplace_point('Elx');
NOTICE: cdb_geocoder_client._cdb_geocode_namedplace_point(3): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_namedplace_point invoked with params (test_user, <NULL>, Elx)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_namedplace_point(username, orgname, city_name)"
PL/pgSQL function cdb_geocode_namedplace_point(text) line 15 at SQL statement
cdb_geocode_namedplace_point
------------------------------
(1 row)
SELECT cdb_geocode_namedplace_point('Elx', 'Spain');
NOTICE: cdb_geocoder_client._cdb_geocode_namedplace_point(4): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_namedplace_point invoked with params (test_user, <NULL>, Elx, Spain)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_namedplace_point(username, orgname, city_name, country_name)"
PL/pgSQL function cdb_geocode_namedplace_point(text,text) line 15 at SQL statement
cdb_geocode_namedplace_point
------------------------------
(1 row)
SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain');
NOTICE: cdb_geocoder_client._cdb_geocode_namedplace_point(5): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_namedplace_point invoked with params (test_user, <NULL>, Elx, Valencia, Spain)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_namedplace_point(username, orgname, city_name, admin1_name, country_name)"
PL/pgSQL function cdb_geocode_namedplace_point(text,text,text) line 15 at SQL statement
cdb_geocode_namedplace_point
------------------------------
(1 row)

36
client/test/0.1.0/expected/40_postalcodes_test.out Normal file
View File

@ -0,0 +1,36 @@
-- Add to the search path the schema
SET search_path TO public,cartodb,cdb_geocoder_client;
-- Mock the server functions
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_polygon(username text, orgname text, postal_code text, country_name text)
RETURNS Geometry AS $$
BEGIN
RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_postalcode_polygon invoked with params (%, %, %, %)', username, orgname, postal_code, country_name;
RETURN NULL;
END;
$$ LANGUAGE 'plpgsql';
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_point(username text, orgname text, postal_code text, country_name text)
RETURNS Geometry AS $$
BEGIN
RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_postalcode_point invoked with params (%, %, %, %)', username, orgname, postal_code, country_name;
RETURN NULL;
END;
$$ LANGUAGE 'plpgsql';
-- Exercise the public and the proxied function
SELECT cdb_geocode_postalcode_polygon('03204', 'Spain');
NOTICE: cdb_geocoder_client._cdb_geocode_postalcode_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_postalcode_polygon invoked with params (test_user, <NULL>, 03204, Spain)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_postalcode_polygon(username, orgname, postal_code, country_name)"
PL/pgSQL function cdb_geocode_postalcode_polygon(text,text) line 15 at SQL statement
cdb_geocode_postalcode_polygon
--------------------------------
(1 row)
SELECT cdb_geocode_postalcode_point('03204', 'Spain');
NOTICE: cdb_geocoder_client._cdb_geocode_postalcode_point(4): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_postalcode_point invoked with params (test_user, <NULL>, 03204, Spain)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_postalcode_point(username, orgname, postal_code, country_name)"
PL/pgSQL function cdb_geocode_postalcode_point(text,text) line 15 at SQL statement
cdb_geocode_postalcode_point
------------------------------
(1 row)

20
client/test/0.1.0/expected/50_ipaddresses_test.out Normal file
View File

@ -0,0 +1,20 @@
-- Add to the search path the schema
SET search_path TO public,cartodb,cdb_geocoder_client;
-- Mock the server functions
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_ipaddress_point(username text, orgname text, ip_address text)
RETURNS Geometry AS $$
BEGIN
RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_ipaddress_point invoked with params (%, %, %)', username, orgname, ip_address;
RETURN NULL;
END;
$$ LANGUAGE 'plpgsql';
-- Exercise the public and the proxied function
SELECT cdb_geocode_ipaddress_point('8.8.8.8');
NOTICE: cdb_geocoder_client._cdb_geocode_ipaddress_point(3): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_ipaddress_point invoked with params (test_user, <NULL>, 8.8.8.8)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_ipaddress_point(username, orgname, ip_address)"
PL/pgSQL function cdb_geocode_ipaddress_point(text) line 15 at SQL statement
cdb_geocode_ipaddress_point
-----------------------------
(1 row)

101
client/test/0.1.0/expected/60_street_v2_test.out Normal file
View File

@ -0,0 +1,101 @@
-- Add to the search path the schema
SET search_path TO public,cartodb,cdb_geocoder_client;
-- Mock the server functions
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_street_point_v2 (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL)
RETURNS Geometry AS $$
BEGIN
RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (%, %, %, %, %, %)', username, orgname, searchtext, city, state_province, country;
RETURN NULL;
END;
$$ LANGUAGE 'plpgsql';
-- Exercise the public and the proxied function
SELECT cdb_geocode_street_point_v2('One street, 1');
NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (test_user, <NULL>, One street, 1, <NULL>, <NULL>, <NULL>)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)"
PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement
cdb_geocode_street_point_v2
-----------------------------
(1 row)
SELECT cdb_geocode_street_point_v2('One street', 'city');
NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (test_user, <NULL>, One street, city, <NULL>, <NULL>)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)"
PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement
cdb_geocode_street_point_v2
-----------------------------
(1 row)
SELECT cdb_geocode_street_point_v2('One street', 'city', 'state');
NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (test_user, <NULL>, One street, city, state, <NULL>)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)"
PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement
cdb_geocode_street_point_v2
-----------------------------
(1 row)
SELECT cdb_geocode_street_point_v2('One street', 'city', 'state', 'country');
NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (test_user, <NULL>, One street, city, state, country)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)"
PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement
cdb_geocode_street_point_v2
-----------------------------
(1 row)
SELECT cdb_geocode_street_point_v2('One street', 'city', NULL, 'country');
NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (test_user, <NULL>, One street, city, <NULL>, country)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)"
PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement
cdb_geocode_street_point_v2
-----------------------------
(1 row)
SELECT cdb_geocode_street_point_v2('One street, 1');
NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (test_user, <NULL>, One street, 1, <NULL>, <NULL>, <NULL>)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)"
PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement
cdb_geocode_street_point_v2
-----------------------------
(1 row)
SELECT cdb_geocode_street_point_v2('One street', 'city');
NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (test_user, <NULL>, One street, city, <NULL>, <NULL>)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)"
PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement
cdb_geocode_street_point_v2
-----------------------------
(1 row)
SELECT cdb_geocode_street_point_v2('One street', 'city', 'state');
NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (test_user, <NULL>, One street, city, state, <NULL>)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)"
PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement
cdb_geocode_street_point_v2
-----------------------------
(1 row)
SELECT cdb_geocode_street_point_v2('One street', 'city', 'state', 'country');
NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (test_user, <NULL>, One street, city, state, country)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)"
PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement
cdb_geocode_street_point_v2
-----------------------------
(1 row)
SELECT cdb_geocode_street_point_v2('One street', 'city', NULL, 'country');
NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (test_user, <NULL>, One street, city, <NULL>, country)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)"
PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement
cdb_geocode_street_point_v2
-----------------------------
(1 row)

117
client/test/0.1.0/expected/90_permissions_test.out Normal file
View File

@ -0,0 +1,117 @@
-- Use regular user role
SET ROLE test_regular_user;
-- Add to the search path the schema
SET search_path TO public,cartodb,cdb_geocoder_client;
-- Exercise the public function
-- it is public, it shall work
SELECT cdb_geocode_admin0_polygon('Spain');
NOTICE: cdb_geocoder_client._cdb_geocode_admin0_polygon(3): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_admin0_polygon invoked with params (test_user, <NULL>, Spain)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_admin0_polygon(username, orgname, country_name)"
PL/pgSQL function cdb_geocode_admin0_polygon(text) line 15 at SQL statement
cdb_geocode_admin0_polygon
----------------------------
(1 row)
SELECT cdb_geocode_admin1_polygon('California');
NOTICE: cdb_geocoder_client._cdb_geocode_admin1_polygon(3): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_admin1_polygon invoked with params (test_user, <NULL>, California)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name)"
PL/pgSQL function cdb_geocode_admin1_polygon(text) line 15 at SQL statement
cdb_geocode_admin1_polygon
----------------------------
(1 row)
SELECT cdb_geocode_admin1_polygon('California', 'United States');
NOTICE: cdb_geocoder_client._cdb_geocode_admin1_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_admin1_polygon invoked with params (test_user, <NULL>, California, United States)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name, country_name)"
PL/pgSQL function cdb_geocode_admin1_polygon(text,text) line 15 at SQL statement
cdb_geocode_admin1_polygon
----------------------------
(1 row)
SELECT cdb_geocode_namedplace_point('Elx');
NOTICE: cdb_geocoder_client._cdb_geocode_namedplace_point(3): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_namedplace_point invoked with params (test_user, <NULL>, Elx)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_namedplace_point(username, orgname, city_name)"
PL/pgSQL function cdb_geocode_namedplace_point(text) line 15 at SQL statement
cdb_geocode_namedplace_point
------------------------------
(1 row)
SELECT cdb_geocode_namedplace_point('Elx', 'Valencia');
NOTICE: cdb_geocoder_client._cdb_geocode_namedplace_point(4): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_namedplace_point invoked with params (test_user, <NULL>, Elx, Valencia)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_namedplace_point(username, orgname, city_name, country_name)"
PL/pgSQL function cdb_geocode_namedplace_point(text,text) line 15 at SQL statement
cdb_geocode_namedplace_point
------------------------------
(1 row)
SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain');
NOTICE: cdb_geocoder_client._cdb_geocode_namedplace_point(5): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_namedplace_point invoked with params (test_user, <NULL>, Elx, Valencia, Spain)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_namedplace_point(username, orgname, city_name, admin1_name, country_name)"
PL/pgSQL function cdb_geocode_namedplace_point(text,text,text) line 15 at SQL statement
cdb_geocode_namedplace_point
------------------------------
(1 row)
SELECT cdb_geocode_postalcode_polygon('03204', 'Spain');
NOTICE: cdb_geocoder_client._cdb_geocode_postalcode_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_postalcode_polygon invoked with params (test_user, <NULL>, 03204, Spain)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_postalcode_polygon(username, orgname, postal_code, country_name)"
PL/pgSQL function cdb_geocode_postalcode_polygon(text,text) line 15 at SQL statement
cdb_geocode_postalcode_polygon
--------------------------------
(1 row)
SELECT cdb_geocode_postalcode_point('03204', 'Spain');
NOTICE: cdb_geocoder_client._cdb_geocode_postalcode_point(4): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_postalcode_point invoked with params (test_user, <NULL>, 03204, Spain)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_postalcode_point(username, orgname, postal_code, country_name)"
PL/pgSQL function cdb_geocode_postalcode_point(text,text) line 15 at SQL statement
cdb_geocode_postalcode_point
------------------------------
(1 row)
SELECT cdb_geocode_ipaddress_point('8.8.8.8');
NOTICE: cdb_geocoder_client._cdb_geocode_ipaddress_point(3): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_ipaddress_point invoked with params (test_user, <NULL>, 8.8.8.8)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_ipaddress_point(username, orgname, ip_address)"
PL/pgSQL function cdb_geocode_ipaddress_point(text) line 15 at SQL statement
cdb_geocode_ipaddress_point
-----------------------------
(1 row)
SELECT cdb_geocode_street_point_v2('one street, 1');
NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (test_user, <NULL>, one street, 1, <NULL>, <NULL>, <NULL>)
CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)"
PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement
cdb_geocode_street_point_v2
-----------------------------
(1 row)
-- Check the regular user has no permissions on private functions
SELECT _cdb_geocode_admin0_polygon('evil_user', 'evil_orgname', 'Hell');
ERROR: permission denied for function _cdb_geocode_admin0_polygon
SELECT _cdb_geocode_admin1_polygon('evil_user', 'evil_orgname', 'Hell');
ERROR: permission denied for function _cdb_geocode_admin1_polygon
SELECT _cdb_geocode_admin1_polygon('evil_user', 'evil_orgname', 'Sheol', 'Hell');
ERROR: permission denied for function _cdb_geocode_admin1_polygon
SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'Sheol');
ERROR: permission denied for function _cdb_geocode_namedplace_point
SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'Sheol', 'Hell');
ERROR: permission denied for function _cdb_geocode_namedplace_point
SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'Sheol', 'Hell', 'Ugly world');
ERROR: permission denied for function _cdb_geocode_namedplace_point
SELECT _cdb_geocode_postalcode_polygon('evil_user', 'evil_orgname', '66666', 'Hell');
ERROR: permission denied for function _cdb_geocode_postalcode_polygon
SELECT _cdb_geocode_postalcode_point('evil_user', 'evil_orgname', '66666', 'Hell');
ERROR: permission denied for function _cdb_geocode_postalcode_point
SELECT _cdb_geocode_ipaddress_point('evil_user', 'evil_orgname', '8.8.8.8');
ERROR: permission denied for function _cdb_geocode_ipaddress_point
SELECT _cdb_geocode_street_point_v2('evil_user', 'evil_orgname', 'one street, 1');
ERROR: permission denied for function _cdb_geocode_street_point_v2

23
client/test/0.1.0/sql/00_installation_test.sql Normal file
View File

@ -0,0 +1,23 @@
-- Install dependencies
CREATE EXTENSION postgis;
CREATE EXTENSION schema_triggers;
CREATE EXTENSION plpythonu;
CREATE EXTENSION cartodb;
CREATE EXTENSION plproxy;
-- Install the extension
CREATE EXTENSION cdb_geocoder_client;
-- Mock the server connection to point to this very test db
SELECT cartodb.cdb_conf_setconf('geocoder_server_config', '{"connection_str": "dbname=contrib_regression host=127.0.0.1 user=postgres"}');
-- Mock the user configuration
SELECT cartodb.cdb_conf_setconf('user_config', '{"is_organization": false, "entity_name": "test_user"}');
-- Mock the server schema
CREATE SCHEMA cdb_geocoder_server;
-- Create a test user to check permissions
DROP ROLE IF EXISTS test_regular_user;
CREATE ROLE test_regular_user;
GRANT publicuser TO test_regular_user;
ALTER ROLE test_regular_user SET search_path TO public,cartodb,cdb_geocoder_client;

15
client/test/0.1.0/sql/10_admin0_test.sql Normal file
View File

@ -0,0 +1,15 @@
-- Add to the search path the schema
SET search_path TO public,cartodb,cdb_geocoder_client;
-- Mock the server function
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text)
RETURNS Geometry AS $$
BEGIN
RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_admin0_polygon invoked with params (%, %, %)', username, orgname, country_name;
RETURN NULL;
END;
$$ LANGUAGE 'plpgsql';
-- Exercise the public and the proxied function
SELECT cdb_geocode_admin0_polygon('Spain');

24
client/test/0.1.0/sql/20_admin1_test.sql Normal file
View File

@ -0,0 +1,24 @@
-- Add to the search path the schema
SET search_path TO public,cartodb,cdb_geocoder_client;
-- Mock the server functions
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text)
RETURNS Geometry AS $$
BEGIN
RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_admin1_polygon invoked with params (%, %, %)', username, orgname, admin1_name;
RETURN NULL;
END;
$$ LANGUAGE 'plpgsql';
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text)
RETURNS Geometry AS $$
BEGIN
RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_admin1_polygon invoked with params (%, %, %, %)', username, orgname, admin1_name, country_name;
RETURN NULL;
END;
$$ LANGUAGE 'plpgsql';
-- Exercise the public and the proxied function
SELECT cdb_geocode_admin1_polygon('California');
SELECT cdb_geocode_admin1_polygon('California', 'United States');

33
client/test/0.1.0/sql/30_namedplaces_test.sql Normal file
View File

@ -0,0 +1,33 @@
-- Add to the search path the schema
SET search_path TO public,cartodb,cdb_geocoder_client;
-- Mock the server functions
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text)
RETURNS Geometry AS $$
BEGIN
RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_namedplace_point invoked with params (%, %, %)', username, orgname, city_name;
RETURN NULL;
END;
$$ LANGUAGE 'plpgsql';
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text)
RETURNS Geometry AS $$
BEGIN
RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_namedplace_point invoked with params (%, %, %, %)', username, orgname, city_name, country_name;
RETURN NULL;
END;
$$ LANGUAGE 'plpgsql';
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text)
RETURNS Geometry AS $$
BEGIN
RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_namedplace_point invoked with params (%, %, %, %, %)', username, orgname, city_name, admin1_name, country_name;
RETURN NULL;
END;
$$ LANGUAGE 'plpgsql';
-- Exercise the public and the proxied function
SELECT cdb_geocode_namedplace_point('Elx');
SELECT cdb_geocode_namedplace_point('Elx', 'Spain');
SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain');

23
client/test/0.1.0/sql/40_postalcodes_test.sql Normal file
View File

@ -0,0 +1,23 @@
-- Add to the search path the schema
SET search_path TO public,cartodb,cdb_geocoder_client;
-- Mock the server functions
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_polygon(username text, orgname text, postal_code text, country_name text)
RETURNS Geometry AS $$
BEGIN
RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_postalcode_polygon invoked with params (%, %, %, %)', username, orgname, postal_code, country_name;
RETURN NULL;
END;
$$ LANGUAGE 'plpgsql';
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_point(username text, orgname text, postal_code text, country_name text)
RETURNS Geometry AS $$
BEGIN
RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_postalcode_point invoked with params (%, %, %, %)', username, orgname, postal_code, country_name;
RETURN NULL;
END;
$$ LANGUAGE 'plpgsql';
-- Exercise the public and the proxied function
SELECT cdb_geocode_postalcode_polygon('03204', 'Spain');
SELECT cdb_geocode_postalcode_point('03204', 'Spain');

15
client/test/0.1.0/sql/50_ipaddresses_test.sql Normal file
View File

@ -0,0 +1,15 @@
-- Add to the search path the schema
SET search_path TO public,cartodb,cdb_geocoder_client;
-- Mock the server functions
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_ipaddress_point(username text, orgname text, ip_address text)
RETURNS Geometry AS $$
BEGIN
RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_ipaddress_point invoked with params (%, %, %)', username, orgname, ip_address;
RETURN NULL;
END;
$$ LANGUAGE 'plpgsql';
-- Exercise the public and the proxied function
SELECT cdb_geocode_ipaddress_point('8.8.8.8');

24
client/test/0.1.0/sql/60_street_v2_test.sql Normal file
View File

@ -0,0 +1,24 @@
-- Add to the search path the schema
SET search_path TO public,cartodb,cdb_geocoder_client;
-- Mock the server functions
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_street_point_v2 (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL)
RETURNS Geometry AS $$
BEGIN
RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (%, %, %, %, %, %)', username, orgname, searchtext, city, state_province, country;
RETURN NULL;
END;
$$ LANGUAGE 'plpgsql';
-- Exercise the public and the proxied function
SELECT cdb_geocode_street_point_v2('One street, 1');
SELECT cdb_geocode_street_point_v2('One street', 'city');
SELECT cdb_geocode_street_point_v2('One street', 'city', 'state');
SELECT cdb_geocode_street_point_v2('One street', 'city', 'state', 'country');
SELECT cdb_geocode_street_point_v2('One street', 'city', NULL, 'country');
SELECT cdb_geocode_street_point_v2('One street, 1');
SELECT cdb_geocode_street_point_v2('One street', 'city');
SELECT cdb_geocode_street_point_v2('One street', 'city', 'state');
SELECT cdb_geocode_street_point_v2('One street', 'city', 'state', 'country');
SELECT cdb_geocode_street_point_v2('One street', 'city', NULL, 'country');

30
client/test/0.1.0/sql/90_permissions_test.sql Normal file
View File

@ -0,0 +1,30 @@
-- Use regular user role
SET ROLE test_regular_user;
-- Add to the search path the schema
SET search_path TO public,cartodb,cdb_geocoder_client;
-- Exercise the public function
-- it is public, it shall work
SELECT cdb_geocode_admin0_polygon('Spain');
SELECT cdb_geocode_admin1_polygon('California');
SELECT cdb_geocode_admin1_polygon('California', 'United States');
SELECT cdb_geocode_namedplace_point('Elx');
SELECT cdb_geocode_namedplace_point('Elx', 'Valencia');
SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain');
SELECT cdb_geocode_postalcode_polygon('03204', 'Spain');
SELECT cdb_geocode_postalcode_point('03204', 'Spain');
SELECT cdb_geocode_ipaddress_point('8.8.8.8');
SELECT cdb_geocode_street_point_v2('one street, 1');
-- Check the regular user has no permissions on private functions
SELECT _cdb_geocode_admin0_polygon('evil_user', 'evil_orgname', 'Hell');
SELECT _cdb_geocode_admin1_polygon('evil_user', 'evil_orgname', 'Hell');
SELECT _cdb_geocode_admin1_polygon('evil_user', 'evil_orgname', 'Sheol', 'Hell');
SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'Sheol');
SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'Sheol', 'Hell');
SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'Sheol', 'Hell', 'Ugly world');
SELECT _cdb_geocode_postalcode_polygon('evil_user', 'evil_orgname', '66666', 'Hell');
SELECT _cdb_geocode_postalcode_point('evil_user', 'evil_orgname', '66666', 'Hell');
SELECT _cdb_geocode_ipaddress_point('evil_user', 'evil_orgname', '8.8.8.8');
SELECT _cdb_geocode_street_point_v2('evil_user', 'evil_orgname', 'one street, 1');

1
interface.yaml → interface_0.0.1.yaml
View File

@ -50,4 +50,3 @@
return_type: Geometry
params:
- { name: ip_address, type: text}

60
interface_0.1.0.yaml Normal file
View File

@ -0,0 +1,60 @@
---
- name: cdb_geocode_admin0_polygon
return_type: Geometry
params:
- { name: country_name, type: text }
- name: cdb_geocode_admin1_polygon
return_type: Geometry
params:
- { name: admin1_name, type: text }
- name: cdb_geocode_admin1_polygon
return_type: Geometry
params:
- { name: admin1_name, type: text }
- { name: country_name, type: text }
- name: cdb_geocode_namedplace_point
return_type: Geometry
params:
- { name: city_name, type: text}
- name: cdb_geocode_namedplace_point
return_type: Geometry
params:
- { name: city_name, type: text}
- { name: country_name, type: text}
- name: cdb_geocode_namedplace_point
return_type: Geometry
params:
- { name: city_name, type: text}
- { name: admin1_name, type: text}
- { name: country_name, type: text}
- name: cdb_geocode_postalcode_polygon
return_type: Geometry
params:
- { name: postal_code, type: text}
- { name: country_name, type: text}
- name: cdb_geocode_postalcode_point
return_type: Geometry
params:
- { name: postal_code, type: text}
- { name: country_name, type: text}
- name: cdb_geocode_ipaddress_point
return_type: Geometry
params:
- { name: ip_address, type: text}
- name: cdb_geocode_street_point_v2
return_type: Geometry
params:
- { name: searchtext, type: text}
- { name: city, type: text, default: 'NULL'}
- { name: state_province, type: text, default: 'NULL'}
- { name: country, type: text, default: 'NULL'}

2
server/extension/.gitignore vendored
View File

@ -2,4 +2,4 @@ results/
regression.diffs
regression.out
cdb_geocoder_server--0.0.1.sql
cdb_geocoder_server--0.1.0.sql

24
server/extension/Makefile
View File

@ -3,24 +3,38 @@
EXTENSION = cdb_geocoder_server
EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/")
DATA = $(EXTENSION)--$(EXTVERSION).sql
# The new version to be generated from templates
NEW_EXTENSION_ARTIFACT = $(EXTENSION)--$(EXTVERSION).sql
REGRESS = $(notdir $(basename $(wildcard sql/*test.sql)))
# DATA is a special variable used by postgres build infrastructure
# These are the files to be installed in the server shared dir,
# for installation from scratch, upgrades and downgrades.
# @see http://www.postgresql.org/docs/current/static/extend-pgxs.html
DATA = $(NEW_EXTENSION_ARTIFACT) \
cdb_geocoder_server--0.0.1.sql \
cdb_geocoder_server--0.1.0--0.0.1.sql \
cdb_geocoder_server--0.0.1--0.1.0.sql
REGRESS = $(notdir $(basename $(wildcard test/$(EXTVERSION)/sql/*test.sql)))
TEST_DIR = test/$(EXTVERSION)
REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)'
# postgres build stuff
PG_CONFIG = pg_config
PGXS := $(shell $(PG_CONFIG) --pgxs)
include $(PGXS)
SOURCES_DATA_DIR = sql/$(EXTVERSION)
SOURCES_DATA = $(wildcard sql/$(EXTVERSION)/*.sql)
$(DATA): $(SOURCES_DATA)
$(NEW_EXTENSION_ARTIFACT): $(SOURCES_DATA)
rm -f $@
cat $(SOURCES_DATA) >> $@
cat $(SOURCES_DATA_DIR)/*.sql >> $@
all: $(DATA)
# Only meant for development time, do not use once a version is released
devclean:
rm -f $(DATA)
rm -f $(NEW_EXTENSION_ARTIFACT)

132
server/extension/cdb_geocoder_server--0.0.1--0.1.0.sql Normal file
View File

@ -0,0 +1,132 @@
-- Get the Redis configuration from the _conf table --
CREATE OR REPLACE FUNCTION cdb_geocoder_server._get_redis_conf_v2(config_key text)
RETURNS cdb_geocoder_server._redis_conf_params AS $$
conf_query = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(config_key)
conf = plpy.execute(conf_query)[0]['conf']
if conf is None:
plpy.error("There is no redis configuration defined")
else:
import json
params = json.loads(conf)
return {
"sentinel_host": params['sentinel_host'],
"sentinel_port": params['sentinel_port'],
"sentinel_master_id": params['sentinel_master_id'],
"timeout": params['timeout'],
"redis_db": params['redis_db']
}
$$ LANGUAGE plpythonu;
-- Get the connection to redis from cache or create a new one
CREATE OR REPLACE FUNCTION cdb_geocoder_server._connect_to_redis(user_id text)
RETURNS boolean AS $$
cache_key = "redis_connection_{0}".format(user_id)
if cache_key in GD:
return False
else:
from cartodb_geocoder import redis_helper
metadata_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port,
c.sentinel_master_id, c.timeout, c.redis_db
from cdb_geocoder_server._get_redis_conf_v2('redis_metadata_config') c;""")[0]
metrics_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port,
c.sentinel_master_id, c.timeout, c.redis_db
from cdb_geocoder_server._get_redis_conf_v2('redis_metrics_config') c;""")[0]
redis_metadata_connection = redis_helper.RedisHelper(metadata_config_params['sentinel_host'],
metadata_config_params['sentinel_port'],
metadata_config_params['sentinel_master_id'],
timeout=metadata_config_params['timeout'],
redis_db=metadata_config_params['redis_db']).redis_connection()
redis_metrics_connection = redis_helper.RedisHelper(metrics_config_params['sentinel_host'],
metrics_config_params['sentinel_port'],
metrics_config_params['sentinel_master_id'],
timeout=metrics_config_params['timeout'],
redis_db=metrics_config_params['redis_db']).redis_connection()
GD[cache_key] = {
'redis_metadata_connection': redis_metadata_connection,
'redis_metrics_connection': redis_metrics_connection,
}
return True
$$ LANGUAGE plpythonu;
-- Get the Redis configuration from the _conf table --
CREATE OR REPLACE FUNCTION cdb_geocoder_server._get_geocoder_config(username text, orgname text)
RETURNS boolean AS $$
cache_key = "user_geocoder_config_{0}".format(username)
if cache_key in GD:
return False
else:
import json
from cartodb_geocoder import config_helper
plpy.execute("SELECT cdb_geocoder_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
heremaps_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as heremaps_conf", 1)[0]['heremaps_conf']
if not heremaps_conf_json:
heremaps_app_id = None
heremaps_app_code = None
else:
heremaps_conf = json.loads(heremaps_conf_json)
heremaps_app_id = heremaps_conf['app_id']
heremaps_app_code = heremaps_conf['app_code']
geocoder_config = config_helper.GeocoderConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code)
# --Think about the security concerns with this kind of global cache, it should be only available
# --for this user session but...
GD[cache_key] = geocoder_config
return True
$$ LANGUAGE plpythonu;
-- Geocodes a street address given a searchtext and a state and/or country
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_street_point_v2(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
plpy.execute("SELECT cdb_geocoder_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_geocoder_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
if user_geocoder_config.heremaps_geocoder:
here_plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
elif user_geocoder_config.google_geocoder:
google_plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
else:
plpy.error('Requested geocoder is not available')
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
from heremaps import heremapsgeocoder
from cartodb_geocoder import quota_service
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
# -- Check the quota
quota_service = quota_service.QuotaService(user_geocoder_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reach the limit of your quota')
try:
geocoder = heremapsgeocoder.Geocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code)
results = geocoder.geocode_address(searchtext=searchtext, city=city, state=state_province, country=country)
coordinates = geocoder.extract_lng_lat_from_result(results[0])
quota_service.increment_success_geocoder_use()
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
return point['st_setsrid']
except heremapsgeocoder.EmptyGeocoderResponse:
quota_service.increment_empty_geocoder_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_geocoder_use()
error_msg = 'There was an error trying to geocode using here maps geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
plpy.error('Google geocoder is not available yet')
return None
$$ LANGUAGE plpythonu;

539
server/extension/cdb_geocoder_server--0.0.1.sql Normal file
View File

@ -0,0 +1,539 @@
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "CREATE EXTENSION cdb_geocoder_server" to load this file. \quit
CREATE TYPE cdb_geocoder_server._redis_conf_params AS (
sentinel_host text,
sentinel_port int,
sentinel_master_id text,
redis_db text,
timeout float
);
-- Get the Redis configuration from the _conf table --
CREATE OR REPLACE FUNCTION cdb_geocoder_server._get_redis_conf()
RETURNS cdb_geocoder_server._redis_conf_params AS $$
conf = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('redis_conf') conf")[0]['conf']
if conf is None:
plpy.error("There is no redis configuration defined")
else:
import json
params = json.loads(conf)
return {
"sentinel_host": params['sentinel_host'],
"sentinel_port": params['sentinel_port'],
"sentinel_master_id": params['sentinel_master_id'],
"timeout": params['timeout'],
"redis_db": params['redis_db']
}
$$ LANGUAGE plpythonu;
-- Get the connection to redis from cache or create a new one
CREATE OR REPLACE FUNCTION cdb_geocoder_server._connect_to_redis(user_id text)
RETURNS boolean AS $$
if user_id in GD and 'redis_connection' in GD[user_id]:
return False
else:
from cartodb_geocoder import redis_helper
config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port,
c.sentinel_master_id, c.timeout, c.redis_db
from cdb_geocoder_server._get_redis_conf() c;""")[0]
redis_connection = redis_helper.RedisHelper(config_params['sentinel_host'],
config_params['sentinel_port'],
config_params['sentinel_master_id'],
timeout=config_params['timeout'],
redis_db=config_params['redis_db']).redis_connection()
GD[user_id] = {'redis_connection': redis_connection}
return True
$$ LANGUAGE plpythonu;-- Geocodes a street address given a searchtext and a state and/or country
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_street_point(searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry
AS $$
import json
from heremaps import heremapsgeocoder
heremaps_conf = json.loads(plpy.execute("SELECT cdb_geocoder_server._get_conf('heremaps')", 1)[0]['get_conf'])
app_id = heremaps_conf['geocoder']['app_id']
app_code = heremaps_conf['geocoder']['app_code']
geocoder = heremapsgeocoder.Geocoder(app_id, app_code)
results = geocoder.geocode_address(searchtext=searchtext, city=city, state=state_province, country=country)
coordinates = geocoder.extract_lng_lat_from_result(results[0])
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
return point['st_setsrid']
$$ LANGUAGE plpythonu;-- Interface of the server extension
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text)
RETURNS Geometry AS $$
plpy.debug('Entering cdb_geocode_admin0_polygons')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_admin0_polygon($1) AS mypolygon", ["text"])
rv = plpy.execute(plan, [country_name], 1)
plpy.debug('Returning from Returning from cdb_geocode_admin0_polygons')
return rv[0]["mypolygon"]
$$ LANGUAGE plpythonu;
--------------------------------------------------------------------------------
-- Implementation of the server extension
-- Note: these functions depend on the cdb_geocoder extension
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_admin0_polygon(country_name text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT n.the_geom as geom INTO ret
FROM (SELECT q, lower(regexp_replace(q, '[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text x
FROM (SELECT country_name q) g) d
LEFT OUTER JOIN admin0_synonyms s ON name_ = d.x
LEFT OUTER JOIN ne_admin0_v3 n ON s.adm0_a3 = n.adm0_a3 GROUP BY d.q, n.the_geom, s.adm0_a3;
RETURN ret;
END
$$ LANGUAGE plpgsql;
-- Interfacess of the server extension
---- cdb_geocode_admin1_polygon(admin1_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text)
RETURNS Geometry AS $$
plpy.debug('Entering cdb_geocode_admin1_polygon(admin1_name text)')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_admin1_polygon($1) AS mypolygon", ["text"])
rv = plpy.execute(plan, [admin1_name], 1)
plpy.debug('Returning from Returning from cdb_geocode_admin1_polygons')
return rv[0]["mypolygon"]
$$ LANGUAGE plpythonu;
---- cdb_geocode_admin1_polygon(admin1_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text)
RETURNS Geometry AS $$
plpy.debug('Entering cdb_geocode_admin1_polygon(admin1_name text, country_name text)')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_admin1_polygon($1, $2) AS mypolygon", ["text", "text"])
rv = plpy.execute(plan, [admin1_name, country_name], 1)
plpy.debug('Returning from Returning from cdb_geocode_admin1_polygon(admin1_name text, country_name text)')
return rv[0]["mypolygon"]
$$ LANGUAGE plpythonu;
--------------------------------------------------------------------------------
-- Implementation of the server extension
-- Note: these functions depend on the cdb_geocoder extension
---- cdb_geocode_admin1_polygon(admin1_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_admin1_polygon(admin1_name text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT geom INTO ret
FROM (
SELECT q, (
SELECT the_geom
FROM global_province_polygons
WHERE d.c = ANY (synonyms)
ORDER BY frequency DESC LIMIT 1
) geom
FROM (
SELECT
trim(replace(lower(admin1_name),'.',' ')) c, admin1_name q
) d
) v;
RETURN ret;
END
$$ LANGUAGE plpgsql;
---- cdb_geocode_admin1_polygon(admin1_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_admin1_polygon(admin1_name text, country_name text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
WITH p AS (SELECT r.c, r.q, (SELECT iso3 FROM country_decoder WHERE lower(country_name) = ANY (synonyms)) i FROM (SELECT trim(replace(lower(admin1_name),'.',' ')) c, country_name q) r)
SELECT
geom INTO ret
FROM (
SELECT
q, (
SELECT the_geom
FROM global_province_polygons
WHERE p.c = ANY (synonyms)
AND iso3 = p.i
ORDER BY frequency DESC LIMIT 1
) geom
FROM p) n;
RETURN ret;
END
$$ LANGUAGE plpgsql;
-- Interfacess of the server extension
---- cdb_geocode_namedplace_point(city_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text)
RETURNS Geometry AS $$
plpy.debug('Entering cdb_geocode_namedplace_point(city_name text)')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_namedplace_point($1) AS mypoint", ["text"])
rv = plpy.execute(plan, [city_name], 1)
plpy.debug('Returning from Returning from geocode_namedplace')
return rv[0]["mypoint"]
$$ LANGUAGE plpythonu;
---- cdb_geocode_namedplace_point(city_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text)
RETURNS Geometry AS $$
plpy.debug('Entering cdb_geocode_namedplace_point(city_name text, country_name text)')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_namedplace_point($1, $2) AS mypoint", ["text", "text"])
rv = plpy.execute(plan, [city_name, country_name], 1)
plpy.debug('Returning from Returning from geocode_namedplace')
return rv[0]["mypoint"]
$$ LANGUAGE plpythonu;
---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text)
RETURNS Geometry AS $$
plpy.debug('Entering cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_namedplace_point($1, $2, $3) AS mypoint", ["text", "text", "text"])
rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1)
plpy.debug('Returning from Returning from geocode_namedplace')
return rv[0]["mypoint"]
$$ LANGUAGE plpythonu;
--------------------------------------------------------------------------------
-- Implementation of the server extension
-- Note: these functions depend on the cdb_geocoder extension
---- cdb_geocode_namedplace_point(city_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_namedplace_point(city_name text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT geom INTO ret
FROM (
WITH best AS (SELECT s AS q, (SELECT the_geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) ORDER BY population DESC LIMIT 1) AS geom FROM (SELECT city_name as s) p),
next AS (SELECT p.s AS q, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM (SELECT city_name as s) p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL))
SELECT q, geom, TRUE AS success FROM best WHERE geom IS NOT NULL
UNION ALL
SELECT q, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next
) v;
RETURN ret;
END
$$ LANGUAGE plpgsql;
---- cdb_geocode_namedplace_point(city_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_namedplace_point(city_name text, country_name text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT geom INTO ret
FROM (
WITH p AS (SELECT r.s, r.c, (SELECT iso2 FROM country_decoder WHERE lower(r.c) = ANY (synonyms)) i FROM (SELECT city_name AS s, country_name::text AS c) r),
best AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.iso2 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p),
next AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND gp.iso2 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE c = p.c AND geom IS NOT NULL))
SELECT geom FROM best WHERE geom IS NOT NULL
UNION ALL
SELECT geom FROM next
) v;
RETURN ret;
END
$$ LANGUAGE plpgsql;
---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT geom INTO ret
FROM (
WITH inputcountry AS (
SELECT iso2 as isoTwo FROM country_decoder WHERE lower(country_name) = ANY (synonyms) LIMIT 1
),
p AS (
SELECT r.s, r.a1, (SELECT admin1 FROM admin1_decoder, inputcountry WHERE lower(r.a1) = ANY (synonyms) AND admin1_decoder.iso2 = inputcountry.isoTwo LIMIT 1) i FROM (SELECT city_name AS s, admin1_name::text AS a1) r),
best AS (SELECT p.s AS q, p.a1 as a1, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.admin1 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p),
next AS (SELECT p.s AS q, p.a1 AS a1, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.admin1 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL))
SELECT geom FROM best WHERE geom IS NOT NULL
UNION ALL
SELECT geom FROM next
) v;
RETURN ret;
END
$$ LANGUAGE plpgsql;
-- Interface of the server extension
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_point(username text, orgname text, code text)
RETURNS Geometry AS $$
plpy.debug('Entering _cdb_geocode_postalcode_point')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_point($1) AS point", ["text"])
rv = plpy.execute(plan, [code], 1)
plpy.debug('Returning from _cdb_geocode_postalcode_point')
return rv[0]["point"]
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text)
RETURNS Geometry AS $$
plpy.debug('Entering _cdb_geocode_postalcode_point')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_point($1, $2) AS point", ["TEXT", "TEXT"])
rv = plpy.execute(plan, [code, country], 1)
plpy.debug('Returning from _cdb_geocode_postalcode_point')
return rv[0]["point"]
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text)
RETURNS Geometry AS $$
plpy.debug('Entering _cdb_geocode_postalcode_polygon')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_polygon($1) AS polygon", ["text"])
rv = plpy.execute(plan, [code], 1)
plpy.debug('Returning from _cdb_geocode_postalcode_polygon')
return rv[0]["polygon"]
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text, country text)
RETURNS Geometry AS $$
plpy.debug('Entering _cdb_geocode_postalcode_point')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_polygon($1, $2) AS polygon", ["TEXT", "TEXT"])
rv = plpy.execute(plan, [code, country], 1)
plpy.debug('Returning from _cdb_geocode_postalcode_point')
return rv[0]["polygon"]
$$ LANGUAGE plpythonu;
--------------------------------------------------------------------------------
-- Implementation of the server extension
-- Note: these functions depend on the cdb_geocoder extension
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_point(code text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT geom INTO ret
FROM (
SELECT
q, (
SELECT the_geom
FROM global_postal_code_points
WHERE postal_code = upper(d.q)
LIMIT 1
) geom
FROM (SELECT code q) d
) v;
RETURN ret;
END
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_point(code text, country text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT geom INTO ret
FROM (
SELECT
q, (
SELECT the_geom
FROM global_postal_code_points
WHERE postal_code = upper(d.q)
AND iso3 = (
SELECT iso3 FROM country_decoder WHERE
lower(country) = ANY (synonyms) LIMIT 1
)
LIMIT 1
) geom
FROM (SELECT code q) d
) v;
RETURN ret;
END
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_polygon(code text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT geom INTO ret
FROM (
SELECT
q, (
SELECT the_geom
FROM global_postal_code_polygons
WHERE postal_code = upper(d.q)
LIMIT 1
) geom
FROM (SELECT code q) d
) v;
RETURN ret;
END
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_polygon(code text, country text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT geom INTO ret
FROM (
SELECT
q, (
SELECT the_geom
FROM global_postal_code_polygons
WHERE postal_code = upper(d.q)
AND iso3 = (
SELECT iso3 FROM country_decoder WHERE
lower(country) = ANY (synonyms) LIMIT 1
)
LIMIT 1
) geom
FROM (SELECT code q) d
) v;
RETURN ret;
END
$$ LANGUAGE plpgsql;
-- Interface of the server extension
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text)
RETURNS Geometry AS $$
plpy.debug('Entering _cdb_geocode_ipaddress_point')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_ipaddress_point($1) AS point", ["TEXT"])
rv = plpy.execute(plan, [ip], 1)
plpy.debug('Returning from _cdb_geocode_ipaddress_point')
return rv[0]["point"]
$$ LANGUAGE plpythonu;
--------------------------------------------------------------------------------
-- Implementation of the server extension
-- Note: these functions depend on the cdb_geocoder extension
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_ipaddress_point(ip text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
new_ip INET;
BEGIN
BEGIN
IF family(ip::inet) = 6 THEN
new_ip := ip::inet;
ELSE
new_ip := ('::ffff:' || ip)::inet;
END IF;
EXCEPTION WHEN OTHERS THEN
SELECT NULL as geom INTO ret;
RETURN ret;
END;
WITH
ips AS (SELECT ip s, new_ip net),
matches AS (SELECT s, (SELECT the_geom FROM ip_address_locations WHERE network_start_ip <= ips.net ORDER BY network_start_ip DESC LIMIT 1) geom FROM ips)
SELECT geom INTO ret
FROM matches;
RETURN ret;
END
$$ LANGUAGE plpgsql;
DO $$
BEGIN
IF NOT EXISTS (
SELECT *
FROM pg_catalog.pg_user
WHERE usename = 'geocoder_api') THEN
CREATE USER geocoder_api;
END IF;
GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_geocoder_server TO geocoder_api;
GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO geocoder_api;
GRANT USAGE ON SCHEMA cdb_geocoder_server TO geocoder_api;
GRANT USAGE ON SCHEMA public TO geocoder_api;
GRANT SELECT ON ALL TABLES IN SCHEMA public TO geocoder_api;
END$$;

6
server/extension/cdb_geocoder_server--0.1.0--0.0.1.sql Normal file
View File

@ -0,0 +1,6 @@
DROP FUNCTION IF EXISTS cdb_geocoder_server._get_redis_conf_v2(text);
DROP FUNCTION IF EXISTS cdb_geocoder_server._connect_to_redis(text);
DROP FUNCTION IF EXISTS cdb_geocoder_server._get_geocoder_config(text, text);
DROP FUNCTION IF EXISTS cdb_geocoder_server.cdb_geocode_street_point_v2(TEXT, TEXT, TEXT, TEXT, TEXT, TEXT);
DROP FUNCTION IF EXISTS cdb_geocoder_server._cdb_here_geocode_street_point(TEXT, TEXT, TEXT, TEXT, TEXT, TEXT);
DROP FUNCTION IF EXISTS cdb_geocoder_server._cdb_google_geocode_street_point(TEXT, TEXT, TEXT, TEXT, TEXT, TEXT);

2
server/extension/cdb_geocoder_server.control
View File

@ -1,6 +1,6 @@
# cdb geocoder server extension
comment = 'CartoDB server geocoder extension'
default_version = '0.0.1'
default_version = '0.1.0'
requires = 'plpythonu, postgis, cdb_geocoder'
superuser = true
schema = cdb_geocoder_server

2
server/extension/sql/0.1.0/00_header.sql Normal file
View File

@ -0,0 +1,2 @@
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "CREATE EXTENSION cdb_geocoder_server" to load this file. \quit

57
server/extension/sql/0.1.0/10_redis_helper.sql Normal file
View File

@ -0,0 +1,57 @@
CREATE TYPE cdb_geocoder_server._redis_conf_params AS (
sentinel_host text,
sentinel_port int,
sentinel_master_id text,
redis_db text,
timeout float
);
-- Get the Redis configuration from the _conf table --
CREATE OR REPLACE FUNCTION cdb_geocoder_server._get_redis_conf_v2(config_key text)
RETURNS cdb_geocoder_server._redis_conf_params AS $$
conf_query = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(config_key)
conf = plpy.execute(conf_query)[0]['conf']
if conf is None:
plpy.error("There is no redis configuration defined")
else:
import json
params = json.loads(conf)
return {
"sentinel_host": params['sentinel_host'],
"sentinel_port": params['sentinel_port'],
"sentinel_master_id": params['sentinel_master_id'],
"timeout": params['timeout'],
"redis_db": params['redis_db']
}
$$ LANGUAGE plpythonu;
-- Get the connection to redis from cache or create a new one
CREATE OR REPLACE FUNCTION cdb_geocoder_server._connect_to_redis(user_id text)
RETURNS boolean AS $$
cache_key = "redis_connection_{0}".format(user_id)
if cache_key in GD:
return False
else:
from cartodb_geocoder import redis_helper
metadata_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port,
c.sentinel_master_id, c.timeout, c.redis_db
from cdb_geocoder_server._get_redis_conf_v2('redis_metadata_config') c;""")[0]
metrics_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port,
c.sentinel_master_id, c.timeout, c.redis_db
from cdb_geocoder_server._get_redis_conf_v2('redis_metrics_config') c;""")[0]
redis_metadata_connection = redis_helper.RedisHelper(metadata_config_params['sentinel_host'],
metadata_config_params['sentinel_port'],
metadata_config_params['sentinel_master_id'],
timeout=metadata_config_params['timeout'],
redis_db=metadata_config_params['redis_db']).redis_connection()
redis_metrics_connection = redis_helper.RedisHelper(metrics_config_params['sentinel_host'],
metrics_config_params['sentinel_port'],
metrics_config_params['sentinel_master_id'],
timeout=metrics_config_params['timeout'],
redis_db=metrics_config_params['redis_db']).redis_connection()
GD[cache_key] = {
'redis_metadata_connection': redis_metadata_connection,
'redis_metrics_connection': redis_metrics_connection,
}
return True
$$ LANGUAGE plpythonu;

25
server/extension/sql/0.1.0/15_config_helper.sql Normal file
View File

@ -0,0 +1,25 @@
-- Get the Redis configuration from the _conf table --
CREATE OR REPLACE FUNCTION cdb_geocoder_server._get_geocoder_config(username text, orgname text)
RETURNS boolean AS $$
cache_key = "user_geocoder_config_{0}".format(username)
if cache_key in GD:
return False
else:
import json
from cartodb_geocoder import config_helper
plpy.execute("SELECT cdb_geocoder_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
heremaps_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as heremaps_conf", 1)[0]['heremaps_conf']
if not heremaps_conf_json:
heremaps_app_id = None
heremaps_app_code = None
else:
heremaps_conf = json.loads(heremaps_conf_json)
heremaps_app_id = heremaps_conf['app_id']
heremaps_app_code = heremaps_conf['app_code']
geocoder_config = config_helper.GeocoderConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code)
# --Think about the security concerns with this kind of global cache, it should be only available
# --for this user session but...
GD[cache_key] = geocoder_config
return True
$$ LANGUAGE plpythonu;

58
server/extension/sql/0.1.0/20_geocode_street.sql Normal file
View File

@ -0,0 +1,58 @@
-- Geocodes a street address given a searchtext and a state and/or country
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_street_point_v2(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
plpy.execute("SELECT cdb_geocoder_server._connect_to_redis('{0}')".format(username))
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
plpy.execute("SELECT cdb_geocoder_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
if user_geocoder_config.heremaps_geocoder:
here_plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
elif user_geocoder_config.google_geocoder:
google_plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"])
return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point']
else:
plpy.error('Requested geocoder is not available')
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
from heremaps import heremapsgeocoder
from cartodb_geocoder import quota_service
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
user_geocoder_config = GD["user_geocoder_config_{0}".format(username)]
# -- Check the quota
quota_service = quota_service.QuotaService(user_geocoder_config, redis_conn)
if not quota_service.check_user_quota():
plpy.error('You have reach the limit of your quota')
try:
geocoder = heremapsgeocoder.Geocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code)
results = geocoder.geocode_address(searchtext=searchtext, city=city, state=state_province, country=country)
coordinates = geocoder.extract_lng_lat_from_result(results[0])
quota_service.increment_success_geocoder_use()
plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"])
point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0]
return point['st_setsrid']
except heremapsgeocoder.EmptyGeocoderResponse:
quota_service.increment_empty_geocoder_use()
return None
except BaseException as e:
import sys, traceback
type_, value_, traceback_ = sys.exc_info()
quota_service.increment_failed_geocoder_use()
error_msg = 'There was an error trying to geocode using here maps geocoder: {0}'.format(e)
plpy.notice(traceback.format_tb(traceback_))
plpy.error(error_msg)
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL)
RETURNS Geometry AS $$
plpy.error('Google geocoder is not available yet')
return None
$$ LANGUAGE plpythonu;

37
server/extension/sql/0.1.0/30_admin0.sql Normal file
View File

@ -0,0 +1,37 @@
-- Interface of the server extension
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text)
RETURNS Geometry AS $$
plpy.debug('Entering cdb_geocode_admin0_polygons')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_admin0_polygon($1) AS mypolygon", ["text"])
rv = plpy.execute(plan, [country_name], 1)
plpy.debug('Returning from Returning from cdb_geocode_admin0_polygons')
return rv[0]["mypolygon"]
$$ LANGUAGE plpythonu;
--------------------------------------------------------------------------------
-- Implementation of the server extension
-- Note: these functions depend on the cdb_geocoder extension
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_admin0_polygon(country_name text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT n.the_geom as geom INTO ret
FROM (SELECT q, lower(regexp_replace(q, '[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text x
FROM (SELECT country_name q) g) d
LEFT OUTER JOIN admin0_synonyms s ON name_ = d.x
LEFT OUTER JOIN ne_admin0_v3 n ON s.adm0_a3 = n.adm0_a3 GROUP BY d.q, n.the_geom, s.adm0_a3;
RETURN ret;
END
$$ LANGUAGE plpgsql;

89
server/extension/sql/0.1.0/40_admin1.sql Normal file
View File

@ -0,0 +1,89 @@
-- Interfacess of the server extension
---- cdb_geocode_admin1_polygon(admin1_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text)
RETURNS Geometry AS $$
plpy.debug('Entering cdb_geocode_admin1_polygon(admin1_name text)')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_admin1_polygon($1) AS mypolygon", ["text"])
rv = plpy.execute(plan, [admin1_name], 1)
plpy.debug('Returning from Returning from cdb_geocode_admin1_polygons')
return rv[0]["mypolygon"]
$$ LANGUAGE plpythonu;
---- cdb_geocode_admin1_polygon(admin1_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text)
RETURNS Geometry AS $$
plpy.debug('Entering cdb_geocode_admin1_polygon(admin1_name text, country_name text)')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_admin1_polygon($1, $2) AS mypolygon", ["text", "text"])
rv = plpy.execute(plan, [admin1_name, country_name], 1)
plpy.debug('Returning from Returning from cdb_geocode_admin1_polygon(admin1_name text, country_name text)')
return rv[0]["mypolygon"]
$$ LANGUAGE plpythonu;
--------------------------------------------------------------------------------
-- Implementation of the server extension
-- Note: these functions depend on the cdb_geocoder extension
---- cdb_geocode_admin1_polygon(admin1_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_admin1_polygon(admin1_name text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT geom INTO ret
FROM (
SELECT q, (
SELECT the_geom
FROM global_province_polygons
WHERE d.c = ANY (synonyms)
ORDER BY frequency DESC LIMIT 1
) geom
FROM (
SELECT
trim(replace(lower(admin1_name),'.',' ')) c, admin1_name q
) d
) v;
RETURN ret;
END
$$ LANGUAGE plpgsql;
---- cdb_geocode_admin1_polygon(admin1_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_admin1_polygon(admin1_name text, country_name text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
WITH p AS (SELECT r.c, r.q, (SELECT iso3 FROM country_decoder WHERE lower(country_name) = ANY (synonyms)) i FROM (SELECT trim(replace(lower(admin1_name),'.',' ')) c, country_name q) r)
SELECT
geom INTO ret
FROM (
SELECT
q, (
SELECT the_geom
FROM global_province_polygons
WHERE p.c = ANY (synonyms)
AND iso3 = p.i
ORDER BY frequency DESC LIMIT 1
) geom
FROM p) n;
RETURN ret;
END
$$ LANGUAGE plpgsql;

121
server/extension/sql/0.1.0/50_namedplaces.sql Normal file
View File

@ -0,0 +1,121 @@
-- Interfacess of the server extension
---- cdb_geocode_namedplace_point(city_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text)
RETURNS Geometry AS $$
plpy.debug('Entering cdb_geocode_namedplace_point(city_name text)')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_namedplace_point($1) AS mypoint", ["text"])
rv = plpy.execute(plan, [city_name], 1)
plpy.debug('Returning from Returning from geocode_namedplace')
return rv[0]["mypoint"]
$$ LANGUAGE plpythonu;
---- cdb_geocode_namedplace_point(city_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text)
RETURNS Geometry AS $$
plpy.debug('Entering cdb_geocode_namedplace_point(city_name text, country_name text)')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_namedplace_point($1, $2) AS mypoint", ["text", "text"])
rv = plpy.execute(plan, [city_name, country_name], 1)
plpy.debug('Returning from Returning from geocode_namedplace')
return rv[0]["mypoint"]
$$ LANGUAGE plpythonu;
---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text)
RETURNS Geometry AS $$
plpy.debug('Entering cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_namedplace_point($1, $2, $3) AS mypoint", ["text", "text", "text"])
rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1)
plpy.debug('Returning from Returning from geocode_namedplace')
return rv[0]["mypoint"]
$$ LANGUAGE plpythonu;
--------------------------------------------------------------------------------
-- Implementation of the server extension
-- Note: these functions depend on the cdb_geocoder extension
---- cdb_geocode_namedplace_point(city_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_namedplace_point(city_name text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT geom INTO ret
FROM (
WITH best AS (SELECT s AS q, (SELECT the_geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) ORDER BY population DESC LIMIT 1) AS geom FROM (SELECT city_name as s) p),
next AS (SELECT p.s AS q, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM (SELECT city_name as s) p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL))
SELECT q, geom, TRUE AS success FROM best WHERE geom IS NOT NULL
UNION ALL
SELECT q, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next
) v;
RETURN ret;
END
$$ LANGUAGE plpgsql;
---- cdb_geocode_namedplace_point(city_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_namedplace_point(city_name text, country_name text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT geom INTO ret
FROM (
WITH p AS (SELECT r.s, r.c, (SELECT iso2 FROM country_decoder WHERE lower(r.c) = ANY (synonyms)) i FROM (SELECT city_name AS s, country_name::text AS c) r),
best AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.iso2 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p),
next AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND gp.iso2 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE c = p.c AND geom IS NOT NULL))
SELECT geom FROM best WHERE geom IS NOT NULL
UNION ALL
SELECT geom FROM next
) v;
RETURN ret;
END
$$ LANGUAGE plpgsql;
---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT geom INTO ret
FROM (
WITH inputcountry AS (
SELECT iso2 as isoTwo FROM country_decoder WHERE lower(country_name) = ANY (synonyms) LIMIT 1
),
p AS (
SELECT r.s, r.a1, (SELECT admin1 FROM admin1_decoder, inputcountry WHERE lower(r.a1) = ANY (synonyms) AND admin1_decoder.iso2 = inputcountry.isoTwo LIMIT 1) i FROM (SELECT city_name AS s, admin1_name::text AS a1) r),
best AS (SELECT p.s AS q, p.a1 as a1, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.admin1 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p),
next AS (SELECT p.s AS q, p.a1 AS a1, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.admin1 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL))
SELECT geom FROM best WHERE geom IS NOT NULL
UNION ALL
SELECT geom FROM next
) v;
RETURN ret;
END
$$ LANGUAGE plpgsql;

162
server/extension/sql/0.1.0/60_postalcodes.sql Normal file
View File

@ -0,0 +1,162 @@
-- Interface of the server extension
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_point(username text, orgname text, code text)
RETURNS Geometry AS $$
plpy.debug('Entering _cdb_geocode_postalcode_point')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_point($1) AS point", ["text"])
rv = plpy.execute(plan, [code], 1)
plpy.debug('Returning from _cdb_geocode_postalcode_point')
return rv[0]["point"]
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text)
RETURNS Geometry AS $$
plpy.debug('Entering _cdb_geocode_postalcode_point')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_point($1, $2) AS point", ["TEXT", "TEXT"])
rv = plpy.execute(plan, [code, country], 1)
plpy.debug('Returning from _cdb_geocode_postalcode_point')
return rv[0]["point"]
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text)
RETURNS Geometry AS $$
plpy.debug('Entering _cdb_geocode_postalcode_polygon')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_polygon($1) AS polygon", ["text"])
rv = plpy.execute(plan, [code], 1)
plpy.debug('Returning from _cdb_geocode_postalcode_polygon')
return rv[0]["polygon"]
$$ LANGUAGE plpythonu;
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text, country text)
RETURNS Geometry AS $$
plpy.debug('Entering _cdb_geocode_postalcode_point')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_polygon($1, $2) AS polygon", ["TEXT", "TEXT"])
rv = plpy.execute(plan, [code, country], 1)
plpy.debug('Returning from _cdb_geocode_postalcode_point')
return rv[0]["polygon"]
$$ LANGUAGE plpythonu;
--------------------------------------------------------------------------------
-- Implementation of the server extension
-- Note: these functions depend on the cdb_geocoder extension
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_point(code text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT geom INTO ret
FROM (
SELECT
q, (
SELECT the_geom
FROM global_postal_code_points
WHERE postal_code = upper(d.q)
LIMIT 1
) geom
FROM (SELECT code q) d
) v;
RETURN ret;
END
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_point(code text, country text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT geom INTO ret
FROM (
SELECT
q, (
SELECT the_geom
FROM global_postal_code_points
WHERE postal_code = upper(d.q)
AND iso3 = (
SELECT iso3 FROM country_decoder WHERE
lower(country) = ANY (synonyms) LIMIT 1
)
LIMIT 1
) geom
FROM (SELECT code q) d
) v;
RETURN ret;
END
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_polygon(code text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT geom INTO ret
FROM (
SELECT
q, (
SELECT the_geom
FROM global_postal_code_polygons
WHERE postal_code = upper(d.q)
LIMIT 1
) geom
FROM (SELECT code q) d
) v;
RETURN ret;
END
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_polygon(code text, country text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
BEGIN
SELECT geom INTO ret
FROM (
SELECT
q, (
SELECT the_geom
FROM global_postal_code_polygons
WHERE postal_code = upper(d.q)
AND iso3 = (
SELECT iso3 FROM country_decoder WHERE
lower(country) = ANY (synonyms) LIMIT 1
)
LIMIT 1
) geom
FROM (SELECT code q) d
) v;
RETURN ret;
END
$$ LANGUAGE plpgsql;

49
server/extension/sql/0.1.0/70_ips.sql Normal file
View File

@ -0,0 +1,49 @@
-- Interface of the server extension
CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text)
RETURNS Geometry AS $$
plpy.debug('Entering _cdb_geocode_ipaddress_point')
plpy.debug('user = %s' % username)
#--TODO: rate limiting check
#--TODO: quota check
#-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html
plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_ipaddress_point($1) AS point", ["TEXT"])
rv = plpy.execute(plan, [ip], 1)
plpy.debug('Returning from _cdb_geocode_ipaddress_point')
return rv[0]["point"]
$$ LANGUAGE plpythonu;
--------------------------------------------------------------------------------
-- Implementation of the server extension
-- Note: these functions depend on the cdb_geocoder extension
CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_ipaddress_point(ip text)
RETURNS Geometry AS $$
DECLARE
ret Geometry;
new_ip INET;
BEGIN
BEGIN
IF family(ip::inet) = 6 THEN
new_ip := ip::inet;
ELSE
new_ip := ('::ffff:' || ip)::inet;
END IF;
EXCEPTION WHEN OTHERS THEN
SELECT NULL as geom INTO ret;
RETURN ret;
END;
WITH
ips AS (SELECT ip s, new_ip net),
matches AS (SELECT s, (SELECT the_geom FROM ip_address_locations WHERE network_start_ip <= ips.net ORDER BY network_start_ip DESC LIMIT 1) geom FROM ips)
SELECT geom INTO ret
FROM matches;
RETURN ret;
END
$$ LANGUAGE plpgsql;

15
server/extension/sql/0.1.0/90_geocoder_server_user.sql Normal file
View File

@ -0,0 +1,15 @@
DO $$
BEGIN
IF NOT EXISTS (
SELECT *
FROM pg_catalog.pg_user
WHERE usename = 'geocoder_api') THEN
CREATE USER geocoder_api;
END IF;
GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_geocoder_server TO geocoder_api;
GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO geocoder_api;
GRANT USAGE ON SCHEMA cdb_geocoder_server TO geocoder_api;
GRANT USAGE ON SCHEMA public TO geocoder_api;
GRANT SELECT ON ALL TABLES IN SCHEMA public TO geocoder_api;
END$$;

0
server/extension/expected/00_install_test.out → server/extension/test/0.0.1/expected/00_install_test.out
View File

0
server/extension/expected/30_admin0_test.out → server/extension/test/0.0.1/expected/30_admin0_test.out
View File

0
server/extension/expected/40_admin1_test.out → server/extension/test/0.0.1/expected/40_admin1_test.out
View File

0
server/extension/expected/50_namedplaces_test.out → server/extension/test/0.0.1/expected/50_namedplaces_test.out
View File

0
server/extension/expected/60_postalcodes_test.out → server/extension/test/0.0.1/expected/60_postalcodes_test.out
View File

0
server/extension/expected/70_ips_test.out → server/extension/test/0.0.1/expected/70_ips_test.out
View File

0
server/extension/expected/90_remove_geocoder_api_user_test.out → server/extension/test/0.0.1/expected/90_remove_geocoder_api_user_test.out
View File

0
server/extension/sql/00_install_test.sql → server/extension/test/0.0.1/sql/00_install_test.sql
View File

0
server/extension/sql/30_admin0_test.sql → server/extension/test/0.0.1/sql/30_admin0_test.sql
View File

0
server/extension/sql/40_admin1_test.sql → server/extension/test/0.0.1/sql/40_admin1_test.sql
View File

0
server/extension/sql/50_namedplaces_test.sql → server/extension/test/0.0.1/sql/50_namedplaces_test.sql
View File

0
server/extension/sql/60_postalcodes_test.sql → server/extension/test/0.0.1/sql/60_postalcodes_test.sql
View File

0
server/extension/sql/70_ips_test.sql → server/extension/test/0.0.1/sql/70_ips_test.sql
View File

0
server/extension/sql/90_remove_geocoder_api_user_test.sql → server/extension/test/0.0.1/sql/90_remove_geocoder_api_user_test.sql
View File

30
server/extension/test/0.1.0/expected/00_install_test.out Normal file
View File

@ -0,0 +1,30 @@
-- Install dependencies
CREATE EXTENSION postgis;
CREATE EXTENSION schema_triggers;
CREATE EXTENSION plpythonu;
CREATE EXTENSION cartodb;
CREATE EXTENSION cdb_geocoder;
-- Install the extension
CREATE EXTENSION cdb_geocoder_server;
-- Mock the redis server connection to point to this very test db
SELECT cartodb.cdb_conf_setconf('redis_conf', '{"sentinel_host": "localhost", "sentinel_port": 26739, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}');
cdb_conf_setconf
------------------
(1 row)
-- Mock the varnish invalidation function
-- (used by cdb_geocoder tests)
CREATE OR REPLACE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS $$
BEGIN
RETURN;
END
$$
LANGUAGE plpgsql;
-- Set user quota
SELECT cartodb.CDB_SetUserQuotaInBytes(0);
cdb_setuserquotainbytes
-------------------------
0
(1 row)

12
server/extension/test/0.1.0/expected/20_street_test.out Normal file
View File

@ -0,0 +1,12 @@
-- Check for namedplaces signatures
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_street_point_v2'
AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text, text');
exists
--------
t
(1 row)

47
server/extension/test/0.1.0/expected/30_admin0_test.out Normal file
View File

@ -0,0 +1,47 @@
-- Check that the public function is callable, even with no data
-- It should return NULL
SELECT cdb_geocoder_server.cdb_geocode_admin0_polygon('test_user', 'test_orgname', 'Spain');
cdb_geocode_admin0_polygon
----------------------------
(1 row)
-- Insert some dummy synonym
INSERT INTO admin0_synonyms (name, adm0_a3) VALUES ('Spain', 'ESP');
-- Insert some dummy geometry to return
INSERT INTO ne_admin0_v3 (adm0_a3, the_geom) VALUES('ESP', ST_GeomFromText(
'POLYGON((-71.1031880899493 42.3152774590236,
-71.1031627617667 42.3152960829043,
-71.102923838298 42.3149156848307,
-71.1031880899493 42.3152774590236))',4326)
);
-- This should return the polygon inserted above
SELECT cdb_geocoder_server.cdb_geocode_admin0_polygon('test_user', 'test_orgname', 'Spain');
cdb_geocode_admin0_polygon
--------------------------------------------------------------------------------------------------------------------------------------------------------------------
0103000020E61000000100000004000000D0EA37A29AC651C00FD603035B284540FEFCFB379AC651C0C0503E9F5B284540FFDDDD4D96C651C033AC3B284F284540D0EA37A29AC651C00FD603035B284540
(1 row)
-- Check for admin0 signatures
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_admin0_polygon'
AND oidvectortypes(p.proargtypes) = 'text, text, text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = '_cdb_geocode_admin0_polygon'
AND oidvectortypes(p.proargtypes) = 'text');
exists
--------
t
(1 row)

81
server/extension/test/0.1.0/expected/40_admin1_test.out Normal file
View File

@ -0,0 +1,81 @@
-- Check that the public function is callable, even with no data
-- It should return NULL
SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California');
cdb_geocode_admin1_polygon
----------------------------
(1 row)
SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California', 'United States');
cdb_geocode_admin1_polygon
----------------------------
(1 row)
-- Insert dummy data into country decoder table
INSERT INTO country_decoder (synonyms, iso3) VALUES (Array['united states'], 'USA');
-- Insert some dummy data and geometry to return
INSERT INTO global_province_polygons (synonyms, iso3, the_geom) VALUES (Array['california'], 'USA', ST_GeomFromText(
'POLYGON((-71.1031880899493 42.3152774590236,
-71.1031627617667 42.3152960829043,
-71.102923838298 42.3149156848307,
-71.1031880899493 42.3152774590236))',4326)
);
-- This should return the polygon inserted above
SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California');
cdb_geocode_admin1_polygon
--------------------------------------------------------------------------------------------------------------------------------------------------------------------
0103000020E61000000100000004000000D0EA37A29AC651C00FD603035B284540FEFCFB379AC651C0C0503E9F5B284540FFDDDD4D96C651C033AC3B284F284540D0EA37A29AC651C00FD603035B284540
(1 row)
SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California', 'United States');
cdb_geocode_admin1_polygon
--------------------------------------------------------------------------------------------------------------------------------------------------------------------
0103000020E61000000100000004000000D0EA37A29AC651C00FD603035B284540FEFCFB379AC651C0C0503E9F5B284540FFDDDD4D96C651C033AC3B284F284540D0EA37A29AC651C00FD603035B284540
(1 row)
-- Check for admin1 signatures
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_admin1_polygon'
AND oidvectortypes(p.proargtypes) = 'text, text, text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_admin1_polygon'
AND oidvectortypes(p.proargtypes) = 'text, text, text, text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = '_cdb_geocode_admin1_polygon'
AND oidvectortypes(p.proargtypes) = 'text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = '_cdb_geocode_admin1_polygon'
AND oidvectortypes(p.proargtypes) = 'text, text');
exists
--------
t
(1 row)

136
server/extension/test/0.1.0/expected/50_namedplaces_test.out Normal file
View File

@ -0,0 +1,136 @@
-- Check that the public function is callable, even with no data
-- It should return NULL
SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx');
cdb_geocode_namedplace_point
------------------------------
(1 row)
SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain');
cdb_geocode_namedplace_point
------------------------------
(1 row)
SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain');
cdb_geocode_namedplace_point
------------------------------
(1 row)
-- Insert dummy data into points table
INSERT INTO global_cities_points_limited (geoname_id, name, iso2, admin1, admin2, population, lowername, the_geom) VALUES (3128760, 'Elche', 'ES', 'Valencia', 'AL', 34534, 'elche', ST_GeomFromText(
'POINT(0.6983 39.26787)',4326)
);
-- Insert dummy data into alternates table
INSERT INTO global_cities_alternates_limited (geoname_id, name, preferred, lowername, admin1_geonameid, iso2, admin1, the_geom) VALUES (3128760, 'Elx', true, 'elx', '000000', 'ES', 'Valencia', ST_GeomFromText(
'POINT(0.6983 39.26787)',4326)
);
-- Insert dummy data into country decoder table
INSERT INTO country_decoder (synonyms, iso2) VALUES (Array['spain'], 'ES');
-- Insert dummy data into admin1 decoder table
INSERT INTO admin1_decoder (admin1, synonyms, iso2) VALUES ('Valencia', Array['valencia', 'Valencia'], 'ES');
-- This should return the point inserted above
SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx');
cdb_geocode_namedplace_point
----------------------------------------------------
0101000020E6100000637FD93D7958E63F2ECA6C9049A24340
(1 row)
SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche');
cdb_geocode_namedplace_point
----------------------------------------------------
0101000020E6100000637FD93D7958E63F2ECA6C9049A24340
(1 row)
SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain');
cdb_geocode_namedplace_point
----------------------------------------------------
0101000020E6100000637FD93D7958E63F2ECA6C9049A24340
(1 row)
SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'Spain');
cdb_geocode_namedplace_point
----------------------------------------------------
0101000020E6100000637FD93D7958E63F2ECA6C9049A24340
(1 row)
SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain');
cdb_geocode_namedplace_point
----------------------------------------------------
0101000020E6100000637FD93D7958E63F2ECA6C9049A24340
(1 row)
SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'valencia', 'Spain');
cdb_geocode_namedplace_point
----------------------------------------------------
0101000020E6100000637FD93D7958E63F2ECA6C9049A24340
(1 row)
-- Check for namedplaces signatures
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_namedplace_point'
AND oidvectortypes(p.proargtypes) = 'text, text, text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_namedplace_point'
AND oidvectortypes(p.proargtypes) = 'text, text, text, text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_namedplace_point'
AND oidvectortypes(p.proargtypes) = 'text, text, text, text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = '_cdb_geocode_namedplace_point'
AND oidvectortypes(p.proargtypes) = 'text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = '_cdb_geocode_namedplace_point'
AND oidvectortypes(p.proargtypes) = 'text, text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = '_cdb_geocode_namedplace_point'
AND oidvectortypes(p.proargtypes) = 'text, text, text');
exists
--------
t
(1 row)

163
server/extension/test/0.1.0/expected/60_postalcodes_test.out Normal file
View File

@ -0,0 +1,163 @@
-- Make sure dbs are clean
DELETE FROM global_postal_code_points;
DELETE FROM global_postal_code_polygons;
DELETE FROM country_decoder;
DELETE FROM available_services;
DELETE FROM admin0_synonyms;
-- Check that the public function is callable, even with no data
-- It should return NULL
SELECT cdb_geocoder_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204');
cdb_geocode_postalcode_point
------------------------------
(1 row)
-- Insert dummy data into ip_address_locations
INSERT INTO global_postal_code_points (the_geom, iso3, postal_code, postal_code_num) VALUES (
'0101000020E61000000000000000E040408036B47414764840',
'ESP',
'03204',
3204
);
INSERT INTO global_postal_code_polygons (the_geom, iso3, postal_code, postal_code_num) VALUES (
'0106000020E610000001000000010300000001000000040000000000000000E000C01F383D7839B740400000000000E000C0AA3C0EDE220F3B4000000000004812404FB7FCCD04893D400000000000E000C01F383D7839B74040',
'ESP',
'03204',
3204
);
INSERT INTO country_decoder (iso3, synonyms) VALUES (
'ESP',
Array['spain', 'Spain', 'ESP']
);
INSERT INTO available_services (adm0_a3, admin0, postal_code_points, postal_code_polygons) VALUES (
'ESP',
't',
't',
't'
);
INSERT INTO admin0_synonyms (adm0_a3, name, name_, rank) VALUES (
'ESP',
'Spain',
'spain',
3
);
-- This should return the polygon inserted above
SELECT cdb_geocoder_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204');
cdb_geocode_postalcode_point
----------------------------------------------------
0101000020E61000000000000000E040408036B47414764840
(1 row)
SELECT cdb_geocoder_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204', 'spain');
cdb_geocode_postalcode_point
----------------------------------------------------
0101000020E61000000000000000E040408036B47414764840
(1 row)
SELECT cdb_geocoder_server.cdb_geocode_postalcode_polygon('test_user', 'test_org', '03204');
cdb_geocode_postalcode_polygon
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
0106000020E610000001000000010300000001000000040000000000000000E000C01F383D7839B740400000000000E000C0AA3C0EDE220F3B4000000000004812404FB7FCCD04893D400000000000E000C01F383D7839B74040
(1 row)
SELECT cdb_geocoder_server.cdb_geocode_postalcode_polygon('test_user', 'test_org', '03204', 'spain');
cdb_geocode_postalcode_polygon
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
0106000020E610000001000000010300000001000000040000000000000000E000C01F383D7839B740400000000000E000C0AA3C0EDE220F3B4000000000004812404FB7FCCD04893D400000000000E000C01F383D7839B74040
(1 row)
-- Clean dbs
DELETE FROM global_postal_code_points;
DELETE FROM global_postal_code_polygons;
DELETE FROM country_decoder;
DELETE FROM available_services;
DELETE FROM admin0_synonyms;
-- Check for namedplaces signatures (point and polygon)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_postalcode_point'
AND oidvectortypes(p.proargtypes) = 'text, text, text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_postalcode_point'
AND oidvectortypes(p.proargtypes) = 'text, text, text, text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_postalcode_polygon'
AND oidvectortypes(p.proargtypes) = 'text, text, text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_postalcode_polygon'
AND oidvectortypes(p.proargtypes) = 'text, text, text, text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = '_cdb_geocode_postalcode_point'
AND oidvectortypes(p.proargtypes) = 'text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = '_cdb_geocode_postalcode_point'
AND oidvectortypes(p.proargtypes) = 'text, text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = '_cdb_geocode_postalcode_polygon'
AND oidvectortypes(p.proargtypes) = 'text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = '_cdb_geocode_postalcode_polygon'
AND oidvectortypes(p.proargtypes) = 'text, text');
exists
--------
t
(1 row)

40
server/extension/test/0.1.0/expected/70_ips_test.out Normal file
View File

@ -0,0 +1,40 @@
-- Check that the public function is callable, even with no data
-- It should return NULL
SELECT cdb_geocoder_server.cdb_geocode_ipaddress_point('test_user', 'test_orgname', '0.0.0.0');
cdb_geocode_ipaddress_point
-----------------------------
(1 row)
-- Insert dummy data into ip_address_locations
INSERT INTO ip_address_locations VALUES ('::ffff:0.0.0.0'::inet, (ST_SetSRID(ST_MakePoint('40.40', '3.71'), 4326)));
-- This should return the polygon inserted above
SELECT cdb_geocoder_server.cdb_geocode_ipaddress_point('test_user', 'test_orgname', '0.0.0.0');
cdb_geocode_ipaddress_point
----------------------------------------------------
0101000020E61000003333333333334440AE47E17A14AE0D40
(1 row)
-- Check for namedplaces signatures (point and polygon)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_ipaddress_point'
AND oidvectortypes(p.proargtypes) = 'text, text, text');
exists
--------
t
(1 row)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = '_cdb_geocode_ipaddress_point'
AND oidvectortypes(p.proargtypes) = 'text');
exists
--------
t
(1 row)

5
server/extension/test/0.1.0/expected/90_remove_geocoder_api_user_test.out Normal file
View File

@ -0,0 +1,5 @@
REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_geocoder_server FROM geocoder_api;
REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA public FROM geocoder_api;
REVOKE USAGE ON SCHEMA cdb_geocoder_server FROM geocoder_api;
REVOKE USAGE ON SCHEMA public FROM geocoder_api;
REVOKE SELECT ON ALL TABLES IN SCHEMA public FROM geocoder_api;

24
server/extension/test/0.1.0/sql/00_install_test.sql Normal file
View File

@ -0,0 +1,24 @@
-- Install dependencies
CREATE EXTENSION postgis;
CREATE EXTENSION schema_triggers;
CREATE EXTENSION plpythonu;
CREATE EXTENSION cartodb;
CREATE EXTENSION cdb_geocoder;
-- Install the extension
CREATE EXTENSION cdb_geocoder_server;
-- Mock the redis server connection to point to this very test db
SELECT cartodb.cdb_conf_setconf('redis_conf', '{"sentinel_host": "localhost", "sentinel_port": 26739, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}');
-- Mock the varnish invalidation function
-- (used by cdb_geocoder tests)
CREATE OR REPLACE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS $$
BEGIN
RETURN;
END
$$
LANGUAGE plpgsql;
-- Set user quota
SELECT cartodb.CDB_SetUserQuotaInBytes(0);

7
server/extension/test/0.1.0/sql/20_street_test.sql Normal file
View File

@ -0,0 +1,7 @@
-- Check for namedplaces signatures
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_street_point_v2'
AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text, text');

32
server/extension/test/0.1.0/sql/30_admin0_test.sql Normal file
View File

@ -0,0 +1,32 @@
-- Check that the public function is callable, even with no data
-- It should return NULL
SELECT cdb_geocoder_server.cdb_geocode_admin0_polygon('test_user', 'test_orgname', 'Spain');
-- Insert some dummy synonym
INSERT INTO admin0_synonyms (name, adm0_a3) VALUES ('Spain', 'ESP');
-- Insert some dummy geometry to return
INSERT INTO ne_admin0_v3 (adm0_a3, the_geom) VALUES('ESP', ST_GeomFromText(
'POLYGON((-71.1031880899493 42.3152774590236,
-71.1031627617667 42.3152960829043,
-71.102923838298 42.3149156848307,
-71.1031880899493 42.3152774590236))',4326)
);
-- This should return the polygon inserted above
SELECT cdb_geocoder_server.cdb_geocode_admin0_polygon('test_user', 'test_orgname', 'Spain');
-- Check for admin0 signatures
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_admin0_polygon'
AND oidvectortypes(p.proargtypes) = 'text, text, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = '_cdb_geocode_admin0_polygon'
AND oidvectortypes(p.proargtypes) = 'text');

48
server/extension/test/0.1.0/sql/40_admin1_test.sql Normal file
View File

@ -0,0 +1,48 @@
-- Check that the public function is callable, even with no data
-- It should return NULL
SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California');
SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California', 'United States');
-- Insert dummy data into country decoder table
INSERT INTO country_decoder (synonyms, iso3) VALUES (Array['united states'], 'USA');
-- Insert some dummy data and geometry to return
INSERT INTO global_province_polygons (synonyms, iso3, the_geom) VALUES (Array['california'], 'USA', ST_GeomFromText(
'POLYGON((-71.1031880899493 42.3152774590236,
-71.1031627617667 42.3152960829043,
-71.102923838298 42.3149156848307,
-71.1031880899493 42.3152774590236))',4326)
);
-- This should return the polygon inserted above
SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California');
SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California', 'United States');
-- Check for admin1 signatures
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_admin1_polygon'
AND oidvectortypes(p.proargtypes) = 'text, text, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_admin1_polygon'
AND oidvectortypes(p.proargtypes) = 'text, text, text, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = '_cdb_geocode_admin1_polygon'
AND oidvectortypes(p.proargtypes) = 'text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = '_cdb_geocode_admin1_polygon'
AND oidvectortypes(p.proargtypes) = 'text, text');

72
server/extension/test/0.1.0/sql/50_namedplaces_test.sql Normal file
View File

@ -0,0 +1,72 @@
-- Check that the public function is callable, even with no data
-- It should return NULL
SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx');
SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain');
SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain');
-- Insert dummy data into points table
INSERT INTO global_cities_points_limited (geoname_id, name, iso2, admin1, admin2, population, lowername, the_geom) VALUES (3128760, 'Elche', 'ES', 'Valencia', 'AL', 34534, 'elche', ST_GeomFromText(
'POINT(0.6983 39.26787)',4326)
);
-- Insert dummy data into alternates table
INSERT INTO global_cities_alternates_limited (geoname_id, name, preferred, lowername, admin1_geonameid, iso2, admin1, the_geom) VALUES (3128760, 'Elx', true, 'elx', '000000', 'ES', 'Valencia', ST_GeomFromText(
'POINT(0.6983 39.26787)',4326)
);
-- Insert dummy data into country decoder table
INSERT INTO country_decoder (synonyms, iso2) VALUES (Array['spain'], 'ES');
-- Insert dummy data into admin1 decoder table
INSERT INTO admin1_decoder (admin1, synonyms, iso2) VALUES ('Valencia', Array['valencia', 'Valencia'], 'ES');
-- This should return the point inserted above
SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx');
SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche');
SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain');
SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'Spain');
SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain');
SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'valencia', 'Spain');
-- Check for namedplaces signatures
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_namedplace_point'
AND oidvectortypes(p.proargtypes) = 'text, text, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_namedplace_point'
AND oidvectortypes(p.proargtypes) = 'text, text, text, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_namedplace_point'
AND oidvectortypes(p.proargtypes) = 'text, text, text, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = '_cdb_geocode_namedplace_point'
AND oidvectortypes(p.proargtypes) = 'text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = '_cdb_geocode_namedplace_point'
AND oidvectortypes(p.proargtypes) = 'text, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = '_cdb_geocode_namedplace_point'
AND oidvectortypes(p.proargtypes) = 'text, text, text');

117
server/extension/test/0.1.0/sql/60_postalcodes_test.sql Normal file
View File

@ -0,0 +1,117 @@
-- Make sure dbs are clean
DELETE FROM global_postal_code_points;
DELETE FROM global_postal_code_polygons;
DELETE FROM country_decoder;
DELETE FROM available_services;
DELETE FROM admin0_synonyms;
-- Check that the public function is callable, even with no data
-- It should return NULL
SELECT cdb_geocoder_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204');
-- Insert dummy data into ip_address_locations
INSERT INTO global_postal_code_points (the_geom, iso3, postal_code, postal_code_num) VALUES (
'0101000020E61000000000000000E040408036B47414764840',
'ESP',
'03204',
3204
);
INSERT INTO global_postal_code_polygons (the_geom, iso3, postal_code, postal_code_num) VALUES (
'0106000020E610000001000000010300000001000000040000000000000000E000C01F383D7839B740400000000000E000C0AA3C0EDE220F3B4000000000004812404FB7FCCD04893D400000000000E000C01F383D7839B74040',
'ESP',
'03204',
3204
);
INSERT INTO country_decoder (iso3, synonyms) VALUES (
'ESP',
Array['spain', 'Spain', 'ESP']
);
INSERT INTO available_services (adm0_a3, admin0, postal_code_points, postal_code_polygons) VALUES (
'ESP',
't',
't',
't'
);
INSERT INTO admin0_synonyms (adm0_a3, name, name_, rank) VALUES (
'ESP',
'Spain',
'spain',
3
);
-- This should return the polygon inserted above
SELECT cdb_geocoder_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204');
SELECT cdb_geocoder_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204', 'spain');
SELECT cdb_geocoder_server.cdb_geocode_postalcode_polygon('test_user', 'test_org', '03204');
SELECT cdb_geocoder_server.cdb_geocode_postalcode_polygon('test_user', 'test_org', '03204', 'spain');
-- Clean dbs
DELETE FROM global_postal_code_points;
DELETE FROM global_postal_code_polygons;
DELETE FROM country_decoder;
DELETE FROM available_services;
DELETE FROM admin0_synonyms;
-- Check for namedplaces signatures (point and polygon)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_postalcode_point'
AND oidvectortypes(p.proargtypes) = 'text, text, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_postalcode_point'
AND oidvectortypes(p.proargtypes) = 'text, text, text, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_postalcode_polygon'
AND oidvectortypes(p.proargtypes) = 'text, text, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_postalcode_polygon'
AND oidvectortypes(p.proargtypes) = 'text, text, text, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = '_cdb_geocode_postalcode_point'
AND oidvectortypes(p.proargtypes) = 'text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = '_cdb_geocode_postalcode_point'
AND oidvectortypes(p.proargtypes) = 'text, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = '_cdb_geocode_postalcode_polygon'
AND oidvectortypes(p.proargtypes) = 'text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = '_cdb_geocode_postalcode_polygon'
AND oidvectortypes(p.proargtypes) = 'text, text');

24
server/extension/test/0.1.0/sql/70_ips_test.sql Normal file
View File

@ -0,0 +1,24 @@
-- Check that the public function is callable, even with no data
-- It should return NULL
SELECT cdb_geocoder_server.cdb_geocode_ipaddress_point('test_user', 'test_orgname', '0.0.0.0');
-- Insert dummy data into ip_address_locations
INSERT INTO ip_address_locations VALUES ('::ffff:0.0.0.0'::inet, (ST_SetSRID(ST_MakePoint('40.40', '3.71'), 4326)));
-- This should return the polygon inserted above
SELECT cdb_geocoder_server.cdb_geocode_ipaddress_point('test_user', 'test_orgname', '0.0.0.0');
-- Check for namedplaces signatures (point and polygon)
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = 'cdb_geocode_ipaddress_point'
AND oidvectortypes(p.proargtypes) = 'text, text, text');
SELECT exists(SELECT *
FROM pg_proc p
INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
WHERE ns.nspname = 'cdb_geocoder_server'
AND proname = '_cdb_geocode_ipaddress_point'
AND oidvectortypes(p.proargtypes) = 'text');

5
server/extension/test/0.1.0/sql/90_remove_geocoder_api_user_test.sql Normal file
View File

@ -0,0 +1,5 @@
REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_geocoder_server FROM geocoder_api;
REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA public FROM geocoder_api;
REVOKE USAGE ON SCHEMA cdb_geocoder_server FROM geocoder_api;
REVOKE USAGE ON SCHEMA public FROM geocoder_api;
REVOKE SELECT ON ALL TABLES IN SCHEMA public FROM geocoder_api;

216
server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py
View File

@ -1,104 +1,148 @@
import json
from dateutil.parser import parse as date_parse
class ConfigException(Exception):
pass
class UserConfig:
USER_CONFIG_KEYS = ['is_organization', 'entity_name']
def __init__(self, user_config_json, db_user_id = None):
config = json.loads(user_config_json)
filtered_config = { key: config[key] for key in self.USER_CONFIG_KEYS if key in config.keys() }
self.__check_config(filtered_config)
self.__parse_config(filtered_config)
self._user_id = self.__extract_uuid(db_user_id)
def __check_config(self, filtered_config):
if len(filtered_config.keys()) != len(self.USER_CONFIG_KEYS):
raise ConfigException("Passed user configuration is not correct, check it please")
return True
@property
def is_organization(self):
return self._is_organization
@property
def entity_name(self):
return self._entity_name
@property
def user_id(self):
return self._user_id
def __parse_config(self, filtered_config):
self._is_organization = filtered_config['is_organization']
self._entity_name = filtered_config['entity_name']
def __extract_uuid(self, db_user_id):
# Format: development_cartodb_user_<UUID>
return db_user_id.split('_')[-1]
class GeocoderConfig:
GEOCODER_CONFIG_KEYS = ['street_geocoder_provider', 'google_maps_private_key',
'nokia_monthly_quota', 'nokia_soft_geocoder_limit']
NOKIA_GEOCODER_MANDATORY_KEYS = ['nokia_monthly_quota', 'nokia_soft_geocoder_limit']
NOKIA_GEOCODER = 'nokia'
NOKIA_QUOTA_KEY = 'nokia_monthly_quota'
NOKIA_SOFT_LIMIT_KEY = 'nokia_soft_geocoder_limit'
GOOGLE_GEOCODER = 'google'
GEOCODER_TYPE = 'street_geocoder_provider'
GOOGLE_GEOCODER_API_KEY = 'google_maps_private_key'
GEOCODER_CONFIG_KEYS = ['google_maps_client_id', 'google_maps_api_key',
'geocoding_quota', 'soft_geocoding_limit',
'geocoder_type', 'period_end_date',
'heremaps_app_id', 'heremaps_app_code', 'username',
'orgname']
NOKIA_GEOCODER_MANDATORY_KEYS = ['geocoding_quota', 'soft_geocoding_limit',
'heremaps_app_id', 'heremaps_app_code']
NOKIA_GEOCODER = 'heremaps'
NOKIA_GEOCODER_APP_ID_KEY = 'heremaps_app_id'
NOKIA_GEOCODER_APP_CODE_KEY = 'heremaps_app_code'
GOOGLE_GEOCODER = 'google'
GOOGLE_GEOCODER_API_KEY = 'google_maps_api_key'
GOOGLE_GEOCODER_CLIENT_ID = 'google_maps_client_id'
GEOCODER_TYPE = 'geocoder_type'
QUOTA_KEY = 'geocoding_quota'
SOFT_LIMIT_KEY = 'soft_geocoding_limit'
USERNAME_KEY = 'username'
ORGNAME_KEY = 'orgname'
PERIOD_END_DATE = 'period_end_date'
def __init__(self, geocoder_config_json):
config = json.loads(geocoder_config_json)
filtered_config = { key: config[key] for key in self.GEOCODER_CONFIG_KEYS if key in config.keys() }
self.__check_config(filtered_config)
self.__parse_config(filtered_config)
def __init__(self, redis_connection, username, orgname=None,
heremaps_app_id=None, heremaps_app_code=None):
self._redis_connection = redis_connection
config = self.__get_user_config(username, orgname, heremaps_app_id,
heremaps_app_code)
filtered_config = {key: config[key] for key in self.GEOCODER_CONFIG_KEYS if key in config.keys()}
self.__check_config(filtered_config)
self.__parse_config(filtered_config)
def __check_config(self, filtered_config):
if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER:
if not set(self.NOKIA_GEOCODER_MANDATORY_KEYS).issubset(set(filtered_config.keys())):
raise ConfigException("""Nokia geocoder need the mandatory parameters 'nokia_monthly_quota' and 'nokia_soft_geocoder_limit'""")
elif filtered_config[self.GEOCODER_TYPE].lower() == self.GOOGLE_GEOCODER:
if self.GOOGLE_GEOCODER_API_KEY not in filtered_config.keys():
raise ConfigException("Google geocoder need the mandatory parameter 'google_maps_private_key'")
def __get_user_config(self, username, orgname=None, heremaps_app_id=None,
heremaps_app_code=None):
user_config = self._redis_connection.hgetall(
"rails:users:{0}".format(username))
if not user_config:
raise ConfigException("""There is no user config available. Please check your configuration.'""")
else:
user_config[self.USERNAME_KEY] = username
user_config[self.ORGNAME_KEY] = orgname
user_config[self.NOKIA_GEOCODER_APP_ID_KEY] = heremaps_app_id
user_config[self.NOKIA_GEOCODER_APP_CODE_KEY] = heremaps_app_code
if orgname:
self.__get_organization_config(orgname, user_config)
return True
return user_config
def __parse_config(self, filtered_config):
self._geocoder_type = filtered_config[self.GEOCODER_TYPE].lower()
self._google_maps_private_key = None
self._nokia_monthly_quota = 0
self._nokia_soft_geocoder_limit = False
if self.GOOGLE_GEOCODER == self._geocoder_type:
self._google_maps_private_key = filtered_config[self.GOOGLE_GEOCODER_API_KEY]
elif self.NOKIA_GEOCODER == self._geocoder_type:
self._nokia_monthly_quota = filtered_config[self.NOKIA_QUOTA_KEY]
self._nokia_soft_geocoder_limit = filtered_config[self.NOKIA_SOFT_LIMIT_KEY]
def __get_organization_config(self, orgname, user_config):
org_config = self._redis_connection.hgetall(
"rails:orgs:{0}".format(orgname))
if not org_config:
raise ConfigException("""There is no organization config available. Please check your configuration.'""")
else:
user_config[self.QUOTA_KEY] = org_config[self.QUOTA_KEY]
user_config[self.PERIOD_END_DATE] = org_config[self.PERIOD_END_DATE]
user_config[self.GOOGLE_GEOCODER_CLIENT_ID] = org_config[self.GOOGLE_GEOCODER_CLIENT_ID]
user_config[self.GOOGLE_GEOCODER_API_KEY] = org_config[self.GOOGLE_GEOCODER_API_KEY]
@property
def service_type(self):
return self._geocoder_type
def __check_config(self, filtered_config):
if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER:
if not set(self.NOKIA_GEOCODER_MANDATORY_KEYS).issubset(set(filtered_config.keys())):
raise ConfigException("""Some mandatory parameter/s for Nokia geocoder are missing. Check it please""")
if not filtered_config[self.NOKIA_GEOCODER_APP_ID_KEY] or not filtered_config[self.NOKIA_GEOCODER_APP_CODE_KEY]:
raise ConfigException("""Nokia geocoder configuration is missing. Check it please""")
elif filtered_config[self.GEOCODER_TYPE].lower() == self.GOOGLE_GEOCODER:
if self.GOOGLE_GEOCODER_API_KEY not in filtered_config.keys():
raise ConfigException("""Google geocoder need the mandatory parameter 'google_maps_private_key'""")
@property
def nokia_geocoder(self):
return self._geocoder_type == self.NOKIA_GEOCODER
return True
@property
def google_geocoder(self):
return self._geocoder_type == self.GOOGLE_GEOCODER
def __parse_config(self, filtered_config):
self._username = filtered_config[self.USERNAME_KEY].lower()
if filtered_config[self.ORGNAME_KEY]:
self._orgname = filtered_config[self.ORGNAME_KEY].lower()
else:
self._orgname = None
self._geocoder_type = filtered_config[self.GEOCODER_TYPE].lower()
self._geocoding_quota = float(filtered_config[self.QUOTA_KEY])
self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE])
if filtered_config[self.SOFT_LIMIT_KEY].lower() == 'true':
self._soft_geocoding_limit = True
else:
self._soft_geocoding_limit = False
if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER:
self._heremaps_app_id = filtered_config[self.NOKIA_GEOCODER_APP_ID_KEY]
self._heremaps_app_code = filtered_config[self.NOKIA_GEOCODER_APP_CODE_KEY]
elif filtered_config[self.GEOCODER_TYPE].lower() == self.GOOGLE_GEOCODER:
self._google_maps_api_key = filtered_config[self.GOOGLE_GEOCODER_API_KEY]
self._google_maps_client_id = filtered_config[self.GOOGLE_GEOCODER_CLIENT_ID]
@property
def google_api_key(self):
return self._google_maps_private_key
@property
def service_type(self):
if self._geocoder_type == self.GOOGLE_GEOCODER:
return 'geocoder_google'
else:
return 'geocoder_here'
@property
def nokia_monthly_quota(self):
return self._nokia_monthly_quota
@property
def heremaps_geocoder(self):
return self._geocoder_type == self.NOKIA_GEOCODER
@property
def nokia_soft_limit(self):
return self._nokia_soft_geocoder_limit
@property
def google_geocoder(self):
return self._geocoder_type == self.GOOGLE_GEOCODER
@property
def google_client_id(self):
return self._google_maps_client_id
@property
def google_api_key(self):
return self._google_maps_api_key
@property
def geocoding_quota(self):
return self._geocoding_quota
@property
def soft_geocoding_limit(self):
return self._soft_geocoding_limit
@property
def period_end_date(self):
return self._period_end_date
@property
def heremaps_app_id(self):
return self._heremaps_app_id
@property
def heremaps_app_code(self):
return self._heremaps_app_code
@property
def username(self):
return self._username
@property
def organization(self):
return self._orgname

55
server/lib/python/cartodb_geocoder/cartodb_geocoder/quota_service.py
View File

@ -1,31 +1,52 @@
import user_service
import config_helper
from datetime import date
class QuotaService:
""" Class to manage all the quota operation for the Geocoder SQL API Extension """
def __init__(self, user_config, geocoder_config, redis_connection):
self._user_config = user_config
self._geocoder_config = geocoder_config
self._user_service = user_service.UserService(self._user_config,
self._geocoder_config.service_type, redis_connection)
class QuotaService:
""" Class to manage all the quota operation for
the Geocoder SQL API Extension """
def __init__(self, user_geocoder_config, redis_connection):
self._user_geocoder_config = user_geocoder_config
self._user_service = user_service.UserService(
self._user_geocoder_config, redis_connection)
def check_user_quota(self):
""" Check if the current user quota surpasses the current quota """
# We don't have quota check for google geocoder
if self._geocoder_config.google_geocoder:
if self._user_geocoder_config.google_geocoder:
return True
user_quota = self._geocoder_config.nokia_monthly_quota
user_quota = self._user_geocoder_config.geocoding_quota
today = date.today()
service_type = self._geocoder_config.service_type
current_used = self._user_service.used_quota(service_type, today.year, today.month)
soft_geocoder_limit = self._geocoder_config.nokia_soft_limit
service_type = self._user_geocoder_config.service_type
current_used = self._user_service.used_quota(service_type, today)
soft_geocoding_limit = self._user_geocoder_config.soft_geocoding_limit
print "User quota: {0} --- current_used: {1} --- limit: {2}".format(user_quota, current_used, soft_geocoder_limit)
if soft_geocoding_limit or current_used <= user_quota:
return True
else:
return False
return True if soft_geocoder_limit or current_used <= user_quota else False
def increment_success_geocoder_use(self, amount=1):
self._user_service.increment_service_use(
self._user_geocoder_config.service_type, "success_responses",
amount=amount)
self.increment_total_geocoder_use(amount)
def increment_geocoder_use(self, amount=1):
self._user_service.increment_service_use(self._geocoder_config.service_type)
def increment_empty_geocoder_use(self, amount=1):
self._user_service.increment_service_use(
self._user_geocoder_config.service_type, "empty_responses",
amount=amount)
self.increment_total_geocoder_use(amount)
def increment_failed_geocoder_use(self, amount=1):
self._user_service.increment_service_use(
self._user_geocoder_config.service_type, "fail_responses",
amount=amount)
self.increment_total_geocoder_use(amount)
def increment_total_geocoder_use(self, amount=1):
self._user_service.increment_service_use(
self._user_geocoder_config.service_type, "total_requests",
amount=amount)

35
server/lib/python/cartodb_geocoder/cartodb_geocoder/redis_helper.py
View File

@ -1,20 +1,29 @@
from redis.sentinel import Sentinel
class RedisHelper:
REDIS_DEFAULT_USER_DB = 5
REDIS_TIMEOUT = 2 #seconds
REDIS_DEFAULT_USER_DB = 5
REDIS_DEFAULT_TIMEOUT = 2 #seconds
REDIS_SENTINEL_DEFAULT_PORT = 26379
def __init__(self, sentinel_host, sentinel_port, sentinel_master_id, redis_db=REDIS_DEFAULT_USER_DB, **kwargs):
self.sentinel_host = sentinel_host
self.sentinel_port = sentinel_port
self.sentinel_master_id = sentinel_master_id
self.timeout = kwargs['timeout'] if 'timeout' in kwargs else REDIS_DEFAULT_TIMEOUT
self.redis_db = redis_db
def __init__(self, sentinel_host, sentinel_port, sentinel_master_id,
redis_db=REDIS_DEFAULT_USER_DB, **kwargs):
self.sentinel_host = sentinel_host
self.sentinel_port = sentinel_port
self.sentinel_master_id = sentinel_master_id
self.timeout = kwargs['timeout'] if 'timeout' in kwargs else self.REDIS_DEFAULT_TIMEOUT
self.redis_db = redis_db
def redis_connection(self):
return self.__create_redis_connection()
def redis_connection(self):
return self.__create_redis_connection()
def __create_redis_connection(self):
sentinel = Sentinel([(self.sentinel_host, 26379)], socket_timeout=self.timeout)
return sentinel.master_for(self.sentinel_master_id, socket_timeout=self.timeout, db=self.redis_db)
def __create_redis_connection(self):
sentinel = Sentinel([(self.sentinel_host,
self.REDIS_SENTINEL_DEFAULT_PORT)],
socket_timeout=self.timeout)
return sentinel.master_for(
self.sentinel_master_id,
socket_timeout=self.timeout,
db=self.redis_db
)

128
server/lib/python/cartodb_geocoder/cartodb_geocoder/user_service.py
View File

@ -1,64 +1,96 @@
import redis_helper
from datetime import date
from datetime import date, timedelta
from dateutil.relativedelta import relativedelta
class UserService:
""" Class to manage all the user info """
""" Class to manage all the user info """
GEOCODING_QUOTA_KEY = "geocoding_quota"
GEOCODING_SOFT_LIMIT_KEY = "soft_geocoder_limit"
SERVICE_GEOCODER_NOKIA = 'geocoder_here'
SERVICE_GEOCODER_GOOGLE = 'geocoder_google'
SERVICE_GEOCODER_CACHE = 'geocoder_cache'
REDIS_CONNECTION_KEY = "redis_connection"
REDIS_CONNECTION_HOST = "redis_host"
REDIS_CONNECTION_PORT = "redis_port"
REDIS_CONNECTION_DB = "redis_db"
GEOCODING_QUOTA_KEY = "geocoding_quota"
GEOCODING_SOFT_LIMIT_KEY = "soft_geocoder_limit"
def __init__(self, user_config, service_type, redis_connection):
self.user_config = user_config
self.service_type = service_type
self._redis_connection = redis_connection
REDIS_CONNECTION_KEY = "redis_connection"
REDIS_CONNECTION_HOST = "redis_host"
REDIS_CONNECTION_PORT = "redis_port"
REDIS_CONNECTION_DB = "redis_db"
def used_quota(self, service_type, year, month, day=None):
""" Recover the used quota for the user in the current month """
redis_key_data = self.__get_redis_key(service_type, year, month, day)
current_use = self._redis_connection.hget(redis_key_data['redis_name'], redis_key_data['redis_key'])
return int(current_use) if current_use else 0
def __init__(self, user_geocoder_config, redis_connection):
self._user_geocoder_config = user_geocoder_config
self._redis_connection = redis_connection
self._username = user_geocoder_config.username
self._orgname = user_geocoder_config.organization
def increment_service_use(self, service_type, date=date.today(), amount=1):
""" Increment the services uses in monthly and daily basis"""
self.__increment_monthly_uses(date, service_type, amount)
self.__increment_daily_uses(date, service_type, amount)
def used_quota(self, service_type, date):
""" Recover the used quota for the user in the current month """
date_from, date_to = self.__current_billing_cycle()
current_use = 0
success_responses = self.get_metrics(service_type,
'success_responses', date_from,
date_to)
empty_responses = self.get_metrics(service_type,
'empty_responses', date_from,
date_to)
current_use += (success_responses + empty_responses)
if service_type == self.SERVICE_GEOCODER_NOKIA:
cache_hits = self.get_metrics(self.SERVICE_GEOCODER_CACHE,
'success_responses', date_from,
date_to)
current_use += cache_hits
# Private functions
return current_use
def __increment_monthly_uses(self, date, service_type, amount):
redis_key_data = self.__get_redis_key(service_type, date.year, date.month)
self._redis_connection.hincrby(redis_key_data['redis_name'],redis_key_data['redis_key'],amount)
def increment_service_use(self, service_type, metric, date=date.today(), amount=1):
""" Increment the services uses in monthly and daily basis"""
self.__increment_user_uses(service_type, metric, date, amount)
if self._orgname:
self.__increment_organization_uses(service_type, metric, date, amount)
def __increment_daily_uses(self, date, service_type, amount):
redis_key_data = self.__get_redis_key(service_type, date.year, date.month, date.day)
self._redis_connection.hincrby(redis_key_data['redis_name'],redis_key_data['redis_key'],amount)
def get_metrics(self, service, metric, date_from, date_to):
aggregated_metric = 0
key_prefix = "org" if self._orgname else "user"
entity_name = self._orgname if self._orgname else self._username
for date in self.__generate_date_range(date_from, date_to):
redis_prefix = self.__parse_redis_prefix(key_prefix, entity_name,
service, metric, date)
score = self._redis_connection.zscore(redis_prefix, date.day)
aggregated_metric += score if score else 0
return aggregated_metric
def __get_redis_key(self, service_type, year, month, day=None):
redis_name = self.__parse_redis_name(service_type,day)
redis_key = self.__parse_redis_key(year,month,day)
# Private functions
return {'redis_name': redis_name, 'redis_key': redis_key}
def __increment_user_uses(self, service_type, metric, date, amount):
redis_prefix = self.__parse_redis_prefix("user", self._username,
service_type, metric, date)
self._redis_connection.zincrby(redis_prefix, date.day, amount)
def __parse_redis_name(self,service_type, day=None):
prefix = "org" if self.user_config.is_organization else "user"
dated_key = "used_quota_day" if day else "used_quota_month"
redis_name = "{0}:{1}:{2}:{3}".format(
prefix, self.user_config.entity_name, service_type, dated_key
)
if self.user_config.is_organization and day:
redis_name = "{0}:{1}".format(redis_name, self.user_config.user_id)
def __increment_organization_uses(self, service_type, metric, date, amount):
redis_prefix = self.__parse_redis_prefix("org", self._orgname,
service_type, metric, date)
self._redis_connection.zincrby(redis_prefix, date.day, amount)
return redis_name
def __parse_redis_prefix(self, prefix, entity_name, service_type, metric, date):
yearmonth_key = date.strftime('%Y%m')
redis_name = "{0}:{1}:{2}:{3}:{4}".format(prefix, entity_name,
service_type, metric,
yearmonth_key)
def __parse_redis_key(self,year,month,day=None):
if day:
redis_key = "{0}_{1}_{2}".format(year,month,day)
else:
redis_key = "{0}_{1}".format(year,month)
return redis_name
return redis_key
def __current_billing_cycle(self):
""" Return the begining and end date for the current billing cycle """
end_period_day = self._user_geocoder_config.period_end_date.day
today = date.today()
if end_period_day > today.day:
temp_date = today + relativedelta(months=-1)
date_from = date(temp_date.year, temp_date.month, end_period_day)
else:
date_from = date(today.year, today.month, end_period_day)
return date_from, today
def __generate_date_range(self, date_from, date_to):
for n in range(int((date_to - date_from).days + 1)):
yield date_from + timedelta(n)

3
server/lib/python/cartodb_geocoder/requirements.txt
View File

@ -1,4 +1,5 @@
redis-py==2.10.5
redis==2.10.5
python-dateutil==2.4.2
# Test
mock==1.3.0

69
server/lib/python/cartodb_geocoder/test/test_config_helper.py
View File

@ -1,48 +1,41 @@
import test_helper
from cartodb_geocoder import config_helper
from unittest import TestCase
from nose.tools import assert_raises
from mockredis import MockRedis
from datetime import datetime, timedelta
class TestConfigHelper(TestCase):
def test_should_return_list_of_user_config_if_its_ok(self):
user_config_json = '{"is_organization": false, "entity_name": "test_user"}'
user_config = config_helper.UserConfig(user_config_json, 'development_cartodb_user_UUID')
assert user_config.is_organization == False
assert user_config.entity_name == 'test_user'
assert user_config.user_id == 'UUID'
def setUp(self):
self.redis_conn = MockRedis()
def test_should_return_raise_config_exception_if_not_ok(self):
user_config_json = '{"is_organization": "false"}'
assert_raises(config_helper.ConfigException, config_helper.UserConfig, user_config_json)
def test_should_return_list_of_nokia_geocoder_config_if_its_ok(self):
test_helper.build_redis_user_config(self.redis_conn, 'test_user')
geocoder_config = config_helper.GeocoderConfig(self.redis_conn,
'test_user', None,
'nokia_id', 'nokia_cod')
assert geocoder_config.heremaps_geocoder == True
assert geocoder_config.geocoding_quota == 100
assert geocoder_config.soft_geocoding_limit == False
def test_should_return_raise_config_exception_if_empty(self):
user_config_json = '{}'
assert_raises(config_helper.ConfigException, config_helper.UserConfig, user_config_json)
def test_should_return_list_of_nokia_geocoder_config_ok_for_org(self):
yesterday = datetime.today() - timedelta(days=1)
test_helper.build_redis_user_config(self.redis_conn, 'test_user')
test_helper.build_redis_org_config(self.redis_conn, 'test_org',
quota=200, end_date=yesterday)
geocoder_config = config_helper.GeocoderConfig(self.redis_conn,
'test_user', 'test_org',
'nokia_id', 'nokia_cod')
assert geocoder_config.heremaps_geocoder == True
assert geocoder_config.geocoding_quota == 200
assert geocoder_config.soft_geocoding_limit == False
assert geocoder_config.period_end_date.date() == yesterday.date()
def test_should_return_list_of_nokia_geocoder_config_if_its_ok(self):
geocoder_config_json = """{"street_geocoder_provider": "Nokia",
"nokia_monthly_quota": 100, "nokia_soft_geocoder_limit": false}"""
geocoder_config = config_helper.GeocoderConfig(geocoder_config_json)
assert geocoder_config.nokia_geocoder == True
assert geocoder_config.nokia_monthly_quota == 100
assert geocoder_config.nokia_soft_limit == False
def test_should_raise_configuration_exception_when_missing_nokia_geocoder_parameters(self):
geocoder_config_json = '{"street_geocoder_provider": "NokiA", "nokia_monthly_quota": "100"}'
assert_raises(config_helper.ConfigException, config_helper.GeocoderConfig, geocoder_config_json)
def test_should_raise_configuration_exception_when_missing_nokia_geocoder_parameters_2(self):
geocoder_config_json = '{"street_geocoder_provider": "NoKia", "nokia_soft_geocoder_limit": "false"}'
assert_raises(config_helper.ConfigException, config_helper.GeocoderConfig, geocoder_config_json)
def test_should_return_list_of_google_geocoder_config_if_its_ok(self):
geocoder_config_json = """{"street_geocoder_provider": "gOOgle",
"google_maps_private_key": "sdasdasda"}"""
geocoder_config = config_helper.GeocoderConfig(geocoder_config_json)
assert geocoder_config.google_geocoder == True
assert geocoder_config.google_api_key == 'sdasdasda'
def test_should_raise_configuration_exception_when_missing_google_api_key(self):
geocoder_config_json = '{"street_geocoder_provider": "google"}'
assert_raises(config_helper.ConfigException, config_helper.GeocoderConfig, geocoder_config_json)
def test_should_raise_configuration_exception_when_missing_nokia_geocoder_parameters(self):
test_helper.build_redis_user_config(self.redis_conn, 'test_user')
assert_raises(config_helper.ConfigException,
config_helper.GeocoderConfig,
self.redis_conn, 'test_user',
None, None, None)

33
server/lib/python/cartodb_geocoder/test/test_helper.py Normal file
View File

@ -0,0 +1,33 @@
from datetime import datetime, date
def build_redis_user_config(redis_conn, username, quota=100, soft_limit=False,
service="heremaps",
end_date=datetime.today()):
user_redis_name = "rails:users:{0}".format(username)
redis_conn.hset(user_redis_name, 'soft_geocoding_limit', soft_limit)
redis_conn.hset(user_redis_name, 'geocoding_quota', quota)
redis_conn.hset(user_redis_name, 'geocoder_type', service)
redis_conn.hset(user_redis_name, 'period_end_date', end_date)
redis_conn.hset(user_redis_name, 'google_maps_client_id', '')
redis_conn.hset(user_redis_name, 'google_maps_api_key', '')
def build_redis_org_config(redis_conn, orgname, quota=100,
end_date=datetime.today()):
org_redis_name = "rails:orgs:{0}".format(orgname)
redis_conn.hset(org_redis_name, 'geocoding_quota', quota)
redis_conn.hset(org_redis_name, 'period_end_date', end_date)
redis_conn.hset(org_redis_name, 'google_maps_client_id', '')
redis_conn.hset(org_redis_name, 'google_maps_api_key', '')
def increment_geocoder_uses(redis_conn, username, orgname=None,
date=date.today(), service='geocoder_here',
metric='success_responses', amount=20):
prefix = 'org' if orgname else 'user'
entity_name = orgname if orgname else username
yearmonth = date.strftime('%Y%m')
redis_name = "{0}:{1}:{2}:{3}:{4}".format(prefix, entity_name,
service, metric, yearmonth)
redis_conn.zincrby(redis_name, date.day, amount)

148
server/lib/python/cartodb_geocoder/test/test_quota_service.py
View File

@ -1,85 +1,109 @@
import test_helper
from mockredis import MockRedis
from cartodb_geocoder import quota_service
from cartodb_geocoder import config_helper
from unittest import TestCase
from nose.tools import assert_raises
from datetime import datetime
from datetime import datetime, date
class TestQuotaService(TestCase):
# single user
# user:<username>:<service>:used_quota_month:year_month
# user:<username>:<service>:used_quota_day:year_month_day
# organization user
# org:<orgname>:<service>:used_quota_month:year_month
# org:<orgname>:<service>:<uuid>:used_quota_day:year_month_day
# single user
# user:<username>:<service>:<metric>:YYYYMM:DD
# organization user
# org:<orgname>:<service>:<metric>:YYYYMM:DD
def setUp(self):
self.fake_redis_connection = MockRedis()
# def increment_geocoder_uses(self, username, orgname=None,
# date=date.today(), service='geocoder_here',
# metric='success_responses', amount=20):
def test_should_return_true_if_user_quota_with_no_use(self):
qs = self.__build_quota_service()
assert qs.check_user_quota() == True
def setUp(self):
self.redis_conn = MockRedis()
def test_should_return_true_if_org_quota_with_no_use(self):
qs = self.__build_quota_service(organization=True)
assert qs.check_user_quota() == True
def test_should_return_true_if_user_quota_with_no_use(self):
qs = self.__build_quota_service('test_user')
assert qs.check_user_quota() is True
def test_should_return_true_if_user_quota_is_not_completely_used(self):
qs = self.__build_quota_service()
self.__increment_geocoder_uses('test_user', '20151111')
assert qs.check_user_quota() == True
def test_should_return_true_if_org_quota_with_no_use(self):
qs = self.__build_quota_service('test_user', orgname='test_org')
assert qs.check_user_quota() is True
def test_should_return_true_if_org_quota_is_not_completely_used(self):
qs = self.__build_quota_service(organization=True)
self.__increment_geocoder_uses('test_user', '20151111', org=True)
assert qs.check_user_quota() == True
def test_should_return_true_if_user_quota_is_not_completely_used(self):
qs = self.__build_quota_service('test_user')
test_helper.increment_geocoder_uses(self.redis_conn, 'test_user')
assert qs.check_user_quota() is True
def test_should_return_false_if_user_quota_is_surpassed(self):
qs = self.__build_quota_service(quota = 1, soft_limit=False)
self.__increment_geocoder_uses('test_user', '20151111')
assert qs.check_user_quota() == False
def test_should_return_true_if_org_quota_is_not_completely_used(self):
qs = self.__build_quota_service('test_user', orgname='test_org')
test_helper.increment_geocoder_uses(self.redis_conn, 'test_user',
orgname='test_org')
assert qs.check_user_quota() is True
def test_should_return_false_if_org_quota_is_surpassed(self):
qs = self.__build_quota_service(organization=True, quota=1)
self.__increment_geocoder_uses('test_user', '20151111', org=True)
assert qs.check_user_quota() == False
def test_should_return_false_if_user_quota_is_surpassed(self):
qs = self.__build_quota_service('test_user')
test_helper.increment_geocoder_uses(self.redis_conn, 'test_user',
amount=300)
assert qs.check_user_quota() is False
def test_should_return_true_if_user_quota_is_surpassed_but_soft_limit_is_enabled(self):
qs = self.__build_quota_service(quota=1, soft_limit=True)
self.__increment_geocoder_uses('test_user', '20151111')
assert qs.check_user_quota() == True
def test_should_return_false_if_org_quota_is_surpassed(self):
qs = self.__build_quota_service('test_user', orgname='test_org')
test_helper.increment_geocoder_uses(self.redis_conn, 'test_user',
orgname='test_org', amount=400)
assert qs.check_user_quota() is False
def test_should_return_true_if_org_quota_is_surpassed_but_soft_limit_is_enabled(self):
qs = self.__build_quota_service(organization=True, quota=1, soft_limit=True)
self.__increment_geocoder_uses('test_user', '20151111', org=True)
assert qs.check_user_quota() == True
def test_should_return_true_if_user_quota_is_surpassed_but_soft_limit_is_enabled(self):
qs = self.__build_quota_service('test_user', soft_limit=True)
test_helper.increment_geocoder_uses(self.redis_conn, 'test_user',
amount=300)
assert qs.check_user_quota() is True
def test_should_check_user_increment_and_quota_check_correctly(self):
qs = self.__build_quota_service(quota=2, soft_limit=False)
qs.increment_geocoder_use()
assert qs.check_user_quota() == True
def test_should_return_true_if_org_quota_is_surpassed_but_soft_limit_is_enabled(self):
qs = self.__build_quota_service('test_user', orgname='test_org',
soft_limit=True)
test_helper.increment_geocoder_uses(self.redis_conn, 'test_user',
orgname='test_org', amount=400)
assert qs.check_user_quota() is True
def test_should_check_org_increment_and_quota_check_correctly(self):
qs = self.__build_quota_service(organization=True, quota=2, soft_limit=False)
qs.increment_geocoder_use()
assert qs.check_user_quota() == True
def test_should_check_user_increment_and_quota_check_correctly(self):
qs = self.__build_quota_service('test_user', quota=2)
qs.increment_success_geocoder_use()
assert qs.check_user_quota() == True
qs.increment_success_geocoder_use(amount=2)
assert qs.check_user_quota() == False
month = date.today().strftime('%Y%m')
name = 'user:test_user:geocoder_here:total_requests:{0}'.format(month)
total_requests = self.redis_conn.zscore(name, date.today().day)
assert total_requests == 3
def __build_quota_service(self, quota=100, service='nokia', organization=False, soft_limit=False):
is_organization = 'true' if organization else 'false'
has_soft_limit = 'true' if soft_limit else 'false'
user_config_json = '{{"is_organization": {0}, "entity_name": "test_user"}}'.format(is_organization)
geocoder_config_json = """{{"street_geocoder_provider": "{0}","nokia_monthly_quota": {1},
"nokia_soft_geocoder_limit": {2}}}""".format(service, quota, has_soft_limit)
user_config = config_helper.UserConfig(user_config_json, 'user_id')
geocoder_config = config_helper.GeocoderConfig(geocoder_config_json)
def test_should_check_org_increment_and_quota_check_correctly(self):
qs = self.__build_quota_service('test_user', orgname='test_org',
quota=2)
qs.increment_success_geocoder_use()
assert qs.check_user_quota() == True
qs.increment_success_geocoder_use(amount=2)
assert qs.check_user_quota() == False
month = date.today().strftime('%Y%m')
org_name = 'org:test_org:geocoder_here:total_requests:{0}'.format(month)
org_total_requests = self.redis_conn.zscore(org_name, date.today().day)
assert org_total_requests == 3
user_name = 'user:test_user:geocoder_here:total_requests:{0}'.format(month)
user_total_requests = self.redis_conn.zscore(user_name, date.today().day)
assert user_total_requests == 3
return quota_service.QuotaService(user_config, geocoder_config, redis_connection = self.fake_redis_connection)
def __build_quota_service(self, username, quota=100, service='heremaps',
orgname=None, soft_limit=False,
end_date = datetime.today()):
test_helper.build_redis_user_config(self.redis_conn, username,
quota = quota, service = service,
soft_limit = soft_limit,
end_date = end_date)
if orgname:
test_helper.build_redis_org_config(self.redis_conn, orgname,
quota=quota, end_date=end_date)
geocoder_config = config_helper.GeocoderConfig(self.redis_conn,
username, orgname,
'nokia_id', 'nokia_cod')
return quota_service.QuotaService(geocoder_config,
redis_connection = self.redis_conn)
def __increment_geocoder_uses(self, entity_name, date_string, service='nokia', amount=20, org=False):
prefix = 'org' if org else 'user'
date = datetime.strptime(date_string, "%Y%m%d")
redis_name = "{0}:{1}:{2}:used_quota_month".format(prefix, entity_name, service)
redis_key_month = "{0}_{1}".format(date.year, date.month)
self.fake_redis_connection.hset(redis_name, redis_key_month, amount)

Some files were not shown because too many files have changed in this diff Show More