diff --git a/README.md b/README.md index 0b12815..d80e222 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,21 @@ # geocoder-api The CartoDB Geocoder SQL API (server and client FTM) + +### Deploy instructions +Steps to deploy a new Geocoder API version : + +- Deploy new version of geocoder API to all servers +- Update the server user using: ALTER EXTENSION cdb_geocoder_server UPDATE TO ''; +- Update the python dependencies if needed: **cartodb_geocoder** and **heremaps** +- Add the needed config in the `cdb_conf` table: + - `redis_metadata_config` and `redis_metrics_conf` + - `{"sentinel_host": "localhost", "sentinel_port": 26739, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}` + - `heremaps_conf` + - `{"app_id": "APP_ID", "app_code": "APP_CODE"}` +- Deploy the client to all the servers with the new version +- Deploy the editor with the new geocoder api version changed (https://github.com/CartoDB/cartodb/blob/master/app/models/user/db_service.rb#L18) +- Execute the rails task to update first the CartoDB team organizaton to test in production + - `RAILS_ENV=production bundle exec rake cartodb:db:configure_geocoder_extension_for_organizations['team']` +- Check if all works perfectly for our team. If so, execute the rake tasks to update all the users and organizations: + - `RAILS_ENV=production bundle exec rake cartodb:db:configure_geocoder_extension_for_organizations['', true]` + - `RAILS_ENV=production bundle exec rake cartodb:db:configure_geocoder_extension_for_non_org_users['', true]` diff --git a/client/.gitignore b/client/.gitignore index 1ff37f5..cf79682 100644 --- a/client/.gitignore +++ b/client/.gitignore @@ -1,4 +1,8 @@ results/ regression.diffs regression.out +20_public_functions.sql +30_plproxy_functions.sql +90_grant_execute.sql cdb_geocoder_client--0.0.1.sql +cdb_geocoder_client--0.1.0.sql diff --git a/client/Makefile b/client/Makefile index 1e0696c..a9ff0c0 100644 --- a/client/Makefile +++ b/client/Makefile @@ -3,9 +3,22 @@ EXTENSION = cdb_geocoder_client EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/") -DATA = $(EXTENSION)--$(EXTVERSION).sql +# The new version to be generated from templates +NEW_EXTENSION_ARTIFACT = $(EXTENSION)--$(EXTVERSION).sql -REGRESS = $(notdir $(basename $(wildcard sql/*test.sql))) +# DATA is a special variable used by postgres build infrastructure +# These are the files to be installed in the server shared dir, +# for installation from scratch, upgrades and downgrades. +# @see http://www.postgresql.org/docs/current/static/extend-pgxs.html +DATA = $(NEW_EXTENSION_ARTIFACT) \ + cdb_geocoder_client--0.0.1.sql \ + cdb_geocoder_client--0.1.0--0.0.1.sql \ + cdb_geocoder_client--0.0.1--0.1.0.sql + + +REGRESS = $(notdir $(basename $(wildcard test/$(EXTVERSION)/sql/*test.sql))) +TEST_DIR = test/$(EXTVERSION) +REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)' # postgres build stuff PG_CONFIG = pg_config @@ -16,7 +29,7 @@ SOURCES_DATA_DIR = sql/$(EXTVERSION) # The interface definition is used along with some templates to automatically generate code RENDERER = ../sql-template-renderer -INTERFACE_FILE = ../interface.yaml +INTERFACE_FILE = ../interface_$(EXTVERSION).yaml TEMPLATE_DIR = templates TEMPLATE_FILES = $(wildcard $(TEMPLATE_DIR)/*.erb) GENERATED_SQL_FILES = $(patsubst $(TEMPLATE_DIR)/%.erb, $(SOURCES_DATA_DIR)/%.sql, $(TEMPLATE_FILES)) @@ -26,7 +39,7 @@ $(GENERATED_SQL_FILES): $(SOURCES_DATA_DIR)/%.sql: $(TEMPLATE_DIR)/%.erb $(INTER SOURCES_DATA = $(wildcard $(SOURCES_DATA_DIR)/*.sql) $(GENERATED_SQL_FILES) -$(DATA): $(SOURCES_DATA) +$(NEW_EXTENSION_ARTIFACT): $(SOURCES_DATA) rm -f $@ cat $(SOURCES_DATA_DIR)/*.sql >> $@ @@ -34,5 +47,5 @@ all: $(DATA) # Only meant for development time, do not use once a version is released devclean: - rm -f $(DATA) + rm -f $(NEW_EXTENSION_ARTIFACT) rm -f $(GENERATED_SQL_FILES) diff --git a/client/cdb_geocoder_client--0.0.1--0.1.0.sql b/client/cdb_geocoder_client--0.0.1--0.1.0.sql new file mode 100644 index 0000000..76032ad --- /dev/null +++ b/client/cdb_geocoder_client--0.0.1--0.1.0.sql @@ -0,0 +1,60 @@ +-- +-- Get entity config function +-- +-- The purpose of this function is to retrieve the username and organization name from +-- a) schema where he/her is the owner in case is an organization user +-- b) entity_name from the cdb_conf database in case is a non organization user +CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_entity_config() +RETURNS record AS $$ +DECLARE + result cdb_geocoder_client._entity_config; + is_organization boolean; + username text; + organization_name text; +BEGIN + SELECT cartodb.cdb_conf_getconf('user_config')->'is_organization' INTO is_organization; + IF is_organization IS NULL THEN + RAISE EXCEPTION 'User must have user configuration in the config table'; + ELSIF is_organization = TRUE THEN + SELECT nspname + FROM pg_namespace s + LEFT JOIN pg_roles r ON s.nspowner = r.oid + WHERE r.rolname = session_user INTO username; + SELECT cartodb.cdb_conf_getconf('user_config')->>'entity_name' INTO organization_name; + ELSE + SELECT cartodb.cdb_conf_getconf('user_config')->>'entity_name' INTO username; + organization_name = NULL; + END IF; + result.username = username; + result.organization_name = organization_name; + RETURN result; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_street_point_v2 (searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_geocoder_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country) INTO ret; + RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_street_point_v2 (username text, organization_name text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ + CONNECT cdb_geocoder_client._server_conn_str(); + SELECT cdb_geocoder_server.cdb_geocode_street_point_v2 (username, organization_name, searchtext, city, state_province, country); +$$ LANGUAGE plproxy; + +GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_street_point_v2(searchtext text, city text, state_province text, country text) TO publicuser; \ No newline at end of file diff --git a/client/cdb_geocoder_client--0.0.1.sql b/client/cdb_geocoder_client--0.0.1.sql new file mode 100644 index 0000000..846931c --- /dev/null +++ b/client/cdb_geocoder_client--0.0.1.sql @@ -0,0 +1,359 @@ +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION cdb_geocoder_client" to load this file. \quit +-- +-- Geocoder server connection config +-- +-- The purpose of this function is provide to the PL/Proxy functions +-- the connection string needed to connect with the server + +CREATE OR REPLACE FUNCTION cdb_geocoder_client._server_conn_str() +RETURNS text AS $$ +DECLARE + db_connection_str text; +BEGIN + SELECT cartodb.cdb_conf_getconf('geocoder_server_config')->'connection_str' INTO db_connection_str; + SELECT trim(both '"' FROM db_connection_str) INTO db_connection_str; + RETURN db_connection_str; +END; +$$ LANGUAGE 'plpgsql';CREATE TYPE cdb_geocoder_client._entity_config AS ( + username text, + organization_name text +); + +-- +-- Get entity config function +-- +-- The purpose of this function is to retrieve the username and organization name from +-- a) schema where he/her is the owner in case is an organization user +-- b) entity_name from the cdb_conf database in case is a non organization user +CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_entity_config() +RETURNS record AS $$ +DECLARE + result cdb_geocoder_client._entity_config; + is_organization boolean; + username text; + organization_name text; +BEGIN + SELECT cartodb.cdb_conf_getconf('user_config')->'is_organization' INTO is_organization; + IF is_organization IS NULL THEN + RAISE EXCEPTION 'User must have user configuration in the config table'; + ELSIF is_organization = TRUE THEN + SELECT nspname + FROM pg_namespace s + LEFT JOIN pg_roles r ON s.nspowner = r.oid + WHERE r.rolname = session_user INTO username; + SELECT cartodb.cdb_conf_getconf('user_config')->'entity_name' INTO organization_name; + ELSE + SELECT cartodb.cdb_conf_getconf('user_config')->'entity_name' INTO username; + organization_name = NULL; + END IF; + result.username = username; + result.organization_name = organization_name; + RETURN result; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER;-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_admin0_polygon (country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_geocoder_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + SELECT cdb_geocoder_client._cdb_geocode_admin0_polygon(username, orgname, country_name) INTO ret; + RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_admin1_polygon (admin1_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_geocoder_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + SELECT cdb_geocoder_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name) INTO ret; + RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_admin1_polygon (admin1_name text, country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_geocoder_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + SELECT cdb_geocoder_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name, country_name) INTO ret; + RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_namedplace_point (city_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_geocoder_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + SELECT cdb_geocoder_client._cdb_geocode_namedplace_point(username, orgname, city_name) INTO ret; + RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_namedplace_point (city_name text, country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_geocoder_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + SELECT cdb_geocoder_client._cdb_geocode_namedplace_point(username, orgname, city_name, country_name) INTO ret; + RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_namedplace_point (city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_geocoder_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + SELECT cdb_geocoder_client._cdb_geocode_namedplace_point(username, orgname, city_name, admin1_name, country_name) INTO ret; + RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_postalcode_polygon (postal_code text, country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_geocoder_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + SELECT cdb_geocoder_client._cdb_geocode_postalcode_polygon(username, orgname, postal_code, country_name) INTO ret; + RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_postalcode_point (postal_code text, country_name text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_geocoder_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + SELECT cdb_geocoder_client._cdb_geocode_postalcode_point(username, orgname, postal_code, country_name) INTO ret; + RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +-- +-- Public geocoder API function +-- +-- These are the only ones with permissions to publicuser role +-- and should also be the only ones with SECURITY DEFINER + +CREATE OR REPLACE FUNCTION cdb_geocoder_client.cdb_geocode_ipaddress_point (ip_address text) +RETURNS Geometry AS $$ +DECLARE + ret Geometry; + username text; + orgname text; +BEGIN + IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN + RAISE EXCEPTION 'The api_key must be provided'; + END IF; + SELECT u, o INTO username, orgname FROM cdb_geocoder_client._cdb_entity_config() AS (u text, o text); + -- JSON value stored "" is taken as literal + IF username IS NULL OR username = '' OR username = '""' THEN + RAISE EXCEPTION 'Username is a mandatory argument, check it out'; + END IF; + SELECT cdb_geocoder_client._cdb_geocode_ipaddress_point(username, orgname, ip_address) INTO ret; + RETURN ret; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; + +CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_admin0_polygon (username text, organization_name text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_geocoder_client._server_conn_str(); + SELECT cdb_geocoder_server.cdb_geocode_admin0_polygon (username, organization_name, country_name); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_admin1_polygon (username text, organization_name text, admin1_name text) +RETURNS Geometry AS $$ + CONNECT cdb_geocoder_client._server_conn_str(); + SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon (username, organization_name, admin1_name); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_admin1_polygon (username text, organization_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_geocoder_client._server_conn_str(); + SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon (username, organization_name, admin1_name, country_name); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_namedplace_point (username text, organization_name text, city_name text) +RETURNS Geometry AS $$ + CONNECT cdb_geocoder_client._server_conn_str(); + SELECT cdb_geocoder_server.cdb_geocode_namedplace_point (username, organization_name, city_name); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_namedplace_point (username text, organization_name text, city_name text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_geocoder_client._server_conn_str(); + SELECT cdb_geocoder_server.cdb_geocode_namedplace_point (username, organization_name, city_name, country_name); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_namedplace_point (username text, organization_name text, city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_geocoder_client._server_conn_str(); + SELECT cdb_geocoder_server.cdb_geocode_namedplace_point (username, organization_name, city_name, admin1_name, country_name); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_postalcode_polygon (username text, organization_name text, postal_code text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_geocoder_client._server_conn_str(); + SELECT cdb_geocoder_server.cdb_geocode_postalcode_polygon (username, organization_name, postal_code, country_name); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_postalcode_point (username text, organization_name text, postal_code text, country_name text) +RETURNS Geometry AS $$ + CONNECT cdb_geocoder_client._server_conn_str(); + SELECT cdb_geocoder_server.cdb_geocode_postalcode_point (username, organization_name, postal_code, country_name); +$$ LANGUAGE plproxy; + +CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_geocode_ipaddress_point (username text, organization_name text, ip_address text) +RETURNS Geometry AS $$ + CONNECT cdb_geocoder_client._server_conn_str(); + SELECT cdb_geocoder_server.cdb_geocode_ipaddress_point (username, organization_name, ip_address); +$$ LANGUAGE plproxy; + +-- Make sure by default there are no permissions for publicuser +-- NOTE: this happens at extension creation time, as part of an implicit transaction. +REVOKE ALL PRIVILEGES ON SCHEMA cdb_geocoder_client FROM PUBLIC, publicuser CASCADE; + +-- Grant permissions on the schema to publicuser (but just the schema) +GRANT USAGE ON SCHEMA cdb_geocoder_client TO publicuser; + +-- Revoke execute permissions on all functions in the schema by default +REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_geocoder_client FROM PUBLIC, publicuser; +GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_admin0_polygon(country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_admin1_polygon(admin1_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_admin1_polygon(admin1_name text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_namedplace_point(city_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_namedplace_point(city_name text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_postalcode_polygon(postal_code text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_postalcode_point(postal_code text, country_name text) TO publicuser; +GRANT EXECUTE ON FUNCTION cdb_geocoder_client.cdb_geocode_ipaddress_point(ip_address text) TO publicuser; diff --git a/client/cdb_geocoder_client--0.1.0--0.0.1.sql b/client/cdb_geocoder_client--0.1.0--0.0.1.sql new file mode 100644 index 0000000..c10a677 --- /dev/null +++ b/client/cdb_geocoder_client--0.1.0--0.0.1.sql @@ -0,0 +1,35 @@ +DROP FUNCTION IF EXISTS cdb_geocoder_client.cdb_geocode_street_point_v2 (text, text, text, text); +DROP FUNCTION IF EXISTS cdb_geocoder_client._cdb_geocode_street_point_v2 (text, text, text, text); + +-- +-- Get entity config function +-- +-- The purpose of this function is to retrieve the username and organization name from +-- a) schema where he/her is the owner in case is an organization user +-- b) entity_name from the cdb_conf database in case is a non organization user +CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_entity_config() +RETURNS record AS $$ +DECLARE + result cdb_geocoder_client._entity_config; + is_organization boolean; + username text; + organization_name text; +BEGIN + SELECT cartodb.cdb_conf_getconf('user_config')->'is_organization' INTO is_organization; + IF is_organization IS NULL THEN + RAISE EXCEPTION 'User must have user configuration in the config table'; + ELSIF is_organization = TRUE THEN + SELECT nspname + FROM pg_namespace s + LEFT JOIN pg_roles r ON s.nspowner = r.oid + WHERE r.rolname = session_user INTO username; + SELECT cartodb.cdb_conf_getconf('user_config')->'entity_name' INTO organization_name; + ELSE + SELECT cartodb.cdb_conf_getconf('user_config')->'entity_name' INTO username; + organization_name = NULL; + END IF; + result.username = username; + result.organization_name = organization_name; + RETURN result; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; \ No newline at end of file diff --git a/client/cdb_geocoder_client.control b/client/cdb_geocoder_client.control index c903511..4d1f047 100644 --- a/client/cdb_geocoder_client.control +++ b/client/cdb_geocoder_client.control @@ -1,6 +1,6 @@ # CartoDB geocoder client API extension comment = 'CartoDB geocoder client API extension' -default_version = '0.0.1' +default_version = '0.1.0' requires = 'plproxy, cartodb' superuser = true schema = cdb_geocoder_client diff --git a/client/sql/0.0.1/.gitignore b/client/sql/0.0.1/.gitignore deleted file mode 100644 index 4cf1e95..0000000 --- a/client/sql/0.0.1/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -20_public_functions.sql -30_plproxy_functions.sql -90_grant_execute.sql diff --git a/client/sql/0.1.0/00_header.sql b/client/sql/0.1.0/00_header.sql new file mode 100644 index 0000000..a140c33 --- /dev/null +++ b/client/sql/0.1.0/00_header.sql @@ -0,0 +1,2 @@ +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION cdb_geocoder_client" to load this file. \quit diff --git a/client/sql/0.1.0/10_geocoder_server_conn.sql b/client/sql/0.1.0/10_geocoder_server_conn.sql new file mode 100644 index 0000000..5f3e1fc --- /dev/null +++ b/client/sql/0.1.0/10_geocoder_server_conn.sql @@ -0,0 +1,16 @@ +-- +-- Geocoder server connection config +-- +-- The purpose of this function is provide to the PL/Proxy functions +-- the connection string needed to connect with the server + +CREATE OR REPLACE FUNCTION cdb_geocoder_client._server_conn_str() +RETURNS text AS $$ +DECLARE + db_connection_str text; +BEGIN + SELECT cartodb.cdb_conf_getconf('geocoder_server_config')->'connection_str' INTO db_connection_str; + SELECT trim(both '"' FROM db_connection_str) INTO db_connection_str; + RETURN db_connection_str; +END; +$$ LANGUAGE 'plpgsql'; \ No newline at end of file diff --git a/client/sql/0.1.0/15_config_management.sql b/client/sql/0.1.0/15_config_management.sql new file mode 100644 index 0000000..34da860 --- /dev/null +++ b/client/sql/0.1.0/15_config_management.sql @@ -0,0 +1,37 @@ +CREATE TYPE cdb_geocoder_client._entity_config AS ( + username text, + organization_name text +); + +-- +-- Get entity config function +-- +-- The purpose of this function is to retrieve the username and organization name from +-- a) schema where he/her is the owner in case is an organization user +-- b) entity_name from the cdb_conf database in case is a non organization user +CREATE OR REPLACE FUNCTION cdb_geocoder_client._cdb_entity_config() +RETURNS record AS $$ +DECLARE + result cdb_geocoder_client._entity_config; + is_organization boolean; + username text; + organization_name text; +BEGIN + SELECT cartodb.cdb_conf_getconf('user_config')->'is_organization' INTO is_organization; + IF is_organization IS NULL THEN + RAISE EXCEPTION 'User must have user configuration in the config table'; + ELSIF is_organization = TRUE THEN + SELECT nspname + FROM pg_namespace s + LEFT JOIN pg_roles r ON s.nspowner = r.oid + WHERE r.rolname = session_user INTO username; + SELECT cartodb.cdb_conf_getconf('user_config')->>'entity_name' INTO organization_name; + ELSE + SELECT cartodb.cdb_conf_getconf('user_config')->>'entity_name' INTO username; + organization_name = NULL; + END IF; + result.username = username; + result.organization_name = organization_name; + RETURN result; +END; +$$ LANGUAGE 'plpgsql' SECURITY DEFINER; \ No newline at end of file diff --git a/client/sql/0.1.0/80_permissions.sql b/client/sql/0.1.0/80_permissions.sql new file mode 100644 index 0000000..16fc021 --- /dev/null +++ b/client/sql/0.1.0/80_permissions.sql @@ -0,0 +1,9 @@ +-- Make sure by default there are no permissions for publicuser +-- NOTE: this happens at extension creation time, as part of an implicit transaction. +REVOKE ALL PRIVILEGES ON SCHEMA cdb_geocoder_client FROM PUBLIC, publicuser CASCADE; + +-- Grant permissions on the schema to publicuser (but just the schema) +GRANT USAGE ON SCHEMA cdb_geocoder_client TO publicuser; + +-- Revoke execute permissions on all functions in the schema by default +REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_geocoder_client FROM PUBLIC, publicuser; diff --git a/client/templates/20_public_functions.erb b/client/templates/20_public_functions.erb index 6ee473d..39a7c06 100644 --- a/client/templates/20_public_functions.erb +++ b/client/templates/20_public_functions.erb @@ -4,7 +4,7 @@ -- These are the only ones with permissions to publicuser role -- and should also be the only ones with SECURITY DEFINER -CREATE OR REPLACE FUNCTION <%= GEOCODER_CLIENT_SCHEMA %>.<%= name %> (<%= params_with_type %>) +CREATE OR REPLACE FUNCTION <%= GEOCODER_CLIENT_SCHEMA %>.<%= name %> (<%= params_with_type_and_default %>) RETURNS <%= return_type %> AS $$ DECLARE ret <%= return_type %>; @@ -14,7 +14,7 @@ BEGIN IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN RAISE EXCEPTION 'The api_key must be provided'; END IF; - SELECT u, o INTO username, orgname FROM _cdb_entity_config() AS (u text, o text); + SELECT u, o INTO username, orgname FROM <%= GEOCODER_CLIENT_SCHEMA %>._cdb_entity_config() AS (u text, o text); -- JSON value stored "" is taken as literal IF username IS NULL OR username = '' OR username = '""' THEN RAISE EXCEPTION 'Username is a mandatory argument, check it out'; diff --git a/client/templates/30_plproxy_functions.erb b/client/templates/30_plproxy_functions.erb index 71ebdc4..906d8a5 100644 --- a/client/templates/30_plproxy_functions.erb +++ b/client/templates/30_plproxy_functions.erb @@ -1,4 +1,4 @@ -CREATE OR REPLACE FUNCTION <%= GEOCODER_CLIENT_SCHEMA %>._<%= name %> (username text, organization_name text, <%= params_with_type %>) +CREATE OR REPLACE FUNCTION <%= GEOCODER_CLIENT_SCHEMA %>._<%= name %> (username text, organization_name text, <%= params_with_type_and_default %>) RETURNS <%= return_type %> AS $$ CONNECT <%= GEOCODER_CLIENT_SCHEMA %>._server_conn_str(); SELECT cdb_geocoder_server.<%= name %> (username, organization_name, <%= params %>); diff --git a/client/expected/00_installation_test.out b/client/test/0.0.1/expected/00_installation_test.out similarity index 100% rename from client/expected/00_installation_test.out rename to client/test/0.0.1/expected/00_installation_test.out diff --git a/client/expected/10_admin0_test.out b/client/test/0.0.1/expected/10_admin0_test.out similarity index 100% rename from client/expected/10_admin0_test.out rename to client/test/0.0.1/expected/10_admin0_test.out diff --git a/client/expected/20_admin1_test.out b/client/test/0.0.1/expected/20_admin1_test.out similarity index 100% rename from client/expected/20_admin1_test.out rename to client/test/0.0.1/expected/20_admin1_test.out diff --git a/client/expected/30_namedplaces_test.out b/client/test/0.0.1/expected/30_namedplaces_test.out similarity index 100% rename from client/expected/30_namedplaces_test.out rename to client/test/0.0.1/expected/30_namedplaces_test.out diff --git a/client/expected/40_postalcodes_test.out b/client/test/0.0.1/expected/40_postalcodes_test.out similarity index 100% rename from client/expected/40_postalcodes_test.out rename to client/test/0.0.1/expected/40_postalcodes_test.out diff --git a/client/expected/50_ipaddresses_test.out b/client/test/0.0.1/expected/50_ipaddresses_test.out similarity index 100% rename from client/expected/50_ipaddresses_test.out rename to client/test/0.0.1/expected/50_ipaddresses_test.out diff --git a/client/expected/90_permissions_test.out b/client/test/0.0.1/expected/90_permissions_test.out similarity index 100% rename from client/expected/90_permissions_test.out rename to client/test/0.0.1/expected/90_permissions_test.out diff --git a/client/sql/00_installation_test.sql b/client/test/0.0.1/sql/00_installation_test.sql similarity index 100% rename from client/sql/00_installation_test.sql rename to client/test/0.0.1/sql/00_installation_test.sql diff --git a/client/sql/10_admin0_test.sql b/client/test/0.0.1/sql/10_admin0_test.sql similarity index 100% rename from client/sql/10_admin0_test.sql rename to client/test/0.0.1/sql/10_admin0_test.sql diff --git a/client/sql/20_admin1_test.sql b/client/test/0.0.1/sql/20_admin1_test.sql similarity index 100% rename from client/sql/20_admin1_test.sql rename to client/test/0.0.1/sql/20_admin1_test.sql diff --git a/client/sql/30_namedplaces_test.sql b/client/test/0.0.1/sql/30_namedplaces_test.sql similarity index 100% rename from client/sql/30_namedplaces_test.sql rename to client/test/0.0.1/sql/30_namedplaces_test.sql diff --git a/client/sql/40_postalcodes_test.sql b/client/test/0.0.1/sql/40_postalcodes_test.sql similarity index 100% rename from client/sql/40_postalcodes_test.sql rename to client/test/0.0.1/sql/40_postalcodes_test.sql diff --git a/client/sql/50_ipaddresses_test.sql b/client/test/0.0.1/sql/50_ipaddresses_test.sql similarity index 100% rename from client/sql/50_ipaddresses_test.sql rename to client/test/0.0.1/sql/50_ipaddresses_test.sql diff --git a/client/sql/90_permissions_test.sql b/client/test/0.0.1/sql/90_permissions_test.sql similarity index 100% rename from client/sql/90_permissions_test.sql rename to client/test/0.0.1/sql/90_permissions_test.sql diff --git a/client/test/0.1.0/expected/00_installation_test.out b/client/test/0.1.0/expected/00_installation_test.out new file mode 100644 index 0000000..cbeef05 --- /dev/null +++ b/client/test/0.1.0/expected/00_installation_test.out @@ -0,0 +1,29 @@ +-- Install dependencies +CREATE EXTENSION postgis; +CREATE EXTENSION schema_triggers; +CREATE EXTENSION plpythonu; +CREATE EXTENSION cartodb; +CREATE EXTENSION plproxy; +-- Install the extension +CREATE EXTENSION cdb_geocoder_client; +-- Mock the server connection to point to this very test db +SELECT cartodb.cdb_conf_setconf('geocoder_server_config', '{"connection_str": "dbname=contrib_regression host=127.0.0.1 user=postgres"}'); + cdb_conf_setconf +------------------ + +(1 row) + +-- Mock the user configuration +SELECT cartodb.cdb_conf_setconf('user_config', '{"is_organization": false, "entity_name": "test_user"}'); + cdb_conf_setconf +------------------ + +(1 row) + +-- Mock the server schema +CREATE SCHEMA cdb_geocoder_server; +-- Create a test user to check permissions +DROP ROLE IF EXISTS test_regular_user; +CREATE ROLE test_regular_user; +GRANT publicuser TO test_regular_user; +ALTER ROLE test_regular_user SET search_path TO public,cartodb,cdb_geocoder_client; diff --git a/client/test/0.1.0/expected/10_admin0_test.out b/client/test/0.1.0/expected/10_admin0_test.out new file mode 100644 index 0000000..46d0b3f --- /dev/null +++ b/client/test/0.1.0/expected/10_admin0_test.out @@ -0,0 +1,20 @@ +-- Add to the search path the schema +SET search_path TO public,cartodb,cdb_geocoder_client; +-- Mock the server function +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_admin0_polygon invoked with params (%, %, %)', username, orgname, country_name; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; +-- Exercise the public and the proxied function +SELECT cdb_geocode_admin0_polygon('Spain'); +NOTICE: cdb_geocoder_client._cdb_geocode_admin0_polygon(3): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_admin0_polygon invoked with params (test_user, , Spain) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_admin0_polygon(username, orgname, country_name)" +PL/pgSQL function cdb_geocode_admin0_polygon(text) line 15 at SQL statement + cdb_geocode_admin0_polygon +---------------------------- + +(1 row) + diff --git a/client/test/0.1.0/expected/20_admin1_test.out b/client/test/0.1.0/expected/20_admin1_test.out new file mode 100644 index 0000000..2b859c3 --- /dev/null +++ b/client/test/0.1.0/expected/20_admin1_test.out @@ -0,0 +1,36 @@ +-- Add to the search path the schema +SET search_path TO public,cartodb,cdb_geocoder_client; +-- Mock the server functions +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_admin1_polygon invoked with params (%, %, %)', username, orgname, admin1_name; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_admin1_polygon invoked with params (%, %, %, %)', username, orgname, admin1_name, country_name; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; +-- Exercise the public and the proxied function +SELECT cdb_geocode_admin1_polygon('California'); +NOTICE: cdb_geocoder_client._cdb_geocode_admin1_polygon(3): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_admin1_polygon invoked with params (test_user, , California) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name)" +PL/pgSQL function cdb_geocode_admin1_polygon(text) line 15 at SQL statement + cdb_geocode_admin1_polygon +---------------------------- + +(1 row) + +SELECT cdb_geocode_admin1_polygon('California', 'United States'); +NOTICE: cdb_geocoder_client._cdb_geocode_admin1_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_admin1_polygon invoked with params (test_user, , California, United States) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name, country_name)" +PL/pgSQL function cdb_geocode_admin1_polygon(text,text) line 15 at SQL statement + cdb_geocode_admin1_polygon +---------------------------- + +(1 row) + diff --git a/client/test/0.1.0/expected/30_namedplaces_test.out b/client/test/0.1.0/expected/30_namedplaces_test.out new file mode 100644 index 0000000..4205dc0 --- /dev/null +++ b/client/test/0.1.0/expected/30_namedplaces_test.out @@ -0,0 +1,52 @@ +-- Add to the search path the schema +SET search_path TO public,cartodb,cdb_geocoder_client; +-- Mock the server functions +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_namedplace_point invoked with params (%, %, %)', username, orgname, city_name; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_namedplace_point invoked with params (%, %, %, %)', username, orgname, city_name, country_name; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_namedplace_point invoked with params (%, %, %, %, %)', username, orgname, city_name, admin1_name, country_name; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; +-- Exercise the public and the proxied function +SELECT cdb_geocode_namedplace_point('Elx'); +NOTICE: cdb_geocoder_client._cdb_geocode_namedplace_point(3): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_namedplace_point invoked with params (test_user, , Elx) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_namedplace_point(username, orgname, city_name)" +PL/pgSQL function cdb_geocode_namedplace_point(text) line 15 at SQL statement + cdb_geocode_namedplace_point +------------------------------ + +(1 row) + +SELECT cdb_geocode_namedplace_point('Elx', 'Spain'); +NOTICE: cdb_geocoder_client._cdb_geocode_namedplace_point(4): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_namedplace_point invoked with params (test_user, , Elx, Spain) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_namedplace_point(username, orgname, city_name, country_name)" +PL/pgSQL function cdb_geocode_namedplace_point(text,text) line 15 at SQL statement + cdb_geocode_namedplace_point +------------------------------ + +(1 row) + +SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain'); +NOTICE: cdb_geocoder_client._cdb_geocode_namedplace_point(5): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_namedplace_point invoked with params (test_user, , Elx, Valencia, Spain) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_namedplace_point(username, orgname, city_name, admin1_name, country_name)" +PL/pgSQL function cdb_geocode_namedplace_point(text,text,text) line 15 at SQL statement + cdb_geocode_namedplace_point +------------------------------ + +(1 row) + diff --git a/client/test/0.1.0/expected/40_postalcodes_test.out b/client/test/0.1.0/expected/40_postalcodes_test.out new file mode 100644 index 0000000..92d39d1 --- /dev/null +++ b/client/test/0.1.0/expected/40_postalcodes_test.out @@ -0,0 +1,36 @@ +-- Add to the search path the schema +SET search_path TO public,cartodb,cdb_geocoder_client; +-- Mock the server functions +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_polygon(username text, orgname text, postal_code text, country_name text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_postalcode_polygon invoked with params (%, %, %, %)', username, orgname, postal_code, country_name; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_point(username text, orgname text, postal_code text, country_name text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_postalcode_point invoked with params (%, %, %, %)', username, orgname, postal_code, country_name; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; +-- Exercise the public and the proxied function +SELECT cdb_geocode_postalcode_polygon('03204', 'Spain'); +NOTICE: cdb_geocoder_client._cdb_geocode_postalcode_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_postalcode_polygon invoked with params (test_user, , 03204, Spain) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_postalcode_polygon(username, orgname, postal_code, country_name)" +PL/pgSQL function cdb_geocode_postalcode_polygon(text,text) line 15 at SQL statement + cdb_geocode_postalcode_polygon +-------------------------------- + +(1 row) + +SELECT cdb_geocode_postalcode_point('03204', 'Spain'); +NOTICE: cdb_geocoder_client._cdb_geocode_postalcode_point(4): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_postalcode_point invoked with params (test_user, , 03204, Spain) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_postalcode_point(username, orgname, postal_code, country_name)" +PL/pgSQL function cdb_geocode_postalcode_point(text,text) line 15 at SQL statement + cdb_geocode_postalcode_point +------------------------------ + +(1 row) + diff --git a/client/test/0.1.0/expected/50_ipaddresses_test.out b/client/test/0.1.0/expected/50_ipaddresses_test.out new file mode 100644 index 0000000..9a10179 --- /dev/null +++ b/client/test/0.1.0/expected/50_ipaddresses_test.out @@ -0,0 +1,20 @@ +-- Add to the search path the schema +SET search_path TO public,cartodb,cdb_geocoder_client; +-- Mock the server functions +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_ipaddress_point(username text, orgname text, ip_address text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_ipaddress_point invoked with params (%, %, %)', username, orgname, ip_address; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; +-- Exercise the public and the proxied function +SELECT cdb_geocode_ipaddress_point('8.8.8.8'); +NOTICE: cdb_geocoder_client._cdb_geocode_ipaddress_point(3): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_ipaddress_point invoked with params (test_user, , 8.8.8.8) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_ipaddress_point(username, orgname, ip_address)" +PL/pgSQL function cdb_geocode_ipaddress_point(text) line 15 at SQL statement + cdb_geocode_ipaddress_point +----------------------------- + +(1 row) + diff --git a/client/test/0.1.0/expected/60_street_v2_test.out b/client/test/0.1.0/expected/60_street_v2_test.out new file mode 100644 index 0000000..6750855 --- /dev/null +++ b/client/test/0.1.0/expected/60_street_v2_test.out @@ -0,0 +1,101 @@ +-- Add to the search path the schema +SET search_path TO public,cartodb,cdb_geocoder_client; +-- Mock the server functions +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_street_point_v2 (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (%, %, %, %, %, %)', username, orgname, searchtext, city, state_province, country; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; +-- Exercise the public and the proxied function +SELECT cdb_geocode_street_point_v2('One street, 1'); +NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (test_user, , One street, 1, , , ) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)" +PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement + cdb_geocode_street_point_v2 +----------------------------- + +(1 row) + +SELECT cdb_geocode_street_point_v2('One street', 'city'); +NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (test_user, , One street, city, , ) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)" +PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement + cdb_geocode_street_point_v2 +----------------------------- + +(1 row) + +SELECT cdb_geocode_street_point_v2('One street', 'city', 'state'); +NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (test_user, , One street, city, state, ) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)" +PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement + cdb_geocode_street_point_v2 +----------------------------- + +(1 row) + +SELECT cdb_geocode_street_point_v2('One street', 'city', 'state', 'country'); +NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (test_user, , One street, city, state, country) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)" +PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement + cdb_geocode_street_point_v2 +----------------------------- + +(1 row) + +SELECT cdb_geocode_street_point_v2('One street', 'city', NULL, 'country'); +NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (test_user, , One street, city, , country) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)" +PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement + cdb_geocode_street_point_v2 +----------------------------- + +(1 row) + +SELECT cdb_geocode_street_point_v2('One street, 1'); +NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (test_user, , One street, 1, , , ) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)" +PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement + cdb_geocode_street_point_v2 +----------------------------- + +(1 row) + +SELECT cdb_geocode_street_point_v2('One street', 'city'); +NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (test_user, , One street, city, , ) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)" +PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement + cdb_geocode_street_point_v2 +----------------------------- + +(1 row) + +SELECT cdb_geocode_street_point_v2('One street', 'city', 'state'); +NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (test_user, , One street, city, state, ) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)" +PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement + cdb_geocode_street_point_v2 +----------------------------- + +(1 row) + +SELECT cdb_geocode_street_point_v2('One street', 'city', 'state', 'country'); +NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (test_user, , One street, city, state, country) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)" +PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement + cdb_geocode_street_point_v2 +----------------------------- + +(1 row) + +SELECT cdb_geocode_street_point_v2('One street', 'city', NULL, 'country'); +NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (test_user, , One street, city, , country) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)" +PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement + cdb_geocode_street_point_v2 +----------------------------- + +(1 row) + diff --git a/client/test/0.1.0/expected/90_permissions_test.out b/client/test/0.1.0/expected/90_permissions_test.out new file mode 100644 index 0000000..7569760 --- /dev/null +++ b/client/test/0.1.0/expected/90_permissions_test.out @@ -0,0 +1,117 @@ +-- Use regular user role +SET ROLE test_regular_user; +-- Add to the search path the schema +SET search_path TO public,cartodb,cdb_geocoder_client; +-- Exercise the public function +-- it is public, it shall work +SELECT cdb_geocode_admin0_polygon('Spain'); +NOTICE: cdb_geocoder_client._cdb_geocode_admin0_polygon(3): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_admin0_polygon invoked with params (test_user, , Spain) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_admin0_polygon(username, orgname, country_name)" +PL/pgSQL function cdb_geocode_admin0_polygon(text) line 15 at SQL statement + cdb_geocode_admin0_polygon +---------------------------- + +(1 row) + +SELECT cdb_geocode_admin1_polygon('California'); +NOTICE: cdb_geocoder_client._cdb_geocode_admin1_polygon(3): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_admin1_polygon invoked with params (test_user, , California) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name)" +PL/pgSQL function cdb_geocode_admin1_polygon(text) line 15 at SQL statement + cdb_geocode_admin1_polygon +---------------------------- + +(1 row) + +SELECT cdb_geocode_admin1_polygon('California', 'United States'); +NOTICE: cdb_geocoder_client._cdb_geocode_admin1_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_admin1_polygon invoked with params (test_user, , California, United States) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_admin1_polygon(username, orgname, admin1_name, country_name)" +PL/pgSQL function cdb_geocode_admin1_polygon(text,text) line 15 at SQL statement + cdb_geocode_admin1_polygon +---------------------------- + +(1 row) + +SELECT cdb_geocode_namedplace_point('Elx'); +NOTICE: cdb_geocoder_client._cdb_geocode_namedplace_point(3): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_namedplace_point invoked with params (test_user, , Elx) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_namedplace_point(username, orgname, city_name)" +PL/pgSQL function cdb_geocode_namedplace_point(text) line 15 at SQL statement + cdb_geocode_namedplace_point +------------------------------ + +(1 row) + +SELECT cdb_geocode_namedplace_point('Elx', 'Valencia'); +NOTICE: cdb_geocoder_client._cdb_geocode_namedplace_point(4): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_namedplace_point invoked with params (test_user, , Elx, Valencia) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_namedplace_point(username, orgname, city_name, country_name)" +PL/pgSQL function cdb_geocode_namedplace_point(text,text) line 15 at SQL statement + cdb_geocode_namedplace_point +------------------------------ + +(1 row) + +SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain'); +NOTICE: cdb_geocoder_client._cdb_geocode_namedplace_point(5): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_namedplace_point invoked with params (test_user, , Elx, Valencia, Spain) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_namedplace_point(username, orgname, city_name, admin1_name, country_name)" +PL/pgSQL function cdb_geocode_namedplace_point(text,text,text) line 15 at SQL statement + cdb_geocode_namedplace_point +------------------------------ + +(1 row) + +SELECT cdb_geocode_postalcode_polygon('03204', 'Spain'); +NOTICE: cdb_geocoder_client._cdb_geocode_postalcode_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_postalcode_polygon invoked with params (test_user, , 03204, Spain) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_postalcode_polygon(username, orgname, postal_code, country_name)" +PL/pgSQL function cdb_geocode_postalcode_polygon(text,text) line 15 at SQL statement + cdb_geocode_postalcode_polygon +-------------------------------- + +(1 row) + +SELECT cdb_geocode_postalcode_point('03204', 'Spain'); +NOTICE: cdb_geocoder_client._cdb_geocode_postalcode_point(4): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_postalcode_point invoked with params (test_user, , 03204, Spain) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_postalcode_point(username, orgname, postal_code, country_name)" +PL/pgSQL function cdb_geocode_postalcode_point(text,text) line 15 at SQL statement + cdb_geocode_postalcode_point +------------------------------ + +(1 row) + +SELECT cdb_geocode_ipaddress_point('8.8.8.8'); +NOTICE: cdb_geocoder_client._cdb_geocode_ipaddress_point(3): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_ipaddress_point invoked with params (test_user, , 8.8.8.8) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_ipaddress_point(username, orgname, ip_address)" +PL/pgSQL function cdb_geocode_ipaddress_point(text) line 15 at SQL statement + cdb_geocode_ipaddress_point +----------------------------- + +(1 row) + +SELECT cdb_geocode_street_point_v2('one street, 1'); +NOTICE: cdb_geocoder_client._cdb_geocode_street_point_v2(6): [contrib_regression] REMOTE NOTICE: cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (test_user, , one street, 1, , , ) +CONTEXT: SQL statement "SELECT cdb_geocoder_client._cdb_geocode_street_point_v2(username, orgname, searchtext, city, state_province, country)" +PL/pgSQL function cdb_geocode_street_point_v2(text,text,text,text) line 15 at SQL statement + cdb_geocode_street_point_v2 +----------------------------- + +(1 row) + +-- Check the regular user has no permissions on private functions +SELECT _cdb_geocode_admin0_polygon('evil_user', 'evil_orgname', 'Hell'); +ERROR: permission denied for function _cdb_geocode_admin0_polygon +SELECT _cdb_geocode_admin1_polygon('evil_user', 'evil_orgname', 'Hell'); +ERROR: permission denied for function _cdb_geocode_admin1_polygon +SELECT _cdb_geocode_admin1_polygon('evil_user', 'evil_orgname', 'Sheol', 'Hell'); +ERROR: permission denied for function _cdb_geocode_admin1_polygon +SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'Sheol'); +ERROR: permission denied for function _cdb_geocode_namedplace_point +SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'Sheol', 'Hell'); +ERROR: permission denied for function _cdb_geocode_namedplace_point +SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'Sheol', 'Hell', 'Ugly world'); +ERROR: permission denied for function _cdb_geocode_namedplace_point +SELECT _cdb_geocode_postalcode_polygon('evil_user', 'evil_orgname', '66666', 'Hell'); +ERROR: permission denied for function _cdb_geocode_postalcode_polygon +SELECT _cdb_geocode_postalcode_point('evil_user', 'evil_orgname', '66666', 'Hell'); +ERROR: permission denied for function _cdb_geocode_postalcode_point +SELECT _cdb_geocode_ipaddress_point('evil_user', 'evil_orgname', '8.8.8.8'); +ERROR: permission denied for function _cdb_geocode_ipaddress_point +SELECT _cdb_geocode_street_point_v2('evil_user', 'evil_orgname', 'one street, 1'); +ERROR: permission denied for function _cdb_geocode_street_point_v2 diff --git a/client/test/0.1.0/sql/00_installation_test.sql b/client/test/0.1.0/sql/00_installation_test.sql new file mode 100644 index 0000000..784fa03 --- /dev/null +++ b/client/test/0.1.0/sql/00_installation_test.sql @@ -0,0 +1,23 @@ +-- Install dependencies +CREATE EXTENSION postgis; +CREATE EXTENSION schema_triggers; +CREATE EXTENSION plpythonu; +CREATE EXTENSION cartodb; +CREATE EXTENSION plproxy; + +-- Install the extension +CREATE EXTENSION cdb_geocoder_client; + +-- Mock the server connection to point to this very test db +SELECT cartodb.cdb_conf_setconf('geocoder_server_config', '{"connection_str": "dbname=contrib_regression host=127.0.0.1 user=postgres"}'); +-- Mock the user configuration +SELECT cartodb.cdb_conf_setconf('user_config', '{"is_organization": false, "entity_name": "test_user"}'); + +-- Mock the server schema +CREATE SCHEMA cdb_geocoder_server; + +-- Create a test user to check permissions +DROP ROLE IF EXISTS test_regular_user; +CREATE ROLE test_regular_user; +GRANT publicuser TO test_regular_user; +ALTER ROLE test_regular_user SET search_path TO public,cartodb,cdb_geocoder_client; \ No newline at end of file diff --git a/client/test/0.1.0/sql/10_admin0_test.sql b/client/test/0.1.0/sql/10_admin0_test.sql new file mode 100644 index 0000000..5c3e488 --- /dev/null +++ b/client/test/0.1.0/sql/10_admin0_test.sql @@ -0,0 +1,15 @@ +-- Add to the search path the schema +SET search_path TO public,cartodb,cdb_geocoder_client; + +-- Mock the server function +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_admin0_polygon invoked with params (%, %, %)', username, orgname, country_name; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; + + +-- Exercise the public and the proxied function +SELECT cdb_geocode_admin0_polygon('Spain'); diff --git a/client/test/0.1.0/sql/20_admin1_test.sql b/client/test/0.1.0/sql/20_admin1_test.sql new file mode 100644 index 0000000..a8a2075 --- /dev/null +++ b/client/test/0.1.0/sql/20_admin1_test.sql @@ -0,0 +1,24 @@ +-- Add to the search path the schema +SET search_path TO public,cartodb,cdb_geocoder_client; + +-- Mock the server functions +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_admin1_polygon invoked with params (%, %, %)', username, orgname, admin1_name; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_admin1_polygon invoked with params (%, %, %, %)', username, orgname, admin1_name, country_name; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; + + +-- Exercise the public and the proxied function +SELECT cdb_geocode_admin1_polygon('California'); +SELECT cdb_geocode_admin1_polygon('California', 'United States'); diff --git a/client/test/0.1.0/sql/30_namedplaces_test.sql b/client/test/0.1.0/sql/30_namedplaces_test.sql new file mode 100644 index 0000000..3309cf5 --- /dev/null +++ b/client/test/0.1.0/sql/30_namedplaces_test.sql @@ -0,0 +1,33 @@ +-- Add to the search path the schema +SET search_path TO public,cartodb,cdb_geocoder_client; + +-- Mock the server functions +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_namedplace_point invoked with params (%, %, %)', username, orgname, city_name; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_namedplace_point invoked with params (%, %, %, %)', username, orgname, city_name, country_name; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_namedplace_point invoked with params (%, %, %, %, %)', username, orgname, city_name, admin1_name, country_name; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; + +-- Exercise the public and the proxied function +SELECT cdb_geocode_namedplace_point('Elx'); +SELECT cdb_geocode_namedplace_point('Elx', 'Spain'); +SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain'); + diff --git a/client/test/0.1.0/sql/40_postalcodes_test.sql b/client/test/0.1.0/sql/40_postalcodes_test.sql new file mode 100644 index 0000000..b5aafc6 --- /dev/null +++ b/client/test/0.1.0/sql/40_postalcodes_test.sql @@ -0,0 +1,23 @@ +-- Add to the search path the schema +SET search_path TO public,cartodb,cdb_geocoder_client; + +-- Mock the server functions +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_polygon(username text, orgname text, postal_code text, country_name text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_postalcode_polygon invoked with params (%, %, %, %)', username, orgname, postal_code, country_name; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_point(username text, orgname text, postal_code text, country_name text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_postalcode_point invoked with params (%, %, %, %)', username, orgname, postal_code, country_name; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; + +-- Exercise the public and the proxied function +SELECT cdb_geocode_postalcode_polygon('03204', 'Spain'); +SELECT cdb_geocode_postalcode_point('03204', 'Spain'); diff --git a/client/test/0.1.0/sql/50_ipaddresses_test.sql b/client/test/0.1.0/sql/50_ipaddresses_test.sql new file mode 100644 index 0000000..09a3342 --- /dev/null +++ b/client/test/0.1.0/sql/50_ipaddresses_test.sql @@ -0,0 +1,15 @@ +-- Add to the search path the schema +SET search_path TO public,cartodb,cdb_geocoder_client; + +-- Mock the server functions +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_ipaddress_point(username text, orgname text, ip_address text) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_ipaddress_point invoked with params (%, %, %)', username, orgname, ip_address; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; + + +-- Exercise the public and the proxied function +SELECT cdb_geocode_ipaddress_point('8.8.8.8'); diff --git a/client/test/0.1.0/sql/60_street_v2_test.sql b/client/test/0.1.0/sql/60_street_v2_test.sql new file mode 100644 index 0000000..99078c9 --- /dev/null +++ b/client/test/0.1.0/sql/60_street_v2_test.sql @@ -0,0 +1,24 @@ +-- Add to the search path the schema +SET search_path TO public,cartodb,cdb_geocoder_client; + +-- Mock the server functions +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_street_point_v2 (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL) +RETURNS Geometry AS $$ +BEGIN + RAISE NOTICE 'cdb_geocoder_server.cdb_geocode_geocoder_street_point_v2 invoked with params (%, %, %, %, %, %)', username, orgname, searchtext, city, state_province, country; + RETURN NULL; +END; +$$ LANGUAGE 'plpgsql'; + + +-- Exercise the public and the proxied function +SELECT cdb_geocode_street_point_v2('One street, 1'); +SELECT cdb_geocode_street_point_v2('One street', 'city'); +SELECT cdb_geocode_street_point_v2('One street', 'city', 'state'); +SELECT cdb_geocode_street_point_v2('One street', 'city', 'state', 'country'); +SELECT cdb_geocode_street_point_v2('One street', 'city', NULL, 'country'); +SELECT cdb_geocode_street_point_v2('One street, 1'); +SELECT cdb_geocode_street_point_v2('One street', 'city'); +SELECT cdb_geocode_street_point_v2('One street', 'city', 'state'); +SELECT cdb_geocode_street_point_v2('One street', 'city', 'state', 'country'); +SELECT cdb_geocode_street_point_v2('One street', 'city', NULL, 'country'); \ No newline at end of file diff --git a/client/test/0.1.0/sql/90_permissions_test.sql b/client/test/0.1.0/sql/90_permissions_test.sql new file mode 100644 index 0000000..675867f --- /dev/null +++ b/client/test/0.1.0/sql/90_permissions_test.sql @@ -0,0 +1,30 @@ +-- Use regular user role +SET ROLE test_regular_user; + +-- Add to the search path the schema +SET search_path TO public,cartodb,cdb_geocoder_client; + +-- Exercise the public function +-- it is public, it shall work +SELECT cdb_geocode_admin0_polygon('Spain'); +SELECT cdb_geocode_admin1_polygon('California'); +SELECT cdb_geocode_admin1_polygon('California', 'United States'); +SELECT cdb_geocode_namedplace_point('Elx'); +SELECT cdb_geocode_namedplace_point('Elx', 'Valencia'); +SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain'); +SELECT cdb_geocode_postalcode_polygon('03204', 'Spain'); +SELECT cdb_geocode_postalcode_point('03204', 'Spain'); +SELECT cdb_geocode_ipaddress_point('8.8.8.8'); +SELECT cdb_geocode_street_point_v2('one street, 1'); + +-- Check the regular user has no permissions on private functions +SELECT _cdb_geocode_admin0_polygon('evil_user', 'evil_orgname', 'Hell'); +SELECT _cdb_geocode_admin1_polygon('evil_user', 'evil_orgname', 'Hell'); +SELECT _cdb_geocode_admin1_polygon('evil_user', 'evil_orgname', 'Sheol', 'Hell'); +SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'Sheol'); +SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'Sheol', 'Hell'); +SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'Sheol', 'Hell', 'Ugly world'); +SELECT _cdb_geocode_postalcode_polygon('evil_user', 'evil_orgname', '66666', 'Hell'); +SELECT _cdb_geocode_postalcode_point('evil_user', 'evil_orgname', '66666', 'Hell'); +SELECT _cdb_geocode_ipaddress_point('evil_user', 'evil_orgname', '8.8.8.8'); +SELECT _cdb_geocode_street_point_v2('evil_user', 'evil_orgname', 'one street, 1'); diff --git a/interface.yaml b/interface_0.0.1.yaml similarity index 96% rename from interface.yaml rename to interface_0.0.1.yaml index c789cd3..9f219fb 100644 --- a/interface.yaml +++ b/interface_0.0.1.yaml @@ -49,5 +49,4 @@ - name: cdb_geocode_ipaddress_point return_type: Geometry params: - - { name: ip_address, type: text} - + - { name: ip_address, type: text} \ No newline at end of file diff --git a/interface_0.1.0.yaml b/interface_0.1.0.yaml new file mode 100644 index 0000000..d5e6fed --- /dev/null +++ b/interface_0.1.0.yaml @@ -0,0 +1,60 @@ +--- +- name: cdb_geocode_admin0_polygon + return_type: Geometry + params: + - { name: country_name, type: text } + +- name: cdb_geocode_admin1_polygon + return_type: Geometry + params: + - { name: admin1_name, type: text } + +- name: cdb_geocode_admin1_polygon + return_type: Geometry + params: + - { name: admin1_name, type: text } + - { name: country_name, type: text } + +- name: cdb_geocode_namedplace_point + return_type: Geometry + params: + - { name: city_name, type: text} + +- name: cdb_geocode_namedplace_point + return_type: Geometry + params: + - { name: city_name, type: text} + - { name: country_name, type: text} + +- name: cdb_geocode_namedplace_point + return_type: Geometry + params: + - { name: city_name, type: text} + - { name: admin1_name, type: text} + - { name: country_name, type: text} + + +- name: cdb_geocode_postalcode_polygon + return_type: Geometry + params: + - { name: postal_code, type: text} + - { name: country_name, type: text} + +- name: cdb_geocode_postalcode_point + return_type: Geometry + params: + - { name: postal_code, type: text} + - { name: country_name, type: text} + +- name: cdb_geocode_ipaddress_point + return_type: Geometry + params: + - { name: ip_address, type: text} + +- name: cdb_geocode_street_point_v2 + return_type: Geometry + params: + - { name: searchtext, type: text} + - { name: city, type: text, default: 'NULL'} + - { name: state_province, type: text, default: 'NULL'} + - { name: country, type: text, default: 'NULL'} diff --git a/server/extension/.gitignore b/server/extension/.gitignore index 7b6ce39..626ea9f 100644 --- a/server/extension/.gitignore +++ b/server/extension/.gitignore @@ -2,4 +2,4 @@ results/ regression.diffs regression.out cdb_geocoder_server--0.0.1.sql - +cdb_geocoder_server--0.1.0.sql \ No newline at end of file diff --git a/server/extension/Makefile b/server/extension/Makefile index b7f4973..c3f5de8 100644 --- a/server/extension/Makefile +++ b/server/extension/Makefile @@ -3,24 +3,38 @@ EXTENSION = cdb_geocoder_server EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/") -DATA = $(EXTENSION)--$(EXTVERSION).sql +# The new version to be generated from templates +NEW_EXTENSION_ARTIFACT = $(EXTENSION)--$(EXTVERSION).sql -REGRESS = $(notdir $(basename $(wildcard sql/*test.sql))) +# DATA is a special variable used by postgres build infrastructure +# These are the files to be installed in the server shared dir, +# for installation from scratch, upgrades and downgrades. +# @see http://www.postgresql.org/docs/current/static/extend-pgxs.html +DATA = $(NEW_EXTENSION_ARTIFACT) \ + cdb_geocoder_server--0.0.1.sql \ + cdb_geocoder_server--0.1.0--0.0.1.sql \ + cdb_geocoder_server--0.0.1--0.1.0.sql + + +REGRESS = $(notdir $(basename $(wildcard test/$(EXTVERSION)/sql/*test.sql))) +TEST_DIR = test/$(EXTVERSION) +REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)' # postgres build stuff PG_CONFIG = pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) include $(PGXS) +SOURCES_DATA_DIR = sql/$(EXTVERSION) SOURCES_DATA = $(wildcard sql/$(EXTVERSION)/*.sql) -$(DATA): $(SOURCES_DATA) +$(NEW_EXTENSION_ARTIFACT): $(SOURCES_DATA) rm -f $@ - cat $(SOURCES_DATA) >> $@ + cat $(SOURCES_DATA_DIR)/*.sql >> $@ all: $(DATA) # Only meant for development time, do not use once a version is released devclean: - rm -f $(DATA) + rm -f $(NEW_EXTENSION_ARTIFACT) diff --git a/server/extension/cdb_geocoder_server--0.0.1--0.1.0.sql b/server/extension/cdb_geocoder_server--0.0.1--0.1.0.sql new file mode 100644 index 0000000..43aa150 --- /dev/null +++ b/server/extension/cdb_geocoder_server--0.0.1--0.1.0.sql @@ -0,0 +1,132 @@ +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_geocoder_server._get_redis_conf_v2(config_key text) +RETURNS cdb_geocoder_server._redis_conf_params AS $$ + conf_query = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(config_key) + conf = plpy.execute(conf_query)[0]['conf'] + if conf is None: + plpy.error("There is no redis configuration defined") + else: + import json + params = json.loads(conf) + return { + "sentinel_host": params['sentinel_host'], + "sentinel_port": params['sentinel_port'], + "sentinel_master_id": params['sentinel_master_id'], + "timeout": params['timeout'], + "redis_db": params['redis_db'] + } +$$ LANGUAGE plpythonu; + +-- Get the connection to redis from cache or create a new one +CREATE OR REPLACE FUNCTION cdb_geocoder_server._connect_to_redis(user_id text) +RETURNS boolean AS $$ + cache_key = "redis_connection_{0}".format(user_id) + if cache_key in GD: + return False + else: + from cartodb_geocoder import redis_helper + metadata_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port, + c.sentinel_master_id, c.timeout, c.redis_db + from cdb_geocoder_server._get_redis_conf_v2('redis_metadata_config') c;""")[0] + metrics_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port, + c.sentinel_master_id, c.timeout, c.redis_db + from cdb_geocoder_server._get_redis_conf_v2('redis_metrics_config') c;""")[0] + redis_metadata_connection = redis_helper.RedisHelper(metadata_config_params['sentinel_host'], + metadata_config_params['sentinel_port'], + metadata_config_params['sentinel_master_id'], + timeout=metadata_config_params['timeout'], + redis_db=metadata_config_params['redis_db']).redis_connection() + redis_metrics_connection = redis_helper.RedisHelper(metrics_config_params['sentinel_host'], + metrics_config_params['sentinel_port'], + metrics_config_params['sentinel_master_id'], + timeout=metrics_config_params['timeout'], + redis_db=metrics_config_params['redis_db']).redis_connection() + GD[cache_key] = { + 'redis_metadata_connection': redis_metadata_connection, + 'redis_metrics_connection': redis_metrics_connection, + } + return True +$$ LANGUAGE plpythonu; +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_geocoder_server._get_geocoder_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + import json + from cartodb_geocoder import config_helper + plpy.execute("SELECT cdb_geocoder_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + heremaps_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as heremaps_conf", 1)[0]['heremaps_conf'] + if not heremaps_conf_json: + heremaps_app_id = None + heremaps_app_code = None + else: + heremaps_conf = json.loads(heremaps_conf_json) + heremaps_app_id = heremaps_conf['app_id'] + heremaps_app_code = heremaps_conf['app_code'] + geocoder_config = config_helper.GeocoderConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code) + # --Think about the security concerns with this kind of global cache, it should be only available + # --for this user session but... + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu; +-- Geocodes a street address given a searchtext and a state and/or country +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_street_point_v2(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_geocoder_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_geocoder_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + plpy.error('Requested geocoder is not available') + +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from heremaps import heremapsgeocoder + from cartodb_geocoder import quota_service + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + # -- Check the quota + quota_service = quota_service.QuotaService(user_geocoder_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reach the limit of your quota') + + try: + geocoder = heremapsgeocoder.Geocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code) + results = geocoder.geocode_address(searchtext=searchtext, city=city, state=state_province, country=country) + coordinates = geocoder.extract_lng_lat_from_result(results[0]) + quota_service.increment_success_geocoder_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + except heremapsgeocoder.EmptyGeocoderResponse: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using here maps geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.error('Google geocoder is not available yet') + return None +$$ LANGUAGE plpythonu; diff --git a/server/extension/cdb_geocoder_server--0.0.1.sql b/server/extension/cdb_geocoder_server--0.0.1.sql new file mode 100644 index 0000000..33325df --- /dev/null +++ b/server/extension/cdb_geocoder_server--0.0.1.sql @@ -0,0 +1,539 @@ +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION cdb_geocoder_server" to load this file. \quit +CREATE TYPE cdb_geocoder_server._redis_conf_params AS ( + sentinel_host text, + sentinel_port int, + sentinel_master_id text, + redis_db text, + timeout float +); + +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_geocoder_server._get_redis_conf() +RETURNS cdb_geocoder_server._redis_conf_params AS $$ + conf = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('redis_conf') conf")[0]['conf'] + if conf is None: + plpy.error("There is no redis configuration defined") + else: + import json + params = json.loads(conf) + return { + "sentinel_host": params['sentinel_host'], + "sentinel_port": params['sentinel_port'], + "sentinel_master_id": params['sentinel_master_id'], + "timeout": params['timeout'], + "redis_db": params['redis_db'] + } +$$ LANGUAGE plpythonu; + +-- Get the connection to redis from cache or create a new one +CREATE OR REPLACE FUNCTION cdb_geocoder_server._connect_to_redis(user_id text) +RETURNS boolean AS $$ + if user_id in GD and 'redis_connection' in GD[user_id]: + return False + else: + from cartodb_geocoder import redis_helper + config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port, + c.sentinel_master_id, c.timeout, c.redis_db + from cdb_geocoder_server._get_redis_conf() c;""")[0] + redis_connection = redis_helper.RedisHelper(config_params['sentinel_host'], + config_params['sentinel_port'], + config_params['sentinel_master_id'], + timeout=config_params['timeout'], + redis_db=config_params['redis_db']).redis_connection() + GD[user_id] = {'redis_connection': redis_connection} + return True +$$ LANGUAGE plpythonu;-- Geocodes a street address given a searchtext and a state and/or country +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_street_point(searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) + RETURNS Geometry +AS $$ + import json + from heremaps import heremapsgeocoder + + heremaps_conf = json.loads(plpy.execute("SELECT cdb_geocoder_server._get_conf('heremaps')", 1)[0]['get_conf']) + + app_id = heremaps_conf['geocoder']['app_id'] + app_code = heremaps_conf['geocoder']['app_code'] + + geocoder = heremapsgeocoder.Geocoder(app_id, app_code) + + results = geocoder.geocode_address(searchtext=searchtext, city=city, state=state_province, country=country) + coordinates = geocoder.extract_lng_lat_from_result(results[0]) + + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + + return point['st_setsrid'] +$$ LANGUAGE plpythonu;-- Interface of the server extension + +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) +RETURNS Geometry AS $$ + plpy.debug('Entering cdb_geocode_admin0_polygons') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_admin0_polygon($1) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [country_name], 1) + + plpy.debug('Returning from Returning from cdb_geocode_admin0_polygons') + return rv[0]["mypolygon"] +$$ LANGUAGE plpythonu; + + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_admin0_polygon(country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT n.the_geom as geom INTO ret + FROM (SELECT q, lower(regexp_replace(q, '[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text x + FROM (SELECT country_name q) g) d + LEFT OUTER JOIN admin0_synonyms s ON name_ = d.x + LEFT OUTER JOIN ne_admin0_v3 n ON s.adm0_a3 = n.adm0_a3 GROUP BY d.q, n.the_geom, s.adm0_a3; + + RETURN ret; + END +$$ LANGUAGE plpgsql; +-- Interfacess of the server extension + +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) +RETURNS Geometry AS $$ + plpy.debug('Entering cdb_geocode_admin1_polygon(admin1_name text)') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_admin1_polygon($1) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [admin1_name], 1) + + plpy.debug('Returning from Returning from cdb_geocode_admin1_polygons') + return rv[0]["mypolygon"] +$$ LANGUAGE plpythonu; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + plpy.debug('Entering cdb_geocode_admin1_polygon(admin1_name text, country_name text)') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_admin1_polygon($1, $2) AS mypolygon", ["text", "text"]) + rv = plpy.execute(plan, [admin1_name, country_name], 1) + + plpy.debug('Returning from Returning from cdb_geocode_admin1_polygon(admin1_name text, country_name text)') + return rv[0]["mypolygon"] +$$ LANGUAGE plpythonu; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_admin1_polygon(admin1_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT q, ( + SELECT the_geom + FROM global_province_polygons + WHERE d.c = ANY (synonyms) + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM ( + SELECT + trim(replace(lower(admin1_name),'.',' ')) c, admin1_name q + ) d + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_admin1_polygon(admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + WITH p AS (SELECT r.c, r.q, (SELECT iso3 FROM country_decoder WHERE lower(country_name) = ANY (synonyms)) i FROM (SELECT trim(replace(lower(admin1_name),'.',' ')) c, country_name q) r) + SELECT + geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_province_polygons + WHERE p.c = ANY (synonyms) + AND iso3 = p.i + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM p) n; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +-- Interfacess of the server extension + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text) +RETURNS Geometry AS $$ + plpy.debug('Entering cdb_geocode_namedplace_point(city_name text)') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_namedplace_point($1) AS mypoint", ["text"]) + rv = plpy.execute(plan, [city_name], 1) + + plpy.debug('Returning from Returning from geocode_namedplace') + return rv[0]["mypoint"] +$$ LANGUAGE plpythonu; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text) +RETURNS Geometry AS $$ + plpy.debug('Entering cdb_geocode_namedplace_point(city_name text, country_name text)') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_namedplace_point($1, $2) AS mypoint", ["text", "text"]) + rv = plpy.execute(plan, [city_name, country_name], 1) + + plpy.debug('Returning from Returning from geocode_namedplace') + return rv[0]["mypoint"] +$$ LANGUAGE plpythonu; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + plpy.debug('Entering cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_namedplace_point($1, $2, $3) AS mypoint", ["text", "text", "text"]) + rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1) + + plpy.debug('Returning from Returning from geocode_namedplace') + return rv[0]["mypoint"] +$$ LANGUAGE plpythonu; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_namedplace_point(city_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH best AS (SELECT s AS q, (SELECT the_geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) ORDER BY population DESC LIMIT 1) AS geom FROM (SELECT city_name as s) p), + next AS (SELECT p.s AS q, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM (SELECT city_name as s) p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT q, geom, TRUE AS success FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT q, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_namedplace_point(city_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH p AS (SELECT r.s, r.c, (SELECT iso2 FROM country_decoder WHERE lower(r.c) = ANY (synonyms)) i FROM (SELECT city_name AS s, country_name::text AS c) r), + best AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.iso2 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND gp.iso2 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE c = p.c AND geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH inputcountry AS ( + SELECT iso2 as isoTwo FROM country_decoder WHERE lower(country_name) = ANY (synonyms) LIMIT 1 + ), + p AS ( + SELECT r.s, r.a1, (SELECT admin1 FROM admin1_decoder, inputcountry WHERE lower(r.a1) = ANY (synonyms) AND admin1_decoder.iso2 = inputcountry.isoTwo LIMIT 1) i FROM (SELECT city_name AS s, admin1_name::text AS a1) r), + best AS (SELECT p.s AS q, p.a1 as a1, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.admin1 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.a1 AS a1, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.admin1 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +-- Interface of the server extension + +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_point(username text, orgname text, code text) +RETURNS Geometry AS $$ + plpy.debug('Entering _cdb_geocode_postalcode_point') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_point($1) AS point", ["text"]) + rv = plpy.execute(plan, [code], 1) + + plpy.debug('Returning from _cdb_geocode_postalcode_point') + return rv[0]["point"] +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + plpy.debug('Entering _cdb_geocode_postalcode_point') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_point($1, $2) AS point", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + + plpy.debug('Returning from _cdb_geocode_postalcode_point') + return rv[0]["point"] +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text) +RETURNS Geometry AS $$ + plpy.debug('Entering _cdb_geocode_postalcode_polygon') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_polygon($1) AS polygon", ["text"]) + rv = plpy.execute(plan, [code], 1) + + plpy.debug('Returning from _cdb_geocode_postalcode_polygon') + return rv[0]["polygon"] +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + plpy.debug('Entering _cdb_geocode_postalcode_point') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_polygon($1, $2) AS polygon", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + + plpy.debug('Returning from _cdb_geocode_postalcode_point') + return rv[0]["polygon"] +$$ LANGUAGE plpythonu; + + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_point(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_point(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_polygon(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_polygon(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; +-- Interface of the server extension + +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text) +RETURNS Geometry AS $$ + plpy.debug('Entering _cdb_geocode_ipaddress_point') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_ipaddress_point($1) AS point", ["TEXT"]) + rv = plpy.execute(plan, [ip], 1) + + plpy.debug('Returning from _cdb_geocode_ipaddress_point') + return rv[0]["point"] +$$ LANGUAGE plpythonu; + + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_ipaddress_point(ip text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + + new_ip INET; + BEGIN + BEGIN + IF family(ip::inet) = 6 THEN + new_ip := ip::inet; + ELSE + new_ip := ('::ffff:' || ip)::inet; + END IF; + EXCEPTION WHEN OTHERS THEN + SELECT NULL as geom INTO ret; + RETURN ret; + END; + + WITH + ips AS (SELECT ip s, new_ip net), + matches AS (SELECT s, (SELECT the_geom FROM ip_address_locations WHERE network_start_ip <= ips.net ORDER BY network_start_ip DESC LIMIT 1) geom FROM ips) + SELECT geom INTO ret + FROM matches; + RETURN ret; +END +$$ LANGUAGE plpgsql; +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT * + FROM pg_catalog.pg_user + WHERE usename = 'geocoder_api') THEN + + CREATE USER geocoder_api; + END IF; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_geocoder_server TO geocoder_api; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO geocoder_api; + GRANT USAGE ON SCHEMA cdb_geocoder_server TO geocoder_api; + GRANT USAGE ON SCHEMA public TO geocoder_api; + GRANT SELECT ON ALL TABLES IN SCHEMA public TO geocoder_api; +END$$; \ No newline at end of file diff --git a/server/extension/cdb_geocoder_server--0.1.0--0.0.1.sql b/server/extension/cdb_geocoder_server--0.1.0--0.0.1.sql new file mode 100644 index 0000000..f626887 --- /dev/null +++ b/server/extension/cdb_geocoder_server--0.1.0--0.0.1.sql @@ -0,0 +1,6 @@ +DROP FUNCTION IF EXISTS cdb_geocoder_server._get_redis_conf_v2(text); +DROP FUNCTION IF EXISTS cdb_geocoder_server._connect_to_redis(text); +DROP FUNCTION IF EXISTS cdb_geocoder_server._get_geocoder_config(text, text); +DROP FUNCTION IF EXISTS cdb_geocoder_server.cdb_geocode_street_point_v2(TEXT, TEXT, TEXT, TEXT, TEXT, TEXT); +DROP FUNCTION IF EXISTS cdb_geocoder_server._cdb_here_geocode_street_point(TEXT, TEXT, TEXT, TEXT, TEXT, TEXT); +DROP FUNCTION IF EXISTS cdb_geocoder_server._cdb_google_geocode_street_point(TEXT, TEXT, TEXT, TEXT, TEXT, TEXT); \ No newline at end of file diff --git a/server/extension/cdb_geocoder_server.control b/server/extension/cdb_geocoder_server.control index 7ac080a..967b8d3 100644 --- a/server/extension/cdb_geocoder_server.control +++ b/server/extension/cdb_geocoder_server.control @@ -1,6 +1,6 @@ # cdb geocoder server extension comment = 'CartoDB server geocoder extension' -default_version = '0.0.1' +default_version = '0.1.0' requires = 'plpythonu, postgis, cdb_geocoder' superuser = true schema = cdb_geocoder_server diff --git a/server/extension/sql/0.0.1/10_redis_helper.sql b/server/extension/sql/0.0.1/10_redis_helper.sql index a8841fa..501a0cc 100644 --- a/server/extension/sql/0.0.1/10_redis_helper.sql +++ b/server/extension/sql/0.0.1/10_redis_helper.sql @@ -41,4 +41,4 @@ RETURNS boolean AS $$ redis_db=config_params['redis_db']).redis_connection() GD[user_id] = {'redis_connection': redis_connection} return True -$$ LANGUAGE plpythonu; \ No newline at end of file +$$ LANGUAGE plpythonu; diff --git a/server/extension/sql/0.1.0/00_header.sql b/server/extension/sql/0.1.0/00_header.sql new file mode 100644 index 0000000..949e06e --- /dev/null +++ b/server/extension/sql/0.1.0/00_header.sql @@ -0,0 +1,2 @@ +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION cdb_geocoder_server" to load this file. \quit diff --git a/server/extension/sql/0.1.0/10_redis_helper.sql b/server/extension/sql/0.1.0/10_redis_helper.sql new file mode 100644 index 0000000..ef5047c --- /dev/null +++ b/server/extension/sql/0.1.0/10_redis_helper.sql @@ -0,0 +1,57 @@ +CREATE TYPE cdb_geocoder_server._redis_conf_params AS ( + sentinel_host text, + sentinel_port int, + sentinel_master_id text, + redis_db text, + timeout float +); + +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_geocoder_server._get_redis_conf_v2(config_key text) +RETURNS cdb_geocoder_server._redis_conf_params AS $$ + conf_query = "SELECT cartodb.CDB_Conf_GetConf('{0}') as conf".format(config_key) + conf = plpy.execute(conf_query)[0]['conf'] + if conf is None: + plpy.error("There is no redis configuration defined") + else: + import json + params = json.loads(conf) + return { + "sentinel_host": params['sentinel_host'], + "sentinel_port": params['sentinel_port'], + "sentinel_master_id": params['sentinel_master_id'], + "timeout": params['timeout'], + "redis_db": params['redis_db'] + } +$$ LANGUAGE plpythonu; + +-- Get the connection to redis from cache or create a new one +CREATE OR REPLACE FUNCTION cdb_geocoder_server._connect_to_redis(user_id text) +RETURNS boolean AS $$ + cache_key = "redis_connection_{0}".format(user_id) + if cache_key in GD: + return False + else: + from cartodb_geocoder import redis_helper + metadata_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port, + c.sentinel_master_id, c.timeout, c.redis_db + from cdb_geocoder_server._get_redis_conf_v2('redis_metadata_config') c;""")[0] + metrics_config_params = plpy.execute("""select c.sentinel_host, c.sentinel_port, + c.sentinel_master_id, c.timeout, c.redis_db + from cdb_geocoder_server._get_redis_conf_v2('redis_metrics_config') c;""")[0] + redis_metadata_connection = redis_helper.RedisHelper(metadata_config_params['sentinel_host'], + metadata_config_params['sentinel_port'], + metadata_config_params['sentinel_master_id'], + timeout=metadata_config_params['timeout'], + redis_db=metadata_config_params['redis_db']).redis_connection() + redis_metrics_connection = redis_helper.RedisHelper(metrics_config_params['sentinel_host'], + metrics_config_params['sentinel_port'], + metrics_config_params['sentinel_master_id'], + timeout=metrics_config_params['timeout'], + redis_db=metrics_config_params['redis_db']).redis_connection() + GD[cache_key] = { + 'redis_metadata_connection': redis_metadata_connection, + 'redis_metrics_connection': redis_metrics_connection, + } + return True +$$ LANGUAGE plpythonu; diff --git a/server/extension/sql/0.1.0/15_config_helper.sql b/server/extension/sql/0.1.0/15_config_helper.sql new file mode 100644 index 0000000..dd37953 --- /dev/null +++ b/server/extension/sql/0.1.0/15_config_helper.sql @@ -0,0 +1,25 @@ +-- Get the Redis configuration from the _conf table -- +CREATE OR REPLACE FUNCTION cdb_geocoder_server._get_geocoder_config(username text, orgname text) +RETURNS boolean AS $$ + cache_key = "user_geocoder_config_{0}".format(username) + if cache_key in GD: + return False + else: + import json + from cartodb_geocoder import config_helper + plpy.execute("SELECT cdb_geocoder_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection'] + heremaps_conf_json = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as heremaps_conf", 1)[0]['heremaps_conf'] + if not heremaps_conf_json: + heremaps_app_id = None + heremaps_app_code = None + else: + heremaps_conf = json.loads(heremaps_conf_json) + heremaps_app_id = heremaps_conf['app_id'] + heremaps_app_code = heremaps_conf['app_code'] + geocoder_config = config_helper.GeocoderConfig(redis_conn, username, orgname, heremaps_app_id, heremaps_app_code) + # --Think about the security concerns with this kind of global cache, it should be only available + # --for this user session but... + GD[cache_key] = geocoder_config + return True +$$ LANGUAGE plpythonu; diff --git a/server/extension/sql/0.1.0/20_geocode_street.sql b/server/extension/sql/0.1.0/20_geocode_street.sql new file mode 100644 index 0000000..b396ef3 --- /dev/null +++ b/server/extension/sql/0.1.0/20_geocode_street.sql @@ -0,0 +1,58 @@ +-- Geocodes a street address given a searchtext and a state and/or country +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_street_point_v2(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.execute("SELECT cdb_geocoder_server._connect_to_redis('{0}')".format(username)) + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + plpy.execute("SELECT cdb_geocoder_server._get_geocoder_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname))) + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + if user_geocoder_config.heremaps_geocoder: + here_plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_here_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(here_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + elif user_geocoder_config.google_geocoder: + google_plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_google_geocode_street_point($1, $2, $3, $4, $5, $6) as point; ", ["text", "text", "text", "text", "text", "text"]) + return plpy.execute(google_plan, [username, orgname, searchtext, city, state_province, country], 1)[0]['point'] + else: + plpy.error('Requested geocoder is not available') + +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_here_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + from heremaps import heremapsgeocoder + from cartodb_geocoder import quota_service + + redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection'] + user_geocoder_config = GD["user_geocoder_config_{0}".format(username)] + + # -- Check the quota + quota_service = quota_service.QuotaService(user_geocoder_config, redis_conn) + if not quota_service.check_user_quota(): + plpy.error('You have reach the limit of your quota') + + try: + geocoder = heremapsgeocoder.Geocoder(user_geocoder_config.heremaps_app_id, user_geocoder_config.heremaps_app_code) + results = geocoder.geocode_address(searchtext=searchtext, city=city, state=state_province, country=country) + coordinates = geocoder.extract_lng_lat_from_result(results[0]) + quota_service.increment_success_geocoder_use() + plan = plpy.prepare("SELECT ST_SetSRID(ST_MakePoint($1, $2), 4326); ", ["double precision", "double precision"]) + point = plpy.execute(plan, [coordinates[0], coordinates[1]], 1)[0] + return point['st_setsrid'] + except heremapsgeocoder.EmptyGeocoderResponse: + quota_service.increment_empty_geocoder_use() + return None + except BaseException as e: + import sys, traceback + type_, value_, traceback_ = sys.exc_info() + quota_service.increment_failed_geocoder_use() + error_msg = 'There was an error trying to geocode using here maps geocoder: {0}'.format(e) + plpy.notice(traceback.format_tb(traceback_)) + plpy.error(error_msg) + +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_google_geocode_street_point(username TEXT, orgname TEXT, searchtext TEXT, city TEXT DEFAULT NULL, state_province TEXT DEFAULT NULL, country TEXT DEFAULT NULL) +RETURNS Geometry AS $$ + plpy.error('Google geocoder is not available yet') + return None +$$ LANGUAGE plpythonu; diff --git a/server/extension/sql/0.1.0/30_admin0.sql b/server/extension/sql/0.1.0/30_admin0.sql new file mode 100644 index 0000000..96bbc43 --- /dev/null +++ b/server/extension/sql/0.1.0/30_admin0.sql @@ -0,0 +1,37 @@ +-- Interface of the server extension + +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin0_polygon(username text, orgname text, country_name text) +RETURNS Geometry AS $$ + plpy.debug('Entering cdb_geocode_admin0_polygons') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_admin0_polygon($1) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [country_name], 1) + + plpy.debug('Returning from Returning from cdb_geocode_admin0_polygons') + return rv[0]["mypolygon"] +$$ LANGUAGE plpythonu; + + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_admin0_polygon(country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT n.the_geom as geom INTO ret + FROM (SELECT q, lower(regexp_replace(q, '[^a-zA-Z\u00C0-\u00ff]+', '', 'g'))::text x + FROM (SELECT country_name q) g) d + LEFT OUTER JOIN admin0_synonyms s ON name_ = d.x + LEFT OUTER JOIN ne_admin0_v3 n ON s.adm0_a3 = n.adm0_a3 GROUP BY d.q, n.the_geom, s.adm0_a3; + + RETURN ret; + END +$$ LANGUAGE plpgsql; diff --git a/server/extension/sql/0.1.0/40_admin1.sql b/server/extension/sql/0.1.0/40_admin1.sql new file mode 100644 index 0000000..44a1953 --- /dev/null +++ b/server/extension/sql/0.1.0/40_admin1.sql @@ -0,0 +1,89 @@ +-- Interfacess of the server extension + +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text) +RETURNS Geometry AS $$ + plpy.debug('Entering cdb_geocode_admin1_polygon(admin1_name text)') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_admin1_polygon($1) AS mypolygon", ["text"]) + rv = plpy.execute(plan, [admin1_name], 1) + + plpy.debug('Returning from Returning from cdb_geocode_admin1_polygons') + return rv[0]["mypolygon"] +$$ LANGUAGE plpythonu; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_admin1_polygon(username text, orgname text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + plpy.debug('Entering cdb_geocode_admin1_polygon(admin1_name text, country_name text)') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_admin1_polygon($1, $2) AS mypolygon", ["text", "text"]) + rv = plpy.execute(plan, [admin1_name, country_name], 1) + + plpy.debug('Returning from Returning from cdb_geocode_admin1_polygon(admin1_name text, country_name text)') + return rv[0]["mypolygon"] +$$ LANGUAGE plpythonu; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_admin1_polygon(admin1_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_admin1_polygon(admin1_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT q, ( + SELECT the_geom + FROM global_province_polygons + WHERE d.c = ANY (synonyms) + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM ( + SELECT + trim(replace(lower(admin1_name),'.',' ')) c, admin1_name q + ) d + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_admin1_polygon(admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_admin1_polygon(admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + WITH p AS (SELECT r.c, r.q, (SELECT iso3 FROM country_decoder WHERE lower(country_name) = ANY (synonyms)) i FROM (SELECT trim(replace(lower(admin1_name),'.',' ')) c, country_name q) r) + SELECT + geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_province_polygons + WHERE p.c = ANY (synonyms) + AND iso3 = p.i + ORDER BY frequency DESC LIMIT 1 + ) geom + FROM p) n; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + diff --git a/server/extension/sql/0.1.0/50_namedplaces.sql b/server/extension/sql/0.1.0/50_namedplaces.sql new file mode 100644 index 0000000..44069f2 --- /dev/null +++ b/server/extension/sql/0.1.0/50_namedplaces.sql @@ -0,0 +1,121 @@ +-- Interfacess of the server extension + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text) +RETURNS Geometry AS $$ + plpy.debug('Entering cdb_geocode_namedplace_point(city_name text)') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_namedplace_point($1) AS mypoint", ["text"]) + rv = plpy.execute(plan, [city_name], 1) + + plpy.debug('Returning from Returning from geocode_namedplace') + return rv[0]["mypoint"] +$$ LANGUAGE plpythonu; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text) +RETURNS Geometry AS $$ + plpy.debug('Entering cdb_geocode_namedplace_point(city_name text, country_name text)') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_namedplace_point($1, $2) AS mypoint", ["text", "text"]) + rv = plpy.execute(plan, [city_name, country_name], 1) + + plpy.debug('Returning from Returning from geocode_namedplace') + return rv[0]["mypoint"] +$$ LANGUAGE plpythonu; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + plpy.debug('Entering cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_namedplace_point($1, $2, $3) AS mypoint", ["text", "text", "text"]) + rv = plpy.execute(plan, [city_name, admin1_name, country_name], 1) + + plpy.debug('Returning from Returning from geocode_namedplace') + return rv[0]["mypoint"] +$$ LANGUAGE plpythonu; + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension + +---- cdb_geocode_namedplace_point(city_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_namedplace_point(city_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH best AS (SELECT s AS q, (SELECT the_geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) ORDER BY population DESC LIMIT 1) AS geom FROM (SELECT city_name as s) p), + next AS (SELECT p.s AS q, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM (SELECT city_name as s) p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT q, geom, TRUE AS success FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT q, geom, CASE WHEN geom IS NULL THEN FALSE ELSE TRUE END AS success FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_namedplace_point(city_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_namedplace_point(city_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH p AS (SELECT r.s, r.c, (SELECT iso2 FROM country_decoder WHERE lower(r.c) = ANY (synonyms)) i FROM (SELECT city_name AS s, country_name::text AS c) r), + best AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.iso2 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.c AS c, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND gp.iso2 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE c = p.c AND geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + +---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + WITH inputcountry AS ( + SELECT iso2 as isoTwo FROM country_decoder WHERE lower(country_name) = ANY (synonyms) LIMIT 1 + ), + p AS ( + SELECT r.s, r.a1, (SELECT admin1 FROM admin1_decoder, inputcountry WHERE lower(r.a1) = ANY (synonyms) AND admin1_decoder.iso2 = inputcountry.isoTwo LIMIT 1) i FROM (SELECT city_name AS s, admin1_name::text AS a1) r), + best AS (SELECT p.s AS q, p.a1 as a1, (SELECT gp.the_geom AS geom FROM global_cities_points_limited gp WHERE gp.lowername = lower(p.s) AND gp.admin1 = p.i ORDER BY population DESC LIMIT 1) AS geom FROM p), + next AS (SELECT p.s AS q, p.a1 AS a1, (SELECT gp.the_geom FROM global_cities_points_limited gp, global_cities_alternates_limited ga WHERE lower(p.s) = ga.lowername AND ga.admin1 = p.i AND ga.geoname_id = gp.geoname_id ORDER BY preferred DESC LIMIT 1) geom FROM p WHERE p.s NOT IN (SELECT q FROM best WHERE geom IS NOT NULL)) + SELECT geom FROM best WHERE geom IS NOT NULL + UNION ALL + SELECT geom FROM next + ) v; + + RETURN ret; + END +$$ LANGUAGE plpgsql; + diff --git a/server/extension/sql/0.1.0/60_postalcodes.sql b/server/extension/sql/0.1.0/60_postalcodes.sql new file mode 100644 index 0000000..1a20379 --- /dev/null +++ b/server/extension/sql/0.1.0/60_postalcodes.sql @@ -0,0 +1,162 @@ +-- Interface of the server extension + +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_point(username text, orgname text, code text) +RETURNS Geometry AS $$ + plpy.debug('Entering _cdb_geocode_postalcode_point') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_point($1) AS point", ["text"]) + rv = plpy.execute(plan, [code], 1) + + plpy.debug('Returning from _cdb_geocode_postalcode_point') + return rv[0]["point"] +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_point(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + plpy.debug('Entering _cdb_geocode_postalcode_point') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_point($1, $2) AS point", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + + plpy.debug('Returning from _cdb_geocode_postalcode_point') + return rv[0]["point"] +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text) +RETURNS Geometry AS $$ + plpy.debug('Entering _cdb_geocode_postalcode_polygon') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_polygon($1) AS polygon", ["text"]) + rv = plpy.execute(plan, [code], 1) + + plpy.debug('Returning from _cdb_geocode_postalcode_polygon') + return rv[0]["polygon"] +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_postalcode_polygon(username text, orgname text, code text, country text) +RETURNS Geometry AS $$ + plpy.debug('Entering _cdb_geocode_postalcode_point') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_postalcode_polygon($1, $2) AS polygon", ["TEXT", "TEXT"]) + rv = plpy.execute(plan, [code, country], 1) + + plpy.debug('Returning from _cdb_geocode_postalcode_point') + return rv[0]["polygon"] +$$ LANGUAGE plpythonu; + + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_point(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_point(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_points + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_polygon(code text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_postalcode_polygon(code text, country text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + BEGIN + SELECT geom INTO ret + FROM ( + SELECT + q, ( + SELECT the_geom + FROM global_postal_code_polygons + WHERE postal_code = upper(d.q) + AND iso3 = ( + SELECT iso3 FROM country_decoder WHERE + lower(country) = ANY (synonyms) LIMIT 1 + ) + LIMIT 1 + ) geom + FROM (SELECT code q) d + ) v; + + RETURN ret; +END +$$ LANGUAGE plpgsql; diff --git a/server/extension/sql/0.1.0/70_ips.sql b/server/extension/sql/0.1.0/70_ips.sql new file mode 100644 index 0000000..5480c2d --- /dev/null +++ b/server/extension/sql/0.1.0/70_ips.sql @@ -0,0 +1,49 @@ +-- Interface of the server extension + +CREATE OR REPLACE FUNCTION cdb_geocoder_server.cdb_geocode_ipaddress_point(username text, orgname text, ip text) +RETURNS Geometry AS $$ + plpy.debug('Entering _cdb_geocode_ipaddress_point') + plpy.debug('user = %s' % username) + + #--TODO: rate limiting check + #--TODO: quota check + + #-- Copied from the doc, see http://www.postgresql.org/docs/9.4/static/plpython-database.html + plan = plpy.prepare("SELECT cdb_geocoder_server._cdb_geocode_ipaddress_point($1) AS point", ["TEXT"]) + rv = plpy.execute(plan, [ip], 1) + + plpy.debug('Returning from _cdb_geocode_ipaddress_point') + return rv[0]["point"] +$$ LANGUAGE plpythonu; + + +-------------------------------------------------------------------------------- + +-- Implementation of the server extension +-- Note: these functions depend on the cdb_geocoder extension +CREATE OR REPLACE FUNCTION cdb_geocoder_server._cdb_geocode_ipaddress_point(ip text) +RETURNS Geometry AS $$ + DECLARE + ret Geometry; + + new_ip INET; + BEGIN + BEGIN + IF family(ip::inet) = 6 THEN + new_ip := ip::inet; + ELSE + new_ip := ('::ffff:' || ip)::inet; + END IF; + EXCEPTION WHEN OTHERS THEN + SELECT NULL as geom INTO ret; + RETURN ret; + END; + + WITH + ips AS (SELECT ip s, new_ip net), + matches AS (SELECT s, (SELECT the_geom FROM ip_address_locations WHERE network_start_ip <= ips.net ORDER BY network_start_ip DESC LIMIT 1) geom FROM ips) + SELECT geom INTO ret + FROM matches; + RETURN ret; +END +$$ LANGUAGE plpgsql; diff --git a/server/extension/sql/0.1.0/90_geocoder_server_user.sql b/server/extension/sql/0.1.0/90_geocoder_server_user.sql new file mode 100644 index 0000000..3c2b354 --- /dev/null +++ b/server/extension/sql/0.1.0/90_geocoder_server_user.sql @@ -0,0 +1,15 @@ +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT * + FROM pg_catalog.pg_user + WHERE usename = 'geocoder_api') THEN + + CREATE USER geocoder_api; + END IF; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_geocoder_server TO geocoder_api; + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO geocoder_api; + GRANT USAGE ON SCHEMA cdb_geocoder_server TO geocoder_api; + GRANT USAGE ON SCHEMA public TO geocoder_api; + GRANT SELECT ON ALL TABLES IN SCHEMA public TO geocoder_api; +END$$; \ No newline at end of file diff --git a/server/extension/expected/00_install_test.out b/server/extension/test/0.0.1/expected/00_install_test.out similarity index 100% rename from server/extension/expected/00_install_test.out rename to server/extension/test/0.0.1/expected/00_install_test.out diff --git a/server/extension/expected/30_admin0_test.out b/server/extension/test/0.0.1/expected/30_admin0_test.out similarity index 100% rename from server/extension/expected/30_admin0_test.out rename to server/extension/test/0.0.1/expected/30_admin0_test.out diff --git a/server/extension/expected/40_admin1_test.out b/server/extension/test/0.0.1/expected/40_admin1_test.out similarity index 100% rename from server/extension/expected/40_admin1_test.out rename to server/extension/test/0.0.1/expected/40_admin1_test.out diff --git a/server/extension/expected/50_namedplaces_test.out b/server/extension/test/0.0.1/expected/50_namedplaces_test.out similarity index 100% rename from server/extension/expected/50_namedplaces_test.out rename to server/extension/test/0.0.1/expected/50_namedplaces_test.out diff --git a/server/extension/expected/60_postalcodes_test.out b/server/extension/test/0.0.1/expected/60_postalcodes_test.out similarity index 100% rename from server/extension/expected/60_postalcodes_test.out rename to server/extension/test/0.0.1/expected/60_postalcodes_test.out diff --git a/server/extension/expected/70_ips_test.out b/server/extension/test/0.0.1/expected/70_ips_test.out similarity index 100% rename from server/extension/expected/70_ips_test.out rename to server/extension/test/0.0.1/expected/70_ips_test.out diff --git a/server/extension/expected/90_remove_geocoder_api_user_test.out b/server/extension/test/0.0.1/expected/90_remove_geocoder_api_user_test.out similarity index 100% rename from server/extension/expected/90_remove_geocoder_api_user_test.out rename to server/extension/test/0.0.1/expected/90_remove_geocoder_api_user_test.out diff --git a/server/extension/sql/00_install_test.sql b/server/extension/test/0.0.1/sql/00_install_test.sql similarity index 100% rename from server/extension/sql/00_install_test.sql rename to server/extension/test/0.0.1/sql/00_install_test.sql diff --git a/server/extension/sql/30_admin0_test.sql b/server/extension/test/0.0.1/sql/30_admin0_test.sql similarity index 100% rename from server/extension/sql/30_admin0_test.sql rename to server/extension/test/0.0.1/sql/30_admin0_test.sql diff --git a/server/extension/sql/40_admin1_test.sql b/server/extension/test/0.0.1/sql/40_admin1_test.sql similarity index 100% rename from server/extension/sql/40_admin1_test.sql rename to server/extension/test/0.0.1/sql/40_admin1_test.sql diff --git a/server/extension/sql/50_namedplaces_test.sql b/server/extension/test/0.0.1/sql/50_namedplaces_test.sql similarity index 100% rename from server/extension/sql/50_namedplaces_test.sql rename to server/extension/test/0.0.1/sql/50_namedplaces_test.sql diff --git a/server/extension/sql/60_postalcodes_test.sql b/server/extension/test/0.0.1/sql/60_postalcodes_test.sql similarity index 100% rename from server/extension/sql/60_postalcodes_test.sql rename to server/extension/test/0.0.1/sql/60_postalcodes_test.sql diff --git a/server/extension/sql/70_ips_test.sql b/server/extension/test/0.0.1/sql/70_ips_test.sql similarity index 100% rename from server/extension/sql/70_ips_test.sql rename to server/extension/test/0.0.1/sql/70_ips_test.sql diff --git a/server/extension/sql/90_remove_geocoder_api_user_test.sql b/server/extension/test/0.0.1/sql/90_remove_geocoder_api_user_test.sql similarity index 100% rename from server/extension/sql/90_remove_geocoder_api_user_test.sql rename to server/extension/test/0.0.1/sql/90_remove_geocoder_api_user_test.sql diff --git a/server/extension/test/0.1.0/expected/00_install_test.out b/server/extension/test/0.1.0/expected/00_install_test.out new file mode 100644 index 0000000..b386bec --- /dev/null +++ b/server/extension/test/0.1.0/expected/00_install_test.out @@ -0,0 +1,30 @@ +-- Install dependencies +CREATE EXTENSION postgis; +CREATE EXTENSION schema_triggers; +CREATE EXTENSION plpythonu; +CREATE EXTENSION cartodb; +CREATE EXTENSION cdb_geocoder; +-- Install the extension +CREATE EXTENSION cdb_geocoder_server; +-- Mock the redis server connection to point to this very test db +SELECT cartodb.cdb_conf_setconf('redis_conf', '{"sentinel_host": "localhost", "sentinel_port": 26739, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); + cdb_conf_setconf +------------------ + +(1 row) + +-- Mock the varnish invalidation function +-- (used by cdb_geocoder tests) +CREATE OR REPLACE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS $$ +BEGIN + RETURN; +END +$$ +LANGUAGE plpgsql; +-- Set user quota +SELECT cartodb.CDB_SetUserQuotaInBytes(0); + cdb_setuserquotainbytes +------------------------- + 0 +(1 row) + diff --git a/server/extension/test/0.1.0/expected/20_street_test.out b/server/extension/test/0.1.0/expected/20_street_test.out new file mode 100644 index 0000000..526cc00 --- /dev/null +++ b/server/extension/test/0.1.0/expected/20_street_test.out @@ -0,0 +1,12 @@ +-- Check for namedplaces signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_street_point_v2' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text, text'); + exists +-------- + t +(1 row) + diff --git a/server/extension/test/0.1.0/expected/30_admin0_test.out b/server/extension/test/0.1.0/expected/30_admin0_test.out new file mode 100644 index 0000000..52850c4 --- /dev/null +++ b/server/extension/test/0.1.0/expected/30_admin0_test.out @@ -0,0 +1,47 @@ +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_geocoder_server.cdb_geocode_admin0_polygon('test_user', 'test_orgname', 'Spain'); + cdb_geocode_admin0_polygon +---------------------------- + +(1 row) + +-- Insert some dummy synonym +INSERT INTO admin0_synonyms (name, adm0_a3) VALUES ('Spain', 'ESP'); +-- Insert some dummy geometry to return +INSERT INTO ne_admin0_v3 (adm0_a3, the_geom) VALUES('ESP', ST_GeomFromText( + 'POLYGON((-71.1031880899493 42.3152774590236, + -71.1031627617667 42.3152960829043, + -71.102923838298 42.3149156848307, + -71.1031880899493 42.3152774590236))',4326) +); +-- This should return the polygon inserted above +SELECT cdb_geocoder_server.cdb_geocode_admin0_polygon('test_user', 'test_orgname', 'Spain'); + cdb_geocode_admin0_polygon +-------------------------------------------------------------------------------------------------------------------------------------------------------------------- + 0103000020E61000000100000004000000D0EA37A29AC651C00FD603035B284540FEFCFB379AC651C0C0503E9F5B284540FFDDDD4D96C651C033AC3B284F284540D0EA37A29AC651C00FD603035B284540 +(1 row) + +-- Check for admin0 signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_admin0_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_admin0_polygon' + AND oidvectortypes(p.proargtypes) = 'text'); + exists +-------- + t +(1 row) + diff --git a/server/extension/test/0.1.0/expected/40_admin1_test.out b/server/extension/test/0.1.0/expected/40_admin1_test.out new file mode 100644 index 0000000..9897f34 --- /dev/null +++ b/server/extension/test/0.1.0/expected/40_admin1_test.out @@ -0,0 +1,81 @@ +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California'); + cdb_geocode_admin1_polygon +---------------------------- + +(1 row) + +SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California', 'United States'); + cdb_geocode_admin1_polygon +---------------------------- + +(1 row) + +-- Insert dummy data into country decoder table +INSERT INTO country_decoder (synonyms, iso3) VALUES (Array['united states'], 'USA'); +-- Insert some dummy data and geometry to return +INSERT INTO global_province_polygons (synonyms, iso3, the_geom) VALUES (Array['california'], 'USA', ST_GeomFromText( + 'POLYGON((-71.1031880899493 42.3152774590236, + -71.1031627617667 42.3152960829043, + -71.102923838298 42.3149156848307, + -71.1031880899493 42.3152774590236))',4326) +); +-- This should return the polygon inserted above +SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California'); + cdb_geocode_admin1_polygon +-------------------------------------------------------------------------------------------------------------------------------------------------------------------- + 0103000020E61000000100000004000000D0EA37A29AC651C00FD603035B284540FEFCFB379AC651C0C0503E9F5B284540FFDDDD4D96C651C033AC3B284F284540D0EA37A29AC651C00FD603035B284540 +(1 row) + +SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California', 'United States'); + cdb_geocode_admin1_polygon +-------------------------------------------------------------------------------------------------------------------------------------------------------------------- + 0103000020E61000000100000004000000D0EA37A29AC651C00FD603035B284540FEFCFB379AC651C0C0503E9F5B284540FFDDDD4D96C651C033AC3B284F284540D0EA37A29AC651C00FD603035B284540 +(1 row) + +-- Check for admin1 signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_admin1_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_admin1_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_admin1_polygon' + AND oidvectortypes(p.proargtypes) = 'text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_admin1_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text'); + exists +-------- + t +(1 row) + diff --git a/server/extension/test/0.1.0/expected/50_namedplaces_test.out b/server/extension/test/0.1.0/expected/50_namedplaces_test.out new file mode 100644 index 0000000..1ac8d73 --- /dev/null +++ b/server/extension/test/0.1.0/expected/50_namedplaces_test.out @@ -0,0 +1,136 @@ +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx'); + cdb_geocode_namedplace_point +------------------------------ + +(1 row) + +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain'); + cdb_geocode_namedplace_point +------------------------------ + +(1 row) + +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain'); + cdb_geocode_namedplace_point +------------------------------ + +(1 row) + +-- Insert dummy data into points table +INSERT INTO global_cities_points_limited (geoname_id, name, iso2, admin1, admin2, population, lowername, the_geom) VALUES (3128760, 'Elche', 'ES', 'Valencia', 'AL', 34534, 'elche', ST_GeomFromText( + 'POINT(0.6983 39.26787)',4326) +); +-- Insert dummy data into alternates table +INSERT INTO global_cities_alternates_limited (geoname_id, name, preferred, lowername, admin1_geonameid, iso2, admin1, the_geom) VALUES (3128760, 'Elx', true, 'elx', '000000', 'ES', 'Valencia', ST_GeomFromText( + 'POINT(0.6983 39.26787)',4326) +); +-- Insert dummy data into country decoder table +INSERT INTO country_decoder (synonyms, iso2) VALUES (Array['spain'], 'ES'); +-- Insert dummy data into admin1 decoder table +INSERT INTO admin1_decoder (admin1, synonyms, iso2) VALUES ('Valencia', Array['valencia', 'Valencia'], 'ES'); +-- This should return the point inserted above +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx'); + cdb_geocode_namedplace_point +---------------------------------------------------- + 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 +(1 row) + +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche'); + cdb_geocode_namedplace_point +---------------------------------------------------- + 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 +(1 row) + +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain'); + cdb_geocode_namedplace_point +---------------------------------------------------- + 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 +(1 row) + +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'Spain'); + cdb_geocode_namedplace_point +---------------------------------------------------- + 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 +(1 row) + +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain'); + cdb_geocode_namedplace_point +---------------------------------------------------- + 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 +(1 row) + +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'valencia', 'Spain'); + cdb_geocode_namedplace_point +---------------------------------------------------- + 0101000020E6100000637FD93D7958E63F2ECA6C9049A24340 +(1 row) + +-- Check for namedplaces signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + exists +-------- + t +(1 row) + diff --git a/server/extension/test/0.1.0/expected/60_postalcodes_test.out b/server/extension/test/0.1.0/expected/60_postalcodes_test.out new file mode 100644 index 0000000..01aa003 --- /dev/null +++ b/server/extension/test/0.1.0/expected/60_postalcodes_test.out @@ -0,0 +1,163 @@ +-- Make sure dbs are clean +DELETE FROM global_postal_code_points; +DELETE FROM global_postal_code_polygons; +DELETE FROM country_decoder; +DELETE FROM available_services; +DELETE FROM admin0_synonyms; +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_geocoder_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204'); + cdb_geocode_postalcode_point +------------------------------ + +(1 row) + +-- Insert dummy data into ip_address_locations +INSERT INTO global_postal_code_points (the_geom, iso3, postal_code, postal_code_num) VALUES ( + '0101000020E61000000000000000E040408036B47414764840', + 'ESP', + '03204', + 3204 +); +INSERT INTO global_postal_code_polygons (the_geom, iso3, postal_code, postal_code_num) VALUES ( + '0106000020E610000001000000010300000001000000040000000000000000E000C01F383D7839B740400000000000E000C0AA3C0EDE220F3B4000000000004812404FB7FCCD04893D400000000000E000C01F383D7839B74040', + 'ESP', + '03204', + 3204 +); +INSERT INTO country_decoder (iso3, synonyms) VALUES ( + 'ESP', + Array['spain', 'Spain', 'ESP'] +); +INSERT INTO available_services (adm0_a3, admin0, postal_code_points, postal_code_polygons) VALUES ( + 'ESP', + 't', + 't', + 't' +); +INSERT INTO admin0_synonyms (adm0_a3, name, name_, rank) VALUES ( + 'ESP', + 'Spain', + 'spain', + 3 +); +-- This should return the polygon inserted above +SELECT cdb_geocoder_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204'); + cdb_geocode_postalcode_point +---------------------------------------------------- + 0101000020E61000000000000000E040408036B47414764840 +(1 row) + +SELECT cdb_geocoder_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204', 'spain'); + cdb_geocode_postalcode_point +---------------------------------------------------- + 0101000020E61000000000000000E040408036B47414764840 +(1 row) + +SELECT cdb_geocoder_server.cdb_geocode_postalcode_polygon('test_user', 'test_org', '03204'); + cdb_geocode_postalcode_polygon +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + 0106000020E610000001000000010300000001000000040000000000000000E000C01F383D7839B740400000000000E000C0AA3C0EDE220F3B4000000000004812404FB7FCCD04893D400000000000E000C01F383D7839B74040 +(1 row) + +SELECT cdb_geocoder_server.cdb_geocode_postalcode_polygon('test_user', 'test_org', '03204', 'spain'); + cdb_geocode_postalcode_polygon +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + 0106000020E610000001000000010300000001000000040000000000000000E000C01F383D7839B740400000000000E000C0AA3C0EDE220F3B4000000000004812404FB7FCCD04893D400000000000E000C01F383D7839B74040 +(1 row) + +-- Clean dbs +DELETE FROM global_postal_code_points; +DELETE FROM global_postal_code_polygons; +DELETE FROM country_decoder; +DELETE FROM available_services; +DELETE FROM admin0_synonyms; +-- Check for namedplaces signatures (point and polygon) +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_postalcode_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_postalcode_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_postalcode_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_postalcode_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_postalcode_point' + AND oidvectortypes(p.proargtypes) = 'text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_postalcode_point' + AND oidvectortypes(p.proargtypes) = 'text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_postalcode_polygon' + AND oidvectortypes(p.proargtypes) = 'text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_postalcode_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text'); + exists +-------- + t +(1 row) + diff --git a/server/extension/test/0.1.0/expected/70_ips_test.out b/server/extension/test/0.1.0/expected/70_ips_test.out new file mode 100644 index 0000000..2386200 --- /dev/null +++ b/server/extension/test/0.1.0/expected/70_ips_test.out @@ -0,0 +1,40 @@ +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_geocoder_server.cdb_geocode_ipaddress_point('test_user', 'test_orgname', '0.0.0.0'); + cdb_geocode_ipaddress_point +----------------------------- + +(1 row) + +-- Insert dummy data into ip_address_locations +INSERT INTO ip_address_locations VALUES ('::ffff:0.0.0.0'::inet, (ST_SetSRID(ST_MakePoint('40.40', '3.71'), 4326))); +-- This should return the polygon inserted above +SELECT cdb_geocoder_server.cdb_geocode_ipaddress_point('test_user', 'test_orgname', '0.0.0.0'); + cdb_geocode_ipaddress_point +---------------------------------------------------- + 0101000020E61000003333333333334440AE47E17A14AE0D40 +(1 row) + +-- Check for namedplaces signatures (point and polygon) +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_ipaddress_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + exists +-------- + t +(1 row) + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_ipaddress_point' + AND oidvectortypes(p.proargtypes) = 'text'); + exists +-------- + t +(1 row) + diff --git a/server/extension/test/0.1.0/expected/90_remove_geocoder_api_user_test.out b/server/extension/test/0.1.0/expected/90_remove_geocoder_api_user_test.out new file mode 100644 index 0000000..c53fcfc --- /dev/null +++ b/server/extension/test/0.1.0/expected/90_remove_geocoder_api_user_test.out @@ -0,0 +1,5 @@ +REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_geocoder_server FROM geocoder_api; +REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA public FROM geocoder_api; +REVOKE USAGE ON SCHEMA cdb_geocoder_server FROM geocoder_api; +REVOKE USAGE ON SCHEMA public FROM geocoder_api; +REVOKE SELECT ON ALL TABLES IN SCHEMA public FROM geocoder_api; diff --git a/server/extension/test/0.1.0/sql/00_install_test.sql b/server/extension/test/0.1.0/sql/00_install_test.sql new file mode 100644 index 0000000..8feac25 --- /dev/null +++ b/server/extension/test/0.1.0/sql/00_install_test.sql @@ -0,0 +1,24 @@ +-- Install dependencies +CREATE EXTENSION postgis; +CREATE EXTENSION schema_triggers; +CREATE EXTENSION plpythonu; +CREATE EXTENSION cartodb; +CREATE EXTENSION cdb_geocoder; + +-- Install the extension +CREATE EXTENSION cdb_geocoder_server; + +-- Mock the redis server connection to point to this very test db +SELECT cartodb.cdb_conf_setconf('redis_conf', '{"sentinel_host": "localhost", "sentinel_port": 26739, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}'); + +-- Mock the varnish invalidation function +-- (used by cdb_geocoder tests) +CREATE OR REPLACE FUNCTION public.cdb_invalidate_varnish(table_name text) RETURNS void AS $$ +BEGIN + RETURN; +END +$$ +LANGUAGE plpgsql; + +-- Set user quota +SELECT cartodb.CDB_SetUserQuotaInBytes(0); diff --git a/server/extension/test/0.1.0/sql/20_street_test.sql b/server/extension/test/0.1.0/sql/20_street_test.sql new file mode 100644 index 0000000..7613f5b --- /dev/null +++ b/server/extension/test/0.1.0/sql/20_street_test.sql @@ -0,0 +1,7 @@ +-- Check for namedplaces signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_street_point_v2' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text, text, text'); \ No newline at end of file diff --git a/server/extension/test/0.1.0/sql/30_admin0_test.sql b/server/extension/test/0.1.0/sql/30_admin0_test.sql new file mode 100644 index 0000000..3851d4f --- /dev/null +++ b/server/extension/test/0.1.0/sql/30_admin0_test.sql @@ -0,0 +1,32 @@ +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_geocoder_server.cdb_geocode_admin0_polygon('test_user', 'test_orgname', 'Spain'); + +-- Insert some dummy synonym +INSERT INTO admin0_synonyms (name, adm0_a3) VALUES ('Spain', 'ESP'); + +-- Insert some dummy geometry to return +INSERT INTO ne_admin0_v3 (adm0_a3, the_geom) VALUES('ESP', ST_GeomFromText( + 'POLYGON((-71.1031880899493 42.3152774590236, + -71.1031627617667 42.3152960829043, + -71.102923838298 42.3149156848307, + -71.1031880899493 42.3152774590236))',4326) +); + +-- This should return the polygon inserted above +SELECT cdb_geocoder_server.cdb_geocode_admin0_polygon('test_user', 'test_orgname', 'Spain'); + +-- Check for admin0 signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_admin0_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_admin0_polygon' + AND oidvectortypes(p.proargtypes) = 'text'); \ No newline at end of file diff --git a/server/extension/test/0.1.0/sql/40_admin1_test.sql b/server/extension/test/0.1.0/sql/40_admin1_test.sql new file mode 100644 index 0000000..d3080bf --- /dev/null +++ b/server/extension/test/0.1.0/sql/40_admin1_test.sql @@ -0,0 +1,48 @@ +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California'); +SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California', 'United States'); + +-- Insert dummy data into country decoder table +INSERT INTO country_decoder (synonyms, iso3) VALUES (Array['united states'], 'USA'); + +-- Insert some dummy data and geometry to return +INSERT INTO global_province_polygons (synonyms, iso3, the_geom) VALUES (Array['california'], 'USA', ST_GeomFromText( + 'POLYGON((-71.1031880899493 42.3152774590236, + -71.1031627617667 42.3152960829043, + -71.102923838298 42.3149156848307, + -71.1031880899493 42.3152774590236))',4326) +); + +-- This should return the polygon inserted above +SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California'); +SELECT cdb_geocoder_server.cdb_geocode_admin1_polygon('test_user', 'test_orgname', 'California', 'United States'); + +-- Check for admin1 signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_admin1_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_admin1_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_admin1_polygon' + AND oidvectortypes(p.proargtypes) = 'text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_admin1_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text'); \ No newline at end of file diff --git a/server/extension/test/0.1.0/sql/50_namedplaces_test.sql b/server/extension/test/0.1.0/sql/50_namedplaces_test.sql new file mode 100644 index 0000000..47c304a --- /dev/null +++ b/server/extension/test/0.1.0/sql/50_namedplaces_test.sql @@ -0,0 +1,72 @@ +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx'); +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain'); +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain'); + +-- Insert dummy data into points table +INSERT INTO global_cities_points_limited (geoname_id, name, iso2, admin1, admin2, population, lowername, the_geom) VALUES (3128760, 'Elche', 'ES', 'Valencia', 'AL', 34534, 'elche', ST_GeomFromText( + 'POINT(0.6983 39.26787)',4326) +); + +-- Insert dummy data into alternates table +INSERT INTO global_cities_alternates_limited (geoname_id, name, preferred, lowername, admin1_geonameid, iso2, admin1, the_geom) VALUES (3128760, 'Elx', true, 'elx', '000000', 'ES', 'Valencia', ST_GeomFromText( + 'POINT(0.6983 39.26787)',4326) +); + +-- Insert dummy data into country decoder table +INSERT INTO country_decoder (synonyms, iso2) VALUES (Array['spain'], 'ES'); + +-- Insert dummy data into admin1 decoder table +INSERT INTO admin1_decoder (admin1, synonyms, iso2) VALUES ('Valencia', Array['valencia', 'Valencia'], 'ES'); + +-- This should return the point inserted above +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx'); +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche'); +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Spain'); +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'Spain'); +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elx', 'Valencia', 'Spain'); +SELECT cdb_geocoder_server.cdb_geocode_namedplace_point('test_user', 'test_orgname', 'Elche', 'valencia', 'Spain'); + +-- Check for namedplaces signatures +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_namedplace_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); \ No newline at end of file diff --git a/server/extension/test/0.1.0/sql/60_postalcodes_test.sql b/server/extension/test/0.1.0/sql/60_postalcodes_test.sql new file mode 100644 index 0000000..2a8192e --- /dev/null +++ b/server/extension/test/0.1.0/sql/60_postalcodes_test.sql @@ -0,0 +1,117 @@ +-- Make sure dbs are clean +DELETE FROM global_postal_code_points; +DELETE FROM global_postal_code_polygons; +DELETE FROM country_decoder; +DELETE FROM available_services; +DELETE FROM admin0_synonyms; + +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_geocoder_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204'); + +-- Insert dummy data into ip_address_locations +INSERT INTO global_postal_code_points (the_geom, iso3, postal_code, postal_code_num) VALUES ( + '0101000020E61000000000000000E040408036B47414764840', + 'ESP', + '03204', + 3204 +); + +INSERT INTO global_postal_code_polygons (the_geom, iso3, postal_code, postal_code_num) VALUES ( + '0106000020E610000001000000010300000001000000040000000000000000E000C01F383D7839B740400000000000E000C0AA3C0EDE220F3B4000000000004812404FB7FCCD04893D400000000000E000C01F383D7839B74040', + 'ESP', + '03204', + 3204 +); + +INSERT INTO country_decoder (iso3, synonyms) VALUES ( + 'ESP', + Array['spain', 'Spain', 'ESP'] +); + +INSERT INTO available_services (adm0_a3, admin0, postal_code_points, postal_code_polygons) VALUES ( + 'ESP', + 't', + 't', + 't' +); + +INSERT INTO admin0_synonyms (adm0_a3, name, name_, rank) VALUES ( + 'ESP', + 'Spain', + 'spain', + 3 +); + +-- This should return the polygon inserted above +SELECT cdb_geocoder_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204'); + +SELECT cdb_geocoder_server.cdb_geocode_postalcode_point('test_user', 'test_org', '03204', 'spain'); + +SELECT cdb_geocoder_server.cdb_geocode_postalcode_polygon('test_user', 'test_org', '03204'); + +SELECT cdb_geocoder_server.cdb_geocode_postalcode_polygon('test_user', 'test_org', '03204', 'spain'); + +-- Clean dbs +DELETE FROM global_postal_code_points; +DELETE FROM global_postal_code_polygons; +DELETE FROM country_decoder; +DELETE FROM available_services; +DELETE FROM admin0_synonyms; + +-- Check for namedplaces signatures (point and polygon) +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_postalcode_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_postalcode_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_postalcode_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_postalcode_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_postalcode_point' + AND oidvectortypes(p.proargtypes) = 'text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_postalcode_point' + AND oidvectortypes(p.proargtypes) = 'text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_postalcode_polygon' + AND oidvectortypes(p.proargtypes) = 'text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_postalcode_polygon' + AND oidvectortypes(p.proargtypes) = 'text, text'); \ No newline at end of file diff --git a/server/extension/test/0.1.0/sql/70_ips_test.sql b/server/extension/test/0.1.0/sql/70_ips_test.sql new file mode 100644 index 0000000..f9875f1 --- /dev/null +++ b/server/extension/test/0.1.0/sql/70_ips_test.sql @@ -0,0 +1,24 @@ +-- Check that the public function is callable, even with no data +-- It should return NULL +SELECT cdb_geocoder_server.cdb_geocode_ipaddress_point('test_user', 'test_orgname', '0.0.0.0'); + +-- Insert dummy data into ip_address_locations +INSERT INTO ip_address_locations VALUES ('::ffff:0.0.0.0'::inet, (ST_SetSRID(ST_MakePoint('40.40', '3.71'), 4326))); + +-- This should return the polygon inserted above +SELECT cdb_geocoder_server.cdb_geocode_ipaddress_point('test_user', 'test_orgname', '0.0.0.0'); + +-- Check for namedplaces signatures (point and polygon) +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = 'cdb_geocode_ipaddress_point' + AND oidvectortypes(p.proargtypes) = 'text, text, text'); + +SELECT exists(SELECT * + FROM pg_proc p + INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid) + WHERE ns.nspname = 'cdb_geocoder_server' + AND proname = '_cdb_geocode_ipaddress_point' + AND oidvectortypes(p.proargtypes) = 'text'); \ No newline at end of file diff --git a/server/extension/test/0.1.0/sql/90_remove_geocoder_api_user_test.sql b/server/extension/test/0.1.0/sql/90_remove_geocoder_api_user_test.sql new file mode 100644 index 0000000..4efb88e --- /dev/null +++ b/server/extension/test/0.1.0/sql/90_remove_geocoder_api_user_test.sql @@ -0,0 +1,5 @@ +REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_geocoder_server FROM geocoder_api; +REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA public FROM geocoder_api; +REVOKE USAGE ON SCHEMA cdb_geocoder_server FROM geocoder_api; +REVOKE USAGE ON SCHEMA public FROM geocoder_api; +REVOKE SELECT ON ALL TABLES IN SCHEMA public FROM geocoder_api; \ No newline at end of file diff --git a/server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py b/server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py index a2b148f..0c3878e 100644 --- a/server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py +++ b/server/lib/python/cartodb_geocoder/cartodb_geocoder/config_helper.py @@ -1,104 +1,148 @@ import json +from dateutil.parser import parse as date_parse + class ConfigException(Exception): pass -class UserConfig: - - USER_CONFIG_KEYS = ['is_organization', 'entity_name'] - - def __init__(self, user_config_json, db_user_id = None): - config = json.loads(user_config_json) - filtered_config = { key: config[key] for key in self.USER_CONFIG_KEYS if key in config.keys() } - self.__check_config(filtered_config) - self.__parse_config(filtered_config) - self._user_id = self.__extract_uuid(db_user_id) - - def __check_config(self, filtered_config): - if len(filtered_config.keys()) != len(self.USER_CONFIG_KEYS): - raise ConfigException("Passed user configuration is not correct, check it please") - - return True - - @property - def is_organization(self): - return self._is_organization - - @property - def entity_name(self): - return self._entity_name - - @property - def user_id(self): - return self._user_id - - def __parse_config(self, filtered_config): - self._is_organization = filtered_config['is_organization'] - self._entity_name = filtered_config['entity_name'] - - def __extract_uuid(self, db_user_id): - # Format: development_cartodb_user_ - return db_user_id.split('_')[-1] class GeocoderConfig: - GEOCODER_CONFIG_KEYS = ['street_geocoder_provider', 'google_maps_private_key', - 'nokia_monthly_quota', 'nokia_soft_geocoder_limit'] - NOKIA_GEOCODER_MANDATORY_KEYS = ['nokia_monthly_quota', 'nokia_soft_geocoder_limit'] - NOKIA_GEOCODER = 'nokia' - NOKIA_QUOTA_KEY = 'nokia_monthly_quota' - NOKIA_SOFT_LIMIT_KEY = 'nokia_soft_geocoder_limit' - GOOGLE_GEOCODER = 'google' - GEOCODER_TYPE = 'street_geocoder_provider' - GOOGLE_GEOCODER_API_KEY = 'google_maps_private_key' + GEOCODER_CONFIG_KEYS = ['google_maps_client_id', 'google_maps_api_key', + 'geocoding_quota', 'soft_geocoding_limit', + 'geocoder_type', 'period_end_date', + 'heremaps_app_id', 'heremaps_app_code', 'username', + 'orgname'] + NOKIA_GEOCODER_MANDATORY_KEYS = ['geocoding_quota', 'soft_geocoding_limit', + 'heremaps_app_id', 'heremaps_app_code'] + NOKIA_GEOCODER = 'heremaps' + NOKIA_GEOCODER_APP_ID_KEY = 'heremaps_app_id' + NOKIA_GEOCODER_APP_CODE_KEY = 'heremaps_app_code' + GOOGLE_GEOCODER = 'google' + GOOGLE_GEOCODER_API_KEY = 'google_maps_api_key' + GOOGLE_GEOCODER_CLIENT_ID = 'google_maps_client_id' + GEOCODER_TYPE = 'geocoder_type' + QUOTA_KEY = 'geocoding_quota' + SOFT_LIMIT_KEY = 'soft_geocoding_limit' + USERNAME_KEY = 'username' + ORGNAME_KEY = 'orgname' + PERIOD_END_DATE = 'period_end_date' - def __init__(self, geocoder_config_json): - config = json.loads(geocoder_config_json) - filtered_config = { key: config[key] for key in self.GEOCODER_CONFIG_KEYS if key in config.keys() } - self.__check_config(filtered_config) - self.__parse_config(filtered_config) + def __init__(self, redis_connection, username, orgname=None, + heremaps_app_id=None, heremaps_app_code=None): + self._redis_connection = redis_connection + config = self.__get_user_config(username, orgname, heremaps_app_id, + heremaps_app_code) + filtered_config = {key: config[key] for key in self.GEOCODER_CONFIG_KEYS if key in config.keys()} + self.__check_config(filtered_config) + self.__parse_config(filtered_config) - def __check_config(self, filtered_config): - if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER: - if not set(self.NOKIA_GEOCODER_MANDATORY_KEYS).issubset(set(filtered_config.keys())): - raise ConfigException("""Nokia geocoder need the mandatory parameters 'nokia_monthly_quota' and 'nokia_soft_geocoder_limit'""") - elif filtered_config[self.GEOCODER_TYPE].lower() == self.GOOGLE_GEOCODER: - if self.GOOGLE_GEOCODER_API_KEY not in filtered_config.keys(): - raise ConfigException("Google geocoder need the mandatory parameter 'google_maps_private_key'") + def __get_user_config(self, username, orgname=None, heremaps_app_id=None, + heremaps_app_code=None): + user_config = self._redis_connection.hgetall( + "rails:users:{0}".format(username)) + if not user_config: + raise ConfigException("""There is no user config available. Please check your configuration.'""") + else: + user_config[self.USERNAME_KEY] = username + user_config[self.ORGNAME_KEY] = orgname + user_config[self.NOKIA_GEOCODER_APP_ID_KEY] = heremaps_app_id + user_config[self.NOKIA_GEOCODER_APP_CODE_KEY] = heremaps_app_code + if orgname: + self.__get_organization_config(orgname, user_config) - return True + return user_config - def __parse_config(self, filtered_config): - self._geocoder_type = filtered_config[self.GEOCODER_TYPE].lower() - self._google_maps_private_key = None - self._nokia_monthly_quota = 0 - self._nokia_soft_geocoder_limit = False - if self.GOOGLE_GEOCODER == self._geocoder_type: - self._google_maps_private_key = filtered_config[self.GOOGLE_GEOCODER_API_KEY] - elif self.NOKIA_GEOCODER == self._geocoder_type: - self._nokia_monthly_quota = filtered_config[self.NOKIA_QUOTA_KEY] - self._nokia_soft_geocoder_limit = filtered_config[self.NOKIA_SOFT_LIMIT_KEY] + def __get_organization_config(self, orgname, user_config): + org_config = self._redis_connection.hgetall( + "rails:orgs:{0}".format(orgname)) + if not org_config: + raise ConfigException("""There is no organization config available. Please check your configuration.'""") + else: + user_config[self.QUOTA_KEY] = org_config[self.QUOTA_KEY] + user_config[self.PERIOD_END_DATE] = org_config[self.PERIOD_END_DATE] + user_config[self.GOOGLE_GEOCODER_CLIENT_ID] = org_config[self.GOOGLE_GEOCODER_CLIENT_ID] + user_config[self.GOOGLE_GEOCODER_API_KEY] = org_config[self.GOOGLE_GEOCODER_API_KEY] - @property - def service_type(self): - return self._geocoder_type + def __check_config(self, filtered_config): + if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER: + if not set(self.NOKIA_GEOCODER_MANDATORY_KEYS).issubset(set(filtered_config.keys())): + raise ConfigException("""Some mandatory parameter/s for Nokia geocoder are missing. Check it please""") + if not filtered_config[self.NOKIA_GEOCODER_APP_ID_KEY] or not filtered_config[self.NOKIA_GEOCODER_APP_CODE_KEY]: + raise ConfigException("""Nokia geocoder configuration is missing. Check it please""") + elif filtered_config[self.GEOCODER_TYPE].lower() == self.GOOGLE_GEOCODER: + if self.GOOGLE_GEOCODER_API_KEY not in filtered_config.keys(): + raise ConfigException("""Google geocoder need the mandatory parameter 'google_maps_private_key'""") - @property - def nokia_geocoder(self): - return self._geocoder_type == self.NOKIA_GEOCODER + return True - @property - def google_geocoder(self): - return self._geocoder_type == self.GOOGLE_GEOCODER + def __parse_config(self, filtered_config): + self._username = filtered_config[self.USERNAME_KEY].lower() + if filtered_config[self.ORGNAME_KEY]: + self._orgname = filtered_config[self.ORGNAME_KEY].lower() + else: + self._orgname = None + self._geocoder_type = filtered_config[self.GEOCODER_TYPE].lower() + self._geocoding_quota = float(filtered_config[self.QUOTA_KEY]) + self._period_end_date = date_parse(filtered_config[self.PERIOD_END_DATE]) + if filtered_config[self.SOFT_LIMIT_KEY].lower() == 'true': + self._soft_geocoding_limit = True + else: + self._soft_geocoding_limit = False + if filtered_config[self.GEOCODER_TYPE].lower() == self.NOKIA_GEOCODER: + self._heremaps_app_id = filtered_config[self.NOKIA_GEOCODER_APP_ID_KEY] + self._heremaps_app_code = filtered_config[self.NOKIA_GEOCODER_APP_CODE_KEY] + elif filtered_config[self.GEOCODER_TYPE].lower() == self.GOOGLE_GEOCODER: + self._google_maps_api_key = filtered_config[self.GOOGLE_GEOCODER_API_KEY] + self._google_maps_client_id = filtered_config[self.GOOGLE_GEOCODER_CLIENT_ID] - @property - def google_api_key(self): - return self._google_maps_private_key + @property + def service_type(self): + if self._geocoder_type == self.GOOGLE_GEOCODER: + return 'geocoder_google' + else: + return 'geocoder_here' - @property - def nokia_monthly_quota(self): - return self._nokia_monthly_quota + @property + def heremaps_geocoder(self): + return self._geocoder_type == self.NOKIA_GEOCODER - @property - def nokia_soft_limit(self): - return self._nokia_soft_geocoder_limit \ No newline at end of file + @property + def google_geocoder(self): + return self._geocoder_type == self.GOOGLE_GEOCODER + + @property + def google_client_id(self): + return self._google_maps_client_id + + @property + def google_api_key(self): + return self._google_maps_api_key + + @property + def geocoding_quota(self): + return self._geocoding_quota + + @property + def soft_geocoding_limit(self): + return self._soft_geocoding_limit + + @property + def period_end_date(self): + return self._period_end_date + + @property + def heremaps_app_id(self): + return self._heremaps_app_id + + @property + def heremaps_app_code(self): + return self._heremaps_app_code + + @property + def username(self): + return self._username + + @property + def organization(self): + return self._orgname diff --git a/server/lib/python/cartodb_geocoder/cartodb_geocoder/quota_service.py b/server/lib/python/cartodb_geocoder/cartodb_geocoder/quota_service.py index 001eb34..2aa832d 100644 --- a/server/lib/python/cartodb_geocoder/cartodb_geocoder/quota_service.py +++ b/server/lib/python/cartodb_geocoder/cartodb_geocoder/quota_service.py @@ -1,31 +1,52 @@ import user_service -import config_helper from datetime import date -class QuotaService: - """ Class to manage all the quota operation for the Geocoder SQL API Extension """ - def __init__(self, user_config, geocoder_config, redis_connection): - self._user_config = user_config - self._geocoder_config = geocoder_config - self._user_service = user_service.UserService(self._user_config, - self._geocoder_config.service_type, redis_connection) +class QuotaService: + """ Class to manage all the quota operation for + the Geocoder SQL API Extension """ + + def __init__(self, user_geocoder_config, redis_connection): + self._user_geocoder_config = user_geocoder_config + self._user_service = user_service.UserService( + self._user_geocoder_config, redis_connection) def check_user_quota(self): """ Check if the current user quota surpasses the current quota """ # We don't have quota check for google geocoder - if self._geocoder_config.google_geocoder: + if self._user_geocoder_config.google_geocoder: return True - user_quota = self._geocoder_config.nokia_monthly_quota + user_quota = self._user_geocoder_config.geocoding_quota today = date.today() - service_type = self._geocoder_config.service_type - current_used = self._user_service.used_quota(service_type, today.year, today.month) - soft_geocoder_limit = self._geocoder_config.nokia_soft_limit + service_type = self._user_geocoder_config.service_type + current_used = self._user_service.used_quota(service_type, today) + soft_geocoding_limit = self._user_geocoder_config.soft_geocoding_limit - print "User quota: {0} --- current_used: {1} --- limit: {2}".format(user_quota, current_used, soft_geocoder_limit) + if soft_geocoding_limit or current_used <= user_quota: + return True + else: + return False - return True if soft_geocoder_limit or current_used <= user_quota else False + def increment_success_geocoder_use(self, amount=1): + self._user_service.increment_service_use( + self._user_geocoder_config.service_type, "success_responses", + amount=amount) + self.increment_total_geocoder_use(amount) - def increment_geocoder_use(self, amount=1): - self._user_service.increment_service_use(self._geocoder_config.service_type) \ No newline at end of file + def increment_empty_geocoder_use(self, amount=1): + self._user_service.increment_service_use( + self._user_geocoder_config.service_type, "empty_responses", + amount=amount) + self.increment_total_geocoder_use(amount) + + def increment_failed_geocoder_use(self, amount=1): + self._user_service.increment_service_use( + self._user_geocoder_config.service_type, "fail_responses", + amount=amount) + self.increment_total_geocoder_use(amount) + + def increment_total_geocoder_use(self, amount=1): + self._user_service.increment_service_use( + self._user_geocoder_config.service_type, "total_requests", + amount=amount) diff --git a/server/lib/python/cartodb_geocoder/cartodb_geocoder/redis_helper.py b/server/lib/python/cartodb_geocoder/cartodb_geocoder/redis_helper.py index 27d4800..0c686bf 100644 --- a/server/lib/python/cartodb_geocoder/cartodb_geocoder/redis_helper.py +++ b/server/lib/python/cartodb_geocoder/cartodb_geocoder/redis_helper.py @@ -1,20 +1,29 @@ from redis.sentinel import Sentinel + class RedisHelper: - REDIS_DEFAULT_USER_DB = 5 - REDIS_TIMEOUT = 2 #seconds + REDIS_DEFAULT_USER_DB = 5 + REDIS_DEFAULT_TIMEOUT = 2 #seconds + REDIS_SENTINEL_DEFAULT_PORT = 26379 - def __init__(self, sentinel_host, sentinel_port, sentinel_master_id, redis_db=REDIS_DEFAULT_USER_DB, **kwargs): - self.sentinel_host = sentinel_host - self.sentinel_port = sentinel_port - self.sentinel_master_id = sentinel_master_id - self.timeout = kwargs['timeout'] if 'timeout' in kwargs else REDIS_DEFAULT_TIMEOUT - self.redis_db = redis_db + def __init__(self, sentinel_host, sentinel_port, sentinel_master_id, + redis_db=REDIS_DEFAULT_USER_DB, **kwargs): + self.sentinel_host = sentinel_host + self.sentinel_port = sentinel_port + self.sentinel_master_id = sentinel_master_id + self.timeout = kwargs['timeout'] if 'timeout' in kwargs else self.REDIS_DEFAULT_TIMEOUT + self.redis_db = redis_db - def redis_connection(self): - return self.__create_redis_connection() + def redis_connection(self): + return self.__create_redis_connection() - def __create_redis_connection(self): - sentinel = Sentinel([(self.sentinel_host, 26379)], socket_timeout=self.timeout) - return sentinel.master_for(self.sentinel_master_id, socket_timeout=self.timeout, db=self.redis_db) \ No newline at end of file + def __create_redis_connection(self): + sentinel = Sentinel([(self.sentinel_host, + self.REDIS_SENTINEL_DEFAULT_PORT)], + socket_timeout=self.timeout) + return sentinel.master_for( + self.sentinel_master_id, + socket_timeout=self.timeout, + db=self.redis_db + ) diff --git a/server/lib/python/cartodb_geocoder/cartodb_geocoder/user_service.py b/server/lib/python/cartodb_geocoder/cartodb_geocoder/user_service.py index 142056e..aa5ca98 100644 --- a/server/lib/python/cartodb_geocoder/cartodb_geocoder/user_service.py +++ b/server/lib/python/cartodb_geocoder/cartodb_geocoder/user_service.py @@ -1,64 +1,96 @@ -import redis_helper -from datetime import date +from datetime import date, timedelta +from dateutil.relativedelta import relativedelta + class UserService: - """ Class to manage all the user info """ + """ Class to manage all the user info """ - GEOCODING_QUOTA_KEY = "geocoding_quota" - GEOCODING_SOFT_LIMIT_KEY = "soft_geocoder_limit" + SERVICE_GEOCODER_NOKIA = 'geocoder_here' + SERVICE_GEOCODER_GOOGLE = 'geocoder_google' + SERVICE_GEOCODER_CACHE = 'geocoder_cache' - REDIS_CONNECTION_KEY = "redis_connection" - REDIS_CONNECTION_HOST = "redis_host" - REDIS_CONNECTION_PORT = "redis_port" - REDIS_CONNECTION_DB = "redis_db" + GEOCODING_QUOTA_KEY = "geocoding_quota" + GEOCODING_SOFT_LIMIT_KEY = "soft_geocoder_limit" - def __init__(self, user_config, service_type, redis_connection): - self.user_config = user_config - self.service_type = service_type - self._redis_connection = redis_connection + REDIS_CONNECTION_KEY = "redis_connection" + REDIS_CONNECTION_HOST = "redis_host" + REDIS_CONNECTION_PORT = "redis_port" + REDIS_CONNECTION_DB = "redis_db" - def used_quota(self, service_type, year, month, day=None): - """ Recover the used quota for the user in the current month """ - redis_key_data = self.__get_redis_key(service_type, year, month, day) - current_use = self._redis_connection.hget(redis_key_data['redis_name'], redis_key_data['redis_key']) - return int(current_use) if current_use else 0 + def __init__(self, user_geocoder_config, redis_connection): + self._user_geocoder_config = user_geocoder_config + self._redis_connection = redis_connection + self._username = user_geocoder_config.username + self._orgname = user_geocoder_config.organization - def increment_service_use(self, service_type, date=date.today(), amount=1): - """ Increment the services uses in monthly and daily basis""" - self.__increment_monthly_uses(date, service_type, amount) - self.__increment_daily_uses(date, service_type, amount) + def used_quota(self, service_type, date): + """ Recover the used quota for the user in the current month """ + date_from, date_to = self.__current_billing_cycle() + current_use = 0 + success_responses = self.get_metrics(service_type, + 'success_responses', date_from, + date_to) + empty_responses = self.get_metrics(service_type, + 'empty_responses', date_from, + date_to) + current_use += (success_responses + empty_responses) + if service_type == self.SERVICE_GEOCODER_NOKIA: + cache_hits = self.get_metrics(self.SERVICE_GEOCODER_CACHE, + 'success_responses', date_from, + date_to) + current_use += cache_hits - # Private functions + return current_use - def __increment_monthly_uses(self, date, service_type, amount): - redis_key_data = self.__get_redis_key(service_type, date.year, date.month) - self._redis_connection.hincrby(redis_key_data['redis_name'],redis_key_data['redis_key'],amount) + def increment_service_use(self, service_type, metric, date=date.today(), amount=1): + """ Increment the services uses in monthly and daily basis""" + self.__increment_user_uses(service_type, metric, date, amount) + if self._orgname: + self.__increment_organization_uses(service_type, metric, date, amount) - def __increment_daily_uses(self, date, service_type, amount): - redis_key_data = self.__get_redis_key(service_type, date.year, date.month, date.day) - self._redis_connection.hincrby(redis_key_data['redis_name'],redis_key_data['redis_key'],amount) + def get_metrics(self, service, metric, date_from, date_to): + aggregated_metric = 0 + key_prefix = "org" if self._orgname else "user" + entity_name = self._orgname if self._orgname else self._username + for date in self.__generate_date_range(date_from, date_to): + redis_prefix = self.__parse_redis_prefix(key_prefix, entity_name, + service, metric, date) + score = self._redis_connection.zscore(redis_prefix, date.day) + aggregated_metric += score if score else 0 + return aggregated_metric - def __get_redis_key(self, service_type, year, month, day=None): - redis_name = self.__parse_redis_name(service_type,day) - redis_key = self.__parse_redis_key(year,month,day) + # Private functions - return {'redis_name': redis_name, 'redis_key': redis_key} + def __increment_user_uses(self, service_type, metric, date, amount): + redis_prefix = self.__parse_redis_prefix("user", self._username, + service_type, metric, date) + self._redis_connection.zincrby(redis_prefix, date.day, amount) - def __parse_redis_name(self,service_type, day=None): - prefix = "org" if self.user_config.is_organization else "user" - dated_key = "used_quota_day" if day else "used_quota_month" - redis_name = "{0}:{1}:{2}:{3}".format( - prefix, self.user_config.entity_name, service_type, dated_key - ) - if self.user_config.is_organization and day: - redis_name = "{0}:{1}".format(redis_name, self.user_config.user_id) + def __increment_organization_uses(self, service_type, metric, date, amount): + redis_prefix = self.__parse_redis_prefix("org", self._orgname, + service_type, metric, date) + self._redis_connection.zincrby(redis_prefix, date.day, amount) - return redis_name + def __parse_redis_prefix(self, prefix, entity_name, service_type, metric, date): + yearmonth_key = date.strftime('%Y%m') + redis_name = "{0}:{1}:{2}:{3}:{4}".format(prefix, entity_name, + service_type, metric, + yearmonth_key) - def __parse_redis_key(self,year,month,day=None): - if day: - redis_key = "{0}_{1}_{2}".format(year,month,day) - else: - redis_key = "{0}_{1}".format(year,month) + return redis_name - return redis_key + def __current_billing_cycle(self): + """ Return the begining and end date for the current billing cycle """ + end_period_day = self._user_geocoder_config.period_end_date.day + today = date.today() + if end_period_day > today.day: + temp_date = today + relativedelta(months=-1) + date_from = date(temp_date.year, temp_date.month, end_period_day) + else: + date_from = date(today.year, today.month, end_period_day) + + return date_from, today + + def __generate_date_range(self, date_from, date_to): + for n in range(int((date_to - date_from).days + 1)): + yield date_from + timedelta(n) diff --git a/server/lib/python/cartodb_geocoder/requirements.txt b/server/lib/python/cartodb_geocoder/requirements.txt index 784f064..3b94e05 100644 --- a/server/lib/python/cartodb_geocoder/requirements.txt +++ b/server/lib/python/cartodb_geocoder/requirements.txt @@ -1,6 +1,7 @@ -redis-py==2.10.5 +redis==2.10.5 +python-dateutil==2.4.2 # Test mock==1.3.0 mockredispy==2.9.0.11 -nose==1.3.7 \ No newline at end of file +nose==1.3.7 diff --git a/server/lib/python/cartodb_geocoder/test/test_config_helper.py b/server/lib/python/cartodb_geocoder/test/test_config_helper.py index 061aab1..72b7d21 100644 --- a/server/lib/python/cartodb_geocoder/test/test_config_helper.py +++ b/server/lib/python/cartodb_geocoder/test/test_config_helper.py @@ -1,48 +1,41 @@ +import test_helper from cartodb_geocoder import config_helper from unittest import TestCase from nose.tools import assert_raises +from mockredis import MockRedis +from datetime import datetime, timedelta class TestConfigHelper(TestCase): - def test_should_return_list_of_user_config_if_its_ok(self): - user_config_json = '{"is_organization": false, "entity_name": "test_user"}' - user_config = config_helper.UserConfig(user_config_json, 'development_cartodb_user_UUID') - assert user_config.is_organization == False - assert user_config.entity_name == 'test_user' - assert user_config.user_id == 'UUID' + def setUp(self): + self.redis_conn = MockRedis() - def test_should_return_raise_config_exception_if_not_ok(self): - user_config_json = '{"is_organization": "false"}' - assert_raises(config_helper.ConfigException, config_helper.UserConfig, user_config_json) + def test_should_return_list_of_nokia_geocoder_config_if_its_ok(self): + test_helper.build_redis_user_config(self.redis_conn, 'test_user') + geocoder_config = config_helper.GeocoderConfig(self.redis_conn, + 'test_user', None, + 'nokia_id', 'nokia_cod') + assert geocoder_config.heremaps_geocoder == True + assert geocoder_config.geocoding_quota == 100 + assert geocoder_config.soft_geocoding_limit == False - def test_should_return_raise_config_exception_if_empty(self): - user_config_json = '{}' - assert_raises(config_helper.ConfigException, config_helper.UserConfig, user_config_json) + def test_should_return_list_of_nokia_geocoder_config_ok_for_org(self): + yesterday = datetime.today() - timedelta(days=1) + test_helper.build_redis_user_config(self.redis_conn, 'test_user') + test_helper.build_redis_org_config(self.redis_conn, 'test_org', + quota=200, end_date=yesterday) + geocoder_config = config_helper.GeocoderConfig(self.redis_conn, + 'test_user', 'test_org', + 'nokia_id', 'nokia_cod') + assert geocoder_config.heremaps_geocoder == True + assert geocoder_config.geocoding_quota == 200 + assert geocoder_config.soft_geocoding_limit == False + assert geocoder_config.period_end_date.date() == yesterday.date() - def test_should_return_list_of_nokia_geocoder_config_if_its_ok(self): - geocoder_config_json = """{"street_geocoder_provider": "Nokia", - "nokia_monthly_quota": 100, "nokia_soft_geocoder_limit": false}""" - geocoder_config = config_helper.GeocoderConfig(geocoder_config_json) - assert geocoder_config.nokia_geocoder == True - assert geocoder_config.nokia_monthly_quota == 100 - assert geocoder_config.nokia_soft_limit == False - - def test_should_raise_configuration_exception_when_missing_nokia_geocoder_parameters(self): - geocoder_config_json = '{"street_geocoder_provider": "NokiA", "nokia_monthly_quota": "100"}' - assert_raises(config_helper.ConfigException, config_helper.GeocoderConfig, geocoder_config_json) - - def test_should_raise_configuration_exception_when_missing_nokia_geocoder_parameters_2(self): - geocoder_config_json = '{"street_geocoder_provider": "NoKia", "nokia_soft_geocoder_limit": "false"}' - assert_raises(config_helper.ConfigException, config_helper.GeocoderConfig, geocoder_config_json) - - def test_should_return_list_of_google_geocoder_config_if_its_ok(self): - geocoder_config_json = """{"street_geocoder_provider": "gOOgle", - "google_maps_private_key": "sdasdasda"}""" - geocoder_config = config_helper.GeocoderConfig(geocoder_config_json) - assert geocoder_config.google_geocoder == True - assert geocoder_config.google_api_key == 'sdasdasda' - - def test_should_raise_configuration_exception_when_missing_google_api_key(self): - geocoder_config_json = '{"street_geocoder_provider": "google"}' - assert_raises(config_helper.ConfigException, config_helper.GeocoderConfig, geocoder_config_json) \ No newline at end of file + def test_should_raise_configuration_exception_when_missing_nokia_geocoder_parameters(self): + test_helper.build_redis_user_config(self.redis_conn, 'test_user') + assert_raises(config_helper.ConfigException, + config_helper.GeocoderConfig, + self.redis_conn, 'test_user', + None, None, None) diff --git a/server/lib/python/cartodb_geocoder/test/test_helper.py b/server/lib/python/cartodb_geocoder/test/test_helper.py new file mode 100644 index 0000000..9f1f50f --- /dev/null +++ b/server/lib/python/cartodb_geocoder/test/test_helper.py @@ -0,0 +1,33 @@ +from datetime import datetime, date + + +def build_redis_user_config(redis_conn, username, quota=100, soft_limit=False, + service="heremaps", + end_date=datetime.today()): + user_redis_name = "rails:users:{0}".format(username) + redis_conn.hset(user_redis_name, 'soft_geocoding_limit', soft_limit) + redis_conn.hset(user_redis_name, 'geocoding_quota', quota) + redis_conn.hset(user_redis_name, 'geocoder_type', service) + redis_conn.hset(user_redis_name, 'period_end_date', end_date) + redis_conn.hset(user_redis_name, 'google_maps_client_id', '') + redis_conn.hset(user_redis_name, 'google_maps_api_key', '') + + +def build_redis_org_config(redis_conn, orgname, quota=100, + end_date=datetime.today()): + org_redis_name = "rails:orgs:{0}".format(orgname) + redis_conn.hset(org_redis_name, 'geocoding_quota', quota) + redis_conn.hset(org_redis_name, 'period_end_date', end_date) + redis_conn.hset(org_redis_name, 'google_maps_client_id', '') + redis_conn.hset(org_redis_name, 'google_maps_api_key', '') + + +def increment_geocoder_uses(redis_conn, username, orgname=None, + date=date.today(), service='geocoder_here', + metric='success_responses', amount=20): + prefix = 'org' if orgname else 'user' + entity_name = orgname if orgname else username + yearmonth = date.strftime('%Y%m') + redis_name = "{0}:{1}:{2}:{3}:{4}".format(prefix, entity_name, + service, metric, yearmonth) + redis_conn.zincrby(redis_name, date.day, amount) diff --git a/server/lib/python/cartodb_geocoder/test/test_quota_service.py b/server/lib/python/cartodb_geocoder/test/test_quota_service.py index a618c67..7c71315 100644 --- a/server/lib/python/cartodb_geocoder/test/test_quota_service.py +++ b/server/lib/python/cartodb_geocoder/test/test_quota_service.py @@ -1,85 +1,109 @@ +import test_helper from mockredis import MockRedis from cartodb_geocoder import quota_service from cartodb_geocoder import config_helper from unittest import TestCase from nose.tools import assert_raises -from datetime import datetime +from datetime import datetime, date class TestQuotaService(TestCase): - # single user - # user:::used_quota_month:year_month - # user:::used_quota_day:year_month_day - # organization user - # org:::used_quota_month:year_month - # org::::used_quota_day:year_month_day + # single user + # user::::YYYYMM:DD + # organization user + # org::::YYYYMM:DD - def setUp(self): - self.fake_redis_connection = MockRedis() +# def increment_geocoder_uses(self, username, orgname=None, +# date=date.today(), service='geocoder_here', +# metric='success_responses', amount=20): - def test_should_return_true_if_user_quota_with_no_use(self): - qs = self.__build_quota_service() - assert qs.check_user_quota() == True + def setUp(self): + self.redis_conn = MockRedis() - def test_should_return_true_if_org_quota_with_no_use(self): - qs = self.__build_quota_service(organization=True) - assert qs.check_user_quota() == True + def test_should_return_true_if_user_quota_with_no_use(self): + qs = self.__build_quota_service('test_user') + assert qs.check_user_quota() is True - def test_should_return_true_if_user_quota_is_not_completely_used(self): - qs = self.__build_quota_service() - self.__increment_geocoder_uses('test_user', '20151111') - assert qs.check_user_quota() == True + def test_should_return_true_if_org_quota_with_no_use(self): + qs = self.__build_quota_service('test_user', orgname='test_org') + assert qs.check_user_quota() is True - def test_should_return_true_if_org_quota_is_not_completely_used(self): - qs = self.__build_quota_service(organization=True) - self.__increment_geocoder_uses('test_user', '20151111', org=True) - assert qs.check_user_quota() == True + def test_should_return_true_if_user_quota_is_not_completely_used(self): + qs = self.__build_quota_service('test_user') + test_helper.increment_geocoder_uses(self.redis_conn, 'test_user') + assert qs.check_user_quota() is True - def test_should_return_false_if_user_quota_is_surpassed(self): - qs = self.__build_quota_service(quota = 1, soft_limit=False) - self.__increment_geocoder_uses('test_user', '20151111') - assert qs.check_user_quota() == False + def test_should_return_true_if_org_quota_is_not_completely_used(self): + qs = self.__build_quota_service('test_user', orgname='test_org') + test_helper.increment_geocoder_uses(self.redis_conn, 'test_user', + orgname='test_org') + assert qs.check_user_quota() is True - def test_should_return_false_if_org_quota_is_surpassed(self): - qs = self.__build_quota_service(organization=True, quota=1) - self.__increment_geocoder_uses('test_user', '20151111', org=True) - assert qs.check_user_quota() == False + def test_should_return_false_if_user_quota_is_surpassed(self): + qs = self.__build_quota_service('test_user') + test_helper.increment_geocoder_uses(self.redis_conn, 'test_user', + amount=300) + assert qs.check_user_quota() is False - def test_should_return_true_if_user_quota_is_surpassed_but_soft_limit_is_enabled(self): - qs = self.__build_quota_service(quota=1, soft_limit=True) - self.__increment_geocoder_uses('test_user', '20151111') - assert qs.check_user_quota() == True + def test_should_return_false_if_org_quota_is_surpassed(self): + qs = self.__build_quota_service('test_user', orgname='test_org') + test_helper.increment_geocoder_uses(self.redis_conn, 'test_user', + orgname='test_org', amount=400) + assert qs.check_user_quota() is False - def test_should_return_true_if_org_quota_is_surpassed_but_soft_limit_is_enabled(self): - qs = self.__build_quota_service(organization=True, quota=1, soft_limit=True) - self.__increment_geocoder_uses('test_user', '20151111', org=True) - assert qs.check_user_quota() == True + def test_should_return_true_if_user_quota_is_surpassed_but_soft_limit_is_enabled(self): + qs = self.__build_quota_service('test_user', soft_limit=True) + test_helper.increment_geocoder_uses(self.redis_conn, 'test_user', + amount=300) + assert qs.check_user_quota() is True - def test_should_check_user_increment_and_quota_check_correctly(self): - qs = self.__build_quota_service(quota=2, soft_limit=False) - qs.increment_geocoder_use() - assert qs.check_user_quota() == True + def test_should_return_true_if_org_quota_is_surpassed_but_soft_limit_is_enabled(self): + qs = self.__build_quota_service('test_user', orgname='test_org', + soft_limit=True) + test_helper.increment_geocoder_uses(self.redis_conn, 'test_user', + orgname='test_org', amount=400) + assert qs.check_user_quota() is True - def test_should_check_org_increment_and_quota_check_correctly(self): - qs = self.__build_quota_service(organization=True, quota=2, soft_limit=False) - qs.increment_geocoder_use() - assert qs.check_user_quota() == True + def test_should_check_user_increment_and_quota_check_correctly(self): + qs = self.__build_quota_service('test_user', quota=2) + qs.increment_success_geocoder_use() + assert qs.check_user_quota() == True + qs.increment_success_geocoder_use(amount=2) + assert qs.check_user_quota() == False + month = date.today().strftime('%Y%m') + name = 'user:test_user:geocoder_here:total_requests:{0}'.format(month) + total_requests = self.redis_conn.zscore(name, date.today().day) + assert total_requests == 3 - def __build_quota_service(self, quota=100, service='nokia', organization=False, soft_limit=False): - is_organization = 'true' if organization else 'false' - has_soft_limit = 'true' if soft_limit else 'false' - user_config_json = '{{"is_organization": {0}, "entity_name": "test_user"}}'.format(is_organization) - geocoder_config_json = """{{"street_geocoder_provider": "{0}","nokia_monthly_quota": {1}, - "nokia_soft_geocoder_limit": {2}}}""".format(service, quota, has_soft_limit) - user_config = config_helper.UserConfig(user_config_json, 'user_id') - geocoder_config = config_helper.GeocoderConfig(geocoder_config_json) + def test_should_check_org_increment_and_quota_check_correctly(self): + qs = self.__build_quota_service('test_user', orgname='test_org', + quota=2) + qs.increment_success_geocoder_use() + assert qs.check_user_quota() == True + qs.increment_success_geocoder_use(amount=2) + assert qs.check_user_quota() == False + month = date.today().strftime('%Y%m') + org_name = 'org:test_org:geocoder_here:total_requests:{0}'.format(month) + org_total_requests = self.redis_conn.zscore(org_name, date.today().day) + assert org_total_requests == 3 + user_name = 'user:test_user:geocoder_here:total_requests:{0}'.format(month) + user_total_requests = self.redis_conn.zscore(user_name, date.today().day) + assert user_total_requests == 3 - return quota_service.QuotaService(user_config, geocoder_config, redis_connection = self.fake_redis_connection) + def __build_quota_service(self, username, quota=100, service='heremaps', + orgname=None, soft_limit=False, + end_date = datetime.today()): + test_helper.build_redis_user_config(self.redis_conn, username, + quota = quota, service = service, + soft_limit = soft_limit, + end_date = end_date) + if orgname: + test_helper.build_redis_org_config(self.redis_conn, orgname, + quota=quota, end_date=end_date) + geocoder_config = config_helper.GeocoderConfig(self.redis_conn, + username, orgname, + 'nokia_id', 'nokia_cod') + return quota_service.QuotaService(geocoder_config, + redis_connection = self.redis_conn) - def __increment_geocoder_uses(self, entity_name, date_string, service='nokia', amount=20, org=False): - prefix = 'org' if org else 'user' - date = datetime.strptime(date_string, "%Y%m%d") - redis_name = "{0}:{1}:{2}:used_quota_month".format(prefix, entity_name, service) - redis_key_month = "{0}_{1}".format(date.year, date.month) - self.fake_redis_connection.hset(redis_name, redis_key_month, amount) \ No newline at end of file diff --git a/server/lib/python/cartodb_geocoder/test/test_user_service.py b/server/lib/python/cartodb_geocoder/test/test_user_service.py index 2e3b969..4aeb1c0 100644 --- a/server/lib/python/cartodb_geocoder/test/test_user_service.py +++ b/server/lib/python/cartodb_geocoder/test/test_user_service.py @@ -1,63 +1,90 @@ +import test_helper from mockredis import MockRedis from cartodb_geocoder import user_service from cartodb_geocoder import config_helper -from datetime import datetime +from datetime import datetime, date from unittest import TestCase from nose.tools import assert_raises +from datetime import timedelta class TestUserService(TestCase): - NOKIA_GEOCODER = 'nokia' + NOKIA_GEOCODER = 'geocoder_here' def setUp(self): - self.fake_redis_connection = MockRedis() + self.redis_conn = MockRedis() - def test_user_used_quota_for_a_month(self): - us = self.__build_user_service() - self.__increment_month_geocoder_uses('test_user', '20151111') - assert us.used_quota(self.NOKIA_GEOCODER, 2015, 11) == 20 + def test_user_used_quota_for_a_day(self): + us = self.__build_user_service('test_user') + test_helper.increment_geocoder_uses(self.redis_conn, 'test_user', + amount=400) + assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 400 - def test_org_used_quota_for_a_month(self): - us = self.__build_user_service(organization=True) - self.__increment_month_geocoder_uses('test_user', '20151111', org=True) - assert us.used_quota(self.NOKIA_GEOCODER, 2015, 11) == 20 + def test_org_used_quota_for_a_day(self): + us = self.__build_user_service('test_user', orgname='test_org') + test_helper.increment_geocoder_uses(self.redis_conn, 'test_user', + orgname='test_org', + amount=400) + assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 400 def test_user_not_amount_in_used_quota_for_month_should_be_0(self): - us = self.__build_user_service() - assert us.used_quota(self.NOKIA_GEOCODER, 2015, 11) == 0 + us = self.__build_user_service('test_user') + assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 0 def test_org_not_amount_in_used_quota_for_month_should_be_0(self): - us = self.__build_user_service(organization=True) - assert us.used_quota(self.NOKIA_GEOCODER, 2015, 11) == 0 + us = self.__build_user_service('test_user', orgname='test_org') + assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 0 - def test_should_increment_user_used_quota(self): - us = self.__build_user_service() - date = datetime.strptime("20151111", "%Y%m%d") - us.increment_service_use(self.NOKIA_GEOCODER, date=date, amount=1) - assert us.used_quota(self.NOKIA_GEOCODER, 2015, 11) == 1 - assert us.used_quota(self.NOKIA_GEOCODER, 2015, 11, 11) == 1 + def test_should_increment_user_used_quota_for_one_date(self): + us = self.__build_user_service('test_user') + us.increment_service_use(self.NOKIA_GEOCODER, 'success_responses') + assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 1 + us.increment_service_use(self.NOKIA_GEOCODER, 'empty_responses') + assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2 + us.increment_service_use(self.NOKIA_GEOCODER, 'fail_responses') + assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2 def test_should_increment_org_used_quota(self): - us = self.__build_user_service(organization=True) - date = datetime.strptime("20151111", "%Y%m%d") - us.increment_service_use(self.NOKIA_GEOCODER, date=date, amount=1) - assert us.used_quota(self.NOKIA_GEOCODER, 2015, 11) == 1 - assert us.used_quota(self.NOKIA_GEOCODER, 2015, 11, 11) == 1 + us = self.__build_user_service('test_user', orgname='test_org') + us.increment_service_use(self.NOKIA_GEOCODER, 'success_responses') + assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 1 + us.increment_service_use(self.NOKIA_GEOCODER, 'empty_responses') + assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2 + us.increment_service_use(self.NOKIA_GEOCODER, 'fail_responses') + assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2 - def test_exception_if_not_redis_config(self): - assert_raises(Exception, user_service.UserService, 'user_id') + def test_should_increment_user_used_quota_in_for_multiples_dates(self): + two_days_ago = datetime.today() - timedelta(days=2) + one_day_ago = datetime.today() - timedelta(days=1) + one_day_after = datetime.today() + timedelta(days=1) + us = self.__build_user_service('test_user', end_date=one_day_ago) + us.increment_service_use(self.NOKIA_GEOCODER, 'success_responses', + date=date.today()) + assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 1 + us.increment_service_use(self.NOKIA_GEOCODER, 'empty_responses', + date=one_day_ago) + assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2 + us.increment_service_use(self.NOKIA_GEOCODER, 'empty_responses', + date=two_days_ago) + assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2 + us.increment_service_use(self.NOKIA_GEOCODER, 'empty_responses', + date=one_day_after) + assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2 + us.increment_service_use(self.NOKIA_GEOCODER, 'fail_responses') + assert us.used_quota(self.NOKIA_GEOCODER, date.today()) == 2 - def __build_user_service(self, organization=False, service='nokia'): - is_organization = 'true' if organization else 'false' - user_config_json = '{{"is_organization": {0}, "entity_name": "test_user"}}'.format(is_organization) - user_config = config_helper.UserConfig(user_config_json, 'user_id') - - return user_service.UserService(user_config, service, redis_connection = self.fake_redis_connection) - - def __increment_month_geocoder_uses(self, entity_name, date_string, service='nokia', amount=20, org=False): - parent_tag = 'org' if org else 'user' - date = datetime.strptime(date_string, "%Y%m%d") - redis_name = "{0}:{1}:{2}:used_quota_month".format(parent_tag, entity_name, service) - redis_key_month = "{0}_{1}".format(date.year, date.month) - self.fake_redis_connection.hset(redis_name, redis_key_month, amount) \ No newline at end of file + def __build_user_service(self, username, quota=100, service='heremaps', + orgname=None, soft_limit=False, + end_date=datetime.today()): + test_helper.build_redis_user_config(self.redis_conn, username, + quota=quota, service=service, + soft_limit=soft_limit, + end_date=end_date) + if orgname: + test_helper.build_redis_org_config(self.redis_conn, orgname, + quota=quota, end_date=end_date) + geocoder_config = config_helper.GeocoderConfig(self.redis_conn, + username, orgname, + 'nokia_id', 'nokia_cod') + return user_service.UserService(geocoder_config, self.redis_conn) diff --git a/server/lib/python/heremaps/heremaps/heremapsexceptions.py b/server/lib/python/heremaps/heremaps/heremapsexceptions.py index 6416872..cb10713 100644 --- a/server/lib/python/heremaps/heremaps/heremapsexceptions.py +++ b/server/lib/python/heremaps/heremaps/heremapsexceptions.py @@ -1,22 +1,26 @@ #!/usr/local/bin/python # -*- coding: utf-8 -*- - import json + class BadGeocodingParams(Exception): def __init__(self, value): self.value = value + def __str__(self): - return repr('Bad geocoding params: ' + json.dumps(value)) + return repr('Bad geocoding params: ' + json.dumps(self.value)) + class NoGeocodingParams(Exception): def __str__(self): return repr('No params for geocoding specified') + class EmptyGeocoderResponse(Exception): def __str__(self): return repr('The request could not be geocoded') + class MalformedResult(Exception): - def __str__(self): - return repr('Result structure is malformed') + def __str__(self): + return repr('Result structure is malformed') diff --git a/server/lib/python/heremaps/heremaps/heremapsgeocoder.py b/server/lib/python/heremaps/heremaps/heremapsgeocoder.py index 696c79f..9ebe1df 100644 --- a/server/lib/python/heremaps/heremaps/heremapsgeocoder.py +++ b/server/lib/python/heremaps/heremaps/heremapsgeocoder.py @@ -2,17 +2,19 @@ # -*- coding: utf-8 -*- import json -import urllib +import requests from heremaps.heremapsexceptions import BadGeocodingParams from heremaps.heremapsexceptions import EmptyGeocoderResponse from heremaps.heremapsexceptions import NoGeocodingParams from heremaps.heremapsexceptions import MalformedResult + class Geocoder: 'A Here Maps Geocoder wrapper for python' - URL_GEOCODE_JSON = 'http://geocoder.api.here.com/6.2/geocode.json' + PRODUCTION_GEOCODE_JSON_URL = 'https://geocoder.api.here.com/6.2/geocode.json' + STAGING_GEOCODE_JSON_URL = 'https://geocoder.cit.api.here.com/6.2/geocode.json' DEFAULT_MAXRESULTS = 1 DEFAULT_GEN = 9 @@ -52,18 +54,18 @@ class Geocoder: app_code = '' maxresults = '' - def __init__(self, app_id, app_code, maxresults=DEFAULT_MAXRESULTS, gen=DEFAULT_GEN): + def __init__(self, app_id, app_code, maxresults=DEFAULT_MAXRESULTS, + gen=DEFAULT_GEN, host=PRODUCTION_GEOCODE_JSON_URL): self.app_id = app_id self.app_code = app_code self.maxresults = maxresults self.gen = gen + self.host = host def geocode(self, params): if not set(params.keys()).issubset(set(self.ADDRESS_PARAMS)): raise BadGeocodingParams(params) - response = self.perform_request(params) - try: results = response['Response']['View'][0]['Result'] except IndexError: @@ -73,29 +75,25 @@ class Geocoder: def perform_request(self, params): request_params = { - 'app_id' : self.app_id, - 'app_code' : self.app_code, - 'maxresults' : self.maxresults, - 'gen' : self.gen - } + 'app_id': self.app_id, + 'app_code': self.app_code, + 'maxresults': self.maxresults, + 'gen': self.gen + } request_params.update(params) - - encoded_request_params = urllib.urlencode(request_params) - - response = json.load( - urllib.urlopen(self.URL_GEOCODE_JSON - + '?' - + encoded_request_params)) - - return response + response = requests.get(self.host, params=request_params) + if response.status_code == requests.codes.ok: + return json.loads(response.text) + else: + response.raise_for_status() def geocode_address(self, **kwargs): params = {} for key, value in kwargs.iteritems(): - if value: params[key] = value - - if not params: raise NoGeocodingParams() - + if value: + params[key] = value + if not params: + raise NoGeocodingParams() return self.geocode(params) def extract_lng_lat_from_result(self, result): diff --git a/server/lib/python/heremaps/heremaps/tests/heremapsgeocoder_tests.py b/server/lib/python/heremaps/heremaps/tests/heremapsgeocoder_tests.py index 6fbd1c0..bca926f 100644 --- a/server/lib/python/heremaps/heremaps/tests/heremapsgeocoder_tests.py +++ b/server/lib/python/heremaps/heremaps/tests/heremapsgeocoder_tests.py @@ -9,73 +9,72 @@ from heremaps.heremapsexceptions import EmptyGeocoderResponse from heremaps.heremapsexceptions import NoGeocodingParams from heremaps.heremapsexceptions import MalformedResult -from secrets import * class GeocoderTestCase(unittest.TestCase): EMPTY_RESPONSE = { - "Response":{ - "MetaInfo":{ - "Timestamp":"2015-11-04T16:31:57.273+0000" + "Response": { + "MetaInfo": { + "Timestamp": "2015-11-04T16:31:57.273+0000" }, - "View":[] + "View": [] } } GOOD_RESPONSE = { "Response": { "MetaInfo": { - "Timestamp":"2015-11-04T16:30:32.187+0000" + "Timestamp": "2015-11-04T16:30:32.187+0000" }, - "View":[{ - "_type":"SearchResultsViewType", - "ViewId":0, - "Result":[{ - "Relevance":0.89, - "MatchLevel":"street", - "MatchQuality":{ - "City":1.0, - "Street":[1.0] + "View": [{ + "_type": "SearchResultsViewType", + "ViewId": 0, + "Result": [{ + "Relevance": 0.89, + "MatchLevel": "street", + "MatchQuality": { + "City": 1.0, + "Street": [1.0] }, - "Location":{ - "LocationId":"NT_yyKB4r3mCWAX4voWgxPcuA", - "LocationType":"address", - "DisplayPosition":{ - "Latitude":40.43433, - "Longitude":-3.70126 + "Location": { + "LocationId": "NT_yyKB4r3mCWAX4voWgxPcuA", + "LocationType": "address", + "DisplayPosition": { + "Latitude": 40.43433, + "Longitude": -3.70126 }, - "NavigationPosition":[{ - "Latitude":40.43433, - "Longitude":-3.70126 + "NavigationPosition": [{ + "Latitude": 40.43433, + "Longitude": -3.70126 }], - "MapView":{ - "TopLeft":{ - "Latitude":40.43493, - "Longitude":-3.70404 + "MapView": { + "TopLeft": { + "Latitude": 40.43493, + "Longitude": -3.70404 }, - "BottomRight":{ - "Latitude":40.43373, - "Longitude":-3.69873 + "BottomRight": { + "Latitude": 40.43373, + "Longitude": -3.69873 } }, - "Address":{ - "Label":"Calle de Eloy Gonzalo, Madrid, España", - "Country":"ESP", - "State":"Comunidad de Madrid", - "County":"Madrid", - "City":"Madrid", - "District":"Trafalgar", - "Street":"Calle de Eloy Gonzalo", - "AdditionalData":[{ - "value":"España", - "key":"CountryName" + "Address": { + "Label": "Calle de Eloy Gonzalo, Madrid, España", + "Country": "ESP", + "State": "Comunidad de Madrid", + "County": "Madrid", + "City": "Madrid", + "District": "Trafalgar", + "Street": "Calle de Eloy Gonzalo", + "AdditionalData": [{ + "value": "España", + "key": "CountryName" }, { - "value":"Comunidad de Madrid", - "key":"StateName" + "value": "Comunidad de Madrid", + "key": "StateName" }, { - "value":"Madrid", - "key":"CountyName" + "value": "Madrid", + "key": "CountyName" }] } } @@ -117,10 +116,7 @@ class GeocoderTestCase(unittest.TestCase): self.assertEqual(coordinates[1], 40.43433) def test_extract_lng_lat_from_result_with_malformed_result(self): - result = {'manolo':'escobar'} + result = {'manolo': 'escobar'} with self.assertRaises(MalformedResult): self.geocoder.extract_lng_lat_from_result(result) - -if __name__ == '__main__': - main() diff --git a/server/lib/python/heremaps/requirements.txt b/server/lib/python/heremaps/requirements.txt new file mode 100644 index 0000000..18860e2 --- /dev/null +++ b/server/lib/python/heremaps/requirements.txt @@ -0,0 +1,4 @@ +requests==2.9.1 + +# Test +nose==1.3.7 diff --git a/sql-template-renderer b/sql-template-renderer index de5f9a7..8938db8 100755 --- a/sql-template-renderer +++ b/sql-template-renderer @@ -40,9 +40,19 @@ class SqlTemplateRenderer end def params_with_type - @function_signature['params'].map { |p| "#{p['name']} #{p['type']}"}.join(', ') + @function_signature['params'].map { |p| "#{p['name']} #{p['type']}" }.join(', ') end + def params_with_type_and_default + parameters = @function_signature['params'].map do |p| + if not p['default'].nil? + "#{p['name']} #{p['type']} DEFAULT #{p['default']}" + else + "#{p['name']} #{p['type']}" + end + end + return parameters.join(', ') + end end diff --git a/test/README.md b/test/README.md index d1e8028..1eb6229 100644 --- a/test/README.md +++ b/test/README.md @@ -18,3 +18,4 @@ This suite of tests test the following parts of the geocoding API through the SQ - Named places functions - Postal code functions - Ip address functions +- Street address functions (This will call Heremaps or Google so it will cost you 2 credits) diff --git a/test/fixtures/geocoder_api_test_dataset.csv b/test/fixtures/geocoder_api_test_dataset.csv index b27327b..072a95c 100644 --- a/test/fixtures/geocoder_api_test_dataset.csv +++ b/test/fixtures/geocoder_api_test_dataset.csv @@ -1,3 +1,3 @@ -id,country,province,city,postalcode,ip -1,Spain,Castilla y León,Valladolid,47010,8.8.8.8 -2,USA,New York,Manhattn,10001,8.8.8.8 \ No newline at end of file +id,country,province,city,postalcode,ip,street +1,Spain,Castilla y León,Valladolid,47010,8.8.8.8,Calle Amor de Dios +2,USA,New York,Manhattn,10001,8.8.8.8,NonExistentStreet diff --git a/test/integration/test_street_functions.py b/test/integration/test_street_functions.py new file mode 100644 index 0000000..5af715c --- /dev/null +++ b/test/integration/test_street_functions.py @@ -0,0 +1,32 @@ +from unittest import TestCase +from nose.tools import assert_raises +from nose.tools import assert_not_equal, assert_equal +from ..helpers.integration_test_helper import IntegrationTestHelper + + +class TestStreetFunctions(TestCase): + + def setUp(self): + self.env_variables = IntegrationTestHelper.get_environment_variables() + self.sql_api_url = "https://{0}.{1}/api/v2/sql".format( + self.env_variables['username'], + self.env_variables['host'], + self.env_variables['api_key'] + ) + + def test_if_select_with_street_point_is_ok(self): + query = "SELECT cdb_geocode_street_point_v2_(street) " \ + "as geometry FROM {0} LIMIT 1&api_key={1}".format( + self.env_variables['table_name'], + self.env_variables['api_key']) + geometry = IntegrationTestHelper.execute_query(self.sql_api_url, query) + assert_not_equal(geometry, None) + + def test_if_select_with_street_without_api_key_raise_error(self): + query = "SELECT cdb_geocode_street_point_v2_(street) " \ + "as geometry FROM {0} LIMIT 1".format( + self.env_variables['table_name']) + try: + IntegrationTestHelper.execute_query(self.sql_api_url, query) + except Exception as e: + assert_equal(e.message[0], "The api_key must be provided")