Merge pull request #220 from CartoDB/bump_version_dev
Bump client to 0.10.0 and server to 0.13.0
This commit is contained in:
Carla
GitHub
commit
261fff7873
@ -13,8 +13,8 @@ OLD_VERSIONS = $(wildcard old_versions/*.sql)
|
||||
# @see http://www.postgresql.org/docs/current/static/extend-pgxs.html
|
||||
DATA = $(NEW_EXTENSION_ARTIFACT) \
|
||||
$(OLD_VERSIONS) \
|
||||
cdb_dataservices_client--0.8.0--0.9.0.sql \
|
||||
cdb_dataservices_client--0.9.0--0.8.0.sql
|
||||
cdb_dataservices_client--0.9.0--0.10.0.sql \
|
||||
cdb_dataservices_client--0.10.0--0.9.0.sql
|
||||
|
||||
|
||||
REGRESS = $(notdir $(basename $(wildcard test/sql/*test.sql)))
|
||||
|
||||
16
client/cdb_dataservices_client--0.10.0--0.9.0.sql
Normal file
16
client/cdb_dataservices_client--0.10.0--0.9.0.sql
Normal file
@ -0,0 +1,16 @@
|
||||
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
|
||||
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
|
||||
\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.9.0'" to load this file. \quit
|
||||
DROP FUNCTION IF EXISTS cdb_dataservices_client._OBS_GetTable(text, text, text, json);
|
||||
DROP FUNCTION IF EXISTS cdb_dataservices_client._OBS_AugmentTable(text, text, json);
|
||||
DROP FUNCTION IF EXISTS cdb_dataservices_client.__OBS_AugmentTable(text, text, text, text, text, text, text, json);
|
||||
DROP FUNCTION IF EXISTS cdb_dataservices_client.__OBS_GetTable(text, text, text, text, text, text, text, text, json);
|
||||
DROP FUNCTION IF EXISTS cdb_dataservices_client._OBS_ConnectUserTable(text, text, text, text, text, text);
|
||||
DROP FUNCTION IF EXISTS cdb_dataservices_client._OBS_GetReturnMetadata(text, text, text, json);
|
||||
DROP FUNCTION IF EXISTS cdb_dataservices_client._OBS_FetchJoinFdwTableData(text, text, text, text, text, json);
|
||||
DROP FUNCTION IF EXISTS cdb_dataservices_client._OBS_DisconnectUserTable(text, text, text, text, text);
|
||||
DROP FUNCTION IF EXISTS cdb_dataservices_client.OBS_DumpVersion();
|
||||
DROP FUNCTION IF EXISTS cdb_dataservices_client._OBS_DumpVersion(text, text);
|
||||
|
||||
DROP TYPE IF EXISTS cdb_dataservices_client.ds_fdw_metadata;
|
||||
DROP TYPE IF EXISTS cdb_dataservices_client.ds_return_metadata;
|
||||
1743
client/cdb_dataservices_client--0.10.0.sql
Normal file
1743
client/cdb_dataservices_client--0.10.0.sql
Normal file
File diff suppressed because it is too large
Load Diff
303
client/cdb_dataservices_client--0.9.0--0.10.0.sql
Normal file
303
client/cdb_dataservices_client--0.9.0--0.10.0.sql
Normal file
@ -0,0 +1,303 @@
|
||||
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
|
||||
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
|
||||
\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.10.0'" to load this file. \quit
|
||||
|
||||
CREATE TYPE cdb_dataservices_client.ds_fdw_metadata as (schemaname text, tabname text, servername text);
|
||||
CREATE TYPE cdb_dataservices_client.ds_return_metadata as (colnames text[], coltypes text[]);
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_GetTable(table_name text, output_table_name text, function_name text, params json)
|
||||
RETURNS boolean AS $$
|
||||
DECLARE
|
||||
username text;
|
||||
user_db_role text;
|
||||
orgname text;
|
||||
dbname text;
|
||||
user_schema text;
|
||||
result boolean;
|
||||
BEGIN
|
||||
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
|
||||
RAISE EXCEPTION 'The api_key must be provided';
|
||||
END IF;
|
||||
|
||||
SELECT session_user INTO user_db_role;
|
||||
|
||||
SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
|
||||
-- JSON value stored "" is taken as literal
|
||||
IF username IS NULL OR username = '' OR username = '""' THEN
|
||||
RAISE EXCEPTION 'Username is a mandatory argument';
|
||||
END IF;
|
||||
|
||||
IF orgname IS NULL OR orgname = '' OR orgname = '""' THEN
|
||||
user_schema := 'public';
|
||||
ELSE
|
||||
user_schema := username;
|
||||
END IF;
|
||||
|
||||
SELECT current_database() INTO dbname;
|
||||
|
||||
SELECT cdb_dataservices_client.__OBS_GetTable(username, orgname, user_db_role, user_schema, dbname, table_name, output_table_name, function_name, params) INTO result;
|
||||
|
||||
RETURN result;
|
||||
END;
|
||||
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
|
||||
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_AugmentTable(table_name text, function_name text, params json)
|
||||
RETURNS boolean AS $$
|
||||
DECLARE
|
||||
username text;
|
||||
user_db_role text;
|
||||
orgname text;
|
||||
dbname text;
|
||||
user_schema text;
|
||||
result boolean;
|
||||
BEGIN
|
||||
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
|
||||
RAISE EXCEPTION 'The api_key must be provided';
|
||||
END IF;
|
||||
|
||||
SELECT session_user INTO user_db_role;
|
||||
|
||||
SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
|
||||
-- JSON value stored "" is taken as literal
|
||||
IF username IS NULL OR username = '' OR username = '""' THEN
|
||||
RAISE EXCEPTION 'Username is a mandatory argument';
|
||||
END IF;
|
||||
|
||||
IF orgname IS NULL OR orgname = '' OR orgname = '""' THEN
|
||||
user_schema := 'public';
|
||||
ELSE
|
||||
user_schema := username;
|
||||
END IF;
|
||||
|
||||
SELECT current_database() INTO dbname;
|
||||
|
||||
SELECT cdb_dataservices_client.__OBS_AugmentTable(username, orgname, user_db_role, user_schema, dbname, table_name, function_name, params) INTO result;
|
||||
|
||||
RETURN result;
|
||||
END;
|
||||
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_client.__OBS_AugmentTable(username text, orgname text, user_db_role text, user_schema text, dbname text, table_name text, function_name text, params json)
|
||||
RETURNS boolean AS $$
|
||||
from time import strftime
|
||||
try:
|
||||
server_table_name = None
|
||||
temporary_table_name = 'ds_tmp_' + str(strftime("%s")) + table_name
|
||||
|
||||
# Obtain return types for augmentation procedure
|
||||
ds_return_metadata = plpy.execute("SELECT colnames, coltypes "
|
||||
"FROM cdb_dataservices_client._OBS_GetReturnMetadata({username}::text, {orgname}::text, {function_name}::text, {params}::json);"
|
||||
.format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), function_name=plpy.quote_literal(function_name), params=plpy.quote_literal(params))
|
||||
)
|
||||
|
||||
colnames_arr = ds_return_metadata[0]["colnames"]
|
||||
coltypes_arr = ds_return_metadata[0]["coltypes"]
|
||||
|
||||
# Prepare column and type strings required in the SQL queries
|
||||
colnames = ','.join(colnames_arr)
|
||||
columns_with_types_arr = [colnames_arr[i] + ' ' + coltypes_arr[i] for i in range(0,len(colnames_arr))]
|
||||
columns_with_types = ','.join(columns_with_types_arr)
|
||||
|
||||
|
||||
# Instruct the OBS server side to establish a FDW
|
||||
# The metadata is obtained as well in order to:
|
||||
# - (a) be able to write the query to grab the actual data to be executed in the remote server via pl/proxy,
|
||||
# - (b) be able to tell OBS to free resources when done.
|
||||
ds_fdw_metadata = plpy.execute("SELECT schemaname, tabname, servername "
|
||||
"FROM cdb_dataservices_client._OBS_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {user_schema}::text, {dbname}::text, {table_name}::text);"
|
||||
.format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), user_schema=plpy.quote_literal(user_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name))
|
||||
)
|
||||
|
||||
server_schema = ds_fdw_metadata[0]["schemaname"]
|
||||
server_table_name = ds_fdw_metadata[0]["tabname"]
|
||||
server_name = ds_fdw_metadata[0]["servername"]
|
||||
|
||||
# Create temporary table with the augmented results
|
||||
plpy.execute('CREATE UNLOGGED TABLE "{user_schema}".{temp_table_name} AS '
|
||||
'(SELECT {columns}, cartodb_id '
|
||||
'FROM cdb_dataservices_client._OBS_FetchJoinFdwTableData('
|
||||
'{username}::text, {orgname}::text, {schema}::text, {table_name}::text, {function_name}::text, {params}::json) '
|
||||
'AS results({columns_with_types}, cartodb_id int) )'
|
||||
.format(columns=colnames, username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname),
|
||||
user_schema=user_schema, schema=plpy.quote_literal(server_schema), table_name=plpy.quote_literal(server_table_name),
|
||||
function_name=plpy.quote_literal(function_name), params=plpy.quote_literal(params), columns_with_types=columns_with_types,
|
||||
temp_table_name=temporary_table_name)
|
||||
)
|
||||
|
||||
# Wipe user FDW data from the server
|
||||
wiped = plpy.execute("SELECT cdb_dataservices_client._OBS_DisconnectUserTable({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, {fdw_server}::text)"
|
||||
.format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), server_schema=plpy.quote_literal(server_schema), server_table_name=plpy.quote_literal(server_table_name), fdw_server=plpy.quote_literal(server_name))
|
||||
)
|
||||
|
||||
# Prepare table to receive augmented results in new columns
|
||||
for idx, column in enumerate(colnames_arr):
|
||||
if colnames_arr[idx] is not 'the_geom':
|
||||
plpy.execute('ALTER TABLE "{user_schema}".{table_name} ADD COLUMN {column_name} {column_type}'
|
||||
.format(user_schema=user_schema, table_name=table_name, column_name=colnames_arr[idx], column_type=coltypes_arr[idx])
|
||||
)
|
||||
|
||||
# Populate the user table with the augmented results
|
||||
plpy.execute('UPDATE "{user_schema}".{table_name} SET {columns} = '
|
||||
'(SELECT {columns} FROM "{user_schema}".{temporary_table_name} '
|
||||
'WHERE "{user_schema}".{temporary_table_name}.cartodb_id = "{user_schema}".{table_name}.cartodb_id)'
|
||||
.format(columns = colnames, username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname),
|
||||
user_schema = user_schema, table_name=table_name, function_name=function_name, params=params, columns_with_types=columns_with_types,
|
||||
temporary_table_name=temporary_table_name)
|
||||
)
|
||||
|
||||
plpy.execute('DROP TABLE IF EXISTS "{user_schema}".{temporary_table_name}'
|
||||
.format(user_schema=user_schema, table_name=table_name, temporary_table_name=temporary_table_name)
|
||||
)
|
||||
|
||||
return True
|
||||
except Exception as e:
|
||||
plpy.warning('Error trying to augment table {0}'.format(e))
|
||||
# Wipe user FDW data from the server in case of failure if the table was connected
|
||||
if server_table_name:
|
||||
# Wipe local temporary table
|
||||
plpy.execute('DROP TABLE IF EXISTS "{user_schema}".{temporary_table_name}'
|
||||
.format(user_schema=user_schema, table_name=table_name, temporary_table_name=temporary_table_name)
|
||||
)
|
||||
|
||||
wiped = plpy.execute("SELECT cdb_dataservices_client._OBS_DisconnectUserTable({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, {fdw_server}::text)"
|
||||
.format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), server_schema=plpy.quote_literal(server_schema), server_table_name=plpy.quote_literal(server_table_name), fdw_server=plpy.quote_literal(server_name))
|
||||
)
|
||||
return False
|
||||
$$ LANGUAGE plpythonu;
|
||||
|
||||
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_client.__OBS_GetTable(username text, orgname text, user_db_role text, user_schema text, dbname text, table_name text, output_table_name text, function_name text, params json)
|
||||
RETURNS boolean AS $$
|
||||
try:
|
||||
server_table_name = None
|
||||
# Obtain return types for augmentation procedure
|
||||
ds_return_metadata = plpy.execute("SELECT colnames, coltypes "
|
||||
"FROM cdb_dataservices_client._OBS_GetReturnMetadata({username}::text, {orgname}::text, {function_name}::text, {params}::json);"
|
||||
.format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), function_name=plpy.quote_literal(function_name), params=plpy.quote_literal(params))
|
||||
)
|
||||
|
||||
colnames_arr = ds_return_metadata[0]["colnames"]
|
||||
coltypes_arr = ds_return_metadata[0]["coltypes"]
|
||||
|
||||
# Prepare column and type strings required in the SQL queries
|
||||
colnames = ','.join(colnames_arr)
|
||||
columns_with_types_arr = [colnames_arr[i] + ' ' + coltypes_arr[i] for i in range(0,len(colnames_arr))]
|
||||
columns_with_types = ','.join(columns_with_types_arr)
|
||||
|
||||
|
||||
# Instruct the OBS server side to establish a FDW
|
||||
# The metadata is obtained as well in order to:
|
||||
# - (a) be able to write the query to grab the actual data to be executed in the remote server via pl/proxy,
|
||||
# - (b) be able to tell OBS to free resources when done.
|
||||
ds_fdw_metadata = plpy.execute("SELECT schemaname, tabname, servername "
|
||||
"FROM cdb_dataservices_client._OBS_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {schema}::text, {dbname}::text, {table_name}::text);"
|
||||
.format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), schema=plpy.quote_literal(user_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name))
|
||||
)
|
||||
|
||||
server_schema = ds_fdw_metadata[0]["schemaname"]
|
||||
server_table_name = ds_fdw_metadata[0]["tabname"]
|
||||
server_name = ds_fdw_metadata[0]["servername"]
|
||||
|
||||
# Get list of user columns to include in the new table
|
||||
user_table_columns = ','.join(
|
||||
plpy.execute('SELECT array_agg(\'user_table.\' || attname) AS columns '
|
||||
'FROM pg_attribute WHERE attrelid = \'"{user_schema}".{table_name}\'::regclass '
|
||||
'AND attnum > 0 AND NOT attisdropped AND attname NOT LIKE \'the_geom_webmercator\' '
|
||||
'AND NOT attname LIKE ANY(string_to_array(\'{colnames}\',\',\'));'
|
||||
.format(user_schema=user_schema, table_name=table_name, colnames=colnames)
|
||||
)[0]["columns"]
|
||||
)
|
||||
|
||||
# Populate a new table with the augmented results
|
||||
plpy.execute('CREATE TABLE "{user_schema}".{output_table_name} AS '
|
||||
'(SELECT results.{columns}, {user_table_columns} '
|
||||
'FROM {table_name} AS user_table '
|
||||
'LEFT JOIN cdb_dataservices_client._OBS_FetchJoinFdwTableData({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, {function_name}::text, {params}::json) as results({columns_with_types}, cartodb_id int) '
|
||||
'ON results.cartodb_id = user_table.cartodb_id)'
|
||||
.format(output_table_name=output_table_name, columns=colnames, user_table_columns=user_table_columns, username=plpy.quote_nullable(username),
|
||||
orgname=plpy.quote_nullable(orgname), user_schema=user_schema, server_schema=plpy.quote_literal(server_schema), server_table_name=plpy.quote_literal(server_table_name),
|
||||
table_name=table_name, function_name=plpy.quote_literal(function_name), params=plpy.quote_literal(params), columns_with_types=columns_with_types)
|
||||
)
|
||||
|
||||
plpy.execute('ALTER TABLE "{schema}".{table_name} OWNER TO "{user}";'
|
||||
.format(schema=user_schema, table_name=output_table_name, user=user_db_role)
|
||||
)
|
||||
|
||||
# Wipe user FDW data from the server
|
||||
wiped = plpy.execute("SELECT cdb_dataservices_client._OBS_DisconnectUserTable({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, {fdw_server}::text)"
|
||||
.format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), server_schema=plpy.quote_literal(server_schema), server_table_name=plpy.quote_literal(server_table_name), fdw_server=plpy.quote_literal(server_name))
|
||||
)
|
||||
|
||||
return True
|
||||
except Exception as e:
|
||||
plpy.warning('Error trying to get table {0}'.format(e))
|
||||
# Wipe user FDW data from the server in case of failure if the table was connected
|
||||
if server_table_name:
|
||||
wiped = plpy.execute("SELECT cdb_dataservices_client._OBS_DisconnectUserTable({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, {fdw_server}::text)"
|
||||
.format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), server_schema=plpy.quote_literal(server_schema), server_table_name=plpy.quote_literal(server_table_name), fdw_server=plpy.quote_literal(server_name))
|
||||
)
|
||||
return False
|
||||
$$ LANGUAGE plpythonu;
|
||||
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_ConnectUserTable(username text, orgname text, user_db_role text, user_schema text, dbname text, table_name text)
|
||||
RETURNS cdb_dataservices_client.ds_fdw_metadata AS $$
|
||||
CONNECT _server_conn_str();
|
||||
TARGET cdb_dataservices_server._OBS_ConnectUserTable;
|
||||
$$ LANGUAGE plproxy;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_GetReturnMetadata(username text, orgname text, function_name text, params json)
|
||||
RETURNS cdb_dataservices_client.ds_return_metadata AS $$
|
||||
CONNECT _server_conn_str();
|
||||
TARGET cdb_dataservices_server._OBS_GetReturnMetadata;
|
||||
$$ LANGUAGE plproxy;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json)
|
||||
RETURNS SETOF record AS $$
|
||||
CONNECT _server_conn_str();
|
||||
TARGET cdb_dataservices_server._OBS_FetchJoinFdwTableData;
|
||||
$$ LANGUAGE plproxy;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_DisconnectUserTable(username text, orgname text, table_schema text, table_name text, server_name text)
|
||||
RETURNS boolean AS $$
|
||||
CONNECT _server_conn_str();
|
||||
TARGET cdb_dataservices_server._OBS_DisconnectUserTable;
|
||||
$$ LANGUAGE plproxy;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_dumpversion ()
|
||||
RETURNS text AS $$
|
||||
DECLARE
|
||||
ret text;
|
||||
username text;
|
||||
orgname text;
|
||||
BEGIN
|
||||
IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
|
||||
RAISE EXCEPTION 'The api_key must be provided';
|
||||
END IF;
|
||||
SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
|
||||
-- JSON value stored "" is taken as literal
|
||||
IF username IS NULL OR username = '' OR username = '""' THEN
|
||||
RAISE EXCEPTION 'Username is a mandatory argument, check it out';
|
||||
END IF;
|
||||
|
||||
SELECT * FROM cdb_dataservices_client._obs_dumpversion(username, orgname) INTO ret;
|
||||
RETURN ret;
|
||||
|
||||
END;
|
||||
$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
|
||||
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_dumpversion (username text, organization_name text)
|
||||
|
||||
RETURNS text AS $$
|
||||
CONNECT cdb_dataservices_client._server_conn_str();
|
||||
|
||||
SELECT * FROM cdb_dataservices_server.obs_dumpversion (username, organization_name);
|
||||
|
||||
$$ LANGUAGE plproxy;
|
||||
|
||||
GRANT EXECUTE ON FUNCTION cdb_dataservices_client._OBS_AugmentTable(text, text, json) TO publicuser;
|
||||
GRANT EXECUTE ON FUNCTION cdb_dataservices_client._OBS_GetTable(text, text, text, json) TO publicuser;
|
||||
GRANT EXECUTE ON FUNCTION cdb_dataservices_client.OBS_DumpVersion() TO publicuser;
|
||||
@ -1,5 +1,5 @@
|
||||
comment = 'CartoDB dataservices client API extension'
|
||||
default_version = '0.9.0'
|
||||
default_version = '0.10.0'
|
||||
requires = 'plproxy, cartodb'
|
||||
superuser = true
|
||||
schema = cdb_dataservices_client
|
||||
|
||||
149
server/extension/cdb_dataservices_server--0.12.0--0.13.0.sql
Normal file
149
server/extension/cdb_dataservices_server--0.12.0--0.13.0.sql
Normal file
@ -0,0 +1,149 @@
|
||||
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
|
||||
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
|
||||
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.13.0'" to load this file. \quit
|
||||
|
||||
-- HERE goes your code to upgrade/downgrade
|
||||
CREATE TYPE cdb_dataservices_server.ds_fdw_metadata as (schemaname text, tabname text, servername text);
|
||||
|
||||
CREATE TYPE cdb_dataservices_server.ds_return_metadata as (colnames text[], coltypes text[]);
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text)
|
||||
RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$
|
||||
return plpy.execute("SELECT * FROM cdb_dataservices_server.__OBS_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {schema}::text, {dbname}::text, {table_name}::text)"
|
||||
.format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), schema=plpy.quote_literal(input_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name))
|
||||
)[0]
|
||||
$$ LANGUAGE plpythonu;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.__OBS_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text)
|
||||
RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$
|
||||
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
|
||||
TARGET cdb_observatory._OBS_ConnectUserTable;
|
||||
$$ LANGUAGE plproxy;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetReturnMetadata(username text, orgname text, function_name text, params json)
|
||||
RETURNS cdb_dataservices_server.ds_return_metadata AS $$
|
||||
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
|
||||
TARGET cdb_observatory._OBS_GetReturnMetadata;
|
||||
$$ LANGUAGE plproxy;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json)
|
||||
RETURNS SETOF record AS $$
|
||||
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
|
||||
TARGET cdb_observatory._OBS_FetchJoinFdwTableData;
|
||||
$$ LANGUAGE plproxy;
|
||||
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_DisconnectUserTable(username text, orgname text, table_schema text, table_name text, servername text)
|
||||
RETURNS boolean AS $$
|
||||
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
|
||||
TARGET cdb_observatory._OBS_DisconnectUserTable;
|
||||
$$ LANGUAGE plproxy;
|
||||
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isolines(
|
||||
username TEXT,
|
||||
orgname TEXT,
|
||||
isotype TEXT,
|
||||
source geometry(Geometry, 4326),
|
||||
mode TEXT,
|
||||
data_range integer[],
|
||||
options text[])
|
||||
RETURNS SETOF cdb_dataservices_server.isoline AS $$
|
||||
import json
|
||||
from cartodb_services.mapzen import MatrixClient
|
||||
from cartodb_services.mapzen import MapzenIsolines
|
||||
from cartodb_services.metrics import QuotaService
|
||||
|
||||
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
user_mapzen_isolines_routing_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)]
|
||||
|
||||
# -- Check the quota
|
||||
quota_service = QuotaService(user_mapzen_isolines_routing_config, redis_conn)
|
||||
if not quota_service.check_user_quota():
|
||||
plpy.error('You have reached the limit of your quota')
|
||||
|
||||
try:
|
||||
client = MatrixClient(user_mapzen_isolines_routing_config.mapzen_matrix_api_key)
|
||||
mapzen_isolines = MapzenIsolines(client)
|
||||
|
||||
if source:
|
||||
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
|
||||
lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
|
||||
origin = {'lat': lat, 'lon': lon}
|
||||
else:
|
||||
raise Exception('source is NULL')
|
||||
|
||||
# -- TODO Support options properly
|
||||
isolines = {}
|
||||
if isotype == 'isodistance':
|
||||
for r in data_range:
|
||||
isoline = mapzen_isolines.calculate_isodistance(origin, mode, r)
|
||||
isolines[r] = isoline
|
||||
elif isotype == 'isochrone':
|
||||
for r in data_range:
|
||||
isoline = mapzen_isolines.calculate_isochrone(origin, mode, r)
|
||||
isolines[r] = isoline
|
||||
|
||||
result = []
|
||||
for r in data_range:
|
||||
|
||||
if len(isolines[r]) >= 3:
|
||||
# -- TODO encapsulate this block into a func/method
|
||||
locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
|
||||
wkt_coordinates = ','.join(["%f %f" % (l['lon'], l['lat']) for l in locations])
|
||||
sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates)
|
||||
multipolygon = plpy.execute(sql, 1)[0]['geom']
|
||||
else:
|
||||
multipolygon = None
|
||||
|
||||
result.append([source, r, multipolygon])
|
||||
|
||||
quota_service.increment_success_service_use()
|
||||
quota_service.increment_isolines_service_use(len(isolines))
|
||||
return result
|
||||
except BaseException as e:
|
||||
import sys, traceback
|
||||
type_, value_, traceback_ = sys.exc_info()
|
||||
quota_service.increment_failed_service_use()
|
||||
error_msg = 'There was an error trying to obtain isolines using mapzen: {0}'.format(e)
|
||||
plpy.debug(traceback.format_tb(traceback_))
|
||||
raise e
|
||||
#plpy.error(error_msg)
|
||||
finally:
|
||||
quota_service.increment_total_service_use()
|
||||
$$ LANGUAGE plpythonu SECURITY DEFINER;
|
||||
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
|
||||
RETURNS SETOF cdb_dataservices_server.isoline AS $$
|
||||
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
|
||||
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_mapzen_isolines_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
|
||||
user_isolines_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)]
|
||||
type = 'isodistance'
|
||||
|
||||
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
|
||||
result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options])
|
||||
|
||||
return result
|
||||
$$ LANGUAGE plpythonu;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
|
||||
RETURNS SETOF cdb_dataservices_server.isoline AS $$
|
||||
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
|
||||
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_mapzen_isolines_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
|
||||
user_isolines_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)]
|
||||
type = 'isochrone'
|
||||
|
||||
mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
|
||||
result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options])
|
||||
|
||||
return result
|
||||
$$ LANGUAGE plpythonu;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_dumpversion(username text, orgname text)
|
||||
RETURNS text AS $$
|
||||
CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
|
||||
SELECT cdb_observatory.obs_dumpversion();
|
||||
$$ LANGUAGE plproxy;
|
||||
124
server/extension/cdb_dataservices_server--0.13.0--0.12.0.sql
Normal file
124
server/extension/cdb_dataservices_server--0.13.0--0.12.0.sql
Normal file
@ -0,0 +1,124 @@
|
||||
--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
|
||||
-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
|
||||
\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.12.0'" to load this file. \quit
|
||||
|
||||
-- HERE goes your code to upgrade/downgrade
|
||||
DROP FUNCTION IF EXISTS cdb_dataservices_server._OBS_ConnectUserTable(text, text, text, text, text, text);
|
||||
DROP FUNCTION IF EXISTS cdb_dataservices_server.__OBS_ConnectUserTable(text, text, text, text, text, text);
|
||||
DROP FUNCTION IF EXISTS cdb_dataservices_server._OBS_GetReturnMetadata(text, text, text, json);
|
||||
DROP FUNCTION IF EXISTS cdb_dataservices_server._OBS_FetchJoinFdwTableData(text, text, text, text, text, json);
|
||||
DROP FUNCTION IF EXISTS cdb_dataservices_server._OBS_DisconnectUserTable(text, text, text, text, text);
|
||||
DROP FUNCTION IF EXISTS cdb_dataservices_server.OBS_DumpVersion(text, text);
|
||||
|
||||
DROP TYPE IF EXISTS cdb_dataservices_server.ds_fdw_metadata;
|
||||
DROP TYPE IF EXISTS cdb_dataservices_server.ds_return_metadata;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
|
||||
RETURNS SETOF cdb_dataservices_server.isoline AS $$
|
||||
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
|
||||
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_mapzen_isolines_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
|
||||
user_isolines_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)]
|
||||
type = 'isodistance'
|
||||
|
||||
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
|
||||
result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options])
|
||||
isolines = []
|
||||
for element in result:
|
||||
isoline = element['isoline']
|
||||
isoline = isoline.translate(None, "()").split(',')
|
||||
isolines.append(isoline)
|
||||
|
||||
return isolines
|
||||
$$ LANGUAGE plpythonu;
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
|
||||
RETURNS SETOF cdb_dataservices_server.isoline AS $$
|
||||
plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
|
||||
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
plpy.execute("SELECT cdb_dataservices_server._get_mapzen_isolines_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
|
||||
user_isolines_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)]
|
||||
type = 'isochrone'
|
||||
|
||||
mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
|
||||
result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options])
|
||||
isolines = []
|
||||
for element in result:
|
||||
isoline = element['isoline']
|
||||
isoline = isoline.translate(None, "()").split(',') #--TODO what is this for?
|
||||
isolines.append(isoline)
|
||||
|
||||
return isolines
|
||||
$$ LANGUAGE plpythonu;
|
||||
|
||||
|
||||
CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isolines(
|
||||
username TEXT,
|
||||
orgname TEXT,
|
||||
isotype TEXT,
|
||||
source geometry(Geometry, 4326),
|
||||
mode TEXT,
|
||||
data_range integer[],
|
||||
options text[])
|
||||
RETURNS SETOF cdb_dataservices_server.isoline AS $$
|
||||
import json
|
||||
from cartodb_services.mapzen import MatrixClient
|
||||
from cartodb_services.mapzen import MapzenIsolines
|
||||
from cartodb_services.metrics import QuotaService
|
||||
|
||||
redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
|
||||
user_mapzen_isolines_routing_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)]
|
||||
|
||||
# -- Check the quota
|
||||
quota_service = QuotaService(user_mapzen_isolines_routing_config, redis_conn)
|
||||
if not quota_service.check_user_quota():
|
||||
plpy.error('You have reached the limit of your quota')
|
||||
|
||||
try:
|
||||
client = MatrixClient(user_mapzen_isolines_routing_config.mapzen_matrix_api_key)
|
||||
mapzen_isolines = MapzenIsolines(client)
|
||||
|
||||
if source:
|
||||
lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
|
||||
lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
|
||||
origin = {'lat': lat, 'lon': lon}
|
||||
else:
|
||||
raise Exception('source is NULL')
|
||||
|
||||
# -- TODO Support options properly
|
||||
isolines = {}
|
||||
if isotype == 'isodistance':
|
||||
for r in data_range:
|
||||
isoline = mapzen_isolines.calculate_isodistance(origin, mode, r)
|
||||
isolines[r] = (isoline)
|
||||
elif isotype == 'isochrone':
|
||||
for r in data_range:
|
||||
isoline = mapzen_isolines.calculate_isochrone(origin, mode, r)
|
||||
isolines[r] = (isoline)
|
||||
|
||||
result = []
|
||||
for r in data_range:
|
||||
|
||||
# -- TODO encapsulate this block into a func/method
|
||||
locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
|
||||
wkt_coordinates = ','.join(["%f %f" % (l['lon'], l['lat']) for l in locations])
|
||||
sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates)
|
||||
multipolygon = plpy.execute(sql, 1)[0]['geom']
|
||||
|
||||
result.append([source, r, multipolygon])
|
||||
|
||||
quota_service.increment_success_service_use()
|
||||
quota_service.increment_isolines_service_use(len(isolines))
|
||||
return result
|
||||
except BaseException as e:
|
||||
import sys, traceback
|
||||
type_, value_, traceback_ = sys.exc_info()
|
||||
quota_service.increment_failed_service_use()
|
||||
error_msg = 'There was an error trying to obtain isolines using mapzen: {0}'.format(e)
|
||||
plpy.debug(traceback.format_tb(traceback_))
|
||||
raise e
|
||||
#plpy.error(error_msg)
|
||||
finally:
|
||||
quota_service.increment_total_service_use()
|
||||
$$ LANGUAGE plpythonu SECURITY DEFINER;
|
||||
|
||||
2293
server/extension/cdb_dataservices_server--0.13.0.sql
Normal file
2293
server/extension/cdb_dataservices_server--0.13.0.sql
Normal file
File diff suppressed because it is too large
Load Diff
@ -1,5 +1,5 @@
|
||||
comment = 'CartoDB dataservices server extension'
|
||||
default_version = '0.12.0'
|
||||
default_version = '0.13.0'
|
||||
requires = 'plpythonu, plproxy, postgis, cdb_geocoder'
|
||||
superuser = true
|
||||
schema = cdb_dataservices_server
|
||||
|
||||
@ -26,7 +26,7 @@ SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"geocoder": {"app_id": "dummy
|
||||
|
||||
(1 row)
|
||||
|
||||
SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing": {"api_key": "routing_dummy_api_key", "monthly_quota": 1500000}, "geocoder": {"api_key": "geocoder_dummy_api_key", "monthly_quota": 1500000}}');
|
||||
SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing": {"api_key": "routing_dummy_api_key", "monthly_quota": 1500000}, "geocoder": {"api_key": "geocoder_dummy_api_key", "monthly_quota": 1500000}, "matrix": {"api_key": "matrix_dummy_api_key", "monthly_quota": 1500000}}');
|
||||
cdb_conf_setconf
|
||||
------------------
|
||||
|
||||
|
||||
@ -13,7 +13,7 @@ CREATE EXTENSION cdb_dataservices_server;
|
||||
SELECT cartodb.cdb_conf_setconf('redis_metrics_config', '{"redis_host": "localhost", "redis_port": 6379, "timeout": 0.1, "redis_db": 5}');
|
||||
SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"redis_host": "localhost", "redis_port": 6379, "timeout": 0.1, "redis_db": 5}');
|
||||
SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"geocoder": {"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "dummy_id", "app_code": "dummy_code"}}');
|
||||
SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing": {"api_key": "routing_dummy_api_key", "monthly_quota": 1500000}, "geocoder": {"api_key": "geocoder_dummy_api_key", "monthly_quota": 1500000}}');
|
||||
SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing": {"api_key": "routing_dummy_api_key", "monthly_quota": 1500000}, "geocoder": {"api_key": "geocoder_dummy_api_key", "monthly_quota": 1500000}, "matrix": {"api_key": "matrix_dummy_api_key", "monthly_quota": 1500000}}');
|
||||
SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/dev/null"}');
|
||||
SELECT cartodb.cdb_conf_setconf('data_observatory_conf', '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}, "monthly_quota": 100000}');
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user