cartodb/cartodb-postgresql
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #392 from Algunenano/test_bionic

Browse Source 
Test bionic
This commit is contained in:
Raúl Marín 2019-12-26 18:11:30 +01:00 committed by GitHub
commit 741d171f4a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 78 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.travis.yml
View File

@ -1,4 +1,3 @@
dist: xenial
language: c
sudo: required
@ -12,10 +11,15 @@ env:
jobs:
include:
- env: POSTGRESQL_VERSION="9.6" POSTGIS_VERSION="2.5"
dist: xenial
- env: POSTGRESQL_VERSION="10" POSTGIS_VERSION="2.5"
dist: xenial
- env: POSTGRESQL_VERSION="11" POSTGIS_VERSION="2.5"
dist: xenial
- env: POSTGRESQL_VERSION="12" POSTGIS_VERSION="2.5"
dist: bionic
- env: POSTGRESQL_VERSION="12" POSTGIS_VERSION="3"
dist: bionic
script:
- sudo service postgresql stop;

6
Makefile
View File

@ -150,7 +150,7 @@ $(EXTENSION)--$(EXTVERSION).sql: $(CDBSCRIPTS) cartodb_version.sql Makefile
cat $(CDBSCRIPTS) | \
$(SED) -e 's/@extschema@/cartodb/g' \
-e 's/@postgisschema@/public/g' \
-e 's/plpythonu/$(PLPYTHONU)/g' >> $@
-e 's/@@plpythonu@@/$(PLPYTHONU)/g' >> $@
echo "GRANT USAGE ON SCHEMA cartodb TO public;" >> $@
cat cartodb_version.sql >> $@
@ -164,10 +164,10 @@ $(EXTENSION)--$(EXTVERSION)--$(EXTVERSION)next.sql: $(EXTENSION)--$(EXTVERSION).
cp $< $@
$(EXTENSION).control: $(EXTENSION).control.in Makefile
$(SED) -e 's/@@VERSION@@/$(EXTVERSION)/g' -e 's/plpythonu/$(PLPYTHONU)/g' $< > $@
$(SED) -e 's/@@VERSION@@/$(EXTVERSION)/g' -e 's/@@plpythonu@@/$(PLPYTHONU)/g' $< > $@
cartodb_version.sql: cartodb_version.sql.in Makefile $(GITDIR)/index
$(SED) -e 's/@@VERSION@@/$(EXTVERSION)/' -e 's/@extschema@/cartodb/g' -e "s/@postgisschema@/public/g" -e 's/plpythonu/$(PLPYTHONU)/g' $< > $@
$(SED) -e 's/@@VERSION@@/$(EXTVERSION)/' -e 's/@extschema@/cartodb/g' -e "s/@postgisschema@/public/g" -e 's/@@plpythonu@@/$(PLPYTHONU)/g' $< > $@
# Needed for consistent `echo` results with backslashes
SHELL = bash

2
cartodb.control.in
View File

@ -3,4 +3,4 @@ comment = 'Turn a database into a cartodb user database.'
superuser = true
relocatable = false
schema = cartodb
requires = 'plpythonu, postgis'
requires = '@@plpythonu@@, postgis'

2
scripts-available/CDB_FederatedServerDiagnostics.sql
View File

@ -191,7 +191,7 @@ AS $$
})
return json.dumps(stats)
$$
LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
LANGUAGE @@plpythonu@@ VOLATILE PARALLEL UNSAFE;
--

6
scripts-available/CDB_GhostTables.sql
View File

@ -32,7 +32,6 @@ AS $$
client = redis.Redis(host=tis_host, port=tis_port, socket_timeout=tis_timeout)
GD['invalidation'] = client
except Exception as err:
error = "client_error - %s" % str(err)
# NOTE: no retries on connection error
plpy.warning('Error trying to connect to Invalidation Service to link Ghost Tables: ' + str(err))
break
@ -41,13 +40,12 @@ AS $$
client.execute_command('DBSCH', db_name, username, event_name)
break
except Exception as err:
error = "request_error - %s" % str(err)
client = GD['invalidation'] = None # force reconnect
if not tis_retry:
plpy.warning('Error calling Invalidation Service to link Ghost Tables: ' + str(err))
break
tis_retry -= 1 # try reconnecting
$$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE;
$$ LANGUAGE '@@plpythonu@@' VOLATILE PARALLEL UNSAFE;
-- Enqueues a job to run Ghost tables linking process for the current user
CREATE OR REPLACE FUNCTION @extschema@.CDB_LinkGhostTables(event_name text DEFAULT 'USER')
@ -61,7 +59,7 @@ AS $$
EXECUTE 'SELECT current_database();' INTO db_name;
PERFORM @extschema@._CDB_LinkGhostTables(username, db_name, event_name);
RAISE NOTICE '_CDB_LinkGhostTables() called with username=%, event_name=%', username, event_name;
RAISE INFO '_CDB_LinkGhostTables() called with username=%, event_name=%', username, event_name;
END;
$$ LANGUAGE plpgsql
VOLATILE

20
scripts-available/CDB_Groups_API.sql
View File

@ -21,7 +21,7 @@ $$
body = '{ "name": "%s", "database_role": "%s" }' % (group_name, group_role)
query = "select @extschema@._CDB_Group_API_Request('POST', '%s', '%s', '{200, 409}') as response_status" % (url, body)
plpy.execute(query)
$$ LANGUAGE 'plpythonu'
$$ LANGUAGE '@@plpythonu@@'
VOLATILE
PARALLEL UNSAFE
SECURITY DEFINER
@ -41,7 +41,7 @@ $$
query = "select @extschema@._CDB_Group_API_Request('DELETE', '%s', '', '{204, 404}') as response_status" % url
plpy.execute(query)
$$ LANGUAGE 'plpythonu'
$$ LANGUAGE '@@plpythonu@@'
VOLATILE
PARALLEL UNSAFE
SECURITY DEFINER
@ -61,7 +61,7 @@ $$
body = '{ "name": "%s", "database_role": "%s" }' % (new_group_name, new_group_role)
query = "select @extschema@._CDB_Group_API_Request('PUT', '%s', '%s', '{200, 409}') as response_status" % (url, body)
plpy.execute(query)
$$ LANGUAGE 'plpythonu'
$$ LANGUAGE '@@plpythonu@@'
VOLATILE
PARALLEL UNSAFE
SECURITY DEFINER
@ -81,7 +81,7 @@ $$
body = "{ \"users\": [\"%s\"] }" % "\",\"".join(usernames)
query = "select @extschema@._CDB_Group_API_Request('POST', '%s', '%s', '{200, 409}') as response_status" % (url, body)
plpy.execute(query)
$$ LANGUAGE 'plpythonu'
$$ LANGUAGE '@@plpythonu@@'
VOLATILE
PARALLEL UNSAFE
SECURITY DEFINER
@ -101,7 +101,7 @@ $$
body = "{ \"users\": [\"%s\"] }" % "\",\"".join(usernames)
query = "select @extschema@._CDB_Group_API_Request('DELETE', '%s', '%s', '{200, 404}') as response_status" % (url, body)
plpy.execute(query)
$$ LANGUAGE 'plpythonu'
$$ LANGUAGE '@@plpythonu@@'
VOLATILE
PARALLEL UNSAFE
SECURITY DEFINER
@ -129,7 +129,7 @@ $$
body = '{ "access": "%s" }' % access
query = "select @extschema@._CDB_Group_API_Request('PUT', '%s', '%s', '{200, 409}') as response_status" % (url, body)
plpy.execute(query)
$$ LANGUAGE 'plpythonu'
$$ LANGUAGE '@@plpythonu@@'
VOLATILE
PARALLEL UNSAFE
SECURITY DEFINER
@ -156,7 +156,7 @@ $$
url = '/api/v1/databases/{0}/groups/%s/permission/%s/tables/%s' % (pathname2url(group_name), username, table_name)
query = "select @extschema@._CDB_Group_API_Request('DELETE', '%s', '', '{200, 404}') as response_status" % url
plpy.execute(query)
$$ LANGUAGE 'plpythonu'
$$ LANGUAGE '@@plpythonu@@'
VOLATILE
PARALLEL UNSAFE
SECURITY DEFINER
@ -191,7 +191,7 @@ $$
params = json.loads(conf)
auth = 'Basic %s' % plpy.execute("SELECT @extschema@._CDB_Group_API_Auth('%s', '%s') as auth" % (params['username'], params['password']))[0]['auth']
return { "host": params['host'], "port": params['port'], 'timeout': params['timeout'], 'auth': auth }
$$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE;
$$ LANGUAGE '@@plpythonu@@' VOLATILE PARALLEL UNSAFE;
CREATE OR REPLACE
FUNCTION @extschema@._CDB_Group_API_Auth(username text, password text)
@ -199,7 +199,7 @@ FUNCTION @extschema@._CDB_Group_API_Auth(username text, password text)
$$
import base64
return base64.encodestring('%s:%s' % (username, password)).replace('\n', '')
$$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE;
$$ LANGUAGE '@@plpythonu@@' VOLATILE PARALLEL UNSAFE;
-- url must contain a '%s' placeholder that will be replaced by current_database, for security reasons.
CREATE OR REPLACE
@ -239,5 +239,5 @@ $$
raise last_err
return None
$$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE;
$$ LANGUAGE '@@plpythonu@@' VOLATILE PARALLEL UNSAFE;
revoke all on function @extschema@._CDB_Group_API_Request(text, text, text, int[]) from public;

2
scripts-available/CDB_QueryStatements.sql
View File

@ -11,4 +11,4 @@ RETURNS SETOF TEXT AS $$
cleaned = match[0].strip()
if ( cleaned ):
yield cleaned
$$ language 'plpythonu' IMMUTABLE STRICT PARALLEL SAFE;
$$ language '@@plpythonu@@' IMMUTABLE STRICT PARALLEL SAFE;

50
test/CDB_GhostTables.sql
View File

@ -1,6 +1,56 @@
-- Create user and enable Ghost tables trigger
\set QUIET on
SET client_min_messages TO error;
-- Recreate the function without extra error messages as it changes depending on the python-redis version
CREATE OR REPLACE FUNCTION cartodb._CDB_LinkGhostTables(username text, db_name text, event_name text)
RETURNS void
AS $$
if not username:
return
if 'json' not in GD:
import json
GD['json'] = json
else:
json = GD['json']
tis_config = plpy.execute("select cartodb.CDB_Conf_GetConf('invalidation_service');")[0]['cdb_conf_getconf']
if not tis_config:
plpy.warning('Invalidation service configuration not found. Skipping Ghost Tables linking.')
return
tis_config_dict = json.loads(tis_config)
tis_host = tis_config_dict.get('host')
tis_port = tis_config_dict.get('port')
tis_timeout = tis_config_dict.get('timeout', 5)
tis_retry = tis_config_dict.get('retry', 5)
client = GD.get('invalidation', None)
while True:
if not client:
try:
import redis
client = redis.Redis(host=tis_host, port=tis_port, socket_timeout=tis_timeout)
GD['invalidation'] = client
except Exception as err:
# NOTE: no retries on connection error
plpy.warning('Error trying to connect to Invalidation Service to link Ghost Tables')
break
try:
client.execute_command('DBSCH', db_name, username, event_name)
break
except Exception as err:
client = GD['invalidation'] = None # force reconnect
if not tis_retry:
plpy.warning('Error calling Invalidation Service to link Ghost Tables')
break
tis_retry -= 1 # try reconnecting
$$ LANGUAGE '@@plpythonu@@' VOLATILE PARALLEL UNSAFE;
SELECT CDB_EnableGhostTablesTrigger();
CREATE ROLE "fulano" LOGIN;
GRANT ALL ON SCHEMA cartodb TO "fulano";

10
test/CDB_GhostTables_expect
View File

@ -1,19 +1,19 @@
WARNING: Invalidation service configuration not found. Skipping Ghost Tables linking.
NOTICE: _CDB_LinkGhostTables() called with username=fulanito, event_name=USER
INFO: _CDB_LinkGhostTables() called with username=fulanito, event_name=USER
WARNING: Error calling Invalidation Service to link Ghost Tables: Error -2 connecting to fake-tis-host:3142. Name or service not known.
NOTICE: _CDB_LinkGhostTables() called with username=fulanito, event_name=USER
WARNING: Error calling Invalidation Service to link Ghost Tables
INFO: _CDB_LinkGhostTables() called with username=fulanito, event_name=USER
BEGIN
cdb_ddl_execution
0
CREATE TABLE
1
WARNING: Error calling Invalidation Service to link Ghost Tables: Error -2 connecting to fake-tis-host:3142. Name or service not known.
NOTICE: _CDB_LinkGhostTables() called with username=fulanito, event_name=CREATE TABLE
WARNING: Error calling Invalidation Service to link Ghost Tables
INFO: _CDB_LinkGhostTables() called with username=fulanito, event_name=CREATE TABLE
COMMIT