diff --git a/.travis.yml b/.travis.yml index f934592..28e2200 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,3 @@ -dist: xenial language: c sudo: required @@ -12,10 +11,15 @@ env: jobs: include: - env: POSTGRESQL_VERSION="9.6" POSTGIS_VERSION="2.5" + dist: xenial - env: POSTGRESQL_VERSION="10" POSTGIS_VERSION="2.5" + dist: xenial - env: POSTGRESQL_VERSION="11" POSTGIS_VERSION="2.5" + dist: xenial - env: POSTGRESQL_VERSION="12" POSTGIS_VERSION="2.5" + dist: bionic - env: POSTGRESQL_VERSION="12" POSTGIS_VERSION="3" + dist: bionic script: - sudo service postgresql stop; diff --git a/Makefile b/Makefile index d25f86c..1d53d14 100644 --- a/Makefile +++ b/Makefile @@ -150,7 +150,7 @@ $(EXTENSION)--$(EXTVERSION).sql: $(CDBSCRIPTS) cartodb_version.sql Makefile cat $(CDBSCRIPTS) | \ $(SED) -e 's/@extschema@/cartodb/g' \ -e 's/@postgisschema@/public/g' \ - -e 's/plpythonu/$(PLPYTHONU)/g' >> $@ + -e 's/@@plpythonu@@/$(PLPYTHONU)/g' >> $@ echo "GRANT USAGE ON SCHEMA cartodb TO public;" >> $@ cat cartodb_version.sql >> $@ @@ -164,10 +164,10 @@ $(EXTENSION)--$(EXTVERSION)--$(EXTVERSION)next.sql: $(EXTENSION)--$(EXTVERSION). cp $< $@ $(EXTENSION).control: $(EXTENSION).control.in Makefile - $(SED) -e 's/@@VERSION@@/$(EXTVERSION)/g' -e 's/plpythonu/$(PLPYTHONU)/g' $< > $@ + $(SED) -e 's/@@VERSION@@/$(EXTVERSION)/g' -e 's/@@plpythonu@@/$(PLPYTHONU)/g' $< > $@ cartodb_version.sql: cartodb_version.sql.in Makefile $(GITDIR)/index - $(SED) -e 's/@@VERSION@@/$(EXTVERSION)/' -e 's/@extschema@/cartodb/g' -e "s/@postgisschema@/public/g" -e 's/plpythonu/$(PLPYTHONU)/g' $< > $@ + $(SED) -e 's/@@VERSION@@/$(EXTVERSION)/' -e 's/@extschema@/cartodb/g' -e "s/@postgisschema@/public/g" -e 's/@@plpythonu@@/$(PLPYTHONU)/g' $< > $@ # Needed for consistent `echo` results with backslashes SHELL = bash diff --git a/cartodb.control.in b/cartodb.control.in index a5dcfa0..d166ac1 100644 --- a/cartodb.control.in +++ b/cartodb.control.in @@ -3,4 +3,4 @@ comment = 'Turn a database into a cartodb user database.' superuser = true relocatable = false schema = cartodb -requires = 'plpythonu, postgis' +requires = '@@plpythonu@@, postgis' diff --git a/scripts-available/CDB_FederatedServerDiagnostics.sql b/scripts-available/CDB_FederatedServerDiagnostics.sql index b1e2e4c..fd55af8 100644 --- a/scripts-available/CDB_FederatedServerDiagnostics.sql +++ b/scripts-available/CDB_FederatedServerDiagnostics.sql @@ -191,7 +191,7 @@ AS $$ }) return json.dumps(stats) $$ -LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; +LANGUAGE @@plpythonu@@ VOLATILE PARALLEL UNSAFE; -- diff --git a/scripts-available/CDB_GhostTables.sql b/scripts-available/CDB_GhostTables.sql index 0b72087..bb45785 100644 --- a/scripts-available/CDB_GhostTables.sql +++ b/scripts-available/CDB_GhostTables.sql @@ -32,7 +32,6 @@ AS $$ client = redis.Redis(host=tis_host, port=tis_port, socket_timeout=tis_timeout) GD['invalidation'] = client except Exception as err: - error = "client_error - %s" % str(err) # NOTE: no retries on connection error plpy.warning('Error trying to connect to Invalidation Service to link Ghost Tables: ' + str(err)) break @@ -41,13 +40,12 @@ AS $$ client.execute_command('DBSCH', db_name, username, event_name) break except Exception as err: - error = "request_error - %s" % str(err) client = GD['invalidation'] = None # force reconnect if not tis_retry: plpy.warning('Error calling Invalidation Service to link Ghost Tables: ' + str(err)) break tis_retry -= 1 # try reconnecting -$$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE '@@plpythonu@@' VOLATILE PARALLEL UNSAFE; -- Enqueues a job to run Ghost tables linking process for the current user CREATE OR REPLACE FUNCTION @extschema@.CDB_LinkGhostTables(event_name text DEFAULT 'USER') @@ -61,7 +59,7 @@ AS $$ EXECUTE 'SELECT current_database();' INTO db_name; PERFORM @extschema@._CDB_LinkGhostTables(username, db_name, event_name); - RAISE NOTICE '_CDB_LinkGhostTables() called with username=%, event_name=%', username, event_name; + RAISE INFO '_CDB_LinkGhostTables() called with username=%, event_name=%', username, event_name; END; $$ LANGUAGE plpgsql VOLATILE diff --git a/scripts-available/CDB_Groups_API.sql b/scripts-available/CDB_Groups_API.sql index b29eb77..d4d9669 100644 --- a/scripts-available/CDB_Groups_API.sql +++ b/scripts-available/CDB_Groups_API.sql @@ -21,7 +21,7 @@ $$ body = '{ "name": "%s", "database_role": "%s" }' % (group_name, group_role) query = "select @extschema@._CDB_Group_API_Request('POST', '%s', '%s', '{200, 409}') as response_status" % (url, body) plpy.execute(query) -$$ LANGUAGE 'plpythonu' +$$ LANGUAGE '@@plpythonu@@' VOLATILE PARALLEL UNSAFE SECURITY DEFINER @@ -41,7 +41,7 @@ $$ query = "select @extschema@._CDB_Group_API_Request('DELETE', '%s', '', '{204, 404}') as response_status" % url plpy.execute(query) -$$ LANGUAGE 'plpythonu' +$$ LANGUAGE '@@plpythonu@@' VOLATILE PARALLEL UNSAFE SECURITY DEFINER @@ -61,7 +61,7 @@ $$ body = '{ "name": "%s", "database_role": "%s" }' % (new_group_name, new_group_role) query = "select @extschema@._CDB_Group_API_Request('PUT', '%s', '%s', '{200, 409}') as response_status" % (url, body) plpy.execute(query) -$$ LANGUAGE 'plpythonu' +$$ LANGUAGE '@@plpythonu@@' VOLATILE PARALLEL UNSAFE SECURITY DEFINER @@ -81,7 +81,7 @@ $$ body = "{ \"users\": [\"%s\"] }" % "\",\"".join(usernames) query = "select @extschema@._CDB_Group_API_Request('POST', '%s', '%s', '{200, 409}') as response_status" % (url, body) plpy.execute(query) -$$ LANGUAGE 'plpythonu' +$$ LANGUAGE '@@plpythonu@@' VOLATILE PARALLEL UNSAFE SECURITY DEFINER @@ -101,7 +101,7 @@ $$ body = "{ \"users\": [\"%s\"] }" % "\",\"".join(usernames) query = "select @extschema@._CDB_Group_API_Request('DELETE', '%s', '%s', '{200, 404}') as response_status" % (url, body) plpy.execute(query) -$$ LANGUAGE 'plpythonu' +$$ LANGUAGE '@@plpythonu@@' VOLATILE PARALLEL UNSAFE SECURITY DEFINER @@ -129,7 +129,7 @@ $$ body = '{ "access": "%s" }' % access query = "select @extschema@._CDB_Group_API_Request('PUT', '%s', '%s', '{200, 409}') as response_status" % (url, body) plpy.execute(query) -$$ LANGUAGE 'plpythonu' +$$ LANGUAGE '@@plpythonu@@' VOLATILE PARALLEL UNSAFE SECURITY DEFINER @@ -156,7 +156,7 @@ $$ url = '/api/v1/databases/{0}/groups/%s/permission/%s/tables/%s' % (pathname2url(group_name), username, table_name) query = "select @extschema@._CDB_Group_API_Request('DELETE', '%s', '', '{200, 404}') as response_status" % url plpy.execute(query) -$$ LANGUAGE 'plpythonu' +$$ LANGUAGE '@@plpythonu@@' VOLATILE PARALLEL UNSAFE SECURITY DEFINER @@ -191,7 +191,7 @@ $$ params = json.loads(conf) auth = 'Basic %s' % plpy.execute("SELECT @extschema@._CDB_Group_API_Auth('%s', '%s') as auth" % (params['username'], params['password']))[0]['auth'] return { "host": params['host'], "port": params['port'], 'timeout': params['timeout'], 'auth': auth } -$$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE '@@plpythonu@@' VOLATILE PARALLEL UNSAFE; CREATE OR REPLACE FUNCTION @extschema@._CDB_Group_API_Auth(username text, password text) @@ -199,7 +199,7 @@ FUNCTION @extschema@._CDB_Group_API_Auth(username text, password text) $$ import base64 return base64.encodestring('%s:%s' % (username, password)).replace('\n', '') -$$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE '@@plpythonu@@' VOLATILE PARALLEL UNSAFE; -- url must contain a '%s' placeholder that will be replaced by current_database, for security reasons. CREATE OR REPLACE @@ -239,5 +239,5 @@ $$ raise last_err return None -$$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE '@@plpythonu@@' VOLATILE PARALLEL UNSAFE; revoke all on function @extschema@._CDB_Group_API_Request(text, text, text, int[]) from public; diff --git a/scripts-available/CDB_QueryStatements.sql b/scripts-available/CDB_QueryStatements.sql index 54b7115..ff60d35 100644 --- a/scripts-available/CDB_QueryStatements.sql +++ b/scripts-available/CDB_QueryStatements.sql @@ -11,4 +11,4 @@ RETURNS SETOF TEXT AS $$ cleaned = match[0].strip() if ( cleaned ): yield cleaned -$$ language 'plpythonu' IMMUTABLE STRICT PARALLEL SAFE; +$$ language '@@plpythonu@@' IMMUTABLE STRICT PARALLEL SAFE; diff --git a/test/CDB_GhostTables.sql b/test/CDB_GhostTables.sql index 1741f01..3d3ac93 100644 --- a/test/CDB_GhostTables.sql +++ b/test/CDB_GhostTables.sql @@ -1,6 +1,56 @@ -- Create user and enable Ghost tables trigger \set QUIET on SET client_min_messages TO error; + +-- Recreate the function without extra error messages as it changes depending on the python-redis version +CREATE OR REPLACE FUNCTION cartodb._CDB_LinkGhostTables(username text, db_name text, event_name text) +RETURNS void +AS $$ + if not username: + return + + if 'json' not in GD: + import json + GD['json'] = json + else: + json = GD['json'] + + tis_config = plpy.execute("select cartodb.CDB_Conf_GetConf('invalidation_service');")[0]['cdb_conf_getconf'] + if not tis_config: + plpy.warning('Invalidation service configuration not found. Skipping Ghost Tables linking.') + return + + tis_config_dict = json.loads(tis_config) + tis_host = tis_config_dict.get('host') + tis_port = tis_config_dict.get('port') + tis_timeout = tis_config_dict.get('timeout', 5) + tis_retry = tis_config_dict.get('retry', 5) + + client = GD.get('invalidation', None) + + while True: + + if not client: + try: + import redis + client = redis.Redis(host=tis_host, port=tis_port, socket_timeout=tis_timeout) + GD['invalidation'] = client + except Exception as err: + # NOTE: no retries on connection error + plpy.warning('Error trying to connect to Invalidation Service to link Ghost Tables') + break + + try: + client.execute_command('DBSCH', db_name, username, event_name) + break + except Exception as err: + client = GD['invalidation'] = None # force reconnect + if not tis_retry: + plpy.warning('Error calling Invalidation Service to link Ghost Tables') + break + tis_retry -= 1 # try reconnecting +$$ LANGUAGE '@@plpythonu@@' VOLATILE PARALLEL UNSAFE; + SELECT CDB_EnableGhostTablesTrigger(); CREATE ROLE "fulano" LOGIN; GRANT ALL ON SCHEMA cartodb TO "fulano"; diff --git a/test/CDB_GhostTables_expect b/test/CDB_GhostTables_expect index 469edeb..a5d2aff 100644 --- a/test/CDB_GhostTables_expect +++ b/test/CDB_GhostTables_expect @@ -1,19 +1,19 @@ WARNING: Invalidation service configuration not found. Skipping Ghost Tables linking. -NOTICE: _CDB_LinkGhostTables() called with username=fulanito, event_name=USER +INFO: _CDB_LinkGhostTables() called with username=fulanito, event_name=USER -WARNING: Error calling Invalidation Service to link Ghost Tables: Error -2 connecting to fake-tis-host:3142. Name or service not known. -NOTICE: _CDB_LinkGhostTables() called with username=fulanito, event_name=USER +WARNING: Error calling Invalidation Service to link Ghost Tables +INFO: _CDB_LinkGhostTables() called with username=fulanito, event_name=USER BEGIN cdb_ddl_execution 0 CREATE TABLE 1 -WARNING: Error calling Invalidation Service to link Ghost Tables: Error -2 connecting to fake-tis-host:3142. Name or service not known. -NOTICE: _CDB_LinkGhostTables() called with username=fulanito, event_name=CREATE TABLE +WARNING: Error calling Invalidation Service to link Ghost Tables +INFO: _CDB_LinkGhostTables() called with username=fulanito, event_name=CREATE TABLE COMMIT