Merge pull request #392 from Algunenano/test_bionic

Test bionic
5 years ago · 741d171f4a
parent 4c15fafd72 093e0bcb36
commit 741d171f4a
9 changed files with 78 additions and 26 deletions
--- a/.travis.yml
+++ b/.travis.yml
@ -1,4 +1,3 @@
-dist: xenial
 language: c
 sudo: required

@ -12,10 +11,15 @@ env:
 jobs:
  include:
    - env: POSTGRESQL_VERSION="9.6" POSTGIS_VERSION="2.5"
+      dist: xenial
    - env: POSTGRESQL_VERSION="10" POSTGIS_VERSION="2.5"
+      dist: xenial
    - env: POSTGRESQL_VERSION="11" POSTGIS_VERSION="2.5"
+      dist: xenial
    - env: POSTGRESQL_VERSION="12" POSTGIS_VERSION="2.5"
+      dist: bionic
    - env: POSTGRESQL_VERSION="12" POSTGIS_VERSION="3"
+      dist: bionic

 script:
  - sudo service postgresql stop;
--- a/6
+++ b/6
@ -150,7 +150,7 @@ $(EXTENSION)--$(EXTVERSION).sql: $(CDBSCRIPTS) cartodb_version.sql Makefile
 	cat $(CDBSCRIPTS) | \
 	$(SED) 	-e 's/@extschema@/cartodb/g' \
 		-e 's/@postgisschema@/public/g' \
-		-e 's/plpythonu/$(PLPYTHONU)/g' >> $@
+		-e 's/@@plpythonu@@/$(PLPYTHONU)/g' >> $@
 	echo "GRANT USAGE ON SCHEMA cartodb TO public;" >> $@
 	cat cartodb_version.sql >> $@

@ -164,10 +164,10 @@ $(EXTENSION)--$(EXTVERSION)--$(EXTVERSION)next.sql: $(EXTENSION)--$(EXTVERSION).
 	cp $< $@

 $(EXTENSION).control: $(EXTENSION).control.in Makefile
-	$(SED) -e 's/@@VERSION@@/$(EXTVERSION)/g' -e 's/plpythonu/$(PLPYTHONU)/g' $< > $@
+	$(SED) -e 's/@@VERSION@@/$(EXTVERSION)/g' -e 's/@@plpythonu@@/$(PLPYTHONU)/g' $< > $@

 cartodb_version.sql: cartodb_version.sql.in Makefile $(GITDIR)/index
-	$(SED) -e 's/@@VERSION@@/$(EXTVERSION)/' -e 's/@extschema@/cartodb/g' -e "s/@postgisschema@/public/g" -e 's/plpythonu/$(PLPYTHONU)/g' $< > $@
+	$(SED) -e 's/@@VERSION@@/$(EXTVERSION)/' -e 's/@extschema@/cartodb/g' -e "s/@postgisschema@/public/g" -e 's/@@plpythonu@@/$(PLPYTHONU)/g' $< > $@

 # Needed for consistent `echo` results with backslashes
 SHELL = bash
--- a/cartodb.control.in
+++ b/cartodb.control.in
@ -3,4 +3,4 @@ comment = 'Turn a database into a cartodb user database.'
 superuser = true
 relocatable = false
 schema = cartodb
-requires = 'plpythonu, postgis'
+requires = '@@plpythonu@@, postgis'
--- a/scripts-available/CDB_FederatedServerDiagnostics.sql
+++ b/scripts-available/CDB_FederatedServerDiagnostics.sql
@ -191,7 +191,7 @@ AS $$
        })
    return json.dumps(stats)
 $$
-LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
+LANGUAGE @@plpythonu@@ VOLATILE PARALLEL UNSAFE;


 --
--- a/scripts-available/CDB_GhostTables.sql
+++ b/scripts-available/CDB_GhostTables.sql
@ -32,7 +32,6 @@ AS $$
          client = redis.Redis(host=tis_host, port=tis_port, socket_timeout=tis_timeout)
          GD['invalidation'] = client
        except Exception as err:
-          error = "client_error - %s" % str(err)
          # NOTE: no retries on connection error
          plpy.warning('Error trying to connect to Invalidation Service to link Ghost Tables: ' +  str(err))
          break
@ -41,13 +40,12 @@ AS $$
      client.execute_command('DBSCH', db_name, username, event_name)
      break
    except Exception as err:
-      error = "request_error - %s" % str(err)
      client = GD['invalidation'] = None # force reconnect
      if not tis_retry:
        plpy.warning('Error calling Invalidation Service to link Ghost Tables: ' +  str(err))
        break
      tis_retry -= 1 # try reconnecting
-$$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE;
+$$ LANGUAGE '@@plpythonu@@' VOLATILE PARALLEL UNSAFE;

 -- Enqueues a job to run Ghost tables linking process for the current user
 CREATE OR REPLACE FUNCTION @extschema@.CDB_LinkGhostTables(event_name text DEFAULT 'USER')
@ -61,7 +59,7 @@ AS $$
    EXECUTE 'SELECT current_database();' INTO db_name;

    PERFORM @extschema@._CDB_LinkGhostTables(username, db_name, event_name);
-    RAISE NOTICE '_CDB_LinkGhostTables() called with username=%, event_name=%', username, event_name;
+    RAISE INFO '_CDB_LinkGhostTables() called with username=%, event_name=%', username, event_name;
  END;
 $$  LANGUAGE plpgsql
    VOLATILE
--- a/scripts-available/CDB_Groups_API.sql
+++ b/scripts-available/CDB_Groups_API.sql
@ -21,7 +21,7 @@ $$
    body = '{ "name": "%s", "database_role": "%s" }' % (group_name, group_role)
    query = "select @extschema@._CDB_Group_API_Request('POST', '%s', '%s', '{200, 409}') as response_status" % (url, body)
    plpy.execute(query)
-$$  LANGUAGE 'plpythonu'
+$$  LANGUAGE '@@plpythonu@@'
    VOLATILE
    PARALLEL UNSAFE
    SECURITY DEFINER
@ -41,7 +41,7 @@ $$

    query = "select @extschema@._CDB_Group_API_Request('DELETE', '%s', '', '{204, 404}') as response_status" % url
    plpy.execute(query)
-$$  LANGUAGE 'plpythonu'
+$$  LANGUAGE '@@plpythonu@@'
    VOLATILE
    PARALLEL UNSAFE
    SECURITY DEFINER
@ -61,7 +61,7 @@ $$
    body = '{ "name": "%s", "database_role": "%s" }' % (new_group_name, new_group_role)
    query = "select @extschema@._CDB_Group_API_Request('PUT', '%s', '%s', '{200, 409}') as response_status" % (url, body)
    plpy.execute(query)
-$$  LANGUAGE 'plpythonu'
+$$  LANGUAGE '@@plpythonu@@'
    VOLATILE
    PARALLEL UNSAFE
    SECURITY DEFINER
@ -81,7 +81,7 @@ $$
    body = "{ \"users\": [\"%s\"] }" % "\",\"".join(usernames)
    query = "select @extschema@._CDB_Group_API_Request('POST', '%s', '%s', '{200, 409}') as response_status" % (url, body)
    plpy.execute(query)
-$$  LANGUAGE 'plpythonu'
+$$  LANGUAGE '@@plpythonu@@'
    VOLATILE
    PARALLEL UNSAFE
    SECURITY DEFINER
@ -101,7 +101,7 @@ $$
    body = "{ \"users\": [\"%s\"] }" % "\",\"".join(usernames)
    query = "select @extschema@._CDB_Group_API_Request('DELETE', '%s', '%s', '{200, 404}') as response_status" % (url, body)
    plpy.execute(query)
-$$  LANGUAGE 'plpythonu'
+$$  LANGUAGE '@@plpythonu@@'
    VOLATILE
    PARALLEL UNSAFE
    SECURITY DEFINER
@ -129,7 +129,7 @@ $$
    body = '{ "access": "%s" }' % access
    query = "select @extschema@._CDB_Group_API_Request('PUT', '%s', '%s', '{200, 409}') as response_status" % (url, body)
    plpy.execute(query)
-$$  LANGUAGE 'plpythonu'
+$$  LANGUAGE '@@plpythonu@@'
    VOLATILE
    PARALLEL UNSAFE
    SECURITY DEFINER
@ -156,7 +156,7 @@ $$
    url = '/api/v1/databases/{0}/groups/%s/permission/%s/tables/%s' % (pathname2url(group_name), username, table_name)
    query = "select @extschema@._CDB_Group_API_Request('DELETE', '%s', '', '{200, 404}') as response_status" % url
    plpy.execute(query)
-$$  LANGUAGE 'plpythonu'
+$$  LANGUAGE '@@plpythonu@@'
    VOLATILE
    PARALLEL UNSAFE
    SECURITY DEFINER
@ -191,7 +191,7 @@ $$
      params = json.loads(conf)
      auth = 'Basic %s' % plpy.execute("SELECT @extschema@._CDB_Group_API_Auth('%s', '%s') as auth" % (params['username'], params['password']))[0]['auth']
      return { "host": params['host'], "port": params['port'], 'timeout': params['timeout'], 'auth': auth }
-$$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE;
+$$ LANGUAGE '@@plpythonu@@' VOLATILE PARALLEL UNSAFE;

 CREATE OR REPLACE
 FUNCTION @extschema@._CDB_Group_API_Auth(username text, password text)
@ -199,7 +199,7 @@ FUNCTION @extschema@._CDB_Group_API_Auth(username text, password text)
 $$
    import base64
    return base64.encodestring('%s:%s' % (username, password)).replace('\n', '')
-$$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE;
+$$ LANGUAGE '@@plpythonu@@' VOLATILE PARALLEL UNSAFE;

 -- url must contain a '%s' placeholder that will be replaced by current_database, for security reasons.
 CREATE OR REPLACE
@ -239,5 +239,5 @@ $$
      raise last_err

    return None
-$$ LANGUAGE 'plpythonu' VOLATILE PARALLEL UNSAFE;
+$$ LANGUAGE '@@plpythonu@@' VOLATILE PARALLEL UNSAFE;
 revoke all on function @extschema@._CDB_Group_API_Request(text, text, text, int[]) from public;
--- a/scripts-available/CDB_QueryStatements.sql
+++ b/scripts-available/CDB_QueryStatements.sql
@ -11,4 +11,4 @@ RETURNS SETOF TEXT AS $$
    cleaned = match[0].strip()
    if ( cleaned ):
      yield cleaned
-$$ language 'plpythonu' IMMUTABLE STRICT PARALLEL SAFE;
+$$ language '@@plpythonu@@' IMMUTABLE STRICT PARALLEL SAFE;
--- a/test/CDB_GhostTables.sql
+++ b/test/CDB_GhostTables.sql
@ -1,6 +1,56 @@
 -- Create user and enable Ghost tables trigger
 \set QUIET on
 SET client_min_messages TO error;
+
+-- Recreate the function without extra error messages as it changes depending on the python-redis version
+CREATE OR REPLACE FUNCTION cartodb._CDB_LinkGhostTables(username text, db_name text, event_name text)
+RETURNS void
+AS $$
+  if not username:
+    return
+
+  if 'json' not in GD:
+    import json
+    GD['json'] = json
+  else:
+    json = GD['json']
+
+  tis_config = plpy.execute("select cartodb.CDB_Conf_GetConf('invalidation_service');")[0]['cdb_conf_getconf']
+  if not tis_config:
+    plpy.warning('Invalidation service configuration not found. Skipping Ghost Tables linking.')
+    return
+
+  tis_config_dict = json.loads(tis_config)
+  tis_host = tis_config_dict.get('host')
+  tis_port = tis_config_dict.get('port')
+  tis_timeout = tis_config_dict.get('timeout', 5)
+  tis_retry = tis_config_dict.get('retry', 5)
+
+  client = GD.get('invalidation', None)
+
+  while True:
+
+    if not client:
+        try:
+          import redis
+          client = redis.Redis(host=tis_host, port=tis_port, socket_timeout=tis_timeout)
+          GD['invalidation'] = client
+        except Exception as err:
+          # NOTE: no retries on connection error
+          plpy.warning('Error trying to connect to Invalidation Service to link Ghost Tables')
+          break
+
+    try:
+      client.execute_command('DBSCH', db_name, username, event_name)
+      break
+    except Exception as err:
+      client = GD['invalidation'] = None # force reconnect
+      if not tis_retry:
+        plpy.warning('Error calling Invalidation Service to link Ghost Tables')
+        break
+      tis_retry -= 1 # try reconnecting
+$$ LANGUAGE '@@plpythonu@@' VOLATILE PARALLEL UNSAFE;
+
 SELECT CDB_EnableGhostTablesTrigger();
 CREATE ROLE "fulano" LOGIN;
 GRANT ALL ON SCHEMA cartodb TO "fulano";
--- a/test/CDB_GhostTables_expect
+++ b/test/CDB_GhostTables_expect
@ -1,19 +1,19 @@


 WARNING:  Invalidation service configuration not found. Skipping Ghost Tables linking.
-NOTICE:  _CDB_LinkGhostTables() called with username=fulanito, event_name=USER
+INFO:  _CDB_LinkGhostTables() called with username=fulanito, event_name=USER


-WARNING:  Error calling Invalidation Service to link Ghost Tables: Error -2 connecting to fake-tis-host:3142. Name or service not known.
-NOTICE:  _CDB_LinkGhostTables() called with username=fulanito, event_name=USER
+WARNING:  Error calling Invalidation Service to link Ghost Tables
+INFO:  _CDB_LinkGhostTables() called with username=fulanito, event_name=USER

 BEGIN
 cdb_ddl_execution
 0
 CREATE TABLE
 1
-WARNING:  Error calling Invalidation Service to link Ghost Tables: Error -2 connecting to fake-tis-host:3142. Name or service not known.
-NOTICE:  _CDB_LinkGhostTables() called with username=fulanito, event_name=CREATE TABLE
+WARNING:  Error calling Invalidation Service to link Ghost Tables
+INFO:  _CDB_LinkGhostTables() called with username=fulanito, event_name=CREATE TABLE
 COMMIT