Merge pull request #104 from CartoDB/103-Extension_Group_API

103 extension group api

.gitignore

@@ -6,3 +6,4 @@ regression.*
 expected/test
 sql/test
 .idea/*
+*.swp

Makefile

@@ -1,7 +1,7 @@
 # cartodb/Makefile
 
 EXTENSION = cartodb
-EXTVERSION = 0.10.2
+EXTVERSION = 0.11.0
 
 SED = sed
 
@@ -50,6 +50,7 @@ UPGRADABLE = \
   0.10.0 \
   0.10.1 \
   0.10.2 \
+  0.11.0 \
   $(EXTVERSION)dev \
   $(EXTVERSION)next \
   $(END)

NEWS.md

@@ -1,6 +1,10 @@
 next (2015-mm-dd)
 -----------------
 
+0.11.0 (2015-09-dd)
+-------------------
+* Groups API
+
 0.10.2 (2015-09-24)
 -------------------
 * Add back the `DROP FUNCTION IF EXISTS CDB_UserTables(text);` to be able to upgrade from `0.7.3` upward [#160](https://github.com/CartoDB/cartodb-postgresql/issues/160)

README.md

@@ -98,3 +98,8 @@ ls `pg_config --sharedir`/extension/cartodb*
 During development the cartodb extension version doesn't change with
 every commit, so testing latest change requires special steps documented
 in the CONTRIBUTING document, under "Testing changes live".
+
+Limitations
+-----------
+
+- The main schema of an organization user must have one only owner (the user).

scripts-available/CDB_Conf.sql

@@ -0,0 +1,48 @@
+----------------------------------
+-- CONF MANAGEMENT FUNCTIONS
+--
+-- Meant to be used by superadmin user.
+-- Functions needing reading configuration should use SECURITY DEFINER.
+----------------------------------
+
+-- This will trigger NOTICE if cartodb.CDB_CONF already exists
+DO LANGUAGE 'plpgsql' $$
+BEGIN
+    CREATE TABLE IF NOT EXISTS cartodb.CDB_CONF ( KEY TEXT PRIMARY KEY, VALUE JSON NOT NULL );
+END
+$$;
+
+-- This can only be called from an SQL script executed by CREATE EXTENSION
+DO LANGUAGE 'plpgsql' $$
+BEGIN
+    PERFORM pg_catalog.pg_extension_config_dump('cartodb.CDB_CONF', '');
+END
+$$;
+
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Conf_SetConf(key text, value JSON)
+    RETURNS void AS $$
+BEGIN
+    PERFORM cartodb.CDB_Conf_RemoveConf(key);
+    EXECUTE 'INSERT INTO cartodb.CDB_CONF (KEY, VALUE) VALUES ($1, $2);' USING key, value;
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Conf_RemoveConf(key text)
+    RETURNS void AS $$
+BEGIN
+    EXECUTE 'DELETE FROM cartodb.CDB_CONF WHERE KEY = $1;' USING key;
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Conf_GetConf(key text)
+    RETURNS JSON AS $$
+DECLARE
+    value JSON;
+BEGIN
+    EXECUTE 'SELECT VALUE FROM cartodb.CDB_CONF WHERE KEY = $1;' INTO value USING key;
+    RETURN value;
+END
+$$ LANGUAGE PLPGSQL STABLE;

scripts-available/CDB_Groups.sql

@@ -0,0 +1,252 @@
+----------------------------------
+-- GROUP MANAGEMENT FUNCTIONS
+--
+-- Meant to be used by org admin. See CDB_Organization_AddAdmin.
+----------------------------------
+
+-- Creates a new group
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Group_CreateGroup(group_name text)
+    RETURNS VOID AS $$
+DECLARE
+    group_role TEXT;
+BEGIN
+    group_role := cartodb._CDB_Group_GroupRole(group_name);
+    EXECUTE format('CREATE ROLE %I NOLOGIN;', group_role);
+    PERFORM cartodb._CDB_Group_CreateGroup_API(group_name, group_role);
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+-- Drops group and everything that role owns
+-- TODO: LIMITATION: in order to drop a role all its owned objects must be dropped before.
+-- Right now this is done with DROP OWNED, which can only be done by a superadmin.
+-- Not even the role creator can drop the role and the objects it owns.
+-- All group owned objects by the group are permissions.
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Group_DropGroup(group_name text)
+    RETURNS VOID AS $$
+DECLARE
+    group_role TEXT;
+BEGIN
+    group_role := cartodb._CDB_Group_GroupRole(group_name);
+    EXECUTE format('DROP OWNED BY %I', group_role);
+    EXECUTE format('DROP ROLE IF EXISTS %I', group_role);
+    PERFORM cartodb._CDB_Group_DropGroup_API(group_name);
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+-- Renames a group
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Group_RenameGroup(old_group_name text, new_group_name text)
+    RETURNS VOID AS $$
+DECLARE
+    old_group_role TEXT;
+    new_group_role TEXT;
+BEGIN
+    old_group_role = cartodb._CDB_Group_GroupRole(old_group_name);
+    new_group_role = cartodb._CDB_Group_GroupRole(new_group_name);
+    EXECUTE format('ALTER ROLE %I RENAME TO %I', old_group_role, new_group_role);
+    PERFORM cartodb._CDB_Group_RenameGroup_API(old_group_name, new_group_name, new_group_role);
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+-- Adds users to a group
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Group_AddUsers(group_name text, usernames text[])
+    RETURNS VOID AS $$
+DECLARE
+    group_role TEXT;
+    user_role TEXT;
+    username TEXT;
+BEGIN
+    group_role := cartodb._CDB_Group_GroupRole(group_name);
+    foreach username in array usernames
+    loop
+      user_role := cartodb._CDB_User_RoleFromUsername(username);
+      IF(group_role IS NULL OR user_role IS NULL)
+      THEN
+        RAISE EXCEPTION 'Group role (%) and user role (%) must be already existing', group_role, user_role;
+      END IF;
+      EXECUTE format('GRANT %I TO %I', group_role, user_role);
+    end loop;
+    PERFORM cartodb._CDB_Group_AddUsers_API(group_name, usernames);
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+-- Removes users from a group
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Group_RemoveUsers(group_name text, usernames text[])
+    RETURNS VOID AS $$
+DECLARE
+    group_role TEXT;
+    user_role TEXT;
+    username TEXT;
+BEGIN
+    group_role := cartodb._CDB_Group_GroupRole(group_name);
+    foreach username in array usernames
+    loop
+      user_role := cartodb._CDB_User_RoleFromUsername(username);
+      EXECUTE format('REVOKE %I FROM %I', group_role, user_role);
+    end loop;
+    PERFORM cartodb._CDB_Group_RemoveUsers_API(group_name, usernames);
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+----------------------------------
+-- TABLE MANAGEMENT FUNCTIONS
+--
+-- Meant to be used by table owners.
+----------------------------------
+
+-- Grants table read permission to a group
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Group_Table_GrantRead(group_name text, username text, table_name text)
+    RETURNS VOID AS $$
+DECLARE
+    group_role TEXT;
+BEGIN
+    PERFORM cartodb._CDB_Group_Table_GrantRead(group_name, username, table_name, true);
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_Table_GrantRead(group_name text, username text, table_name text, sync boolean)
+    RETURNS VOID AS $$
+DECLARE
+    group_role TEXT;
+BEGIN
+    group_role := cartodb._CDB_Group_GroupRole(group_name);
+    EXECUTE format('GRANT USAGE ON SCHEMA %I TO %I', username, group_role);
+    EXECUTE format('GRANT SELECT ON TABLE %I.%I TO %I', username, table_name, group_role );
+    IF(sync) THEN
+      PERFORM cartodb._CDB_Group_Table_GrantPermission_API(group_name, username, table_name, 'r');
+    END IF;
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+-- Grants table write permission to a group
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Group_Table_GrantReadWrite(group_name text, username text, table_name text)
+    RETURNS VOID AS $$
+DECLARE
+    group_role TEXT;
+BEGIN
+    PERFORM cartodb._CDB_Group_Table_GrantReadWrite(group_name, username, table_name, true);
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_Table_GrantReadWrite(group_name text, username text, table_name text, sync boolean)
+    RETURNS VOID AS $$
+DECLARE
+    group_role TEXT;
+BEGIN
+    group_role := cartodb._CDB_Group_GroupRole(group_name);
+    EXECUTE format('GRANT USAGE ON SCHEMA %I TO %I', username, group_role);
+    EXECUTE format('GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE %I.%I TO %I', username, table_name, group_role);
+    PERFORM cartodb._CDB_Group_TableSequences_Permission(group_name, username, table_name, true);
+    IF(sync) THEN
+      PERFORM cartodb._CDB_Group_Table_GrantPermission_API(group_name, username, table_name, 'w');
+    END IF;
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+-- Granting and revoking permissions on sequences
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_TableSequences_Permission(group_name text, username text, table_name text, do_grant bool)
+    RETURNS VOID AS $$
+DECLARE
+    column_name TEXT;
+    sequence_name TEXT;
+    group_role TEXT;
+BEGIN
+    group_role := cartodb._CDB_Group_GroupRole(group_name);
+    FOR column_name IN EXECUTE 'SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_CATALOG = current_database() AND TABLE_SCHEMA = $1 AND TABLE_NAME = $2 AND COLUMN_DEFAULT LIKE ''nextval%''' USING username, table_name
+    LOOP
+        EXECUTE 'SELECT PG_GET_SERIAL_SEQUENCE($1, $2)' USING table_name, column_name INTO sequence_name;
+        IF sequence_name IS NOT NULL THEN
+          IF do_grant THEN
+            -- Here %s is needed since sequence_name has quotes
+            EXECUTE format('GRANT USAGE, SELECT, UPDATE ON SEQUENCE %s TO %I', sequence_name, group_role);
+          ELSE
+            EXECUTE format('REVOKE ALL ON SEQUENCE %s FROM %I', sequence_name, group_role);
+          END IF;
+        END IF;
+    END LOOP;
+    RETURN;
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+-- Revokes all permissions on a table from a group
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Group_Table_RevokeAll(group_name text, username text, table_name text)
+    RETURNS VOID AS $$
+DECLARE
+    group_role TEXT;
+BEGIN
+    PERFORM cartodb._CDB_Group_Table_RevokeAll(group_name, username, table_name, true);
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_Table_RevokeAll(group_name text, username text, table_name text, sync boolean)
+    RETURNS VOID AS $$
+DECLARE
+    group_role TEXT;
+BEGIN
+    group_role := cartodb._CDB_Group_GroupRole(group_name);
+    EXECUTE format('REVOKE ALL ON TABLE %I.%I FROM %I', username, table_name, group_role);
+    PERFORM cartodb._CDB_Group_TableSequences_Permission(group_name, username, table_name, false);
+    IF(sync) THEN
+      PERFORM cartodb._CDB_Group_Table_RevokeAllPermission_API(group_name, username, table_name);
+    END IF;
+END
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+-----------------------
+-- Helper functions
+-----------------------
+-- Given a group name returns a role. group_name must be a valid PostgreSQL idenfifier. See http://www.postgresql.org/docs/9.2/static/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_GroupRole(group_name text)
+    RETURNS TEXT AS $$
+DECLARE
+    group_role TEXT;
+    prefix TEXT;
+    max_length constant INTEGER := 63;
+BEGIN
+    prefix = format('%s_g_', cartodb._CDB_Group_ShortDatabaseName());
+    group_role := format('%s%s', prefix, group_name);
+    IF LENGTH(group_role) > max_length
+    THEN
+        RAISE EXCEPTION 'Group name must be shorter. It can''t have more than % characters, but it is longer (%): %', max_length - LENGTH(prefix), length(group_name), group_name;
+    END IF;
+    RETURN group_role;
+END
+$$ LANGUAGE PLPGSQL;
+
+-- Returns the first owner of the schema matching username. Organization user schemas must have one only owner.
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_User_RoleFromUsername(username text)
+    RETURNS TEXT AS $$
+DECLARE
+    user_role TEXT;
+BEGIN
+    -- This was preferred, but non-superadmins won't get results
+    -- SELECT SCHEMA_OWNER FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME = $1 LIMIT 1'
+    SELECT pg_get_userbyid(nspowner) FROM pg_namespace WHERE nspname = username INTO user_role;
+    RETURN user_role;
+END
+$$ LANGUAGE PLPGSQL;
+
+-- Database names are too long, we need a shorter version for composing role names
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_ShortDatabaseName()
+    RETURNS TEXT AS $$
+DECLARE
+    short_database_name TEXT;
+BEGIN
+    SELECT md5(current_database()) INTO short_database_name;
+    RETURN short_database_name;
+END
+$$ LANGUAGE PLPGSQL;

scripts-available/CDB_Groups_API.sql

@@ -0,0 +1,195 @@
+----------------------------------
+-- GROUP METADATA API FUNCTIONS
+--
+-- Meant to be used by CDB_Group_* functions to sync data with the editor.
+-- Requires configuration parameter. Example: SELECT cartodb.CDB_Conf_SetConf('groups_api', '{ "host": "127.0.0.1", "port": 3000, "timeout": 10, "username": "extension", "password": "elephant" }');
+----------------------------------
+
+-- TODO: delete this development cleanup before final merge
+DROP FUNCTION IF EXISTS cartodb.CDB_Group_AddMember(group_name text, username text);
+DROP FUNCTION IF EXISTS cartodb.CDB_Group_RemoveMember(group_name text, username text);
+DROP FUNCTION IF EXISTS cartodb._CDB_Group_AddMember_API(group_name text, username text);
+DROP FUNCTION IF EXISTS cartodb._CDB_Group_RemoveMember_API(group_name text, username text);
+
+-- Sends the create group request
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_CreateGroup_API(group_name text, group_role text)
+    RETURNS VOID AS
+$$
+    import string
+
+    url = '/api/v1/databases/{0}/groups'
+    body = '{ "name": "%s", "database_role": "%s" }' % (group_name, group_role)
+    query = "select cartodb._CDB_Group_API_Request('POST', '%s', '%s', '{200, 409}') as response_status" % (url, body)
+    plpy.execute(query)
+$$ LANGUAGE 'plpythonu' VOLATILE SECURITY DEFINER;
+
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_DropGroup_API(group_name text)
+    RETURNS VOID AS
+$$
+    import string
+    import urllib
+
+    url = '/api/v1/databases/{0}/groups/%s' % (urllib.pathname2url(group_name))
+
+    query = "select cartodb._CDB_Group_API_Request('DELETE', '%s', '', '{204, 404}') as response_status" % url
+    plpy.execute(query)
+$$ LANGUAGE 'plpythonu' VOLATILE SECURITY DEFINER;
+
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_RenameGroup_API(old_group_name text, new_group_name text, new_group_role text)
+    RETURNS VOID AS
+$$
+    import string
+    import urllib
+
+    url = '/api/v1/databases/{0}/groups/%s' % (urllib.pathname2url(old_group_name))
+    body = '{ "name": "%s", "database_role": "%s" }' % (new_group_name, new_group_role)
+    query = "select cartodb._CDB_Group_API_Request('PUT', '%s', '%s', '{200, 409}') as response_status" % (url, body)
+    plpy.execute(query)
+$$ LANGUAGE 'plpythonu' VOLATILE SECURITY DEFINER;
+
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_AddUsers_API(group_name text, usernames text[])
+    RETURNS VOID AS
+$$
+    import string
+    import urllib
+
+    url = '/api/v1/databases/{0}/groups/%s/users' % (urllib.pathname2url(group_name))
+    body = "{ \"users\": [\"%s\"] }" % "\",\"".join(usernames)
+    query = "select cartodb._CDB_Group_API_Request('POST', '%s', '%s', '{200, 409}') as response_status" % (url, body)
+    plpy.execute(query)
+$$ LANGUAGE 'plpythonu' VOLATILE SECURITY DEFINER;
+
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_RemoveUsers_API(group_name text, usernames text[])
+    RETURNS VOID AS
+$$
+    import string
+    import urllib
+
+    url = '/api/v1/databases/{0}/groups/%s/users' % (urllib.pathname2url(group_name))
+    body = "{ \"users\": [\"%s\"] }" % "\",\"".join(usernames)
+    query = "select cartodb._CDB_Group_API_Request('DELETE', '%s', '%s', '{200, 404}') as response_status" % (url, body)
+    plpy.execute(query)
+$$ LANGUAGE 'plpythonu' VOLATILE SECURITY DEFINER;
+
+DO LANGUAGE 'plpgsql' $$
+BEGIN
+    -- Needed for dropping type
+    DROP FUNCTION IF EXISTS cartodb._CDB_Group_API_Conf();
+    DROP TYPE IF EXISTS _CDB_Group_API_Params;
+END
+$$;
+
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_Table_GrantPermission_API(group_name text, username text, table_name text, access text)
+    RETURNS VOID AS
+$$
+    import string
+    import urllib
+
+    url = '/api/v1/databases/{0}/groups/%s/permission/%s/tables/%s' % (urllib.pathname2url(group_name), username, table_name)
+    body = '{ "access": "%s" }' % access
+    query = "select cartodb._CDB_Group_API_Request('PUT', '%s', '%s', '{200, 409}') as response_status" % (url, body)
+    plpy.execute(query)
+$$ LANGUAGE 'plpythonu' VOLATILE SECURITY DEFINER;
+
+DO LANGUAGE 'plpgsql' $$
+BEGIN
+    -- Needed for dropping type
+    DROP FUNCTION IF EXISTS cartodb._CDB_Group_API_Conf();
+    DROP TYPE IF EXISTS _CDB_Group_API_Params;
+END
+$$;
+
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_Table_RevokeAllPermission_API(group_name text, username text, table_name text)
+    RETURNS VOID AS
+$$
+    import string
+    import urllib
+
+    url = '/api/v1/databases/{0}/groups/%s/permission/%s/tables/%s' % (urllib.pathname2url(group_name), username, table_name)
+    query = "select cartodb._CDB_Group_API_Request('DELETE', '%s', '', '{200, 404}') as response_status" % url
+    plpy.execute(query)
+$$ LANGUAGE 'plpythonu' VOLATILE SECURITY DEFINER;
+
+DO LANGUAGE 'plpgsql' $$
+BEGIN
+    -- Needed for dropping type
+    DROP FUNCTION IF EXISTS cartodb._CDB_Group_API_Conf();
+    DROP TYPE IF EXISTS _CDB_Group_API_Params;
+END
+$$;
+
+CREATE TYPE _CDB_Group_API_Params AS (
+    host text,
+    port int,
+    timeout int,
+    auth text
+);
+
+-- This must be explicitally extracted because "composite types are currently not supported".
+-- See http://www.postgresql.org/docs/9.3/static/plpython-database.html.
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_API_Conf()
+    RETURNS _CDB_Group_API_Params AS
+$$
+    conf = plpy.execute("SELECT cartodb.CDB_Conf_GetConf('groups_api') conf")[0]['conf']
+    if conf is None:
+      return None
+    else:
+      import json
+      params = json.loads(conf)
+      auth = 'Basic %s' % plpy.execute("SELECT cartodb._CDB_Group_API_Auth('%s', '%s') as auth" % (params['username'], params['password']))[0]['auth']
+      return { "host": params['host'], "port": params['port'], 'timeout': params['timeout'], 'auth': auth }
+$$ LANGUAGE 'plpythonu' VOLATILE;
+
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_API_Auth(username text, password text)
+    RETURNS TEXT AS
+$$
+    import base64
+    return base64.encodestring('%s:%s' % (username, password)).replace('\n', '')
+$$ LANGUAGE 'plpythonu' VOLATILE;
+
+-- url must contain a '%s' placeholder that will be replaced by current_database, for security reasons.
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Group_API_Request(method text, url text, body text, valid_return_codes int[])
+    RETURNS int AS
+$$
+    import httplib
+
+    params = plpy.execute("select c.host, c.port, c.timeout, c.auth from cartodb._CDB_Group_API_Conf() c;")[0]
+    if params['host'] is None:
+      return None
+
+    headers = { 'Authorization': params['auth'], 'Content-Type': 'application/json', 'X-Forwarded-Proto': 'https' }
+
+    retry = 3
+
+    last_err = None
+    while retry > 0:
+      try:
+        client = SD['groups_api_client'] = httplib.HTTPConnection(params['host'], params['port'], False, params['timeout'])
+        database_name = plpy.execute("select current_database();")[0]['current_database']
+        client.request(method, url.format(database_name), body, headers)
+        response = client.getresponse()
+        assert response.status in valid_return_codes
+        return response.status
+      except Exception as err:
+        retry -= 1
+        last_err = err
+        plpy.warning('Retrying after: ' + str(err))
+        client = SD['groups_api_client'] = None
+
+    if last_err is not None:
+      plpy.error('Fatal Group API error: ' + str(last_err))
+      raise last_err
+
+    return None
+$$ LANGUAGE 'plpythonu' VOLATILE;
+revoke all on function cartodb._CDB_Group_API_Request(text, text, text, int[]) from public;

scripts-available/CDB_Organizations.sql

@@ -1,7 +1,9 @@
 CREATE OR REPLACE
 FUNCTION cartodb.CDB_Organization_Member_Group_Role_Member_Name()
     RETURNS TEXT
-AS 'SELECT ''cdb_org_member''::text || ''_'' || md5(current_database());'
+AS $$
+    SELECT 'cdb_org_member'::text || '_' || md5(current_database());
+$$
 LANGUAGE SQL IMMUTABLE;
 
 DO LANGUAGE 'plpgsql' $$
@@ -25,6 +27,60 @@ BEGIN
 END
 $$ LANGUAGE PLPGSQL VOLATILE;
 
+-------------------------------------------------------------------------------
+-- Administrator
+-------------------------------------------------------------------------------
+CREATE OR REPLACE
+FUNCTION cartodb._CDB_Organization_Admin_Role_Name()
+    RETURNS TEXT
+AS $$
+    SELECT current_database() || '_a'::text;
+$$
+LANGUAGE SQL IMMUTABLE;
+
+-- Administrator role creation on extension install
+DO LANGUAGE 'plpgsql' $$
+DECLARE
+    cdb_org_admin_role_name TEXT;
+BEGIN
+    cdb_org_admin_role_name := cartodb._CDB_Organization_Admin_Role_Name();
+    IF NOT EXISTS ( SELECT * FROM pg_roles WHERE rolname= cdb_org_admin_role_name )
+    THEN
+        EXECUTE format('CREATE ROLE %I CREATEROLE NOLOGIN;', cdb_org_admin_role_name);
+    END IF;
+END
+$$;
+
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Organization_AddAdmin(username text)
+    RETURNS void
+AS $$
+DECLARE
+    cdb_user_role TEXT;
+    cdb_admin_role TEXT;
+BEGIN
+    cdb_admin_role := cartodb._CDB_Organization_Admin_Role_Name();
+    cdb_user_role := cartodb._CDB_User_RoleFromUsername(username);
+    EXECUTE format('GRANT %I TO %I WITH ADMIN OPTION', cdb_admin_role, cdb_user_role);
+    -- CREATEROLE is not inherited, and is needed for user creation
+    EXECUTE format('ALTER ROLE %I CREATEROLE', cdb_user_role);
+END
+$$ LANGUAGE PLPGSQL;
+
+CREATE OR REPLACE
+FUNCTION cartodb.CDB_Organization_RemoveAdmin(username text)
+    RETURNS void
+AS $$
+DECLARE
+    cdb_user_role TEXT;
+    cdb_admin_role TEXT;
+BEGIN
+    cdb_admin_role := cartodb._CDB_Organization_Admin_Role_Name();
+    cdb_user_role := cartodb._CDB_User_RoleFromUsername(username);
+    EXECUTE format('ALTER ROLE %I NOCREATEROLE', cdb_user_role);
+    EXECUTE format('REVOKE %I FROM %I', cdb_admin_role, cdb_user_role);
+END
+$$ LANGUAGE PLPGSQL;
 
 -------------------------------------------------------------------------------
 -- Sharing tables

scripts-available/CDB_UserTables.sql

@@ -15,7 +15,7 @@ FROM pg_class c
 JOIN pg_namespace n ON n.oid = c.relnamespace
 WHERE c.relkind = 'r' 
 AND c.relname NOT IN ('cdb_tablemetadata', 'spatial_ref_sys')
-AND n.nspname NOT IN ('pg_catalog', 'information_schema', 'topology')
+AND n.nspname NOT IN ('pg_catalog', 'information_schema', 'topology', 'cartodb')
 AND CASE WHEN perm = 'public' THEN has_table_privilege('publicuser', c.oid, 'SELECT')
          WHEN perm = 'private' THEN has_table_privilege(current_user, c.oid, 'SELECT') AND NOT has_table_privilege('publicuser', c.oid, 'SELECT')
          WHEN perm = 'all' THEN has_table_privilege(current_user, c.oid, 'SELECT') OR has_table_privilege('publicuser', c.oid, 'SELECT')

scripts-enabled/CDB_Conf.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_Conf.sql

scripts-enabled/CDB_Groups.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_Groups.sql

scripts-enabled/CDB_Groups_API.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_Groups_API.sql

test/organization/test.sh

@@ -28,6 +28,7 @@ function clear_partial_result() {
 function sql() {
     local ROLE
     local QUERY
+    ERROR_OUTPUT_FILE='/tmp/test_error.log'
     if [[ $# -ge 2 ]]
     then
         ROLE="$1"
@@ -38,14 +39,39 @@ function sql() {
 
     if [ -n "${ROLE}" ]; then
         log_debug "Executing query '${QUERY}' as ${ROLE}"
-      RESULT=`${CMD} -U "${ROLE}" ${DATABASE} -c "${QUERY}" -A -t`
+        RESULT=`${CMD} -U "${ROLE}" ${DATABASE} -c "${QUERY}" -A -t 2>"${ERROR_OUTPUT_FILE}"`
     else
         log_debug "Executing query '${QUERY}'"
-      RESULT=`${CMD} ${DATABASE} -c "${QUERY}" -A -t`
+        RESULT=`${CMD} ${DATABASE} -c "${QUERY}" -A -t 2>"${ERROR_OUTPUT_FILE}"`
     fi
     CODERESULT=$?
+    ERROR_OUTPUT=`cat "${ERROR_OUTPUT_FILE}"`
+    rm ${ERROR_OUTPUT_FILE}
 
-    echo ${RESULT}
+    echo -n "> Code Result: "
+    echo -n ${CODERESULT}
+    echo -n "; Result: "
+    echo -n ${RESULT}
+    echo -n "; Error output: "
+    echo -n ${ERROR_OUTPUT}
+
+    # Some warnings should actually be failures
+    if [[ ${CODERESULT} == "0" ]]
+    then
+        case "${ERROR_OUTPUT}" in
+            WARNING:*no*privileges*were*granted*for*)
+                echo -n "FAILED BECAUSE OF PRIVILEGES GRANTING WARNING"
+                CODERESULT=1
+            ;;
+            WARNING:*no*privileges*could*be*revoked*for*)
+                echo -n "FAILED BECAUSE OF PRIVILEGES REVOKING WARNING"
+                CODERESULT=1
+            ;;
+            *) ;;
+        esac
+        echo -n "; Code result after warnings: "
+        echo -n ${CODERESULT}
+    fi
     echo
 
     if [[ ${CODERESULT} -ne 0 ]]
@@ -133,13 +159,18 @@ function create_table() {
 function setup() {
     ${CMD} -c "CREATE DATABASE ${DATABASE}"
     sql "CREATE SCHEMA cartodb;"
+    sql "CREATE EXTENSION plpythonu;"
     sql "GRANT USAGE ON SCHEMA cartodb TO public;"
 
     log_info "########################### BOOTSTRAP ###########################"
     ${CMD} -d ${DATABASE} -f scripts-available/CDB_Organizations.sql
-
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_Conf.sql
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_Groups.sql
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_Groups_API.sql
 
     log_info "############################# SETUP #############################"
+    create_role_and_schema cdb_org_admin
+    sql "SELECT cartodb.CDB_Organization_AddAdmin('cdb_org_admin');"
     create_role_and_schema cdb_testmember_1
     create_role_and_schema cdb_testmember_2
     sql "CREATE ROLE publicuser LOGIN;"
@@ -152,8 +183,16 @@ function setup() {
     create_table cdb_testmember_2 bar
     sql cdb_testmember_2 'INSERT INTO bar VALUES (1), (2), (3), (4), (5);'
     sql cdb_testmember_2 'SELECT * FROM cdb_testmember_2.bar;'
+
+    sql "SELECT cartodb.CDB_Group_CreateGroup('group_a_tmp')"
+    sql "SELECT cartodb.CDB_Group_RenameGroup('group_a_tmp', 'group_a')"
+
+    sql "SELECT cartodb.CDB_Group_AddUsers('group_a', ARRAY['cdb_testmember_1'])"
+
+    sql "SELECT cartodb.CDB_Group_CreateGroup('group_b')"
 }
 
+
 function tear_down() {
     log_info "########################### USER TEAR DOWN ###########################"
     sql cdb_testmember_1 "SELECT * FROM cartodb.CDB_Organization_Remove_Access_Permission('cdb_testmember_1', 'foo', 'cdb_testmember_2');"
@@ -162,19 +201,29 @@ function tear_down() {
     sql cdb_testmember_1 'DROP TABLE cdb_testmember_1.foo;'
     sql cdb_testmember_2 'DROP TABLE cdb_testmember_2.bar;'
 
+    sql "select cartodb.CDB_Group_DropGroup('group_b')"
+
+    sql "SELECT cartodb.CDB_Group_RemoveUsers('group_a', ARRAY['cdb_testmember_1'])"
+
+    sql "select cartodb.CDB_Group_DropGroup('group_a')"
+    sql "SELECT cartodb.CDB_Organization_RemoveAdmin('cdb_org_admin');"
+
     sql "DROP SCHEMA cartodb CASCADE"
 
     log_info "########################### TEAR DOWN ###########################"
     sql 'DROP SCHEMA cdb_testmember_1;'
     sql 'DROP SCHEMA cdb_testmember_2;'
+    sql 'DROP SCHEMA cdb_org_admin;'
 
     sql "REVOKE CONNECT ON DATABASE \"${DATABASE}\" FROM cdb_testmember_1;"
     sql "REVOKE CONNECT ON DATABASE \"${DATABASE}\" FROM cdb_testmember_2;"
     sql "REVOKE CONNECT ON DATABASE \"${DATABASE}\" FROM publicuser;"
+    sql "REVOKE CONNECT ON DATABASE \"${DATABASE}\" FROM cdb_org_admin;"
 
     sql 'DROP ROLE cdb_testmember_1;'
     sql 'DROP ROLE cdb_testmember_2;'
     sql 'DROP ROLE publicuser;'
+    sql 'DROP ROLE cdb_org_admin;'
 
     ${CMD} -c "DROP DATABASE ${DATABASE}"
 }
@@ -198,6 +247,7 @@ function run_tests() {
         echo "####################################################################"
         clear_partial_result
         setup
+        log_info "############################# TESTS #############################"
         eval ${t}
         if [[ ${PARTIALOK} -ne 0 ]]
         then
@@ -330,21 +380,18 @@ function test_user_can_read_when_it_has_permission_after_organization_permission
 }
 
 function test_cdb_querytables_returns_schema_and_table_name() {
-    sql "CREATE EXTENSION plpythonu;"
     ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryStatements.sql
     ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryTables.sql
     sql cdb_testmember_1 "select * from CDB_QueryTables('select * from foo');" should "{cdb_testmember_1.foo}"
 }
 
 function test_cdb_querytables_returns_schema_and_table_name_for_several_schemas() {
-    sql "CREATE EXTENSION plpythonu;"
     ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryStatements.sql
     ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryTables.sql
     sql postgres "select * from CDB_QueryTables('select * from cdb_testmember_1.foo, cdb_testmember_2.bar');" should "{cdb_testmember_1.foo,cdb_testmember_2.bar}"
 }
 
 function test_cdb_querytables_does_not_return_functions_as_part_of_the_resultset() {
-    sql "CREATE EXTENSION plpythonu;"
     ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryStatements.sql
     ${CMD} -d ${DATABASE} -f scripts-available/CDB_QueryTables.sql
     sql postgres "select * from CDB_QueryTables('select * from cdb_testmember_1.foo, cdb_testmember_2.bar, plainto_tsquery(''foo'')');" should "{cdb_testmember_1.foo,cdb_testmember_2.bar}"
@@ -393,6 +440,107 @@ function test_cdb_usertables_should_work_with_orgusers() {
     sql cdb_testmember_1 "DROP TABLE test_perms_priv"
 }
 
+function test_CDB_Group_Table_GrantRead_should_grant_select_and_RevokeAll_should_remove_it() {
+    create_table cdb_testmember_2 shared_with_group
+
+    sql cdb_testmember_1 'SELECT count(*) FROM cdb_testmember_2.shared_with_group;' fails
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_2.shared_with_group;'
+    sql cdb_testmember_2 "select cartoDB.CDB_Group_Table_GrantRead('group_a', 'cdb_testmember_2', 'shared_with_group')"
+    sql cdb_testmember_1 'SELECT count(*) FROM cdb_testmember_2.shared_with_group;'
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_2.shared_with_group;'
+    sql cdb_testmember_2 "select cartoDB.CDB_Group_Table_RevokeAll('group_a', 'cdb_testmember_2', 'shared_with_group')"
+    sql cdb_testmember_1 'SELECT count(*) FROM cdb_testmember_2.shared_with_group;' fails
+    sql cdb_testmember_2 'SELECT count(*) FROM cdb_testmember_2.shared_with_group;'
+
+    sql cdb_testmember_2 'DROP TABLE cdb_testmember_2.shared_with_group;'
+}
+
+function test_CDB_Group_Table_GrantReadWrite_should_grant_insert_and_RevokeAll_should_remove_it() {
+    create_table cdb_testmember_2 shared_with_group
+
+    sql cdb_testmember_1 'INSERT INTO cdb_testmember_2.shared_with_group VALUES (1), (2), (3), (4), (5)' fails
+    sql cdb_testmember_2 'INSERT INTO cdb_testmember_2.shared_with_group VALUES (1), (2), (3), (4), (5)'
+    sql cdb_testmember_2 "select cartoDB.CDB_Group_Table_GrantReadWrite('group_a', 'cdb_testmember_2', 'shared_with_group')"
+    sql cdb_testmember_1 'INSERT INTO cdb_testmember_2.shared_with_group VALUES (1), (2), (3), (4), (5)'
+    sql cdb_testmember_2 'INSERT INTO cdb_testmember_2.shared_with_group VALUES (1), (2), (3), (4), (5)'
+    sql cdb_testmember_2 "select cartoDB.CDB_Group_Table_RevokeAll('group_a', 'cdb_testmember_2', 'shared_with_group')"
+    sql cdb_testmember_1 'INSERT INTO cdb_testmember_2.shared_with_group VALUES (1), (2), (3), (4), (5)' fails
+    sql cdb_testmember_2 'INSERT INTO cdb_testmember_2.shared_with_group VALUES (1), (2), (3), (4), (5)'
+
+    sql cdb_testmember_2 'DROP TABLE cdb_testmember_2.shared_with_group;'
+}
+
+function test_group_management_functions_cant_be_used_by_normal_members() {
+    sql cdb_testmember_1 "SELECT cartodb.CDB_Group_CreateGroup('group_x_1');" fails
+    sql cdb_testmember_1 "SELECT cartodb.CDB_Group_RenameGroup('group_a', 'group_x_2');" fails
+    sql cdb_testmember_1 "SELECT cartodb.CDB_Group_DropGroup('group_a');" fails
+    sql cdb_testmember_1 "SELECT cartodb.CDB_Group_AddUsers('group_a', ARRAY['cdb_testmember_2']);" fails
+    sql cdb_testmember_1 "SELECT cartodb.CDB_Group_RemoveUsers('group_a', ARRAY['cdb_testmember_1']);" fails
+}
+
+function test_group_permission_functions_cant_be_used_by_normal_members() {
+    create_table cdb_testmember_2 shared_with_group
+
+    sql cdb_testmember_1 "select cartoDB.CDB_Group_Table_GrantRead('group_a', 'cdb_testmember_2', 'shared_with_group');" fails
+    sql cdb_testmember_1 "select cartoDB.CDB_Group_Table_GrantReadWrite('group_a', 'cdb_testmember_2', 'shared_with_group');" fails
+
+    # Checks that you can't grant even if your group has RW permissions
+    sql cdb_testmember_2 "select cartoDB.CDB_Group_Table_GrantReadWrite('group_a', 'cdb_testmember_2', 'shared_with_group')"
+    sql cdb_testmember_1 "select cartoDB.CDB_Group_Table_GrantRead('group_a', 'cdb_testmember_2', 'shared_with_group');" fails
+    sql cdb_testmember_1 "select cartoDB.CDB_Group_Table_GrantReadWrite('group_b', 'cdb_testmember_2', 'shared_with_group');" fails
+    sql cdb_testmember_1 "select cartoDB.CDB_Group_Table_RevokeAll('group_b', 'cdb_testmember_2', 'shared_with_group');" fails
+
+    sql cdb_testmember_2 'DROP TABLE cdb_testmember_2.shared_with_group;'
+}
+
+function test_group_management_functions_can_be_used_by_org_admin() {
+    sql cdb_org_admin "SELECT cartodb.CDB_Group_CreateGroup('group_x_tmp');"
+    sql cdb_org_admin "SELECT cartodb.CDB_Group_RenameGroup('group_x_tmp', 'group_x');"
+    sql cdb_org_admin "SELECT cartodb.CDB_Group_AddUsers('group_x', ARRAY['cdb_testmember_1', 'cdb_testmember_2']);"
+    sql cdb_org_admin "SELECT cartodb.CDB_Group_RemoveUsers('group_x', ARRAY['cdb_testmember_1', 'cdb_testmember_2']);"
+    # TODO: workaround superadmin limitation
+    sql "SELECT cartodb.CDB_Group_DropGroup('group_x');"
+}
+
+function test_org_admin_cant_grant_permissions_on_tables_he_does_not_own() {
+    create_table cdb_testmember_2 shared_with_group
+
+    sql cdb_org_admin "select cartoDB.CDB_Group_Table_GrantRead('group_a', 'cdb_testmember_2', 'shared_with_group');" fails
+    sql cdb_org_admin "select cartoDB.CDB_Group_Table_GrantReadWrite('group_a', 'cdb_testmember_2', 'shared_with_group');" fails
+
+    # Checks that you can't grant even if your group has RW permissions
+    sql cdb_testmember_2 "select cartoDB.CDB_Group_Table_GrantReadWrite('group_a', 'cdb_testmember_2', 'shared_with_group')"
+    sql cdb_org_admin "select cartoDB.CDB_Group_Table_GrantRead('group_a', 'cdb_testmember_2', 'shared_with_group');" fails
+    sql cdb_org_admin "select cartoDB.CDB_Group_Table_GrantReadWrite('group_b', 'cdb_testmember_2', 'shared_with_group');" fails
+    sql cdb_org_admin "select cartoDB.CDB_Group_Table_RevokeAll('group_b', 'cdb_testmember_2', 'shared_with_group');" fails
+
+    sql cdb_testmember_2 'DROP TABLE cdb_testmember_2.shared_with_group;'
+}
+
+function test_valid_group_names() {
+    sql postgres "select cartodb._CDB_Group_GroupRole('group_1$_a');"
+    sql postgres "select cartodb._CDB_Group_GroupRole('GROUP_1$_A');"
+    sql postgres "select cartodb._CDB_Group_GroupRole('_group_1$_a');"
+}
+
+function test_administrator_name_generation() {
+    sql postgres "select cartodb._CDB_Organization_Admin_Role_Name();"
+}
+
+function test_conf() {
+    sql postgres "SELECT cartodb.CDB_Conf_GetConf('test_conf')" should ''
+    sql postgres "SELECT cartodb.CDB_Conf_GetConf('test_conf_2')" should ''
+
+    sql postgres "SELECT cartodb.CDB_Conf_SetConf('test_conf', '{ \"a_key\": \"test_val\" }')"
+
+    sql postgres "SELECT cartodb.CDB_Conf_GetConf('test_conf')" should '{ "a_key": "test_val" }'
+    sql postgres "SELECT cartodb.CDB_Conf_GetConf('test_conf_2')" should ''
+
+    sql postgres "SELECT cartodb.CDB_Conf_RemoveConf('test_conf')"
+
+    sql postgres "SELECT cartodb.CDB_Conf_GetConf('test_conf')" should ''
+    sql postgres "SELECT cartodb.CDB_Conf_GetConf('test_conf_2')" should ''
+}
 
 #################################################### TESTS END HERE ####################################################