Server: Simplify PG12/Postgis 3 interaction

server/extension/.gitignore

@@ -6,4 +6,6 @@ cdb_geocoder_server--0.1.0.sql
 cdb_geocoder_server--0.2.0.sql
 cdb_geocoder_server--0.3.0.sql
 cdb_geocoder_server--0.4.0.sql
-cdb_geocoder_server--0.5.0.sql
+cdb_geocoder_server--0.5.0.sql
+cdb_dataservices_server.control
+./test/sql/366_empty_table_test.sql

server/extension/Makefile

@@ -1,17 +1,32 @@
 # Makefile to generate the extension out of separate sql source files.
 # Once a version is released, it is not meant to be changed. E.g: once version 0.0.1 is out, it SHALL NOT be changed.
 EXTENSION = cdb_dataservices_server
-EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/")
+EXTVERSION = 0.38.0
+
 # The new version to be generated from templates
-SED = sed
-ERB = erb
-NEW_EXTENSION_ARTIFACT = $(EXTENSION)--$(EXTVERSION).sql
-AWK = awk
+SED ?= sed
+ERB ?= erb
+AWK ?= awk
+PG_CONFIG ?= pg_config
 
 # Parallel support macros
-PG_CONFIG         = pg_config
+PG_CONFIG        ?= pg_config
 PG_PARALLEL      := $(shell $(PG_CONFIG) --version | ($(AWK) '{$$2*=1000; if ($$2 >= 9600) print 1; else print 0;}' 2> /dev/null || echo 0))
 
+# PG12 compatibility
+PG_VERSION := $(shell $(PG_CONFIG) --version | $(AWK) '{split($$2,a,"."); print a[1]}')
+PG_12_GE := $(shell [ $(PG_VERSION) -ge 12 ] && echo true)
+PLPYTHONU := plpythonu
+POSTGIS := postgis
+ifeq ($(PG_12_GE), true)
+PLPYTHONU := plpython3u
+POSTGIS := postgis, postgis_raster
+endif
+
+REPLACEMENTS = -e 's/@@EXTVERSION@@/$(EXTVERSION)/g' -e 's/@@plpythonu@@/$(PLPYTHONU)/g' -e 's/@@postgis@@/$(POSTGIS)/g'
+REPLACEMENTS_TEST_REVERSE = -e 's/$(PLPYTHONU)/@@plpythonu@@/g'
+NEW_EXTENSION_ARTIFACT = $(EXTENSION)--$(EXTVERSION).sql
+
 REGRESS = $(notdir $(basename $(sort $(wildcard test/sql/*test.sql))))
 REGRESS_EXPEC = $(notdir $(basename $(sort $(wildcard test/expected/*test.out))))
 TEST_DIR = test
@@ -29,48 +44,29 @@ SOURCES_DATA_DIR = sql/
 SOURCES_DATA = $(wildcard sql/*.sql)
 
 # postgres build stuff
-PG_CONFIG = pg_config
+
+# The targets listed under REGRESS_PREP are executed before regress
+PYTHON_TESTS = ./test/sql/366_empty_table_test.sql
+REGRESS_PREP = $(PYTHON_TESTS)
+
+./test/sql/366_empty_table_test.sql: ./test/sql/366_empty_table_test.sql.template
+	$(SED) $(REPLACEMENTS) $@.template > $@
+
 PGXS := $(shell $(PG_CONFIG) --pgxs)
 include $(PGXS)
 
-# PG12 compatibility
-PG_VERSION := $(shell $(PG_CONFIG) --version | $(AWK) '{split($$2,a,"."); print a[1]}')
-PG_12_GE := $(shell [ $(PG_VERSION) -ge 12 ] && echo true)
-PLPYTHONU := plpythonu
-ifeq ($(PG_12_GE), true)
-PLPYTHONU := plpython3u
-endif
-
-REPLACEMENTS = -e 's/@@plpythonu@@/$(PLPYTHONU)/g'
-ifneq ($(NEW_VERSION),)
-REPLACEMENTS += -e 's/$(EXTVERSION)/$(NEW_VERSION)/g'
-endif
 
 $(NEW_EXTENSION_ARTIFACT): $(SOURCES_DATA)
 	rm -f $@
 	cat $(SOURCES_DATA_DIR)/*.sql >> $@
-ifeq ($(PG_PARALLEL), 0)
-# Remove PARALLEL in aggregates and functions
-	$(eval TMPFILE := $(shell mktemp /tmp/$(basename $0).XXXXXXXX))
-	$(SED) -e 's/PARALLEL \= [A-Z]*,/''/g' \
-		-e 's/PARALLEL [A-Z]*/''/g' $@ > $(TMPFILE)
-	mv $(TMPFILE) $@
-endif
-
-# $(EXTENSION).control: $(EXTENSION).control.in Makefile
-# 	$(SED) $(REPLACEMENTS)  $< > $@
 
 .PHONY: all
 all: $(DATA)
 
 .PHONY: release
-release: $(EXTENSION).control $(SOURCES_DATA)
+release: $(SOURCES_DATA)
 	test -n "$(NEW_VERSION)"  # $$NEW_VERSION VARIABLE MISSING. Eg. make release NEW_VERSION=0.x.0
-	for f in $(wildcard *.sql); do \
-		git mv $${f} old_versions/$${f}; \
-	done
-	$(SED) -i 's/$(EXTVERSION)/$(NEW_VERSION)/g' $(EXTENSION).control
-	git add $(EXTENSION).control
+	git mv *.sql old_versions
 	cat $(SOURCES_DATA_DIR)/*.sql > $(EXTENSION)--$(NEW_VERSION).sql
 	$(ERB) version=$(NEW_VERSION) upgrade_downgrade_template.erb > $(EXTENSION)--$(EXTVERSION)--$(NEW_VERSION).sql
 	$(ERB) version=$(EXTVERSION) upgrade_downgrade_template.erb > $(EXTENSION)--$(NEW_VERSION)--$(EXTVERSION).sql
@@ -84,78 +80,32 @@ release: $(EXTENSION).control $(SOURCES_DATA)
 .PHONY: devclean
 devclean:
 	rm -f $(NEW_EXTENSION_ARTIFACT)
+	rm -f $(PYTHON_TESTS)
 
-clean: restore_copies
-
-# If needed remove PARALLEL tags from the release files
-release_remove_parallel_deploy:
-ifeq ($(PG_PARALLEL), 0)
+# Replace variables (and PARALLEL tags if necessary) and deploy files
+.PHONY: replace_variables_and_deploy
+replace_variables_and_deploy: $(NEW_EXTENSION_ARTIFACT)
+	mkdir -p '$(DESTDIR)$(datadir)/extension/'; \
 	for n in $(wildcard old_versions/*.sql *.sql); do \
 		$(eval TMPFILE := $(shell mktemp /tmp/XXXXXXXXXX)) \
-		$(SED) -e 's/PARALLEL \= [A-Z]*,/''/g' -e 's/PARALLEL [A-Z]*/''/g' $$n > $(TMPFILE); \
-		mv $(TMPFILE) $$n; \
+		if [ "$(PG_PARALLEL)" -eq "0" ]; then \
+			$(SED) -e 's/PARALLEL \= [A-Z]*,/''/g' -e 's/PARALLEL [A-Z]*/''/g' $$n > $(TMPFILE); \
+			mv $(TMPFILE) $$n; \
+		fi; \
+		$(SED) $(REPLACEMENTS) $$n > $(TMPFILE); \
+		mv $(TMPFILE) '$(DESTDIR)$(datadir)/extension/'$$(basename $$n); \
 	done
-endif
+	
+
+.PHONY: $(EXTENSION).control
+$(EXTENSION).control:
+	$(SED) $(REPLACEMENTS) $(EXTENSION).control.in > $(EXTENSION).control
 
 # Install the current release into the PostgreSQL extensions directory
-deploy: release_remove_parallel_deploy
+.PHONY: deploy
+deploy: replace_variables_and_deploy $(EXTENSION).control
 	$(INSTALL_DATA) $(EXTENSION).control '$(DESTDIR)$(datadir)/extension/'
-	$(INSTALL_DATA) old_versions/*.sql *.sql '$(DESTDIR)$(datadir)/extension/'
 
-restore_copies:
-	# tests
-	for f in $(basename $(wildcard test/sql/*.copy)); do \
-		cat $${f}.copy > $${f}; \
-	done
-	for f in $(basename $(wildcard test/expected/*.copy)); do \
-		cat $${f}.copy > $${f}; \
-	done
-	rm -f test/sql/*.copy;
-	rm -f test/expected/*.copy;
-	# data
-	for f in $(basename $(wildcard sql/*.copy)); do \
-		cat $${f}.copy > $${f}; \
-	done
-	rm -f sql/*.copy;
-	# old_versions
-	for f in $(basename $(wildcard old_versions/*.copy)); do \
-		cat $${f}.copy > $${f}; \
-	done
-	rm -f old_versions/*.copy;
-	# current scripts
-	for f in $(basename $(wildcard *.copy)); do \
-		cat $${f}.copy > $${f}; \
-	done
-	rm -f *.copy;
-
-
-# %.sql:
-# 	$(SED) $(REPLACEMENTS)  $< > $@
-
-# Replacing variables defined within test files
-replace_variables: restore_copies
-	# tests
-	for f in $(sort $(wildcard test/sql/*test.sql)); do \
-		sed --in-place=.copy -e 's/@@plpythonu@@/$(PLPYTHONU)/g' $${f}; \
-	done
-	for f in $(sort $(wildcard test/expected/*test.out)); do \
-		sed --in-place=.copy -e 's/@@plpythonu@@/$(PLPYTHONU)/g' $${f}; \
-	done
-	# data
-	for f in $(SOURCES_DATA); do \
-		sed --in-place=.copy -e 's/@@plpythonu@@/$(PLPYTHONU)/g' $${f}; \
-	done
-	# old_versions
-	for f in $(wildcard old_versions/*.sql); do \
-		sed --in-place=.copy -e 's/@@plpythonu@@/$(PLPYTHONU)/g' $${f}; \
-	done
-	# current scripts
-	for f in $(wildcard *.sql); do \
-		sed --in-place=.copy -e 's/@@plpythonu@@/$(PLPYTHONU)/g' $${f}; \
-	done
-	sed --in-place=.copy -e 's/@@plpythonu@@/$(PLPYTHONU)/g' $(EXTENSION).control;
-
-install: replace_variables deploy
-
-reinstall: install
-	psql -U postgres -d dataservices_db -c "drop extension if exists cdb_dataservices_server; create extension cdb_dataservices_server;"
+.PHONY: install
+install: deploy
+	@echo "Install overriden"

server/extension/cdb_dataservices_server.control.in

@@ -1,5 +1,5 @@
 comment = 'CartoDB dataservices server extension'
-default_version = '0.38.0'
-requires = '@@plpythonu@@, plproxy, postgis, cdb_geocoder'
+default_version = '@@EXTVERSION@@'
+requires = '@@plpythonu@@, plproxy, @@postgis@@, cdb_geocoder'
 superuser = true
 schema = cdb_dataservices_server

server/extension/sql/05_postgis3_adapter.sql

@@ -1,11 +0,0 @@
--- PG12_DEPRECATED
--- Create geomval if it doesn't exist (in postgis 3+ it only exists in postgis_raster)
-DO $$
-BEGIN
-    IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'geomval') THEN
-        CREATE TYPE cdb_dataservices_server.geomval AS (
-            geom geometry,
-            val double precision
-        );
-    END IF;
-END$$;

server/extension/test/expected/00_install_test.out

@@ -1,13 +1,9 @@
 -- Only show warning or error messages in the tests output
+\set QUIET on
 SET client_min_messages TO WARNING;
--- Install dependencies
-CREATE EXTENSION postgis;
-CREATE EXTENSION @@plpythonu@@;
-CREATE EXTENSION plproxy;
-CREATE EXTENSION cartodb;
-CREATE EXTENSION cdb_geocoder;
 -- Install the extension
-CREATE EXTENSION cdb_dataservices_server;
+CREATE EXTENSION cdb_dataservices_server CASCADE;
+\unset QUIET
 -- Mock the redis server connection to point to this very test db
 SELECT cartodb.cdb_conf_setconf('redis_metrics_config', '{"redis_host": "localhost", "redis_port": 6379, "timeout": 0.1, "redis_db": 5}');
  cdb_conf_setconf 
@@ -71,6 +67,7 @@ BEGIN
 END
 $$
 LANGUAGE plpgsql;
+CREATE FUNCTION
 -- Set user quota
 SELECT cartodb.CDB_SetUserQuotaInBytes(0);
  cdb_setuserquotainbytes 

server/extension/test/sql/00_install_test.sql

@@ -1,14 +1,10 @@
 -- Only show warning or error messages in the tests output
+\set QUIET on
 SET client_min_messages TO WARNING;
--- Install dependencies
-CREATE EXTENSION postgis;
-CREATE EXTENSION @@plpythonu@@;
-CREATE EXTENSION plproxy;
-CREATE EXTENSION cartodb;
-CREATE EXTENSION cdb_geocoder;
 
 -- Install the extension
-CREATE EXTENSION cdb_dataservices_server;
+CREATE EXTENSION cdb_dataservices_server CASCADE;
+\unset QUIET
 
 -- Mock the redis server connection to point to this very test db
 SELECT cartodb.cdb_conf_setconf('redis_metrics_config', '{"redis_host": "localhost", "redis_port": 6379, "timeout": 0.1, "redis_db": 5}');