WIP2: Python3 + PG12

release/crankshaft.control

@@ -1,5 +1,5 @@
 comment = 'CartoDB Spatial Analysis extension'
 default_version = '0.8.2'
-requires = 'plpythonu, postgis'
+requires = 'plpython3u, postgis'
 superuser = true
 schema = cdb_crankshaft

src/pg/Makefile

@@ -4,7 +4,7 @@ include ../../Makefile.global
 #
 # * install generates the control & script files into src/pg/
 #   and installs then into the PostgreSQL extensions directory;
-#   requires sudo. In additionof the current development version
+#   requires sudo. In addition of the current development version
 #   named 'dev', an alias 'current' is generating for ease of
 #   update (upgrade to 'current', then to 'dev').
 # * test runs the tests for the currently generated Development
@@ -20,21 +20,27 @@ SOURCES_DATA_DIR = sql
 SOURCES_DATA = $(wildcard $(SOURCES_DATA_DIR)/*.sql)
 
 REPLACEMENTS = -e 's/@@VERSION@@/$(EXTVERSION)/g'
+ifneq ($(PYTHON3), true)
+	REPLACEMENTS = -e 's/@@VERSION@@/$(EXTVERSION)/g' -e
+endif
 
 $(DATA): $(SOURCES_DATA)
 	$(SED) $(REPLACEMENTS) $(SOURCES_DATA_DIR)/*.sql > $@
-ifeq ($(PG_PARALLEL), 0)
+ifneq ($(PG_PARALLEL), true)
 	$(eval TMPFILE := $(shell mktemp /tmp/$@.XXXXXXXXXX))
 	$(SED) -e 's/PARALLEL \= [A-Z]*,/''/g' -e 's/PARALLEL [A-Z]*/''/g' $@ > $(TMPFILE);
 	mv $(TMPFILE) $@
 endif
-
+ifneq ($(PYTHON3), true)
+	$(eval TMPFILE := $(shell mktemp /tmp/$@.XXXXXXXXXX))
+	$(SED) -e 's/plpython3u/plpythonu/g' $$n > $(TMPFILE);
+	mv $(TMPFILE) $@
+endif
 
 TEST_DIR = test
 REGRESS = $(sort $(notdir $(basename $(wildcard $(TEST_DIR)/sql/*test.sql))))
 REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)'
 
-PG_CONFIG = pg_config
 PGXS := $(shell $(PG_CONFIG) --pgxs)
 include $(PGXS)
 
@@ -58,7 +64,7 @@ release: ../../release/$(EXTENSION).control $(SOURCES_DATA)
 
 # If needed remove PARALLEL tags from the release files
 release_remove_parallel_deploy:
-ifeq ($(PG_PARALLEL), 0)
+ifneq ($(PG_PARALLEL), true)
 	for n in $(wildcard ../../release/*.sql); do \
 		$(eval TMPFILE := $(shell mktemp /tmp/XXXXXXXXXX)) \
 		$(SED) -e 's/PARALLEL \= [A-Z]*,/''/g' -e 's/PARALLEL [A-Z]*/''/g' $$n > $(TMPFILE); \
@@ -66,8 +72,19 @@ ifeq ($(PG_PARALLEL), 0)
 	done
 endif
 
+release_adapt_plpython:
+ifneq ($(PYTHON3), true)
+	for n in $(wildcard ../../release/*.sql); do \
+		$(eval TMPFILE := $(shell mktemp /tmp/XXXXXXXXXX)) \
+		$(SED) -e 's/plpython3u/plpythonu/g' $$n > $(TMPFILE); \
+		mv $(TMPFILE) $$n; \
+	done
+endif
+
 # Install the current release into the PostgreSQL extensions directory
-deploy: release_remove_parallel_deploy
+# We execute these 3 tasks to modify release files in place so they are compatible with
+# the PG and python releases available during deployment, and not release
+deploy: release_remove_parallel_deploy release_adapt_plpython ../../release/$(EXTENSION).control
 	$(INSTALL_DATA) ../../release/$(EXTENSION).control '$(DESTDIR)$(datadir)/extension/'
 	$(INSTALL_DATA) ../../release/*.sql '$(DESTDIR)$(datadir)/extension/'
 

src/pg/crankshaft.control

@@ -1,5 +1,5 @@
 comment = 'CartoDB Spatial Analysis extension'
 default_version = '0.8.2'
-requires = 'plpythonu, postgis'
+requires = 'plpython3u, postgis'
 superuser = true
 schema = cdb_crankshaft

src/pg/sql/03_random_seeds.sql

@@ -6,4 +6,4 @@ _cdb_random_seeds (seed_value INTEGER) RETURNS VOID
 AS $$
   from crankshaft import random_seeds
   random_seeds.set_random_seeds(seed_value)
-$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
+$$ LANGUAGE plpython3u VOLATILE PARALLEL UNSAFE;

src/pg/sql/05_segmentation.sql

@@ -35,7 +35,7 @@ AS $$
         target_ids,
         model_params)
 
-$$ LANGUAGE plpythonu VOLATILE PARALLEL RESTRICTED;
+$$ LANGUAGE plpython3u VOLATILE PARALLEL RESTRICTED;
 
 CREATE OR REPLACE FUNCTION
   CDB_CreateAndPredictSegment(
@@ -68,7 +68,7 @@ AS $$
         target_table,
         model_params
     )
-$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
+$$ LANGUAGE plpython3u VOLATILE PARALLEL UNSAFE;
 
 CREATE OR REPLACE FUNCTION
   CDB_CreateAndPredictSegment(
@@ -99,4 +99,4 @@ AS $$
         target_query,
         model_params
     )
-$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
+$$ LANGUAGE plpython3u VOLATILE PARALLEL UNSAFE;

src/pg/sql/10_moran.sql

@@ -15,7 +15,7 @@ AS $$
   moran = Moran()
   return moran.global_stat(subquery, column_name, w_type,
                            num_ngbrs, permutations, geom_col, id_col)
-$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
+$$ LANGUAGE plpython3u VOLATILE PARALLEL UNSAFE;
 
 -- Moran's I Local (internal function) - DEPRECATED
 CREATE OR REPLACE FUNCTION
@@ -40,7 +40,7 @@ AS $$
                             num_ngbrs, permutations, geom_col, id_col)
   # remove spatial lag
   return [(r[6], r[0], r[1], r[7], r[5]) for r in result]
-$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
+$$ LANGUAGE plpython3u VOLATILE PARALLEL UNSAFE;
 
 -- Moran's I Local (internal function)
 CREATE OR REPLACE FUNCTION
@@ -68,7 +68,7 @@ moran = Moran()
 return moran.local_stat(subquery, column_name, w_type,
                         num_ngbrs, permutations, geom_col, id_col)
 
-$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
+$$ LANGUAGE plpython3u VOLATILE PARALLEL UNSAFE;
 
 
 -- Moran's I Local (public-facing function)
@@ -195,7 +195,7 @@ AS $$
   # TODO: use named parameters or a dictionary
   return moran.global_rate_stat(subquery, numerator, denominator, w_type,
                                 num_ngbrs, permutations, geom_col, id_col)
-$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
+$$ LANGUAGE plpython3u VOLATILE PARALLEL UNSAFE;
 
 
 -- Moran's I Local Rate (internal function) - DEPRECATED
@@ -223,7 +223,7 @@ AS $$
   result = moran.local_rate_stat(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col)
   # remove spatial lag
   return [(r[6], r[0], r[1], r[7], r[4]) for r in result]
-$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
+$$ LANGUAGE plpython3u VOLATILE PARALLEL UNSAFE;
 
 -- Moran's I Local Rate (public-facing function) - DEPRECATED
 CREATE OR REPLACE FUNCTION
@@ -279,7 +279,7 @@ return moran.local_rate_stat(
     geom_col,
     id_col
 )
-$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
+$$ LANGUAGE plpython3u VOLATILE PARALLEL UNSAFE;
 
 -- Moran's I Rate
 -- Replaces CDB_AreasOfInterestLocalRate

src/pg/sql/11_kmeans.sql

@@ -14,7 +14,7 @@ from crankshaft.clustering import Kmeans
 kmeans = Kmeans()
 return kmeans.spatial(query, no_clusters, no_init)
 
-$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
+$$ LANGUAGE plpython3u VOLATILE PARALLEL UNSAFE;
 
 -- Non-spatial k-means clustering
 -- query: sql query to retrieve all the needed data
@@ -44,7 +44,7 @@ kmeans = Kmeans()
 return kmeans.nonspatial(query, colnames, no_clusters,
                          standardize=standardize,
                          id_col=id_col)
-$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
+$$ LANGUAGE plpython3u VOLATILE PARALLEL UNSAFE;
 
 
 CREATE OR REPLACE FUNCTION CDB_WeightedMeanS(

src/pg/sql/11_markov.sql

@@ -27,7 +27,7 @@ AS $$
 
   ## TODO: use named parameters or a dictionary
   return markov.spatial_trend(subquery, time_cols, num_classes, w_type, num_ngbrs, permutations, geom_col, id_col)
-$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
+$$ LANGUAGE plpython3u VOLATILE PARALLEL UNSAFE;
 
 -- input table format: identical to above but in a predictable format
 -- Sample function call:
@@ -53,7 +53,7 @@ $$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
 --   from crankshaft.clustering import moran_local
 --   # TODO: use named parameters or a dictionary
 --   return spatial_markov(subquery, time_cols, permutations, geom_column, id_col, w_type, num_ngbrs)
--- $$ LANGUAGE plpythonu;
+-- $$ LANGUAGE plpython3u;
 --
 -- -- input table format:
 -- -- id | geom | date  | measurement
@@ -79,4 +79,4 @@ $$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
 --   from crankshaft.clustering import moran_local
 --   # TODO: use named parameters or a dictionary
 --   return spatial_markov(subquery, time_cols, permutations, geom_column, id_col, w_type, num_ngbrs)
--- $$ LANGUAGE plpythonu;
+-- $$ LANGUAGE plpython3u;

src/pg/sql/16_getis.sql

@@ -14,6 +14,6 @@ AS $$
   from crankshaft.clustering import Getis
   getis = Getis()
   return getis.getis_ord(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col)
-$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
+$$ LANGUAGE plpython3u VOLATILE PARALLEL UNSAFE;
 
 -- TODO: make a version that accepts the values as arrays

src/pg/sql/21_gwr.sql

@@ -15,7 +15,7 @@ gwr = GWR()
 
 return gwr.gwr(subquery, dep_var, ind_vars, bw, fixed, kernel, geom_col, id_col)
 
-$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
+$$ LANGUAGE plpython3u VOLATILE PARALLEL UNSAFE;
 
 
 CREATE OR REPLACE FUNCTION
@@ -33,4 +33,4 @@ gwr = GWR()
 
 return gwr.gwr_predict(subquery, dep_var, ind_vars, bw, fixed, kernel, geom_col, id_col)
 
-$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
+$$ LANGUAGE plpython3u VOLATILE PARALLEL UNSAFE;

src/pg/test/expected/01_install_test.out

@@ -1,6 +1,3 @@
--- Install dependencies
-CREATE EXTENSION plpythonu;
-CREATE EXTENSION postgis;
 -- Create role publicuser if it does not exist
 DO
 $$
@@ -15,4 +12,4 @@ BEGIN
 END
 $$ LANGUAGE plpgsql;
 -- Install the extension
-CREATE EXTENSION crankshaft VERSION 'dev';
+CREATE EXTENSION crankshaft VERSION 'dev' CASCADE;

src/pg/test/sql/01_install_test.sql

@@ -1,7 +1,3 @@
--- Install dependencies
-CREATE EXTENSION plpythonu;
-CREATE EXTENSION postgis;
-
 -- Create role publicuser if it does not exist
 DO
 $$
@@ -17,4 +13,4 @@ END
 $$ LANGUAGE plpgsql;
 
 -- Install the extension
-CREATE EXTENSION crankshaft VERSION 'dev';
+CREATE EXTENSION crankshaft VERSION 'dev' CASCADE;