From 6d50c125f96acf245f5bf3456dfebe6e0d504811 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 19 Dec 2019 16:30:23 +0100 Subject: [PATCH] WIP2: Python3 + PG12 --- release/crankshaft.control | 2 +- src/pg/Makefile | 29 +++++++++++++++++++----- src/pg/crankshaft.control | 2 +- src/pg/sql/03_random_seeds.sql | 2 +- src/pg/sql/05_segmentation.sql | 6 ++--- src/pg/sql/10_moran.sql | 12 +++++----- src/pg/sql/11_kmeans.sql | 4 ++-- src/pg/sql/11_markov.sql | 6 ++--- src/pg/sql/16_getis.sql | 2 +- src/pg/sql/21_gwr.sql | 4 ++-- src/pg/test/expected/01_install_test.out | 5 +--- src/pg/test/sql/01_install_test.sql | 6 +---- 12 files changed, 45 insertions(+), 35 deletions(-) diff --git a/release/crankshaft.control b/release/crankshaft.control index 79d68b4..8194bed 100644 --- a/release/crankshaft.control +++ b/release/crankshaft.control @@ -1,5 +1,5 @@ comment = 'CartoDB Spatial Analysis extension' default_version = '0.8.2' -requires = 'plpythonu, postgis' +requires = 'plpython3u, postgis' superuser = true schema = cdb_crankshaft diff --git a/src/pg/Makefile b/src/pg/Makefile index 0103c84..d523765 100644 --- a/src/pg/Makefile +++ b/src/pg/Makefile @@ -4,7 +4,7 @@ include ../../Makefile.global # # * install generates the control & script files into src/pg/ # and installs then into the PostgreSQL extensions directory; -# requires sudo. In additionof the current development version +# requires sudo. In addition of the current development version # named 'dev', an alias 'current' is generating for ease of # update (upgrade to 'current', then to 'dev'). # * test runs the tests for the currently generated Development @@ -20,21 +20,27 @@ SOURCES_DATA_DIR = sql SOURCES_DATA = $(wildcard $(SOURCES_DATA_DIR)/*.sql) REPLACEMENTS = -e 's/@@VERSION@@/$(EXTVERSION)/g' +ifneq ($(PYTHON3), true) + REPLACEMENTS = -e 's/@@VERSION@@/$(EXTVERSION)/g' -e +endif $(DATA): $(SOURCES_DATA) $(SED) $(REPLACEMENTS) $(SOURCES_DATA_DIR)/*.sql > $@ -ifeq ($(PG_PARALLEL), 0) +ifneq ($(PG_PARALLEL), true) $(eval TMPFILE := $(shell mktemp /tmp/$@.XXXXXXXXXX)) $(SED) -e 's/PARALLEL \= [A-Z]*,/''/g' -e 's/PARALLEL [A-Z]*/''/g' $@ > $(TMPFILE); mv $(TMPFILE) $@ endif - +ifneq ($(PYTHON3), true) + $(eval TMPFILE := $(shell mktemp /tmp/$@.XXXXXXXXXX)) + $(SED) -e 's/plpython3u/plpythonu/g' $$n > $(TMPFILE); + mv $(TMPFILE) $@ +endif TEST_DIR = test REGRESS = $(sort $(notdir $(basename $(wildcard $(TEST_DIR)/sql/*test.sql)))) REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)' -PG_CONFIG = pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) include $(PGXS) @@ -58,7 +64,7 @@ release: ../../release/$(EXTENSION).control $(SOURCES_DATA) # If needed remove PARALLEL tags from the release files release_remove_parallel_deploy: -ifeq ($(PG_PARALLEL), 0) +ifneq ($(PG_PARALLEL), true) for n in $(wildcard ../../release/*.sql); do \ $(eval TMPFILE := $(shell mktemp /tmp/XXXXXXXXXX)) \ $(SED) -e 's/PARALLEL \= [A-Z]*,/''/g' -e 's/PARALLEL [A-Z]*/''/g' $$n > $(TMPFILE); \ @@ -66,8 +72,19 @@ ifeq ($(PG_PARALLEL), 0) done endif +release_adapt_plpython: +ifneq ($(PYTHON3), true) + for n in $(wildcard ../../release/*.sql); do \ + $(eval TMPFILE := $(shell mktemp /tmp/XXXXXXXXXX)) \ + $(SED) -e 's/plpython3u/plpythonu/g' $$n > $(TMPFILE); \ + mv $(TMPFILE) $$n; \ + done +endif + # Install the current release into the PostgreSQL extensions directory -deploy: release_remove_parallel_deploy +# We execute these 3 tasks to modify release files in place so they are compatible with +# the PG and python releases available during deployment, and not release +deploy: release_remove_parallel_deploy release_adapt_plpython ../../release/$(EXTENSION).control $(INSTALL_DATA) ../../release/$(EXTENSION).control '$(DESTDIR)$(datadir)/extension/' $(INSTALL_DATA) ../../release/*.sql '$(DESTDIR)$(datadir)/extension/' diff --git a/src/pg/crankshaft.control b/src/pg/crankshaft.control index 79d68b4..8194bed 100644 --- a/src/pg/crankshaft.control +++ b/src/pg/crankshaft.control @@ -1,5 +1,5 @@ comment = 'CartoDB Spatial Analysis extension' default_version = '0.8.2' -requires = 'plpythonu, postgis' +requires = 'plpython3u, postgis' superuser = true schema = cdb_crankshaft diff --git a/src/pg/sql/03_random_seeds.sql b/src/pg/sql/03_random_seeds.sql index 6607398..e33d546 100644 --- a/src/pg/sql/03_random_seeds.sql +++ b/src/pg/sql/03_random_seeds.sql @@ -6,4 +6,4 @@ _cdb_random_seeds (seed_value INTEGER) RETURNS VOID AS $$ from crankshaft import random_seeds random_seeds.set_random_seeds(seed_value) -$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plpython3u VOLATILE PARALLEL UNSAFE; diff --git a/src/pg/sql/05_segmentation.sql b/src/pg/sql/05_segmentation.sql index 4a0cfa0..9b0eb08 100644 --- a/src/pg/sql/05_segmentation.sql +++ b/src/pg/sql/05_segmentation.sql @@ -35,7 +35,7 @@ AS $$ target_ids, model_params) -$$ LANGUAGE plpythonu VOLATILE PARALLEL RESTRICTED; +$$ LANGUAGE plpython3u VOLATILE PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION CDB_CreateAndPredictSegment( @@ -68,7 +68,7 @@ AS $$ target_table, model_params ) -$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plpython3u VOLATILE PARALLEL UNSAFE; CREATE OR REPLACE FUNCTION CDB_CreateAndPredictSegment( @@ -99,4 +99,4 @@ AS $$ target_query, model_params ) -$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plpython3u VOLATILE PARALLEL UNSAFE; diff --git a/src/pg/sql/10_moran.sql b/src/pg/sql/10_moran.sql index bc75aff..4a13c52 100644 --- a/src/pg/sql/10_moran.sql +++ b/src/pg/sql/10_moran.sql @@ -15,7 +15,7 @@ AS $$ moran = Moran() return moran.global_stat(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col) -$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plpython3u VOLATILE PARALLEL UNSAFE; -- Moran's I Local (internal function) - DEPRECATED CREATE OR REPLACE FUNCTION @@ -40,7 +40,7 @@ AS $$ num_ngbrs, permutations, geom_col, id_col) # remove spatial lag return [(r[6], r[0], r[1], r[7], r[5]) for r in result] -$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plpython3u VOLATILE PARALLEL UNSAFE; -- Moran's I Local (internal function) CREATE OR REPLACE FUNCTION @@ -68,7 +68,7 @@ moran = Moran() return moran.local_stat(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col) -$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plpython3u VOLATILE PARALLEL UNSAFE; -- Moran's I Local (public-facing function) @@ -195,7 +195,7 @@ AS $$ # TODO: use named parameters or a dictionary return moran.global_rate_stat(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) -$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plpython3u VOLATILE PARALLEL UNSAFE; -- Moran's I Local Rate (internal function) - DEPRECATED @@ -223,7 +223,7 @@ AS $$ result = moran.local_rate_stat(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) # remove spatial lag return [(r[6], r[0], r[1], r[7], r[4]) for r in result] -$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plpython3u VOLATILE PARALLEL UNSAFE; -- Moran's I Local Rate (public-facing function) - DEPRECATED CREATE OR REPLACE FUNCTION @@ -279,7 +279,7 @@ return moran.local_rate_stat( geom_col, id_col ) -$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plpython3u VOLATILE PARALLEL UNSAFE; -- Moran's I Rate -- Replaces CDB_AreasOfInterestLocalRate diff --git a/src/pg/sql/11_kmeans.sql b/src/pg/sql/11_kmeans.sql index f17e395..cf0311e 100644 --- a/src/pg/sql/11_kmeans.sql +++ b/src/pg/sql/11_kmeans.sql @@ -14,7 +14,7 @@ from crankshaft.clustering import Kmeans kmeans = Kmeans() return kmeans.spatial(query, no_clusters, no_init) -$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plpython3u VOLATILE PARALLEL UNSAFE; -- Non-spatial k-means clustering -- query: sql query to retrieve all the needed data @@ -44,7 +44,7 @@ kmeans = Kmeans() return kmeans.nonspatial(query, colnames, no_clusters, standardize=standardize, id_col=id_col) -$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plpython3u VOLATILE PARALLEL UNSAFE; CREATE OR REPLACE FUNCTION CDB_WeightedMeanS( diff --git a/src/pg/sql/11_markov.sql b/src/pg/sql/11_markov.sql index 0a930c3..e74d9b2 100644 --- a/src/pg/sql/11_markov.sql +++ b/src/pg/sql/11_markov.sql @@ -27,7 +27,7 @@ AS $$ ## TODO: use named parameters or a dictionary return markov.spatial_trend(subquery, time_cols, num_classes, w_type, num_ngbrs, permutations, geom_col, id_col) -$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plpython3u VOLATILE PARALLEL UNSAFE; -- input table format: identical to above but in a predictable format -- Sample function call: @@ -53,7 +53,7 @@ $$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; -- from crankshaft.clustering import moran_local -- # TODO: use named parameters or a dictionary -- return spatial_markov(subquery, time_cols, permutations, geom_column, id_col, w_type, num_ngbrs) --- $$ LANGUAGE plpythonu; +-- $$ LANGUAGE plpython3u; -- -- -- input table format: -- -- id | geom | date | measurement @@ -79,4 +79,4 @@ $$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; -- from crankshaft.clustering import moran_local -- # TODO: use named parameters or a dictionary -- return spatial_markov(subquery, time_cols, permutations, geom_column, id_col, w_type, num_ngbrs) --- $$ LANGUAGE plpythonu; +-- $$ LANGUAGE plpython3u; diff --git a/src/pg/sql/16_getis.sql b/src/pg/sql/16_getis.sql index b5feb9f..762a228 100644 --- a/src/pg/sql/16_getis.sql +++ b/src/pg/sql/16_getis.sql @@ -14,6 +14,6 @@ AS $$ from crankshaft.clustering import Getis getis = Getis() return getis.getis_ord(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col) -$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plpython3u VOLATILE PARALLEL UNSAFE; -- TODO: make a version that accepts the values as arrays diff --git a/src/pg/sql/21_gwr.sql b/src/pg/sql/21_gwr.sql index 2fcad7a..7d29b45 100644 --- a/src/pg/sql/21_gwr.sql +++ b/src/pg/sql/21_gwr.sql @@ -15,7 +15,7 @@ gwr = GWR() return gwr.gwr(subquery, dep_var, ind_vars, bw, fixed, kernel, geom_col, id_col) -$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plpython3u VOLATILE PARALLEL UNSAFE; CREATE OR REPLACE FUNCTION @@ -33,4 +33,4 @@ gwr = GWR() return gwr.gwr_predict(subquery, dep_var, ind_vars, bw, fixed, kernel, geom_col, id_col) -$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE; +$$ LANGUAGE plpython3u VOLATILE PARALLEL UNSAFE; diff --git a/src/pg/test/expected/01_install_test.out b/src/pg/test/expected/01_install_test.out index 79ff047..1d48563 100644 --- a/src/pg/test/expected/01_install_test.out +++ b/src/pg/test/expected/01_install_test.out @@ -1,6 +1,3 @@ --- Install dependencies -CREATE EXTENSION plpythonu; -CREATE EXTENSION postgis; -- Create role publicuser if it does not exist DO $$ @@ -15,4 +12,4 @@ BEGIN END $$ LANGUAGE plpgsql; -- Install the extension -CREATE EXTENSION crankshaft VERSION 'dev'; +CREATE EXTENSION crankshaft VERSION 'dev' CASCADE; diff --git a/src/pg/test/sql/01_install_test.sql b/src/pg/test/sql/01_install_test.sql index 98d7db4..dc97bfa 100644 --- a/src/pg/test/sql/01_install_test.sql +++ b/src/pg/test/sql/01_install_test.sql @@ -1,7 +1,3 @@ --- Install dependencies -CREATE EXTENSION plpythonu; -CREATE EXTENSION postgis; - -- Create role publicuser if it does not exist DO $$ @@ -17,4 +13,4 @@ END $$ LANGUAGE plpgsql; -- Install the extension -CREATE EXTENSION crankshaft VERSION 'dev'; +CREATE EXTENSION crankshaft VERSION 'dev' CASCADE;