cartodb/crankshaft
1
0
Fork 0
mirror of https://github.com/CartoDB/crankshaft.git synced 2024-11-01 10:20:48 +08:00
Code Issues Packages Projects Releases Wiki Activity

WIP2: Python3 + PG12

Browse Source
This commit is contained in:
Raúl Marín 2019-12-19 16:30:23 +01:00
parent dfdbacfb30
commit 6d50c125f9
12 changed files with 45 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
release/crankshaft.control
View File

@ -1,5 +1,5 @@
comment = 'CartoDB Spatial Analysis extension'
default_version = '0.8.2'
requires = 'plpythonu, postgis'
requires = 'plpython3u, postgis'
superuser = true
schema = cdb_crankshaft

27
src/pg/Makefile
View File

@ -20,21 +20,27 @@ SOURCES_DATA_DIR = sql
SOURCES_DATA = $(wildcard $(SOURCES_DATA_DIR)/*.sql)
REPLACEMENTS = -e 's/@@VERSION@@/$(EXTVERSION)/g'
ifneq ($(PYTHON3), true)
REPLACEMENTS = -e 's/@@VERSION@@/$(EXTVERSION)/g' -e
endif
$(DATA): $(SOURCES_DATA)
$(SED) $(REPLACEMENTS) $(SOURCES_DATA_DIR)/*.sql > $@
ifeq ($(PG_PARALLEL), 0)
ifneq ($(PG_PARALLEL), true)
$(eval TMPFILE := $(shell mktemp /tmp/$@.XXXXXXXXXX))
$(SED) -e 's/PARALLEL \= [A-Z]*,/''/g' -e 's/PARALLEL [A-Z]*/''/g' $@ > $(TMPFILE);
mv $(TMPFILE) $@
endif
ifneq ($(PYTHON3), true)
$(eval TMPFILE := $(shell mktemp /tmp/$@.XXXXXXXXXX))
$(SED) -e 's/plpython3u/plpythonu/g' $$n > $(TMPFILE);
mv $(TMPFILE) $@
endif
TEST_DIR = test
REGRESS = $(sort $(notdir $(basename $(wildcard $(TEST_DIR)/sql/*test.sql))))
REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)'
PG_CONFIG = pg_config
PGXS := $(shell $(PG_CONFIG) --pgxs)
include $(PGXS)
@ -58,7 +64,7 @@ release: ../../release/$(EXTENSION).control $(SOURCES_DATA)
# If needed remove PARALLEL tags from the release files
release_remove_parallel_deploy:
ifeq ($(PG_PARALLEL), 0)
ifneq ($(PG_PARALLEL), true)
for n in $(wildcard ../../release/*.sql); do \
$(eval TMPFILE := $(shell mktemp /tmp/XXXXXXXXXX)) \
$(SED) -e 's/PARALLEL \= [A-Z]*,/''/g' -e 's/PARALLEL [A-Z]*/''/g' $$n > $(TMPFILE); \
@ -66,8 +72,19 @@ ifeq ($(PG_PARALLEL), 0)
done
endif
release_adapt_plpython:
ifneq ($(PYTHON3), true)
for n in $(wildcard ../../release/*.sql); do \
$(eval TMPFILE := $(shell mktemp /tmp/XXXXXXXXXX)) \
$(SED) -e 's/plpython3u/plpythonu/g' $$n > $(TMPFILE); \
mv $(TMPFILE) $$n; \
done
endif
# Install the current release into the PostgreSQL extensions directory
deploy: release_remove_parallel_deploy
# We execute these 3 tasks to modify release files in place so they are compatible with
# the PG and python releases available during deployment, and not release
deploy: release_remove_parallel_deploy release_adapt_plpython ../../release/$(EXTENSION).control
$(INSTALL_DATA) ../../release/$(EXTENSION).control '$(DESTDIR)$(datadir)/extension/'
$(INSTALL_DATA) ../../release/*.sql '$(DESTDIR)$(datadir)/extension/'

2
src/pg/crankshaft.control
View File

@ -1,5 +1,5 @@
comment = 'CartoDB Spatial Analysis extension'
default_version = '0.8.2'
requires = 'plpythonu, postgis'
requires = 'plpython3u, postgis'
superuser = true
schema = cdb_crankshaft

2
src/pg/sql/03_random_seeds.sql
View File

@ -6,4 +6,4 @@ _cdb_random_seeds (seed_value INTEGER) RETURNS VOID
AS $$
from crankshaft import random_seeds
random_seeds.set_random_seeds(seed_value)
$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
$$ LANGUAGE plpython3u VOLATILE PARALLEL UNSAFE;

6
src/pg/sql/05_segmentation.sql
View File

@ -35,7 +35,7 @@ AS $$
target_ids,
model_params)
$$ LANGUAGE plpythonu VOLATILE PARALLEL RESTRICTED;
$$ LANGUAGE plpython3u VOLATILE PARALLEL RESTRICTED;
CREATE OR REPLACE FUNCTION
CDB_CreateAndPredictSegment(
@ -68,7 +68,7 @@ AS $$
target_table,
model_params
)
$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
$$ LANGUAGE plpython3u VOLATILE PARALLEL UNSAFE;
CREATE OR REPLACE FUNCTION
CDB_CreateAndPredictSegment(
@ -99,4 +99,4 @@ AS $$
target_query,
model_params
)
$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
$$ LANGUAGE plpython3u VOLATILE PARALLEL UNSAFE;

12
src/pg/sql/10_moran.sql
View File

@ -15,7 +15,7 @@ AS $$
moran = Moran()
return moran.global_stat(subquery, column_name, w_type,
num_ngbrs, permutations, geom_col, id_col)
$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
$$ LANGUAGE plpython3u VOLATILE PARALLEL UNSAFE;
-- Moran's I Local (internal function) - DEPRECATED
CREATE OR REPLACE FUNCTION
@ -40,7 +40,7 @@ AS $$
num_ngbrs, permutations, geom_col, id_col)
# remove spatial lag
return [(r[6], r[0], r[1], r[7], r[5]) for r in result]
$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
$$ LANGUAGE plpython3u VOLATILE PARALLEL UNSAFE;
-- Moran's I Local (internal function)
CREATE OR REPLACE FUNCTION
@ -68,7 +68,7 @@ moran = Moran()
return moran.local_stat(subquery, column_name, w_type,
num_ngbrs, permutations, geom_col, id_col)
$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
$$ LANGUAGE plpython3u VOLATILE PARALLEL UNSAFE;
-- Moran's I Local (public-facing function)
@ -195,7 +195,7 @@ AS $$
# TODO: use named parameters or a dictionary
return moran.global_rate_stat(subquery, numerator, denominator, w_type,
num_ngbrs, permutations, geom_col, id_col)
$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
$$ LANGUAGE plpython3u VOLATILE PARALLEL UNSAFE;
-- Moran's I Local Rate (internal function) - DEPRECATED
@ -223,7 +223,7 @@ AS $$
result = moran.local_rate_stat(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col)
# remove spatial lag
return [(r[6], r[0], r[1], r[7], r[4]) for r in result]
$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
$$ LANGUAGE plpython3u VOLATILE PARALLEL UNSAFE;
-- Moran's I Local Rate (public-facing function) - DEPRECATED
CREATE OR REPLACE FUNCTION
@ -279,7 +279,7 @@ return moran.local_rate_stat(
geom_col,
id_col
)
$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
$$ LANGUAGE plpython3u VOLATILE PARALLEL UNSAFE;
-- Moran's I Rate
-- Replaces CDB_AreasOfInterestLocalRate

4
src/pg/sql/11_kmeans.sql
View File

@ -14,7 +14,7 @@ from crankshaft.clustering import Kmeans
kmeans = Kmeans()
return kmeans.spatial(query, no_clusters, no_init)
$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
$$ LANGUAGE plpython3u VOLATILE PARALLEL UNSAFE;
-- Non-spatial k-means clustering
-- query: sql query to retrieve all the needed data
@ -44,7 +44,7 @@ kmeans = Kmeans()
return kmeans.nonspatial(query, colnames, no_clusters,
standardize=standardize,
id_col=id_col)
$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
$$ LANGUAGE plpython3u VOLATILE PARALLEL UNSAFE;
CREATE OR REPLACE FUNCTION CDB_WeightedMeanS(

6
src/pg/sql/11_markov.sql
View File

@ -27,7 +27,7 @@ AS $$
## TODO: use named parameters or a dictionary
return markov.spatial_trend(subquery, time_cols, num_classes, w_type, num_ngbrs, permutations, geom_col, id_col)
$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
$$ LANGUAGE plpython3u VOLATILE PARALLEL UNSAFE;
-- input table format: identical to above but in a predictable format
-- Sample function call:
@ -53,7 +53,7 @@ $$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
-- from crankshaft.clustering import moran_local
-- # TODO: use named parameters or a dictionary
-- return spatial_markov(subquery, time_cols, permutations, geom_column, id_col, w_type, num_ngbrs)
-- $$ LANGUAGE plpythonu;
-- $$ LANGUAGE plpython3u;
--
-- -- input table format:
-- -- id | geom | date | measurement
@ -79,4 +79,4 @@ $$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
-- from crankshaft.clustering import moran_local
-- # TODO: use named parameters or a dictionary
-- return spatial_markov(subquery, time_cols, permutations, geom_column, id_col, w_type, num_ngbrs)
-- $$ LANGUAGE plpythonu;
-- $$ LANGUAGE plpython3u;

2
src/pg/sql/16_getis.sql
View File

@ -14,6 +14,6 @@ AS $$
from crankshaft.clustering import Getis
getis = Getis()
return getis.getis_ord(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col)
$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
$$ LANGUAGE plpython3u VOLATILE PARALLEL UNSAFE;
-- TODO: make a version that accepts the values as arrays

4
src/pg/sql/21_gwr.sql
View File

@ -15,7 +15,7 @@ gwr = GWR()
return gwr.gwr(subquery, dep_var, ind_vars, bw, fixed, kernel, geom_col, id_col)
$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
$$ LANGUAGE plpython3u VOLATILE PARALLEL UNSAFE;
CREATE OR REPLACE FUNCTION
@ -33,4 +33,4 @@ gwr = GWR()
return gwr.gwr_predict(subquery, dep_var, ind_vars, bw, fixed, kernel, geom_col, id_col)
$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
$$ LANGUAGE plpython3u VOLATILE PARALLEL UNSAFE;

5
src/pg/test/expected/01_install_test.out
View File

@ -1,6 +1,3 @@
-- Install dependencies
CREATE EXTENSION plpythonu;
CREATE EXTENSION postgis;
-- Create role publicuser if it does not exist
DO
$$
@ -15,4 +12,4 @@ BEGIN
END
$$ LANGUAGE plpgsql;
-- Install the extension
CREATE EXTENSION crankshaft VERSION 'dev';
CREATE EXTENSION crankshaft VERSION 'dev' CASCADE;

6
src/pg/test/sql/01_install_test.sql
View File

@ -1,7 +1,3 @@
-- Install dependencies
CREATE EXTENSION plpythonu;
CREATE EXTENSION postgis;
-- Create role publicuser if it does not exist
DO
$$
@ -17,4 +13,4 @@ END
$$ LANGUAGE plpgsql;
-- Install the extension
CREATE EXTENSION crankshaft VERSION 'dev';
CREATE EXTENSION crankshaft VERSION 'dev' CASCADE;