Merge pull request #62 from CartoDB/60-remove-venv-activation

Remove virtualenv activation #60

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,6 +1,6 @@
 
 - [ ] All declared geometries are `geometry(Geometry, 4326)` for general geoms, or `geometry(Point, 4326)`
-- [ ] Include python is activated for new functions. Include this before importing modules: `plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()')`
+- [ ] Existing functions in crankshaft python library called from the extension are kept at least from version N to version N+1 (to avoid breakage during upgrades).
 - [ ] Docs for public-facing functions are written
 - [ ] New functions follow the naming conventions: `CDB_NameOfFunction`. Where internal functions begin with an underscore `_`.
 - [ ] If appropriate, new functions accepts an arbitrary query as an input (see [Crankshaft Issue #6](https://github.com/CartoDB/crankshaft/issues/6) for more information)

Makefile

@@ -11,7 +11,6 @@ PYP_DIR = src/py
 # Generate and install developmet versions of the extension
 # and python package.
 # The extension is named 'dev' with a 'current' alias for easily upgrading.
-# The Python package is installed in a virtual environment envs/dev/
 # Requires sudo.
 install: ## Generate and install development version of the extension; requires sudo.
 	$(MAKE) -C $(PYP_DIR) install
@@ -29,7 +28,6 @@ release: ## Generate a new release of the extension. Only for telease manager
 	$(MAKE) -C $(PYP_DIR) release
 
 # Install the current release.
-# The Python package is installed in a virtual environment envs/X.Y.Z/
 # Requires sudo.
 # Use the RELEASE_VERSION environment variable to deploy a specific version:
 #     sudo make deploy RELEASE_VERSION=1.0.0
@@ -52,11 +50,7 @@ clean-release: ## clean up current release
 	rm -rf release/python/$(RELEASE_VERSION)
 	rm -f release/$(RELEASE_VERSION)--*.sql
 
-# Cleanup all virtual environments
+clean-all: clean-dev clean-release
-clean-environments: ## clean up all virtual environments
-	rm -rf envs/*
-
-clean-all: clean-dev clean-release clean-environments
 
 help:
 	@IFS=$$'\n' ; \

README.md

@@ -9,11 +9,10 @@ CartoDB Spatial Analysis extension for PostgreSQL.
 * - *src/pg* contains the PostgreSQL extension source code
 * - *src/py* Python module source code
 * *release* reseleased versions
-* *env* base directory for Python virtual environments
 
 ## Requirements
 
-* pip, virtualenv, PostgreSQL
+* pip, PostgreSQL
 * python-scipy system package (see [src/py/README.md](https://github.com/CartoDB/crankshaft/blob/master/src/py/README.md))
 
 # Working Process -- Quickstart Guide

src/pg/Makefile

@@ -7,7 +7,6 @@ include ../../Makefile.global
 #   requires sudo. In additionof the current development version
 #   named 'dev', an alias 'current' is generating for ease of
 #   update (upgrade to 'current', then to 'dev').
-#   the python module is installed in a virtualenv in envs/dev/
 # * test runs the tests for the currently generated Development
 #   extension.
 
@@ -18,11 +17,8 @@ DATA         = $(EXTENSION)--dev.sql \
 SOURCES_DATA_DIR = sql
 SOURCES_DATA = $(wildcard $(SOURCES_DATA_DIR)/*.sql)
 
-VIRTUALENV_PATH = $(realpath ../../envs)
-ESC_VIRVIRTUALENV_PATH = $(subst /,\/,$(VIRTUALENV_PATH))
 
-REPLACEMENTS = -e 's/@@VERSION@@/$(EXTVERSION)/g' \
+REPLACEMENTS = -e 's/@@VERSION@@/$(EXTVERSION)/g'
-               -e 's/@@VIRTUALENV_PATH@@/$(ESC_VIRVIRTUALENV_PATH)/g'
 
 $(DATA): $(SOURCES_DATA)
 	$(SED) $(REPLACEMENTS) $(SOURCES_DATA_DIR)/*.sql > $@
@@ -54,7 +50,6 @@ release: ../../release/$(EXTENSION).control $(SOURCES_DATA)
 	$(SED) $(REPLACEMENTS) $(SOURCES_DATA_DIR)/*.sql > ../../release/$(EXTENSION)--$(EXTVERSION).sql
 
 # Install the current relese into the PostgreSQL extensions directory
-# and the Python package in a virtual environment envs/X.Y.Z
 deploy:
 	$(INSTALL_DATA) ../../release/$(EXTENSION).control '$(DESTDIR)$(datadir)/extension/'
 	$(INSTALL_DATA) ../../release/*.sql '$(DESTDIR)$(datadir)/extension/'

src/pg/sql/02_py.sql

@@ -1,23 +0,0 @@
-CREATE OR REPLACE FUNCTION _cdb_crankshaft_virtualenvs_path()
-RETURNS text
-AS $$
-  BEGIN
-    -- RETURN '/opt/virtualenvs/crankshaft';
-    RETURN '@@VIRTUALENV_PATH@@';
-  END;
-$$ language plpgsql IMMUTABLE STRICT;
-
--- Use the crankshaft python module
-CREATE OR REPLACE FUNCTION _cdb_crankshaft_activate_py()
-RETURNS VOID
-AS $$
-    import os
-    # plpy.notice('%',str(os.environ))
-    # activate virtualenv
-    crankshaft_version = plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_internal_version()')[0]['_cdb_crankshaft_internal_version']
-    base_path = plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_virtualenvs_path()')[0]['_cdb_crankshaft_virtualenvs_path']
-    default_venv_path = os.path.join(base_path, crankshaft_version)
-    venv_path =  os.environ.get('CRANKSHAFT_VENV', default_venv_path)
-    activate_path = venv_path + '/bin/activate_this.py'
-    exec(open(activate_path).read(), dict(__file__=activate_path))
-$$ LANGUAGE plpythonu;

src/pg/sql/03_random_seeds.sql

@@ -4,7 +4,6 @@
 CREATE OR REPLACE FUNCTION
 _cdb_random_seeds (seed_value INTEGER) RETURNS VOID
 AS $$
-  plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()')
   from crankshaft import random_seeds
   random_seeds.set_random_seeds(seed_value)
 $$ LANGUAGE plpythonu;

src/pg/sql/10_moran.sql

@@ -10,7 +10,6 @@ CREATE OR REPLACE FUNCTION
       id_col TEXT DEFAULT 'cartodb_id')
 RETURNS TABLE (moran NUMERIC, significance NUMERIC)
 AS $$
-  plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()')
   from crankshaft.clustering import moran_local
   # TODO: use named parameters or a dictionary
   return moran(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col)
@@ -28,7 +27,6 @@ CREATE OR REPLACE FUNCTION
       id_col TEXT)
 RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC)
 AS $$
-  plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()')
   from crankshaft.clustering import moran_local
   # TODO: use named parameters or a dictionary
   return moran_local(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col)
@@ -122,7 +120,6 @@ CREATE OR REPLACE FUNCTION
       id_col TEXT DEFAULT 'cartodb_id')
 RETURNS TABLE (moran FLOAT, significance FLOAT)
 AS $$
-  plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()')
   from crankshaft.clustering import moran_local
   # TODO: use named parameters or a dictionary
   return moran_rate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col)
@@ -143,7 +140,6 @@ CREATE OR REPLACE FUNCTION
 RETURNS
 TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC)
 AS $$
-  plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()')
   from crankshaft.clustering import moran_local_rate
   # TODO: use named parameters or a dictionary
   return moran_local_rate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col)

src/py/Makefile

@@ -2,14 +2,11 @@ include ../../Makefile.global
 
 # Install the package locally for development
 install:
-	virtualenv --system-site-packages ../../envs/dev
+	pip install ./crankshaft
-	# source ../../envs/dev/bin/activate
-	../../envs/dev/bin/pip install -I ./crankshaft
-	../../envs/dev/bin/pip install -I nose
 
 # Test develpment install
 test:
-	../../envs/dev/bin/nosetests crankshaft/test/
+	nosetests crankshaft/test/
 
 release: ../../release/$(EXTENSION).control $(SOURCES_DATA)
 	mkdir -p ../../release/python/$(EXTVERSION)
@@ -17,6 +14,4 @@ release: ../../release/$(EXTENSION).control $(SOURCES_DATA)
 	$(SED) -i -r 's/version='"'"'[0-9]+\.[0-9]+\.[0-9]+'"'"'/version='"'"'$(EXTVERSION)'"'"'/g'  ../../release/python/$(EXTVERSION)/$(PACKAGE)/setup.py
 
 deploy:
-	virtualenv --system-site-packages $(VIRTUALENV_PATH)/$(RELEASE_VERSION)
+	pip install --upgrade ../../release/python/$(RELEASE_VERSION)/$(PACKAGE)
-	$(VIRTUALENV_PATH)/$(RELEASE_VERSION)/bin/pip install -I -U ../../release/python/$(RELEASE_VERSION)/$(PACKAGE)
-	$(VIRTUALENV_PATH)/$(RELEASE_VERSION)/bin/pip install -I nose

src/py/README.md

@@ -10,7 +10,6 @@ nosetests test/
 
 ## Notes about Python dependencies
 * This extension is targeted at production databases. Therefore certain restrictions must be assumed about the production environment vs other experimental environments.
-* We're using `pip` and `virtualenv` to generate a suitable isolated environment for python code that has  all the dependencies
 * Every dependency should be:
   - Added to the `setup.py` file
   - Installed through it
@@ -30,21 +29,7 @@ PySAL 1.10 or later, so we'll stick to 1.9.1.
 apt-get install -y python-scipy
 ```
 
-We'll use virtual environments to install our packages,
+#### Test the libraries
-but configued to use also system modules so that the
-mentioned scipy and numpy are used.
-
-    # Create a virtual environment for python
-    $ virtualenv --system-site-packages dev
-
-    # Activate the virtualenv
-    $ source dev/bin/activate
-
-    # Install all the requirements
-    # expect this to take a while, as it will trigger a few compilations
-    (dev) $ pip install -I ./crankshaft
-
-#### Test the libraries with that virtual env
 
 ##### Test numpy library dependency: