Release tasks using release-specific virtual environments

Refine the development process and define the procedure for
releasing new versions.

.gitignore

@@ -0,0 +1 @@
+envs/

Makefile

@@ -3,11 +3,31 @@ PYP_DIR = src/py
 
 .PHONY: install
 .PHONY: run_tests
+.PHONY: release
+.PHONY: deploy
 
+
+# Generate and install developmet versions of the extension
+# and python package.
+# The extension is named 'dev' with a 'current' alias for easily upgrading.
+# The Python package is installed in a virtual environment envs/dev/
+# Requires sudo.
 install:
 	$(MAKE) -C $(PYP_DIR) install
 	$(MAKE) -C $(EXT_DIR) install
 
-testinstalled:
-	$(MAKE) -C $(PYP_DIR) testinstalled
-	$(MAKE) -C $(EXT_DIR) installcheck
+# Run the tests for the installed development extension and
+# python package
+test:
+	$(MAKE) -C $(PYP_DIR) test
+	$(MAKE) -C $(EXT_DIR) test
+
+# Generate a new release into release
+release:
+	$(MAKE) -C $(EXT_DIR) release
+
+# Install the current release.
+# The Python package is installed in a virtual environment envs/X.Y.Z/
+# Requires sudo.
+deploy:
+	$(MAKE) -C $(EXT_DIR) deploy

NEWS.md

@@ -0,0 +1,3 @@
+0.0.1 (2016-03-15)
+------------------
+* Preliminar release

README.md

@@ -9,6 +9,7 @@ CartoDB Spatial Analysis extension for PostgreSQL.
 * - *src/pg* contains the PostgreSQL extension source code
 * - *src/py* Python module source code
 * *release* reseleased versions
+* *env* base directory for Python virtual environments
 
 ## Requirements
 
@@ -17,16 +18,24 @@ CartoDB Spatial Analysis extension for PostgreSQL.
 
 # Working Process
 
+We use the default `develop` branch as the basis for development.
+This branch and `master` are maintained by the *Release Manager*.
+The `master` branch is used to merge and tag releases to be
+deployed in production.
+
+In addition to these two permanent branches, temporal *topic*
+branches will be used for all modifications.
+
 ## Development
 
-Work in `src/pg/sql`, `src/py/crankshaft`;
-use a topic branch. See src/py/README.md
-for the procedure to work with the Python local environment.
+A topic branch should be created out of the `develop` branch
+and be used for the development process; see src/py/README.md
 
+Modifications are done inside `src/pg/sql` and `src/py/crankshaft`.
 Take into account:
 
-*  Always remember to add tests for any new functionality
-   documentation.
+*  Always remember to add tests (`src/pg/test`, `src/py/crankshaft/test`)
+   for any new functionality.
 *  Add or modify the corresponding documentation files in the `doc` folder.
    Since we expect to have highly technical functions here, an extense
    background explanation would be of great help to users of this extension.
@@ -35,12 +44,22 @@ Take into account:
    and private functions (to be used only internally inside
    the extension)  with `_cdb_`.
 
-Update local installation with `sudo make install`
-(this will update the 'dev' version of the extension in 'src/pg/')
+Update the local development installation with `sudo make install`.
+This will update the 'dev' version of the extension in 'src/pg/' and
+make it available to PostgreSQL.
+It will also install the python package (crankshaft) in a virtual
+environment `env/dev`.
 
-Run the tests with `PGUSER=postgres make test`
+Run the tests with `make test`
 
-Update extension in working database with
+To use the python extension for custom tests, activate the virtual
+environment with:
+
+```
+source envs/dev/bin/activate
+```
+
+Update extension in a working database with:
 
 * `ALTER EXTENSION crankshaft VERSION TO 'current';`
   `ALTER EXTENSION crankshaft VERSION TO 'dev';`
@@ -57,31 +76,37 @@ we can:
 
 * `CREATE EXTENSION crankshaft WITH VERSION 'dev';`
 
+Note: the development extension uses the development pyhton virtual
+environment automatically.
+
 Once the tests are succeeding a new Pull-Request can be created.
-CI-tests must be checked to be successfull.
-
-Before merging a topic branch peer code reviewing of the code is a must.
+CI-tests must be checked to be successful.
 
+Before proceeding to the release process peer code reviewing of the code is a must.
 
 ## Release
 
 The release process of a new version of the extension
 shall by performed by the designated *Release Manager*.
 
-Note that we expect to gradually automate this process.
+Note that we expect to gradually automate more of this process.
 
-Having checkout the topic branch of the PR to be released:
+Having checked the topic branch of the PR to be released it shall be
+merged back into the `develop` branch to prepare the new release.
 
 The version number in `pg/cranckshaft.control` must first be updated.
 To do so [Semantic Versioning 2.0](http://semver.org/) is in order.
 
+Thew `NEWS.md` will be updated.
+
 We now will explain the process for the case of backwards-compatible
 releases (updating the minor or patch version numbers).
 
 TODO: document the complex case of major releases.
 
 The next command must be executed to produce the main installation
-script for the new release, `release/cranckshaft--X.Y.Z.sql`.
+script for the new release, `release/cranckshaft--X.Y.Z.sql` and
+also to copy the python package to `release/python/X.Y.Z/crankshaft`.
 
 ```
 make release
@@ -93,10 +118,32 @@ releases this simply consist in extracting the functions that have changed
 and placing them in the proper `release/cranckshaft--X.Y.Z--A.B.C.sql`
 file.
 
-TODO: configure the local enviroment to be used by the release;
-currently should be directory `src/py/X.Y.Z`, but this must be fixed;
-a possibility to explore is to use the `cdb_conf` table.
+The new release can be deployed for staging/smoke tests with this command:
 
-TODO: testing procedure for the new release
+```
+sudo make deploy
+```
 
-TODO: push, merge, tag, deploy procedures.
+This will make the 'X.Y.Z' version of the extension to PostgreSQL.
+The corresponding Python extension will be installed in a
+virtual environment in `envs/X.Y.Z`
+
+It can be activated with:
+
+```
+source envs/X.Y.Z/bin/activate
+```
+
+But note that this is needed only for using the package directly;
+the 'X.Y.Z' version of the extension will automatically use the
+python package from this virtual environment.
+
+The `sudo make deploy` operation can be also used for installing
+the new version after it has been released.
+
+TODO: testing procedure for the new release.
+
+TODO: procedure for staging deployment.
+
+TODO: procedure for merging to master, tagging and deploying
+in production.

TODO.md

@@ -1,9 +0,0 @@
-* [x] Support versioning
-* [x] Test use of `plpy` from python Package
-* [x] Add `pysal` etc. dependencies
-* [x] Define documentation practices (general, per extension/package?)
-* [x] Add initial function set (WIP)
-* Unify style of function comments
-* [x] Add integration tests
-* Make target to open a new version development (create symlinks, etc.)
-* [x] Should add cartodb ext. as a dependency?

src/pg/Makefile

@@ -1,9 +1,13 @@
-# Generation of a new development version 'dev' (with an alias 'current' for
-# updating easily by upgrading to 'current', then 'dev')
-
-# sudo make install -- generate the 'dev' version from current source
-#                      and make it available to PostgreSQL
-# PGUSER=postgres make installcheck -- test the 'dev' extension
+# Development tasks:
+#
+# * install generates the control & script files into src/pg/
+#   and installs then into the PostgreSQL extensions directory;
+#   requires sudo. In additionof the current development version
+#   named 'dev', an alias 'current' is generating for ease of
+#   update (upgrade to 'current', then to 'dev').
+#   the python module is installed in a virtualenv in envs/dev/
+# * test runs the tests for the currently generated Development
+#   extension.
 
 SED = sed
 
@@ -16,7 +20,7 @@ DATA         = $(EXTENSION)--dev.sql \
 SOURCES_DATA_DIR = sql
 SOURCES_DATA = $(wildcard $(SOURCES_DATA_DIR)/*.sql)
 
-VIRTUALENV_PATH = $(realpath ../py/)
+VIRTUALENV_PATH = $(realpath ../../envs)
 ESC_VIRVIRTUALENV_PATH = $(subst /,\/,$(VIRTUALENV_PATH))
 
 REPLACEMENTS = -e 's/@@VERSION@@/$(EXTVERSION)/g' \
@@ -36,13 +40,46 @@ include $(PGXS)
 # This seems to be needed at least for PG 9.3.11
 all: $(DATA)
 
+test: export PGUSER=postgres
+test: installcheck
 
-# WIP: goals for releasing the extension...
+# Release tasks
 
+PACKAGE = crankshaft
 EXTVERSION   = $(shell grep default_version $(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/")
 
-../release/$(EXTENSION).control: $(EXTENSION).control
+../../release/$(EXTENSION).control: $(EXTENSION).control
 	cp $< $@
 
-release: ../release/$(EXTENSION).control
-	cp $(EXTENSION)--dev.sql $(EXTENSION)--$(EXTVERSION).sql
+# Prepare new release from the currently installed development version,
+# for the current version X.Y.Z (defined in the control file)
+# producing the extension script and control files in releases/
+# and the python package in releases/python/X.Y.Z/crankshaft/
+release: ../../release/$(EXTENSION).control
+	cp $(EXTENSION)--dev.sql ../../release/$(EXTENSION)--$(EXTVERSION).sql
+	mkdir -p ../../release/python/$(EXTVERSION)
+	cp -r ../py/$(PACKAGE) ../../release/python/$(EXTVERSION)/
+	$(SED) -i -r 's/version='"'"'[0-9]+\.[0-9]+\.[0-9]+'"'"'/version='"'"'$(EXTVERSION)'"'"'/g'  ../../release/python/$(EXTVERSION)/$(PACKAGE)/setup.py
+
+# Install the current relese into the PostgreSQL extensions directory
+# and the Python package in a virtual environment envs/X.Y.Z
+deploy:
+	$(INSTALL_DATA) ../../release/$(EXTENSION).control '$(DESTDIR)$(datadir)/extension/'
+	$(INSTALL_DATA) ../../release/*.sql '$(DESTDIR)$(datadir)/extension/'
+	# TODO: install also upgrade/downgrade files (manually generated)
+	virtualenv --system-site-packages $(VIRTUALENV_PATH)/$(EXTVERSION)
+	$(VIRTUALENV_PATH)/$(EXTVERSION)/bin/pip install -I -U ../../release/python/$(EXTVERSION)/$(PACKAGE)
+	$(VIRTUALENV_PATH)/$(EXTVERSION)/bin/pip install -I nose
+
+clean-dev:
+	rm $(EXTNAME)--*.sql
+
+clean-releases:
+	rm -rf ../../release/python/*
+	rm ../../release/$(EXTNAME)--*.sql
+	rm ../../release/$(EXTNAME).control
+
+clean-environments:
+	rm -rf ../../envs/*
+
+clean-all: clean-dev clean-releases clean-environments

src/pg/README.md

@@ -1,7 +0,0 @@
-
-# Running the tests:
-
-```
-sudo make install
-PGUSER=postgres make installcheck
-```

src/py/.gitignore

@@ -1,2 +1 @@
 *.pyc
-dev/

src/py/Makefile

@@ -1,9 +1,10 @@
 # Install the package locally for development
 install:
-	virtualenv --system-site-packages dev
-	./dev/bin/pip install -I ./crankshaft
-	./dev/bin/pip install -I nose
+	virtualenv --system-site-packages ../../envs/dev
+	# source ../../envs/dev/bin/activate
+	../../envs/dev/bin/pip install -I ./crankshaft
+	../../envs/dev/bin/pip install -I nose
 
 # Test develpment install
-testinstalled:
-	./dev/bin/nosetests crankshaft/test/
+test:
+	../../envs/dev/bin/nosetests crankshaft/test/

src/py/crankshaft/setup.py

@@ -10,7 +10,7 @@ from setuptools import setup, find_packages
 setup(
     name='crankshaft',
 
-    version='0.0.1',
+    version='0.0.0',
 
     description='CartoDB Spatial Analysis Python Library',