From e801c9cb60cdb2774238da313bdbfd6c58bb21c7 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Tue, 15 Mar 2016 18:48:46 +0100 Subject: [PATCH] Release tasks using release-specific virtual environments Refine the development process and define the procedure for releasing new versions. --- .gitignore | 1 + Makefile | 26 ++++++++++-- NEWS.md | 3 ++ README.md | 87 +++++++++++++++++++++++++++++--------- TODO.md | 9 ---- release/.gitignore | 0 release/python/.gitignore | 0 src/pg/Makefile | 59 +++++++++++++++++++++----- src/pg/README.md | 7 --- src/py/.gitignore | 1 - src/py/Makefile | 11 ++--- src/py/crankshaft/setup.py | 2 +- 12 files changed, 149 insertions(+), 57 deletions(-) create mode 100644 .gitignore create mode 100644 NEWS.md delete mode 100644 TODO.md create mode 100644 release/.gitignore create mode 100644 release/python/.gitignore delete mode 100644 src/pg/README.md diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..37819f7 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +envs/ diff --git a/Makefile b/Makefile index 5cae2cb..43b8480 100644 --- a/Makefile +++ b/Makefile @@ -3,11 +3,31 @@ PYP_DIR = src/py .PHONY: install .PHONY: run_tests +.PHONY: release +.PHONY: deploy + +# Generate and install developmet versions of the extension +# and python package. +# The extension is named 'dev' with a 'current' alias for easily upgrading. +# The Python package is installed in a virtual environment envs/dev/ +# Requires sudo. install: $(MAKE) -C $(PYP_DIR) install $(MAKE) -C $(EXT_DIR) install -testinstalled: - $(MAKE) -C $(PYP_DIR) testinstalled - $(MAKE) -C $(EXT_DIR) installcheck +# Run the tests for the installed development extension and +# python package +test: + $(MAKE) -C $(PYP_DIR) test + $(MAKE) -C $(EXT_DIR) test + +# Generate a new release into release +release: + $(MAKE) -C $(EXT_DIR) release + +# Install the current release. +# The Python package is installed in a virtual environment envs/X.Y.Z/ +# Requires sudo. +deploy: + $(MAKE) -C $(EXT_DIR) deploy diff --git a/NEWS.md b/NEWS.md new file mode 100644 index 0000000..201da51 --- /dev/null +++ b/NEWS.md @@ -0,0 +1,3 @@ +0.0.1 (2016-03-15) +------------------ +* Preliminar release diff --git a/README.md b/README.md index c9dd529..75f7c2e 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ CartoDB Spatial Analysis extension for PostgreSQL. * - *src/pg* contains the PostgreSQL extension source code * - *src/py* Python module source code * *release* reseleased versions +* *env* base directory for Python virtual environments ## Requirements @@ -17,16 +18,24 @@ CartoDB Spatial Analysis extension for PostgreSQL. # Working Process +We use the default `develop` branch as the basis for development. +This branch and `master` are maintained by the *Release Manager*. +The `master` branch is used to merge and tag releases to be +deployed in production. + +In addition to these two permanent branches, temporal *topic* +branches will be used for all modifications. + ## Development -Work in `src/pg/sql`, `src/py/crankshaft`; -use a topic branch. See src/py/README.md -for the procedure to work with the Python local environment. +A topic branch should be created out of the `develop` branch +and be used for the development process; see src/py/README.md +Modifications are done inside `src/pg/sql` and `src/py/crankshaft`. Take into account: -* Always remember to add tests for any new functionality - documentation. +* Always remember to add tests (`src/pg/test`, `src/py/crankshaft/test`) + for any new functionality. * Add or modify the corresponding documentation files in the `doc` folder. Since we expect to have highly technical functions here, an extense background explanation would be of great help to users of this extension. @@ -35,12 +44,22 @@ Take into account: and private functions (to be used only internally inside the extension) with `_cdb_`. -Update local installation with `sudo make install` -(this will update the 'dev' version of the extension in 'src/pg/') +Update the local development installation with `sudo make install`. +This will update the 'dev' version of the extension in 'src/pg/' and +make it available to PostgreSQL. +It will also install the python package (crankshaft) in a virtual +environment `env/dev`. + +Run the tests with `make test` -Run the tests with `PGUSER=postgres make test` +To use the python extension for custom tests, activate the virtual +environment with: + +``` +source envs/dev/bin/activate +``` -Update extension in working database with +Update extension in a working database with: * `ALTER EXTENSION crankshaft VERSION TO 'current';` `ALTER EXTENSION crankshaft VERSION TO 'dev';` @@ -57,31 +76,37 @@ we can: * `CREATE EXTENSION crankshaft WITH VERSION 'dev';` -Once the tests are succeeding a new Pull-Request can be created. -CI-tests must be checked to be successfull. +Note: the development extension uses the development pyhton virtual +environment automatically. -Before merging a topic branch peer code reviewing of the code is a must. +Once the tests are succeeding a new Pull-Request can be created. +CI-tests must be checked to be successful. +Before proceeding to the release process peer code reviewing of the code is a must. ## Release The release process of a new version of the extension shall by performed by the designated *Release Manager*. -Note that we expect to gradually automate this process. +Note that we expect to gradually automate more of this process. -Having checkout the topic branch of the PR to be released: +Having checked the topic branch of the PR to be released it shall be +merged back into the `develop` branch to prepare the new release. The version number in `pg/cranckshaft.control` must first be updated. To do so [Semantic Versioning 2.0](http://semver.org/) is in order. +Thew `NEWS.md` will be updated. + We now will explain the process for the case of backwards-compatible releases (updating the minor or patch version numbers). TODO: document the complex case of major releases. The next command must be executed to produce the main installation -script for the new release, `release/cranckshaft--X.Y.Z.sql`. +script for the new release, `release/cranckshaft--X.Y.Z.sql` and +also to copy the python package to `release/python/X.Y.Z/crankshaft`. ``` make release @@ -93,10 +118,32 @@ releases this simply consist in extracting the functions that have changed and placing them in the proper `release/cranckshaft--X.Y.Z--A.B.C.sql` file. -TODO: configure the local enviroment to be used by the release; -currently should be directory `src/py/X.Y.Z`, but this must be fixed; -a possibility to explore is to use the `cdb_conf` table. +The new release can be deployed for staging/smoke tests with this command: + +``` +sudo make deploy +``` + +This will make the 'X.Y.Z' version of the extension to PostgreSQL. +The corresponding Python extension will be installed in a +virtual environment in `envs/X.Y.Z` + +It can be activated with: + +``` +source envs/X.Y.Z/bin/activate +``` + +But note that this is needed only for using the package directly; +the 'X.Y.Z' version of the extension will automatically use the +python package from this virtual environment. + +The `sudo make deploy` operation can be also used for installing +the new version after it has been released. + +TODO: testing procedure for the new release. -TODO: testing procedure for the new release +TODO: procedure for staging deployment. -TODO: push, merge, tag, deploy procedures. +TODO: procedure for merging to master, tagging and deploying +in production. diff --git a/TODO.md b/TODO.md deleted file mode 100644 index 8a708e2..0000000 --- a/TODO.md +++ /dev/null @@ -1,9 +0,0 @@ -* [x] Support versioning -* [x] Test use of `plpy` from python Package -* [x] Add `pysal` etc. dependencies -* [x] Define documentation practices (general, per extension/package?) -* [x] Add initial function set (WIP) -* Unify style of function comments -* [x] Add integration tests -* Make target to open a new version development (create symlinks, etc.) -* [x] Should add cartodb ext. as a dependency? diff --git a/release/.gitignore b/release/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/release/python/.gitignore b/release/python/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/src/pg/Makefile b/src/pg/Makefile index ed65eba..fea7386 100644 --- a/src/pg/Makefile +++ b/src/pg/Makefile @@ -1,9 +1,13 @@ -# Generation of a new development version 'dev' (with an alias 'current' for -# updating easily by upgrading to 'current', then 'dev') - -# sudo make install -- generate the 'dev' version from current source -# and make it available to PostgreSQL -# PGUSER=postgres make installcheck -- test the 'dev' extension +# Development tasks: +# +# * install generates the control & script files into src/pg/ +# and installs then into the PostgreSQL extensions directory; +# requires sudo. In additionof the current development version +# named 'dev', an alias 'current' is generating for ease of +# update (upgrade to 'current', then to 'dev'). +# the python module is installed in a virtualenv in envs/dev/ +# * test runs the tests for the currently generated Development +# extension. SED = sed @@ -16,7 +20,7 @@ DATA = $(EXTENSION)--dev.sql \ SOURCES_DATA_DIR = sql SOURCES_DATA = $(wildcard $(SOURCES_DATA_DIR)/*.sql) -VIRTUALENV_PATH = $(realpath ../py/) +VIRTUALENV_PATH = $(realpath ../../envs) ESC_VIRVIRTUALENV_PATH = $(subst /,\/,$(VIRTUALENV_PATH)) REPLACEMENTS = -e 's/@@VERSION@@/$(EXTVERSION)/g' \ @@ -36,13 +40,46 @@ include $(PGXS) # This seems to be needed at least for PG 9.3.11 all: $(DATA) +test: export PGUSER=postgres +test: installcheck -# WIP: goals for releasing the extension... +# Release tasks +PACKAGE = crankshaft EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/") -../release/$(EXTENSION).control: $(EXTENSION).control +../../release/$(EXTENSION).control: $(EXTENSION).control cp $< $@ -release: ../release/$(EXTENSION).control - cp $(EXTENSION)--dev.sql $(EXTENSION)--$(EXTVERSION).sql +# Prepare new release from the currently installed development version, +# for the current version X.Y.Z (defined in the control file) +# producing the extension script and control files in releases/ +# and the python package in releases/python/X.Y.Z/crankshaft/ +release: ../../release/$(EXTENSION).control + cp $(EXTENSION)--dev.sql ../../release/$(EXTENSION)--$(EXTVERSION).sql + mkdir -p ../../release/python/$(EXTVERSION) + cp -r ../py/$(PACKAGE) ../../release/python/$(EXTVERSION)/ + $(SED) -i -r 's/version='"'"'[0-9]+\.[0-9]+\.[0-9]+'"'"'/version='"'"'$(EXTVERSION)'"'"'/g' ../../release/python/$(EXTVERSION)/$(PACKAGE)/setup.py + +# Install the current relese into the PostgreSQL extensions directory +# and the Python package in a virtual environment envs/X.Y.Z +deploy: + $(INSTALL_DATA) ../../release/$(EXTENSION).control '$(DESTDIR)$(datadir)/extension/' + $(INSTALL_DATA) ../../release/*.sql '$(DESTDIR)$(datadir)/extension/' + # TODO: install also upgrade/downgrade files (manually generated) + virtualenv --system-site-packages $(VIRTUALENV_PATH)/$(EXTVERSION) + $(VIRTUALENV_PATH)/$(EXTVERSION)/bin/pip install -I -U ../../release/python/$(EXTVERSION)/$(PACKAGE) + $(VIRTUALENV_PATH)/$(EXTVERSION)/bin/pip install -I nose + +clean-dev: + rm $(EXTNAME)--*.sql + +clean-releases: + rm -rf ../../release/python/* + rm ../../release/$(EXTNAME)--*.sql + rm ../../release/$(EXTNAME).control + +clean-environments: + rm -rf ../../envs/* + +clean-all: clean-dev clean-releases clean-environments diff --git a/src/pg/README.md b/src/pg/README.md deleted file mode 100644 index 511fdae..0000000 --- a/src/pg/README.md +++ /dev/null @@ -1,7 +0,0 @@ - -# Running the tests: - -``` -sudo make install -PGUSER=postgres make installcheck -``` diff --git a/src/py/.gitignore b/src/py/.gitignore index fb50fe5..0d20b64 100644 --- a/src/py/.gitignore +++ b/src/py/.gitignore @@ -1,2 +1 @@ *.pyc -dev/ diff --git a/src/py/Makefile b/src/py/Makefile index 16be269..ab787ee 100644 --- a/src/py/Makefile +++ b/src/py/Makefile @@ -1,9 +1,10 @@ # Install the package locally for development install: - virtualenv --system-site-packages dev - ./dev/bin/pip install -I ./crankshaft - ./dev/bin/pip install -I nose + virtualenv --system-site-packages ../../envs/dev + # source ../../envs/dev/bin/activate + ../../envs/dev/bin/pip install -I ./crankshaft + ../../envs/dev/bin/pip install -I nose # Test develpment install -testinstalled: - ./dev/bin/nosetests crankshaft/test/ +test: + ../../envs/dev/bin/nosetests crankshaft/test/ diff --git a/src/py/crankshaft/setup.py b/src/py/crankshaft/setup.py index aaa76a6..8d5e622 100644 --- a/src/py/crankshaft/setup.py +++ b/src/py/crankshaft/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages setup( name='crankshaft', - version='0.0.1', + version='0.0.0', description='CartoDB Spatial Analysis Python Library',