Merge pull request #1145 from CartoDB/no-makefile

Improve project usability
remotes/origin/1117-camshaft-update
Daniel G. Aubert 5 years ago committed by GitHub
commit c7effbccb4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

1
.gitignore vendored

@ -11,3 +11,4 @@ redis.pid
*.log
coverage/
.DS_Store
.nyc_output

@ -9,4 +9,4 @@ env:
services:
- docker
before_install: docker pull ${DOCKER_IMAGE}
script: npm run docker-test -- ${DOCKER_IMAGE} ${NODE_VERSION}
script: npm run test:docker

@ -1,11 +0,0 @@
Contributing
---
The issue tracker is at [github.com/CartoDB/Windshaft-cartodb](https://github.com/CartoDB/Windshaft-cartodb).
We love pull requests from everyone, see [Contributing to Open Source on GitHub](https://guides.github.com/activities/contributing-to-open-source/#contributing).
## Submitting Contributions
* You will need to sign a Contributor License Agreement (CLA) before making a submission. [Learn more here](https://carto.com/contributions).

@ -1,18 +0,0 @@
1. Test (make clean all check), fix if broken before proceeding
2. Ensure proper version in package.json and package-lock.json
3. Ensure NEWS section exists for the new version, review it, add release date
4. If there are modified dependencies in package.json, update them with `npm upgrade {{package_name}}@{{version}}`
5. Commit package.json, package-lock.json, NEWS
6. git tag -a Major.Minor.Patch # use NEWS section as content
7. Stub NEWS/package for next version
Versions:
Bugfix releases increment Patch component of version.
Feature releases increment Minor and set Patch to zero.
If backward compatibility is broken, increment Major and
set to zero Minor and Patch.
Branches named 'b<Major>.<Minor>' are kept for any critical
fix that might need to be shipped before next feature release
is ready.

@ -0,0 +1,16 @@
How to release:
1. Test (npm test), fix if broken before proceeding.
2. Ensure proper version in `package.json` and `package-lock.json`.
3. Ensure NEWS section exists for the new version, review it, add release date.
4. If there are modified dependencies in `package.json`, update them with `npm upgrade {{package_name}}@{{version}}`.
5. Commit `package.json`, `package-lock.json`, NEWS.
6. Run `git tag -a Major.Minor.Patch`. Use NEWS section as content.
7. Stub NEWS/package for next version.
Versions:
* Bugfix releases increment Patch component of version.
* Feature releases increment Minor and set Patch to zero.
* If backward compatibility is broken, increment Major and set to zero Minor and Patch.
* Branches named 'b<Major>.<Minor>' are kept for any critical fix that might need to be shipped before next feature release is ready.

@ -1,41 +0,0 @@
# Installing Windshaft-CartoDB
## Requirements
Make sure that you have the requirements needed. These are:
- Node 10.x
- npm 6.x
- PostgreSQL >= 10.0
- PostGIS >= 2.4
- CARTO Postgres Extension >= 0.24.1
- Redis >= 4
- libcairo2-dev, libpango1.0-dev, libjpeg8-dev and libgif-dev for server side canvas support
- C++11 (to build internal dependencies if needed)
### Optional
- Varnish (http://www.varnish-cache.org)
## PostGIS setup
A `template_postgis` database is expected. One can be set up with
```shell
createdb --owner postgres --template template0 template_postgis
psql -d template_postgis -c 'CREATE EXTENSION postgis;'
```
## Build/install
To fetch and build all node-based dependencies, run:
```shell
npm install
```
Note that the ```npm``` step will populate the node_modules/
directory with modules, some of which being compiled on demand. If you
happen to have startup errors you may need to force rebuilding those
modules. At any time just wipe out the node_modules/ directory and run
```npm``` again.

@ -1,53 +0,0 @@
SHELL=/bin/bash
pre-install:
@$(SHELL) ./scripts/check-node-canvas.sh
all:
@$(SHELL) ./scripts/install.sh
clean:
rm -rf node_modules/
distclean: clean
rm config.status*
config.status--test:
./configure --environment=test
config/environments/test.js: config.status--test
./config.status--test
TEST_SUITE := $(shell find test/{acceptance,integration,unit} -name "*.js")
TEST_SUITE_UNIT := $(shell find test/unit -name "*.js")
TEST_SUITE_INTEGRATION := $(shell find test/integration -name "*.js")
TEST_SUITE_ACCEPTANCE := $(shell find test/acceptance -name "*.js")
test: config/environments/test.js
@echo "***tests***"
@$(SHELL) ./run_tests.sh ${RUNTESTFLAGS} $(TEST_SUITE)
test-unit: config/environments/test.js
@echo "***tests***"
@$(SHELL) ./run_tests.sh ${RUNTESTFLAGS} $(TEST_SUITE_UNIT)
test-integration: config/environments/test.js
@echo "***tests***"
@$(SHELL) ./run_tests.sh ${RUNTESTFLAGS} $(TEST_SUITE_INTEGRATION)
test-acceptance: config/environments/test.js
@echo "***tests***"
@$(SHELL) ./run_tests.sh ${RUNTESTFLAGS} $(TEST_SUITE_ACCEPTANCE)
lint:
@echo "***eslint***"
@./node_modules/.bin/eslint app.js "lib/**/*.js" "test/**/*.js"
test-all: test lint
coverage:
@RUNTESTFLAGS=--with-coverage make test
check: test
.PHONY: pre-install test lint coverage

@ -1,11 +1,22 @@
# Changelog
## 8.0.1
## 8.1.0
Released 2019-mm-dd
Announcements:
- Removed `jshint` as linter in favour of `eslint` to check syntax, find problems, and enforce code style.
- Upgrade `camshaft` to [`0.65.0`](https://github.com/CartoDB/camshaft/blob/9e8e5cd09dbfc6886c40f3f4a356fb7f235e82be/CHANGELOG.md#0650): Use quoted identifiers for column names.
- Stop using two different tools for package management, testing, and any other developer workflow.
- Removes Makefile and related bash scripts
- Use npm scripts as the only tool for testing, CI and linting.
- Simplified CI configuration.
- Improved documentation:
- Centralized several documents into README.md
- Remove outdated sections
- Update old sections
- Added missing sections.
- Remove deprecated coverage tool istanbul, using nyc instead.
- Removed unused dockerfiles
## 8.0.0
Released 2019-11-13

@ -1,80 +1,126 @@
Windshaft-CartoDB
==================
# Windshaft-CartoDB [![Build Status](https://travis-ci.org/CartoDB/Windshaft-cartodb.svg?branch=master)](https://travis-ci.org/CartoDB/Windshaft-cartodb)
[![Build Status](https://travis-ci.org/CartoDB/Windshaft-cartodb.svg?branch=master)](https://travis-ci.org/CartoDB/Windshaft-cartodb)
The [`CARTO Maps API`](http://docs.cartodb.com/cartodb-platform/maps-api.html) tiler. It extends [`Windshaft`](https://github.com/CartoDB/Windshaft) and exposes a web service with extra functionality:
This is the [CartoDB Maps API](http://docs.cartodb.com/cartodb-platform/maps-api.html) tiler. It extends
[Windshaft](https://github.com/CartoDB/Windshaft) with some extra functionality and custom filters for authentication.
* Instantiate [`Anonymous Maps`](https://github.com/CartoDB/Windshaft-cartodb/blob/master/docs/guides/03-anonymous-maps.md) through CARTO's map configuration ([`MapConfig`](https://github.com/CartoDB/Windshaft/blob/master/doc/MapConfig-specification.md)).
* Create [`Named Maps`](https://github.com/CartoDB/Windshaft-cartodb/blob/master/docs/guides/04-named-maps.md) based on customizable templates.
* Get map previews through [`Static Maps`](https://github.com/CartoDB/Windshaft-cartodb/blob/master/docs/guides/05-static-maps-API.md) API.
* Render maps with a large amount of data faster using [`Tile Aggregation`](https://github.com/CartoDB/Windshaft-cartodb/blob/master/docs/guides/06-tile-aggregation.md).
* Build advanced maps with enriched data through [`Analyses Extension`](https://github.com/CartoDB/Windshaft-cartodb/blob/master/docs/guides/09-MapConfig-analyses-extension.md).
* Fetch tabular data from analysis nodes with [`Dataviews`](https://github.com/CartoDB/Windshaft-cartodb/blob/master/docs/guides/10-MapConfig-dataviews-extension.md)
* reads dbname from subdomain and cartodb redis for pretty tile urls
* configures windshaft to publish `cartodb_id` as the interactivity layer
* gets the default geometry type from the cartodb redis store
* allows tiles to be styled individually
* provides a link to varnish high speed cache
* provides a [template maps API](https://github.com/CartoDB/Windshaft-cartodb/blob/master/docs/Template-maps.md)
## Build
Install
-------
See [INSTALL.md](INSTALL.md) for detailed installation instructions.
Requirements:
Configure
---------
* [`Node 10.x (npm 6.x)`](https://nodejs.org/dist/latest-v10.x/)
* [`PostgreSQL >= 10.0`](https://www.postgresql.org/download/)
* [`PostGIS >= 2.4`](https://postgis.net/install/)
* [`CARTO Postgres Extension >= 0.24.1`](https://github.com/CartoDB/cartodb-postgresql)
* [`Redis >= 4`](https://redis.io/download)
* `libcairo2-dev`, `libpango1.0-dev`, `libjpeg8-dev` and `libgif-dev` for server side canvas support
* `C++11` to build internal dependencies. When there's no pre-built binaries for your OS/architecture distribution.
Create the config/environments/<env>.js files (there are .example files
to start from). You can optionally use the ./configure script for this,
see ```./configure --help``` to see available options.
Optional:
Look at lib/cartodb/server-options.js for more on config
* [`Varnish`](http://www.varnish-cache.org)
* [`Statsd`](https://github.com/statsd/statsd)
Upgrading
---------
### PostGIS setup
Checkout your commit/branch. If you need to reinstall dependencies (you can check [NEWS](NEWS.md)) do the following:
A `template_postgis` database is expected. One can be set up with
```sh
$ rm -rf node_modules
```shell
$ createdb --owner postgres --template template0 template_postgis
$ psql -d template_postgis -c 'CREATE EXTENSION postgis;'
```
### Install
To fetch and build all node-based dependencies, run:
```shell
$ npm install
```
### Run
Create the `./config/environments/<env>.js` file (there are `.example` files to start from). Look at `./lib/cartodb/server-options.js` for more on config.
```shell
$ node app.js <env>
```
node app.js <env>
Where `<env>` is the name of a configuration file under `./config/environments/`.
### Test
```shell
$ npm test
```
Where <env> is the name of a configuration file under config/environments/.
### Coverage
Note that caches are kept in redis. If you're not seeing what you expect
there may be out-of-sync records in there.
Take a look: http://redis.io/commands
```shell
$ npm run cover
```
Open `./coverage/lcov-report/index.html`.
Documentation
-------------
### Docker support
The [docs directory](https://github.com/CartoDB/Windshaft-cartodb/tree/master/docs) contains different documentation
resources, from higher level to more detailed ones:
The [Maps API](https://github.com/CartoDB/Windshaft-cartodb/blob/master/docs/Map-API.md) defined the endpoints and their
expected parameters and outputs.
We provide docker images just for testing and continuous integration purposes:
* [`nodejs-xenial-pg1121`](https://hub.docker.com/r/carto/nodejs-xenial-pg1121/tags)
* [`nodejs-xenial-pg101`](https://hub.docker.com/r/carto/nodejs-xenial-pg101/tags)
Examples
--------
You can find instructions to install Docker, download, and update images [here](https://github.com/CartoDB/Windshaft-cartodb/blob/master/docker/reference.md).
[CartoDB's Map Gallery](http://cartodb.com/gallery/) showcases several examples of visualisations built on top of this.
### Useful `npm` scripts
Contributing
---
Run test in a docker image with a specific Node.js version:
See [CONTRIBUTING.md](CONTRIBUTING.md).
```shell
$ DOCKER_IMAGE=<docker-image-tag> NODE_VERSION=<nodejs-version> npm run test:docker
```
### Developing with a custom windshaft version
Where:
If you plan or want to use a custom / not released yet version of windshaft (or any other dependency) the best option is
to use `npm link`. You can read more about it at [npm-link: Symlink a package folder](https://docs.npmjs.com/cli/link.html).
* `<docker-image-tag>`: the tag of required docker image, e.g. `carto/nodejs-xenial-pg1121:latest`
* `<nodejs-version>`: the Node.js version, e.g. `10.15.1`
**Quick start**:
In case you need to debug:
```shell
~/windshaft-directory $ npm install
~/windshaft-directory $ npm link
~/windshaft-cartodb-directory $ npm link windshaft
$ DOCKER_IMAGE=<docker-image-tag> npm run docker:bash
```
## Documentation
You can find an overview, guides, full reference, and support in [`CARTO's developer center`](https://carto.com/developers/maps-api/). The [docs directory](https://github.com/CartoDB/Windshaft-cartodb/tree/master/docs) contains different documentation resources, from a higher level to more detailed ones.
## Contributing
* The issue tracker: [`Github`](https://github.com/CartoDB/Windshaft-cartodb/issues).
* We love Pull Requests from everyone, see [contributing to Open Source on GitHub](https://guides.github.com/activities/contributing-to-open-source/#contributing).
* You'll need to sign a Contributor License Agreement (CLA) before submitting a Pull Request. [Learn more here](https://carto.com/contributions).
## Developing with a custom `Windshaft` version
If you plan or want to use a custom / not released yet version of windshaft (or any other dependency), the best option is to use `npm link`. You can read more about it at `npm-link`: [symlink a package folder](https://docs.npmjs.com/cli/link.html).
```shell
$ cd /path/to/Windshaft
$ npm install
$ npm link
$ cd /path/to/Windshaft-cartodb
$ npm link windshaft
```
## Versioning
We follow [`SemVer`](http://semver.org/) for versioning. For available versions, see the [tags on this repository](https://github.com/Windshaft-cartodb/cartonik/tags).
## License
This project is licensed under the BSD 3-clause "New" or "Revised" License. See the [LICENSE](LICENSE) file for details.

81
configure vendored

@ -1,81 +0,0 @@
#!/bin/sh
#
# This script creates config/environments/*.js files using
# config/environments/*.js.example files as input and performing
# settings substitutions.
#
# It relies on a known format of the .js.example files which haven't
# been made easier to parse to still let humans copy them manually and
# do further editing or leave them as such to get the same setup as before
# the introduction of this script.
#
# The script is a work in progress. Available switches are printed
# by invoking with the --help switch. More switches will be added
# as the need/request for them arises.
#
# --strk(2012-07-23)
#
ENVDIR=config/environments
PGPORT=
MAPNIK_VERSION=
ENVIRONMENT=development
STATUS="$0 $*"
usage() {
echo "Usage: $0 [OPTION]"
echo
echo "Configuration:"
echo " --help display this help and exit"
echo " --with-pgport=NUM access PostgreSQL server on TCP port NUM [$PGPORT]"
echo " --with-mapnik-version=STRING set mapnik version string [$MAPNIK_VERSION]"
echo " --environment=STRING set output environment name [$ENVIRONMENT]"
}
while test -n "$1"; do
case "$1" in
--help|-h)
usage
exit 0
;;
--with-pgport=*)
PGPORT=`echo "$1" | cut -d= -f2`
;;
--with-mapnik-version=*)
MAPNIK_VERSION=`echo "$1" | cut -d= -f2`
;;
--environment=*)
ENVIRONMENT=`echo "$1" | cut -d= -f2`
;;
*)
echo "Unused option '$1'" >&2
;;
esac
shift
done
ENVEX=./${ENVDIR}/${ENVIRONMENT}.js.example
if [ -z "$PGPORT" ]; then
PGPORT=`node -e "console.log(require('${ENVEX}').postgres.port)"`
fi
echo "PGPORT: $PGPORT"
echo "MAPNIK_VERSION: $MAPNIK_VERSION"
echo "ENVIRONMENT: $ENVIRONMENT"
o=`dirname "${ENVEX}"`/`basename "${ENVEX}" .example`
echo "Writing $o"
# See http://austinmatzko.com/2008/04/26/sed-multi-line-search-and-replace/
sed -n "1h;1!H;\${;g;s/\(,postgres: {[^}]*port: *'\?\)[^',]*\('\?,\)/\1$PGPORT\2/;p;}" < "${ENVEX}" \
| sed "s/mapnik_version:.*/mapnik_version: '$MAPNIK_VERSION'/" \
> "$o"
STATUSFILE=config.status--${ENVIRONMENT}
echo "Writing ${STATUSFILE}"
echo ${STATUS} > ${STATUSFILE} && chmod +x ${STATUSFILE}

@ -1,13 +0,0 @@
#!/bin/bash
echo "*********************"
echo "To install Node.js, run:"
echo "/src/nodejs-install.sh"
echo "Use NODEJS_VERSION env var to select the Node.js version"
echo " "
echo "To start postgres, run:"
echo "/etc/init.d/postgresql start"
echo "*********************"
echo " "
docker run -it -v `pwd`:/srv carto/nodejs-xenial-pg101:latest bash

@ -1,4 +0,0 @@
#!/bin/bash
docker run -e "NODEJS_VERSION=${2}" -v `pwd`:/srv ${1} bash run_tests_docker.sh && \
docker ps --filter status=dead --filter status=exited -aq | xargs docker rm -v

@ -1,88 +0,0 @@
FROM ubuntu:xenial
# Use UTF8 to avoid encoding problems with pgsql
ENV LANG C.UTF-8
ENV NPROCS 1
ENV JOBS 1
ENV CXX g++-4.9
ENV PGUSER postgres
# Add external repos
RUN set -ex \
&& apt-get update \
&& apt-get install -y \
curl \
software-properties-common \
locales \
&& add-apt-repository -y ppa:ubuntu-toolchain-r/test \
&& add-apt-repository -y ppa:cartodb/postgresql-10 \
&& add-apt-repository -y ppa:cartodb/gis \
&& curl -sL https://deb.nodesource.com/setup_10.x | bash \
&& locale-gen en_US.UTF-8 \
&& update-locale LANG=en_US.UTF-8
RUN set -ex \
&& apt-get update \
&& apt-get install -y \
g++-4.9 \
gcc-4.9 \
git \
libcairo2-dev \
libgdal-dev \
libgdal1i \
libgdal20 \
libgeos-dev \
libgif-dev \
libjpeg8-dev \
libjson-c-dev \
libpango1.0-dev \
libpixman-1-dev \
libproj-dev \
libprotobuf-c-dev \
libxml2-dev \
gdal-bin \
make \
nodejs \
protobuf-c-compiler \
pkg-config \
wget \
zip \
postgresql-10 \
postgresql-10-plproxy \
postgis=2.4.4.5+carto-1 \
postgresql-10-postgis-2.4=2.4.4.5+carto-1 \
postgresql-10-postgis-2.4-scripts=2.4.4.5+carto-1 \
postgresql-10-postgis-scripts=2.4.4.5+carto-1 \
postgresql-client-10 \
postgresql-client-common \
postgresql-common \
postgresql-contrib \
postgresql-plpython-10 \
postgresql-server-dev-10 \
&& wget http://download.redis.io/releases/redis-4.0.8.tar.gz \
&& tar xvzf redis-4.0.8.tar.gz \
&& cd redis-4.0.8 \
&& make \
&& make install \
&& cd .. \
&& rm redis-4.0.8.tar.gz \
&& rm -R redis-4.0.8 \
&& apt-get purge -y wget protobuf-c-compiler \
&& apt-get autoremove -y
# Configure PostgreSQL
RUN set -ex \
&& echo "listen_addresses='*'" >> /etc/postgresql/10/main/postgresql.conf \
&& echo "local all all trust" > /etc/postgresql/10/main/pg_hba.conf \
&& echo "host all all 0.0.0.0/0 trust" >> /etc/postgresql/10/main/pg_hba.conf \
&& echo "host all all ::1/128 trust" >> /etc/postgresql/10/main/pg_hba.conf \
&& /etc/init.d/postgresql start \
&& createdb template_postgis \
&& createuser publicuser \
&& psql -c "CREATE EXTENSION postgis" template_postgis \
&& /etc/init.d/postgresql stop
WORKDIR /srv
EXPOSE 5858
CMD /etc/init.d/postgresql start

@ -1,89 +0,0 @@
FROM ubuntu:xenial
# Use UTF8 to avoid encoding problems with pgsql
ENV LANG C.UTF-8
ENV NPROCS 1
ENV JOBS 1
ENV CXX g++-4.9
ENV PGUSER postgres
# Add external repos
RUN set -ex \
&& apt-get update \
&& apt-get install -y \
curl \
software-properties-common \
locales \
&& add-apt-repository -y ppa:ubuntu-toolchain-r/test \
&& add-apt-repository -y ppa:cartodb/postgresql-10 \
&& add-apt-repository -y ppa:cartodb/gis \
&& curl -sL https://deb.nodesource.com/setup_6.x | bash \
&& locale-gen en_US.UTF-8 \
&& update-locale LANG=en_US.UTF-8
# Install dependencies and PostGIS 2.4 from sources
RUN set -ex \
&& apt-get update \
&& apt-get install -y \
g++-4.9 \
gcc-4.9 \
git \
libcairo2-dev \
libgdal-dev \
libgdal1i \
libgdal20 \
libgeos-dev \
libgif-dev \
libjpeg8-dev \
libjson-c-dev \
libpango1.0-dev \
libpixman-1-dev \
libproj-dev \
libprotobuf-c-dev \
libxml2-dev \
gdal-bin \
make \
nodejs \
protobuf-c-compiler \
pkg-config \
wget \
zip \
postgresql-10 \
postgresql-10-plproxy \
postgresql-10-postgis-2.4 \
postgresql-10-postgis-2.4-scripts \
postgresql-10-postgis-scripts \
postgresql-client-10 \
postgresql-client-common \
postgresql-common \
postgresql-contrib \
postgresql-plpython-10 \
postgresql-server-dev-10 \
postgis \
&& wget http://download.redis.io/releases/redis-4.0.8.tar.gz \
&& tar xvzf redis-4.0.8.tar.gz \
&& cd redis-4.0.8 \
&& make \
&& make install \
&& cd .. \
&& rm redis-4.0.8.tar.gz \
&& rm -R redis-4.0.8 \
&& apt-get purge -y wget protobuf-c-compiler \
&& apt-get autoremove -y
# Configure PostgreSQL
RUN set -ex \
&& echo "listen_addresses='*'" >> /etc/postgresql/10/main/postgresql.conf \
&& echo "local all all trust" > /etc/postgresql/10/main/pg_hba.conf \
&& echo "host all all 0.0.0.0/0 trust" >> /etc/postgresql/10/main/pg_hba.conf \
&& echo "host all all ::1/128 trust" >> /etc/postgresql/10/main/pg_hba.conf \
&& /etc/init.d/postgresql start \
&& createdb template_postgis \
&& createuser publicuser \
&& psql -c "CREATE EXTENSION postgis" template_postgis \
&& /etc/init.d/postgresql stop
WORKDIR /srv
EXPOSE 5858
CMD /etc/init.d/postgresql start

@ -1,89 +0,0 @@
FROM ubuntu:xenial
# Use UTF8 to avoid encoding problems with pgsql
ENV LANG C.UTF-8
ENV NPROCS 1
ENV JOBS 1
ENV CXX g++-4.9
ENV PGUSER postgres
# Add external repos
RUN set -ex \
&& apt-get update \
&& apt-get install -y \
curl \
software-properties-common \
locales \
&& add-apt-repository -y ppa:ubuntu-toolchain-r/test \
&& add-apt-repository -y ppa:cartodb/postgresql-10 \
&& add-apt-repository -y ppa:cartodb/gis \
&& curl -sL https://deb.nodesource.com/setup_6.x | bash \
&& locale-gen en_US.UTF-8 \
&& update-locale LANG=en_US.UTF-8
# Install dependencies and PostGIS 2.4 from sources
RUN set -ex \
&& apt-get update \
&& apt-get install -y \
g++-4.9 \
gcc-4.9 \
git \
libcairo2-dev \
libgdal-dev \
libgdal1i \
libgdal20 \
libgeos-dev \
libgif-dev \
libjpeg8-dev \
libjson-c-dev \
libpango1.0-dev \
libpixman-1-dev \
libproj-dev \
libprotobuf-c-dev \
libxml2-dev \
gdal-bin \
make \
nodejs \
protobuf-c-compiler \
pkg-config \
wget \
zip \
postgresql-10 \
postgresql-10-plproxy \
postgresql-10-postgis-2.4 \
postgresql-10-postgis-2.4-scripts \
postgresql-10-postgis-scripts \
postgresql-client-10 \
postgresql-client-common \
postgresql-common \
postgresql-contrib \
postgresql-plpython-10 \
postgresql-server-dev-10 \
postgis \
&& wget http://download.redis.io/releases/redis-4.0.8.tar.gz \
&& tar xvzf redis-4.0.8.tar.gz \
&& cd redis-4.0.8 \
&& make \
&& make install \
&& cd .. \
&& rm redis-4.0.8.tar.gz \
&& rm -R redis-4.0.8 \
&& apt-get purge -y wget protobuf-c-compiler \
&& apt-get autoremove -y
# Configure PostgreSQL
RUN set -ex \
&& echo "listen_addresses='*'" >> /etc/postgresql/10/main/postgresql.conf \
&& echo "local all all trust" > /etc/postgresql/10/main/pg_hba.conf \
&& echo "host all all 0.0.0.0/0 trust" >> /etc/postgresql/10/main/pg_hba.conf \
&& echo "host all all ::1/128 trust" >> /etc/postgresql/10/main/pg_hba.conf \
&& /etc/init.d/postgresql start \
&& createdb template_postgis \
&& createuser publicuser \
&& psql -c "CREATE EXTENSION postgis" template_postgis \
&& /etc/init.d/postgresql stop
WORKDIR /srv
EXPOSE 5858
CMD /etc/init.d/postgresql start

@ -1,88 +0,0 @@
FROM ubuntu:xenial
# Use UTF8 to avoid encoding problems with pgsql
ENV LANG C.UTF-8
ENV NPROCS 1
ENV JOBS 1
ENV CXX g++-4.9
ENV PGUSER postgres
# Add external repos
RUN set -ex \
&& apt-get update \
&& apt-get install -y \
curl \
software-properties-common \
locales \
&& add-apt-repository -y ppa:ubuntu-toolchain-r/test \
&& add-apt-repository -y ppa:cartodb/postgresql-10 \
&& add-apt-repository -y ppa:cartodb/gis \
&& curl -sL https://deb.nodesource.com/setup_6.x | bash \
&& locale-gen en_US.UTF-8 \
&& update-locale LANG=en_US.UTF-8
RUN set -ex \
&& apt-get update \
&& apt-get install -y \
g++-4.9 \
gcc-4.9 \
git \
libcairo2-dev \
libgdal-dev \
libgdal1i \
libgdal20 \
libgeos-dev \
libgif-dev \
libjpeg8-dev \
libjson-c-dev \
libpango1.0-dev \
libpixman-1-dev \
libproj-dev \
libprotobuf-c-dev \
libxml2-dev \
gdal-bin \
make \
nodejs \
protobuf-c-compiler \
pkg-config \
wget \
zip \
postgresql-10 \
postgresql-10-plproxy \
postgis=2.4.4.5+carto-1 \
postgresql-10-postgis-2.4=2.4.4.5+carto-1 \
postgresql-10-postgis-2.4-scripts=2.4.4.5+carto-1 \
postgresql-10-postgis-scripts=2.4.4.5+carto-1 \
postgresql-client-10 \
postgresql-client-common \
postgresql-common \
postgresql-contrib \
postgresql-plpython-10 \
postgresql-server-dev-10 \
&& wget http://download.redis.io/releases/redis-4.0.8.tar.gz \
&& tar xvzf redis-4.0.8.tar.gz \
&& cd redis-4.0.8 \
&& make \
&& make install \
&& cd .. \
&& rm redis-4.0.8.tar.gz \
&& rm -R redis-4.0.8 \
&& apt-get purge -y wget protobuf-c-compiler \
&& apt-get autoremove -y
# Configure PostgreSQL
RUN set -ex \
&& echo "listen_addresses='*'" >> /etc/postgresql/10/main/postgresql.conf \
&& echo "local all all trust" > /etc/postgresql/10/main/pg_hba.conf \
&& echo "host all all 0.0.0.0/0 trust" >> /etc/postgresql/10/main/pg_hba.conf \
&& echo "host all all ::1/128 trust" >> /etc/postgresql/10/main/pg_hba.conf \
&& /etc/init.d/postgresql start \
&& createdb template_postgis \
&& createuser publicuser \
&& psql -c "CREATE EXTENSION postgis" template_postgis \
&& /etc/init.d/postgresql stop
WORKDIR /srv
EXPOSE 5858
CMD /etc/init.d/postgresql start

@ -1,23 +1,33 @@
After running the tests with docker, you will need Docker installed and the docker image downloaded.
# Testing with Docker
Before running the tests with docker, you'll need Docker installed and the docker image downloaded.
## Install docker
`sudo apt install docker.io && sudo usermod -aG docker $(whoami)`
```shell
$ sudo apt install docker.io && sudo usermod -aG docker $(whoami)
```
## Download image
`docker pull carto/IMAGE`
```shell
docker pull carto/IMAGE
```
## Carto account
https://hub.docker.com/r/carto/
* `https://hub.docker.com/r/carto/`
## Update image
- Edit the docker image file with your desired changes
- Build image:
- `docker build -t carto/IMAGE -f docker/DOCKER_FILE docker/`
- Upload to docker hub:
- Login into docker hub:
- `docker login`
- Create tag:
- `docker tag carto/IMAGE carto/IMAGE`
- Upload:
- `docker push carto/IMAGE`
* Edit the docker image file
* Build image:
* `docker build -t carto/IMAGE -f docker/DOCKER_FILE docker/`
* Upload to docker hub:
* Login into docker hub:
* `docker login`
* Create tag:
* `docker tag carto/IMAGE carto/IMAGE`
* Upload:
* `docker push carto/IMAGE`

@ -8,15 +8,7 @@ source /src/nodejs-install.sh
git clone https://github.com/CartoDB/cartodb-postgresql.git
cd cartodb-postgresql && make && make install && cd ..
echo "Node.js version: "
node -v
cp config/environments/test.js.example config/environments/test.js
echo "npm version: "
npm -v
echo "Clean install: "
npm ci
npm ls
# run tests
npm test

1041
package-lock.json generated

File diff suppressed because it is too large Load Diff

@ -1,10 +1,18 @@
{
"private": true,
"name": "windshaft-cartodb",
"version": "8.0.1",
"description": "A map tile server for CartoDB",
"version": "8.1.0",
"description": "CARTO Maps API tiler",
"keywords": [
"cartodb"
"carto",
"tiler",
"maps",
"anonymous maps",
"named maps",
"static maps",
"smart aggregations",
"analysis",
"dataviews"
],
"url": "https://github.com/CartoDB/Windshaft-cartodb",
"license": "BSD-3-Clause",
@ -59,10 +67,10 @@
"eslint-plugin-node": "^10.0.0",
"eslint-plugin-promise": "^4.2.1",
"eslint-plugin-standard": "^4.0.1",
"istanbul": "0.4.5",
"mocha": "5.2.0",
"moment": "2.22.1",
"nock": "9.2.6",
"nyc": "^14.1.1",
"redis": "2.8.0",
"step": "1.0.0",
"strftime": "0.10.0"
@ -70,11 +78,22 @@
"scripts": {
"lint:fix": "eslint --fix app.js \"lib/**/*.js\" \"test/**/*.js\"",
"lint": "eslint app.js \"lib/**/*.js\" \"test/**/*.js\"",
"preinstall": "make pre-install",
"test": "make test-all",
"update-internal-deps": "rm -rf node_modules && npm install",
"docker-test": "./docker-test.sh",
"docker-bash": "./docker-bash.sh"
"preinstall": "scripts/darwin-pre-install.sh",
"pretest:setup": "npm run lint",
"test:setup": "NODE_ENV=test node test setup",
"pretest": "npm run test:setup",
"test": "npm run test:all",
"test:all": "npm run test:unit && npm run test:integration && npm run test:acceptance",
"test:acceptance": "NODE_ENV=test mocha -t 5000 --exit --recursive test/acceptance",
"test:integration": "NODE_ENV=test mocha --exit --recursive test/integration",
"test:unit": "NODE_ENV=test mocha --exit --recursive test/unit",
"posttest": "npm run test:teardown",
"test:teardown": "NODE_ENV=test node test teardown",
"precover": "npm run test:setup",
"cover": "NODE_ENV=test nyc --reporter=lcov npm run test:all",
"postcover": "npm run test:teardown",
"test:docker": "docker run -e \"NODEJS_VERSION=$NODE_VERSION\" -v `pwd`:/srv $DOCKER_IMAGE bash docker/scripts/test-setup.sh && docker ps --filter status=dead --filter status=exited -aq | xargs docker rm -v",
"docker:bash": "docker run -it -v `pwd`:/srv $DOCKER_IMAGE bash"
},
"engines": {
"node": "^10.15.1",

@ -1,149 +0,0 @@
#!/bin/bash
OPT_CREATE_REDIS=yes # create the redis test environment
OPT_CREATE_PGSQL=yes # create the PostgreSQL test environment
OPT_DROP_REDIS=yes # drop the redis test environment
OPT_DROP_PGSQL=yes # drop the PostgreSQL test environment
OPT_COVERAGE=no # run tests with coverage
OPT_REDIS_CELL=yes # download redis cell
export PGAPPNAME=cartodb_tiler_tester
cd $(dirname $0)
BASEDIR=$(pwd)
cd -
REDIS_PORT=`node -e "console.log(require('${BASEDIR}/config/environments/test.js').redis.port)"`
export REDIS_PORT
cleanup() {
if test x"$OPT_DROP_REDIS" = xyes; then
if test x"$PID_REDIS" = x; then
PID_REDIS=$(cat ${BASEDIR}/redis.pid)
if test x"$PID_REDIS" = x; then
echo "Could not find a test redis pid to kill it"
return;
fi
fi
redis-cli -p ${REDIS_PORT} info stats
redis-cli -p ${REDIS_PORT} info keyspace
echo "Killing test redis pid ${PID_REDIS}"
#kill ${PID_REDIS_MONITOR}
kill ${PID_REDIS}
fi
if test x"$OPT_DROP_PGSQL" = xyes; then
# TODO: drop postgresql ?
echo "Dropping PostgreSQL test database isn't implemented yet"
fi
}
cleanup_and_exit() {
cleanup
exit
}
die() {
msg=$1
echo "${msg}" >&2
cleanup
exit 1
}
trap 'cleanup_and_exit' 1 2 3 5 9 13
while [ -n "$1" ]; do
# This is kept for backward compatibility
if test "$1" = "--nodrop"; then
OPT_DROP_REDIS=no
OPT_DROP_PGSQL=no
shift
continue
elif test "$1" = "--nodrop-pg"; then
OPT_DROP_PGSQL=no
shift
continue
elif test "$1" = "--nodrop-redis"; then
OPT_DROP_REDIS=no
shift
continue
elif test "$1" = "--nocreate-pg"; then
OPT_CREATE_PGSQL=no
shift
continue
elif test "$1" = "--nocreate-redis"; then
OPT_CREATE_REDIS=no
shift
continue
elif test "$1" = "--with-coverage"; then
OPT_COVERAGE=yes
shift
continue
# This is kept for backward compatibility
elif test "$1" = "--nocreate"; then
OPT_CREATE_REDIS=no
OPT_CREATE_PGSQL=no
shift
continue
elif test "$1" = "--norediscell"; then
OPT_REDIS_CELL=no
shift
continue
else
break
fi
done
if [ -z "$1" ]; then
echo "Usage: $0 [<options>] <test> [<test>]" >&2
echo "Options:" >&2
echo " --nocreate do not create the test environment on start" >&2
echo " --nodrop do not drop the test environment on exit" >&2
echo " --with-coverage use istanbul to determine code coverage" >&2
echo " --norediscell do not download redis-cell" >&2
exit 1
fi
TESTS=$@
if test x"$OPT_CREATE_REDIS" = xyes; then
echo "Starting redis on port ${REDIS_PORT}"
REDIS_CELL_PATH="${BASEDIR}/test/support/libredis_cell.so"
if [[ "$OSTYPE" == "darwin"* ]]; then
REDIS_CELL_PATH="${BASEDIR}/test/support/libredis_cell.dylib"
fi
echo "port ${REDIS_PORT}" | redis-server - --loadmodule ${REDIS_CELL_PATH} > ${BASEDIR}/test.log &
PID_REDIS=$!
echo ${PID_REDIS} > ${BASEDIR}/redis.pid
fi
PREPARE_DB_OPTS=
if test x"$OPT_CREATE_PGSQL" != xyes; then
PREPARE_DB_OPTS="$PREPARE_DB_OPTS --skip-pg"
fi
if test x"$OPT_CREATE_REDIS" != xyes; then
PREPARE_DB_OPTS="$PREPARE_DB_OPTS --skip-redis"
fi
echo "Preparing the environment"
cd ${BASEDIR}/test/support
source prepare_db.sh "${PREPARE_DB_OPTS}" || die "database preparation failure"
cd -
PATH=node_modules/.bin/:$PATH
#redis-cli -p ${REDIS_PORT} monitor > /tmp/windshaft-cartodb.redis.log &
#PID_REDIS_MONITOR=$!
if test x"$OPT_COVERAGE" = xyes; then
echo "Running tests with coverage"
./node_modules/.bin/istanbul cover node_modules/.bin/_mocha -- -u tdd -t 5000 --exit ${TESTS}
else
echo "Running tests"
./node_modules/.bin/_mocha -c -u tdd -t 5000 --exit ${TESTS}
fi
ret=$?
cleanup
exit $ret

@ -1,6 +1,7 @@
#!/bin/bash
if [[ "$OSTYPE" == "darwin"* ]]; then
export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:/opt/X11/lib/pkgconfig
CAIRO_PKG_CONFIG=`pkg-config cairo --cflags-only-I 2> /dev/null`
RESULT=$?

@ -1,7 +0,0 @@
#!/bin/bash
if [[ "$OSTYPE" == "darwin"* ]]; then
export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:/opt/X11/lib/pkgconfig
fi
npm install

@ -0,0 +1,180 @@
'use strict';
const util = require('util');
const path = require('path');
const exec = util.promisify(require('child_process').exec);
if (!process.env.NODE_ENV) {
console.error('Please set "NODE_ENV" variable, e.g.: "NODE_ENV=test"');
process.exit(1);
}
const environment = require(`../config/environments/${process.env.NODE_ENV}.js`);
const REDIS_PORT = environment.redis.port;
const REDIS_CELL_PATH = path.resolve(
process.platform === 'darwin'
? './test/support/libredis_cell.dylib'
: './test/support/libredis_cell.so'
);
const TEST_USER_ID = 1;
const TEST_USER = environment.postgres_auth_user.replace('<%= user_id %>', TEST_USER_ID);
const TEST_PASSWORD = environment.postgres_auth_pass.replace('<%= user_id %>', TEST_USER_ID);
const PUBLIC_USER = environment.postgres.user;
const PUBLIC_USER_PASSWORD = environment.postgres.password;
const TEST_DB = `${TEST_USER}_db`;
async function startRedis () {
await exec(`redis-server --port ${REDIS_PORT} --loadmodule ${REDIS_CELL_PATH} --logfile ${__dirname}/redis-server.log --daemonize yes`);
}
async function stopRedis () {
await exec(`redis-cli -p ${REDIS_PORT} shutdown`);
}
async function dropDatabase () {
await exec(`dropdb --if-exists ${TEST_DB}`, {
env: {
PGUSER: 'postgres'
}
});
}
async function createDatabase () {
await exec(`createdb -T template_postgis -EUTF8 "${TEST_DB}"`, {
env: {
PGUSER: 'postgres'
}
});
}
async function createDatabaseExtension () {
await exec(`psql -c "CREATE EXTENSION IF NOT EXISTS cartodb CASCADE;" ${TEST_DB}`, {
env: {
PGUSER: 'postgres'
}
});
}
async function populateDatabase () {
const filenames = [
'analysis_catalog',
'windshaft.test',
'gadm4',
'countries_null_values',
'ported/populated_places_simple_reduced',
'cdb_analysis_check',
'cdb_invalidate_varnish'
].map(filename => `${__dirname}/support/sql/${filename}.sql`);
const populateDatabaseCmd = `
cat ${filenames.join(' ')} |
sed -e "s/:PUBLICUSER/${PUBLIC_USER}/g" |
sed -e "s/:PUBLICPASS/${PUBLIC_USER_PASSWORD}/g" |
sed -e "s/:TESTUSER/${TEST_USER}/g" |
sed -e "s/:TESTPASS/${TEST_PASSWORD}/g" |
PGOPTIONS='--client-min-messages=WARNING' psql -q -v ON_ERROR_STOP=1 ${TEST_DB}
`;
await exec(populateDatabaseCmd, {
env: {
PGUSER: 'postgres'
}
});
}
async function populateRedis () {
const commands = `
HMSET rails:users:localhost \
id ${TEST_USER_ID} \
database_name "${TEST_DB}" \
database_host localhost \
map_key 1234
HMSET rails:users:cartodb250user \
id ${TEST_USER_ID} \
database_name "${TEST_DB}" \
database_host "localhost" \
database_password "${TEST_PASSWORD}" \
map_key 4321
HMSET api_keys:localhost:1234 \
user "localhost" \
type "master" \
grants_sql "true" \
grants_maps "true" \
database_role "${TEST_USER}" \
database_password "${TEST_PASSWORD}"
HMSET api_keys:localhost:default_public \
user "localhost" \
type "default" \
grants_sql "true" \
grants_maps "true" \
database_role "test_windshaft_publicuser" \
database_password "public"
HMSET api_keys:localhost:regular1 \
user "localhost" \
type "regular" \
grants_sql "true" \
grants_maps "true" \
database_role "test_windshaft_regular1" \
database_password "regular1"
HMSET api_keys:localhost:regular2 \
user "localhost" \
type "regular" \
grants_sql "true" \
grants_maps "false" \
database_role "test_windshaft_publicuser" \
database_password "public"
HMSET api_keys:cartodb250user:4321 \
user "localhost" \
type "master" \
grants_sql "true" \
grants_maps "true" \
database_role "${TEST_USER}" \
database_password "${TEST_PASSWORD}"
HMSET api_keys:cartodb250user:default_public \
user "localhost" \
type "default" \
grants_sql "true" \
grants_maps "true" \
database_role "test_windshaft_publicuser" \
database_password "public"
`;
await exec(`echo "${commands}" | redis-cli -p ${REDIS_PORT} -n 5`);
}
async function main (args) {
let code = 0;
try {
switch (args[0]) {
case 'setup':
await startRedis();
await populateRedis();
await dropDatabase();
await createDatabase();
await createDatabaseExtension();
await populateDatabase();
break;
case 'teardown':
await stopRedis();
break;
default:
throw new Error('Missing "mode" argument. Valid ones: "setup" or "teardown"');
}
} catch (err) {
console.error(err);
code = 1;
} finally {
process.exit(code);
}
}
main(process.argv.slice(2));
Loading…
Cancel
Save