cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1194 from CartoDB/clean

Browse Source 
Use Github actions for builds
This commit is contained in:
Álvaro 2020-12-21 09:06:52 +01:00 committed by GitHub
commit 5ed1a3a2d1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 1444 additions and 346 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
.github/workflows/main.yml vendored Normal file
View File

@ -0,0 +1,58 @@
name: continuous integration
on:
pull_request:
paths-ignore:
- 'LICENSE'
- 'README**'
- 'HOW_TO_RELEASE**'
- 'LOGGING**'
env:
GCLOUD_VERSION: '306.0.0'
ARTIFACTS_PROJECT_ID: cartodb-on-gcp-main-artifacts
jobs:
build-test-docker:
runs-on: ubuntu-18.04
timeout-minutes: 10
steps:
- uses: actions/checkout@v2
with:
submodules: true
token: ${{ secrets.CARTOFANTE_PERSONAL_TOKEN }}
- name: Build image
# we tag with "latest" but we don't push it on purpose. We use it as a base for the testing image
run: |
echo ${GITHUB_SHA::7}
echo ${GITHUB_REF##*/}
docker build -f private/Dockerfile -t gcr.io/$ARTIFACTS_PROJECT_ID/windshaft:latest -t gcr.io/$ARTIFACTS_PROJECT_ID/windshaft:${GITHUB_REF##*/} -t gcr.io/$ARTIFACTS_PROJECT_ID/windshaft:${GITHUB_SHA::7} -t gcr.io/$ARTIFACTS_PROJECT_ID/windshaft:${GITHUB_REF##*/}--${GITHUB_SHA::7} .
- name: Build testing image
# here it uses the lastest from prev step to add the needed parts on top
run: |
docker build -t gcr.io/$ARTIFACTS_PROJECT_ID/windshaft-test:latest -f private/Dockerfile.test .
- name: Setup gcloud authentication
uses: google-github-actions/setup-gcloud@master
with:
version: ${{env.GCLOUD_VERSION}}
service_account_key: ${{ secrets.ARTIFACTS_GCLOUD_ACCOUNT_BASE64 }}
- name: Configure docker and pull images
# we pull images manually, as if done in next step using docker-compose it fails because missing openssl
run: |
gcloud auth configure-docker
docker pull gcr.io/cartodb-on-gcp-main-artifacts/postgres:latest
docker pull gcr.io/cartodb-on-gcp-main-artifacts/redis:latest
- name: Run tests inside container
run: docker-compose -f private/ci/docker-compose.yml run windshaft-tests
- name: Upload image
run: |
docker push gcr.io/$ARTIFACTS_PROJECT_ID/windshaft:${GITHUB_REF##*/}
docker push gcr.io/$ARTIFACTS_PROJECT_ID/windshaft:${GITHUB_SHA::7}
docker push gcr.io/$ARTIFACTS_PROJECT_ID/windshaft:${GITHUB_REF##*/}--${GITHUB_SHA::7}

43
.github/workflows/master.yml vendored Normal file
View File

@ -0,0 +1,43 @@
# in this workflow we don't run the tests. Only build image, tag (also latests) and upload. The tests are not run because they are run
# on each pull request, and there is a branch protection that forces to have branch up to date before merging, so tests are always run
# with the latest code
name: master build image
on:
push:
branches:
- master
env:
GCLOUD_VERSION: '306.0.0'
ARTIFACTS_PROJECT_ID: cartodb-on-gcp-main-artifacts
jobs:
build-master:
runs-on: ubuntu-18.04
timeout-minutes: 5
steps:
- uses: actions/checkout@v2
with:
submodules: true
token: ${{ secrets.CARTOFANTE_PERSONAL_TOKEN }}
- name: Build image
run: |
echo ${GITHUB_SHA::7}
echo ${GITHUB_REF##*/}
docker build -f private/Dockerfile -t gcr.io/$ARTIFACTS_PROJECT_ID/windshaft:latest -t gcr.io/$ARTIFACTS_PROJECT_ID/windshaft:${GITHUB_REF##*/} -t gcr.io/$ARTIFACTS_PROJECT_ID/windshaft:${GITHUB_SHA::7} .
- name: Setup gcloud authentication
uses: google-github-actions/setup-gcloud@master
with:
version: ${{env.GCLOUD_VERSION}}
service_account_key: ${{ secrets.ARTIFACTS_GCLOUD_ACCOUNT_BASE64 }}
- name: Upload image
run: |
docker push gcr.io/$ARTIFACTS_PROJECT_ID/windshaft:${GITHUB_REF##*/}
docker push gcr.io/$ARTIFACTS_PROJECT_ID/windshaft:${GITHUB_SHA::7}
docker push gcr.io/$ARTIFACTS_PROJECT_ID/windshaft:latest

1
.gitignore vendored
View File

@ -12,6 +12,5 @@ redis.pid
coverage/
.DS_Store
.nyc_output
Dockerfile*
build_resources/
.dockerignore

4
.gitmodules vendored Normal file
View File

@ -0,0 +1,4 @@
[submodule "private"]
path = private
url = git@github.com:CartoDB/Windshaft-cartodb-private.git
branch = master

63
.travis.yml
View File

@ -1,63 +0,0 @@
sudo: false
language: node_js
services:
- redis-server
env:
global:
- PGUSER=postgres
- PGDATABASE=postgres
- PGOPTIONS='-c client_min_messages=NOTICE'
jobs:
include:
- env: POSTGRESQL_VERSION="10" POSTGIS_VERSION="2.4"
dist: xenial
- env: POSTGRESQL_VERSION="11" POSTGIS_VERSION="2.5"
dist: xenial
- env: POSTGRESQL_VERSION="12" POSTGIS_VERSION="3"
dist: bionic
node_js:
- "12"
install:
- npm ci
script:
# Ensure dev dependencies are installed
- sudo apt-get install -y libpangocairo-1.0-0 libpango1.0-dev pkg-config
# Remove old packages
- sudo apt-get remove postgresql-$POSTGRESQL_VERSION postgresql-client-$POSTGRESQL_VERSION postgresql-server-dev-$POSTGRESQL_VERSION postgresql-common postgresql-client-common postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION-scripts postgis
# Install CARTO packages
- if [[ $POSTGRESQL_VERSION == '10' ]]; then sudo add-apt-repository -y ppa:cartodb/gis; fi;
- sudo add-apt-repository -y ppa:cartodb/postgresql-$POSTGRESQL_VERSION
- sudo apt-get -q update
# We use -t $TRAVIS_DIST to give preference to our ppa's (which are called as the ${dist}), instead of
# pgdg repos (which are called ${dist}-pgdg. Nasty but it works.
- sudo apt-get install -y --allow-unauthenticated --no-install-recommends --no-install-suggests postgresql-$POSTGRESQL_VERSION postgresql-client-$POSTGRESQL_VERSION postgresql-server-dev-$POSTGRESQL_VERSION postgresql-common postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION-scripts postgis -t $TRAVIS_DIST
# For pre12, install plpython2. For PG12 install plpython3
- if [[ $POSTGRESQL_VERSION != '12' ]]; then sudo apt-get install -y postgresql-plpython-$POSTGRESQL_VERSION python python-redis -t $TRAVIS_DIST; else sudo apt-get install -y postgresql-plpython3-12 python3 python3-redis -t $TRAVIS_DIST; fi;
# Remove old clusters and create the new one
- for i in $(pg_lsclusters | tail -n +2 | awk '{print $1}'); do sudo pg_dropcluster --stop $i main; done;
- sudo rm -rf /etc/postgresql/$POSTGRESQL_VERSION /var/lib/postgresql/$POSTGRESQL_VERSION /var/ramfs/postgresql/$POSTGRESQL_VERSION
- sudo pg_createcluster -u postgres $POSTGRESQL_VERSION main --start -p 5432 -- --auth-local trust
- git clone https://github.com/CartoDB/cartodb-postgresql.git
- cd cartodb-postgresql && make && sudo make install && cd ..
- createdb template_postgis
- psql -c "CREATE EXTENSION postgis" template_postgis
- if [[ $POSTGRESQL_VERSION == '12' ]]; then psql -c "CREATE EXTENSION postgis_raster" template_postgis; fi;
- cp config/environments/test.js.example config/environments/test.js
- npm test
after_failure:
- pg_lsclusters
- sudo cat /var/log/postgresql/postgresql-$POSTGRESQL_VERSION-main.log

28
README.md
View File

@ -13,8 +13,8 @@ The [`CARTO Maps API`](https://carto.com/developers/maps-api/) tiler. It extends
Requirements:
* [`Node 10.x (npm 6.x)`](https://nodejs.org/dist/latest-v10.x/)
* [`PostgreSQL >= 10.0`](https://www.postgresql.org/download/)
* [`Node 12.x `](https://nodejs.org/dist/latest-v10.x/)
* [`PostgreSQL >= 11.0`](https://www.postgresql.org/download/)
* [`PostGIS >= 2.4`](https://postgis.net/install/)
* [`CARTO Postgres Extension >= 0.24.1`](https://github.com/CartoDB/cartodb-postgresql)
* [`Redis >= 4`](https://redis.io/download)
@ -45,7 +45,11 @@ $ npm install
### Run
Create the `./config/environments/<env>.js` file (there are `.example` files to start from). Look at `./lib/server-options.js` for more on config.
You can inject the configuration through environment variables at run time. Check the file `./config/environments/config.js` to see the ones you have available.
While the migration to the new environment based configuration, you can still use the old method of copying a config file. To enabled the one with environment variables you need to pass `CARTO_WINDSHAFT_ENV_BASED_CONF=true`. You can use the docker image to run it.
Old way:
```shell
$ node app.js <env>
@ -55,10 +59,26 @@ Where `<env>` is the name of a configuration file under `./config/environments/`
### Test
You can easily run the tests against the dependencies from the `dev-env`. To do so, you need to build the test docker image:
```shell
$ npm test
$ docker-compose build
```
Then you can run the tests like:
```shell
$ docker-compose run windshaft-tests
```
It will mount your code inside a volume. In case you want to play and run `npm test` or something else you can do:
```shell
$ docker-compose run --entrypoint bash windshaft-tests
```
So you will have a bash shell inside the test container, with the code from your host.
### Coverage
```shell

9
app.js
View File

@ -25,7 +25,12 @@ const argv = require('yargs')
.argv;
const environmentArg = argv._[0] || process.env.NODE_ENV || 'development';
const configurationFile = path.resolve(argv.config || `./config/environments/${environmentArg}.js`);
let configFileName = environmentArg;
if (process.env.CARTO_WINDSHAFT_ENV_BASED_CONF) {
// we override the file with the one with env vars
configFileName = 'config';
}
const configurationFile = path.resolve(argv.config || `./config/environments/${configFileName}.js`);
global.environment = require(configurationFile);
process.env.NODE_ENV = argv._[0] || process.env.NODE_ENV || global.environment.environment;
@ -59,7 +64,7 @@ const availableEnvironments = {
};
if (!availableEnvironments[process.env.NODE_ENV]) {
logger.fatal(new Error(`Invalid environment argument, valid ones: ${Object.keys(availableEnvironments).join(', ')}`));
logger.fatal(new Error(`Invalid environment ${process.env.NODE_ENV} argument, valid ones: ${Object.keys(availableEnvironments).join(', ')}`));
process.exit(1);
}

411
config/environments/config.js Normal file
View File

@ -0,0 +1,411 @@
var config = {
environment: process.env.CARTO_WINDSHAFT_NODE_ENV,
port: 8181,
host: null, // null on purpouse so it listens to whatever address docker assigns
// Size of the threadpool which can be used to run user code and get notified in the loop thread
// Its default size is 4, but it can be changed at startup time (the absolute maximum is 128).
// See http://docs.libuv.org/en/latest/threadpool.html
uv_threadpool_size: undefined,
// Time in milliseconds to force GC cycle.
// Disable by using <=0 value.
gc_interval: 10000,
// Regular expression pattern to extract username
// from hostname. Must have a single grabbing block.
user_from_host: process.env.CARTO_WINDSHAFT_USER_FROM_HOST || '^(.*)\\.cartodb\\.com$',
// Base URLs for the APIs
//
// See https://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API
//
// Note: each entry corresponds with an express' router.
// You must define at least one path. However, middlewares are optional.
routes: {
api: [{
paths: [
'/api/v1',
'/user/:user/api/v1'
],
// Optional: attach middlewares at the begining of the router
// to perform custom operations.
middlewares: [
function noop () {
return function noopMiddleware (req, res, next) {
next();
};
}
],
// Base url for the Detached Maps API
// "/api/v1/map" is the new API,
map: [{
paths: [
'/map'
],
middlewares: [] // Optional
}],
// Base url for the Templated Maps API
// "/api/v1/map/named" is the new API,
template: [{
paths: [
'/map/named'
],
middlewares: [] // Optional
}]
}]
},
// Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status.
//
// This URLs depend on how `routes` and `user_from_host` are configured: the application can be
// configured to accept request with the {user} in the header host or in the request path.
// It also might depend on the configured cdn_url via `serverMetadata.cdn_url`.
//
// This template allows to make the endpoints generation more flexible, the template exposes the following params:
// 1. {{=it.cdn_url}}: will be used when `serverMetadata.cdn_url` exists.
// 2. {{=it.user}}: will use the username as extraced from `user_from_host` or `routes`.
// 3. {{=it.port}}: will use the `port` from this very same configuration file.
resources_url_templates: {
http: 'http://{{=it.cdn_url}}/{{=it.user}}/api/v1/map',
https: 'https://{{=it.cdn_url}}/{{=it.user}}/api/v1/map'
},
// Specify the maximum length of the queue of pending connections for the HTTP server.
// The actual length will be determined by the OS through sysctl settings such as tcp_max_syn_backlog and somaxconn on Linux.
// The default value of this parameter is 511 (not 512).
// See: https://nodejs.org/docs/latest/api/net.html#net_server_listen
maxConnections: 128,
// Maximum number of templates per user. Unlimited by default.
maxUserTemplates: 1024,
// Seconds since "last creation" before a detached
// or template instance map expires. Or: how long do you want
// to be able to navigate the map without a reload ?
// Defaults to 7200 (2 hours)
mapConfigTTL: 7200,
// idle socket timeout, in milliseconds
socket_timeout: 600000,
enable_cors: true,
cache_enabled: true,
// Templated database username for authorized user
// Supported labels: 'user_id' (read from redis)
postgres_auth_user: process.env.CARTO_WINDSHAFT_DB_USER || 'cartodb_user_<%= user_id %>',
// Templated database password for authorized user
// Supported labels: 'user_id', 'user_password' (both read from redis)
postgres_auth_pass: '<%= user_password %>',
postgres: {
user: 'publicuser',
password: 'public',
host: process.env.CARTO_WINDSHAFT_POSTGRES_HOST || 'localhost',
port: process.env.CARTO_WINDSHAFT_POSTGRES_PORT || 5432,
pool: {
// maximum number of resources to create at any given time
size: 16,
// max milliseconds a resource can go unused before it should be destroyed
idleTimeout: 3000,
// frequency to check for idle resources
reapInterval: 1000
}
},
mapnik_version: undefined,
mapnik_tile_format: 'png8:m=h',
statsd: {
host: process.env.CARTO_WINDSHAFT_STATSD_HOST || 'localhost',
port: 8125,
prefix: process.env.CARTO_WINDSHAFT_STATSD_PREFIX || ':host.', // could be hostname, better not containing dots
cacheDns: true
// support all allowed node-statsd options
},
renderer: {
// Milliseconds since last access before renderer cache item expires
cache_ttl: 60000,
statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
mvt: {
// If enabled, MVTs will be generated with PostGIS directly
// If disabled, MVTs will be generated with Mapnik MVT
usePostGIS: true
},
mapnik: {
// The size of the pool of internal mapnik backend
// This pool size is per mapnik renderer created in Windshaft's RendererFactory
// See https://github.com/CartoDB/Windshaft/blob/master/lib/windshaft/renderers/renderer_factory.js
// Important: check the configuration of uv_threadpool_size to use suitable value
poolSize: 8,
// The maximum number of waiting clients of the pool of internal mapnik backend
// This maximum number is per mapnik renderer created in Windshaft's RendererFactory
poolMaxWaitingClients: 64,
// Whether grainstore will use a child process or not to transform CartoCSS into Mapnik XML.
// This will prevent blocking the main thread.
useCartocssWorkers: false,
// Metatile is the number of tiles-per-side that are going
// to be rendered at once. If all of them will be requested
// we'd have saved time. If only one will be used, we'd have
// wasted time.
metatile: 2,
// tilelive-mapnik uses an internal cache to store tiles/grids
// generated when using metatile. This options allow to tune
// the behaviour for that internal cache.
metatileCache: {
// Time an object must stay in the cache until is removed
ttl: 0,
// Whether an object must be removed after the first hit
// Usually you want to use `true` here when ttl>0.
deleteOnHit: false
},
// Override metatile behaviour depending on the format
formatMetatile: {
png: 2,
'grid.json': 1
},
// Buffer size is the tickness in pixel of a buffer
// around the rendered (meta?)tile.
//
// This is important for labels and other marker that overlap tile boundaries.
// Setting to 128 ensures no render artifacts.
// 64 may have artifacts but is faster.
// Less important if we can turn metatiling on.
bufferSize: 64,
// SQL queries will be wrapped with ST_SnapToGrid
// Snapping all points of the geometry to a regular grid
snapToGrid: false,
// SQL queries will be wrapped with ST_ClipByBox2D
// Returning the portion of a geometry falling within a rectangle
// It will only work if snapToGrid is enabled
clipByBox2d: true,
postgis: {
// Parameters to pass to datasource plugin of mapnik
// See http://github.com/mapnik/mapnik/wiki/PostGIS
user: 'publicuser',
password: 'public',
host: process.env.CARTO_WINDSHAFT_POSTGRES_HOST || '127.0.0.1',
port: process.env.CARTO_WINDSHAFT_POSTGRES_PORT || 5432,
extent: '-20037508.3,-20037508.3,20037508.3,20037508.3',
// max number of rows to return when querying data, 0 means no limit
row_limit: 65535,
/*
* Set persist_connection to false if you want
* database connections to be closed on renderer
* expiration (1 minute after last use).
* Setting to true (the default) would never
* close any connection for the server's lifetime
*/
persist_connection: false,
simplify_geometries: true,
use_overviews: true, // use overviews to retrieve raster
max_size: 500,
twkb_encoding: true
},
limits: {
// Time in milliseconds a render request can take before it fails, some notes:
// - 0 means no render limit
// - it considers metatiling, naive implementation: (render timeout) * (number of tiles in metatile)
render: 0,
// As the render request will finish even if timed out, whether it should be placed in the internal
// cache or it should be fully discarded. When placed in the internal cache another attempt to retrieve
// the same tile will result in an immediate response, however that will use a lot of more application
// memory. If we want to enforce this behaviour we have to implement a cache eviction policy for the
// internal cache.
cacheOnTimeout: true
},
// If enabled Mapnik will reuse the features retrieved from the database
// instead of requesting them once per style inside a layer
'cache-features': true,
// Require metrics to the renderer
metrics: false,
// Options for markers attributes, ellipses and images caches
markers_symbolizer_caches: {
disabled: false
}
},
http: {
timeout: 2000, // the timeout in ms for a http tile request
proxy: undefined, // the url for a proxy server
whitelist: [ // the whitelist of urlTemplates that can be used
'.*', // will enable any URL
'http://{s}.example.com/{z}/{x}/{y}.png'
],
// image to use as placeholder when urlTemplate is not in the whitelist
// if provided the http renderer will use it instead of throw an error
fallbackImage: {
type: 'fs', // 'fs' and 'url' supported
src: __dirname + '/../../assets/default-placeholder.png'
}
},
torque: {}
},
// anything analyses related
analysis: {
// batch configuration
batch: {
// Inline execution avoid the use of SQL API as batch endpoint
// When set to true it will run all analysis queries in series, with a direct connection to the DB
// This might be useful for:
// - testing
// - running an standalone server without any dependency on external services
inlineExecution: false,
// where the SQL API is running, it will use a custom Host header to specify the username.
endpoint: 'http://127.0.0.1:8080/api/v2/sql/job',
// the template to use for adding the host header in the batch api requests
hostHeaderTemplate: '{{=it.username}}.localhost.lan'
},
// Define max execution time in ms for analyses or tags
// If analysis or tag are not found in redis this values will be used as default.
limits: {
moran: { timeout: 120000, maxNumberOfRows: 1e5 },
cpu2x: { timeout: 60000 }
}
},
millstone: {
// Needs to be writable by server user
cache_basedir: process.env.CARTO_WINDSHAFT_TILE_CACHE || '/home/ubuntu/tile_assets/'
},
redis: {
host: process.env.CARTO_WINDSHAFT_REDIS_HOST || '127.0.0.1',
port: process.env.CARTO_WINDSHAFT_REDIS_PORT || 6379,
// Max number of connections in each pool.
// Users will be put on a queue when the limit is hit.
// Set to maxConnection to have no possible queues.
// There are currently 2 pools involved in serving
// windshaft-cartodb requests so multiply this number
// by 2 to know how many possible connections will be
// kept open by the servelsr. The default is 50.
max: 50,
returnToHead: true, // defines the behaviour of the pool: false => queue, true => stack
idleTimeoutMillis: 30000, // idle time before dropping connection
reapIntervalMillis: 1000, // time between cleanups
slowQueries: {
log: true,
elapsedThreshold: 200
},
slowPool: {
log: true, // whether a slow acquire must be logged or not
elapsedThreshold: 25 // the threshold to determine an slow acquire must be reported or not
},
emitter: {
statusInterval: 5000 // time, in ms, between each status report is emitted from the pool, status is sent to statsd
},
unwatchOnRelease: false, // Send unwatch on release, see http://github.com/CartoDB/Windshaft-cartodb/issues/161
noReadyCheck: true // Check `no_ready_check` at https://github.com/mranney/node_redis/tree/v0.12.1#overloading
},
// For more details about this options check https://nodejs.org/api/http.html#http_new_agent_options
httpAgent: {
keepAlive: true,
keepAliveMsecs: 1000,
maxSockets: 25,
maxFreeSockets: 256
},
varnish: {
host: process.env.CARTO_WINDSHAFT_VARNISH_PORT || 'localhost',
port: process.env.CARTO_WINDSHAFT_VARNISH_PORT || 6082, // the por for the telnet interface where varnish is listening to
http_port: 6081, // the port for the HTTP interface where varnish is listening to
purge_enabled: process.env.CARTO_WINDSHAFT_VARNISH_PURGE_ENABLED === 'true' || false, // whether the purge/invalidation mechanism is enabled in varnish or not
secret: 'xxx',
ttl: 86400,
fallbackTtl: 300,
layergroupTtl: 86400 // the max-age for cache-control header in layergroup responses
},
// this [OPTIONAL] configuration enables invalidating by surrogate key in fastly
fastly: {
// whether the invalidation is enabled or not
enabled: false,
// the fastly api key
apiKey: 'wadus_api_key',
// the service that will get surrogate key invalidation
serviceId: 'wadus_service_id'
},
// If useProfiler is true every response will be served with an
// X-Tiler-Profile header containing elapsed timing for various
// steps taken for producing the response.
useProfiler: false,
serverMetadata: {
cdn_url: {
http: 'api.cartocdn.com',
https: 'cartocdn.global.ssl.fastly.net'
}
},
// Settings for the health check available at /health
health: {
enabled: process.env.CARTO_WINDSHAFT_HEALTH_ENABLED === 'true' || false,
username: 'localhost',
z: 0,
x: 0,
y: 0
},
disabled_file: 'pids/disabled',
// Use this as a feature flags enabling/disabling mechanism
enabledFeatures: {
// whether it should intercept tile render errors an act based on them, enabled by default.
onTileErrorStrategy: false,
// whether the affected tables for a given SQL must query directly postgresql or use the SQL API
cdbQueryTablesFromPostgres: true,
// whether in mapconfig is available stats & metadata for each layer
layerStats: process.env.CARTO_WINDSHAFT_LAYERSTATS_ENABLED === 'true' || false,
// whether it should rate limit endpoints (global configuration)
rateLimitsEnabled: false,
// whether it should rate limit one or more endpoints (only if rateLimitsEnabled = true)
rateLimitsByEndpoint: {
anonymous: false,
static: false,
static_named: false,
dataview: false,
dataview_search: false,
analysis: false,
analysis_catalog: false,
tile: false,
attributes: false,
named_list: false,
named_create: false,
named_get: false,
named: false,
named_update: false,
named_delete: false,
named_tiles: false
}
},
pubSubMetrics: {
enabled: process.env.CARTO_WINDSHAFT_METRICS_ENABLED === 'true' || false,
project_id: process.env.CARTO_WINDSHAFT_METRICS_PROJECT_ID || 'avid-wavelet-844',
credentials: '',
topic: process.env.CARTO_WINDSHAFT_METRICS_PROJECT_ID || 'raw-metric-events'
}
};
// override some defaults for tests
if (process.env.NODE_ENV === 'test') {
config.user_from_host = '(.*)';
config.postgres_auth_pass = 'test_windshaft_cartodb_user_<%= user_id %>_pass';
config.millstone.cache_basedir = '/tmp/tile_assets';
config.postgres.user = 'test_windshaft_publicuser';
config.resources_url_templates = {
http: 'http://{{=it.user}}.localhost.lan:{{=it.port}}/api/v1/map',
https: 'https://{{=it.user}}.localhost.lan:{{=it.port}}/api/v1/map'
};
config.cache_enabled = false;
config.postgres_auth_user = 'test_windshaft_cartodb_user_<%= user_id %>';
config.renderer.mapnik.postgis.twkb_encoding = false;
config.renderer.mapnik['cache-features'] = false;
config.renderer.http.whitelist = [ // the whitelist of urlTemplates that can be used
'.*', // will enable any URL
'http://{s}.example.com/{z}/{x}/{y}.png',
// for testing purposes
'http://{s}.basemaps.cartocdn.com/dark_nolabels/{z}/{x}/{y}.png'
];
config.analysis.batch.inlineExecution = true;
config.redis.idleTimeoutMillis = 1;
config.redis.reapIntervalMillis = 1;
config.varnish.purge_enabled = false;
config.health.enabled = false;
config.enabledFeatures.layerStats = true;
}
module.exports = config;

92
docker/Dockerfile-nodejs-xenial-pg101:latest
View File

@ -1,92 +0,0 @@
FROM ubuntu:xenial
# Use UTF8 to avoid encoding problems with pgsql
ENV LANG C.UTF-8
ENV NPROCS 1
ENV JOBS 1
ENV CXX g++-4.9
ENV PGUSER postgres
# Add external repos
RUN set -ex \
&& apt-get update \
&& apt-get install -y \
curl \
software-properties-common \
locales \
&& add-apt-repository -y ppa:ubuntu-toolchain-r/test \
&& add-apt-repository -y ppa:cartodb/postgresql-10 \
&& add-apt-repository -y ppa:cartodb/gis \
&& curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.34.0/install.sh | bash \
&& . ~/.nvm/nvm.sh \
&& locale-gen en_US.UTF-8 \
&& update-locale LANG=en_US.UTF-8
RUN set -ex \
&& apt-get update \
&& apt-get install -y \
g++-4.9 \
gcc-4.9 \
git \
libcairo2-dev \
libgdal-dev \
libgdal1i \
libgdal20 \
libgeos-dev \
libgif-dev \
libjpeg8-dev \
libjson-c-dev \
libpango1.0-dev \
libpixman-1-dev \
libproj-dev \
libprotobuf-c-dev \
libxml2-dev \
gdal-bin \
make \
nodejs \
protobuf-c-compiler \
pkg-config \
wget \
zip \
postgresql-10 \
postgresql-10-plproxy \
postgis=2.4.4.6+carto-1 \
postgresql-10-postgis-2.4=2.4.4.6+carto-1 \
postgresql-10-postgis-2.4-scripts=2.4.4.6+carto-1 \
postgresql-10-postgis-scripts=2.4.4.6+carto-1 \
postgresql-client-10 \
postgresql-client-common \
postgresql-common \
postgresql-contrib \
postgresql-plpython-10 \
postgresql-server-dev-10 \
&& wget http://download.redis.io/releases/redis-4.0.8.tar.gz \
&& tar xvzf redis-4.0.8.tar.gz \
&& cd redis-4.0.8 \
&& make \
&& make install \
&& cd .. \
&& rm redis-4.0.8.tar.gz \
&& rm -R redis-4.0.8 \
&& apt-get purge -y wget protobuf-c-compiler \
&& apt-get autoremove -y
# Configure PostgreSQL
RUN set -ex \
&& echo "listen_addresses='*'" >> /etc/postgresql/10/main/postgresql.conf \
&& echo "local all all trust" > /etc/postgresql/10/main/pg_hba.conf \
&& echo "host all all 0.0.0.0/0 trust" >> /etc/postgresql/10/main/pg_hba.conf \
&& echo "host all all ::1/128 trust" >> /etc/postgresql/10/main/pg_hba.conf \
&& /etc/init.d/postgresql start \
&& createdb template_postgis \
&& createuser publicuser \
&& psql -c "CREATE EXTENSION postgis" template_postgis \
&& /etc/init.d/postgresql stop
WORKDIR /srv
EXPOSE 5858
COPY ./scripts/nodejs-install.sh /src/nodejs-install.sh
RUN chmod 777 /src/nodejs-install.sh
CMD /src/nodejs-install.sh

85
docker/Dockerfile-nodejs-xenial-pg1121:latest
View File

@ -1,85 +0,0 @@
FROM ubuntu:xenial
# Use UTF8 to avoid encoding problems with pgsql
ENV LANG C.UTF-8
ENV NPROCS 1
ENV JOBS 1
ENV CXX g++-4.9
ENV PGUSER postgres
# Add external repos
RUN set -ex \
&& apt-get update \
&& apt-get install -y \
curl \
software-properties-common \
locales \
&& add-apt-repository -y ppa:ubuntu-toolchain-r/test \
&& add-apt-repository -y ppa:cartodb/postgresql-11 \
&& add-apt-repository -y ppa:cartodb/redis-next \
&& curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.34.0/install.sh | bash \
&& . ~/.nvm/nvm.sh \
&& locale-gen en_US.UTF-8 \
&& update-locale LANG=en_US.UTF-8
RUN set -ex \
&& apt-get update \
&& apt-get install -y \
g++-4.9 \
gcc-4.9 \
git \
libcairo2-dev \
libgdal-dev=2.3.2+dfsg-2build2~carto1 \
libgdal20=2.3.2+dfsg-2build2~carto1 \
libgeos-dev=3.7.1~carto1 \
libgif-dev \
libjpeg8-dev \
libjson-c-dev \
libpango1.0-dev \
libpixman-1-dev \
libproj-dev \
libprotobuf-c-dev \
libxml2-dev \
gdal-bin=2.3.2+dfsg-2build2~carto1 \
make \
nodejs \
protobuf-c-compiler \
pkg-config \
wget \
zip \
libopenscenegraph100v5 \
libsfcgal1 \
liblwgeom-2.5.0=2.5.1.4+carto-1 \
postgresql-11 \
postgresql-11-plproxy \
postgis=2.5.1.4+carto-1 \
postgresql-11-postgis-2.5=2.5.1.4+carto-1 \
postgresql-11-postgis-2.5-scripts=2.5.1.4+carto-1 \
postgresql-client-11 \
postgresql-client-common \
postgresql-common \
postgresql-contrib \
postgresql-plpython-11 \
postgresql-server-dev-11 \
redis=5:4.0.9-1carto1~xenial1 \
&& apt-get purge -y wget protobuf-c-compiler \
&& apt-get autoremove -y
# Configure PostgreSQL
RUN set -ex \
&& echo "listen_addresses='*'" >> /etc/postgresql/11/main/postgresql.conf \
&& echo "local all all trust" > /etc/postgresql/11/main/pg_hba.conf \
&& echo "host all all 0.0.0.0/0 trust" >> /etc/postgresql/11/main/pg_hba.conf \
&& echo "host all all ::1/128 trust" >> /etc/postgresql/11/main/pg_hba.conf \
&& /etc/init.d/postgresql start \
&& createdb template_postgis \
&& createuser publicuser \
&& psql -c "CREATE EXTENSION postgis" template_postgis \
&& /etc/init.d/postgresql stop
WORKDIR /srv
EXPOSE 5858
COPY ./scripts/nodejs-install.sh /src/nodejs-install.sh
RUN chmod 777 /src/nodejs-install.sh
CMD /src/nodejs-install.sh

33
docker/reference.md
View File

@ -1,33 +0,0 @@
# Testing with Docker
Before running the tests with docker, you'll need Docker installed and the docker image downloaded.
## Install docker
```shell
$ sudo apt install docker.io && sudo usermod -aG docker $(whoami)
```
## Download image
```shell
docker pull carto/IMAGE
```
## Carto account
* `https://hub.docker.com/r/carto/`
## Update image
* Edit the docker image file
* Build image:
* `docker build -t carto/IMAGE -f docker/DOCKER_FILE docker/`
* Upload to docker hub:
* Login into docker hub:
* `docker login`
* Create tag:
* `docker tag carto/IMAGE carto/IMAGE`
* Upload:
* `docker push carto/IMAGE`

13
docker/scripts/nodejs-install.sh
View File

@ -1,13 +0,0 @@
#!/bin/bash
export NVM_DIR="$HOME/.nvm"
[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
if [ -z $NODEJS_VERSION ]; then
NODEJS_VERSION="10"
NODEJS_VERSION_OPTIONS="--lts"
fi
nvm install $NODEJS_VERSION $NODEJS_VERSION_OPTIONS
nvm alias default $NODEJS_VERSION
nvm use default

14
docker/scripts/test-setup.sh
View File

@ -1,14 +0,0 @@
#!/bin/bash
/etc/init.d/postgresql start
source /src/nodejs-install.sh
# Install cartodb-postgresql extension
git clone https://github.com/CartoDB/cartodb-postgresql.git
cd cartodb-postgresql && make && make install && cd ..
cp config/environments/test.js.example config/environments/test.js
npm ci
npm test

2
lib/api/api-router.js
View File

@ -199,8 +199,8 @@ module.exports = class ApiRouter {
const apiRouter = router({ mergeParams: true });
const { paths, middlewares = [] } = route;
apiRouter.use(user(this.metadataBackend));
apiRouter.use(initLogger({ logger: this.serverOptions.logger }));
apiRouter.use(user(this.metadataBackend));
apiRouter.use(profiler({
enabled: this.serverOptions.useProfiler,
statsClient: global.statsClient

822
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

4
package.json
View File

@ -77,6 +77,7 @@
"mocha": "^7.2.0",
"moment": "2.22.1",
"nock": "9.2.6",
"nodemon": "^2.0.6",
"nyc": "^14.1.1",
"pino-pretty": "^4.0.0",
"redis": "2.8.0",
@ -94,8 +95,7 @@
"posttest": "npm run test:teardown",
"test:teardown": "NODE_ENV=test node test teardown",
"cover": "nyc --reporter=lcov npm test",
"test:docker": "docker run -e \"NODEJS_VERSION=$NODE_VERSION\" -v `pwd`:/srv $DOCKER_IMAGE bash docker/scripts/test-setup.sh && docker ps --filter status=dead --filter status=exited -aq | xargs docker rm -v",
"docker:bash": "docker run -it -v `pwd`:/srv $DOCKER_IMAGE bash"
"dev": "NODE_ENV=development nodemon app.js"
},
"engines": {
"node": "^12.16.3",

1
private Submodule

@ -0,0 +1 @@
Subproject commit bd86ec2ba2a51579add0dd7a556612ed0a8c05ca

6
test/acceptance/multilayer-test.js
View File

@ -512,7 +512,11 @@ var QueryTables = require('cartodb-query-tables').queryTables;
]
};
var statskey = 'user:localhost:mapviews';
var redisStatsClient = redis.createClient(global.environment.redis.port);
var redisStatsClient = redis.createClient(
{
port: global.environment.redis.port,
host: global.environment.redis.host
});
var expectedToken; // will be set on first post and checked on second
var now = strftime('%Y%m%d', new Date());
step(

18
test/acceptance/rate-limit-test.js
View File

@ -164,7 +164,11 @@ describe('rate limit', function () {
global.environment.enabledFeatures.rateLimitsEnabled = true;
global.environment.enabledFeatures.rateLimitsByEndpoint.anonymous = true;
redisClient = redis.createClient(global.environment.redis.port);
redisClient = redis.createClient(
{
port: global.environment.redis.port,
host: global.environment.redis.host
});
testClient = new TestClient(createMapConfig(), 1234);
});
@ -226,7 +230,11 @@ describe('rate limit middleware', function () {
});
rateLimit = rateLimitMiddleware(userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS);
redisClient = redis.createClient(global.environment.redis.port);
redisClient = redis.createClient(
{
port: global.environment.redis.port,
host: global.environment.redis.host
});
testClient = new TestClient(createMapConfig(), 1234);
const count = 1;
@ -290,7 +298,11 @@ function rateLimitAndVectorTilesTest (usePostGIS) {
global.environment.enabledFeatures.rateLimitsEnabled = true;
global.environment.enabledFeatures.rateLimitsByEndpoint.tile = true;
redisClient = redis.createClient(global.environment.redis.port);
redisClient = redis.createClient(
{
port: global.environment.redis.port,
host: global.environment.redis.host
});
const count = 1;
const period = 1;
const burst = 0;

6
test/acceptance/templates-test.js
View File

@ -1506,7 +1506,11 @@ describe('template_api', function () {
layergroup: layergroup
};
var statskey = 'user:localhost:mapviews';
var redisStatsClient = redis.createClient(global.environment.redis.port);
var redisStatsClient = redis.createClient(
{
port: global.environment.redis.port,
host: global.environment.redis.host
});
var templateId; // will be set on template post
var now = strftime('%Y%m%d', new Date());
var errors = [];

41
test/index.js
View File

@ -1,21 +1,24 @@
'use strict';
const util = require('util');
const path = require('path');
const exec = util.promisify(require('child_process').exec);
if (!process.env.NODE_ENV) {
console.error('Please set "NODE_ENV" variable, e.g.: "NODE_ENV=test"');
process.exit(1);
}
let configFileName = process.env.NODE_ENV;
if (process.env.CARTO_WINDSHAFT_ENV_BASED_CONF) {
// we override the file with the one with env vars
configFileName = 'config';
}
const environment = require(`../config/environments/${process.env.NODE_ENV}.js`);
const environment = require(`../config/environments/${configFileName}.js`);
const REDIS_PORT = environment.redis.port;
const REDIS_CELL_PATH = path.resolve(
process.platform === 'darwin'
? './test/support/libredis_cell.dylib'
: './test/support/libredis_cell.so'
);
const REDIS_HOST = environment.redis.host;
const PGHOST = environment.postgres.host;
const PGPORT = environment.postgres.port;
const TEST_USER_ID = 1;
const TEST_USER = environment.postgres_auth_user.replace('<%= user_id %>', TEST_USER_ID);
@ -24,29 +27,21 @@ const PUBLIC_USER = environment.postgres.user;
const PUBLIC_USER_PASSWORD = environment.postgres.password;
const TEST_DB = `${TEST_USER}_db`;
async function startRedis () {
await exec(`redis-server --port ${REDIS_PORT} --loadmodule ${REDIS_CELL_PATH} --logfile ${__dirname}/redis-server.log --daemonize yes`);
}
async function stopRedis () {
await exec(`redis-cli -p ${REDIS_PORT} shutdown`);
}
async function dropDatabase () {
await exec(`dropdb --if-exists ${TEST_DB}`, {
env: Object.assign({ PGUSER: 'postgres' }, process.env)
env: Object.assign({ PGUSER: 'postgres', PGHOST: PGHOST, PGPORT: PGPORT }, process.env)
});
}
async function createDatabase () {
await exec(`createdb -T template_postgis -EUTF8 "${TEST_DB}"`, {
env: Object.assign({ PGUSER: 'postgres' }, process.env)
env: Object.assign({ PGUSER: 'postgres', PGHOST: PGHOST, PGPORT: PGPORT }, process.env)
});
}
async function createDatabaseExtension () {
await exec(`psql -c "CREATE EXTENSION IF NOT EXISTS cartodb CASCADE;" ${TEST_DB}`, {
env: Object.assign({ PGUSER: 'postgres' }, process.env)
env: Object.assign({ PGUSER: 'postgres', PGHOST: PGHOST, PGPORT: PGPORT }, process.env)
});
}
@ -71,7 +66,7 @@ async function populateDatabase () {
`;
await exec(populateDatabaseCmd, {
env: Object.assign({ PGUSER: 'postgres' }, process.env)
env: Object.assign({ PGUSER: 'postgres', PGHOST: PGHOST, PGPORT: PGPORT }, process.env)
});
}
@ -82,13 +77,13 @@ async function populateRedis () {
HMSET rails:users:localhost \
id ${TEST_USER_ID} \
database_name "${TEST_DB}" \
database_host localhost \
database_host "${PGHOST}" \
map_key 1234
HMSET rails:users:cartodb250user \
id ${TEST_USER_ID} \
database_name "${TEST_DB}" \
database_host "localhost" \
database_host "${PGHOST}" \
database_password "${TEST_PASSWORD}" \
map_key 4321
@ -141,7 +136,7 @@ async function populateRedis () {
database_password "public"
`;
await exec(`echo "${commands}" | redis-cli -p ${REDIS_PORT} -n 5`);
await exec(`echo "${commands}" | redis-cli -h ${REDIS_HOST} -p ${REDIS_PORT} -n 5`);
}
async function main (args) {
@ -150,7 +145,6 @@ async function main (args) {
try {
switch (args[0]) {
case 'setup':
await startRedis();
await populateRedis();
await dropDatabase();
await createDatabase();
@ -158,7 +152,6 @@ async function main (args) {
await populateDatabase();
break;
case 'teardown':
await stopRedis();
break;
default:
throw new Error('Missing "mode" argument. Valid ones: "setup" or "teardown"');

6
test/integration/analysis-backend-limits-test.js
View File

@ -16,7 +16,11 @@ describe('analysis-backend limits', function () {
var user = 'localhost';
beforeEach(function () {
redisClient = redis.createClient(global.environment.redis.port);
redisClient = redis.createClient(
{
port: global.environment.redis.port,
host: global.environment.redis.host
});
keysToDelete = {};
var redisPool = new RedisPool(global.environment.redis);
this.metadataBackend = cartodbRedis({ pool: redisPool });

6
test/integration/template-maps-limits-test.js
View File

@ -30,7 +30,11 @@ describe('TemplateMaps limits', function () {
};
}
var redisClient = redis.createClient(global.environment.redis.port);
var redisClient = redis.createClient(
{
port: global.environment.redis.port,
host: global.environment.redis.host
});
var redisPool = new RedisPool(global.environment.redis);
afterEach(function (done) {

22
test/support/test-helper.js
View File

@ -10,8 +10,14 @@ var redis = require('redis');
const setICUEnvVariable = require('../../lib/utils/icu-data-env-setter');
// set environment specific variables
global.environment = require('../../config/environments/test');
global.environment.name = 'test';
let configFileName = process.env.NODE_ENV;
if (process.env.CARTO_WINDSHAFT_ENV_BASED_CONF) {
// we override the file with the one with env vars
configFileName = 'config';
}
global.environment = require(`../../config/environments/${configFileName}.js`);
process.env.NODE_ENV = 'test';
setICUEnvVariable();
@ -82,7 +88,11 @@ var redisClient;
beforeEach(function () {
if (!redisClient) {
redisClient = redis.createClient(global.environment.redis.port);
redisClient = redis.createClient(
{
port: global.environment.redis.port,
host: global.environment.redis.host
});
}
});
@ -145,7 +155,11 @@ function deleteRedisKeys (keysToDelete, callback) {
}
Object.keys(keysToDelete).forEach(function (k) {
var redisClient = redis.createClient(global.environment.redis.port);
var redisClient = redis.createClient(
{
port: global.environment.redis.port,
host: global.environment.redis.host
});
redisClient.select(keysToDelete[k], function () {
redisClient.del(k, function (err, deletedKeysCount) {
assert.ifError(err);

2
test/unit/ported/windshaft-server-test.js
View File

@ -8,7 +8,7 @@ var serverOptions = require('../../../lib/server-options');
describe('windshaft', function () {
it('should have valid global environment', function () {
assert.strictEqual(global.environment.name, 'test');
assert.strictEqual(global.environment.environment, 'test');
});
it('can instantiate a Windshaft object (configured express instance)', function () {