cartodb/CartoDB-SQL-API
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

build onprem image (#678)

Browse Source 
* build onprem image

Co-authored-by: Jorge Tarrero <thedae@gmail.com>
This commit is contained in:
Alberto Hernández 2021-03-22 13:39:33 +01:00 committed by GitHub
parent 62558e1c7f
commit 42eeefd451
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 126 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
.github/workflows/main.yml vendored
View File

@ -11,6 +11,7 @@ env:
ARTIFACTS_PROJECT_ID: cartodb-on-gcp-main-artifacts
NODE_VERSION: '12.18.3'
_DOCKER_IMAGE_TO_REPLACE: 'sql-api'
COMPOSE_PROJECT_NAME: 'carto'
jobs:
build-and-test:
@ -99,7 +100,10 @@ jobs:
BRANCH_NAME=${GITHUB_HEAD_REF#refs/heads/}
BRANCH_NAME=${BRANCH_NAME//\//-}
echo "Building sql-api image from branch: $BRANCH_NAME, commit: ${GITHUB_SHA::7}..."
docker build -f private/Dockerfile --label="org.opencontainers.image.created=$(date --rfc-3339=seconds)" --label=org.opencontainers.image.revision=${GITHUB_SHA} -t gcr.io/$ARTIFACTS_PROJECT_ID/sql-api:${BRANCH_NAME} -t gcr.io/$ARTIFACTS_PROJECT_ID/sql-api:${GITHUB_SHA::7} -t gcr.io/$ARTIFACTS_PROJECT_ID/sql-api:${BRANCH_NAME}--${GITHUB_SHA::7} .
docker build -f private/Dockerfile --label="org.opencontainers.image.created=$(date --rfc-3339=seconds)" --label=org.opencontainers.image.revision=${GITHUB_SHA} -t gcr.io/$ARTIFACTS_PROJECT_ID/sql-api:latest -t gcr.io/$ARTIFACTS_PROJECT_ID/sql-api:${BRANCH_NAME} -t gcr.io/$ARTIFACTS_PROJECT_ID/sql-api:${GITHUB_SHA::7} -t gcr.io/$ARTIFACTS_PROJECT_ID/sql-api:${BRANCH_NAME}--${GITHUB_SHA::7} .
echo "Building sql-api-onprem image from branch: $BRANCH_NAME, commit: ${GITHUB_SHA::7}..."
docker build -f private/Dockerfile.onprem --label="org.opencontainers.image.created=$(date --rfc-3339=seconds)" --label=org.opencontainers.image.revision=${GITHUB_SHA} -t gcr.io/$ARTIFACTS_PROJECT_ID/sql-api-onprem:${BRANCH_NAME} -t gcr.io/$ARTIFACTS_PROJECT_ID/sql-api-onprem:${GITHUB_SHA::7} -t gcr.io/$ARTIFACTS_PROJECT_ID/sql-api-onprem:${BRANCH_NAME}--${GITHUB_SHA::7} .
- name: Set up gcloud I
uses: google-github-actions/setup-gcloud@master
@ -116,13 +120,15 @@ jobs:
run: |
./scripts/init.sh
- name: Test networking
- name: Run all tests
run: |
docker-compose -f docker-compose.yml -f docker-compose.test.yml run --rm test-networking
- name: Test DB seed
run: |
docker-compose -f docker-compose.yml -f docker-compose.test.yml run --rm test-db
docker run --rm --name tavern-tester \
--env CARTO_USER=carto \
--env CARTO_DOMAIN=localhost.lan \
--network host \
--env-file \
.env gcr.io/cartodb-on-gcp-main-artifacts/tavern-tester:latest \
all dev
- name: Push image
run: |
@ -134,4 +140,8 @@ jobs:
docker push gcr.io/$ARTIFACTS_PROJECT_ID/sql-api:${GITHUB_SHA::7}
docker push gcr.io/$ARTIFACTS_PROJECT_ID/sql-api:${BRANCH_NAME}--${GITHUB_SHA::7}
echo 'Pushing onprem images to the registry...'
docker push gcr.io/$ARTIFACTS_PROJECT_ID/sql-api-onprem:${BRANCH_NAME}
docker push gcr.io/$ARTIFACTS_PROJECT_ID/sql-api-onprem:${GITHUB_SHA::7}
docker push gcr.io/$ARTIFACTS_PROJECT_ID/sql-api-onprem:${BRANCH_NAME}--${GITHUB_SHA::7}

6
.github/workflows/master.yml vendored
View File

@ -28,6 +28,7 @@ jobs:
echo ${GITHUB_SHA::7}
echo ${GITHUB_REF##*/}
docker build -f private/Dockerfile --label="org.opencontainers.image.created=$(date --rfc-3339=seconds)" --label=org.opencontainers.image.revision=${GITHUB_SHA} -t gcr.io/$ARTIFACTS_PROJECT_ID/sql-api:latest -t gcr.io/$ARTIFACTS_PROJECT_ID/sql-api:${GITHUB_REF##*/} -t gcr.io/$ARTIFACTS_PROJECT_ID/sql-api:${GITHUB_SHA::7} -t gcr.io/$ARTIFACTS_PROJECT_ID/sql-api:${GITHUB_REF##*/}--${GITHUB_SHA::7} .
docker build -f private/Dockerfile.onprem --label="org.opencontainers.image.created=$(date --rfc-3339=seconds)" --label=org.opencontainers.image.revision=${GITHUB_SHA} -t gcr.io/$ARTIFACTS_PROJECT_ID/sql-api-onprem:latest -t gcr.io/$ARTIFACTS_PROJECT_ID/sql-api-onprem:${GITHUB_REF##*/} -t gcr.io/$ARTIFACTS_PROJECT_ID/sql-api-onprem:${GITHUB_SHA::7} -t gcr.io/$ARTIFACTS_PROJECT_ID/sql-api-onprem:${GITHUB_REF##*/}--${GITHUB_SHA::7} .
- name: Setup gcloud authentication
uses: google-github-actions/setup-gcloud@master
@ -42,5 +43,10 @@ jobs:
run: |
docker push gcr.io/$ARTIFACTS_PROJECT_ID/sql-api:${GITHUB_REF##*/}
docker push gcr.io/$ARTIFACTS_PROJECT_ID/sql-api:${GITHUB_SHA::7}
docker push gcr.io/$ARTIFACTS_PROJECT_ID/sql-api:${GITHUB_REF##*/}--${GITHUB_SHA::7}
docker push gcr.io/$ARTIFACTS_PROJECT_ID/sql-api:latest
docker push gcr.io/$ARTIFACTS_PROJECT_ID/sql-api-onprem:${GITHUB_REF##*/}
docker push gcr.io/$ARTIFACTS_PROJECT_ID/sql-api-onprem:${GITHUB_SHA::7}
docker push gcr.io/$ARTIFACTS_PROJECT_ID/sql-api-onprem:${GITHUB_REF##*/}--${GITHUB_SHA::7}
docker push gcr.io/$ARTIFACTS_PROJECT_ID/sql-api-onprem:latest

1
README.md
View File

@ -64,6 +64,7 @@ $ npm test
You can try to run the tests against the dependencies from the `dev-env`. To do so, you need to build the test docker image:
```shell
$ docker build -t sqlapi -f private/Dockerfile .
$ docker-compose -f private/docker-compose.yml build
```

13
config/environments/config.js
View File

@ -17,20 +17,17 @@ module.exports.routes = {
],
// Optional: attach middlewares at the begining of the router
// to perform custom operations.
middlewares: [
function noop () {
return function noopMiddleware (req, res, next) {
next();
}
}
],
// This must be a **comma-separated string** with the list of paths to the middlewares. Pe:
// CARTO_SQL_PREROUTING_MIDDLEWARES=/usr/src/lib/mw1.js,/usr/src/lib/mw2.js
// Note: The list order is kept when loading the middlewares!
middlewares: process.env.CARTO_SQL_PREROUTING_MIDDLEWARES || '',
sql: [{
// Required
paths: [
'/sql'
],
// Optional
middlewares: []
middlewares: process.env.CARTO_SQL_SQLROUTING_MIDDLEWARES || ''
}]
}]
};

13
config/environments/development.js.example
View File

@ -14,20 +14,17 @@ module.exports.routes = {
],
// Optional: attach middlewares at the begining of the router
// to perform custom operations.
middlewares: [
function noop () {
return function noopMiddleware (req, res, next) {
next();
}
}
],
// This must be a **comma-separated string** with the list of paths to the middlewares. Pe:
// CARTO_SQL_PREROUTING_MIDDLEWARES=/usr/src/lib/mw1.js,/usr/src/lib/mw2.js
// Note: The list order is kept when loading the middlewares!
middlewares: process.env.CARTO_SQL_PREROUTING_MIDDLEWARES || '',
sql: [{
// Required
paths: [
'/sql'
],
// Optional
middlewares: []
middlewares: process.env.CARTO_SQL_SQLROUTING_MIDDLEWARES || ''
}]
}]
};

13
config/environments/production.js.example
View File

@ -14,20 +14,17 @@ module.exports.routes = {
],
// Optional: attach middlewares at the begining of the router
// to perform custom operations.
middlewares: [
function noop () {
return function noopMiddleware (req, res, next) {
next();
}
}
],
// This must be a **comma-separated string** with the list of paths to the middlewares. Pe:
// CARTO_SQL_PREROUTING_MIDDLEWARES=/usr/src/lib/mw1.js,/usr/src/lib/mw2.js
// Note: The list order is kept when loading the middlewares!
middlewares: process.env.CARTO_SQL_PREROUTING_MIDDLEWARES || '',
sql: [{
// Required
paths: [
'/sql'
],
// Optional
middlewares: []
middlewares: process.env.CARTO_SQL_SQLROUTING_MIDDLEWARES || ''
}]
}]
};

13
config/environments/staging.js.example
View File

@ -14,20 +14,17 @@ module.exports.routes = {
],
// Optional: attach middlewares at the begining of the router
// to perform custom operations.
middlewares: [
function noop () {
return function noopMiddleware (req, res, next) {
next();
}
}
],
// This must be a **comma-separated string** with the list of paths to the middlewares. Pe:
// CARTO_SQL_PREROUTING_MIDDLEWARES=/usr/src/lib/mw1.js,/usr/src/lib/mw2.js
// Note: The list order is kept when loading the middlewares!
middlewares: process.env.CARTO_SQL_PREROUTING_MIDDLEWARES || '',
sql: [{
// Required
paths: [
'/sql'
],
// Optional
middlewares: []
middlewares: process.env.CARTO_SQL_SQLROUTING_MIDDLEWARES || ''
}]
}]
};

13
config/environments/test.js.example
View File

@ -14,20 +14,17 @@ module.exports.routes = {
],
// Optional: attach middlewares at the begining of the router
// to perform custom operations.
middlewares: [
function noop () {
return function noopMiddleware (req, res, next) {
next();
}
}
],
// This must be a **comma-separated string** with the list of paths to the middlewares. Pe:
// CARTO_SQL_PREROUTING_MIDDLEWARES=/usr/src/lib/mw1.js,/usr/src/lib/mw2.js
// Note: The list order is kept when loading the middlewares!
middlewares: process.env.CARTO_SQL_PREROUTING_MIDDLEWARES || '',
sql: [{
// Required
paths: [
'/sql'
],
// Optional
middlewares: []
middlewares: process.env.CARTO_SQL_SQLROUTING_MIDDLEWARES || ''
}]
}]
};

21
lib/api/api-router.js
View File

@ -29,6 +29,7 @@ module.exports = class ApiRouter {
this.versionController = new VersionController();
this.jobsWipController = new JobsWipController({ jobService });
this.pubSubMetricsService = PubSubMetricsService.build();
this.logger = logger;
this.sqlRouter = new SqlRouter({
metadataBackend,
@ -41,11 +42,23 @@ module.exports = class ApiRouter {
route (app, routes) {
routes.forEach(route => {
const apiRouter = router({ mergeParams: true });
const paths = route.paths;
const middlewares = route.middlewares || [];
const preRoutingMiddlewares = route.middlewares;
middlewares.forEach(middleware => apiRouter.use(middleware()));
if (preRoutingMiddlewares !== undefined && preRoutingMiddlewares.trim()) {
preRoutingMiddlewares.split(',').forEach(middlewarePath => {
try {
const middleware = require(middlewarePath).middlewares;
if (Array.isArray(middleware)) {
middleware.forEach(m => apiRouter.use(m()));
} else {
apiRouter.use(middleware());
}
} catch (e) {
this.logger.error({ exception: e, path: middlewarePath }, 'Prerouting middleware not found, skipping');
}
});
}
// FIXME: version controller should be attached to the main entry point: "/"
// instead of "/api/:version" or "/user/:user/api/:version"
@ -61,7 +74,7 @@ module.exports = class ApiRouter {
paths.forEach(path => app.use(path, apiRouter));
apiRouter.use(error());
apiRouter.use(error({ logger: this.logger }));
apiRouter.use(pubSubMetrics(this.pubSubMetricsService));
});
}

2
lib/api/jobs-wip-controller.js
View File

@ -13,7 +13,7 @@ module.exports = class JobsWipController {
bodyParser(),
listWorkInProgressJobs(this.jobService),
sendResponse(),
error()
error({ logger: null })
]);
}
};

6
lib/api/middlewares/error.js
View File

@ -2,13 +2,13 @@
const errorHandlerFactory = require('../../services/error-handler-factory');
module.exports = function error () {
module.exports = function error ({ logger }) {
return function errorMiddleware (err, req, res, next) {
const { logger } = res.locals;
const errorHandler = errorHandlerFactory(err);
const errorResponse = errorHandler.getResponse();
const errorLogger = res.locals.logger || logger;
logger.error({ exception: err }, 'Error while handling the request');
errorLogger.error({ exception: err }, 'Error while handling the request');
// Force inline content disposition
res.header('Content-Disposition', 'inline');

18
lib/api/sql/sql-router.js
View File

@ -24,6 +24,7 @@ module.exports = class SqlRouter {
const userDatabaseService = new UserDatabaseService(metadataBackend);
const userLimitsService = new UserLimitsService(metadataBackend, userLimitsServiceOptions);
this.logger = logger;
this.queryController = new QueryController({
metadataBackend,
userDatabaseService,
@ -55,9 +56,22 @@ module.exports = class SqlRouter {
const sqlRouter = router({ mergeParams: true });
const paths = route.paths;
const middlewares = route.middlewares || [];
const preRoutingMiddlewares = route.middlewares;
middlewares.forEach(middleware => sqlRouter.use(middleware()));
if (preRoutingMiddlewares !== undefined && preRoutingMiddlewares.trim()) {
preRoutingMiddlewares.split(',').forEach(middlewarePath => {
try {
const middleware = require(middlewarePath).middlewares;
if (Array.isArray(middleware)) {
middleware.forEach(m => apiRouter.use(m()));
} else {
apiRouter.use(middleware());
}
} catch (e) {
this.logger.error({ exception: e, path: middlewarePath }, 'SQL prerouting middleware not found, skipping');
}
});
}
sqlRouter.use(socketTimeout());
sqlRouter.use(cors());

2
private

@ -1 +1 @@
Subproject commit d50798990e04a12e0e84b08c53e025e2f56cfd08
Subproject commit cb59185b1ad0654b0e805741aaf29d41ec688357

28
test/acceptance/custom-middlewares-test.js
View File

@ -16,12 +16,6 @@ describe('custom middlewares', function () {
statusCode: 418
};
const customMiddleware = function teapot () {
return function teapotMiddleware (req, res) {
res.status(418).send('I\'m a teapot');
};
};
describe('wired in /api/v1/', function () {
before(function () {
this.backupRoutes = global.settings.routes;
@ -32,9 +26,7 @@ describe('custom middlewares', function () {
'/api/:version',
'/user/:user/api/:version'
],
middlewares: [
customMiddleware
],
middlewares: '../../test/support/middlewares/teapot-headers.js,../../test/support/middlewares/teapot-response.js',
sql: [{
paths: [
'/sql'
@ -59,6 +51,8 @@ describe('custom middlewares', function () {
return done(err);
}
assert.strictEqual(res.headers['x-what-am-i'], 'I\'m a teapot');
assert.strictEqual(res.headers['x-again-what-am-i'], 'I\'m a teapot');
assert.strictEqual(res.body, 'I\'m a teapot');
done();
});
@ -73,6 +67,8 @@ describe('custom middlewares', function () {
return done(err);
}
assert.strictEqual(res.headers['x-what-am-i'], 'I\'m a teapot');
assert.strictEqual(res.headers['x-again-what-am-i'], 'I\'m a teapot');
assert.strictEqual(res.body, 'I\'m a teapot');
done();
});
@ -87,6 +83,8 @@ describe('custom middlewares', function () {
return done(err);
}
assert.strictEqual(res.headers['x-what-am-i'], 'I\'m a teapot');
assert.strictEqual(res.headers['x-again-what-am-i'], 'I\'m a teapot');
assert.strictEqual(res.body, 'I\'m a teapot');
done();
});
@ -101,6 +99,8 @@ describe('custom middlewares', function () {
return done(err);
}
assert.strictEqual(res.headers['x-what-am-i'], 'I\'m a teapot');
assert.strictEqual(res.headers['x-again-what-am-i'], 'I\'m a teapot');
assert.strictEqual(res.body, 'I\'m a teapot');
done();
});
@ -122,6 +122,8 @@ describe('custom middlewares', function () {
return done(err);
}
assert.strictEqual(res.headers['x-what-am-i'], 'I\'m a teapot');
assert.strictEqual(res.headers['x-again-what-am-i'], 'I\'m a teapot');
assert.strictEqual(res.body, 'I\'m a teapot');
done();
});
@ -136,6 +138,8 @@ describe('custom middlewares', function () {
return done(err);
}
assert.strictEqual(res.headers['x-what-am-i'], 'I\'m a teapot');
assert.strictEqual(res.headers['x-again-what-am-i'], 'I\'m a teapot');
assert.strictEqual(res.body, 'I\'m a teapot');
done();
});
@ -156,6 +160,8 @@ describe('custom middlewares', function () {
return done(err);
}
assert.strictEqual(res.headers['x-what-am-i'], 'I\'m a teapot');
assert.strictEqual(res.headers['x-again-what-am-i'], 'I\'m a teapot');
assert.strictEqual(res.body, 'I\'m a teapot');
done();
});
@ -175,6 +181,8 @@ describe('custom middlewares', function () {
return done(err);
}
assert.strictEqual(res.headers['x-what-am-i'], 'I\'m a teapot');
assert.strictEqual(res.headers['x-again-what-am-i'], 'I\'m a teapot');
assert.strictEqual(res.body, 'I\'m a teapot');
done();
});
@ -195,7 +203,7 @@ describe('custom middlewares', function () {
paths: [
'/sql'
],
middlewares: [customMiddleware]
middlewares: '../../../test/support/middlewares/teapot-response.js'
}]
}]
};

14
test/support/middlewares/teapot-headers.js Normal file
View File

@ -0,0 +1,14 @@
exports.middlewares = [
function () {
return function teapotHeaderMiddleware (req, res, next) {
res.header('X-What-Am-I', 'I\'m a teapot');
return next();
};
},
function () {
return function teapotAnotherHeaderMiddleware (req, res, next) {
res.header('X-Again-What-Am-I', 'I\'m a teapot');
return next();
};
}
];

5
test/support/middlewares/teapot-response.js Normal file
View File

@ -0,0 +1,5 @@
exports.middlewares = function () {
return function teapotMiddleware (req, res) {
res.status(418).send('I\'m a teapot');
};
};