cartodb/CartoDB-SQL-API
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #599 from CartoDB/max-age-directive

Browse Source 
Set directive 'max-age' to a fallback value
This commit is contained in:
Daniel G. Aubert 2019-07-05 16:19:14 +02:00 committed by GitHub
commit a15ee27932
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 143 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
NEWS.md
View File

@ -4,6 +4,7 @@
Released 2019-mm-dd
Announcements:
* Cache control header fine tuning. Set a shorter value for "max-age" directive if there is no way to know when to trigger the invalidation.
* Upgrade devel dependency `sqlite3` to version `4.0.6`
* Log queries (https://github.com/CartoDB/CartoDB-SQL-API/pull/574)
* Improve batch-queries draining while exiting the process #582

12
app/controllers/query_controller.js
View File

@ -23,7 +23,8 @@ const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimitsMiddleware;
const handleQueryMiddleware = require('../middlewares/handle-query');
const logMiddleware = require('../middlewares/log');
var ONE_YEAR_IN_SECONDS = 31536000; // 1 year time to live by default
const ONE_YEAR_IN_SECONDS = 31536000; // ttl in cache provider
const FIVE_MINUTES_IN_SECONDS = 60 * 5; // ttl in cache provider
function QueryController(metadataBackend, userDatabaseService, tableCache, statsd_client, userLimitsService) {
this.metadataBackend = metadataBackend;
@ -190,8 +191,13 @@ QueryController.prototype.handleQuery = function (req, res, next) {
if (cachePolicy === 'persist') {
res.header('Cache-Control', 'public,max-age=' + ONE_YEAR_IN_SECONDS);
} else {
var maxAge = (mayWrite) ? 0 : ONE_YEAR_IN_SECONDS;
res.header('Cache-Control', 'no-cache,max-age='+maxAge+',must-revalidate,public');
if (affectedTables && affectedTables.getTables().every(table => table.updated_at !== null)) {
const maxAge = mayWrite ? 0 : (global.settings.cache.ttl || ONE_YEAR_IN_SECONDS);
res.header('Cache-Control', `no-cache,max-age=${maxAge},must-revalidate,public`);
} else {
const maxAge = global.settings.cache.fallbackTtl || FIVE_MINUTES_IN_SECONDS;
res.header('Cache-Control', `no-cache,max-age=${maxAge},must-revalidate,public`);
}
}
// Only set an X-Cache-Channel for responses we want Varnish to cache.

5
config/environments/development.js.example
View File

@ -131,3 +131,8 @@ module.exports.validatePGEntitiesAccess = false;
module.exports.dataIngestionLogPath = 'logs/data-ingestion.log';
module.exports.logQueries = true;
module.exports.maxQueriesLogLength = 2000;
module.exports.cache = {
ttl: 60 * 60 * 24 * 365, // one year in seconds
fallbackTtl: 60 * 5 // five minutes in seconds
};

5
config/environments/production.js.example
View File

@ -135,3 +135,8 @@ module.exports.validatePGEntitiesAccess = false;
module.exports.dataIngestionLogPath = 'logs/data-ingestion.log';
module.exports.logQueries = true;
module.exports.maxQueriesLogLength = 1024;
module.exports.cache = {
ttl: 60 * 60 * 24 * 365, // one year in seconds
fallbackTtl: 60 * 5 // five minutes in seconds
};

5
config/environments/staging.js.example
View File

@ -132,3 +132,8 @@ module.exports.validatePGEntitiesAccess = false;
module.exports.dataIngestionLogPath = 'logs/data-ingestion.log';
module.exports.logQueries = true;
module.exports.maxQueriesLogLength = 1024;
module.exports.cache = {
ttl: 60 * 60 * 24 * 365, // one year in seconds
fallbackTtl: 60 * 5 // five minutes in seconds
};

5
config/environments/test.js.example
View File

@ -132,3 +132,8 @@ module.exports.validatePGEntitiesAccess = false;
module.exports.dataIngestionLogPath = 'logs/data-ingestion.log';
module.exports.logQueries = true;
module.exports.maxQueriesLogLength = 1024;
module.exports.cache = {
ttl: 60 * 60 * 24 * 365, // one year in seconds
fallbackTtl: 60 * 5 // five minutes in seconds
};

2
test/acceptance/app.test.js
View File

@ -246,7 +246,7 @@ it('TRUNCATE TABLE with GET and auth', function(done){
assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
// table should not get a cache channel as it won't get invalidated
assert.ok(!res.headers.hasOwnProperty('x-cache-channel'));
assert.equal(res.headers['cache-control'], expected_cache_control);
assert.equal(res.headers['cache-control'], 'no-cache,max-age=300,must-revalidate,public');
var pbody = JSON.parse(res.body);
assert.equal(pbody.total_rows, 1);
assert.equal(pbody.rows[0].count, 0);

112
test/acceptance/cache-headers.js Normal file
View File

@ -0,0 +1,112 @@
'use strict';
const server = require('../../app/server')();
const assert = require('../support/assert');
const qs = require('querystring');
describe('cache headers', function () {
it('should return a Vary header', function (done) {
assert.response(server, {
url: `/api/v1/sql?${qs.encode({
api_key: '1234',
q: 'select * from untitle_table_4'
})}`,
headers: {
host: 'vizzuality.cartodb.com'
},
method: 'GET'
},
{},
function (err, res) {
assert.equal(res.headers.vary, 'Authorization');
done();
});
});
it('should return a proper max-age when CDB_TableMetadata table includes the last updated time', function (done) {
const ONE_YEAR_IN_SECONDS = 60 * 60 * 24 * 365;
const noTtl = 0;
const fallbackTtl = global.settings.cache.fallbackTtl;
const ttl = global.settings.cache.ttl || ONE_YEAR_IN_SECONDS;
const tableName = `wadus_table_${Date.now()}`;
assert.response(server, {
url: `/api/v1/sql?${qs.encode({
api_key: '1234',
q: `create table ${tableName}()`
})}`,
headers: {
host: 'vizzuality.cartodb.com'
},
method: 'GET'
},
{},
function(err, res) {
assert.equal(res.headers['cache-control'], `no-cache,max-age=${noTtl},must-revalidate,public`);
assert.response(server, {
url: `/api/v1/sql?${qs.encode({
api_key: '1234',
q: `select * from ${tableName}`
})}`,
headers: {
host: 'vizzuality.cartodb.com'
},
method: 'GET'
}, {},
function(err, res) {
assert.equal(res.headers['cache-control'], `no-cache,max-age=${fallbackTtl},must-revalidate,public`);
assert.response(server, {
url: `/api/v1/sql?${qs.encode({
api_key: '1234',
q: `select CDB_TableMetadataTouch('${tableName}'::regclass)`
})}`,
headers: {
host: 'vizzuality.cartodb.com'
},
method: 'GET'
}, {},
function(err, res) {
assert.equal(res.headers['cache-control'], `no-cache,max-age=${ttl},must-revalidate,public`);
assert.response(server, {
url: `/api/v1/sql?${qs.encode({
api_key: '1234',
q: `select * from ${tableName}`
})}`,
headers: {
host: 'vizzuality.cartodb.com'
},
method: 'GET'
}, {},
function(err, res) {
assert.equal(res.headers['cache-control'], `no-cache,max-age=${ttl},must-revalidate,public`);
done();
});
});
});
});
});
it('should return a proper max-age when the query doesn\'t use any table', function (done) {
const ONE_YEAR_IN_SECONDS = 60 * 60 * 24 * 365;
const ttl = global.settings.cache.ttl || ONE_YEAR_IN_SECONDS;
assert.response(server, {
url: `/api/v1/sql?${qs.encode({
api_key: '1234',
q: `select 1`
})}`,
headers: {
host: 'vizzuality.cartodb.com'
},
method: 'GET'
},
{},
function (err, res) {
assert.equal(res.headers['cache-control'], `no-cache,max-age=${ttl},must-revalidate,public`);
done();
});
});
});

21
test/acceptance/cache.js
View File

@ -1,21 +0,0 @@
'use strict';
var server = require('../../app/server')();
const assert = require('../support/assert');
describe('Cache', function () {
it('should return a Vary header', function (done) {
assert.response(server, {
url: '/api/v1/sql?api_key=1234&g=select%20*%20from%20untitle_table_4',
headers: {
host: 'vizzuality.cartodb.com'
},
method: 'GET'
},
{},
function(err, res) {
assert.equal(res.headers.vary, 'Authorization');
done();
});
});
});