From 0a7edc2e11d8d58a8165ae1d6db0181798a62e42 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Thu, 26 Jul 2018 11:13:16 +0200 Subject: [PATCH 1/2] Add missing COPY config parameters to staging.js.example I forgot to add these to the staging example file and missed them when testing config changes there. Related to https://github.com/CartoDB/CartoDB-SQL-API/pull/523 --- config/environments/staging.js.example | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index 2c5929a6..6ffbab2d 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -36,6 +36,9 @@ module.exports.db_batch_port = '5432'; module.exports.finished_jobs_ttl_in_seconds = 2 * 3600; // 2 hours module.exports.batch_query_timeout = 12 * 3600 * 1000; // 12 hours in milliseconds module.exports.batch_log_filename = 'logs/batch-queries.log'; +module.exports.copy_timeout = "'5h'"; +module.exports.copy_from_max_post_size = 1.99 * 1024 * 1024 * 1024 // 1.99 GB; +module.exports.copy_from_max_post_size_pretty = '2 GB'; // Max number of queued jobs a user can have at a given time module.exports.batch_max_queued_jobs = 64; // Capacity strategy to use. From a709e8f26f0ea20556e99b202165740dda19046f Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Thu, 26 Jul 2018 12:52:18 +0200 Subject: [PATCH 2/2] Set the default copy_from_max_post_size to 2 GB Set the default copy_from_max_post_size to exactly 2 GB, as the nginx limitation has been removed for that endpoint. So, the application is now in control and this keeps the code clear and clean. --- app/controllers/copy_controller.js | 2 +- config/environments/development.js.example | 2 +- config/environments/production.js.example | 2 +- config/environments/staging.js.example | 2 +- config/environments/test.js.example | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js index 41740d27..25bd85cb 100644 --- a/app/controllers/copy_controller.js +++ b/app/controllers/copy_controller.js @@ -102,7 +102,7 @@ function handleCopyFrom (logger) { const sql = req.query.q; const { userDbParams, user, dbRemainingQuota } = res.locals; const isGzip = req.get('content-encoding') === 'gzip'; - const COPY_FROM_MAX_POST_SIZE = global.settings.copy_from_max_post_size || 1.99 * 1024 * 1024 * 1024; // 1.99 GB + const COPY_FROM_MAX_POST_SIZE = global.settings.copy_from_max_post_size || 2 * 1024 * 1024 * 1024; // 2 GB const COPY_FROM_MAX_POST_SIZE_PRETTY = global.settings.copy_from_max_post_size_pretty || '2 GB'; const streamCopy = new StreamCopy(sql, userDbParams); diff --git a/config/environments/development.js.example b/config/environments/development.js.example index ad521cbb..70463359 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -36,7 +36,7 @@ module.exports.finished_jobs_ttl_in_seconds = 2 * 3600; // 2 hours module.exports.batch_query_timeout = 12 * 3600 * 1000; // 12 hours in milliseconds module.exports.batch_log_filename = 'logs/batch-queries.log'; module.exports.copy_timeout = "'5h'"; -module.exports.copy_from_max_post_size = 1.99 * 1024 * 1024 * 1024 // 1.99 GB; +module.exports.copy_from_max_post_size = 2 * 1024 * 1024 * 1024 // 2 GB; module.exports.copy_from_max_post_size_pretty = '2 GB'; // Max number of queued jobs a user can have at a given time module.exports.batch_max_queued_jobs = 64; diff --git a/config/environments/production.js.example b/config/environments/production.js.example index 488b75bf..aea4011e 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -37,7 +37,7 @@ module.exports.finished_jobs_ttl_in_seconds = 2 * 3600; // 2 hours module.exports.batch_query_timeout = 12 * 3600 * 1000; // 12 hours in milliseconds module.exports.batch_log_filename = 'logs/batch-queries.log'; module.exports.copy_timeout = "'5h'"; -module.exports.copy_from_max_post_size = 1.99 * 1024 * 1024 * 1024 // 1.99 GB; +module.exports.copy_from_max_post_size = 2 * 1024 * 1024 * 1024 // 2 GB; module.exports.copy_from_max_post_size_pretty = '2 GB'; // Max number of queued jobs a user can have at a given time module.exports.batch_max_queued_jobs = 64; diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index 6ffbab2d..4fc7e89e 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -37,7 +37,7 @@ module.exports.finished_jobs_ttl_in_seconds = 2 * 3600; // 2 hours module.exports.batch_query_timeout = 12 * 3600 * 1000; // 12 hours in milliseconds module.exports.batch_log_filename = 'logs/batch-queries.log'; module.exports.copy_timeout = "'5h'"; -module.exports.copy_from_max_post_size = 1.99 * 1024 * 1024 * 1024 // 1.99 GB; +module.exports.copy_from_max_post_size = 2 * 1024 * 1024 * 1024 // 2 GB; module.exports.copy_from_max_post_size_pretty = '2 GB'; // Max number of queued jobs a user can have at a given time module.exports.batch_max_queued_jobs = 64; diff --git a/config/environments/test.js.example b/config/environments/test.js.example index f390120f..a849b3c6 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -34,7 +34,7 @@ module.exports.finished_jobs_ttl_in_seconds = 2 * 3600; // 2 hours module.exports.batch_query_timeout = 5 * 1000; // 5 seconds in milliseconds module.exports.batch_log_filename = 'logs/batch-queries.log'; module.exports.copy_timeout = "'5h'"; -module.exports.copy_from_max_post_size = 1.99 * 1024 * 1024 * 1024 // 1.99 GB; +module.exports.copy_from_max_post_size = 2 * 1024 * 1024 * 1024 // 2 GB; module.exports.copy_from_max_post_size_pretty = '2 GB'; // Max number of queued jobs a user can have at a given time module.exports.batch_max_queued_jobs = 64;