diff --git a/lib/cartodb/models/aggregation/aggregation-query.js b/lib/cartodb/models/aggregation/aggregation-query.js index 7cdbe265..3e6ba873 100644 --- a/lib/cartodb/models/aggregation/aggregation-query.js +++ b/lib/cartodb/models/aggregation/aggregation-query.js @@ -41,13 +41,15 @@ const templateForOptions = (options) => { * When placement, columns or dimensions are specified, columns are aggregated as requested * (by default only _cdb_feature_count) and with the_geom_webmercator as defined by placement. */ -const queryForOptions = (options) => templateForOptions(options)({ +const queryForOptions = (options) => { + return templateForOptions(options)({ sourceQuery: options.query, res: 256/options.resolution, columns: options.columns, dimensions: options.dimensions, filters: options.filters -}); + }); +}; module.exports = queryForOptions; @@ -190,17 +192,17 @@ const timeDimensionParameters = definition => { return { time: `to_timestamp("${definition.column}")`, timezone: definition.timezone || 'utc', - granularity: group_by_units, - multiplicity: group_by_count || 1, - cycle: group_by_cycle, - offset: definition.offset || 0 + group_by_units, + group_by_count, + group_by_cycle, + epoch: definition.starting }; }; // Adapt old-style dimension definitions for backwards compatibility const adaptDimensionDefinition = definition => { if (typeof(definition) === 'string') { - return { column: definition } + return { column: definition }; } return definition; }; diff --git a/lib/cartodb/models/aggregation/time-dimension.js b/lib/cartodb/models/aggregation/time-dimension.js index 16eb468c..476fc4ca 100644 --- a/lib/cartodb/models/aggregation/time-dimension.js +++ b/lib/cartodb/models/aggregation/time-dimension.js @@ -1,4 +1,4 @@ -const MONTH_SECONDS = 365.2425 / 12 * 24 * 3600 // PG intervals use 30 * 24 * 3600 +const MONTH_SECONDS = 365.2425 / 12 * 24 * 3600; // PG intervals use 30 * 24 * 3600 const YEAR_SECONDS = 12 * MONTH_SECONDS; // time unit durations @@ -19,50 +19,54 @@ const usecs = { millennium: 1000 * YEAR_SECONDS }; -serialParts = { +const YEARSPAN = "(date_part('year', $t)-date_part('year', $epoch))"; +// Note that SECONDSPAN is not a UTC epoch, but an epoch in the specified TZ, +// so we can use it to compute any multiple of seconds with it without using date_part or date_trunc +const SECONDSPAN = "(date_part('epoch', $t) - date_part('epoch', $epoch))"; + +const serialParts = { second: { - sql: `FLOOR(date_part('epoch', $t))`, + sql: `FLOOR(${SECONDSPAN})`, zeroBased: true }, minute: { - sql: `date_part('epoch', date_trunc('day', $t))/60`, + sql: `FLOOR(${SECONDSPAN}/60)`, zeroBased: true }, hour: { - sql: `date_part('epoch', date_trunc('day', $t))/(60*60)`, + sql: `FLOOR(${SECONDSPAN}/3600)`, zeroBased: true }, day: { - sql: `date_part('epoch', date_trunc('day', $t))/(24*60*60) + 1`, + sql: `1 + FLOOR(${SECONDSPAN}/86400)`, zeroBased: false }, week: { - sql: `date_part('epoch', date_trunc('week', $t))/(7*24*60*60) + 1`, - zeroBaseed: false + sql: `1 + FLOOR(${SECONDSPAN}/(7*86400))`, + zeroBased: false }, month: { - sql: `date_part('month', $t) + 12*(date_part('year', $t)-date_part('year', to_timestamp(0.0)))`, + sql: `1 + date_part('month', $t) - date_part('month', $epoch) + 12*${YEARSPAN}`, zeroBased: false }, quarter: { - sql: `date_part('quarter', $t) + 4*(date_part('year', $t)-date_part('year', to_timestamp(0.0)))`, + sql: `1 + date_part('quarter', $t) - date_part('quarter', $epoch) + 4*${YEARSPAN}`, zeroBased: false }, year: { - sql: `date_part('year', $t)-date_part('year', to_timestamp(0.0))`, + // TODO: isn't more meaningful to ignore the epoch here and return date_part('year', $t) + sql: `1 + ${YEARSPAN}`, zeroBased: false } }; -function serialSqlExpr(t, tz, u, m = 1, u_offset = 0, m_offset = 0) { - [u, m, u_offset] = serialNormalize(u, m, u_offset); +function serialSqlExpr(t, tz, u, m = 1, starting = undefined) { + [u, m] = serialNormalize(u, m); let { sql, zeroBased } = serialParts[u]; const column = timeExpression(t, tz); - const serial = sql.replace(/\$t/g, column); + const epoch = epochExpression(starting); + const serial = sql.replace(/\$t/g, column).replace(/\$epoch/g, epoch); let expr = serial; - if (u_offset !== 0) { - expr = `expr - ${u_offset}`; - } if (m !== 1) { if (zeroBased) { expr = `FLOOR((${expr})/(${m}::double precision))::int`; @@ -70,37 +74,54 @@ function serialSqlExpr(t, tz, u, m = 1, u_offset = 0, m_offset = 0) { expr = `CEIL((${expr})/(${m}::double precision))::int`; } } else { - expr = `ROUND(${expr})::int`; - } - if (m_offset !== 0) { - expr = `(${expr} - 1)`; + expr = `(${expr})::int`; } return expr; } -function serialNormalize(u, m, u_offset) { +const isoParts = { + second: `to_char($t, 'YYYY-MM-DD"T"HH:MI:SS')`, + minute: `to_char($t, 'YYYY-MM-DD"T"HH:MI')`, + hour: `to_char($t, 'YYYY-MM-DD"T"HH')`, + day: `to_char($t, 'YYYY-MM-DD')`, + month: `to_char($t, 'YYYY-MM')`, + year: `to_char($t, 'YYYY')`, + week: `to_char($t, 'IYYY-"W"IW')`, + quarter: `to_char($t, 'YYYY-"Q"Q')`, + semester: `to_char($t, 'YYYY"S"') || to_char(CEIL(date_part('month', $t)/6), '9')`, + trimester: `to_char($t, 'YYYY"t"') || to_char(CEIL(date_part('month', $t)/4), '9')`, + decade: `to_char(date_part('decade', $t), '"D"999')`, + century: `to_char($t, '"C"CC')`, + millennium: `to_char(date_part('millenium', $t), '"M"999')` +}; + +function isoSqlExpr(t, tz, u, m = 1) { + const column = timeExpression(t, tz); + if (m > 1) { + // TODO: it would be sensible to return the ISO of the firt unit in the period + throw new Error('Multiple time units not supported for ISO format'); + } + return isoParts[u].replace(/\$t/g, column); +} + +function serialNormalize(u, m) { if (u === 'semester') { u = 'month'; m *= 6; - u_offset *= 6; } else if (u === 'trimester') { u = 'month'; m *= 4; - u_offset *= 4; } else if (u === 'decade') { u = 'year'; m *= 10; - u_offset *= 10 } else if (u === 'century') { u = 'year'; m *= 100; - u_offset *= 100 } else if (u === 'millenium') { u = 'year'; m *= 1000; - u_offset *= 1000 } - return [u, m, u_offset]; + return [u, m]; } function cyclicNormalize(u, m, c, c_offset) { @@ -128,7 +149,7 @@ function timezone(tz) { if (isFinite(tz)) { return `INTERVAL '${tz} seconds'`; } - return `'${tz}'` + return `'${tz}'`; } // We assume t is a TIMESTAMP WITH TIME ZONE. @@ -141,35 +162,48 @@ function timezone(tz) { // (converted) column. function timeExpression(t, tz) { if (tz !== undefined) { - return `timezone(${timezone(tz)}, ${t})` + return `timezone(${timezone(tz)}, ${t})`; } return t; } +// Epoch should be an ISO timestamp literal without time zone +// (it is interpreted as in the defined timzezone for the input time) +// It can be partial, e.g. 'YYYY', 'YYYY-MM', 'YYYY-MM-DDTHH', etc. +// Defaults are applied: YYYY=0001, MM=01, DD=01, HH=00, MM=00, S=00 +// It returns a timestamp without time zone +function epochExpression(epoch) { + const format = /^(\d\d\d\d)(?:\-?(\d\d)(?:\-?(\d\d)(?:[T\s]?(\d\d)(?:(\d\d)(?:\:(\d\d))?)?)?)?)?$/; + const match = epoch.match(format) || []; + const year = match[1] || '0001'; + const month = match[2] || '01'; + const day = match[3] || '01'; + const hour = match[4] || '00'; + const minute = match[5] || '00'; + const second = match[6] || '00'; + epoch = `${year}-${month}-${day}T${hour}:${minute}:${second}`; + return `TIMESTAMP '${epoch}'`; + } + function cyclicSqlExpr(t, tz, u, c, c_offset = 0, m = 1) { [u, m, c, c_offset] = cyclicNormalize(u, m, c, c_offset); const comb = `${u}/${c}`; const column = timeExpression(t, tz); let expr; + // TODO: drop offset support + if (m === 1) { switch (comb) { case 'day/week': - // result: 0-6 - // c_offset = 0 => 0 = sunday; 1 => 0 = monday... - // let expr = `EXTRACT(DOW FROM ${column})`; - expr = `date_part('dow', ${column})`; - if (c_offset !== 0) { - expr = `(${expr} - ${c_offset}) % 7`; + if (c_offset === 0) { + // 1 = monday; 7 = sunday; + return `date_part('isodow', ${column})`; } - return expr; - - // iso dow monday=1, no offset: - // `EXTRACT(ISODOW FROM ${column})` - // iso dow 1-6, offset 0 => 1 = mondayº - // expr = `date_part('dow, ${column})`; - // c_offset += 1; - // expr = `(${expr} - ${c_offset}) % 7 + 1`; + // iso dow 1-6, offset 0 => 1 = monday + expr = `date_part('dow, ${column})`; + c_offset += 1; + return `(${expr} - ${c_offset}) % 7 + 1`; case 'day/month': // result: 1-31 @@ -222,10 +256,10 @@ function cyclicSqlExpr(t, tz, u, c, c_offset = 0, m = 1) { return expr; case 'week/year': - // result 1-52 + // result 1-53 expr = `date_part('week', ${column})`; if (c_offset !== 0) { - expr = `((${expr} - ${c_offset} - 1) % 52) + 1`; + expr = `((${expr} - ${c_offset} - 1) % 53) + 1`; } return expr; @@ -238,6 +272,7 @@ function cyclicSqlExpr(t, tz, u, c, c_offset = 0, m = 1) { return expr; } } + // TODO: remove generic expression, reaching here should error return genericCyclicSqlExpr(t, u, c, c_offset, m); } @@ -248,29 +283,37 @@ function genericCyclicSqlExpr(t, tz, u, c, c_offset = 0, m = 1) { return `((FLOOR(date_part('epoch', ${column})/(${usec*m}))*(${usec*m})+${c_offset}) % ${csec})/${usec*m}`; } -function validateParameters(params) { +function validateParameters(_params) { return true; } function classificationSql(params) { validateParameters(params); - if (params.cycle) { + if (params.group_by_cycle) { + // TODO: validate group_by_count === 1, No epoch return cyclicSqlExpr( params.time, params.timezone || 'utc', - params.granularity, - params.cycle, - params.offset || 0, - params.multiplicity || 1 + params.group_by_units, + params.group_by_cycle, + 0, + params.group_by_count || 1 + ); + } else if (params.format === 'iso') { + // TODO: validate group_by_count === 1, No epoch + return isoSqlExpr( + params.time, + params.timezone || 'utc', + params.group_by_units, + params.group_by_count || 1 ); } else { return serialSqlExpr( params.time, params.timezone || 'utc', - params.granularity, - params.multiplicity || 1, - params.offset || 0, - 0 + params.group_by_units, + params.group_by_count || 1, + params.epoch ); }