cartodb/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Changes in time dimensions API

Browse Source 
Use single `starting` epoch instead of various offsets.
Add ISO text representation.
Adopt ISO conventions for day of week and week of year.
Rename internal parameters for consistency with external API.
This commit is contained in:
Javier Goizueta 2018-10-03 17:05:58 +02:00
parent fbf3fd9d8c
commit dede22c915
2 changed files with 108 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
lib/cartodb/models/aggregation/aggregation-query.js
View File

@ -41,13 +41,15 @@ const templateForOptions = (options) => {
* When placement, columns or dimensions are specified, columns are aggregated as requested
* (by default only _cdb_feature_count) and with the_geom_webmercator as defined by placement.
*/
const queryForOptions = (options) => templateForOptions(options)({
const queryForOptions = (options) => {
return templateForOptions(options)({
sourceQuery: options.query,
res: 256/options.resolution,
columns: options.columns,
dimensions: options.dimensions,
filters: options.filters
});
});
};
module.exports = queryForOptions;
@ -190,17 +192,17 @@ const timeDimensionParameters = definition => {
return {
time: `to_timestamp("${definition.column}")`,
timezone: definition.timezone || 'utc',
granularity: group_by_units,
multiplicity: group_by_count || 1,
cycle: group_by_cycle,
offset: definition.offset || 0
group_by_units,
group_by_count,
group_by_cycle,
epoch: definition.starting
};
};
// Adapt old-style dimension definitions for backwards compatibility
const adaptDimensionDefinition = definition => {
if (typeof(definition) === 'string') {
return { column: definition }
return { column: definition };
}
return definition;
};

155
lib/cartodb/models/aggregation/time-dimension.js
View File

@ -1,4 +1,4 @@
const MONTH_SECONDS = 365.2425 / 12 * 24 * 3600 // PG intervals use 30 * 24 * 3600
const MONTH_SECONDS = 365.2425 / 12 * 24 * 3600; // PG intervals use 30 * 24 * 3600
const YEAR_SECONDS = 12 * MONTH_SECONDS;
// time unit durations
@ -19,50 +19,54 @@ const usecs = {
millennium: 1000 * YEAR_SECONDS
};
serialParts = {
const YEARSPAN = "(date_part('year', $t)-date_part('year', $epoch))";
// Note that SECONDSPAN is not a UTC epoch, but an epoch in the specified TZ,
// so we can use it to compute any multiple of seconds with it without using date_part or date_trunc
const SECONDSPAN = "(date_part('epoch', $t) - date_part('epoch', $epoch))";
const serialParts = {
second: {
sql: `FLOOR(date_part('epoch', $t))`,
sql: `FLOOR(${SECONDSPAN})`,
zeroBased: true
},
minute: {
sql: `date_part('epoch', date_trunc('day', $t))/60`,
sql: `FLOOR(${SECONDSPAN}/60)`,
zeroBased: true
},
hour: {
sql: `date_part('epoch', date_trunc('day', $t))/(60*60)`,
sql: `FLOOR(${SECONDSPAN}/3600)`,
zeroBased: true
},
day: {
sql: `date_part('epoch', date_trunc('day', $t))/(24*60*60) + 1`,
sql: `1 + FLOOR(${SECONDSPAN}/86400)`,
zeroBased: false
},
week: {
sql: `date_part('epoch', date_trunc('week', $t))/(7*24*60*60) + 1`,
zeroBaseed: false
sql: `1 + FLOOR(${SECONDSPAN}/(7*86400))`,
zeroBased: false
},
month: {
sql: `date_part('month', $t) + 12*(date_part('year', $t)-date_part('year', to_timestamp(0.0)))`,
sql: `1 + date_part('month', $t) - date_part('month', $epoch) + 12*${YEARSPAN}`,
zeroBased: false
},
quarter: {
sql: `date_part('quarter', $t) + 4*(date_part('year', $t)-date_part('year', to_timestamp(0.0)))`,
sql: `1 + date_part('quarter', $t) - date_part('quarter', $epoch) + 4*${YEARSPAN}`,
zeroBased: false
},
year: {
sql: `date_part('year', $t)-date_part('year', to_timestamp(0.0))`,
// TODO: isn't more meaningful to ignore the epoch here and return date_part('year', $t)
sql: `1 + ${YEARSPAN}`,
zeroBased: false
}
};
function serialSqlExpr(t, tz, u, m = 1, u_offset = 0, m_offset = 0) {
[u, m, u_offset] = serialNormalize(u, m, u_offset);
function serialSqlExpr(t, tz, u, m = 1, starting = undefined) {
[u, m] = serialNormalize(u, m);
let { sql, zeroBased } = serialParts[u];
const column = timeExpression(t, tz);
const serial = sql.replace(/\$t/g, column);
const epoch = epochExpression(starting);
const serial = sql.replace(/\$t/g, column).replace(/\$epoch/g, epoch);
let expr = serial;
if (u_offset !== 0) {
expr = `expr - ${u_offset}`;
}
if (m !== 1) {
if (zeroBased) {
expr = `FLOOR((${expr})/(${m}::double precision))::int`;
@ -70,37 +74,54 @@ function serialSqlExpr(t, tz, u, m = 1, u_offset = 0, m_offset = 0) {
expr = `CEIL((${expr})/(${m}::double precision))::int`;
}
} else {
expr = `ROUND(${expr})::int`;
}
if (m_offset !== 0) {
expr = `(${expr} - 1)`;
expr = `(${expr})::int`;
}
return expr;
}
function serialNormalize(u, m, u_offset) {
const isoParts = {
second: `to_char($t, 'YYYY-MM-DD"T"HH:MI:SS')`,
minute: `to_char($t, 'YYYY-MM-DD"T"HH:MI')`,
hour: `to_char($t, 'YYYY-MM-DD"T"HH')`,
day: `to_char($t, 'YYYY-MM-DD')`,
month: `to_char($t, 'YYYY-MM')`,
year: `to_char($t, 'YYYY')`,
week: `to_char($t, 'IYYY-"W"IW')`,
quarter: `to_char($t, 'YYYY-"Q"Q')`,
semester: `to_char($t, 'YYYY"S"') || to_char(CEIL(date_part('month', $t)/6), '9')`,
trimester: `to_char($t, 'YYYY"t"') || to_char(CEIL(date_part('month', $t)/4), '9')`,
decade: `to_char(date_part('decade', $t), '"D"999')`,
century: `to_char($t, '"C"CC')`,
millennium: `to_char(date_part('millenium', $t), '"M"999')`
};
function isoSqlExpr(t, tz, u, m = 1) {
const column = timeExpression(t, tz);
if (m > 1) {
// TODO: it would be sensible to return the ISO of the firt unit in the period
throw new Error('Multiple time units not supported for ISO format');
}
return isoParts[u].replace(/\$t/g, column);
}
function serialNormalize(u, m) {
if (u === 'semester') {
u = 'month';
m *= 6;
u_offset *= 6;
} else if (u === 'trimester') {
u = 'month';
m *= 4;
u_offset *= 4;
} else if (u === 'decade') {
u = 'year';
m *= 10;
u_offset *= 10
} else if (u === 'century') {
u = 'year';
m *= 100;
u_offset *= 100
} else if (u === 'millenium') {
u = 'year';
m *= 1000;
u_offset *= 1000
}
return [u, m, u_offset];
return [u, m];
}
function cyclicNormalize(u, m, c, c_offset) {
@ -128,7 +149,7 @@ function timezone(tz) {
if (isFinite(tz)) {
return `INTERVAL '${tz} seconds'`;
}
return `'${tz}'`
return `'${tz}'`;
}
// We assume t is a TIMESTAMP WITH TIME ZONE.
@ -141,35 +162,48 @@ function timezone(tz) {
// (converted) column.
function timeExpression(t, tz) {
if (tz !== undefined) {
return `timezone(${timezone(tz)}, ${t})`
return `timezone(${timezone(tz)}, ${t})`;
}
return t;
}
// Epoch should be an ISO timestamp literal without time zone
// (it is interpreted as in the defined timzezone for the input time)
// It can be partial, e.g. 'YYYY', 'YYYY-MM', 'YYYY-MM-DDTHH', etc.
// Defaults are applied: YYYY=0001, MM=01, DD=01, HH=00, MM=00, S=00
// It returns a timestamp without time zone
function epochExpression(epoch) {
const format = /^(\d\d\d\d)(?:\-?(\d\d)(?:\-?(\d\d)(?:[T\s]?(\d\d)(?:(\d\d)(?:\:(\d\d))?)?)?)?)?$/;
const match = epoch.match(format) || [];
const year = match[1] || '0001';
const month = match[2] || '01';
const day = match[3] || '01';
const hour = match[4] || '00';
const minute = match[5] || '00';
const second = match[6] || '00';
epoch = `${year}-${month}-${day}T${hour}:${minute}:${second}`;
return `TIMESTAMP '${epoch}'`;
}
function cyclicSqlExpr(t, tz, u, c, c_offset = 0, m = 1) {
[u, m, c, c_offset] = cyclicNormalize(u, m, c, c_offset);
const comb = `${u}/${c}`;
const column = timeExpression(t, tz);
let expr;
// TODO: drop offset support
if (m === 1) {
switch (comb) {
case 'day/week':
// result: 0-6
// c_offset = 0 => 0 = sunday; 1 => 0 = monday...
// let expr = `EXTRACT(DOW FROM ${column})`;
expr = `date_part('dow', ${column})`;
if (c_offset !== 0) {
expr = `(${expr} - ${c_offset}) % 7`;
if (c_offset === 0) {
// 1 = monday; 7 = sunday;
return `date_part('isodow', ${column})`;
}
return expr;
// iso dow monday=1, no offset:
// `EXTRACT(ISODOW FROM ${column})`
// iso dow 1-6, offset 0 => 1 = mondayº
// expr = `date_part('dow, ${column})`;
// c_offset += 1;
// expr = `(${expr} - ${c_offset}) % 7 + 1`;
// iso dow 1-6, offset 0 => 1 = monday
expr = `date_part('dow, ${column})`;
c_offset += 1;
return `(${expr} - ${c_offset}) % 7 + 1`;
case 'day/month':
// result: 1-31
@ -222,10 +256,10 @@ function cyclicSqlExpr(t, tz, u, c, c_offset = 0, m = 1) {
return expr;
case 'week/year':
// result 1-52
// result 1-53
expr = `date_part('week', ${column})`;
if (c_offset !== 0) {
expr = `((${expr} - ${c_offset} - 1) % 52) + 1`;
expr = `((${expr} - ${c_offset} - 1) % 53) + 1`;
}
return expr;
@ -238,6 +272,7 @@ function cyclicSqlExpr(t, tz, u, c, c_offset = 0, m = 1) {
return expr;
}
}
// TODO: remove generic expression, reaching here should error
return genericCyclicSqlExpr(t, u, c, c_offset, m);
}
@ -248,29 +283,37 @@ function genericCyclicSqlExpr(t, tz, u, c, c_offset = 0, m = 1) {
return `((FLOOR(date_part('epoch', ${column})/(${usec*m}))*(${usec*m})+${c_offset}) % ${csec})/${usec*m}`;
}
function validateParameters(params) {
function validateParameters(_params) {
return true;
}
function classificationSql(params) {
validateParameters(params);
if (params.cycle) {
if (params.group_by_cycle) {
// TODO: validate group_by_count === 1, No epoch
return cyclicSqlExpr(
params.time,
params.timezone || 'utc',
params.granularity,
params.cycle,
params.offset || 0,
params.multiplicity || 1
params.group_by_units,
params.group_by_cycle,
0,
params.group_by_count || 1
);
} else if (params.format === 'iso') {
// TODO: validate group_by_count === 1, No epoch
return isoSqlExpr(
params.time,
params.timezone || 'utc',
params.group_by_units,
params.group_by_count || 1
);
} else {
return serialSqlExpr(
params.time,
params.timezone || 'utc',
params.granularity,
params.multiplicity || 1,
params.offset || 0,
0
params.group_by_units,
params.group_by_count || 1,
params.epoch
);
}