Skip to content

Commit

Permalink
Handle "ambiguous durations" (apache#5785)
Browse files Browse the repository at this point in the history
* WIP

* Update interface

* Fix truncate

* Improve unit tests

* Improve code

* Use moment.js to parse ISO durations

* Fix typo and React props

(cherry picked from commit f740974)
(cherry picked from commit 25ac26d)
  • Loading branch information
betodealmeida committed Oct 30, 2018
1 parent 7bb1777 commit c568cfa
Show file tree
Hide file tree
Showing 7 changed files with 1,855 additions and 178 deletions.
1 change: 0 additions & 1 deletion superset/assets/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,6 @@
"mousetrap": "^1.6.1",
"mustache": "^2.2.1",
"nvd3": "1.8.6",
"parse-iso-duration": "^1.0.0",
"prop-types": "^15.6.0",
"re-resizable": "^4.3.1",
"react": "^15.6.2",
Expand Down
100 changes: 63 additions & 37 deletions superset/assets/spec/javascripts/modules/time_spec.js
Original file line number Diff line number Diff line change
@@ -1,6 +1,32 @@
import { it, describe } from 'mocha';
import { expect } from 'chai';
import { getPlaySliderParams } from '../../../src/modules/time';

import moment from 'moment';
import { getPlaySliderParams, truncate } from '../../../src/modules/time';

describe('truncate', () => {
it('truncates timestamps', () => {
const timestamp = moment('2018-03-03T03:03:03.333');
const isoDurations = [
// basic units
[moment.duration('PT1S'), moment('2018-03-03T03:03:03')],
[moment.duration('PT1M'), moment('2018-03-03T03:03:00')],
[moment.duration('PT1H'), moment('2018-03-03T03:00:00')],
[moment.duration('P1D'), moment('2018-03-03T00:00:00')],
[moment.duration('P1M'), moment('2018-03-01T00:00:00')],
[moment.duration('P1Y'), moment('2018-01-01T00:00:00')],

// durations that are multiples
[moment.duration('PT2H'), moment('2018-03-03T02:00:00')],
[moment.duration('P2D'), moment('2018-03-03T00:00:00')],
];
let result;
isoDurations.forEach(([step, expected]) => {
result = truncate(timestamp, step);
expect(result.format()).to.equal(expected.format());
});
});
});

describe('getPlaySliderParams', () => {
it('is a function', () => {
Expand All @@ -9,49 +35,49 @@ describe('getPlaySliderParams', () => {

it('handles durations', () => {
const timestamps = [
new Date('2018-01-01'),
new Date('2018-01-02'),
new Date('2018-01-03'),
new Date('2018-01-04'),
new Date('2018-01-05'),
new Date('2018-01-06'),
new Date('2018-01-07'),
new Date('2018-01-08'),
new Date('2018-01-09'),
new Date('2018-01-10'),
].map(d => d.getTime());
const { start, end, step, values, disabled } = getPlaySliderParams(timestamps, 'P2D');
expect(new Date(start)).to.eql(new Date('2018-01-01'));
expect(new Date(end)).to.eql(new Date('2018-01-11'));
expect(step).to.equal(2 * 24 * 60 * 60 * 1000);
expect(values.map(v => new Date(v))).to.eql([
new Date('2018-01-01'),
new Date('2018-01-03'),
moment('2018-01-01T00:00:00'),
moment('2018-01-02T00:00:00'),
moment('2018-01-03T00:00:00'),
moment('2018-01-04T00:00:00'),
moment('2018-01-05T00:00:00'),
moment('2018-01-06T00:00:00'),
moment('2018-01-07T00:00:00'),
moment('2018-01-08T00:00:00'),
moment('2018-01-09T00:00:00'),
moment('2018-01-10T00:00:00'),
].map(d => parseInt(d.format('x'), 10));
const { start, end, getStep, values, disabled } = getPlaySliderParams(timestamps, 'P2D');
expect(moment(start).format()).to.equal(moment('2018-01-01T00:00:00').format());
expect(moment(end).format()).to.equal(moment('2018-01-11T00:00:00').format());
expect(getStep(start)).to.equal(2 * 24 * 60 * 60 * 1000);
expect(values.map(v => moment(v).format())).to.eql([
moment('2018-01-01T00:00:00').format(),
moment('2018-01-03T00:00:00').format(),
]);
expect(disabled).to.equal(false);
});

it('handles intervals', () => {
const timestamps = [
new Date('2018-01-01'),
new Date('2018-01-02'),
new Date('2018-01-03'),
new Date('2018-01-04'),
new Date('2018-01-05'),
new Date('2018-01-06'),
new Date('2018-01-07'),
new Date('2018-01-08'),
new Date('2018-01-09'),
new Date('2018-01-10'),
].map(d => d.getTime());
moment('2018-01-01T00:00:00'),
moment('2018-01-02T00:00:00'),
moment('2018-01-03T00:00:00'),
moment('2018-01-04T00:00:00'),
moment('2018-01-05T00:00:00'),
moment('2018-01-06T00:00:00'),
moment('2018-01-07T00:00:00'),
moment('2018-01-08T00:00:00'),
moment('2018-01-09T00:00:00'),
moment('2018-01-10T00:00:00'),
].map(d => parseInt(d.format('x'), 10));
// 1970-01-03 was a Saturday
const { start, end, step, values, disabled } = getPlaySliderParams(timestamps, 'P1W/1970-01-03T00:00:00Z');
expect(new Date(start)).to.eql(new Date('2017-12-30')); // Saturday
expect(new Date(end)).to.eql(new Date('2018-01-13')); // Saturday
expect(step).to.equal(7 * 24 * 60 * 60 * 1000);
expect(values.map(v => new Date(v))).to.eql([
new Date('2017-12-30'),
new Date('2018-01-06'),
const { start, end, getStep, values, disabled } = getPlaySliderParams(timestamps, 'P1W/1970-01-03T00:00:00Z');
expect(moment(start).format()).to.equal(moment('2017-12-30T00:00:00Z').format()); // Saturday
expect(moment(end).format()).to.equal(moment('2018-01-13T00:00:00Z').format()); // Saturday
expect(getStep(start)).to.equal(7 * 24 * 60 * 60 * 1000);
expect(values.map(v => moment(v).format())).to.eql([
moment('2017-12-30T00:00:00Z').format(),
moment('2018-01-06T00:00:00Z').format(),
]);
expect(disabled).to.equal(false);
});
Expand Down
104 changes: 86 additions & 18 deletions superset/assets/src/modules/time.js
Original file line number Diff line number Diff line change
@@ -1,38 +1,106 @@
import parseIsoDuration from 'parse-iso-duration';
import moment from 'moment';


// array with the minimum values of each part of a timestamp -- note that
// months are zero-indexed in Javascript
const truncatePartTo = [
1, // year
0, // month
1, // day
0, // hour
0, // minute
0, // second
0, // millisecond
];


export function truncate(timestamp, step) {
/*
* Truncate timestamp down to duration resolution.
*/
const lowerBound = moment(timestamp).subtract(step);
const explodedTimestamp = timestamp.toArray();
const explodedLowerBound = lowerBound.toArray();

const firstDiffIndex = explodedTimestamp
.map((part, i) => (explodedLowerBound[i] !== part))
.indexOf(true);
const dateParts = explodedTimestamp.map((part, i) => {
if (i === firstDiffIndex) {
// truncate down to closest `truncatePartTo[i] + n * step`
const difference = part - explodedLowerBound[i];
return part - ((part - truncatePartTo[i]) % difference);
} else if (i < firstDiffIndex || firstDiffIndex === -1) {
return part;
}
return truncatePartTo[i];
});

return moment(dateParts);
}

function getStepSeconds(step, start) {
/* Return number of seconds in a step.
*
* The step might be ambigous, eg, "1 month" has a variable number of
* seconds, which is why we need to know the start time.
*/
const startMillliseconds = parseInt(moment(start).format('x'), 10);
const endMilliseconds = parseInt(moment(start).add(step).format('x'), 10);
return endMilliseconds - startMillliseconds;
}

export const getPlaySliderParams = function (timestamps, timeGrain) {
let start = Math.min(...timestamps);
let end = Math.max(...timestamps);
const minTimestamp = moment(Math.min(...timestamps));
const maxTimestamp = moment(Math.max(...timestamps));
let step;
let reference;

if (timeGrain.indexOf('/') > 0) {
if (timeGrain.indexOf('/') !== -1) {
// Here, time grain is a time interval instead of a simple duration, either
// `reference/duration` or `duration/reference`. We need to parse the
// duration and make sure that start and end are in the right places. For
// example, if `reference` is a Saturday and `duration` is 1 week (P1W)
// then both start and end should be Saturdays.
const parts = timeGrain.split('/', 2);
let reference;
if (parts[0].endsWith('Z')) { // ISO string
reference = new Date(parts[0]).getTime();
step = parseIsoDuration(parts[1]);
reference = moment(parts[0]);
step = moment.duration(parts[1]);
} else {
reference = new Date(parts[1]).getTime();
step = parseIsoDuration(parts[0]);
reference = moment(parts[1]);
step = moment.duration(parts[0]);
}
start = reference + step * Math.floor((start - reference) / step);
end = reference + step * (Math.floor((end - reference) / step) + 1);
} else {
// lock start and end to the closest steps
step = parseIsoDuration(timeGrain);
start -= start % step;
end += step - end % step;
step = moment.duration(timeGrain);
reference = truncate(minTimestamp, step);
}

const values = timeGrain != null ? [start, start + step] : [start, end];
// find the largest `reference + n * step` smaller than the minimum timestamp
const start = moment(reference);
while (start < minTimestamp) {
start.add(step);
}
while (start > minTimestamp) {
start.subtract(step);
}

// find the smallest `reference + n * step` larger than the maximum timestamp
const end = moment(reference);
while (end > maxTimestamp) {
end.subtract(step);
}
while (end < maxTimestamp) {
end.add(step);
}

const values = timeGrain != null ? [start, moment(start).add(step)] : [start, end];
const disabled = timestamps.every(timestamp => timestamp === null);

return { start, end, step, values, disabled };
return {
start: parseInt(start.format('x'), 10),
end: parseInt(end.format('x'), 10),
getStep: getStepSeconds.bind(this, step),
values: values.map(v => parseInt(v.format('x'), 10)),
disabled,
};
};

Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ const propTypes = {
getLayers: PropTypes.func.isRequired,
start: PropTypes.number.isRequired,
end: PropTypes.number.isRequired,
step: PropTypes.number.isRequired,
getStep: PropTypes.func,
values: PropTypes.array.isRequired,
aggregation: PropTypes.bool,
disabled: PropTypes.bool,
Expand All @@ -27,7 +27,7 @@ const defaultProps = {
export default class AnimatableDeckGLContainer extends React.Component {
constructor(props) {
super(props);
const { getLayers, start, end, step, values, disabled, viewport, ...other } = props;
const { getLayers, start, end, getStep, values, disabled, viewport, ...other } = props;
this.state = { values, viewport };
this.other = other;
this.onChange = this.onChange.bind(this);
Expand All @@ -39,11 +39,11 @@ export default class AnimatableDeckGLContainer extends React.Component {
this.setState({
values: Array.isArray(newValues)
? newValues
: [newValues, newValues + this.props.step],
: [newValues, this.props.getStep(newValues)],
});
}
render() {
const { start, end, step, disabled, aggregation, children, getLayers } = this.props;
const { start, end, getStep, disabled, aggregation, children, getLayers } = this.props;
const { values, viewport } = this.state;
const layers = getLayers(values);
return (
Expand All @@ -58,7 +58,7 @@ export default class AnimatableDeckGLContainer extends React.Component {
<PlaySlider
start={start}
end={end}
step={step}
step={getStep(start)}
values={values}
range={!aggregation}
onChange={this.onChange}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,10 +51,10 @@ export default class CategoricalDeckGLContainer extends React.PureComponent {

const timeGrain = fd.time_grain_sqla || fd.granularity || 'PT1M';
const timestamps = nextProps.payload.data.features.map(f => f.__timestamp);
const { start, end, step, values, disabled } = getPlaySliderParams(timestamps, timeGrain);
const { start, end, getStep, values, disabled } = getPlaySliderParams(timestamps, timeGrain);
const categories = getCategories(fd, nextProps.payload.data.features);

return { start, end, step, values, disabled, categories, viewport: nextProps.viewport };
return { start, end, getStep, values, disabled, categories, viewport: nextProps.viewport };
}
constructor(props) {
super(props);
Expand Down Expand Up @@ -141,7 +141,7 @@ export default class CategoricalDeckGLContainer extends React.PureComponent {
getLayers={this.getLayers}
start={this.state.start}
end={this.state.end}
step={this.state.step}
getStep={this.state.getStep}
values={this.state.values}
disabled={this.state.disabled}
viewport={this.state.viewport}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -65,9 +65,9 @@ class DeckGLScreenGrid extends React.PureComponent {

const timeGrain = fd.time_grain_sqla || fd.granularity || 'PT1M';
const timestamps = nextProps.payload.data.features.map(f => f.__timestamp);
const { start, end, step, values, disabled } = getPlaySliderParams(timestamps, timeGrain);
const { start, end, getStep, values, disabled } = getPlaySliderParams(timestamps, timeGrain);

return { start, end, step, values, disabled, viewport: nextProps.viewport };
return { start, end, getStep, values, disabled, viewport: nextProps.viewport };
}
constructor(props) {
super(props);
Expand Down Expand Up @@ -107,7 +107,7 @@ class DeckGLScreenGrid extends React.PureComponent {
getLayers={this.getLayers}
start={this.state.start}
end={this.state.end}
step={this.state.step}
getStep={this.state.getStep}
values={this.state.values}
disabled={this.state.disabled}
viewport={this.state.viewport}
Expand Down
Loading

0 comments on commit c568cfa

Please sign in to comment.