Skip to content

Commit

Permalink
Add Time Machine to Charting
Browse files Browse the repository at this point in the history
  • Loading branch information
@JafarMirzaie
JafarMirzaie committed Aug 19, 2024
1 parent 18bdfda commit 4a7cc10
Show file tree
Hide file tree
Showing 32 changed files with 301 additions and 115 deletions.
5 changes: 3 additions & 2 deletions Extensions/Signum.Chart/ChartButton.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import { FilterOptionParsed, FilterGroupOptionParsed, isFilterGroup } from '@fra
import * as AppContext from '@framework/AppContext'
import { Navigator } from '@framework/Navigator'
import { default as SearchControlLoaded } from '@framework/SearchControl/SearchControlLoaded'
import { ChartMessage, ChartRequestModel } from './Signum.Chart'
import { ChartMessage, ChartRequestModel, ChartTimeSeriesEmbedded } from './Signum.Chart'
import { ChartClient } from './ChartClient'
import { Button } from 'react-bootstrap'
import { Finder } from '@framework/Finder';
Expand All @@ -31,7 +31,8 @@ export default class ChartButton extends React.Component<ChartButtonProps> {
const path = ChartClient.Encoder.chartPath({
queryName: fo.queryName,
orderOptions: [],
filterOptions: fo.filterOptions
filterOptions: fo.filterOptions,
timeSeries: ChartClient.cloneChartTimeSeries(fo.systemTime as any),
})

if (sc.props.avoidChangeUrl)
Expand Down
62 changes: 57 additions & 5 deletions Extensions/Signum.Chart/ChartClient.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import { Constructor } from '@framework/Constructor'
import { Entity, getToString, is, Lite, liteKey, MList, SelectorMessage, toLite, translated } from '@framework/Signum.Entities'
import { getQueryKey, getEnumInfo, QueryTokenString, tryGetTypeInfos, timeToString, toFormatWithFixes } from '@framework/Reflection'
import {
FilterOption, OrderOption, QueryRequest, QueryToken, SubTokensOptions, ResultTable, OrderRequest, OrderType, FilterOptionParsed, hasAggregate, ColumnOption, withoutAggregate, FilterConditionOption, QueryDescription, FindOptions, withoutPinned
FilterOption, OrderOption, QueryRequest, QueryToken, SubTokensOptions, ResultTable, OrderRequest, OrderType, FilterOptionParsed, hasAggregate, ColumnOption, withoutAggregate, FilterConditionOption, QueryDescription, FindOptions, withoutPinned, SystemTime
} from '@framework/FindOptions'
import { AuthClient } from '../Signum.Authorization/AuthClient'
import ChartButton from './ChartButton'
Expand All @@ -27,7 +27,7 @@ import { Dic, softCast } from '@framework/Globals';
import { colorInterpolators, colorSchemes } from './ColorPalette/ColorUtils';
import { getColorInterpolation } from './D3Scripts/Components/ChartUtils';
import { UserQueryEntity } from '../Signum.UserQueries/Signum.UserQueries';
import { ChartColumnEmbedded, ChartColumnType, ChartParameterEmbedded, ChartParameterType, ChartPermission, ChartRequestModel, ChartScriptSymbol, D3ChartScript, GoogleMapsChartScript, HtmlChartScript, SpecialParameterType, SvgMapsChartScript } from './Signum.Chart';
import { ChartColumnEmbedded, ChartColumnType, ChartParameterEmbedded, ChartParameterType, ChartPermission, ChartRequestModel, ChartScriptSymbol, ChartTimeSeriesEmbedded, D3ChartScript, GoogleMapsChartScript, HtmlChartScript, SpecialParameterType, SvgMapsChartScript } from './Signum.Chart';
import { IChartBase, UserChartEntity } from './UserChart/Signum.Chart.UserChart';
import { UserChartPartHandler } from './Dashboard/View/UserChartPart';
import SelectorModal from '@framework/SelectorModal';
Expand Down Expand Up @@ -516,12 +516,23 @@ export namespace ChartClient {
return clone;
}


export function cloneChartTimeSeries(ts : ChartTimeSeriesEmbedded | null): ChartTimeSeriesEmbedded | null {
if(!ts)
return null;
return ChartTimeSeriesEmbedded.New({
timeSeriesStep: ts.timeSeriesStep,
timeSeriesUnit: ts.timeSeriesUnit,
startDate: ts.startDate,
endDate: ts.endDate,
timeSeriesMaxRowsPerStep: ts.timeSeriesMaxRowsPerStep});
}

export interface ChartOptions {
queryName: any,
chartScript?: string,
maxRows?: number | null,
groupResults?: boolean,
timeSeries?: ChartTimeSeriesEmbedded | null | undefined;
filterOptions?: (FilterOption | null | undefined)[];
orderOptions?: (OrderOption | null | undefined)[];
columnOptions?: (ChartColumnOption | null | undefined)[];
Expand Down Expand Up @@ -575,6 +586,7 @@ export namespace ChartClient {
queryName: cr.queryKey,
chartScript: cr.chartScript?.key.after(".") ?? undefined,
maxRows: cr.maxRows,
timeSeries: cloneChartTimeSeries(cr.chartTimeSeries),
filterOptions: Finder.toFilterOptions(cr.filterOptions),
columnOptions: cr.columns.map(co => ({
token: co.element.token && co.element.token.tokenString,
Expand Down Expand Up @@ -610,15 +622,17 @@ export namespace ChartClient {
maxRows:
co.maxRows === null ? "null" :
co.maxRows === undefined || co.maxRows == Decoder.DefaultMaxRows ? undefined : co.maxRows,
groupResults: co.groupResults,
groupResults: co.groupResults,
userChart: userChart && liteKey(userChart)
};

encodeTimeSeries(query, co.timeSeries);
Finder.Encoder.encodeFilters(query, co.filterOptions?.notNull());
Finder.Encoder.encodeOrders(query, co.orderOptions?.notNull());
encodeParameters(query, co.parameters?.notNull());

encodeColumn(query, co.columnOptions?.notNull());


return `/chart/${getQueryKey(co.queryName)}?` + QueryString.stringify(query);

Expand All @@ -639,6 +653,17 @@ export namespace ChartClient {
if (parameters)
parameters.map((p, i) => query["param" + i] = scapeTilde(p.name!) + "~" + scapeTilde(p.value!));
}

export function encodeTimeSeries(query: any, ts: ChartTimeSeriesEmbedded | null | undefined): void {
if (ts)
{
query['systemTimeStartDate'] = ts.startDate;
query['systemTimeEndDate'] = ts.endDate;
query['timeSeriesStep'] = ts.timeSeriesStep;
query['timeSeriesUnit'] = ts.timeSeriesUnit;
query['timeSeriesMaxRowsPerStep'] = ts.timeSeriesMaxRowsPerStep;
}
}
}

export module Decoder {
Expand All @@ -658,8 +683,10 @@ export namespace ChartClient {
const oos = Finder.Decoder.decodeOrders(query);
oos.forEach(oo => completer.request(oo.token.toString(), SubTokensOptions.CanElement | SubTokensOptions.CanAggregate));

const ts = Decoder.decodeTimeSeries(query);

const cols = Decoder.decodeColumns(query);
cols.map(a => a.element.token).filter(te => te != undefined).forEach(te => completer.request(te!.tokenString!, SubTokensOptions.CanAggregate | SubTokensOptions.CanElement));
cols.map(a => a.element.token).filter(te => te != undefined).forEach(te => completer.request(te!.tokenString!, SubTokensOptions.CanAggregate | SubTokensOptions.CanElement | (ts ? SubTokensOptions.CanTimeSeries : 0)));

return completer.finished().then(() => {

Expand All @@ -677,6 +704,7 @@ export namespace ChartClient {
filterOptions: fos.map(fo => completer.toFilterOptionParsed(fo)),
columns: cols,
parameters: Decoder.decodeParameters(query),
chartTimeSeries: ts,
});

synchronizeColumns(chartRequest, cr);
Expand Down Expand Up @@ -731,16 +759,40 @@ export namespace ChartClient {
})
}));
}

export function decodeTimeSeries(query: any): ChartTimeSeriesEmbedded | null {
if(!query.timeSeriesUnit)
return null;
return ChartTimeSeriesEmbedded.New({
startDate: query.systemTimeStartDate,
endDate: query.systemTimeEndDate,
timeSeriesUnit: query.timeSeriesUnit,
timeSeriesStep: query.timeSeriesStep && parseInt(query.timeSeriesStep),
timeSeriesMaxRowsPerStep: query.timeSeriesMaxRowsPerStep && parseInt(query.timeSeriesMaxRowsPerStep),
});
}
}


export module API {

export function getRequest(request: ChartRequestModel): QueryRequest {
var ts = request.chartTimeSeries;
var systemTime : SystemTime | undefined = ts == null ? undefined :
softCast<SystemTime>({
joinMode: 'AllCompatible',
mode : 'TimeSeries',
timeSeriesStep: ts.timeSeriesStep!,
timeSeriesUnit: ts.timeSeriesUnit!,
startDate: ts.startDate!,
endDate: ts.endDate!,
timeSeriesMaxRowsPerStep: ts.timeSeriesMaxRowsPerStep!,
});

return {
queryKey: request.queryKey,
groupResults: hasAggregates(request),
systemTime: systemTime,
filters: Finder.toFilterRequests(request.filterOptions),
columns: request.columns.map(mle => mle.element).filter(cce => cce.token != null).map(co => ({ token: co.token!.token!.fullKey }) as ColumnRequest),
orders: request.columns.filter(mle => mle.element.orderByType != null && mle.element.token != null).orderBy(mle => mle.element.orderByIndex).map(mle => ({ token: mle.element.token!.token!.fullKey, orderType: mle.element.orderByType! }) as OrderRequest),
Expand Down
1 change: 1 addition & 0 deletions Extensions/Signum.Chart/ChartLogic.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ public static QueryRequest ToQueryRequest(this ChartRequestModel request)
{
QueryName = request.QueryName,
GroupResults = request.HasAggregates(),
SystemTime = request.ChartTimeSeries?.ToSystemTimeRequest(),
Columns = request.GetQueryColumns(),
Filters = request.Filters,
Orders = request.GetQueryOrders(),
Expand Down
66 changes: 65 additions & 1 deletion Extensions/Signum.Chart/ChartRequest.cs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
using Signum.DynamicQuery.Tokens;
using Signum.UserAssets;
using System.Xml.Linq;

namespace Signum.Chart;

Expand All @@ -11,6 +13,8 @@ public interface IChartBase
MList<ChartColumnEmbedded> Columns { get; }
MList<ChartParameterEmbedded> Parameters { get; }

ChartTimeSeriesEmbedded? ChartTimeSeries { get; }

void FixParameters(ChartColumnEmbedded chartColumnEntity);
}

Expand Down Expand Up @@ -61,6 +65,8 @@ public ChartScript GetChartScript()

public int? MaxRows { get; set; }

public ChartTimeSeriesEmbedded? ChartTimeSeries { get; set; }

public List<Column> GetQueryColumns()
{
return Columns.Where(c => c.Token != null).Select(t => t.CreateColumn()).ToList();
Expand Down Expand Up @@ -94,7 +100,7 @@ public List<CollectionElementToken> Multiplications
{
get { return CollectionElementToken.GetElements(new HashSet<QueryToken>(AllTokens())); }
}

public void FixParameters(ChartColumnEmbedded chartColumn)
{
ChartUtils.FixParameters(this, chartColumn);
Expand All @@ -105,3 +111,61 @@ public bool HasAggregates()
return Filters.Any(a=>a.IsAggregate()) || Columns.Any(a=>a.Token?.Token is AggregateToken);
}
}

public class ChartTimeSeriesEmbedded : EmbeddedEntity
{
[StringLengthValidator(Max = 100)]
public string? StartDate { get; set; }

[StringLengthValidator(Max = 100)]
public string? EndDate { get; set; }

public TimeSeriesUnit? TimeSeriesUnit { get; set; }

[NumberIsValidator(ComparisonType.GreaterThan, 0)]
public int? TimeSeriesStep { get; set; }

[NumberIsValidator(ComparisonType.GreaterThan, 0)]
public int? TimeSeriesMaxRowsPerStep { get; set; }

internal ChartTimeSeriesEmbedded? FromXml(XElement xml)
{
StartDate = xml.Attribute("StartDate")?.Value;
EndDate = xml.Attribute("EndDate")?.Value;
TimeSeriesUnit = xml.Attribute("TimeSeriesUnit")?.Value.ToEnum<TimeSeriesUnit>();
TimeSeriesStep = xml.Attribute("TimeSeriesStep")?.Value.ToInt();
TimeSeriesMaxRowsPerStep = xml.Attribute("TimeSeriesMaxRowsPerStep")?.Value.ToInt();
return this;
}

internal XElement ToXml()
{
return new XElement("SystemTime",
StartDate == null ? null : new XAttribute("StartDate", StartDate),
EndDate == null ? null : new XAttribute("EndDate", EndDate),
TimeSeriesUnit == null ? null : new XAttribute("TimeSeriesUnit", TimeSeriesUnit.ToString()!),
TimeSeriesStep == null ? null : new XAttribute("TimeSeriesStep", TimeSeriesStep.ToString()!),
TimeSeriesMaxRowsPerStep == null ? null : new XAttribute("TimeSeriesMaxRowsPerStep", TimeSeriesMaxRowsPerStep.ToString()!)
);
}

internal SystemTimeRequest ToSystemTimeRequest() => new SystemTimeRequest
{
mode = SystemTimeMode.TimeSeries,
joinMode = SystemTimeJoinMode.AllCompatible,
endDate = ParseDate(this.EndDate),
startDate = ParseDate(this.StartDate),
timeSeriesStep = this.TimeSeriesStep,
timeSeriesUnit = this.TimeSeriesUnit,
timeSeriesMaxRowsPerStep = this.TimeSeriesMaxRowsPerStep,
};

DateTime? ParseDate(string? date)
{
if (date.IsNullOrEmpty())
return null;


return (DateTime)FilterValueConverter.Parse(date, typeof(DateTime), false)!;
}
}
6 changes: 4 additions & 2 deletions Extensions/Signum.Chart/ChartServer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,9 @@ public static void Start(IApplicationBuilder app)

if (cr.Columns != null)
foreach (var c in cr.Columns)
c.ParseData(cr, qd, SubTokensOptions.CanElement | SubTokensOptions.CanAggregate);
{
c.ParseData(cr, qd, SubTokensOptions.CanElement | SubTokensOptions.CanAggregate | (cr.ChartTimeSeries != null ? SubTokensOptions.CanTimeSeries : 0));
}
}
});

Expand Down Expand Up @@ -93,7 +95,7 @@ private static void CustomizeChartRequest()

var qd = QueryLogic.Queries.QueryDescription(cr.QueryName);

cr.Filters = list.Select(l => l.ToFilter(qd, canAggregate: true, SignumServer.JsonSerializerOptions)).ToList();
cr.Filters = list.Select(l => l.ToFilter(qd, canAggregate: true, SignumServer.JsonSerializerOptions, cr.ChartTimeSeries != null)).ToList();
},
CustomWriteJsonProperty = (Utf8JsonWriter writer, WriteJsonPropertyContext ctx) =>
{
Expand Down
11 changes: 11 additions & 0 deletions Extensions/Signum.Chart/Signum.Chart.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -307,13 +307,24 @@ export interface ChartRequestModel extends Entities.ModelEntity {
columns: Entities.MList<ChartColumnEmbedded>;
parameters: Entities.MList<ChartParameterEmbedded>;
maxRows: number | null;
chartTimeSeries: ChartTimeSeriesEmbedded | null;
}

export const ChartScriptSymbol: Type<ChartScriptSymbol> = new Type<ChartScriptSymbol>("ChartScript");
export interface ChartScriptSymbol extends Basics.Symbol {
Type: "ChartScript";
}

export const ChartTimeSeriesEmbedded: Type<ChartTimeSeriesEmbedded> = new Type<ChartTimeSeriesEmbedded>("ChartTimeSeriesEmbedded");
export interface ChartTimeSeriesEmbedded extends Entities.EmbeddedEntity {
Type: "ChartTimeSeriesEmbedded";
startDate: string | null;
endDate: string | null;
timeSeriesUnit: DynamicQuery.TimeSeriesUnit | null;
timeSeriesStep: number | null;
timeSeriesMaxRowsPerStep: number | null;
}

export module D3ChartScript {
export const Bars : ChartScriptSymbol = registerSymbol("ChartScript", "D3ChartScript.Bars");
export const Columns : ChartScriptSymbol = registerSymbol("ChartScript", "D3ChartScript.Columns");
Expand Down
Loading

4 comments on commit 4a7cc10

@olmobrutall
Copy link
Collaborator

@olmobrutall olmobrutall commented on 4a7cc10 Nov 22, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Time Machine in Charting!

This change added the feature of making charts on database changes using Time Machine.

It's easier to understand with an example: You have an Order entity, or any other transactional or master entity that can be modified. One day you get a requirement to make a chart about the evolution of all theorders: The boss wants to see many orders we had on each state... every month to see hot it changed over time, this means that the same order should be counted in state "Ordered" for May 2024, but then in status "Shipped" in April 2024 and in any following month.

You could rebuild everything using some CQRS bullshit, but you have been doing normal UPDATES / DELETEs like a normal dev. Fortunately, you activated the time machine some months ago, so you have all the changes in the history table, so making this chart is a few clicks:

In the chart UI, if the table has system-versioning enabled a new Time Machine checkbox adds some new options:

Image

With this new blue bar you can configure:

  • How often you want to make a photo, in this case every 1 day.
  • From which day (min) to which day (max).
  • Limit, for every photo, the max rows you want (TOP).
  • A small calculator gives you an estimation of the total number of rows.

Then, in the charting configuration, you probably want to use the new [Time series] token in your horizontal axis.

In this case, look how we over estimated the TOP 20, while in reality there are only 3 values per photo (the count of Ordered, Shipped and Cancelled).

Of course, we are not making 327 queries to the database! Instead is doing a SelectMany with a Tabled-Value-Function
called GetDatesInRange.

SELECT TOP (@p0) 
  dv.Date, 
  [s9].[c01], 
  [s9].[c0]
FROM dbo.GetDatesInRange(@p1, @p2, @p3, @p4) AS dv
CROSS APPLY (
  (SELECT TOP (@p5) 
    [s5].[c0] as [c01], 
    [s5].[agg1] as [c0]
  FROM (
    (SELECT 
      o.StateID as [c0], 
      COUNT(*) as [agg1]
    FROM dbo.[Order] FOR SYSTEM_TIME ALL AS o
    WHERE ((o.SysStartDate <= dv.Date) AND (dv.Date < o.SysEndDate))
    GROUP BY o.StateID)
  ) AS [s5])
) AS [s9]

Oh! And you can also use Time Series from the Search Control!

Image

Also, to make this chart I had to simulate the values in the history table I had to make some cool UnsafeUpdates, maybe is usefull for someone importing data...

That's it! Enjoy the coding.

@MehdyKarimpour
Copy link
Contributor

@MehdyKarimpour MehdyKarimpour commented on 4a7cc10 Nov 23, 2024 via email

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@MehdyKarimpour
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Great feature! 💯

@KonstantinLukaschenko
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Image

Please sign in to comment.