Support for debounced and bulk uploading of track messages

CHANGELOG.md

@@ -3,27 +3,32 @@
 ## 69.0.0-SNAPSHOT - unreleased
 
 ### 💥 Breaking Changes (upgrade difficulty: 🟢 LOW )
-* The `INITIALIZING` AppState has been replaced with more fine-grained states (see below).  This
-is not expected to affect any applications.
 
-### 🎁 New Features
+* Requires `hoist-core >= 23.1` to support bulk upload of activity tracking logs to server.
+* `AppState.INITIALIZING` replaced with finer-grained states (not expected to impact most apps).
 
+### 🎁 New Features
 * Added new AppStates `AUTHENTICATING`, `INITIALIZING_HOIST`, and `INITIALIZING_APP` to support
 more granular tracking and timing of app startup lifecycle.
 * Improved the default "Loaded App" activity tracking entry with more granular data on load timing.
 * `RestGrid` now displays an optional refresh button in its toolbar.
-
-### ⚙️ Technical
-
-* Improvements to the typing of `HoistBase.addReaction`.  Note that applications may need to adjust
-typescript slightly in calls to this method to conform to the tighter typing.
+* Enhanced tracking data posted with the built-in "Loaded App" entry to include a new `timings`
+  block that breaks down the overall initial load time into more discrete phases. Supported by
+  new `AppState` enums `AUTHENTICATING`, `INITIALIZING_HOIST`, and `INITIALIZING_APP`.
 * The filter field in the top toolbar of Grid's Column Filter Values tab now filters with `any`,
   instead of `startsWith`.
 
-### 🐞 Bug Fixes
 
-* Added a workaround for a bug where Panel drag resizing was broken in Safari.
+### ⚙️ Typescript API Adjustments
+* Improved typing of `HoistBase.addReaction` to flow types returned by the `track` closure through
+  to the `run` closure that receives them.
+  * Note that apps might need to adjust their reaction signatures slightly to accommodate the more
+    accurate typing, specifically if they are tracking an array of values, destructuring those
+    values in their `run` closure, and passing them on to typed APIs. Look out for `tsc` warnings.
+
+### 🐞 Bug Fixes
 
+* Fixed broken `Panel` resizing in Safari. (Other browsers were not affected.)
 
 ## 68.1.0 - 2024-09-27
 
@@ -33,6 +38,7 @@ typescript slightly in calls to this method to conform to the tighter typing.
   props to the underlying `reactMarkdown` instance.
 
 ### ⚙️ Technical
+
 * Misc. Improvements to Cluster Tab in Admin Panel.
 
 ## 68.0.0 - 2024-09-18
@@ -61,7 +67,6 @@ typescript slightly in calls to this method to conform to the tighter typing.
 * mobx  `6.9.1 -> 6.13.2`,
 * mobx-react-lite  `3.4.3 -> 4.0.7`,
 
-
 ## 67.0.0 - 2024-09-03
 
 ### 💥 Breaking Changes (upgrade difficulty: 🟢 LOW - Hoist Core update only)

appcontainer/AppStateModel.ts

@@ -85,6 +85,7 @@ export class AppStateModel extends HoistModel {
                 XH.track({
                     category: 'App',
                     message: `Loaded ${XH.clientAppCode}`,
+                    timestamp: loadStarted,
                     elapsed: Date.now() - loadStarted - (timings.LOGIN_REQUIRED ?? 0),
                     data: {
                         appVersion: XH.appVersion,

core/types/Interfaces.ts

@@ -234,6 +234,9 @@ export interface TrackOptions {
     /** Optional LoadSpec associated with this track.*/
     loadSpec?: LoadSpec;
 
+    /** Timestamp for action. */
+    timestamp?: number;
+
     /** Elapsed time (ms) for action. */
     elapsed?: number;
 

promise/Promise.ts

@@ -198,8 +198,11 @@ const enhancePromise = promisePrototype => {
 
             const startTime = Date.now();
             return this.finally(() => {
-                options.elapsed = Date.now() - startTime;
-                XH.track(options);
+                XH.track({
+                    timestamp: startTime,
+                    elapsed: Date.now() - startTime,
+                    ...options
+                });
             });
         },
 

svc/PrefService.ts

@@ -6,8 +6,8 @@
  */
 import {HoistService, XH} from '@xh/hoist/core';
 import {SECONDS} from '@xh/hoist/utils/datetime';
-import {deepFreeze, throwIf} from '@xh/hoist/utils/js';
-import {cloneDeep, debounce, forEach, isEmpty, isEqual, size} from 'lodash';
+import {debounced, deepFreeze, throwIf} from '@xh/hoist/utils/js';
+import {cloneDeep, forEach, isEmpty, isEqual, size} from 'lodash';
 
 /**
  * Service to read and set user-specific preference values.
@@ -30,16 +30,9 @@ export class PrefService extends HoistService {
 
     private _data = {};
     private _updates = {};
-    private readonly pushPendingBuffered: any;
-
-    constructor() {
-        super();
-        const pushFn = () => this.pushPendingAsync();
-        window.addEventListener('beforeunload', pushFn);
-        this.pushPendingBuffered = debounce(pushFn, 5 * SECONDS);
-    }
 
     override async initAsync() {
+        window.addEventListener('beforeunload', () => this.pushPendingAsync());
         await this.migrateLocalPrefsAsync();
         return this.loadPrefsAsync();
     }
@@ -139,23 +132,25 @@ export class PrefService extends HoistService {
 
         if (isEmpty(updates)) return;
 
-        // clear obj state immediately to allow picking up next batch during async operation
         this._updates = {};
 
-        if (!isEmpty(updates)) {
-            await XH.fetchJson({
-                url: 'xh/setPrefs',
-                params: {
-                    updates: JSON.stringify(updates),
-                    clientUsername: XH.getUsername()
-                }
-            });
-        }
+        await XH.postJson({
+            url: 'xh/setPrefs',
+            body: updates,
+            params: {
+                clientUsername: XH.getUsername()
+            }
+        });
     }
 
     //-------------------
     //  Implementation
     //-------------------
+    @debounced(5 * SECONDS)
+    private pushPendingBuffered() {
+        this.pushPendingAsync();
+    }
+
     private async loadPrefsAsync() {
         const data = await XH.fetchJson({
             url: 'xh/getPrefs',

svc/TrackService.ts

@@ -4,10 +4,11 @@
  *
  * Copyright © 2024 Extremely Heavy Industries Inc.
  */
-import {HoistService, TrackOptions, XH} from '@xh/hoist/core';
+import {HoistService, PlainObject, TrackOptions, XH} from '@xh/hoist/core';
+import {SECONDS} from '@xh/hoist/utils/datetime';
 import {isOmitted} from '@xh/hoist/utils/impl';
-import {stripTags, withDefault} from '@xh/hoist/utils/js';
-import {isNil, isString} from 'lodash';
+import {debounced, stripTags, withDefault} from '@xh/hoist/utils/js';
+import {isEmpty, isNil, isString} from 'lodash';
 
 /**
  * Primary service for tracking any activity that an application's admins want to track.
@@ -17,7 +18,12 @@ import {isNil, isString} from 'lodash';
 export class TrackService extends HoistService {
     static instance: TrackService;
 
-    private _oncePerSessionSent = new Map();
+    private oncePerSessionSent = new Map();
+    private pending: PlainObject[] = [];
+
+    override async initAsync() {
+        window.addEventListener('beforeunload', () => this.pushPendingAsync());
+    }
 
     get conf() {
         return XH.getConf('xhActivityTrackingConfig', {
@@ -40,84 +46,102 @@ export class TrackService extends HoistService {
         // Normalize string form, msg -> message, default severity.
         if (isString(options)) options = {message: options};
         if (isOmitted(options)) return;
-        options.message = withDefault(options.message, (options as any).msg);
-        options.severity = withDefault(options.severity, 'INFO');
+        options = {
+            message: withDefault(options.message, (options as any).msg),
+            severity: withDefault(options.severity, 'INFO'),
+            timestamp: withDefault(options.timestamp, Date.now()),
+            ...options
+        };
 
         // Short-circuit if disabled...
         if (!this.enabled) {
             this.logDebug('Activity tracking disabled - activity will not be tracked.', options);
             return;
         }
 
-        // ...or invalid request (with warning for developer)...
+        // ...or invalid request (with warning for developer)
         if (!options.message) {
             this.logWarn('Required message not provided - activity will not be tracked.', options);
             return;
         }
 
-        // ...or if auto-refresh...
+        // ...or if auto-refresh
         if (options.loadSpec?.isAutoRefresh) return;
 
-        // ...or if unauthenticated user...
+        // ...or if unauthenticated user
         if (!XH.getUsername()) return;
 
-        // ...or if already-sent once-per-session messages.
-        const key = options.message + '_' + (options.category ?? '');
-        if (options.oncePerSession && this._oncePerSessionSent.has(key)) return;
-
-        // Otherwise - fire off (but do not await) request.
-        this.doTrackAsync(options);
-
+        // ...or if already-sent once-per-session messages
         if (options.oncePerSession) {
-            this._oncePerSessionSent.set(key, true);
+            const sent = this.oncePerSessionSent,
+                key = options.message + '_' + (options.category ?? '');
+            if (sent.has(key)) return;
+            sent.set(key, true);
         }
+
+        // Otherwise - log and for next batch,
+        this.logMessage(options);
+
+        this.pending.push(this.toServerJson(options));
+        this.pushPendingBuffered();
     }
 
     //------------------
     // Implementation
     //------------------
-    private async doTrackAsync(options: TrackOptions) {
-        try {
-            const query: any = {
-                msg: stripTags(options.message),
-                clientUsername: XH.getUsername(),
-                appVersion: XH.getEnv('clientVersion'),
-                url: window.location.href
-            };
-
-            if (options.category) query.category = options.category;
-            if (options.correlationId) query.correlationId = options.correlationId;
-            if (options.data) query.data = options.data;
-            if (options.severity) query.severity = options.severity;
-            if (options.logData !== undefined) query.logData = options.logData;
-            if (options.elapsed !== undefined) query.elapsed = options.elapsed;
-
-            const {maxDataLength} = this.conf;
-            if (query.data?.length > maxDataLength) {
-                this.logWarn(
-                    `Track log includes ${query.data.length} chars of JSON data`,
-                    `exceeds limit of ${maxDataLength}`,
-                    'data will not be persisted',
-                    options.data
-                );
-                query.data = null;
-            }
-
-            const elapsedStr = query.elapsed != null ? `${query.elapsed}ms` : null,
-                consoleMsgs = [query.category, query.msg, query.correlationId, elapsedStr].filter(
-                    it => !isNil(it)
-                );
-
-            this.logInfo(...consoleMsgs);
-
-            await XH.fetchService.postJson({
-                url: 'xh/track',
-                body: query,
-                // Post clientUsername as a parameter to ensure client username matches session.
-                params: {clientUsername: query.clientUsername}
-            });
-        } catch (e) {
-            this.logError('Failed to persist track log', options, e);
+    private async pushPendingAsync() {
+        const {pending} = this;
+        if (isEmpty(pending)) return;
+
+        this.pending = [];
+        await XH.fetchService.postJson({
+            url: 'xh/track',
+            body: {entries: pending},
+            params: {clientUsername: XH.getUsername()}
+        });
+    }
+
+    @debounced(10 * SECONDS)
+    private pushPendingBuffered() {
+        this.pushPendingAsync();
+    }
+
+    private toServerJson(options: TrackOptions): PlainObject {
+        const ret: PlainObject = {
+            msg: stripTags(options.message),
+            clientUsername: XH.getUsername(),
+            appVersion: XH.getEnv('clientVersion'),
+            url: window.location.href,
+            timestamp: Date.now()
+        };
+
+        if (options.category) ret.category = options.category;
+        if (options.correlationId) ret.correlationId = options.correlationId;
+        if (options.data) ret.data = options.data;
+        if (options.severity) ret.severity = options.severity;
+        if (options.logData !== undefined) ret.logData = options.logData;
+        if (options.elapsed !== undefined) ret.elapsed = options.elapsed;
+
+        const {maxDataLength} = this.conf;
+        if (ret.data?.length > maxDataLength) {
+            this.logWarn(
+                `Track log includes ${ret.data.length} chars of JSON data`,
+                `exceeds limit of ${maxDataLength}`,
+                'data will not be persisted',
+                options.data
+            );
+            ret.data = null;
         }
+
+        return ret;
+    }
+
+    private logMessage(opts: TrackOptions) {
+        const elapsedStr = opts.elapsed != null ? `${opts.elapsed}ms` : null,
+            consoleMsgs = [opts.category, opts.message, opts.correlationId, elapsedStr].filter(
+                it => !isNil(it)
+            );
+
+        this.logInfo(...consoleMsgs);
     }
 }