feat(history): Added progress bar

OptimistikSAS · Feb 5, 2024 · fcff359 · fcff359
1 parent 49535f8
commit fcff359
Show file tree

Hide file tree

Showing 14 changed files with 213 additions and 47 deletions.
diff --git a/backend/src/south/south-connector.spec.ts b/backend/src/south/south-connector.spec.ts
@@ -53,7 +53,8 @@ jest.mock(
         },
         metrics: {
           numberOfValuesRetrieved: 1,
-          numberOfFilesRetrieved: 1
+          numberOfFilesRetrieved: 1,
+          historyMetrics: {}
         }
       };
     }

diff --git a/backend/src/south/south-connector.ts b/backend/src/south/south-connector.ts
@@ -262,10 +262,9 @@ export default class SouthConnector<T extends SouthSettings = any, I extends Sou
       for (const [index, item] of itemsToRead.entries()) {
         if (this.stopping) {
           this.logger.debug(`Connector is stopping. Exiting history query at item ${item.name}`);
-          this.metricsService!.updateMetrics(this.connector.id, {
-            ...this.metricsService!.metrics,
-            historyMetrics: { running: false }
-          });
+          const stoppedMetrics = structuredClone(this.metricsService!.metrics);
+          stoppedMetrics.historyMetrics.running = false;
+          this.metricsService!.updateMetrics(this.connector.id, stoppedMetrics);
           this.historyIsRunning = false;
           return;
         }
@@ -293,10 +292,9 @@ export default class SouthConnector<T extends SouthSettings = any, I extends Sou
 
       await this.queryIntervals(intervals, itemsToRead, southCache, startTime);
     }
-    this.metricsService!.updateMetrics(this.connector.id, {
-      ...this.metricsService!.metrics,
-      historyMetrics: { running: false }
-    });
+    const stoppedMetrics = structuredClone(this.metricsService!.metrics);
+    stoppedMetrics.historyMetrics.running = false;
+    this.metricsService!.updateMetrics(this.connector.id, stoppedMetrics);
     this.historyIsRunning = false;
   }
 
@@ -310,44 +308,39 @@ export default class SouthConnector<T extends SouthSettings = any, I extends Sou
       ...this.metricsService!.metrics,
       historyMetrics: {
         running: true,
-        intervalProgress:
-          1 -
-          (DateTime.fromISO(intervals[intervals.length - 1].end).toMillis() - DateTime.fromISO(intervals[0].start).toMillis()) /
-            (DateTime.fromISO(intervals[intervals.length - 1].end).toMillis() - DateTime.fromISO(startTime).toMillis())
+        intervalProgress: this.calculateIntervalProgress(intervals, 0, startTime)
       }
     });
+
     for (const [index, interval] of intervals.entries()) {
       // @ts-ignore
       const lastInstantRetrieved = await this.historyQuery(items, interval.start, interval.end);
-      if (index !== intervals.length - 1) {
-        this.cacheService!.createOrUpdateCacheScanMode({
-          southId: this.connector.id,
-          scanModeId: southCache.scanModeId,
-          itemId: southCache.itemId,
-          maxInstant: lastInstantRetrieved
-        });
-        this.metricsService!.updateMetrics(this.connector.id, {
-          ...this.metricsService!.metrics,
-          historyMetrics: {
-            running: true,
-            intervalProgress:
-              1 -
-              (DateTime.fromISO(intervals[intervals.length - 1].end).toMillis() - DateTime.fromISO(interval.start).toMillis()) /
-                (DateTime.fromISO(intervals[intervals.length - 1].end).toMillis() - DateTime.fromISO(startTime).toMillis())
-          }
-        });
-        if (this.stopping) {
-          this.logger.debug(`Connector is stopping. Exiting history query at interval ${index}: [${interval.start}, ${interval.end}]`);
-          return;
+      this.cacheService!.createOrUpdateCacheScanMode({
+        southId: this.connector.id,
+        scanModeId: southCache.scanModeId,
+        itemId: southCache.itemId,
+        maxInstant: lastInstantRetrieved
+      });
+
+      this.metricsService!.updateMetrics(this.connector.id, {
+        ...this.metricsService!.metrics,
+        historyMetrics: {
+          running: true,
+          intervalProgress: this.calculateIntervalProgress(intervals, index, startTime),
+          currentIntervalStart: interval.start,
+          currentIntervalEnd: interval.end,
+          currentIntervalNumber: index + 1,
+          numberOfIntervals: intervals.length
         }
+      });
+
+      if (this.stopping) {
+        this.logger.debug(`Connector is stopping. Exiting history query at interval ${index}: [${interval.start}, ${interval.end}]`);
+        return;
+      }
+
+      if (index !== intervals.length - 1) {
         await delay(this.connector.history.readDelay);
-      } else {
-        this.cacheService!.createOrUpdateCacheScanMode({
-          southId: this.connector.id,
-          scanModeId: southCache.scanModeId,
-          itemId: southCache.itemId,
-          maxInstant: lastInstantRetrieved
-        });
       }
     }
   }
@@ -372,6 +365,28 @@ export default class SouthConnector<T extends SouthSettings = any, I extends Sou
     }
   }
 
+  /**
+   * Calculate the progress of the current interval
+   */
+  private calculateIntervalProgress(intervals: Array<Interval>, currentIntervalIndex: number, startTime: Instant) {
+    // calculate progress based on time
+    const progress =
+      1 -
+      (DateTime.fromISO(intervals[intervals.length - 1].end).toMillis() -
+        DateTime.fromISO(intervals[currentIntervalIndex].start).toMillis()) /
+        (DateTime.fromISO(intervals[intervals.length - 1].end).toMillis() - DateTime.fromISO(startTime).toMillis());
+
+    // round to 2 decimals
+    const roundedProgress = Math.round((progress + Number.EPSILON) * 100) / 100;
+
+    // in the chance that the rounded progress is 0.99, but it's the last interval, we want to return 1
+    if (currentIntervalIndex === intervals.length - 1) {
+      return 1;
+    }
+
+    return roundedProgress;
+  }
+
   /**
    * Add new values to the South connector buffer.
    */

diff --git a/frontend/proxy.conf.mjs b/frontend/proxy.conf.mjs
@@ -1,7 +1,8 @@
 const PROXY_CONFIG = [
   {
     // proxy all the requests starting with /api to the NodeJS backend
-    context: ['/api'],
+    // note: we need to add the /sse path for the server sent events
+    context: ['/api', '/sse'],
     target: 'http://localhost:2223',
     secure: false
   }

diff --git a/...src/app/history-query/history-query-detail/history-metrics/history-metrics.component.html b/...src/app/history-query/history-query-detail/history-metrics/history-metrics.component.html
@@ -14,6 +14,13 @@
         </tr>
       </thead>
       <tbody>
+        <!-- north interval progressbar -->
+        <tr *ngIf="historyMetrics.south.historyMetrics.intervalProgress === 1 && northProgress > 0">
+          <td translate="history-query.monitoring.north.data-sent"></td>
+          <td>
+            <oib-progressbar [value]="northProgress" [max]="1" [animated]="northProgressbarAnimated" />
+          </td>
+        </tr>
         <!-- number of values for point connectors -->
         <tr *ngIf="northManifest.modes.points">
           <td style="width: 33%" translate="history-query.monitoring.north.number-of-values"></td>
@@ -64,6 +71,30 @@
         </tr>
       </thead>
       <tbody>
+        <!-- south interval progressbar -->
+        <tr *ngIf="historyMetrics.south.historyMetrics.intervalProgress">
+          <td translate="history-query.monitoring.south.data-retrieved"></td>
+          <td>
+            <oib-progressbar [value]="historyMetrics.south.historyMetrics.intervalProgress" [animated]="southProgressbarAnimated" />
+          </td>
+        </tr>
+        <!-- south interval progress -->
+        <tr *ngIf="historyMetrics.south.historyMetrics.currentIntervalNumber && historyMetrics.south.historyMetrics.numberOfIntervals">
+          <td translate="history-query.monitoring.south.interval-progress"></td>
+          <td>
+            {{ historyMetrics.south.historyMetrics.currentIntervalNumber }} / {{ historyMetrics.south.historyMetrics.numberOfIntervals }}
+          </td>
+        </tr>
+        <!-- south interval start -->
+        <tr *ngIf="historyMetrics.south.historyMetrics.currentIntervalStart">
+          <td translate="history-query.monitoring.south.interval-start"></td>
+          <td>{{ historyMetrics.south.historyMetrics.currentIntervalStart }}</td>
+        </tr>
+        <!-- south interval end -->
+        <tr *ngIf="historyMetrics.south.historyMetrics.currentIntervalEnd">
+          <td translate="history-query.monitoring.south.interval-end"></td>
+          <td>{{ historyMetrics.south.historyMetrics.currentIntervalEnd }}</td>
+        </tr>
         <!-- number of values for point connectors -->
         <tr>
           <td style="width: 33%" translate="history-query.monitoring.south.number-of-values"></td>

diff --git a/.../app/history-query/history-query-detail/history-metrics/history-metrics.component.spec.ts b/.../app/history-query/history-query-detail/history-metrics/history-metrics.component.spec.ts
@@ -118,7 +118,28 @@ class TestComponent {
       retentionDuration: 0
     }
   };
-  historyMetrics: HistoryMetrics = { north: {}, south: {} } as HistoryMetrics;
+  historyMetrics: HistoryMetrics = {
+    north: {
+      numberOfValuesSent: 10,
+      numberOfFilesSent: 0,
+      lastValueSent: null,
+      lastFileSent: null,
+      cacheSize: 0,
+      metricsStart: '2023-01-01T00:00:00.000Z',
+      lastConnection: null,
+      lastRunStart: null,
+      lastRunDuration: null
+    },
+    south: {
+      numberOfValuesRetrieved: 20,
+      numberOfFilesRetrieved: 0,
+      lastValueRetrieved: null,
+      lastFileRetrieved: null,
+      historyMetrics: {
+        intervalProgress: 1
+      }
+    }
+  } as HistoryMetrics;
 }
 
 class HistoryMetricsComponentTester extends ComponentTester<TestComponent> {

diff --git a/...d/src/app/history-query/history-query-detail/history-metrics/history-metrics.component.ts b/...d/src/app/history-query/history-query-detail/history-metrics/history-metrics.component.ts
@@ -8,12 +8,13 @@ import { BoxComponent, BoxTitleDirective } from '../../../shared/box/box.compone
 import { HistoryQueryDTO } from '../../../../../../shared/model/history-query.model';
 import { NorthConnectorManifest } from '../../../../../../shared/model/north-connector.model';
 import { SouthConnectorManifest } from '../../../../../../shared/model/south-connector.model';
+import { ProgressbarComponent } from './progressbar/progressbar.component';
 
 @Component({
   selector: 'oib-history-metrics',
   templateUrl: './history-metrics.component.html',
   styleUrl: './history-metrics.component.scss',
-  imports: [TranslateModule, NgIf, DatetimePipe, DurationPipe, BoxComponent, BoxTitleDirective, JsonPipe],
+  imports: [TranslateModule, NgIf, DatetimePipe, DurationPipe, BoxComponent, BoxTitleDirective, JsonPipe, ProgressbarComponent],
   standalone: true
 })
 export class HistoryMetricsComponent {
@@ -23,4 +24,25 @@ export class HistoryMetricsComponent {
   @Input({ required: true }) historyMetrics!: HistoryMetrics;
 
   constructor() {}
+
+  get southProgressbarAnimated(): boolean {
+    if (this.historyQuery.status === 'RUNNING' && this.historyMetrics.south.historyMetrics.intervalProgress !== 1) {
+      return true;
+    }
+    return false;
+  }
+
+  get northProgress() {
+    const valueProgress = this.historyMetrics.north.numberOfValuesSent / this.historyMetrics.south.numberOfValuesRetrieved;
+    const fileProgress = this.historyMetrics.north.numberOfFilesSent / this.historyMetrics.south.numberOfFilesRetrieved;
+
+    return valueProgress > 0 ? valueProgress : fileProgress;
+  }
+
+  get northProgressbarAnimated(): boolean {
+    if (this.historyQuery.status === 'RUNNING' && this.northProgress !== 1) {
+      return true;
+    }
+    return false;
+  }
 }
diff --git a/...history-query/history-query-detail/history-metrics/progressbar/progressbar.component.html b/...history-query/history-query-detail/history-metrics/progressbar/progressbar.component.html
@@ -0,0 +1,4 @@
+<div class="oib-progressbar">
+  <span>{{value | percent}}</span>
+  <ngb-progressbar [striped]="animated" [animated]="animated" [value]="value" [max]="max" />
+</div>
diff --git a/...history-query/history-query-detail/history-metrics/progressbar/progressbar.component.scss b/...history-query/history-query-detail/history-metrics/progressbar/progressbar.component.scss
@@ -0,0 +1,10 @@
+.oib-progressbar {
+  position: relative;
+}
+
+.oib-progressbar span {
+  position: absolute;
+  left: 50%;
+  top: 50%;
+  transform: translate(-50%, -50%);
+}
diff --git a/...tory-query/history-query-detail/history-metrics/progressbar/progressbar.component.spec.ts b/...tory-query/history-query-detail/history-metrics/progressbar/progressbar.component.spec.ts
@@ -0,0 +1,22 @@
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { ProgressbarComponent } from './progressbar.component';
+
+describe('ProgressbarComponent', () => {
+  let component: ProgressbarComponent;
+  let fixture: ComponentFixture<ProgressbarComponent>;
+
+  beforeEach(async () => {
+    await TestBed.configureTestingModule({
+      imports: [ProgressbarComponent]
+    }).compileComponents();
+
+    fixture = TestBed.createComponent(ProgressbarComponent);
+    component = fixture.componentInstance;
+    fixture.detectChanges();
+  });
+
+  it('should create', () => {
+    expect(component).toBeTruthy();
+  });
+});
diff --git a/...p/history-query/history-query-detail/history-metrics/progressbar/progressbar.component.ts b/...p/history-query/history-query-detail/history-metrics/progressbar/progressbar.component.ts
@@ -0,0 +1,16 @@
+import { PercentPipe } from '@angular/common';
+import { Component, Input } from '@angular/core';
+import { NgbProgressbarModule } from '@ng-bootstrap/ng-bootstrap';
+
+@Component({
+  selector: 'oib-progressbar',
+  standalone: true,
+  imports: [NgbProgressbarModule, PercentPipe],
+  templateUrl: './progressbar.component.html',
+  styleUrl: './progressbar.component.scss'
+})
+export class ProgressbarComponent {
+  @Input({ required: true }) value!: number;
+  @Input() max: number = 1;
+  @Input({ required: true }) animated!: boolean;
+}
diff --git a/frontend/src/app/history-query/history-query-detail/history-query-detail.component.ts b/frontend/src/app/history-query/history-query-detail/history-query-detail.component.ts
@@ -159,6 +159,10 @@ export class HistoryQueryDetailComponent implements OnInit, OnDestroy {
   }
 
   connectToEventSource(): void {
+    if (this.historyStream) {
+      this.historyStream.close();
+    }
+
     const token = this.windowService.getStorageItem('oibus-token');
     this.historyStream = new EventSource(`/sse/history-queries/${this.historyQuery!.id}?token=${token}`, { withCredentials: true });
     this.historyStream.addEventListener('message', (event: MessageEvent) => {
@@ -186,6 +190,7 @@ export class HistoryQueryDetailComponent implements OnInit, OnDestroy {
         .subscribe(updatedHistoryQuery => {
           this.historyQuery = updatedHistoryQuery;
           this.notificationService.success('history-query.started', { name: this.historyQuery!.name });
+          this.connectToEventSource();
         });
     } else {
       this.historyQueryService
@@ -199,6 +204,7 @@ export class HistoryQueryDetailComponent implements OnInit, OnDestroy {
         .subscribe(updatedHistoryQuery => {
           this.historyQuery = updatedHistoryQuery;
           this.notificationService.success('history-query.paused', { name: this.historyQuery!.name });
+          this.historyStream?.close();
         });
     }
   }

diff --git a/frontend/src/i18n/en.json b/frontend/src/i18n/en.json
@@ -722,7 +722,8 @@
         "last-value": "Last value sent",
         "last-value-content": "{{ pointId }} at {{ timestamp }} with content {{ data }}",
         "last-run-start": "Last run",
-        "last-run-duration": "Last run duration"
+        "last-run-duration": "Last run duration",
+        "data-sent": "Data sent"
       },
       "south": {
         "title": "South metrics",
@@ -736,7 +737,11 @@
         "last-value": "Last value retrieved",
         "last-value-content": "{{ pointId }} at {{ timestamp }} with content {{ data }}",
         "last-run-start": "Last run",
-        "last-run-duration": "Last run duration"
+        "last-run-duration": "Last run duration",
+        "data-retrieved": "Data retrieved",
+        "interval-start": "Interval start",
+        "interval-end": "Interval end",
+        "interval-progress": "Interval progress"
       }
     }
   },

diff --git a/shared/model/engine.model.ts b/shared/model/engine.model.ts
@@ -111,7 +111,19 @@ export interface NorthConnectorMetrics extends BaseConnectorMetrics {
   cacheSize: number;
 }
 
-export interface SouthHistoryMetrics {}
+export interface SouthHistoryMetrics {
+  running?: boolean;
+  // Percentage of the current interval that has been processed [0,1]
+  intervalProgress?: number;
+  // Start of the current interval
+  currentIntervalStart?: Instant;
+  // End of the current interval
+  currentIntervalEnd?: Instant;
+  // Number of the current interval
+  currentIntervalNumber?: number;
+  // Maximum number of intervals
+  numberOfIntervals?: number;
+}
 
 export interface SouthConnectorMetrics extends BaseConnectorMetrics {
   numberOfValuesRetrieved: number;

diff --git a/shared/model/history-query.model.ts b/shared/model/history-query.model.ts
@@ -2,7 +2,7 @@ import { NorthArchiveSettings, NorthCacheSettingsDTO } from './north-connector.m
 import { SouthConnectorHistorySettings, SouthConnectorItemDTO } from './south-connector.model';
 import { BaseEntity } from './types';
 
-export const HISTORY_QUERY_STATUS = ['PENDING', 'RUNNING', 'PAUSED', 'FINISHED', 'ERRORED'];
+export const HISTORY_QUERY_STATUS = ['PENDING', 'RUNNING', 'PAUSED', 'FINISHED', 'ERRORED'] as const;
 export type HistoryQueryStatus = (typeof HISTORY_QUERY_STATUS)[number];
 
 /**