Skip to content

Commit

Permalink
fix(guardian-prover-health-check-ui): fix status glitches (#15560)
Browse files Browse the repository at this point in the history
  • Loading branch information
KorbinianK authored Jan 25, 2024
1 parent 6d4cfe5 commit 5902e38
Show file tree
Hide file tree
Showing 6 changed files with 111 additions and 56 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
import DesktopOrLarger from '$components/DesktopOrLarger/DesktopOrLarger.svelte';
import { classNames } from '$lib/util/classNames';
import { loading as loadingStore } from '$stores';
export let refreshData: () => void;
export let filteredGuardianProvers: Guardian[] = [];
export let loading: boolean = false;
Expand All @@ -31,6 +33,8 @@
$: selectedStatus = null;
$: spin = loading || $loadingStore;
$: if (selectedStatus !== null) {
filtered = true;
filterByStatus(selectedStatus);
Expand All @@ -47,7 +51,7 @@

<div class={classes}>
<button class="btn btn-xs w-[36px] h-[36px] rounded-full" on:click={refreshData}
><RotatingIcon type="refresh" bind:loading /></button
><RotatingIcon type="refresh" bind:loading={spin} /></button
>
<StatusFilterDropdown bind:selectedStatus />
</div>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
const options = [
{ value: null, label: $t('filter.guardian_status.all') },
{ value: GuardianProverStatus.DEAD, label: $t('filter.guardian_status.dead') },
{ value: GuardianProverStatus.UNHEALTHY, label: $t('filter.guardian_status.unhealthy') },
{ value: GuardianProverStatus.ALIVE, label: $t('filter.guardian_status.alive') }
];
Expand Down
4 changes: 2 additions & 2 deletions packages/guardian-prover-health-check-ui/src/i18n/en.json
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,6 @@
"configuration_error": "Amount of provers does not match configuration!",
"critical": "CRITICAL: only {online} of required {required} provers online",
"degraded": "WARNING: only {online} of {total} provers online (min: {required})",
"operational": " {online} of {total} proves online. (min: {required})"
"operational": " {online} of {total} provers online. (min: {required})"
}
}
}
118 changes: 70 additions & 48 deletions packages/guardian-prover-health-check-ui/src/lib/dataFetcher.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { fetchGuardianProversFromContract } from './guardianProver/fetchGuardianProversFromContract';
import { GuardianProverStatus, type SignedBlocks } from './types';
import { GuardianProverStatus, type Guardian, type SignedBlocks } from './types';
import { fetchSignedBlocksFromApi } from './api/signedBlocksApiCalls';
import { getGuardianProverIdsPerBlockNumber } from './blocks/getGuardianProverIdsPerBlockNumber';
import { sortSignedBlocksDescending } from './blocks/sortSignedBlocks';
Expand All @@ -16,81 +16,97 @@ import {
loading,
totalGuardianProvers
} from '$stores';
import { get } from 'svelte/store';
import { get, writable } from 'svelte/store';

const BLOCKS_TO_CHECK = 20;
const THRESHOLD = BLOCKS_TO_CHECK / 2;
const HEALTHCHECK_TIMEOUT_IN_SECONDS = 60;

const tempGuardianStore = writable<Guardian[]>([]);

export function startFetching() {
if (get(loading) === true) return;
// Fetch all data immediately
refreshData();
let guardiansIntervalId;
let blocksAndLivelinessIntervalId;

export async function startFetching() {
if (get(loading)) return;

await refreshData();

// Set up an interval to fetch guardians every 30 seconds
const guardiansInterval = setInterval(() => {
guardiansIntervalId = setInterval(() => {
fetchGuardians();
}, 30000);
}, 10000);

// Set up an interval to fetch signed block and liveliness stats every 12 seconds
const blocksAndLivelinessInterval = setInterval(() => {
blocksAndLivelinessIntervalId = setInterval(() => {
fetchSignedBlockStats();
determineLiveliness();
}, 12000);
// Return a function to clear all intervals
return () => {
clearInterval(guardiansInterval);
clearInterval(blocksAndLivelinessInterval);
};
}

export function stopFetching() {
if (guardiansIntervalId) {
clearInterval(guardiansIntervalId);
guardiansIntervalId = null;
}
if (blocksAndLivelinessIntervalId) {
clearInterval(blocksAndLivelinessIntervalId);
blocksAndLivelinessIntervalId = null;
}
}

export async function refreshData() {
if (get(loading) === true) return;
loading.set(true);
await fetchSignedBlockStats();
await fetchGuardians();
await determineLiveliness();

if (get(guardianProvers)?.length === 0) {
await fetchGuardians();
const block = fetchSignedBlockStats();
const liveness = determineLiveliness();
await Promise.all([block, liveness]);
} else {
const guardian = fetchGuardians();
const block = fetchSignedBlockStats();
const liveness = determineLiveliness();
await Promise.all([block, guardian, liveness]);
}

loading.set(false);
}

async function fetchGuardians() {
const rawData = await fetchGuardianProversFromContract();
const required = await fetchGuardianProverRequirementsFromContract();

const [rawData, required] = await Promise.all([
fetchGuardianProversFromContract(),
fetchGuardianProverRequirementsFromContract()
]);
minGuardianRequirement.set(required);
totalGuardianProvers.set(rawData.length);

const guardians = [];
for (const guardian of rawData) {
const guardianFetchPromises = rawData.map(async (guardian) => {
const balance = await publicClient.getBalance({
address: guardian.address as Address
});

const balanceAsEther = formatEther(balance);
guardian.balance = balanceAsEther;

const status = await fetchLatestGuardianProverHealtCheckFromApi(
import.meta.env.VITE_GUARDIAN_PROVER_API_URL,
guardian.id
);
guardian.latestHealthCheck = status;
const [status, uptime] = await Promise.all([
fetchLatestGuardianProverHealtCheckFromApi(
import.meta.env.VITE_GUARDIAN_PROVER_API_URL,
guardian.id
),
fetchUptimeFromApi(import.meta.env.VITE_GUARDIAN_PROVER_API_URL, guardian.id)
]);

const uptime = await fetchUptimeFromApi(
import.meta.env.VITE_GUARDIAN_PROVER_API_URL,
guardian.id
);
guardian.uptime = uptime;
if (uptime > 100) guardian.uptime = 100;
guardian.latestHealthCheck = status;
guardian.uptime = Math.min(uptime, 100);
// guardian.alive = status.alive ? GuardianProverStatus.ALIVE : GuardianProverStatus.DEAD;

guardian.alive = status.alive ? GuardianProverStatus.ALIVE : GuardianProverStatus.DEAD;
return guardian;
});

guardians.push(guardian);
}
const data = await Promise.all(guardianFetchPromises);
tempGuardianStore.set(data);

lastGuardianFetchTimestamp.set(Date.now());

guardianProvers.set(guardians);
}

async function fetchSignedBlockStats() {
Expand All @@ -105,19 +121,19 @@ async function fetchSignedBlockStats() {
}

async function determineLiveliness(): Promise<void> {
const tempData = get(tempGuardianStore);
const now = new Date();
if (!tempData) return;

const guardians = get(guardianProvers);
if (!guardians) return;
for (const guardian of guardians) {
const guardians = tempData.map((guardian) => {
const latestCheck = guardian.latestHealthCheck;
const createdAt = new Date(latestCheck.createdAt);
const secondsSinceLastCheck = (now.getTime() - createdAt.getTime()) / 1000;

if (secondsSinceLastCheck > HEALTHCHECK_TIMEOUT_IN_SECONDS) {
guardian.alive = GuardianProverStatus.DEAD;
break;
return { ...guardian, alive: GuardianProverStatus.DEAD };
}

let countSignedBlocks = 0;
const recentSignedBlocks = get(signedBlocks).slice(0, BLOCKS_TO_CHECK);

Expand All @@ -127,8 +143,14 @@ async function determineLiveliness(): Promise<void> {
}
}

// Update status based on whether the guardian signed at least half of the configured blocks to check
guardian.alive =
const status =
countSignedBlocks >= THRESHOLD ? GuardianProverStatus.ALIVE : GuardianProverStatus.UNHEALTHY;
}

return {
...guardian,
alive: status
};
});

guardianProvers.set(guardians);
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,12 @@
import Header from '$components/Header/Header.svelte';
import { Page } from '$components/Page';
import { onMount, onDestroy } from 'svelte';
import { startFetching } from '$lib/dataFetcher';
let stopFetching: () => void;
import { startFetching, stopFetching } from '$lib/dataFetcher';
import { loading } from '$stores/stores';
onMount(() => {
stopFetching = startFetching();
loading.set(false);
startFetching();
});
onDestroy(() => {
Expand Down
30 changes: 29 additions & 1 deletion packages/guardian-prover-health-check-ui/src/stores/stores.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import {
type GuardianProverIdsMap,
PageTabs
} from '$lib/types';
import { type Writable, writable } from 'svelte/store';
import { type Writable, writable, derived } from 'svelte/store';

// Healtchecks
export const apiResponse: Writable<HealthCheck[]> = writable([]);
Expand All @@ -24,3 +24,31 @@ export const signerPerBlock: Writable<GuardianProverIdsMap> = writable({});
export const loading: Writable<boolean> = writable(false);
export const selectedTab = writable<PageTabs>(PageTabs.GUARDIAN_PROVER);
export const selectedGuardianProver = writable<Guardian>();

interface StatusCounts {
dead: number;
alive: number;
unhealthy: number;
}

export const guardianStatusCounts = derived(guardianProvers, ($guardianProvers): StatusCounts => {
if (!$guardianProvers) return { dead: 0, alive: 0, unhealthy: 0 };

return $guardianProvers.reduce(
(acc, guardian) => {
switch (guardian.alive) {
case 0: // DEAD
acc.dead += 1;
break;
case 1: // ALIVE
acc.alive += 1;
break;
case 2: // UNHEALTHY
acc.unhealthy += 1;
break;
}
return acc;
},
{ dead: 0, alive: 0, unhealthy: 0 }
);
});

0 comments on commit 5902e38

Please sign in to comment.