Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(guardian-prover-health-check-ui): fix status glitches #15560

Merged
merged 3 commits into from
Jan 25, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
import DesktopOrLarger from '$components/DesktopOrLarger/DesktopOrLarger.svelte';
import { classNames } from '$lib/util/classNames';

import { loading as loadingStore } from '$stores';

export let refreshData: () => void;
export let filteredGuardianProvers: Guardian[] = [];
export let loading: boolean = false;
Expand All @@ -31,6 +33,8 @@

$: selectedStatus = null;

$: spin = loading || $loadingStore;

$: if (selectedStatus !== null) {
filtered = true;
filterByStatus(selectedStatus);
Expand All @@ -47,7 +51,7 @@

<div class={classes}>
<button class="btn btn-xs w-[36px] h-[36px] rounded-full" on:click={refreshData}
><RotatingIcon type="refresh" bind:loading /></button
><RotatingIcon type="refresh" bind:loading={spin} /></button
>
<StatusFilterDropdown bind:selectedStatus />
</div>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
const options = [
{ value: null, label: $t('filter.guardian_status.all') },
{ value: GuardianProverStatus.DEAD, label: $t('filter.guardian_status.dead') },
{ value: GuardianProverStatus.UNHEALTHY, label: $t('filter.guardian_status.unhealthy') },
{ value: GuardianProverStatus.ALIVE, label: $t('filter.guardian_status.alive') }
];

Expand Down
4 changes: 2 additions & 2 deletions packages/guardian-prover-health-check-ui/src/i18n/en.json
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,6 @@
"configuration_error": "Amount of provers does not match configuration!",
"critical": "CRITICAL: only {online} of required {required} provers online",
"degraded": "WARNING: only {online} of {total} provers online (min: {required})",
"operational": " {online} of {total} proves online. (min: {required})"
"operational": " {online} of {total} provers online. (min: {required})"
}
}
}
118 changes: 70 additions & 48 deletions packages/guardian-prover-health-check-ui/src/lib/dataFetcher.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { fetchGuardianProversFromContract } from './guardianProver/fetchGuardianProversFromContract';
import { GuardianProverStatus, type SignedBlocks } from './types';
import { GuardianProverStatus, type Guardian, type SignedBlocks } from './types';
import { fetchSignedBlocksFromApi } from './api/signedBlocksApiCalls';
import { getGuardianProverIdsPerBlockNumber } from './blocks/getGuardianProverIdsPerBlockNumber';
import { sortSignedBlocksDescending } from './blocks/sortSignedBlocks';
Expand All @@ -16,81 +16,97 @@ import {
loading,
totalGuardianProvers
} from '$stores';
import { get } from 'svelte/store';
import { get, writable } from 'svelte/store';

const BLOCKS_TO_CHECK = 20;
const THRESHOLD = BLOCKS_TO_CHECK / 2;
const HEALTHCHECK_TIMEOUT_IN_SECONDS = 60;

const tempGuardianStore = writable<Guardian[]>([]);

export function startFetching() {
if (get(loading) === true) return;
// Fetch all data immediately
refreshData();
let guardiansIntervalId;
let blocksAndLivelinessIntervalId;

export async function startFetching() {
if (get(loading)) return;

await refreshData();

// Set up an interval to fetch guardians every 30 seconds
const guardiansInterval = setInterval(() => {
guardiansIntervalId = setInterval(() => {
fetchGuardians();
}, 30000);
}, 10000);

// Set up an interval to fetch signed block and liveliness stats every 12 seconds
const blocksAndLivelinessInterval = setInterval(() => {
blocksAndLivelinessIntervalId = setInterval(() => {
fetchSignedBlockStats();
determineLiveliness();
}, 12000);
// Return a function to clear all intervals
return () => {
clearInterval(guardiansInterval);
clearInterval(blocksAndLivelinessInterval);
};
}

export function stopFetching() {
if (guardiansIntervalId) {
clearInterval(guardiansIntervalId);
guardiansIntervalId = null;
}
if (blocksAndLivelinessIntervalId) {
clearInterval(blocksAndLivelinessIntervalId);
blocksAndLivelinessIntervalId = null;
}
}

export async function refreshData() {
if (get(loading) === true) return;
loading.set(true);
await fetchSignedBlockStats();
await fetchGuardians();
await determineLiveliness();

if (get(guardianProvers)?.length === 0) {
await fetchGuardians();
const block = fetchSignedBlockStats();
const liveness = determineLiveliness();
await Promise.all([block, liveness]);
} else {
const guardian = fetchGuardians();
const block = fetchSignedBlockStats();
const liveness = determineLiveliness();
await Promise.all([block, guardian, liveness]);
}

loading.set(false);
}

async function fetchGuardians() {
const rawData = await fetchGuardianProversFromContract();
const required = await fetchGuardianProverRequirementsFromContract();

const [rawData, required] = await Promise.all([
fetchGuardianProversFromContract(),
fetchGuardianProverRequirementsFromContract()
]);
minGuardianRequirement.set(required);
totalGuardianProvers.set(rawData.length);

const guardians = [];
for (const guardian of rawData) {
const guardianFetchPromises = rawData.map(async (guardian) => {
const balance = await publicClient.getBalance({
address: guardian.address as Address
});

const balanceAsEther = formatEther(balance);
guardian.balance = balanceAsEther;

const status = await fetchLatestGuardianProverHealtCheckFromApi(
import.meta.env.VITE_GUARDIAN_PROVER_API_URL,
guardian.id
);
guardian.latestHealthCheck = status;
const [status, uptime] = await Promise.all([
fetchLatestGuardianProverHealtCheckFromApi(
import.meta.env.VITE_GUARDIAN_PROVER_API_URL,
guardian.id
),
fetchUptimeFromApi(import.meta.env.VITE_GUARDIAN_PROVER_API_URL, guardian.id)
]);

const uptime = await fetchUptimeFromApi(
import.meta.env.VITE_GUARDIAN_PROVER_API_URL,
guardian.id
);
guardian.uptime = uptime;
if (uptime > 100) guardian.uptime = 100;
guardian.latestHealthCheck = status;
guardian.uptime = Math.min(uptime, 100);
// guardian.alive = status.alive ? GuardianProverStatus.ALIVE : GuardianProverStatus.DEAD;

guardian.alive = status.alive ? GuardianProverStatus.ALIVE : GuardianProverStatus.DEAD;
return guardian;
});

guardians.push(guardian);
}
const data = await Promise.all(guardianFetchPromises);
tempGuardianStore.set(data);

lastGuardianFetchTimestamp.set(Date.now());

guardianProvers.set(guardians);
}

async function fetchSignedBlockStats() {
Expand All @@ -105,19 +121,19 @@ async function fetchSignedBlockStats() {
}

async function determineLiveliness(): Promise<void> {
const tempData = get(tempGuardianStore);
const now = new Date();
if (!tempData) return;

const guardians = get(guardianProvers);
if (!guardians) return;
for (const guardian of guardians) {
const guardians = tempData.map((guardian) => {
const latestCheck = guardian.latestHealthCheck;
const createdAt = new Date(latestCheck.createdAt);
const secondsSinceLastCheck = (now.getTime() - createdAt.getTime()) / 1000;

if (secondsSinceLastCheck > HEALTHCHECK_TIMEOUT_IN_SECONDS) {
guardian.alive = GuardianProverStatus.DEAD;
break;
return { ...guardian, alive: GuardianProverStatus.DEAD };
}

let countSignedBlocks = 0;
const recentSignedBlocks = get(signedBlocks).slice(0, BLOCKS_TO_CHECK);

Expand All @@ -127,8 +143,14 @@ async function determineLiveliness(): Promise<void> {
}
}

// Update status based on whether the guardian signed at least half of the configured blocks to check
guardian.alive =
const status =
countSignedBlocks >= THRESHOLD ? GuardianProverStatus.ALIVE : GuardianProverStatus.UNHEALTHY;
}

return {
...guardian,
alive: status
};
});

guardianProvers.set(guardians);
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,12 @@
import Header from '$components/Header/Header.svelte';
import { Page } from '$components/Page';
import { onMount, onDestroy } from 'svelte';
import { startFetching } from '$lib/dataFetcher';

let stopFetching: () => void;
import { startFetching, stopFetching } from '$lib/dataFetcher';
import { loading } from '$stores/stores';

onMount(() => {
stopFetching = startFetching();
loading.set(false);
startFetching();
});

onDestroy(() => {
Expand Down
30 changes: 29 additions & 1 deletion packages/guardian-prover-health-check-ui/src/stores/stores.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import {
type GuardianProverIdsMap,
PageTabs
} from '$lib/types';
import { type Writable, writable } from 'svelte/store';
import { type Writable, writable, derived } from 'svelte/store';

// Healtchecks
export const apiResponse: Writable<HealthCheck[]> = writable([]);
Expand All @@ -24,3 +24,31 @@ export const signerPerBlock: Writable<GuardianProverIdsMap> = writable({});
export const loading: Writable<boolean> = writable(false);
export const selectedTab = writable<PageTabs>(PageTabs.GUARDIAN_PROVER);
export const selectedGuardianProver = writable<Guardian>();

interface StatusCounts {
dead: number;
alive: number;
unhealthy: number;
}

export const guardianStatusCounts = derived(guardianProvers, ($guardianProvers): StatusCounts => {
if (!$guardianProvers) return { dead: 0, alive: 0, unhealthy: 0 };

return $guardianProvers.reduce(
(acc, guardian) => {
switch (guardian.alive) {
case 0: // DEAD
acc.dead += 1;
break;
case 1: // ALIVE
acc.alive += 1;
break;
case 2: // UNHEALTHY
acc.unhealthy += 1;
break;
}
return acc;
},
{ dead: 0, alive: 0, unhealthy: 0 }
);
});
Loading