fix: simplify query if signature validation skipped (#56)

* fix: simplify query of signature validation skipped * fix * fix: ch query --------- Co-authored-by: Eugene Formanenko <[email protected]>
subsquid · Oct 18, 2024 · fc97220 · fc97220 · github-actions · Oct 18, 2024
1 parent 7cf1ea4
commit fc97220
Show file tree

Hide file tree

Showing 4 changed files with 49 additions and 4 deletions.
diff --git a/packages/rewards-calculator/src/clickhouseClient.ts b/packages/rewards-calculator/src/clickhouseClient.ts
@@ -32,6 +32,44 @@ export class ClickhouseClient {
   }
 
   public async getActiveWorkers(shouldSkipSignatureValidation = false) {
+    if (shouldSkipSignatureValidation) {
+      const columns = [
+        "worker_id",
+        "sum(num_read_chunks) as num_read_chunks",
+        "sum(output_size) as output_size",
+        "count(*) as totalRequests",
+      ];
+      await this.logTotalQueries();
+      const query = `
+        select ${columns.join(",")}
+        from ${
+          config.clickhouse.logsTableName
+        }
+        where
+          ${config.clickhouse.logsTableName}.worker_timestamp >= '${formatDate(this.from)}' and  
+          ${config.clickhouse.logsTableName}.worker_timestamp <= '${formatDate(this.to)}' and
+          (toUnixTimestamp64Micro(collector_timestamp) - toUnixTimestamp64Micro(worker_timestamp)) / 60000000 < 20
+        group by worker_id
+      `;
+      const res: any[] = await clickhouse.query(query).toPromise()
+
+
+
+      let rows = 0
+      for await (const row of res) {
+        const worker = this.workers.add(row.worker_id);
+        worker.totalRequests = row.totalRequests;
+        worker.requestsProcessed = row.totalRequests;
+        worker.bytesSent = row.output_size;
+        worker.chunksRead = row.num_read_chunks;
+        rows+= row.totalRequests
+      }
+
+      console.log('ROWS', rows)
+
+      return this.workers;
+    } 
+
     const columns = [
       "client_id",
       "worker_id",
@@ -62,10 +100,15 @@ export class ClickhouseClient {
     }.worker_timestamp >= '${formatDate(this.from)}' and ${
       config.clickhouse.logsTableName
     }.worker_timestamp <= '${formatDate(this.to)}' and timeDiff < 20 order by query_hash`;
+
+    let rows = 0
     for await (const row of clickhouse.query(query).stream()) {
+      rows++
       const worker = this.workers.add(row.worker_id);
-      await worker.processQuery(row, shouldSkipSignatureValidation);
+      await worker.processQuery(row, false);
     }
+    console.log('ROWS', rows)
+
     return this.workers;
   }
 
@@ -85,6 +128,7 @@ export class ClickhouseClient {
     for await (const row of clickhouse.query(query).stream()) {
       pings[row.worker_id] = row.timestamps;
     }
+
     return pings;
   }
 

diff --git a/packages/rewards-calculator/src/config.ts b/packages/rewards-calculator/src/config.ts
@@ -30,6 +30,7 @@ export const config = {
   rewardEpochLength: Number(env("REWARD_EPOCH_LENGTH_BLOCKS", 7000)),
   epochConfirmationBlocks: Number(env("EPOCH_CONFIRMATION_BLOCKS", 150)),
   maxEpochsPerCommit: Number(env("MAX_EPOCHS_PER_COMMIT", 1)),
+  skipSignatureValidation: env<string>("SKIP_SIGNATURE_VALIDATION", 'false') === 'true',
   clickhouse: {
     username: env("CLICKHOUSE_USERNAME", "sqd_read"),
     password: env("CLICKHOUSE_PASSWORD"),

diff --git a/packages/rewards-calculator/src/endpoints.ts b/packages/rewards-calculator/src/endpoints.ts
@@ -49,7 +49,7 @@ async function rewards(
     const _epochStats = await epochStats(
       Number(fromBlock),
       Number(toBlock),
-      true,
+      config.skipSignatureValidation,
     );
     const _duration = await duration(BigInt(fromBlock), BigInt(toBlock));
     const workerStats = _epochStats.map((worker) => ({

diff --git a/packages/rewards-calculator/src/rewardBot.ts b/packages/rewards-calculator/src/rewardBot.ts
@@ -45,7 +45,7 @@ export class RewardBot {
          * We need to calculate 2 epochs to get the correct period for the rewards
          * because of splitting the rewards to chunks
          */
-        const workers = await epochStats(fromBlock - epochLen, toBlock);
+        const workers = await epochStats(fromBlock - epochLen, toBlock, config.skipSignatureValidation);
 
         /**
          * We send to blockchain original epoch length due to a flaw in the contract
@@ -117,7 +117,7 @@ export class RewardBot {
          * We need to calculate 2 epochs to get the correct period for the rewards
          * because of splitting the rewards to chunks
          */
-        const workers = await epochStats(ranges.fromBlock - ranges.epochLen, ranges.toBlock);
+        const workers = await epochStats(ranges.fromBlock - ranges.epochLen, ranges.toBlock, config.skipSignatureValidation);
 
         /**
          * We send to blockchain original epoch length due to a flaw in the contract