agent,common: Move subgraph health safety check to reconcileActions

- Previously the safety check occurred later in the process. We should
identify early that a subgraph is failed and safety = true to not
add the action to the queue in the first place. The other benefit
of moving this check is so it has access to the corresponding rule
so the safety property can be used here for control.

packages/indexer-agent/src/agent.ts

@@ -1073,8 +1073,15 @@ export class Agent {
           epoch,
         )
       case true: {
-        // If no active allocations, create one
-        if (activeDeploymentAllocations.length === 0) {
+        // If no active allocations and subgraph health passes safety check, create one
+        const indexingStatus = await this.graphNode.indexingStatus([deploymentAllocationDecision.deployment])
+        const failsHealthCheck = indexingStatus && indexingStatus.health == 'failed' && deploymentAllocationDecision.ruleMatch.rule?.safety
+        if (failsHealthCheck) {
+          logger.warn('Subgraph deployment has failed health check, skipping allocate', {
+            indexingStatus,
+            safety: deploymentAllocationDecision.ruleMatch.rule?.safety
+          })
+        } else if (activeDeploymentAllocations.length === 0) {
           // Fetch the latest closed allocation, if any
           const mostRecentlyClosedAllocation = (
             await network.networkMonitor.closedAllocations(

packages/indexer-common/src/indexer-management/allocations.ts

@@ -356,12 +356,6 @@ export class AllocationManager {
         `Subgraph deployment, '${deployment.ipfsHash}', is not syncing`,
       )
     }
-    if (status && status.health == 'failed') {
-      throw indexerError(
-        IndexerErrorCode.IE020,
-        `Subgraph deployment, '${deployment.ipfsHash}', failed during syncing`,
-      )
-    }
 
     logger.debug('Obtain a unique Allocation ID')
     const { allocationSigner, allocationId } = uniqueAllocationID(