Ignore version mismatches when pushing

This prevents the background worker from getting stuck when a remote backend is running an incompatible or broken instance.
wireapp · Mar 4, 2024 · 4da4075 · 4da4075
1 parent 4674b57
commit 4da4075
Show file tree

Hide file tree

Showing 2 changed files with 22 additions and 2 deletions.
diff --git a/services/background-worker/background-worker.cabal b/services/background-worker/background-worker.cabal
@@ -50,6 +50,7 @@ library
     , types-common
     , unliftio
     , wai-utilities
+    , wire-api
     , wire-api-federation
 
   default-extensions:

diff --git a/services/background-worker/src/Wire/BackendNotificationPusher.hs b/services/background-worker/src/Wire/BackendNotificationPusher.hs
@@ -24,7 +24,9 @@ import UnliftIO
 import Wire.API.Federation.API
 import Wire.API.Federation.BackendNotifications
 import Wire.API.Federation.Client
+import Wire.API.Federation.Error
 import Wire.API.Federation.Version
+import Wire.API.RawJson
 import Wire.BackgroundWorker.Env
 import Wire.BackgroundWorker.Options
 import Wire.BackgroundWorker.Util
@@ -116,7 +118,7 @@ pushNotification runningFlag targetDomain (msg, envelope) = do
                 cveEnv = FederatorClientEnv {..}
                 cveVersion = Just V0 -- V0 is assumed for non-versioned queue messages
                 fcEnv = FederatorClientVersionedEnv {..}
-            liftIO $ either throwM pure =<< sendNotification fcEnv notif.targetComponent notif.path notif.body
+            sendNotificationIgnoringVersionMismatch fcEnv notif.targetComponent notif.path notif.body
             lift $ ack envelope
             metrics <- asks backendNotificationMetrics
             withLabel metrics.pushedCounter (domainText targetDomain) incCounter
@@ -165,12 +167,29 @@ pushNotification runningFlag targetDomain (msg, envelope) = do
                 ceOriginRequestId = fromMaybe (RequestId "N/A") notif.requestId
                 cveEnv = FederatorClientEnv {..}
                 fcEnv = FederatorClientVersionedEnv {..}
-            liftIO $ either throwM pure =<< sendNotification fcEnv notif.targetComponent notif.path notif.body
+            sendNotificationIgnoringVersionMismatch fcEnv notif.targetComponent notif.path notif.body
             lift $ ack envelope
             metrics <- asks backendNotificationMetrics
             withLabel metrics.pushedCounter (domainText targetDomain) incCounter
             withLabel metrics.stuckQueuesGauge (domainText targetDomain) (flip setGauge 0)
 
+sendNotificationIgnoringVersionMismatch ::
+  FederatorClientVersionedEnv ->
+  Component ->
+  Text ->
+  RawJson ->
+  AppT IO ()
+sendNotificationIgnoringVersionMismatch env comp path body =
+  liftIO (sendNotification env comp path body) >>= \case
+    Left (FederatorClientVersionNegotiationError v) -> do
+      Log.fatal $
+        Log.msg (Log.val "Federator version negotiation error")
+          . Log.field "domain" (domainText env.cveEnv.ceTargetDomain)
+          . Log.field "error" (show v)
+      pure ()
+    Left e -> throwM e
+    Right () -> pure ()
+
 -- | Find the pair that maximises b.
 pairedMaximumOn :: Ord b => (a -> b) -> [a] -> (a, b)
 pairedMaximumOn f = maximumBy (compare `on` snd) . map (id &&& f)