frag-backend-routing.xml

<fragment>
    <retry condition="@(context.Response != null && (context.Response.StatusCode == 429 || context.Response.StatusCode >= 500) && ((Int32)context.Variables["remainingRoutes"]) > 0)" count="3" interval="0">
        <!-- Before picking the route, let's verify if there is any that should be set to not throttling anymore -->
        <set-variable name="routes" value="@{
                JArray routes = (JArray)context.Variables["routes"];
                
                for (int i = 0; i < routes.Count; i++)
                {
                    JObject route = (JObject)routes[i];

                    if (route.Value<bool>("isThrottling") && DateTime.Now >= route.Value<DateTime>("retryAfter"))
                    {
                        route["isThrottling"] = false;
                        route["retryAfter"] = DateTime.MinValue;
                    }
                }

                return routes; 
            }" />
        <cache-store-value key="@((string)context.Variables.GetValueOrDefault<string>("routesCacheKey", "ALL-ROUTES"))" value="@((JArray)context.Variables["routes"])" duration="86400" />
        <!-- This is the main logic to pick the route to be used -->
        <set-variable name="routeIndex" value="@{
                JArray routes = (JArray)context.Variables["routes"];
                int selectedPriority = Int32.MaxValue;
                List<int> availableRoutesIndexes = new List<int>();

                for (int i = 0; i < routes.Count; i++)
                {
                    JObject route = (JObject)routes[i];

                    if (!route.Value<bool>("isThrottling"))
                    {
                        int routePriority = route.Value<int>("priority");

                        if (routePriority < selectedPriority)
                        {
                            selectedPriority = routePriority;
                            availableRoutesIndexes.Clear();
                            availableRoutesIndexes.Add(i);
                        } 
                        else if (routePriority == selectedPriority)
                        {
                            availableRoutesIndexes.Add(i);
                        }
                    }
                }

                if (availableRoutesIndexes.Count == 1)
                {
                    return availableRoutesIndexes[0];
                }
            
                if (availableRoutesIndexes.Count > 0)
                {
                    //Returns a random route from the list if we have more than one available with the same priority
                    return availableRoutesIndexes[new Random().Next(0, availableRoutesIndexes.Count)];
                }
                else
                {
                    //If there are no available routes, the request will be sent to the first one
                    return 0;    
                }
                }" />
        <set-variable name="backendId" value="@(((JObject)((JArray)context.Variables["routes"])[(Int32)context.Variables["routeIndex"]]).Value<string>("backend-id"))" />
        <set-variable name="routeLocation" value="@(((JObject)((JArray)context.Variables["routes"])[(Int32)context.Variables["routeIndex"]]).Value<string>("location"))" />
        <set-variable name="routeName" value="@(((JObject)((JArray)context.Variables["routes"])[(Int32)context.Variables["routeIndex"]]).Value<string>("name"))" />
        <set-variable name="deploymentName" value="@((string)context.Variables["deployment-id"])" />
        <set-backend-service backend-id="@((string)context.Variables["backendId"])" />
        <choose>
            <when condition="@(context.Variables.GetValueOrDefault<string>("isStream", "false") == "true")">
                <forward-request buffer-request-body="false" />
            </when>
            <otherwise>
                <forward-request buffer-request-body="true" />
            </otherwise>
        </choose>
        <choose>
            <!-- In case we got 429 or 5xx from a route, update the list with its status -->
            <when condition="@(context.Response != null && (context.Response.StatusCode == 429 || context.Response.StatusCode >= 500) )">
                <cache-lookup-value key="@(context.Request.MatchedParameters["deployment-id"] + "Routes" + context.Deployment.Region + context.Api.Revision)" variable-name="routes" />
                <set-variable name="routes" value="@{
                        JArray routes = (JArray)context.Variables["routes"];
                        int currentrouteIndex = context.Variables.GetValueOrDefault<int>("routeIndex");
                        int retryAfter = Convert.ToInt32(context.Response.Headers.GetValueOrDefault("Retry-After", "-1"));

                        if (retryAfter == -1)
                        {
                            retryAfter = Convert.ToInt32(context.Response.Headers.GetValueOrDefault("x-ratelimit-reset-requests", "-1"));
                        }

                        if (retryAfter == -1)
                        {
                            retryAfter = Convert.ToInt32(context.Response.Headers.GetValueOrDefault("x-ratelimit-reset-tokens", "10"));
                        }

                        JObject route = (JObject)routes[currentrouteIndex];
                        route["isThrottling"] = true;
                        route["retryAfter"] = DateTime.Now.AddSeconds(retryAfter);

                        return routes;      
                    }" />
                <cache-store-value key="@((string)context.Variables.GetValueOrDefault<string>("routesCacheKey", "ALL-ROUTES"))" value="@((JArray)context.Variables["routes"])" duration="86400" />
                <set-variable name="remainingRoutes" value="@{
                        JArray routes = (JArray)context.Variables["routes"];

                        int remainingRoutes = 0;

                        for (int i = 0; i < routes.Count; i++)
                        {
                            JObject route = (JObject)routes[i];

                            if (!route.Value<bool>("isThrottling"))
                            {
                                remainingRoutes++;
                            }
                        }

                        return remainingRoutes;
                    }" />
            </when>
        </choose>
    </retry>
</fragment>