fix: Remove follow_redirects override in HttpxHttpClient (#1015)

### Description

Allow users to disable redirect following for HttpxHttpClient.

### Issues

- Closes: #1013 

### Testing

After removing `follow_redirects=True` a couple of tests actually
failed, updating default `kwargs` fixed them.

### Checklist

- [ ] CI passed

src/crawlee/http_clients/_httpx.py

@@ -163,7 +163,7 @@ async def crawl(
         )
 
         try:
-            response = await client.send(http_request, follow_redirects=True)
+            response = await client.send(http_request)
         except httpx.TransportError as exc:
             if self._is_proxy_error(exc):
                 raise ProxyError from exc
@@ -235,6 +235,7 @@ def _get_client(self, proxy_url: str | None) -> httpx.AsyncClient:
                 'proxy': proxy_url,
                 'http1': self._http1,
                 'http2': self._http2,
+                'follow_redirects': True,
             }
 
             # Update the default kwargs with any additional user-provided kwargs.

tests/unit/http_clients/test_httpx.py

@@ -106,3 +106,28 @@ async def test_common_headers_and_user_agent(httpbin: URL, header_network: dict)
     assert 'User-Agent' in response_headers
     assert 'python-httpx' not in response_headers['User-Agent']
     assert response_headers['User-Agent'] in get_available_header_values(header_network, {'User-Agent', 'user-agent'})
+
+
+async def test_crawl_follow_redirects_by_default(http_client: HttpxHttpClient, httpbin: URL) -> None:
+    final_url = str(httpbin / 'get')
+    redirect_url = str((httpbin / 'redirect-to').with_query(url=final_url))
+    request = Request.from_url(redirect_url)
+
+    crawling_result = await http_client.crawl(request)
+
+    assert crawling_result.http_response.status_code == 200
+    assert request.loaded_url == final_url
+
+
+async def test_crawl_follow_redirects_false(httpbin: URL) -> None:
+    http_client = HttpxHttpClient(follow_redirects=False)
+
+    final_url = str(httpbin / 'get')
+    redirect_url = str((httpbin / 'redirect-to').with_query(url=final_url))
+    request = Request.from_url(redirect_url)
+
+    crawling_result = await http_client.crawl(request)
+
+    assert crawling_result.http_response.status_code == 302
+    assert crawling_result.http_response.headers['Location'] == final_url
+    assert request.loaded_url == redirect_url