follows redirects on checking robots.txt

2024-11-30 23:31:30 +09:00 · 2024-11-30 23:31:30 +09:00 · d5bae8759f
commit d5bae8759f
parent 168522f34d
1 changed files with 3 additions and 1 deletions
--- a/src/fetch/src/mcp_server_fetch/server.py
+++ b/src/fetch/src/mcp_server_fetch/server.py
@ -74,7 +74,9 @@ async def check_may_autonomously_fetch_url(url: AnyUrl | str, user_agent: str) -
    async with AsyncClient() as client:
        try:
            response = await client.get(
-                robot_txt_url, headers={"User-Agent": user_agent}
+                robot_txt_url,
+                follow_redirects=True,
+                headers={"User-Agent": user_agent},
            )
        except HTTPError:
            raise McpError(