modelcontextprotocol/servers
Watch
1
0
Fork
You've already forked servers
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #144 from SkywaveTM/main

Browse source 
fix(fetch): properly handles robots.txt
This commit is contained in:
Justin Spahr-Summers 2024-12-03 07:18:33 -06:00 • committed by GitHub
commit d1650ce5fb
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 4 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

6
src/fetch/src/mcp_server_fetch/server.py
View file

@ -74,7 +74,9 @@ async def check_may_autonomously_fetch_url(url: str, user_agent: str) -> None:
async with AsyncClient() as client: async with AsyncClient() as client:
try: try:
response = await client.get( response = await client.get(
robot_txt_url, headers={"User-Agent": user_agent} robot_txt_url,
follow_redirects=True,
headers={"User-Agent": user_agent},
) )
except HTTPError: except HTTPError:
raise McpError( raise McpError(
@ -93,7 +95,7 @@ async def check_may_autonomously_fetch_url(url: str, user_agent: str) -> None:
line for line in robot_txt.splitlines() if not line.strip().startswith("#") line for line in robot_txt.splitlines() if not line.strip().startswith("#")
) )
robot_parser = Protego.parse(processed_robot_txt) robot_parser = Protego.parse(processed_robot_txt)
if not robot_parser.can_fetch(url, user_agent): if not robot_parser.can_fetch(str(url), user_agent):
raise McpError( raise McpError(
INTERNAL_ERROR, INTERNAL_ERROR,
f"The sites robots.txt ({robot_txt_url}), specifies that autonomous fetching of this page is not allowed, " f"The sites robots.txt ({robot_txt_url}), specifies that autonomous fetching of this page is not allowed, "