modelcontextprotocol/servers-archived
Watch
1
0
Fork
You've already forked servers-archived
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

make the fetch mcp-server obey robots.txt

Browse source
This commit is contained in:
Jack Adamson 2024-11-22 18:51:39 +00:00
commit 84321c89cf
No known key found for this signature in database
1 changed files with 85 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

3
src/fetch/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "mcp-server-fetch"
version = "0.1.1"
version = "0.1.2"
description = "A Model Context Protocol server providing tools to fetch and convert web content for usage by LLMs"
readme = "README.md"
requires-python = ">=3.10"
@ -18,6 +18,7 @@ classifiers = [
dependencies = [
"markdownify>=0.13.1",
"mcp>=0.6.0",
"protego>=0.3.1",
"pydantic>=2.0.0",
"readabilipy>=0.2.0",
"requests>=2.32.3",