modelcontextprotocol/ext-apps
Watch
1
0
Fork
You've already forked ext-apps
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'origin/main' into ochafik/readme-mcp-config

Browse source
This commit is contained in:
Olivier Chafik 2026-01-20 10:17:45 +00:00
commit 5ef3120e0c
No known key found for this signature in database
6 changed files with 2711 additions and 217 deletions
Download patch file Download diff file Expand all files Collapse all files

54
examples/shadertoy-server/README.md
View file

@ -36,6 +36,7 @@ Add to your MCP client configuration (stdio transport):
- **Real-time Rendering**: Renders GLSL shaders using WebGL 2.0
- **ShaderToy Compatibility**: Uses the standard `mainImage(out vec4 fragColor, in vec2 fragCoord)` entry point
- **Multi-pass Rendering**: Supports buffers A-D for feedback effects, blur chains, and simulations
- **Mouse & Touch Interaction**: Full iMouse support with click detection (works on mobile)
- **Standard Uniforms**: iResolution, iTime, iTimeDelta, iFrame, iMouse, iDate, iChannel0-3
## Running
@ -122,13 +123,20 @@ _Tool input:_
}
```
**Interactive Julia Set** (mouse controls the fractal's c parameter):
**Interactive Julia Set** (click and drag to control the fractal's c parameter):
```glsl
void mainImage(out vec4 fragColor, in vec2 fragCoord) {
vec2 uv = (fragCoord - 0.5 * iResolution.xy) / iResolution.y * 2.5;
vec2 mouse = (iMouse.xy / iResolution.xy - 0.5) * 2.0;
vec2 c = mouse;
// Use mouse position if clicked, otherwise use animated default
vec2 c;
if (iMouse.z > 0.0) {
// Mouse is pressed - use mouse position
c = (iMouse.xy / iResolution.xy - 0.5) * 2.0;
} else {
// Not pressed - animate around an interesting region
c = vec2(-0.8 + 0.2 * sin(iTime * 0.5), 0.156 + 0.1 * cos(iTime * 0.7));
}
vec2 z = uv;
float iter = 0.0;
for (int i = 0; i < 100; i++) {
@ -147,25 +155,33 @@ _Tool input:_
```json
{
"fragmentShader": "void mainImage(out vec4 fragColor, in vec2 fragCoord) {
vec2 uv = (fragCoord - 0.5 * iResolution.xy) / iResolution.y * 2.5;
vec2 mouse = (iMouse.xy / iResolution.xy - 0.5) * 2.0;
vec2 c = mouse;
vec2 z = uv;
float iter = 0.0;
for (int i = 0; i < 100; i++) {
z = vec2(z.x * z.x - z.y * z.y, 2.0 * z.x * z.y) + c;
if (dot(z, z) > 4.0) break;
iter++;
}
float t = iter / 100.0;
vec3 col = 0.5 + 0.5 * cos(3.0 + t * 6.28 * 2.0 + vec3(0.0, 0.6, 1.0));
if (iter == 100.0) col = vec3(0.0);
fragColor = vec4(col, 1.0);
}"
"fragmentShader": "void mainImage(out vec4 fragColor, in vec2 fragCoord) {\n vec2 uv = (fragCoord - 0.5 * iResolution.xy) / iResolution.y * 2.5;\n vec2 c;\n if (iMouse.z > 0.0) {\n c = (iMouse.xy / iResolution.xy - 0.5) * 2.0;\n } else {\n c = vec2(-0.8 + 0.2 * sin(iTime * 0.5), 0.156 + 0.1 * cos(iTime * 0.7));\n }\n vec2 z = uv;\n float iter = 0.0;\n for (int i = 0; i < 100; i++) {\n z = vec2(z.x * z.x - z.y * z.y, 2.0 * z.x * z.y) + c;\n if (dot(z, z) > 4.0) break;\n iter++;\n }\n float t = iter / 100.0;\n vec3 col = 0.5 + 0.5 * cos(3.0 + t * 6.28 * 2.0 + vec3(0.0, 0.6, 1.0));\n if (iter == 100.0) col = vec3(0.0);\n fragColor = vec4(col, 1.0);\n}"
}
```
## Mouse & Touch Interaction
The `iMouse` uniform provides interactive input, compatible with the official Shadertoy specification:
| Component | When Button Down | After Release | Never Clicked |
| ----------- | ---------------------- | ---------------- | ------------- |
| `iMouse.xy` | Current position | Last position | `(0, 0)` |
| `iMouse.zw` | Click start (positive) | Negated (-x, -y) | `(0, 0)` |
**Detecting button state:**
```glsl
if (iMouse.z > 0.0) {
// Button/touch is currently held down
} else if (iMouse.z < 0.0) {
// Button was released (can use abs(iMouse.zw) for last click position)
} else {
// Never clicked - show default state or animate
}
```
Touch events are automatically supported for mobile devices.
## Architecture
### Server (`server.ts`)

1
examples/shadertoy-server/mcp-app.html
View file

@ -9,6 +9,7 @@
<body>
<main class="main">
<canvas id="canvas"></canvas>
<pre id="code-preview"></pre>
<button id="fullscreen-btn" class="fullscreen-btn" title="Toggle fullscreen">
<svg class="expand-icon" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
<path d="M8 3H5a2 2 0 0 0-2 2v3m18 0V5a2 2 0 0 0-2-2h-3m0 18h3a2 2 0 0 0 2-2v-3M3 16v3a2 2 0 0 0 2 2h3"/>

16
examples/shadertoy-server/server.ts
View file

@ -33,10 +33,24 @@ AVAILABLE UNIFORMS:
- iTime (float): elapsed time in seconds
- iTimeDelta (float): time since last frame
- iFrame (int): frame counter
- iMouse (vec4): mouse position (xy=current, zw=click)
- iMouse (vec4): mouse/touch position in pixels (see MOUSE INTERACTION below)
- iDate (vec4): year, month, day, seconds
- iChannel0-3 (sampler2D): buffer inputs for multi-pass shaders
MOUSE INTERACTION:
The iMouse uniform provides interactive mouse/touch input (works on mobile):
- iMouse.xy: Current position while button/touch is held down (frozen on release)
- iMouse.zw: Click start position (positive when down, negative when released)
- iMouse.z > 0: Button is currently pressed
- iMouse.z < 0: Button was released
- iMouse.z == 0: Never clicked
Example - camera control:
vec2 uv = iMouse.xy / iResolution.xy; // normalized 0-1
Example - detect click:
if (iMouse.z > 0.0) { /* button is down */ }
MULTI-PASS RENDERING:
- Use bufferA-D parameters for feedback effects, blur chains, simulations
- BufferA output -> iChannel0, BufferB -> iChannel1, etc.

34
examples/shadertoy-server/src/mcp-app.css
View file

@ -11,6 +11,12 @@ html, body {
width: 100%;
height: 100%;
min-height: 400px;
border-radius: var(--border-radius-lg);
overflow: hidden;
}
.main.fullscreen {
border-radius: 0;
}
#canvas {
@ -19,6 +25,34 @@ html, body {
display: block;
}
#canvas.hidden {
display: none;
}
#code-preview {
display: none;
width: 100%;
height: 100%;
margin: 0;
padding: 16px;
box-sizing: border-box;
overflow: auto;
background: linear-gradient(
135deg,
var(--color-background-secondary) 0%,
var(--color-background-tertiary) 100%
);
color: var(--color-text-ghost);
font-family: var(--font-mono);
font-size: var(--font-text-xs-size);
line-height: var(--font-text-xs-line-height);
white-space: pre-wrap;
}
#code-preview.visible {
display: block;
}
/* Fullscreen button */
.fullscreen-btn {
position: absolute;

39
examples/shadertoy-server/src/mcp-app.ts
View file

@ -1,7 +1,12 @@
/**
* ShaderToy renderer MCP App using ShaderToyLite.js
*/
import { App, type McpUiHostContext } from "@modelcontextprotocol/ext-apps";
import {
App,
type McpUiHostContext,
applyHostStyleVariables,
applyDocumentTheme,
} from "@modelcontextprotocol/ext-apps";
import "./global.css";
import "./mcp-app.css";
import ShaderToyLite, {
@ -34,6 +39,7 @@ const log = {
// Get element references
const mainEl = document.querySelector(".main") as HTMLElement;
const canvas = document.getElementById("canvas") as HTMLCanvasElement;
const codePreview = document.getElementById("code-preview") as HTMLPreElement;
const fullscreenBtn = document.getElementById(
"fullscreen-btn",
) as HTMLButtonElement;
@ -49,8 +55,12 @@ function resizeCanvas() {
resizeCanvas();
window.addEventListener("resize", resizeCanvas);
// Handle host context changes (display mode)
// Handle host context changes (display mode, styling)
function handleHostContextChanged(ctx: McpUiHostContext) {
// Apply host styling
if (ctx.theme) applyDocumentTheme(ctx.theme);
if (ctx.styles?.variables) applyHostStyleVariables(ctx.styles.variables);
// Note: We ignore safeAreaInsets to maximize shader display area
// Show fullscreen button if available (only update if field is present)
@ -112,9 +122,22 @@ app.onteardown = async () => {
return {};
};
app.ontoolinputpartial = (params) => {
// Show code preview, hide canvas
codePreview.classList.add("visible");
canvas.classList.add("hidden");
const code = params.arguments?.fragmentShader;
codePreview.textContent = typeof code === "string" ? code : "";
codePreview.scrollTop = codePreview.scrollHeight;
};
app.ontoolinput = (params) => {
log.info("Received shader input");
// Hide code preview, show canvas
codePreview.classList.remove("visible");
canvas.classList.remove("hidden");
if (!isShaderInput(params.arguments)) {
log.error("Invalid tool input");
return;
@ -162,6 +185,18 @@ app.onerror = log.error;
app.onhostcontextchanged = handleHostContextChanged;
// Pause/resume shader based on visibility
const observer = new IntersectionObserver((entries) => {
entries.forEach((entry) => {
if (entry.isIntersecting) {
shaderToy?.play();
} else {
shaderToy?.pause();
}
});
});
observer.observe(mainEl);
// Connect to host
app.connect().then(() => {
log.info("Connected to host");

71
examples/shadertoy-server/src/vendor/ShaderToyLite.js vendored
View file

@ -90,7 +90,10 @@ function ShaderToyLite(canvasId) {
// uniforms
var iFrame = 0;
// iMouse state: xy = current position (only updates when button down)
// zw = click start position (positive when down, negative when released)
var iMouse = {x: 0, y: 0, clickX: 0, clickY: 0};
var isMouseDown = false;
// shader common source
var common = "";
@ -145,19 +148,75 @@ function ShaderToyLite(canvasId) {
gl.viewport(0, 0, gl.canvas.width, gl.canvas.height);
});
// Mouse event handlers - match official Shadertoy iMouse behavior:
// iMouse.xy: current position (only updates when button is down)
// iMouse.zw: click position (positive when down, negated when released)
canvas.addEventListener("mousemove", (event) => {
iMouse.x = event.offsetX;
iMouse.y = canvas.height - event.offsetY;
if (isMouseDown) {
iMouse.x = event.offsetX;
iMouse.y = canvas.height - event.offsetY;
}
});
canvas.addEventListener("mousedown", (event) => {
iMouse.clickX = event.offsetX;
iMouse.clickY = canvas.height - event.offsetY;
isMouseDown = true;
var x = event.offsetX;
var y = canvas.height - event.offsetY;
iMouse.x = x;
iMouse.y = y;
iMouse.clickX = x;
iMouse.clickY = y;
});
canvas.addEventListener("mouseup", () => {
iMouse.clickX = 0;
iMouse.clickY = 0;
isMouseDown = false;
// Negate click position to indicate button released (Shadertoy convention)
iMouse.clickX = -Math.abs(iMouse.clickX);
iMouse.clickY = -Math.abs(iMouse.clickY);
});
// Prevent context menu on right-click to avoid interrupting interaction
canvas.addEventListener("contextmenu", (event) => {
event.preventDefault();
});
// Touch support for mobile devices
canvas.addEventListener("touchstart", (event) => {
event.preventDefault();
if (event.touches.length > 0) {
var touch = event.touches[0];
var rect = canvas.getBoundingClientRect();
var x = touch.clientX - rect.left;
var y = canvas.height - (touch.clientY - rect.top);
isMouseDown = true;
iMouse.x = x;
iMouse.y = y;
iMouse.clickX = x;
iMouse.clickY = y;
}
}, { passive: false });
canvas.addEventListener("touchmove", (event) => {
event.preventDefault();
if (isMouseDown && event.touches.length > 0) {
var touch = event.touches[0];
var rect = canvas.getBoundingClientRect();
iMouse.x = touch.clientX - rect.left;
iMouse.y = canvas.height - (touch.clientY - rect.top);
}
}, { passive: false });
canvas.addEventListener("touchend", (event) => {
event.preventDefault();
isMouseDown = false;
iMouse.clickX = -Math.abs(iMouse.clickX);
iMouse.clickY = -Math.abs(iMouse.clickY);
}, { passive: false });
canvas.addEventListener("touchcancel", () => {
isMouseDown = false;
iMouse.clickX = -Math.abs(iMouse.clickX);
iMouse.clickY = -Math.abs(iMouse.clickY);
});
}