Merge pull request #297 from modelcontextprotocol/feat/shadertoy-streaming-visibility
feat(shadertoy): add streaming partial input, visibility optimization, host styling
This commit is contained in:
Anton Pidkuiko
GitHub
commit
73ab853dc5
4 changed files with 187 additions and 3 deletions
|
|
@ -9,6 +9,7 @@
|
|||
<body>
|
||||
<main class="main">
|
||||
<canvas id="canvas"></canvas>
|
||||
<pre id="code-preview"></pre>
|
||||
<button id="fullscreen-btn" class="fullscreen-btn" title="Toggle fullscreen">
|
||||
<svg class="expand-icon" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
|
||||
<path d="M8 3H5a2 2 0 0 0-2 2v3m18 0V5a2 2 0 0 0-2-2h-3m0 18h3a2 2 0 0 0 2-2v-3M3 16v3a2 2 0 0 0 2 2h3"/>
|
||||
|
|
|
|||
|
|
@ -11,6 +11,12 @@ html, body {
|
|||
width: 100%;
|
||||
height: 100%;
|
||||
min-height: 400px;
|
||||
border-radius: var(--border-radius-lg);
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.main.fullscreen {
|
||||
border-radius: 0;
|
||||
}
|
||||
|
||||
#canvas {
|
||||
|
|
@ -19,6 +25,34 @@ html, body {
|
|||
display: block;
|
||||
}
|
||||
|
||||
#canvas.hidden {
|
||||
display: none;
|
||||
}
|
||||
|
||||
#code-preview {
|
||||
display: none;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
margin: 0;
|
||||
padding: 16px;
|
||||
box-sizing: border-box;
|
||||
overflow: auto;
|
||||
background: linear-gradient(
|
||||
135deg,
|
||||
var(--color-background-secondary) 0%,
|
||||
var(--color-background-tertiary) 100%
|
||||
);
|
||||
color: var(--color-text-ghost);
|
||||
font-family: var(--font-mono);
|
||||
font-size: var(--font-text-xs-size);
|
||||
line-height: var(--font-text-xs-line-height);
|
||||
white-space: pre-wrap;
|
||||
}
|
||||
|
||||
#code-preview.visible {
|
||||
display: block;
|
||||
}
|
||||
|
||||
/* Fullscreen button */
|
||||
.fullscreen-btn {
|
||||
position: absolute;
|
||||
|
|
|
|||
|
|
@ -1,7 +1,12 @@
|
|||
/**
|
||||
* ShaderToy renderer MCP App using ShaderToyLite.js
|
||||
*/
|
||||
import { App, type McpUiHostContext } from "@modelcontextprotocol/ext-apps";
|
||||
import {
|
||||
App,
|
||||
type McpUiHostContext,
|
||||
applyHostStyleVariables,
|
||||
applyDocumentTheme,
|
||||
} from "@modelcontextprotocol/ext-apps";
|
||||
import "./global.css";
|
||||
import "./mcp-app.css";
|
||||
import ShaderToyLite, {
|
||||
|
|
@ -34,6 +39,7 @@ const log = {
|
|||
// Get element references
|
||||
const mainEl = document.querySelector(".main") as HTMLElement;
|
||||
const canvas = document.getElementById("canvas") as HTMLCanvasElement;
|
||||
const codePreview = document.getElementById("code-preview") as HTMLPreElement;
|
||||
const fullscreenBtn = document.getElementById(
|
||||
"fullscreen-btn",
|
||||
) as HTMLButtonElement;
|
||||
|
|
@ -49,8 +55,12 @@ function resizeCanvas() {
|
|||
resizeCanvas();
|
||||
window.addEventListener("resize", resizeCanvas);
|
||||
|
||||
// Handle host context changes (display mode)
|
||||
// Handle host context changes (display mode, styling)
|
||||
function handleHostContextChanged(ctx: McpUiHostContext) {
|
||||
// Apply host styling
|
||||
if (ctx.theme) applyDocumentTheme(ctx.theme);
|
||||
if (ctx.styles?.variables) applyHostStyleVariables(ctx.styles.variables);
|
||||
|
||||
// Note: We ignore safeAreaInsets to maximize shader display area
|
||||
|
||||
// Show fullscreen button if available (only update if field is present)
|
||||
|
|
@ -112,9 +122,22 @@ app.onteardown = async () => {
|
|||
return {};
|
||||
};
|
||||
|
||||
app.ontoolinputpartial = (params) => {
|
||||
// Show code preview, hide canvas
|
||||
codePreview.classList.add("visible");
|
||||
canvas.classList.add("hidden");
|
||||
const code = params.arguments?.fragmentShader;
|
||||
codePreview.textContent = typeof code === "string" ? code : "";
|
||||
codePreview.scrollTop = codePreview.scrollHeight;
|
||||
};
|
||||
|
||||
app.ontoolinput = (params) => {
|
||||
log.info("Received shader input");
|
||||
|
||||
// Hide code preview, show canvas
|
||||
codePreview.classList.remove("visible");
|
||||
canvas.classList.remove("hidden");
|
||||
|
||||
if (!isShaderInput(params.arguments)) {
|
||||
log.error("Invalid tool input");
|
||||
return;
|
||||
|
|
@ -162,6 +185,18 @@ app.onerror = log.error;
|
|||
|
||||
app.onhostcontextchanged = handleHostContextChanged;
|
||||
|
||||
// Pause/resume shader based on visibility
|
||||
const observer = new IntersectionObserver((entries) => {
|
||||
entries.forEach((entry) => {
|
||||
if (entry.isIntersecting) {
|
||||
shaderToy?.play();
|
||||
} else {
|
||||
shaderToy?.pause();
|
||||
}
|
||||
});
|
||||
});
|
||||
observer.observe(mainEl);
|
||||
|
||||
// Connect to host
|
||||
app.connect().then(() => {
|
||||
log.info("Connected to host");
|
||||
|
|
|
|||
|
|
@ -86,6 +86,7 @@ Read JSDoc documentation directly from source:
|
|||
|
||||
| Example | Pattern Demonstrated |
|
||||
|---------|---------------------|
|
||||
| `examples/shadertoy-server/` | **Streaming partial input** + visibility-based pause/play (best practice for large inputs) |
|
||||
| `examples/wiki-explorer-server/` | `callServerTool` for interactive data fetching |
|
||||
| `examples/system-monitor-server/` | Polling pattern with interval management |
|
||||
| `examples/video-resource-server/` | Binary/blob resources |
|
||||
|
|
@ -173,9 +174,31 @@ app.onhostcontextchanged = (ctx) => {
|
|||
import { useApp, useHostStyles } from "@modelcontextprotocol/ext-apps/react";
|
||||
|
||||
const { app } = useApp({ appInfo, capabilities, onAppCreated });
|
||||
useHostStyles(app); // Handles theme, styles, fonts automatically
|
||||
useHostStyles(app); // Injects CSS variables to document, making var(--*) available
|
||||
```
|
||||
|
||||
**Using variables in CSS** - After applying, use `var()`:
|
||||
```css
|
||||
.container {
|
||||
background: var(--color-background-secondary);
|
||||
color: var(--color-text-primary);
|
||||
font-family: var(--font-sans);
|
||||
border-radius: var(--border-radius-md);
|
||||
}
|
||||
.code {
|
||||
font-family: var(--font-mono);
|
||||
font-size: var(--font-text-sm-size);
|
||||
line-height: var(--font-text-sm-line-height);
|
||||
color: var(--color-text-secondary);
|
||||
}
|
||||
.heading {
|
||||
font-size: var(--font-heading-lg-size);
|
||||
font-weight: var(--font-weight-semibold);
|
||||
}
|
||||
```
|
||||
|
||||
Key variable groups: `--color-background-*`, `--color-text-*`, `--color-border-*`, `--font-sans`, `--font-mono`, `--font-text-*-size`, `--font-heading-*-size`, `--border-radius-*`. See `src/spec.types.ts` for full list.
|
||||
|
||||
### Safe Area Handling
|
||||
|
||||
Always respect `safeAreaInsets`:
|
||||
|
|
@ -189,6 +212,96 @@ app.onhostcontextchanged = (ctx) => {
|
|||
};
|
||||
```
|
||||
|
||||
### Streaming Partial Input
|
||||
|
||||
For large tool inputs, use `ontoolinputpartial` to show progress during LLM generation. The partial JSON is healed (always valid), enabling progressive UI updates.
|
||||
|
||||
**Spec:** [ui/notifications/tool-input-partial](https://github.com/modelcontextprotocol/ext-apps/blob/main/specification/draft/apps.mdx#streaming-tool-input)
|
||||
|
||||
```typescript
|
||||
app.ontoolinputpartial = (params) => {
|
||||
const args = params.arguments; // Healed partial JSON - always valid, fields appear as generated
|
||||
// Use args directly for progressive rendering
|
||||
};
|
||||
|
||||
app.ontoolinput = (params) => {
|
||||
// Final complete input - switch from preview to full render
|
||||
};
|
||||
```
|
||||
|
||||
**Use cases:**
|
||||
| Pattern | Example |
|
||||
|---------|---------|
|
||||
| Code preview | Show streaming code in `<pre>`, render on complete (`examples/shadertoy-server/`) |
|
||||
| Progressive form | Fill form fields as they stream in |
|
||||
| Live chart | Add data points to chart as array grows |
|
||||
| Partial render | Render incomplete structured data (tables, lists, trees) |
|
||||
|
||||
**Simple pattern (code preview):**
|
||||
```typescript
|
||||
app.ontoolinputpartial = (params) => {
|
||||
codePreview.textContent = params.arguments?.code ?? "";
|
||||
codePreview.style.display = "block";
|
||||
canvas.style.display = "none";
|
||||
};
|
||||
app.ontoolinput = (params) => {
|
||||
codePreview.style.display = "none";
|
||||
canvas.style.display = "block";
|
||||
render(params.arguments);
|
||||
};
|
||||
```
|
||||
|
||||
### Visibility-Based Resource Management
|
||||
|
||||
Pause expensive operations (animations, WebGL, polling) when widget scrolls out of viewport:
|
||||
|
||||
```typescript
|
||||
const observer = new IntersectionObserver((entries) => {
|
||||
entries.forEach((entry) => {
|
||||
if (entry.isIntersecting) {
|
||||
animation.play(); // or: startPolling(), shaderToy.play()
|
||||
} else {
|
||||
animation.pause(); // or: stopPolling(), shaderToy.pause()
|
||||
}
|
||||
});
|
||||
});
|
||||
observer.observe(document.querySelector(".main"));
|
||||
```
|
||||
|
||||
### Fullscreen Mode
|
||||
|
||||
Request fullscreen via `app.requestDisplayMode()`. Check availability in host context:
|
||||
|
||||
```typescript
|
||||
let currentMode: "inline" | "fullscreen" = "inline";
|
||||
|
||||
app.onhostcontextchanged = (ctx) => {
|
||||
// Check if fullscreen available
|
||||
if (ctx.availableDisplayModes?.includes("fullscreen")) {
|
||||
fullscreenBtn.style.display = "block";
|
||||
}
|
||||
// Track current mode
|
||||
if (ctx.displayMode) {
|
||||
currentMode = ctx.displayMode;
|
||||
container.classList.toggle("fullscreen", currentMode === "fullscreen");
|
||||
}
|
||||
};
|
||||
|
||||
async function toggleFullscreen() {
|
||||
const newMode = currentMode === "fullscreen" ? "inline" : "fullscreen";
|
||||
const result = await app.requestDisplayMode({ mode: newMode });
|
||||
currentMode = result.mode;
|
||||
}
|
||||
```
|
||||
|
||||
**CSS pattern** - Remove border radius in fullscreen:
|
||||
```css
|
||||
.main { border-radius: var(--border-radius-lg); overflow: hidden; }
|
||||
.main.fullscreen { border-radius: 0; }
|
||||
```
|
||||
|
||||
See `examples/shadertoy-server/` for complete implementation.
|
||||
|
||||
## Common Mistakes to Avoid
|
||||
|
||||
1. **Handlers after connect()** - Register ALL handlers BEFORE calling `app.connect()`
|
||||
|
|
@ -198,6 +311,7 @@ app.onhostcontextchanged = (ctx) => {
|
|||
5. **Ignoring safe area insets** - Always handle `ctx.safeAreaInsets`
|
||||
6. **No text fallback** - Always provide `content` array for non-UI hosts
|
||||
7. **Hardcoded styles** - Use host CSS variables for theme integration
|
||||
8. **No streaming for large inputs** - Use `ontoolinputpartial` to show progress during generation
|
||||
|
||||
## Testing
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue