feat(shadertoy): add streaming partial input, visibility optimization, host styling
Shadertoy server improvements: - Add ontoolinputpartial handler to show code preview during generation - Add IntersectionObserver to pause/play shader based on visibility - Apply host style variables (theme, colors, fonts) - Add border-radius with fullscreen mode support - Use gradient background and ghost text color for code preview Skill documentation updates: - Add streaming partial input section with use cases - Add visibility-based resource management pattern - Add fullscreen mode API and CSS patterns - Expand host styling with CSS variable usage examples
This commit is contained in:
parent
8c3b1daf7e
commit
f1a00f9e15
4 changed files with 187 additions and 3 deletions
|
|
@ -9,6 +9,7 @@
|
||||||
<body>
|
<body>
|
||||||
<main class="main">
|
<main class="main">
|
||||||
<canvas id="canvas"></canvas>
|
<canvas id="canvas"></canvas>
|
||||||
|
<pre id="code-preview"></pre>
|
||||||
<button id="fullscreen-btn" class="fullscreen-btn" title="Toggle fullscreen">
|
<button id="fullscreen-btn" class="fullscreen-btn" title="Toggle fullscreen">
|
||||||
<svg class="expand-icon" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
|
<svg class="expand-icon" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
|
||||||
<path d="M8 3H5a2 2 0 0 0-2 2v3m18 0V5a2 2 0 0 0-2-2h-3m0 18h3a2 2 0 0 0 2-2v-3M3 16v3a2 2 0 0 0 2 2h3"/>
|
<path d="M8 3H5a2 2 0 0 0-2 2v3m18 0V5a2 2 0 0 0-2-2h-3m0 18h3a2 2 0 0 0 2-2v-3M3 16v3a2 2 0 0 0 2 2h3"/>
|
||||||
|
|
|
||||||
|
|
@ -11,6 +11,12 @@ html, body {
|
||||||
width: 100%;
|
width: 100%;
|
||||||
height: 100%;
|
height: 100%;
|
||||||
min-height: 400px;
|
min-height: 400px;
|
||||||
|
border-radius: var(--border-radius-lg);
|
||||||
|
overflow: hidden;
|
||||||
|
}
|
||||||
|
|
||||||
|
.main.fullscreen {
|
||||||
|
border-radius: 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
#canvas {
|
#canvas {
|
||||||
|
|
@ -19,6 +25,34 @@ html, body {
|
||||||
display: block;
|
display: block;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#canvas.hidden {
|
||||||
|
display: none;
|
||||||
|
}
|
||||||
|
|
||||||
|
#code-preview {
|
||||||
|
display: none;
|
||||||
|
width: 100%;
|
||||||
|
height: 100%;
|
||||||
|
margin: 0;
|
||||||
|
padding: 16px;
|
||||||
|
box-sizing: border-box;
|
||||||
|
overflow: auto;
|
||||||
|
background: linear-gradient(
|
||||||
|
135deg,
|
||||||
|
var(--color-background-secondary) 0%,
|
||||||
|
var(--color-background-tertiary) 100%
|
||||||
|
);
|
||||||
|
color: var(--color-text-ghost);
|
||||||
|
font-family: var(--font-mono);
|
||||||
|
font-size: var(--font-text-xs-size);
|
||||||
|
line-height: var(--font-text-xs-line-height);
|
||||||
|
white-space: pre-wrap;
|
||||||
|
}
|
||||||
|
|
||||||
|
#code-preview.visible {
|
||||||
|
display: block;
|
||||||
|
}
|
||||||
|
|
||||||
/* Fullscreen button */
|
/* Fullscreen button */
|
||||||
.fullscreen-btn {
|
.fullscreen-btn {
|
||||||
position: absolute;
|
position: absolute;
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,12 @@
|
||||||
/**
|
/**
|
||||||
* ShaderToy renderer MCP App using ShaderToyLite.js
|
* ShaderToy renderer MCP App using ShaderToyLite.js
|
||||||
*/
|
*/
|
||||||
import { App, type McpUiHostContext } from "@modelcontextprotocol/ext-apps";
|
import {
|
||||||
|
App,
|
||||||
|
type McpUiHostContext,
|
||||||
|
applyHostStyleVariables,
|
||||||
|
applyDocumentTheme,
|
||||||
|
} from "@modelcontextprotocol/ext-apps";
|
||||||
import "./global.css";
|
import "./global.css";
|
||||||
import "./mcp-app.css";
|
import "./mcp-app.css";
|
||||||
import ShaderToyLite, {
|
import ShaderToyLite, {
|
||||||
|
|
@ -34,6 +39,7 @@ const log = {
|
||||||
// Get element references
|
// Get element references
|
||||||
const mainEl = document.querySelector(".main") as HTMLElement;
|
const mainEl = document.querySelector(".main") as HTMLElement;
|
||||||
const canvas = document.getElementById("canvas") as HTMLCanvasElement;
|
const canvas = document.getElementById("canvas") as HTMLCanvasElement;
|
||||||
|
const codePreview = document.getElementById("code-preview") as HTMLPreElement;
|
||||||
const fullscreenBtn = document.getElementById(
|
const fullscreenBtn = document.getElementById(
|
||||||
"fullscreen-btn",
|
"fullscreen-btn",
|
||||||
) as HTMLButtonElement;
|
) as HTMLButtonElement;
|
||||||
|
|
@ -49,8 +55,12 @@ function resizeCanvas() {
|
||||||
resizeCanvas();
|
resizeCanvas();
|
||||||
window.addEventListener("resize", resizeCanvas);
|
window.addEventListener("resize", resizeCanvas);
|
||||||
|
|
||||||
// Handle host context changes (display mode)
|
// Handle host context changes (display mode, styling)
|
||||||
function handleHostContextChanged(ctx: McpUiHostContext) {
|
function handleHostContextChanged(ctx: McpUiHostContext) {
|
||||||
|
// Apply host styling
|
||||||
|
if (ctx.theme) applyDocumentTheme(ctx.theme);
|
||||||
|
if (ctx.styles?.variables) applyHostStyleVariables(ctx.styles.variables);
|
||||||
|
|
||||||
// Note: We ignore safeAreaInsets to maximize shader display area
|
// Note: We ignore safeAreaInsets to maximize shader display area
|
||||||
|
|
||||||
// Show fullscreen button if available (only update if field is present)
|
// Show fullscreen button if available (only update if field is present)
|
||||||
|
|
@ -112,9 +122,22 @@ app.onteardown = async () => {
|
||||||
return {};
|
return {};
|
||||||
};
|
};
|
||||||
|
|
||||||
|
app.ontoolinputpartial = (params) => {
|
||||||
|
// Show code preview, hide canvas
|
||||||
|
codePreview.classList.add("visible");
|
||||||
|
canvas.classList.add("hidden");
|
||||||
|
const code = params.arguments?.fragmentShader;
|
||||||
|
codePreview.textContent = typeof code === "string" ? code : "";
|
||||||
|
codePreview.scrollTop = codePreview.scrollHeight;
|
||||||
|
};
|
||||||
|
|
||||||
app.ontoolinput = (params) => {
|
app.ontoolinput = (params) => {
|
||||||
log.info("Received shader input");
|
log.info("Received shader input");
|
||||||
|
|
||||||
|
// Hide code preview, show canvas
|
||||||
|
codePreview.classList.remove("visible");
|
||||||
|
canvas.classList.remove("hidden");
|
||||||
|
|
||||||
if (!isShaderInput(params.arguments)) {
|
if (!isShaderInput(params.arguments)) {
|
||||||
log.error("Invalid tool input");
|
log.error("Invalid tool input");
|
||||||
return;
|
return;
|
||||||
|
|
@ -162,6 +185,18 @@ app.onerror = log.error;
|
||||||
|
|
||||||
app.onhostcontextchanged = handleHostContextChanged;
|
app.onhostcontextchanged = handleHostContextChanged;
|
||||||
|
|
||||||
|
// Pause/resume shader based on visibility
|
||||||
|
const observer = new IntersectionObserver((entries) => {
|
||||||
|
entries.forEach((entry) => {
|
||||||
|
if (entry.isIntersecting) {
|
||||||
|
shaderToy?.play();
|
||||||
|
} else {
|
||||||
|
shaderToy?.pause();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
observer.observe(mainEl);
|
||||||
|
|
||||||
// Connect to host
|
// Connect to host
|
||||||
app.connect().then(() => {
|
app.connect().then(() => {
|
||||||
log.info("Connected to host");
|
log.info("Connected to host");
|
||||||
|
|
|
||||||
|
|
@ -86,6 +86,7 @@ Read JSDoc documentation directly from source:
|
||||||
|
|
||||||
| Example | Pattern Demonstrated |
|
| Example | Pattern Demonstrated |
|
||||||
|---------|---------------------|
|
|---------|---------------------|
|
||||||
|
| `examples/shadertoy-server/` | **Streaming partial input** + visibility-based pause/play (best practice for large inputs) |
|
||||||
| `examples/wiki-explorer-server/` | `callServerTool` for interactive data fetching |
|
| `examples/wiki-explorer-server/` | `callServerTool` for interactive data fetching |
|
||||||
| `examples/system-monitor-server/` | Polling pattern with interval management |
|
| `examples/system-monitor-server/` | Polling pattern with interval management |
|
||||||
| `examples/video-resource-server/` | Binary/blob resources |
|
| `examples/video-resource-server/` | Binary/blob resources |
|
||||||
|
|
@ -173,9 +174,31 @@ app.onhostcontextchanged = (ctx) => {
|
||||||
import { useApp, useHostStyles } from "@modelcontextprotocol/ext-apps/react";
|
import { useApp, useHostStyles } from "@modelcontextprotocol/ext-apps/react";
|
||||||
|
|
||||||
const { app } = useApp({ appInfo, capabilities, onAppCreated });
|
const { app } = useApp({ appInfo, capabilities, onAppCreated });
|
||||||
useHostStyles(app); // Handles theme, styles, fonts automatically
|
useHostStyles(app); // Injects CSS variables to document, making var(--*) available
|
||||||
```
|
```
|
||||||
|
|
||||||
|
**Using variables in CSS** - After applying, use `var()`:
|
||||||
|
```css
|
||||||
|
.container {
|
||||||
|
background: var(--color-background-secondary);
|
||||||
|
color: var(--color-text-primary);
|
||||||
|
font-family: var(--font-sans);
|
||||||
|
border-radius: var(--border-radius-md);
|
||||||
|
}
|
||||||
|
.code {
|
||||||
|
font-family: var(--font-mono);
|
||||||
|
font-size: var(--font-text-sm-size);
|
||||||
|
line-height: var(--font-text-sm-line-height);
|
||||||
|
color: var(--color-text-secondary);
|
||||||
|
}
|
||||||
|
.heading {
|
||||||
|
font-size: var(--font-heading-lg-size);
|
||||||
|
font-weight: var(--font-weight-semibold);
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Key variable groups: `--color-background-*`, `--color-text-*`, `--color-border-*`, `--font-sans`, `--font-mono`, `--font-text-*-size`, `--font-heading-*-size`, `--border-radius-*`. See `src/spec.types.ts` for full list.
|
||||||
|
|
||||||
### Safe Area Handling
|
### Safe Area Handling
|
||||||
|
|
||||||
Always respect `safeAreaInsets`:
|
Always respect `safeAreaInsets`:
|
||||||
|
|
@ -189,6 +212,96 @@ app.onhostcontextchanged = (ctx) => {
|
||||||
};
|
};
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Streaming Partial Input
|
||||||
|
|
||||||
|
For large tool inputs, use `ontoolinputpartial` to show progress during LLM generation. The partial JSON is healed (always valid), enabling progressive UI updates.
|
||||||
|
|
||||||
|
**Spec:** [ui/notifications/tool-input-partial](https://github.com/modelcontextprotocol/ext-apps/blob/main/specification/draft/apps.mdx#streaming-tool-input)
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
app.ontoolinputpartial = (params) => {
|
||||||
|
const args = params.arguments; // Healed partial JSON - always valid, fields appear as generated
|
||||||
|
// Use args directly for progressive rendering
|
||||||
|
};
|
||||||
|
|
||||||
|
app.ontoolinput = (params) => {
|
||||||
|
// Final complete input - switch from preview to full render
|
||||||
|
};
|
||||||
|
```
|
||||||
|
|
||||||
|
**Use cases:**
|
||||||
|
| Pattern | Example |
|
||||||
|
|---------|---------|
|
||||||
|
| Code preview | Show streaming code in `<pre>`, render on complete (`examples/shadertoy-server/`) |
|
||||||
|
| Progressive form | Fill form fields as they stream in |
|
||||||
|
| Live chart | Add data points to chart as array grows |
|
||||||
|
| Partial render | Render incomplete structured data (tables, lists, trees) |
|
||||||
|
|
||||||
|
**Simple pattern (code preview):**
|
||||||
|
```typescript
|
||||||
|
app.ontoolinputpartial = (params) => {
|
||||||
|
codePreview.textContent = params.arguments?.code ?? "";
|
||||||
|
codePreview.style.display = "block";
|
||||||
|
canvas.style.display = "none";
|
||||||
|
};
|
||||||
|
app.ontoolinput = (params) => {
|
||||||
|
codePreview.style.display = "none";
|
||||||
|
canvas.style.display = "block";
|
||||||
|
render(params.arguments);
|
||||||
|
};
|
||||||
|
```
|
||||||
|
|
||||||
|
### Visibility-Based Resource Management
|
||||||
|
|
||||||
|
Pause expensive operations (animations, WebGL, polling) when widget scrolls out of viewport:
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
const observer = new IntersectionObserver((entries) => {
|
||||||
|
entries.forEach((entry) => {
|
||||||
|
if (entry.isIntersecting) {
|
||||||
|
animation.play(); // or: startPolling(), shaderToy.play()
|
||||||
|
} else {
|
||||||
|
animation.pause(); // or: stopPolling(), shaderToy.pause()
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
observer.observe(document.querySelector(".main"));
|
||||||
|
```
|
||||||
|
|
||||||
|
### Fullscreen Mode
|
||||||
|
|
||||||
|
Request fullscreen via `app.requestDisplayMode()`. Check availability in host context:
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
let currentMode: "inline" | "fullscreen" = "inline";
|
||||||
|
|
||||||
|
app.onhostcontextchanged = (ctx) => {
|
||||||
|
// Check if fullscreen available
|
||||||
|
if (ctx.availableDisplayModes?.includes("fullscreen")) {
|
||||||
|
fullscreenBtn.style.display = "block";
|
||||||
|
}
|
||||||
|
// Track current mode
|
||||||
|
if (ctx.displayMode) {
|
||||||
|
currentMode = ctx.displayMode;
|
||||||
|
container.classList.toggle("fullscreen", currentMode === "fullscreen");
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
async function toggleFullscreen() {
|
||||||
|
const newMode = currentMode === "fullscreen" ? "inline" : "fullscreen";
|
||||||
|
const result = await app.requestDisplayMode({ mode: newMode });
|
||||||
|
currentMode = result.mode;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**CSS pattern** - Remove border radius in fullscreen:
|
||||||
|
```css
|
||||||
|
.main { border-radius: var(--border-radius-lg); overflow: hidden; }
|
||||||
|
.main.fullscreen { border-radius: 0; }
|
||||||
|
```
|
||||||
|
|
||||||
|
See `examples/shadertoy-server/` for complete implementation.
|
||||||
|
|
||||||
## Common Mistakes to Avoid
|
## Common Mistakes to Avoid
|
||||||
|
|
||||||
1. **Handlers after connect()** - Register ALL handlers BEFORE calling `app.connect()`
|
1. **Handlers after connect()** - Register ALL handlers BEFORE calling `app.connect()`
|
||||||
|
|
@ -198,6 +311,7 @@ app.onhostcontextchanged = (ctx) => {
|
||||||
5. **Ignoring safe area insets** - Always handle `ctx.safeAreaInsets`
|
5. **Ignoring safe area insets** - Always handle `ctx.safeAreaInsets`
|
||||||
6. **No text fallback** - Always provide `content` array for non-UI hosts
|
6. **No text fallback** - Always provide `content` array for non-UI hosts
|
||||||
7. **Hardcoded styles** - Use host CSS variables for theme integration
|
7. **Hardcoded styles** - Use host CSS variables for theme integration
|
||||||
|
8. **No streaming for large inputs** - Use `ontoolinputpartial` to show progress during generation
|
||||||
|
|
||||||
## Testing
|
## Testing
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue