From 6c5aa52c95ea1c8154d266caf0a2fcc2c116d5c8 Mon Sep 17 00:00:00 2001 From: Olivier Chafik Date: Mon, 19 Jan 2026 19:11:53 +0000 Subject: [PATCH] feat(shadertoy): add proper mouse/touch interaction support (#293) * feat(shadertoy): add proper mouse/touch interaction support - Fix iMouse uniform to match official Shadertoy specification: - iMouse.xy only updates while button/touch is held down - iMouse.zw uses positive values when down, negative when released - Enables proper click detection via iMouse.z > 0 - Add touch event support for mobile devices: - touchstart/touchmove/touchend/touchcancel handlers - Same behavior as mouse events - Update tool description with mouse interaction documentation - Update README with: - Mouse & Touch Interaction feature - Improved Julia Set example with click detection - iMouse specification table and usage examples The iMouse uniform is a core Shadertoy feature used by popular shaders like Star Nest, raymarching examples, and particle systems. * style: fix Prettier formatting in shadertoy README --- examples/shadertoy-server/README.md | 54 +++++++++----- examples/shadertoy-server/server.ts | 16 ++++- .../src/vendor/ShaderToyLite.js | 71 +++++++++++++++++-- 3 files changed, 115 insertions(+), 26 deletions(-) diff --git a/examples/shadertoy-server/README.md b/examples/shadertoy-server/README.md index 5f965cb..445ca6c 100644 --- a/examples/shadertoy-server/README.md +++ b/examples/shadertoy-server/README.md @@ -15,6 +15,7 @@ A demo MCP App that renders [ShaderToy](https://www.shadertoy.com/)-compatible G - **Real-time Rendering**: Renders GLSL shaders using WebGL 2.0 - **ShaderToy Compatibility**: Uses the standard `mainImage(out vec4 fragColor, in vec2 fragCoord)` entry point - **Multi-pass Rendering**: Supports buffers A-D for feedback effects, blur chains, and simulations +- **Mouse & Touch Interaction**: Full iMouse support with click detection (works on mobile) - **Standard Uniforms**: iResolution, iTime, iTimeDelta, iFrame, iMouse, iDate, iChannel0-3 ## Running @@ -101,13 +102,20 @@ _Tool input:_ } ``` -**Interactive Julia Set** (mouse controls the fractal's c parameter): +**Interactive Julia Set** (click and drag to control the fractal's c parameter): ```glsl void mainImage(out vec4 fragColor, in vec2 fragCoord) { vec2 uv = (fragCoord - 0.5 * iResolution.xy) / iResolution.y * 2.5; - vec2 mouse = (iMouse.xy / iResolution.xy - 0.5) * 2.0; - vec2 c = mouse; + // Use mouse position if clicked, otherwise use animated default + vec2 c; + if (iMouse.z > 0.0) { + // Mouse is pressed - use mouse position + c = (iMouse.xy / iResolution.xy - 0.5) * 2.0; + } else { + // Not pressed - animate around an interesting region + c = vec2(-0.8 + 0.2 * sin(iTime * 0.5), 0.156 + 0.1 * cos(iTime * 0.7)); + } vec2 z = uv; float iter = 0.0; for (int i = 0; i < 100; i++) { @@ -126,25 +134,33 @@ _Tool input:_ ```json { - "fragmentShader": "void mainImage(out vec4 fragColor, in vec2 fragCoord) { - vec2 uv = (fragCoord - 0.5 * iResolution.xy) / iResolution.y * 2.5; - vec2 mouse = (iMouse.xy / iResolution.xy - 0.5) * 2.0; - vec2 c = mouse; - vec2 z = uv; - float iter = 0.0; - for (int i = 0; i < 100; i++) { - z = vec2(z.x * z.x - z.y * z.y, 2.0 * z.x * z.y) + c; - if (dot(z, z) > 4.0) break; - iter++; - } - float t = iter / 100.0; - vec3 col = 0.5 + 0.5 * cos(3.0 + t * 6.28 * 2.0 + vec3(0.0, 0.6, 1.0)); - if (iter == 100.0) col = vec3(0.0); - fragColor = vec4(col, 1.0); -}" + "fragmentShader": "void mainImage(out vec4 fragColor, in vec2 fragCoord) {\n vec2 uv = (fragCoord - 0.5 * iResolution.xy) / iResolution.y * 2.5;\n vec2 c;\n if (iMouse.z > 0.0) {\n c = (iMouse.xy / iResolution.xy - 0.5) * 2.0;\n } else {\n c = vec2(-0.8 + 0.2 * sin(iTime * 0.5), 0.156 + 0.1 * cos(iTime * 0.7));\n }\n vec2 z = uv;\n float iter = 0.0;\n for (int i = 0; i < 100; i++) {\n z = vec2(z.x * z.x - z.y * z.y, 2.0 * z.x * z.y) + c;\n if (dot(z, z) > 4.0) break;\n iter++;\n }\n float t = iter / 100.0;\n vec3 col = 0.5 + 0.5 * cos(3.0 + t * 6.28 * 2.0 + vec3(0.0, 0.6, 1.0));\n if (iter == 100.0) col = vec3(0.0);\n fragColor = vec4(col, 1.0);\n}" } ``` +## Mouse & Touch Interaction + +The `iMouse` uniform provides interactive input, compatible with the official Shadertoy specification: + +| Component | When Button Down | After Release | Never Clicked | +| ----------- | ---------------------- | ---------------- | ------------- | +| `iMouse.xy` | Current position | Last position | `(0, 0)` | +| `iMouse.zw` | Click start (positive) | Negated (-x, -y) | `(0, 0)` | + +**Detecting button state:** + +```glsl +if (iMouse.z > 0.0) { + // Button/touch is currently held down +} else if (iMouse.z < 0.0) { + // Button was released (can use abs(iMouse.zw) for last click position) +} else { + // Never clicked - show default state or animate +} +``` + +Touch events are automatically supported for mobile devices. + ## Architecture ### Server (`server.ts`) diff --git a/examples/shadertoy-server/server.ts b/examples/shadertoy-server/server.ts index 1c96fea..a4bfe9e 100644 --- a/examples/shadertoy-server/server.ts +++ b/examples/shadertoy-server/server.ts @@ -33,10 +33,24 @@ AVAILABLE UNIFORMS: - iTime (float): elapsed time in seconds - iTimeDelta (float): time since last frame - iFrame (int): frame counter -- iMouse (vec4): mouse position (xy=current, zw=click) +- iMouse (vec4): mouse/touch position in pixels (see MOUSE INTERACTION below) - iDate (vec4): year, month, day, seconds - iChannel0-3 (sampler2D): buffer inputs for multi-pass shaders +MOUSE INTERACTION: +The iMouse uniform provides interactive mouse/touch input (works on mobile): +- iMouse.xy: Current position while button/touch is held down (frozen on release) +- iMouse.zw: Click start position (positive when down, negative when released) +- iMouse.z > 0: Button is currently pressed +- iMouse.z < 0: Button was released +- iMouse.z == 0: Never clicked + +Example - camera control: + vec2 uv = iMouse.xy / iResolution.xy; // normalized 0-1 + +Example - detect click: + if (iMouse.z > 0.0) { /* button is down */ } + MULTI-PASS RENDERING: - Use bufferA-D parameters for feedback effects, blur chains, simulations - BufferA output -> iChannel0, BufferB -> iChannel1, etc. diff --git a/examples/shadertoy-server/src/vendor/ShaderToyLite.js b/examples/shadertoy-server/src/vendor/ShaderToyLite.js index 1b03e8b..b54c745 100644 --- a/examples/shadertoy-server/src/vendor/ShaderToyLite.js +++ b/examples/shadertoy-server/src/vendor/ShaderToyLite.js @@ -90,7 +90,10 @@ function ShaderToyLite(canvasId) { // uniforms var iFrame = 0; + // iMouse state: xy = current position (only updates when button down) + // zw = click start position (positive when down, negative when released) var iMouse = {x: 0, y: 0, clickX: 0, clickY: 0}; + var isMouseDown = false; // shader common source var common = ""; @@ -145,19 +148,75 @@ function ShaderToyLite(canvasId) { gl.viewport(0, 0, gl.canvas.width, gl.canvas.height); }); + // Mouse event handlers - match official Shadertoy iMouse behavior: + // iMouse.xy: current position (only updates when button is down) + // iMouse.zw: click position (positive when down, negated when released) canvas.addEventListener("mousemove", (event) => { - iMouse.x = event.offsetX; - iMouse.y = canvas.height - event.offsetY; + if (isMouseDown) { + iMouse.x = event.offsetX; + iMouse.y = canvas.height - event.offsetY; + } }); canvas.addEventListener("mousedown", (event) => { - iMouse.clickX = event.offsetX; - iMouse.clickY = canvas.height - event.offsetY; + isMouseDown = true; + var x = event.offsetX; + var y = canvas.height - event.offsetY; + iMouse.x = x; + iMouse.y = y; + iMouse.clickX = x; + iMouse.clickY = y; }); canvas.addEventListener("mouseup", () => { - iMouse.clickX = 0; - iMouse.clickY = 0; + isMouseDown = false; + // Negate click position to indicate button released (Shadertoy convention) + iMouse.clickX = -Math.abs(iMouse.clickX); + iMouse.clickY = -Math.abs(iMouse.clickY); + }); + + // Prevent context menu on right-click to avoid interrupting interaction + canvas.addEventListener("contextmenu", (event) => { + event.preventDefault(); + }); + + // Touch support for mobile devices + canvas.addEventListener("touchstart", (event) => { + event.preventDefault(); + if (event.touches.length > 0) { + var touch = event.touches[0]; + var rect = canvas.getBoundingClientRect(); + var x = touch.clientX - rect.left; + var y = canvas.height - (touch.clientY - rect.top); + isMouseDown = true; + iMouse.x = x; + iMouse.y = y; + iMouse.clickX = x; + iMouse.clickY = y; + } + }, { passive: false }); + + canvas.addEventListener("touchmove", (event) => { + event.preventDefault(); + if (isMouseDown && event.touches.length > 0) { + var touch = event.touches[0]; + var rect = canvas.getBoundingClientRect(); + iMouse.x = touch.clientX - rect.left; + iMouse.y = canvas.height - (touch.clientY - rect.top); + } + }, { passive: false }); + + canvas.addEventListener("touchend", (event) => { + event.preventDefault(); + isMouseDown = false; + iMouse.clickX = -Math.abs(iMouse.clickX); + iMouse.clickY = -Math.abs(iMouse.clickY); + }, { passive: false }); + + canvas.addEventListener("touchcancel", () => { + isMouseDown = false; + iMouse.clickX = -Math.abs(iMouse.clickX); + iMouse.clickY = -Math.abs(iMouse.clickY); }); }