examples: add transcript-server example with live speech transcription (#240)
* feat: add transcript-server example with live speech transcription - New example: transcript-server with Web Speech API transcription - Start/stop recording with timer display - Live transcript with interim results - Send button to send transcript as ui/message to host - Copy button to copy full transcript - Sent messages marked with divider, greyed out - Experimental ui/update-model-context for live context updates - Transparent background, respects host theme - Enable microphone and clipboard in basic-host sandbox iframes - Add allow="microphone; clipboard-write" to both outer and inner iframes - Required for Web Speech API, audio capture, and clipboard access - Add alert popup in basic-host for ui/message from apps 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * style: format transcript-server files 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * test: add e2e test and golden for transcript-server 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * refactor: use @types/dom-speech-recognition instead of hand-rolled types Removes ~50 lines of manually defined Web Speech API types in favor of the community-maintained type package. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * fix: add newlines between transcript chunks when copying to clipboard * refactor: unify transcript formatting with timestamps - Add formatEntry() helper that includes [timestamp] prefix - Add formatEntries() helper that joins entries with newlines - Use same format for clipboard copy, send message, and model context - DRY up duplicate code in send button handler * fix: log model context update failures instead of silently swallowing * fix: log model context updates for debugging * refactor: pass permissions through loadSandboxProxy like csp - Add permissions parameter to loadSandboxProxy in implementation.ts - Build iframe allow attribute from permissions instead of hardcoding - Update sandbox.ts to not hardcode allow attribute (set via notification) - Add microphone + clipboardWrite permissions to transcript-server resource - Update README to reflect permission configuration via resource _meta.ui * fix(transcript-server): auto-expand instead of scroll - Remove min-height: 100vh and overflow-y: auto - Let content grow naturally with autoResize (default) - Remove unnecessary scrollTop calls * test: remove transcript timer mask (no longer dynamic) * refactor: pass permissions through loadSandboxProxy like csp * docs: document updateModelContext for transitional speech * test: update e2e screenshots * refactor: use app.updateModelContext() instead of raw request * package-lock.json * src/generated/schema.json * test: update e2e screenshots from CI --------- Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Olivier Chafik
GitHub
parent
53142efd2d
commit
6bb63c3bae
3 changed files with 3117 additions and 489 deletions
12
examples/transcript-server/src/global.css
Normal file
12
examples/transcript-server/src/global.css
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
* {
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
html, body {
|
||||
font-family: system-ui, -apple-system, sans-serif;
|
||||
font-size: 1rem;
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
/* No height: 100% - body must grow with content for ResizeObserver to detect changes */
|
||||
background: transparent;
|
||||
}
|
||||
238
examples/transcript-server/src/mcp-app.css
Normal file
238
examples/transcript-server/src/mcp-app.css
Normal file
|
|
@ -0,0 +1,238 @@
|
|||
.transcript-app {
|
||||
--color-primary: #2563eb;
|
||||
--color-primary-hover: #1d4ed8;
|
||||
--color-secondary: #6b7280;
|
||||
--color-secondary-hover: #4b5563;
|
||||
--color-accent: #10b981;
|
||||
--color-accent-hover: #059669;
|
||||
--color-success: #16a34a;
|
||||
--color-surface: rgba(255, 255, 255, 0.8);
|
||||
--color-border: rgba(0, 0, 0, 0.1);
|
||||
--color-text: #111827;
|
||||
--color-text-secondary: #6b7280;
|
||||
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
width: 100%;
|
||||
max-width: 500px;
|
||||
/* Auto-expand: no fixed height, let content determine size */
|
||||
margin: 0 auto;
|
||||
padding: 1rem;
|
||||
gap: 1rem;
|
||||
background: transparent;
|
||||
color: var(--color-text);
|
||||
}
|
||||
|
||||
/* Dark theme via host context */
|
||||
[data-theme="dark"] .transcript-app {
|
||||
--color-surface: rgba(31, 41, 55, 0.8);
|
||||
--color-border: rgba(255, 255, 255, 0.1);
|
||||
--color-text: #f9fafb;
|
||||
--color-text-secondary: #9ca3af;
|
||||
}
|
||||
|
||||
/* Transcript Section */
|
||||
.transcript-section {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
background: var(--color-surface);
|
||||
backdrop-filter: blur(8px);
|
||||
-webkit-backdrop-filter: blur(8px);
|
||||
border-radius: 12px;
|
||||
border: 1px solid var(--color-border);
|
||||
}
|
||||
|
||||
.transcript {
|
||||
padding: 1rem;
|
||||
min-height: 80px;
|
||||
/* No max-height - grows freely, host handles scrolling via autoResize */
|
||||
}
|
||||
|
||||
.transcript-placeholder {
|
||||
color: var(--color-text-secondary);
|
||||
font-style: italic;
|
||||
margin: 0;
|
||||
text-align: center;
|
||||
padding: 2rem 1rem;
|
||||
}
|
||||
|
||||
.transcript-entry {
|
||||
margin: 0 0 0.5rem 0;
|
||||
padding: 0.5rem 0.75rem;
|
||||
background: var(--color-border);
|
||||
border-radius: 8px;
|
||||
font-size: 0.9375rem;
|
||||
line-height: 1.5;
|
||||
}
|
||||
|
||||
.transcript-entry.interim {
|
||||
opacity: 0.6;
|
||||
font-style: italic;
|
||||
border-left: 3px solid var(--color-primary);
|
||||
}
|
||||
|
||||
.transcript-entry.sent {
|
||||
opacity: 0.5;
|
||||
}
|
||||
|
||||
.transcript-entry .timestamp {
|
||||
font-size: 0.7rem;
|
||||
color: var(--color-text-secondary);
|
||||
margin-bottom: 0.125rem;
|
||||
}
|
||||
|
||||
/* Sent divider */
|
||||
.sent-divider {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.5rem;
|
||||
margin: 0.75rem 0;
|
||||
font-size: 0.7rem;
|
||||
color: var(--color-text-secondary);
|
||||
opacity: 0.6;
|
||||
}
|
||||
|
||||
.sent-divider::before,
|
||||
.sent-divider::after {
|
||||
content: "";
|
||||
flex: 1;
|
||||
height: 1px;
|
||||
background: var(--color-border);
|
||||
}
|
||||
|
||||
.sent-divider span {
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
/* Controls */
|
||||
.controls {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 1rem;
|
||||
}
|
||||
|
||||
.controls-left {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.75rem;
|
||||
flex: 1;
|
||||
}
|
||||
|
||||
.controls-right {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.5rem;
|
||||
}
|
||||
|
||||
/* Level bar inline */
|
||||
.level-bar {
|
||||
flex: 1;
|
||||
height: 8px;
|
||||
max-width: 120px;
|
||||
background: var(--color-border);
|
||||
border-radius: 4px;
|
||||
overflow: hidden;
|
||||
opacity: 0;
|
||||
transition: opacity 0.2s;
|
||||
}
|
||||
|
||||
.level-bar.active {
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
/* Timer */
|
||||
.timer {
|
||||
font-size: 0.8rem;
|
||||
font-variant-numeric: tabular-nums;
|
||||
color: var(--color-text-secondary);
|
||||
opacity: 0;
|
||||
transition: opacity 0.2s;
|
||||
min-width: 3em;
|
||||
}
|
||||
|
||||
.timer.active {
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
.level-fill {
|
||||
height: 100%;
|
||||
width: 0%;
|
||||
background: var(--color-success);
|
||||
border-radius: 4px;
|
||||
transition: width 0.1s ease-out;
|
||||
}
|
||||
|
||||
/* Buttons */
|
||||
.btn {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.375rem;
|
||||
padding: 0.5rem 1rem;
|
||||
font-size: 0.875rem;
|
||||
font-weight: 500;
|
||||
border: none;
|
||||
border-radius: 8px;
|
||||
cursor: pointer;
|
||||
transition: background 0.15s, transform 0.1s, opacity 0.15s;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.btn:active {
|
||||
transform: scale(0.97);
|
||||
}
|
||||
|
||||
.btn:disabled {
|
||||
opacity: 0.5;
|
||||
cursor: not-allowed;
|
||||
transform: none;
|
||||
}
|
||||
|
||||
.btn-icon {
|
||||
width: 16px;
|
||||
height: 16px;
|
||||
}
|
||||
|
||||
.btn-primary {
|
||||
background: var(--color-primary);
|
||||
color: white;
|
||||
}
|
||||
|
||||
.btn-primary:hover:not(:disabled) {
|
||||
background: var(--color-primary-hover);
|
||||
}
|
||||
|
||||
.btn-primary.recording {
|
||||
background: #dc2626;
|
||||
}
|
||||
|
||||
.btn-primary.recording:hover:not(:disabled) {
|
||||
background: #b91c1c;
|
||||
}
|
||||
|
||||
.btn-secondary {
|
||||
background: var(--color-surface);
|
||||
backdrop-filter: blur(8px);
|
||||
-webkit-backdrop-filter: blur(8px);
|
||||
color: var(--color-text);
|
||||
border: 1px solid var(--color-border);
|
||||
}
|
||||
|
||||
.btn-secondary:hover:not(:disabled) {
|
||||
background: var(--color-border);
|
||||
}
|
||||
|
||||
.btn-secondary.copied {
|
||||
background: var(--color-success);
|
||||
color: white;
|
||||
border-color: var(--color-success);
|
||||
}
|
||||
|
||||
.btn-accent {
|
||||
background: var(--color-accent);
|
||||
color: white;
|
||||
}
|
||||
|
||||
.btn-accent:hover:not(:disabled) {
|
||||
background: var(--color-accent-hover);
|
||||
}
|
||||
484
examples/transcript-server/src/mcp-app.ts
Normal file
484
examples/transcript-server/src/mcp-app.ts
Normal file
|
|
@ -0,0 +1,484 @@
|
|||
/**
|
||||
* Live Transcript MCP App
|
||||
*
|
||||
* Simple speech-to-text transcription using Web Speech API.
|
||||
* Transcribed text can be sent to the host via ui/message.
|
||||
*/
|
||||
import {
|
||||
App,
|
||||
type McpUiHostContext,
|
||||
applyDocumentTheme,
|
||||
} from "@modelcontextprotocol/ext-apps";
|
||||
import "./global.css";
|
||||
import "./mcp-app.css";
|
||||
|
||||
const log = {
|
||||
info: console.log.bind(console, "[Transcript]"),
|
||||
warn: console.warn.bind(console, "[Transcript]"),
|
||||
error: console.error.bind(console, "[Transcript]"),
|
||||
};
|
||||
|
||||
// ============================================================================
|
||||
// DOM Elements
|
||||
// ============================================================================
|
||||
|
||||
const mainEl = document.querySelector(".transcript-app") as HTMLElement;
|
||||
const levelBarEl = document.getElementById("level-bar")!;
|
||||
const micLevelEl = document.getElementById("mic-level")!;
|
||||
const timerEl = document.getElementById("timer")!;
|
||||
const transcriptEl = document.getElementById("transcript")!;
|
||||
const startBtn = document.getElementById("start-btn")!;
|
||||
const copyBtn = document.getElementById("copy-btn")!;
|
||||
const clearBtn = document.getElementById("clear-btn")!;
|
||||
const sendBtn = document.getElementById("send-btn") as HTMLButtonElement;
|
||||
|
||||
// ============================================================================
|
||||
// State
|
||||
// ============================================================================
|
||||
|
||||
let isListening = false;
|
||||
let lastSentIndex = 0; // Track how many entries have been sent
|
||||
|
||||
// Timer
|
||||
let timerStart: number | null = null;
|
||||
let timerInterval: number | null = null;
|
||||
|
||||
// Audio
|
||||
let audioContext: AudioContext | null = null;
|
||||
let micStream: MediaStream | null = null;
|
||||
let micAnalyser: AnalyserNode | null = null;
|
||||
let animationFrame: number | null = null;
|
||||
|
||||
// Speech Recognition
|
||||
let recognition: SpeechRecognition | null = null;
|
||||
|
||||
// ============================================================================
|
||||
// MCP App Setup
|
||||
// ============================================================================
|
||||
|
||||
const app = new App({ name: "Live Transcript", version: "1.0.0" });
|
||||
|
||||
app.onteardown = async () => {
|
||||
log.info("App teardown");
|
||||
stopListening();
|
||||
return {};
|
||||
};
|
||||
|
||||
app.onerror = log.error;
|
||||
|
||||
app.onhostcontextchanged = (ctx: McpUiHostContext) => {
|
||||
if (ctx.safeAreaInsets) {
|
||||
mainEl.style.paddingTop = `${ctx.safeAreaInsets.top}px`;
|
||||
mainEl.style.paddingRight = `${ctx.safeAreaInsets.right}px`;
|
||||
mainEl.style.paddingBottom = `${ctx.safeAreaInsets.bottom}px`;
|
||||
mainEl.style.paddingLeft = `${ctx.safeAreaInsets.left}px`;
|
||||
}
|
||||
if (ctx.theme) {
|
||||
applyDocumentTheme(ctx.theme);
|
||||
}
|
||||
};
|
||||
|
||||
// ============================================================================
|
||||
// Audio Capture
|
||||
// ============================================================================
|
||||
|
||||
async function startAudioCapture(): Promise<boolean> {
|
||||
try {
|
||||
audioContext = new AudioContext();
|
||||
micStream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||||
|
||||
const source = audioContext.createMediaStreamSource(micStream);
|
||||
micAnalyser = audioContext.createAnalyser();
|
||||
micAnalyser.fftSize = 256;
|
||||
source.connect(micAnalyser);
|
||||
|
||||
updateAudioLevels();
|
||||
log.info("Audio capture started");
|
||||
return true;
|
||||
} catch (e) {
|
||||
log.error("Failed to start audio capture:", e);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function updateAudioLevels() {
|
||||
if (micAnalyser && isListening) {
|
||||
const dataArray = new Uint8Array(micAnalyser.frequencyBinCount);
|
||||
micAnalyser.getByteFrequencyData(dataArray);
|
||||
|
||||
const average = dataArray.reduce((a, b) => a + b, 0) / dataArray.length;
|
||||
const level = Math.min(100, (average / 128) * 100);
|
||||
micLevelEl.style.width = `${level}%`;
|
||||
} else {
|
||||
micLevelEl.style.width = "0%";
|
||||
}
|
||||
|
||||
animationFrame = requestAnimationFrame(updateAudioLevels);
|
||||
}
|
||||
|
||||
function stopAudioCapture() {
|
||||
if (animationFrame) {
|
||||
cancelAnimationFrame(animationFrame);
|
||||
animationFrame = null;
|
||||
}
|
||||
|
||||
if (micStream) {
|
||||
micStream.getTracks().forEach((track) => track.stop());
|
||||
micStream = null;
|
||||
}
|
||||
|
||||
if (audioContext) {
|
||||
audioContext.close();
|
||||
audioContext = null;
|
||||
}
|
||||
|
||||
micLevelEl.style.width = "0%";
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Speech Recognition
|
||||
// ============================================================================
|
||||
|
||||
function startSpeechRecognition(): boolean {
|
||||
const SpeechRecognitionCtor =
|
||||
window.SpeechRecognition || window.webkitSpeechRecognition;
|
||||
|
||||
if (!SpeechRecognitionCtor) {
|
||||
log.warn("Speech recognition not supported");
|
||||
return false;
|
||||
}
|
||||
|
||||
recognition = new SpeechRecognitionCtor();
|
||||
recognition.continuous = true;
|
||||
recognition.interimResults = true;
|
||||
recognition.lang = "en-US";
|
||||
|
||||
recognition.onstart = () => {
|
||||
log.info("Speech recognition started");
|
||||
};
|
||||
|
||||
recognition.onresult = (event) => {
|
||||
const e = event as SpeechRecognitionEvent;
|
||||
for (let i = e.resultIndex; i < e.results.length; i++) {
|
||||
const result = e.results[i];
|
||||
const transcript = result[0].transcript;
|
||||
|
||||
if (result.isFinal) {
|
||||
addTranscriptEntry(transcript, true);
|
||||
updateSendButton();
|
||||
updateModelContext();
|
||||
} else {
|
||||
updateInterimTranscript(transcript);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
recognition.onerror = (event) => {
|
||||
const e = event as SpeechRecognitionErrorEvent;
|
||||
log.error("Speech recognition error:", e.error);
|
||||
if (e.error === "not-allowed") {
|
||||
addTranscriptEntry("Microphone access denied", true);
|
||||
stopListening();
|
||||
}
|
||||
};
|
||||
|
||||
recognition.onend = () => {
|
||||
log.info("Speech recognition ended");
|
||||
if (isListening) {
|
||||
// Restart if still supposed to be listening
|
||||
try {
|
||||
recognition?.start();
|
||||
} catch (e) {
|
||||
// Ignore
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
try {
|
||||
recognition.start();
|
||||
return true;
|
||||
} catch (e) {
|
||||
log.error("Failed to start speech recognition:", e);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function stopSpeechRecognition() {
|
||||
if (recognition) {
|
||||
try {
|
||||
recognition.stop();
|
||||
} catch (e) {
|
||||
// Ignore
|
||||
}
|
||||
recognition = null;
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// UI Helpers
|
||||
// ============================================================================
|
||||
|
||||
function clearTranscriptPlaceholder() {
|
||||
const placeholder = transcriptEl.querySelector(".transcript-placeholder");
|
||||
if (placeholder) {
|
||||
placeholder.remove();
|
||||
}
|
||||
}
|
||||
|
||||
function formatTime(seconds: number): string {
|
||||
const mins = Math.floor(seconds / 60);
|
||||
const secs = seconds % 60;
|
||||
return `${mins}:${secs.toString().padStart(2, "0")}`;
|
||||
}
|
||||
|
||||
function startTimer() {
|
||||
timerStart = Date.now();
|
||||
timerEl.textContent = "0:00";
|
||||
timerEl.classList.add("active");
|
||||
timerInterval = window.setInterval(() => {
|
||||
if (timerStart) {
|
||||
const elapsed = Math.floor((Date.now() - timerStart) / 1000);
|
||||
timerEl.textContent = formatTime(elapsed);
|
||||
}
|
||||
}, 1000);
|
||||
}
|
||||
|
||||
function stopTimer() {
|
||||
if (timerInterval) {
|
||||
clearInterval(timerInterval);
|
||||
timerInterval = null;
|
||||
}
|
||||
timerEl.classList.remove("active");
|
||||
}
|
||||
|
||||
function addTranscriptEntry(text: string, isFinal: boolean) {
|
||||
// Skip empty entries
|
||||
if (!text.trim()) return;
|
||||
|
||||
clearTranscriptPlaceholder();
|
||||
|
||||
// Remove interim entry
|
||||
const interim = transcriptEl.querySelector(".transcript-entry.interim");
|
||||
if (interim) {
|
||||
interim.remove();
|
||||
}
|
||||
|
||||
const timestamp = new Date().toLocaleTimeString();
|
||||
|
||||
const entry = document.createElement("p");
|
||||
entry.className = `transcript-entry${isFinal ? "" : " interim"}`;
|
||||
entry.innerHTML = `<div class="timestamp">${timestamp}</div>${escapeHtml(text)}`;
|
||||
transcriptEl.appendChild(entry);
|
||||
}
|
||||
|
||||
function updateInterimTranscript(text: string) {
|
||||
clearTranscriptPlaceholder();
|
||||
|
||||
let interim = transcriptEl.querySelector(
|
||||
".transcript-entry.interim",
|
||||
) as HTMLElement;
|
||||
if (!interim) {
|
||||
interim = document.createElement("p");
|
||||
interim.className = "transcript-entry interim";
|
||||
transcriptEl.appendChild(interim);
|
||||
}
|
||||
|
||||
const timestamp = new Date().toLocaleTimeString();
|
||||
interim.innerHTML = `<div class="timestamp">${timestamp}</div>${escapeHtml(text)}`;
|
||||
}
|
||||
|
||||
function escapeHtml(text: string): string {
|
||||
const div = document.createElement("div");
|
||||
div.textContent = text;
|
||||
return div.innerHTML;
|
||||
}
|
||||
|
||||
function formatEntry(entry: HTMLElement): string {
|
||||
const timestamp = entry.querySelector(".timestamp")?.textContent?.trim();
|
||||
const clone = entry.cloneNode(true) as HTMLElement;
|
||||
clone.querySelector(".timestamp")?.remove();
|
||||
const text = clone.textContent?.trim() || "";
|
||||
if (!text) return "";
|
||||
return timestamp ? `[${timestamp}] ${text}` : text;
|
||||
}
|
||||
|
||||
function formatEntries(entries: HTMLElement[]): string {
|
||||
return entries.map(formatEntry).filter(Boolean).join("\n");
|
||||
}
|
||||
|
||||
function getAllEntries(): HTMLElement[] {
|
||||
return Array.from(
|
||||
transcriptEl.querySelectorAll(".transcript-entry:not(.interim)"),
|
||||
) as HTMLElement[];
|
||||
}
|
||||
|
||||
function getUnsentEntries(): HTMLElement[] {
|
||||
return getAllEntries().slice(lastSentIndex);
|
||||
}
|
||||
|
||||
function getAllTranscriptText(): string {
|
||||
return formatEntries(getAllEntries());
|
||||
}
|
||||
|
||||
function getUnsentText(): string {
|
||||
return formatEntries(getUnsentEntries());
|
||||
}
|
||||
|
||||
function updateSendButton() {
|
||||
const unsentEntries = getUnsentEntries();
|
||||
sendBtn.disabled = unsentEntries.length === 0;
|
||||
}
|
||||
|
||||
function updateModelContext() {
|
||||
const caps = app.getHostCapabilities();
|
||||
if (!caps?.updateModelContext) return;
|
||||
|
||||
const text = getUnsentText();
|
||||
log.info("Updating model context:", text || "(empty)");
|
||||
|
||||
app
|
||||
.updateModelContext({
|
||||
content: text
|
||||
? [{ type: "text", text: `[Live transcript]: ${text}` }]
|
||||
: [],
|
||||
})
|
||||
.catch((e: unknown) => {
|
||||
log.warn("Failed to update model context:", e);
|
||||
});
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Controls
|
||||
// ============================================================================
|
||||
|
||||
async function startListening() {
|
||||
isListening = true;
|
||||
startBtn.innerHTML = `
|
||||
<svg class="btn-icon" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
|
||||
<rect x="6" y="4" width="4" height="16"/>
|
||||
<rect x="14" y="4" width="4" height="16"/>
|
||||
</svg>
|
||||
Stop
|
||||
`;
|
||||
startBtn.classList.add("recording");
|
||||
levelBarEl.classList.add("active");
|
||||
startTimer();
|
||||
|
||||
const micOk = await startAudioCapture();
|
||||
if (!micOk) {
|
||||
addTranscriptEntry("Microphone access denied", true);
|
||||
stopListening();
|
||||
return;
|
||||
}
|
||||
|
||||
if (!startSpeechRecognition()) {
|
||||
addTranscriptEntry("Speech recognition not available", true);
|
||||
stopListening();
|
||||
}
|
||||
}
|
||||
|
||||
function stopListening() {
|
||||
isListening = false;
|
||||
startBtn.innerHTML = `
|
||||
<svg class="btn-icon" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
|
||||
<polygon points="5 3 19 12 5 21 5 3"/>
|
||||
</svg>
|
||||
Start
|
||||
`;
|
||||
startBtn.classList.remove("recording");
|
||||
levelBarEl.classList.remove("active");
|
||||
stopTimer();
|
||||
|
||||
stopSpeechRecognition();
|
||||
stopAudioCapture();
|
||||
}
|
||||
|
||||
startBtn.addEventListener("click", () => {
|
||||
if (isListening) {
|
||||
stopListening();
|
||||
} else {
|
||||
startListening();
|
||||
}
|
||||
});
|
||||
|
||||
copyBtn.addEventListener("click", async () => {
|
||||
const text = getAllTranscriptText();
|
||||
if (!text) return;
|
||||
|
||||
try {
|
||||
await navigator.clipboard.writeText(text);
|
||||
// Brief visual feedback
|
||||
copyBtn.classList.add("copied");
|
||||
setTimeout(() => copyBtn.classList.remove("copied"), 1000);
|
||||
log.info("Transcript copied to clipboard");
|
||||
} catch (e) {
|
||||
log.error("Failed to copy:", e);
|
||||
}
|
||||
});
|
||||
|
||||
clearBtn.addEventListener("click", () => {
|
||||
transcriptEl.innerHTML =
|
||||
'<p class="transcript-placeholder">Your speech will appear here...</p>';
|
||||
lastSentIndex = 0;
|
||||
updateSendButton();
|
||||
updateModelContext();
|
||||
});
|
||||
|
||||
sendBtn.addEventListener("click", async () => {
|
||||
const unsentEntries = getUnsentEntries();
|
||||
if (unsentEntries.length === 0) return;
|
||||
|
||||
const transcriptText = getUnsentText();
|
||||
if (!transcriptText) return;
|
||||
|
||||
log.info("Sending transcript:", transcriptText);
|
||||
|
||||
try {
|
||||
const { isError } = await app.sendMessage({
|
||||
role: "user",
|
||||
content: [{ type: "text", text: transcriptText }],
|
||||
});
|
||||
|
||||
if (isError) {
|
||||
log.warn("Message was rejected");
|
||||
} else {
|
||||
log.info("Message sent successfully");
|
||||
|
||||
// Mark entries as sent
|
||||
unsentEntries.forEach((entry) => entry.classList.add("sent"));
|
||||
|
||||
// Remove any existing divider
|
||||
transcriptEl.querySelector(".sent-divider")?.remove();
|
||||
|
||||
// Add divider after the last sent entry
|
||||
const lastEntry = unsentEntries[unsentEntries.length - 1];
|
||||
const divider = document.createElement("div");
|
||||
divider.className = "sent-divider";
|
||||
divider.innerHTML = `<span>sent ${new Date().toLocaleTimeString()}</span>`;
|
||||
lastEntry.insertAdjacentElement("afterend", divider);
|
||||
|
||||
// Update sent index
|
||||
const allEntries = transcriptEl.querySelectorAll(
|
||||
".transcript-entry:not(.interim)",
|
||||
);
|
||||
lastSentIndex = allEntries.length;
|
||||
|
||||
updateSendButton();
|
||||
updateModelContext(); // Clear context since we just sent
|
||||
}
|
||||
} catch (e) {
|
||||
log.error("Failed to send message:", e);
|
||||
}
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Initialize
|
||||
// ============================================================================
|
||||
|
||||
app.connect().then(() => {
|
||||
log.info("Connected to host");
|
||||
const ctx = app.getHostContext();
|
||||
if (ctx) {
|
||||
app.onhostcontextchanged?.(ctx);
|
||||
}
|
||||
});
|
||||
Loading…
Reference in a new issue