Skip to content

in chat and studio -> when capturing dictation/voice input -> over top #989

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jefking merged 1 commit into from
May 23, 2026
Merged

in chat and studio -> when capturing dictation/voice input -> over top #989

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 48 additions & 1 deletion internal/web/server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -689,6 +689,43 @@ func (s Server) handleChat(w http.ResponseWriter, r *http.Request) {
return grid.querySelector(".chat-repo-card[aria-expanded='true'] .chat-repo-prompt");
}

function activeChatSpeechStream() {
const input = activeChatPromptInput();
return input ? input.closest(".speech-stream-wrap")?.querySelector(".speech-stream") : null;
}

function clampSpeechStreamLevel(level) {
if (!Number.isFinite(level)) return 0;
return Math.max(0, Math.min(1, level));
}

function setSpeechStreamActive(stream, active) {
if (!(stream instanceof HTMLElement)) return;
stream.classList.toggle("is-active", Boolean(active));
if (!active) {
stream.style.setProperty("--speech-level", "0");
}
}

function setSpeechStreamLevel(stream, level) {
if (!(stream instanceof HTMLElement)) return;
stream.style.setProperty("--speech-level", clampSpeechStreamLevel(level).toFixed(3));
}

function speechStreamLevelFromSamples(input) {
if (!input || !input.length) return 0;
let sum = 0;
let peak = 0;
for (let i = 0; i < input.length; i += 1) {
const sample = Math.max(-1, Math.min(1, Number(input[i]) || 0));
const absSample = Math.abs(sample);
if (absSample > peak) peak = absSample;
sum += sample * sample;
}
const rms = Math.sqrt(sum / input.length);
return Math.max(peak * 1.8, rms * 16);
}

function setChatSpeechStatus(tone, message) {
status.textContent = message;
status.dataset.tone = tone || "";
Expand Down Expand Up @@ -754,6 +791,7 @@ func (s Server) handleChat(w http.ResponseWriter, r *http.Request) {
setChatSpeechStatus("error", "Open a repository prompt before dictating.");
return;
}
const stream = activeChatSpeechStream();
if (!browserSpeechCaptureSupported()) {
setChatSpeechStatus("error", "This browser does not support microphone capture.");
return;
Expand Down Expand Up @@ -790,13 +828,15 @@ func (s Server) handleChat(w http.ResponseWriter, r *http.Request) {
const inputBuffer = event.inputBuffer.getChannelData(0);
const downsampled = downsampleSpeech(inputBuffer, speech.audioContext.sampleRate, SPEECH_SAMPLE_RATE);
updateSpeechCaptureStats(downsampled);
setSpeechStreamLevel(stream, speechStreamLevelFromSamples(downsampled));
speech.chunks.push(floatToSpeechPCM(downsampled));
};
speech.source.connect(speech.processor);
speech.processor.connect(speech.zeroGain);
speech.zeroGain.connect(speech.audioContext.destination);
speech.recording = true;
syncChatSpeechButton();
setSpeechStreamActive(stream, true);
input.focus();
setChatSpeechStatus("warn", "Listening. Press the mic again to stop.");
trackChatEvent("chat_speech_started");
Expand Down Expand Up @@ -871,6 +911,7 @@ func (s Server) handleChat(w http.ResponseWriter, r *http.Request) {
speech.zeroGain = null;
speech.chunks = [];
resetSpeechCaptureStats();
setSpeechStreamActive(activeChatSpeechStream(), false);
syncChatSpeechButton();
}

Expand Down Expand Up @@ -1078,6 +1119,12 @@ func (s Server) handleChat(w http.ResponseWriter, r *http.Request) {
input.rows = 3;
input.placeholder = "Prompt for default branch";
input.setAttribute("aria-label", "Prompt for " + String(repo.full_name || repo.name || "repository"));
const inputWrap = document.createElement("span");
inputWrap.className = "speech-stream-wrap chat-repo-prompt-wrap";
const speechStream = document.createElement("span");
speechStream.className = "speech-stream chat-speech-stream";
speechStream.setAttribute("aria-hidden", "true");
inputWrap.append(input, speechStream);

const panelStatus = document.createElement("span");
panelStatus.className = "chat-repo-submit-status";
Expand All @@ -1096,7 +1143,7 @@ func (s Server) handleChat(w http.ResponseWriter, r *http.Request) {
input.addEventListener("pointerdown", (event) => {
event.stopPropagation();
});
panel.append(input, panelStatus);
panel.append(inputWrap, panelStatus);
card.appendChild(panel);

const openPrompt = () => {
Expand Down
59 changes: 52 additions & 7 deletions internal/web/static/index.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -185,12 +185,15 @@
</div>
<label id="builder-prompt-field" class="prompt-field">
<span class="prompt-label">Prompt</span>
<textarea
id="builder-prompt-input"
class="prompt-text prompt-control"
rows="6"
spellcheck="false"
placeholder="Describe the change you want"></textarea>
<span class="speech-stream-wrap">
<textarea
id="builder-prompt-input"
class="prompt-text prompt-control"
rows="6"
spellcheck="false"
placeholder="Describe the change you want"></textarea>
<span id="builder-speech-stream" class="speech-stream" aria-hidden="true"></span>
</span>
</label>
</div>

Expand Down Expand Up @@ -1913,6 +1916,7 @@ <h2 id="hub-setup-title">Connect to Hub</h2>
const localPromptStatusDefaultText = String(localPromptStatus?.dataset.defaultText || "").trim();
let localPromptStatusTimer = 0;
const promptSpeechToggle = document.getElementById("prompt-speech-toggle");
const builderSpeechStream = document.getElementById("builder-speech-stream");
const promptModeBuilder = document.getElementById("prompt-mode-builder");
const promptModeLibrary = document.getElementById("prompt-mode-library");
const promptModeJSON = document.getElementById("prompt-mode-json");
Expand Down Expand Up @@ -5167,6 +5171,38 @@ <h2 id="hub-setup-title">Connect to Hub</h2>
replaceLucideIcons(promptSpeechToggle);
}

function clampSpeechStreamLevel(level) {
if (!Number.isFinite(level)) return 0;
return Math.max(0, Math.min(1, level));
}

function setSpeechStreamActive(stream, active) {
if (!(stream instanceof HTMLElement)) return;
stream.classList.toggle("is-active", Boolean(active));
if (!active) {
stream.style.setProperty("--speech-level", "0");
}
}

function setSpeechStreamLevel(stream, level) {
if (!(stream instanceof HTMLElement)) return;
stream.style.setProperty("--speech-level", clampSpeechStreamLevel(level).toFixed(3));
}

function speechStreamLevelFromSamples(input) {
if (!input || !input.length) return 0;
let sum = 0;
let peak = 0;
for (let i = 0; i < input.length; i += 1) {
const sample = Math.max(-1, Math.min(1, Number(input[i]) || 0));
const absSample = Math.abs(sample);
if (absSample > peak) peak = absSample;
sum += sample * sample;
}
const rms = Math.sqrt(sum / input.length);
return Math.max(peak * 1.8, rms * 16);
}

async function refreshSpeechStatus() {
try {
const response = await fetch("/api/speech/status", { cache: "no-store" });
Expand Down Expand Up @@ -5239,13 +5275,15 @@ <h2 id="hub-setup-title">Connect to Hub</h2>
const input = event.inputBuffer.getChannelData(0);
const downsampled = downsampleSpeech(input, state.speech.audioContext.sampleRate, SPEECH_SAMPLE_RATE);
updateSpeechCaptureStats(downsampled);
setSpeechStreamLevel(builderSpeechStream, speechStreamLevelFromSamples(downsampled));
state.speech.chunks.push(floatToSpeechPCM(downsampled));
};
state.speech.source.connect(state.speech.processor);
state.speech.processor.connect(state.speech.zeroGain);
state.speech.zeroGain.connect(state.speech.audioContext.destination);
state.speech.recording = true;
syncPromptSpeechButton();
setSpeechStreamActive(builderSpeechStream, true);
setLocalPromptStatus("warn", "Listening. Press the mic again to stop.");
trackAnalyticsEvent("prompt_speech_started", { prompt_mode: state.promptMode });
} catch (err) {
Expand Down Expand Up @@ -5324,6 +5362,7 @@ <h2 id="hub-setup-title">Connect to Hub</h2>
state.speech.zeroGain = null;
state.speech.chunks = [];
resetSpeechCaptureStats();
setSpeechStreamActive(builderSpeechStream, false);
syncPromptSpeechButton();
}

Expand Down Expand Up @@ -6556,6 +6595,12 @@ <h2 id="hub-setup-title">Connect to Hub</h2>
if (repoKey && state.chatPromptDrafts.has(repoKey)) {
prompt.value = state.chatPromptDrafts.get(repoKey);
}
const promptWrap = document.createElement("span");
promptWrap.className = "speech-stream-wrap chat-repo-prompt-wrap";
const speechStream = document.createElement("span");
speechStream.className = "speech-stream chat-speech-stream";
speechStream.setAttribute("aria-hidden", "true");
promptWrap.append(prompt, speechStream);

const pasteTarget = document.createElement("div");
pasteTarget.className = "prompt-control prompt-action-paste chat-repo-image-paste";
Expand Down Expand Up @@ -6634,7 +6679,7 @@ <h2 id="hub-setup-title">Connect to Hub</h2>
event.stopPropagation();
});
imageActions.append(pasteTarget, submitStatus);
panel.append(promptLog, prompt, imageActions);
panel.append(promptLog, promptWrap, imageActions);
card.appendChild(panel);

const openPrompt = () => {
Expand Down
62 changes: 62 additions & 0 deletions internal/web/static/style.css
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1711,6 +1711,64 @@ select.prompt-control {
background: var(--surface-control-bg-strong);
}

.speech-stream-wrap {
position: relative;
display: block;
min-width: 0;
}

.speech-stream {
--speech-level: 0;
position: absolute;
top: 9px;
left: 10px;
right: 10px;
height: 18px;
overflow: hidden;
border-radius: 999px;
opacity: 0;
pointer-events: none;
transform: translateY(-2px);
transition: opacity 0.16s ease, transform 0.16s ease;
}

.speech-stream.is-active {
opacity: calc(0.28 + (var(--speech-level) * 0.72));
transform: translateY(0);
}

.speech-stream::before {
content: "";
position: absolute;
inset: 0;
width: 220%;
background:
linear-gradient(90deg, transparent 0%, rgba(10, 132, 255, 0.2) 12%, rgba(10, 132, 255, 0.68) 22%, transparent 34%),
repeating-linear-gradient(90deg, transparent 0 8px, rgba(10, 132, 255, 0.32) 8px 10px, transparent 10px 16px);
filter: drop-shadow(0 0 5px rgba(10, 132, 255, 0.28));
transform: translateX(0);
animation: speech-stream-flow 1.1s linear infinite;
}

.speech-stream::after {
content: "";
position: absolute;
inset: 4px 0;
background: linear-gradient(90deg, transparent, rgba(10, 132, 255, 0.54), transparent);
opacity: calc(0.16 + var(--speech-level) * 0.38);
transform: scaleY(calc(0.35 + (var(--speech-level) * 0.9)));
transform-origin: center;
}

@keyframes speech-stream-flow {
from {
transform: translateX(0);
}
to {
transform: translateX(-50%);
}
}

#builder-prompt-input,
#local-prompt-input {
width: 100%;
Expand Down Expand Up @@ -3699,6 +3757,10 @@ select.prompt-control {
line-height: 1.35;
}

.chat-repo-prompt-wrap {
display: block;
}

.chat-repo-prompt:focus {
border-color: var(--border-strong);
outline: none;
Expand Down