"use client"; import { Button } from '@/components/ui/button'; import { ScrollArea } from '@/components/ui/scroll-area'; import { Textarea } from '@/components/ui/textarea'; import { Bot, Paperclip, SendHorizonal, MessageSquarePlus } from 'lucide-react'; import React, { useEffect, useRef, useState } from 'react'; import { ChatMessage, LoadingMessage, ImageLoadingMessage } from './chat-message'; import { useIsMobile } from '@/hooks/use-mobile'; import type { Message } from '@/app/page'; type ChatPanelProps = { messages: Message[]; onSendMessage: (input: string, temperature: number, file: File | null) => Promise; isLoading: boolean; isChatSelected: boolean; onNewChat: () => void; onConsciousnessDimensionSelect: (dimension: string) => void; selectedConsciousnessDimension: string; temperature: number; isWelcomeMode?: boolean; consciousnessColor?: string; onImageGeneratingChange?: (isGenerating: boolean) => void; isImageGenerating?: boolean; onOpenEditor?: (b64: string, prompt?: string) => void; imageProgress?: number; imageElapsed?: number; imageStatus?: string; }; export function ChatPanel({ messages, onSendMessage, isLoading, isChatSelected, onNewChat, onConsciousnessDimensionSelect, selectedConsciousnessDimension, temperature, isWelcomeMode = false, consciousnessColor = 'text-blue-600', onImageGeneratingChange, isImageGenerating, onOpenEditor, imageProgress = 0, imageElapsed = 0, imageStatus = 'starting', }: ChatPanelProps) { const isMobile = useIsMobile(); const [input, setInput] = useState(''); const [file, setFile] = useState(null); const [imageMode, setImageMode] = useState(false); const [safeMode, setSafeMode] = useState(true); const scrollAreaRef = useRef(null); const messagesEndRef = useRef(null); const textareaRef = useRef(null); const fileInputRef = useRef(null); // Compute backend/proxy base. Always use same-origin Next API proxy in the browser // to avoid CORS/mixed-origin issues (e.g., when dev host is 127.0.0.2). // Fallback to direct backend only during SSR where window is not available. function getApiUrl(path: string): string { const p = path.startsWith('/api') ? path : `/api${path}`; if (typeof window !== 'undefined') { return p; } const direct = process.env.NEXT_PUBLIC_BACKEND_URL || ''; return direct ? `${direct}${path}` : p; } // Compute backend base with runtime fallback when env is missing function getBackendBase(): string { const fromEnv = process.env.NEXT_PUBLIC_BACKEND_URL || ''; if (fromEnv) return fromEnv; if (typeof window !== 'undefined') { if (window.location.hostname === 'localhost' || window.location.hostname === '127.0.0.1') { return 'http://localhost:5000'; } } return ''; } // Speech mode state const [speechMode, setSpeechMode] = useState(false); const [isRecording, setIsRecording] = useState(false); const mediaStreamRef = useRef(null); const audioCtxRef = useRef(null); const sourceRef = useRef(null); const processorRef = useRef(null); const analyserRef = useRef(null); const pcmChunksRef = useRef([]); const mediaRecorderRef = useRef(null); const recChunksRef = useRef([]); const levelsRef = useRef([0,0,0,0,0]); const [, forceRender] = useState(0); // force rerender for animation const lastActiveRef = useRef(Date.now()); const startedAtRef = useRef(Date.now()); const SILENCE_MS = 1500; const MIN_CAPTURE_MS = 800; const SILENCE_THRESHOLD = 0.008; // less aggressive // Track last-spoken assistant message to avoid repeated TTS const lastSpokenIdRef = useRef(null); const ttsAudioRef = useRef(null); // Auto-scroll to bottom when messages change useEffect(() => { const scrollToBottom = () => { if (messagesEndRef.current) { messagesEndRef.current.scrollIntoView({ behavior: 'smooth' }); } }; // Scroll immediately scrollToBottom(); // Also scroll after a short delay to handle any dynamic content loading const timeoutId = setTimeout(scrollToBottom, 100); return () => clearTimeout(timeoutId); }, [messages, isLoading]); // Focus textarea when not loading useEffect(() => { if (!isLoading && textareaRef.current) { textareaRef.current.focus(); } }, [isLoading]); // Animated waveform (5 bars) + silence detection useEffect(() => { let rafId: number; const tick = () => { if (isRecording && analyserRef.current) { const analyser = analyserRef.current; const arr = new Uint8Array(analyser.fftSize); analyser.getByteTimeDomainData(arr); // RMS amplitude let sum = 0; for (let i = 0; i < arr.length; i++) { const v = (arr[i] - 128) / 128; sum += v * v; } const rms = Math.sqrt(sum / arr.length); const level = Math.min(1, rms * 4); const prev = levelsRef.current.slice(); prev.shift(); prev.push(level); levelsRef.current = prev; forceRender((x) => x + 1); // Silence detection if (rms > SILENCE_THRESHOLD) { lastActiveRef.current = Date.now(); } else if ( Date.now() - lastActiveRef.current > SILENCE_MS && Date.now() - startedAtRef.current > MIN_CAPTURE_MS ) { // Auto stop after 1.5s silence stopRecording(); } } rafId = requestAnimationFrame(tick); }; rafId = requestAnimationFrame(tick); return () => cancelAnimationFrame(rafId); }, [isRecording]); const startRecording = async () => { try { const stream = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true, channelCount: 1, } as any, } as any); mediaStreamRef.current = stream; const audioCtx = new (window.AudioContext || (window as any).webkitAudioContext)(); audioCtxRef.current = audioCtx; if (audioCtx.state === 'suspended') { try { await audioCtx.resume(); } catch {} } const source = audioCtx.createMediaStreamSource(stream); sourceRef.current = source; const analyser = audioCtx.createAnalyser(); analyser.fftSize = 1024; analyserRef.current = analyser; // MediaRecorder fallback (WebM/Opus) try { const mr = new MediaRecorder(stream, { mimeType: 'audio/webm;codecs=opus' } as any); mediaRecorderRef.current = mr; recChunksRef.current = []; mr.ondataavailable = (ev) => { if (ev.data && ev.data.size > 0) recChunksRef.current.push(ev.data); }; mr.start(250); // gather in 250ms chunks } catch {} // Sink to keep processing running while staying silent const silentGain = audioCtx.createGain(); silentGain.gain.value = 0.0; pcmChunksRef.current = []; const now = Date.now(); lastActiveRef.current = now; startedAtRef.current = now; const processor = audioCtx.createScriptProcessor(4096, 1, 1); processorRef.current = processor; processor.onaudioprocess = (e: AudioProcessingEvent) => { if (!isRecording) return; const input = e.inputBuffer.getChannelData(0); // Copy the buffer since input is a live view const buf = new Float32Array(input.length); buf.set(input); pcmChunksRef.current.push(buf); }; // Connect graph: source -> analyser (for levels) and -> processor (for capture) source.connect(analyser); source.connect(processor); processor.connect(silentGain); silentGain.connect(audioCtx.destination); setIsRecording(true); } catch (err) { console.error('Mic access failed:', err); setSpeechMode(false); } }; const stopRecording = async () => { if (!isRecording) return; setIsRecording(false); // stop tracks mediaStreamRef.current?.getTracks().forEach((t) => t.stop()); mediaStreamRef.current = null; // disconnect audio graph try { processorRef.current?.disconnect(); analyserRef.current?.disconnect(); sourceRef.current?.disconnect(); if (mediaRecorderRef.current && mediaRecorderRef.current.state !== 'inactive') { mediaRecorderRef.current.stop(); } await audioCtxRef.current?.close(); } catch {} const sampleRate = audioCtxRef.current?.sampleRate || 48000; processorRef.current = null; analyserRef.current = null; sourceRef.current = null; audioCtxRef.current = null; // Build mono WAV from Float32 chunks using native sample rate const floatData = concatFloat32(pcmChunksRef.current); let wavBlob: Blob; if (!floatData || floatData.length === 0) { // Fallback: decode MediaRecorder chunks to PCM via AudioContext try { const webmBlob = new Blob(recChunksRef.current, { type: 'audio/webm' }); recChunksRef.current = []; const arr = await webmBlob.arrayBuffer(); const decodeCtx = new (window.AudioContext || (window as any).webkitAudioContext)(); const audioBuf = await decodeCtx.decodeAudioData(arr); const ch0 = audioBuf.getChannelData(0); const copy = new Float32Array(ch0.length); copy.set(ch0); wavBlob = encodeWAV(copy, 1, audioBuf.sampleRate || sampleRate); try { await decodeCtx.close(); } catch {} } catch (e) { console.warn('No audio captured (empty buffer) - aborting ASR request'); return; } } else { wavBlob = encodeWAV(floatData, 1, sampleRate); } pcmChunksRef.current = []; mediaRecorderRef.current = null; // Send to ASR backend const base64 = await blobToBase64(wavBlob); try { const res = await fetch(getApiUrl('/asr/transcribe'), { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ audio_base64: base64, language: 'en', vad: false, beam_size: 5 }), }); const ct = res.headers.get('content-type') || ''; const payload = ct.includes('application/json') ? await res.json() : { success: false, error: `${res.status} ${await res.text()}` }; if (payload?.success) { const primary = (payload?.text ?? '').toString(); const fallback = Array.isArray(payload?.segments) ? payload.segments.map((s: any) => s?.text || '').join(' ') : ''; const text = (primary || fallback).trim(); if (text.length > 0) { setInput(text); // Auto-send when in speech mode if (speechMode) { const messageToSend = imageMode ? `[[IMAGE_MODE]] ${text}${safeMode ? ' [[SAFE_MODE=ON]]' : ' [[SAFE_MODE=OFF]]'}` : text; await onSendMessage(messageToSend, temperature, file); setInput(''); setFile(null); if (fileInputRef.current) fileInputRef.current.value = ''; } } else { console.warn('ASR returned empty text', payload); } } else { console.warn('ASR failed:', payload?.error || `status ${res.status}`); } } catch (e) { console.error('ASR request failed:', e); } }; const toggleSpeech = async () => { const next = !speechMode; setSpeechMode(next); if (next) { await startRecording(); } else if (isRecording) { await stopRecording(); } }; // Auto TTS of assistant replies in speech mode useEffect(() => { if (!speechMode) return; if (!messages || messages.length === 0) return; const last = messages[messages.length - 1]; if (last.role !== 'assistant' || !last.content?.trim()) return; // Compose a simple id from content + index const msgId = `${messages.length}-${last.content.length}`; if (lastSpokenIdRef.current === msgId) return; lastSpokenIdRef.current = msgId; const runTts = async () => { try { const res = await fetch(getApiUrl('/tts/synthesize'), { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ text: last.content }), }); const ct = res.headers.get('content-type') || ''; const payload = ct.includes('application/json') ? await res.json() : { success: false, error: `${res.status} ${await res.text()}` }; if (payload?.success && payload?.audio_base64_wav) { const audio = new Audio(`data:audio/wav;base64,${payload.audio_base64_wav}`); ttsAudioRef.current?.pause(); ttsAudioRef.current = audio; audio.play().catch(() => {}); } else { console.warn('TTS failed:', payload?.error || `status ${res.status}`); } } catch (e) { console.error('TTS request failed:', e); } }; runTts(); }, [messages, speechMode]); // Helpers: concat, WAV encode, base64 function concatFloat32(chunks: Float32Array[]): Float32Array { let length = 0; for (const c of chunks) length += c.length; const out = new Float32Array(length); let o = 0; for (const c of chunks) { out.set(c, o); o += c.length; } return out; } function encodeWAV(samples: Float32Array, numChannels: number, sampleRate: number): Blob { // Convert float to 16-bit PCM const pcm = new Int16Array(samples.length); for (let i = 0; i < samples.length; i++) { let s = Math.max(-1, Math.min(1, samples[i])); pcm[i] = s < 0 ? s * 0x8000 : s * 0x7fff; } const byteRate = (sampleRate * numChannels * 16) / 8; const blockAlign = (numChannels * 16) / 8; const buffer = new ArrayBuffer(44 + pcm.byteLength); const view = new DataView(buffer); let offset = 0; // RIFF header writeString(view, offset, 'RIFF'); offset += 4; view.setUint32(offset, 36 + pcm.byteLength, true); offset += 4; writeString(view, offset, 'WAVE'); offset += 4; writeString(view, offset, 'fmt '); offset += 4; view.setUint32(offset, 16, true); offset += 4; // fmt chunk size view.setUint16(offset, 1, true); offset += 2; // PCM view.setUint16(offset, numChannels, true); offset += 2; view.setUint32(offset, sampleRate, true); offset += 4; view.setUint32(offset, byteRate, true); offset += 4; view.setUint16(offset, blockAlign, true); offset += 2; view.setUint16(offset, 16, true); offset += 2; writeString(view, offset, 'data'); offset += 4; view.setUint32(offset, pcm.byteLength, true); offset += 4; // PCM data const pcmView = new DataView(buffer, 44); for (let i = 0; i < pcm.length; i++) { pcmView.setInt16(i * 2, pcm[i], true); } return new Blob([buffer], { type: 'audio/wav' }); } function writeString(view: DataView, offset: number, str: string) { for (let i = 0; i < str.length; i++) view.setUint8(offset + i, str.charCodeAt(i)); } function blobToBase64(blob: Blob): Promise { return new Promise((resolve) => { const reader = new FileReader(); reader.onloadend = () => { const res = (reader.result as string) || ''; const base64 = res.split(',')[1] || ''; resolve(base64); }; reader.readAsDataURL(blob); }); } const handleFileChange = (e: React.ChangeEvent) => { const selectedFile = e.target.files?.[0]; if (selectedFile) { setFile(selectedFile); } }; const handleSubmit = async (e: React.FormEvent) => { e.preventDefault(); if (!input.trim() || isLoading) return; // Only attach IMAGE_MODE when the toggle is ON. No auto intent here. const messageToSend = imageMode ? `[[IMAGE_MODE]] ${input}${safeMode ? ' [[SAFE_MODE=ON]]' : ' [[SAFE_MODE=OFF]]'}` : input; // Notify image generation start/end to parent if (imageMode) onImageGeneratingChange?.(true); await onSendMessage(messageToSend, temperature, file); if (imageMode) onImageGeneratingChange?.(false); setInput(''); setFile(null); if (fileInputRef.current) { fileInputRef.current.value = ''; } }; const handleKeyDown = (e: React.KeyboardEvent) => { if (e.key === 'Enter' && !e.shiftKey) { e.preventDefault(); handleSubmit(e as any); } }; const removeFile = () => { setFile(null); if (fileInputRef.current) { fileInputRef.current.value = ''; } }; if (isWelcomeMode) { // Welcome mode - just show the input form return (
{/* File attachment display */} {file && (
{file.name}
)} {/* Input form */}
{/* Mic/Wave button (opposite side of send) */}