[Feat] 발화자 하이라이팅 구현

apps/web/src/components/live/StreamView/List/SubVideoGrid.tsx

@@ -4,18 +4,6 @@ import { useEffect, useRef, useState } from 'react';
 import { StreamData } from '@/components/live/StreamView';
 import VideoPlayer from '@/components/live/StreamView/List/VideoPlayer';
 
-const highlightVariants = cva(`h-full w-full overflow-hidden rounded-lg border-2`, {
-  variants: {
-    pinned: {
-      true: 'border-primary',
-      false: 'border-transparent',
-    },
-  },
-  defaultVariants: {
-    pinned: false,
-  },
-});
-
 interface SubVideoGridProps {
   videoStreamData: StreamData[];
   pinnedVideoStreamData: StreamData | null;
@@ -31,6 +19,7 @@ function SubVideoGrid({
 }: SubVideoGridProps) {
   const containerRef = useRef<HTMLDivElement>(null);
   const [videoMaxWidth, setVideoMaxWidth] = useState(0);
+
   useEffect(() => {
     // TODO: throttle resize event
     const adjustSize = () => {
@@ -54,9 +43,7 @@ function SubVideoGrid({
         <div
           key={`${streamData.socketId}${idx}`}
           style={{ maxWidth: videoMaxWidth }}
-          className={highlightVariants({
-            pinned: pinnedVideoStreamData?.stream?.id === streamData.stream?.id,
-          })}
+          className="h-full w-full overflow-hidden rounded-lg"
           onClick={() => streamData.stream && onVideoClick(streamData)}
         >
           <VideoPlayer
@@ -66,6 +53,8 @@ function SubVideoGrid({
             stream={streamData.stream ?? null}
             isMicOn={streamData && getAudioMutedState(streamData)}
             mediaType={streamData.consumer?.appData?.mediaTypes}
+            socketId={streamData.socketId}
+            isPinned={pinnedVideoStreamData?.stream?.id === streamData.stream?.id}
           />
         </div>
       ))}

apps/web/src/components/live/StreamView/List/VideoGrid.tsx

@@ -37,6 +37,7 @@ function VideoGrid({ videoStreamData, onVideoClick, getAudioMutedState }: VideoG
             stream={streamData.stream ?? null}
             isMicOn={streamData && getAudioMutedState(streamData)}
             mediaType={streamData.consumer?.appData?.mediaTypes}
+            socketId={streamData.socketId}
           />
         </div>
       ))}

apps/web/src/components/live/StreamView/List/VideoPlayer.tsx

@@ -3,22 +3,32 @@ import { memo, useEffect, useRef, useState } from 'react';
 
 import MicOffIc from '@/assets/icons/mic-off.svg?react';
 import MicOnIc from '@/assets/icons/mic-on.svg?react';
+import PinIc from '@/assets/icons/pin.svg?react';
 import Avatar from '@/components/common/Avatar';
 import Badge from '@/components/common/Badge';
 import Loading from '@/components/common/Loading';
+import useAudioLevelDetector from '@/hooks/mediasoup/useAudioLevelDetector';
 import cn from '@/utils/cn';
 
-const videoVariants = cva('absolute h-full w-full object-cover transition-opacity duration-300', {
-  variants: {
-    loading: {
-      true: 'opacity-0',
-      false: 'opacity-100',
+const videoVariants = cva(
+  'absolute h-full w-full rounded-lg object-cover transition-opacity duration-300 [transform:rotateY(180deg)]',
+  {
+    variants: {
+      loading: {
+        true: 'opacity-0',
+        false: 'opacity-100',
+      },
+      isSpeaking: {
+        true: 'border-4 border-primary',
+        false: 'border-4 border-alt',
+      },
     },
-  },
-  defaultVariants: {
-    loading: true,
-  },
-});
+    defaultVariants: {
+      loading: true,
+      isSpeaking: false,
+    },
+  }
+);
 
 export interface VideoPlayerProps {
   stream: MediaStream | null;
@@ -27,6 +37,8 @@ export interface VideoPlayerProps {
   avatarSize?: 'sm' | 'md' | 'lg';
   mediaType?: string;
   nickname: string;
+  socketId?: string;
+  isPinned?: boolean;
 }
 
 function VideoPlayer({
@@ -36,10 +48,15 @@ function VideoPlayer({
   isMicOn = false,
   avatarSize = 'md',
   nickname,
+  socketId,
+  isPinned = false,
 }: VideoPlayerProps) {
   const [isLoading, setIsLoading] = useState(true);
   const videoRef = useRef<HTMLVideoElement>(null);
 
+  const { activeSocketId } = useAudioLevelDetector();
+  const isSpeaking = activeSocketId === socketId;
+
   useEffect(() => {
     if (!videoRef.current) return;
 
@@ -60,7 +77,10 @@ function VideoPlayer({
             autoPlay
             playsInline
             preload="metadata"
-            className={videoVariants({ loading: isLoading })}
+            className={videoVariants({
+              loading: isLoading,
+              isSpeaking,
+            })}
             onLoadedData={onLoadedData}
           >
             <track default kind="captions" srcLang="en" src="SUBTITLE_PATH" />
@@ -69,7 +89,7 @@ function VideoPlayer({
             Your browser does not support the video.
           </video>
         ) : (
-          <div className={videoVariants({ loading: false })}>
+          <div className={videoVariants({ loading: false, isSpeaking })}>
             <Avatar
               size={avatarSize}
               className="absolute left-1/2 top-1/2 -translate-x-1/2 -translate-y-1/2 transform"
@@ -92,9 +112,17 @@ function VideoPlayer({
       {stream && (
         <>
           {mediaType === 'video' && (
-            <div className="absolute right-3 top-3 flex h-8 w-8 items-center justify-center rounded-full bg-altWeak p-1">
-              {isMicOn ? <MicOnIc className="text-white" /> : <MicOffIc className="fill-white" />}
-            </div>
+            <>
+              <div className="absolute right-3 top-3 flex h-8 w-8 items-center justify-center rounded-full bg-altWeak p-1">
+                {isMicOn ? <MicOnIc className="text-white" /> : <MicOffIc className="fill-white" />}
+              </div>
+
+              {isPinned && (
+                <div className="absolute left-3 top-3 flex h-8 w-8 items-center justify-center rounded-full bg-primary p-1">
+                  <PinIc className="fill-white" />
+                </div>
+              )}
+            </>
           )}
         </>
       )}

apps/web/src/components/live/StreamView/index.tsx

@@ -1,12 +1,13 @@
 import { types } from 'mediasoup-client';
 import { MediaTypes } from '@repo/mediasoup';
 
-import AudioStreams from '@/components/live/StreamView/AudioStreams';
 import PinnedGrid from '@/components/live/StreamView/List/Pinned';
 import UnPinnedGrid from '@/components/live/StreamView/List/UnPinned';
 import useAudioState from '@/hooks/useAudioState';
 import usePinnedVideo from '@/hooks/usePinnedVideo';
 
+import AudioStreams from './AudioStreams';
+
 export interface StreamData {
   socketId: string;
   nickname: string;

apps/web/src/hooks/mediasoup/useAudioLevelDetector.ts

@@ -0,0 +1,187 @@
+import { useEffect, useRef, useState } from 'react';
+import { client } from '@repo/mediasoup';
+
+import { useRemoteStreamState } from '@/contexts/remoteStream/context';
+
+interface AudioLevelData {
+  socketId: string;
+  audioLevel: number;
+  analyser: AnalyserNode;
+  dataArray: Float32Array;
+}
+
+const useAudioLevelDetector = () => {
+  const { audioStreams } = useRemoteStreamState();
+
+  const audioContextRef = useRef<AudioContext | null>(null);
+  const audioLevelsRef = useRef<AudioLevelData[]>([]);
+  const intervalRef = useRef<number>();
+
+  const lastActiveTimeRef = useRef<number>(0);
+  const currentSpeakerRef = useRef<string | null>(null);
+
+  const [activeSocketId, setActiveSocketId] = useState<string | null>(null);
+
+  const startAudioLevelDetection = () => {
+    const AUDIO_THRESHOLD = 0.01;
+    const SPEECH_END_DELAY = 1000;
+
+    const detectAudioLevels = () => {
+      const unmutedStreamIds = new Set(
+        audioStreams.filter((stream) => !stream.paused).map((stream) => stream.socketId)
+      );
+
+      const unmutedAudioLevels = audioLevelsRef.current.filter((data) =>
+        unmutedStreamIds.has(data.socketId)
+      );
+
+      if (!unmutedAudioLevels.length) return;
+
+      let maxLevel = 0;
+      let maxLevelSocketId = null;
+
+      unmutedAudioLevels.forEach((levelData) => {
+        const { analyser, dataArray, socketId } = levelData;
+        analyser.getFloatTimeDomainData(dataArray);
+
+        let sum = 0;
+        for (const amplitude of dataArray) {
+          sum += amplitude * amplitude;
+        }
+        const level = Math.sqrt(sum / dataArray.length);
+
+        if (level > maxLevel) {
+          maxLevel = level;
+          maxLevelSocketId = socketId;
+        }
+      });
+
+      if (maxLevel > AUDIO_THRESHOLD) {
+        lastActiveTimeRef.current = Date.now();
+
+        if (currentSpeakerRef.current === null) {
+          currentSpeakerRef.current = maxLevelSocketId;
+          setActiveSocketId(maxLevelSocketId);
+        } else if (maxLevelSocketId !== currentSpeakerRef.current) {
+          currentSpeakerRef.current = maxLevelSocketId;
+          setActiveSocketId(maxLevelSocketId);
+        }
+      } else {
+        const activeTime = Date.now() - lastActiveTimeRef.current;
+
+        if (activeTime > SPEECH_END_DELAY && currentSpeakerRef.current !== null) {
+          currentSpeakerRef.current = null;
+          setActiveSocketId(null);
+        }
+      }
+    };
+
+    if (intervalRef.current) {
+      clearInterval(intervalRef.current);
+    }
+
+    intervalRef.current = setInterval(detectAudioLevels, 300);
+  };
+
+  const resetAudioContext = () => {
+    if (audioContextRef.current?.state === 'closed') {
+      audioContextRef.current = new AudioContext();
+
+      const audioContext = audioContextRef.current;
+
+      audioStreams.forEach((stream) => {
+        if (stream.kind === 'audio') {
+          const source = audioContext.createMediaStreamSource(stream.stream);
+          const analyser = audioContext.createAnalyser();
+          analyser.fftSize = 256;
+          source.connect(analyser);
+
+          const dataArray = new Float32Array(analyser.frequencyBinCount);
+
+          const audioLevelData: AudioLevelData = {
+            socketId: stream.socketId,
+            audioLevel: 0,
+            analyser,
+            dataArray,
+          };
+
+          audioLevelsRef.current = [...audioLevelsRef.current, audioLevelData];
+        }
+      });
+
+      startAudioLevelDetection();
+    }
+  };
+
+  const createAudioLevel = (remoteStream: client.RemoteStream) => {
+    resetAudioContext();
+
+    if (audioContextRef.current?.state === 'closed') {
+      audioContextRef.current = null;
+    }
+
+    if (!audioContextRef.current) {
+      audioContextRef.current = new AudioContext();
+    }
+
+    const audioContext = audioContextRef.current;
+    const audioLevels = audioLevelsRef.current;
+
+    const isExist = audioLevels.some((data) => data.socketId === remoteStream.socketId);
+    if (isExist || remoteStream.kind !== 'audio') return;
+
+    const source = audioContext.createMediaStreamSource(remoteStream.stream);
+    const analyser = audioContext.createAnalyser();
+    analyser.fftSize = 256;
+    source.connect(analyser);
+
+    const dataArray = new Float32Array(analyser.frequencyBinCount);
+
+    const audioLevelData: AudioLevelData = {
+      socketId: remoteStream.socketId,
+      audioLevel: 0,
+      analyser,
+      dataArray,
+    };
+
+    audioLevelsRef.current = [...audioLevels, audioLevelData];
+
+    if (audioLevelsRef.current.length === 1) {
+      startAudioLevelDetection();
+    }
+  };
+
+  useEffect(() => {
+    const streams = audioStreams.filter(
+      (stream) => !audioLevelsRef.current.some((level) => level.socketId === stream.socketId)
+    );
+
+    streams.forEach((stream) => {
+      if (stream.paused) return;
+      createAudioLevel(stream);
+    });
+  }, [audioStreams]);
+
+  useEffect(() => {
+    return () => {
+      if (intervalRef.current) {
+        clearInterval(intervalRef.current);
+      }
+
+      if (audioContextRef.current && audioContextRef.current.state !== 'closed') {
+        audioContextRef.current.close();
+      }
+
+      currentSpeakerRef.current = null;
+      setActiveSocketId(null);
+    };
+  }, []);
+
+  return {
+    audioLevelsRef,
+    activeSocketId,
+    createAudioLevel,
+  };
+};
+
+export default useAudioLevelDetector;

apps/web/tailwind.config.ts

@@ -38,6 +38,8 @@ const tailwindConfig: Config = {
       borderColor: {
         main: 'var(--grey-300)',
         primary: 'var(--purple-500)',
+        black: 'var(--black)',
+        alt: 'var(--grey-700)',
         error: 'var(--red)',
       },
       fill: {