-refactor room

Dockerfile

@@ -3,8 +3,6 @@ FROM node:20-alpine AS frontend
 RUN apk add --no-cache libc6-compat
 WORKDIR /app
 
-ARG DUMMY_ARG=1
-
 COPY streaming-react-app .
 RUN \
     if [ -f yarn.lock ]; then yarn --frozen-lockfile; \
@@ -47,55 +45,21 @@ RUN apt-get update && \
     # gradio dependencies \
     ffmpeg \
     # fairseq2 dependencies \
+    libjpeg8-dev \
+    libpng-dev \
     libsndfile-dev && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
-
-# RUN apt-get install zlib1g-dev &&\
-#     wget https://sourceforge.net/projects/libpng/files/libpng15/1.5.30/libpng-1.5.30.tar.gz &&\
-#     tar -xzvf libpng-1.5.30.tar.gz &&\
-#     cd libpng-1.5.30 &&\ 
-#     ./configure --prefix=/usr/local/libpng &&\
-#     make &&\
-#     make install &&\
-#     echo "Contents of /usr/local/lib:" && ls /usr/local/lib && \
-#     echo "Contents of /usr/local/libpng/lib:" && ls /usr/local/libpng/lib && \
-# #    ls /usr/local/lib/libpng* &&\
-#     ldconfig &&\
-#     cd ..
-# ENV LD_LIBRARY_PATH=/usr/local/libpng/lib:
-
-# RUN wget http://www.ijg.org/files/jpegsrc.v9a.tar.gz &&\
-#     tar -xzvf jpegsrc.v9a.tar.gz &&\
-#     cd jpeg-9a &&\ 
-#     ./configure --prefix=/usr/local/libjpeg &&\
-#     make &&\
-#     make install 
-# RUN cd .. &&\
-#     echo "Contents of /usr/local/lib :" && ls /usr/local && \
-#     echo "Contents of /usr/local/libjpeg/lib :" && ls /usr/local/libjpeg/lib && \
-#     ls /usr/local/lib/libpng* &&\
-#     ldconfig
-# # ENV LD_LIBRARY_PATH=/usr/local/libpng/lib:
-
-# RUN wget libjpeg62-turbo_2.0.6-4_amd64.deb &&\
-#     tar -xzvf jpegsrc.v9a.tar.gz &&\
-#     cd jpeg-9a &&\ 
-#     ./configure --prefix=/usr/local/libjpeg &&\
-#     make &&\
-#     make install 
-# RUN cd .. &&\
-#     echo "Contents of /usr/local/lib :" && ls /usr/local && \
-#     echo "Contents of /usr/local/libjpeg/lib :" && ls /usr/local/libjpeg/lib && \
-#     ls /usr/local/lib/libpng* &&\
-#     ldconfig
-# ENV LD_LIBRARY_PATH=/usr/local/libpng/lib:
-
-# RUN apt-get update &&\
-#     apt-get -y install libjpeg62-turbo-dev &&\
-#     apt-get install libjpeg8 libbodfile1
-
+USER root
+RUN ln -s /usr/lib/x86_64-linux-gnu/libsox.so.3 /usr/lib/x86_64-linux-gnu/libsox.so
+# install older versions libjpeg62-turbo and libpng15
+RUN wget http://ftp.us.debian.org/debian/pool/main/libj/libjpeg-turbo/libjpeg62-turbo_2.1.5-2_amd64.deb && \
+    dpkg -i libjpeg62-turbo_2.1.5-2_amd64.deb && \
+    rm libjpeg62-turbo_2.1.5-2_amd64.deb
+RUN wget https://master.dl.sourceforge.net/project/libpng/libpng15/1.5.30/libpng-1.5.30.tar.gz && \
+    tar -xvf libpng-1.5.30.tar.gz && cd libpng-1.5.30 && ./configure && make && make install && cd .. && rm -rf libpng-1.5.30.tar.gz libpng-1.5.30
+    
 RUN useradd -m -u 1000 user
 USER user
 ENV HOME=/home/user \
@@ -113,8 +77,7 @@ RUN pyenv install $PYTHON_VERSION && \
 COPY --chown=user:user ./seamless_server ./seamless_server
 # change dir since pip needs to seed whl folder
 RUN cd seamless_server && \
-    pip install fairseq2 &&\
-#    pip install fairseq2 --pre --extra-index-url https://fair.pkg.atmeta.com/fairseq2/whl/nightly/pt2.1.1/cu118 && \
+    pip install fairseq2 --pre --extra-index-url https://fair.pkg.atmeta.com/fairseq2/whl/nightly/pt2.1.1/cu118 && \
     pip install --no-cache-dir --upgrade -r requirements.txt
 COPY --from=frontend /app/dist ./streaming-react-app/dist
 
@@ -124,10 +87,7 @@ RUN --mount=type=secret,id=HF_TOKEN,mode=0444,required=false \
     huggingface-cli download meta-private/SeamlessExpressive pretssel_melhifigan_wm-final.pt  --local-dir ./models/Seamless/ || echo "HF_TOKEN error" && \
     ln -s $(readlink -f models/Seamless/pretssel_melhifigan_wm-final.pt) models/Seamless/pretssel_melhifigan_wm.pt || true;
 
-USER root
-RUN ln -s /usr/lib/x86_64-linux-gnu/libsox.so.3 /usr/lib/x86_64-linux-gnu/libsox.so
 USER user
 RUN ["chmod", "+x", "./run_docker.sh"]
 CMD ./run_docker.sh
 
-

README.md

@@ -10,10 +10,6 @@ models:
  - facebook/seamless-streaming
 ---
 
-# RESOURCES AWS
-I use a g4 machine with 35GB of volume. You could delete the volume and create a new one and connect it to the instance when needed. Obv you'd need to re-pull the docker image. 
-The same is true for the elastic IP. Both the volume and the elastic IP costs me ~0.10$ per day, for an overall of ~0.22$. It does not make sense to keep them if you don't need to use it for a couple of months.
-
 # Seamless Streaming demo
 ## Running on HF spaces
 You can simply duplicate the space to run it.
@@ -71,4 +67,4 @@ To enable additional logging from uvicorn pass `--log-level debug` or `--log-lev
 
 ### Debuging
 
-If you enable "Server Debug Flag" when starting streaming from the client, this enables extensive debug logging and it saves audio files in /debug folder. 
+If you enable "Server Debug Flag" when starting streaming from the client, this enables extensive debug logging and it saves audio files in /debug folder. 

seamless_server/src/simuleval_transcoder.py

@@ -19,8 +19,6 @@ import traceback
 import time
 import random
 import colorlog
-import nltk
-nltk.download('averaged_perceptron_tagger_eng')
 
 from .speech_and_text_output import SpeechAndTextOutput
 

streaming-react-app/src/RoomConfig.tsx

@@ -87,7 +87,6 @@ export default function RoomConfig({
         'join_room',
         clientID,
         createNewRoom ? null : roomID,
-        true,
         configObject,
         (result) => {
           console.log('join_room result:', result);
@@ -151,7 +150,7 @@ export default function RoomConfig({
   return (
     <Stack direction="column" spacing="12px">
       <Stack direction="row" spacing="12px" sx={{alignItems: 'center'}}>
-        {/* <TextField
+        <TextField
           size="small"
           label="Room Code"
           variant="outlined"
@@ -168,9 +167,9 @@ export default function RoomConfig({
             }
           }}
           sx={{width: '8em'}}
-        /> */}
+        />
 
-        {/* <div>
+        <div>
           <Button
             variant="contained"
             disabled={
@@ -181,7 +180,7 @@ export default function RoomConfig({
             onClick={() => onJoinRoom(false)}>
             {roomState?.room_id != null ? 'Update Roles' : 'Join Room'}
           </Button>
-        </div> */}
+        </div>
 
         {roomState?.room_id == null && (
           <div>
@@ -193,7 +192,7 @@ export default function RoomConfig({
                 streamingStatus !== 'stopped'
               }
               onClick={() => onJoinRoom(true)}>
-              {'Start translating'}
+              {'Create New Room'}
             </Button>
           </div>
         )}

streaming-react-app/src/StreamingInterface.tsx

@@ -6,7 +6,7 @@ import FormControl from '@mui/material/FormControl';
 import Select, {SelectChangeEvent} from '@mui/material/Select';
 import MenuItem from '@mui/material/MenuItem';
 import Stack from '@mui/material/Stack';
-import seamlessLogoUrl from './assets/illy.svg';
+import seamlessLogoUrl from './assets/seamless.svg';
 import {
   AgentCapabilities,
   BaseResponse,
@@ -744,44 +744,42 @@ export default function StreamingInterface() {
                 src={seamlessLogoUrl}
                 className="header-icon-sra"
                 alt="Seamless Translation Logo"
-                height={150}
-                width={225}
+                height={24}
+                width={24}
               />
 
               <div>
-                <Typography variant="h1" sx={{color: '#800020'}}>
-                  Illy's translator
-                </Typography>
-                <Typography variant="body2" sx={{color: '#800020'}}>
-                  <span style={{ fontStyle: 'italic' }}>
-                    Natale 2023 
-                  </span>
+                <Typography variant="h1" sx={{color: '#65676B'}}>
+                  Seamless Translation
                 </Typography>
               </div>
             </div>
             <div className="header-container-sra">
               <div>
                 <Typography variant="body2" sx={{color: '#65676B'}}>
-                  Hey <strong style={{ fontWeight: 'bold' }}>Illy</strong>, <strong style={{ fontWeight: 'bold' }}>it's lovely to see you!</strong>
-                  <br/>
-                  You can use this platform to translate from/to Italian and many some other languages.
-                  <br/>
+                  Welcome! This space is limited to one speaker at a time. 
+                  If using the live HF space, sharing room code to listeners on another 
+                  IP address may not work because it's running on different replicas. 
                   Use headphones if you are both speaker and listener to prevent feedback.
                   <br/>
+                  If max speakers reached, please duplicate the space <a target="_blank" rel="noopener noreferrer" href="https://huggingface.co/spaces/facebook/seamless-streaming?duplicate=true">here</a>. 
+                  In your duplicated space, join a room as speaker or listener (or both), 
+                  and share the room code to invite listeners.
                   <br/>
-                  <a target="_blank" rel="noopener noreferrer" href="https://ai.meta.com/research/seamless-communication/">SeamlessStreaming</a> is
-                  a research model and  streaming quality works best if you pause
-                  every couple of sentences. The real-time performance will degrade
-                  if you try streaming multiple speakers at the same time.
-                  <br/>
+                  Check out the seamless_communication <a target="_blank" rel="noopener noreferrer" href="https://github.com/facebookresearch/seamless_communication/tree/main">README</a> for more information.
                   <br/>
-                  Let's try!
+                  SeamlessStreaming model is a research model and is not released
+                  for production deployment. It is important to use a microphone with 
+                  noise cancellation (for e.g. a smartphone), otherwise you may see model hallucination on noises. 
+                  It works best if you pause every couple of sentences, or you may wish adjust the VAD threshold
+                  in the model config. The real-time performance will degrade
+                  if you try streaming multiple speakers at the same time.
                 </Typography>
               </div>
             </div>
             <Stack spacing="22px" direction="column">
               <Box>
-                { <RoomConfig
+                <RoomConfig
                   roomState={roomState}
                   serverState={serverState}
                   streamingStatus={streamingStatus}
@@ -790,7 +788,7 @@ export default function StreamingInterface() {
                     // player to play eagerly, since currently the listener doesn't have any stop/start controls
                     bufferedSpeechPlayer.start();
                   }}
-                /> }
+                />
 
                 {isListener && !isSpeaker && (
                   <Box
@@ -812,9 +810,9 @@ export default function StreamingInterface() {
                   <Divider />
 
                   <Stack spacing="12px" direction="column">
-                    {/* <FormLabel id="output-modes-radio-group-label">
+                    <FormLabel id="output-modes-radio-group-label">
                       Model
-                    </FormLabel> */}
+                    </FormLabel>
                     <FormControl
                       disabled={
                         streamFixedConfigOptionsDisabled ||
@@ -822,10 +820,10 @@ export default function StreamingInterface() {
                       }
                       fullWidth
                       sx={{minWidth: '14em'}}>
-                      {/* <InputLabel id="model-selector-input-label">
+                      <InputLabel id="model-selector-input-label">
                         Model
-                      </InputLabel> */}
-                      {/* <Select
+                      </InputLabel>
+                      <Select
                         labelId="model-selector-input-label"
                         label="Model"
                         onChange={(e: SelectChangeEvent) => {
@@ -847,14 +845,14 @@ export default function StreamingInterface() {
                             {agent.name}
                           </MenuItem>
                         ))}
-                      </Select> */}
+                      </Select>
                     </FormControl>
 
                   </Stack>
 
                   <Stack spacing={0.5}>
                     <FormLabel id="output-modes-radio-group-label">
-                      Illy, can you please select the target language?
+                      Output
                     </FormLabel>
 
                     <Box sx={{paddingTop: 2, paddingBottom: 1}}>
@@ -919,7 +917,7 @@ export default function StreamingInterface() {
                           spacing={1}
                           alignItems="flex-start"
                           sx={{flexGrow: 1}}>
-                          {/* {currentAgent?.dynamicParams?.includes(
+                          {currentAgent?.dynamicParams?.includes(
                             'expressive',
                           ) && (
                             <FormControlLabel
@@ -939,7 +937,7 @@ export default function StreamingInterface() {
                               }
                               label="Expressive"
                             />
-                          )} */}
+                          )}
 
                           {isListener && (
                             <Box
@@ -963,10 +961,10 @@ export default function StreamingInterface() {
                     justifyContent="space-between">
                     <Box sx={{flex: 1}}>
                       <FormControl disabled={streamFixedConfigOptionsDisabled}>
-                        {/* <FormLabel id="input-source-radio-group-label">
+                        <FormLabel id="input-source-radio-group-label">
                           Input Source
-                        </FormLabel> */}
-                        {/* <RadioGroup
+                        </FormLabel>
+                        <RadioGroup
                           aria-labelledby="input-source-radio-group-label"
                           value={inputSource}
                           onChange={(e: React.ChangeEvent<HTMLInputElement>) =>
@@ -982,8 +980,8 @@ export default function StreamingInterface() {
                               control={<Radio />}
                               label={label}
                             />
-                          ))} */}
-                        {/* </RadioGroup> */}
+                          ))}
+                        </RadioGroup>
                       </FormControl>
                     </Box>
 

streaming-react-app/src/react-xr/XRDialog.tsx

@@ -64,7 +64,7 @@ export default function XRDialog(props: XRConfigProps) {
 
   return (
     <>
-      {/* <Button variant="contained" onClick={() => setIsDialogOpen(true)}>
+      <Button variant="contained" onClick={() => setIsDialogOpen(true)}>
         Enter AR Experience
       </Button>
       {isDialogOpen && (
@@ -85,7 +85,7 @@ export default function XRDialog(props: XRConfigProps) {
           </IconButton>
           <XRContent {...props} />
         </Dialog>
-      )} */}
+      )}
     </>
   );
 }

streaming-react-app/src/theme.ts

@@ -2,12 +2,12 @@ import {createTheme} from '@mui/material/styles';
 
 const themeObject = {
   palette: {
-    background: {default: '#999999' },//'#383838'},
+    background: {default: '#383838'},
     primary: {
       main: '#465A69',
     },
     info: {
-      main: '#9f0000'// '#0064E0',
+      main: '#0064E0',
     },
     text: {primary: '#1C2A33'},
   },