Merge pull request #41 from joelmathewthomas/api/spectrogram

Api/spectrogram

api/api/tasks.py

@@ -1,5 +1,7 @@
 import os
 import shutil
+import json
+import numpy as np
 from pathlib import Path
 from celery import shared_task
 from freqsplit.input.file_reader import read_audio
@@ -10,6 +12,7 @@ from freqsplit.preprocessing.resample import resample
 from freqsplit.postprocessing.audio_writer import export_audio
 from freqsplit.separation.demucs_wrapper import separate_audio_with_demucs
 from freqsplit.refinement.deepfilternet_wrapper import noisereduce
+from freqsplit.spectrogram.generator import generate_spectrogram
 
 
 @shared_task
@@ -25,7 +28,13 @@ def save_and_classify(file_path, file_content):
     # Classify the audio
     audio_class = classify_audio(waveform, sr)
 
-    return audio_class, org_sr
+    # Generate spectrogram
+    spec_db, plot_data = generate_spectrogram(file_path)
+    # Convert numpy array to JSON-safe list
+    spec_db = np.nan_to_num(spec_db, nan=-80.0, posinf=-80.0, neginf=-80.0)
+    spec_data_json = json.dumps(spec_db.tolist())
+
+    return audio_class, org_sr, spec_data_json, plot_data['sr']
 
 @shared_task
 def normalize_audio_task(file_path):
@@ -122,6 +131,20 @@ def noisereduce_task(file_path):
         return False
 
 @shared_task
+def generate_spectrogram_task(file_path):
+    """Celery task to generate spectrogram"""
+    try:
+        file_path = Path(file_path)
+    
+        # Generate spectrogram
+        spec_db, plot_data = generate_spectrogram(file_path)
+        spec_db = np.nan_to_num(spec_db, nan=-80.0, posinf=-80.0, neginf=-80.0)
+        spec_data_json = json.dumps(spec_db.tolist())
+
+        return True, spec_data_json, plot_data['sr'] 
+    except Exception as e:
+        return False
+@shared_task
 def cleanup_task(file_path):
     """Celery task to cleanup files"""
     file_path = Path(file_path)

api/api/views.py

@@ -12,6 +12,7 @@ from .tasks import trim_audio_task
 from .tasks import resample_audio_task
 from .tasks import music_separation_task
 from .tasks import noisereduce_task
+from .tasks import generate_spectrogram_task
 from .tasks import cleanup_task
 from freqsplit.input.format_checker import is_supported_format
 
@@ -47,17 +48,21 @@ def upload_audio(request):
     # Save the uploaded file
     task = save_and_classify.apply(args=(file_path, audio_file.read()))
     
-    if task.successful():
+    if task.ready() and task.successful():
-        audio_class = task.result[0]
+        result = task.result
         return Response(
             {
                 "Status": "File uploaded successfully",
                 "file_uuid": file_uuid,
-                "audio_class": audio_class,
+                "audio_class": result[0],
-                "sr": task.result[1]
+                "sr": result[1],
+                "spectrogram": result[2],
+                "spec_sr": result[3]
             }, 
             status=status.HTTP_201_CREATED,
         )
+    else:
+        return Response({"error": "Processing failed"}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
 
 # Endpoint to normalize audio
 @api_view(['POST'])
@@ -143,6 +148,35 @@ def noisereduce(request):
     else:
         return Response({"error": "Failed to remove noise from audio"}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
 
+# Endpoint to generate spectrograms
+@api_view(['POST'])
+def generate_spectrogram(request):
+    """Handle generation of spectrogram"""
+    file_uuid = request.data.get("file_uuid")
+    file_name = request.data.get("file_name")
+    file_path = os.path.join(UPLOAD_DIR, file_uuid, file_name)
+    
+    # Check if file exists
+    if os.path.exists(file_path):
+        # Call Celery task synchronously
+        task = generate_spectrogram_task.apply(args=(file_path,))
+
+        if task.ready() and task.successful():
+            result = task.result
+            if result[0]:
+                return Response(
+                    {
+                        "Status": "Spectrogram generated successfully",
+                        "spectrogram": result[1],
+                        "spec_sr": result[2]
+                    },
+                    status=status.HTTP_200_OK
+                )
+            else:
+                return Response({"error": "Failed to generate spectrogram"}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
+    else:
+        return Response({"error": "File Not Found"}, status=status.HTTP_404_NOT_FOUND)
+    
 # Endpoint to download audio file or zipped directory
 @api_view(['GET'])
 def download_audio(request):

api/backend/urls.py

@@ -23,6 +23,7 @@ from api.views import resample_audio
 from api.views import separate_music
 from api.views import noisereduce
 from api.views import download_audio
+from api.views import generate_spectrogram
 from api.views import cleanup
 from api.views import cleanup_zip
 
@@ -35,6 +36,7 @@ urlpatterns = [
     path('api/separate', separate_music, name="separate_music"),
     path('api/noisereduce', noisereduce, name="noisreduce"),
     path('api/download', download_audio, name="download_audio"),
+    path('api/spectrogram', generate_spectrogram, name="generate_spectrogram"),
     path('api/cleanup', cleanup, name="cleanup"),
     path('api/cleanup_zip', cleanup_zip, name="cleanup_zip")
 ]

client/package.json

@@ -18,6 +18,7 @@
     "axios": "^1.8.3",
     "jszip": "^3.10.1",
     "react": "^18.2.0",
+    "react-audio-spectrogram-player": "^2.0.1",
     "react-dom": "^18.2.0",
     "react-router-dom": "^6.22.1"
   },

client/src/Pages/PreviewPage.tsx

@@ -11,6 +11,8 @@ import {
 import { VolumeUp as VolumeUpIcon, ErrorOutline as ErrorIcon } from '@mui/icons-material';
 import StepperComponent from '../components/StepperComponent';
 import { useMediaContext } from '../contexts/MediaContext';
+// @ts-ignore
+import SpectrogramPlayer from "react-audio-spectrogram-player"
 
 function PreviewPage() {
   const navigate = useNavigate();
@@ -83,12 +85,15 @@ function PreviewPage() {
               <Typography variant="h6" gutterBottom>
                 {mediaFile.name}
               </Typography>
-              <Box sx={{ width: '100%', mt: 2 }}>
+              <Box sx={{ width: '100%', mt: 2, border: `1px solid gray`, p:2, borderRadius: 2}}>
-                <audio
+                <SpectrogramPlayer
-                  ref={videoRef}
                   src={mediaFile.url}
-                  style={{ width: '100%' }}
+                  sxx={JSON.parse(response.spectrogram)}
-                  controls
+                  SampleRate={response.spec_sr}
+                  colormap={'magma'}
+                  settings={true}
+                  transparent={false}
+                  navigator={true}
                 />
               </Box>
               <p>Audio Classification: {audioClass || "No data received"}</p>

client/src/Pages/ProcessingPage.tsx

@@ -8,7 +8,7 @@ import JSZip from "jszip";
 
 function ProcessingPage() {
   const navigate = useNavigate();
-  const { mediaFile, response, setExtractedFiles, setDownloadedFileURL } = useMediaContext();
+  const { mediaFile, response, setExtractedFiles, setDownloadedFileURL, setDownloadedFileSpectrogram } = useMediaContext();
   const [progress, setProgress] = useState(0);
   const [statusText, setStatusText] = useState("Analyzing media...");
 
@@ -73,7 +73,36 @@ function ProcessingPage() {
         const fileURL = URL.createObjectURL(blob);
 
         setDownloadedFileURL( fileURL  );
+        setProgress(90);
+
+        // Get spectrogram
+        setProgress(95);
+        setStatusText("Calculating Spectrogram");
+
+        const formData = new FormData();
+        formData.append("file_uuid", response.file_uuid);
+        if (mediaFile?.name) {
+          formData.append("file_name", mediaFile?.name);
+        }
+
+        const startTime = Date.now();
+        const resp = await axios.post<{
+          Status: String;
+          spectrogram: string;
+          spec_sr: number;
+        }>("/api/spectrogram", formData, {
+          headers: {
+            "Content-Type": "multipart/form-data",
+          }
+        })
+
+        if (resp.status === 200 && resp.data) {
+          const elapsedTime = Date.now() - startTime;
+          if(elapsedTime < 5000) await delay(5000 - elapsedTime);
+          setDownloadedFileSpectrogram({spectrogram: resp.data.spectrogram, spec_sr: resp.data.spec_sr})
           setProgress(100);
+        }
+
 
       } else {
         console.log("Failed to download the file");
@@ -94,9 +123,32 @@ function ProcessingPage() {
       if (!fileData.dir) {
         const fileBlob = await fileData.async("blob");
         const fileURL = URL.createObjectURL(fileBlob);
-        fileURLs.push({ name: filename, url: fileURL });
+        
+        // Get spectrograms
+        setProgress(95);
+        setStatusText("Calculating Spectrograms");
+
+        const formData = new FormData();
+        formData.append("file_uuid", response.file_uuid);
+        formData.append("file_name", filename);
+
+        const res = await axios.post<{
+          Status: string;
+          spectrogram: string;
+          spec_sr: number;
+        }>("/api/spectrogram", formData, {
+          headers: {
+            "Content-Type": "multipart/form-data",
+          },
+        })
+
+        if (res.status === 200 && res.data){
+
+        }
+        fileURLs.push({ name: filename, url: fileURL, spectrogram: res.data.spectrogram, spec_sr: res.data.spec_sr });
       }
     }
+    console.log(fileURLs)
     setExtractedFiles(fileURLs);
     setProgress(100);
   };

client/src/Pages/ResultsPage.tsx

@@ -1,4 +1,4 @@
-import { useEffect, useRef } from 'react';
+import { useEffect } from 'react';
 import { useNavigate } from 'react-router-dom';
 //import axios from 'axios';
 import { 
@@ -18,14 +18,14 @@ import {
 } from '@mui/icons-material';
 import StepperComponent from '../components/StepperComponent';
 import { useMediaContext } from '../contexts/MediaContext';
+// @ts-ignore
+import SpectrogramPlayer from "react-audio-spectrogram-player"
 
 function ResultsPage() {
   const navigate = useNavigate();
-  const { mediaFile, response, extractedFiles, downloadedFileURL } = useMediaContext();
+  const { mediaFile, response, extractedFiles, downloadedFileURL, downloadedFileSpectrogram } = useMediaContext();
   console.log("Extracted files are", extractedFiles);
 //  const [isPlaying, setIsPlaying] = useState(false);
-  const audioRefs = [useRef(null), useRef(null), useRef(null),useRef(null)];
-  const mediaFileRef = useRef(null);
   const audioClass = response.audio_class
   const isVideo = mediaFile?.type.includes('video');
 
@@ -97,15 +97,18 @@ function ResultsPage() {
               <Typography variant="h6" gutterBottom>
                 {mediaFile.name} (Original)
               </Typography>
-              <Box sx={{ width: '100%', mt: 2, mb: 2 }}>
+              <Box sx={{ width: '100%', mt: 2, mb: 2, border: `1px solid gray`, p:2, borderRadius: 2 }}>
                 <Typography variant="body2" color="textSecondary" sx={{ mb: 1 }}>
                   {mediaFile.name}
                 </Typography>
-                <audio
+                <SpectrogramPlayer
-                  ref={mediaFileRef}
                   src={mediaFile.url}
-                  style={{ width: '100%', borderRadius: '8px', boxShadow: '0 2px 10px rgba(0,0,0,0.2)' }}
+                  sxx={JSON.parse(response.spectrogram)}
-                  controls
+                  SampleRate={response.spec_sr}
+                  colormap={'magma'}
+                  settings={true}
+                  transparent={false}
+                  navigator={true}
                 />
               </Box>
               {audioClass === "Music" ? (
@@ -116,15 +119,18 @@ function ResultsPage() {
                   </Typography>
                 </Box>
                 {extractedFiles.map((file, index) => (
-                  <Box key={index} sx={{ width: '100%', mt: 2 }}>
+                  <Box key={index} sx={{ width: '100%', mt: 2, border: `1px solid gray`, p:2, borderRadius: 2 }}>
                     <Typography variant="body2" color="textSecondary" sx={{ mb: 1 }}>
                       {file.name}
                     </Typography>
-                    <audio
+                    <SpectrogramPlayer
-                      ref={audioRefs[index]}
                       src={file.url}
-                      style={{ width: '100%', borderRadius: '8px', boxShadow: '0 2px 10px rgba(0,0,0,0.2)' }}
+                      sxx={JSON.parse(file.spectrogram)}
-                      controls
+                      SampleRate={file.spec_sr}
+                      colormap={'magma'}
+                      settings={true}
+                      transparent={false}
+                      navigator={true}
                     />
                   </Box>
                 ))}
@@ -136,15 +142,18 @@ function ResultsPage() {
                       Processed File
                     </Typography>
                   </Box>
-                  <Box sx={{ width: '100%', mt: 2 }}>
+                  <Box sx={{ width: '100%', mt: 2, border: `1px solid gray`, p:2, borderRadius: 2 }}>
                     <Typography variant="body2" color="textSecondary" sx={{ mb: 1 }}>
                         {mediaFile?.name}
                       </Typography>
-                    <audio
+                      <SpectrogramPlayer
-                      ref={audioRefs[0]}
                       src={downloadedFileURL}
-                      style={{ width: '100%', borderRadius: '8px', boxShadow: '0 2px 10px rgba(0,0,0,0.2)' }}
+                      sxx={JSON.parse(downloadedFileSpectrogram.spectrogram)}
-                      controls
+                      SampleRate={downloadedFileSpectrogram.spec_sr}
+                      colormap={'magma'}
+                      settings={true}
+                      transparent={false}
+                      navigator={true}
                     />
                   </Box>
                 </>

client/src/Pages/UploadPage.tsx

@@ -91,6 +91,8 @@ function UploadPage() {
         file_uuid: string;
         sr: number;
         audio_class: string;
+        spectrogram: string;
+        spec_sr: number;
       }>("/api/upload", formData, {
         headers: {
           "Content-Type": "multipart/form-data",
@@ -104,6 +106,8 @@ function UploadPage() {
           audio_class: res.data.audio_class,
           file_uuid: res.data.file_uuid,
           sr: res.data.sr,
+          spectrogram: res.data.spectrogram,
+          spec_sr: res.data.spec_sr
         }));
         setUpload(true);
       }

client/src/contexts/MediaContext.tsx

@@ -3,12 +3,14 @@ import React, { createContext, useState, useContext } from 'react';
 interface MediaContextType {
   mediaFile: { name: string; url: string; type: string } | null;
   setMediaFile: (file: { name: string; url: string; type: string }) => void;
-  response: { file_uuid: string; sr: number; audio_class: string }; 
+  response: { file_uuid: string; sr: number; audio_class: string, spectrogram: string, spec_sr: number }; 
-  setResponse: (response: { file_uuid: string; sr: number; audio_class: string }) => void;
+  setResponse: (response: { file_uuid: string; sr: number; audio_class: string, spectrogram: string, spec_sr: number }) => void;
-  extractedFiles: { name: string; url: string }[];
+  extractedFiles: { name: string; url: string, spectrogram: string, spec_sr: number }[];
-  setExtractedFiles: (files: {name: string; url: string }[]) => void;
+  setExtractedFiles: (files: {name: string; url: string, spectrogram: string, spec_sr: number}[]) => void;
   downloadedFileURL:  string;
   setDownloadedFileURL: ( file: string) => void;
+  downloadedFileSpectrogram: { spectrogram: string, spec_sr: number};
+  setDownloadedFileSpectrogram: (spectrogram: {spectrogram: string, spec_sr: number}) => void;
 }
 
 
@@ -20,13 +22,19 @@ export const MediaProvider: React.FC<{ children: React.ReactNode }> = ({ childre
     audio_class: "",
     file_uuid: "",
     sr: 0,
+    spectrogram: "",
+    spec_sr: 0
   });
   const [extractedFiles, setExtractedFiles] = useState<MediaContextType["extractedFiles"]>([]);
   const [downloadedFileURL, setDownloadedFileURL] = useState<MediaContextType["downloadedFileURL"]>("");
+  const [downloadedFileSpectrogram, setDownloadedFileSpectrogram] = useState<MediaContextType["downloadedFileSpectrogram"]>({
+    spectrogram: "",
+    spec_sr: 0
+  });
   
 
   return (
-    <MediaContext.Provider value={{ mediaFile, setMediaFile, response, setResponse, extractedFiles, setExtractedFiles, downloadedFileURL, setDownloadedFileURL }}>
+    <MediaContext.Provider value={{ mediaFile, setMediaFile, response, setResponse, extractedFiles, setExtractedFiles, downloadedFileURL, setDownloadedFileURL, downloadedFileSpectrogram, setDownloadedFileSpectrogram }}>
       {children}
     </MediaContext.Provider>
   );

src/freqsplit/spectrogram/__init__.py

@@ -9,4 +9,4 @@ logging.basicConfig(
     level = logging.INFO
 )
 
-logging.info("freqsplit/spectogram package has been imported.")
+logging.info("freqsplit/spectrogram package has been imported.")

src/freqsplit/spectrogram/generator.py

@@ -1,7 +1,7 @@
 import librosa
 import numpy as np
 
-def generate_spectogram(audio_file: str, spectogram_type: str = 'mel', sr: int = 22050):
+def generate_spectrogram(audio_file: str, spectrogram_type: str = 'mel', sr: int = 22050):
     """
     Generates a spectrogram array from an audio file.
 
@@ -19,16 +19,16 @@ def generate_spectogram(audio_file: str, spectogram_type: str = 'mel', sr: int =
     # Load the audio file
     waveform, sr = librosa.load(audio_file, sr=sr)
 
-    # Create the spectogram
+    # Create the spectrogram
-    if spectogram_type == 'mel':
+    if spectrogram_type == 'mel':
         spec = librosa.feature.melspectrogram(y=waveform, sr=sr)
         spec_db = librosa.power_to_db(spec, ref=np.max) # Convert to decibels
         plot_data = {'sr': sr, 'x_axis': 'time', 'y_axis': 'mel'}
-    elif spectogram_type == 'stft':
+    elif spectrogram_type == 'stft':
         spec = np.abs(librosa.stft(waveform))
         spec_db = librosa.amplitude_to_db(spec, ref=np.max)
         plot_data = {'sr': sr, 'x_axis': 'time', 'y_axis': 'log'}
     else:
-        raise ValueError(f"Unsupported spectogram type: {spectogram_type}")
+        raise ValueError(f"Unsupported spectrogram type: {spectrogram_type}")
 
     return spec_db, plot_data