From 4a5296cb18c4ef6d5e41347754d8195d769aefe4 Mon Sep 17 00:00:00 2001
From: Joel Mathew Thomas <90510078+joelmathewthomas@users.noreply.github.com>
Date: Tue, 18 Mar 2025 21:32:33 +0530
Subject: [PATCH] preview spectrograms for non-music files

---
 client/src/Pages/ProcessingPage.tsx  | 33 ++++++++++++++++++++++++++--
 client/src/Pages/ResultsPage.tsx     | 16 ++++++++------
 client/src/contexts/MediaContext.tsx |  8 ++++++-
 3 files changed, 47 insertions(+), 10 deletions(-)

diff --git a/client/src/Pages/ProcessingPage.tsx b/client/src/Pages/ProcessingPage.tsx
index f04206b..8a44a90 100644
--- a/client/src/Pages/ProcessingPage.tsx
+++ b/client/src/Pages/ProcessingPage.tsx
@@ -8,7 +8,7 @@ import JSZip from "jszip";
 
 function ProcessingPage() {
   const navigate = useNavigate();
-  const { mediaFile, response, setExtractedFiles, setDownloadedFileURL } = useMediaContext();
+  const { mediaFile, response, setExtractedFiles, setDownloadedFileURL, setDownloadedFileSpectrogram } = useMediaContext();
   const [progress, setProgress] = useState(0);
   const [statusText, setStatusText] = useState("Analyzing media...");
 
@@ -73,7 +73,36 @@ function ProcessingPage() {
         const fileURL = URL.createObjectURL(blob);
 
         setDownloadedFileURL( fileURL  );
-        setProgress(100);
+        setProgress(90);
+
+        // Get spectrogram
+        setProgress(95);
+        setStatusText("Calculating Spectrogram");
+
+        const formData = new FormData();
+        formData.append("file_uuid", response.file_uuid);
+        if (mediaFile?.name) {
+          formData.append("file_name", mediaFile?.name);
+        }
+
+        const startTime = Date.now();
+        const resp = await axios.post<{
+          Status: String;
+          spectrogram: string;
+          spec_sr: number;
+        }>("/api/spectrogram", formData, {
+          headers: {
+            "Content-Type": "multipart/form-data",
+          }
+        })
+
+        if (resp.status === 200 && resp.data) {
+          const elapsedTime = Date.now() - startTime;
+          if(elapsedTime < 5000) await delay(5000 - elapsedTime);
+          setDownloadedFileSpectrogram({spectrogram: resp.data.spectrogram, spec_sr: resp.data.spec_sr})
+          setProgress(100);
+        }
+
 
       } else {
         console.log("Failed to download the file");
diff --git a/client/src/Pages/ResultsPage.tsx b/client/src/Pages/ResultsPage.tsx
index 4044b74..90128b9 100644
--- a/client/src/Pages/ResultsPage.tsx
+++ b/client/src/Pages/ResultsPage.tsx
@@ -1,4 +1,4 @@
-import { useEffect, useRef } from 'react';
+import { useEffect } from 'react';
 import { useNavigate } from 'react-router-dom';
 //import axios from 'axios';
 import { 
@@ -23,10 +23,9 @@ import SpectrogramPlayer from "react-audio-spectrogram-player"
 
 function ResultsPage() {
   const navigate = useNavigate();
-  const { mediaFile, response, extractedFiles, downloadedFileURL } = useMediaContext();
+  const { mediaFile, response, extractedFiles, downloadedFileURL, downloadedFileSpectrogram } = useMediaContext();
   console.log("Extracted files are", extractedFiles);
 //  const [isPlaying, setIsPlaying] = useState(false);
-  const audioRefs = [useRef(null), useRef(null), useRef(null),useRef(null)];
   const audioClass = response.audio_class
   const isVideo = mediaFile?.type.includes('video');
 
@@ -147,11 +146,14 @@ function ResultsPage() {
                     <Typography variant="body2" color="textSecondary" sx={{ mb: 1 }}>
                         {mediaFile?.name}
                       </Typography>
-                    <audio
-                      ref={audioRefs[0]}
+                      <SpectrogramPlayer
                       src={downloadedFileURL}
-                      style={{ width: '100%', borderRadius: '8px', boxShadow: '0 2px 10px rgba(0,0,0,0.2)' }}
-                      controls
+                      sxx={JSON.parse(downloadedFileSpectrogram.spectrogram)}
+                      SampleRate={downloadedFileSpectrogram.spec_sr}
+                      colormap={'magma'}
+                      settings={true}
+                      transparent={false}
+                      navigator={true}
                     />
                   </Box>
                 </>
diff --git a/client/src/contexts/MediaContext.tsx b/client/src/contexts/MediaContext.tsx
index 94f63dd..b7afcc1 100644
--- a/client/src/contexts/MediaContext.tsx
+++ b/client/src/contexts/MediaContext.tsx
@@ -9,6 +9,8 @@ interface MediaContextType {
   setExtractedFiles: (files: {name: string; url: string, spectrogram: string, spec_sr: number}[]) => void;
   downloadedFileURL:  string;
   setDownloadedFileURL: ( file: string) => void;
+  downloadedFileSpectrogram: { spectrogram: string, spec_sr: number};
+  setDownloadedFileSpectrogram: (spectrogram: {spectrogram: string, spec_sr: number}) => void;
 }
 
 
@@ -25,10 +27,14 @@ export const MediaProvider: React.FC<{ children: React.ReactNode }> = ({ childre
   });
   const [extractedFiles, setExtractedFiles] = useState<MediaContextType["extractedFiles"]>([]);
   const [downloadedFileURL, setDownloadedFileURL] = useState<MediaContextType["downloadedFileURL"]>("");
+  const [downloadedFileSpectrogram, setDownloadedFileSpectrogram] = useState<MediaContextType["downloadedFileSpectrogram"]>({
+    spectrogram: "",
+    spec_sr: 0
+  });
   
 
   return (
-    <MediaContext.Provider value={{ mediaFile, setMediaFile, response, setResponse, extractedFiles, setExtractedFiles, downloadedFileURL, setDownloadedFileURL }}>
+    <MediaContext.Provider value={{ mediaFile, setMediaFile, response, setResponse, extractedFiles, setExtractedFiles, downloadedFileURL, setDownloadedFileURL, downloadedFileSpectrogram, setDownloadedFileSpectrogram }}>
       {children}
     </MediaContext.Provider>
   );