joel/freqsplit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #41 from joelmathewthomas/api/spectrogram

Browse Source 
Api/spectrogram
This commit is contained in:
Joel Mathew Thomas
2025-03-19 01:13:35 +05:30
committed by GitHub
parent 33b61b9a45 7754af83e8
commit 9fdc3c8b3c
12 changed files with 184 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/api/tasks.py
+25 -2
View File
@@ -1,5 +1,7 @@
import os
import shutil
import json
import numpy as np
from pathlib import Path
from celery import shared_task
from freqsplit.input.file_reader import read_audio
@@ -10,6 +12,7 @@ from freqsplit.preprocessing.resample import resample
from freqsplit.postprocessing.audio_writer import export_audio
from freqsplit.separation.demucs_wrapper import separate_audio_with_demucs
from freqsplit.refinement.deepfilternet_wrapper import noisereduce
from freqsplit.spectrogram.generator import generate_spectrogram
@shared_task
@@ -24,8 +27,14 @@ def save_and_classify(file_path, file_content):
# Classify the audio
audio_class = classify_audio(waveform, sr)
return audio_class, org_sr
# Generate spectrogram
spec_db, plot_data = generate_spectrogram(file_path)
# Convert numpy array to JSON-safe list
spec_db = np.nan_to_num(spec_db, nan=-80.0, posinf=-80.0, neginf=-80.0)
spec_data_json = json.dumps(spec_db.tolist())
return audio_class, org_sr, spec_data_json, plot_data['sr']
@shared_task
def normalize_audio_task(file_path):
@@ -122,6 +131,20 @@ def noisereduce_task(file_path):
return False
@shared_task
def generate_spectrogram_task(file_path):
"""Celery task to generate spectrogram"""
try:
file_path = Path(file_path)
# Generate spectrogram
spec_db, plot_data = generate_spectrogram(file_path)
spec_db = np.nan_to_num(spec_db, nan=-80.0, posinf=-80.0, neginf=-80.0)
spec_data_json = json.dumps(spec_db.tolist())
return True, spec_data_json, plot_data['sr']
except Exception as e:
return False
@shared_task
def cleanup_task(file_path):
"""Celery task to cleanup files"""
file_path = Path(file_path)
api/api/views.py
+40 -6
View File
@@ -12,6 +12,7 @@ from .tasks import trim_audio_task
from .tasks import resample_audio_task
from .tasks import music_separation_task
from .tasks import noisereduce_task
from .tasks import generate_spectrogram_task
from .tasks import cleanup_task
from freqsplit.input.format_checker import is_supported_format
@@ -47,17 +48,21 @@ def upload_audio(request):
# Save the uploaded file
task = save_and_classify.apply(args=(file_path, audio_file.read()))
if task.successful():
audio_class = task.result[0]
if task.ready() and task.successful():
result = task.result
return Response(
{
"Status": "File uploaded successfully",
"file_uuid": file_uuid,
"audio_class": audio_class,
"sr": task.result[1]
},
"audio_class": result[0],
"sr": result[1],
"spectrogram": result[2],
"spec_sr": result[3]
},
status=status.HTTP_201_CREATED,
)
)
else:
return Response({"error": "Processing failed"}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
# Endpoint to normalize audio
@api_view(['POST'])
@@ -143,6 +148,35 @@ def noisereduce(request):
else:
return Response({"error": "Failed to remove noise from audio"}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
# Endpoint to generate spectrograms
@api_view(['POST'])
def generate_spectrogram(request):
"""Handle generation of spectrogram"""
file_uuid = request.data.get("file_uuid")
file_name = request.data.get("file_name")
file_path = os.path.join(UPLOAD_DIR, file_uuid, file_name)
# Check if file exists
if os.path.exists(file_path):
# Call Celery task synchronously
task = generate_spectrogram_task.apply(args=(file_path,))
if task.ready() and task.successful():
result = task.result
if result[0]:
return Response(
{
"Status": "Spectrogram generated successfully",
"spectrogram": result[1],
"spec_sr": result[2]
},
status=status.HTTP_200_OK
)
else:
return Response({"error": "Failed to generate spectrogram"}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
else:
return Response({"error": "File Not Found"}, status=status.HTTP_404_NOT_FOUND)
# Endpoint to download audio file or zipped directory
@api_view(['GET'])
def download_audio(request):
api/backend/urls.py
+2
View File
@@ -23,6 +23,7 @@ from api.views import resample_audio
from api.views import separate_music
from api.views import noisereduce
from api.views import download_audio
from api.views import generate_spectrogram
from api.views import cleanup
from api.views import cleanup_zip
@@ -35,6 +36,7 @@ urlpatterns = [
path('api/separate', separate_music, name="separate_music"),
path('api/noisereduce', noisereduce, name="noisreduce"),
path('api/download', download_audio, name="download_audio"),
path('api/spectrogram', generate_spectrogram, name="generate_spectrogram"),
path('api/cleanup', cleanup, name="cleanup"),
path('api/cleanup_zip', cleanup_zip, name="cleanup_zip")
]
client/package.json
+1
View File
@@ -18,6 +18,7 @@
"axios": "^1.8.3",
"jszip": "^3.10.1",
"react": "^18.2.0",
"react-audio-spectrogram-player": "^2.0.1",
"react-dom": "^18.2.0",
"react-router-dom": "^6.22.1"
},
client/src/Pages/PreviewPage.tsx
+10 -5
View File
@@ -11,6 +11,8 @@ import {
import { VolumeUp as VolumeUpIcon, ErrorOutline as ErrorIcon } from '@mui/icons-material';
import StepperComponent from '../components/StepperComponent';
import { useMediaContext } from '../contexts/MediaContext';
// @ts-ignore
import SpectrogramPlayer from "react-audio-spectrogram-player"
function PreviewPage() {
const navigate = useNavigate();
@@ -83,12 +85,15 @@ function PreviewPage() {
<Typography variant="h6" gutterBottom>
{mediaFile.name}
</Typography>
<Box sx={{ width: '100%', mt: 2 }}>
<audio
ref={videoRef}
<Box sx={{ width: '100%', mt: 2, border: `1px solid gray`, p:2, borderRadius: 2}}>
<SpectrogramPlayer
src={mediaFile.url}
style={{ width: '100%' }}
controls
sxx={JSON.parse(response.spectrogram)}
SampleRate={response.spec_sr}
colormap={'magma'}
settings={true}
transparent={false}
navigator={true}
/>
</Box>
<p>Audio Classification: {audioClass || "No data received"}</p>
client/src/Pages/ProcessingPage.tsx
+55 -3
View File
@@ -8,7 +8,7 @@ import JSZip from "jszip";
function ProcessingPage() {
const navigate = useNavigate();
const { mediaFile, response, setExtractedFiles, setDownloadedFileURL } = useMediaContext();
const { mediaFile, response, setExtractedFiles, setDownloadedFileURL, setDownloadedFileSpectrogram } = useMediaContext();
const [progress, setProgress] = useState(0);
const [statusText, setStatusText] = useState("Analyzing media...");
@@ -73,7 +73,36 @@ function ProcessingPage() {
const fileURL = URL.createObjectURL(blob);
setDownloadedFileURL( fileURL );
setProgress(100);
setProgress(90);
// Get spectrogram
setProgress(95);
setStatusText("Calculating Spectrogram");
const formData = new FormData();
formData.append("file_uuid", response.file_uuid);
if (mediaFile?.name) {
formData.append("file_name", mediaFile?.name);
}
const startTime = Date.now();
const resp = await axios.post<{
Status: String;
spectrogram: string;
spec_sr: number;
}>("/api/spectrogram", formData, {
headers: {
"Content-Type": "multipart/form-data",
}
})
if (resp.status === 200 && resp.data) {
const elapsedTime = Date.now() - startTime;
if(elapsedTime < 5000) await delay(5000 - elapsedTime);
setDownloadedFileSpectrogram({spectrogram: resp.data.spectrogram, spec_sr: resp.data.spec_sr})
setProgress(100);
}
} else {
console.log("Failed to download the file");
@@ -94,9 +123,32 @@ function ProcessingPage() {
if (!fileData.dir) {
const fileBlob = await fileData.async("blob");
const fileURL = URL.createObjectURL(fileBlob);
fileURLs.push({ name: filename, url: fileURL });
// Get spectrograms
setProgress(95);
setStatusText("Calculating Spectrograms");
const formData = new FormData();
formData.append("file_uuid", response.file_uuid);
formData.append("file_name", filename);
const res = await axios.post<{
Status: string;
spectrogram: string;
spec_sr: number;
}>("/api/spectrogram", formData, {
headers: {
"Content-Type": "multipart/form-data",
},
})
if (res.status === 200 && res.data){
}
fileURLs.push({ name: filename, url: fileURL, spectrogram: res.data.spectrogram, spec_sr: res.data.spec_sr });
}
}
console.log(fileURLs)
setExtractedFiles(fileURLs);
setProgress(100);
};
client/src/Pages/ResultsPage.tsx
+28 -19
View File
@@ -1,4 +1,4 @@
import { useEffect, useRef } from 'react';
import { useEffect } from 'react';
import { useNavigate } from 'react-router-dom';
//import axios from 'axios';
import {
@@ -18,14 +18,14 @@ import {
} from '@mui/icons-material';
import StepperComponent from '../components/StepperComponent';
import { useMediaContext } from '../contexts/MediaContext';
// @ts-ignore
import SpectrogramPlayer from "react-audio-spectrogram-player"
function ResultsPage() {
const navigate = useNavigate();
const { mediaFile, response, extractedFiles, downloadedFileURL } = useMediaContext();
const { mediaFile, response, extractedFiles, downloadedFileURL, downloadedFileSpectrogram } = useMediaContext();
console.log("Extracted files are", extractedFiles);
// const [isPlaying, setIsPlaying] = useState(false);
const audioRefs = [useRef(null), useRef(null), useRef(null),useRef(null)];
const mediaFileRef = useRef(null);
const audioClass = response.audio_class
const isVideo = mediaFile?.type.includes('video');
@@ -97,15 +97,18 @@ function ResultsPage() {
<Typography variant="h6" gutterBottom>
{mediaFile.name} (Original)
</Typography>
<Box sx={{ width: '100%', mt: 2, mb: 2 }}>
<Box sx={{ width: '100%', mt: 2, mb: 2, border: `1px solid gray`, p:2, borderRadius: 2 }}>
<Typography variant="body2" color="textSecondary" sx={{ mb: 1 }}>
{mediaFile.name}
</Typography>
<audio
ref={mediaFileRef}
<SpectrogramPlayer
src={mediaFile.url}
style={{ width: '100%', borderRadius: '8px', boxShadow: '0 2px 10px rgba(0,0,0,0.2)' }}
controls
sxx={JSON.parse(response.spectrogram)}
SampleRate={response.spec_sr}
colormap={'magma'}
settings={true}
transparent={false}
navigator={true}
/>
</Box>
{audioClass === "Music" ? (
@@ -116,15 +119,18 @@ function ResultsPage() {
</Typography>
</Box>
{extractedFiles.map((file, index) => (
<Box key={index} sx={{ width: '100%', mt: 2 }}>
<Box key={index} sx={{ width: '100%', mt: 2, border: `1px solid gray`, p:2, borderRadius: 2 }}>
<Typography variant="body2" color="textSecondary" sx={{ mb: 1 }}>
{file.name}
</Typography>
<audio
ref={audioRefs[index]}
<SpectrogramPlayer
src={file.url}
style={{ width: '100%', borderRadius: '8px', boxShadow: '0 2px 10px rgba(0,0,0,0.2)' }}
controls
sxx={JSON.parse(file.spectrogram)}
SampleRate={file.spec_sr}
colormap={'magma'}
settings={true}
transparent={false}
navigator={true}
/>
</Box>
))}
@@ -136,15 +142,18 @@ function ResultsPage() {
Processed File
</Typography>
</Box>
<Box sx={{ width: '100%', mt: 2 }}>
<Box sx={{ width: '100%', mt: 2, border: `1px solid gray`, p:2, borderRadius: 2 }}>
<Typography variant="body2" color="textSecondary" sx={{ mb: 1 }}>
{mediaFile?.name}
</Typography>
<audio
ref={audioRefs[0]}
<SpectrogramPlayer
src={downloadedFileURL}
style={{ width: '100%', borderRadius: '8px', boxShadow: '0 2px 10px rgba(0,0,0,0.2)' }}
controls
sxx={JSON.parse(downloadedFileSpectrogram.spectrogram)}
SampleRate={downloadedFileSpectrogram.spec_sr}
colormap={'magma'}
settings={true}
transparent={false}
navigator={true}
/>
</Box>
</>
client/src/Pages/UploadPage.tsx
+4
View File
@@ -91,6 +91,8 @@ function UploadPage() {
file_uuid: string;
sr: number;
audio_class: string;
spectrogram: string;
spec_sr: number;
}>("/api/upload", formData, {
headers: {
"Content-Type": "multipart/form-data",
@@ -104,6 +106,8 @@ function UploadPage() {
audio_class: res.data.audio_class,
file_uuid: res.data.file_uuid,
sr: res.data.sr,
spectrogram: res.data.spectrogram,
spec_sr: res.data.spec_sr
}));
setUpload(true);
}
client/src/contexts/MediaContext.tsx
+13 -5
View File
@@ -3,12 +3,14 @@ import React, { createContext, useState, useContext } from 'react';
interface MediaContextType {
mediaFile: { name: string; url: string; type: string } | null;
setMediaFile: (file: { name: string; url: string; type: string }) => void;
response: { file_uuid: string; sr: number; audio_class: string };
setResponse: (response: { file_uuid: string; sr: number; audio_class: string }) => void;
extractedFiles: { name: string; url: string }[];
setExtractedFiles: (files: {name: string; url: string }[]) => void;
response: { file_uuid: string; sr: number; audio_class: string, spectrogram: string, spec_sr: number };
setResponse: (response: { file_uuid: string; sr: number; audio_class: string, spectrogram: string, spec_sr: number }) => void;
extractedFiles: { name: string; url: string, spectrogram: string, spec_sr: number }[];
setExtractedFiles: (files: {name: string; url: string, spectrogram: string, spec_sr: number}[]) => void;
downloadedFileURL: string;
setDownloadedFileURL: ( file: string) => void;
downloadedFileSpectrogram: { spectrogram: string, spec_sr: number};
setDownloadedFileSpectrogram: (spectrogram: {spectrogram: string, spec_sr: number}) => void;
}
@@ -20,13 +22,19 @@ export const MediaProvider: React.FC<{ children: React.ReactNode }> = ({ childre
audio_class: "",
file_uuid: "",
sr: 0,
spectrogram: "",
spec_sr: 0
});
const [extractedFiles, setExtractedFiles] = useState<MediaContextType["extractedFiles"]>([]);
const [downloadedFileURL, setDownloadedFileURL] = useState<MediaContextType["downloadedFileURL"]>("");
const [downloadedFileSpectrogram, setDownloadedFileSpectrogram] = useState<MediaContextType["downloadedFileSpectrogram"]>({
spectrogram: "",
spec_sr: 0
});
return (
<MediaContext.Provider value={{ mediaFile, setMediaFile, response, setResponse, extractedFiles, setExtractedFiles, downloadedFileURL, setDownloadedFileURL }}>
<MediaContext.Provider value={{ mediaFile, setMediaFile, response, setResponse, extractedFiles, setExtractedFiles, downloadedFileURL, setDownloadedFileURL, downloadedFileSpectrogram, setDownloadedFileSpectrogram }}>
{children}
</MediaContext.Provider>
);
src/freqsplit/spectogram/__init__.py → src/freqsplit/spectrogram/__init__.py
+1 -1
View File
@@ -9,4 +9,4 @@ logging.basicConfig(
level = logging.INFO
)
logging.info("freqsplit/spectogram package has been imported.")
logging.info("freqsplit/spectrogram package has been imported.")
src/freqsplit/spectogram/display.py → src/freqsplit/spectrogram/display.py
View File
src/freqsplit/spectogram/generator.py → src/freqsplit/spectrogram/generator.py
+5 -5
View File
@@ -1,7 +1,7 @@
import librosa
import numpy as np
def generate_spectogram(audio_file: str, spectogram_type: str = 'mel', sr: int = 22050):
def generate_spectrogram(audio_file: str, spectrogram_type: str = 'mel', sr: int = 22050):
"""
Generates a spectrogram array from an audio file.
@@ -19,16 +19,16 @@ def generate_spectogram(audio_file: str, spectogram_type: str = 'mel', sr: int =
# Load the audio file
waveform, sr = librosa.load(audio_file, sr=sr)
# Create the spectogram
if spectogram_type == 'mel':
# Create the spectrogram
if spectrogram_type == 'mel':
spec = librosa.feature.melspectrogram(y=waveform, sr=sr)
spec_db = librosa.power_to_db(spec, ref=np.max) # Convert to decibels
plot_data = {'sr': sr, 'x_axis': 'time', 'y_axis': 'mel'}
elif spectogram_type == 'stft':
elif spectrogram_type == 'stft':
spec = np.abs(librosa.stft(waveform))
spec_db = librosa.amplitude_to_db(spec, ref=np.max)
plot_data = {'sr': sr, 'x_axis': 'time', 'y_axis': 'log'}
else:
raise ValueError(f"Unsupported spectogram type: {spectogram_type}")
raise ValueError(f"Unsupported spectrogram type: {spectrogram_type}")
return spec_db, plot_data