fix: multi-stage key/bpm detection pipeline, fuzzy matching and eliminate 8B fallback

server/db.js

@@ -77,26 +77,82 @@ async function query(text, params = []) {
   return getPool().query(text, params);
 }
 
-// Find track by artist and title (case insensitive)
+function normalizeTrackTitle(str) {
+  if (!str) return '';
+  return str
+    .toLowerCase()
+    .replace(/\(.*?\)|\[.*?\]/g, '') // strip (feat. ...), [feat. ...]
+    .replace(/ - .*$/g, '')          // strip trailing - Radio Edit, etc.
+    .replace(/[^a-z0-9]/g, ' ')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+function normalizeTrackArtist(str) {
+  if (!str) return '';
+  const first = str.split(/,|&|feat\.|with|vs\./i)[0];
+  return first
+    .toLowerCase()
+    .replace(/[^a-z0-9]/g, ' ')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+// Find track by artist and title (case insensitive, with fuzzy fallback)
 async function findTrack(artist, title) {
   if (!artist || !title) return null;
 
+  const cleanArt = artist.trim();
+  const cleanTitle = title.trim();
+
   if (isConnected) {
     try {
+      // 1. Exact match
       const res = await getPool().query(
         `SELECT * FROM tracks 
          WHERE LOWER(artist) = LOWER($1) AND LOWER(title) = LOWER($2)
          LIMIT 1`,
-        [artist.trim(), title.trim()]
+        [cleanArt, cleanTitle]
       );
       if (res.rows.length > 0) return res.rows[0];
+
+      // 2. Fuzzy match with stripped/normalized title & primary artist
+      const normTitle = normalizeTrackTitle(cleanTitle);
+      const normArt = normalizeTrackArtist(cleanArt);
+
+      if (normTitle.length >= 2 && normArt.length >= 2) {
+        const fuzzyRes = await getPool().query(
+          `SELECT * FROM tracks 
+           WHERE (LOWER(artist) LIKE $1 OR LOWER(artist) LIKE $2)
+             AND (LOWER(title) LIKE $3 OR LOWER(title) LIKE $4)
+           ORDER BY is_corrected DESC, play_count DESC
+           LIMIT 1`,
+          [`%${normArt}%`, `${normArt}%`, `%${normTitle}%`, `${normTitle}%`]
+        );
+        if (fuzzyRes.rows.length > 0) return fuzzyRes.rows[0];
+      }
     } catch (e) {
       console.error('[DB] findTrack error:', e.message);
     }
   }
 
-  const key = `${artist.trim().toLowerCase()} - ${title.trim().toLowerCase()}`;
-  return memoryTracks.get(key) || null;
+  // Memory fallback
+  const exactKey = `${cleanArt.toLowerCase()} - ${cleanTitle.toLowerCase()}`;
+  if (memoryTracks.has(exactKey)) return memoryTracks.get(exactKey);
+
+  const normTitle = normalizeTrackTitle(cleanTitle);
+  const normArt = normalizeTrackArtist(cleanArt);
+  for (const track of memoryTracks.values()) {
+    const tNormTitle = normalizeTrackTitle(track.title);
+    const tNormArt = normalizeTrackArtist(track.artist);
+    if (tNormTitle.includes(normTitle) || normTitle.includes(tNormTitle)) {
+      if (tNormArt.includes(normArt) || normArt.includes(tNormArt)) {
+        return track;
+      }
+    }
+  }
+
+  return null;
 }
 
 // Save or update track

server/routes/metadata.js

@@ -1,6 +1,7 @@
 const express = require('express');
 const router = express.Router();
 const axios = require('axios');
+const { spawn } = require('child_process');
 const db = require('../db');
 const camelotToMusicKey = require('../../shared/camelot-to-musickey');
 
@@ -99,6 +100,217 @@ function estimateAudioFeatures(bpm, camelotKey, tags = []) {
   };
 }
 
+const MAJOR_PROFILE = [6.35, 2.23, 3.48, 2.33, 4.38, 4.09, 2.52, 5.19, 2.39, 3.66, 2.29, 2.88];
+const MINOR_PROFILE = [6.33, 2.68, 3.52, 5.38, 2.60, 3.53, 2.54, 4.75, 3.98, 2.69, 3.34, 3.17];
+const NOTE_NAMES = ['C', 'C#', 'D', 'D#', 'E', 'F', 'F#', 'G', 'G#', 'A', 'A#', 'B'];
+
+function normalizeTitle(str) {
+  if (!str) return '';
+  return str
+    .toLowerCase()
+    .replace(/\(.*?\)|\[.*?\]/g, '') // strip (feat. ...), [feat. ...]
+    .replace(/ - .*$/g, '')          // strip trailing - Radio Edit, etc.
+    .replace(/[^a-z0-9]/g, ' ')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+function normalizeArtist(str) {
+  if (!str) return '';
+  const first = str.split(/,|&|feat\.|with|vs\./i)[0];
+  return first
+    .toLowerCase()
+    .replace(/[^a-z0-9]/g, ' ')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+function pearsonCorrelation(x, y) {
+  const n = x.length;
+  let sX = 0, sY = 0, sXY = 0, sX2 = 0, sY2 = 0;
+  for (let i = 0; i < n; i++) {
+    sX += x[i]; sY += y[i];
+    sXY += x[i] * y[i];
+    sX2 += x[i] * x[i]; sY2 += y[i] * y[i];
+  }
+  const den = Math.sqrt((n * sX2 - sX * sX) * (n * sY2 - sY * sY));
+  return den === 0 ? 0 : (n * sXY - sX * sY) / den;
+}
+
+function detectBpmFromSamples(samples, sampleRate) {
+  const frameSize = 512;
+  const numFrames = Math.floor(samples.length / frameSize);
+  if (numFrames < 20) return null;
+
+  const energies = new Float32Array(numFrames);
+  for (let i = 0; i < numFrames; i++) {
+    let sum = 0;
+    const off = i * frameSize;
+    for (let j = 0; j < frameSize; j++) {
+      const s = samples[off + j];
+      sum += s * s;
+    }
+    energies[i] = sum;
+  }
+
+  const onsets = new Float32Array(numFrames);
+  for (let i = 1; i < numFrames; i++) {
+    const diff = energies[i] - energies[i - 1];
+    onsets[i] = diff > 0 ? diff : 0;
+  }
+
+  const fps = sampleRate / frameSize;
+  const minLag = Math.floor((60 * fps) / 175);
+  const maxLag = Math.floor((60 * fps) / 75);
+
+  let bestLag = minLag;
+  let maxCorr = -Infinity;
+
+  for (let lag = minLag; lag <= maxLag; lag++) {
+    let corr = 0;
+    for (let i = 0; i < numFrames - lag; i++) {
+      corr += onsets[i] * onsets[i + lag];
+    }
+    if (corr > maxCorr) {
+      maxCorr = corr;
+      bestLag = lag;
+    }
+  }
+
+  return Math.round((60 * fps) / bestLag);
+}
+
+function detectKeyFromSamples(samples, sampleRate) {
+  const chroma = new Float64Array(12);
+  for (let midi = 36; midi <= 84; midi++) {
+    const freq = 440 * Math.pow(2, (midi - 69) / 12);
+    const pitchClass = midi % 12;
+    const k = Math.round((samples.length * freq) / sampleRate);
+    const omega = (2 * Math.PI * k) / samples.length;
+    const coeff = 2 * Math.cos(omega);
+
+    let sPrev = 0, sPrev2 = 0;
+    for (let i = 0; i < samples.length; i += 4) {
+      const s = samples[i] + coeff * sPrev - sPrev2;
+      sPrev2 = sPrev;
+      sPrev = s;
+    }
+    const power = sPrev2 * sPrev2 + sPrev * sPrev - coeff * sPrev * sPrev2;
+    if (power > 0) chroma[pitchClass] += Math.sqrt(power);
+  }
+
+  const maxChroma = Math.max(...chroma) || 1;
+  const normChroma = Array.from(chroma).map(v => v / maxChroma);
+
+  let bestScore = -Infinity;
+  let bestKeyStr = 'c major';
+
+  for (let root = 0; root < 12; root++) {
+    const shifted = [];
+    for (let i = 0; i < 12; i++) shifted.push(normChroma[(root + i) % 12]);
+
+    const corrMaj = pearsonCorrelation(shifted, MAJOR_PROFILE);
+    if (corrMaj > bestScore) {
+      bestScore = corrMaj;
+      bestKeyStr = `${NOTE_NAMES[root].toLowerCase()} major`;
+    }
+    const corrMin = pearsonCorrelation(shifted, MINOR_PROFILE);
+    if (corrMin > bestScore) {
+      bestScore = corrMin;
+      bestKeyStr = `${NOTE_NAMES[root].toLowerCase()} minor`;
+    }
+  }
+
+  const camelot = standardizeKey(bestKeyStr) || '8B';
+  return {
+    camelotKey: camelot,
+    confidence: Math.round(bestScore * 100)
+  };
+}
+
+function analyzeAudioPreview(previewUrl) {
+  return new Promise((resolve) => {
+    if (!previewUrl) return resolve(null);
+
+    const ffmpeg = spawn('ffmpeg', [
+      '-i', previewUrl,
+      '-f', 's16le',
+      '-ac', '1',
+      '-ar', '11025',
+      '-t', '25',
+      '-'
+    ]);
+
+    const chunks = [];
+    ffmpeg.stdout.on('data', chunk => chunks.push(chunk));
+    ffmpeg.on('close', code => {
+      if (code !== 0 || chunks.length === 0) return resolve(null);
+      try {
+        const buf = Buffer.concat(chunks);
+        const numSamples = Math.floor(buf.length / 2);
+        const samples = new Float32Array(numSamples);
+        for (let i = 0; i < numSamples; i++) {
+          samples[i] = buf.readInt16LE(i * 2) / 32768.0;
+        }
+
+        const bpm = detectBpmFromSamples(samples, 11025);
+        const keyInfo = detectKeyFromSamples(samples, 11025);
+
+        if (bpm && keyInfo && keyInfo.camelotKey) {
+          return resolve({
+            bpm,
+            camelotKey: keyInfo.camelotKey,
+            confidence: keyInfo.confidence,
+            source: 'audio_analysis'
+          });
+        }
+      } catch (e) {
+        console.warn('[Metadata] Audio analysis error:', e.message);
+      }
+      resolve(null);
+    });
+    ffmpeg.on('error', (err) => {
+      console.warn('[Metadata] ffmpeg spawn error:', err.message);
+      resolve(null);
+    });
+  });
+}
+
+async function fetchSongBpm(artist, title) {
+  const normArt = normalizeArtist(artist);
+  const normTitle = normalizeTitle(title);
+  const variants = [
+    { a: normArt, t: normTitle },
+    { a: artist, t: title }
+  ];
+
+  for (const v of variants) {
+    const aSlug = v.a.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '');
+    const tSlug = v.t.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '');
+    if (!aSlug || !tSlug) continue;
+
+    try {
+      const r = await axios.get(`https://songbpm.com/@${aSlug}/${tSlug}`, {
+        headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)' },
+        timeout: 3500
+      });
+      const kMatch = r.data.match(/<span class="font-semibold">([A-G][b#]?)<\/span>\s*key\s*and\s*a\s*<span class="font-semibold">(minor|major)<\/span>\s*mode/i);
+      const bpmMatch = r.data.match(/(\d+)\s*<span class="pl-1 text-sm">\s*BPM<\/span>/i);
+      if (kMatch && bpmMatch) {
+        const fullKey = `${kMatch[1].toLowerCase()} ${kMatch[2].toLowerCase()}`;
+        const camelot = standardizeKey(fullKey);
+        const bpm = parseInt(bpmMatch[1], 10);
+        if (camelot && bpm) {
+          return { bpm, camelotKey: camelot, source: 'songbpm' };
+        }
+      }
+    } catch (e) {
+      // ignore
+    }
+  }
+  return null;
+}
+
 router.get('/', async (req, res) => {
   const track = req.query.track;
   const artist = req.query.artist;
@@ -108,7 +320,7 @@ router.get('/', async (req, res) => {
   }
 
   try {
-    // 1. Check PostgreSQL first!
+    // 1. Check PostgreSQL first (exact + fuzzy match on 26,000+ tracks)
     const local = await db.findTrack(artist, track);
     if (local && local.camelot_key && local.bpm) {
       const est = estimateAudioFeatures(local.bpm, local.camelot_key, local.tags);
@@ -128,7 +340,7 @@ router.get('/', async (req, res) => {
           energy: local.energy || est.energy,
           danceability: local.danceability || est.danceability,
           happiness: local.happiness || est.happiness,
-          source: 'postgresql'
+          source: local.source || 'postgresql'
         }
       });
     }
@@ -137,69 +349,16 @@ router.get('/', async (req, res) => {
     let camelotKey = null;
     let tags = [];
     let coverUrl = null;
-
-    // 2. Query GetSongBPM API if key available
-    if (process.env.GETSONGBPM_API_KEY) {
-      try {
-        const gsbRes = await axios.get('https://api.getsong.co/search/', {
-          params: {
-            api_key: process.env.GETSONGBPM_API_KEY,
-            type: 'both',
-            lookup: `song:${track} artist:${artist}`
-          },
-          timeout: 4000
-        });
-        
-        const searchResults = gsbRes.data && gsbRes.data.search;
-        if (Array.isArray(searchResults) && searchResults.length > 0) {
-          const song = searchResults[0];
-          if (song.tempo) {
-            bpm = parseFloat(song.tempo);
-          }
-          if (song.key_of) {
-            camelotKey = standardizeKey(song.key_of);
-          }
-          if (song.artist && song.artist.genres && tags.length === 0) {
-            tags = song.artist.genres.slice(0, 4);
-          }
-        }
-      } catch (err) {
-        console.warn('[Metadata] GetSongBPM Error:', err.message);
-      }
-    }
-
-    // 3. Last.fm API for genre tags
-    if (process.env.LASTFM_API_KEY) {
-      try {
-        const lfmRes = await axios.get('https://ws.audioscrobbler.com/2.0/', {
-          params: {
-            method: 'track.getInfo',
-            api_key: process.env.LASTFM_API_KEY,
-            artist: artist,
-            track: track,
-            format: 'json'
-          },
-          timeout: 4000
-        });
-        
-        if (lfmRes.data && lfmRes.data.track && lfmRes.data.track.toptags && lfmRes.data.track.toptags.tag) {
-          tags = lfmRes.data.track.toptags.tag
-            .slice(0, 4)
-            .map(t => t.name.toLowerCase());
-        }
-      } catch (err) {
-        console.warn('[Metadata] Last.fm Error:', err.message);
-      }
-    }
-
-    // 4. Query Deezer for cover, duration, preview, and genre fallback
     let durationSec = 210;
     let previewUrl = null;
     let deezerLink = null;
     let deezerId = null;
+    let detectedSource = null;
+
+    // 2. Query Deezer for cover, duration, preview, and deezer links
     try {
       const deezerRes = await axios.get(`https://api.deezer.com/search?q=${encodeURIComponent(artist + ' ' + track)}&limit=1`, {
-        timeout: 3000
+        timeout: 3500
       });
       if (deezerRes.data && deezerRes.data.data && deezerRes.data.data[0]) {
         const dTrack = deezerRes.data.data[0];
@@ -224,37 +383,121 @@ router.get('/', async (req, res) => {
       // ignore
     }
 
-    // Defaults if not found online
-    if (!bpm) bpm = 120;
-    if (!camelotKey) camelotKey = '8B';
+    // 3. Try SongBPM lookup (exact calibrated musical data)
+    const songBpmHit = await fetchSongBpm(artist, track);
+    if (songBpmHit) {
+      bpm = songBpmHit.bpm;
+      camelotKey = songBpmHit.camelotKey;
+      detectedSource = songBpmHit.source;
+    }
+
+    // 4. Query GetSongBPM API with normalized clean parameters if key still missing
+    if (!camelotKey && process.env.GETSONGBPM_API_KEY) {
+      try {
+        const cleanTitle = normalizeTitle(track);
+        const cleanArtist = normalizeArtist(artist);
+        const gsbRes = await axios.get('https://api.getsong.co/search/', {
+          params: {
+            api_key: process.env.GETSONGBPM_API_KEY,
+            type: 'both',
+            lookup: `song:${cleanTitle} artist:${cleanArtist}`
+          },
+          timeout: 4000
+        });
+        
+        const searchResults = gsbRes.data && gsbRes.data.search;
+        if (Array.isArray(searchResults) && searchResults.length > 0) {
+          const song = searchResults[0];
+          if (song.tempo && !bpm) {
+            bpm = parseFloat(song.tempo);
+          }
+          if (song.key_of) {
+            camelotKey = standardizeKey(song.key_of);
+            if (camelotKey) detectedSource = 'getsongbpm';
+          }
+          if (song.artist && song.artist.genres && tags.length === 0) {
+            tags = song.artist.genres.slice(0, 4);
+          }
+        }
+      } catch (err) {
+        console.warn('[Metadata] GetSongBPM Error:', err.message);
+      }
+    }
+
+    // 5. Real-time Audio Preview Analysis via ffmpeg if key still not determined
+    if ((!camelotKey || !bpm) && previewUrl) {
+      try {
+        const audioHit = await analyzeAudioPreview(previewUrl);
+        if (audioHit) {
+          if (!bpm && audioHit.bpm) bpm = audioHit.bpm;
+          if (!camelotKey && audioHit.camelotKey) {
+            camelotKey = audioHit.camelotKey;
+            detectedSource = audioHit.source;
+          }
+        }
+      } catch (err) {
+        console.warn('[Metadata] Audio analysis failed:', err.message);
+      }
+    }
+
+    // 6. Last.fm API for genre tags
+    if (process.env.LASTFM_API_KEY && tags.length === 0) {
+      try {
+        const lfmRes = await axios.get('https://ws.audioscrobbler.com/2.0/', {
+          params: {
+            method: 'track.getInfo',
+            api_key: process.env.LASTFM_API_KEY,
+            artist: normalizeArtist(artist),
+            track: normalizeTitle(track),
+            format: 'json'
+          },
+          timeout: 3500
+        });
+        
+        if (lfmRes.data && lfmRes.data.track && lfmRes.data.track.toptags && lfmRes.data.track.toptags.tag) {
+          tags = lfmRes.data.track.toptags.tag
+            .slice(0, 4)
+            .map(t => t.name.toLowerCase());
+        }
+      } catch (err) {
+        console.warn('[Metadata] Last.fm Error:', err.message);
+      }
+    }
+
     if (tags.length === 0) tags = ['electronic', 'dance'];
 
-    const musicKey = camelotToMusicKey[camelotKey] || '';
-    const audioAttrs = estimateAudioFeatures(bpm, camelotKey, tags);
+    const hasRealKey = !!camelotKey;
+    const finalBpm = bpm ? Math.round(bpm) : 120;
+    const finalKey = camelotKey || '8B';
+    const finalSource = detectedSource || (hasRealKey ? 'online' : 'unverified');
+    const musicKey = camelotToMusicKey[finalKey] || '';
+    const audioAttrs = estimateAudioFeatures(finalBpm, finalKey, tags);
 
-    // 5. Persist newly fetched track into PostgreSQL
-    await db.saveTrack({
-      title: track,
-      artist: artist,
-      camelot_key: camelotKey,
-      bpm: Math.round(bpm),
-      music_key: musicKey,
-      cover_url: coverUrl,
-      preview_url: previewUrl,
-      duration_sec: durationSec,
-      energy: audioAttrs.energy,
-      danceability: audioAttrs.danceability,
-      happiness: audioAttrs.happiness,
-      tags: tags,
-      is_corrected: false,
-      source: 'online'
-    });
+    // 7. Persist only if key was legitimately determined (do NOT pollute DB with blind unverified 8B)
+    if (hasRealKey) {
+      await db.saveTrack({
+        title: track,
+        artist: artist,
+        camelot_key: finalKey,
+        bpm: finalBpm,
+        music_key: musicKey,
+        cover_url: coverUrl,
+        preview_url: previewUrl,
+        duration_sec: durationSec,
+        energy: audioAttrs.energy,
+        danceability: audioAttrs.danceability,
+        happiness: audioAttrs.happiness,
+        tags: tags,
+        is_corrected: false,
+        source: finalSource
+      });
+    }
 
     res.json({
       status: 'success',
       data: {
-        key: camelotKey,
-        bpm: Math.round(bpm),
+        key: finalKey,
+        bpm: finalBpm,
         music_key: musicKey,
         tags: tags,
         is_corrected: false,
@@ -266,7 +509,7 @@ router.get('/', async (req, res) => {
         energy: audioAttrs.energy,
         danceability: audioAttrs.danceability,
         happiness: audioAttrs.happiness,
-        source: 'online'
+        source: finalSource
       }
     });
 

server/routes/search.js

@@ -77,7 +77,10 @@ router.get('/', async (req, res) => {
       if (seen.has(key)) continue;
       seen.add(key);
 
-      const existingLocal = localMap.get(key);
+      let existingLocal = localMap.get(key);
+      if (!existingLocal) {
+        existingLocal = await db.findTrack(ot.artist, ot.title);
+      }
       merged.push({
         title: ot.title,
         artist: ot.artist,