KeySelector/server/importer.js

const fs = require('fs');
const readline = require('readline');
const path = require('path');

const PITCH_MAJOR = ['8B', '3B', '10B', '5B', '12B', '7B', '2B', '9B', '4B', '11B', '6B', '1B'];
const PITCH_MINOR = ['5A', '12A', '7A', '2A', '9A', '4A', '11A', '6A', '1A', '8A', '3A', '10A'];
const MUSIC_MAJOR = ['C', 'Db', 'D', 'Eb', 'E', 'F', 'F#', 'G', 'Ab', 'A', 'Bb', 'B'];
const MUSIC_MINOR = ['Cm', 'C#m', 'Dm', 'D#m', 'Em', 'Fm', 'F#m', 'Gm', 'G#m', 'Am', 'Bbm', 'Bm'];

function parseCsvLine(line) {
  const result = [];
  let cur = '';
  let inQuotes = false;
  for (let i = 0; i < line.length; i++) {
    const ch = line[i];
    if (ch === '"') {
      if (inQuotes && line[i + 1] === '"') {
        cur += '"';
        i++;
      } else {
        inQuotes = !inQuotes;
      }
    } else if (ch === ',' && !inQuotes) {
      result.push(cur);
      cur = '';
    } else {
      cur += ch;
    }
  }
  result.push(cur);
  return result;
}

async function importCatalogIfEmpty(pool) {
  if (!pool) return;

  try {
    const countRes = await pool.query('SELECT COUNT(*) FROM tracks');
    const count = parseInt(countRes.rows[0].count, 10);

    if (count >= 5000) {
      console.log(`[Catalog Importer] Database already contains ${count} tracks. Skipping seed import.`);
      return;
    }

    const csvPath = path.join(__dirname, '../db/spotify_songs.csv');
    if (!fs.existsSync(csvPath)) {
      console.warn(`[Catalog Importer] Dataset not found at ${csvPath}. Skipping.`);
      return;
    }

    console.log(`[Catalog Importer] Database has only ${count} tracks. Importing full 26,000+ catalog...`);

    const fileStream = fs.createReadStream(csvPath);
    const rl = readline.createInterface({ input: fileStream, crlfDelay: Infinity });
    let isHeader = true;

    const rows = [];
    const seen = new Set();

    for await (const line of rl) {
      if (isHeader) {
        isHeader = false;
        continue;
      }
      const cols = parseCsvLine(line);
      const title = cols[1]?.trim();
      const artist = cols[2]?.trim();
      const genre = cols[9]?.trim();
      const subgenre = cols[10]?.trim();
      const dance = Math.round(parseFloat(cols[11] || '0.7') * 100);
      const energy = Math.round(parseFloat(cols[12] || '0.7') * 100);
      const keyNum = parseInt(cols[13], 10);
      const mode = parseInt(cols[15], 10);
      const happy = Math.round(parseFloat(cols[20] || '0.5') * 100);
      const bpm = Math.round(parseFloat(cols[21] || '120'));
      const durationSec = Math.round(parseInt(cols[22] || '210000', 10) / 1000);

      if (!title || !artist || isNaN(keyNum) || isNaN(mode) || !bpm) continue;
      const cKey = mode === 1 ? PITCH_MAJOR[keyNum] : PITCH_MINOR[keyNum];
      const mKey = mode === 1 ? MUSIC_MAJOR[keyNum] : MUSIC_MINOR[keyNum];
      if (!cKey) continue;

      const dedup = `${artist.toLowerCase()} - ${title.toLowerCase()}`;
      if (seen.has(dedup)) continue;
      seen.add(dedup);

      const tags = [];
      if (genre) tags.push(genre.toLowerCase());
      if (subgenre && subgenre !== genre) tags.push(subgenre.toLowerCase());

      rows.push([artist, title, cKey, bpm, mKey, durationSec, energy, dance, happy, tags, 'spotify']);
    }

    console.log(`[Catalog Importer] Loaded ${rows.length} unique tracks from dataset. Inserting into PostgreSQL...`);
    const BATCH_SIZE = 500;
    let inserted = 0;

    for (let i = 0; i < rows.length; i += BATCH_SIZE) {
      const chunk = rows.slice(i, i + BATCH_SIZE);
      const valuePlaceholders = [];
      const params = [];
      let pIdx = 1;

      for (const r of chunk) {
        valuePlaceholders.push(`($${pIdx}, $${pIdx + 1}, $${pIdx + 2}, $${pIdx + 3}, $${pIdx + 4}, $${pIdx + 5}, $${pIdx + 6}, $${pIdx + 7}, $${pIdx + 8}, $${pIdx + 9}, $${pIdx + 10})`);
        params.push(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10]);
        pIdx += 11;
      }

      const sql = `
        INSERT INTO tracks (artist, title, camelot_key, bpm, music_key, duration_sec, energy, danceability, happiness, tags, source)
        VALUES ${valuePlaceholders.join(',\n')}
        ON CONFLICT (artist, title) DO NOTHING
      `;

      await pool.query(sql, params);
      inserted += chunk.length;
    }

    const finalRes = await pool.query('SELECT COUNT(*) FROM tracks');
    console.log(`[Catalog Importer] Successfully initialized catalog! Total tracks in DB: ${finalRes.rows[0].count}`);

  } catch (err) {
    console.error('[Catalog Importer] Error importing dataset:', err.message);
  }
}

module.exports = { importCatalogIfEmpty };