divan/api/src/import.ts

// Load divan-data (ganjoor-data layout: manifest.json, poets/**/_cat.json, poems) into PostgreSQL.
// Source: a local divan-data folder or the public CDN. Upserts, so re-running applies sync changes.
//   node src/import.ts [../../divan-data | https://cdn.jsdelivr.net/gh/anas-rashid/divan-data@main/]
import { readFile } from 'node:fs/promises';
import { join } from 'node:path';
import { pool } from './db.ts';
import { normalise } from './urdu.ts';
import { loadTags } from './tags.ts';

const SOURCE = process.argv[2] ?? new URL('../../../divan-data', import.meta.url).pathname;
const isHttp = /^https?:/.test(SOURCE);

async function read(path: string): Promise<any> {
  if (isHttp) {
    const res = await fetch(SOURCE.replace(/\/?$/, '/') + path);
    if (!res.ok) throw new Error(`${res.status} ${path}`);
    return res.json();
  }
  return JSON.parse(await readFile(join(SOURCE, path), 'utf8'));
}

const schema = await readFile(new URL('../../db/schema.sql', import.meta.url), 'utf8');
await pool.query(schema);

const manifest = await read('manifest.json');
console.log(`${manifest.PoetsCount} poets, ${manifest.PoemsCount} poems from ${SOURCE}`);

let poems = 0;
for (const entry of manifest.Poets) {
  const poet = await read(`poets${entry.FullUrl}/poet.json`);
  const client = await pool.connect();
  try {
    await client.query('BEGIN');
    await client.query(
      `INSERT INTO poets (id, url, name, nickname, description, birth_year_ah, death_year_ah, birth_year_ce, death_year_ce)
       VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)
       ON CONFLICT (id) DO UPDATE SET url=$2, name=$3, nickname=$4, description=$5, birth_year_ah=$6, death_year_ah=$7,
         birth_year_ce=$8, death_year_ce=$9`,
      [poet.Id, poet.FullUrl, poet.Name, poet.Nickname, poet.Description,
       poet.ValidBirthDate ? poet.BirthYearInLHijri : null, poet.ValidDeathDate ? poet.DeathYearInLHijri : null,
       poet.BirthYearCE ?? null, poet.DeathYearCE ?? null],
    );
    // categories depth-first so parents exist before children
    const walk = async (url: string, position: number): Promise<void> => {
      const cat = await read(`poets${url}/_cat.json`);
      await client.query(
        `INSERT INTO categories (id, poet_id, parent_id, url, title, position) VALUES ($1,$2,$3,$4,$5,$6)
         ON CONFLICT (id) DO UPDATE SET poet_id=$2, parent_id=$3, url=$4, title=$5, position=$6`,
        [cat.Id, cat.PoetId, cat.ParentId, cat.FullUrl, cat.Title, position],
      );
      for (const [i, ref] of cat.Poems.entries()) {
        const p = await read(`poets${ref.FullUrl}.json`);
        const searchText = normalise([p.Title, ...p.Verses.map((v: any) => v.Text)].join(' '));
        await client.query(
          `INSERT INTO poems (id, category_id, poet_id, url, title, full_title, source_url, position, search_text,
             radif, matla, maqta, radif_letter)
           VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13)
           ON CONFLICT (id) DO UPDATE SET category_id=$2, poet_id=$3, url=$4, title=$5, full_title=$6,
             source_url=$7, position=$8, search_text=$9, radif=$10, matla=$11, maqta=$12, radif_letter=$13`,
          [p.Id, cat.Id, poet.Id, p.FullUrl, p.Title, p.FullTitle, p.SourceUrl ?? null, i, searchText,
           p.Radif ?? null, p.Matla ?? null, p.Maqta ?? null, ref.RadifLetter ?? null],
        );
        await client.query('DELETE FROM verses WHERE poem_id = $1', [p.Id]);
        if (p.Verses.length) {
          // one multi-row insert per poem
          const values: unknown[] = [];
          const rows = p.Verses.map((v: any, j: number) => {
            values.push(p.Id, v.VOrder, v.Position, v.CoupletIndex ?? 0, v.Text);
            const o = j * 5;
            return `($${o + 1},$${o + 2},$${o + 3},$${o + 4},$${o + 5})`;
          });
          await client.query(`INSERT INTO verses (poem_id, vorder, position, couplet, text) VALUES ${rows.join(',')}`, values);
        }
        poems++;
      }
      for (const [i, child] of cat.ChildCats.entries()) await walk(child.FullUrl, i);
    };
    await walk(entry.FullUrl, 0);
    await client.query('COMMIT');
  } catch (e) {
    await client.query('ROLLBACK');
    throw e;
  } finally {
    client.release();
  }
  if (process.stdout.isTTY) process.stdout.write(`\r${poems} poems`); // progress only when interactive (cron logs stay clean)
}
await pool.query(await readFile(new URL('../../db/featured.sql', import.meta.url), 'utf8'));
// published tags (divan-data/divan/*.tags) back onto the pages they belong to
if (!isHttp) console.log(`\ntags on ${await loadTags(SOURCE)} pages`);
console.log(`\nimported ${manifest.Poets.length} poets, ${poems} poems`);
await pool.end();