Publishing commits to divan-data git (#34); public names only; 'last edited' on poems

- api/src/git.ts: each published version is a commit in the divan-data checkout (queued, one at a
  time): the moderator as author by public name with a placeholder email, the summary and version
  in the message, Reviewed-by / Approved-by / Divan-Revision / Divan-Version trailers;
  DIVAN_GIT_PUSH=1 pushes. The commit id is kept with the revision.
- Privacy fix: published files and commits carry public names (profile name, else 'موڈریٹر <id>'),
  never email addresses (divan-data is public).
- Poem pages: 'دیوان کا ورژن …' with who edited, reviewed and published, the date, and a link to
  the commit diff (DIVAN_DATA_COMMIT_URL).
- deploy/sync.sh: pull --rebase so publishing commits are kept.
- Admins are super moderators (documented).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

README.md

@@ -35,7 +35,7 @@ The import upserts, so re-running it after a divan-data sync applies the changes
 
 **Accounts and admin.** Readers sign up with an email address and password (no email is sent). The first admin is made on the server: sign up on the site, then `npm run make-admin -- you@example.com` in `api/`. Admins manage users at `/admin` (search, password reset on a reader's request, disable, roles, delete) and see every admin action at `/admin/audit`. Moderators (L2 junior, L1 senior) get scoped permissions from admins: a scope (all poets, a poet, a book with everything in it, or one work), content types (poets, books, works, dictionary) and actions (create, edit, delete, arrange); `can()` in `api/src/permissions.ts` is the one check for moderation.
 
-**Moderation.** Moderators open **ترمیم کریں** on a work they may edit, change its Divan text (see `docs/content-model.md`) with an edit summary and submit it. An L1 moderator covering that work approves, returns (with a reason) or rejects it; an admin publishes. L1 drafts go straight to the admin and an admin's own edits publish directly. Every step is recorded (`/mod`, the activity log, each work's history at `/mod/work/<id>`). Publishing numbers the version, writes it to divan-data's `divan/` folder (`DIVAN_DATA_DIR`, default `../divan-data`) and shows it on the site at once; a draft started before a newer version was published cannot be published.
+**Moderation.** Moderators open **ترمیم کریں** on a work they may edit, change its Divan text (see `docs/content-model.md`) with an edit summary and submit it. An L1 moderator covering that work approves, returns (with a reason) or rejects it; an admin publishes. L1 drafts go straight to the admin and an admin's own edits publish directly. Every step is recorded (`/mod`, the activity log, each work's history at `/mod/work/<id>`). Admins are super moderators: they edit, review and publish any work without grants. Publishing numbers the version, writes it to divan-data's `divan/` folder (`DIVAN_DATA_DIR`, default `../divan-data`), **commits it there** (the moderator as author by public name, with `Reviewed-by:` and `Approved-by:` trailers; `DIVAN_GIT_PUSH=1` also pushes), and shows it on the site at once with a link to the commit (`DIVAN_DATA_COMMIT_URL`, `{sha}` replaced). Public content never carries email addresses. A draft started before a newer version was published cannot be published. The daily sync rebases on pull so these commits are kept.
 
 ## Daily content sync (server)
 

api/src/auth.ts

@@ -60,6 +60,9 @@ const signinByIp = limiter(20, 15 * 60_000), signinByEmail = limiter(8, 15 * 60_
 function sha(t: string) {
   return createHash('sha256').update(t).digest('hex');
 }
+// a person's public name (shown on published content and in divan-data's git history): never their email
+export const publicName = (u: { id: unknown; full_name?: string | null }) => u.full_name?.trim() || `موڈریٹر ${u.id}`;
+
 export const publicUser = (u: any) => ({
   id: Number(u.id), email: u.email, role: u.role as string, created_at: u.created_at, full_name: u.full_name ?? '', bio: u.bio ?? '',
 });

api/src/git.ts

@@ -0,0 +1,31 @@
+// Publishing to git (#34): each published version is a commit in the divan-data checkout, so the public git
+// history is the record of every change. Commits run one at a time (ponytail: in-process queue; one API process).
+// Moderators appear by their public name with a placeholder email, never their real address.
+//   DIVAN_GIT_PUSH=1        push after each commit (else the daily sync or a release pushes)
+//   DIVAN_GIT_EMAIL_DOMAIN  domain of the placeholder emails (default users.noreply.divan)
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
+
+const run = promisify(execFile);
+const git = (dir: string, ...args: string[]) => run('git', ['-C', dir, ...args], { maxBuffer: 16 * 1024 * 1024 });
+let queue: Promise<unknown> = Promise.resolve();
+
+export type Person = { id: number | string; name: string };
+// a moderator as a git identity: public name, placeholder email
+export const identity = (p: Person) => `${p.name.replace(/[<>\n]/g, '')} <moderator-${p.id}@${process.env.DIVAN_GIT_EMAIL_DOMAIN ?? 'users.noreply.divan'}>`;
+
+// commit these files (paths relative to dir) as the author; returns the commit id, or null if dir is not a git
+// checkout (publishing still works, there is just no commit)
+export function commit(dir: string, files: string[], author: Person, message: string): Promise<string | null> {
+  const job = queue.then(async () => {
+    try { await git(dir, 'rev-parse', '--is-inside-work-tree'); } catch { return null; }
+    await git(dir, 'add', '--', ...files);
+    await git(dir, '-c', 'user.name=Divan', '-c', `user.email=publish@${process.env.DIVAN_GIT_EMAIL_DOMAIN ?? 'users.noreply.divan'}`,
+      'commit', '--quiet', `--author=${identity(author)}`, '-m', message, '--', ...files);
+    const sha = (await git(dir, 'rev-parse', 'HEAD')).stdout.trim();
+    if (process.env.DIVAN_GIT_PUSH === '1') await git(dir, 'push', '--quiet').catch((e) => console.error('divan-data push failed:', e.message));
+    return sha;
+  });
+  queue = job.catch(() => {});
+  return job;
+}

api/src/moderation.test.ts

@@ -9,6 +9,7 @@ import { adminRoutes } from './admin.ts';
 import { permissionRoutes } from './permissions.ts';
 import { moderationRoutes } from './moderation.ts';
 import { diffLines } from './diff.ts';
+import { execFileSync } from 'node:child_process';
 import { pool } from './db.ts';
 
 after(() => pool.end());
@@ -20,6 +21,8 @@ test('line diff: kept, removed and added lines', () => {
 test('pipeline: L2 drafts, L1 approves, admin publishes; returns, rejects, permissions, conflicts, history', async () => {
   const data = await mkdtemp(join(tmpdir(), 'divan-data-'));
   process.env.DIVAN_DATA_DIR = data;
+  const git = (...a: string[]) => execFileSync('git', ['-C', data, ...a], { encoding: 'utf8' });
+  git('init', '-q'); git('-c', 'user.name=t', '-c', 'user.email=t@t', 'commit', '-q', '--allow-empty', '-m', 'start');
   const app = Fastify();
   authRoutes(app); adminRoutes(app); permissionRoutes(app); moderationRoutes(app);
   const run = Date.now();
@@ -31,6 +34,7 @@ test('pipeline: L2 drafts, L1 approves, admin publishes; returns, rejects, permi
     return { ...s, id: s.user.id };
   };
   const admin = await person('admin', 'admin'), l1 = await person('l1', 'mod-l1'), l2 = await person('l2', 'mod-l2');
+  await call('POST', '/api/auth/profile', { full_name: 'نیا موڈریٹر' }, l2.token); // a public name; the L1 has none
   const other = await person('l1other', 'mod-l1'), reader = await person('reader', 'reader');
 
   // a Ghalib ghazal; the work is restored at the end
@@ -74,9 +78,20 @@ test('pipeline: L2 drafts, L1 approves, admin publishes; returns, rejects, permi
     // admin publishes: version 1 in divan-data and on the site
     assert.deepEqual((await call('POST', `/api/mod/revisions/${id}/publish`, {}, admin.token)).json(), { status: 'published', version: 1 });
     const file = JSON.parse(await readFile(join(data, 'divan', poem.url.slice(1) + '.json'), 'utf8'));
-    assert.equal(file.Edited.by, `l2-${run}@divan.test`);
+    // public names only in divan-data (it is public); a commit with the moderator as author and review trailers
-    assert.equal(file.Edited.reviewedBy, `l1-${run}@divan.test`);
+    assert.equal(file.Edited.by, 'نیا موڈریٹر');
-    assert.equal(file.Edited.publishedBy, `admin-${run}@divan.test`);
+    assert.equal(file.Edited.reviewedBy, `موڈریٹر ${l1.id}`);
+    assert.equal(file.Edited.publishedBy, `موڈریٹر ${admin.id}`);
+    assert.ok(!JSON.stringify(file).includes('@'), 'no email addresses');
+    const log = git('log', '-1', '--format=%an <%ae>%n%B');
+    assert.match(log, new RegExp(`^نیا موڈریٹر <moderator-${l2.id}@users\\.noreply\\.divan>`));
+    assert.match(log, /\(ورژن 1\)/);
+    assert.match(log, new RegExp(`Reviewed-by: موڈریٹر ${l1.id} <moderator-${l1.id}@`));
+    assert.match(log, new RegExp(`Approved-by: موڈریٹر ${admin.id} <moderator-${admin.id}@`));
+    assert.ok(!log.includes('divan.test'), 'no real emails in git');
+    assert.deepEqual(git('show', '--name-only', '--format=', 'HEAD').trim().split('\n').sort(), [`divan${poem.url}.dtx`, `divan${poem.url}.json`]);
+    const committed = (await pool.query(`SELECT commit FROM revisions WHERE id = $1`, [id])).rows[0].commit;
+    assert.equal(committed, git('rev-parse', 'HEAD').trim(), 'the commit is recorded with the version');
     assert.ok((await readFile(join(data, 'divan', poem.url.slice(1) + '.dtx'), 'utf8')).includes(misra + ' (ترمیم)'));
     assert.equal((await pool.query('SELECT text FROM verses WHERE poem_id = $1 AND couplet = 1 AND position = $2', [poem.id, 'Left'])).rows[0].text, misra + ' (ترمیم)');
 

api/src/moderation.ts

@@ -5,7 +5,9 @@
 //   L1 moderator (grant covering the work): approves, returns (with a comment) or rejects; an L1's own draft
 //                goes straight to the admin step  -> approved
 //   admin: publishes (an admin's own draft publishes directly), returns or rejects
-// Publishing numbers the version, writes it to divan-data as Divan-owned content (owned.ts) and updates the site.
+// Admins are super moderators: they can edit, review and publish any work without grants.
+// Publishing numbers the version, writes it to divan-data as Divan-owned content (owned.ts), commits it there (git.ts)
+// and updates the site.
 // If another version was published after the draft started, publishing is refused until the draft is redone.
 //   GET  /api/mod/can?poem=                  what the reader may do on a work
 //   GET  /api/mod/queue                      my drafts, drafts to review, drafts to publish
@@ -23,6 +25,8 @@ import { fromPoem, writeOwned } from './owned.ts';
 import { parse, toVerses } from './divantext.ts';
 import { normalise } from './urdu.ts';
 import { diffLines, changed } from './diff.ts';
+import { commit, identity } from './git.ts';
+import { publicName } from './auth.ts';
 
 const dataDir = () => process.env.DIVAN_DATA_DIR ?? new URL('../../../divan-data', import.meta.url).pathname;
 const isModerator = (u: any) => ['mod-l2', 'mod-l1', 'admin'].includes(u?.role);
@@ -79,13 +83,22 @@ async function publish(r: any, u: any, comment?: string) {
     throw Object.assign(new Error('اس دوران اس کلام کا نیا ورژن شائع ہو چکا ہے۔ مسودہ واپس بھیج کر تازہ متن پر دوبارہ بنوائیں۔'), { code: 409 });
   const doc = parse(r.content), verses = toVerses(doc);
   const title = doc.meta['عنوان'] || cur.poem.title, version = cur.version + 1, at = new Date().toISOString();
-  // divan-data first (the published record), then the site's database
+  // who did it, by public name (divan-data is public: never email addresses)
-  await writeOwned(dataDir(), cur.poem.url, r.content, { by: r.author_email, at, version, reviewedBy: r.reviewer_email, publishedBy: u.email });
+  const people = async (id: unknown) => id ? (await pool.query('SELECT id, full_name FROM users WHERE id = $1', [id])).rows[0] ?? null : null;
+  const [author, reviewer] = await Promise.all([people(r.author_id), people(r.reviewer_id)]);
+  const who = (p: any) => p && { id: Number(p.id), name: publicName(p) };
+  const credits = { by: who(author)?.name ?? 'موڈریٹر', reviewedBy: who(reviewer)?.name ?? null, publishedBy: publicName(u) };
+  // divan-data first (the published record, committed to git), then the site's database
+  const files = await writeOwned(dataDir(), cur.poem.url, r.content, { ...credits, at, version, revision: Number(r.id) });
+  const trailers = [`Divan-Revision: ${r.id}`, `Divan-Version: ${version}`,
+    ...(reviewer ? [`Reviewed-by: ${identityOf(reviewer)}`] : []), `Approved-by: ${identityOf(u)}`];
+  const sha = await commit(dataDir(), files.map((f) => f.slice(dataDir().replace(/\/$/, '').length + 1)),
+    who(author) ?? { id: 0, name: 'موڈریٹر' }, `${title}: ${r.summary || 'ترمیم'} (ورژن ${version})\n\n${trailers.join('\n')}`);
   const client = await pool.connect();
   try {
     await client.query('BEGIN');
-    await client.query(`UPDATE revisions SET status = 'published', version = $2, publisher_email = $3, published_at = $4, updated_at = now() WHERE id = $1`,
+    await client.query(`UPDATE revisions SET status = 'published', version = $2, publisher_email = $3, published_at = $4, commit = $5, credits = $6, updated_at = now()
-      [r.id, version, u.email, at]);
+                        WHERE id = $1`, [r.id, version, u.email, at, sha, credits]);
     await client.query('UPDATE poems SET title = $2, search_text = $3 WHERE id = $1',
       [r.entity_id, title, normalise([title, ...verses.map((v) => v.Text)].join(' '))]);
     await client.query('DELETE FROM verses WHERE poem_id = $1', [r.entity_id]);
@@ -104,6 +117,8 @@ async function publish(r: any, u: any, comment?: string) {
   return version;
 }
 
+const identityOf = (p: any) => identity({ id: Number(p.id), name: publicName(p) });
+
 export function moderationRoutes(app: FastifyInstance) {
   app.get<{ Querystring: { poem?: string } }>('/api/mod/can', async (req) => {
     const u = await sessionUser(req), poemId = Number(req.query.poem) || 0;
@@ -202,7 +217,7 @@ export function moderationRoutes(app: FastifyInstance) {
         return { status: u.role === 'mod-l1' ? 'approved' : 'submitted' };
       }
       if (action === 'approve') {
-        await pool.query(`UPDATE revisions SET status = 'approved', reviewer_email = $2, updated_at = now() WHERE id = $1`, [r.id, u.email]);
+        await pool.query(`UPDATE revisions SET status = 'approved', reviewer_id = $3, reviewer_email = $2, updated_at = now() WHERE id = $1`, [r.id, u.email, u.id]);
         await event(r.id, u, 'approved', comment);
         return { status: 'approved' };
       }

api/src/owned.ts

@@ -31,7 +31,7 @@ export function fromPoem(poem: { Title: string; Verses: Verse[]; SourceUrl?: str
 
 // write a Divan-owned work (the .dtx and the generated .json); returns the paths written
 export async function writeOwned(dataDir: string, url: string, dtx: string,
-  edited: { by: string; at?: string; version?: number; reviewedBy?: string | null; publishedBy?: string }) {
+  edited: { by: string; at?: string; version?: number; revision?: number; reviewedBy?: string | null; publishedBy?: string }) {
   const doc = parse(dtx);
   const verses = toVerses(doc);
   if (!verses.length) throw new Error('the text has no verses or paragraphs');

api/src/server.ts

@@ -53,7 +53,7 @@ app.get<{ Querystring: { url?: string } }>('/api/page', async (req, reply) => {
 
   const poem = (await pool.query('SELECT * FROM poems WHERE url = $1', [url])).rows[0];
   if (poem) {
-    const [verses, poet, crumbs, siblings] = await Promise.all([
+    const [verses, poet, crumbs, siblings, edited] = await Promise.all([
       pool.query('SELECT vorder, position, couplet, text FROM verses WHERE poem_id = $1 ORDER BY vorder', [poem.id]),
       pool.query('SELECT id, url, nickname FROM poets WHERE id = $1', [poem.poet_id]),
       ancestors(poem.category_id),
@@ -62,9 +62,15 @@ app.get<{ Querystring: { url?: string } }>('/api/page', async (req, reply) => {
                 (SELECT url FROM poems WHERE category_id = $1 AND position > $2 ORDER BY position LIMIT 1) AS next`,
         [poem.category_id, poem.position],
       ),
+      // the latest Divan version (public names only) and its divan-data commit
+      pool.query(`SELECT version, published_at, credits, commit FROM revisions WHERE entity = 'work' AND entity_id = $1 AND status = 'published'
+                  ORDER BY version DESC LIMIT 1`, [poem.id]),
     ]);
+    const e = edited.rows[0];
+    const divan = e && { version: e.version, at: e.published_at, ...e.credits,
+      commit_url: e.commit ? (process.env.DIVAN_DATA_COMMIT_URL ?? 'https://git.anasrashid.net/anas/divan-data/commit/{sha}').replace('{sha}', e.commit) : null };
     const { search_text, ...rest } = poem;
-    return { type: 'poem', poem: rest, poet: poet.rows[0], breadcrumbs: crumbs, verses: verses.rows, ...siblings.rows[0] };
+    return { type: 'poem', poem: rest, poet: poet.rows[0], breadcrumbs: crumbs, verses: verses.rows, ...siblings.rows[0], divan };
   }
 
   const cat = (await pool.query('SELECT * FROM categories WHERE url = $1', [url])).rows[0];

db/schema.sql

@@ -170,6 +170,9 @@ CREATE TABLE IF NOT EXISTS revisions (
     published_at timestamptz
 );
 ALTER TABLE revisions ADD COLUMN IF NOT EXISTS base_content text NOT NULL DEFAULT '';
+ALTER TABLE revisions ADD COLUMN IF NOT EXISTS reviewer_id bigint REFERENCES users(id) ON DELETE SET NULL;
+ALTER TABLE revisions ADD COLUMN IF NOT EXISTS commit text;     -- divan-data commit of a published version
+ALTER TABLE revisions ADD COLUMN IF NOT EXISTS credits jsonb;   -- public names at publishing: by, reviewedBy, publishedBy
 CREATE INDEX IF NOT EXISTS revisions_entity ON revisions(entity, entity_id);
 CREATE INDEX IF NOT EXISTS revisions_status ON revisions(status);
 CREATE UNIQUE INDEX IF NOT EXISTS revisions_version ON revisions(entity, entity_id, version) WHERE version IS NOT NULL;

deploy/sync.sh

@@ -26,7 +26,7 @@ if command -v flock >/dev/null && ! flock -n 9; then echo "$(date -u +%FT%TZ) sy
 echo "== $(date -u +%FT%TZ) divan sync"
 [ -d "$DATA_DIR/.git" ] || git clone -q https://github.com/anas-rashid/divan-data.git "$DATA_DIR"
 cd "$DATA_DIR"
-git pull -q --ff-only
+git pull -q --rebase   # keep commits made by publishing in the Divan app (git.ts)
 
 if [ "${DIVAN_DATA_PUSH:-0}" = 1 ]; then
   ./update.sh                                   # fetch + rebuild, commit and push if the data changed

web/src/pages/[...path].astro

@@ -136,6 +136,13 @@ const title = page.type === 'poem' ? page.poem.title : page.type === 'poet' ? pa
         {page.next ? <a href={page.next}>اگلا ›</a> : <span />}
         {page.prev ? <a href={page.prev}>‹ پچھلا</a> : <span />}
       </nav>
+      {page.divan && (
+        <p class="source divan-edit">
+          دیوان کا ورژن {ud(page.divan.version)} · ترمیم: {page.divan.by}{page.divan.reviewedBy && <> · جائزہ: {page.divan.reviewedBy}</>} · اشاعت: {page.divan.publishedBy}
+          · <bdi dir="ltr">{new Date(page.divan.at).toISOString().slice(0, 10)}</bdi>
+          {page.divan.commit_url && <> · <a href={page.divan.commit_url}>تبدیلی دیکھیں</a></>}
+        </p>
+      )}
       {page.poem.source_url && (
         <p class="source">ماخذ: <a href={page.poem.source_url}>ویکی ماخذ</a> · {page.poet && <a href={page.poet.url}>{page.poet.nickname}</a>}</p>
       )}