Merge pull request 'Research: content model and moderation (no wiki engine embedded; Divan text format)' (#49) from research/content-model into main

api/src/divantext.test.ts

@@ -0,0 +1,63 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { readFileSync, existsSync } from 'node:fs';
+import { spawnSync } from 'node:child_process';
+import { parse, toVerses, toTEI, words, inline } from './divantext.ts';
+
+const doc = (f: string) => readFileSync(new URL(`../../docs/content-model/${f}`, import.meta.url), 'utf8');
+const ghazal = parse(doc('ghalib-ghazal.dtx')), prose = parse(doc('sir-syed-azadi-e-rai.dtx'));
+
+test('a ghazal: data points, couplets, labels, dictionary links, footnote, variant', () => {
+  assert.equal(ghazal.meta['شاعر'], 'مرزا غالب');
+  assert.equal(ghazal.meta['ردیف'], 'ہوتا');
+  assert.equal(ghazal.blocks.length, 11);
+  assert.ok(ghazal.blocks.every((b) => b.type === 'couplet'), 'two lines = a couplet');
+  const first = ghazal.blocks[0] as any, fourth = ghazal.blocks[3] as any, sixth = ghazal.blocks[5] as any;
+  assert.equal(first.label, 'مطلع');
+  assert.deepEqual(first.lines[0].words, [{ lemma: 'وصال', shown: 'وصال' }]);
+  assert.equal(first.lines[0].text, 'یہ نہ تھی ہماری قسمت کہ وصال یار ہوتا', 'markup removed from the text');
+  assert.equal(fourth.lines[0].notes.length, 1);
+  assert.deepEqual(sixth.lines[0].variants, [{ shown: 'تھمتا', others: ['رکتا'], source: 'مثال کے لیے' }]);
+});
+
+test('the same verses as the current export (the site would show the same text)', { skip: !existsSync(new URL('../../../divan-data/poets/p266/ghazal', import.meta.url)) }, () => {
+  const dir = new URL('../../../divan-data/poets/p266/ghazal/', import.meta.url);
+  const cat = JSON.parse(readFileSync(new URL('_cat.json', dir), 'utf8'));
+  const ref = cat.Poems.find((p: any) => p.Title === 'یہ نہ تھی ہماری قسمت کہ وصال یار ہوتا');
+  const exported = JSON.parse(readFileSync(new URL(`../../../divan-data/poets${ref.FullUrl}.json`, import.meta.url), 'utf8')).Verses;
+  const pick = (v: any) => [v.Position, v.Text, v.CoupletIndex];
+  assert.deepEqual(toVerses(ghazal).map(pick), exported.map(pick));
+});
+
+test('Wikisource markup reads as-is ({{header}}, <poem align>)', () => {
+  const ws = parse(`{{header\n | title = غزل\n | author = مرزا غالب\n}}\n\n<poem align="right" dir="rtl">\nپہلا مصرع\nدوسرا مصرع\n\nاکیلا مصرع\n</poem>`);
+  assert.equal(ws.meta.author, 'مرزا غالب');
+  assert.deepEqual(ws.blocks.map((b) => b.type), ['couplet', 'line']);
+});
+
+test('prose: chapter heading, paragraphs, footnote', () => {
+  assert.equal(prose.meta['مصنف'], 'سید احمد خان');
+  assert.deepEqual(prose.blocks.map((b) => b.type), ['heading', 'para', 'para', 'para']);
+  assert.equal((prose.blocks[1] as any).line.notes.length, 1);
+  assert.deepEqual(toVerses(prose).map((v) => v.Position), ['Paragraph', 'Paragraph', 'Paragraph']);
+});
+
+test('stanzas and single lines', () => {
+  const d = parse('<poem>\nا\nب\nج\nد\nہ\n\nو\n</poem>');
+  assert.deepEqual(d.blocks.map((b) => b.type), ['stanza', 'line']);
+});
+
+test('what is a word: spaces split, a ZWNJ keeps a compound, punctuation and ؔ are not part of words', () => {
+  assert.deepEqual(words('یہ مسائل تصوف، یہ ترا بیان غالبؔ!'), ['یہ', 'مسائل', 'تصوف', 'یہ', 'ترا', 'بیان', 'غالب']);
+  assert.deepEqual(words('بے‌ثبوت بات۔'), ['بے‌ثبوت', 'بات']);
+  assert.deepEqual(inline('[[لغت:دیوانہ|دیوانے]] ہیں').words, [{ lemma: 'دیوانہ', shown: 'دیوانے' }]);
+});
+
+test('TEI export is well-formed XML with couplets as <lg type="sher"> of two <l>', () => {
+  const tei = toTEI(ghazal);
+  assert.match(tei, /<lg type="sher" n="1" ana="#matla"><l n="1">یہ نہ تھی ہماری قسمت کہ <w lemma="وصال">وصال<\/w> یار ہوتا<\/l>/);
+  assert.match(tei, /<app><lem>تھمتا<\/lem><rdg source="مثال کے لیے">رکتا<\/rdg><\/app>/);
+  const lint = spawnSync('xmllint', ['--noout', '-'], { input: tei + '\n' + '' });
+  if (!lint.error) assert.equal(lint.status, 0, String(lint.stderr));
+  if (!lint.error) assert.equal(spawnSync('xmllint', ['--noout', '-'], { input: toTEI(prose) }).status, 0);
+});

api/src/divantext.ts

@@ -0,0 +1,142 @@
+// Divan text (research prototype, #38): the source format for content Divan edits and publishes.
+// A small, Wikisource-compatible subset of wikitext, written as plain Urdu lines so git diffs show exactly
+// which misra or paragraph changed. Parsed into blocks (what is a couplet, stanza, paragraph, chapter, word)
+// and converted to the site's verse JSON and to TEI P5 for scholars.
+//
+//   {{دیوان | عنوان = … | شاعر = … | صنف = غزل | ردیف = … | ماخذ = …}}     work data points (one per line)
+//   == باب ==                                                             chapter/section heading (prose)
+//   <poem> … </poem>                                                       verse: units split by blank lines;
+//                                                                          2 lines = شعر (couplet), 1 = مصرع, 3+ = بند
+//   blank-line paragraphs outside <poem>                                   prose
+//   {{مطلع}} {{حسن مطلع}} {{مقطع}} at the start of a unit                  overrides the computed labels
+//   [[لفظ]]  [[لغت:دیوانہ|دیوانے]]                                         word linked to the dictionary (lemma|shown)
+//   <ref>…</ref>                                                           footnote
+//   {{نسخہ|متن|دوسرا متن|ماخذ=…}}                                           variant reading (first is shown)
+// Words: split on spaces; a zero-width non-joiner keeps a compound one word (بے‌ثبوت); punctuation ، ۔ ؟ ! is
+// not part of a word; the izafat kasra stays on its word.
+
+export type Inline = { text: string; words: { shown: string; lemma: string }[]; notes: string[]; variants: { shown: string; others: string[]; source?: string }[] };
+export type Block =
+  | { type: 'heading'; text: string }
+  | { type: 'para'; line: Inline }
+  | { type: 'couplet' | 'line' | 'stanza'; lines: Inline[]; label?: string };
+export type Doc = { meta: Record<string, string>; blocks: Block[] };
+
+const LABELS = ['مطلع', 'حسن مطلع', 'مقطع'];
+
+// {{name | a = b | …}} → fields (positional ones as 1, 2, …); nested templates and links kept as text
+function template(src: string) {
+  const body = src.slice(2, -2), parts: string[] = [];
+  let depth = 0, cur = '';
+  for (const ch of body) {
+    if (ch === '{' || ch === '[') depth++;
+    if (ch === '}' || ch === ']') depth--;
+    if (ch === '|' && depth === 0) { parts.push(cur); cur = ''; } else cur += ch;
+  }
+  parts.push(cur);
+  const fields: Record<string, string> = {};
+  let n = 0;
+  for (const p of parts.slice(1)) {
+    const m = p.match(/^\s*([^=]+?)\s*=\s*([\s\S]*)$/);
+    if (m) fields[m[1]] = m[2].trim(); else fields[String(++n)] = p.trim();
+  }
+  return { name: parts[0].trim(), fields };
+}
+
+// one line of text: links, notes and variants out, plain text in
+export function inline(src: string): Inline {
+  const words: Inline['words'] = [], notes: string[] = [], variants: Inline['variants'] = [];
+  let text = src
+    .replace(/<ref>([\s\S]*?)<\/ref>/g, (_, n) => { notes.push(n.trim()); return ''; })
+    .replace(/\{\{نسخہ\|[^{}]*\}\}/g, (t) => {
+      const { fields } = template(t), others = Object.keys(fields).filter((k) => /^\d+$/.test(k) && k !== '1').map((k) => fields[k]);
+      variants.push({ shown: fields['1'] ?? '', others, source: fields['ماخذ'] });
+      return fields['1'] ?? '';
+    })
+    .replace(/\[\[(?:لغت:)?([^\]|]+)(?:\|([^\]]+))?\]\]/g, (_, lemma, shown) => {
+      words.push({ lemma: lemma.trim(), shown: (shown ?? lemma).trim() });
+      return (shown ?? lemma).trim();
+    });
+  text = text.replace(/[ \t]+/g, ' ').trim();
+  return { text, words, notes, variants };
+}
+
+export function parse(src: string): Doc {
+  const meta: Record<string, string> = {};
+  let rest = src.replace(/\r\n?/g, '\n').replace(/^\s*\{\{\s*(?:دیوان|header)(?=[\s|])[\s\S]*?\n\}\}\s*\n?/u, (t) => {
+    Object.assign(meta, template(t.trim()).fields);
+    return '';
+  });
+  const blocks: Block[] = [];
+  // verse inside <poem>, prose outside
+  for (const piece of rest.split(/(<poem[^>]*>[\s\S]*?<\/poem>)/)) {
+    const verse = piece.match(/^<poem[^>]*>([\s\S]*?)<\/poem>$/);
+    for (const unit of (verse ? verse[1] : piece).split(/\n\s*\n/)) {
+      let lines = unit.split('\n').map((l) => l.trim()).filter(Boolean);
+      if (!lines.length) continue;
+      if (!verse) {
+        for (const l of lines) {
+          const h = l.match(/^=+\s*(.+?)\s*=+$/);
+          blocks.push(h ? { type: 'heading', text: h[1] } : { type: 'para', line: inline(l) });
+        }
+        continue;
+      }
+      let label: string | undefined;
+      const m = lines[0].match(/^\{\{(مطلع|حسن مطلع|مقطع)\}\}\s*/);
+      if (m) { label = m[1]; lines[0] = lines[0].slice(m[0].length); }
+      lines = lines.filter(Boolean);
+      const type = lines.length === 2 ? 'couplet' : lines.length === 1 ? 'line' : 'stanza';
+      blocks.push({ type, lines: lines.map(inline), ...(label && { label }) });
+    }
+  }
+  return { meta, blocks };
+}
+
+// what counts as a word (for indexing, the word sidebar and dictionary links)
+export const words = (text: string) =>
+  text.split(/[\s،۔؟!؛:«»()"]+/).map((w) => w.replace(/ؔ/g, '')).filter(Boolean);
+
+// → the site's verse JSON (the ganjoor-data layout divan-data already exports)
+export function toVerses(doc: Doc) {
+  const out: { VOrder: number; Position: string; Text: string; CoupletIndex: number }[] = [];
+  let couplet = 0;
+  for (const b of doc.blocks) {
+    if (b.type === 'heading') continue;
+    if (b.type === 'para') out.push({ VOrder: 0, Position: 'Paragraph', Text: b.line.text, CoupletIndex: couplet++ });
+    else if (b.type === 'couplet') { b.lines.forEach((l, i) => out.push({ VOrder: 0, Position: i ? 'Left' : 'Right', Text: l.text, CoupletIndex: couplet })); couplet++; }
+    else for (const l of b.lines) out.push({ VOrder: 0, Position: 'Single', Text: l.text, CoupletIndex: couplet++ });
+  }
+  return out.map((v, i) => ({ ...v, VOrder: i + 1 }));
+}
+
+const x = (s: string) => s.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/"/g, '&quot;');
+// a line with its linked words as <w lemma>, variants as <app>, notes as <note>
+function teiLine(l: Inline) {
+  let s = x(l.text);
+  for (const w of l.words) s = s.replace(x(w.shown), `<w lemma="${x(w.lemma)}">${x(w.shown)}</w>`);
+  for (const v of l.variants) s = s.replace(x(v.shown), `<app><lem>${x(v.shown)}</lem>${v.others.map((o) => `<rdg${v.source ? ` source="${x(v.source)}"` : ''}>${x(o)}</rdg>`).join('')}</app>`);
+  return s + l.notes.map((n) => `<note place="foot">${x(n)}</note>`).join('');
+}
+
+// → TEI P5: verse as line groups (a couplet is <lg type="sher"> with two <l>), prose as <div>/<p>
+export function toTEI(doc: Doc) {
+  const m = doc.meta, body: string[] = [];
+  let n = 0;
+  for (const b of doc.blocks) {
+    if (b.type === 'heading') body.push(`<head>${x(b.text)}</head>`);
+    else if (b.type === 'para') body.push(`<p>${teiLine(b.line)}</p>`);
+    else {
+      const type = b.type === 'couplet' ? 'sher' : b.type === 'stanza' ? 'band' : 'misra';
+      const ana = b.label ? ` ana="#${b.label === 'مطلع' ? 'matla' : b.label === 'مقطع' ? 'maqta' : 'husn-e-matla'}"` : '';
+      body.push(`<lg type="${type}" n="${++n}"${ana}>${b.lines.map((l, i) => `<l n="${i + 1}">${teiLine(l)}</l>`).join('')}</lg>`);
+    }
+  }
+  const genre = m['صنف'] ?? '', poem = doc.blocks.some((b) => b.type !== 'para' && b.type !== 'heading');
+  return `<TEI xmlns="http://www.tei-c.org/ns/1.0" xml:lang="ur"><teiHeader><fileDesc>`
+    + `<titleStmt><title>${x(m['عنوان'] ?? m['title'] ?? '')}</title><author>${x(m['شاعر'] ?? m['مصنف'] ?? m['author'] ?? '')}</author></titleStmt>`
+    + `<publicationStmt><publisher>دیوان</publisher><availability><licence target="https://creativecommons.org/licenses/by-sa/4.0/"/></availability></publicationStmt>`
+    + `<sourceDesc><p>${x(m['ماخذ'] ?? 'ویکی ماخذ')}</p></sourceDesc></fileDesc></teiHeader>`
+    + `<text><body>${poem ? `<lg type="${x(genre || 'poem')}"${m['ردیف'] ? ` rhyme="${x(m['ردیف'])}"` : ''}>${body.join('')}</lg>` : `<div type="chapter">${body.join('')}</div>`}</body></text></TEI>`;
+}
+
+export const LABELS_ = LABELS;

docs/content-model.md

@@ -0,0 +1,143 @@
+# Content model and moderation: research and recommendation (#38)
+
+October 2026. Question from the owner: content moderation should work like a wiki, with tags and data points
+that say what a couplet, a stanza, a chapter and a word are, and we should see whether an open-source wiki
+content-management module can be integrated.
+
+## What Divan needs
+
+- **Structure**: poet › book › chapter/section › work; verse as couplets (شعر, two misras), single lines and
+  stanzas (بند); prose as paragraphs under headings; words (for the dictionary, search and the word book).
+- **Data points**: genre (غزل, نظم, رباعی, …), radif, qafiya, matla/maqta, pen name, footnotes, variant
+  readings between editions, links from words to dictionary entries, sources.
+- **Moderation**: L2 → L1 → admin approval (#31), scoped per poet/book/work and per action, including arranging
+  (#29, #41); Divan's copy takes precedence over Wikisource (#30); upstream changes reviewed (#36).
+- **Publishing**: the published version is committed to the open divan-data repo, so its git history is the
+  public version history and every change is a readable diff (#34).
+- **Clients**: the site and mobile apps read the same API (#44).
+
+## Options considered (checked October 2026)
+
+| Option | Status | Licence | Fit | Why |
+|---|---|---|---|---|
+| MediaWiki 1.47 LTS (due Nov 2026) as a module | Stable; runs Urdu Wikisource | GPL-2.0+ | Poor | A second stack (PHP) and a second user/permission system. FlaggedRevs is "not recommended for production use" (its own documentation). Approved Revs has one approval level only. Content would be unstructured wikitext, not couplets. |
+| + ProofreadPage, Semantic MediaWiki, Wikibase | Maintained | GPL-2.0+ | Poor/partial | ProofreadPage is for page scans, which we do not have. SMW and Wikibase keep data in the wiki, not in our schema. Wikibase is worth revisiting later for linked-data records of poets and works. |
+| Wiki.js | v3 still in beta ("not for production") | AGPL-3.0 | Poor | Syncs to git, but stores Markdown pages, with no review workflow and no verse structure. |
+| BookStack, Outline, Docmost, DokuWiki | Maintained | MIT / BSL / AGPL / GPL | Poor | No multi-level approval, no git publishing, no verse model. Outline is not open source (BSL). |
+| **wikiparser-node** (library) | v1.48, very active | **GPL-3.0** | **Good** | A full wikitext parser for Node: `<poem>`, `<ref>`, templates; tested against MediaWiki's parser tests. For importing and re-syncing from Wikisource and Wiktionary. |
+| **Tiptap 3 core / ProseMirror** (editor) | Active | **MIT** | **Good** | A schema-first editor: our own nodes (ghazal › sher › misra, paragraph, heading), RTL. Tiptap's paid "Pro" add-ons (comments, history, collaboration) are not needed: review and history live in our database and in git. |
+| CodeMirror 6 (editor) | Active | MIT | Good | A plain-text "source" view with bidi support; wikiparser-node has a CodeMirror integration. |
+| **TEI P5 4.12** | The standard for literary texts | Open | Export | A couplet is `<lg>` with two `<l>`. No established Urdu or Persian TEI corpus was found. Good as an export for scholars, not as storage. |
+| **Ganjoor's correction system** (GanjoorService) | Active | GPL-3.0 | Reference | The closest working example: per-poem and per-section correction records that moderators accept or reject field by field, kept as history. We follow its data design, not its code. |
+
+**Conclusion: no wiki engine is embedded.** Divan already has what a wiki engine would duplicate: accounts,
+scoped permissions and the site. None of the engines offers the three things Divan needs together: verse
+structure, L2 → L1 → admin review scoped per poet and book, and git publishing. The "wiki" qualities (open
+history, diffs, notes per page, typed data points) are built into Divan's own moderation, using the libraries
+above.
+
+## Recommendation
+
+1. **Divan text as the source format** (prototype: `api/src/divantext.ts`). It is a small subset of Wikisource's
+   own markup, so Wikisource pages read as-is and moderators who know Wikisource feel at home:
+
+   ```
+   {{دیوان
+   | عنوان = یہ نہ تھی ہماری قسمت کہ وصال یار ہوتا
+   | شاعر = مرزا غالب
+   | صنف = غزل
+   | ردیف = ہوتا
+   | ماخذ = ویکی ماخذ
+   }}
+   <poem>
+   {{مطلع}} یہ نہ تھی ہماری قسمت کہ [[وصال]] یار ہوتا
+   اگر اور جیتے رہتے یہی انتظار ہوتا
+
+   کوئی میرے دل سے پوچھے ترے تیر [[نیم کش]] کو<ref>نیم کش: آدھا کھنچا ہوا تیر</ref>
+   یہ خلش کہاں سے ہوتی جو جگر کے پار ہوتا
+   </poem>
+   ```
+
+   | Mark-up | Meaning |
+   |---|---|
+   | `{{دیوان \| …}}` | The work's data points: title, poet/author, genre, radif, qafiya, metre, sources. |
+   | `<poem>` | Verse. Blank lines separate units: **2 lines = شعر** (couplet: misra 1 and 2), **1 line = مصرع**, **3 or more = بند** (stanza). |
+   | Text outside `<poem>` | Prose. Blank lines separate **paragraphs**; `== … ==` is a **chapter or section heading**. |
+   | `{{مطلع}}` `{{حسن مطلع}}` `{{مقطع}}` | At the start of a couplet: overrides the computed labels. |
+   | `[[لفظ]]` `[[لغت:دیوانہ\|دیوانے]]` | A **word** linked to its dictionary entry (lemma, then shown form). |
+   | `<ref>…</ref>` | Footnote. |
+   | `{{نسخہ\|متن\|دوسرا متن\|ماخذ=…}}` | Variant reading between editions; the first is shown. |
+
+   **What a word is**: text split on spaces. A zero-width non-joiner keeps a compound as one word (بے‌ثبوت).
+   Punctuation (، ۔ ؟ !) and the pen-name sign ؔ are not part of a word, and the izafat kasra stays on its word.
+   The same rule is used by search, the word sidebar and the word book.
+
+2. **Git diffs stay readable.** The same one-word correction in one misra, in each format:
+   - **Divan text**: a one-line diff showing that misra (`-کبھی تو نہ توڑ سکتا اگر استوار ہوتا` /
+     `+کبھی تو نہ توڑ سکتا جو استوار ہوتا`).
+   - **The current JSON export**: the change also rewrites a field holding the whole poem on one line, which is
+     unreadable in a diff.
+
+   divan-data therefore keeps **one `.dtx` file per work** as the published source. The JSON layout and TEI are
+   generated from it.
+
+3. **Arranging is a list in a file.** Each book or chapter keeps its contents as an ordered list of works. A
+   moderator who reorders Ghalib's ghazals (#41) changes the order of lines in that list, which is again a
+   readable diff.
+
+4. **The moderation workflow lives in PostgreSQL** (#31), modelled on Ganjoor's correction records:
+   - a `drafts` table holds the proposed Divan text with its base version, author, scope, status (draft,
+     submitted, L1-approved, published, returned, rejected) and comments;
+   - reviewers see a side-by-side diff;
+   - on publish, the `.dtx` file is written to divan-data and committed with the moderator as author, and the
+     reviewer and admin in `Reviewed-by:` / `Approved-by:` trailers (#34);
+   - the importer reads the published files, so the site and apps show the new text.
+
+5. **Editors** (#28, #32):
+   - the main editor is a structured Urdu editor on **Tiptap 3's MIT core**: couplet, misra, stanza, paragraph
+     and heading blocks; the Urdu keyboard and Roman typing; a diacritics palette; and buttons for the data
+     points above;
+   - a **source view** (CodeMirror 6) shows the Divan text itself for experienced moderators;
+   - both read and write the same Divan text.
+
+6. **Wikisource sync** (#30, #36) uses **wikiparser-node** to read Wikisource pages into Divan text. Works that
+   Divan has published are never overwritten; a changed upstream page becomes a draft in the review queue.
+
+7. **TEI P5 export** for scholars:
+   - a couplet is `<lg type="sher">` with two `<l>`;
+   - matla/maqta are `@ana`;
+   - dictionary-linked words are `<w lemma>`;
+   - variants are `<app>`;
+   - footnotes are `<note>`.
+
+   Rendering with CETEIcean or publishing through TEI Publisher can be added if scholars ask for them.
+
+8. **Later**: Wikibase as linked-data records for poets and works, if that becomes useful; OpenITI-style
+   stable identifiers for works and versions.
+
+## Prototype results
+
+`api/src/divantext.ts` parses Divan text into blocks, converts them to the site's verse JSON, and exports TEI.
+The tests (`api/src/divantext.test.ts`, 7 passing) run on the real Ghalib ghazal and Sir Syed's آزادی رائے
+(`docs/content-model/*.dtx`):
+
+- **Data points**: genre, radif, the مطلع override, a dictionary link, a footnote and a variant reading are read
+  from the annotated ghazal.
+- **Same text on the site**: the annotated ghazal produces **exactly the verses of today's export** (position,
+  text and couplet number of all 22 lines), so adopting the format changes nothing for readers.
+- **Wikisource compatibility**: raw Wikisource markup (`{{header}}`, `<poem align>`) reads unchanged.
+- **Prose and stanzas**: headings and paragraphs (with a footnote) from the prose chapter; stanzas and single
+  lines.
+- **Words**: the rule above, including compounds and the pen-name sign.
+- **TEI**: the export validates as well-formed XML (`xmllint`).
+
+## Effect on the plan
+
+- **#30** (Divan-owned content): `.dtx` files in divan-data, marked as Divan's once published; the export and
+  sync respect them.
+- **#31** (pipeline): drafts of Divan text, with diffs and the L2 → L1 → admin states.
+- **#32, #41** (editors, arranging): Tiptap editor and source view; contents lists as ordered files.
+- **#28, #33** (Urdu editor, document import): document import converts to Divan text.
+- **#34** (git publishing): commits the `.dtx` files and the contents lists.
+- **#36** (upstream changes): Wikisource re-reads through wikiparser-node become drafts.
+- **New**: the TEI export, and a word index built on the word rule above.

docs/content-model/ghalib-ghazal.dtx

@@ -0,0 +1,42 @@
+{{دیوان
+| عنوان = یہ نہ تھی ہماری قسمت کہ وصال یار ہوتا
+| شاعر = مرزا غالب
+| صنف = غزل
+| ردیف = ہوتا
+| ماخذ = ویکی ماخذ
+| ماخذ_ربط = https://ur.wikisource.org/wiki/%DB%8C%DB%81_%D9%86%DB%81_%D8%AA%DA%BE%DB%8C_%DB%81%D9%85%D8%A7%D8%B1%DB%8C_%D9%82%D8%B3%D9%85%D8%AA_%DA%A9%DB%81_%D9%88%D8%B5%D8%A7%D9%84_%DB%8C%D8%A7%D8%B1_%DB%81%D9%88%D8%AA%D8%A7
+}}
+<poem>
+{{مطلع}} یہ نہ تھی ہماری قسمت کہ [[وصال]] یار ہوتا
+اگر اور جیتے رہتے یہی انتظار ہوتا
+
+ترے وعدے پر جیے ہم تو یہ جان جھوٹ جانا
+کہ خوشی سے مر نہ جاتے اگر اعتبار ہوتا
+
+تری نازکی سے جانا کہ بندھا تھا عہد بودا
+کبھی تو نہ توڑ سکتا اگر استوار ہوتا
+
+کوئی میرے دل سے پوچھے ترے تیر [[نیم کش]] کو<ref>نیم کش: آدھا کھنچا ہوا تیر، جو دل میں اٹک کر رہ جائے۔</ref>
+یہ خلش کہاں سے ہوتی جو جگر کے پار ہوتا
+
+یہ کہاں کی دوستی ہے کہ بنے ہیں دوست ناصح
+کوئی چارہ ساز ہوتا کوئی غم گسار ہوتا
+
+رگ سنگ سے ٹپکتا وہ لہو کہ پھر نہ {{نسخہ|تھمتا|رکتا|ماخذ=مثال کے لیے}}
+جسے غم سمجھ رہے ہو یہ اگر شرار ہوتا
+
+غم اگرچہ جاں گسل ہے پہ کہاں بچیں کہ دل ہے
+غم عشق گر نہ ہوتا غم روزگار ہوتا
+
+کہوں کس سے میں کہ کیا ہے شب غم بری بلا ہے
+مجھے کیا برا تھا مرنا اگر ایک بار ہوتا
+
+ہوئے مر کے ہم جو رسوا ہوئے کیوں نہ غرق دریا
+نہ کبھی جنازہ اٹھتا نہ کہیں مزار ہوتا
+
+اسے کون دیکھ سکتا کہ یگانہ ہے وہ یکتا
+جو دوئی کی بو بھی ہوتی تو کہیں دو چار ہوتا
+
+یہ مسائل تصوف یہ ترا بیان غالبؔ
+تجھے ہم ولی سمجھتے جو نہ بادہ خوار ہوتا
+</poem>

docs/content-model/sir-syed-azadi-e-rai.dtx

@@ -0,0 +1,14 @@
+{{دیوان
+| عنوان = آزادی رائے
+| مصنف = سید احمد خان
+| صنف = مضمون
+| ماخذ = ویکی ماخذ
+| ماخذ_ربط = https://ur.wikisource.org/wiki/%D8%A2%D8%B2%D8%A7%D8%AF%DB%8C_%D8%B1%D8%A7%D8%A6%DB%92
+}}
+== آزادی رائے ==
+
+ہم اپنے اس آرٹیکل کو ایک بڑے لائق اور قابل زمانہ حال کے [[فیلسوف]] کی تحریر سے اخذ کرتے ہیں۔ رائے کی آزادی ایک ایسی چیز ہے کہ ہر ایک انسان اس پر پورا پورا حق رکھتا ہے۔ فرض کرو کہ تمام آدمی بجز ایک شخص کے کسی بات پر متفق الرائے ہیں، مگر صرف وہی ایک شخص ان کے بر خلاف رائے رکھتا ہے تو ان تمام آدمیوں کو اس ایک شخص کی رائے کو غلط ٹھہرانے کے لیے اس سے زیادہ کچھ استحقاق نہیں ہے جتنا کہ اس ایک شخص کو ان تمام آدمیوں کی رائے کے غلط ثابت کرنے کا (اگر وہ ثابت کر سکے) استحقاق حاصل ہے۔<ref>مثال کے لیے حاشیہ: یہ مضمون تہذیب الاخلاق میں شائع ہوا۔</ref>
+
+کوئی وجہ اس بات کی نہیں ہے کہ پانچ آدمیوں کو تو بمقابلہ پانچ آدمیوں کی رایوں کے غلط ٹھہرانے کا استحقاق ہو اور ایک آدمی کو بمقابل نو آدمیوں کے یہ استحقاق نہ ہو۔ رائے کی غلطی آدمیوں کی تعداد کی کمی بیشی پر منحصر نہیں ہے، بلکہ قوت استدلال پر منحصر ہے۔ جیسے کہ یہ بات ممکن ہے کہ نو آدمیوں کی رائے بمقابلہ ایک شخص کے صحیح ہو ویسے یہ بھی ممکن ہے کہ ایک شخص کی رائے بمقابل نو کے صحیح ہو۔
+
+رایوں کا بند رہنا خواہ بسبب کسی مذہبی خوف کے اور خواہ بسبب اندیشہ برادری وقوم کے اور خواہ بدنامی کے ڈر سے اور یا گورنمنٹ کے ظلم سے نہایت ہی بری چیز ہے۔ اگر رائے اس قسم کی کوئی چیز ہوتی جس کی قدر و قیمت صرف اس رائے والے کی ذات ہی سے متعلق اور اسی میں محصور ہوتی تو رایوں کے بند رہنے سے ایک خاص شخص یا معدودے چند کا نقصان متصور ہوتا مگر رایوں کے بند رہنے سے تمام انسانوں کی حق تلفی ہوتی ہے اور کل انسانوں کو نقصان پہنچتا ہے اور نہ صرف موجودہ انسانوں کو، بلکہ ان کو بھی جو آئندہ پیدا ہوں گے۔