robots.txt, sitemap.xml and an RSS feed of new versions, e-books, arrangements and tags (rss.xml, linked in the head and footer)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

api/src/feeds.test.ts

@@ -0,0 +1,33 @@
+import { test, after } from 'node:test';
+import assert from 'node:assert/strict';
+import Fastify from 'fastify';
+import { feedRoutes } from './feeds.ts';
+import { pool } from './db.ts';
+
+after(() => pool.end());
+
+test('sitemap and feed: every public page; a published version appears with its date, summary, author and lines', async () => {
+  const app = Fastify(); feedRoutes(app);
+  const work = (await pool.query(`SELECT id, url FROM poems WHERE url LIKE '/p266/ghazal/%' ORDER BY id LIMIT 1`)).rows[0];
+  const when = new Date(Date.now() + 60_000).toISOString(); // newer than anything else, so it comes first
+  const rev = (await pool.query(
+    `INSERT INTO revisions (entity, entity_id, version, base_version, content, status, author_email, summary, published_at, credits)
+     VALUES ('work', $1, 99, 98, '', 'published', 'feed-test@divan.test', 'آزمائشی ترمیم', $2, $3) RETURNING id`,
+    [work.id, when, { by: 'آزمائشی موڈریٹر', publishedBy: 'ایڈمن' }])).rows[0].id;
+  try {
+    const s = (await app.inject('/api/sitemap')).json();
+    assert.ok(s.poets.length > 300 && s.categories.length > 500 && s.works.length > 10000, 'all the public pages');
+    assert.ok(s.poets.includes('/p266'));
+    assert.equal(new Date(s.works.find((w: any) => w.url === work.url).updated).toISOString(), when, "a work's date is its latest version");
+
+    const feed = (await app.inject('/api/feed?limit=5')).json();
+    assert.ok(feed.length >= 1 && feed.length <= 5);
+    const first = feed[0];
+    assert.deepEqual([first.kind, first.url, first.version, first.summary, first.by], ['work', work.url, 99, 'آزمائشی ترمیم', 'آزمائشی موڈریٹر']);
+    assert.ok(first.lines.length >= 1, 'the opening lines');
+    assert.equal((await app.inject('/api/feed?limit=999')).json().length <= 100, true, 'at most 100');
+  } finally {
+    await pool.query('DELETE FROM revisions WHERE id = $1', [rev]);
+    await app.close();
+  }
+});

api/src/feeds.ts

@@ -0,0 +1,60 @@
+// What the site's sitemap.xml and rss.xml are made from (#83, #87).
+//   GET /api/sitemap      every public page: poets, books/chapters, works (with the date of their latest Divan
+//                         version), tags, e-books
+//   GET /api/feed?limit=  the latest published changes, newest first: new versions of works (version, summary, the
+//                         opening lines), e-books, arrangements and tags; who made each (public names)
+import type { FastifyInstance } from 'fastify';
+import { pool } from './db.ts';
+
+export function feedRoutes(app: FastifyInstance) {
+  app.get('/api/sitemap', async (_req, reply) => {
+    const [poets, cats, works, tags, ebooks] = await Promise.all([
+      pool.query('SELECT url FROM poets ORDER BY id'),
+      pool.query('SELECT url FROM categories WHERE parent_id IS NOT NULL ORDER BY id'),
+      pool.query(`SELECT p.url, max(r.published_at) AS updated FROM poems p
+                  LEFT JOIN revisions r ON r.entity = 'work' AND r.entity_id = p.id AND r.status = 'published'
+                  GROUP BY p.id ORDER BY p.id`),
+      pool.query('SELECT DISTINCT t.type, t.name FROM tags t JOIN entity_tags e ON e.tag_id = t.id ORDER BY t.type, t.name'),
+      pool.query('SELECT b.id, t.url AS poet_url FROM ebooks b JOIN poets t ON t.id = b.poet_id WHERE b.published ORDER BY b.id'),
+    ]);
+    reply.header('cache-control', 'public, max-age=3600');
+    return {
+      poets: poets.rows.map((r) => r.url),
+      categories: cats.rows.map((r) => r.url),
+      works: works.rows.map((r) => ({ url: r.url, updated: r.updated })),
+      tags: tags.rows,
+      ebooks: ebooks.rows,
+    };
+  });
+
+  app.get<{ Querystring: { limit?: string } }>('/api/feed', async (req, reply) => {
+    const limit = Math.min(Math.max(Number(req.query.limit) || 50, 1), 100);
+    const { rows } = await pool.query(
+      `SELECT r.id, r.entity, r.entity_id, r.version, r.summary, r.published_at, r.credits,
+              p.title AS work_title, p.url AS work_url, pp.nickname AS work_poet,
+              c.title AS cat_title, c.url AS cat_url, cp.nickname AS cat_poet,
+              b.title AS ebook_title, bp.nickname AS ebook_poet
+       FROM revisions r
+       LEFT JOIN poems p ON r.entity IN ('work', 'tags-work') AND p.id = r.entity_id LEFT JOIN poets pp ON pp.id = p.poet_id
+       LEFT JOIN categories c ON r.entity IN ('order', 'tags-category') AND c.id = r.entity_id LEFT JOIN poets cp ON cp.id = c.poet_id
+       LEFT JOIN ebooks b ON r.entity = 'ebook' AND b.id = r.entity_id LEFT JOIN poets bp ON bp.id = b.poet_id
+       WHERE r.status = 'published' AND r.published_at IS NOT NULL
+       ORDER BY r.published_at DESC, r.id DESC LIMIT $1`, [limit]);
+    // the opening lines of each new version of a work, for the feed's description
+    const workIds = rows.filter((r) => r.entity === 'work').map((r) => r.entity_id);
+    const lines = workIds.length ? (await pool.query(
+      `SELECT poem_id, array_agg(text ORDER BY vorder) AS lines FROM verses WHERE poem_id = ANY($1) AND vorder <= 2 GROUP BY poem_id`,
+      [workIds])).rows : [];
+    reply.header('cache-control', 'public, max-age=300');
+    return rows.map((r) => {
+      const kind = r.entity === 'work' ? 'work' : r.entity === 'ebook' ? 'ebook' : r.entity === 'order' ? 'order' : 'tags';
+      const title = r.work_title ?? r.cat_title ?? r.ebook_title ?? '';
+      const url = r.work_url ?? r.cat_url ?? (r.entity === 'ebook' ? `/ebook/${r.entity_id}` : '/');
+      return {
+        id: Number(r.id), kind, version: r.version, title, url, poet: r.work_poet ?? r.cat_poet ?? r.ebook_poet ?? null,
+        summary: r.summary ?? null, published: r.published_at, by: r.credits?.by ?? null,
+        lines: lines.find((l) => l.poem_id === r.entity_id)?.lines ?? [],
+      };
+    });
+  });
+}

api/src/server.ts

@@ -22,6 +22,7 @@ import { moderationRoutes } from './moderation.ts';
 import { siteRoutes } from './site.ts';
 import { tagRoutes, pageTags } from './tags.ts';
 import { ebookRoutes } from './ebooks.ts';
+import { feedRoutes } from './feeds.ts';
 
 const app = Fastify({ logger: { level: process.env.LOG_LEVEL ?? 'info' } });
 const PAGE_SIZE = 20;
@@ -199,6 +200,7 @@ moderationRoutes(app);
 siteRoutes(app);
 tagRoutes(app);
 ebookRoutes(app);
+feedRoutes(app);
 
 const port = Number(process.env.PORT ?? 4100);
 await app.listen({ port, host: process.env.HOST ?? '127.0.0.1' });

web/src/components/Icon.astro

@@ -55,6 +55,7 @@ const PATHS = {
   search: '<circle cx="10.5" cy="10.5" r="6.5"/><path d="m20 20-4.9-4.9"/>',
   chevron: '<path d="m15 18-6-6 6-6"/>',
   // footer
+  rss: '<path d="M4 11a9 9 0 0 1 9 9"/><path d="M4 4a16 16 0 0 1 16 16"/><circle cx="5" cy="19" r="1"/>',
   info: '<circle cx="12" cy="12" r="10"/><path d="M12 16v-4"/><path d="M12 8h.01"/>',
   globe: '<circle cx="12" cy="12" r="10"/><path d="M12 2a14.5 14.5 0 0 0 0 20 14.5 14.5 0 0 0 0-20"/><path d="M2 12h20"/>',
   database: '<ellipse cx="12" cy="5" rx="9" ry="3"/><path d="M3 5v14a9 3 0 0 0 18 0V5"/><path d="M3 12a9 3 0 0 0 18 0"/>',

web/src/layouts/Base.astro

@@ -21,6 +21,7 @@ import { brand } from '../lib/brand';
     <title>{fullTitle}</title>
     <meta name="description" content={description} />
     <link rel="icon" href="/favicon.svg" type="image/svg+xml" />
+    <link rel="alternate" type="application/rss+xml" title={`${brand.name}: نئی تبدیلیاں`} href="/rss.xml" />
     <link rel="icon" href="/favicon-32.png" type="image/png" sizes="32x32" />
     <link rel="apple-touch-icon" href="/apple-touch-icon.png" />
     <link rel="manifest" href="/site.webmanifest" />
@@ -134,6 +135,7 @@ import { brand } from '../lib/brand';
           <div><h2>{brand.name}</h2><ul>
             <li><a href="/about"><Icon name="info" size={18} /><span>{brand.name} کے بارے میں<small>مقصد اور طریقۂ کار</small></span></a></li>
             <li><a href="/tags"><Icon name="tag" size={18} /><span>ٹیگ<small>موضوعات سے کلام</small></span></a></li>
+            <li><a href="/rss.xml"><Icon name="rss" size={18} /><span>نئی تبدیلیاں<small>RSS: نئے ورژن اور ای بکس</small></span></a></li>
           </ul></div>
           <div><h2>ماخذ</h2><ul>
             <li><a href="https://ur.wikisource.org" rel="noopener"><Icon name="dict" size={18} /><span>ویکی ماخذ<small>متن · عوامی ملکیت</small></span><Icon name="external" size={13} class="ext" /></a></li>

web/src/lib/site-url.ts

@@ -0,0 +1,4 @@
+// The site's public address for links in feeds and the sitemap: SITE_URL if set (e.g. https://divan.pk), else the
+// address the request came to (behind Caddy that is the public one).
+export const siteUrl = (url: URL) => (process.env.SITE_URL ?? url.origin).replace(/\/$/, '');
+export const xml = (s: unknown) => String(s ?? '').replace(/[<>&'"]/g, (c) => ({ '<': '&lt;', '>': '&gt;', '&': '&amp;', "'": '&apos;', '"': '&quot;' })[c]!);

web/src/pages/robots.txt.ts

@@ -0,0 +1,15 @@
+// robots.txt: search engines may read the reading pages, not accounts, moderation or search results (endless and
+// private). DIVAN_NOINDEX=1 (the private preview) keeps everything out.
+import type { APIRoute } from 'astro';
+import { siteUrl } from '../lib/site-url';
+
+export const GET: APIRoute = ({ url }) => {
+  const closed = process.env.DIVAN_NOINDEX === '1';
+  const body = closed ? 'User-agent: *\nDisallow: /\n' : [
+    'User-agent: *',
+    ...['/admin', '/mod', '/account', '/library', '/writers', '/words', '/notes', '/signin', '/signup', '/search', '/api/', '/ebooks/file/', '/health']
+      .map((p) => `Disallow: ${p}`),
+    '', `Sitemap: ${siteUrl(url)}/sitemap.xml`, '',
+  ].join('\n');
+  return new Response(body, { headers: { 'content-type': 'text/plain; charset=utf-8', 'cache-control': 'public, max-age=3600' } });
+};

web/src/pages/rss.xml.ts

@@ -0,0 +1,33 @@
+// rss.xml (RSS 2.0): the latest published changes, to follow or share: new versions of works, new e-books,
+// arrangements and tags, each with who made it (public names) and, for a work, its opening lines.
+import type { APIRoute } from 'astro';
+import { api } from '../lib/api';
+import { brand } from '../lib/brand';
+import { siteUrl, xml } from '../lib/site-url';
+import { ud } from '../lib/urdu';
+
+const KIND: Record<string, string> = { work: 'نیا ورژن', ebook: 'نئی ای بک', order: 'نئی ترتیب', tags: 'ٹیگ' };
+
+export const GET: APIRoute = async ({ url }) => {
+  const base = siteUrl(url), items = (await api<any[]>('/api/feed?limit=50')) ?? [];
+  const item = (i: any) => {
+    const title = `${KIND[i.kind]}: ${i.title}${i.poet ? ` (${i.poet})` : ''}${i.kind === 'work' && i.version ? ` · ورژن ${ud(i.version)}` : ''}`;
+    const text = [i.lines.join(' / '), i.summary, i.by && `از ${i.by}`].filter(Boolean).join(' — ');
+    return `<item><title>${xml(title)}</title><link>${xml(base + encodeURI(i.url))}</link>` +
+      `<guid isPermaLink="false">divan-${i.id}</guid><pubDate>${new Date(i.published).toUTCString()}</pubDate>` +
+      `${i.by ? `<dc:creator>${xml(i.by)}</dc:creator>` : ''}<category>${xml(KIND[i.kind])}</category><description>${xml(text)}</description></item>`;
+  };
+  const body = `<?xml version="1.0" encoding="UTF-8"?>
+<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom" xmlns:dc="http://purl.org/dc/elements/1.1/">
+<channel>
+<title>${xml(`${brand.name}: نئی تبدیلیاں`)}</title>
+<link>${xml(base)}/</link>
+<description>${xml(`${brand.name} (${brand.tagline}) میں شائع ہونے والے نئے ورژن، ای بکس اور تبدیلیاں`)}</description>
+<language>ur-PK</language>
+<atom:link href="${xml(base)}/rss.xml" rel="self" type="application/rss+xml" />
+${items.length ? `<lastBuildDate>${new Date(items[0].published).toUTCString()}</lastBuildDate>\n` : ''}${items.map(item).join('\n')}
+</channel>
+</rss>
+`;
+  return new Response(body, { headers: { 'content-type': 'application/rss+xml; charset=utf-8', 'cache-control': 'public, max-age=300' } });
+};

web/src/pages/sitemap.xml.ts

@@ -0,0 +1,22 @@
+// sitemap.xml: every public page (poets, books/chapters, works, tags, e-books, and the fixed pages), for search
+// engines. A work's date is when its latest Divan version was published. About 13,000 addresses (one file holds 50,000).
+import type { APIRoute } from 'astro';
+import { api } from '../lib/api';
+import { siteUrl, xml } from '../lib/site-url';
+
+export const GET: APIRoute = async ({ url }) => {
+  const base = siteUrl(url), s = await api<any>('/api/sitemap');
+  const loc = (path: string, updated?: string | null) =>
+    `<url><loc>${xml(base + encodeURI(path))}</loc>${updated ? `<lastmod>${new Date(updated).toISOString().slice(0, 10)}</lastmod>` : ''}</url>`;
+  const urls = [
+    ...['/', '/about', '/tags', '/privacy', '/terms'].map((p) => loc(p)),
+    ...s.poets.map((p: string) => loc(p)),
+    ...s.poets.map((p: string) => loc(`${p}/ebooks`)),
+    ...s.categories.map((c: string) => loc(c)),
+    ...s.works.map((w: any) => loc(w.url, w.updated)),
+    ...s.tags.map((t: any) => loc(`/tag/${t.type}/${t.name}`)),
+    ...s.ebooks.map((b: any) => loc(`/ebook/${b.id}`)),
+  ];
+  const body = `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n${urls.join('\n')}\n</urlset>\n`;
+  return new Response(body, { headers: { 'content-type': 'application/xml; charset=utf-8', 'cache-control': 'public, max-age=3600' } });
+};