From 13eaf9fe90be06b333fc3046fb14589239e4486d Mon Sep 17 00:00:00 2001 From: Anas Rashid Date: Fri, 9 Oct 2026 19:32:55 +0200 Subject: [PATCH] robots.txt, sitemap.xml and an RSS feed of new versions, e-books, arrangements and tags (rss.xml, linked in the head and footer) Co-Authored-By: Claude Opus 5.5 --- api/src/feeds.test.ts | 33 +++++++++++++++++++ api/src/feeds.ts | 60 +++++++++++++++++++++++++++++++++++ api/src/server.ts | 2 ++ web/src/components/Icon.astro | 1 + web/src/layouts/Base.astro | 2 ++ web/src/lib/site-url.ts | 4 +++ web/src/pages/robots.txt.ts | 15 +++++++++ web/src/pages/rss.xml.ts | 33 +++++++++++++++++++ web/src/pages/sitemap.xml.ts | 22 +++++++++++++ 9 files changed, 172 insertions(+) create mode 100644 api/src/feeds.test.ts create mode 100644 api/src/feeds.ts create mode 100644 web/src/lib/site-url.ts create mode 100644 web/src/pages/robots.txt.ts create mode 100644 web/src/pages/rss.xml.ts create mode 100644 web/src/pages/sitemap.xml.ts diff --git a/api/src/feeds.test.ts b/api/src/feeds.test.ts new file mode 100644 index 00000000..19eaad7b --- /dev/null +++ b/api/src/feeds.test.ts @@ -0,0 +1,33 @@ +import { test, after } from 'node:test'; +import assert from 'node:assert/strict'; +import Fastify from 'fastify'; +import { feedRoutes } from './feeds.ts'; +import { pool } from './db.ts'; + +after(() => pool.end()); + +test('sitemap and feed: every public page; a published version appears with its date, summary, author and lines', async () => { + const app = Fastify(); feedRoutes(app); + const work = (await pool.query(`SELECT id, url FROM poems WHERE url LIKE '/p266/ghazal/%' ORDER BY id LIMIT 1`)).rows[0]; + const when = new Date(Date.now() + 60_000).toISOString(); // newer than anything else, so it comes first + const rev = (await pool.query( + `INSERT INTO revisions (entity, entity_id, version, base_version, content, status, author_email, summary, published_at, credits) + VALUES ('work', $1, 99, 98, '', 'published', 'feed-test@divan.test', 'آزمائشی ترمیم', $2, $3) RETURNING id`, + [work.id, when, { by: 'آزمائشی موڈریٹر', publishedBy: 'ایڈمن' }])).rows[0].id; + try { + const s = (await app.inject('/api/sitemap')).json(); + assert.ok(s.poets.length > 300 && s.categories.length > 500 && s.works.length > 10000, 'all the public pages'); + assert.ok(s.poets.includes('/p266')); + assert.equal(new Date(s.works.find((w: any) => w.url === work.url).updated).toISOString(), when, "a work's date is its latest version"); + + const feed = (await app.inject('/api/feed?limit=5')).json(); + assert.ok(feed.length >= 1 && feed.length <= 5); + const first = feed[0]; + assert.deepEqual([first.kind, first.url, first.version, first.summary, first.by], ['work', work.url, 99, 'آزمائشی ترمیم', 'آزمائشی موڈریٹر']); + assert.ok(first.lines.length >= 1, 'the opening lines'); + assert.equal((await app.inject('/api/feed?limit=999')).json().length <= 100, true, 'at most 100'); + } finally { + await pool.query('DELETE FROM revisions WHERE id = $1', [rev]); + await app.close(); + } +}); diff --git a/api/src/feeds.ts b/api/src/feeds.ts new file mode 100644 index 00000000..b2463a3f --- /dev/null +++ b/api/src/feeds.ts @@ -0,0 +1,60 @@ +// What the site's sitemap.xml and rss.xml are made from (#83, #87). +// GET /api/sitemap every public page: poets, books/chapters, works (with the date of their latest Divan +// version), tags, e-books +// GET /api/feed?limit= the latest published changes, newest first: new versions of works (version, summary, the +// opening lines), e-books, arrangements and tags; who made each (public names) +import type { FastifyInstance } from 'fastify'; +import { pool } from './db.ts'; + +export function feedRoutes(app: FastifyInstance) { + app.get('/api/sitemap', async (_req, reply) => { + const [poets, cats, works, tags, ebooks] = await Promise.all([ + pool.query('SELECT url FROM poets ORDER BY id'), + pool.query('SELECT url FROM categories WHERE parent_id IS NOT NULL ORDER BY id'), + pool.query(`SELECT p.url, max(r.published_at) AS updated FROM poems p + LEFT JOIN revisions r ON r.entity = 'work' AND r.entity_id = p.id AND r.status = 'published' + GROUP BY p.id ORDER BY p.id`), + pool.query('SELECT DISTINCT t.type, t.name FROM tags t JOIN entity_tags e ON e.tag_id = t.id ORDER BY t.type, t.name'), + pool.query('SELECT b.id, t.url AS poet_url FROM ebooks b JOIN poets t ON t.id = b.poet_id WHERE b.published ORDER BY b.id'), + ]); + reply.header('cache-control', 'public, max-age=3600'); + return { + poets: poets.rows.map((r) => r.url), + categories: cats.rows.map((r) => r.url), + works: works.rows.map((r) => ({ url: r.url, updated: r.updated })), + tags: tags.rows, + ebooks: ebooks.rows, + }; + }); + + app.get<{ Querystring: { limit?: string } }>('/api/feed', async (req, reply) => { + const limit = Math.min(Math.max(Number(req.query.limit) || 50, 1), 100); + const { rows } = await pool.query( + `SELECT r.id, r.entity, r.entity_id, r.version, r.summary, r.published_at, r.credits, + p.title AS work_title, p.url AS work_url, pp.nickname AS work_poet, + c.title AS cat_title, c.url AS cat_url, cp.nickname AS cat_poet, + b.title AS ebook_title, bp.nickname AS ebook_poet + FROM revisions r + LEFT JOIN poems p ON r.entity IN ('work', 'tags-work') AND p.id = r.entity_id LEFT JOIN poets pp ON pp.id = p.poet_id + LEFT JOIN categories c ON r.entity IN ('order', 'tags-category') AND c.id = r.entity_id LEFT JOIN poets cp ON cp.id = c.poet_id + LEFT JOIN ebooks b ON r.entity = 'ebook' AND b.id = r.entity_id LEFT JOIN poets bp ON bp.id = b.poet_id + WHERE r.status = 'published' AND r.published_at IS NOT NULL + ORDER BY r.published_at DESC, r.id DESC LIMIT $1`, [limit]); + // the opening lines of each new version of a work, for the feed's description + const workIds = rows.filter((r) => r.entity === 'work').map((r) => r.entity_id); + const lines = workIds.length ? (await pool.query( + `SELECT poem_id, array_agg(text ORDER BY vorder) AS lines FROM verses WHERE poem_id = ANY($1) AND vorder <= 2 GROUP BY poem_id`, + [workIds])).rows : []; + reply.header('cache-control', 'public, max-age=300'); + return rows.map((r) => { + const kind = r.entity === 'work' ? 'work' : r.entity === 'ebook' ? 'ebook' : r.entity === 'order' ? 'order' : 'tags'; + const title = r.work_title ?? r.cat_title ?? r.ebook_title ?? ''; + const url = r.work_url ?? r.cat_url ?? (r.entity === 'ebook' ? `/ebook/${r.entity_id}` : '/'); + return { + id: Number(r.id), kind, version: r.version, title, url, poet: r.work_poet ?? r.cat_poet ?? r.ebook_poet ?? null, + summary: r.summary ?? null, published: r.published_at, by: r.credits?.by ?? null, + lines: lines.find((l) => l.poem_id === r.entity_id)?.lines ?? [], + }; + }); + }); +} diff --git a/api/src/server.ts b/api/src/server.ts index b5fcb65c..29155658 100644 --- a/api/src/server.ts +++ b/api/src/server.ts @@ -22,6 +22,7 @@ import { moderationRoutes } from './moderation.ts'; import { siteRoutes } from './site.ts'; import { tagRoutes, pageTags } from './tags.ts'; import { ebookRoutes } from './ebooks.ts'; +import { feedRoutes } from './feeds.ts'; const app = Fastify({ logger: { level: process.env.LOG_LEVEL ?? 'info' } }); const PAGE_SIZE = 20; @@ -199,6 +200,7 @@ moderationRoutes(app); siteRoutes(app); tagRoutes(app); ebookRoutes(app); +feedRoutes(app); const port = Number(process.env.PORT ?? 4100); await app.listen({ port, host: process.env.HOST ?? '127.0.0.1' }); diff --git a/web/src/components/Icon.astro b/web/src/components/Icon.astro index 0e779352..3a2915ef 100644 --- a/web/src/components/Icon.astro +++ b/web/src/components/Icon.astro @@ -55,6 +55,7 @@ const PATHS = { search: '', chevron: '', // footer + rss: '', info: '', globe: '', database: '', diff --git a/web/src/layouts/Base.astro b/web/src/layouts/Base.astro index 17a74a44..b6bb27a0 100644 --- a/web/src/layouts/Base.astro +++ b/web/src/layouts/Base.astro @@ -21,6 +21,7 @@ import { brand } from '../lib/brand'; {fullTitle} + @@ -134,6 +135,7 @@ import { brand } from '../lib/brand';

ماخذ

  • ویکی ماخذمتن · عوامی ملکیت
    • diff --git a/web/src/lib/site-url.ts b/web/src/lib/site-url.ts new file mode 100644 index 00000000..2e5286d5 --- /dev/null +++ b/web/src/lib/site-url.ts @@ -0,0 +1,4 @@ +// The site's public address for links in feeds and the sitemap: SITE_URL if set (e.g. https://divan.pk), else the +// address the request came to (behind Caddy that is the public one). +export const siteUrl = (url: URL) => (process.env.SITE_URL ?? url.origin).replace(/\/$/, ''); +export const xml = (s: unknown) => String(s ?? '').replace(/[<>&'"]/g, (c) => ({ '<': '<', '>': '>', '&': '&', "'": ''', '"': '"' })[c]!); diff --git a/web/src/pages/robots.txt.ts b/web/src/pages/robots.txt.ts new file mode 100644 index 00000000..fc8f460e --- /dev/null +++ b/web/src/pages/robots.txt.ts @@ -0,0 +1,15 @@ +// robots.txt: search engines may read the reading pages, not accounts, moderation or search results (endless and +// private). DIVAN_NOINDEX=1 (the private preview) keeps everything out. +import type { APIRoute } from 'astro'; +import { siteUrl } from '../lib/site-url'; + +export const GET: APIRoute = ({ url }) => { + const closed = process.env.DIVAN_NOINDEX === '1'; + const body = closed ? 'User-agent: *\nDisallow: /\n' : [ + 'User-agent: *', + ...['/admin', '/mod', '/account', '/library', '/writers', '/words', '/notes', '/signin', '/signup', '/search', '/api/', '/ebooks/file/', '/health'] + .map((p) => `Disallow: ${p}`), + '', `Sitemap: ${siteUrl(url)}/sitemap.xml`, '', + ].join('\n'); + return new Response(body, { headers: { 'content-type': 'text/plain; charset=utf-8', 'cache-control': 'public, max-age=3600' } }); +}; diff --git a/web/src/pages/rss.xml.ts b/web/src/pages/rss.xml.ts new file mode 100644 index 00000000..cffa3fe2 --- /dev/null +++ b/web/src/pages/rss.xml.ts @@ -0,0 +1,33 @@ +// rss.xml (RSS 2.0): the latest published changes, to follow or share: new versions of works, new e-books, +// arrangements and tags, each with who made it (public names) and, for a work, its opening lines. +import type { APIRoute } from 'astro'; +import { api } from '../lib/api'; +import { brand } from '../lib/brand'; +import { siteUrl, xml } from '../lib/site-url'; +import { ud } from '../lib/urdu'; + +const KIND: Record = { work: 'نیا ورژن', ebook: 'نئی ای بک', order: 'نئی ترتیب', tags: 'ٹیگ' }; + +export const GET: APIRoute = async ({ url }) => { + const base = siteUrl(url), items = (await api('/api/feed?limit=50')) ?? []; + const item = (i: any) => { + const title = `${KIND[i.kind]}: ${i.title}${i.poet ? ` (${i.poet})` : ''}${i.kind === 'work' && i.version ? ` · ورژن ${ud(i.version)}` : ''}`; + const text = [i.lines.join(' / '), i.summary, i.by && `از ${i.by}`].filter(Boolean).join(' — '); + return `${xml(title)}${xml(base + encodeURI(i.url))}` + + `divan-${i.id}${new Date(i.published).toUTCString()}` + + `${i.by ? `${xml(i.by)}` : ''}${xml(KIND[i.kind])}${xml(text)}`; + }; + const body = ` + + +${xml(`${brand.name}: نئی تبدیلیاں`)} +${xml(base)}/ +${xml(`${brand.name} (${brand.tagline}) میں شائع ہونے والے نئے ورژن، ای بکس اور تبدیلیاں`)} +ur-PK + +${items.length ? `${new Date(items[0].published).toUTCString()}\n` : ''}${items.map(item).join('\n')} + + +`; + return new Response(body, { headers: { 'content-type': 'application/rss+xml; charset=utf-8', 'cache-control': 'public, max-age=300' } }); +}; diff --git a/web/src/pages/sitemap.xml.ts b/web/src/pages/sitemap.xml.ts new file mode 100644 index 00000000..ab596a46 --- /dev/null +++ b/web/src/pages/sitemap.xml.ts @@ -0,0 +1,22 @@ +// sitemap.xml: every public page (poets, books/chapters, works, tags, e-books, and the fixed pages), for search +// engines. A work's date is when its latest Divan version was published. About 13,000 addresses (one file holds 50,000). +import type { APIRoute } from 'astro'; +import { api } from '../lib/api'; +import { siteUrl, xml } from '../lib/site-url'; + +export const GET: APIRoute = async ({ url }) => { + const base = siteUrl(url), s = await api('/api/sitemap'); + const loc = (path: string, updated?: string | null) => + `${xml(base + encodeURI(path))}${updated ? `${new Date(updated).toISOString().slice(0, 10)}` : ''}`; + const urls = [ + ...['/', '/about', '/tags', '/privacy', '/terms'].map((p) => loc(p)), + ...s.poets.map((p: string) => loc(p)), + ...s.poets.map((p: string) => loc(`${p}/ebooks`)), + ...s.categories.map((c: string) => loc(c)), + ...s.works.map((w: any) => loc(w.url, w.updated)), + ...s.tags.map((t: any) => loc(`/tag/${t.type}/${t.name}`)), + ...s.ebooks.map((b: any) => loc(`/ebook/${b.id}`)), + ]; + const body = `\n\n${urls.join('\n')}\n\n`; + return new Response(body, { headers: { 'content-type': 'application/xml; charset=utf-8', 'cache-control': 'public, max-age=3600' } }); +};