diff options
| -rw-r--r-- | CLAUDE.md | 3 | ||||
| -rw-r--r-- | public/static/style.css | 21 | ||||
| -rw-r--r-- | src/app.test.ts | 98 | ||||
| -rw-r--r-- | src/app.ts | 32 | ||||
| -rw-r--r-- | src/scraper.test.ts | 80 | ||||
| -rw-r--r-- | src/scraper.ts | 115 | ||||
| -rw-r--r-- | views/_wikiNav.eta | 4 | ||||
| -rw-r--r-- | views/layout.eta | 1 | ||||
| -rw-r--r-- | views/wiki.eta | 37 | ||||
| -rw-r--r-- | views/wikiPages.eta | 16 |
10 files changed, 404 insertions, 3 deletions
@@ -33,6 +33,9 @@ pnpm test - `GET /:owner/:repo` - Repository info with directory listing - `GET /:owner/:repo/tree/:branch/:path*` - Directory listing - `GET /:owner/:repo/blob/:branch/:path` - File content view +- `GET /:owner/:repo/wiki` - Wiki home page +- `GET /:owner/:repo/wiki/:page` - Wiki page (also `/:page/:oid` for a revision) +- `GET /:owner/:repo/wiki/_pages` - Wiki page index ## Instructions diff --git a/public/static/style.css b/public/static/style.css index 157cc15..a83e128 100644 --- a/public/static/style.css +++ b/public/static/style.css @@ -288,7 +288,7 @@ pre.box { width: fit-content; } -.overview, .release { +.overview, .release, .wiki { overflow-wrap: break-word; /* This is a bit convoluted: we want the effect of padding but using margins @@ -332,7 +332,7 @@ pre.box { font-weight: bold; } -.release > header { +.release > header, .wiki > header { display: flex; align-items: baseline; > :first-child { @@ -341,6 +341,23 @@ pre.box { } } +.wiki > footer { + /* The wiki's _Footer page is rendered content, not a caption for the page. */ + font-size: .9rem; +} + +.wiki-pages { + summary { + cursor: pointer; + } + + ul { + columns: 16rem; + list-style: none; + padding: 0; + } +} + .timestamp { white-space: nowrap; font-size: .8rem; diff --git a/src/app.test.ts b/src/app.test.ts index e28c375..3ecebd0 100644 --- a/src/app.test.ts +++ b/src/app.test.ts @@ -178,6 +178,104 @@ describe("embedded files", () => { }); }); +describe("wiki", () => { + const repo = { owner: "ariya", name: "phantomjs" }; + const page = { + repo, + title: "Home", + updatedBy: "pixiuPL", + updatedAt: "2018-02-08T18:57:56Z", + bodyHtml: "<p>New to <strong>PhantomJS</strong>?</p>", + sidebarHtml: null, + footerHtml: null, + pages: [ + { name: "Home", href: "/ariya/phantomjs/wiki" }, + { name: "API Reference WebPage", href: "/ariya/phantomjs/wiki/API-Reference-WebPage" }, + ], + }; + + it("should render the page beside who last edited it", () => { + const html = eta.render("wiki.eta", page); + + assert.match(html, /<p>New to <strong>PhantomJS<\/strong>\?<\/p>/); + assert.match(html, /pixiuPL edited this page/); + assert.match(html, /<time title="2018-02-08T18:57:56Z">2018-02-08<\/time>/); + }); + + // The _Sidebar and _Footer pages a wiki can define, which GitHub renders + // alongside every page rather than as pages of their own. + it("should render the sidebar and footer only when the wiki has them", () => { + // The page's own content is the last thing in its box, and the box is the + // only one on the page. + const bare = eta.render("wiki.eta", page); + assert.match(bare, /New to <strong>PhantomJS<\/strong>\?<\/p>\s*<\/div>/); + assert.doesNotMatch(bare, /Sidebar/); + + const html = eta.render("wiki.eta", { + ...page, + sidebarHtml: "<ul><li>nav</li></ul>", + footerHtml: "<p>fine print</p>", + }); + assert.match(html, /<footer>\s*<p>fine print<\/p>/); + assert.match(html, /<header><span>Sidebar<\/span><\/header>\s*<ul><li>nav<\/li><\/ul>/); + }); + + it("should link every page of the wiki", () => { + const html = eta.render("wiki.eta", page); + + assert.match(html, /<summary>2 pages<\/summary>/); + assert.match( + html, + /href="\/ariya\/phantomjs\/wiki\/API-Reference-WebPage">API Reference WebPage/, + ); + }); + + it("should list the pages with when each was last updated", () => { + const html = eta.render("wikiPages.eta", { + repo, + pages: [ + { + name: "Home", + href: "/ariya/phantomjs/wiki", + updatedAt: "2018-02-08T18:57:56Z", + }, + ], + }); + + assert.match(html, /href="\/ariya\/phantomjs\/wiki">Home/); + assert.match(html, /<time title="2018-02-08T18:57:56Z">2018-02-08<\/time>/); + }); + + // The editor and the timestamp are scraped independently, so a page missing + // one of them should still show the other. + it("should render each part of the header meta it has", () => { + assert.doesNotMatch(eta.render("wiki.eta", { ...page, updatedAt: null }), /<time/); + assert.match(eta.render("wiki.eta", { ...page, updatedAt: null }), /pixiuPL edited this page/); + assert.doesNotMatch(eta.render("wiki.eta", { ...page, updatedBy: null }), /edited this page/); + }); + + // GitHub reserves underscore-prefixed names for its own actions, so a page + // can never be one; the rest of them are for signed-in users only. + it("should not scrape a reserved wiki name as a page", async () => { + const reserved = ["/ariya/phantomjs/wiki/_new", "/ariya/phantomjs/wiki/Home/_edit"]; + for (const path of reserved) { + const res = await app.request(`http://cgithub.example${path}`); + + assert.strictEqual(res.status, 200); + assert.match(await res.text(), new RegExp(`url=https://github.com${path}"`)); + } + }); + + it("should only read a revision off a path that names one", async () => { + const res = await app.request("http://cgithub.example/ariya/phantomjs/wiki/Home/latest"); + + assert.match( + await res.text(), + /url=https:\/\/github.com\/ariya\/phantomjs\/wiki\/Home\/latest"/, + ); + }); +}); + describe("redirects to GitHub", () => { // An HTTP redirect would be caught by the extension's declarativeNetRequest // rules and bounced straight back here. @@ -16,6 +16,8 @@ import { getGitHubReleases, getGitHubRepo, getGitHubTree, + getGitHubWiki, + getGitHubWikiPages, GitHubHTTPError, RedirectError, } from "./scraper.ts"; @@ -146,6 +148,36 @@ export function createApp(eta: Eta) { return new Response(response.body, { headers: { "Content-Type": contentType } }); }); + // GitHub's own wiki URLs, in the order it resolves them: names beginning with + // an underscore are reserved for its actions (_pages, _new, _edit, ...), so a + // page can never be mistaken for one. + app.get("/:owner/:repo/wiki", async (c) => { + const { owner, repo } = c.req.param(); + return tryRender(c, "wiki.eta", getGitHubWiki(owner, repo)); + }); + + app.get("/:owner/:repo/wiki/_pages", async (c) => { + const { owner, repo } = c.req.param(); + return tryRender(c, "wikiPages.eta", getGitHubWikiPages(owner, repo)); + }); + + // An older revision of a page renders exactly like its current one. + app.get("/:owner/:repo/wiki/:page/:oid", async (c) => { + const { owner, repo, page, oid } = c.req.param(); + if (page.startsWith("_") || !/^[0-9a-f]{40}$/.test(oid)) { + return redirectToGitHub(c, githubUrlFor(c)); + } + return tryRender(c, "wiki.eta", getGitHubWiki(owner, repo, page, oid)); + }); + + app.get("/:owner/:repo/wiki/:page", async (c) => { + const { owner, repo, page } = c.req.param(); + if (page.startsWith("_")) { + return redirectToGitHub(c, githubUrlFor(c)); + } + return tryRender(c, "wiki.eta", getGitHubWiki(owner, repo, page)); + }); + app.get("/:owner/:repo/issues", async (c) => { const { owner, repo } = c.req.param(); const q = c.req.query("q"); diff --git a/src/scraper.test.ts b/src/scraper.test.ts index 88e9271..372772b 100644 --- a/src/scraper.test.ts +++ b/src/scraper.test.ts @@ -14,6 +14,8 @@ import { getGitHubRepo, getGitHubOwner, getGitHubTree, + getGitHubWiki, + getGitHubWikiPages, GitHubHTTPError, RedirectError, } from "./scraper.ts"; @@ -384,6 +386,84 @@ describe("GitHub scraper", () => { }); }); + describe("wikis", () => { + it("should fetch the home page a wiki serves off /wiki itself", async () => { + const data = await getGitHubWiki("microsoft", "vscode"); + + assert.strictEqual(data.repo.owner, "microsoft"); + assert.strictEqual(data.repo.name, "vscode"); + assert.strictEqual(data.title, "Home"); + assert.strictEqual(data.updatedBy, "Greg Van Liew"); + assert.strictEqual(data.updatedAt, "2019-11-22T03:08:32Z"); + + assert.match(data.bodyHtml, /Welcome to the Visual Studio Code Wiki/); + // The _Sidebar and _Footer pages this wiki defines. + assert.match(data.sidebarHtml ?? "", /href="\/microsoft\/vscode\/wiki\/Roadmap"/); + assert.match(data.footerHtml ?? "", /Want to contribute to this Wiki\?/); + }); + + it("should fetch a page by its slug", async () => { + const data = await getGitHubWiki("microsoft", "vscode", "Commit-Signing"); + + assert.strictEqual(data.title, "Commit Signing"); + assert.match(data.bodyHtml, /GPG/); + }); + + // The slug arrives from our route already decoded, so it has to be encoded + // again rather than interpolated into the URL as-is. + it("should fetch a page whose slug is not path-safe", async () => { + const data = await getGitHubWiki( + "microsoft", + "vscode", + "[DEV]-Perf-Tools-for-VS-Code-Development", + ); + + assert.strictEqual(data.title, "[DEV] Perf Tools for VS Code Development"); + assert.ok(data.bodyHtml.length > 0); + }); + + it("should fetch an older revision of a page", async () => { + const oid = "f1c60035ba38b68ba383f58e482c312218daf86c"; + const data = await getGitHubWiki("microsoft", "vscode", "Commit-Signing", oid); + + assert.strictEqual(data.title, "Commit Signing"); + assert.strictEqual(data.updatedBy, "Joaqu\u00edn Ruales"); + assert.strictEqual(data.updatedAt, "2025-10-17T15:07:14Z"); + }); + + // The sidebar list is complete; its "Show N more pages…" button only + // unhides the rest, so no second request is needed to link every page. + it("should collect every page from the sidebar of any page", async () => { + const data = await getGitHubWiki("microsoft", "vscode"); + + assert.ok(data.pages.length > 15); + assert.deepStrictEqual(data.pages[0], { name: "Home", href: "/microsoft/vscode/wiki" }); + assert.ok(data.pages.some((page) => page.href === "/microsoft/vscode/wiki/Commit-Signing")); + }); + + // An archived repository, so neither its wiki nor these dates can move. + it("should fetch the page index with its last-updated dates", async () => { + const data = await getGitHubWikiPages("ariya", "phantomjs"); + + assert.strictEqual(data.repo.owner, "ariya"); + assert.ok(data.pages.length > 1); + assert.ok(data.pages.every((page) => page.updatedAt)); + + const home = data.pages.find((page) => page.name === "Home"); + assert.ok(home); + assert.strictEqual(home.href, "/ariya/phantomjs/wiki"); + assert.strictEqual(home.updatedAt, "2018-02-08T18:57:56Z"); + }); + + it("should redirect to the repository when it has no wiki", async () => { + await assert.rejects(getGitHubWiki("git", "git"), (err) => { + assert(err instanceof RedirectError, "error should be a RedirectError"); + assert.strictEqual(err.location, "/git/git"); + return true; + }); + }); + }); + describe("owner pages", () => { it("should fetch user profile", async () => { const data = await getGitHubOwner("torvalds"); diff --git a/src/scraper.ts b/src/scraper.ts index c02cb5e..6461b19 100644 --- a/src/scraper.ts +++ b/src/scraper.ts @@ -1,5 +1,5 @@ import { parseDocument } from "htmlparser2"; -import { type AnyNode, type Element } from "domhandler"; +import { isText, type AnyNode, type Element } from "domhandler"; import { getAttributeValue, getInnerHTML, textContent } from "domutils"; import * as cssSelect from "css-select"; @@ -352,6 +352,26 @@ export interface GitHubRelease extends GitHubCommon { release: Release; } +export interface WikiPage { + name: string; + href: string; + updatedAt?: string; +} + +export interface GitHubWiki extends GitHubCommon { + title: string; + updatedBy: string | null; + updatedAt: string | null; + bodyHtml: string; + sidebarHtml: string | null; + footerHtml: string | null; + pages: WikiPage[]; +} + +export interface GitHubWikiPages extends GitHubCommon { + pages: WikiPage[]; +} + interface OwnerRepo { owner: string; name: string; @@ -1109,3 +1129,96 @@ export async function getGitHubOwner(owner: string): Promise<GitHubOwner> { // Not a profile page (marketing pages like github.com/features, etc.). throw new RedirectError(`https://github.com/${owner}`); } + +// The text directly inside an element, ignoring any nested elements' text. +function directText(elem: Element): string { + return elem.children + .filter(isText) + .map((node) => node.data) + .join("") + .replace(/\s+/g, " ") + .trim(); +} + +// The sidebar holds every wiki page, which are revealed by JavaScript. +function parseWikiSidebarPages(document: AnyNode): WikiPage[] { + const pages: WikiPage[] = []; + const anchors = cssSelect.selectAll( + ".js-wiki-sidebar-page-container > div > a[href]", + document, + ) as unknown as Element[]; + for (const anchor of anchors) { + const href = getAttributeValue(anchor, "href"); + const name = textContent(anchor).replace(/\s+/g, " ").trim(); + if (href && name) pages.push({ name, href }); + } + return pages; +} + +function wikiHtml(selector: string, document: AnyNode): string | null { + const elem = cssSelect.selectOne(selector, document); + if (!elem) return null; + const html = getInnerHTML(elem).trim(); + return html || null; +} + +export async function getGitHubWiki( + owner: string, + repo: string, + // Empty for the wiki's home page, which GitHub serves off /wiki itself. + page = "", + // A revision SHA renders that older version of the page. + oid = "", +): Promise<GitHubWiki> { + // Page names keep characters that are not path-safe (C#, [DEV] ...), so the + // slug has to be re-encoded rather than interpolated the way a ref is. + const path = [`${owner}/${repo}/wiki`, page && encodeURIComponent(page), oid] + .filter(Boolean) + .join("/"); + const html = await fetchGitHubPage(path); + const document = parseDocument(html); + + const heading = cssSelect.selectOne("h1.gh-header-title", document) as Element | null; + if (!heading) { + throw new Error("Could not find the wiki page heading"); + } + + // "<author> edited this page <date> · <n> revisions", where the author is the + // only part not wrapped in an element of its own. + const meta = cssSelect.selectOne(".gh-header-meta", document) as Element | null; + const updatedBy = meta ? (/^(.+?) edited this page\b/.exec(directText(meta))?.[1] ?? null) : null; + const time = meta ? cssSelect.selectOne("relative-time", meta) : null; + + return { + repo: { owner, name: repo }, + title: directText(heading), + updatedBy, + updatedAt: time ? getAttributeValue(time, "datetime") || null : null, + bodyHtml: wikiHtml("#wiki-body .markdown-body", document) ?? "", + // Wikis can define _Sidebar and _Footer pages, rendered alongside every page. + sidebarHtml: wikiHtml(".wiki-custom-sidebar", document), + footerHtml: wikiHtml("#wiki-footer .markdown-body", document), + pages: parseWikiSidebarPages(document), + }; +} + +export async function getGitHubWikiPages(owner: string, repo: string): Promise<GitHubWikiPages> { + const html = await fetchGitHubPage(`${owner}/${repo}/wiki/_pages`); + const document = parseDocument(html); + + const pages: WikiPage[] = []; + for (const row of cssSelect.selectAll( + "#wiki-content li.Box-row", + document, + ) as unknown as Element[]) { + const anchor = cssSelect.selectOne("a[href]", row) as Element | null; + const href = anchor ? getAttributeValue(anchor, "href") : undefined; + const name = anchor ? textContent(anchor).replace(/\s+/g, " ").trim() : ""; + if (!href || !name) continue; + + const time = cssSelect.selectOne("relative-time", row) as Element | null; + pages.push({ name, href, updatedAt: time ? getAttributeValue(time, "datetime") : undefined }); + } + + return { repo: { owner, name: repo }, pages }; +} diff --git a/views/_wikiNav.eta b/views/_wikiNav.eta new file mode 100644 index 0000000..cc177c7 --- /dev/null +++ b/views/_wikiNav.eta @@ -0,0 +1,4 @@ +<nav> + <a href="/<%= it.repo.owner %>/<%= it.repo.name %>/wiki">Wiki home</a> + <a href="/<%= it.repo.owner %>/<%= it.repo.name %>/wiki/_pages">Pages</a> +</nav> diff --git a/views/layout.eta b/views/layout.eta index 6d1b052..1ac82cc 100644 --- a/views/layout.eta +++ b/views/layout.eta @@ -17,6 +17,7 @@ git clone git@github.com:<%= it.repo.owner %>/<%= it.repo.name %>.git</code></pr <a href="/<%= it.repo.owner %>/<%= it.repo.name %>/issues">Issues</a> <a href="/<%= it.repo.owner %>/<%= it.repo.name %>/pulls">Pull requests</a> <a href="/<%= it.repo.owner %>/<%= it.repo.name %>/releases">Releases</a> + <a href="/<%= it.repo.owner %>/<%= it.repo.name %>/wiki">Wiki</a> <% if (it.branch) { %> <a href="/<%= it.repo.owner %>/<%= it.repo.name %>/commits/<%= it.branch %><%= it.path && it.path !== '/' ? '/' + it.path : '' %>">Commits</a> <% } %> diff --git a/views/wiki.eta b/views/wiki.eta new file mode 100644 index 0000000..d6047e1 --- /dev/null +++ b/views/wiki.eta @@ -0,0 +1,37 @@ +<% layout("./layout.eta"); it.path = "wiki/" + it.title %> + +<%~ include("./_wikiNav.eta", it) %> + +<div class="box wiki"> + <header> + <span><%= it.title %></span> + <% if (it.updatedBy || it.updatedAt) { %> + <span class="timestamp"> + <% if (it.updatedBy) { %><%= it.updatedBy %> edited this page<% } %> + <% if (it.updatedAt) { %><time title="<%= it.updatedAt %>"><%= it.updatedAt.split("T")[0] %></time><% } %> + </span> + <% } %> + </header> + <%~ it.bodyHtml %> + <% if (it.footerHtml) { %> + <footer><%~ it.footerHtml %></footer> + <% } %> +</div> + +<% if (it.sidebarHtml) { %> + <div class="box wiki"> + <header><span>Sidebar</span></header> + <%~ it.sidebarHtml %> + </div> +<% } %> + +<% if (it.pages.length > 0) { %> + <details class="box wiki-pages"> + <summary><%= it.pages.length %> page<%= it.pages.length === 1 ? "" : "s" %></summary> + <ul> + <% for (const page of it.pages) { %> + <li><a href="<%= page.href %>"><%= page.name %></a></li> + <% } %> + </ul> + </details> +<% } %> diff --git a/views/wikiPages.eta b/views/wikiPages.eta new file mode 100644 index 0000000..58d6d21 --- /dev/null +++ b/views/wikiPages.eta @@ -0,0 +1,16 @@ +<% layout("./layout.eta"); it.path = "wiki/_pages" %> + +<%~ include("./_wikiNav.eta", it) %> + +<table> + <% for (const page of it.pages) { %> + <tr> + <td><a href="<%= page.href %>"><%= page.name %></a></td> + <td class="timestamp"> + <% if (page.updatedAt) { %> + <time title="<%= page.updatedAt %>"><%= page.updatedAt.split("T")[0] %></time> + <% } %> + </td> + </tr> + <% } %> +</table> |