summaryrefslogtreecommitdiff
path: root/cgithub/src
diff options
context:
space:
mode:
Diffstat (limited to 'cgithub/src')
-rw-r--r--cgithub/src/app.test.ts98
-rw-r--r--cgithub/src/app.ts32
-rw-r--r--cgithub/src/scraper.test.ts80
-rw-r--r--cgithub/src/scraper.ts115
4 files changed, 324 insertions, 1 deletions
diff --git a/cgithub/src/app.test.ts b/cgithub/src/app.test.ts
index e28c375..3ecebd0 100644
--- a/cgithub/src/app.test.ts
+++ b/cgithub/src/app.test.ts
@@ -178,6 +178,104 @@ describe("embedded files", () => {
});
});
+describe("wiki", () => {
+ const repo = { owner: "ariya", name: "phantomjs" };
+ const page = {
+ repo,
+ title: "Home",
+ updatedBy: "pixiuPL",
+ updatedAt: "2018-02-08T18:57:56Z",
+ bodyHtml: "<p>New to <strong>PhantomJS</strong>?</p>",
+ sidebarHtml: null,
+ footerHtml: null,
+ pages: [
+ { name: "Home", href: "/ariya/phantomjs/wiki" },
+ { name: "API Reference WebPage", href: "/ariya/phantomjs/wiki/API-Reference-WebPage" },
+ ],
+ };
+
+ it("should render the page beside who last edited it", () => {
+ const html = eta.render("wiki.eta", page);
+
+ assert.match(html, /<p>New to <strong>PhantomJS<\/strong>\?<\/p>/);
+ assert.match(html, /pixiuPL edited this page/);
+ assert.match(html, /<time title="2018-02-08T18:57:56Z">2018-02-08<\/time>/);
+ });
+
+ // The _Sidebar and _Footer pages a wiki can define, which GitHub renders
+ // alongside every page rather than as pages of their own.
+ it("should render the sidebar and footer only when the wiki has them", () => {
+ // The page's own content is the last thing in its box, and the box is the
+ // only one on the page.
+ const bare = eta.render("wiki.eta", page);
+ assert.match(bare, /New to <strong>PhantomJS<\/strong>\?<\/p>\s*<\/div>/);
+ assert.doesNotMatch(bare, /Sidebar/);
+
+ const html = eta.render("wiki.eta", {
+ ...page,
+ sidebarHtml: "<ul><li>nav</li></ul>",
+ footerHtml: "<p>fine print</p>",
+ });
+ assert.match(html, /<footer>\s*<p>fine print<\/p>/);
+ assert.match(html, /<header><span>Sidebar<\/span><\/header>\s*<ul><li>nav<\/li><\/ul>/);
+ });
+
+ it("should link every page of the wiki", () => {
+ const html = eta.render("wiki.eta", page);
+
+ assert.match(html, /<summary>2 pages<\/summary>/);
+ assert.match(
+ html,
+ /href="\/ariya\/phantomjs\/wiki\/API-Reference-WebPage">API Reference WebPage/,
+ );
+ });
+
+ it("should list the pages with when each was last updated", () => {
+ const html = eta.render("wikiPages.eta", {
+ repo,
+ pages: [
+ {
+ name: "Home",
+ href: "/ariya/phantomjs/wiki",
+ updatedAt: "2018-02-08T18:57:56Z",
+ },
+ ],
+ });
+
+ assert.match(html, /href="\/ariya\/phantomjs\/wiki">Home/);
+ assert.match(html, /<time title="2018-02-08T18:57:56Z">2018-02-08<\/time>/);
+ });
+
+ // The editor and the timestamp are scraped independently, so a page missing
+ // one of them should still show the other.
+ it("should render each part of the header meta it has", () => {
+ assert.doesNotMatch(eta.render("wiki.eta", { ...page, updatedAt: null }), /<time/);
+ assert.match(eta.render("wiki.eta", { ...page, updatedAt: null }), /pixiuPL edited this page/);
+ assert.doesNotMatch(eta.render("wiki.eta", { ...page, updatedBy: null }), /edited this page/);
+ });
+
+ // GitHub reserves underscore-prefixed names for its own actions, so a page
+ // can never be one; the rest of them are for signed-in users only.
+ it("should not scrape a reserved wiki name as a page", async () => {
+ const reserved = ["/ariya/phantomjs/wiki/_new", "/ariya/phantomjs/wiki/Home/_edit"];
+ for (const path of reserved) {
+ const res = await app.request(`http://cgithub.example${path}`);
+
+ assert.strictEqual(res.status, 200);
+ assert.match(await res.text(), new RegExp(`url=https://github.com${path}"`));
+ }
+ });
+
+ it("should only read a revision off a path that names one", async () => {
+ const res = await app.request("http://cgithub.example/ariya/phantomjs/wiki/Home/latest");
+
+ assert.match(
+ await res.text(),
+ /url=https:\/\/github.com\/ariya\/phantomjs\/wiki\/Home\/latest"/,
+ );
+ });
+});
+
describe("redirects to GitHub", () => {
// An HTTP redirect would be caught by the extension's declarativeNetRequest
// rules and bounced straight back here.
diff --git a/cgithub/src/app.ts b/cgithub/src/app.ts
index 33b408e..956a8cb 100644
--- a/cgithub/src/app.ts
+++ b/cgithub/src/app.ts
@@ -16,6 +16,8 @@ import {
getGitHubReleases,
getGitHubRepo,
getGitHubTree,
+ getGitHubWiki,
+ getGitHubWikiPages,
GitHubHTTPError,
RedirectError,
} from "./scraper.ts";
@@ -146,6 +148,36 @@ export function createApp(eta: Eta) {
return new Response(response.body, { headers: { "Content-Type": contentType } });
});
+ // GitHub's own wiki URLs, in the order it resolves them: names beginning with
+ // an underscore are reserved for its actions (_pages, _new, _edit, ...), so a
+ // page can never be mistaken for one.
+ app.get("/:owner/:repo/wiki", async (c) => {
+ const { owner, repo } = c.req.param();
+ return tryRender(c, "wiki.eta", getGitHubWiki(owner, repo));
+ });
+
+ app.get("/:owner/:repo/wiki/_pages", async (c) => {
+ const { owner, repo } = c.req.param();
+ return tryRender(c, "wikiPages.eta", getGitHubWikiPages(owner, repo));
+ });
+
+ // An older revision of a page renders exactly like its current one.
+ app.get("/:owner/:repo/wiki/:page/:oid", async (c) => {
+ const { owner, repo, page, oid } = c.req.param();
+ if (page.startsWith("_") || !/^[0-9a-f]{40}$/.test(oid)) {
+ return redirectToGitHub(c, githubUrlFor(c));
+ }
+ return tryRender(c, "wiki.eta", getGitHubWiki(owner, repo, page, oid));
+ });
+
+ app.get("/:owner/:repo/wiki/:page", async (c) => {
+ const { owner, repo, page } = c.req.param();
+ if (page.startsWith("_")) {
+ return redirectToGitHub(c, githubUrlFor(c));
+ }
+ return tryRender(c, "wiki.eta", getGitHubWiki(owner, repo, page));
+ });
+
app.get("/:owner/:repo/issues", async (c) => {
const { owner, repo } = c.req.param();
const q = c.req.query("q");
diff --git a/cgithub/src/scraper.test.ts b/cgithub/src/scraper.test.ts
index 88e9271..372772b 100644
--- a/cgithub/src/scraper.test.ts
+++ b/cgithub/src/scraper.test.ts
@@ -14,6 +14,8 @@ import {
getGitHubRepo,
getGitHubOwner,
getGitHubTree,
+ getGitHubWiki,
+ getGitHubWikiPages,
GitHubHTTPError,
RedirectError,
} from "./scraper.ts";
@@ -384,6 +386,84 @@ describe("GitHub scraper", () => {
});
});
+ describe("wikis", () => {
+ it("should fetch the home page a wiki serves off /wiki itself", async () => {
+ const data = await getGitHubWiki("microsoft", "vscode");
+
+ assert.strictEqual(data.repo.owner, "microsoft");
+ assert.strictEqual(data.repo.name, "vscode");
+ assert.strictEqual(data.title, "Home");
+ assert.strictEqual(data.updatedBy, "Greg Van Liew");
+ assert.strictEqual(data.updatedAt, "2019-11-22T03:08:32Z");
+
+ assert.match(data.bodyHtml, /Welcome to the Visual Studio Code Wiki/);
+ // The _Sidebar and _Footer pages this wiki defines.
+ assert.match(data.sidebarHtml ?? "", /href="\/microsoft\/vscode\/wiki\/Roadmap"/);
+ assert.match(data.footerHtml ?? "", /Want to contribute to this Wiki\?/);
+ });
+
+ it("should fetch a page by its slug", async () => {
+ const data = await getGitHubWiki("microsoft", "vscode", "Commit-Signing");
+
+ assert.strictEqual(data.title, "Commit Signing");
+ assert.match(data.bodyHtml, /GPG/);
+ });
+
+ // The slug arrives from our route already decoded, so it has to be encoded
+ // again rather than interpolated into the URL as-is.
+ it("should fetch a page whose slug is not path-safe", async () => {
+ const data = await getGitHubWiki(
+ "microsoft",
+ "vscode",
+ "[DEV]-Perf-Tools-for-VS-Code-Development",
+ );
+
+ assert.strictEqual(data.title, "[DEV] Perf Tools for VS Code Development");
+ assert.ok(data.bodyHtml.length > 0);
+ });
+
+ it("should fetch an older revision of a page", async () => {
+ const oid = "f1c60035ba38b68ba383f58e482c312218daf86c";
+ const data = await getGitHubWiki("microsoft", "vscode", "Commit-Signing", oid);
+
+ assert.strictEqual(data.title, "Commit Signing");
+ assert.strictEqual(data.updatedBy, "Joaqu\u00edn Ruales");
+ assert.strictEqual(data.updatedAt, "2025-10-17T15:07:14Z");
+ });
+
+ // The sidebar list is complete; its "Show N more pages…" button only
+ // unhides the rest, so no second request is needed to link every page.
+ it("should collect every page from the sidebar of any page", async () => {
+ const data = await getGitHubWiki("microsoft", "vscode");
+
+ assert.ok(data.pages.length > 15);
+ assert.deepStrictEqual(data.pages[0], { name: "Home", href: "/microsoft/vscode/wiki" });
+ assert.ok(data.pages.some((page) => page.href === "/microsoft/vscode/wiki/Commit-Signing"));
+ });
+
+ // An archived repository, so neither its wiki nor these dates can move.
+ it("should fetch the page index with its last-updated dates", async () => {
+ const data = await getGitHubWikiPages("ariya", "phantomjs");
+
+ assert.strictEqual(data.repo.owner, "ariya");
+ assert.ok(data.pages.length > 1);
+ assert.ok(data.pages.every((page) => page.updatedAt));
+
+ const home = data.pages.find((page) => page.name === "Home");
+ assert.ok(home);
+ assert.strictEqual(home.href, "/ariya/phantomjs/wiki");
+ assert.strictEqual(home.updatedAt, "2018-02-08T18:57:56Z");
+ });
+
+ it("should redirect to the repository when it has no wiki", async () => {
+ await assert.rejects(getGitHubWiki("git", "git"), (err) => {
+ assert(err instanceof RedirectError, "error should be a RedirectError");
+ assert.strictEqual(err.location, "/git/git");
+ return true;
+ });
+ });
+ });
+
describe("owner pages", () => {
it("should fetch user profile", async () => {
const data = await getGitHubOwner("torvalds");
diff --git a/cgithub/src/scraper.ts b/cgithub/src/scraper.ts
index c02cb5e..6461b19 100644
--- a/cgithub/src/scraper.ts
+++ b/cgithub/src/scraper.ts
@@ -1,5 +1,5 @@
import { parseDocument } from "htmlparser2";
-import { type AnyNode, type Element } from "domhandler";
+import { isText, type AnyNode, type Element } from "domhandler";
import { getAttributeValue, getInnerHTML, textContent } from "domutils";
import * as cssSelect from "css-select";
@@ -352,6 +352,26 @@ export interface GitHubRelease extends GitHubCommon {
release: Release;
}
+export interface WikiPage {
+ name: string;
+ href: string;
+ updatedAt?: string;
+}
+
+export interface GitHubWiki extends GitHubCommon {
+ title: string;
+ updatedBy: string | null;
+ updatedAt: string | null;
+ bodyHtml: string;
+ sidebarHtml: string | null;
+ footerHtml: string | null;
+ pages: WikiPage[];
+}
+
+export interface GitHubWikiPages extends GitHubCommon {
+ pages: WikiPage[];
+}
+
interface OwnerRepo {
owner: string;
name: string;
@@ -1109,3 +1129,96 @@ export async function getGitHubOwner(owner: string): Promise<GitHubOwner> {
// Not a profile page (marketing pages like github.com/features, etc.).
throw new RedirectError(`https://github.com/${owner}`);
}
+
+// The text directly inside an element, ignoring any nested elements' text.
+function directText(elem: Element): string {
+ return elem.children
+ .filter(isText)
+ .map((node) => node.data)
+ .join("")
+ .replace(/\s+/g, " ")
+ .trim();
+}
+
+// The sidebar holds every wiki page, which are revealed by JavaScript.
+function parseWikiSidebarPages(document: AnyNode): WikiPage[] {
+ const pages: WikiPage[] = [];
+ const anchors = cssSelect.selectAll(
+ ".js-wiki-sidebar-page-container > div > a[href]",
+ document,
+ ) as unknown as Element[];
+ for (const anchor of anchors) {
+ const href = getAttributeValue(anchor, "href");
+ const name = textContent(anchor).replace(/\s+/g, " ").trim();
+ if (href && name) pages.push({ name, href });
+ }
+ return pages;
+}
+
+function wikiHtml(selector: string, document: AnyNode): string | null {
+ const elem = cssSelect.selectOne(selector, document);
+ if (!elem) return null;
+ const html = getInnerHTML(elem).trim();
+ return html || null;
+}
+
+export async function getGitHubWiki(
+ owner: string,
+ repo: string,
+ // Empty for the wiki's home page, which GitHub serves off /wiki itself.
+ page = "",
+ // A revision SHA renders that older version of the page.
+ oid = "",
+): Promise<GitHubWiki> {
+ // Page names keep characters that are not path-safe (C#, [DEV] ...), so the
+ // slug has to be re-encoded rather than interpolated the way a ref is.
+ const path = [`${owner}/${repo}/wiki`, page && encodeURIComponent(page), oid]
+ .filter(Boolean)
+ .join("/");
+ const html = await fetchGitHubPage(path);
+ const document = parseDocument(html);
+
+ const heading = cssSelect.selectOne("h1.gh-header-title", document) as Element | null;
+ if (!heading) {
+ throw new Error("Could not find the wiki page heading");
+ }
+
+ // "<author> edited this page <date> · <n> revisions", where the author is the
+ // only part not wrapped in an element of its own.
+ const meta = cssSelect.selectOne(".gh-header-meta", document) as Element | null;
+ const updatedBy = meta ? (/^(.+?) edited this page\b/.exec(directText(meta))?.[1] ?? null) : null;
+ const time = meta ? cssSelect.selectOne("relative-time", meta) : null;
+
+ return {
+ repo: { owner, name: repo },
+ title: directText(heading),
+ updatedBy,
+ updatedAt: time ? getAttributeValue(time, "datetime") || null : null,
+ bodyHtml: wikiHtml("#wiki-body .markdown-body", document) ?? "",
+ // Wikis can define _Sidebar and _Footer pages, rendered alongside every page.
+ sidebarHtml: wikiHtml(".wiki-custom-sidebar", document),
+ footerHtml: wikiHtml("#wiki-footer .markdown-body", document),
+ pages: parseWikiSidebarPages(document),
+ };
+}
+
+export async function getGitHubWikiPages(owner: string, repo: string): Promise<GitHubWikiPages> {
+ const html = await fetchGitHubPage(`${owner}/${repo}/wiki/_pages`);
+ const document = parseDocument(html);
+
+ const pages: WikiPage[] = [];
+ for (const row of cssSelect.selectAll(
+ "#wiki-content li.Box-row",
+ document,
+ ) as unknown as Element[]) {
+ const anchor = cssSelect.selectOne("a[href]", row) as Element | null;
+ const href = anchor ? getAttributeValue(anchor, "href") : undefined;
+ const name = anchor ? textContent(anchor).replace(/\s+/g, " ").trim() : "";
+ if (!href || !name) continue;
+
+ const time = cssSelect.selectOne("relative-time", row) as Element | null;
+ pages.push({ name, href, updatedAt: time ? getAttributeValue(time, "datetime") : undefined });
+ }
+
+ return { repo: { owner, name: repo }, pages };
+}
generated by cgit v1.3.1 (git 2.54.0) at 2026-10-04 21:24:45 +0000