diff options
Diffstat (limited to 'cgithub/src')
| -rw-r--r-- | cgithub/src/app.test.ts | 483 | ||||
| -rw-r--r-- | cgithub/src/app.ts | 284 | ||||
| -rw-r--r-- | cgithub/src/scraper.test.ts | 624 | ||||
| -rw-r--r-- | cgithub/src/scraper.ts | 1228 | ||||
| -rw-r--r-- | cgithub/src/worker.ts | 30 |
5 files changed, 2649 insertions, 0 deletions
diff --git a/cgithub/src/app.test.ts b/cgithub/src/app.test.ts new file mode 100644 index 0000000..450c1cc --- /dev/null +++ b/cgithub/src/app.test.ts @@ -0,0 +1,483 @@ +import assert from "node:assert"; +import path from "node:path"; +import { describe, it, type TestContext } from "node:test"; +import { Eta } from "eta"; +import { createApp } from "./app.ts"; + +// The routes exercised here redirect without scraping, so no network is needed. +const eta = new Eta({ views: path.join(import.meta.dirname, "..", "views") }); +const app = createApp(eta); + +describe("branch picker", () => { + const repo = { owner: "actions", name: "deploy-pages" }; + const refs = [{ name: "main", isDefault: true }, { name: "v4" }]; + + it("should keep the current view and path", () => { + const blob = eta.render("_refs.eta", { + repo, + refs, + type: "branches", + view: "blob", + path: "src/main.ts", + }); + const tree = eta.render("_refs.eta", { + repo, + refs, + type: "branches", + view: "tree", + path: "src", + }); + + assert.match(blob, /href="\/actions\/deploy-pages\/blob\/main\/src\/main.ts"/); + assert.match(blob, /href="\/actions\/deploy-pages\/blob\/v4\/src\/main.ts"/); + assert.match(tree, /href="\/actions\/deploy-pages\/tree\/v4\/src"/); + }); + + it("should link to the root tree without a path", () => { + const html = eta.render("_refs.eta", { repo, refs, type: "branches", view: "tree", path: "" }); + + assert.match(html, /href="\/actions\/deploy-pages\/tree\/main"/); + }); + + it("should not build ref links for views it does not serve", async () => { + const res = await app.request("http://cgithub.example/api/a/b/refs/branches/raw/README.md"); + + assert.strictEqual(res.status, 404); + }); + + it("should tell the refs endpoint where the picker was opened", () => { + const nav = { repo, branch: "main", path: "src" }; + + const blob = eta.render("_nav.eta", { ...nav, view: "blob", path: "src/main.ts" }); + assert.match(blob, /data-view="blob" data-path="src\/main.ts"/); + assert.match( + eta.render("_nav.eta", { ...nav, view: "tree" }), + /data-view="tree" data-path="src"/, + ); + assert.doesNotMatch(eta.render("_nav.eta", { ...nav, view: "tree", path: "" }), /data-path=/); + }); +}); + +describe("sidebar fragment", () => { + const repo = { owner: "actions", name: "deploy-pages" }; + const languages = [{ name: "JavaScript", percentage: 100, color: "#f1e05a" }]; + const latestRelease = { tag: "v5.0.0", publishedAt: "2026-03-25T16:59:14Z" }; + + // The release replaces the empty element it names, beside the release count. + it("should render the latest release and the languages", () => { + const html = eta.render("_sidebar.eta", { repo, languages, latestRelease }); + + assert.match(html, /<span data-target="latest-release">\(<a href/); + assert.match(html, /href="\/actions\/deploy-pages\/releases\/tag\/v5.0.0"/); + assert.match(html, /<time class="timestamp" title="2026-03-25T16:59:14Z">2026-03-25<\/time>/); + assert.match(html, /<div class="languages box">/); + }); + + it("should name a spot the repo page holds open", () => { + const html = eta.render("repo.eta", { + repo, + branch: "main", + path: "/", + items: [], + info: { description: null, website: null, stars: null, forks: null, numReleases: 38 }, + }); + + assert.match(html, /<span id="latest-release"><\/span>/); + }); + + it("should render nothing when the repository has neither", () => { + const html = eta.render("_sidebar.eta", { repo, languages: [], latestRelease: null }); + + assert.strictEqual(html.trim(), ""); + }); +}); + +describe("blob view", () => { + const blob = { + repo: { owner: "justrajdeep", name: "fonts" }, + branch: "master", + path: "Times New Roman.ttf", + language: null, + size: "815 KB", + image: false, + renderFileType: null, + textLines: null, + htmlLines: null, + htmlContent: null, + }; + + it("should embed a PDF", () => { + const html = eta.render("blob.eta", { + ...blob, + path: "test/pdfs/basicapi.pdf", + renderFileType: "pdf", + }); + + assert.match(html, /<object[^>]*type="application\/pdf"/); + assert.match(html, /data="\/justrajdeep\/fonts\/embed\/pdf\/master\/test\/pdfs\/basicapi.pdf"/); + }); + + it("should not embed a file type it has no viewer for", () => { + const html = eta.render("blob.eta", { ...blob, renderFileType: "stl" }); + + assert.doesNotMatch(html, /<object/); + assert.match(html, /This file cannot be displayed\./); + }); + + it("should give each syntax-highlighted line an anchor", () => { + const html = eta.render("blob.eta", { + ...blob, + path: "src/app.ts", + htmlLines: ['<span class="pl-k">const</span> x;', "x++;"], + }); + + assert.match( + html, + /<code><span id="L1"><span class="pl-k">const<\/span> x;<\/span>\n<span id="L2">x\+\+;<\/span><\/code>/, + ); + }); + + it("should give each plain-text line an anchor", () => { + const html = eta.render("blob.eta", { + ...blob, + path: "notes.txt", + textLines: ["a < b", "c"], + }); + + assert.match(html, /<code><span id="L1">a < b<\/span>\n<span id="L2">c<\/span><\/code>/); + }); + + it("should offer the raw file when there is nothing to display", () => { + const html = eta.render("blob.eta", blob); + + assert.match(html, /This file cannot be displayed\./); + assert.match(html, /href="\/justrajdeep\/fonts\/raw\/master\/Times New Roman.ttf">View raw/); + }); +}); + +describe("commit history", () => { + const repo = { owner: "actions", name: "deploy-pages" }; + const oid = "b39c421b98f49d83ae50ec502c3ddfc3bf28f2c6"; + const commitGroups = [ + { + title: "Apr 18, 2023", + commits: [ + { oid, shortMessage: "Remove deprecated input parameters", authors: [{ login: "jmg" }] }, + ], + }, + ]; + const history = { repo, branch: "main", path: "src/index.js", commitGroups }; + + // A commits payload never says whether its path is a file or a directory, so + // every link goes to the tree and GitHub redirects a file's to its blob. + it("should browse a file's history at each of its commits", () => { + const html = eta.render("commits.eta", history); + + assert.match(html, new RegExp(`href="/actions/deploy-pages/blob/${oid}/src/index.js">Browse`)); + }); + + it("should browse a directory's history the same way", () => { + const html = eta.render("commits.eta", { ...history, path: "src" }); + + assert.match(html, new RegExp(`href="/actions/deploy-pages/blob/${oid}/src">Browse`)); + }); + + it("should browse the whole repository when the history has no path", () => { + const html = eta.render("commits.eta", { ...history, path: "" }); + + assert.match(html, new RegExp(`href="/actions/deploy-pages/blob/${oid}">Browse`)); + }); + + // Walking up from a history stays in the history, and its last crumb is the + // page you are already on, just as a blob or a tree ends on itself. + it("should walk up through the history of each parent directory", () => { + const html = eta.render("commits.eta", { ...history, path: "src/lib/index.js" }); + + assert.match(html, /href="\/actions\/deploy-pages\/commits\/main">\(top\)</); + assert.match(html, /href="\/actions\/deploy-pages\/commits\/main\/src">src</); + assert.match( + html, + /href="\/actions\/deploy-pages\/commits\/main\/src\/lib">lib<\/a>\/index\.js/, + ); + }); + + it("should say so when a path has no history", () => { + const html = eta.render("commits.eta", { ...history, commitGroups: [] }); + + assert.match(html, /No commits found\./); + }); + + it("should link a blob to its own history", () => { + const html = eta.render("blob.eta", { ...history, textLines: ["x"] }); + + assert.match(html, /href="\/actions\/deploy-pages\/commits\/main\/src\/index.js">History/); + }); +}); + +describe("embedded files", () => { + function stubGitHub(t: TestContext, body: string, contentType: string, status = 200) { + t.mock.method( + globalThis, + "fetch", + async () => new Response(body, { status, headers: { "Content-Type": contentType } }), + ); + } + + it("should retype a PDF GitHub sends as a download", async (t) => { + stubGitHub(t, "%PDF-1.6", "application/octet-stream"); + + const res = await app.request("http://cgithub.example/mozilla/pdf.js/embed/pdf/master/doc.pdf"); + + assert.strictEqual(res.status, 200); + assert.strictEqual(res.headers.get("Content-Type"), "application/pdf"); + assert.strictEqual(await res.text(), "%PDF-1.6"); + }); + + it("should pass on a type GitHub already got right", async (t) => { + stubGitHub(t, "%PDF-1.6", "application/pdf"); + + const res = await app.request("http://cgithub.example/mozilla/pdf.js/embed/pdf/master/doc.pdf"); + + assert.strictEqual(res.headers.get("Content-Type"), "application/pdf"); + }); + + it("should not serve a file type it has no viewer for", async (t) => { + stubGitHub(t, "<script>", "text/html"); + + const res = await app.request("http://cgithub.example/mozilla/pdf.js/embed/html/master/x.html"); + + assert.strictEqual(res.status, 404); + }); + + it("should pass on the status of a file GitHub does not have", async (t) => { + stubGitHub(t, "404: Not Found", "text/plain", 404); + + const res = await app.request( + "http://cgithub.example/mozilla/pdf.js/embed/pdf/master/nope.pdf", + ); + + assert.strictEqual(res.status, 404); + }); +}); + +describe("wiki", () => { + const repo = { owner: "ariya", name: "phantomjs" }; + const page = { + repo, + title: "Home", + updatedBy: "pixiuPL", + updatedAt: "2018-02-08T18:57:56Z", + bodyHtml: "<p>New to <strong>PhantomJS</strong>?</p>", + sidebarHtml: null, + footerHtml: null, + pages: [ + { name: "Home", href: "/ariya/phantomjs/wiki" }, + { name: "API Reference WebPage", href: "/ariya/phantomjs/wiki/API-Reference-WebPage" }, + ], + }; + + it("should render the page beside who last edited it", () => { + const html = eta.render("wiki.eta", page); + + assert.match(html, /<p>New to <strong>PhantomJS<\/strong>\?<\/p>/); + assert.match(html, /pixiuPL edited this page/); + assert.match(html, /<time title="2018-02-08T18:57:56Z">2018-02-08<\/time>/); + }); + + // The _Sidebar and _Footer pages a wiki can define, which GitHub renders + // alongside every page rather than as pages of their own. + it("should render the sidebar and footer only when the wiki has them", () => { + // The page's own content is the last thing in its box, and the box is the + // only one on the page. + const bare = eta.render("wiki.eta", page); + assert.match(bare, /New to <strong>PhantomJS<\/strong>\?<\/p>\s*<\/div>/); + assert.doesNotMatch(bare, /Sidebar/); + + const html = eta.render("wiki.eta", { + ...page, + sidebarHtml: "<ul><li>nav</li></ul>", + footerHtml: "<p>fine print</p>", + }); + assert.match(html, /<footer>\s*<p>fine print<\/p>/); + assert.match(html, /<header><span>Sidebar<\/span><\/header>\s*<ul><li>nav<\/li><\/ul>/); + }); + + it("should link every page of the wiki", () => { + const html = eta.render("wiki.eta", page); + + assert.match(html, /<summary>2 pages<\/summary>/); + assert.match( + html, + /href="\/ariya\/phantomjs\/wiki\/API-Reference-WebPage">API Reference WebPage/, + ); + }); + + it("should list the pages with when each was last updated", () => { + const html = eta.render("wikiPages.eta", { + repo, + pages: [ + { + name: "Home", + href: "/ariya/phantomjs/wiki", + updatedAt: "2018-02-08T18:57:56Z", + }, + ], + }); + + assert.match(html, /href="\/ariya\/phantomjs\/wiki">Home/); + assert.match(html, /<time title="2018-02-08T18:57:56Z">2018-02-08<\/time>/); + }); + + // The editor and the timestamp are scraped independently, so a page missing + // one of them should still show the other. + it("should render each part of the header meta it has", () => { + assert.doesNotMatch(eta.render("wiki.eta", { ...page, updatedAt: null }), /<time/); + assert.match(eta.render("wiki.eta", { ...page, updatedAt: null }), /pixiuPL edited this page/); + assert.doesNotMatch(eta.render("wiki.eta", { ...page, updatedBy: null }), /edited this page/); + }); + + // GitHub reserves underscore-prefixed names for its own actions, so a page + // can never be one; the rest of them are for signed-in users only. + it("should not scrape a reserved wiki name as a page", async () => { + const reserved = ["/ariya/phantomjs/wiki/_new", "/ariya/phantomjs/wiki/Home/_edit"]; + for (const path of reserved) { + const res = await app.request(`http://cgithub.example${path}`); + + assert.strictEqual(res.status, 200); + assert.match(await res.text(), new RegExp(`url=https://github.com${path}"`)); + } + }); + + it("should only read a revision off a path that names one", async () => { + const res = await app.request("http://cgithub.example/ariya/phantomjs/wiki/Home/latest"); + + assert.match( + await res.text(), + /url=https:\/\/github.com\/ariya\/phantomjs\/wiki\/Home\/latest"/, + ); + }); +}); + +describe("global search", () => { + it("should keep the rest of the query around the qualifier", async () => { + const res = await app.request( + "http://cgithub.example/search?q=" + + encodeURIComponent("is:issue repo:actions/deploy-pages label:bug") + + "&type=issues", + ); + + assert.strictEqual( + res.headers.get("location"), + "/actions/deploy-pages/search?q=is%3Aissue+label%3Abug&type=issues", + ); + }); + + it("should redirect searches it cannot scope to a repository", async () => { + for (const url of [ + "http://cgithub.example/search?q=dms&type=code", + // An owner-wide search is not a single repository. + "http://cgithub.example/search?q=" + + encodeURIComponent("org:nix-community dms") + + "&type=code", + // Without a type there is no search to hand over. + "http://cgithub.example/search?q=" + encodeURIComponent("repo:actions/deploy-pages dms"), + ]) { + const res = await app.request(url); + + assert.strictEqual(res.status, 200, url); + assert.match(await res.text(), /url=https:\/\/github.com\/search\?/); + } + }); +}); + +describe("redirects to GitHub", () => { + // An HTTP redirect would be caught by the extension's declarativeNetRequest + // rules and bounced straight back here. + it("should redirect unknown paths with a meta refresh", async () => { + const res = await app.request("http://cgithub.example/a/b/c?x=1"); + + assert.strictEqual(res.status, 200); + assert.strictEqual(res.headers.get("location"), null); + + const body = await res.text(); + assert.match(body, /<meta name="referrer" content="no-referrer">/); + assert.match( + body, + /<meta http-equiv="refresh" content="1; url=https:\/\/github.com\/a\/b\/c\?x=1">/, + ); + assert.doesNotMatch(body, /<script/); + }); + + // Same-origin requests are mostly subresources, which render an HTML page as + // a broken image rather than following its refresh. + it("should redirect over HTTP when the referer is one of our own pages", async () => { + const res = await app.request("http://cgithub.example/a/b/c?x=1", { + headers: { Referer: "http://cgithub.example/a/b" }, + }); + + assert.strictEqual(res.status, 302); + assert.strictEqual(res.headers.get("location"), "https://github.com/a/b/c?x=1"); + assert.strictEqual(res.headers.get("Referrer-Policy"), "no-referrer"); + }); + + it("should use a meta refresh when the referer is another site", async () => { + const res = await app.request("http://cgithub.example/a/b/c?x=1", { + headers: { Referer: "https://github.example/a/b" }, + }); + + assert.strictEqual(res.status, 200); + assert.strictEqual(res.headers.get("location"), null); + }); + + it("should redirect unhandled search types", async () => { + const res = await app.request( + "http://cgithub.example/actions/deploy-pages/search?q=x&type=code", + ); + + assert.strictEqual(res.status, 200); + assert.match( + await res.text(), + /https:\/\/github.com\/actions\/deploy-pages\/search\?q=x&type=code/, + ); + }); + + it("should redirect notifications without scraping them as a profile", async () => { + const res = await app.request("http://cgithub.example/notifications?query=is%3Aunread"); + + assert.strictEqual(res.status, 200); + assert.match(await res.text(), /url=https:\/\/github.com\/notifications\?query=is%3Aunread"/); + }); + + it("should redirect raw file requests to raw.githubusercontent.com", async () => { + const res = await app.request("http://cgithub.example/actions/deploy-pages/raw/main/README.md"); + + assert.strictEqual(res.status, 200); + assert.match( + await res.text(), + /url=https:\/\/raw.githubusercontent.com\/actions\/deploy-pages\/main\/README.md"/, + ); + }); +}); + +describe("error page", () => { + it("should render inside the page shell", () => { + const html = eta.render("error.eta", { + title: "Not Found", + message: "GitHub responded with HTTP 404 Not Found", + buildCommit: "v0-g1234567", + githubUrl: "https://github.com/actions/deploy-pages", + }); + + assert.match(html, /<title>Not Found<\/title>/); + assert.match(html, /GitHub responded with HTTP 404 Not Found/); + assert.match(html, /<footer>Generated by cgithub commit <code>v0-g1234567<\/code>/); + assert.match(html, /href="https:\/\/github.com\/actions\/deploy-pages"/); + }); + + // The title falls back to the message for errors that carry no status text. + it("should title itself with the message when it has no title", () => { + const html = eta.render("error.eta", { message: "Error: boom" }); + + assert.match(html, /<title>Error: boom<\/title>/); + }); +}); diff --git a/cgithub/src/app.ts b/cgithub/src/app.ts new file mode 100644 index 0000000..0b9a7e6 --- /dev/null +++ b/cgithub/src/app.ts @@ -0,0 +1,284 @@ +import type { Eta } from "eta"; +import { type Context, Hono } from "hono"; +import type { StatusCode } from "hono/utils/http-status"; +import { buildCommit } from "./build.generated.ts"; +import { + getGitHubBlob, + getGitHubCommits, + getGitHubIssues, + getGitHubSidebar, + getGitHubLatestCommit, + getGitHubOwner, + getGitHubPulls, + getGitHubRaw, + getGitHubRefs, + getGitHubRelease, + getGitHubReleases, + getGitHubRepo, + getGitHubTree, + getGitHubWiki, + getGitHubWikiPages, + ExternalRedirectError, + GitHubHTTPError, + InternalRedirectError, + RedirectError, +} from "./scraper.ts"; + +declare module "hono" { + interface ContextRenderer { + (template: string, data?: object): Response; + } +} + +function githubUrlFor(c: Context) { + const { pathname, search } = new URL(c.req.url); + return `https://github.com${pathname}${search}`; +} + +export function createApp(eta: Eta) { + const app = new Hono(); + + app.use(async (c, next) => { + // Include render data for views/layouts/base.eta. + c.setRenderer((template, data) => + c.html(eta.render(template, { ...data, githubUrl: githubUrlFor(c), buildCommit })), + ); + await next(); + }); + + // Redirect off-domain. + function redirectTo(c: Context, location: string) { + const referer = c.req.header("Referer"); + // If we are the referer, redirect directly so we can hotlink resources + // (e.g., <img> in README). + if (referer && URL.parse(referer)?.origin === new URL(c.req.url).origin) { + c.header("Referrer-Policy", "no-referrer"); + return c.redirect(location); + } + + // Otherwise use a meta refresh to make ourself the initiator. This will + // avoid redirect loops if we are part of excludedInitiatorDomains in a + // declarativeNetRequest. + return c.render("redirect.eta", { location }); + } + + function redirectToGitHub(c: Context) { + return redirectTo(c, githubUrlFor(c)); + } + + app.onError((e, c) => { + // Fragments fetched asynchronously by client-side JS (see public/static/refs.js) + // fail as an empty body rather than the full error.eta page. + if (c.req.path.startsWith("/api/")) { + c.status( + e instanceof GitHubHTTPError + ? (e.status as StatusCode) + : e instanceof RedirectError + ? 404 + : 500, + ); + return c.body(null); + } + + if (e instanceof InternalRedirectError) return c.redirect(e.location); + if (e instanceof ExternalRedirectError) return redirectTo(c, e.location); + if (e instanceof GitHubHTTPError) { + c.status(e.status as StatusCode); + const message = `GitHub responded with HTTP ${e.status} ${e.message}`; + return c.render("error.eta", { title: e.message, message }); + } + c.status(500); + return c.render("error.eta", { message: "" + e }); + }); + + app.get("/", async (c) => { + return c.render("home.eta"); + }); + + // Exclusion that immediately redirects to GitHub. + app.get("/notifications", async (c) => { + return redirectToGitHub(c); + }); + + // GitHub's global search, which we only serve when it is scoped to a single + // repository: strip the qualifier and let that repository's search route + // decide what to do with the rest. + app.get("/search", async (c) => { + const q = c.req.query("q") ?? ""; + const type = c.req.query("type"); + + const repoQualifier = /(?:^|\s)repo:([^/\s]+)\/([^/\s]+)(?=\s|$)/.exec(q); + if (type && repoQualifier) { + const [qualifier, owner, repo] = repoQualifier; + const rest = + q.slice(0, repoQualifier.index) + q.slice(repoQualifier.index + qualifier.length); + const params = new URLSearchParams({ q: rest.trim(), type }); + return c.redirect(`/${owner}/${repo}/search?${params}`); + } + + return redirectToGitHub(c); + }); + + app.get("/:owner", async (c) => { + const { owner } = c.req.param(); + return c.render("owner.eta", await getGitHubOwner(owner)); + }); + + app.get("/:owner/:repo", async (c) => { + const { owner, repo } = c.req.param(); + return c.render("repo.eta", await getGitHubRepo(owner, repo)); + }); + + app.get("/:owner/:repo/tree/:branch/:path{.*}?", async (c) => { + const { owner, repo, branch, path = "" } = c.req.param(); + return c.render("tree.eta", await getGitHubTree(owner, repo, branch, path)); + }); + + app.get("/:owner/:repo/blob/:branch/:path{.*}", async (c) => { + const { owner, repo, branch, path } = c.req.param(); + return c.render("blob.eta", await getGitHubBlob(owner, repo, branch, path)); + }); + + app.get("/:owner/:repo/raw/:branch/:path{.*}", async (c) => { + const { owner, repo, branch, path } = c.req.param(); + return redirectTo(c, `https://raw.githubusercontent.com/${owner}/${repo}/${branch}/${path}`); + }); + + // This is not a native GitHub route, but instead it lets us render blob + // content whose raw headers prevent hotlinking. + app.get("/:owner/:repo/embed/:type/:branch/:path{.*}", async (c) => { + const { owner, repo, type, branch, path } = c.req.param(); + + if (type !== "pdf") { + return c.notFound(); + } + + let response: Response; + try { + response = await getGitHubRaw(owner, repo, branch, path); + } catch (e) { + c.status(e instanceof GitHubHTTPError ? (e.status as StatusCode) : 500); + return c.body(null); + } + + let contentType = response.headers.get("Content-Type"); + if (contentType === null || contentType === "application/octet-stream") { + contentType = "application/pdf"; + } + + return new Response(response.body, { headers: { "Content-Type": contentType } }); + }); + + // GitHub's own wiki URLs, in the order it resolves them: names beginning with + // an underscore are reserved for its actions (_pages, _new, _edit, ...), so a + // page can never be mistaken for one. + app.get("/:owner/:repo/wiki", async (c) => { + const { owner, repo } = c.req.param(); + return c.render("wiki.eta", await getGitHubWiki(owner, repo)); + }); + + app.get("/:owner/:repo/wiki/_pages", async (c) => { + const { owner, repo } = c.req.param(); + return c.render("wikiPages.eta", await getGitHubWikiPages(owner, repo)); + }); + + // An older revision of a page renders exactly like its current one. + app.get("/:owner/:repo/wiki/:page/:oid", async (c) => { + const { owner, repo, page, oid } = c.req.param(); + if (page.startsWith("_") || !/^[0-9a-f]{40}$/.test(oid)) { + return redirectToGitHub(c); + } + return c.render("wiki.eta", await getGitHubWiki(owner, repo, page, oid)); + }); + + app.get("/:owner/:repo/wiki/:page", async (c) => { + const { owner, repo, page } = c.req.param(); + if (page.startsWith("_")) { + return redirectToGitHub(c); + } + return c.render("wiki.eta", await getGitHubWiki(owner, repo, page)); + }); + + app.get("/:owner/:repo/issues", async (c) => { + const { owner, repo } = c.req.param(); + const q = c.req.query("q"); + + // GitHub supports combined issue/PR search, apparently for legacy. We just + // redirect to issue-only search. + if (q && !/\bis:issue\b/.test(q)) { + return c.redirect(`/${owner}/${repo}/issues?q=${encodeURIComponent(`is:issue ${q}`)}`); + } + + return c.render("issues.eta", await getGitHubIssues(owner, repo, { q })); + }); + + app.get("/:owner/:repo/pulls", async (c) => { + const { owner, repo } = c.req.param(); + const q = c.req.query("q"); + return c.render("pulls.eta", await getGitHubPulls(owner, repo, { q })); + }); + + app.get("/:owner/:repo/search", async (c) => { + const { owner, repo } = c.req.param(); + const q = c.req.query("q") ?? ""; + const type = c.req.query("type") ?? "issues"; + + if (type === "issues") { + return c.redirect(`/${owner}/${repo}/issues?q=${encodeURIComponent(q)}`); + } else if (type === "pullrequests") { + return c.redirect(`/${owner}/${repo}/pulls?q=${encodeURIComponent(q)}`); + } + + // Redirect unhandled search types (like code, which requires sign in anyway). + return redirectToGitHub(c); + }); + + app.get("/:owner/:repo/commits/:branch/:path{.*}?", async (c) => { + const { owner, repo, branch, path = "" } = c.req.param(); + return c.render("commits.eta", await getGitHubCommits(owner, repo, branch, path)); + }); + + // Internal endpoints fetched asynchronously by client-side JS, not meant to + // be visited directly. Keep the site's initial page loads plain server- + // rendered HTML while still surfacing data (branches/tags, latest commit) + // that would otherwise require an extra scrape on every page view. + // The view and path are where the picker was opened, so each ref can link to + // the same page on another ref. + app.get("/api/:owner/:repo/refs/:type/:view/:path{.*}?", async (c) => { + const { owner, repo, type, view, path = "" } = c.req.param(); + if (type !== "branches" && type !== "tags") { + return c.notFound(); + } + if (view !== "tree" && view !== "blob" && view !== "commits") { + return c.notFound(); + } + const refs = await getGitHubRefs(owner, repo, type); + return c.render("_refs.eta", { ...refs, view, path }); + }); + + app.get("/api/:owner/:repo/latest-commit/:branch/:path{.*}?", async (c) => { + const { owner, repo, branch, path = "" } = c.req.param(); + return c.render("_latestCommit.eta", await getGitHubLatestCommit(owner, repo, branch, path)); + }); + + app.get("/api/:owner/:repo/sidebar", async (c) => { + const { owner, repo } = c.req.param(); + return c.render("_sidebar.eta", await getGitHubSidebar(owner, repo)); + }); + + app.get("/:owner/:repo/releases/tag/:tag", async (c) => { + const { owner, repo, tag } = c.req.param(); + return c.render("release.eta", await getGitHubRelease(owner, repo, tag)); + }); + + app.get("/:owner/:repo/releases", async (c) => { + const { owner, repo } = c.req.param(); + return c.render("releases.eta", await getGitHubReleases(owner, repo)); + }); + + app.all("*", async (c) => { + return redirectToGitHub(c); + }); + + return app; +} diff --git a/cgithub/src/scraper.test.ts b/cgithub/src/scraper.test.ts new file mode 100644 index 0000000..8dcb0b8 --- /dev/null +++ b/cgithub/src/scraper.test.ts @@ -0,0 +1,624 @@ +import assert from "node:assert"; +import { describe, it } from "node:test"; +import { + getGitHubBlob, + getGitHubCommits, + getGitHubIssues, + getGitHubSidebar, + getGitHubLatestCommit, + getGitHubPulls, + getGitHubRaw, + getGitHubRefs, + getGitHubRelease, + getGitHubReleases, + getGitHubRepo, + getGitHubOwner, + getGitHubTree, + getGitHubWiki, + getGitHubWikiPages, + ExternalRedirectError, + GitHubHTTPError, + InternalRedirectError, +} from "./scraper.ts"; + +describe("GitHub scraper", () => { + describe("actions/deploy-pages repository", () => { + it("should fetch repository", async () => { + const data = await getGitHubRepo("actions", "deploy-pages"); + + assert.strictEqual(data.repo.owner, "actions"); + assert.strictEqual(data.repo.name, "deploy-pages"); + assert.strictEqual(data.branch, "main"); + assert.strictEqual(data.path, "/"); + + assert.strictEqual( + data.info.description, + "GitHub Action to publish artifacts to GitHub Pages for deployments", + ); + assert.strictEqual(data.info.website, "https://pages.github.com"); + assert.strictEqual(data.info.stars?.length, 3); + assert.strictEqual(data.info.forks?.length, 3); + assert.strictEqual(data.info.numReleases, 39); + + assert.ok(data.items.length > 0); + assert.ok(typeof data.overviewHtml?.["README.md"] === "string"); + }); + + it("should fetch tree", async () => { + const data = await getGitHubTree("github", "rally", "main", "lib"); + + assert.strictEqual(data.repo.owner, "github"); + assert.strictEqual(data.repo.name, "rally"); + assert.strictEqual(data.branch, "main"); + assert.strictEqual(data.path, "lib"); + + assert.deepStrictEqual(data.items, [ + { + name: "README.md", + path: "lib/README.md", + contentType: "file", + }, + { + name: "RallyValidate.js", + path: "lib/RallyValidate.js", + contentType: "file", + }, + ]); + + assert.ok(typeof data.overviewHtml?.["README.md"] === "string"); + }); + + it("should fetch text blob", async () => { + const data = await getGitHubBlob("actions", "deploy-pages", "main", "LICENSE"); + + assert.strictEqual(data.repo.owner, "actions"); + assert.strictEqual(data.repo.name, "deploy-pages"); + assert.strictEqual(data.branch, "main"); + assert.strictEqual(data.path, "LICENSE"); + + assert.strictEqual(data.size, "1.04 KB / 21 lines / 17 loc"); + assert.strictEqual(data.language, "Text"); + assert.strictEqual(data.image, false); + + assert.strictEqual(data.textLines?.length, 21); + assert.strictEqual(data.textLines[0], "MIT License"); + assert.strictEqual(data.htmlLines?.length, 21); + assert.strictEqual(data.htmlLines[0], "MIT License"); + assert.strictEqual(data.htmlContent, null); + }); + + it("should fetch Markdown blob", async () => { + const data = await getGitHubBlob("actions", "deploy-pages", "main", "README.md"); + + assert.strictEqual(data.repo.owner, "actions"); + assert.strictEqual(data.repo.name, "deploy-pages"); + assert.strictEqual(data.branch, "main"); + assert.strictEqual(data.path, "README.md"); + + assert.strictEqual(data.size, "9.12 KB / 133 lines / 94 loc"); + assert.strictEqual(data.language, "Markdown"); + assert.strictEqual(data.image, false); + + assert.strictEqual(data.textLines?.[0], "# deploy-pages ๐"); + assert.strictEqual( + data.htmlLines?.[0], + '<span class="pl-mh"># <span class="pl-en">deploy-pages ๐</span></span>', + ); + const firstLine = + /^<article class="markdown-body entry-content container-lg" itemprop="text"><div class="markdown-heading" dir="auto"><h1 tabindex="-1" class="heading-element" dir="auto">deploy-pages ๐<\/h1>/; + assert.match(data.htmlContent ?? "", firstLine); + }); + + it("should fetch code blob", async () => { + const data = await getGitHubBlob("actions", "deploy-pages", "main", ".gitattributes"); + + assert.strictEqual(data.repo.owner, "actions"); + assert.strictEqual(data.repo.name, "deploy-pages"); + assert.strictEqual(data.branch, "main"); + assert.strictEqual(data.path, ".gitattributes"); + + assert.strictEqual(data.size, "39 Bytes / 1 lines / 1 loc"); + assert.strictEqual(data.language, "Git Attributes"); + assert.strictEqual(data.image, false); + + assert.deepStrictEqual(data.textLines, ["dist/** -diff linguist-generated=true "]); + assert.deepStrictEqual(data.htmlLines, [ + '\u003cspan class="pl-e"\u003edist\u003c/span\u003e/\u003cspan class="pl-k"\u003e**\u003c/span\u003e \u003cspan class="pl-k"\u003e-\u003c/span\u003e\u003cspan class="pl-v"\u003ediff\u003c/span\u003e \u003cspan class="pl-v"\u003elinguist-generated\u003c/span\u003e=\u003cspan class="pl-c1"\u003etrue\u003c/span\u003e ', + ]); + assert.strictEqual(data.htmlContent, null); + }); + + it("should fetch issues", async () => { + const data = await getGitHubIssues("actions", "deploy-pages"); + + assert.strictEqual(data.repo.owner, "actions"); + assert.strictEqual(data.repo.name, "deploy-pages"); + assert.strictEqual(data.q, undefined); + + const issue402 = data.issues.find((issue) => issue.number === 402); + assert.ok(issue402); + assert.strictEqual(issue402.title, "Dry Run"); + assert.strictEqual(issue402.state, "OPEN"); + assert.strictEqual(issue402.createdAt, "2025-07-03T19:07:08Z"); + }); + + it("should fetch filtered issues with q", async () => { + const data = await getGitHubIssues("actions", "deploy-pages", { + q: "is:issue state:closed", + }); + + assert.strictEqual(data.repo.owner, "actions"); + assert.strictEqual(data.repo.name, "deploy-pages"); + assert.strictEqual(data.q, "is:issue state:closed"); + const issue417 = data.issues.find((issue) => issue.number === 417); + assert.ok(issue417); + assert.strictEqual(issue417.title, "Chats"); + assert.ok(data.issues.every((issue) => issue.title !== "")); + }); + + it("should fetch pull requests", async () => { + const data = await getGitHubPulls("actions", "deploy-pages", { q: "state:closed" }); + + assert.strictEqual(data.repo.owner, "actions"); + assert.strictEqual(data.repo.name, "deploy-pages"); + const pr411 = data.pulls.find((pull) => pull.number === 411); + assert.ok(pr411); + assert.strictEqual(pr411.title, "update node version"); + }); + + it("should fetch commits", async () => { + const data = await getGitHubCommits("actions", "deploy-pages", "main", ""); + + assert.strictEqual(data.repo.owner, "actions"); + assert.strictEqual(data.repo.name, "deploy-pages"); + assert.strictEqual(data.branch, "main"); + assert.strictEqual(data.path, ""); + + assert.ok(data.commitGroups.length > 0); + const firstGroup = data.commitGroups[0]; + assert.ok(firstGroup.title.length > 0); + assert.ok(firstGroup.commits.length > 0); + + const firstCommit = firstGroup.commits[0]; + assert.ok(firstCommit.oid.length === 40); + assert.ok(firstCommit.shortMessage.length > 0); + assert.ok(firstCommit.authors.length > 0); + }); + + it("should fetch commits with path", async () => { + const data = await getGitHubCommits("actions", "deploy-pages", "main", "src"); + + assert.strictEqual(data.repo.owner, "actions"); + assert.strictEqual(data.repo.name, "deploy-pages"); + assert.strictEqual(data.branch, "main"); + assert.strictEqual(data.path, "src"); + + assert.ok(data.commitGroups.length > 0); + }); + + // GitHub serves a file's history from the same route as a directory's. + it("should fetch the history of a blob", async () => { + const data = await getGitHubCommits("actions", "deploy-pages", "main", "src/index.js"); + + assert.strictEqual(data.path, "src/index.js"); + + assert.ok(data.commitGroups.length > 0); + const commit = data.commitGroups[0].commits[0]; + assert.strictEqual(commit.oid.length, 40); + assert.ok(commit.shortMessage.length > 0); + }); + + // What a history page's "Browse" link resolves to: the file as it stood at + // one of its commits. Nothing in a commits payload says whether the path is + // a file or a directory, so those links all point at the tree and rely on + // GitHub redirecting a file's to its blob. + it("should redirect a blob browsed at a commit to its blob view", async () => { + const oid = "b39c421b98f49d83ae50ec502c3ddfc3bf28f2c6"; + + await assert.rejects(getGitHubTree("actions", "deploy-pages", oid, "src/index.js"), (err) => { + assert(err instanceof InternalRedirectError, "error should be an InternalRedirectError"); + assert.strictEqual(err.location, `/actions/deploy-pages/blob/${oid}/src/index.js`); + return true; + }); + + const data = await getGitHubBlob("actions", "deploy-pages", oid, "src/index.js"); + + assert.strictEqual(data.branch, oid); + assert.strictEqual(data.path, "src/index.js"); + assert.strictEqual(data.size, "1.37 KB / 51 lines / 40 loc"); + assert.strictEqual(data.textLines?.length, 51); + }); + + it("should fetch releases", async () => { + const data = await getGitHubReleases("actions", "deploy-pages"); + + assert.strictEqual(data.repo.owner, "actions"); + assert.strictEqual(data.repo.name, "deploy-pages"); + + assert.ok(data.releases.length > 0); + const firstRelease = data.releases[0]; + assert.ok(firstRelease.tagName.length > 0); + assert.ok(firstRelease.title.match(/^v\d/)); + assert.ok(firstRelease.publishedAt.length > 0); + assert.ok(firstRelease.bodyHtml.length > 0); + }); + + it("should fetch latest commit for a path", async () => { + const data = await getGitHubLatestCommit("actions", "deploy-pages", "main", "src"); + + assert.strictEqual(data.repo.owner, "actions"); + assert.strictEqual(data.repo.name, "deploy-pages"); + assert.ok(data.commit); + assert.strictEqual(data.commit.oid.length, 40); + assert.ok(data.commit.shortMessageHtml.length > 0); + assert.ok(data.commit.date.length > 0); + assert.ok(data.commit.authors.length > 0); + }); + + it("should fetch the sidebar", async () => { + const data = await getGitHubSidebar("actions", "deploy-pages"); + + assert.strictEqual(data.repo.owner, "actions"); + assert.strictEqual(data.repo.name, "deploy-pages"); + + assert.deepStrictEqual(data.languages, [ + { name: "JavaScript", percentage: 100, color: "#f1e05a" }, + ]); + + assert.strictEqual(data.latestRelease?.tag, "v5.0.1"); + assert.ok(data.latestRelease?.publishedAt?.startsWith("2026-09-01")); + }); + + it("should fetch the sidebar for a multi-language repository", async () => { + const data = await getGitHubSidebar("rust-lang", "rust"); + + const rust = data.languages.find((language) => language.name === "Rust"); + assert.ok(rust); + assert.ok(rust.percentage > 50); + assert.strictEqual(rust.color, "#dea584"); + + // The aggregate bucket comes back without a name. + const other = data.languages.find((language) => language.name === "Other"); + assert.ok(other); + + const total = data.languages.reduce((sum, language) => sum + language.percentage, 0); + assert.ok(Math.abs(total - 100) < 1); + }); + + it("should fetch branches", async () => { + const data = await getGitHubRefs("actions", "deploy-pages", "branches"); + + assert.strictEqual(data.repo.owner, "actions"); + assert.strictEqual(data.repo.name, "deploy-pages"); + assert.strictEqual(data.type, "branches"); + + assert.ok(data.refs.length > 0); + const main = data.refs.find((ref) => ref.name === "main"); + assert.ok(main); + assert.strictEqual(main.isDefault, true); + }); + + it("should fetch tags", async () => { + const data = await getGitHubRefs("actions", "deploy-pages", "tags"); + + assert.strictEqual(data.repo.owner, "actions"); + assert.strictEqual(data.repo.name, "deploy-pages"); + assert.strictEqual(data.type, "tags"); + + assert.ok(data.refs.length > 0); + const tag = data.refs.find((ref) => ref.name === "v5.0.0"); + assert.ok(tag); + assert.ok(tag.date?.length); + assert.strictEqual(data.hasMore, true); + }); + + it("should fetch single release", async () => { + const data = await getGitHubRelease("actions", "deploy-pages", "v4.0.5"); + + assert.strictEqual(data.repo.owner, "actions"); + assert.strictEqual(data.repo.name, "deploy-pages"); + + assert.ok(data.release.tagName.length > 0); + assert.ok(data.release.title.length > 0); + assert.ok(data.release.publishedAt.length > 0); + assert.ok(data.release.bodyHtml.length > 0); + assert.ok(data.release.assets && data.release.assets.length >= 2); + assert.strictEqual(data.release.assets[0].name, "Source code (zip)"); + assert.strictEqual( + data.release.assets[0].url, + "https://github.com/actions/deploy-pages/archive/refs/tags/v4.0.5.zip", + ); + }); + }); + + describe("dustinlyons/nixos-config repository", () => { + it("should fetch nix blob with combined overlapping styling directives", async () => { + const data = await getGitHubBlob("dustinlyons", "nixos-config", "main", "flake.nix"); + + assert.strictEqual(data.repo.owner, "dustinlyons"); + assert.strictEqual(data.repo.name, "nixos-config"); + assert.strictEqual(data.path, "flake.nix"); + assert.strictEqual(data.image, false); + assert.strictEqual(data.htmlContent, null); + assert.ok(data.htmlLines !== null && data.htmlLines.length > 0); + + // Line 2 has directives [2,13,"pl-c1"] and [2,13,"pl-s1"] with identical ranges. + // They should be combined into one span, not duplicated. + const line2 = data.htmlLines[1]; + assert.ok(line2.includes('class="pl-c1 pl-s1"'), `expected combined classes in: ${line2}`); + assert.ok( + (line2.match(/description/g) ?? []).length === 1, + `expected "description" to appear once in: ${line2}`, + ); + }); + }); + + describe("rich file types", () => { + it("should support PNG", async () => { + const data = await getGitHubBlob("github", "docs", "main", "assets/images/site/logo.png"); + + assert.strictEqual(data.repo.owner, "github"); + assert.strictEqual(data.repo.name, "docs"); + assert.strictEqual(data.branch, "main"); + assert.strictEqual(data.path, "assets/images/site/logo.png"); + + assert.strictEqual(data.size, "1.26 KB"); + assert.strictEqual(data.language, null); + assert.strictEqual(data.image, true); + assert.strictEqual(data.renderFileType, null); + assert.strictEqual(data.textLines, null); + assert.strictEqual(data.htmlLines, null); + assert.strictEqual(data.htmlContent, null); + }); + + // GitHub renders a PDF with a viewer of its own, so the page carries no + // content for it beyond the file type. + it("should support PDF", async () => { + const data = await getGitHubBlob("mozilla", "pdf.js", "master", "test/pdfs/basicapi.pdf"); + + assert.strictEqual(data.repo.owner, "mozilla"); + assert.strictEqual(data.repo.name, "pdf.js"); + assert.strictEqual(data.branch, "master"); + assert.strictEqual(data.path, "test/pdfs/basicapi.pdf"); + + assert.strictEqual(data.size, "103 KB"); + assert.strictEqual(data.language, null); + assert.strictEqual(data.image, false); + assert.strictEqual(data.renderFileType, "pdf"); + assert.strictEqual(data.textLines, null); + assert.strictEqual(data.htmlLines, null); + assert.strictEqual(data.htmlContent, null); + }); + + // Why the embed route exists rather than the blob view pointing a viewer + // at GitHub: as application/octet-stream a browser downloads the file + // instead of displaying it. + it("should fetch a raw PDF that GitHub types as a download", async () => { + const response = await getGitHubRaw("mozilla", "pdf.js", "master", "test/pdfs/basicapi.pdf"); + + assert.strictEqual(response.headers.get("Content-Type"), "application/octet-stream"); + assert.ok((await response.text()).startsWith("%PDF-")); + }); + + // A binary file GitHub has no preview for: no content of any kind, and the + // path exercises escaping of a space. + it("should support an unpreviewable binary file", async () => { + const data = await getGitHubBlob("justrajdeep", "fonts", "master", "Times New Roman.ttf"); + + assert.strictEqual(data.repo.owner, "justrajdeep"); + assert.strictEqual(data.repo.name, "fonts"); + assert.strictEqual(data.branch, "master"); + assert.strictEqual(data.path, "Times New Roman.ttf"); + + assert.strictEqual(data.size, "815 KB"); + assert.strictEqual(data.language, null); + assert.strictEqual(data.image, false); + assert.strictEqual(data.renderFileType, null); + assert.strictEqual(data.textLines, null); + assert.strictEqual(data.htmlLines, null); + assert.strictEqual(data.htmlContent, null); + }); + }); + + describe("wikis", () => { + it("should fetch the home page a wiki serves off /wiki itself", async () => { + const data = await getGitHubWiki("microsoft", "vscode"); + + assert.strictEqual(data.repo.owner, "microsoft"); + assert.strictEqual(data.repo.name, "vscode"); + assert.strictEqual(data.title, "Home"); + assert.strictEqual(data.updatedBy, "Greg Van Liew"); + assert.strictEqual(data.updatedAt, "2019-11-22T03:08:32Z"); + + assert.match(data.bodyHtml, /Welcome to the Visual Studio Code Wiki/); + // The _Sidebar and _Footer pages this wiki defines. + assert.match(data.sidebarHtml ?? "", /href="\/microsoft\/vscode\/wiki\/Roadmap"/); + assert.match(data.footerHtml ?? "", /Want to contribute to this Wiki\?/); + }); + + it("should fetch a page by its slug", async () => { + const data = await getGitHubWiki("microsoft", "vscode", "Commit-Signing"); + + assert.strictEqual(data.title, "Commit Signing"); + assert.match(data.bodyHtml, /GPG/); + }); + + // The slug arrives from our route already decoded, so it has to be encoded + // again rather than interpolated into the URL as-is. + it("should fetch a page whose slug is not path-safe", async () => { + const data = await getGitHubWiki( + "microsoft", + "vscode", + "[DEV]-Perf-Tools-for-VS-Code-Development", + ); + + assert.strictEqual(data.title, "[DEV] Perf Tools for VS Code Development"); + assert.ok(data.bodyHtml.length > 0); + }); + + it("should fetch an older revision of a page", async () => { + const oid = "f1c60035ba38b68ba383f58e482c312218daf86c"; + const data = await getGitHubWiki("microsoft", "vscode", "Commit-Signing", oid); + + assert.strictEqual(data.title, "Commit Signing"); + assert.strictEqual(data.updatedBy, "Joaqu\u00edn Ruales"); + assert.strictEqual(data.updatedAt, "2025-10-17T15:07:14Z"); + }); + + // The sidebar list is complete; its "Show N more pagesโฆ" button only + // unhides the rest, so no second request is needed to link every page. + it("should collect every page from the sidebar of any page", async () => { + const data = await getGitHubWiki("microsoft", "vscode"); + + assert.ok(data.pages.length > 15); + assert.deepStrictEqual(data.pages[0], { name: "Home", href: "/microsoft/vscode/wiki" }); + assert.ok(data.pages.some((page) => page.href === "/microsoft/vscode/wiki/Commit-Signing")); + }); + + // An archived repository, so neither its wiki nor these dates can move. + it("should fetch the page index with its last-updated dates", async () => { + const data = await getGitHubWikiPages("ariya", "phantomjs"); + + assert.strictEqual(data.repo.owner, "ariya"); + assert.ok(data.pages.length > 1); + assert.ok(data.pages.every((page) => page.updatedAt)); + + const home = data.pages.find((page) => page.name === "Home"); + assert.ok(home); + assert.strictEqual(home.href, "/ariya/phantomjs/wiki"); + assert.strictEqual(home.updatedAt, "2018-02-08T18:57:56Z"); + }); + + it("should redirect to the repository when it has no wiki", async () => { + await assert.rejects(getGitHubWiki("git", "git"), (err) => { + assert(err instanceof InternalRedirectError, "error should be an InternalRedirectError"); + assert.strictEqual(err.location, "/git/git"); + return true; + }); + }); + }); + + describe("owner pages", () => { + it("should fetch user profile", async () => { + const data = await getGitHubOwner("torvalds"); + + assert.strictEqual(data.type, "user"); + assert.strictEqual(data.login, "torvalds"); + assert.strictEqual(data.name, "Linus Torvalds"); + assert.strictEqual(data.company, "Linux Foundation"); + assert.strictEqual(data.location, "Portland, OR"); + assert.ok(data.avatarUrl?.startsWith("https://avatars.githubusercontent.com/")); + assert.ok(data.followers); + assert.ok(data.following !== null); + + assert.ok(data.pinned.length > 0); + const linux = data.pinned.find((repo) => repo.name === "linux"); + assert.ok(linux); + assert.strictEqual(linux.owner, "torvalds"); + assert.strictEqual(linux.description, "Linux kernel source tree"); + assert.strictEqual(linux.language, "C"); + assert.ok(linux.stars); + assert.ok(linux.forks); + + assert.deepStrictEqual(data.repos, []); + const navTexts = data.nav.map((item) => item.text); + assert.deepStrictEqual(navTexts, ["Repositories", "Projects", "Packages"]); + const reposTab = data.nav.find((item) => item.text === "Repositories"); + assert.strictEqual(reposTab?.href, "/torvalds?tab=repositories"); + }); + + it("should fetch user bio and links", async () => { + const data = await getGitHubOwner("sindresorhus"); + + assert.strictEqual(data.login, "sindresorhus"); + assert.ok(data.bio && data.bio.length > 0); + assert.ok(data.links.length > 0); + assert.ok(data.links.every((link) => link.href.startsWith("https://"))); + assert.ok(data.nav.some((item) => item.text === "Sponsoring")); + }); + + it("should fetch organization profile", async () => { + const data = await getGitHubOwner("github"); + + assert.strictEqual(data.type, "organization"); + assert.strictEqual(data.login, "github"); + assert.strictEqual(data.name, "GitHub"); + assert.strictEqual(data.bio, "How people build software."); + assert.strictEqual(data.location, "United States of America"); + assert.ok(data.avatarUrl?.startsWith("https://avatars.githubusercontent.com/")); + assert.ok(data.followers); + assert.strictEqual(data.following, null); + assert.ok(data.links.some((link) => link.href === "https://github.com/about")); + + assert.ok(data.pinned.length > 0); + assert.ok(data.pinned.length <= 6, "pinned should not include the repository list"); + assert.ok(data.pinned.every((repo) => repo.owner === "github")); + assert.ok(data.pinned.some((repo) => repo.stars && repo.forks)); + + assert.ok(data.repos.length > 0); + assert.ok(data.repos.every((repo) => repo.owner === "github")); + + const navTexts = data.nav.map((item) => item.text); + assert.deepStrictEqual(navTexts, [ + "Repositories", + "Projects", + "Packages", + "People", + "Sponsoring", + ]); + const reposTab = data.nav.find((item) => item.text === "Repositories"); + assert.strictEqual(reposTab?.href, "/orgs/github/repositories"); + }); + + it("should redirect to GitHub for non-profile pages", async () => { + await assert.rejects(getGitHubOwner("features"), (err) => { + assert(err instanceof ExternalRedirectError, "error should be an ExternalRedirectError"); + assert.strictEqual(err.location, "https://github.com/features"); + return true; + }); + }); + + it("should throw for non-existent owner", async () => { + await assert.rejects(getGitHubOwner("nosuchuserzzzz"), (err) => { + assert(err instanceof GitHubHTTPError, "error should be an HTTPError"); + assert.strictEqual(err.status, 404); + return true; + }); + }); + }); + + describe("error handling", () => { + it("should throw for non-existent repository", async () => { + await assert.rejects(getGitHubRepo("nosuchowner", "nosuchrepo"), (err) => { + assert(err instanceof GitHubHTTPError, "error should be an HTTPError"); + assert.strictEqual(err.status, 404); + return true; + }); + }); + + it("should redirect when fetching a directory as a blob", async () => { + await assert.rejects( + getGitHubBlob("earendil-works", "pi", "main", "packages/agent"), + (err) => { + assert(err instanceof InternalRedirectError, "error should be an InternalRedirectError"); + assert.strictEqual(err.location, "/earendil-works/pi/tree/main/packages/agent"); + return true; + }, + ); + }); + + it("should redirect when fetching a file as a tree", async () => { + await assert.rejects( + getGitHubTree("earendil-works", "pi", "main", "packages/agent/package.json"), + (err) => { + assert(err instanceof InternalRedirectError, "error should be an InternalRedirectError"); + assert.strictEqual( + err.location, + "/earendil-works/pi/blob/main/packages/agent/package.json", + ); + return true; + }, + ); + }); + }); +}); diff --git a/cgithub/src/scraper.ts b/cgithub/src/scraper.ts new file mode 100644 index 0000000..fb48700 --- /dev/null +++ b/cgithub/src/scraper.ts @@ -0,0 +1,1228 @@ +import { parseDocument } from "htmlparser2"; +import { isText, type AnyNode, type Element } from "domhandler"; +import { getAttributeValue, getInnerHTML, textContent } from "domutils"; +import * as cssSelect from "css-select"; + +export class GitHubHTTPError extends Error { + status: number; + + constructor(status: number, message: string) { + super(message); + this.name = "GitHubHTTPError"; + this.status = status; + } +} + +export abstract class RedirectError extends Error { + location: string; + + constructor(location: string) { + super(`Redirect to ${location}`); + this.name = this.constructor.name; + this.location = location; + } +} + +export class InternalRedirectError extends RedirectError {} + +export class ExternalRedirectError extends RedirectError {} + +interface TreeItem { + contentType: "directory" | "file"; + name: string; + path: string; +} + +interface OverviewFile { + displayName: string; + // null when GitHub includes the file without rendered content. + richText?: string | null; +} + +interface IssueNode { + number: number; + title?: string; + titleHtml?: string; + author?: { + login: string; + }; + createdAt: string; + state: string; + labels?: { + edges: { + node: { + name: string; + }; + }[]; + }; +} + +interface IssueIndexPageQuery { + queryName: "IssueIndexPageQuery"; + result: { + data: { + repository: { + search: { + edges: { + node: IssueNode; + }[]; + }; + }; + }; + }; +} + +type StylingDirective = [number, number, string]; + +export interface Commit { + oid: string; + url: string; + authoredDate: string; + committedDate: string; + shortMessage: string; + bodyMessageHtml: string; + authors: { + login: string; + displayName: string; + avatarUrl: string; + path: string; + }[]; +} + +export interface CommitGroup { + title: string; + commits: Commit[]; +} + +interface CodeViewLayoutRoute { + repo: { + ownerLogin: string; + name: string; + }; + refInfo: { + name: string; + }; + path: string; +} + +interface CodeViewRepoRoute { + tree: { + items: TreeItem[]; + }; + overview?: { + overviewFiles?: OverviewFile[]; + }; +} + +interface CodeViewTreeRoute { + tree: { + items: TreeItem[]; + readme?: OverviewFile; + }; +} + +interface BlobHeaderInfo { + blobSize: string; + lineInfo?: { + truncatedLoc: string | null; + truncatedSloc: string | null; + }; +} + +interface CodeViewBlobLayoutRoute { + blob: { + headerInfo: BlobHeaderInfo; + language: string | null; + image: boolean; + }; +} + +interface CodeViewBlobLayoutRouteStyledBlob { + rawLines: string[] | null; + colorizedLines: string[] | null; + stylingDirectives: StylingDirective[][] | null; +} + +interface CommitsRefRoute { + commitGroups: CommitGroup[]; +} + +interface CodeViewBlobRoute { + richText: string | null; + renderedFileInfo?: { renderFileType: string | null } | null; +} + +interface SidebarAbout { + description?: string | null; + website?: string | null; + stargazerCount?: number; + forksCount?: number; + // A section is false when the repository has none of that thing. + sections?: { releases?: { releaseCount?: number } | false }; +} + +interface AppPayload { + sidebarAbout?: SidebarAbout; + codeViewLayoutRoute?: CodeViewLayoutRoute; + codeViewRepoRoute?: CodeViewRepoRoute; + codeViewTreeRoute?: CodeViewTreeRoute; + codeViewBlobLayoutRoute?: CodeViewBlobLayoutRoute; + "codeViewBlobLayoutRoute.StyledBlob"?: CodeViewBlobLayoutRouteStyledBlob; + codeViewBlobRoute?: CodeViewBlobRoute; + commitsRefRoute?: CommitsRefRoute; + // Actually there may be other query types. + preloadedQueries?: IssueIndexPageQuery[]; +} + +function escapeHtml(text: string): string { + return text + .replace(/&/g, "&") + .replace(/</g, "<") + .replace(/>/g, ">") + .replace(/"/g, """) + .replace(/'/g, "'"); +} + +// Renders directives[i..] within [rangeStart, rangeEnd], producing nested spans. +// Returns the rendered content and the index of the first directive not consumed. +function renderRange( + line: string, + directives: StylingDirective[], + i: number, + rangeStart: number, + rangeEnd: number, +): [string, number] { + let result = ""; + let pos = rangeStart; + + while (i < directives.length) { + const [start, end, className] = directives[i]; + // Stop at directives that start outside or extend past this range. + if (start >= rangeEnd || end > rangeEnd) break; + + // Combine classes for directives with identical ranges. + let classes = className; + let j = i + 1; + while (j < directives.length && directives[j][0] === start && directives[j][1] === end) { + classes += " " + directives[j][2]; + j++; + } + + result += escapeHtml(line.substring(pos, start)); + const [innerContent, nextI] = renderRange(line, directives, j, start, end); + result += `<span class="${classes}">${innerContent}</span>`; + pos = end; + i = nextI; + } + + result += escapeHtml(line.substring(pos, rangeEnd)); + return [result, i]; +} + +function applyStyling(line: string, directives: StylingDirective[]): string { + // Sort by start ascending, then end descending so outer spans precede nested ones. + const sorted = [...directives].sort((a, b) => (a[0] !== b[0] ? a[0] - b[0] : b[1] - a[1])); + const [result] = renderRange(line, sorted, 0, 0, line.length); + return result; +} + +interface GitHubCommon { + repo: { + owner: string; + name: string; + }; +} + +interface GitHubNav extends GitHubCommon { + branch: string; + path: string; +} + +export interface GitHubTree extends GitHubNav { + items: TreeItem[]; + overviewHtml?: Record<string, string | null>; +} + +export interface RepoInfo { + description: string | null; + website: string | null; + stars: string | null; + forks: string | null; + numReleases?: number; +} + +export interface GitHubRepo extends GitHubTree { + info: RepoInfo; +} + +export interface GitHubBlob extends GitHubNav { + language: string | null; + size: string; + image: boolean; + renderFileType: string | null; + textLines: string[] | null; + htmlLines: string[] | null; + htmlContent: string | null; +} + +interface Issue { + number: number; + title: string; + state: string; + createdAt: string; + numReplies?: number; + numPRs?: number; +} + +export interface GitHubIssues extends GitHubCommon { + issues: Issue[]; + q?: string; +} + +export interface GitHubPulls extends GitHubCommon { + pulls: Issue[]; + q?: string; +} + +export interface GitHubCommits extends GitHubNav { + commitGroups: CommitGroup[]; +} + +export interface LatestCommit { + oid: string; + date: string; + shortMessageHtml: string; + authors: { + login: string; + displayName: string; + avatarUrl: string; + }[]; +} + +export interface GitHubLatestCommit extends GitHubCommon { + commit: LatestCommit; +} + +export interface Language { + // "Other" for the aggregate bucket GitHub reports without a name. + name: string; + percentage: number; + color: string | null; +} + +export interface LatestRelease { + tag: string; + publishedAt: string | null; +} + +export interface GitHubSidebar extends GitHubCommon { + languages: Language[]; + latestRelease: LatestRelease | null; +} + +export interface Ref { + name: string; + isDefault?: boolean; + date?: string; + authorLogin?: string; +} + +export interface GitHubRefs extends GitHubCommon { + type: "branches" | "tags"; + refs: Ref[]; + hasMore: boolean; +} + +interface Release { + tagName: string; + title: string; + publishedAt: string; + bodyHtml: string; + assets?: ReleaseAsset[]; +} + +export interface ReleaseAsset { + name: string; + url: string; + size?: string; + publishedAt?: string; +} + +export interface GitHubReleases extends GitHubCommon { + releases: Release[]; +} + +export interface GitHubRelease extends GitHubCommon { + release: Release; +} + +export interface WikiPage { + name: string; + href: string; + updatedAt?: string; +} + +export interface GitHubWiki extends GitHubCommon { + title: string; + updatedBy: string | null; + updatedAt: string | null; + bodyHtml: string; + sidebarHtml: string | null; + footerHtml: string | null; + pages: WikiPage[]; +} + +export interface GitHubWikiPages extends GitHubCommon { + pages: WikiPage[]; +} + +interface OwnerRepo { + owner: string; + name: string; + description?: string; + language?: string; + stars?: string; + forks?: string; +} + +interface OwnerNavItem { + text: string; + href: string; +} + +export interface GitHubOwner { + type: "user" | "organization"; + login: string; + name: string | null; + bio: string | null; + avatarUrl: string | null; + followers: string | null; + // Users only; always null for organizations. + following: string | null; + company: string | null; + location: string | null; + links: { text: string; href: string }[]; + nav: OwnerNavItem[]; + pinned: OwnerRepo[]; + // The repository list on organization overviews; empty for users. + repos: OwnerRepo[]; +} + +// GitHub throttles/blocks requests without realistic browser headers. +// These headers make the request appear as a standard browser visit. +function browserHeaders(accept: string): Record<string, string> { + return { + "User-Agent": + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", + Accept: accept, + "Accept-Language": "en-US,en;q=0.5", + }; +} + +async function fetchGitHubResponse(path: string, accept: string): Promise<Response> { + const headers = browserHeaders(accept); + const response = await fetch(`https://github.com/${path}`, { headers, redirect: "manual" }); + if (response.status === 301 || response.status === 302) { + const location = response.headers.get("location"); + if (location) { + const url = new URL(location); + if (url.hostname === "github.com") { + throw new InternalRedirectError(url.pathname); + } + } + } + if (!response.ok) throw new GitHubHTTPError(response.status, response.statusText); + return response; +} + +async function fetchGitHubPage(path: string): Promise<string> { + const response = await fetchGitHubResponse( + path, + "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", + ); + return response.text(); +} + +// Some routes (e.g. latest-commit) are the same endpoints the React UI itself +// calls to lazily fill in data; they 406 without an explicit JSON Accept. +async function fetchGitHubJson<T>(path: string): Promise<T> { + const response = await fetchGitHubResponse(path, "application/json"); + return (await response.json()) as T; +} + +function parseEmbeddedPayload<T>(html: string): T | null { + const regex = new RegExp( + `<script type="application/json" data-target="react-app.embeddedData">([^<]+)</script>`, + "g", + ); + + match: for (const match of html.matchAll(regex)) { + let data; + try { + data = JSON.parse(match[1]); + } catch { + continue; + } + if (data.payload === undefined) continue match; + return data.payload; + } + return null; +} + +function parsePayload(html: string): AppPayload | null { + return parseEmbeddedPayload<AppPayload>(html); +} + +function extractOverviewHtml( + overviewFiles: OverviewFile[] = [], +): Record<string, string | null> | undefined { + const result: Record<string, string | null> = {}; + for (const file of overviewFiles) { + if (file.richText !== undefined) { + result[file.displayName] = file.richText; + } + } + return Object.keys(result).length > 0 ? result : undefined; +} + +function extractGitHub<T>(payload: AppPayload | null, extra: T): GitHubNav & T { + const layoutRoute = payload?.codeViewLayoutRoute; + if (layoutRoute === undefined) { + throw new Error("Missing codeViewLayoutRoute"); + } + + return { + repo: { + owner: layoutRoute.repo.ownerLogin, + name: layoutRoute.repo.name, + }, + branch: layoutRoute.refInfo.name, + path: layoutRoute.path, + ...extra, + }; +} + +export async function getGitHubRepo(owner: string, repo: string): Promise<GitHubRepo> { + const html = await fetchGitHubPage(`${owner}/${repo}`); + + const payload = parsePayload(html); + const repoRoute = payload?.codeViewRepoRoute; + + if (repoRoute?.tree === undefined) { + throw new Error("Could not find tree data in embedded JSON"); + } + + const about = payload?.sidebarAbout; + const releases = about?.sections?.releases; + const info = { + description: about?.description ?? null, + website: about?.website ?? null, + stars: about?.stargazerCount?.toLocaleString("en-US") ?? null, + forks: about?.forksCount?.toLocaleString("en-US") ?? null, + numReleases: releases === false ? undefined : releases?.releaseCount, + }; + const overviewHtml = extractOverviewHtml(repoRoute.overview?.overviewFiles); + + return extractGitHub(payload, { items: repoRoute.tree.items, info, overviewHtml }); +} + +export async function getGitHubTree( + owner: string, + repo: string, + branch: string, + path: string, +): Promise<GitHubTree> { + const html = await fetchGitHubPage(`${owner}/${repo}/tree/${branch}/${path}`); + + const payload = parsePayload(html); + const treeRoute = payload?.codeViewTreeRoute; + + if (treeRoute?.tree === undefined) { + throw new Error("Could not find tree data in embedded JSON"); + } + + const overviewHtml = extractOverviewHtml(treeRoute.tree.readme ? [treeRoute.tree.readme] : []); + + return extractGitHub(payload, { items: treeRoute.tree.items, overviewHtml }); +} + +export async function getGitHubBlob( + owner: string, + repo: string, + branch: string, + path: string, +): Promise<GitHubBlob> { + const html = await fetchGitHubPage(`${owner}/${repo}/blob/${branch}/${path}`); + + const payload = parsePayload(html); + const blob = payload?.codeViewBlobLayoutRoute?.blob; + const styledBlob = payload?.["codeViewBlobLayoutRoute.StyledBlob"]; + + if (blob === undefined) { + throw new Error("Could not find blob data in embedded JSON"); + } + + let htmlLines = styledBlob?.colorizedLines ?? null; + const stylingDirectives = styledBlob?.stylingDirectives; + if (!htmlLines && stylingDirectives && styledBlob?.rawLines) { + htmlLines = styledBlob.rawLines.map((line, i) => applyStyling(line, stylingDirectives[i])); + } + + let size = blob.headerInfo.blobSize; + if (blob.headerInfo.lineInfo?.truncatedLoc) { + size += ` / ${blob.headerInfo.lineInfo.truncatedLoc} lines`; + } + if (blob.headerInfo.lineInfo?.truncatedSloc) { + size += ` / ${blob.headerInfo.lineInfo.truncatedSloc} loc`; + } + + return extractGitHub(payload, { + language: blob.language || null, + size, + image: blob.image, + renderFileType: payload?.codeViewBlobRoute?.renderedFileInfo?.renderFileType ?? null, + textLines: styledBlob?.rawLines ?? null, + htmlLines, + htmlContent: payload?.codeViewBlobRoute?.richText ?? null, + }); +} + +// For proxying embeds. +export async function getGitHubRaw( + owner: string, + repo: string, + branch: string, + path: string, +): Promise<Response> { + const url = `https://raw.githubusercontent.com/${owner}/${repo}/${branch}/${path}`; + const response = await fetch(url, { headers: browserHeaders("*/*") }); + if (!response.ok) throw new GitHubHTTPError(response.status, response.statusText); + return response; +} + +export async function getGitHubIssues( + owner: string, + repo: string, + // We only support is:issue searches (no PR results). + { q }: { q?: string } = {}, +): Promise<GitHubIssues> { + const suffix = q ? `?q=${encodeURIComponent(q)}` : ""; + const html = await fetchGitHubPage(`${owner}/${repo}/issues${suffix}`); + const payload = parsePayload(html); + + let issuesQuery: IssueIndexPageQuery | undefined; + for (const query of payload?.preloadedQueries ?? []) { + if (query.queryName === "IssueIndexPageQuery") { + issuesQuery = query; + } + } + if (issuesQuery === undefined) { + throw new Error("Could not find IssueIndexPageQuery in embedded JSON"); + } + + const issues: Issue[] = issuesQuery.result.data.repository.search.edges.map((edge) => { + const { number, title, titleHtml, state, createdAt } = edge.node; + const finalTitle = (title || titleHtml || "").replace(/<[^>]+>/g, ""); + return { number, title: finalTitle, state, createdAt }; + }); + + return { repo: { owner, name: repo }, issues, q }; +} + +export async function getGitHubPulls( + owner: string, + repo: string, + { q }: { q?: string } = {}, +): Promise<GitHubPulls> { + const suffix = q ? `?q=${encodeURIComponent(q)}` : ""; + const html = await fetchGitHubPage(`${owner}/${repo}/pulls${suffix}`); + const document = parseDocument(html); + + const pulls: Issue[] = []; + for (const row of cssSelect.selectAll(".js-issue-row", document) as unknown as Element[]) { + const id = getAttributeValue(row, "id"); // "issue_NNN" + const number = id ? parseInt(id.replace("issue_", ""), 10) : 0; + if (!number) continue; + + const titleLink = cssSelect.selectOne("a.markdown-title", row); + const title = titleLink ? textContent(titleLink).trim() : ""; + + const timeElem = cssSelect.selectOne("relative-time", row); + const createdAt = timeElem ? getAttributeValue(timeElem, "datetime") || "" : ""; + + const stateSpan = cssSelect.selectOne(".tooltipped[aria-label]", row) as Element | null; + const ariaLabel = stateSpan ? (getAttributeValue(stateSpan, "aria-label") ?? "") : ""; + const state = ariaLabel.includes("Merged") + ? "MERGED" + : ariaLabel.includes("Closed") + ? "CLOSED" + : "OPEN"; + + pulls.push({ number, title, state, createdAt }); + } + + return { repo: { owner, name: repo }, pulls, q }; +} + +export async function getGitHubCommits( + owner: string, + repo: string, + branch: string, + path: string, +): Promise<GitHubCommits> { + const urlPath = path + ? `${owner}/${repo}/commits/${branch}/${path}` + : `${owner}/${repo}/commits/${branch}`; + const html = await fetchGitHubPage(urlPath); + + const payload = parsePayload(html); + const commitsRoute = payload?.commitsRefRoute; + + if (commitsRoute === undefined) { + throw new Error("Could not find commit history in embedded JSON"); + } + + return { + repo: { + owner, + name: repo, + }, + branch, + path, + commitGroups: commitsRoute.commitGroups, + }; +} + +interface LatestCommitPayload { + oid: string; + date: string; + shortMessageHtmlLink: string; + authors: { + login: string; + displayName: string; + avatarUrl: string; + }[]; +} + +// This is the same endpoint the React UI itself calls to lazily fill in the +// commit bar above a file listing, rather than the much heavier /commits +// history page: it returns just the single latest commit as JSON. +export async function getGitHubLatestCommit( + owner: string, + repo: string, + branch: string, + path: string, +): Promise<GitHubLatestCommit> { + const urlPath = path + ? `${owner}/${repo}/latest-commit/${branch}/${path}` + : `${owner}/${repo}/latest-commit/${branch}`; + const payload = await fetchGitHubJson<LatestCommitPayload>(urlPath); + + return { + repo: { owner, name: repo }, + commit: { + oid: payload.oid, + date: payload.date, + shortMessageHtml: payload.shortMessageHtmlLink, + authors: payload.authors, + }, + }; +} + +interface SidebarPayload { + languages?: { + languages: { + // Absent for the "Other" bucket. + name?: string; + percentage: number; + color?: string; + }[]; + }; + releases?: { + // The name is the release title, which need not be the tag; the path is. + latestRelease?: { name?: string; path: string; publishedAt?: string } | null; + }; +} + +// The repository sidebar (languages, releases, contributors, ...) is not +// rendered into the repo page at all; GitHub's own UI lazily fetches it as +// JSON from here. +export async function getGitHubSidebar(owner: string, repo: string): Promise<GitHubSidebar> { + const payload = await fetchGitHubJson<SidebarPayload>(`${owner}/${repo}/_sidebar`); + + const languages: Language[] = []; + for (const entry of payload.languages?.languages ?? []) { + languages.push({ + name: entry.name ?? "Other", + percentage: entry.percentage, + color: entry.color ?? null, + }); + } + + const release = payload.releases?.latestRelease; + const latestRelease = release + ? { + tag: decodeURIComponent(release.path.split("/releases/tag/")[1] ?? release.name ?? ""), + publishedAt: release.publishedAt ?? null, + } + : null; + + return { repo: { owner, name: repo }, languages, latestRelease }; +} + +interface BranchRefEntry { + name: string; + isDefault: boolean; + authoredDate?: string; + author?: { login: string }; +} + +interface BranchesPayload { + branches: { + default?: BranchRefEntry; + yours?: BranchRefEntry[]; + active?: BranchRefEntry[]; + stale?: BranchRefEntry[]; + }; + hasMore?: Record<string, boolean>; +} + +async function getGitHubBranchRefs(owner: string, repo: string): Promise<GitHubRefs> { + const html = await fetchGitHubPage(`${owner}/${repo}/branches`); + const payload = parseEmbeddedPayload<BranchesPayload>(html); + + if (payload?.branches === undefined) { + throw new Error("Could not find branches data in embedded JSON"); + } + + const { default: defaultBranch, yours = [], active = [], stale = [] } = payload.branches; + const entries = [...(defaultBranch ? [defaultBranch] : []), ...yours, ...active, ...stale]; + + const seen = new Set<string>(); + const refs: Ref[] = []; + for (const entry of entries) { + if (seen.has(entry.name)) continue; + seen.add(entry.name); + refs.push({ + name: entry.name, + isDefault: entry.isDefault || undefined, + date: entry.authoredDate, + authorLogin: entry.author?.login, + }); + } + + const hasMore = Object.values(payload.hasMore ?? {}).some(Boolean); + + return { repo: { owner, name: repo }, type: "branches", refs, hasMore }; +} + +async function getGitHubTagRefs(owner: string, repo: string): Promise<GitHubRefs> { + const html = await fetchGitHubPage(`${owner}/${repo}/tags`); + const document = parseDocument(html); + + const refs: Ref[] = []; + for (const row of cssSelect.selectAll(".Box-row", document) as unknown as Element[]) { + const nameElem = cssSelect.selectOne("h2 a", row); + if (!nameElem) continue; + const name = textContent(nameElem).trim(); + if (!name) continue; + + // The row has two <relative-time> elements: a release/tag creation + // timestamp, then the tagged commit's authored date. We want the latter. + const timeElems = cssSelect.selectAll("relative-time", row); + const timeElem = timeElems[1] as Element | undefined; + const date = timeElem ? getAttributeValue(timeElem, "datetime") || undefined : undefined; + + refs.push({ name, date }); + } + + const hasMore = cssSelect.selectOne(".pagination a", document) !== null; + + return { repo: { owner, name: repo }, type: "tags", refs, hasMore }; +} + +export async function getGitHubRefs( + owner: string, + repo: string, + type: "branches" | "tags", +): Promise<GitHubRefs> { + return type === "branches" ? getGitHubBranchRefs(owner, repo) : getGitHubTagRefs(owner, repo); +} + +function parseRelease(section: Element, title: string): Release { + // Extract tag name from link + const tagLink = cssSelect.selectOne('a[href*="/tree/"]', section); + let tagName = title; + if (tagLink) { + const href = getAttributeValue(tagLink, "href"); + if (href) { + tagName = href.split("/").pop() || title; + } + } + + // Extract published date + const relativeTime = cssSelect.selectOne("relative-time", section); + const publishedAt = relativeTime ? getAttributeValue(relativeTime, "datetime") || "" : ""; + + // Extract body HTML (markdown content) + const body = cssSelect.selectOne(".markdown-body", section); + const bodyHtml = body ? getInnerHTML(body).trim() : ""; + + return { + tagName, + title, + publishedAt, + bodyHtml, + }; +} + +export async function getGitHubReleases(owner: string, repo: string): Promise<GitHubReleases> { + const html = await fetchGitHubPage(`${owner}/${repo}/releases`); + + const document = parseDocument(html); + const releases: Release[] = []; + + // Not really sure the right way to express this type. + const sections = cssSelect.selectAll("section", document) as unknown as Element[]; + + for (const section of sections) { + const h2 = cssSelect.selectOne("h2[id]", section); + if (!h2) continue; + + const title = textContent(h2).trim(); + + releases.push(parseRelease(section, title)); + } + + return { + repo: { + owner, + name: repo, + }, + releases, + }; +} + +export async function getGitHubReleaseBase( + owner: string, + repo: string, + tag: string, +): Promise<GitHubRelease> { + const html = await fetchGitHubPage(`${owner}/${repo}/releases/tag/${tag}`); + + const document = parseDocument(html); + + const box = cssSelect.selectOne(".Box:has(h1)", document) as Element | null; + if (!box) throw new Error("Could not find release box"); + + const h1 = cssSelect.selectOne("h1", box); + const title = h1 ? textContent(h1).trim() : ""; + + const release = parseRelease(box, title); + + return { + repo: { + owner, + name: repo, + }, + release, + }; +} + +export async function getGitHubReleaseAssets( + owner: string, + repo: string, + tag: string, +): Promise<ReleaseAsset[]> { + const html = await fetchGitHubPage(`${owner}/${repo}/releases/expanded_assets/${tag}`); + const document = parseDocument(html); + const assets: ReleaseAsset[] = []; + + for (const row of cssSelect.selectAll("li", document) as unknown as Element[]) { + const link = cssSelect.selectOne("a[href]", row); + if (!link) continue; + + const url = getAttributeValue(link, "href") || ""; + // Filename is in the link, often split into multiple spans + const name = textContent(link).replace(/\s+/g, " ").trim(); + + let size: string | undefined; + let publishedAt: string | undefined; + for (const s of cssSelect.selectAll("span.color-fg-muted.text-right", row)) { + // Size is in a span with color-fg-muted and text-right, but avoid the one with relative-time + const timeElem = cssSelect.selectOne("relative-time", row); + if (timeElem) { + publishedAt = timeElem ? getAttributeValue(timeElem, "datetime") || "" : ""; + } else { + const text = textContent(s).trim(); + if (text) size = text; + } + } + + assets.push({ + name, + url: url.startsWith("/") ? `https://github.com${url}` : url, + size, + publishedAt, + }); + } + + return assets; +} + +export async function getGitHubRelease( + owner: string, + repo: string, + tag: string, +): Promise<GitHubRelease> { + const [base, assets] = await Promise.all([ + getGitHubReleaseBase(owner, repo, tag), + getGitHubReleaseAssets(owner, repo, tag), + ]); + base.release.assets = assets; + return base; +} + +function selectText(selector: string, context: AnyNode): string | null { + const elem = cssSelect.selectOne(selector, context); + if (!elem) return null; + return textContent(elem).replace(/\s+/g, " ").trim() || null; +} + +function parseRepoItems(items: Element[]): OwnerRepo[] { + const repos: OwnerRepo[] = []; + for (const item of items) { + const link = cssSelect.selectOne( + "a:has(span.repo), a[itemprop~='codeRepository']", + item, + ) as Element | null; + const href = link ? getAttributeValue(link, "href") : undefined; + if (!href) continue; + const [owner, name] = href.replace(/^\//, "").split("/"); + if (!owner || !name) continue; + + repos.push({ + owner, + name, + description: selectText("p.pinned-item-desc, [itemprop='description']", item) ?? undefined, + language: selectText("[itemprop='programmingLanguage']", item) ?? undefined, + stars: selectText("a[href$='/stargazers']", item) ?? undefined, + forks: selectText("a[href$='/forks']", item) ?? undefined, + }); + } + return repos; +} + +function parsePinnedRepos(document: AnyNode): OwnerRepo[] { + return parseRepoItems( + cssSelect.selectAll(".pinned-item-list-item", document) as unknown as Element[], + ); +} + +// The tabs we surface from the profile page's own navigation. Stars and +// Overview are deliberately excluded. +const OWNER_NAV_TABS = ["Repositories", "Projects", "Packages", "People", "Sponsoring"]; + +function parseOwnerNav(document: AnyNode): OwnerNavItem[] { + const nav: OwnerNavItem[] = []; + const seen = new Set<string>(); + const anchors = cssSelect.selectAll("a.UnderlineNav-item", document) as unknown as Element[]; + for (const anchor of anchors) { + const href = getAttributeValue(anchor, "href"); + // Label text is e.g. "Repositories 12" including the counter. + const text = textContent(anchor).replace(/\s+/g, " ").trim().split(" ")[0]; + if (!href || seen.has(href) || !OWNER_NAV_TABS.includes(text)) continue; + seen.add(href); + nav.push({ text, href }); + } + return nav; +} + +function parseUser(document: AnyNode, login: string): GitHubOwner { + const avatarElem = cssSelect.selectOne("a[itemprop='image'] img", document) as Element | null; + const avatarUrl = avatarElem ? getAttributeValue(avatarElem, "src") || null : null; + + const followers = + selectText("a[href$='tab=followers']", document)?.replace(/ followers?$/, "") ?? null; + const following = + selectText("a[href$='tab=following']", document)?.replace(/ following$/, "") ?? null; + + const links: { text: string; href: string }[] = []; + const linkItems = cssSelect.selectAll( + "li[itemprop='url'], li[itemprop='social']", + document, + ) as unknown as Element[]; + for (const item of linkItems) { + const anchor = cssSelect.selectOne("a[href]", item) as Element | null; + const href = anchor ? getAttributeValue(anchor, "href") : undefined; + if (!anchor || !href) continue; + links.push({ text: textContent(anchor).replace(/\s+/g, " ").trim(), href }); + } + + return { + type: "user", + login, + name: selectText(".p-name.vcard-fullname", document), + bio: selectText(".p-note.user-profile-bio", document), + avatarUrl, + followers, + following, + company: selectText("li[itemprop='worksFor']", document), + location: selectText("li[itemprop='homeLocation']", document), + links, + nav: parseOwnerNav(document), + pinned: parsePinnedRepos(document), + repos: [], + }; +} + +function parseOrganization(document: AnyNode, owner: string): GitHubOwner { + const avatarElem = cssSelect.selectOne( + ".orghead img[itemprop='image']", + document, + ) as Element | null; + const avatarUrl = avatarElem ? getAttributeValue(avatarElem, "src") || null : null; + // The login only appears in the avatar's "@login" alt text. + const alt = avatarElem ? getAttributeValue(avatarElem, "alt") : undefined; + const login = alt?.replace(/^@/, "") || owner; + + const followers = + selectText("a[href$='/followers']", document)?.replace(/ followers?$/, "") ?? null; + + const links: { text: string; href: string }[] = []; + const anchors = cssSelect.selectAll( + ".orghead a[itemprop='url'], .orghead [itemprop='email'] a", + document, + ) as unknown as Element[]; + for (const anchor of anchors) { + const href = getAttributeValue(anchor, "href"); + if (!href) continue; + links.push({ text: textContent(anchor).replace(/\s+/g, " ").trim(), href }); + } + + return { + type: "organization", + login, + name: selectText(".orghead h1", document), + bio: selectText(".orghead h1 + div", document), + avatarUrl, + followers, + following: null, + company: null, + location: selectText(".orghead [itemprop='location']", document), + links, + nav: parseOwnerNav(document), + pinned: parsePinnedRepos(document), + repos: parseRepoItems( + cssSelect.selectAll("#org-repositories [itemprop='owns']", document) as unknown as Element[], + ), + }; +} + +export async function getGitHubOwner(owner: string): Promise<GitHubOwner> { + const html = await fetchGitHubPage(owner); + const document = parseDocument(html); + + const login = selectText(".p-nickname.vcard-username", document); + if (login) { + return parseUser(document, login); + } + if (cssSelect.selectOne(".orghead", document)) { + return parseOrganization(document, owner); + } + // Not a profile page (marketing pages like github.com/features, etc.). + throw new ExternalRedirectError(`https://github.com/${owner}`); +} + +// The text directly inside an element, ignoring any nested elements' text. +function directText(elem: Element): string { + return elem.children + .filter(isText) + .map((node) => node.data) + .join("") + .replace(/\s+/g, " ") + .trim(); +} + +// The sidebar holds every wiki page, which are revealed by JavaScript. +function parseWikiSidebarPages(document: AnyNode): WikiPage[] { + const pages: WikiPage[] = []; + const anchors = cssSelect.selectAll( + ".js-wiki-sidebar-page-container > div > a[href]", + document, + ) as unknown as Element[]; + for (const anchor of anchors) { + const href = getAttributeValue(anchor, "href"); + const name = textContent(anchor).replace(/\s+/g, " ").trim(); + if (href && name) pages.push({ name, href }); + } + return pages; +} + +function wikiHtml(selector: string, document: AnyNode): string | null { + const elem = cssSelect.selectOne(selector, document); + if (!elem) return null; + const html = getInnerHTML(elem).trim(); + return html || null; +} + +export async function getGitHubWiki( + owner: string, + repo: string, + // Empty for the wiki's home page, which GitHub serves off /wiki itself. + page = "", + // A revision SHA renders that older version of the page. + oid = "", +): Promise<GitHubWiki> { + // Page names keep characters that are not path-safe (C#, [DEV] ...), so the + // slug has to be re-encoded rather than interpolated the way a ref is. + const path = [`${owner}/${repo}/wiki`, page && encodeURIComponent(page), oid] + .filter(Boolean) + .join("/"); + const html = await fetchGitHubPage(path); + const document = parseDocument(html); + + const heading = cssSelect.selectOne("h1.gh-header-title", document) as Element | null; + if (!heading) { + throw new Error("Could not find the wiki page heading"); + } + + // "<author> edited this page <date> ยท <n> revisions", where the author is the + // only part not wrapped in an element of its own. + const meta = cssSelect.selectOne(".gh-header-meta", document) as Element | null; + const updatedBy = meta ? (/^(.+?) edited this page\b/.exec(directText(meta))?.[1] ?? null) : null; + const time = meta ? cssSelect.selectOne("relative-time", meta) : null; + + return { + repo: { owner, name: repo }, + title: directText(heading), + updatedBy, + updatedAt: time ? getAttributeValue(time, "datetime") || null : null, + bodyHtml: wikiHtml("#wiki-body .markdown-body", document) ?? "", + // Wikis can define _Sidebar and _Footer pages, rendered alongside every page. + sidebarHtml: wikiHtml(".wiki-custom-sidebar", document), + footerHtml: wikiHtml("#wiki-footer .markdown-body", document), + pages: parseWikiSidebarPages(document), + }; +} + +export async function getGitHubWikiPages(owner: string, repo: string): Promise<GitHubWikiPages> { + const html = await fetchGitHubPage(`${owner}/${repo}/wiki/_pages`); + const document = parseDocument(html); + + const pages: WikiPage[] = []; + for (const row of cssSelect.selectAll( + "#wiki-content li.Box-row", + document, + ) as unknown as Element[]) { + const anchor = cssSelect.selectOne("a[href]", row) as Element | null; + const href = anchor ? getAttributeValue(anchor, "href") : undefined; + const name = anchor ? textContent(anchor).replace(/\s+/g, " ").trim() : ""; + if (!href || !name) continue; + + const time = cssSelect.selectOne("relative-time", row) as Element | null; + pages.push({ name, href, updatedAt: time ? getAttributeValue(time, "datetime") : undefined }); + } + + return { repo: { owner, name: repo }, pages }; +} diff --git a/cgithub/src/worker.ts b/cgithub/src/worker.ts new file mode 100644 index 0000000..e59c690 --- /dev/null +++ b/cgithub/src/worker.ts @@ -0,0 +1,30 @@ +// Cloudflare Workers entry point. Static files under public/ are served by +// Workers Static Assets before the worker runs (see wrangler.jsonc). +import { Eta } from "eta"; +import { createApp } from "./app.ts"; +import { views } from "./views.generated.ts"; + +// Workers disallow runtime code generation, so templates are precompiled by +// scripts/build.ts and preloaded into Eta's cache here. With +// cache: true, Eta serves them from the cache and never compiles. +const eta = new Eta({ cache: true }); + +// Templates name each other as siblings ("./_tree") or from the views root +// ("/layouts/base.eta"); either way the name is the path under views/ that +// scripts/build.ts keyed the template by. +eta.resolvePath = (templatePath: string) => { + let name = templatePath.replace(/^\.?\//, ""); + if (!name.endsWith(".eta")) name += ".eta"; + return name; +}; + +// Only reached on a cache miss, i.e. a template missing from the build. +eta.readFile = (path: string) => { + throw new Error(`Unknown template: ${path}`); +}; + +for (const [name, fn] of Object.entries(views)) { + eta.loadTemplate(name, fn); +} + +export default createApp(eta); |