diff options
| -rw-r--r-- | public/static/refs.js | 60 | ||||
| -rw-r--r-- | public/static/style.css | 62 | ||||
| -rw-r--r-- | src/app.ts | 46 | ||||
| -rw-r--r-- | src/scraper.test.ts | 41 | ||||
| -rw-r--r-- | src/scraper.ts | 177 | ||||
| -rw-r--r-- | views/_latestCommit.eta | 16 | ||||
| -rw-r--r-- | views/_nav.eta | 11 | ||||
| -rw-r--r-- | views/_refs.eta | 17 | ||||
| -rw-r--r-- | views/_tree.eta | 3 | ||||
| -rw-r--r-- | views/layout.eta | 1 |
10 files changed, 426 insertions, 8 deletions
diff --git a/public/static/refs.js b/public/static/refs.js new file mode 100644 index 0000000..28b8373 --- /dev/null +++ b/public/static/refs.js @@ -0,0 +1,60 @@ +// Progressive enhancement for two bits of data GitHub itself loads lazily: +// the branch/tag dropdown and the latest-commit bar above a file listing. +// Both degrade gracefully without JS: the button's command/commandfor still +// opens the (empty) dialog natively; the commit placeholder just never fills +// in. +document.addEventListener("DOMContentLoaded", () => { + const dialog = document.querySelector(".ref-selector-menu"); + if (dialog) { + const list = dialog.querySelector(".ref-selector-list"); + const { owner, repo } = dialog.dataset; + const cache = {}; + + async function load(type) { + if (Object.prototype.hasOwnProperty.call(cache, type)) { + list.innerHTML = cache[type]; + return; + } + list.textContent = "Loading…"; + try { + const res = await fetch(`/api/${owner}/${repo}/refs/${type}`); + const html = res.ok ? await res.text() : ""; + cache[type] = html || "Failed to load."; + list.innerHTML = cache[type]; + } catch { + list.textContent = "Failed to load."; + } + } + + let opened = false; + dialog.addEventListener("toggle", () => { + if (dialog.open && !opened) { + opened = true; + load("branches"); + } + }); + + for (const tab of dialog.querySelectorAll("[data-ref-type]")) { + tab.addEventListener("click", () => { + for (const other of dialog.querySelectorAll("[data-ref-type]")) { + other.classList.toggle("active", other === tab); + } + load(tab.dataset.refType); + }); + } + } + + const commitPlaceholder = document.querySelector(".latest-commit-placeholder"); + if (commitPlaceholder) { + fetch(commitPlaceholder.dataset.src) + .then((res) => (res.ok ? res.text() : "")) + .then((html) => { + if (html.trim()) { + commitPlaceholder.outerHTML = html; + } else { + commitPlaceholder.remove(); + } + }) + .catch(() => commitPlaceholder.remove()); + } +}); diff --git a/public/static/style.css b/public/static/style.css index e698993..983cb86 100644 --- a/public/static/style.css +++ b/public/static/style.css @@ -160,6 +160,68 @@ body > footer { border: 1px solid var(--border-color); } +.ref-selector-button { + anchor-name: --ref-selector-anchor; +} + +.ref-selector-menu { + position: absolute; + position-anchor: --ref-selector-anchor; + position-area: bottom span-right; + width: 16rem; + max-height: 20rem; + overflow-y: auto; + overflow-wrap: break-word; + background: var(--body-background); + border: 1px solid var(--border-color); + padding: 0.5rem; + + &::backdrop { + background: transparent; + } +} + +.ref-selector-tabs { + display: flex; + margin-bottom: 0.5rem; + + button { + flex: 1; + padding: 0.25rem; + cursor: pointer; + &.active { + font-weight: bold; + border-bottom: 2px solid var(--link-color); + } + } +} + +.ref-selector-items { + list-style: none; + margin: 0; + padding: 0; + + li { + padding: 0.2rem 0; + } +} + +.ref-selector-badge { + margin-left: 0.3rem; + font-size: 0.75rem; + opacity: 0.7; +} + +.ref-selector-empty, +.ref-selector-more { + padding: 0.2rem 0; + font-size: 0.85rem; +} + +.latest-commit-placeholder { + opacity: 0.7; +} + .repo-info { .description { margin-bottom: .2rem; @@ -5,8 +5,10 @@ import { getGitHubBlob, getGitHubCommits, getGitHubIssues, + getGitHubLatestCommit, getGitHubOwner, getGitHubPulls, + getGitHubRefs, getGitHubRelease, getGitHubReleases, getGitHubRepo, @@ -44,6 +46,29 @@ export function createApp(eta: Eta) { return c.html(eta.render(template, data)); } + // For fragments fetched asynchronously by client-side JS (see public/static/refs.js): + // errors render as an empty body rather than the full error.eta page. + async function tryRenderFragment<T extends object>( + c: Context, + template: string, + promise: Promise<T>, + ) { + let data: T; + try { + data = await promise; + } catch (e) { + const status = + e instanceof GitHubHTTPError + ? (e.status as StatusCode) + : e instanceof RedirectError + ? 404 + : 500; + c.status(status); + return c.body(null); + } + return c.html(eta.render(template, data)); + } + app.get("/:owner", async (c) => { const { owner } = c.req.param(); return tryRender(c, "owner.eta", getGitHubOwner(owner)); @@ -112,6 +137,27 @@ export function createApp(eta: Eta) { return tryRender(c, "commits.eta", getGitHubCommits(owner, repo, branch, path)); }); + // Internal endpoints fetched asynchronously by client-side JS, not meant to + // be visited directly. Keep the site's initial page loads plain server- + // rendered HTML while still surfacing data (branches/tags, latest commit) + // that would otherwise require an extra scrape on every page view. + app.get("/api/:owner/:repo/refs/:type", async (c) => { + const { owner, repo, type } = c.req.param(); + if (type !== "branches" && type !== "tags") { + return c.notFound(); + } + return tryRenderFragment(c, "_refs.eta", getGitHubRefs(owner, repo, type)); + }); + + app.get("/api/:owner/:repo/latest-commit/:branch/:path{.*}?", async (c) => { + const { owner, repo, branch, path = "" } = c.req.param(); + return tryRenderFragment( + c, + "_latestCommit.eta", + getGitHubLatestCommit(owner, repo, branch, path), + ); + }); + app.get("/:owner/:repo/releases/tag/:tag", async (c) => { const { owner, repo, tag } = c.req.param(); return tryRender(c, "release.eta", getGitHubRelease(owner, repo, tag)); diff --git a/src/scraper.test.ts b/src/scraper.test.ts index f9b60e8..56f14b0 100644 --- a/src/scraper.test.ts +++ b/src/scraper.test.ts @@ -4,7 +4,9 @@ import { getGitHubBlob, getGitHubCommits, getGitHubIssues, + getGitHubLatestCommit, getGitHubPulls, + getGitHubRefs, getGitHubRelease, getGitHubReleases, getGitHubRepo, @@ -205,6 +207,45 @@ describe("GitHub scraper", () => { assert.ok(firstRelease.bodyHtml.length > 0); }); + it("should fetch latest commit for a path", async () => { + const data = await getGitHubLatestCommit("actions", "deploy-pages", "main", "src"); + + assert.strictEqual(data.repo.owner, "actions"); + assert.strictEqual(data.repo.name, "deploy-pages"); + assert.ok(data.commit); + assert.strictEqual(data.commit.oid.length, 40); + assert.ok(data.commit.shortMessageHtml.length > 0); + assert.ok(data.commit.date.length > 0); + assert.ok(data.commit.authors.length > 0); + }); + + it("should fetch branches", async () => { + const data = await getGitHubRefs("actions", "deploy-pages", "branches"); + + assert.strictEqual(data.repo.owner, "actions"); + assert.strictEqual(data.repo.name, "deploy-pages"); + assert.strictEqual(data.type, "branches"); + + assert.ok(data.refs.length > 0); + const main = data.refs.find((ref) => ref.name === "main"); + assert.ok(main); + assert.strictEqual(main.isDefault, true); + }); + + it("should fetch tags", async () => { + const data = await getGitHubRefs("actions", "deploy-pages", "tags"); + + assert.strictEqual(data.repo.owner, "actions"); + assert.strictEqual(data.repo.name, "deploy-pages"); + assert.strictEqual(data.type, "tags"); + + assert.ok(data.refs.length > 0); + const tag = data.refs.find((ref) => ref.name === "v5.0.0"); + assert.ok(tag); + assert.ok(tag.date?.length); + assert.strictEqual(data.hasMore, true); + }); + it("should fetch single release", async () => { const data = await getGitHubRelease("actions", "deploy-pages", "v4.0.5"); diff --git a/src/scraper.ts b/src/scraper.ts index 53f2bb1..39ad2cd 100644 --- a/src/scraper.ts +++ b/src/scraper.ts @@ -70,7 +70,7 @@ interface IssueIndexPageQuery { type StylingDirective = [number, number, string]; -interface Commit { +export interface Commit { oid: string; url: string; authoredDate: string; @@ -270,6 +270,34 @@ export interface GitHubCommits extends GitHubNav { commitGroups: CommitGroup[]; } +export interface LatestCommit { + oid: string; + date: string; + shortMessageHtml: string; + authors: { + login: string; + displayName: string; + avatarUrl: string; + }[]; +} + +export interface GitHubLatestCommit extends GitHubCommon { + commit: LatestCommit; +} + +export interface Ref { + name: string; + isDefault?: boolean; + date?: string; + authorLogin?: string; +} + +export interface GitHubRefs extends GitHubCommon { + type: "branches" | "tags"; + refs: Ref[]; + hasMore: boolean; +} + interface Release { tagName: string; title: string; @@ -325,13 +353,13 @@ export interface GitHubOwner { repos: OwnerRepo[]; } -async function fetchGitHubPage(path: string): Promise<string> { +async function fetchGitHubResponse(path: string, accept: string): Promise<Response> { // GitHub throttles/blocks requests without realistic browser headers. // These headers make the request appear as a standard browser visit. const headers = { "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", - Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", + Accept: accept, "Accept-Language": "en-US,en;q=0.5", }; const response = await fetch(`https://github.com/${path}`, { headers, redirect: "manual" }); @@ -345,10 +373,25 @@ async function fetchGitHubPage(path: string): Promise<string> { } } if (!response.ok) throw new GitHubHTTPError(response.status, response.statusText); + return response; +} + +async function fetchGitHubPage(path: string): Promise<string> { + const response = await fetchGitHubResponse( + path, + "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", + ); return response.text(); } -function parsePayload(html: string): AppPayload | null { +// Some routes (e.g. latest-commit) are the same endpoints the React UI itself +// calls to lazily fill in data; they 406 without an explicit JSON Accept. +async function fetchGitHubJson<T>(path: string): Promise<T> { + const response = await fetchGitHubResponse(path, "application/json"); + return (await response.json()) as T; +} + +function parseEmbeddedPayload<T>(html: string): T | null { const regex = new RegExp( `<script type="application/json" data-target="react-app.embeddedData">([^<]+)</script>`, "g", @@ -367,6 +410,10 @@ function parsePayload(html: string): AppPayload | null { return null; } +function parsePayload(html: string): AppPayload | null { + return parseEmbeddedPayload<AppPayload>(html); +} + function parseRepoInfo(html: string): RepoInfo { const document = parseDocument(html); @@ -639,6 +686,121 @@ export async function getGitHubCommits( }; } +interface LatestCommitPayload { + oid: string; + date: string; + shortMessageHtmlLink: string; + authors: { + login: string; + displayName: string; + avatarUrl: string; + }[]; +} + +// This is the same endpoint the React UI itself calls to lazily fill in the +// commit bar above a file listing, rather than the much heavier /commits +// history page: it returns just the single latest commit as JSON. +export async function getGitHubLatestCommit( + owner: string, + repo: string, + branch: string, + path: string, +): Promise<GitHubLatestCommit> { + const urlPath = path + ? `${owner}/${repo}/latest-commit/${branch}/${path}` + : `${owner}/${repo}/latest-commit/${branch}`; + const payload = await fetchGitHubJson<LatestCommitPayload>(urlPath); + + return { + repo: { owner, name: repo }, + commit: { + oid: payload.oid, + date: payload.date, + shortMessageHtml: payload.shortMessageHtmlLink, + authors: payload.authors, + }, + }; +} + +interface BranchRefEntry { + name: string; + isDefault: boolean; + authoredDate?: string; + author?: { login: string }; +} + +interface BranchesPayload { + branches: { + default?: BranchRefEntry; + yours?: BranchRefEntry[]; + active?: BranchRefEntry[]; + stale?: BranchRefEntry[]; + }; + hasMore?: Record<string, boolean>; +} + +async function getGitHubBranchRefs(owner: string, repo: string): Promise<GitHubRefs> { + const html = await fetchGitHubPage(`${owner}/${repo}/branches`); + const payload = parseEmbeddedPayload<BranchesPayload>(html); + + if (payload?.branches === undefined) { + throw new Error("Could not find branches data in embedded JSON"); + } + + const { default: defaultBranch, yours = [], active = [], stale = [] } = payload.branches; + const entries = [...(defaultBranch ? [defaultBranch] : []), ...yours, ...active, ...stale]; + + const seen = new Set<string>(); + const refs: Ref[] = []; + for (const entry of entries) { + if (seen.has(entry.name)) continue; + seen.add(entry.name); + refs.push({ + name: entry.name, + isDefault: entry.isDefault || undefined, + date: entry.authoredDate, + authorLogin: entry.author?.login, + }); + } + + const hasMore = Object.values(payload.hasMore ?? {}).some(Boolean); + + return { repo: { owner, name: repo }, type: "branches", refs, hasMore }; +} + +async function getGitHubTagRefs(owner: string, repo: string): Promise<GitHubRefs> { + const html = await fetchGitHubPage(`${owner}/${repo}/tags`); + const document = parseDocument(html); + + const refs: Ref[] = []; + for (const row of cssSelect.selectAll(".Box-row", document) as unknown as Element[]) { + const nameElem = cssSelect.selectOne("h2 a", row); + if (!nameElem) continue; + const name = textContent(nameElem).trim(); + if (!name) continue; + + // The row has two <relative-time> elements: a release/tag creation + // timestamp, then the tagged commit's authored date. We want the latter. + const timeElems = cssSelect.selectAll("relative-time", row); + const timeElem = timeElems[1] as Element | undefined; + const date = timeElem ? getAttributeValue(timeElem, "datetime") || undefined : undefined; + + refs.push({ name, date }); + } + + const hasMore = cssSelect.selectOne(".pagination a", document) !== null; + + return { repo: { owner, name: repo }, type: "tags", refs, hasMore }; +} + +export async function getGitHubRefs( + owner: string, + repo: string, + type: "branches" | "tags", +): Promise<GitHubRefs> { + return type === "branches" ? getGitHubBranchRefs(owner, repo) : getGitHubTagRefs(owner, repo); +} + function parseRelease(section: Element, title: string): Release { // Extract tag name from link const tagLink = cssSelect.selectOne('a[href*="/tree/"]', section); @@ -867,9 +1029,10 @@ function parseUser(document: AnyNode, login: string): GitHubOwner { } function parseOrganization(document: AnyNode, owner: string): GitHubOwner { - const avatarElem = cssSelect.selectOne(".orghead img[itemprop='image']", document) as - | Element - | null; + const avatarElem = cssSelect.selectOne( + ".orghead img[itemprop='image']", + document, + ) as Element | null; const avatarUrl = avatarElem ? getAttributeValue(avatarElem, "src") || null : null; // The login only appears in the avatar's "@login" alt text. const alt = avatarElem ? getAttributeValue(avatarElem, "alt") : undefined; diff --git a/views/_latestCommit.eta b/views/_latestCommit.eta new file mode 100644 index 0000000..18d22a1 --- /dev/null +++ b/views/_latestCommit.eta @@ -0,0 +1,16 @@ +<div class="commit box"> + <div> + <div> + <%~ it.commit.shortMessageHtml %> + </div> + <div class="commit-author"> + <% for (const author of it.commit.authors) { %> + <a href="/<%= author.login %>"><%= author.login %></a> + <% } %> + <time class="timestamp" title="<%= it.commit.date %>"><%= it.commit.date.split("T")[0] %></time> + </div> + </div> + <div class="commit-sha"> + <code><%= it.commit.oid.substring(0, 7) %></code> + </div> +</div> diff --git a/views/_nav.eta b/views/_nav.eta index 338a6a6..e3fbdcf 100644 --- a/views/_nav.eta +++ b/views/_nav.eta @@ -1,5 +1,14 @@ <section> - <div class="branch"><img src="/static/icons/git-branch.svg" alt="branch" class="icon"> <%= it.branch %></div> + <button type="button" class="ref-selector-button branch" command="show-modal" commandfor="ref-selector-menu"> + <img src="/static/icons/git-branch.svg" alt="branch" class="icon"> <%= it.branch %> ▾ + </button> + <dialog class="ref-selector-menu" id="ref-selector-menu" closedby="any" data-owner="<%= it.repo.owner %>" data-repo="<%= it.repo.name %>"> + <div class="ref-selector-tabs"> + <button type="button" class="active" data-ref-type="branches">Branches</button> + <button type="button" data-ref-type="tags">Tags</button> + </div> + <div class="ref-selector-list">Loading…</div> + </dialog> <% if (it.path && it.path !== "/") { %> <a href="/<%= it.repo.owner %>/<%= it.repo.name %>/tree/<%= it.branch %>">(top)</a><% const pathParts = it.path.split('/').filter(p => p); diff --git a/views/_refs.eta b/views/_refs.eta new file mode 100644 index 0000000..2450c4b --- /dev/null +++ b/views/_refs.eta @@ -0,0 +1,17 @@ +<% if (it.refs.length === 0) { %> + <div class="ref-selector-empty">No <%= it.type %> found.</div> +<% } else { %> + <ul class="ref-selector-items"> + <% for (const ref of it.refs) { %> + <li> + <a href="/<%= it.repo.owner %>/<%= it.repo.name %>/tree/<%= ref.name %>"><%= ref.name %></a> + <% if (ref.isDefault) { %><span class="ref-selector-badge">default</span><% } %> + </li> + <% } %> + </ul> +<% } %> +<% if (it.hasMore) { %> + <div class="ref-selector-more"> + <a href="/<%= it.repo.owner %>/<%= it.repo.name %>/<%= it.type %>">View all <%= it.type %></a> + </div> +<% } %> diff --git a/views/_tree.eta b/views/_tree.eta index ed6eee9..a0ecfab 100644 --- a/views/_tree.eta +++ b/views/_tree.eta @@ -1,5 +1,8 @@ <%~ include("./_nav.eta", {...it, isTree: true}) %> +<% const commitPath = it.path && it.path !== "/" ? it.path : ""; %> +<div class="latest-commit-placeholder box" data-src="/api/<%= it.repo.owner %>/<%= it.repo.name %>/latest-commit/<%= it.branch %>/<%= commitPath %>">Loading latest commit…</div> + <table> <% for (const item of it.items) { %> <tr> diff --git a/views/layout.eta b/views/layout.eta index dd523d1..a80bd1d 100644 --- a/views/layout.eta +++ b/views/layout.eta @@ -5,6 +5,7 @@ <meta name="viewport" content="width=device-width, initial-scale=1.0"> <title><%= it.repo.owner %>/<%= it.repo.name %> - <%= it.path || '/' %></title> <link rel="stylesheet" href="/static/style.css"> + <script src="/static/refs.js" defer></script> </head> <body> <div class="box"> |