aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/app.ts46
-rw-r--r--src/scraper.test.ts41
-rw-r--r--src/scraper.ts177
3 files changed, 257 insertions, 7 deletions
diff --git a/src/app.ts b/src/app.ts
index 78b59ae..ca79b1f 100644
--- a/src/app.ts
+++ b/src/app.ts
@@ -5,8 +5,10 @@ import {
getGitHubBlob,
getGitHubCommits,
getGitHubIssues,
+ getGitHubLatestCommit,
getGitHubOwner,
getGitHubPulls,
+ getGitHubRefs,
getGitHubRelease,
getGitHubReleases,
getGitHubRepo,
@@ -44,6 +46,29 @@ export function createApp(eta: Eta) {
return c.html(eta.render(template, data));
}
+ // For fragments fetched asynchronously by client-side JS (see public/static/refs.js):
+ // errors render as an empty body rather than the full error.eta page.
+ async function tryRenderFragment<T extends object>(
+ c: Context,
+ template: string,
+ promise: Promise<T>,
+ ) {
+ let data: T;
+ try {
+ data = await promise;
+ } catch (e) {
+ const status =
+ e instanceof GitHubHTTPError
+ ? (e.status as StatusCode)
+ : e instanceof RedirectError
+ ? 404
+ : 500;
+ c.status(status);
+ return c.body(null);
+ }
+ return c.html(eta.render(template, data));
+ }
+
app.get("/:owner", async (c) => {
const { owner } = c.req.param();
return tryRender(c, "owner.eta", getGitHubOwner(owner));
@@ -112,6 +137,27 @@ export function createApp(eta: Eta) {
return tryRender(c, "commits.eta", getGitHubCommits(owner, repo, branch, path));
});
+ // Internal endpoints fetched asynchronously by client-side JS, not meant to
+ // be visited directly. Keep the site's initial page loads plain server-
+ // rendered HTML while still surfacing data (branches/tags, latest commit)
+ // that would otherwise require an extra scrape on every page view.
+ app.get("/api/:owner/:repo/refs/:type", async (c) => {
+ const { owner, repo, type } = c.req.param();
+ if (type !== "branches" && type !== "tags") {
+ return c.notFound();
+ }
+ return tryRenderFragment(c, "_refs.eta", getGitHubRefs(owner, repo, type));
+ });
+
+ app.get("/api/:owner/:repo/latest-commit/:branch/:path{.*}?", async (c) => {
+ const { owner, repo, branch, path = "" } = c.req.param();
+ return tryRenderFragment(
+ c,
+ "_latestCommit.eta",
+ getGitHubLatestCommit(owner, repo, branch, path),
+ );
+ });
+
app.get("/:owner/:repo/releases/tag/:tag", async (c) => {
const { owner, repo, tag } = c.req.param();
return tryRender(c, "release.eta", getGitHubRelease(owner, repo, tag));
diff --git a/src/scraper.test.ts b/src/scraper.test.ts
index f9b60e8..56f14b0 100644
--- a/src/scraper.test.ts
+++ b/src/scraper.test.ts
@@ -4,7 +4,9 @@ import {
getGitHubBlob,
getGitHubCommits,
getGitHubIssues,
+ getGitHubLatestCommit,
getGitHubPulls,
+ getGitHubRefs,
getGitHubRelease,
getGitHubReleases,
getGitHubRepo,
@@ -205,6 +207,45 @@ describe("GitHub scraper", () => {
assert.ok(firstRelease.bodyHtml.length > 0);
});
+ it("should fetch latest commit for a path", async () => {
+ const data = await getGitHubLatestCommit("actions", "deploy-pages", "main", "src");
+
+ assert.strictEqual(data.repo.owner, "actions");
+ assert.strictEqual(data.repo.name, "deploy-pages");
+ assert.ok(data.commit);
+ assert.strictEqual(data.commit.oid.length, 40);
+ assert.ok(data.commit.shortMessageHtml.length > 0);
+ assert.ok(data.commit.date.length > 0);
+ assert.ok(data.commit.authors.length > 0);
+ });
+
+ it("should fetch branches", async () => {
+ const data = await getGitHubRefs("actions", "deploy-pages", "branches");
+
+ assert.strictEqual(data.repo.owner, "actions");
+ assert.strictEqual(data.repo.name, "deploy-pages");
+ assert.strictEqual(data.type, "branches");
+
+ assert.ok(data.refs.length > 0);
+ const main = data.refs.find((ref) => ref.name === "main");
+ assert.ok(main);
+ assert.strictEqual(main.isDefault, true);
+ });
+
+ it("should fetch tags", async () => {
+ const data = await getGitHubRefs("actions", "deploy-pages", "tags");
+
+ assert.strictEqual(data.repo.owner, "actions");
+ assert.strictEqual(data.repo.name, "deploy-pages");
+ assert.strictEqual(data.type, "tags");
+
+ assert.ok(data.refs.length > 0);
+ const tag = data.refs.find((ref) => ref.name === "v5.0.0");
+ assert.ok(tag);
+ assert.ok(tag.date?.length);
+ assert.strictEqual(data.hasMore, true);
+ });
+
it("should fetch single release", async () => {
const data = await getGitHubRelease("actions", "deploy-pages", "v4.0.5");
diff --git a/src/scraper.ts b/src/scraper.ts
index 53f2bb1..39ad2cd 100644
--- a/src/scraper.ts
+++ b/src/scraper.ts
@@ -70,7 +70,7 @@ interface IssueIndexPageQuery {
type StylingDirective = [number, number, string];
-interface Commit {
+export interface Commit {
oid: string;
url: string;
authoredDate: string;
@@ -270,6 +270,34 @@ export interface GitHubCommits extends GitHubNav {
commitGroups: CommitGroup[];
}
+export interface LatestCommit {
+ oid: string;
+ date: string;
+ shortMessageHtml: string;
+ authors: {
+ login: string;
+ displayName: string;
+ avatarUrl: string;
+ }[];
+}
+
+export interface GitHubLatestCommit extends GitHubCommon {
+ commit: LatestCommit;
+}
+
+export interface Ref {
+ name: string;
+ isDefault?: boolean;
+ date?: string;
+ authorLogin?: string;
+}
+
+export interface GitHubRefs extends GitHubCommon {
+ type: "branches" | "tags";
+ refs: Ref[];
+ hasMore: boolean;
+}
+
interface Release {
tagName: string;
title: string;
@@ -325,13 +353,13 @@ export interface GitHubOwner {
repos: OwnerRepo[];
}
-async function fetchGitHubPage(path: string): Promise<string> {
+async function fetchGitHubResponse(path: string, accept: string): Promise<Response> {
// GitHub throttles/blocks requests without realistic browser headers.
// These headers make the request appear as a standard browser visit.
const headers = {
"User-Agent":
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
- Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+ Accept: accept,
"Accept-Language": "en-US,en;q=0.5",
};
const response = await fetch(`https://github.com/${path}`, { headers, redirect: "manual" });
@@ -345,10 +373,25 @@ async function fetchGitHubPage(path: string): Promise<string> {
}
}
if (!response.ok) throw new GitHubHTTPError(response.status, response.statusText);
+ return response;
+}
+
+async function fetchGitHubPage(path: string): Promise<string> {
+ const response = await fetchGitHubResponse(
+ path,
+ "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+ );
return response.text();
}
-function parsePayload(html: string): AppPayload | null {
+// Some routes (e.g. latest-commit) are the same endpoints the React UI itself
+// calls to lazily fill in data; they 406 without an explicit JSON Accept.
+async function fetchGitHubJson<T>(path: string): Promise<T> {
+ const response = await fetchGitHubResponse(path, "application/json");
+ return (await response.json()) as T;
+}
+
+function parseEmbeddedPayload<T>(html: string): T | null {
const regex = new RegExp(
`<script type="application/json" data-target="react-app.embeddedData">([^<]+)</script>`,
"g",
@@ -367,6 +410,10 @@ function parsePayload(html: string): AppPayload | null {
return null;
}
+function parsePayload(html: string): AppPayload | null {
+ return parseEmbeddedPayload<AppPayload>(html);
+}
+
function parseRepoInfo(html: string): RepoInfo {
const document = parseDocument(html);
@@ -639,6 +686,121 @@ export async function getGitHubCommits(
};
}
+interface LatestCommitPayload {
+ oid: string;
+ date: string;
+ shortMessageHtmlLink: string;
+ authors: {
+ login: string;
+ displayName: string;
+ avatarUrl: string;
+ }[];
+}
+
+// This is the same endpoint the React UI itself calls to lazily fill in the
+// commit bar above a file listing, rather than the much heavier /commits
+// history page: it returns just the single latest commit as JSON.
+export async function getGitHubLatestCommit(
+ owner: string,
+ repo: string,
+ branch: string,
+ path: string,
+): Promise<GitHubLatestCommit> {
+ const urlPath = path
+ ? `${owner}/${repo}/latest-commit/${branch}/${path}`
+ : `${owner}/${repo}/latest-commit/${branch}`;
+ const payload = await fetchGitHubJson<LatestCommitPayload>(urlPath);
+
+ return {
+ repo: { owner, name: repo },
+ commit: {
+ oid: payload.oid,
+ date: payload.date,
+ shortMessageHtml: payload.shortMessageHtmlLink,
+ authors: payload.authors,
+ },
+ };
+}
+
+interface BranchRefEntry {
+ name: string;
+ isDefault: boolean;
+ authoredDate?: string;
+ author?: { login: string };
+}
+
+interface BranchesPayload {
+ branches: {
+ default?: BranchRefEntry;
+ yours?: BranchRefEntry[];
+ active?: BranchRefEntry[];
+ stale?: BranchRefEntry[];
+ };
+ hasMore?: Record<string, boolean>;
+}
+
+async function getGitHubBranchRefs(owner: string, repo: string): Promise<GitHubRefs> {
+ const html = await fetchGitHubPage(`${owner}/${repo}/branches`);
+ const payload = parseEmbeddedPayload<BranchesPayload>(html);
+
+ if (payload?.branches === undefined) {
+ throw new Error("Could not find branches data in embedded JSON");
+ }
+
+ const { default: defaultBranch, yours = [], active = [], stale = [] } = payload.branches;
+ const entries = [...(defaultBranch ? [defaultBranch] : []), ...yours, ...active, ...stale];
+
+ const seen = new Set<string>();
+ const refs: Ref[] = [];
+ for (const entry of entries) {
+ if (seen.has(entry.name)) continue;
+ seen.add(entry.name);
+ refs.push({
+ name: entry.name,
+ isDefault: entry.isDefault || undefined,
+ date: entry.authoredDate,
+ authorLogin: entry.author?.login,
+ });
+ }
+
+ const hasMore = Object.values(payload.hasMore ?? {}).some(Boolean);
+
+ return { repo: { owner, name: repo }, type: "branches", refs, hasMore };
+}
+
+async function getGitHubTagRefs(owner: string, repo: string): Promise<GitHubRefs> {
+ const html = await fetchGitHubPage(`${owner}/${repo}/tags`);
+ const document = parseDocument(html);
+
+ const refs: Ref[] = [];
+ for (const row of cssSelect.selectAll(".Box-row", document) as unknown as Element[]) {
+ const nameElem = cssSelect.selectOne("h2 a", row);
+ if (!nameElem) continue;
+ const name = textContent(nameElem).trim();
+ if (!name) continue;
+
+ // The row has two <relative-time> elements: a release/tag creation
+ // timestamp, then the tagged commit's authored date. We want the latter.
+ const timeElems = cssSelect.selectAll("relative-time", row);
+ const timeElem = timeElems[1] as Element | undefined;
+ const date = timeElem ? getAttributeValue(timeElem, "datetime") || undefined : undefined;
+
+ refs.push({ name, date });
+ }
+
+ const hasMore = cssSelect.selectOne(".pagination a", document) !== null;
+
+ return { repo: { owner, name: repo }, type: "tags", refs, hasMore };
+}
+
+export async function getGitHubRefs(
+ owner: string,
+ repo: string,
+ type: "branches" | "tags",
+): Promise<GitHubRefs> {
+ return type === "branches" ? getGitHubBranchRefs(owner, repo) : getGitHubTagRefs(owner, repo);
+}
+
function parseRelease(section: Element, title: string): Release {
// Extract tag name from link
const tagLink = cssSelect.selectOne('a[href*="/tree/"]', section);
@@ -867,9 +1029,10 @@ function parseUser(document: AnyNode, login: string): GitHubOwner {
}
function parseOrganization(document: AnyNode, owner: string): GitHubOwner {
- const avatarElem = cssSelect.selectOne(".orghead img[itemprop='image']", document) as
- | Element
- | null;
+ const avatarElem = cssSelect.selectOne(
+ ".orghead img[itemprop='image']",
+ document,
+ ) as Element | null;
const avatarUrl = avatarElem ? getAttributeValue(avatarElem, "src") || null : null;
// The login only appears in the avatar's "@login" alt text.
const alt = avatarElem ? getAttributeValue(avatarElem, "alt") : undefined;
generated by cgit v1.3.1 (git 2.54.0) at 2026-10-04 16:01:59 +0000