From 5e45f900974589c8eead15fc352d50dc45b1a25a Mon Sep 17 00:00:00 2001 From: Joe Mou Date: Thu, 22 Jan 2026 12:10:36 -0500 Subject: GitHub directory scraper Extracts directory listings from GitHub's server-rendered HTML. Avoids using the rate-limited API. Parses JSON embedded within script tags. Co-Authored-By: Claude Sonnet 4.5 --- cgithub/README.md | 81 +++++++++++++++++++++++++++++++++++++++++++++++ cgithub/github-scraper.js | 74 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 155 insertions(+) create mode 100644 cgithub/README.md create mode 100755 cgithub/github-scraper.js diff --git a/cgithub/README.md b/cgithub/README.md new file mode 100644 index 0000000..0f0ab43 --- /dev/null +++ b/cgithub/README.md @@ -0,0 +1,81 @@ +# GitHub Directory Listing Without API + +## Summary + +GitHub repository directory listings can be obtained **without using the API and without rate limits** by scraping the server-rendered HTML that GitHub sends to browsers. + +## How It Works + +When you visit a GitHub repository page in a browser, GitHub server-side renders the directory listing and embeds it as JSON data within ` +``` + +**Access Path:** `data.payload.tree.items` + +## Item Structure + +Each item in the `items` array has this structure: + +```json +{ + "name": "filename.txt", + "path": "path/to/filename.txt", + "contentType": "file" // or "directory" +} +``` + +## Complete Example + +See `github-scraper.js` for a working implementation that: +- Fetches the HTML from GitHub +- Extracts the embedded JSON data +- Parses and returns the directory listing +- Works for both root and subdirectories +- Has zero API rate limits + +## Usage + +```bash +node github-scraper.js torvalds linux # Root directory +node github-scraper.js torvalds linux Documentation # Subdirectory +node github-scraper.js git git Documentation/git # Nested subdirectory +``` + +## Advantages Over API + +| Feature | HTML Scraping | GitHub API | +|---------|--------------|------------| +| Rate Limits | None | 60/hour (unauth), 5000/hour (auth) | +| Authentication | Not required | Optional, but limits apply | +| Access | Any public repo | Any public repo | +| Stability | Depends on HTML structure | Stable API contract | + +## Important Notes + +1. **No authentication needed** - Works for all public repositories +2. **No rate limits** - Can scrape as many repos as needed +3. **Less stable** - HTML structure may change with GitHub updates +4. **Best for:** One-off scripts, personal tools, exploratory work +5. **Use API for:** Production applications, long-term projects + +## Why This Works + +GitHub uses server-side rendering to provide fast initial page loads. The directory data is embedded in the HTML so the page can render immediately without waiting for additional API calls. This embedded data is the same data that would come from the API, just pre-rendered in the page. diff --git a/cgithub/github-scraper.js b/cgithub/github-scraper.js new file mode 100755 index 0000000..7e07adc --- /dev/null +++ b/cgithub/github-scraper.js @@ -0,0 +1,74 @@ +#!/usr/bin/env node +/** + * GitHub Directory Listing Scraper + * + * Gets directory listings from GitHub without using the API (no rate limits!) + * Works by parsing the server-rendered HTML that GitHub sends to browsers. + */ + +async function getGitHubDirectory(owner, repo, path = '') { + // Always use the tree URL pattern, even for root directory + const url = `https://github.com/${owner}/${repo}/tree/main/${path}`; + + console.error(`Fetching: ${url}`); + + const response = await fetch(url); + const html = await response.text(); + + // GitHub embeds the directory data in a JSON script tag + // Using the tree (subdirectory) pattern for all cases + const subdirMatch = html.match(/`, + "g", + ); + + match: for (const match of html.matchAll(regex)) { + let data; try { - const data = JSON.parse(match[1]); - if (data.props?.initialPayload?.tree) { - payload = data.props.initialPayload; - break; - } - } catch (e) { + data = JSON.parse(match[1]); + } catch { continue; } - } - - // Try subdirectory format. - if (!payload) { - const dirFormatMatch = html.match( - /`, "g", @@ -99,9 +105,7 @@ function parsePayload( function parseRepoInfo(html: string): RepoInfo { const descMatch = html.match(/]*class="[^"]*f4[^"]*"[^>]*>([\s\S]*?)<\/p>/); const description = descMatch?.[1]?.replace(/<[^>]+>/g, "").trim() || null; - const websiteMatch = html.match( - /]*rel="noopener noreferrer nofollow"[^>]*href="([^"]+)"/, - ); + const websiteMatch = html.match(/]*rel="noopener noreferrer nofollow"[^>]*href="([^"]+)"/); const starsMatch = html.match(/id="repo-stars-counter-star"[^>]*>([^<]+)/); const watchersMatch = html.match(/(\d+)<\/strong>\s*watching/); const forksMatch = html.match(/id="repo-network-counter"[^>]*>([^<]+)/); @@ -115,6 +119,16 @@ function parseRepoInfo(html: string): RepoInfo { }; } +function extractOverviewHtml(payload: RawPayload): Record | undefined { + const result: Record = {}; + for (const file of payload.overview?.overviewFiles ?? []) { + if (file.loaded && file.richText !== undefined) { + result[file.displayName] = file.richText; + } + } + return Object.keys(result).length > 0 ? result : undefined; +} + function extractGitHub(payload: RawPayload, extra: T): GitHubCommon & T { return { repo: { @@ -127,24 +141,19 @@ function extractGitHub(payload: RawPayload, extra: T): GitHubCommon & T { }; } -export async function getGitHubRepo( - owner: string, - repo: string, -): Promise { +export async function getGitHubRepo(owner: string, repo: string): Promise { const html = await fetchGitHubPage(`${owner}/${repo}`); - const payload = parsePayload(html, "react-partial.embeddedData", [ - "props", - "initialPayload", - ]); + const payload = parsePayload(html, "react-partial.embeddedData", ["props", "initialPayload"]); if (payload?.tree === undefined) { throw new Error("Could not find tree data in embedded JSON"); } const info = parseRepoInfo(html); + const overviewHtml = extractOverviewHtml(payload); - return extractGitHub(payload, { items: payload.tree.items, info }); + return extractGitHub(payload, { items: payload.tree.items, info, overviewHtml }); } export async function getGitHubTree( @@ -158,10 +167,7 @@ export async function getGitHubTree( // GitHub uses different formats for root vs subdirectories. const payload = // Root directory format. - parsePayload(html, "react-partial.embeddedData", [ - "props", - "initialPayload", - ]) || + parsePayload(html, "react-partial.embeddedData", ["props", "initialPayload"]) || // Subdirectory format. parsePayload(html, "react-app.embeddedData", ["payload"]); @@ -169,7 +175,9 @@ export async function getGitHubTree( throw new Error("Could not find tree data in embedded JSON"); } - return extractGitHub(payload, { items: payload.tree.items }); + const overviewHtml = extractOverviewHtml(payload); + + return extractGitHub(payload, { items: payload.tree.items, overviewHtml }); } export async function getGitHubBlob( diff --git a/cgithub/static/style.css b/cgithub/static/style.css index 1229fce..ff787d6 100644 --- a/cgithub/static/style.css +++ b/cgithub/static/style.css @@ -3,6 +3,7 @@ body { font-family: Helvetica, Arial, sans-serif; --border-color: #d0d7de; + --shade-background: #eeeeee; } @media (prefers-color-scheme: dark) { @@ -23,10 +24,20 @@ a { text-decoration: none; } -header, .box { +body > header, .box { + --box-padding: .5rem; + margin: 1rem 0; border: 1px solid var(--border-color); - padding: .5rem; + padding: var(--box-padding); + + > header:first-child { + margin: calc(-1 * var(--box-padding)); + margin-bottom: 0; + padding: .5rem; + border-bottom: 1px solid var(--border-color); + font-weight: bold; + } } footer { @@ -64,3 +75,31 @@ pre.box { /* Instead of scrolling overflow for long lines, grow the box. */ width: fit-content; } + +.overview { + overflow-wrap: break-word; + &.box { + --box-padding: 1rem; + } + + a { + text-decoration: underline; + } + + h1, h2 { + border-bottom: 1px solid var(--border-color); + } + + img { + max-width: 100%; + } + + code { + background: var(--shade-background); + } + + /* TODO Strip permalinks? */ + svg.octicon-link { + display: none; + } +} diff --git a/cgithub/views/_tree.eta b/cgithub/views/_tree.eta index 2549bb8..f287c03 100644 --- a/cgithub/views/_tree.eta +++ b/cgithub/views/_tree.eta @@ -13,3 +13,12 @@ <% } %> + +<% if (it.overviewHtml) { %> + <% for (const [filename, html] of Object.entries(it.overviewHtml)) { %> +
+
<%= filename %>
+ <%~ html %> +
+ <% } %> +<% } %> diff --git a/cgithub/views/blob.eta b/cgithub/views/blob.eta index 9b478b5..7328a62 100644 --- a/cgithub/views/blob.eta +++ b/cgithub/views/blob.eta @@ -7,6 +7,9 @@ <% if (it.size !== undefined) { %>
size: <%= it.size %> bytes
<% } %> +
+ Raw +
<%= it.rawContent %>
-- cgit v1.3.1 From 51558d04a5ca53b2d2b6f088d1220167ee480b39 Mon Sep 17 00:00:00 2001 From: Joe Mou Date: Mon, 26 Jan 2026 13:12:28 -0500 Subject: git clone commands --- cgithub/static/style.css | 7 +++++++ cgithub/views/layout.eta | 2 ++ 2 files changed, 9 insertions(+) diff --git a/cgithub/static/style.css b/cgithub/static/style.css index ff787d6..5c27bbc 100644 --- a/cgithub/static/style.css +++ b/cgithub/static/style.css @@ -31,6 +31,13 @@ body > header, .box { border: 1px solid var(--border-color); padding: var(--box-padding); + > *:first-child { + margin-top: 0; + } + > *:last-child { + margin-bottom: 0; + } + > header:first-child { margin: calc(-1 * var(--box-padding)); margin-bottom: 0; diff --git a/cgithub/views/layout.eta b/cgithub/views/layout.eta index a191301..bfcd370 100644 --- a/cgithub/views/layout.eta +++ b/cgithub/views/layout.eta @@ -26,6 +26,8 @@ } %> <% } %> +
git clone https://github.com/<%= it.repo.owner %>/<%= it.repo.name %>.git
+git clone git@github.com:<%= it.repo.owner %>/<%= it.repo.name %>.git
<%~ it.body %> -- cgit v1.3.1 From 6484840a5412de4b003f9007eadf3e120dae5f3c Mon Sep 17 00:00:00 2001 From: Joe Mou Date: Mon, 26 Jan 2026 13:27:28 -0500 Subject: Append / to directories in tree listings --- cgithub/views/_tree.eta | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cgithub/views/_tree.eta b/cgithub/views/_tree.eta index f287c03..d0c3453 100644 --- a/cgithub/views/_tree.eta +++ b/cgithub/views/_tree.eta @@ -4,7 +4,7 @@ <% if (item.contentType === 'directory') { %> tree - <%= item.name %> + <%= item.name %>/ <% } else { %> blob <%= item.name %> -- cgit v1.3.1 From d91b89f89bd7dcf3258dc5802d6a97c5e341e272 Mon Sep 17 00:00:00 2001 From: Joe Mou Date: Mon, 26 Jan 2026 13:41:55 -0500 Subject: Redirect any unhandled routes to GitHub --- cgithub/src/index.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cgithub/src/index.ts b/cgithub/src/index.ts index 15d464a..38d6d13 100644 --- a/cgithub/src/index.ts +++ b/cgithub/src/index.ts @@ -41,6 +41,12 @@ app.get("/:owner/:repo/raw/:branch/:path{.*}", async (c) => { return c.redirect(`https://raw.githubusercontent.com/${owner}/${repo}/${branch}/${path}`); }); +app.all("*", async (c) => { + const path = c.req.path; + c.header("Referrer-Policy", "no-referrer"); + return c.redirect(`https://github.com${path}`); +}); + if (process.env.LISTEN_FDS === "1") { const server = createAdaptorServer(app); // Listen on SD_LISTEN_FDS_START. -- cgit v1.3.1 From fee5c6f6438316a8d407699370729ac2d0abd9d1 Mon Sep 17 00:00:00 2001 From: Joe Mou Date: Mon, 26 Jan 2026 15:55:52 -0500 Subject: Use Node native TypeScript elision See https://nodejs.org/api/typescript.html#type-stripping --- cgithub/.gitignore | 1 - cgithub/package.json | 3 +-- cgithub/src/index.ts | 6 +++--- cgithub/tsconfig.json | 21 +++++++-------------- 4 files changed, 11 insertions(+), 20 deletions(-) diff --git a/cgithub/.gitignore b/cgithub/.gitignore index b2636c7..3d2bc62 100644 --- a/cgithub/.gitignore +++ b/cgithub/.gitignore @@ -1,3 +1,2 @@ -/build/ /dist/ /node_modules/ diff --git a/cgithub/package.json b/cgithub/package.json index 461a9bd..b3d778b 100644 --- a/cgithub/package.json +++ b/cgithub/package.json @@ -4,7 +4,7 @@ "description": "Lightweight GitHub alternative frontend", "type": "module", "scripts": { - "dev": "concurrently 'tsc --watch --preserveWatchOutput' 'node --watch --watch-preserve-output build/index.js'", + "dev": "node --watch --watch-preserve-output src/index.ts", "build": "rm -rf dist/* && esbuild --bundle --format=esm --platform=node --outdir=dist/build --sourcemap src/index.ts && cp -R static views dist/" }, "dependencies": { @@ -14,7 +14,6 @@ }, "devDependencies": { "@types/node": "^22.10.2", - "concurrently": "^9.1.2", "esbuild": "^0.27.2", "prettier": "^3.8.1", "typescript": "^5.7.2" diff --git a/cgithub/src/index.ts b/cgithub/src/index.ts index 38d6d13..388b1e0 100644 --- a/cgithub/src/index.ts +++ b/cgithub/src/index.ts @@ -2,9 +2,9 @@ import { createAdaptorServer, serve } from "@hono/node-server"; import { serveStatic } from "@hono/node-server/serve-static"; import { Eta } from "eta"; import { Hono } from "hono"; -import * as path from "node:path"; -import * as url from "node:url"; -import { getGitHubBlob, getGitHubRepo, getGitHubTree } from "./scraper.js"; +import path from "node:path"; +import url from "node:url"; +import { getGitHubBlob, getGitHubRepo, getGitHubTree } from "./scraper.ts"; const __dirname = path.dirname(url.fileURLToPath(import.meta.url)); diff --git a/cgithub/tsconfig.json b/cgithub/tsconfig.json index c2f75c1..2e9fe6a 100644 --- a/cgithub/tsconfig.json +++ b/cgithub/tsconfig.json @@ -1,19 +1,12 @@ { "compilerOptions": { - "target": "ES2024", + "target": "ESNext", "module": "NodeNext", - "moduleResolution": "NodeNext", - "lib": ["ES2024"], - "outDir": "./build", - "rootDir": "./src", + "lib": ["ESNext"], + "noEmit": true, "strict": true, - "esModuleInterop": true, - "skipLibCheck": true, - "forceConsistentCasingInFileNames": true, - "resolveJsonModule": true, - "allowSyntheticDefaultImports": true, - "types": ["node"] - }, - "include": ["src/**/*"], - "exclude": ["node_modules", "build"] + "erasableSyntaxOnly": true, + "rewriteRelativeImportExtensions": true, + "verbatimModuleSyntax": true + } } -- cgit v1.3.1 From 744f42a2ae7968881e4d8f1ce0ec0d5209c8d7a4 Mon Sep 17 00:00:00 2001 From: Joe Mou Date: Mon, 26 Jan 2026 13:27:28 -0500 Subject: Show releases --- cgithub/src/scraper.ts | 10 ++++++++++ cgithub/views/repo.eta | 3 +++ 2 files changed, 13 insertions(+) diff --git a/cgithub/src/scraper.ts b/cgithub/src/scraper.ts index c1bdef5..4ffdc43 100644 --- a/cgithub/src/scraper.ts +++ b/cgithub/src/scraper.ts @@ -54,6 +54,7 @@ export interface RepoInfo { stars: string | null; watchers: string | null; forks: string | null; + numReleases?: number; } export interface GitHubRepo extends GitHubTree { @@ -110,12 +111,21 @@ function parseRepoInfo(html: string): RepoInfo { const watchersMatch = html.match(/(\d+)<\/strong>\s*watching/); const forksMatch = html.match(/id="repo-network-counter"[^>]*>([^<]+)/); + let numReleases: number | undefined; + const releaseCountMatch = html.match( + /]*href="[^"]*\/releases"[^>]*>[\s\S]*?]*title="(\d+)"[^>]*(?:class="[^"]*Counter|data-view-component="true")[^>]*>/, + ); + if (releaseCountMatch) { + numReleases = parseInt(releaseCountMatch[1], 10); + } + return { description, website: websiteMatch?.[1] || null, stars: starsMatch?.[1]?.trim() || null, watchers: watchersMatch?.[1] || null, forks: forksMatch?.[1]?.trim() || null, + numReleases, }; } diff --git a/cgithub/views/repo.eta b/cgithub/views/repo.eta index 4965e75..5bebb95 100644 --- a/cgithub/views/repo.eta +++ b/cgithub/views/repo.eta @@ -17,6 +17,9 @@ <% if (it.info.forks) { %> forks <%= it.info.forks %> <% } %> + <% if (it.info.numReleases) { %> + + <% } %> <% } %> -- cgit v1.3.1 From dac0a13f247fa17667ac5983ec65df0756e0829f Mon Sep 17 00:00:00 2001 From: Joe Mou Date: Mon, 26 Jan 2026 15:36:23 -0500 Subject: GitHub scraper test --- cgithub/package.json | 3 +- cgithub/src/scraper.test.ts | 70 +++++++++++++++++++++++++++++++++++++++++++++ cgithub/src/scraper.ts | 7 +++-- 3 files changed, 77 insertions(+), 3 deletions(-) create mode 100644 cgithub/src/scraper.test.ts diff --git a/cgithub/package.json b/cgithub/package.json index b3d778b..71dfb4d 100644 --- a/cgithub/package.json +++ b/cgithub/package.json @@ -5,7 +5,8 @@ "type": "module", "scripts": { "dev": "node --watch --watch-preserve-output src/index.ts", - "build": "rm -rf dist/* && esbuild --bundle --format=esm --platform=node --outdir=dist/build --sourcemap src/index.ts && cp -R static views dist/" + "build": "rm -rf dist/* && esbuild --bundle --format=esm --platform=node --outdir=dist/build --sourcemap src/index.ts && cp -R static views dist/", + "test": "node --test src/scraper.test.ts" }, "dependencies": { "@hono/node-server": "^1.13.7", diff --git a/cgithub/src/scraper.test.ts b/cgithub/src/scraper.test.ts new file mode 100644 index 0000000..3784190 --- /dev/null +++ b/cgithub/src/scraper.test.ts @@ -0,0 +1,70 @@ +import assert from "node:assert"; +import { describe, it } from "node:test"; +import { getGitHubBlob, getGitHubRepo, getGitHubTree } from "./scraper.ts"; + +describe("GitHub scraper", () => { + describe("actions/deploy-pages repository", () => { + it("should fetch repository", async () => { + const data = await getGitHubRepo("actions", "deploy-pages"); + + assert.strictEqual(data.repo.owner, "actions"); + assert.strictEqual(data.repo.name, "deploy-pages"); + assert.strictEqual(data.branch, "main"); + assert.strictEqual(data.path, "/"); + + assert.strictEqual( + data.info.description, + "GitHub Action to publish artifacts to GitHub Pages for deployments", + ); + assert.strictEqual(data.info.website, "https://pages.github.com"); + assert.strictEqual(data.info.stars?.length, 3); + assert.strictEqual(data.info.watchers?.length, 2); + assert.strictEqual(data.info.forks?.length, 3); + assert.strictEqual(data.info.numReleases, 36); + + assert.ok(data.items.length > 0); + assert.ok(Object.keys(data.overviewHtml ?? {}).includes("README.md")); + }); + + it("should fetch tree", async () => { + const data = await getGitHubTree("actions", "deploy-pages", "main", "src"); + + assert.strictEqual(data.repo.owner, "actions"); + assert.strictEqual(data.repo.name, "deploy-pages"); + assert.strictEqual(data.branch, "main"); + assert.strictEqual(data.path, "src"); + + assert.deepStrictEqual(data.items, [ + { + name: "__tests__", + path: "src/__tests__", + contentType: "directory", + }, + { + name: "internal", + path: "src/internal", + contentType: "directory", + }, + { + name: "index.js", + path: "src/index.js", + contentType: "file", + }, + ]); + }); + + it("should fetch blob", async () => { + const data = await getGitHubBlob("actions", "deploy-pages", "main", "LICENSE"); + + assert.strictEqual(data.repo.owner, "actions"); + assert.strictEqual(data.repo.name, "deploy-pages"); + assert.strictEqual(data.branch, "main"); + assert.strictEqual(data.path, "LICENSE"); + + assert.strictEqual(data.size, 1068); + assert.strictEqual(data.language, "Text"); + assert.match(data.rawContent, /^MIT License\n\n/); + assert.match(data.content, /^MIT License\n\n/); + }); + }); +}); diff --git a/cgithub/src/scraper.ts b/cgithub/src/scraper.ts index 4ffdc43..4feda73 100644 --- a/cgithub/src/scraper.ts +++ b/cgithub/src/scraper.ts @@ -63,7 +63,7 @@ export interface GitHubRepo extends GitHubTree { export interface GitHubBlob extends GitHubCommon { language: string | null; - size: number; + size: number | null; content: string; rawContent: string; } @@ -210,10 +210,13 @@ export async function getGitHubBlob( const highlightedLines = blob.highlightedLines || []; const content = highlightedLines.map((line) => line.text || "").join("\n"); + // Calculate size from raw content if not provided + const size = blob.size ?? (rawContent.length > 0 ? Buffer.byteLength(rawContent, "utf8") : null); + return extractGitHub(payload, { content: content || rawContent, rawContent, language: blob.language || null, - size: blob.size, + size, }); } -- cgit v1.3.1 From bac0eb8ed91400fbaf58edc2c33fdc854f88db48 Mon Sep 17 00:00:00 2001 From: Joe Mou Date: Mon, 26 Jan 2026 18:10:01 -0500 Subject: Color schemes --- cgithub/static/style.css | 55 +++++++++++++++++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 17 deletions(-) diff --git a/cgithub/static/style.css b/cgithub/static/style.css index 5c27bbc..d21bd32 100644 --- a/cgithub/static/style.css +++ b/cgithub/static/style.css @@ -1,27 +1,42 @@ +:root { + color-scheme: light dark; + + --body-background: light-dark( + oklch(95% 0.01 250), + oklch(20% 0.01 250) + ); + + --text-color: light-dark( + oklch(5% 0.01 250), + oklch(90% 0.01 250) + ); + + --border-color: light-dark( + oklch(85% 0.01 250), + oklch(40% 0.01 250) + ); + + --shade-background: light-dark( + oklch(90% 0.01 250), + oklch(30% 0.01 250) + ); + + --link-color: light-dark( + oklch(50% 0.15 250), + oklch(60% 0.15 250) + ); +} + body { margin: 0 1rem; font-family: Helvetica, Arial, sans-serif; - - --border-color: #d0d7de; - --shade-background: #eeeeee; -} - -@media (prefers-color-scheme: dark) { - body { - background: #0d1117; - color: #e6edf3; - --border-color: #30363d; - } - a { - color: #4493f8; - } - .blob { - background: #161b22; - } + background: var(--body-background); + color: var(--text-color); } a { text-decoration: none; + color: var(--link-color); } body > header, .box { @@ -57,6 +72,12 @@ footer { vertical-align: text-bottom; } +@media (prefers-color-scheme: dark) { + .icon { + filter: brightness(0) invert(1); + } +} + .repo { margin-bottom: 1rem; font-size: 120%; -- cgit v1.3.1 From 1e339a892761b7a190c3548411ca0ec212cded53 Mon Sep 17 00:00:00 2001 From: Joe Mou Date: Tue, 27 Jan 2026 09:14:31 -0500 Subject: MIT license --- cgithub/LICENSE | 21 +++++++++++++++++++++ cgithub/package.json | 1 + 2 files changed, 22 insertions(+) create mode 100644 cgithub/LICENSE diff --git a/cgithub/LICENSE b/cgithub/LICENSE new file mode 100644 index 0000000..8375820 --- /dev/null +++ b/cgithub/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Joe Mou + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/cgithub/package.json b/cgithub/package.json index 71dfb4d..6faa90b 100644 --- a/cgithub/package.json +++ b/cgithub/package.json @@ -2,6 +2,7 @@ "name": "cgithub", "version": "1.0.0", "description": "Lightweight GitHub alternative frontend", + "license": "MIT", "type": "module", "scripts": { "dev": "node --watch --watch-preserve-output src/index.ts", -- cgit v1.3.1 From ad2871ca617899aa36a2d85b41974bae9796ace6 Mon Sep 17 00:00:00 2001 From: Joe Mou Date: Tue, 27 Jan 2026 09:30:28 -0500 Subject: Rewrite README --- cgithub/README.md | 111 ++++++++++++++++++++++-------------------------------- 1 file changed, 45 insertions(+), 66 deletions(-) diff --git a/cgithub/README.md b/cgithub/README.md index 0f0ab43..9ba2213 100644 --- a/cgithub/README.md +++ b/cgithub/README.md @@ -1,81 +1,60 @@ -# GitHub Directory Listing Without API +cgithub is a lightweight GitHub alternative frontend. It has no bloated +JavaScript and doesn't require API keys. To use it, host cgithub on your own +server then replace `github.com` in your URL with your domain. -## Summary +## Development -GitHub repository directory listings can be obtained **without using the API and without rate limits** by scraping the server-rendered HTML that GitHub sends to browsers. - -## How It Works - -When you visit a GitHub repository page in a browser, GitHub server-side renders the directory listing and embeds it as JSON data within ` +To run the server locally: ``` - -**Access Path:** `data.payload.tree.items` - -## Item Structure - -Each item in the `items` array has this structure: - -```json -{ - "name": "filename.txt", - "path": "path/to/filename.txt", - "contentType": "file" // or "directory" -} +$ pnpm run dev ``` -## Complete Example +To run tests: +``` +$ pnpm test +``` -See `github-scraper.js` for a working implementation that: -- Fetches the HTML from GitHub -- Extracts the embedded JSON data -- Parses and returns the directory listing -- Works for both root and subdirectories -- Has zero API rate limits +## Deployment -## Usage +Assuming you have a server `jupiter`, you might deploy like: +``` +$ pnpm run build +$ rsync -av --del dist jupiter:/opt/cgithub +``` -```bash -node github-scraper.js torvalds linux # Root directory -node github-scraper.js torvalds linux Documentation # Subdirectory -node github-scraper.js git git Documentation/git # Nested subdirectory +To configure systemd socket activation use a `cgithub.socket` file: ``` +[Socket] +ListenStream=/run/cgithub/socket +SocketGroup=nginx +SocketMode=0660 -## Advantages Over API +[Install] +WantedBy=sockets.target +``` -| Feature | HTML Scraping | GitHub API | -|---------|--------------|------------| -| Rate Limits | None | 60/hour (unauth), 5000/hour (auth) | -| Authentication | Not required | Optional, but limits apply | -| Access | Any public repo | Any public repo | -| Stability | Depends on HTML structure | Stable API contract | +And a service file `cgithub.service`: +``` +[Unit] +After=cgithub.socket +Requires=cgithub.socket + +[Service] +Type=exec +WorkingDirectory=/opt/cgithub +ExecStart=/usr/bin/node dist/build/index.js +DynamicUser=true +``` -## Important Notes +An nginx config snippet might be: +``` +location / { + proxy_pass http://unix:/run/cgithub/socket; +} +``` -1. **No authentication needed** - Works for all public repositories -2. **No rate limits** - Can scrape as many repos as needed -3. **Less stable** - HTML structure may change with GitHub updates -4. **Best for:** One-off scripts, personal tools, exploratory work -5. **Use API for:** Production applications, long-term projects +## Disclosures -## Why This Works +AI coding assistants, in particular Claude, are used in the development process. -GitHub uses server-side rendering to provide fast initial page loads. The directory data is embedded in the HTML so the page can render immediately without waiting for additional API calls. This embedded data is the same data that would come from the API, just pre-rendered in the page. +[Phospor Icons](https://phosphoricons.com/) used under MIT license. -- cgit v1.3.1 From 04bbba0a7b71f1fb090d3e3a73110a00c36bf765 Mon Sep 17 00:00:00 2001 From: Joe Mou Date: Tue, 27 Jan 2026 11:23:11 -0500 Subject: Propagate GitHub HTTP errors Render `error.eta` on HTTP error from GitHub. Implemented with Gemini. --- cgithub/GEMINI.md | 1 + cgithub/package.json | 2 +- cgithub/src/index.ts | 31 +++++++++++++++++++++++-------- cgithub/src/scraper.test.ts | 12 +++++++++++- cgithub/src/scraper.ts | 11 +++++++++++ cgithub/views/error.eta | 11 +++++++++++ 6 files changed, 58 insertions(+), 10 deletions(-) create mode 120000 cgithub/GEMINI.md create mode 100644 cgithub/views/error.eta diff --git a/cgithub/GEMINI.md b/cgithub/GEMINI.md new file mode 120000 index 0000000..681311e --- /dev/null +++ b/cgithub/GEMINI.md @@ -0,0 +1 @@ +CLAUDE.md \ No newline at end of file diff --git a/cgithub/package.json b/cgithub/package.json index 6faa90b..3d339a4 100644 --- a/cgithub/package.json +++ b/cgithub/package.json @@ -7,7 +7,7 @@ "scripts": { "dev": "node --watch --watch-preserve-output src/index.ts", "build": "rm -rf dist/* && esbuild --bundle --format=esm --platform=node --outdir=dist/build --sourcemap src/index.ts && cp -R static views dist/", - "test": "node --test src/scraper.test.ts" + "test": "node --test --test-concurrency=4 src/scraper.test.ts" }, "dependencies": { "@hono/node-server": "^1.13.7", diff --git a/cgithub/src/index.ts b/cgithub/src/index.ts index 388b1e0..51e8cf2 100644 --- a/cgithub/src/index.ts +++ b/cgithub/src/index.ts @@ -1,10 +1,11 @@ import { createAdaptorServer, serve } from "@hono/node-server"; import { serveStatic } from "@hono/node-server/serve-static"; import { Eta } from "eta"; -import { Hono } from "hono"; +import { type Context, Hono } from "hono"; +import type { StatusCode } from "hono/utils/http-status"; import path from "node:path"; import url from "node:url"; -import { getGitHubBlob, getGitHubRepo, getGitHubTree } from "./scraper.ts"; +import { getGitHubBlob, getGitHubRepo, getGitHubTree, GitHubHTTPError } from "./scraper.ts"; const __dirname = path.dirname(url.fileURLToPath(import.meta.url)); @@ -17,22 +18,36 @@ app.get("/", async (c) => { return c.html(await eta.renderAsync("home.eta", {})); }); +async function tryRender(c: Context, template: string, promise: Promise) { + let data: T; + try { + data = await promise; + } catch (e) { + if (e instanceof GitHubHTTPError) { + c.status(e.status as StatusCode); + const message = `GitHub responded with HTTP ${e.status} ${e.message}`; + return c.html(await eta.renderAsync("error.eta", { title: e.message, message })); + } else { + c.status(500); + return c.html(await eta.renderAsync("error.eta", { message: "" + e })); + } + } + return c.html(await eta.renderAsync(template, data)); +} + app.get("/:owner/:repo", async (c) => { const { owner, repo } = c.req.param(); - const data = await getGitHubRepo(owner, repo); - return c.html(await eta.renderAsync("repo.eta", data)); + return tryRender(c, "repo.eta", getGitHubRepo(owner, repo)); }); app.get("/:owner/:repo/tree/:branch/:path{.*}?", async (c) => { const { owner, repo, branch, path = "" } = c.req.param(); - const data = await getGitHubTree(owner, repo, branch, path); - return c.html(await eta.renderAsync("tree.eta", data)); + return tryRender(c, "tree.eta", getGitHubTree(owner, repo, branch, path)); }); app.get("/:owner/:repo/blob/:branch/:path{.*}", async (c) => { const { owner, repo, branch, path } = c.req.param(); - const data = await getGitHubBlob(owner, repo, branch, path); - return c.html(await eta.renderAsync("blob.eta", data)); + return tryRender(c, "blob.eta", getGitHubBlob(owner, repo, branch, path)); }); app.get("/:owner/:repo/raw/:branch/:path{.*}", async (c) => { diff --git a/cgithub/src/scraper.test.ts b/cgithub/src/scraper.test.ts index 3784190..adf70ae 100644 --- a/cgithub/src/scraper.test.ts +++ b/cgithub/src/scraper.test.ts @@ -1,6 +1,6 @@ import assert from "node:assert"; import { describe, it } from "node:test"; -import { getGitHubBlob, getGitHubRepo, getGitHubTree } from "./scraper.ts"; +import { getGitHubBlob, getGitHubRepo, getGitHubTree, GitHubHTTPError } from "./scraper.ts"; describe("GitHub scraper", () => { describe("actions/deploy-pages repository", () => { @@ -67,4 +67,14 @@ describe("GitHub scraper", () => { assert.match(data.content, /^MIT License\n\n/); }); }); + + describe("error handling", () => { + it("should throw for non-existent repository", async () => { + await assert.rejects(getGitHubRepo("nosuchowner", "nosuchrepo"), (err) => { + assert(err instanceof GitHubHTTPError, "error should be an HTTPError"); + assert.strictEqual(err.status, 404); + return true; + }); + }); + }); }); diff --git a/cgithub/src/scraper.ts b/cgithub/src/scraper.ts index 4feda73..b76f8c8 100644 --- a/cgithub/src/scraper.ts +++ b/cgithub/src/scraper.ts @@ -1,3 +1,13 @@ +export class GitHubHTTPError extends Error { + status: number; + + constructor(status: number, message: string) { + super(message); + this.name = "GitHubHTTPError"; + this.status = status; + } +} + interface TreeItem { contentType: "directory" | "file"; name: string; @@ -78,6 +88,7 @@ async function fetchGitHubPage(path: string): Promise { "Accept-Language": "en-US,en;q=0.5", }; const response = await fetch(`https://github.com/${path}`, { headers }); + if (!response.ok) throw new GitHubHTTPError(response.status, response.statusText); return response.text(); } diff --git a/cgithub/views/error.eta b/cgithub/views/error.eta new file mode 100644 index 0000000..784a843 --- /dev/null +++ b/cgithub/views/error.eta @@ -0,0 +1,11 @@ + + + + + <%= it.title ?? it.message %> + + + + <%= it.message %> + + -- cgit v1.3.1 From 64c397e256b01a2e542e84529e947e237c07edd6 Mon Sep 17 00:00:00 2001 From: Joe Mou Date: Tue, 27 Jan 2026 16:58:56 -0500 Subject: Split branch and path bread crumbs into separate section --- cgithub/static/style.css | 7 +++++-- cgithub/views/_nav.eta | 18 ++++++++++++++++++ cgithub/views/_tree.eta | 2 ++ cgithub/views/blob.eta | 4 +++- cgithub/views/layout.eta | 22 ++-------------------- cgithub/views/repo.eta | 2 +- cgithub/views/tree.eta | 4 ++-- 7 files changed, 33 insertions(+), 26 deletions(-) create mode 100644 cgithub/views/_nav.eta diff --git a/cgithub/static/style.css b/cgithub/static/style.css index d21bd32..cccb1ba 100644 --- a/cgithub/static/style.css +++ b/cgithub/static/style.css @@ -39,7 +39,11 @@ a { color: var(--link-color); } -body > header, .box { +section { + margin: 1rem 0; +} + +.box { --box-padding: .5rem; margin: 1rem 0; @@ -79,7 +83,6 @@ footer { } .repo { - margin-bottom: 1rem; font-size: 120%; font-weight: bold; } diff --git a/cgithub/views/_nav.eta b/cgithub/views/_nav.eta new file mode 100644 index 0000000..338a6a6 --- /dev/null +++ b/cgithub/views/_nav.eta @@ -0,0 +1,18 @@ +
+
branch <%= it.branch %>
+ <% if (it.path && it.path !== "/") { %> + (top)<% + const pathParts = it.path.split('/').filter(p => p); + let currentPath = ''; + for (let i = 0; i < pathParts.length; i++) { + const part = pathParts[i]; + currentPath += (currentPath ? '/' : '') + part; + const isLast = i === pathParts.length - 1; + if (isLast) { + %>/<%= part %><%= it.isTree ? "/" : "" %><% + } else { + %>/<%= part %><% + } + } %> + <% } %> +
diff --git a/cgithub/views/_tree.eta b/cgithub/views/_tree.eta index d0c3453..311762f 100644 --- a/cgithub/views/_tree.eta +++ b/cgithub/views/_tree.eta @@ -1,3 +1,5 @@ +<%~ include("./_nav.eta", {...it, isTree: true}) %> + <% for (const item of it.items) { %> diff --git a/cgithub/views/blob.eta b/cgithub/views/blob.eta index 7328a62..277b46d 100644 --- a/cgithub/views/blob.eta +++ b/cgithub/views/blob.eta @@ -1,4 +1,6 @@ -<% layout("./layout") %> +<% layout("./layout.eta") %> + +<%~ include("./_nav.eta", it) %>
<% if (it.language) { %> diff --git a/cgithub/views/layout.eta b/cgithub/views/layout.eta index bfcd370..f053b84 100644 --- a/cgithub/views/layout.eta +++ b/cgithub/views/layout.eta @@ -6,29 +6,11 @@ -
+
<%= it.repo.owner %>/<%= it.repo.name %>
-
branch <%= it.branch %>
- <% if (it.path) { %> - - (top)<% - const pathParts = it.path.split('/').filter(p => p); - let currentPath = ''; - for (let i = 0; i < pathParts.length; i++) { - const part = pathParts[i]; - currentPath += (currentPath ? '/' : '') + part; - const isLast = i === pathParts.length - 1; - if (isLast) { - %>/<%= part %><% - } else { - %>/<%= part %><% - } - } %> - - <% } %>
git clone https://github.com/<%= it.repo.owner %>/<%= it.repo.name %>.git
 git clone git@github.com:<%= it.repo.owner %>/<%= it.repo.name %>.git
-
+
<%~ it.body %> diff --git a/cgithub/views/repo.eta b/cgithub/views/repo.eta index 5bebb95..9122775 100644 --- a/cgithub/views/repo.eta +++ b/cgithub/views/repo.eta @@ -1,4 +1,4 @@ -<% layout("./layout") %> +<% layout("./layout.eta") %> <% if (it.info) { %>
diff --git a/cgithub/views/tree.eta b/cgithub/views/tree.eta index 7068754..6e675e1 100644 --- a/cgithub/views/tree.eta +++ b/cgithub/views/tree.eta @@ -1,3 +1,3 @@ -<% layout("./layout") %> +<% layout("./layout.eta") %> -<%~ include("./_tree", it) %> +<%~ include("./_tree.eta", it) %> -- cgit v1.3.1 From d29bdad2f5c8326d111901373f0a56b3a09a2cd8 Mon Sep 17 00:00:00 2001 From: Joe Mou Date: Tue, 27 Jan 2026 15:05:36 -0500 Subject: List issues --- cgithub/CLAUDE.md | 18 ++++------ cgithub/src/index.ts | 13 ++++++- cgithub/src/scraper.test.ts | 21 ++++++++++- cgithub/src/scraper.ts | 85 +++++++++++++++++++++++++++++++++++++++++++-- cgithub/static/style.css | 40 +++++++++++++++++++++ cgithub/views/issues.eta | 20 +++++++++++ cgithub/views/layout.eta | 4 +++ cgithub/views/repo.eta | 2 +- 8 files changed, 185 insertions(+), 18 deletions(-) create mode 100644 cgithub/views/issues.eta diff --git a/cgithub/CLAUDE.md b/cgithub/CLAUDE.md index e258e55..8feb529 100644 --- a/cgithub/CLAUDE.md +++ b/cgithub/CLAUDE.md @@ -1,10 +1,6 @@ -# CLAUDE.md - -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. - ## Project Overview -cgithub is a lightweight GitHub alternative frontend that scrapes GitHub's server-rendered HTML instead of using the API. This approach avoids API rate limits entirely by extracting embedded JSON data from ``, + ``, "g", ); @@ -241,11 +271,8 @@ function parsePayload(html: string, dataTarget: string, prop: string[]): RawPayl } catch { continue; } - for (const key of prop) { - data = data[key]; - if (data === undefined) continue match; - } - return data; + if (data.payload === undefined) continue match; + return data.payload; } return null; } @@ -329,24 +356,31 @@ function parseRepoInfo(html: string): RepoInfo { }; } -function extractOverviewHtml(payload: RawPayload): Record | undefined { +function extractOverviewHtml( + overviewFiles: OverviewFile[] = [], +): Record | undefined { const result: Record = {}; - for (const file of payload.overview?.overviewFiles ?? []) { - if (file.loaded && file.richText !== undefined) { + for (const file of overviewFiles) { + if (file.richText !== undefined) { result[file.displayName] = file.richText; } } return Object.keys(result).length > 0 ? result : undefined; } -function extractGitHub(payload: RawPayload, extra: T): GitHubNav & T { +function extractGitHub(payload: AppPayload | null, extra: T): GitHubNav & T { + const layoutRoute = payload?.codeViewLayoutRoute; + if (layoutRoute === undefined) { + throw new Error("Missing codeViewLayoutRoute"); + } + return { repo: { - owner: payload.repo.ownerLogin, - name: payload.repo.name, + owner: layoutRoute.repo.ownerLogin, + name: layoutRoute.repo.name, }, - branch: payload.refInfo.name, - path: payload.path, + branch: layoutRoute.refInfo.name, + path: layoutRoute.path, ...extra, }; } @@ -354,16 +388,17 @@ function extractGitHub(payload: RawPayload, extra: T): GitHubNav & T { export async function getGitHubRepo(owner: string, repo: string): Promise { const html = await fetchGitHubPage(`${owner}/${repo}`); - const payload = parsePayload(html, "react-partial.embeddedData", ["props", "initialPayload"]); + const payload = parsePayload(html); + const repoRoute = payload?.codeViewRepoRoute; - if (payload?.tree === undefined) { + if (repoRoute?.tree === undefined) { throw new Error("Could not find tree data in embedded JSON"); } const info = parseRepoInfo(html); - const overviewHtml = extractOverviewHtml(payload); + const overviewHtml = extractOverviewHtml(repoRoute.overview?.overviewFiles); - return extractGitHub(payload, { items: payload.tree.items, info, overviewHtml }); + return extractGitHub(payload, { items: repoRoute.tree.items, info, overviewHtml }); } export async function getGitHubTree( @@ -374,20 +409,14 @@ export async function getGitHubTree( ): Promise { const html = await fetchGitHubPage(`${owner}/${repo}/tree/${branch}/${path}`); - // GitHub uses different formats for root vs subdirectories. - const payload = - // Root directory format. - parsePayload(html, "react-partial.embeddedData", ["props", "initialPayload"]) || - // Subdirectory format. - parsePayload(html, "react-app.embeddedData", ["payload"]); + const payload = parsePayload(html); + const treeRoute = payload?.codeViewTreeRoute; - if (payload?.tree === undefined) { + if (treeRoute?.tree === undefined) { throw new Error("Could not find tree data in embedded JSON"); } - const overviewHtml = extractOverviewHtml(payload); - - return extractGitHub(payload, { items: payload.tree.items, overviewHtml }); + return extractGitHub(payload, { items: treeRoute.tree.items }); } export async function getGitHubBlob( @@ -398,42 +427,42 @@ export async function getGitHubBlob( ): Promise { const html = await fetchGitHubPage(`${owner}/${repo}/blob/${branch}/${path}`); - const payload = parsePayload(html, "react-app.embeddedData", ["payload"]); + const payload = parsePayload(html); + const blob = payload?.codeViewBlobLayoutRoute?.blob; + const styledBlob = payload?.["codeViewBlobLayoutRoute.StyledBlob"]; - if (payload?.blob === undefined) { + if (blob === undefined) { throw new Error("Could not find blob data in embedded JSON"); } - const blob = payload.blob; - const { headerInfo, stylingDirectives } = blob; - - let htmlLines = blob.colorizedLines; - if (!htmlLines && stylingDirectives && blob.rawLines) { - htmlLines = blob.rawLines.map((line, i) => applyStyling(line, stylingDirectives[i])); + let htmlLines = styledBlob?.colorizedLines ?? null; + const stylingDirectives = styledBlob?.stylingDirectives; + if (!htmlLines && stylingDirectives && styledBlob?.rawLines) { + htmlLines = styledBlob.rawLines.map((line, i) => applyStyling(line, stylingDirectives[i])); } - let size = headerInfo.blobSize; - if (headerInfo.lineInfo.truncatedLoc) { - size += ` / ${headerInfo.lineInfo.truncatedLoc} lines`; + let size = blob.headerInfo.blobSize; + if (blob.headerInfo.lineInfo?.truncatedLoc) { + size += ` / ${blob.headerInfo.lineInfo.truncatedLoc} lines`; } - if (headerInfo.lineInfo.truncatedSloc) { - size += ` / ${headerInfo.lineInfo.truncatedSloc} loc`; + if (blob.headerInfo.lineInfo?.truncatedSloc) { + size += ` / ${blob.headerInfo.lineInfo.truncatedSloc} loc`; } return extractGitHub(payload, { language: blob.language || null, size, image: blob.image, - textLines: blob.rawLines, + textLines: styledBlob?.rawLines ?? null, htmlLines, - htmlContent: blob.richText, + htmlContent: payload?.codeViewBlobRoute?.richText ?? null, }); } export async function getGitHubIssues(owner: string, repo: string): Promise { const html = await fetchGitHubPage(`${owner}/${repo}/issues`); - const payload = parsePayload(html, "react-app.embeddedData", ["payload"]); + const payload = parsePayload(html); let issuesQuery: IssueIndexPageQuery | undefined; for (const query of payload?.preloadedQueries ?? []) { @@ -470,7 +499,7 @@ export async function getGitHubCommits( : `${owner}/${repo}/commits/${branch}`; const html = await fetchGitHubPage(urlPath); - const payload = parsePayload(html, "react-app.embeddedData", ["payload"]); + const payload = parsePayload(html); if (payload?.commitGroups === undefined) { throw new Error("Could not find commit history in embedded JSON"); -- cgit v1.3.1 From 4662a229e06468a0805a18c6175bc0fd3a4ea985 Mon Sep 17 00:00:00 2001 From: Joe Mou Date: Sat, 14 Mar 2026 00:35:48 -0400 Subject: Fix tree overviewHtml: use tree.readme instead of overview.overviewFiles --- cgithub/src/scraper.test.ts | 25 +++++++++++-------------- cgithub/src/scraper.ts | 5 ++++- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/cgithub/src/scraper.test.ts b/cgithub/src/scraper.test.ts index ad3ed46..1abab29 100644 --- a/cgithub/src/scraper.test.ts +++ b/cgithub/src/scraper.test.ts @@ -38,30 +38,27 @@ describe("GitHub scraper", () => { }); it("should fetch tree", async () => { - const data = await getGitHubTree("actions", "deploy-pages", "main", "src"); + const data = await getGitHubTree("github", "rally", "main", "lib"); - assert.strictEqual(data.repo.owner, "actions"); - assert.strictEqual(data.repo.name, "deploy-pages"); + assert.strictEqual(data.repo.owner, "github"); + assert.strictEqual(data.repo.name, "rally"); assert.strictEqual(data.branch, "main"); - assert.strictEqual(data.path, "src"); + assert.strictEqual(data.path, "lib"); assert.deepStrictEqual(data.items, [ { - name: "__tests__", - path: "src/__tests__", - contentType: "directory", - }, - { - name: "internal", - path: "src/internal", - contentType: "directory", + name: "README.md", + path: "lib/README.md", + contentType: "file", }, { - name: "index.js", - path: "src/index.js", + name: "RallyValidate.js", + path: "lib/RallyValidate.js", contentType: "file", }, ]); + + assert.ok(Object.keys(data.overviewHtml ?? {}).includes("README.md")); }); it("should fetch text blob", async () => { diff --git a/cgithub/src/scraper.ts b/cgithub/src/scraper.ts index ac30259..cd743a0 100644 --- a/cgithub/src/scraper.ts +++ b/cgithub/src/scraper.ts @@ -101,6 +101,7 @@ interface CodeViewRepoRoute { interface CodeViewTreeRoute { tree: { items: TreeItem[]; + readme?: OverviewFile; }; } @@ -416,7 +417,9 @@ export async function getGitHubTree( throw new Error("Could not find tree data in embedded JSON"); } - return extractGitHub(payload, { items: treeRoute.tree.items }); + const overviewHtml = extractOverviewHtml(treeRoute.tree.readme ? [treeRoute.tree.readme] : []); + + return extractGitHub(payload, { items: treeRoute.tree.items, overviewHtml }); } export async function getGitHubBlob( -- cgit v1.3.1 From daa17f2542a56cae4e15bfd9da2e8ade9147fbdb Mon Sep 17 00:00:00 2001 From: Joe Mou Date: Mon, 16 Mar 2026 23:38:05 -0400 Subject: Add viewport meta tag for mobile --- cgithub/views/layout.eta | 1 + 1 file changed, 1 insertion(+) diff --git a/cgithub/views/layout.eta b/cgithub/views/layout.eta index c42d593..ec3c7b8 100644 --- a/cgithub/views/layout.eta +++ b/cgithub/views/layout.eta @@ -2,6 +2,7 @@ + <%= it.repo.owner %>/<%= it.repo.name %> - <%= it.path || '/' %> -- cgit v1.3.1 From 294c8f2802c92ab1d85cb6a160740dbf8983a7e2 Mon Sep 17 00:00:00 2001 From: Joe Mou Date: Mon, 20 Apr 2026 21:24:15 -0400 Subject: Update for unexpected actions/deploy-pages release --- cgithub/src/scraper.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cgithub/src/scraper.test.ts b/cgithub/src/scraper.test.ts index 1abab29..a843561 100644 --- a/cgithub/src/scraper.test.ts +++ b/cgithub/src/scraper.test.ts @@ -29,9 +29,9 @@ describe("GitHub scraper", () => { assert.strictEqual(data.info.stars?.length, 3); assert.strictEqual(data.info.watchers?.length, 2); assert.strictEqual(data.info.forks?.length, 3); - assert.strictEqual(data.info.numReleases, 36); - assert.strictEqual(data.info.latestReleaseTag, "v4.0.5"); - assert.ok(data.info.latestReleaseDate?.startsWith("2024-03-18")); + assert.strictEqual(data.info.numReleases, 38); + assert.strictEqual(data.info.latestReleaseTag, "v5.0.0"); + assert.ok(data.info.latestReleaseDate?.startsWith("2026-03-25")); assert.ok(data.items.length > 0); assert.ok(Object.keys(data.overviewHtml ?? {}).includes("README.md")); -- cgit v1.3.1 From d9a6f6cc7c6a374647486530c6246df980c4be6b Mon Sep 17 00:00:00 2001 From: Joe Mou Date: Mon, 20 Apr 2026 17:14:11 -0400 Subject: Fallback to issue titleHtml --- cgithub/src/scraper.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cgithub/src/scraper.ts b/cgithub/src/scraper.ts index cd743a0..5d17e3f 100644 --- a/cgithub/src/scraper.ts +++ b/cgithub/src/scraper.ts @@ -26,7 +26,8 @@ interface OverviewFile { interface IssueNode { number: number; - title: string; + title?: string; + titleHtml?: string; author?: { login: string; }; @@ -478,8 +479,9 @@ export async function getGitHubIssues(owner: string, repo: string): Promise { - const { number, title, state, createdAt } = edge.node; - return { number, title, state, createdAt }; + const { number, title, titleHtml, state, createdAt } = edge.node; + const finalTitle = (title || titleHtml || "").replace(/<[^>]+>/g, ""); + return { number, title: finalTitle, state, createdAt }; }); return { -- cgit v1.3.1 From dbe8141dce07559c1e16f17800dfd505b0ecdb46 Mon Sep 17 00:00:00 2001 From: Joe Mou Date: Mon, 20 Apr 2026 17:14:11 -0400 Subject: Apply overlapping styling drectives instead of duplicating content --- cgithub/src/scraper.test.ts | 24 +++++++++++++++++++++++- cgithub/src/scraper.ts | 45 ++++++++++++++++++++++++++++++++++++++------- 2 files changed, 61 insertions(+), 8 deletions(-) diff --git a/cgithub/src/scraper.test.ts b/cgithub/src/scraper.test.ts index a843561..4432231 100644 --- a/cgithub/src/scraper.test.ts +++ b/cgithub/src/scraper.test.ts @@ -95,7 +95,7 @@ describe("GitHub scraper", () => { assert.strictEqual(data.textLines?.[0], "# deploy-pages 🚀"); assert.strictEqual( data.htmlLines?.[0], - '# deploy-pages 🚀deploy-pages 🚀deploy-pages 🚀', + '# deploy-pages 🚀', ); const firstLine = /^

deploy-pages 🚀<\/h1>/; @@ -197,6 +197,28 @@ describe("GitHub scraper", () => { }); }); + describe("dustinlyons/nixos-config repository", () => { + it("should fetch nix blob with combined overlapping styling directives", async () => { + const data = await getGitHubBlob("dustinlyons", "nixos-config", "main", "flake.nix"); + + assert.strictEqual(data.repo.owner, "dustinlyons"); + assert.strictEqual(data.repo.name, "nixos-config"); + assert.strictEqual(data.path, "flake.nix"); + assert.strictEqual(data.image, false); + assert.strictEqual(data.htmlContent, null); + assert.ok(data.htmlLines !== null && data.htmlLines.length > 0); + + // Line 2 has directives [2,13,"pl-c1"] and [2,13,"pl-s1"] with identical ranges. + // They should be combined into one span, not duplicated. + const line2 = data.htmlLines[1]; + assert.ok(line2.includes('class="pl-c1 pl-s1"'), `expected combined classes in: ${line2}`); + assert.ok( + (line2.match(/description/g) ?? []).length === 1, + `expected "description" to appear once in: ${line2}`, + ); + }); + }); + describe("rich file types", () => { it("should support PNG", async () => { const data = await getGitHubBlob("github", "docs", "main", "assets/images/site/logo.png"); diff --git a/cgithub/src/scraper.ts b/cgithub/src/scraper.ts index 5d17e3f..2dcd1d0 100644 --- a/cgithub/src/scraper.ts +++ b/cgithub/src/scraper.ts @@ -153,15 +153,46 @@ function escapeHtml(text: string): string { .replace(/'/g, "'"); } -function applyStyling(line: string, directives: StylingDirective[]): string { +// Renders directives[i..] within [rangeStart, rangeEnd], producing nested spans. +// Returns the rendered content and the index of the first directive not consumed. +function renderRange( + line: string, + directives: StylingDirective[], + i: number, + rangeStart: number, + rangeEnd: number, +): [string, number] { let result = ""; - let lastPos = 0; - for (const [start, end, className] of directives) { - result += escapeHtml(line.substring(lastPos, start)); - result += `${escapeHtml(line.substring(start, end))}`; - lastPos = end; + let pos = rangeStart; + + while (i < directives.length) { + const [start, end, className] = directives[i]; + // Stop at directives that start outside or extend past this range. + if (start >= rangeEnd || end > rangeEnd) break; + + // Combine classes for directives with identical ranges. + let classes = className; + let j = i + 1; + while (j < directives.length && directives[j][0] === start && directives[j][1] === end) { + classes += " " + directives[j][2]; + j++; + } + + result += escapeHtml(line.substring(pos, start)); + const [innerContent, nextI] = renderRange(line, directives, j, start, end); + result += `${innerContent}`; + pos = end; + i = nextI; } - result += escapeHtml(line.substring(lastPos)); + + result += escapeHtml(line.substring(pos, rangeEnd)); + return [result, i]; +} + +function applyStyling(line: string, directives: StylingDirective[]): string { + // Sort by start ascending, then end descending so outer spans precede nested ones. + const sorted = [...directives].sort((a, b) => (a[0] !== b[0] ? a[0] - b[0] : b[1] - a[1])); + const [result] = renderRange(line, sorted, 0, 0, line.length); return result; } -- cgit v1.3.1 From cfe74db6c177151da3d1bc8549732e3d39448040 Mon Sep 17 00:00:00 2001 From: Joe Mou Date: Sat, 25 Apr 2026 13:35:17 -0400 Subject: Support redirects --- cgithub/src/index.ts | 5 ++++- cgithub/src/scraper.test.ts | 26 ++++++++++++++++++++++++++ cgithub/src/scraper.ts | 21 ++++++++++++++++++++- 3 files changed, 50 insertions(+), 2 deletions(-) diff --git a/cgithub/src/index.ts b/cgithub/src/index.ts index 59c98ee..eb3a0ee 100644 --- a/cgithub/src/index.ts +++ b/cgithub/src/index.ts @@ -14,6 +14,7 @@ import { getGitHubRepo, getGitHubTree, GitHubHTTPError, + RedirectError, } from "./scraper.ts"; const __dirname = path.dirname(url.fileURLToPath(import.meta.url)); @@ -32,7 +33,9 @@ async function tryRender(c: Context, template: string, promise try { data = await promise; } catch (e) { - if (e instanceof GitHubHTTPError) { + if (e instanceof RedirectError) { + return c.redirect(e.location); + } else if (e instanceof GitHubHTTPError) { c.status(e.status as StatusCode); const message = `GitHub responded with HTTP ${e.status} ${e.message}`; return c.html(await eta.renderAsync("error.eta", { title: e.message, message })); diff --git a/cgithub/src/scraper.test.ts b/cgithub/src/scraper.test.ts index 4432231..c20a5ff 100644 --- a/cgithub/src/scraper.test.ts +++ b/cgithub/src/scraper.test.ts @@ -9,6 +9,7 @@ import { getGitHubRepo, getGitHubTree, GitHubHTTPError, + RedirectError, } from "./scraper.ts"; describe("GitHub scraper", () => { @@ -245,5 +246,30 @@ describe("GitHub scraper", () => { return true; }); }); + + it("should redirect when fetching a directory as a blob", async () => { + await assert.rejects( + getGitHubBlob("badlogic", "pi-mono", "main", "packages/agent"), + (err) => { + assert(err instanceof RedirectError, "error should be a RedirectError"); + assert.strictEqual(err.location, "/badlogic/pi-mono/tree/main/packages/agent"); + return true; + }, + ); + }); + + it("should redirect when fetching a file as a tree", async () => { + await assert.rejects( + getGitHubTree("badlogic", "pi-mono", "main", "packages/agent/package.json"), + (err) => { + assert(err instanceof RedirectError, "error should be a RedirectError"); + assert.strictEqual( + err.location, + "/badlogic/pi-mono/blob/main/packages/agent/package.json", + ); + return true; + }, + ); + }); }); }); diff --git a/cgithub/src/scraper.ts b/cgithub/src/scraper.ts index 2dcd1d0..04a685b 100644 --- a/cgithub/src/scraper.ts +++ b/cgithub/src/scraper.ts @@ -13,6 +13,16 @@ export class GitHubHTTPError extends Error { } } +export class RedirectError extends Error { + location: string; + + constructor(location: string) { + super(`Redirect to ${location}`); + this.name = "RedirectError"; + this.location = location; + } +} + interface TreeItem { contentType: "directory" | "file"; name: string; @@ -286,7 +296,16 @@ async function fetchGitHubPage(path: string): Promise { Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", "Accept-Language": "en-US,en;q=0.5", }; - const response = await fetch(`https://github.com/${path}`, { headers }); + const response = await fetch(`https://github.com/${path}`, { headers, redirect: "manual" }); + if (response.status === 301 || response.status === 302) { + const location = response.headers.get("location"); + if (location) { + const url = new URL(location); + if (url.hostname === "github.com") { + throw new RedirectError(url.pathname); + } + } + } if (!response.ok) throw new GitHubHTTPError(response.status, response.statusText); return response.text(); } -- cgit v1.3.1 From 123da65350888dc0fbe04003c5ffc36fdded3a46 Mon Sep 17 00:00:00 2001 From: Joe Mou Date: Wed, 27 May 2026 14:10:19 -0400 Subject: Update unit tests --- cgithub/src/scraper.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cgithub/src/scraper.test.ts b/cgithub/src/scraper.test.ts index c20a5ff..42ddd6e 100644 --- a/cgithub/src/scraper.test.ts +++ b/cgithub/src/scraper.test.ts @@ -249,10 +249,10 @@ describe("GitHub scraper", () => { it("should redirect when fetching a directory as a blob", async () => { await assert.rejects( - getGitHubBlob("badlogic", "pi-mono", "main", "packages/agent"), + getGitHubBlob("earendil-works", "pi", "main", "packages/agent"), (err) => { assert(err instanceof RedirectError, "error should be a RedirectError"); - assert.strictEqual(err.location, "/badlogic/pi-mono/tree/main/packages/agent"); + assert.strictEqual(err.location, "/earendil-works/pi/tree/main/packages/agent"); return true; }, ); @@ -260,12 +260,12 @@ describe("GitHub scraper", () => { it("should redirect when fetching a file as a tree", async () => { await assert.rejects( - getGitHubTree("badlogic", "pi-mono", "main", "packages/agent/package.json"), + getGitHubTree("earendil-works", "pi", "main", "packages/agent/package.json"), (err) => { assert(err instanceof RedirectError, "error should be a RedirectError"); assert.strictEqual( err.location, - "/badlogic/pi-mono/blob/main/packages/agent/package.json", + "/earendil-works/pi/blob/main/packages/agent/package.json", ); return true; }, -- cgit v1.3.1 From c57123139794c3ef174b62c6bd2a54a60bbf3327 Mon Sep 17 00:00:00 2001 From: Joe Mou Date: Wed, 27 May 2026 14:02:36 -0400 Subject: Add repo-scoped search support for issues and PRs We do not support hybrid issue/PR search. Any searches on the issues endpoint will redirect to an explicit issue-only search. --- cgithub/src/index.ts | 26 +++++++++++++++++++++++++- cgithub/src/scraper.test.ts | 15 +++++++++++++++ cgithub/src/scraper.ts | 20 ++++++++++---------- cgithub/static/style.css | 13 ++++++++++--- cgithub/views/blob.eta | 2 +- cgithub/views/issues.eta | 2 +- cgithub/views/layout.eta | 8 ++++++++ cgithub/views/tree.eta | 2 +- 8 files changed, 71 insertions(+), 17 deletions(-) diff --git a/cgithub/src/index.ts b/cgithub/src/index.ts index eb3a0ee..db9915f 100644 --- a/cgithub/src/index.ts +++ b/cgithub/src/index.ts @@ -70,7 +70,31 @@ app.get("/:owner/:repo/raw/:branch/:path{.*}", async (c) => { app.get("/:owner/:repo/issues", async (c) => { const { owner, repo } = c.req.param(); - return tryRender(c, "issues.eta", getGitHubIssues(owner, repo)); + const q = c.req.query("q"); + + // GitHub supports combined issue/PR search, apparently for legacy. We just + // redirect to issue-only search. + if (q && !/\bis:issue\b/.test(q)) { + return c.redirect(`/${owner}/${repo}/issues?q=${encodeURIComponent(`is:issue ${q}`)}`); + } + + return tryRender(c, "issues.eta", getGitHubIssues(owner, repo, { q })); +}); + +app.get("/:owner/:repo/search", async (c) => { + const { owner, repo } = c.req.param(); + const q = c.req.query("q") ?? ""; + const type = c.req.query("type") ?? "issues"; + + if (type === "issues") { + return c.redirect(`/${owner}/${repo}/issues?q=${encodeURIComponent(q)}`); + } + + // Redirect unhandled search types (like code, which requires sign in anyway). + c.header("Referrer-Policy", "no-referrer"); + return c.redirect( + `https://github.com/${owner}/${repo}/search?${new URLSearchParams(c.req.query()).toString()}`, + ); }); app.get("/:owner/:repo/commits/:branch/:path{.*}?", async (c) => { diff --git a/cgithub/src/scraper.test.ts b/cgithub/src/scraper.test.ts index 42ddd6e..236b0f5 100644 --- a/cgithub/src/scraper.test.ts +++ b/cgithub/src/scraper.test.ts @@ -127,6 +127,7 @@ describe("GitHub scraper", () => { assert.strictEqual(data.repo.owner, "actions"); assert.strictEqual(data.repo.name, "deploy-pages"); + assert.strictEqual(data.q, undefined); const issue402 = data.issues.find((issue) => issue.number === 402); assert.ok(issue402); @@ -135,6 +136,20 @@ describe("GitHub scraper", () => { assert.strictEqual(issue402.createdAt, "2025-07-03T19:07:08Z"); }); + it("should fetch filtered issues with q", async () => { + const data = await getGitHubIssues("actions", "deploy-pages", { + q: "is:issue state:closed", + }); + + assert.strictEqual(data.repo.owner, "actions"); + assert.strictEqual(data.repo.name, "deploy-pages"); + assert.strictEqual(data.q, "is:issue state:closed"); + const issue417 = data.issues.find((issue) => issue.number === 417); + assert.ok(issue417); + assert.strictEqual(issue417.title, "Chats"); + assert.ok(data.issues.every((issue) => issue.title !== "")); + }); + it("should fetch commits", async () => { const data = await getGitHubCommits("actions", "deploy-pages", "main", ""); diff --git a/cgithub/src/scraper.ts b/cgithub/src/scraper.ts index 04a685b..2412e49 100644 --- a/cgithub/src/scraper.ts +++ b/cgithub/src/scraper.ts @@ -258,6 +258,7 @@ interface Issue { export interface GitHubIssues extends GitHubCommon { issues: Issue[]; + q?: string; } export interface GitHubCommits extends GitHubNav { @@ -513,9 +514,14 @@ export async function getGitHubBlob( }); } -export async function getGitHubIssues(owner: string, repo: string): Promise { - const html = await fetchGitHubPage(`${owner}/${repo}/issues`); - +export async function getGitHubIssues( + owner: string, + repo: string, + // We only support is:issue searches (no PR results). + { q }: { q?: string } = {}, +): Promise { + const suffix = q ? `?q=${encodeURIComponent(q)}` : ""; + const html = await fetchGitHubPage(`${owner}/${repo}/issues${suffix}`); const payload = parsePayload(html); let issuesQuery: IssueIndexPageQuery | undefined; @@ -534,13 +540,7 @@ export async function getGitHubIssues(owner: string, repo: string): Promise a { display: block; padding-right: .5rem; diff --git a/cgithub/views/blob.eta b/cgithub/views/blob.eta index ea07059..50f92a9 100644 --- a/cgithub/views/blob.eta +++ b/cgithub/views/blob.eta @@ -1,4 +1,4 @@ -<% layout("./layout.eta") %> +<% layout("./layout.eta"); it.searchType = "code" %> <%~ include("./_nav.eta", it) %> diff --git a/cgithub/views/issues.eta b/cgithub/views/issues.eta index 6e4c9ba..646da46 100644 --- a/cgithub/views/issues.eta +++ b/cgithub/views/issues.eta @@ -1,4 +1,4 @@ -<% layout("./layout") %> +<% layout("./layout"); it.searchType = "issues" %>

<% for (const issue of it.issues) { %> diff --git a/cgithub/views/layout.eta b/cgithub/views/layout.eta index ec3c7b8..f31c6e0 100644 --- a/cgithub/views/layout.eta +++ b/cgithub/views/layout.eta @@ -19,6 +19,14 @@ git clone git@github.com:<%= it.repo.owner %>/<%= it.repo.name %>.gitCommits <% } %> + + "> + + + <%~ it.body %> diff --git a/cgithub/views/tree.eta b/cgithub/views/tree.eta index 6e675e1..41ded06 100644 --- a/cgithub/views/tree.eta +++ b/cgithub/views/tree.eta @@ -1,3 +1,3 @@ -<% layout("./layout.eta") %> +<% layout("./layout.eta"); it.searchType = "code" %> <%~ include("./_tree.eta", it) %> -- cgit v1.3.1 From 745ac9d83234cb060bf802527efd90d301ecc491 Mon Sep 17 00:00:00 2001 From: Joe Mou Date: Wed, 27 May 2026 14:02:36 -0400 Subject: Pull request index --- cgithub/src/index.ts | 9 +++++++++ cgithub/src/scraper.test.ts | 11 +++++++++++ cgithub/src/scraper.ts | 40 ++++++++++++++++++++++++++++++++++++++++ cgithub/views/layout.eta | 2 ++ cgithub/views/pulls.eta | 15 +++++++++++++++ 5 files changed, 77 insertions(+) create mode 100644 cgithub/views/pulls.eta diff --git a/cgithub/src/index.ts b/cgithub/src/index.ts index db9915f..514f2e2 100644 --- a/cgithub/src/index.ts +++ b/cgithub/src/index.ts @@ -9,6 +9,7 @@ import { getGitHubBlob, getGitHubCommits, getGitHubIssues, + getGitHubPulls, getGitHubRelease, getGitHubReleases, getGitHubRepo, @@ -81,6 +82,12 @@ app.get("/:owner/:repo/issues", async (c) => { return tryRender(c, "issues.eta", getGitHubIssues(owner, repo, { q })); }); +app.get("/:owner/:repo/pulls", async (c) => { + const { owner, repo } = c.req.param(); + const q = c.req.query("q"); + return tryRender(c, "pulls.eta", getGitHubPulls(owner, repo, { q })); +}); + app.get("/:owner/:repo/search", async (c) => { const { owner, repo } = c.req.param(); const q = c.req.query("q") ?? ""; @@ -88,6 +95,8 @@ app.get("/:owner/:repo/search", async (c) => { if (type === "issues") { return c.redirect(`/${owner}/${repo}/issues?q=${encodeURIComponent(q)}`); + } else if (type === "pullrequests") { + return c.redirect(`/${owner}/${repo}/pulls?q=${encodeURIComponent(q)}`); } // Redirect unhandled search types (like code, which requires sign in anyway). diff --git a/cgithub/src/scraper.test.ts b/cgithub/src/scraper.test.ts index 236b0f5..a4b834e 100644 --- a/cgithub/src/scraper.test.ts +++ b/cgithub/src/scraper.test.ts @@ -4,6 +4,7 @@ import { getGitHubBlob, getGitHubCommits, getGitHubIssues, + getGitHubPulls, getGitHubRelease, getGitHubReleases, getGitHubRepo, @@ -150,6 +151,16 @@ describe("GitHub scraper", () => { assert.ok(data.issues.every((issue) => issue.title !== "")); }); + it("should fetch pull requests", async () => { + const data = await getGitHubPulls("actions", "deploy-pages", { q: "state:closed" }); + + assert.strictEqual(data.repo.owner, "actions"); + assert.strictEqual(data.repo.name, "deploy-pages"); + const pr411 = data.pulls.find((pull) => pull.number === 411); + assert.ok(pr411); + assert.strictEqual(pr411.title, "update node version"); + }); + it("should fetch commits", async () => { const data = await getGitHubCommits("actions", "deploy-pages", "main", ""); diff --git a/cgithub/src/scraper.ts b/cgithub/src/scraper.ts index 2412e49..f32a8da 100644 --- a/cgithub/src/scraper.ts +++ b/cgithub/src/scraper.ts @@ -261,6 +261,11 @@ export interface GitHubIssues extends GitHubCommon { q?: string; } +export interface GitHubPulls extends GitHubCommon { + pulls: Issue[]; + q?: string; +} + export interface GitHubCommits extends GitHubNav { commitGroups: CommitGroup[]; } @@ -543,6 +548,41 @@ export async function getGitHubIssues( return { repo: { owner, name: repo }, issues, q }; } +export async function getGitHubPulls( + owner: string, + repo: string, + { q }: { q?: string } = {}, +): Promise { + const suffix = q ? `?q=${encodeURIComponent(q)}` : ""; + const html = await fetchGitHubPage(`${owner}/${repo}/pulls${suffix}`); + const document = parseDocument(html); + + const pulls: Issue[] = []; + for (const row of cssSelect.selectAll(".js-issue-row", document) as unknown as Element[]) { + const id = getAttributeValue(row, "id"); // "issue_NNN" + const number = id ? parseInt(id.replace("issue_", ""), 10) : 0; + if (!number) continue; + + const titleLink = cssSelect.selectOne("a.markdown-title", row); + const title = titleLink ? textContent(titleLink).trim() : ""; + + const timeElem = cssSelect.selectOne("relative-time", row); + const createdAt = timeElem ? getAttributeValue(timeElem, "datetime") || "" : ""; + + const stateSpan = cssSelect.selectOne(".tooltipped[aria-label]", row) as Element | null; + const ariaLabel = stateSpan ? (getAttributeValue(stateSpan, "aria-label") ?? "") : ""; + const state = ariaLabel.includes("Merged") + ? "MERGED" + : ariaLabel.includes("Closed") + ? "CLOSED" + : "OPEN"; + + pulls.push({ number, title, state, createdAt }); + } + + return { repo: { owner, name: repo }, pulls, q }; +} + export async function getGitHubCommits( owner: string, repo: string, diff --git a/cgithub/views/layout.eta b/cgithub/views/layout.eta index f31c6e0..85f431c 100644 --- a/cgithub/views/layout.eta +++ b/cgithub/views/layout.eta @@ -14,6 +14,7 @@ git clone git@github.com:<%= it.repo.owner %>/<%= it.repo.name %>.git Code Issues + Pull requests Releases <% if (it.branch) { %> Commits @@ -23,6 +24,7 @@ git clone git@github.com:<%= it.repo.owner %>/<%= it.repo.name %>.git"> diff --git a/cgithub/views/pulls.eta b/cgithub/views/pulls.eta new file mode 100644 index 0000000..9ec5ca1 --- /dev/null +++ b/cgithub/views/pulls.eta @@ -0,0 +1,15 @@ +<% layout("./layout"); it.searchType = "pullrequests" %> + +
+ <% for (const pull of it.pulls) { %> + + + + + + <% } %> +
#<%= pull.number %> + + <%= pull.title %> + +
-- cgit v1.3.1 From 4b929ed4b27993874567516c40765ca5b53291df Mon Sep 17 00:00:00 2001 From: Joe Mou Date: Wed, 27 May 2026 23:17:30 -0400 Subject: Basic keyboard shortcut handling --- cgithub/views/layout.eta | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/cgithub/views/layout.eta b/cgithub/views/layout.eta index 85f431c..dd523d1 100644 --- a/cgithub/views/layout.eta +++ b/cgithub/views/layout.eta @@ -21,7 +21,7 @@ git clone git@github.com:<%= it.repo.owner %>/<%= it.repo.name %>.git
- "> + "> "> - - -
- - - <%~ it.body %> - -
Generated by cgithub commit <%= it.buildCommit %> · View original
- - - - diff --git a/cgithub/views/layouts/base.eta b/cgithub/views/layouts/base.eta new file mode 100644 index 0000000..fb644f5 --- /dev/null +++ b/cgithub/views/layouts/base.eta @@ -0,0 +1,17 @@ + + + + + + <%= it.title %> + + <% for (const src of it.scripts ?? []) { %> + + <% } %> + + + <%~ it.body %> + +
Generated by cgithub commit <%= it.buildCommit %> · View original
+ + diff --git a/cgithub/views/layouts/repo.eta b/cgithub/views/layouts/repo.eta new file mode 100644 index 0000000..e4f7880 --- /dev/null +++ b/cgithub/views/layouts/repo.eta @@ -0,0 +1,59 @@ +<% +layout("/layouts/base.eta", { + title: `${it.repo.owner}/${it.repo.name} - ${it.path || "/"}`, + scripts: ["/static/refs.js"], +}) +%> + +
+ +
git clone https://github.com/<%= it.repo.owner %>/<%= it.repo.name %>.git
+git clone git@github.com:<%= it.repo.owner %>/<%= it.repo.name %>.git
+ +
+ "> + + +
+
+ +<%~ it.body %> + + diff --git a/cgithub/views/owner.eta b/cgithub/views/owner.eta index e5667fb..b3304fb 100644 --- a/cgithub/views/owner.eta +++ b/cgithub/views/owner.eta @@ -1,65 +1,54 @@ - - - - - - <%= it.name ? `${it.name} (${it.login})` : it.login %> - - - -
- <% if (it.avatarUrl) { %> - - <% } %> -
<%= it.name || it.login %>
-
<%= it.login %><%= it.type === "organization" ? " (organization)" : "" %>
- <% if (it.location) { %> -
<%= it.location %>
- <% } %> - <% if (it.nav.length > 0) { %> - - <% } %> -
+<% layout("/layouts/base.eta", { title: it.name ? `${it.name} (${it.login})` : it.login }) %> - <% if (it.bio || it.links.length > 0 || it.followers || it.company) { %> -
- <% if (it.bio) { %> -
<%= it.bio %>
- <% } %> - <% for (const link of it.links) { %> - - <% } %> - <% if (it.followers) { %> -
<%= it.followers %> followers<%= it.following ? ` · ${it.following} following` : "" %>
- <% } %> - <% if (it.company) { %> -
<%= it.company %>
- <% } %> -
+
+ <% if (it.avatarUrl) { %> + <% } %> - - <% if (it.pinned.length > 0) { %> -
-

Pinned

- <% for (const repo of it.pinned) { %> - <%~ include("./_ownerRepo", { repo, login: it.login }) %> - <% } %> -
+
<%= it.name || it.login %>
+
<%= it.login %><%= it.type === "organization" ? " (organization)" : "" %>
+ <% if (it.location) { %> +
<%= it.location %>
<% } %> - - <% if (it.repos.length > 0) { %> -
-

Repositories

- <% for (const repo of it.repos) { %> - <%~ include("./_ownerRepo", { repo, login: it.login }) %> + <% if (it.nav.length > 0) { %> +
+ <% } %> +
+ +<% if (it.bio || it.links.length > 0 || it.followers || it.company) { %> +
+ <% if (it.bio) { %> +
<%= it.bio %>
+ <% } %> + <% for (const link of it.links) { %> + + <% } %> + <% if (it.followers) { %> +
<%= it.followers %> followers<%= it.following ? ` · ${it.following} following` : "" %>
+ <% } %> + <% if (it.company) { %> +
<%= it.company %>
+ <% } %> +
+<% } %> -
Generated by cgithub
- - +<% if (it.pinned.length > 0) { %> +
+

Pinned

+ <% for (const repo of it.pinned) { %> + <%~ include("./_ownerRepo", { repo, login: it.login }) %> + <% } %> +
+<% } %> + +<% if (it.repos.length > 0) { %> +
+

Repositories

+ <% for (const repo of it.repos) { %> + <%~ include("./_ownerRepo", { repo, login: it.login }) %> + <% } %> +
+<% } %> diff --git a/cgithub/views/pulls.eta b/cgithub/views/pulls.eta index 9ec5ca1..697f6b1 100644 --- a/cgithub/views/pulls.eta +++ b/cgithub/views/pulls.eta @@ -1,4 +1,4 @@ -<% layout("./layout"); it.searchType = "pullrequests" %> +<% layout("/layouts/repo.eta"); it.searchType = "pullrequests" %> <% for (const pull of it.pulls) { %> diff --git a/cgithub/views/release.eta b/cgithub/views/release.eta index 46dd5db..64cca35 100644 --- a/cgithub/views/release.eta +++ b/cgithub/views/release.eta @@ -1,3 +1,3 @@ -<% layout("./layout.eta") %> +<% layout("/layouts/repo.eta") %> <%~ include("./_release.eta", { repo: it.repo, release: it.release, linkToDetail: false }) %> diff --git a/cgithub/views/releases.eta b/cgithub/views/releases.eta index 840908c..4ede28b 100644 --- a/cgithub/views/releases.eta +++ b/cgithub/views/releases.eta @@ -1,4 +1,4 @@ -<% layout("./layout.eta") %> +<% layout("/layouts/repo.eta") %> <% for (const release of it.releases) { %> <%~ include("./_release.eta", { repo: it.repo, release, linkToDetail: true }) %> diff --git a/cgithub/views/repo.eta b/cgithub/views/repo.eta index 5dc6beb..60746df 100644 --- a/cgithub/views/repo.eta +++ b/cgithub/views/repo.eta @@ -1,4 +1,4 @@ -<% layout("./layout.eta") %> +<% layout("/layouts/repo.eta") %> <% if (it.info) { %>
diff --git a/cgithub/views/tree.eta b/cgithub/views/tree.eta index 41ded06..fabf5b9 100644 --- a/cgithub/views/tree.eta +++ b/cgithub/views/tree.eta @@ -1,3 +1,3 @@ -<% layout("./layout.eta"); it.searchType = "code" %> +<% layout("/layouts/repo.eta"); it.searchType = "code" %> <%~ include("./_tree.eta", it) %> diff --git a/cgithub/views/wiki.eta b/cgithub/views/wiki.eta index d6047e1..781caff 100644 --- a/cgithub/views/wiki.eta +++ b/cgithub/views/wiki.eta @@ -1,4 +1,4 @@ -<% layout("./layout.eta"); it.path = "wiki/" + it.title %> +<% layout("/layouts/repo.eta"); it.path = "wiki/" + it.title %> <%~ include("./_wikiNav.eta", it) %> diff --git a/cgithub/views/wikiPages.eta b/cgithub/views/wikiPages.eta index 58d6d21..eb90781 100644 --- a/cgithub/views/wikiPages.eta +++ b/cgithub/views/wikiPages.eta @@ -1,4 +1,4 @@ -<% layout("./layout.eta"); it.path = "wiki/_pages" %> +<% layout("/layouts/repo.eta"); it.path = "wiki/_pages" %> <%~ include("./_wikiNav.eta", it) %> -- cgit v1.3.1 From 484cdb8dbceb572e4b858c80109cb81b671fd2fd Mon Sep 17 00:00:00 2001 From: Joe Mou Date: Sat, 5 Sep 2026 17:31:53 -0400 Subject: Add screenshot --- cgithub/README.md | 2 ++ cgithub/docs/screenshot.png | Bin 0 -> 91956 bytes 2 files changed, 2 insertions(+) create mode 100644 cgithub/docs/screenshot.png diff --git a/cgithub/README.md b/cgithub/README.md index a625116..cfcdccd 100644 --- a/cgithub/README.md +++ b/cgithub/README.md @@ -3,6 +3,8 @@ JavaScript and doesn't require API keys. It runs on Cloudflare Workers; to use it, deploy your own instance then replace `github.com` in your URL with your domain. +![Screenshot of cgithub repository page.](docs/screenshot.png) + ## Development To run the server locally: diff --git a/cgithub/docs/screenshot.png b/cgithub/docs/screenshot.png new file mode 100644 index 0000000..68d230b Binary files /dev/null and b/cgithub/docs/screenshot.png differ -- cgit v1.3.1 From a7bc82cad42b7b8c4c8c52e1217f7f005b632e6d Mon Sep 17 00:00:00 2001 From: Joe Mou Date: Wed, 9 Sep 2026 11:03:02 -0400 Subject: Serve GitHub's global search when it names one repository /search?q=repo:owner/name+...&type=code matched the /:owner route and rendered a profile page for an owner named "search", which GitHub 404s. It is the URL GitHub's own search box produces, so it arrives whenever a search is redirected here. A /search route now takes it: when the query carries a repo: qualifier and a type, it strips the qualifier and redirects to that repository's search route, which already maps issues and pull requests to pages we render and hands the rest to GitHub. Searches that name no single repository -- bare queries, org:/user: scopes, a missing type -- keep redirecting to github.com/search as they did before. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01GkGX6puS45xD5KzdV9tHEA --- cgithub/src/app.test.ts | 32 ++++++++++++++++++++++++++++++++ cgithub/src/app.ts | 19 +++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/cgithub/src/app.test.ts b/cgithub/src/app.test.ts index 274b2d5..8720a23 100644 --- a/cgithub/src/app.test.ts +++ b/cgithub/src/app.test.ts @@ -358,6 +358,38 @@ describe("wiki", () => { }); }); +describe("global search", () => { + it("should keep the rest of the query around the qualifier", async () => { + const res = await app.request( + "http://cgithub.example/search?q=" + + encodeURIComponent("is:issue repo:actions/deploy-pages label:bug") + + "&type=issues", + ); + + assert.strictEqual( + res.headers.get("location"), + "/actions/deploy-pages/search?q=is%3Aissue+label%3Abug&type=issues", + ); + }); + + it("should redirect searches it cannot scope to a repository", async () => { + for (const url of [ + "http://cgithub.example/search?q=dms&type=code", + // An owner-wide search is not a single repository. + "http://cgithub.example/search?q=" + + encodeURIComponent("org:nix-community dms") + + "&type=code", + // Without a type there is no search to hand over. + "http://cgithub.example/search?q=" + encodeURIComponent("repo:actions/deploy-pages dms"), + ]) { + const res = await app.request(url); + + assert.strictEqual(res.status, 200, url); + assert.match(await res.text(), /url=https:\/\/github.com\/search\?/); + } + }); +}); + describe("redirects to GitHub", () => { // An HTTP redirect would be caught by the extension's declarativeNetRequest // rules and bounced straight back here. diff --git a/cgithub/src/app.ts b/cgithub/src/app.ts index a040aaa..7e5e9e8 100644 --- a/cgithub/src/app.ts +++ b/cgithub/src/app.ts @@ -95,6 +95,25 @@ export function createApp(eta: Eta) { return c.render("home.eta"); }); + // GitHub's global search, which we only serve when it is scoped to a single + // repository: strip the qualifier and let that repository's search route + // decide what to do with the rest. + app.get("/search", async (c) => { + const q = c.req.query("q") ?? ""; + const type = c.req.query("type"); + + const repoQualifier = /(?:^|\s)repo:([^/\s]+)\/([^/\s]+)(?=\s|$)/.exec(q); + if (type && repoQualifier) { + const [qualifier, owner, repo] = repoQualifier; + const rest = + q.slice(0, repoQualifier.index) + q.slice(repoQualifier.index + qualifier.length); + const params = new URLSearchParams({ q: rest.trim(), type }); + return c.redirect(`/${owner}/${repo}/search?${params}`); + } + + return redirectToGitHub(c); + }); + app.get("/:owner", async (c) => { const { owner } = c.req.param(); return c.render("owner.eta", await getGitHubOwner(owner)); -- cgit v1.3.1 From 95f4269f1a1c3b93ae2b060c7935b03c773e9f4a Mon Sep 17 00:00:00 2001 From: Joe Mou Date: Wed, 9 Sep 2026 14:47:12 -0400 Subject: Add feature list to README.md --- cgithub/CLAUDE.md | 3 ++- cgithub/README.md | 59 +++++++++++++++++++++++++++++-------------------------- 2 files changed, 33 insertions(+), 29 deletions(-) diff --git a/cgithub/CLAUDE.md b/cgithub/CLAUDE.md index 6c6dc60..ef7b133 100644 --- a/cgithub/CLAUDE.md +++ b/cgithub/CLAUDE.md @@ -41,4 +41,5 @@ pnpm test ## Instructions - Use `for (const ... of ...)` instead of `.forEach()` -- Update `scraper.test.ts` \ No newline at end of file +- Update `scraper.test.ts` +- When implementing a new feature update `README.md` diff --git a/cgithub/README.md b/cgithub/README.md index cfcdccd..533c88f 100644 --- a/cgithub/README.md +++ b/cgithub/README.md @@ -5,41 +5,40 @@ domain. ![Screenshot of cgithub repository page.](docs/screenshot.png) -## Development +## Features -To run the server locally: -``` -$ pnpm run dev -``` +Implemented: -To run tests: -``` -$ pnpm test -``` +- Repository overview +- Directory listing and file browsing (`tree`/`blob`) +- Commit history (but this often results in 429 Too Many Requests on Cloudflare) +- Issue and pull request lists and search +- Release lists and details +- Wiki +- User and organization profiles -## Deployment +Not implemented: -To deploy to Cloudflare Workers: -``` -$ pnpm run deploy -``` - -The worker name and other settings live in `wrangler.jsonc`. +- Issue, pull request, and commit details +- Issue and pull request status and labels +- Code search (GitHub requires signing in) +- Pagination (only the first page of issues, pull requests, commits, releases, + branches, and tags is shown) +- Blame, contributors, stargazer/fork lists, network graphs, and insights +- Actions, checks, projects, packages, discussions, and gists +- Profile tabs other than the overview (repositories, projects, packages, + people, sponsoring) +- Anything requiring an account: signing in, starring, commenting, editing, or + any other write action +- Private repositories ## Browser extension redirects -Pages cgithub can't render (marketing pages, code search, anything unrouted) are -redirected back to github.com. A browser extension that rewrites github.com URLs -to a cgithub instance with `declarativeNetRequest` will match those redirects -too and send the request straight back, looping forever. - -To avoid this, cgithub never answers with an HTTP redirect: it serves a small -page that navigates to github.com with a two-second ``. -Because that navigation is initiated by the cgithub origin, the extension can -let it through with `excludedInitiatorDomains`; without that, the user at least -lands on a page they can stop on, with a link, instead of an endless loop. - -A rule that redirects to a cgithub instance should therefore look like this: +cgithub redirects unsupported routes back to github.com. A browser extension +that rewrites github.com URLs to a cgithub instance with `declarativeNetRequest` +will match those redirects and send the request back, looping forever. To avoid +this, the extension should configure `excludedInitiatorDomains` with the cgithub +domain: ```jsonc { @@ -56,6 +55,10 @@ A rule that redirects to a cgithub instance should therefore look like this: } ``` +## Related + +- [GotHub](https://codeberg.org/gothub/gothub) + ## Disclosures AI coding assistants, in particular Claude, are used in the development process. -- cgit v1.3.1 From 198f2a07f68a348dc0823b2bde3bec541e06db01 Mon Sep 17 00:00:00 2001 From: Joe Mou Date: Thu, 10 Sep 2026 00:53:48 -0400 Subject: Redirect /notifications instead of scraping it as an owner A signed-out request for github.com/notifications is bounced to a login page, so the owner route followed that redirect and left the visitor on /login. Hand the path over to GitHub before it reaches the route. --- cgithub/README.md | 4 ++-- cgithub/src/app.test.ts | 7 +++++++ cgithub/src/app.ts | 5 +++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/cgithub/README.md b/cgithub/README.md index 533c88f..45c94cf 100644 --- a/cgithub/README.md +++ b/cgithub/README.md @@ -28,8 +28,8 @@ Not implemented: - Actions, checks, projects, packages, discussions, and gists - Profile tabs other than the overview (repositories, projects, packages, people, sponsoring) -- Anything requiring an account: signing in, starring, commenting, editing, or - any other write action +- Anything requiring an account: signing in, notifications, starring, + commenting, editing, or any other write action - Private repositories ## Browser extension redirects diff --git a/cgithub/src/app.test.ts b/cgithub/src/app.test.ts index 8720a23..c028e1b 100644 --- a/cgithub/src/app.test.ts +++ b/cgithub/src/app.test.ts @@ -441,6 +441,13 @@ describe("redirects to GitHub", () => { ); }); + it("should redirect notifications without scraping them as a profile", async () => { + const res = await app.request("http://cgithub.example/notifications?query=is%3Aunread"); + + assert.strictEqual(res.status, 200); + assert.match(await res.text(), /url=https:\/\/github.com\/notifications\?query=is%3Aunread"/); + }); + it("should redirect raw file requests to raw.githubusercontent.com", async () => { const res = await app.request("http://cgithub.example/actions/deploy-pages/raw/main/README.md"); diff --git a/cgithub/src/app.ts b/cgithub/src/app.ts index 7e5e9e8..0b9a7e6 100644 --- a/cgithub/src/app.ts +++ b/cgithub/src/app.ts @@ -95,6 +95,11 @@ export function createApp(eta: Eta) { return c.render("home.eta"); }); + // Exclusion that immediately redirects to GitHub. + app.get("/notifications", async (c) => { + return redirectToGitHub(c); + }); + // GitHub's global search, which we only serve when it is scoped to a single // repository: strip the qualifier and let that repository's search route // decide what to do with the rest. -- cgit v1.3.1 From 01494e6fa1345a509560e5e19a7f08cb2aac5cc8 Mon Sep 17 00:00:00 2001 From: Joe Mou Date: Thu, 10 Sep 2026 14:57:44 -0400 Subject: Render the error page in the shared page shell error.eta carried its own copy of the HTML shell, so error pages missed the viewport meta tag, the build-commit footer, and the link to the original GitHub page. The renderer already passes githubUrl and buildCommit to every template, so the layout needs nothing new from app.ts. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01JZAT3YzYP85DihSvQbDKB6 --- cgithub/src/app.test.ts | 23 +++++++++++++++++++++++ cgithub/views/error.eta | 14 +++----------- 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/cgithub/src/app.test.ts b/cgithub/src/app.test.ts index c028e1b..450c1cc 100644 --- a/cgithub/src/app.test.ts +++ b/cgithub/src/app.test.ts @@ -458,3 +458,26 @@ describe("redirects to GitHub", () => { ); }); }); + +describe("error page", () => { + it("should render inside the page shell", () => { + const html = eta.render("error.eta", { + title: "Not Found", + message: "GitHub responded with HTTP 404 Not Found", + buildCommit: "v0-g1234567", + githubUrl: "https://github.com/actions/deploy-pages", + }); + + assert.match(html, /Not Found<\/title>/); + assert.match(html, /GitHub responded with HTTP 404 Not Found/); + assert.match(html, /<footer>Generated by cgithub commit <code>v0-g1234567<\/code>/); + assert.match(html, /href="https:\/\/github.com\/actions\/deploy-pages"/); + }); + + // The title falls back to the message for errors that carry no status text. + it("should title itself with the message when it has no title", () => { + const html = eta.render("error.eta", { message: "Error: boom" }); + + assert.match(html, /<title>Error: boom<\/title>/); + }); +}); diff --git a/cgithub/views/error.eta b/cgithub/views/error.eta index 784a843..a46411e 100644 --- a/cgithub/views/error.eta +++ b/cgithub/views/error.eta @@ -1,11 +1,3 @@ -<!DOCTYPE html> -<html lang="en"> -<head> - <meta charset="UTF-8"> - <title><%= it.title ?? it.message %> - - - - <%= it.message %> - - +<% layout("/layouts/base.eta", { title: it.title ?? it.message }) %> + +<%= it.message %> -- cgit v1.3.1