diff options
| -rw-r--r-- | CLAUDE.md | 18 | ||||
| -rw-r--r-- | src/index.ts | 13 | ||||
| -rw-r--r-- | src/scraper.test.ts | 21 | ||||
| -rw-r--r-- | src/scraper.ts | 85 | ||||
| -rw-r--r-- | static/style.css | 40 | ||||
| -rw-r--r-- | views/issues.eta | 20 | ||||
| -rw-r--r-- | views/layout.eta | 4 | ||||
| -rw-r--r-- | views/repo.eta | 2 |
8 files changed, 185 insertions, 18 deletions
@@ -1,10 +1,6 @@ -# CLAUDE.md - -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. - ## Project Overview -cgithub is a lightweight GitHub alternative frontend that scrapes GitHub's server-rendered HTML instead of using the API. This approach avoids API rate limits entirely by extracting embedded JSON data from `<script type="application/json">` tags that GitHub includes for hydration. +cgithub is a lightweight GitHub alternative frontend that scrapes GitHub's server-rendered HTML instead of using the API. Data is usually extracted from embedded JSON data from `<script type="application/json">` tags that GitHub includes for hydration. ## Commands @@ -15,8 +11,8 @@ pnpm run dev # Production build (bundles to dist/) pnpm run build -# Deploy to production server -make deploy +# Run tests +pnpm test ``` ## Architecture @@ -30,13 +26,11 @@ make deploy **Templates:** `views/*.eta` - Eta templates with layout inheritance (`layout.eta` as base) **Routes:** -- `GET /` - Home page - `GET /:owner/:repo` - Repository info with directory listing - `GET /:owner/:repo/tree/:branch/:path*` - Directory listing - `GET /:owner/:repo/blob/:branch/:path` - File content view -**Scraper pattern:** GitHub embeds tree/blob data in two formats depending on page type: -- `react-partial.embeddedData` (props.initialPayload) for root directories -- `react-app.embeddedData` (payload) for subdirectories and blobs +## Instructions -The scraper uses browser-like headers to avoid throttling. +- Use `for (const ... of ...)` instead of `.forEach()` +- Update `scraper.test.ts`
\ No newline at end of file diff --git a/src/index.ts b/src/index.ts index 51e8cf2..d4229fd 100644 --- a/src/index.ts +++ b/src/index.ts @@ -5,7 +5,13 @@ import { type Context, Hono } from "hono"; import type { StatusCode } from "hono/utils/http-status"; import path from "node:path"; import url from "node:url"; -import { getGitHubBlob, getGitHubRepo, getGitHubTree, GitHubHTTPError } from "./scraper.ts"; +import { + getGitHubBlob, + getGitHubIssues, + getGitHubRepo, + getGitHubTree, + GitHubHTTPError, +} from "./scraper.ts"; const __dirname = path.dirname(url.fileURLToPath(import.meta.url)); @@ -56,6 +62,11 @@ app.get("/:owner/:repo/raw/:branch/:path{.*}", async (c) => { return c.redirect(`https://raw.githubusercontent.com/${owner}/${repo}/${branch}/${path}`); }); +app.get("/:owner/:repo/issues", async (c) => { + const { owner, repo } = c.req.param(); + return tryRender(c, "issues.eta", getGitHubIssues(owner, repo)); +}); + app.all("*", async (c) => { const path = c.req.path; c.header("Referrer-Policy", "no-referrer"); diff --git a/src/scraper.test.ts b/src/scraper.test.ts index adf70ae..e7e65ba 100644 --- a/src/scraper.test.ts +++ b/src/scraper.test.ts @@ -1,6 +1,12 @@ import assert from "node:assert"; import { describe, it } from "node:test"; -import { getGitHubBlob, getGitHubRepo, getGitHubTree, GitHubHTTPError } from "./scraper.ts"; +import { + getGitHubBlob, + getGitHubIssues, + getGitHubRepo, + getGitHubTree, + GitHubHTTPError, +} from "./scraper.ts"; describe("GitHub scraper", () => { describe("actions/deploy-pages repository", () => { @@ -66,6 +72,19 @@ describe("GitHub scraper", () => { assert.match(data.rawContent, /^MIT License\n\n/); assert.match(data.content, /^MIT License\n\n/); }); + + it("should fetch issues", async () => { + const data = await getGitHubIssues("actions", "deploy-pages"); + + assert.strictEqual(data.repo.owner, "actions"); + assert.strictEqual(data.repo.name, "deploy-pages"); + + const issue402 = data.issues.find((issue) => issue.number === 402); + assert.ok(issue402); + assert.strictEqual(issue402.title, "Dry Run"); + assert.strictEqual(issue402.state, "OPEN"); + assert.strictEqual(issue402.createdAt, "2025-07-03T19:07:08Z"); + }); }); describe("error handling", () => { diff --git a/src/scraper.ts b/src/scraper.ts index b76f8c8..bbf7511 100644 --- a/src/scraper.ts +++ b/src/scraper.ts @@ -21,6 +21,38 @@ interface OverviewFile { loaded: boolean; } +interface IssueNode { + number: number; + title: string; + author?: { + login: string; + }; + createdAt: string; + state: string; + labels?: { + edges: { + node: { + name: string; + }; + }[]; + }; +} + +interface IssueIndexPageQuery { + queryName: "IssueIndexPageQuery"; + result: { + data: { + repository: { + search: { + edges: { + node: IssueNode; + }[]; + }; + }; + }; + }; +} + interface RawPayload { repo: { ownerLogin: string; @@ -42,6 +74,8 @@ interface RawPayload { overview?: { overviewFiles?: OverviewFile[]; }; + // Actually there may be other query types. + preloadedQueries?: IssueIndexPageQuery[]; } interface GitHubCommon { @@ -49,11 +83,14 @@ interface GitHubCommon { owner: string; name: string; }; +} + +interface GitHubNav extends GitHubCommon { branch: string; path: string; } -export interface GitHubTree extends GitHubCommon { +export interface GitHubTree extends GitHubNav { items: TreeItem[]; overviewHtml?: Record<string, string>; } @@ -71,13 +108,26 @@ export interface GitHubRepo extends GitHubTree { info: RepoInfo; } -export interface GitHubBlob extends GitHubCommon { +export interface GitHubBlob extends GitHubNav { language: string | null; size: number | null; content: string; rawContent: string; } +interface Issue { + number: number; + title: string; + state: string; + createdAt: string; + numReplies?: number; + numPRs?: number; +} + +export interface GitHubIssues extends GitHubCommon { + issues: Issue[]; +} + async function fetchGitHubPage(path: string): Promise<string> { // GitHub throttles/blocks requests without realistic browser headers. // These headers make the request appear as a standard browser visit. @@ -150,7 +200,7 @@ function extractOverviewHtml(payload: RawPayload): Record<string, string> | unde return Object.keys(result).length > 0 ? result : undefined; } -function extractGitHub<T>(payload: RawPayload, extra: T): GitHubCommon & T { +function extractGitHub<T>(payload: RawPayload, extra: T): GitHubNav & T { return { repo: { owner: payload.repo.ownerLogin, @@ -231,3 +281,32 @@ export async function getGitHubBlob( size, }); } + +export async function getGitHubIssues(owner: string, repo: string): Promise<GitHubIssues> { + const html = await fetchGitHubPage(`${owner}/${repo}/issues`); + + const payload = parsePayload(html, "react-app.embeddedData", ["payload"]); + + let issuesQuery: IssueIndexPageQuery | undefined; + for (const query of payload?.preloadedQueries ?? []) { + if (query.queryName === "IssueIndexPageQuery") { + issuesQuery = query; + } + } + if (issuesQuery === undefined) { + throw new Error("Could not find IssueIndexPageQuery in embedded JSON"); + } + + const issues: Issue[] = issuesQuery.result.data.repository.search.edges.map((edge) => { + const { number, title, state, createdAt } = edge.node; + return { number, title, state, createdAt }; + }); + + return { + repo: { + owner, + name: repo, + }, + issues, + }; +} diff --git a/static/style.css b/static/style.css index cccb1ba..7618937 100644 --- a/static/style.css +++ b/static/style.css @@ -39,6 +39,41 @@ a { color: var(--link-color); } +table { + width: 100%; + border-collapse: collapse; +} + +tr { + border: 1px solid var(--border-color); +} + +td { + padding: .25rem; + vertical-align: top; + &:first-child { + padding-left: .5rem; + } + &:last-child { + padding-right: .5rem; + } +} + +nav { + display: flex; + > a { + display: block; + padding-right: .5rem; + + a { + padding-left: .5rem; + border-left: 1px solid var(--border-color); + } + &:last-child { + padding-right: 0; + } + } +} + section { margin: 1rem 0; } @@ -134,3 +169,8 @@ pre.box { display: none; } } + +.timestamp { + white-space: nowrap; + font-size: .8rem; +} diff --git a/views/issues.eta b/views/issues.eta new file mode 100644 index 0000000..6e4c9ba --- /dev/null +++ b/views/issues.eta @@ -0,0 +1,20 @@ +<% layout("./layout") %> + +<table> + <% for (const issue of it.issues) { %> + <tr> + <td>#<%= issue.number %></td> + <td> + <a href="/<%= it.repo.owner %>/<%= it.repo.name %>/issues/<%= issue.number %>"> + <%= issue.title %> + </a> + </td> + <td> + <% if (issue.commentCount > 0) { %> + <img src="/static/icons/link.svg" alt="comments" class="icon"> <%= issue.commentCount %> + <% } %> + </td> + <td class="timestamp"><time title="<%= issue.createdAt %>"><%= issue.createdAt.split("T")[0] %></time></td> + </tr> + <% } %> +</table> diff --git a/views/layout.eta b/views/layout.eta index f053b84..f4535df 100644 --- a/views/layout.eta +++ b/views/layout.eta @@ -10,6 +10,10 @@ <div class="repo"><%= it.repo.owner %>/<%= it.repo.name %></div> <pre><code>git clone https://github.com/<%= it.repo.owner %>/<%= it.repo.name %>.git git clone git@github.com:<%= it.repo.owner %>/<%= it.repo.name %>.git</code></pre> + <nav> + <a href="/<%= it.repo.owner %>/<%= it.repo.name %>">Code</a> + <a href="/<%= it.repo.owner %>/<%= it.repo.name %>/issues">Issues</a> + </nav> </div> <%~ it.body %> diff --git a/views/repo.eta b/views/repo.eta index 9122775..fbca299 100644 --- a/views/repo.eta +++ b/views/repo.eta @@ -18,7 +18,7 @@ <span class="count"><img src="/static/icons/git-fork.svg" alt="forks" class="icon"> <%= it.info.forks %></span> <% } %> <% if (it.info.numReleases) { %> - <div><a href="/<%= it.repo.owner %>/<%= it.repo.name %>/releases" rel="noreferrer"><%= it.info.numReleases %> releases</a></div> + <div><a href="/<%= it.repo.owner %>/<%= it.repo.name %>/releases"><%= it.info.numReleases %> releases</a></div> <% } %> </div> <% } %> |