diff options
Diffstat (limited to 'cgithub/src')
| -rw-r--r-- | cgithub/src/index.ts | 42 | ||||
| -rw-r--r-- | cgithub/src/scraper.ts | 212 |
2 files changed, 103 insertions, 151 deletions
diff --git a/cgithub/src/index.ts b/cgithub/src/index.ts index 525ac74..fcd0069 100644 --- a/cgithub/src/index.ts +++ b/cgithub/src/index.ts @@ -14,45 +14,15 @@ const eta = new Eta({ views: path.join(__dirname, "..", "views") }); app.use("/static/*", serveStatic({ root: path.join(__dirname, "..") })); app.get("/:owner/:repo/tree/:branch/:path{.*}?", async (c) => { - try { - const owner = c.req.param("owner"); - const repo = c.req.param("repo"); - const branch = c.req.param("branch"); - const path = c.req.param("path") || ""; - - const data = await getGitHubTree(owner, repo, branch, path); - const html = await eta.renderAsync("tree.eta", data); - - return c.html(html); - } catch (error) { - const errorMessage = - error instanceof Error ? error.message : "Unknown error"; - return c.html( - `<html><body><h1>Error</h1><p>${errorMessage}</p></body></html>`, - 500, - ); - } + const { owner, repo, branch, path = "" } = c.req.param(); + const data = await getGitHubTree(owner, repo, branch, path); + return c.html(await eta.renderAsync("tree.eta", data)); }); app.get("/:owner/:repo/blob/:branch/:path{.*}", async (c) => { - try { - const owner = c.req.param("owner"); - const repo = c.req.param("repo"); - const branch = c.req.param("branch"); - const path = c.req.param("path"); - - const data = await getGitHubBlob(owner, repo, branch, path); - const html = await eta.renderAsync("blob.eta", data); - - return c.html(html); - } catch (error) { - const errorMessage = - error instanceof Error ? error.message : "Unknown error"; - return c.html( - `<html><body><h1>Error</h1><p>${errorMessage}</p></body></html>`, - 500, - ); - } + const { owner, repo, branch, path } = c.req.param(); + const data = await getGitHubBlob(owner, repo, branch, path); + return c.html(await eta.renderAsync("blob.eta", data)); }); if (process.env.LISTEN_FDS === "1") { diff --git a/cgithub/src/scraper.ts b/cgithub/src/scraper.ts index fa8d655..d6702cc 100644 --- a/cgithub/src/scraper.ts +++ b/cgithub/src/scraper.ts @@ -1,146 +1,137 @@ -export interface GitHubTreeItem { +interface TreeItem { + contentType: "directory" | "file"; name: string; path: string; - contentType: "directory" | "file"; } -export interface GitHubTree { - path: string; - branch: string; - items: GitHubTreeItem[]; +interface RawPayload { repo: { + ownerLogin: string; name: string; - owner: string; - isPublic: boolean; + }; + refInfo: { name: string }; + path: string; + tree?: { + items: TreeItem[]; + }; + blob?: { + language?: string; + rawLines?: string[]; + highlightedLines?: { + text?: string; + }[]; + size: number; }; } -export interface GitHubBlob { - path: string; - branch: string; - content: string; - rawContent: string; - language: string | null; - size: number; +interface GitHubCommon { repo: { - name: string; owner: string; - isPublic: boolean; + name: string; }; + branch: string; + path: string; } -export async function getGitHubTree( - owner: string, - repo: string, - branch: string, - path: string, -): Promise<GitHubTree> { - const url = `https://github.com/${owner}/${repo}/tree/${branch}/${path}`; +export interface GitHubTree extends GitHubCommon { + items: TreeItem[]; +} + +export interface GitHubBlob extends GitHubCommon { + language: string | null; + size: number; + content: string; + rawContent: string; +} +async function fetchGitHubPage(path: string): Promise<string> { // GitHub throttles/blocks requests without realistic browser headers. // These headers make the request appear as a standard browser visit. - const response = await fetch(url, { - headers: { - "User-Agent": - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", - Accept: - "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", - "Accept-Language": "en-US,en;q=0.5", - }, - }); - const html = await response.text(); + const headers = { + "User-Agent": + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", + Accept: + "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.5", + }; + const response = await fetch(`https://github.com/${path}`, { headers }); + return response.text(); +} - // GitHub uses different formats for root vs subdirectories. - let payload; +function parsePayload( + html: string, + dataTarget: string, + prop: string[], +): RawPayload | null { + const regex = new RegExp( + `<script type="application/json" data-target="${dataTarget}">([^<]+)</script>`, + "g", + ); - // Try root format first. - const rootFormatMatches = [ - ...html.matchAll( - /<script type="application\/json" data-target="react-partial\.embeddedData">([^<]+)<\/script>/g, - ), - ]; - for (const match of rootFormatMatches) { + match: for (const match of html.matchAll(regex)) { + let data; try { - const data = JSON.parse(match[1]); - if (data.props?.initialPayload?.tree) { - payload = data.props.initialPayload; - break; - } - } catch (e) { + data = JSON.parse(match[1]); + } catch { continue; } - } - - // Try subdirectory format. - if (!payload) { - const dirFormatMatch = html.match( - /<script type="application\/json" data-target="react-app\.embeddedData">([^<]+)<\/script>/, - ); - if (dirFormatMatch) { - try { - const data = JSON.parse(dirFormatMatch[1]); - if (data.payload?.tree) { - payload = data.payload; - } - } catch (e) { - // Skip invalid JSON - } + for (const key of prop) { + data = data[key]; + if (data === undefined) continue match; } + return data; } + return null; +} - if (!payload) { - throw new Error("Could not find tree data in embedded JSON"); - } - +function extractGitHub<T>(payload: RawPayload, extra: T): GitHubCommon & T { return { - path: payload.path, - branch: payload.refInfo.name, - items: payload.tree.items, repo: { - name: payload.repo.name, owner: payload.repo.ownerLogin, - isPublic: payload.repo.public, + name: payload.repo.name, }, + branch: payload.refInfo.name, + path: payload.path, + ...extra, }; } -export async function getGitHubBlob( +export async function getGitHubTree( owner: string, repo: string, branch: string, path: string, -): Promise<GitHubBlob> { - const url = `https://github.com/${owner}/${repo}/blob/${branch}/${path}`; - - const response = await fetch(url, { - headers: { - "User-Agent": - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", - Accept: - "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", - "Accept-Language": "en-US,en;q=0.5", - }, - }); - const html = await response.text(); +): Promise<GitHubTree> { + const html = await fetchGitHubPage(`${owner}/${repo}/tree/${branch}/${path}`); - // GitHub embeds file data in a JSON script tag - let payload; + // GitHub uses different formats for root vs subdirectories. + const payload = + // Root directory format. + parsePayload(html, "react-partial.embeddedData", [ + "props", + "initialPayload", + ]) || + // Subdirectory format. + parsePayload(html, "react-app.embeddedData", ["payload"]); - const match = html.match( - /<script type="application\/json" data-target="react-app\.embeddedData">([^<]+)<\/script>/, - ); - if (match) { - try { - const data = JSON.parse(match[1]); - if (data.payload?.blob) { - payload = data.payload; - } - } catch (e) { - // Skip invalid JSON - } + if (payload?.tree === undefined) { + throw new Error("Could not find tree data in embedded JSON"); } - if (!payload) { + return extractGitHub(payload, { items: payload.tree.items }); +} + +export async function getGitHubBlob( + owner: string, + repo: string, + branch: string, + path: string, +): Promise<GitHubBlob> { + const html = await fetchGitHubPage(`${owner}/${repo}/blob/${branch}/${path}`); + + const payload = parsePayload(html, "react-app.embeddedData", ["payload"]); + + if (payload?.blob === undefined) { throw new Error("Could not find blob data in embedded JSON"); } @@ -148,21 +139,12 @@ export async function getGitHubBlob( const rawLines = blob.rawLines || []; const rawContent = rawLines.join("\n"); const highlightedLines = blob.highlightedLines || []; - const content = highlightedLines - .map((line: any) => line.text || "") - .join("\n"); + const content = highlightedLines.map((line) => line.text || "").join("\n"); - return { - path: payload.path, - branch: payload.refInfo.name, + return extractGitHub(payload, { content: content || rawContent, rawContent, language: blob.language || null, size: blob.size, - repo: { - name: payload.repo.name, - owner: payload.repo.ownerLogin, - isPublic: payload.repo.public, - }, - }; + }); } |