interface TreeItem { contentType: "directory" | "file"; name: string; path: string; } interface RawPayload { repo: { ownerLogin: string; name: string; }; refInfo: { name: string }; path: string; tree?: { items: TreeItem[]; }; blob?: { language?: string; rawLines?: string[]; highlightedLines?: { text?: string; }[]; size: number; }; } interface GitHubCommon { repo: { owner: string; name: string; }; branch: string; path: string; } export interface GitHubTree extends GitHubCommon { items: TreeItem[]; } export interface GitHubBlob extends GitHubCommon { language: string | null; size: number; content: string; rawContent: string; } async function fetchGitHubPage(path: string): Promise { // GitHub throttles/blocks requests without realistic browser headers. // These headers make the request appear as a standard browser visit. const headers = { "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", "Accept-Language": "en-US,en;q=0.5", }; const response = await fetch(`https://github.com/${path}`, { headers }); return response.text(); } function parsePayload( html: string, dataTarget: string, prop: string[], ): RawPayload | null { const regex = new RegExp( ``, "g", ); match: for (const match of html.matchAll(regex)) { let data; try { data = JSON.parse(match[1]); } catch { continue; } for (const key of prop) { data = data[key]; if (data === undefined) continue match; } return data; } return null; } function extractGitHub(payload: RawPayload, extra: T): GitHubCommon & T { return { repo: { owner: payload.repo.ownerLogin, name: payload.repo.name, }, branch: payload.refInfo.name, path: payload.path, ...extra, }; } export async function getGitHubTree( owner: string, repo: string, branch: string, path: string, ): Promise { const html = await fetchGitHubPage(`${owner}/${repo}/tree/${branch}/${path}`); // GitHub uses different formats for root vs subdirectories. const payload = // Root directory format. parsePayload(html, "react-partial.embeddedData", [ "props", "initialPayload", ]) || // Subdirectory format. parsePayload(html, "react-app.embeddedData", ["payload"]); if (payload?.tree === undefined) { throw new Error("Could not find tree data in embedded JSON"); } return extractGitHub(payload, { items: payload.tree.items }); } export async function getGitHubBlob( owner: string, repo: string, branch: string, path: string, ): Promise { const html = await fetchGitHubPage(`${owner}/${repo}/blob/${branch}/${path}`); const payload = parsePayload(html, "react-app.embeddedData", ["payload"]); if (payload?.blob === undefined) { throw new Error("Could not find blob data in embedded JSON"); } const blob = payload.blob; const rawLines = blob.rawLines || []; const rawContent = rawLines.join("\n"); const highlightedLines = blob.highlightedLines || []; const content = highlightedLines.map((line) => line.text || "").join("\n"); return extractGitHub(payload, { content: content || rawContent, rawContent, language: blob.language || null, size: blob.size, }); }