interface TreeItem { contentType: "directory" | "file"; name: string; path: string; } interface RawPayload { repo: { ownerLogin: string; name: string; }; refInfo: { name: string }; path: string; tree?: { items: TreeItem[]; }; blob?: { language?: string; rawLines?: string[]; highlightedLines?: { text?: string; }[]; size: number; }; } interface GitHubCommon { repo: { owner: string; name: string; }; branch: string; path: string; } export interface GitHubTree extends GitHubCommon { items: TreeItem[]; } export interface RepoInfo { description: string | null; website: string | null; stars: string | null; watchers: string | null; forks: string | null; } export interface GitHubRepo extends GitHubTree { info: RepoInfo; } export interface GitHubBlob extends GitHubCommon { language: string | null; size: number; content: string; rawContent: string; } async function fetchGitHubPage(path: string): Promise { // GitHub throttles/blocks requests without realistic browser headers. // These headers make the request appear as a standard browser visit. const headers = { "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", "Accept-Language": "en-US,en;q=0.5", }; const response = await fetch(`https://github.com/${path}`, { headers }); return response.text(); } function parsePayload( html: string, dataTarget: string, prop: string[], ): RawPayload | null { const regex = new RegExp( ``, "g", ); match: for (const match of html.matchAll(regex)) { let data; try { data = JSON.parse(match[1]); } catch { continue; } for (const key of prop) { data = data[key]; if (data === undefined) continue match; } return data; } return null; } function parseRepoInfo(html: string): RepoInfo { const descMatch = html.match(/]*class="[^"]*f4[^"]*"[^>]*>([\s\S]*?)<\/p>/); const description = descMatch?.[1]?.replace(/<[^>]+>/g, "").trim() || null; const websiteMatch = html.match( /]*rel="noopener noreferrer nofollow"[^>]*href="([^"]+)"/, ); const starsMatch = html.match(/id="repo-stars-counter-star"[^>]*>([^<]+)/); const watchersMatch = html.match(/(\d+)<\/strong>\s*watching/); const forksMatch = html.match(/id="repo-network-counter"[^>]*>([^<]+)/); return { description, website: websiteMatch?.[1] || null, stars: starsMatch?.[1]?.trim() || null, watchers: watchersMatch?.[1] || null, forks: forksMatch?.[1]?.trim() || null, }; } function extractGitHub(payload: RawPayload, extra: T): GitHubCommon & T { return { repo: { owner: payload.repo.ownerLogin, name: payload.repo.name, }, branch: payload.refInfo.name, path: payload.path, ...extra, }; } export async function getGitHubRepo( owner: string, repo: string, ): Promise { const html = await fetchGitHubPage(`${owner}/${repo}`); const payload = parsePayload(html, "react-partial.embeddedData", [ "props", "initialPayload", ]); if (payload?.tree === undefined) { throw new Error("Could not find tree data in embedded JSON"); } const info = parseRepoInfo(html); return extractGitHub(payload, { items: payload.tree.items, info }); } export async function getGitHubTree( owner: string, repo: string, branch: string, path: string, ): Promise { const html = await fetchGitHubPage(`${owner}/${repo}/tree/${branch}/${path}`); // GitHub uses different formats for root vs subdirectories. const payload = // Root directory format. parsePayload(html, "react-partial.embeddedData", [ "props", "initialPayload", ]) || // Subdirectory format. parsePayload(html, "react-app.embeddedData", ["payload"]); if (payload?.tree === undefined) { throw new Error("Could not find tree data in embedded JSON"); } return extractGitHub(payload, { items: payload.tree.items }); } export async function getGitHubBlob( owner: string, repo: string, branch: string, path: string, ): Promise { const html = await fetchGitHubPage(`${owner}/${repo}/blob/${branch}/${path}`); const payload = parsePayload(html, "react-app.embeddedData", ["payload"]); if (payload?.blob === undefined) { throw new Error("Could not find blob data in embedded JSON"); } const blob = payload.blob; const rawLines = blob.rawLines || []; const rawContent = rawLines.join("\n"); const highlightedLines = blob.highlightedLines || []; const content = highlightedLines.map((line) => line.text || "").join("\n"); return extractGitHub(payload, { content: content || rawContent, rawContent, language: blob.language || null, size: blob.size, }); }