;
}
export interface RepoInfo {
description: string | null;
website: string | null;
stars: string | null;
watchers: string | null;
forks: string | null;
numReleases?: number;
}
export interface GitHubRepo extends GitHubTree {
info: RepoInfo;
}
export interface GitHubBlob extends GitHubNav {
language: string | null;
size: string;
textLines: string[] | null;
htmlLines: string[] | null;
htmlContent: string | null;
}
interface Issue {
number: number;
title: string;
state: string;
createdAt: string;
numReplies?: number;
numPRs?: number;
}
export interface GitHubIssues extends GitHubCommon {
issues: Issue[];
}
async function fetchGitHubPage(path: string): Promise {
// GitHub throttles/blocks requests without realistic browser headers.
// These headers make the request appear as a standard browser visit.
const headers = {
"User-Agent":
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"Accept-Language": "en-US,en;q=0.5",
};
const response = await fetch(`https://github.com/${path}`, { headers });
if (!response.ok) throw new GitHubHTTPError(response.status, response.statusText);
return response.text();
}
function parsePayload(html: string, dataTarget: string, prop: string[]): RawPayload | null {
const regex = new RegExp(
``,
"g",
);
match: for (const match of html.matchAll(regex)) {
let data;
try {
data = JSON.parse(match[1]);
} catch {
continue;
}
for (const key of prop) {
data = data[key];
if (data === undefined) continue match;
}
return data;
}
return null;
}
function parseRepoInfo(html: string): RepoInfo {
const descMatch = html.match(/]*class="[^"]*f4[^"]*"[^>]*>([\s\S]*?)<\/p>/);
const description = descMatch?.[1]?.replace(/<[^>]+>/g, "").trim() || null;
const websiteMatch = html.match(/]*rel="noopener noreferrer nofollow"[^>]*href="([^"]+)"/);
const starsMatch = html.match(/id="repo-stars-counter-star"[^>]*>([^<]+)/);
const watchersMatch = html.match(/(\d+)<\/strong>\s*watching/);
const forksMatch = html.match(/id="repo-network-counter"[^>]*>([^<]+)/);
let numReleases: number | undefined;
const releaseCountMatch = html.match(
/]*href="[^"]*\/releases"[^>]*>[\s\S]*?]*title="(\d+)"[^>]*(?:class="[^"]*Counter|data-view-component="true")[^>]*>/,
);
if (releaseCountMatch) {
numReleases = parseInt(releaseCountMatch[1], 10);
}
return {
description,
website: websiteMatch?.[1] || null,
stars: starsMatch?.[1]?.trim() || null,
watchers: watchersMatch?.[1] || null,
forks: forksMatch?.[1]?.trim() || null,
numReleases,
};
}
function extractOverviewHtml(payload: RawPayload): Record | undefined {
const result: Record = {};
for (const file of payload.overview?.overviewFiles ?? []) {
if (file.loaded && file.richText !== undefined) {
result[file.displayName] = file.richText;
}
}
return Object.keys(result).length > 0 ? result : undefined;
}
function extractGitHub(payload: RawPayload, extra: T): GitHubNav & T {
return {
repo: {
owner: payload.repo.ownerLogin,
name: payload.repo.name,
},
branch: payload.refInfo.name,
path: payload.path,
...extra,
};
}
export async function getGitHubRepo(owner: string, repo: string): Promise {
const html = await fetchGitHubPage(`${owner}/${repo}`);
const payload = parsePayload(html, "react-partial.embeddedData", ["props", "initialPayload"]);
if (payload?.tree === undefined) {
throw new Error("Could not find tree data in embedded JSON");
}
const info = parseRepoInfo(html);
const overviewHtml = extractOverviewHtml(payload);
return extractGitHub(payload, { items: payload.tree.items, info, overviewHtml });
}
export async function getGitHubTree(
owner: string,
repo: string,
branch: string,
path: string,
): Promise {
const html = await fetchGitHubPage(`${owner}/${repo}/tree/${branch}/${path}`);
// GitHub uses different formats for root vs subdirectories.
const payload =
// Root directory format.
parsePayload(html, "react-partial.embeddedData", ["props", "initialPayload"]) ||
// Subdirectory format.
parsePayload(html, "react-app.embeddedData", ["payload"]);
if (payload?.tree === undefined) {
throw new Error("Could not find tree data in embedded JSON");
}
const overviewHtml = extractOverviewHtml(payload);
return extractGitHub(payload, { items: payload.tree.items, overviewHtml });
}
export async function getGitHubBlob(
owner: string,
repo: string,
branch: string,
path: string,
): Promise {
const html = await fetchGitHubPage(`${owner}/${repo}/blob/${branch}/${path}`);
const payload = parsePayload(html, "react-app.embeddedData", ["payload"]);
if (payload?.blob === undefined) {
throw new Error("Could not find blob data in embedded JSON");
}
const blob = payload.blob;
const { headerInfo, stylingDirectives } = blob;
let htmlLines = blob.colorizedLines;
if (!htmlLines && stylingDirectives && blob.rawLines) {
htmlLines = blob.rawLines.map((line, i) => applyStyling(line, stylingDirectives[i]));
}
// TODO fine tune
const size = `${headerInfo.blobSize} / ${headerInfo.lineInfo.truncatedLoc} lines / ${headerInfo.lineInfo.truncatedSloc} loc`;
return extractGitHub(payload, {
language: blob.language || null,
size,
textLines: blob.rawLines,
htmlLines,
htmlContent: blob.richText,
});
}
export async function getGitHubIssues(owner: string, repo: string): Promise {
const html = await fetchGitHubPage(`${owner}/${repo}/issues`);
const payload = parsePayload(html, "react-app.embeddedData", ["payload"]);
let issuesQuery: IssueIndexPageQuery | undefined;
for (const query of payload?.preloadedQueries ?? []) {
if (query.queryName === "IssueIndexPageQuery") {
issuesQuery = query;
}
}
if (issuesQuery === undefined) {
throw new Error("Could not find IssueIndexPageQuery in embedded JSON");
}
const issues: Issue[] = issuesQuery.result.data.repository.search.edges.map((edge) => {
const { number, title, state, createdAt } = edge.node;
return { number, title, state, createdAt };
});
return {
repo: {
owner,
name: repo,
},
issues,
};
}