import { parseDocument } from "htmlparser2";
import { isText, type AnyNode, type Element } from "domhandler";
import { getAttributeValue, getInnerHTML, textContent } from "domutils";
import * as cssSelect from "css-select";
export class GitHubHTTPError extends Error {
status: number;
constructor(status: number, message: string) {
super(message);
this.name = "GitHubHTTPError";
this.status = status;
}
}
export abstract class RedirectError extends Error {
location: string;
constructor(location: string) {
super(`Redirect to ${location}`);
this.name = this.constructor.name;
this.location = location;
}
}
export class InternalRedirectError extends RedirectError {}
export class ExternalRedirectError extends RedirectError {}
interface TreeItem {
contentType: "directory" | "file";
name: string;
path: string;
}
interface OverviewFile {
displayName: string;
// null when GitHub includes the file without rendered content.
richText?: string | null;
}
interface IssueNode {
number: number;
title?: string;
titleHtml?: string;
author?: {
login: string;
};
createdAt: string;
state: string;
labels?: {
edges: {
node: IssueLabelNode;
}[];
};
}
interface IssueIndexPageQuery {
queryName: "IssueIndexPageQuery";
result: {
data: {
repository: {
search: {
edges: {
node: IssueNode;
}[];
};
};
};
};
}
interface IssueActor {
login: string;
}
interface IssueLabelNode {
name: string;
color: string;
description: string | null;
}
// One entry of an issue's timeline. Every event type has its own fields, so all
// but the ones every item carries are optional here.
interface IssueTimelineNode {
__typename: string;
createdAt: string;
databaseId?: number;
// Comments only; events name whoever performed them as the actor.
author?: IssueActor | null;
url?: string;
authorAssociation?: string;
bodyHTML?: string;
actor?: IssueActor | null;
label?: IssueLabelNode;
assignee?: IssueActor | null;
blockedUser?: IssueActor | null;
milestoneTitle?: string;
stateReason?: string | null;
lockReason?: string | null;
previousTitle?: string;
currentTitle?: string;
commit?: { url?: string; abbreviatedOid?: string } | null;
}
interface IssueViewerNode {
number: number;
databaseId: number;
// Titles are Markdown, so they can carry code spans and emoji.
titleHTML: string;
state: string;
stateReason: string | null;
createdAt: string;
author: IssueActor | null;
bodyHTML: string;
labels?: { edges: { node: IssueLabelNode }[] };
assignedActors?: { nodes: IssueActor[] };
milestone?: { title: string; url: string } | null;
// The oldest page of the timeline; GitHub fetches the rest as you scroll.
frontTimelineItems: {
totalCount: number;
edges: { node: IssueTimelineNode }[];
};
}
interface IssueViewerViewQuery {
queryName: "IssueViewerViewQuery";
result: {
data: {
repository: {
issue: IssueViewerNode;
};
};
};
}
type PreloadedQuery = IssueIndexPageQuery | IssueViewerViewQuery;
type StylingDirective = [number, number, string];
export interface Commit {
oid: string;
url: string;
authoredDate: string;
committedDate: string;
shortMessage: string;
bodyMessageHtml: string;
authors: {
login: string;
displayName: string;
avatarUrl: string;
path: string;
}[];
}
export interface CommitGroup {
title: string;
commits: Commit[];
}
interface CodeViewLayoutRoute {
repo: {
ownerLogin: string;
name: string;
};
refInfo: {
name: string;
};
path: string;
}
interface CodeViewRepoRoute {
tree: {
items: TreeItem[];
};
overview?: {
overviewFiles?: OverviewFile[];
};
}
interface CodeViewTreeRoute {
tree: {
items: TreeItem[];
readme?: OverviewFile;
};
}
interface BlobHeaderInfo {
blobSize: string;
lineInfo?: {
truncatedLoc: string | null;
truncatedSloc: string | null;
};
}
interface CodeViewBlobLayoutRoute {
blob: {
headerInfo: BlobHeaderInfo;
language: string | null;
image: boolean;
};
}
interface CodeViewBlobLayoutRouteStyledBlob {
rawLines: string[] | null;
colorizedLines: string[] | null;
stylingDirectives: StylingDirective[][] | null;
}
interface CommitsRefRoute {
commitGroups: CommitGroup[];
}
interface RepoPullsDashboardContentRoute {
results: {
number: number;
title: string;
createdAt: string;
// REPO_PULL_REQUEST_DISPLAY_STATE_OPEN, _CLOSED, _MERGED, or _DRAFT.
displayState: string;
labels?: IssueLabelNode[];
}[];
}
interface CodeViewBlobRoute {
richText: string | null;
renderedFileInfo?: { renderFileType: string | null } | null;
}
interface SidebarAbout {
description?: string | null;
website?: string | null;
stargazerCount?: number;
forksCount?: number;
// A section is false when the repository has none of that thing.
sections?: { releases?: { releaseCount?: number } | false };
}
interface AppPayload {
sidebarAbout?: SidebarAbout;
codeViewLayoutRoute?: CodeViewLayoutRoute;
codeViewRepoRoute?: CodeViewRepoRoute;
codeViewTreeRoute?: CodeViewTreeRoute;
codeViewBlobLayoutRoute?: CodeViewBlobLayoutRoute;
"codeViewBlobLayoutRoute.StyledBlob"?: CodeViewBlobLayoutRouteStyledBlob;
codeViewBlobRoute?: CodeViewBlobRoute;
commitsRefRoute?: CommitsRefRoute;
repoPullsDashboardContentRoute?: RepoPullsDashboardContentRoute;
// Actually there may be other query types.
preloadedQueries?: PreloadedQuery[];
}
function escapeHtml(text: string): string {
return text
.replace(/&/g, "&")
.replace(//g, ">")
.replace(/"/g, """)
.replace(/'/g, "'");
}
// Renders directives[i..] within [rangeStart, rangeEnd], producing nested spans.
// Returns the rendered content and the index of the first directive not consumed.
function renderRange(
line: string,
directives: StylingDirective[],
i: number,
rangeStart: number,
rangeEnd: number,
): [string, number] {
let result = "";
let pos = rangeStart;
while (i < directives.length) {
const [start, end, className] = directives[i];
// Stop at directives that start outside or extend past this range.
if (start >= rangeEnd || end > rangeEnd) break;
// Combine classes for directives with identical ranges.
let classes = className;
let j = i + 1;
while (j < directives.length && directives[j][0] === start && directives[j][1] === end) {
classes += " " + directives[j][2];
j++;
}
result += escapeHtml(line.substring(pos, start));
const [innerContent, nextI] = renderRange(line, directives, j, start, end);
result += `${innerContent}`;
pos = end;
i = nextI;
}
result += escapeHtml(line.substring(pos, rangeEnd));
return [result, i];
}
function applyStyling(line: string, directives: StylingDirective[]): string {
// Sort by start ascending, then end descending so outer spans precede nested ones.
const sorted = [...directives].sort((a, b) => (a[0] !== b[0] ? a[0] - b[0] : b[1] - a[1]));
const [result] = renderRange(line, sorted, 0, 0, line.length);
return result;
}
interface GitHubCommon {
repo: {
owner: string;
name: string;
};
}
interface GitHubRepoPage extends GitHubCommon {
// Enabled features, like issues or wiki.
tabs: string[] | null;
}
interface GitHubNav extends GitHubRepoPage {
branch: string;
path: string;
}
export interface GitHubTree extends GitHubNav {
items: TreeItem[];
overviewHtml?: Record;
}
export interface RepoInfo {
description: string | null;
website: string | null;
stars: string | null;
forks: string | null;
numReleases?: number;
}
export interface GitHubRepo extends GitHubTree {
info: RepoInfo;
}
export interface GitHubBlob extends GitHubNav {
language: string | null;
size: string;
image: boolean;
renderFileType: string | null;
textLines: string[] | null;
htmlLines: string[] | null;
htmlContent: string | null;
}
export interface IssueLabel {
name: string;
// GitHub's own hex color for the label, without a leading "#".
color: string;
description: string | null;
}
interface Issue {
number: number;
title: string;
// OPEN or CLOSED for an issue; a pull request is also DRAFT or MERGED.
state: string;
createdAt: string;
labels: IssueLabel[];
numReplies?: number;
numPRs?: number;
}
export interface GitHubIssues extends GitHubRepoPage {
issues: Issue[];
q?: string;
}
export interface GitHubPulls extends GitHubRepoPage {
pulls: Issue[];
q?: string;
}
export interface IssueComment {
// What GitHub's permalinks point at, e.g. "issuecomment-3090316111".
anchor: string;
author: string | null;
// The badge beside the author: Owner, Member, Contributor, ...
association: string | null;
createdAt: string;
bodyHtml: string;
}
export interface IssueEvent {
// What GitHub's permalinks point at, e.g. "event-25375447940".
anchor: string;
actor: string | null;
createdAt: string;
// What happened, in GitHub's own words, e.g. "closed this as completed".
description: string;
// What the description ends on, when it names one: a label, or anything else
// worth linking (the assignee, the commit that referenced the issue, ...).
label: IssueLabel | null;
link: { text: string; href: string } | null;
}
export type IssueTimelineItem =
| ({ kind: "comment" } & IssueComment)
| ({ kind: "event" } & IssueEvent);
export interface GitHubIssue extends GitHubRepoPage {
number: number;
titleHtml: string;
state: string;
// Why a closed issue was closed: COMPLETED, NOT_PLANNED, DUPLICATE.
stateReason: string | null;
createdAt: string;
labels: IssueLabel[];
assignees: string[];
milestone: { title: string; href: string } | null;
// The issue itself, rendered like the comments that follow it.
body: IssueComment;
timeline: IssueTimelineItem[];
// Timeline items GitHub fetches as you scroll rather than sending with the
// page, so we never see them.
hiddenItems: number;
}
interface DiscussionCategory {
slug: string;
name: string;
// Every category has an emoji, but GitHub only prints it in some places.
emoji: string | null;
}
interface DiscussionSummary {
number: number;
// Titles are Markdown, so they can carry code spans and emoji.
titleHtml: string;
author: string | null;
createdAt: string | null;
category: DiscussionCategory | null;
answered: boolean;
}
export interface DiscussionComment {
// What GitHub's permalinks point at, e.g. "discussioncomment-9174663".
anchor: string;
author: string | null;
createdAt: string | null;
// The badges beside the author: Author, Maintainer, Collaborator, ...
labels: string[];
bodyHtml: string;
// How GitHub words a comment it keeps collapsed, e.g. "This comment has been
// hidden." or "This comment was marked as off-topic."; null when the comment
// is shown as usual. A collapsed comment arrives with no author or body.
hiddenReason: string | null;
isAnswer: boolean;
replies: DiscussionComment[];
// Replies GitHub keeps collapsed behind a "show more" button of its own.
hiddenReplies: number;
}
export interface GitHubDiscussions extends GitHubRepoPage {
discussions: DiscussionSummary[];
categories: DiscussionCategory[];
// The category the list is narrowed to, if any.
category: DiscussionCategory | null;
q?: string;
}
export interface GitHubDiscussion extends GitHubRepoPage {
number: number;
titleHtml: string;
author: string | null;
category: DiscussionCategory | null;
// The comment a Q&A discussion is answered by, as an anchor on this page.
answerAnchor: string | null;
body: DiscussionComment;
comments: DiscussionComment[];
}
export interface GitHubCommits extends GitHubNav {
commitGroups: CommitGroup[];
}
export interface LatestCommit {
oid: string;
date: string;
shortMessageHtml: string;
authors: {
login: string;
displayName: string;
avatarUrl: string;
}[];
}
export interface GitHubLatestCommit extends GitHubCommon {
commit: LatestCommit;
}
export interface Language {
// "Other" for the aggregate bucket GitHub reports without a name.
name: string;
percentage: number;
color: string | null;
}
export interface LatestRelease {
tag: string;
publishedAt: string | null;
}
export interface GitHubSidebar extends GitHubCommon {
languages: Language[];
latestRelease: LatestRelease | null;
}
export interface Ref {
name: string;
isDefault?: boolean;
date?: string;
authorLogin?: string;
}
export interface GitHubRefs extends GitHubCommon {
type: "branches" | "tags";
refs: Ref[];
hasMore: boolean;
}
interface Release {
tagName: string;
title: string;
publishedAt: string;
bodyHtml: string;
assets?: ReleaseAsset[];
}
export interface ReleaseAsset {
name: string;
url: string;
size?: string;
publishedAt?: string;
}
export interface GitHubReleases extends GitHubRepoPage {
releases: Release[];
}
export interface GitHubRelease extends GitHubRepoPage {
release: Release;
}
export interface WikiPage {
name: string;
href: string;
updatedAt?: string;
}
export interface GitHubWiki extends GitHubRepoPage {
title: string;
updatedBy: string | null;
updatedAt: string | null;
bodyHtml: string;
sidebarHtml: string | null;
footerHtml: string | null;
pages: WikiPage[];
}
export interface GitHubWikiPages extends GitHubRepoPage {
pages: WikiPage[];
}
interface OwnerRepo {
owner: string;
name: string;
description?: string;
language?: string;
stars?: string;
forks?: string;
}
interface OwnerNavItem {
text: string;
href: string;
}
export interface GitHubOwner {
type: "user" | "organization";
login: string;
name: string | null;
bio: string | null;
avatarUrl: string | null;
followers: string | null;
// Users only; always null for organizations.
following: string | null;
company: string | null;
location: string | null;
links: { text: string; href: string }[];
nav: OwnerNavItem[];
pinned: OwnerRepo[];
// The repository list on organization overviews; empty for users.
repos: OwnerRepo[];
}
// GitHub throttles/blocks requests without realistic browser headers.
// These headers make the request appear as a standard browser visit.
function browserHeaders(accept: string): Record {
return {
"User-Agent":
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
Accept: accept,
"Accept-Language": "en-US,en;q=0.5",
};
}
async function fetchGitHubResponse(path: string, accept: string): Promise {
const headers = browserHeaders(accept);
const response = await fetch(`https://github.com/${path}`, { headers, redirect: "manual" });
if (response.status === 301 || response.status === 302) {
const location = response.headers.get("location");
if (location) {
const url = new URL(location);
if (url.hostname === "github.com") {
throw new InternalRedirectError(url.pathname);
}
}
}
if (!response.ok) throw new GitHubHTTPError(response.status, response.statusText);
return response;
}
async function fetchGitHubPage(path: string): Promise {
const response = await fetchGitHubResponse(
path,
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
);
return response.text();
}
// Some routes (e.g. latest-commit) are the same endpoints the React UI itself
// calls to lazily fill in data; they 406 without an explicit JSON Accept.
async function fetchGitHubJson(path: string): Promise {
const response = await fetchGitHubResponse(path, "application/json");
return (await response.json()) as T;
}
function parseEmbeddedPayload(html: string): T | null {
const regex = new RegExp(
``,
"g",
);
match: for (const match of html.matchAll(regex)) {
let data;
try {
data = JSON.parse(match[1]);
} catch {
continue;
}
if (data.payload === undefined) continue match;
return data.payload;
}
return null;
}
function parsePayload(html: string): AppPayload | null {
return parseEmbeddedPayload(html);
}
function extractOverviewHtml(
overviewFiles: OverviewFile[] = [],
): Record | undefined {
const result: Record = {};
for (const file of overviewFiles) {
if (file.richText !== undefined) {
result[file.displayName] = file.richText;
}
}
return Object.keys(result).length > 0 ? result : undefined;
}
function parseRepoTabs(html: string): string[] | null {
const nav = /