path: root/cgithub/src/scraper.ts

export class GitHubHTTPError extends Error {
  status: number;

  constructor(status: number, message: string) {
    super(message);
    this.name = "GitHubHTTPError";
    this.status = status;
  }
}

interface TreeItem {
  contentType: "directory" | "file";
  name: string;
  path: string;
}

interface OverviewFile {
  displayName: string;
  richText?: string;
  loaded: boolean;
}

interface IssueNode {
  number: number;
  title: string;
  author?: {
    login: string;
  };
  createdAt: string;
  state: string;
  labels?: {
    edges: {
      node: {
        name: string;
      };
    }[];
  };
}

interface IssueIndexPageQuery {
  queryName: "IssueIndexPageQuery";
  result: {
    data: {
      repository: {
        search: {
          edges: {
            node: IssueNode;
          }[];
        };
      };
    };
  };
}

type StylingDirective = [number, number, string];

interface Commit {
  oid: string;
  url: string;
  authoredDate: string;
  committedDate: string;
  shortMessage: string;
  bodyMessageHtml: string;
  authors: {
    login: string;
    displayName: string;
    avatarUrl: string;
    path: string;
  }[];
}

export interface CommitGroup {
  title: string;
  commits: Commit[];
}

interface RawPayload {
  repo: {
    ownerLogin: string;
    name: string;
  };
  refInfo: { name: string };
  path: string;
  tree?: {
    items: TreeItem[];
  };
  blob?: {
    headerInfo: {
      blobSize: string;
      lineInfo: {
        truncatedLoc: string | null;
        truncatedSloc: string | null;
      };
    };
    language: string | null;
    image: boolean;
    rawLines: string[] | null;
    colorizedLines: string[] | null;
    richText: string | null;
    stylingDirectives: StylingDirective[][] | null;
  };
  overview?: {
    overviewFiles?: OverviewFile[];
  };
  commitGroups?: CommitGroup[];
  // Actually there may be other query types.
  preloadedQueries?: IssueIndexPageQuery[];
}

function escapeHtml(text: string): string {
  return text
    .replace(/&/g, "&")
    .replace(/</g, "&lt;")
    .replace(/>/g, "&gt;")
    .replace(/"/g, "&quot;")
    .replace(/'/g, "&#039;");
}

function applyStyling(line: string, directives: StylingDirective[]): string {
  let result = "";
  let lastPos = 0;
  for (const [start, end, className] of directives) {
    result += escapeHtml(line.substring(lastPos, start));
    result += `<span class="${className}">${escapeHtml(line.substring(start, end))}</span>`;
    lastPos = end;
  }
  result += escapeHtml(line.substring(lastPos));
  return result;
}

interface GitHubCommon {
  repo: {
    owner: string;
    name: string;
  };
}

interface GitHubNav extends GitHubCommon {
  branch: string;
  path: string;
}

export interface GitHubTree extends GitHubNav {
  items: TreeItem[];
  overviewHtml?: Record<string, string>;
}

export interface RepoInfo {
  description: string | null;
  website: string | null;
  stars: string | null;
  watchers: string | null;
  forks: string | null;
  numReleases?: number;
}

export interface GitHubRepo extends GitHubTree {
  info: RepoInfo;
}

export interface GitHubBlob extends GitHubNav {
  language: string | null;
  size: string;
  image: boolean;
  textLines: string[] | null;
  htmlLines: string[] | null;
  htmlContent: string | null;
}

interface Issue {
  number: number;
  title: string;
  state: string;
  createdAt: string;
  numReplies?: number;
  numPRs?: number;
}

export interface GitHubIssues extends GitHubCommon {
  issues: Issue[];
}

export interface GitHubCommits extends GitHubNav {
  commitGroups: CommitGroup[];
}

interface Release {
  tagName: string;
  title: string;
  publishedAt: string;
  bodyHtml: string;
}

export interface GitHubReleases extends GitHubCommon {
  releases: Release[];
}

async function fetchGitHubPage(path: string): Promise<string> {
  // GitHub throttles/blocks requests without realistic browser headers.
  // These headers make the request appear as a standard browser visit.
  const headers = {
    "User-Agent":
      "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
    Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
    "Accept-Language": "en-US,en;q=0.5",
  };
  const response = await fetch(`https://github.com/${path}`, { headers });
  if (!response.ok) throw new GitHubHTTPError(response.status, response.statusText);
  return response.text();
}

function parsePayload(html: string, dataTarget: string, prop: string[]): RawPayload | null {
  const regex = new RegExp(
    `<script type="application/json" data-target="${dataTarget}">([^<]+)</script>`,
    "g",
  );

  match: for (const match of html.matchAll(regex)) {
    let data;
    try {
      data = JSON.parse(match[1]);
    } catch {
      continue;
    }
    for (const key of prop) {
      data = data[key];
      if (data === undefined) continue match;
    }
    return data;
  }
  return null;
}

function parseRepoInfo(html: string): RepoInfo {
  const descMatch = html.match(/<p[^>]*class="[^"]*f4[^"]*"[^>]*>([\s\S]*?)<\/p>/);
  const description = descMatch?.[1]?.replace(/<[^>]+>/g, "").trim() || null;
  const websiteMatch = html.match(/<a[^>]*rel="noopener noreferrer nofollow"[^>]*href="([^"]+)"/);
  const starsMatch = html.match(/id="repo-stars-counter-star"[^>]*>([^<]+)/);
  const watchersMatch = html.match(/<strong>(\d+)<\/strong>\s*watching/);
  const forksMatch = html.match(/id="repo-network-counter"[^>]*>([^<]+)/);

  let numReleases: number | undefined;
  const releaseCountMatch = html.match(
    /<a[^>]*href="[^"]*\/releases"[^>]*>[\s\S]*?<span[^>]*title="(\d+)"[^>]*(?:class="[^"]*Counter|data-view-component="true")[^>]*>/,
  );
  if (releaseCountMatch) {
    numReleases = parseInt(releaseCountMatch[1], 10);
  }

  return {
    description,
    website: websiteMatch?.[1] || null,
    stars: starsMatch?.[1]?.trim() || null,
    watchers: watchersMatch?.[1] || null,
    forks: forksMatch?.[1]?.trim() || null,
    numReleases,
  };
}

function extractOverviewHtml(payload: RawPayload): Record<string, string> | undefined {
  const result: Record<string, string> = {};
  for (const file of payload.overview?.overviewFiles ?? []) {
    if (file.loaded && file.richText !== undefined) {
      result[file.displayName] = file.richText;
    }
  }
  return Object.keys(result).length > 0 ? result : undefined;
}

function extractGitHub<T>(payload: RawPayload, extra: T): GitHubNav & T {
  return {
    repo: {
      owner: payload.repo.ownerLogin,
      name: payload.repo.name,
    },
    branch: payload.refInfo.name,
    path: payload.path,
    ...extra,
  };
}

export async function getGitHubRepo(owner: string, repo: string): Promise<GitHubRepo> {
  const html = await fetchGitHubPage(`${owner}/${repo}`);

  const payload = parsePayload(html, "react-partial.embeddedData", ["props", "initialPayload"]);

  if (payload?.tree === undefined) {
    throw new Error("Could not find tree data in embedded JSON");
  }

  const info = parseRepoInfo(html);
  const overviewHtml = extractOverviewHtml(payload);

  return extractGitHub(payload, { items: payload.tree.items, info, overviewHtml });
}

export async function getGitHubTree(
  owner: string,
  repo: string,
  branch: string,
  path: string,
): Promise<GitHubTree> {
  const html = await fetchGitHubPage(`${owner}/${repo}/tree/${branch}/${path}`);

  // GitHub uses different formats for root vs subdirectories.
  const payload =
    // Root directory format.
    parsePayload(html, "react-partial.embeddedData", ["props", "initialPayload"]) ||
    // Subdirectory format.
    parsePayload(html, "react-app.embeddedData", ["payload"]);

  if (payload?.tree === undefined) {
    throw new Error("Could not find tree data in embedded JSON");
  }

  const overviewHtml = extractOverviewHtml(payload);

  return extractGitHub(payload, { items: payload.tree.items, overviewHtml });
}

export async function getGitHubBlob(
  owner: string,
  repo: string,
  branch: string,
  path: string,
): Promise<GitHubBlob> {
  const html = await fetchGitHubPage(`${owner}/${repo}/blob/${branch}/${path}`);

  const payload = parsePayload(html, "react-app.embeddedData", ["payload"]);

  if (payload?.blob === undefined) {
    throw new Error("Could not find blob data in embedded JSON");
  }

  const blob = payload.blob;
  const { headerInfo, stylingDirectives } = blob;

  let htmlLines = blob.colorizedLines;
  if (!htmlLines && stylingDirectives && blob.rawLines) {
    htmlLines = blob.rawLines.map((line, i) => applyStyling(line, stylingDirectives[i]));
  }

  let size = headerInfo.blobSize;
  if (headerInfo.lineInfo.truncatedLoc) {
    size += ` / ${headerInfo.lineInfo.truncatedLoc} lines`;
  }
  if (headerInfo.lineInfo.truncatedSloc) {
    size += ` / ${headerInfo.lineInfo.truncatedSloc} loc`;
  }

  return extractGitHub(payload, {
    language: blob.language || null,
    size,
    image: blob.image,
    textLines: blob.rawLines,
    htmlLines,
    htmlContent: blob.richText,
  });
}

export async function getGitHubIssues(owner: string, repo: string): Promise<GitHubIssues> {
  const html = await fetchGitHubPage(`${owner}/${repo}/issues`);

  const payload = parsePayload(html, "react-app.embeddedData", ["payload"]);

  let issuesQuery: IssueIndexPageQuery | undefined;
  for (const query of payload?.preloadedQueries ?? []) {
    if (query.queryName === "IssueIndexPageQuery") {
      issuesQuery = query;
    }
  }
  if (issuesQuery === undefined) {
    throw new Error("Could not find IssueIndexPageQuery in embedded JSON");
  }

  const issues: Issue[] = issuesQuery.result.data.repository.search.edges.map((edge) => {
    const { number, title, state, createdAt } = edge.node;
    return { number, title, state, createdAt };
  });

  return {
    repo: {
      owner,
      name: repo,
    },
    issues,
  };
}

export async function getGitHubCommits(
  owner: string,
  repo: string,
  branch: string,
  path: string,
): Promise<GitHubCommits> {
  const urlPath = path
    ? `${owner}/${repo}/commits/${branch}/${path}`
    : `${owner}/${repo}/commits/${branch}`;
  const html = await fetchGitHubPage(urlPath);

  const payload = parsePayload(html, "react-app.embeddedData", ["payload"]);

  if (payload?.commitGroups === undefined) {
    throw new Error("Could not find commit history in embedded JSON");
  }

  return {
    repo: {
      owner,
      name: repo,
    },
    branch,
    path,
    commitGroups: payload.commitGroups,
  };
}

export async function getGitHubReleases(owner: string, repo: string): Promise<GitHubReleases> {
  const html = await fetchGitHubPage(`${owner}/${repo}/releases`);

  // Releases page doesn't have embedded JSON, so we parse HTML directly
  const releases: Release[] = [];

  // Find all h2 tags with version numbers
  const h2Regex = /<h2[^>]*id="([^"]*)"[^>]*>(v[^<]*)<\/h2>/g;

  for (const h2Match of html.matchAll(h2Regex)) {
    const id = h2Match[1];
    const title = h2Match[2].trim();

    // Find the section containing this h2
    const sectionRegex = new RegExp(
      `<section[^>]*>\\s*<h2[^>]*id="${id}"[^>]*>[^<]*<\\/h2>([\\s\\S]*?)<\\/section>`,
      "s",
    );
    const sectionMatch = html.match(sectionRegex);
    if (!sectionMatch) continue;

    const sectionHtml = sectionMatch[1];

    // Extract tag name from link
    const tagMatch = sectionHtml.match(/<a[^>]*href="[^"]*\/tree\/([^"]+)"[^>]*>/);
    const tagName = tagMatch?.[1] || title;

    // Extract published date
    const dateMatch = sectionHtml.match(/<relative-time[^>]*datetime="([^"]*)"[^>]*>/);
    const publishedAt = dateMatch?.[1] || "";

    // Extract body HTML (markdown content)
    const bodyMatch = sectionHtml.match(/<div[^>]*class="[^"]*markdown-body[^"]*"[^>]*>([\s\S]*?)<\/div>/);
    const bodyHtml = bodyMatch?.[1]?.trim() || "";

    releases.push({
      tagName,
      title,
      publishedAt,
      bodyHtml,
    });
  }

  return {
    repo: {
      owner,
      name: repo,
    },
    releases,
  };
}