diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/app.test.ts | 112 | ||||
| -rw-r--r-- | src/app.ts | 11 | ||||
| -rw-r--r-- | src/scraper.test.ts | 77 | ||||
| -rw-r--r-- | src/scraper.ts | 300 |
4 files changed, 499 insertions, 1 deletions
diff --git a/src/app.test.ts b/src/app.test.ts index 3fdacd5..2211adf 100644 --- a/src/app.test.ts +++ b/src/app.test.ts @@ -358,6 +358,118 @@ describe("wiki", () => { }); }); +describe("issues", () => { + const repo = { owner: "cli", name: "cli" }; + const label = { name: "bug", color: "d73a4a", description: "Something isn't working" }; + const issue = { + repo, + number: 10000, + titleHtml: "<code>--allow-forking=false</code> not interpreted correctly", + state: "CLOSED", + stateReason: "NOT_PLANNED", + createdAt: "2024-12-03T23:01:40Z", + labels: [label], + assignees: ["BagToad"], + milestone: { title: "gh v2.64", href: "/cli/cli/milestone/42" }, + body: { + anchor: "issue-2716088121", + author: "celloza", + association: null, + createdAt: "2024-12-03T23:01:40Z", + bodyHtml: "<p>Forking is disabled.</p>", + }, + timeline: [ + { + kind: "comment", + anchor: "issuecomment-2517193151", + author: "BagToad", + association: "Member", + createdAt: "2024-12-06T20:49:46Z", + bodyHtml: "<p>Working as intended.</p>", + }, + { + kind: "event", + anchor: "event-15562412541", + actor: "BagToad", + createdAt: "2024-12-06T20:49:46Z", + description: "added the", + label, + link: null, + }, + { + kind: "event", + anchor: "", + actor: "BagToad", + createdAt: "2024-12-06T20:49:46Z", + description: "closed this as not planned", + label: null, + link: null, + }, + ], + hiddenItems: 0, + }; + + it("should render the issue above its timeline", () => { + const html = eta.render("issue.eta", issue); + + assert.match(html, /<title>cli\/cli - issues\/10000<\/title>/); + assert.match(html, /<h2><code>--allow-forking=false<\/code> not interpreted correctly<\/h2>/); + assert.match(html, /#10000/); + assert.match(html, /<span class="pill state state-closed">Closed as not planned<\/span>/); + assert.match(html, /<div class="box comment" id="issue-2716088121">\s*<header>\s*<span>\s*<a href="\/celloza">celloza<\/a>/); + assert.match(html, /<div class="box comment" id="issuecomment-2517193151">/); + assert.match(html, /<span class="pill">Member<\/span>/); + // The search box searches issues from an issue. + assert.match(html, /<option value="issues" selected>/); + }); + + it("should render the labels, assignees, and milestone beside the state", () => { + const html = eta.render("issue.eta", issue); + + assert.match(html, /<span class="pill-dot" style="background: #d73a4a"><\/span>bug/); + assert.match(html, /title="Something isn't working"/); + assert.match(html, /assigned to\s+<a href="\/BagToad">BagToad<\/a>/); + assert.match(html, /milestone <a href="\/cli\/cli\/milestone\/42">gh v2.64<\/a>/); + }); + + // Events are a line each, rather than a box like the comments around them. + it("should render an event per timeline entry that is not a comment", () => { + const html = eta.render("issue.eta", issue); + + assert.match(html, /<div class="issue-event" id="event-15562412541">/); + assert.match(html, /<div class="issue-event">/); + assert.match(html, /added the <span class="pill"[^>]*>.*<\/span>\s+label/); + assert.match(html, /closed this as not planned/); + }); + + it("should link timestamps to their comment or event", () => { + const html = eta.render("issue.eta", issue); + + assert.match(html, /<a href="#issue-2716088121"><time/); + assert.match(html, /<a href="#issuecomment-2517193151"><time/); + assert.match(html, /<a href="#event-15562412541"><time/); + }); + + it("should say what GitHub keeps to itself", () => { + assert.match( + eta.render("issue.eta", { ...issue, hiddenItems: 8 }), + /8 later\s+comments and events are only on GitHub\./, + ); + assert.match( + eta.render("issue.eta", { ...issue, hiddenItems: 1 }), + /1 later\s+comment or event is only on GitHub\./, + ); + }); + + // GitHub's own actions live where a number goes, so they are not ours. + it("should only scrape an issue path that names a number", async () => { + const res = await app.request("http://cgithub.example/cli/cli/issues/new"); + + assert.strictEqual(res.status, 200); + assert.match(await res.text(), /url=https:\/\/github.com\/cli\/cli\/issues\/new"/); + }); +}); + describe("discussions", () => { const repo = { owner: "jj-vcs", name: "jj" }; const categories = [ @@ -7,6 +7,7 @@ import { getGitHubCommits, getGitHubDiscussion, getGitHubDiscussions, + getGitHubIssue, getGitHubIssues, getGitHubSidebar, getGitHubLatestCommit, @@ -214,6 +215,16 @@ export function createApp(eta: Eta) { return c.render("issues.eta", await getGitHubIssues(owner, repo, { q })); }); + // GitHub puts its own actions (new, assigned, ...) where a number goes, so + // anything that isn't one is not an issue of ours to render. + app.get("/:owner/:repo/issues/:number", async (c) => { + const { owner, repo, number } = c.req.param(); + if (!/^\d+$/.test(number)) { + return redirectToGitHub(c); + } + return c.render("issue.eta", await getGitHubIssue(owner, repo, Number(number))); + }); + app.get("/:owner/:repo/pulls", async (c) => { const { owner, repo } = c.req.param(); const q = c.req.query("q"); diff --git a/src/scraper.test.ts b/src/scraper.test.ts index 37415b9..b8797bf 100644 --- a/src/scraper.test.ts +++ b/src/scraper.test.ts @@ -5,6 +5,7 @@ import { getGitHubCommits, getGitHubDiscussion, getGitHubDiscussions, + getGitHubIssue, getGitHubIssues, getGitHubSidebar, getGitHubLatestCommit, @@ -500,6 +501,82 @@ describe("GitHub scraper", () => { }); }); + describe("issues", () => { + it("should fetch an issue with its comments", async () => { + const data = await getGitHubIssue("actions", "deploy-pages", 402); + + assert.strictEqual(data.repo.owner, "actions"); + assert.strictEqual(data.repo.name, "deploy-pages"); + assert.strictEqual(data.number, 402); + assert.strictEqual(data.titleHtml, "Dry Run"); + assert.strictEqual(data.state, "OPEN"); + assert.strictEqual(data.stateReason, null); + assert.strictEqual(data.createdAt, "2025-07-03T19:07:08Z"); + + // The issue's own post reads like the comments that follow it. + assert.strictEqual(data.body.anchor, "issue-3200386867"); + assert.strictEqual(data.body.author, "wmcnamee-coreweave"); + assert.strictEqual(data.body.createdAt, "2025-07-03T19:07:08Z"); + assert.match(data.body.bodyHtml, /I'd like to do a dry run/); + + const comments = data.timeline.filter((item) => item.kind === "comment"); + const comment = comments.find((item) => item.anchor === "issuecomment-3090316111"); + assert.ok(comment); + assert.strictEqual(comment.author, "yoannchaudet"); + assert.strictEqual(comment.association, "Contributor"); + assert.match(comment.bodyHtml, /dry run/); + }); + + it("should fetch the events an issue's timeline carries", async () => { + const data = await getGitHubIssue("actions", "deploy-pages", 417); + + assert.strictEqual(data.state, "CLOSED"); + assert.strictEqual(data.stateReason, "COMPLETED"); + + const events = data.timeline.filter((item) => item.kind === "event"); + const closed = events.find((event) => event.description === "closed this as completed"); + assert.ok(closed); + assert.strictEqual(closed.anchor, "event-25375447940"); + assert.strictEqual(closed.actor, "beingme8583-netizen"); + assert.strictEqual(closed.label, null); + assert.strictEqual(closed.link, null); + }); + + it("should fetch the labels, assignees, and milestone of an issue", async () => { + const data = await getGitHubIssue("microsoft", "vscode", 199954); + + assert.strictEqual(data.state, "CLOSED"); + assert.deepStrictEqual(data.assignees, ["jrieken"]); + assert.deepStrictEqual(data.milestone, { + title: "June 2025", + href: "/microsoft/vscode/milestone/308", + }); + + const label = data.labels.find((label) => label.name === "feature-request"); + assert.ok(label); + assert.strictEqual(label.color, "dcdcdc"); + assert.strictEqual(label.description, "Request for new features or functionality"); + + // A labelling event names the label it applied. + const events = data.timeline.filter((item) => item.kind === "event"); + const labelled = events.find((event) => event.label?.name === "feature-request"); + assert.ok(labelled); + assert.strictEqual(labelled.description, "added the"); + assert.strictEqual(labelled.actor, "jrieken"); + + // GitHub only sends the oldest page of a long timeline with the page. + assert.ok(data.hiddenItems > 0); + }); + + it("should redirect an issue number that names a pull request", async () => { + await assert.rejects(getGitHubIssue("actions", "deploy-pages", 411), (err) => { + assert(err instanceof InternalRedirectError, "error should be an InternalRedirectError"); + assert.strictEqual(err.location, "/actions/deploy-pages/pull/411"); + return true; + }); + }); + }); + describe("discussions", () => { it("should fetch the discussion list with its categories", async () => { const data = await getGitHubDiscussions("jj-vcs", "jj"); diff --git a/src/scraper.ts b/src/scraper.ts index d1a1763..3670034 100644 --- a/src/scraper.ts +++ b/src/scraper.ts @@ -72,6 +72,72 @@ interface IssueIndexPageQuery { }; } +interface IssueActor { + login: string; +} + +interface IssueLabelNode { + name: string; + color: string; + description: string | null; +} + +// One entry of an issue's timeline. Every event type has its own fields, so all +// but the ones every item carries are optional here. +interface IssueTimelineNode { + __typename: string; + createdAt: string; + databaseId?: number; + // Comments only; events name whoever performed them as the actor. + author?: IssueActor | null; + url?: string; + authorAssociation?: string; + bodyHTML?: string; + actor?: IssueActor | null; + label?: IssueLabelNode; + assignee?: IssueActor | null; + blockedUser?: IssueActor | null; + milestoneTitle?: string; + stateReason?: string | null; + lockReason?: string | null; + previousTitle?: string; + currentTitle?: string; + commit?: { url?: string; abbreviatedOid?: string } | null; +} + +interface IssueViewerNode { + number: number; + databaseId: number; + // Titles are Markdown, so they can carry code spans and emoji. + titleHTML: string; + state: string; + stateReason: string | null; + createdAt: string; + author: IssueActor | null; + bodyHTML: string; + labels?: { edges: { node: IssueLabelNode }[] }; + assignedActors?: { nodes: IssueActor[] }; + milestone?: { title: string; url: string } | null; + // The oldest page of the timeline; GitHub fetches the rest as you scroll. + frontTimelineItems: { + totalCount: number; + edges: { node: IssueTimelineNode }[]; + }; +} + +interface IssueViewerViewQuery { + queryName: "IssueViewerViewQuery"; + result: { + data: { + repository: { + issue: IssueViewerNode; + }; + }; + }; +} + +type PreloadedQuery = IssueIndexPageQuery | IssueViewerViewQuery; + type StylingDirective = [number, number, string]; export interface Commit { @@ -171,7 +237,7 @@ interface AppPayload { codeViewBlobRoute?: CodeViewBlobRoute; commitsRefRoute?: CommitsRefRoute; // Actually there may be other query types. - preloadedQueries?: IssueIndexPageQuery[]; + preloadedQueries?: PreloadedQuery[]; } function escapeHtml(text: string): string { @@ -284,6 +350,58 @@ export interface GitHubPulls extends GitHubCommon { q?: string; } +export interface IssueLabel { + name: string; + // GitHub's own hex color for the label, without a leading "#". + color: string; + description: string | null; +} + +export interface IssueComment { + // What GitHub's permalinks point at, e.g. "issuecomment-3090316111". + anchor: string; + author: string | null; + // The badge beside the author: Owner, Member, Contributor, ... + association: string | null; + createdAt: string; + bodyHtml: string; +} + +export interface IssueEvent { + // What GitHub's permalinks point at, e.g. "event-25375447940". + anchor: string; + actor: string | null; + createdAt: string; + // What happened, in GitHub's own words, e.g. "closed this as completed". + description: string; + // What the description ends on, when it names one: a label, or anything else + // worth linking (the assignee, the commit that referenced the issue, ...). + label: IssueLabel | null; + link: { text: string; href: string } | null; +} + +export type IssueTimelineItem = + | ({ kind: "comment" } & IssueComment) + | ({ kind: "event" } & IssueEvent); + +export interface GitHubIssue extends GitHubCommon { + number: number; + titleHtml: string; + state: string; + // Why a closed issue was closed: COMPLETED, NOT_PLANNED, DUPLICATE. + stateReason: string | null; + createdAt: string; + labels: IssueLabel[]; + assignees: string[]; + milestone: { title: string; href: string } | null; + // The issue itself, rendered like the comments that follow it. + body: IssueComment; + timeline: IssueTimelineItem[]; + // Timeline items GitHub fetches as you scroll rather than sending with the + // page, so we never see them. + hiddenItems: number; +} + interface DiscussionCategory { slug: string; name: string; @@ -683,6 +801,186 @@ export async function getGitHubIssues( return { repo: { owner, name: repo }, issues, q }; } +// GitHub's payloads carry absolute URLs; the same paths on our host are ours. +function githubPath(url: string | undefined): string | null { + const parsed = url ? URL.parse(url) : null; + return parsed?.hostname === "github.com" ? parsed.pathname : null; +} + +// "FIRST_TIME_CONTRIBUTOR" is a badge; "NONE" is everybody else, and no badge. +function issueAssociation(association: string | undefined): string | null { + if (!association || association === "NONE") return null; + const words = association.toLowerCase().replace(/_/g, " "); + return words.charAt(0).toUpperCase() + words.slice(1); +} + +function parseIssueLabel(label: IssueLabelNode): IssueLabel { + return { name: label.name, color: label.color, description: label.description ?? null }; +} + +function parseIssueComment(node: IssueTimelineNode): IssueComment { + return { + anchor: node.url?.split("#")[1] ?? "", + author: node.author?.login ?? null, + association: issueAssociation(node.authorAssociation), + createdAt: node.createdAt, + bodyHtml: node.bodyHTML ?? "", + }; +} + +function userLink(user: IssueActor | null | undefined): { text: string; href: string } | null { + return user ? { text: user.login, href: `/${user.login}` } : null; +} + +// GitHub's timeline carries more event types than it shows, and shows more than +// are worth a line here; anything this does not describe is left out. +function parseIssueEvent(node: IssueTimelineNode): IssueEvent | null { + const event = { + anchor: node.databaseId ? `event-${node.databaseId}` : "", + actor: node.actor?.login ?? null, + createdAt: node.createdAt, + label: null, + link: null, + }; + + switch (node.__typename) { + case "ClosedEvent": + return { + ...event, + description: + node.stateReason === "NOT_PLANNED" + ? "closed this as not planned" + : node.stateReason === "DUPLICATE" + ? "closed this as a duplicate" + : "closed this as completed", + }; + case "ReopenedEvent": + return { ...event, description: "reopened this" }; + case "LabeledEvent": + case "UnlabeledEvent": + if (!node.label) return null; + return { + ...event, + description: node.__typename === "LabeledEvent" ? "added the" : "removed the", + label: parseIssueLabel(node.label), + }; + case "AssignedEvent": + case "UnassignedEvent": { + const assigned = node.__typename === "AssignedEvent"; + // GitHub words an assignment to oneself without naming anybody. + if (node.assignee && node.assignee.login === event.actor) { + return { + ...event, + description: assigned ? "self-assigned this" : "removed their assignment", + }; + } + return { + ...event, + description: assigned ? "assigned this to" : "unassigned", + link: userLink(node.assignee), + }; + } + case "MilestonedEvent": + return { ...event, description: `added this to the ${node.milestoneTitle} milestone` }; + case "DemilestonedEvent": + return { ...event, description: `removed this from the ${node.milestoneTitle} milestone` }; + case "RenamedTitleEvent": + return { ...event, description: `changed the title to ${node.currentTitle}` }; + case "LockedEvent": + return { + ...event, + description: node.lockReason + ? `locked this conversation as ${node.lockReason.toLowerCase().replace(/_/g, " ")}` + : "locked this conversation", + }; + case "UnlockedEvent": + return { ...event, description: "unlocked this conversation" }; + case "PinnedEvent": + return { ...event, description: "pinned this" }; + case "UnpinnedEvent": + return { ...event, description: "unpinned this" }; + case "ReferencedEvent": { + const href = githubPath(node.commit?.url); + if (!href || !node.commit?.abbreviatedOid) return null; + return { + ...event, + description: "referenced this in commit", + link: { text: node.commit.abbreviatedOid, href }, + }; + } + case "UserBlockedEvent": + return { ...event, description: "blocked", link: userLink(node.blockedUser) }; + default: + return null; + } +} + +export async function getGitHubIssue( + owner: string, + repo: string, + number: number, +): Promise<GitHubIssue> { + const html = await fetchGitHubPage(`${owner}/${repo}/issues/${number}`); + const payload = parsePayload(html); + + let issue: IssueViewerNode | undefined; + for (const query of payload?.preloadedQueries ?? []) { + if (query.queryName === "IssueViewerViewQuery") { + issue = query.result.data.repository.issue; + } + } + if (issue === undefined) { + throw new Error("Could not find IssueViewerViewQuery in embedded JSON"); + } + + const timeline: IssueTimelineItem[] = []; + for (const edge of issue.frontTimelineItems.edges) { + if (edge.node.__typename === "IssueComment") { + timeline.push({ kind: "comment", ...parseIssueComment(edge.node) }); + continue; + } + const event = parseIssueEvent(edge.node); + if (event) timeline.push({ kind: "event", ...event }); + } + + const labels: IssueLabel[] = []; + for (const edge of issue.labels?.edges ?? []) { + labels.push(parseIssueLabel(edge.node)); + } + + const assignees: string[] = []; + for (const assignee of issue.assignedActors?.nodes ?? []) { + assignees.push(assignee.login); + } + + const milestoneHref = githubPath(issue.milestone?.url); + + return { + repo: { owner, name: repo }, + number: issue.number, + titleHtml: issue.titleHTML, + state: issue.state, + stateReason: issue.stateReason, + createdAt: issue.createdAt, + labels, + assignees, + milestone: + issue.milestone && milestoneHref + ? { title: issue.milestone.title, href: milestoneHref } + : null, + body: { + // The issue's own permalink, which GitHub anchors on its database id. + anchor: `issue-${issue.databaseId}`, + author: issue.author?.login ?? null, + association: null, + createdAt: issue.createdAt, + bodyHtml: issue.bodyHTML, + }, + timeline, + hiddenItems: issue.frontTimelineItems.totalCount - issue.frontTimelineItems.edges.length, + }; +} + export async function getGitHubPulls( owner: string, repo: string, |
