diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/index.ts | 8 | ||||
| -rw-r--r-- | src/scraper.test.ts | 37 | ||||
| -rw-r--r-- | src/scraper.ts | 121 |
3 files changed, 120 insertions, 46 deletions
diff --git a/src/index.ts b/src/index.ts index d8bdfec..f676bd6 100644 --- a/src/index.ts +++ b/src/index.ts @@ -9,12 +9,12 @@ import { getGitHubBlob, getGitHubCommits, getGitHubIssues, + getGitHubOwner, getGitHubPulls, getGitHubRelease, getGitHubReleases, getGitHubRepo, getGitHubTree, - getGitHubUser, GitHubHTTPError, RedirectError, } from "./scraper.ts"; @@ -52,9 +52,9 @@ async function tryRender<T extends object>(c: Context, template: string, promise return c.html(await eta.renderAsync(template, data)); } -app.get("/:user", async (c) => { - const { user } = c.req.param(); - return tryRender(c, "user.eta", getGitHubUser(user)); +app.get("/:owner", async (c) => { + const { owner } = c.req.param(); + return tryRender(c, "owner.eta", getGitHubOwner(owner)); }); app.get("/:owner/:repo", async (c) => { diff --git a/src/scraper.test.ts b/src/scraper.test.ts index 2438361..3048915 100644 --- a/src/scraper.test.ts +++ b/src/scraper.test.ts @@ -8,8 +8,8 @@ import { getGitHubRelease, getGitHubReleases, getGitHubRepo, + getGitHubOwner, getGitHubTree, - getGitHubUser, GitHubHTTPError, RedirectError, } from "./scraper.ts"; @@ -264,10 +264,11 @@ describe("GitHub scraper", () => { }); }); - describe("user pages", () => { + describe("owner pages", () => { it("should fetch user profile", async () => { - const data = await getGitHubUser("torvalds"); + const data = await getGitHubOwner("torvalds"); + assert.strictEqual(data.type, "user"); assert.strictEqual(data.login, "torvalds"); assert.strictEqual(data.name, "Linus Torvalds"); assert.strictEqual(data.company, "Linux Foundation"); @@ -287,7 +288,7 @@ describe("GitHub scraper", () => { }); it("should fetch user bio and links", async () => { - const data = await getGitHubUser("sindresorhus"); + const data = await getGitHubOwner("sindresorhus"); assert.strictEqual(data.login, "sindresorhus"); assert.ok(data.bio && data.bio.length > 0); @@ -295,16 +296,34 @@ describe("GitHub scraper", () => { assert.ok(data.links.every((link) => link.href.startsWith("https://"))); }); - it("should redirect to GitHub for organizations", async () => { - await assert.rejects(getGitHubUser("github"), (err) => { + it("should fetch organization profile", async () => { + const data = await getGitHubOwner("github"); + + assert.strictEqual(data.type, "organization"); + assert.strictEqual(data.login, "github"); + assert.strictEqual(data.name, "GitHub"); + assert.strictEqual(data.bio, "How people build software."); + assert.strictEqual(data.location, "United States of America"); + assert.ok(data.avatarUrl?.startsWith("https://avatars.githubusercontent.com/")); + assert.ok(data.followers); + assert.strictEqual(data.following, null); + assert.ok(data.links.some((link) => link.href === "https://github.com/about")); + + assert.ok(data.pinned.length > 0); + assert.ok(data.pinned.every((repo) => repo.owner === "github")); + assert.ok(data.pinned.some((repo) => repo.stars && repo.forks)); + }); + + it("should redirect to GitHub for non-profile pages", async () => { + await assert.rejects(getGitHubOwner("features"), (err) => { assert(err instanceof RedirectError, "error should be a RedirectError"); - assert.strictEqual(err.location, "https://github.com/github"); + assert.strictEqual(err.location, "https://github.com/features"); return true; }); }); - it("should throw for non-existent user", async () => { - await assert.rejects(getGitHubUser("nosuchuserzzzz"), (err) => { + it("should throw for non-existent owner", async () => { + await assert.rejects(getGitHubOwner("nosuchuserzzzz"), (err) => { assert(err instanceof GitHubHTTPError, "error should be an HTTPError"); assert.strictEqual(err.status, 404); return true; diff --git a/src/scraper.ts b/src/scraper.ts index 3434434..3e3ad75 100644 --- a/src/scraper.ts +++ b/src/scraper.ts @@ -292,7 +292,7 @@ export interface GitHubRelease extends GitHubCommon { release: Release; } -interface UserPinnedRepo { +interface OwnerPinnedRepo { owner: string; name: string; description?: string; @@ -301,17 +301,19 @@ interface UserPinnedRepo { forks?: string; } -export interface GitHubUser { +export interface GitHubOwner { + type: "user" | "organization"; login: string; name: string | null; bio: string | null; avatarUrl: string | null; followers: string | null; + // Users only; always null for organizations. following: string | null; company: string | null; location: string | null; links: { text: string; href: string }[]; - pinned: UserPinnedRepo[]; + pinned: OwnerPinnedRepo[]; } async function fetchGitHubPage(path: string): Promise<string> { @@ -768,16 +770,37 @@ function selectText(selector: string, context: AnyNode): string | null { return textContent(elem).replace(/\s+/g, " ").trim() || null; } -export async function getGitHubUser(user: string): Promise<GitHubUser> { - const html = await fetchGitHubPage(user); - const document = parseDocument(html); +// Pinned repositories appear as .pinned-item-list-item on user pages and as +// [itemprop='owns'] blocks on organization pages. +function parsePinnedRepos(document: AnyNode): OwnerPinnedRepo[] { + const pinned: OwnerPinnedRepo[] = []; + const items = cssSelect.selectAll( + ".pinned-item-list-item, [itemprop='owns']", + document, + ) as unknown as Element[]; + for (const item of items) { + const link = cssSelect.selectOne( + "a:has(span.repo), a[itemprop~='codeRepository']", + item, + ) as Element | null; + const href = link ? getAttributeValue(link, "href") : undefined; + if (!href) continue; + const [owner, name] = href.replace(/^\//, "").split("/"); + if (!owner || !name) continue; - // Organizations and other non-profile pages lack the user vcard. - const login = selectText(".p-nickname.vcard-username", document); - if (!login) { - throw new RedirectError(`https://github.com/${user}`); + pinned.push({ + owner, + name, + description: selectText("p.pinned-item-desc, [itemprop='description']", item) ?? undefined, + language: selectText("[itemprop='programmingLanguage']", item) ?? undefined, + stars: selectText("a[href$='/stargazers']", item) ?? undefined, + forks: selectText("a[href$='/forks']", item) ?? undefined, + }); } + return pinned; +} +function parseUser(document: AnyNode, login: string): GitHubOwner { const avatarElem = cssSelect.selectOne("a[itemprop='image'] img", document) as Element | null; const avatarUrl = avatarElem ? getAttributeValue(avatarElem, "src") || null : null; @@ -798,29 +821,8 @@ export async function getGitHubUser(user: string): Promise<GitHubUser> { links.push({ text: textContent(anchor).replace(/\s+/g, " ").trim(), href }); } - const pinned: UserPinnedRepo[] = []; - const pinnedItems = cssSelect.selectAll( - ".pinned-item-list-item", - document, - ) as unknown as Element[]; - for (const item of pinnedItems) { - const link = cssSelect.selectOne("a:has(span.repo)", item) as Element | null; - const href = link ? getAttributeValue(link, "href") : undefined; - if (!href) continue; - const [owner, name] = href.replace(/^\//, "").split("/"); - if (!owner || !name) continue; - - pinned.push({ - owner, - name, - description: selectText("p.pinned-item-desc", item) ?? undefined, - language: selectText("[itemprop='programmingLanguage']", item) ?? undefined, - stars: selectText("a[href$='/stargazers']", item) ?? undefined, - forks: selectText("a[href$='/forks']", item) ?? undefined, - }); - } - return { + type: "user", login, name: selectText(".p-name.vcard-fullname", document), bio: selectText(".p-note.user-profile-bio", document), @@ -830,6 +832,59 @@ export async function getGitHubUser(user: string): Promise<GitHubUser> { company: selectText("li[itemprop='worksFor']", document), location: selectText("li[itemprop='homeLocation']", document), links, - pinned, + pinned: parsePinnedRepos(document), }; } + +function parseOrganization(document: AnyNode, owner: string): GitHubOwner { + const avatarElem = cssSelect.selectOne(".orghead img[itemprop='image']", document) as + | Element + | null; + const avatarUrl = avatarElem ? getAttributeValue(avatarElem, "src") || null : null; + // The login only appears in the avatar's "@login" alt text. + const alt = avatarElem ? getAttributeValue(avatarElem, "alt") : undefined; + const login = alt?.replace(/^@/, "") || owner; + + const followers = + selectText("a[href$='/followers']", document)?.replace(/ followers?$/, "") ?? null; + + const links: { text: string; href: string }[] = []; + const anchors = cssSelect.selectAll( + ".orghead a[itemprop='url'], .orghead [itemprop='email'] a", + document, + ) as unknown as Element[]; + for (const anchor of anchors) { + const href = getAttributeValue(anchor, "href"); + if (!href) continue; + links.push({ text: textContent(anchor).replace(/\s+/g, " ").trim(), href }); + } + + return { + type: "organization", + login, + name: selectText(".orghead h1", document), + bio: selectText(".orghead h1 + div", document), + avatarUrl, + followers, + following: null, + company: null, + location: selectText(".orghead [itemprop='location']", document), + links, + pinned: parsePinnedRepos(document), + }; +} + +export async function getGitHubOwner(owner: string): Promise<GitHubOwner> { + const html = await fetchGitHubPage(owner); + const document = parseDocument(html); + + const login = selectText(".p-nickname.vcard-username", document); + if (login) { + return parseUser(document, login); + } + if (cssSelect.selectOne(".orghead", document)) { + return parseOrganization(document, owner); + } + // Not a profile page (marketing pages like github.com/features, etc.). + throw new RedirectError(`https://github.com/${owner}`); +} |