diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/index.ts | 9 | ||||
| -rw-r--r-- | src/scraper.test.ts | 49 | ||||
| -rw-r--r-- | src/scraper.ts | 96 |
3 files changed, 153 insertions, 1 deletions
diff --git a/src/index.ts b/src/index.ts index 514f2e2..d8bdfec 100644 --- a/src/index.ts +++ b/src/index.ts @@ -14,6 +14,7 @@ import { getGitHubReleases, getGitHubRepo, getGitHubTree, + getGitHubUser, GitHubHTTPError, RedirectError, } from "./scraper.ts"; @@ -35,6 +36,9 @@ async function tryRender<T extends object>(c: Context, template: string, promise data = await promise; } catch (e) { if (e instanceof RedirectError) { + if (e.location.startsWith("https://")) { + c.header("Referrer-Policy", "no-referrer"); + } return c.redirect(e.location); } else if (e instanceof GitHubHTTPError) { c.status(e.status as StatusCode); @@ -48,6 +52,11 @@ async function tryRender<T extends object>(c: Context, template: string, promise return c.html(await eta.renderAsync(template, data)); } +app.get("/:user", async (c) => { + const { user } = c.req.param(); + return tryRender(c, "user.eta", getGitHubUser(user)); +}); + app.get("/:owner/:repo", async (c) => { const { owner, repo } = c.req.param(); return tryRender(c, "repo.eta", getGitHubRepo(owner, repo)); diff --git a/src/scraper.test.ts b/src/scraper.test.ts index a4b834e..4a01853 100644 --- a/src/scraper.test.ts +++ b/src/scraper.test.ts @@ -9,6 +9,7 @@ import { getGitHubReleases, getGitHubRepo, getGitHubTree, + getGitHubUser, GitHubHTTPError, RedirectError, } from "./scraper.ts"; @@ -264,6 +265,54 @@ describe("GitHub scraper", () => { }); }); + describe("user pages", () => { + it("should fetch user profile", async () => { + const data = await getGitHubUser("torvalds"); + + assert.strictEqual(data.login, "torvalds"); + assert.strictEqual(data.name, "Linus Torvalds"); + assert.strictEqual(data.company, "Linux Foundation"); + assert.strictEqual(data.location, "Portland, OR"); + assert.ok(data.avatarUrl?.startsWith("https://avatars.githubusercontent.com/")); + assert.ok(data.followers); + assert.ok(data.following !== null); + + assert.ok(data.pinned.length > 0); + const linux = data.pinned.find((repo) => repo.name === "linux"); + assert.ok(linux); + assert.strictEqual(linux.owner, "torvalds"); + assert.strictEqual(linux.description, "Linux kernel source tree"); + assert.strictEqual(linux.language, "C"); + assert.ok(linux.stars); + assert.ok(linux.forks); + }); + + it("should fetch user bio and links", async () => { + const data = await getGitHubUser("sindresorhus"); + + assert.strictEqual(data.login, "sindresorhus"); + assert.ok(data.bio && data.bio.length > 0); + assert.ok(data.links.length > 0); + assert.ok(data.links.every((link) => link.href.startsWith("https://"))); + }); + + it("should redirect to GitHub for organizations", async () => { + await assert.rejects(getGitHubUser("github"), (err) => { + assert(err instanceof RedirectError, "error should be a RedirectError"); + assert.strictEqual(err.location, "https://github.com/github"); + return true; + }); + }); + + it("should throw for non-existent user", async () => { + await assert.rejects(getGitHubUser("nosuchuserzzzz"), (err) => { + assert(err instanceof GitHubHTTPError, "error should be an HTTPError"); + assert.strictEqual(err.status, 404); + return true; + }); + }); + }); + describe("error handling", () => { it("should throw for non-existent repository", async () => { await assert.rejects(getGitHubRepo("nosuchowner", "nosuchrepo"), (err) => { diff --git a/src/scraper.ts b/src/scraper.ts index f32a8da..a44f0a7 100644 --- a/src/scraper.ts +++ b/src/scraper.ts @@ -1,5 +1,5 @@ import { parseDocument } from "htmlparser2"; -import { type Element } from "domhandler"; +import { type AnyNode, type Element } from "domhandler"; import { getAttributeValue, getInnerHTML, textContent } from "domutils"; import * as cssSelect from "css-select"; @@ -293,6 +293,28 @@ export interface GitHubRelease extends GitHubCommon { release: Release; } +interface UserPinnedRepo { + owner: string; + name: string; + description?: string; + language?: string; + stars?: string; + forks?: string; +} + +export interface GitHubUser { + login: string; + name: string | null; + bio: string | null; + avatarUrl: string | null; + followers: string | null; + following: string | null; + company: string | null; + location: string | null; + links: { text: string; href: string }[]; + pinned: UserPinnedRepo[]; +} + async function fetchGitHubPage(path: string): Promise<string> { // GitHub throttles/blocks requests without realistic browser headers. // These headers make the request appear as a standard browser visit. @@ -744,3 +766,75 @@ export async function getGitHubRelease( base.release.assets = assets; return base; } + +function selectText(selector: string, context: AnyNode): string | null { + const elem = cssSelect.selectOne(selector, context); + if (!elem) return null; + return textContent(elem).replace(/\s+/g, " ").trim() || null; +} + +export async function getGitHubUser(user: string): Promise<GitHubUser> { + const html = await fetchGitHubPage(user); + const document = parseDocument(html); + + // Organizations and other non-profile pages lack the user vcard. + const login = selectText(".p-nickname.vcard-username", document); + if (!login) { + throw new RedirectError(`https://github.com/${user}`); + } + + const avatarElem = cssSelect.selectOne("a[itemprop='image'] img", document) as Element | null; + const avatarUrl = avatarElem ? getAttributeValue(avatarElem, "src") || null : null; + + const followers = + selectText("a[href$='tab=followers']", document)?.replace(/ followers?$/, "") ?? null; + const following = + selectText("a[href$='tab=following']", document)?.replace(/ following$/, "") ?? null; + + const links: { text: string; href: string }[] = []; + const linkItems = cssSelect.selectAll( + "li[itemprop='url'], li[itemprop='social']", + document, + ) as unknown as Element[]; + for (const item of linkItems) { + const anchor = cssSelect.selectOne("a[href]", item) as Element | null; + const href = anchor ? getAttributeValue(anchor, "href") : undefined; + if (!anchor || !href) continue; + links.push({ text: textContent(anchor).replace(/\s+/g, " ").trim(), href }); + } + + const pinned: UserPinnedRepo[] = []; + const pinnedItems = cssSelect.selectAll( + ".pinned-item-list-item", + document, + ) as unknown as Element[]; + for (const item of pinnedItems) { + const link = cssSelect.selectOne("a:has(span.repo)", item) as Element | null; + const href = link ? getAttributeValue(link, "href") : undefined; + if (!href) continue; + const [owner, name] = href.replace(/^\//, "").split("/"); + if (!owner || !name) continue; + + pinned.push({ + owner, + name, + description: selectText("p.pinned-item-desc", item) ?? undefined, + language: selectText("[itemprop='programmingLanguage']", item) ?? undefined, + stars: selectText("a[href$='/stargazers']", item) ?? undefined, + forks: selectText("a[href$='/forks']", item) ?? undefined, + }); + } + + return { + login, + name: selectText(".p-name.vcard-fullname", document), + bio: selectText(".p-note.user-profile-bio", document), + avatarUrl, + followers, + following, + company: selectText("li[itemprop='worksFor']", document), + location: selectText("li[itemprop='homeLocation']", document), + links, + pinned, + }; +} |