diff options
| -rw-r--r-- | src/scraper.test.ts | 22 | ||||
| -rw-r--r-- | src/scraper.ts | 57 | ||||
| -rw-r--r-- | views/_ownerRepo.eta | 15 | ||||
| -rw-r--r-- | views/owner.eta | 65 |
4 files changed, 118 insertions, 41 deletions
diff --git a/src/scraper.test.ts b/src/scraper.test.ts index 3048915..d6991d7 100644 --- a/src/scraper.test.ts +++ b/src/scraper.test.ts @@ -285,6 +285,12 @@ describe("GitHub scraper", () => { assert.strictEqual(linux.language, "C"); assert.ok(linux.stars); assert.ok(linux.forks); + + assert.deepStrictEqual(data.repos, []); + const navTexts = data.nav.map((item) => item.text); + assert.deepStrictEqual(navTexts, ["Repositories", "Projects", "Packages"]); + const reposTab = data.nav.find((item) => item.text === "Repositories"); + assert.strictEqual(reposTab?.href, "/torvalds?tab=repositories"); }); it("should fetch user bio and links", async () => { @@ -294,6 +300,7 @@ describe("GitHub scraper", () => { assert.ok(data.bio && data.bio.length > 0); assert.ok(data.links.length > 0); assert.ok(data.links.every((link) => link.href.startsWith("https://"))); + assert.ok(data.nav.some((item) => item.text === "Sponsoring")); }); it("should fetch organization profile", async () => { @@ -310,8 +317,23 @@ describe("GitHub scraper", () => { assert.ok(data.links.some((link) => link.href === "https://github.com/about")); assert.ok(data.pinned.length > 0); + assert.ok(data.pinned.length <= 6, "pinned should not include the repository list"); assert.ok(data.pinned.every((repo) => repo.owner === "github")); assert.ok(data.pinned.some((repo) => repo.stars && repo.forks)); + + assert.ok(data.repos.length > 0); + assert.ok(data.repos.every((repo) => repo.owner === "github")); + + const navTexts = data.nav.map((item) => item.text); + assert.deepStrictEqual(navTexts, [ + "Repositories", + "Projects", + "Packages", + "People", + "Sponsoring", + ]); + const reposTab = data.nav.find((item) => item.text === "Repositories"); + assert.strictEqual(reposTab?.href, "/orgs/github/repositories"); }); it("should redirect to GitHub for non-profile pages", async () => { diff --git a/src/scraper.ts b/src/scraper.ts index 3e3ad75..760a306 100644 --- a/src/scraper.ts +++ b/src/scraper.ts @@ -292,7 +292,7 @@ export interface GitHubRelease extends GitHubCommon { release: Release; } -interface OwnerPinnedRepo { +interface OwnerRepo { owner: string; name: string; description?: string; @@ -301,6 +301,11 @@ interface OwnerPinnedRepo { forks?: string; } +interface OwnerNavItem { + text: string; + href: string; +} + export interface GitHubOwner { type: "user" | "organization"; login: string; @@ -313,7 +318,10 @@ export interface GitHubOwner { company: string | null; location: string | null; links: { text: string; href: string }[]; - pinned: OwnerPinnedRepo[]; + nav: OwnerNavItem[]; + pinned: OwnerRepo[]; + // The repository list on organization overviews; empty for users. + repos: OwnerRepo[]; } async function fetchGitHubPage(path: string): Promise<string> { @@ -770,14 +778,8 @@ function selectText(selector: string, context: AnyNode): string | null { return textContent(elem).replace(/\s+/g, " ").trim() || null; } -// Pinned repositories appear as .pinned-item-list-item on user pages and as -// [itemprop='owns'] blocks on organization pages. -function parsePinnedRepos(document: AnyNode): OwnerPinnedRepo[] { - const pinned: OwnerPinnedRepo[] = []; - const items = cssSelect.selectAll( - ".pinned-item-list-item, [itemprop='owns']", - document, - ) as unknown as Element[]; +function parseRepoItems(items: Element[]): OwnerRepo[] { + const repos: OwnerRepo[] = []; for (const item of items) { const link = cssSelect.selectOne( "a:has(span.repo), a[itemprop~='codeRepository']", @@ -788,7 +790,7 @@ function parsePinnedRepos(document: AnyNode): OwnerPinnedRepo[] { const [owner, name] = href.replace(/^\//, "").split("/"); if (!owner || !name) continue; - pinned.push({ + repos.push({ owner, name, description: selectText("p.pinned-item-desc, [itemprop='description']", item) ?? undefined, @@ -797,7 +799,32 @@ function parsePinnedRepos(document: AnyNode): OwnerPinnedRepo[] { forks: selectText("a[href$='/forks']", item) ?? undefined, }); } - return pinned; + return repos; +} + +function parsePinnedRepos(document: AnyNode): OwnerRepo[] { + return parseRepoItems( + cssSelect.selectAll(".pinned-item-list-item", document) as unknown as Element[], + ); +} + +// The tabs we surface from the profile page's own navigation. Stars and +// Overview are deliberately excluded. +const OWNER_NAV_TABS = ["Repositories", "Projects", "Packages", "People", "Sponsoring"]; + +function parseOwnerNav(document: AnyNode): OwnerNavItem[] { + const nav: OwnerNavItem[] = []; + const seen = new Set<string>(); + const anchors = cssSelect.selectAll("a.UnderlineNav-item", document) as unknown as Element[]; + for (const anchor of anchors) { + const href = getAttributeValue(anchor, "href"); + // Label text is e.g. "Repositories 12" including the counter. + const text = textContent(anchor).replace(/\s+/g, " ").trim().split(" ")[0]; + if (!href || seen.has(href) || !OWNER_NAV_TABS.includes(text)) continue; + seen.add(href); + nav.push({ text, href }); + } + return nav; } function parseUser(document: AnyNode, login: string): GitHubOwner { @@ -832,7 +859,9 @@ function parseUser(document: AnyNode, login: string): GitHubOwner { company: selectText("li[itemprop='worksFor']", document), location: selectText("li[itemprop='homeLocation']", document), links, + nav: parseOwnerNav(document), pinned: parsePinnedRepos(document), + repos: [], }; } @@ -870,7 +899,11 @@ function parseOrganization(document: AnyNode, owner: string): GitHubOwner { company: null, location: selectText(".orghead [itemprop='location']", document), links, + nav: parseOwnerNav(document), pinned: parsePinnedRepos(document), + repos: parseRepoItems( + cssSelect.selectAll("#org-repositories [itemprop='owns']", document) as unknown as Element[], + ), }; } diff --git a/views/_ownerRepo.eta b/views/_ownerRepo.eta new file mode 100644 index 0000000..a857aff --- /dev/null +++ b/views/_ownerRepo.eta @@ -0,0 +1,15 @@ +<div class="box repo-info"> + <div><a href="/<%= it.repo.owner %>/<%= it.repo.name %>"><%= it.repo.owner === it.login ? it.repo.name : `${it.repo.owner}/${it.repo.name}` %></a></div> + <% if (it.repo.description) { %> + <div class="description"><%= it.repo.description %></div> + <% } %> + <% if (it.repo.language) { %> + <span class="detail"><%= it.repo.language %></span> + <% } %> + <% if (it.repo.stars) { %> + <span class="detail"><img src="/static/icons/star.svg" alt="stars" class="icon"> <%= it.repo.stars %></span> + <% } %> + <% if (it.repo.forks) { %> + <span class="detail"><img src="/static/icons/git-fork.svg" alt="forks" class="icon"> <%= it.repo.forks %></span> + <% } %> +</div> diff --git a/views/owner.eta b/views/owner.eta index 4426400..e5667fb 100644 --- a/views/owner.eta +++ b/views/owner.eta @@ -7,48 +7,55 @@ <link rel="stylesheet" href="/static/style.css"> </head> <body> - <div class="box repo-info"> + <div class="box"> <% if (it.avatarUrl) { %> <img class="avatar" src="<%= it.avatarUrl %>" alt="" width="64" height="64"> <% } %> <div class="repo"><%= it.name || it.login %></div> - <div class="detail"><%= it.login %><%= it.type === "organization" ? " (organization)" : "" %></div> - <% if (it.bio) { %> - <div class="description"><%= it.bio %></div> - <% } %> - <% if (it.followers) { %> - <div class="detail"><img src="/static/icons/eye.svg" alt="" class="icon"> <%= it.followers %> followers<%= it.following ? ` · ${it.following} following` : "" %></div> - <% } %> - <% if (it.company) { %> - <div class="detail"><%= it.company %></div> - <% } %> + <div><%= it.login %><%= it.type === "organization" ? " (organization)" : "" %></div> <% if (it.location) { %> - <div class="detail"><%= it.location %></div> + <div><%= it.location %></div> <% } %> - <% for (const link of it.links) { %> - <div class="detail"><img src="/static/icons/link.svg" alt="" class="icon"> <a href="<%= link.href %>" rel="noopener noreferrer"><%= link.text %></a></div> + <% if (it.nav.length > 0) { %> + <nav> + <% for (const item of it.nav) { %> + <a href="<%= item.href %>"><%= item.text %></a> + <% } %> + </nav> <% } %> </div> + <% if (it.bio || it.links.length > 0 || it.followers || it.company) { %> + <div class="box repo-info"> + <% if (it.bio) { %> + <div class="description"><%= it.bio %></div> + <% } %> + <% for (const link of it.links) { %> + <div class="detail"><img src="/static/icons/link.svg" alt="" class="icon"> <a href="<%= link.href %>" rel="noopener noreferrer"><%= link.text %></a></div> + <% } %> + <% if (it.followers) { %> + <div class="detail"><img src="/static/icons/eye.svg" alt="" class="icon"> <%= it.followers %> followers<%= it.following ? ` · ${it.following} following` : "" %></div> + <% } %> + <% if (it.company) { %> + <div class="detail"><%= it.company %></div> + <% } %> + </div> + <% } %> + <% if (it.pinned.length > 0) { %> <section> <h2>Pinned</h2> <% for (const repo of it.pinned) { %> - <div class="box repo-info"> - <div><a href="/<%= repo.owner %>/<%= repo.name %>"><%= repo.owner === it.login ? repo.name : `${repo.owner}/${repo.name}` %></a></div> - <% if (repo.description) { %> - <div class="description"><%= repo.description %></div> - <% } %> - <% if (repo.language) { %> - <span class="detail"><%= repo.language %></span> - <% } %> - <% if (repo.stars) { %> - <span class="detail"><img src="/static/icons/star.svg" alt="stars" class="icon"> <%= repo.stars %></span> - <% } %> - <% if (repo.forks) { %> - <span class="detail"><img src="/static/icons/git-fork.svg" alt="forks" class="icon"> <%= repo.forks %></span> - <% } %> - </div> + <%~ include("./_ownerRepo", { repo, login: it.login }) %> + <% } %> + </section> + <% } %> + + <% if (it.repos.length > 0) { %> + <section> + <h2>Repositories</h2> + <% for (const repo of it.repos) { %> + <%~ include("./_ownerRepo", { repo, login: it.login }) %> <% } %> </section> <% } %> |