diff options
| -rw-r--r-- | .gitignore | 2 | ||||
| -rwxr-xr-x | github-scraper.js | 74 | ||||
| -rw-r--r-- | package.json | 22 | ||||
| -rw-r--r-- | pnpm-lock.yaml | 273 | ||||
| -rw-r--r-- | src/index.ts | 36 | ||||
| -rw-r--r-- | src/scraper.ts | 91 | ||||
| -rw-r--r-- | static/style.css | 14 | ||||
| -rw-r--r-- | tsconfig.json | 19 | ||||
| -rw-r--r-- | views/tree.eta | 42 |
9 files changed, 499 insertions, 74 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..34b23da --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/build/ +/node_modules/ diff --git a/github-scraper.js b/github-scraper.js deleted file mode 100755 index 7e07adc..0000000 --- a/github-scraper.js +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env node -/** - * GitHub Directory Listing Scraper - * - * Gets directory listings from GitHub without using the API (no rate limits!) - * Works by parsing the server-rendered HTML that GitHub sends to browsers. - */ - -async function getGitHubDirectory(owner, repo, path = '') { - // Always use the tree URL pattern, even for root directory - const url = `https://github.com/${owner}/${repo}/tree/main/${path}`; - - console.error(`Fetching: ${url}`); - - const response = await fetch(url); - const html = await response.text(); - - // GitHub embeds the directory data in a JSON script tag - // Using the tree (subdirectory) pattern for all cases - const subdirMatch = html.match(/<script type="application\/json" data-target="react-app\.embeddedData">([^<]+)<\/script>/); - - if (!subdirMatch) { - throw new Error('Could not find embedded data in HTML'); - } - - const data = JSON.parse(subdirMatch[1]); - const payload = data.payload; - - return { - path: payload.path, - branch: payload.refInfo.name, - items: payload.tree.items, - repo: { - name: payload.repo.name, - owner: payload.repo.ownerLogin, - isPublic: payload.repo.public - } - }; -} - -// Example usage -if (require.main === module) { - const [owner, repo, path] = process.argv.slice(2); - - if (!owner || !repo) { - console.log('Usage: node github-scraper.js <owner> <repo> [path]'); - console.log('Example: node github-scraper.js torvalds linux'); - console.log('Example: node github-scraper.js torvalds linux Documentation'); - process.exit(1); - } - - getGitHubDirectory(owner, repo, path) - .then(result => { - console.log('\nš Directory listing'); - console.log(`Repository: ${result.repo.owner}/${result.repo.name}`); - console.log(`Branch: ${result.branch}`); - console.log(`Path: ${result.path}`); - console.log(`Items: ${result.items.length}`); - console.log('\nContents:'); - - result.items.forEach(item => { - const icon = item.contentType === 'directory' ? 'š' : 'š'; - console.log(` ${icon} ${item.name}`); - }); - - console.log('\nā No API rate limits - this scrapes the HTML!'); - }) - .catch(err => { - console.error('Error:', err.message); - process.exit(1); - }); -} - -module.exports = { getGitHubDirectory }; diff --git a/package.json b/package.json new file mode 100644 index 0000000..3e62cba --- /dev/null +++ b/package.json @@ -0,0 +1,22 @@ +{ + "name": "cgithub", + "version": "1.0.0", + "description": "Lightweight GitHub alternative frontend", + "type": "module", + "scripts": { + "dev": "concurrently 'tsc --watch --preserveWatchOutput' 'node --watch --watch-preserve-output build/index.js'", + "build": "tsc", + "start": "node build/index.js" + }, + "dependencies": { + "@hono/node-server": "^1.13.7", + "eta": "^3.5.0", + "hono": "^4.6.14" + }, + "devDependencies": { + "@types/node": "^22.10.2", + "concurrently": "^9.1.2", + "prettier": "^3.8.1", + "typescript": "^5.7.2" + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml new file mode 100644 index 0000000..15de9fe --- /dev/null +++ b/pnpm-lock.yaml @@ -0,0 +1,273 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + '@hono/node-server': + specifier: ^1.13.7 + version: 1.19.9(hono@4.11.5) + eta: + specifier: ^3.5.0 + version: 3.5.0 + hono: + specifier: ^4.6.14 + version: 4.11.5 + devDependencies: + '@types/node': + specifier: ^22.10.2 + version: 22.19.7 + concurrently: + specifier: ^9.1.2 + version: 9.2.1 + prettier: + specifier: ^3.8.1 + version: 3.8.1 + typescript: + specifier: ^5.7.2 + version: 5.9.3 + +packages: + + '@hono/node-server@1.19.9': + resolution: {integrity: sha512-vHL6w3ecZsky+8P5MD+eFfaGTyCeOHUIFYMGpQGbrBTSmNNoxv0if69rEZ5giu36weC5saFuznL411gRX7bJDw==} + engines: {node: '>=18.14.1'} + peerDependencies: + hono: ^4 + + '@types/node@22.19.7': + resolution: {integrity: sha512-MciR4AKGHWl7xwxkBa6xUGxQJ4VBOmPTF7sL+iGzuahOFaO0jHCsuEfS80pan1ef4gWId1oWOweIhrDEYLuaOw==} + + ansi-regex@5.0.1: + resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} + engines: {node: '>=8'} + + ansi-styles@4.3.0: + resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==} + engines: {node: '>=8'} + + chalk@4.1.2: + resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} + engines: {node: '>=10'} + + cliui@8.0.1: + resolution: {integrity: sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==} + engines: {node: '>=12'} + + color-convert@2.0.1: + resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} + engines: {node: '>=7.0.0'} + + color-name@1.1.4: + resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} + + concurrently@9.2.1: + resolution: {integrity: sha512-fsfrO0MxV64Znoy8/l1vVIjjHa29SZyyqPgQBwhiDcaW8wJc2W3XWVOGx4M3oJBnv/zdUZIIp1gDeS98GzP8Ng==} + engines: {node: '>=18'} + hasBin: true + + emoji-regex@8.0.0: + resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==} + + escalade@3.2.0: + resolution: {integrity: sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==} + engines: {node: '>=6'} + + eta@3.5.0: + resolution: {integrity: sha512-e3x3FBvGzeCIHhF+zhK8FZA2vC5uFn6b4HJjegUbIWrDb4mJ7JjTGMJY9VGIbRVpmSwHopNiaJibhjIr+HfLug==} + engines: {node: '>=6.0.0'} + + get-caller-file@2.0.5: + resolution: {integrity: sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==} + engines: {node: 6.* || 8.* || >= 10.*} + + has-flag@4.0.0: + resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} + engines: {node: '>=8'} + + hono@4.11.5: + resolution: {integrity: sha512-WemPi9/WfyMwZs+ZUXdiwcCh9Y+m7L+8vki9MzDw3jJ+W9Lc+12HGsd368Qc1vZi1xwW8BWMMsnK5efYKPdt4g==} + engines: {node: '>=16.9.0'} + + is-fullwidth-code-point@3.0.0: + resolution: {integrity: sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==} + engines: {node: '>=8'} + + prettier@3.8.1: + resolution: {integrity: sha512-UOnG6LftzbdaHZcKoPFtOcCKztrQ57WkHDeRD9t/PTQtmT0NHSeWWepj6pS0z/N7+08BHFDQVUrfmfMRcZwbMg==} + engines: {node: '>=14'} + hasBin: true + + require-directory@2.1.1: + resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==} + engines: {node: '>=0.10.0'} + + rxjs@7.8.2: + resolution: {integrity: sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==} + + shell-quote@1.8.3: + resolution: {integrity: sha512-ObmnIF4hXNg1BqhnHmgbDETF8dLPCggZWBjkQfhZpbszZnYur5DUljTcCHii5LC3J5E0yeO/1LIMyH+UvHQgyw==} + engines: {node: '>= 0.4'} + + string-width@4.2.3: + resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} + engines: {node: '>=8'} + + strip-ansi@6.0.1: + resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==} + engines: {node: '>=8'} + + supports-color@7.2.0: + resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==} + engines: {node: '>=8'} + + supports-color@8.1.1: + resolution: {integrity: sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==} + engines: {node: '>=10'} + + tree-kill@1.2.2: + resolution: {integrity: sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==} + hasBin: true + + tslib@2.8.1: + resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + + typescript@5.9.3: + resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} + engines: {node: '>=14.17'} + hasBin: true + + undici-types@6.21.0: + resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} + + wrap-ansi@7.0.0: + resolution: {integrity: sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==} + engines: {node: '>=10'} + + y18n@5.0.8: + resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==} + engines: {node: '>=10'} + + yargs-parser@21.1.1: + resolution: {integrity: sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==} + engines: {node: '>=12'} + + yargs@17.7.2: + resolution: {integrity: sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==} + engines: {node: '>=12'} + +snapshots: + + '@hono/node-server@1.19.9(hono@4.11.5)': + dependencies: + hono: 4.11.5 + + '@types/node@22.19.7': + dependencies: + undici-types: 6.21.0 + + ansi-regex@5.0.1: {} + + ansi-styles@4.3.0: + dependencies: + color-convert: 2.0.1 + + chalk@4.1.2: + dependencies: + ansi-styles: 4.3.0 + supports-color: 7.2.0 + + cliui@8.0.1: + dependencies: + string-width: 4.2.3 + strip-ansi: 6.0.1 + wrap-ansi: 7.0.0 + + color-convert@2.0.1: + dependencies: + color-name: 1.1.4 + + color-name@1.1.4: {} + + concurrently@9.2.1: + dependencies: + chalk: 4.1.2 + rxjs: 7.8.2 + shell-quote: 1.8.3 + supports-color: 8.1.1 + tree-kill: 1.2.2 + yargs: 17.7.2 + + emoji-regex@8.0.0: {} + + escalade@3.2.0: {} + + eta@3.5.0: {} + + get-caller-file@2.0.5: {} + + has-flag@4.0.0: {} + + hono@4.11.5: {} + + is-fullwidth-code-point@3.0.0: {} + + prettier@3.8.1: {} + + require-directory@2.1.1: {} + + rxjs@7.8.2: + dependencies: + tslib: 2.8.1 + + shell-quote@1.8.3: {} + + string-width@4.2.3: + dependencies: + emoji-regex: 8.0.0 + is-fullwidth-code-point: 3.0.0 + strip-ansi: 6.0.1 + + strip-ansi@6.0.1: + dependencies: + ansi-regex: 5.0.1 + + supports-color@7.2.0: + dependencies: + has-flag: 4.0.0 + + supports-color@8.1.1: + dependencies: + has-flag: 4.0.0 + + tree-kill@1.2.2: {} + + tslib@2.8.1: {} + + typescript@5.9.3: {} + + undici-types@6.21.0: {} + + wrap-ansi@7.0.0: + dependencies: + ansi-styles: 4.3.0 + string-width: 4.2.3 + strip-ansi: 6.0.1 + + y18n@5.0.8: {} + + yargs-parser@21.1.1: {} + + yargs@17.7.2: + dependencies: + cliui: 8.0.1 + escalade: 3.2.0 + get-caller-file: 2.0.5 + require-directory: 2.1.1 + string-width: 4.2.3 + y18n: 5.0.8 + yargs-parser: 21.1.1 diff --git a/src/index.ts b/src/index.ts new file mode 100644 index 0000000..df0a116 --- /dev/null +++ b/src/index.ts @@ -0,0 +1,36 @@ +import { Hono } from "hono"; +import { serve } from "@hono/node-server"; +import { serveStatic } from "@hono/node-server/serve-static"; +import { Eta } from "eta"; +import { getGitHubTree } from "./scraper.js"; + +const app = new Hono(); +const eta = new Eta({ views: "./views" }); + +app.use("/static/*", serveStatic({ root: "." })); + +app.get("/:owner/:repo/tree/:branch/:path{.*}?", async (c) => { + try { + const owner = c.req.param("owner"); + const repo = c.req.param("repo"); + const branch = c.req.param("branch"); + const path = c.req.param("path") || ""; + + const data = await getGitHubTree(owner, repo, branch, path); + const html = await eta.renderAsync("tree.eta", data); + + return c.html(html); + } catch (error) { + const errorMessage = + error instanceof Error ? error.message : "Unknown error"; + return c.html( + `<html><body><h1>Error</h1><p>${errorMessage}</p></body></html>`, + 500, + ); + } +}); + +const port = parseInt(process.env.PORT || "3000", 10); +serve({ fetch: app.fetch, port }, (info) => { + console.log(`Server running on http://localhost:${info.port}`); +}); diff --git a/src/scraper.ts b/src/scraper.ts new file mode 100644 index 0000000..add03b2 --- /dev/null +++ b/src/scraper.ts @@ -0,0 +1,91 @@ +export interface GitHubTreeItem { + name: string; + path: string; + contentType: "directory" | "file"; +} + +export interface GitHubTree { + path: string; + branch: string; + items: GitHubTreeItem[]; + repo: { + name: string; + owner: string; + isPublic: boolean; + }; +} + +export async function getGitHubTree( + owner: string, + repo: string, + branch: string, + path: string, +): Promise<GitHubTree> { + const url = `https://github.com/${owner}/${repo}/tree/${branch}/${path}`; + + // GitHub throttles/blocks requests without realistic browser headers. + // These headers make the request appear as a standard browser visit. + const response = await fetch(url, { + headers: { + "User-Agent": + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", + Accept: + "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.5", + }, + }); + const html = await response.text(); + + // GitHub uses different formats for root vs subdirectories. + let payload; + + // Try root format first. + const rootFormatMatches = [ + ...html.matchAll( + /<script type="application\/json" data-target="react-partial\.embeddedData">([^<]+)<\/script>/g, + ), + ]; + for (const match of rootFormatMatches) { + try { + const data = JSON.parse(match[1]); + if (data.props?.initialPayload?.tree) { + payload = data.props.initialPayload; + break; + } + } catch (e) { + continue; + } + } + + // Try subdirectory format. + if (!payload) { + const dirFormatMatch = html.match( + /<script type="application\/json" data-target="react-app\.embeddedData">([^<]+)<\/script>/, + ); + if (dirFormatMatch) { + try { + const data = JSON.parse(dirFormatMatch[1]); + if (data.payload?.tree) { + payload = data.payload; + } + } catch (e) { + // Skip invalid JSON + } + } + } + + if (!payload) { + throw new Error("Could not find tree data in embedded JSON"); + } + + return { + path: payload.path, + branch: payload.refInfo.name, + items: payload.tree.items, + repo: { + name: payload.repo.name, + owner: payload.repo.ownerLogin, + isPublic: payload.repo.public, + }, + }; +} diff --git a/static/style.css b/static/style.css new file mode 100644 index 0000000..3e19563 --- /dev/null +++ b/static/style.css @@ -0,0 +1,14 @@ +body { + margin: 0; + padding: 1rem; +} + +@media (prefers-color-scheme: dark) { + body { + background: #0d1117; + color: #e6edf3; + } + a { + color: #4493f8; + } +} diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..c2f75c1 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,19 @@ +{ + "compilerOptions": { + "target": "ES2024", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": ["ES2024"], + "outDir": "./build", + "rootDir": "./src", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "resolveJsonModule": true, + "allowSyntheticDefaultImports": true, + "types": ["node"] + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "build"] +} diff --git a/views/tree.eta b/views/tree.eta new file mode 100644 index 0000000..4194216 --- /dev/null +++ b/views/tree.eta @@ -0,0 +1,42 @@ +<!DOCTYPE html> +<html lang="en"> +<head> + <meta charset="UTF-8"> + <title><%= it.repo.owner %>/<%= it.repo.name %> - <%= it.path || 'root' %></title> + <link rel="stylesheet" href="/static/style.css"> +</head> +<body> + <header> + <h1><%= it.repo.owner %>/<%= it.repo.name %></h1> + <div>branch: <%= it.branch %></div> + <% if (it.path) { %> + <div> + path: <a href="/<%= it.repo.owner %>/<%= it.repo.name %>/tree/<%= it.branch %>">root</a><% + const pathParts = it.path.split('/').filter(p => p); + let currentPath = ''; + for (const part of pathParts) { + currentPath += (currentPath ? '/' : '') + part; + %>/<a href="/<%= it.repo.owner %>/<%= it.repo.name %>/tree/<%= it.branch %>/<%= currentPath %>"><%= part %></a><% + } %> + </div> + <% } %> + </header> + + <table> + <% for (const item of it.items) { %> + <tr> + <td> + <span class="icon"><%= item.contentType === 'directory' ? 'š' : 'š' %></span> + <% if (item.contentType === 'directory') { %> + <a href="/<%= it.repo.owner %>/<%= it.repo.name %>/tree/<%= it.branch %>/<%= item.path %>"><%= item.name %></a> + <% } else { %> + <%= item.name %> + <% } %> + </td> + </tr> + <% } %> + </table> + + <footer>Generated by cgithub</footer> +</body> +</html> |