From 5e45f900974589c8eead15fc352d50dc45b1a25a Mon Sep 17 00:00:00 2001 From: Joe Mou Date: Thu, 22 Jan 2026 12:10:36 -0500 Subject: GitHub directory scraper Extracts directory listings from GitHub's server-rendered HTML. Avoids using the rate-limited API. Parses JSON embedded within script tags. Co-Authored-By: Claude Sonnet 4.5 --- cgithub/README.md | 81 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 cgithub/README.md (limited to 'cgithub/README.md') diff --git a/cgithub/README.md b/cgithub/README.md new file mode 100644 index 0000000..0f0ab43 --- /dev/null +++ b/cgithub/README.md @@ -0,0 +1,81 @@ +# GitHub Directory Listing Without API + +## Summary + +GitHub repository directory listings can be obtained **without using the API and without rate limits** by scraping the server-rendered HTML that GitHub sends to browsers. + +## How It Works + +When you visit a GitHub repository page in a browser, GitHub server-side renders the directory listing and embeds it as JSON data within ` +``` + +**Access Path:** `data.payload.tree.items` + +## Item Structure + +Each item in the `items` array has this structure: + +```json +{ + "name": "filename.txt", + "path": "path/to/filename.txt", + "contentType": "file" // or "directory" +} +``` + +## Complete Example + +See `github-scraper.js` for a working implementation that: +- Fetches the HTML from GitHub +- Extracts the embedded JSON data +- Parses and returns the directory listing +- Works for both root and subdirectories +- Has zero API rate limits + +## Usage + +```bash +node github-scraper.js torvalds linux # Root directory +node github-scraper.js torvalds linux Documentation # Subdirectory +node github-scraper.js git git Documentation/git # Nested subdirectory +``` + +## Advantages Over API + +| Feature | HTML Scraping | GitHub API | +|---------|--------------|------------| +| Rate Limits | None | 60/hour (unauth), 5000/hour (auth) | +| Authentication | Not required | Optional, but limits apply | +| Access | Any public repo | Any public repo | +| Stability | Depends on HTML structure | Stable API contract | + +## Important Notes + +1. **No authentication needed** - Works for all public repositories +2. **No rate limits** - Can scrape as many repos as needed +3. **Less stable** - HTML structure may change with GitHub updates +4. **Best for:** One-off scripts, personal tools, exploratory work +5. **Use API for:** Production applications, long-term projects + +## Why This Works + +GitHub uses server-side rendering to provide fast initial page loads. The directory data is embedded in the HTML so the page can render immediately without waiting for additional API calls. This embedded data is the same data that would come from the API, just pre-rendered in the page. -- cgit v1.3.1 From ad2871ca617899aa36a2d85b41974bae9796ace6 Mon Sep 17 00:00:00 2001 From: Joe Mou Date: Tue, 27 Jan 2026 09:30:28 -0500 Subject: Rewrite README --- cgithub/README.md | 111 ++++++++++++++++++++++-------------------------------- 1 file changed, 45 insertions(+), 66 deletions(-) (limited to 'cgithub/README.md') diff --git a/cgithub/README.md b/cgithub/README.md index 0f0ab43..9ba2213 100644 --- a/cgithub/README.md +++ b/cgithub/README.md @@ -1,81 +1,60 @@ -# GitHub Directory Listing Without API +cgithub is a lightweight GitHub alternative frontend. It has no bloated +JavaScript and doesn't require API keys. To use it, host cgithub on your own +server then replace `github.com` in your URL with your domain. -## Summary +## Development -GitHub repository directory listings can be obtained **without using the API and without rate limits** by scraping the server-rendered HTML that GitHub sends to browsers. - -## How It Works - -When you visit a GitHub repository page in a browser, GitHub server-side renders the directory listing and embeds it as JSON data within ` +To run the server locally: ``` - -**Access Path:** `data.payload.tree.items` - -## Item Structure - -Each item in the `items` array has this structure: - -```json -{ - "name": "filename.txt", - "path": "path/to/filename.txt", - "contentType": "file" // or "directory" -} +$ pnpm run dev ``` -## Complete Example +To run tests: +``` +$ pnpm test +``` -See `github-scraper.js` for a working implementation that: -- Fetches the HTML from GitHub -- Extracts the embedded JSON data -- Parses and returns the directory listing -- Works for both root and subdirectories -- Has zero API rate limits +## Deployment -## Usage +Assuming you have a server `jupiter`, you might deploy like: +``` +$ pnpm run build +$ rsync -av --del dist jupiter:/opt/cgithub +``` -```bash -node github-scraper.js torvalds linux # Root directory -node github-scraper.js torvalds linux Documentation # Subdirectory -node github-scraper.js git git Documentation/git # Nested subdirectory +To configure systemd socket activation use a `cgithub.socket` file: ``` +[Socket] +ListenStream=/run/cgithub/socket +SocketGroup=nginx +SocketMode=0660 -## Advantages Over API +[Install] +WantedBy=sockets.target +``` -| Feature | HTML Scraping | GitHub API | -|---------|--------------|------------| -| Rate Limits | None | 60/hour (unauth), 5000/hour (auth) | -| Authentication | Not required | Optional, but limits apply | -| Access | Any public repo | Any public repo | -| Stability | Depends on HTML structure | Stable API contract | +And a service file `cgithub.service`: +``` +[Unit] +After=cgithub.socket +Requires=cgithub.socket + +[Service] +Type=exec +WorkingDirectory=/opt/cgithub +ExecStart=/usr/bin/node dist/build/index.js +DynamicUser=true +``` -## Important Notes +An nginx config snippet might be: +``` +location / { + proxy_pass http://unix:/run/cgithub/socket; +} +``` -1. **No authentication needed** - Works for all public repositories -2. **No rate limits** - Can scrape as many repos as needed -3. **Less stable** - HTML structure may change with GitHub updates -4. **Best for:** One-off scripts, personal tools, exploratory work -5. **Use API for:** Production applications, long-term projects +## Disclosures -## Why This Works +AI coding assistants, in particular Claude, are used in the development process. -GitHub uses server-side rendering to provide fast initial page loads. The directory data is embedded in the HTML so the page can render immediately without waiting for additional API calls. This embedded data is the same data that would come from the API, just pre-rendered in the page. +[Phospor Icons](https://phosphoricons.com/) used under MIT license. -- cgit v1.3.1 From 2b4a69720b53480d82cf296ac5dbe4614d9470b9 Mon Sep 17 00:00:00 2001 From: Joe Mou Date: Tue, 11 Aug 2026 08:27:26 -0400 Subject: Deploy only to Cloudflare Workers The Node server entry, esbuild bundle, and Makefile rsync/systemd deploy are gone; wrangler dev and wrangler deploy are the only paths now, so the dev:cf/deploy:cf scripts take over the plain dev/deploy names. @hono/ node-server and esbuild drop out with src/index.ts, and the README's systemd socket and nginx snippets go with them. Co-Authored-By: Claude Opus 5 --- cgithub/.gitignore | 1 - cgithub/CLAUDE.md | 19 ++-- cgithub/Makefile | 5 - cgithub/README.md | 41 ++----- cgithub/package.json | 12 +-- cgithub/pnpm-lock.yaml | 284 ------------------------------------------------- cgithub/src/index.ts | 28 ----- 7 files changed, 17 insertions(+), 373 deletions(-) delete mode 100644 cgithub/Makefile delete mode 100644 cgithub/src/index.ts (limited to 'cgithub/README.md') diff --git a/cgithub/.gitignore b/cgithub/.gitignore index e502860..5a5e404 100644 --- a/cgithub/.gitignore +++ b/cgithub/.gitignore @@ -1,4 +1,3 @@ -/dist/ /node_modules/ /.wrangler/ /src/views.generated.ts diff --git a/cgithub/CLAUDE.md b/cgithub/CLAUDE.md index 1bf4d88..37774cc 100644 --- a/cgithub/CLAUDE.md +++ b/cgithub/CLAUDE.md @@ -5,33 +5,28 @@ cgithub is a lightweight GitHub alternative frontend that scrapes GitHub's serve ## Commands ```bash -# Development (watch mode with hot reload) +# Local dev server (Cloudflare Workers runtime) pnpm run dev -# Production build (bundles to dist/) -pnpm run build +# Deploy to Cloudflare Workers +pnpm run deploy # Run tests pnpm test - -# Cloudflare Workers: local dev and deploy -pnpm run dev:cf -pnpm run deploy:cf ``` ## Architecture -**Stack:** Hono (web framework) + Eta (templating) + TypeScript, bundled with esbuild +**Stack:** Hono (web framework) + Eta (templating) + TypeScript, running on Cloudflare Workers **Source files:** -- `src/app.ts` - Hono routes, shared by both entry points -- `src/index.ts` - Node server entry (supports systemd socket activation via `LISTEN_FDS=1`) +- `src/app.ts` - Hono routes - `src/worker.ts` - Cloudflare Workers entry (config in `wrangler.jsonc`) - `src/scraper.ts` - GitHub HTML fetching and JSON extraction logic -**Templates:** `views/*.eta` - Eta templates with layout inheritance (`layout.eta` as base). On Workers, runtime code generation is disallowed, so `scripts/compile-views.ts` precompiles templates to `src/views.generated.ts` (gitignored, regenerated by the `dev:cf`/`deploy:cf` scripts). +**Templates:** `views/*.eta` - Eta templates with layout inheritance (`layout.eta` as base). On Workers, runtime code generation is disallowed, so `scripts/compile-views.ts` precompiles templates to `src/views.generated.ts` (gitignored, regenerated by the `dev`/`deploy` scripts). -**Static files:** `public/static/*`, served by `@hono/node-server` on Node and by Workers Static Assets on Cloudflare. +**Static files:** `public/static/*`, served by Workers Static Assets. **Routes:** - `GET /:owner` - User or organization profile with pinned repositories diff --git a/cgithub/Makefile b/cgithub/Makefile deleted file mode 100644 index a25ee23..0000000 --- a/cgithub/Makefile +++ /dev/null @@ -1,5 +0,0 @@ -deploy: - pnpm run build - rsync -av --del dist elmo:/opt/cgithub - ssh elmo sudo systemctl restart cgithub -.PHONY: deploy diff --git a/cgithub/README.md b/cgithub/README.md index 9ba2213..3bcf017 100644 --- a/cgithub/README.md +++ b/cgithub/README.md @@ -1,6 +1,7 @@ cgithub is a lightweight GitHub alternative frontend. It has no bloated -JavaScript and doesn't require API keys. To use it, host cgithub on your own -server then replace `github.com` in your URL with your domain. +JavaScript and doesn't require API keys. It runs on Cloudflare Workers; to use +it, deploy your own instance then replace `github.com` in your URL with your +domain. ## Development @@ -16,42 +17,12 @@ $ pnpm test ## Deployment -Assuming you have a server `jupiter`, you might deploy like: +To deploy to Cloudflare Workers: ``` -$ pnpm run build -$ rsync -av --del dist jupiter:/opt/cgithub +$ pnpm run deploy ``` -To configure systemd socket activation use a `cgithub.socket` file: -``` -[Socket] -ListenStream=/run/cgithub/socket -SocketGroup=nginx -SocketMode=0660 - -[Install] -WantedBy=sockets.target -``` - -And a service file `cgithub.service`: -``` -[Unit] -After=cgithub.socket -Requires=cgithub.socket - -[Service] -Type=exec -WorkingDirectory=/opt/cgithub -ExecStart=/usr/bin/node dist/build/index.js -DynamicUser=true -``` - -An nginx config snippet might be: -``` -location / { - proxy_pass http://unix:/run/cgithub/socket; -} -``` +The worker name and other settings live in `wrangler.jsonc`. ## Disclosures diff --git a/cgithub/package.json b/cgithub/package.json index b5349ca..337fabc 100644 --- a/cgithub/package.json +++ b/cgithub/package.json @@ -5,14 +5,11 @@ "license": "MIT", "type": "module", "scripts": { - "dev": "node --watch --watch-preserve-output src/index.ts", - "build": "rm -rf dist/* && esbuild --bundle --format=esm --platform=node --outdir=dist/build --sourcemap src/index.ts && cp -R public views dist/", - "test": "node --test --test-concurrency=4 src/scraper.test.ts", - "dev:cf": "node scripts/compile-views.ts && wrangler dev", - "deploy:cf": "node scripts/compile-views.ts && wrangler deploy" + "dev": "node scripts/compile-views.ts && wrangler dev", + "deploy": "node scripts/compile-views.ts && wrangler deploy", + "test": "node --test --test-concurrency=4 src/scraper.test.ts" }, "dependencies": { - "@hono/node-server": "^1.13.7", "css-select": "^6.0.0", "domhandler": "^5.0.3", "domutils": "^3.2.2", @@ -21,11 +18,10 @@ "htmlparser2": "^10.1.0" }, "pnpm": { - "onlyBuiltDependencies": ["esbuild", "workerd"] + "onlyBuiltDependencies": ["workerd"] }, "devDependencies": { "@types/node": "^22.10.2", - "esbuild": "^0.27.2", "prettier": "^3.8.1", "typescript": "^5.7.2", "wrangler": "^4.110.0" diff --git a/cgithub/pnpm-lock.yaml b/cgithub/pnpm-lock.yaml index b3abd2a..6f4ff25 100644 --- a/cgithub/pnpm-lock.yaml +++ b/cgithub/pnpm-lock.yaml @@ -8,9 +8,6 @@ importers: .: dependencies: - '@hono/node-server': - specifier: ^1.13.7 - version: 1.19.9(hono@4.11.5) css-select: specifier: ^6.0.0 version: 6.0.0 @@ -33,9 +30,6 @@ importers: '@types/node': specifier: ^22.10.2 version: 22.19.7 - esbuild: - specifier: ^0.27.2 - version: 0.27.2 prettier: specifier: ^3.8.1 version: 3.8.1 @@ -98,324 +92,162 @@ packages: '@emnapi/runtime@1.11.2': resolution: {integrity: sha512-kyOl3X0DuTiT1h2ft8r2fYO8JYtU9a9Xis/zBSiGArNaagCOWx90N1k2wxp18czFDH+OgcWGb5ZP/XMt3dcyPA==} - '@esbuild/aix-ppc64@0.27.2': - resolution: {integrity: sha512-GZMB+a0mOMZs4MpDbj8RJp4cw+w1WV5NYD6xzgvzUJ5Ek2jerwfO2eADyI6ExDSUED+1X8aMbegahsJi+8mgpw==} - engines: {node: '>=18'} - cpu: [ppc64] - os: [aix] - '@esbuild/aix-ppc64@0.28.1': resolution: {integrity: sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ==} engines: {node: '>=18'} cpu: [ppc64] os: [aix] - '@esbuild/android-arm64@0.27.2': - resolution: {integrity: sha512-pvz8ZZ7ot/RBphf8fv60ljmaoydPU12VuXHImtAs0XhLLw+EXBi2BLe3OYSBslR4rryHvweW5gmkKFwTiFy6KA==} - engines: {node: '>=18'} - cpu: [arm64] - os: [android] - '@esbuild/android-arm64@0.28.1': resolution: {integrity: sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg==} engines: {node: '>=18'} cpu: [arm64] os: [android] - '@esbuild/android-arm@0.27.2': - resolution: {integrity: sha512-DVNI8jlPa7Ujbr1yjU2PfUSRtAUZPG9I1RwW4F4xFB1Imiu2on0ADiI/c3td+KmDtVKNbi+nffGDQMfcIMkwIA==} - engines: {node: '>=18'} - cpu: [arm] - os: [android] - '@esbuild/android-arm@0.28.1': resolution: {integrity: sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ==} engines: {node: '>=18'} cpu: [arm] os: [android] - '@esbuild/android-x64@0.27.2': - resolution: {integrity: sha512-z8Ank4Byh4TJJOh4wpz8g2vDy75zFL0TlZlkUkEwYXuPSgX8yzep596n6mT7905kA9uHZsf/o2OJZubl2l3M7A==} - engines: {node: '>=18'} - cpu: [x64] - os: [android] - '@esbuild/android-x64@0.28.1': resolution: {integrity: sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng==} engines: {node: '>=18'} cpu: [x64] os: [android] - '@esbuild/darwin-arm64@0.27.2': - resolution: {integrity: sha512-davCD2Zc80nzDVRwXTcQP/28fiJbcOwvdolL0sOiOsbwBa72kegmVU0Wrh1MYrbuCL98Omp5dVhQFWRKR2ZAlg==} - engines: {node: '>=18'} - cpu: [arm64] - os: [darwin] - '@esbuild/darwin-arm64@0.28.1': resolution: {integrity: sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q==} engines: {node: '>=18'} cpu: [arm64] os: [darwin] - '@esbuild/darwin-x64@0.27.2': - resolution: {integrity: sha512-ZxtijOmlQCBWGwbVmwOF/UCzuGIbUkqB1faQRf5akQmxRJ1ujusWsb3CVfk/9iZKr2L5SMU5wPBi1UWbvL+VQA==} - engines: {node: '>=18'} - cpu: [x64] - os: [darwin] - '@esbuild/darwin-x64@0.28.1': resolution: {integrity: sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ==} engines: {node: '>=18'} cpu: [x64] os: [darwin] - '@esbuild/freebsd-arm64@0.27.2': - resolution: {integrity: sha512-lS/9CN+rgqQ9czogxlMcBMGd+l8Q3Nj1MFQwBZJyoEKI50XGxwuzznYdwcav6lpOGv5BqaZXqvBSiB/kJ5op+g==} - engines: {node: '>=18'} - cpu: [arm64] - os: [freebsd] - '@esbuild/freebsd-arm64@0.28.1': resolution: {integrity: sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw==} engines: {node: '>=18'} cpu: [arm64] os: [freebsd] - '@esbuild/freebsd-x64@0.27.2': - resolution: {integrity: sha512-tAfqtNYb4YgPnJlEFu4c212HYjQWSO/w/h/lQaBK7RbwGIkBOuNKQI9tqWzx7Wtp7bTPaGC6MJvWI608P3wXYA==} - engines: {node: '>=18'} - cpu: [x64] - os: [freebsd] - '@esbuild/freebsd-x64@0.28.1': resolution: {integrity: sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ==} engines: {node: '>=18'} cpu: [x64] os: [freebsd] - '@esbuild/linux-arm64@0.27.2': - resolution: {integrity: sha512-hYxN8pr66NsCCiRFkHUAsxylNOcAQaxSSkHMMjcpx0si13t1LHFphxJZUiGwojB1a/Hd5OiPIqDdXONia6bhTw==} - engines: {node: '>=18'} - cpu: [arm64] - os: [linux] - '@esbuild/linux-arm64@0.28.1': resolution: {integrity: sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g==} engines: {node: '>=18'} cpu: [arm64] os: [linux] - '@esbuild/linux-arm@0.27.2': - resolution: {integrity: sha512-vWfq4GaIMP9AIe4yj1ZUW18RDhx6EPQKjwe7n8BbIecFtCQG4CfHGaHuh7fdfq+y3LIA2vGS/o9ZBGVxIDi9hw==} - engines: {node: '>=18'} - cpu: [arm] - os: [linux] - '@esbuild/linux-arm@0.28.1': resolution: {integrity: sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ==} engines: {node: '>=18'} cpu: [arm] os: [linux] - '@esbuild/linux-ia32@0.27.2': - resolution: {integrity: sha512-MJt5BRRSScPDwG2hLelYhAAKh9imjHK5+NE/tvnRLbIqUWa+0E9N4WNMjmp/kXXPHZGqPLxggwVhz7QP8CTR8w==} - engines: {node: '>=18'} - cpu: [ia32] - os: [linux] - '@esbuild/linux-ia32@0.28.1': resolution: {integrity: sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w==} engines: {node: '>=18'} cpu: [ia32] os: [linux] - '@esbuild/linux-loong64@0.27.2': - resolution: {integrity: sha512-lugyF1atnAT463aO6KPshVCJK5NgRnU4yb3FUumyVz+cGvZbontBgzeGFO1nF+dPueHD367a2ZXe1NtUkAjOtg==} - engines: {node: '>=18'} - cpu: [loong64] - os: [linux] - '@esbuild/linux-loong64@0.28.1': resolution: {integrity: sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg==} engines: {node: '>=18'} cpu: [loong64] os: [linux] - '@esbuild/linux-mips64el@0.27.2': - resolution: {integrity: sha512-nlP2I6ArEBewvJ2gjrrkESEZkB5mIoaTswuqNFRv/WYd+ATtUpe9Y09RnJvgvdag7he0OWgEZWhviS1OTOKixw==} - engines: {node: '>=18'} - cpu: [mips64el] - os: [linux] - '@esbuild/linux-mips64el@0.28.1': resolution: {integrity: sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ==} engines: {node: '>=18'} cpu: [mips64el] os: [linux] - '@esbuild/linux-ppc64@0.27.2': - resolution: {integrity: sha512-C92gnpey7tUQONqg1n6dKVbx3vphKtTHJaNG2Ok9lGwbZil6DrfyecMsp9CrmXGQJmZ7iiVXvvZH6Ml5hL6XdQ==} - engines: {node: '>=18'} - cpu: [ppc64] - os: [linux] - '@esbuild/linux-ppc64@0.28.1': resolution: {integrity: sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ==} engines: {node: '>=18'} cpu: [ppc64] os: [linux] - '@esbuild/linux-riscv64@0.27.2': - resolution: {integrity: sha512-B5BOmojNtUyN8AXlK0QJyvjEZkWwy/FKvakkTDCziX95AowLZKR6aCDhG7LeF7uMCXEJqwa8Bejz5LTPYm8AvA==} - engines: {node: '>=18'} - cpu: [riscv64] - os: [linux] - '@esbuild/linux-riscv64@0.28.1': resolution: {integrity: sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ==} engines: {node: '>=18'} cpu: [riscv64] os: [linux] - '@esbuild/linux-s390x@0.27.2': - resolution: {integrity: sha512-p4bm9+wsPwup5Z8f4EpfN63qNagQ47Ua2znaqGH6bqLlmJ4bx97Y9JdqxgGZ6Y8xVTixUnEkoKSHcpRlDnNr5w==} - engines: {node: '>=18'} - cpu: [s390x] - os: [linux] - '@esbuild/linux-s390x@0.28.1': resolution: {integrity: sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag==} engines: {node: '>=18'} cpu: [s390x] os: [linux] - '@esbuild/linux-x64@0.27.2': - resolution: {integrity: sha512-uwp2Tip5aPmH+NRUwTcfLb+W32WXjpFejTIOWZFw/v7/KnpCDKG66u4DLcurQpiYTiYwQ9B7KOeMJvLCu/OvbA==} - engines: {node: '>=18'} - cpu: [x64] - os: [linux] - '@esbuild/linux-x64@0.28.1': resolution: {integrity: sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA==} engines: {node: '>=18'} cpu: [x64] os: [linux] - '@esbuild/netbsd-arm64@0.27.2': - resolution: {integrity: sha512-Kj6DiBlwXrPsCRDeRvGAUb/LNrBASrfqAIok+xB0LxK8CHqxZ037viF13ugfsIpePH93mX7xfJp97cyDuTZ3cw==} - engines: {node: '>=18'} - cpu: [arm64] - os: [netbsd] - '@esbuild/netbsd-arm64@0.28.1': resolution: {integrity: sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw==} engines: {node: '>=18'} cpu: [arm64] os: [netbsd] - '@esbuild/netbsd-x64@0.27.2': - resolution: {integrity: sha512-HwGDZ0VLVBY3Y+Nw0JexZy9o/nUAWq9MlV7cahpaXKW6TOzfVno3y3/M8Ga8u8Yr7GldLOov27xiCnqRZf0tCA==} - engines: {node: '>=18'} - cpu: [x64] - os: [netbsd] - '@esbuild/netbsd-x64@0.28.1': resolution: {integrity: sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg==} engines: {node: '>=18'} cpu: [x64] os: [netbsd] - '@esbuild/openbsd-arm64@0.27.2': - resolution: {integrity: sha512-DNIHH2BPQ5551A7oSHD0CKbwIA/Ox7+78/AWkbS5QoRzaqlev2uFayfSxq68EkonB+IKjiuxBFoV8ESJy8bOHA==} - engines: {node: '>=18'} - cpu: [arm64] - os: [openbsd] - '@esbuild/openbsd-arm64@0.28.1': resolution: {integrity: sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q==} engines: {node: '>=18'} cpu: [arm64] os: [openbsd] - '@esbuild/openbsd-x64@0.27.2': - resolution: {integrity: sha512-/it7w9Nb7+0KFIzjalNJVR5bOzA9Vay+yIPLVHfIQYG/j+j9VTH84aNB8ExGKPU4AzfaEvN9/V4HV+F+vo8OEg==} - engines: {node: '>=18'} - cpu: [x64] - os: [openbsd] - '@esbuild/openbsd-x64@0.28.1': resolution: {integrity: sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw==} engines: {node: '>=18'} cpu: [x64] os: [openbsd] - '@esbuild/openharmony-arm64@0.27.2': - resolution: {integrity: sha512-LRBbCmiU51IXfeXk59csuX/aSaToeG7w48nMwA6049Y4J4+VbWALAuXcs+qcD04rHDuSCSRKdmY63sruDS5qag==} - engines: {node: '>=18'} - cpu: [arm64] - os: [openharmony] - '@esbuild/openharmony-arm64@0.28.1': resolution: {integrity: sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg==} engines: {node: '>=18'} cpu: [arm64] os: [openharmony] - '@esbuild/sunos-x64@0.27.2': - resolution: {integrity: sha512-kMtx1yqJHTmqaqHPAzKCAkDaKsffmXkPHThSfRwZGyuqyIeBvf08KSsYXl+abf5HDAPMJIPnbBfXvP2ZC2TfHg==} - engines: {node: '>=18'} - cpu: [x64] - os: [sunos] - '@esbuild/sunos-x64@0.28.1': resolution: {integrity: sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ==} engines: {node: '>=18'} cpu: [x64] os: [sunos] - '@esbuild/win32-arm64@0.27.2': - resolution: {integrity: sha512-Yaf78O/B3Kkh+nKABUF++bvJv5Ijoy9AN1ww904rOXZFLWVc5OLOfL56W+C8F9xn5JQZa3UX6m+IktJnIb1Jjg==} - engines: {node: '>=18'} - cpu: [arm64] - os: [win32] - '@esbuild/win32-arm64@0.28.1': resolution: {integrity: sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA==} engines: {node: '>=18'} cpu: [arm64] os: [win32] - '@esbuild/win32-ia32@0.27.2': - resolution: {integrity: sha512-Iuws0kxo4yusk7sw70Xa2E2imZU5HoixzxfGCdxwBdhiDgt9vX9VUCBhqcwY7/uh//78A1hMkkROMJq9l27oLQ==} - engines: {node: '>=18'} - cpu: [ia32] - os: [win32] - '@esbuild/win32-ia32@0.28.1': resolution: {integrity: sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg==} engines: {node: '>=18'} cpu: [ia32] os: [win32] - '@esbuild/win32-x64@0.27.2': - resolution: {integrity: sha512-sRdU18mcKf7F+YgheI/zGf5alZatMUTKj/jNS6l744f9u3WFu4v7twcUI9vu4mknF4Y9aDlblIie0IM+5xxaqQ==} - engines: {node: '>=18'} - cpu: [x64] - os: [win32] - '@esbuild/win32-x64@0.28.1': resolution: {integrity: sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A==} engines: {node: '>=18'} cpu: [x64] os: [win32] - '@hono/node-server@1.19.9': - resolution: {integrity: sha512-vHL6w3ecZsky+8P5MD+eFfaGTyCeOHUIFYMGpQGbrBTSmNNoxv0if69rEZ5giu36weC5saFuznL411gRX7bJDw==} - engines: {node: '>=18.14.1'} - peerDependencies: - hono: ^4 - '@img/colour@1.1.0': resolution: {integrity: sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==} engines: {node: '>=18'} @@ -627,11 +459,6 @@ packages: error-stack-parser-es@1.0.5: resolution: {integrity: sha512-5qucVt2XcuGMcEGgWI7i+yZpmpByQ8J1lHhcL7PwqCwu9FPP3VUXzT4ltHe5i2z9dePwEHcDVOAfSnHsOlCXRA==} - esbuild@0.27.2: - resolution: {integrity: sha512-HyNQImnsOC7X9PMNaCIeAm4ISCQXs5a5YasTXVliKv4uuBo1dKrG0A+uQS8M5eXjVMnLg3WgXaKvprHlFJQffw==} - engines: {node: '>=18'} - hasBin: true - esbuild@0.28.1: resolution: {integrity: sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==} engines: {node: '>=18'} @@ -774,166 +601,84 @@ snapshots: tslib: 2.8.1 optional: true - '@esbuild/aix-ppc64@0.27.2': - optional: true - '@esbuild/aix-ppc64@0.28.1': optional: true - '@esbuild/android-arm64@0.27.2': - optional: true - '@esbuild/android-arm64@0.28.1': optional: true - '@esbuild/android-arm@0.27.2': - optional: true - '@esbuild/android-arm@0.28.1': optional: true - '@esbuild/android-x64@0.27.2': - optional: true - '@esbuild/android-x64@0.28.1': optional: true - '@esbuild/darwin-arm64@0.27.2': - optional: true - '@esbuild/darwin-arm64@0.28.1': optional: true - '@esbuild/darwin-x64@0.27.2': - optional: true - '@esbuild/darwin-x64@0.28.1': optional: true - '@esbuild/freebsd-arm64@0.27.2': - optional: true - '@esbuild/freebsd-arm64@0.28.1': optional: true - '@esbuild/freebsd-x64@0.27.2': - optional: true - '@esbuild/freebsd-x64@0.28.1': optional: true - '@esbuild/linux-arm64@0.27.2': - optional: true - '@esbuild/linux-arm64@0.28.1': optional: true - '@esbuild/linux-arm@0.27.2': - optional: true - '@esbuild/linux-arm@0.28.1': optional: true - '@esbuild/linux-ia32@0.27.2': - optional: true - '@esbuild/linux-ia32@0.28.1': optional: true - '@esbuild/linux-loong64@0.27.2': - optional: true - '@esbuild/linux-loong64@0.28.1': optional: true - '@esbuild/linux-mips64el@0.27.2': - optional: true - '@esbuild/linux-mips64el@0.28.1': optional: true - '@esbuild/linux-ppc64@0.27.2': - optional: true - '@esbuild/linux-ppc64@0.28.1': optional: true - '@esbuild/linux-riscv64@0.27.2': - optional: true - '@esbuild/linux-riscv64@0.28.1': optional: true - '@esbuild/linux-s390x@0.27.2': - optional: true - '@esbuild/linux-s390x@0.28.1': optional: true - '@esbuild/linux-x64@0.27.2': - optional: true - '@esbuild/linux-x64@0.28.1': optional: true - '@esbuild/netbsd-arm64@0.27.2': - optional: true - '@esbuild/netbsd-arm64@0.28.1': optional: true - '@esbuild/netbsd-x64@0.27.2': - optional: true - '@esbuild/netbsd-x64@0.28.1': optional: true - '@esbuild/openbsd-arm64@0.27.2': - optional: true - '@esbuild/openbsd-arm64@0.28.1': optional: true - '@esbuild/openbsd-x64@0.27.2': - optional: true - '@esbuild/openbsd-x64@0.28.1': optional: true - '@esbuild/openharmony-arm64@0.27.2': - optional: true - '@esbuild/openharmony-arm64@0.28.1': optional: true - '@esbuild/sunos-x64@0.27.2': - optional: true - '@esbuild/sunos-x64@0.28.1': optional: true - '@esbuild/win32-arm64@0.27.2': - optional: true - '@esbuild/win32-arm64@0.28.1': optional: true - '@esbuild/win32-ia32@0.27.2': - optional: true - '@esbuild/win32-ia32@0.28.1': optional: true - '@esbuild/win32-x64@0.27.2': - optional: true - '@esbuild/win32-x64@0.28.1': optional: true - '@hono/node-server@1.19.9(hono@4.11.5)': - dependencies: - hono: 4.11.5 - '@img/colour@1.1.0': {} '@img/sharp-darwin-arm64@0.34.5': @@ -1101,35 +846,6 @@ snapshots: error-stack-parser-es@1.0.5: {} - esbuild@0.27.2: - optionalDependencies: - '@esbuild/aix-ppc64': 0.27.2 - '@esbuild/android-arm': 0.27.2 - '@esbuild/android-arm64': 0.27.2 - '@esbuild/android-x64': 0.27.2 - '@esbuild/darwin-arm64': 0.27.2 - '@esbuild/darwin-x64': 0.27.2 - '@esbuild/freebsd-arm64': 0.27.2 - '@esbuild/freebsd-x64': 0.27.2 - '@esbuild/linux-arm': 0.27.2 - '@esbuild/linux-arm64': 0.27.2 - '@esbuild/linux-ia32': 0.27.2 - '@esbuild/linux-loong64': 0.27.2 - '@esbuild/linux-mips64el': 0.27.2 - '@esbuild/linux-ppc64': 0.27.2 - '@esbuild/linux-riscv64': 0.27.2 - '@esbuild/linux-s390x': 0.27.2 - '@esbuild/linux-x64': 0.27.2 - '@esbuild/netbsd-arm64': 0.27.2 - '@esbuild/netbsd-x64': 0.27.2 - '@esbuild/openbsd-arm64': 0.27.2 - '@esbuild/openbsd-x64': 0.27.2 - '@esbuild/openharmony-arm64': 0.27.2 - '@esbuild/sunos-x64': 0.27.2 - '@esbuild/win32-arm64': 0.27.2 - '@esbuild/win32-ia32': 0.27.2 - '@esbuild/win32-x64': 0.27.2 - esbuild@0.28.1: optionalDependencies: '@esbuild/aix-ppc64': 0.28.1 diff --git a/cgithub/src/index.ts b/cgithub/src/index.ts deleted file mode 100644 index 65c46d9..0000000 --- a/cgithub/src/index.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { createAdaptorServer, serve } from "@hono/node-server"; -import { serveStatic } from "@hono/node-server/serve-static"; -import { Eta } from "eta"; -import { Hono } from "hono"; -import path from "node:path"; -import url from "node:url"; -import { createApp } from "./app.ts"; - -const __dirname = path.dirname(url.fileURLToPath(import.meta.url)); - -const eta = new Eta({ views: path.join(__dirname, "..", "views") }); - -const app = new Hono(); -app.use("/static/*", serveStatic({ root: path.join(__dirname, "..", "public") })); -app.route("/", createApp(eta)); - -if (process.env.LISTEN_FDS === "1") { - const server = createAdaptorServer(app); - // Listen on SD_LISTEN_FDS_START. - server.listen({ fd: 3 }, () => { - console.log("Server running via systemd socket activation"); - }); -} else { - const port = parseInt(process.env.PORT || "3000", 10); - serve({ fetch: app.fetch, port }, (info) => { - console.log(`Server running on http://localhost:${info.port}`); - }); -} -- cgit v1.3.1 From 9731ebd1bece1187f943a29da474f4aed6e7ee3b Mon Sep 17 00:00:00 2001 From: Joe Mou Date: Tue, 11 Aug 2026 11:39:01 -0400 Subject: Redirect to GitHub with a meta refresh A browser extension redirects github.com to cgithub with declarative net requests. Its rules match our HTTP redirects back to github.com too, so those bounce straight back here and loop forever. A meta refresh makes cgithub the initiator origin instead, which the extension's rules can exclude, and its delay leaves a page the user can stop on. Co-Authored-By: Claude Opus 5 --- cgithub/README.md | 30 ++++++++++++++++++++++++++++ cgithub/package.json | 6 ++++-- cgithub/src/app.test.ts | 50 ++++++++++++++++++++++++++++++++++++++++++++++ cgithub/src/app.ts | 35 +++++++++++++++++++------------- cgithub/views/redirect.eta | 14 +++++++++++++ 5 files changed, 119 insertions(+), 16 deletions(-) create mode 100644 cgithub/src/app.test.ts create mode 100644 cgithub/views/redirect.eta (limited to 'cgithub/README.md') diff --git a/cgithub/README.md b/cgithub/README.md index 3bcf017..a625116 100644 --- a/cgithub/README.md +++ b/cgithub/README.md @@ -24,6 +24,36 @@ $ pnpm run deploy The worker name and other settings live in `wrangler.jsonc`. +## Browser extension redirects + +Pages cgithub can't render (marketing pages, code search, anything unrouted) are +redirected back to github.com. A browser extension that rewrites github.com URLs +to a cgithub instance with `declarativeNetRequest` will match those redirects +too and send the request straight back, looping forever. + +To avoid this, cgithub never answers with an HTTP redirect: it serves a small +page that navigates to github.com with a two-second ``. +Because that navigation is initiated by the cgithub origin, the extension can +let it through with `excludedInitiatorDomains`; without that, the user at least +lands on a page they can stop on, with a link, instead of an endless loop. + +A rule that redirects to a cgithub instance should therefore look like this: + +```jsonc +{ + "action": { + "type": "redirect", + "redirect": { "transform": { "host": "cgithub.example" } }, + }, + "condition": { + "urlFilter": "||github.com", + "resourceTypes": ["main_frame"], + // So cgithub's own meta refresh to github.com isn't redirected back here. + "excludedInitiatorDomains": ["cgithub.example"], + }, +} +``` + ## Disclosures AI coding assistants, in particular Claude, are used in the development process. diff --git a/cgithub/package.json b/cgithub/package.json index 7673612..ce25701 100644 --- a/cgithub/package.json +++ b/cgithub/package.json @@ -7,7 +7,7 @@ "scripts": { "dev": "node scripts/build.ts && wrangler dev", "deploy": "node scripts/build.ts && wrangler deploy", - "test": "node --test --test-concurrency=4 src/scraper.test.ts" + "test": "node --test --test-concurrency=4 src/app.test.ts src/scraper.test.ts" }, "dependencies": { "css-select": "^6.0.0", @@ -18,7 +18,9 @@ "htmlparser2": "^10.1.0" }, "pnpm": { - "onlyBuiltDependencies": ["workerd"] + "onlyBuiltDependencies": [ + "workerd" + ] }, "devDependencies": { "@types/node": "^22.10.2", diff --git a/cgithub/src/app.test.ts b/cgithub/src/app.test.ts new file mode 100644 index 0000000..498d8a6 --- /dev/null +++ b/cgithub/src/app.test.ts @@ -0,0 +1,50 @@ +import assert from "node:assert"; +import path from "node:path"; +import { describe, it } from "node:test"; +import { Eta } from "eta"; +import { createApp } from "./app.ts"; + +// The routes exercised here redirect without scraping, so no network is needed. +const eta = new Eta({ views: path.join(import.meta.dirname, "..", "views") }); +const app = createApp(eta); + +describe("redirects to GitHub", () => { + // An HTTP redirect would be caught by the extension's declarativeNetRequest + // rules and bounced straight back here. + it("should redirect unknown paths with a meta refresh", async () => { + const res = await app.request("http://cgithub.example/a/b/c?x=1"); + + assert.strictEqual(res.status, 200); + assert.strictEqual(res.headers.get("location"), null); + assert.strictEqual(res.headers.get("Referrer-Policy"), "no-referrer"); + + const body = await res.text(); + assert.match( + body, + //, + ); + assert.doesNotMatch(body, /