From 5e45f900974589c8eead15fc352d50dc45b1a25a Mon Sep 17 00:00:00 2001
From: Joe Mou <dev@mou.fo>
Date: Thu, 22 Jan 2026 12:10:36 -0500
Subject: GitHub directory scraper

Extracts directory listings from GitHub's server-rendered HTML. Avoids
using the rate-limited API. Parses JSON embedded within script tags.

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
---
 cgithub/README.md         | 81 +++++++++++++++++++++++++++++++++++++++++++++++
 cgithub/github-scraper.js | 74 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 155 insertions(+)
 create mode 100644 cgithub/README.md
 create mode 100755 cgithub/github-scraper.js

diff --git a/cgithub/README.md b/cgithub/README.md
new file mode 100644
index 0000000..0f0ab43
--- /dev/null
+++ b/cgithub/README.md
@@ -0,0 +1,81 @@
+# GitHub Directory Listing Without API
+
+## Summary
+
+GitHub repository directory listings can be obtained **without using the API and without rate limits** by scraping the server-rendered HTML that GitHub sends to browsers.
+
+## How It Works
+
+When you visit a GitHub repository page in a browser, GitHub server-side renders the directory listing and embeds it as JSON data within `<script>` tags in the HTML. This approach avoids API rate limits entirely.
+
+## The Pattern
+
+**URL Pattern:** `https://github.com/{owner}/{repo}/tree/main/{path}`
+
+This single URL pattern works for both root directories (with empty path) and subdirectories.
+
+**Data Location:**
+```html
+<script type="application/json" data-target="react-app.embeddedData">
+  {
+    "payload": {
+      "tree": {
+        "items": [...]
+      },
+      ...
+    }
+  }
+</script>
+```
+
+**Access Path:** `data.payload.tree.items`
+
+## Item Structure
+
+Each item in the `items` array has this structure:
+
+```json
+{
+  "name": "filename.txt",
+  "path": "path/to/filename.txt",
+  "contentType": "file"  // or "directory"
+}
+```
+
+## Complete Example
+
+See `github-scraper.js` for a working implementation that:
+- Fetches the HTML from GitHub
+- Extracts the embedded JSON data
+- Parses and returns the directory listing
+- Works for both root and subdirectories
+- Has zero API rate limits
+
+## Usage
+
+```bash
+node github-scraper.js torvalds linux              # Root directory
+node github-scraper.js torvalds linux Documentation # Subdirectory
+node github-scraper.js git git Documentation/git    # Nested subdirectory
+```
+
+## Advantages Over API
+
+| Feature | HTML Scraping | GitHub API |
+|---------|--------------|------------|
+| Rate Limits | None | 60/hour (unauth), 5000/hour (auth) |
+| Authentication | Not required | Optional, but limits apply |
+| Access | Any public repo | Any public repo |
+| Stability | Depends on HTML structure | Stable API contract |
+
+## Important Notes
+
+1. **No authentication needed** - Works for all public repositories
+2. **No rate limits** - Can scrape as many repos as needed
+3. **Less stable** - HTML structure may change with GitHub updates
+4. **Best for:** One-off scripts, personal tools, exploratory work
+5. **Use API for:** Production applications, long-term projects
+
+## Why This Works
+
+GitHub uses server-side rendering to provide fast initial page loads. The directory data is embedded in the HTML so the page can render immediately without waiting for additional API calls. This embedded data is the same data that would come from the API, just pre-rendered in the page.
diff --git a/cgithub/github-scraper.js b/cgithub/github-scraper.js
new file mode 100755
index 0000000..7e07adc
--- /dev/null
+++ b/cgithub/github-scraper.js
@@ -0,0 +1,74 @@
+#!/usr/bin/env node
+/**
+ * GitHub Directory Listing Scraper
+ *
+ * Gets directory listings from GitHub without using the API (no rate limits!)
+ * Works by parsing the server-rendered HTML that GitHub sends to browsers.
+ */
+
+async function getGitHubDirectory(owner, repo, path = '') {
+  // Always use the tree URL pattern, even for root directory
+  const url = `https://github.com/${owner}/${repo}/tree/main/${path}`;
+
+  console.error(`Fetching: ${url}`);
+
+  const response = await fetch(url);
+  const html = await response.text();
+
+  // GitHub embeds the directory data in a JSON script tag
+  // Using the tree (subdirectory) pattern for all cases
+  const subdirMatch = html.match(/<script type="application\/json" data-target="react-app\.embeddedData">([^<]+)<\/script>/);
+
+  if (!subdirMatch) {
+    throw new Error('Could not find embedded data in HTML');
+  }
+
+  const data = JSON.parse(subdirMatch[1]);
+  const payload = data.payload;
+
+  return {
+    path: payload.path,
+    branch: payload.refInfo.name,
+    items: payload.tree.items,
+    repo: {
+      name: payload.repo.name,
+      owner: payload.repo.ownerLogin,
+      isPublic: payload.repo.public
+    }
+  };
+}
+
+// Example usage
+if (require.main === module) {
+  const [owner, repo, path] = process.argv.slice(2);
+
+  if (!owner || !repo) {
+    console.log('Usage: node github-scraper.js <owner> <repo> [path]');
+    console.log('Example: node github-scraper.js torvalds linux');
+    console.log('Example: node github-scraper.js torvalds linux Documentation');
+    process.exit(1);
+  }
+
+  getGitHubDirectory(owner, repo, path)
+    .then(result => {
+      console.log('\n📂 Directory listing');
+      console.log(`Repository: ${result.repo.owner}/${result.repo.name}`);
+      console.log(`Branch: ${result.branch}`);
+      console.log(`Path: ${result.path}`);
+      console.log(`Items: ${result.items.length}`);
+      console.log('\nContents:');
+
+      result.items.forEach(item => {
+        const icon = item.contentType === 'directory' ? '📁' : '📄';
+        console.log(`  ${icon} ${item.name}`);
+      });
+
+      console.log('\n✓ No API rate limits - this scrapes the HTML!');
+    })
+    .catch(err => {
+      console.error('Error:', err.message);
+      process.exit(1);
+    });
+}
+
+module.exports = { getGitHubDirectory };
-- 
cgit v1.3.1