GitHub scraper test

3 files changed, 77 insertions, 3 deletions

diff --git a/package.json b/package.json
index b3d778b..71dfb4d 100644
--- a/package.json
+++ b/package.json
@@ -5,7 +5,8 @@
   "type": "module",
   "scripts": {
     "dev": "node --watch --watch-preserve-output src/index.ts",
-    "build": "rm -rf dist/* && esbuild --bundle --format=esm --platform=node --outdir=dist/build --sourcemap src/index.ts && cp -R static views dist/"
+    "build": "rm -rf dist/* && esbuild --bundle --format=esm --platform=node --outdir=dist/build --sourcemap src/index.ts && cp -R static views dist/",
+    "test": "node --test src/scraper.test.ts"
   },
   "dependencies": {
     "@hono/node-server": "^1.13.7",
diff --git a/src/scraper.test.ts b/src/scraper.test.ts
new file mode 100644
index 0000000..3784190
--- /dev/null
+++ b/src/scraper.test.ts
@@ -0,0 +1,70 @@
+import assert from "node:assert";
+import { describe, it } from "node:test";
+import { getGitHubBlob, getGitHubRepo, getGitHubTree } from "./scraper.ts";
+
+describe("GitHub scraper", () => {
+  describe("actions/deploy-pages repository", () => {
+    it("should fetch repository", async () => {
+      const data = await getGitHubRepo("actions", "deploy-pages");
+
+      assert.strictEqual(data.repo.owner, "actions");
+      assert.strictEqual(data.repo.name, "deploy-pages");
+      assert.strictEqual(data.branch, "main");
+      assert.strictEqual(data.path, "/");
+
+      assert.strictEqual(
+        data.info.description,
+        "GitHub Action to publish artifacts to GitHub Pages for deployments",
+      );
+      assert.strictEqual(data.info.website, "https://pages.github.com");
+      assert.strictEqual(data.info.stars?.length, 3);
+      assert.strictEqual(data.info.watchers?.length, 2);
+      assert.strictEqual(data.info.forks?.length, 3);
+      assert.strictEqual(data.info.numReleases, 36);
+
+      assert.ok(data.items.length > 0);
+      assert.ok(Object.keys(data.overviewHtml ?? {}).includes("README.md"));
+    });
+
+    it("should fetch tree", async () => {
+      const data = await getGitHubTree("actions", "deploy-pages", "main", "src");
+
+      assert.strictEqual(data.repo.owner, "actions");
+      assert.strictEqual(data.repo.name, "deploy-pages");
+      assert.strictEqual(data.branch, "main");
+      assert.strictEqual(data.path, "src");
+
+      assert.deepStrictEqual(data.items, [
+        {
+          name: "__tests__",
+          path: "src/__tests__",
+          contentType: "directory",
+        },
+        {
+          name: "internal",
+          path: "src/internal",
+          contentType: "directory",
+        },
+        {
+          name: "index.js",
+          path: "src/index.js",
+          contentType: "file",
+        },
+      ]);
+    });
+
+    it("should fetch blob", async () => {
+      const data = await getGitHubBlob("actions", "deploy-pages", "main", "LICENSE");
+
+      assert.strictEqual(data.repo.owner, "actions");
+      assert.strictEqual(data.repo.name, "deploy-pages");
+      assert.strictEqual(data.branch, "main");
+      assert.strictEqual(data.path, "LICENSE");
+
+      assert.strictEqual(data.size, 1068);
+      assert.strictEqual(data.language, "Text");
+      assert.match(data.rawContent, /^MIT License\n\n/);
+      assert.match(data.content, /^MIT License\n\n/);
+    });
+  });
+});
diff --git a/src/scraper.ts b/src/scraper.ts
index 4ffdc43..4feda73 100644
--- a/src/scraper.ts
+++ b/src/scraper.ts
@@ -63,7 +63,7 @@ export interface GitHubRepo extends GitHubTree {
 
 export interface GitHubBlob extends GitHubCommon {
   language: string | null;
-  size: number;
+  size: number | null;
   content: string;
   rawContent: string;
 }
@@ -210,10 +210,13 @@ export async function getGitHubBlob(
   const highlightedLines = blob.highlightedLines || [];
   const content = highlightedLines.map((line) => line.text || "").join("\n");
 
+  // Calculate size from raw content if not provided
+  const size = blob.size ?? (rawContent.length > 0 ? Buffer.byteLength(rawContent, "utf8") : null);
+
   return extractGitHub(payload, {
     content: content || rawContent,
     rawContent,
     language: blob.language || null,
-    size: blob.size,
+    size,
   });
 }