2 files changed, 48 insertions, 8 deletions

diff --git a/src/scraper.test.ts b/src/scraper.test.ts
index f3a472d..9326747 100644
--- a/src/scraper.test.ts
+++ b/src/scraper.test.ts
@@ -69,7 +69,10 @@ describe("GitHub scraper", () => {
 
       assert.strictEqual(data.size, "1.04 KB / 21 lines / 17 loc");
       assert.strictEqual(data.language, "Text");
-      assert.strictEqual(data.rawLines?.[0], "MIT License");
+      assert.strictEqual(data.textLines?.length, 21);
+      assert.strictEqual(data.textLines[0], "MIT License");
+      assert.strictEqual(data.htmlLines?.length, 21);
+      assert.strictEqual(data.htmlLines[0], "MIT License");
       assert.strictEqual(data.htmlContent, null);
     });
 
@@ -83,7 +86,8 @@ describe("GitHub scraper", () => {
 
       assert.strictEqual(data.size, "8.95 KB / 133 lines / 94 loc");
       assert.strictEqual(data.language, "Markdown");
-      assert.strictEqual(data.rawLines, null);
+      assert.strictEqual(data.textLines, null);
+      assert.strictEqual(data.htmlLines, null);
       const firstLine =
         /^<article class="markdown-body entry-content container-lg" itemprop="text"><div class="markdown-heading" dir="auto"><h1 tabindex="-1" class="heading-element" dir="auto">deploy-pages 🚀<\/h1>/;
       assert.match(data.htmlContent ?? "", firstLine);
@@ -99,7 +103,10 @@ describe("GitHub scraper", () => {
 
       assert.strictEqual(data.size, "39 Bytes / 1 lines / 1 loc");
       assert.strictEqual(data.language, "Git Attributes");
-      assert.deepStrictEqual(data.rawLines, ["dist/** -diff linguist-generated=true "]);
+      assert.deepStrictEqual(data.textLines, ["dist/** -diff linguist-generated=true "]);
+      assert.deepStrictEqual(data.htmlLines, [
+        '\u003cspan class="pl-e"\u003edist\u003c/span\u003e/\u003cspan class="pl-k"\u003e**\u003c/span\u003e \u003cspan class="pl-k"\u003e-\u003c/span\u003e\u003cspan class="pl-v"\u003ediff\u003c/span\u003e \u003cspan class="pl-v"\u003elinguist-generated\u003c/span\u003e=\u003cspan class="pl-c1"\u003etrue\u003c/span\u003e ',
+      ]);
       assert.strictEqual(data.htmlContent, null);
     });
 
diff --git a/src/scraper.ts b/src/scraper.ts
index c2983c9..05afc33 100644
--- a/src/scraper.ts
+++ b/src/scraper.ts
@@ -52,6 +52,8 @@ interface IssueIndexPageQuery {
   };
 }
 
+type StylingDirective = [number, number, string];
+
 interface RawPayload {
   repo: {
     ownerLogin: string;
@@ -74,6 +76,7 @@ interface RawPayload {
     rawLines: string[] | null;
     colorizedLines: string[] | null;
     richText: string | null;
+    stylingDirectives: StylingDirective[][] | null;
   };
   overview?: {
     overviewFiles?: OverviewFile[];
@@ -82,6 +85,27 @@ interface RawPayload {
   preloadedQueries?: IssueIndexPageQuery[];
 }
 
+function escapeHtml(text: string): string {
+  return text
+    .replace(/&/g, "&amp;")
+    .replace(/</g, "&lt;")
+    .replace(/>/g, "&gt;")
+    .replace(/"/g, "&quot;")
+    .replace(/'/g, "&#039;");
+}
+
+function applyStyling(line: string, directives: StylingDirective[]): string {
+  let result = "";
+  let lastPos = 0;
+  for (const [start, end, className] of directives) {
+    result += escapeHtml(line.substring(lastPos, start));
+    result += `<span class="${className}">${escapeHtml(line.substring(start, end))}</span>`;
+    lastPos = end;
+  }
+  result += escapeHtml(line.substring(lastPos));
+  return result;
+}
+
 interface GitHubCommon {
   repo: {
     owner: string;
@@ -115,7 +139,8 @@ export interface GitHubRepo extends GitHubTree {
 export interface GitHubBlob extends GitHubNav {
   language: string | null;
   size: string;
-  rawLines: string[] | null;
+  textLines: string[] | null;
+  htmlLines: string[] | null;
   htmlContent: string | null;
 }
 
@@ -269,15 +294,23 @@ export async function getGitHubBlob(
     throw new Error("Could not find blob data in embedded JSON");
   }
 
-  const { headerInfo, language, rawLines, richText } = payload.blob;
+  const blob = payload.blob;
+  const { headerInfo, stylingDirectives } = blob;
+
+  let htmlLines = blob.colorizedLines;
+  if (!htmlLines && stylingDirectives && blob.rawLines) {
+    htmlLines = blob.rawLines.map((line, i) => applyStyling(line, stylingDirectives[i]));
+  }
+
   // TODO fine tune
   const size = `${headerInfo.blobSize} / ${headerInfo.lineInfo.truncatedLoc} lines / ${headerInfo.lineInfo.truncatedSloc} loc`;
 
   return extractGitHub(payload, {
-    language: language || null,
+    language: blob.language || null,
     size,
-    rawLines,
-    htmlContent: richText,
+    textLines: blob.rawLines,
+    htmlLines,
+    htmlContent: blob.richText,
   });
 }