ttcMD: mostly working nested grid parser

lib/tcmd/TokenIdentifiers.ts

@@ -1,7 +1,18 @@
-export const TokenIdentifiers = new Map<string, {
+type TokenIdentifier = {
   rx: RegExp;
   parse: (s: string) => Token;
-}>();
+  search?: (s: string, start: number, end: number) => {
+    start: number;
+    end: number;
+    text: string;
+    lastIndex: number;
+  };
+};
+
+export const TokenIdentifiers = new Map<
+  string,
+  TokenIdentifier
+>();
 
 // TokenIdentifiers.set("p", {
 //   rx: /\n{2,}((?:.|\n)*?)\n{2,}/g,
@@ -81,14 +92,44 @@ TokenIdentifiers.set("list-item", {
   },
 });
 TokenIdentifiers.set("grid", {
+  search(s, start, end) {
+    const rx = /(?<!\/)(?:\[\])+/g;
+    const closeRx = /\/\[\]/g;
+
+    const oldEnd = end;
+
+    const newEnd = findMatchingClosedParenthesis(
+      s,
+      rx,
+      closeRx,
+    );
+
+    if (newEnd === null) throw Error("BAD BAD BAD BAD");
+
+    end = newEnd + start;
+
+    return {
+      start,
+      end,
+      text: s.substring(0, newEnd),
+      lastIndex: oldEnd === end
+        ? end
+        : start + s.match(new RegExp(rx))![0].length,
+    };
+  },
   rx: /(?:\[\])+\n+((?:.|\n)*?)\n+\/\[\]/g,
   parse(s) {
+    const rx = /((?:\[\])+)\n+([\s\S]*)\n+\/\[\]/;
+    console.log(s);
+    const [_, columns, content] = s.match(rx) ||
+      ["", "..", "Unable to parse grid"];
+
+    console.log(columns);
     return {
-      content: s.match(new RegExp(this.rx, ""))?.at(1) ||
-        "Unable to parse grid",
+      content,
       raw: s,
       metadata: {
-        columns: s.split("\n").at(0)?.match(/\[\]/g)?.length.toString() || "1",
+        columns: (columns.length / 2).toString(),
       },
       type: "grid",
       uuid: crypto.randomUUID(),
@@ -254,3 +295,49 @@ TokenIdentifiers.set("p", {
 // const p = TokenIdentifiers.get("p");
 // TokenIdentifiers.clear();
 // p && TokenIdentifiers.set("p", p);
+
+function findMatchingClosedParenthesis(
+  str: string,
+  openRegex: RegExp,
+  closedRegex: RegExp,
+): number | null {
+  let openings = 0;
+  let closings = 0;
+
+  openRegex = new RegExp(openRegex, "g");
+  closedRegex = new RegExp(closedRegex, "g");
+
+  let lastOpeningSuccessIndex = 0;
+  let lastClosingSuccessIndex = 0;
+
+  debugger;
+  do {
+    const openingMatch = openRegex.exec(str);
+    const closingMatch = closedRegex.exec(str);
+
+    if ((openingMatch && !closingMatch)) {
+      throw Error("Things have gone horribly wrong");
+    }
+
+    // if ((!openingMatch && closingMatch) || (!openingMatch && !closingMatch)) break;
+
+    if (
+      openingMatch && closingMatch && openingMatch.index < closingMatch.index
+    ) {
+      openings++;
+      lastOpeningSuccessIndex = openingMatch.index + openingMatch[0].length;
+      closedRegex.lastIndex = lastClosingSuccessIndex;
+    } else if (
+      (!openingMatch && closingMatch) ||
+      (openingMatch && closingMatch && openingMatch.index > closingMatch.index)
+    ) {
+      closings++;
+      lastClosingSuccessIndex = closingMatch.index + closingMatch[0].length;
+      openRegex.lastIndex = lastOpeningSuccessIndex;
+    } else {
+      return closingMatch?.index ?? null;
+    }
+  } while (openings > closings);
+
+  return closedRegex.lastIndex;
+}

lib/tcmd/index.ts

@@ -35,6 +35,19 @@ const tokenize = (body: string) => {
       const start = match.index;
       const end = rx.lastIndex;
 
+      if (token.search) {
+        const found = token.search(body.substring(start), start, end);
+        rx.lastIndex = found.lastIndex;
+
+        addToken({
+          start: found.start,
+          end: found.end,
+          type,
+          token: token.parse(found.text),
+        });
+        continue;
+      }
+
       addToken({
         start,
         end,
@@ -69,34 +82,6 @@ function buildAbstractSyntaxTree(markers: TokenMarker[]) {
   return markers;
 }
 
-// function establishClosestParent(blocks: TokenMarker[]): void {
-//   blocks.sort((a, b) => a.start - b.start); // Sort blocks by start position
-
-//   for (let i = 0; i < blocks.length; i++) {
-//     const block = blocks[i];
-//     if (block.parent) continue; // Skip blocks that already have a parent
-
-//     let closestParent: TokenMarker | undefined = undefined;
-//     let minDistance = Number.MAX_SAFE_INTEGER;
-
-//     // Find the closest parent block for each block
-//     for (let j = 0; j < i; j++) {
-//       const otherBlock = blocks[j];
-//       if (otherBlock.end >= block.start && otherBlock.start <= block.start) {
-//         const distance = block.start - otherBlock.start;
-//         if (distance < minDistance) {
-//           minDistance = distance;
-//           closestParent = otherBlock;
-//         }
-//       }
-//     }
-
-//     if (closestParent) {
-//       block.parent = closestParent; // Assign the closest parent block
-//     }
-//   }
-// }
-
 function establishClosestParent(blocks: TokenMarker[]): void {
   blocks.sort((a, b) => a.start - b.start); // Sort blocks by start position
 
@@ -166,6 +151,7 @@ function filterOverlappingPBlocks(blocks: TokenMarker[]): TokenMarker[] {
 
 const contentToChildren = (token: Token) => {
   let content = token.content;
+  if (!content) return;
 
   const wasSpecialCase = handleSpecial(token);
 
@@ -260,7 +246,6 @@ function processChunks(chunks: Token[][]) {
     }
   }
 
-  // console.log(chunks);
   // Filter out chunks that were merged into others
   return chunks.filter((c) => !mergedChunks.find((c2) => c === c2));
 }