tcmd: Fixes regex reuse in inline tokenizer,

tcmd: changes popover syntax to allow for embeddable markdown

lib/tcmd/index.ts

@@ -1,7 +1,7 @@
 import { zipArrays } from "../zip";
-import { inlineTokens } from "./inlineTokens";
-import { singleLineTokens } from "./singleLineTokens";
+import { singleLineTokens, tokenizeLine } from "./tokenizeLine";
 import { tokenizeBlock } from "./tokenizeBlock";
+import { tokenizeInline } from "./tokenizeInline";
 import { tokenizeParagraph } from "./tokenizeParagraph";
 
 export const createElements = (body: string) => {
@@ -11,6 +11,8 @@ export const createElements = (body: string) => {
 };
 
 const tokenize = (body: string) => {
+  body = body.replace(/\n?<!--(.*?)-->\n?/gs, "");
+
   const paragraphs = body.split("\n\n");
 
   const blockTokens: BlockToken[] = [];
@@ -97,129 +99,3 @@ const tokenize = (body: string) => {
 
   return blockTokens.filter((b) => !b.parent);
 };
-
-// const __tokenize = (md: string) => {
-//   const tokens: (Token)[] = [];
-//   // md = md.replace(/(?<=[a-z])\n(?=[a-z])/g, " ");
-//   const lines = md.split("\n");
-//   let preserveEmpty = false;
-//   let multilineLines;
-//   let tokenSettings;
-
-//   for (let line of lines) {
-//     if (!line && !preserveEmpty) continue;
-//     let foundLine = false;
-
-//     if (!multilineLines) {
-//       token:
-//       for (const token of multilineTokens) {
-//         if (!token.rx.test(line)) continue token;
-//         tokenSettings = token;
-//         multilineLines = token.create(tokens);
-//         preserveEmpty = true;
-//         foundLine = true;
-//         multilineLines.push({
-//           type: "text",
-//           line: token.replace(line),
-//         });
-//       }
-//     } else {
-//       foundLine = true;
-//       if (tokenSettings?.closeRx?.test(line) || tokenSettings?.rx.test(line)) {
-//         tokenSettings = undefined;
-//         multilineLines = undefined;
-//         preserveEmpty = false;
-//       } else {
-//         multilineLines.push({
-//           type: "text",
-//           line,
-//         });
-//       }
-//     }
-
-//     if (!multilineLines) {
-//       token:
-//       for (const token of singleLineTokens) {
-//         if (!token.rx.test(line)) continue token;
-//         foundLine = true;
-//         line = line.replace(token.replaceRx, "").trim();
-
-//         const lineContent = tokenizeInline(line);
-//         token.create(lineContent, tokens);
-//       }
-//     }
-
-//     if (foundLine) continue;
-
-//     tokens.push({
-//       type: "text",
-//       line: tokenizeInline(line),
-//     });
-//   }
-
-//   return tokens;
-// };
-
-const tokenizeLine = (
-  line: string,
-  previous?: SingleLineToken,
-): SingleLineToken => {
-  for (const token of singleLineTokens) {
-    if (!token.rx.test(line)) continue;
-
-    const t = token.create(line);
-
-    if (t.type === "h2") {
-    }
-
-    t.line = tokenizeInline(line.replace(token.replaceRx, ""));
-    return t;
-  }
-
-  if (previous?.mends) {
-    previous.raw += " " + line;
-    previous.line = tokenizeInline(previous.raw.replace(previous.cfg!.rx, ""));
-    return previous;
-  }
-
-  return {
-    line: tokenizeInline(line),
-    type: "text",
-    raw: line,
-  };
-};
-
-const tokenizeInline = (line: string) => {
-  line = line.trim();
-  const originalLine = line;
-  const insertMarker = "\u{03A9}";
-  const tokens: InlineTokenInsert[] = [];
-
-  for (const token of inlineTokens) {
-    token.rx.lastIndex = 0;
-    let match;
-    while ((match = token.rx.exec(line)) !== null) {
-      const tokenStart = match.index;
-      const tokenEnd = match.index + match[0].length;
-
-      token.create(match, tokenStart, tokenEnd, tokens);
-    }
-  }
-
-  if (tokens.length) {
-    for (const insert of tokens) {
-      line = line.slice(0, insert.start) +
-        "".padStart(insert.end - insert.start, insertMarker) +
-        line.slice(insert.end, line.length);
-    }
-
-    return zipArrays(
-      line.split(new RegExp(insertMarker + "{2,}")).map((t): InlineToken => ({
-        content: t,
-        type: "text",
-      })),
-      tokens,
-    ).filter((t) => t.content);
-  }
-  return originalLine;
-};

lib/tcmd/tokenizeInline.ts

@@ -1,3 +1,55 @@
+import { zipArrays } from "../zip";
+
+export const tokenizeInline = (line: string, recursive?: boolean) => {
+  if (recursive) console.log("recursive call");
+  line = line.trim();
+  const originalLine = line;
+  const insertMarker = "\u{03A9}";
+  const tokens: InlineTokenInsert[] = [];
+
+  for (const token of inlineTokens) {
+    const rx = new RegExp(token.rx);
+    let match;
+    while ((match = rx.exec(line)) !== null) {
+      const tokenStart = match.index;
+      const tokenEnd = match.index + match[0].length;
+
+      const wrappingToken = tokens.find((t) =>
+        t.start < tokenStart && t.end > tokenStart
+      );
+      if (wrappingToken) continue;
+
+      let wrappedToken;
+      while (
+        (wrappedToken = tokens.findIndex((t) =>
+          t.start > tokenStart && t.start < tokenEnd
+        )) !== -1
+      ) {
+        tokens.splice(wrappedToken, 1);
+      }
+
+      token.create(match, tokenStart, tokenEnd, tokens);
+    }
+  }
+
+  if (tokens.length) {
+    for (const insert of tokens) {
+      line = line.slice(0, insert.start) +
+        "".padStart(insert.end - insert.start, insertMarker) +
+        line.slice(insert.end, line.length);
+    }
+
+    return zipArrays(
+      line.split(new RegExp(insertMarker + "{2,}")).map((t): InlineToken => ({
+        content: t,
+        type: "text",
+      })),
+      tokens,
+    ).filter((t) => t.content);
+  }
+  return originalLine;
+};
+
 const joiner = "<><>";
 export const inlineTokens: {
   rx: RegExp;
@@ -61,16 +113,17 @@ export const inlineTokens: {
     },
   },
   {
-    rx: /\^\[(.*?)\]\((.*?)\)/g,
+    rx: /\^\[(.*?)\]<<(.*?)>>/gm,
     create(content, start, end, tokens) {
       const [_, text, popover] = content;
+      // tokenizeInline("", true);
       tokens.push({
         content: text,
         end,
         start,
         type: "popover",
         data: {
-          popover,
+          popover: tokenizeInline(popover),
         },
       });
     },

lib/tcmd/tokenizeLine.ts

@@ -1,3 +1,34 @@
+import { tokenizeInline } from "./tokenizeInline";
+
+export const tokenizeLine = (
+  line: string,
+  previous?: SingleLineToken,
+): SingleLineToken => {
+  for (const token of singleLineTokens) {
+    if (!token.rx.test(line)) continue;
+
+    const t = token.create(line);
+
+    if (t.type === "h2") {
+    }
+
+    t.line = tokenizeInline(line.replace(token.replaceRx, ""));
+    return t;
+  }
+
+  if (previous?.mends) {
+    previous.raw += " " + line;
+    previous.line = tokenizeInline(previous.raw.replace(previous.cfg!.rx, ""));
+    return previous;
+  }
+
+  return {
+    line: tokenizeInline(line),
+    type: "text",
+    raw: line,
+  };
+};
+
 export const singleLineTokens: SingleLineCfg[] = [
   {
     rx: /^#\s/,