I'm really sick of not making any progress

.gitignore

@@ -37,3 +37,5 @@ next-env.d.ts
 
 # vscode
 .vscode
+
+temp.json

app/help/[article]/page.tsx

@@ -1,6 +1,6 @@
 import { TTCMD } from "@/components/ttcmd";
+import { ArrowLeftCircleIcon } from "@heroicons/react/24/solid";
 import { readFile } from "fs/promises";
-import Error from "next/error";
 import { Suspense } from "react";
 
 export default async function Help({
@@ -8,6 +8,7 @@ export default async function Help({
 }: {
   params: { article: string };
 }) {
+  if (!params.article.endsWith(".md")) return <></>;
   const body = readFile(
     "./md/help articles/" + decodeURIComponent(params.article),
     "utf-8",
@@ -18,9 +19,13 @@ export default async function Help({
         <h2 className="strapline">Help</h2>
         <h1>How to use TTCMD</h1>
       </section>
+      <section className="grid grid-cols-3 gap-x-8 gap-y-6 my-6">
+        <div className="col-span-2 card">
           <Suspense>
             <TTCMD body={body} />
           </Suspense>
+        </div>
+      </section>
     </>
   );
 }

app/layout.tsx

@@ -23,16 +23,16 @@ export default function RootLayout({
   children: React.ReactNode;
 }>) {
   const navItems = [
-    {
-      to: "/schemas",
-      icon: CircleStackIcon,
-      text: "Schemas",
-    },
     {
       to: "/game-systems",
       icon: PuzzlePieceIcon,
       text: "Game Systems",
     },
+    {
+      to: "/schemas",
+      icon: CircleStackIcon,
+      text: "Schemas",
+    },
     {
       to: "/publications",
       icon: BookOpenIcon,

components/ttcmd/index.tsx

@@ -2,31 +2,185 @@
 
 import { Accordion, AccordionContent } from "@/lib/accordion";
 import { Poppable } from "@/lib/poppables/components/poppable";
-import { createElements } from "@/lib/tcmd";
+import { buildAbstractSyntaxTree, createElements } from "@/lib/tcmd";
 import Link from "next/link";
 import React, { FC, Fragment, ReactNode, use, useMemo } from "react";
 
 import { sanitize } from "isomorphic-dompurify";
+import StaticGenerationSearchParamsBailoutProvider from "next/dist/client/components/static-generation-searchparams-bailout-provider";
 
 export const TTCMD: FC<{ body: Promise<string> }> = ({ body }) => {
   const text = use(body);
-  const elements = useMemo(() => createElements(text), [text]);
+  const [elements, tabSpacing] = useMemo(() => createElements(text), [text]);
   return (
-    // <div className="grid grid-cols-2">
+    <div className="text-lg col-span-2">
-    //   <pre>{JSON.stringify(elements,null,2)}</pre>
+      <div>
-    // </div>
+        <button
-    <div className="flex flex-col gap-6 my-6">
+          className="btn-primary"
-      {elements.map((e, i) => <Fragment key={e.uuid}>{renderBlock(e)}
+          onClick={() =>
-      </Fragment>)}
+            navigator.clipboard.writeText(JSON.stringify(elements, null, 2))}
+        >
+          copy ast
+        </button>
       </div>
+      {/* {elements.map((e, i) => <Fragment key={e.uuid}>{render(e)}</Fragment>)} */}
+      {renderer(elements, tabSpacing)}
+    </div>
+    // <div className="grid grid-cols-3">
+    //   {/* <pre suppressHydrationWarning>{JSON.stringify(elements,null,2)}</pre> */}
+    // </div>
   );
 };
 
-const renderBlock = (block: BlockChildren): ReactNode => {
+const renderer = (tokens: Token[], tabSpacing: number) => {
+  const usedIds: string[] = [];
+  return tokens.map((t) => (
+    <Fragment key={t.uuid}>{render(t, usedIds, tabSpacing)}</Fragment>
+  ));
+};
+
+const render = (token: Token, usedIds: string[], tabSpacing: number) => {
+  switch (token.type) {
+    case "heading":
+      return (
+        <div
+          id={generateId(token.raw, usedIds)}
+          data-strength={token.metadata.strength}
+          className={`
+          font-bold
+          data-[strength="1"]:text-4xl
+          data-[strength="2"]:text-3xl
+          data-[strength="3"]:text-2xl
+          p
+          `}
+        >
+          {token.content}
+        </div>
+      );
+    case "grid":
+      return (
+        <div
+          style={{
+            "--grid-cols": token.metadata.columns,
+          } as React.CSSProperties}
+          className="grid grid-cols-dynamic gap-x-8 gap-y-6 mb-6"
+        >
+          {token.children?.map((c, i) => (
+            <div key={c.uuid}>
+              {render(c, usedIds, tabSpacing)}
+            </div>
+          ))}
+        </div>
+      );
+    case "code":
+      return (
+        <pre className="whitespace-pre-wrap bg-black/20 p-2 rounded-md">
+          {token.content}
+        </pre>
+      );
+    case "card":
+      return (
+        <div className="card mb-6">
+          {token.children?.map((e) => (
+            <Fragment key={e.uuid}>
+              {render(e, usedIds, tabSpacing)}
+            </Fragment>
+          ))}
+        </div>
+      );
+    case "anchor":
+      return (
+        <Link
+          className={token.metadata.classes ||
+            "dark:text-primary-600 underline dark:no-underline"}
+          href={token.metadata.href}
+        >
+          {token.content}
+        </Link>
+      );
+    case "image": {
+      token.metadata.src = token.metadata.src as string;
+      if (token.metadata.src.startsWith("<svg")) {
+        return (
+          <div
+            dangerouslySetInnerHTML={{
+              __html: sanitize(token.metadata.src, {
+                USE_PROFILES: { svg: true },
+              }),
+            }}
+          >
+          </div>
+        );
+      }
+      // eslint-disable-next-line @next/next/no-img-element
+      return <img src={token.metadata.src} alt={token.content} />;
+    }
+    case "inline-code":
+      return (
+        <span className="p-1 rounded-md font-mono bg-black/20 border border-mixed-100/20 mx-2">
+          {token.content}
+        </span>
+      );
+    case "popover":
+      return (
+        <Poppable
+          content={token.children?.map((c) => render(c, usedIds, tabSpacing)) ||
+            token.content}
+          preferredAlign="centered"
+          preferredEdge="bottom"
+          className="cursor-pointer mx-2"
+        >
+          <span className="border-b-2 border-dotted border-white">
+            {token.metadata.title}
+          </span>
+        </Poppable>
+      );
+    case "text":
+      return <span>{token.content}</span>;
+    case "p":
+      return (
+        <div className="p">
+          {token.children?.map((e, i) => (
+            <Fragment key={e.uuid}>
+              {render(e, usedIds, tabSpacing)}
+            </Fragment>
+          ))}
+        </div>
+      );
+    case "accordion":
+      return (
+        <div className="bg-black/20 p-1 rounded-md">
+          <Accordion
+            title={token.metadata.title || "Expand"}
+          >
+            <AccordionContent>
+              {token.children?.map((e, i) => (
+                <Fragment key={e.uuid}>
+                  {render(e, usedIds, tabSpacing)}
+                </Fragment>
+              ))}
+            </AccordionContent>
+          </Accordion>
+        </div>
+      );
+    default:
+      return (
+        <div className="p bg-red-600 text-white">
+          Block or paragraph missing implementation: {token.type}
+        </div>
+      );
+  }
+};
+
+const renderBlock = (
+  block: BlockChildren,
+  usedIds: string[] = [],
+): ReactNode => {
+  usedIds = usedIds || [];
   switch (block.type) {
     case "block":
       return block.children.map((e, i) => (
-        <Fragment key={e.uuid}>{renderBlock(e)}</Fragment>
+        <Fragment key={e.uuid}>{renderBlock(e, usedIds)}</Fragment>
       ));
     case "grid":
       return (
@@ -38,50 +192,52 @@ const renderBlock = (block: BlockChildren): ReactNode => {
         >
           {block.children.map((c, i) => (
             <div key={c.uuid}>
-              {renderBlock(c)}
+              {renderBlock(c, usedIds)}
             </div>
           ))}
         </div>
       );
     case "card":
       return (
-        <div className="card">
+        <div className="card mb-6">
           {block.children.map((e, i) => (
             <Fragment key={e.uuid}>
-              {renderBlock(e)}
+              {renderBlock(e, usedIds)}
             </Fragment>
           ))}
         </div>
       );
     case "accordion":
       return (
+        <div className="bg-black/20 p-1 rounded-md">
           <Accordion
             title={block.metadata.title || "Expand"}
           >
             <AccordionContent>
               {block.children.map((e, i) => (
                 <Fragment key={e.uuid}>
-                {renderBlock(e)}
+                  {renderBlock(e, usedIds)}
                 </Fragment>
               ))}
             </AccordionContent>
           </Accordion>
+        </div>
       );
     default:
       return (
-        renderParagraph(block as ParagraphToken)
+        renderParagraph(block as ParagraphToken, usedIds)
       );
   }
 };
 
-const renderParagraph = (p: ParagraphToken) => {
+const renderParagraph = (p: ParagraphToken, usedIds: string[]) => {
   switch (p.type) {
     case "p":
       return (
         <div className="p">
           {p.content.map((e, i) => (
             <Fragment key={e.uuid}>
-              {renderToken(e)}
+              {renderToken(e, usedIds)}
             </Fragment>
           ))}
         </div>
@@ -101,44 +257,52 @@ const renderParagraph = (p: ParagraphToken) => {
   }
 };
 
-const renderToken = (t: Token) => {
+const generateId = (t: string, usedIds: string[]) => {
-  switch (t.type) {
+  let id = t.toLowerCase().replace(/[^a-z\s]/ig, "").trim().replaceAll(
-    case "h1":
-      return (
-        <div
-          id={t.raw.toLowerCase().replace(/[^a-z\s]/ig, "").trim().replaceAll(
     " ",
     "-",
-          )}
+  );
+  let idNum = 1;
+  while (usedIds.includes(id)) {
+    id = id.replace(/-[0-9]+$/g, "");
+    id += "-" + idNum;
+    idNum++;
+  }
+  return id;
+};
+
+const renderToken = (t: Token, usedIds: string[]) => {
+  switch (t.type) {
+    case "h1": {
+      return (
+        <div
+          id={generateId(t.raw, usedIds)}
           className="font-bold text-2xl p"
         >
           {renderInlineToken(t.line)}
         </div>
       );
-    case "h2":
+    }
+    case "h2": {
       return (
         <div
-          id={t.raw.toLowerCase().replace(/[^a-z\s]/ig, "").trim().replaceAll(
+          id={generateId(t.raw, usedIds)}
-            " ",
-            "-",
-          )}
           className="font-bold text-xl p"
         >
           {renderInlineToken(t.line)}
         </div>
       );
-    case "h3":
+    }
+    case "h3": {
       return (
         <div
-          id={t.raw.toLowerCase().replace(/[^a-z\s]/ig, "").trim().replaceAll(
+          id={generateId(t.raw, usedIds)}
-            " ",
-            "-",
-          )}
           className="font-bold text-lg p"
         >
           {renderInlineToken(t.line)}
         </div>
       );
+    }
     case "p":
       return (
         <div className="p">
@@ -223,6 +387,12 @@ const renderInlineToken = (l: Line) => {
                 </span>
               </Poppable>
             );
+          case "inline-code":
+            return (
+              <span className="p-1 rounded-md font-mono bg-black/20 border border-mixed-100/20 mx-2">
+                {token.content}
+              </span>
+            );
           default:
             return (
               <span className="bg-red-500">

lib/portal/components/index.ts

@@ -1,3 +1,5 @@
+"use client";
+
 import { FC, PropsWithChildren, useEffect, useState } from "react";
 import { createPortal } from "react-dom";
 
@@ -12,9 +14,10 @@ export const Portal: FC<PropsWithChildren<IProps>> = (
   const [container] = useState(() => {
     // This will be executed only on the initial render
     // https://reactjs.org/docs/hooks-reference.html#lazy-initial-state
-    return document.getElementById("root-portal")!;
+    return document.getElementById("root-portal") || document.createElement(el);
   });
 
+  // todo: this smells. appending the same element?
   useEffect(() => {
     container.classList.add(className);
     document.body.appendChild(container);

lib/tcmd/TokenIdentifiers.ts

@@ -0,0 +1,192 @@
+export const TokenIdentifiers = new Map<string, {
+  rx: RegExp;
+  parse: (s: string) => Token;
+}>();
+
+// TokenIdentifiers.set("p", {
+//   rx: /\n{2,}((?:.|\n)*?)\n{2,}/g,
+//   parse(s) {
+//     const [_, content] = s.match(new RegExp(this.rx, ""))!;
+
+//     return {
+//       // content,
+//       content,
+//       raw: s,
+//       metadata: {},
+//       type: "p",
+//       uuid: crypto.randomUUID(),
+//     };
+//   },
+// });
+const rendersContentOnly = true;
+const rendersChildrenOnly = true;
+TokenIdentifiers.set("card", {
+  rx: /\[{2}\n+(\n|.*?)\n+\]{2}/g,
+  parse(s) {
+    return {
+      content: s.match(new RegExp(this.rx, ""))?.at(1) ||
+        "Unable to parse card",
+      raw: s,
+      metadata: {},
+      type: "card",
+      uuid: crypto.randomUUID(),
+    };
+  },
+});
+TokenIdentifiers.set("code", {
+  rx: /`{3}\n+((?:.|\n)*?)\n+`{3}/g,
+  parse(s) {
+    return {
+      content: s.match(new RegExp(this.rx, ""))?.at(1) ||
+        "Unable to parse code",
+      raw: s,
+      metadata: {},
+      type: "code",
+      uuid: crypto.randomUUID(),
+      rendersContentOnly,
+    };
+  },
+});
+TokenIdentifiers.set("grid", {
+  rx: /(?:\[\])+\n+((?:.|\n)*?)\n+\/\[\]/g,
+  parse(s) {
+    return {
+      content: s.match(new RegExp(this.rx, ""))?.at(1) ||
+        "Unable to parse grid",
+      raw: s,
+      metadata: {
+        columns: s.split("\n").at(0)?.match(/\[\]/g)?.length.toString() || "1",
+      },
+      type: "grid",
+      uuid: crypto.randomUUID(),
+      rendersChildrenOnly,
+    };
+  },
+});
+TokenIdentifiers.set("heading", {
+  rx: /^#+\s(.*?)$/gm,
+  parse(s) {
+    return {
+      content: s.match(new RegExp(this.rx, ""))?.at(1) ||
+        "Unable to parse heading",
+      raw: s,
+      metadata: {
+        strength: s.match(/#/g)?.length.toString() || "1",
+      },
+      type: "heading",
+      uuid: crypto.randomUUID(),
+      rendersContentOnly,
+    };
+  },
+});
+TokenIdentifiers.set("image", {
+  rx: /\!\[(.*?)\]\((.*?)\)/g,
+  parse(s) {
+    const [_, title, src] = s.match(new RegExp(this.rx, ""))!;
+
+    return {
+      // content: inline,
+      content: title.trim(),
+      raw: s,
+      metadata: {
+        src,
+      },
+      type: "image",
+      uuid: crypto.randomUUID(),
+      rendersContentOnly,
+    };
+  },
+});
+TokenIdentifiers.set("anchor", {
+  rx: /(?<![\!^])\[(.*?)\]\((.*?)\)/g,
+  parse(s) {
+    let preset, [_, title, href] = s.match(new RegExp(this.rx, ""))!;
+    const match = title.match(/`{3}(cta|button)?(.*)/);
+
+    if (match) {
+      [_, preset, title] = match;
+    }
+
+    const classes = {
+      button: "btn-primary inline-block",
+      cta: "btn-secondary inline-block uppercase",
+    };
+    return {
+      // content: inline,
+      content: title.trim(),
+      raw: s,
+      metadata: {
+        href,
+        classes: classes[preset as keyof typeof classes],
+      },
+      type: "anchor",
+      uuid: crypto.randomUUID(),
+      rendersContentOnly,
+    };
+  },
+});
+TokenIdentifiers.set("inline-code", {
+  rx: /\s?`(.{3,}|[a-z0-9]*?)`[^`a-z0-9\n]/gi,
+  parse(s) {
+    return {
+      // content: inline,
+      content: s.match(new RegExp(this.rx, "i"))?.at(1) ||
+        "Unable to parse inline-code",
+      raw: s,
+      metadata: {},
+      type: "inline-code",
+      uuid: crypto.randomUUID(),
+      rendersContentOnly,
+    };
+  },
+});
+TokenIdentifiers.set("popover", {
+  rx: /\^\[(.*?)\]\<<(.*?)\>>/g,
+  parse(s) {
+    const [_, title, content] = s.match(new RegExp(this.rx, ""))!;
+
+    return {
+      // content,
+      content,
+      raw: s,
+      metadata: { title },
+      type: "popover",
+      uuid: crypto.randomUUID(),
+      rendersContentOnly,
+    };
+  },
+});
+TokenIdentifiers.set("accordion", {
+  rx: /\[accordion(\s.*?)?]\n+((?:.|\n)*?)\n+\[\/accordion\]/g,
+  parse(s) {
+    const [_, title, content] = s.match(new RegExp(this.rx, ""))!;
+
+    return {
+      // content,
+      content,
+      raw: s,
+      metadata: { title },
+      type: "accordion",
+      uuid: crypto.randomUUID(),
+    };
+  },
+});
+TokenIdentifiers.set("p", {
+  rx: /(?<=\n\n)([\s\S]*?)(?=\n\n)/g,
+  parse(s) {
+    // const [_, content] = s.match(new RegExp(this.rx, ""))!;
+
+    return {
+      // content,
+      content: s,
+      raw: s,
+      metadata: {},
+      type: "p",
+      uuid: crypto.randomUUID(),
+    };
+  },
+});
+
+// const p = TokenIdentifiers.get("p");
+// TokenIdentifiers.clear();
+// p && TokenIdentifiers.set("p", p);

lib/tcmd/index.ts

@@ -1,103 +1,286 @@
+"use client";
+
 import { zipArrays } from "../zip";
-import { tokenizeLine } from "./tokenizeLine";
+import { TokenIdentifiers } from "./TokenIdentifiers";
-import { tokenizeBlock } from "./tokenizeBlock";
-import { tokenizeParagraph } from "./tokenizeParagraph";
 
-export const createElements = (body: string) => {
+export const createElements = (body: string): [Token[], number] => {
+  const tabOptions = [
+    /^\s{2}(?!\s|\t)/m,
+    /^\s{4}(?!\s|\t)/m,
+    /^\t(?!\s|\t)]/m,
+  ];
+  let tabSpacing = 0;
+
+  for (const [i, tabOption] of tabOptions.entries()) {
+    if (body.match(tabOption)) {
+      tabSpacing = i;
+      break;
+    }
+  }
   const tokens = tokenize(body);
-
+  return [buildAbstractSyntaxTree(tokens, body), tabSpacing];
-  return tokens;
 };
 
 const tokenize = (body: string) => {
-  body = body.replace(/\n?<!--(.*?)-->\n?/gs, "");
+  const tokenizedBody: tokenMarker[] = [];
 
-  const paragraphs = body.split("\n\n");
+  const addToken = (thing: tokenMarker) => {
-
+    tokenizedBody.push(thing);
-  const blockTokens: BlockToken[] = [];
-  const paragraphTokens: ParagraphToken[] = [];
-
-  for (const paragraph of paragraphs) {
-    const block = tokenizeBlock(paragraph);
-    let openBT = blockTokens.findLast((bt) => !bt.closed);
-    if (block) {
-      if (typeof block === "string") {
-        if (openBT) {
-          openBT.closed = true;
-        }
-        continue;
-      }
-
-      if (openBT) {
-        openBT.children.push(block);
-        block.parent = openBT.type;
-      }
-      blockTokens.push(block);
-      continue;
-    }
-
-    if (!openBT) {
-      openBT = {
-        children: [],
-        closed: false,
-        metadata: {},
-        type: "block",
-        uuid: crypto.randomUUID(),
   };
-      blockTokens.push(openBT);
-    }
 
-    const multiline = tokenizeParagraph(paragraph);
+  for (const [type, token] of TokenIdentifiers.entries()) {
-    let openP = paragraphTokens.findLast((p) => !p.closed);
+    const rx = new RegExp(token.rx);
-    if (multiline) {
+    let match;
-      if (Array.isArray(multiline)) {
+    while ((match = rx.exec(body)) !== null) {
-        if (openP) {
+      const start = match.index;
-          openP.closed = true;
+      const end = rx.lastIndex;
-          openP.content = openP.content.concat(multiline);
-        }
-        continue;
-      }
 
-      openBT.children.push(multiline);
+      if (type !== "p" || !tokenizedBody.find((i) => i.start === start)) {
-      paragraphTokens.push(multiline);
+        addToken({
-      continue;
+          start,
-    } else if (openP && !openP?.allowsInline) {
+          end,
-      openP.content.push({
+          type,
-        line: paragraph,
-        raw: paragraph,
-        type: "text",
-        uuid: crypto.randomUUID(),
         });
       }
+    }
+  }
+  return tokenizedBody;
+};
 
-    // I don't think the closed check is necessary, but just in case
+export const buildAbstractSyntaxTree = (
-    // if (openP && !openP.closed && !openP.allowsInline) continue;
+  markers: tokenMarker[],
-    if (!openP) {
+  body: string,
-      openP = {
+): Token[] => {
-        allowsInline: true,
+  ensureNoOrphans(markers);
-        closed: true,
+
-        content: [],
+  markers.sort((a, b) => {
+    if (a.start === b.start) {
+      console.log(a, b);
+      if (a.type === "p") return -1;
+      if (b.type === "p") return 1;
+    }
+    // if (a.type === "p" && a.start === b.start) return -1;
+    // if (b.type === "p" && a.start === b.start) return 1;
+    return a.start - b.start;
+  });
+
+  for (const marker of markers) {
+    marker.token = TokenIdentifiers.get(marker.type)?.parse(
+      body.substring(marker.start, marker.end),
+    );
+    // if (marker.type === "p" && marker.parent && marker.parent?.type !== "p") {
+    //   marker.parent = undefined;
+    //   continue;
+    // }
+    if (!marker.token) {
+      throw new Error("Failed to parse token. Token type not found?");
+    }
+    if (!marker.parent) continue;
+
+    if (!marker.parent.token) {
+      // debugger;
+      throw new Error("Failed to parse token. Child tokenized before parent");
+    }
+
+    marker.parent.token.children = marker.parent.token.children || [];
+    marker.parent.token.children.push(marker.token);
+    // marker.token.parent = marker.parent.token;
+  }
+
+  const tokens = markers.filter((m) =>
+    markers.filter((a) => a !== m && (a.end === m.end || a.start === m.start))
+      .length || m.type !== "p"
+  ).map((t) => t.token!);
+
+  for (const token of tokens) {
+    contentToChildren(token);
+  }
+
+  return tokens.filter((t) => !t.parent);
+};
+
+const ensureNoOrphansOld = (tokens: tokenMarker[]) => {
+  for (const token of tokens) {
+    const parentPs = tokens.filter((t) => (
+      t.type === "p" && (
+        // any p that fully encapsulates the token
+        (t.start <= token.start && t.end >= token.end) ||
+        // any p that contains the start of the token
+        (t.start <= token.start && t.end >= token.start) ||
+        // any p that contains the end of the token
+        (t.start <= token.end && t.end >= token.end)
+      )
+    )).sort((a, b) => (a.start - b.start));
+
+    if (parentPs.length > 1) {
+      parentPs[0].end = parentPs.at(-1)!.end;
+      const remainingParents = parentPs.slice(1);
+      for (const token of tokens) {
+        if (token.parent && remainingParents.includes(token.parent)) {
+          token.parent = parentPs[0];
+        }
+      }
+      if (parentPs[0] && parentPs[0].end < token.end) {
+        parentPs[0].end = token.end;
+      }
+      tokens = tokens.filter((t) => !remainingParents.includes(t));
+    }
+
+    const potentialParents = tokens.filter((t) =>
+      (t.start < token.start && t.end > token.end) ||
+      (t.type === "p" && t.start <= token.start &&
+        t.end >= token.end && t !== token)
+    ).sort((a, b) => {
+      if (token.start - a.start < token.start - b.start) return -1;
+      return 1;
+    });
+
+    token.parent = potentialParents.find((p) => p.type !== "p") ??
+      potentialParents[0];
+
+    if (token.type === "grid") {
+      debugger;
+    }
+  }
+};
+
+const ensureNoOrphans = (tokens: tokenMarker[]) => {
+  ensureNoOrphansOld(tokens);
+};
+
+const contentToChildren = (token: Token) => {
+  const children: Token[] = [];
+  let part, content = token.content;
+
+  // for (const child of token.children || []) {
+  //   if (!content) continue;
+  //   [part, content] = content.split(child.raw);
+  //   part && children.push({
+  //     content: part.trim(),
+  //     metadata: {},
+  //     raw: part,
+  //     type: "text",
+  //     uuid: crypto.randomUUID(),
+  //   });
+  //   children.push(child);
+  // }
+
+  // if (content) {
+  //   children.push({
+  //     content: content.trim(),
+  //     metadata: {},
+  //     raw: content,
+  //     type: "text",
+  //     uuid: crypto.randomUUID(),
+  //   });
+  // }
+  const splitMarker = "{{^^}}";
+  for (const child of token.children || []) {
+    content = content.replace(child.raw, splitMarker);
+  }
+
+  token.children = zipArrays(
+    content.split(splitMarker).map((c): Token => ({
+      content: c.trim(),
       metadata: {},
-        type: "p",
+      raw: c,
+      type: "text",
       uuid: crypto.randomUUID(),
+      rendersContentOnly: true,
+    })),
+    token.children || [],
+  ).filter((c) => c.children?.length || (c.rendersContentOnly && c.content));
 };
-      openBT.children.push(openP);
-      paragraphTokens.push(openP);
-    }
 
-    const lines = paragraph.split("\n");
+// const tokenize = (body: string) => {
-    let previous;
+//   body = body.replace(/\n?<!--(.*?)-->\n?/gs, "");
-    for (const line of lines) {
-      const singleLine = tokenizeLine(line, previous);
 
-      if (singleLine) {
+//   const paragraphs = body.split("\n\n");
-        if (singleLine !== previous) {
-          openP.content.push(singleLine);
-        }
-        previous = singleLine;
-      }
-    }
-  }
 
-  return blockTokens.filter((b) => !b.parent);
+//   const blockTokens: BlockToken[] = [];
-};
+//   const paragraphTokens: ParagraphToken[] = [];
+
+//   for (const paragraph of paragraphs) {
+//     const block = tokenizeBlock(paragraph);
+//     let openBT = blockTokens.findLast((bt) => !bt.closed);
+//     if (block) {
+//       if (typeof block === "string") {
+//         if (openBT) {
+//           openBT.closed = true;
+//         }
+//         continue;
+//       }
+
+//       if (openBT) {
+//         openBT.children.push(block);
+//         block.parent = openBT.type;
+//       }
+//       blockTokens.push(block);
+//       continue;
+//     }
+
+//     if (!openBT) {
+//       openBT = {
+//         children: [],
+//         closed: false,
+//         metadata: {},
+//         type: "block",
+//         uuid: crypto.randomUUID(),
+//       };
+//       blockTokens.push(openBT);
+//     }
+
+//     const multiline = tokenizeParagraph(paragraph);
+//     let openP = paragraphTokens.findLast((p) => !p.closed);
+//     if (multiline) {
+//       if (Array.isArray(multiline)) {
+//         if (openP) {
+//           openP.closed = true;
+//           openP.content = openP.content.concat(multiline);
+//         }
+//         continue;
+//       }
+
+//       openBT.children.push(multiline);
+//       paragraphTokens.push(multiline);
+//       continue;
+//     } else if (openP && !openP?.allowsInline) {
+//       openP.content.push({
+//         line: paragraph,
+//         raw: paragraph,
+//         type: "text",
+//         uuid: crypto.randomUUID(),
+//       });
+//     }
+
+//     // I don't think the closed check is necessary, but just in case
+//     // if (openP && !openP.closed && !openP.allowsInline) continue;
+//     if (!openP) {
+//       openP = {
+//         allowsInline: true,
+//         closed: true,
+//         content: [],
+//         metadata: {},
+//         type: "p",
+//         uuid: crypto.randomUUID(),
+//       };
+//       openBT.children.push(openP);
+//       paragraphTokens.push(openP);
+//     }
+
+//     const lines = paragraph.split("\n");
+//     let previous;
+//     for (const line of lines) {
+//       const singleLine = tokenizeLine(line, previous);
+
+//       if (singleLine) {
+//         if (singleLine !== previous) {
+//           openP.content.push(singleLine);
+//         }
+//         previous = singleLine;
+//       }
+//     }
+//   }
+
+//   return blockTokens.filter((b) => !b.parent);
+// };

lib/tcmd/tokenizeBlock.ts

@@ -44,8 +44,8 @@ const blockTokens: {
     },
   },
   {
-    rx: /\[accordion\s?([a-z\s]*)\]/,
+    rx: /^\[accordion\s?([a-z\s]*)\]/i,
-    closeRx: /\[\/accordion\]/,
+    closeRx: /^\[\/accordion\]/,
     create(line) {
       const title = line.match(this.rx)?.at(1);
       return {

lib/tcmd/tokenizeInline.ts

@@ -61,6 +61,21 @@ export const inlineTokens: {
   ) => void;
   replace: (line: string) => string;
 }[] = [
+  {
+    rx: /\s?`(.*?)`[^a-z0-9`]\s?/gi,
+    create(content, start, end, tokens) {
+      tokens.push({
+        content: this.replace(content[0]),
+        type: "inline-code",
+        end,
+        start,
+        uuid: crypto.randomUUID(),
+      });
+    },
+    replace(l) {
+      return l.replace(this.rx, (...all) => all[1]);
+    },
+  },
   {
     rx: /(\*\*)(.*?)(\*\*)/g,
     create(content, start, end, tokens) {

lib/tcmd/tokenizeParagraph.ts

@@ -1,37 +1,37 @@
 export const tokenizeParagraph = (paragraph: string) => {
-  for (const block of blockTokens) {
+  for (const pgraph of paragraphTokens) {
-    const openTest = block.rx.test(paragraph),
+    const openTest = pgraph.rx.test(paragraph),
-      closeTest = block.closeRx.test(paragraph);
+      closeTest = pgraph.closeRx.test(paragraph);
     if (openTest && closeTest) {
-      const p = block.create(paragraph);
+      const p = pgraph.create(paragraph);
       p.closed = true;
       return p;
     }
-    if (closeTest) return block.create(paragraph).content;
+    if (closeTest) return pgraph.create(paragraph).content;
 
     if (openTest) {
-      return block.create(paragraph);
+      return pgraph.create(paragraph);
     }
   }
 };
 
-const blockTokens: {
+const paragraphTokens: {
   rx: RegExp;
   closeRx: RegExp;
   create: (line: string) => ParagraphToken;
 }[] = [
   {
-    rx: /^```/g,
+    rx: /\n```/g,
     closeRx: /\n```/g,
     create(line) {
       return {
         type: "code",
         metadata: {
-          language: line.split("\n").at(0)!.replace(this.rx, ""),
+          // language: line.split("\n").at(0)!.replace(this.rx, ""),
         },
         closed: false,
         content: [{
-          line: line.replace(/```.*?\n/g, "").replace(/\n```/, ""),
+          line: line.match(/```(.*?)\n```/g)?.at(1) || line,
           type: "text",
           raw: line,
           uuid: crypto.randomUUID(),

md/help articles/How to use ttcMD.md

@@ -1 +1,87 @@
-# Henlo, am help
+# Table of Contents
+- [Table of Contents](#table-of-contents)
+- [How do you use ttcMD?](#how-do-you-use-ttcmd)
+  - [Enhanced Standard Elements](#enhanced-standard-elements)
+    - [Links](#links)
+  - [Custom Elements](#custom-elements)
+    - [Pop-outs](#pop-outs)
+  - [Block-level Elements](#block-level-elements)
+    - [Accordions](#accordions)
+
+# How do you use ttcMD?
+
+ttcMD is a flavor of markdown that has been specifically designed to use with [ttcQuery](/help/ttcQuery.md). It has all of the basic syntax of [markdown](https://www.markdownguide.org/cheat-sheet/), but also includes Tables, basic Fenced Code Blocks and a slew of custom elements and styling annotations.
+
+## Enhanced Standard Elements
+
+This section will cover all of the enhancements that are added for basic markdown elements
+
+### Links
+
+You can use the typical link syntax: `[link name](/link/location)`, but there are a few presets that allow you to style them to look a bit nicer.
+
+**Primary Button:**
+Prefix the link name with ````button` to create a button.
+`[```button link name](#links)` produces:
+
+[```button link name](#links)
+
+**Call to Action:**
+Prefix the link name with ````cta` to create a modestly styled button/call to action.
+`[```cta link name](#links)` produces:
+
+[```cta link name](#links)
+
+## Custom Elements
+
+This section will cover the specific elements custom built for Tabletop Commander.
+
+### Pop-outs
+
+Pop-outs, or popovers, are the little cards that "pop out" when you hover over an item.
+
+The syntax is thus: `^[pop-out title]<<pop-out content>>`. The pop-out title will be rendered inline, just like a link, and the content will be included in the pop-out itself. Content can also include inline markdown elements as well, so you can format the content within as well.
+
+Example:
+
+This syntax `^[goofy!]<<This is my *favorite* picture: ![goofy](https://yt3.ggpht.com/a/AATXAJwbIW0TwEhqdT2ZPeSB1AtdtWD2ZXam80oijg=s900-c-k-c0xffffffff-no-rj-mo)>>` will produce this element: ^[goofy!]<<This is my *favorite* picture: ![goofy](https://yt3.ggpht.com/a/AATXAJwbIW0TwEhqdT2ZPeSB1AtdtWD2ZXam80oijg=s900-c-k-c0xffffffff-no-rj-mo)>>
+
+Note: currently, only inline elements are available, so formatting is limited
+
+## Block-level Elements
+
+Block-level elements have a slightly different syntax than the single-line and inline elements we've seen so far. In order to use block-level elements, they *must* be formatted correctly, including the empty lines. As a general rule, you cannot nest block-level elements within themselves, but you can nest different block-level elements within it.
+
+### Accordions
+
+Accordions are when you can click an item to expand it to show additional information.
+
+Syntax: 
+
+[][][]
+
+```
+[accordion title]
+
+whatever markdown you desire, including non-accordion block elements
+
+[/accordion]
+```
+
+[accordion this is what an accordion looks like]
+
+This is the body of the accordion.
+
+As you can see, I can do normal markdown in here.
+
+I can include a [link](#accordions), or *italic* and **bold** text.
+
+[[
+
+I can even include a card, like this one
+
+]]
+
+[/accordion]
+
+/[]

md/home.md

@@ -74,13 +74,13 @@ parts of the publication through context based pop-overs.
 
 **For the techies (again):**
 
-Publications use an enhanced markdown syntax (tcMD) that
+Publications use an enhanced markdown syntax (ttcMD) that
 implements tcQuery, and adds a bit of custom syntax for things
 like pop-overs and styling hints for rendering.
 
 The styling aspect is similar to a very trimmed down CSS, but
 can accomplish quite a lot. For example, this page is actually
-built using tcMD!
+built using ttcMD!
 
 [```cta Learn More](/help/Publications.md)
 

package.json

@@ -11,6 +11,7 @@
   "dependencies": {
     "@heroicons/react": "^2.1.1",
     "isomorphic-dompurify": "^2.4.0",
+    "marked": "^12.0.1",
     "next": "14.1.0",
     "react": "^18",
     "react-dom": "^18"

tsconfig.json

@@ -1,6 +1,10 @@
 {
   "compilerOptions": {
-    "lib": ["dom", "dom.iterable", "esnext"],
+    "lib": [
+      "dom",
+      "dom.iterable",
+      "esnext"
+    ],
     "allowJs": true,
     "skipLibCheck": true,
     "strict": true,
@@ -18,9 +22,19 @@
       }
     ],
     "paths": {
-      "@/*": ["./*"]
+      "@/*": [
-    }
+        "./*"
+      ]
     },
-  "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
+    "target": "es2022"
-  "exclude": ["node_modules"]
+  },
+  "include": [
+    "next-env.d.ts",
+    "**/*.ts",
+    "**/*.tsx",
+    ".next/types/**/*.ts"
+  ],
+  "exclude": [
+    "node_modules"
+  ]
 }

types.d.ts

@@ -1,5 +1,12 @@
 type InlineToken = {
-  type: "text" | "bold" | "anchor" | "image" | "popover" | "italic";
+  type:
+    | "text"
+    | "bold"
+    | "anchor"
+    | "image"
+    | "popover"
+    | "italic"
+    | "inline-code";
   content: string;
   data?: any;
   uuid: string;
@@ -32,7 +39,25 @@ type SingleLineToken = {
   cfg?: SingleLineCfg;
   uuid: string;
 };
-type Token = SingleLineToken | MultilineToken;
+type Token = {
+  type: string;
+  metadata: Record<string, string>;
+  parent?: Token;
+  children?: Token[];
+  uuid: string;
+  raw: string;
+  content: string;
+  rendersChildrenOnly?: boolean;
+  rendersContentOnly?: boolean;
+};
+
+type tokenMarker = {
+  start: number;
+  end: number;
+  type: string;
+  parent?: tokenMarker;
+  token?: Token;
+};
 
 type MultilineCfg = {
   rx: RegExp;