Incredible majestic beautiful md parser

This commit is contained in:
Emmaline Autumn 2024-02-28 01:08:34 -07:00
parent e1ed37d733
commit 6ef8c20149
12 changed files with 674 additions and 128 deletions

View File

@ -1,4 +1,5 @@
This is a [Next.js](https://nextjs.org/) project bootstrapped with [`create-next-app`](https://github.com/vercel/next.js/tree/canary/packages/create-next-app).
This is a [Next.js](https://nextjs.org/) project bootstrapped with
[`create-next-app`](https://github.com/vercel/next.js/tree/canary/packages/create-next-app).
## Getting Started
@ -14,23 +15,34 @@ pnpm dev
bun dev
```
Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.
Open [http://localhost:3000](http://localhost:3000) with your browser to see the
result.
You can start editing the page by modifying `app/page.tsx`. The page auto-updates as you edit the file.
You can start editing the page by modifying `app/page.tsx`. The page
auto-updates as you edit the file.
This project uses [`next/font`](https://nextjs.org/docs/basic-features/font-optimization) to automatically optimize and load Inter, a custom Google Font.
This project uses
[`next/font`](https://nextjs.org/docs/basic-features/font-optimization) to
automatically optimize and load Inter, a custom Google Font.
## Learn More
To learn more about Next.js, take a look at the following resources:
- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API.
- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js
features and API.
- [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial.
You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js/) - your feedback and contributions are welcome!
You can check out
[the Next.js GitHub repository](https://github.com/vercel/next.js/) - your
feedback and contributions are welcome!
## Deploy on Vercel
The easiest way to deploy your Next.js app is to use the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme) from the creators of Next.js.
The easiest way to deploy your Next.js app is to use the
[Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme)
from the creators of Next.js.
Check out our [Next.js deployment documentation](https://nextjs.org/docs/deployment) for more details.
Check out our
[Next.js deployment documentation](https://nextjs.org/docs/deployment) for more
details.

View File

@ -11,8 +11,8 @@ import {
const inter = Inter({ subsets: ["latin"] });
export const metadata: Metadata = {
title: "Create Next App",
description: "Generated by create next app",
title: "Tabletop Commander",
description: "Rules and tools for tabletop games!",
};
export default function RootLayout({

View File

@ -4,7 +4,10 @@ import {
CircleStackIcon,
PuzzlePieceIcon,
} from "@heroicons/react/24/solid";
import { readFileSync } from "fs";
import { readFile } from "fs/promises";
import Image from "next/image";
import { Suspense } from "react";
export default function Home() {
return (
@ -133,17 +136,14 @@ export default function Home() {
turn the home page into magic
</cite>
</section>
<section>
<Suspense>
<TCMD
body={`
# spicy sandwiches
## taste good
### I like them
**A lot**
`}
body={(async () => {
"use server";
return await readFile("./test.md", "utf-8");
})()}
/>
</section>
</Suspense>
</>
);
}

View File

@ -1,123 +1,135 @@
import { zipArrays } from "@/lib/zip";
import { FC, PropsWithChildren, useMemo } from "react";
"use client";
export const TCMD: FC<{ body: string }> = ({ body }) => {
const elements = useMemo(() => createElements(body), [body]);
import { createElements } from "@/lib/tcmd";
import Link from "next/link";
import React, { FC, ReactNode, use, useMemo } from "react";
export const TCMD: FC<{ body: Promise<string> }> = ({ body }) => {
const text = use(body);
const elements = useMemo(() => createElements(text), [text]);
return (
<>
<pre>{JSON.stringify(elements,null,2)}</pre>
</>
// <div className="grid grid-cols-2">
// <pre>{JSON.stringify(elements,null,2)}</pre>
// </div>
<div>
{elements.map(renderBlock)}
</div>
);
};
const createElements = (body: string) => {
const tokens = tokenize(body);
return tokens;
};
type InlineToken = {
type: "text" | "bold";
content: string;
};
type Line = string | InlineToken[];
type MultilineToken = {
type: "code";
lines: Token[];
};
type Token = {
type: "h1" | "h2" | "h3" | "p";
line: Line;
};
const tokenize = (md: string) => {
const tokens: (Token | MultilineToken)[] = [];
md = md.replace(/(?<=[a-z])\n(?=[a-z])/g, " ");
const lines = md.split("\n");
const multilineFlags = {
heading: 0,
};
const tokenMatches = [
{
rx: /^\s*#\s/,
create: (line: Line) => tokens.push({ type: "h1", line }),
},
{
rx: /^\s*##\s/,
create: (line: Line) => tokens.push({ type: "h2", line }),
},
{
rx: /^\s*###\s/,
create: (line: Line) => tokens.push({ type: "h3", line }),
},
];
for (let line of lines) {
let foundLine = false;
token:
for (const token of tokenMatches) {
if (!token.rx.test(line)) continue token;
foundLine = true;
line = line.replace(token.rx, "").trim();
const lineContent = tokenizeInline(line);
token.create(lineContent);
const renderBlock = (block: BlockChildren): ReactNode => {
switch (block.type) {
case "block":
return block.children.map(renderBlock);
case "grid":
return (
<div
style={{
"--grid-cols": block.metadata.columns,
} as React.CSSProperties}
className="grid grid-cols-dynamic gap-x-8 gap-y-6"
>
{block.children.map((c, i) => (
<div key={"block-grid" + c.type + i}>{renderBlock(c)}</div>
))}
</div>
);
case "card":
return <div className="card">{block.children.map(renderBlock)}</div>;
default:
return (
renderParagraph(block as ParagraphToken)
);
}
};
if (foundLine) continue;
const renderParagraph = (p: ParagraphToken) => {
switch (p.type) {
case "p":
return <p>{p.content.map(renderToken)}</p>;
case "code":
return (
<pre className="whitespace-pre-wrap">
{p.content.map((c) => c.line.toString()).join("\n\n")}
</pre>
);
default:
return (
<p className="bg-red-600 text-white">
Block or paragraph missing implementation: {p.type}
</p>
);
}
};
tokens.push({
type: "p",
line: tokenizeInline(line),
const renderToken = (t: Token) => {
switch (t.type) {
case "h1":
return (
<p className="font-bold text-2xl">
{renderInlineToken(t.line)}
</p>
);
case "h2":
return (
<p className="font-bold text-xl">
{renderInlineToken(t.line)}
</p>
);
case "h3":
return (
<p className="font-bold text-lg">
{renderInlineToken(t.line)}
</p>
);
case "p":
return (
<p>
{t.lines.map(renderToken)}
</p>
);
case "text":
return (
<>
{renderInlineToken(t.line)}
&nbsp;
</>
);
case "list1":
return <li className="list-disc ml-6">{renderInlineToken(t.line)}</li>;
case "list2":
return <li className="list-disc ml-8">{renderInlineToken(t.line)}</li>;
default:
return (
<p className="text-white bg-red-500">
Missing implementation for tcMD element `{(t as { type: string })
.type}`
</p>
);
}
};
const renderInlineToken = (l: Line) => {
if (typeof l === "string") return l;
return l.map((token) => {
switch (token.type) {
case "text":
return <span>{token.content}</span>;
case "bold":
return <span className="font-bold">{token.content}</span>;
case "anchor":
return (
<Link className="text-primary-600" href={token.data.href}>
{token.content}
</Link>
);
default:
return (
<span className="bg-red-500">
Inline element not implemented: {token.type}
</span>
);
}
});
}
console.log(tokens);
return tokens.filter((t) => (t as Token).line || (t as MultilineToken).lines);
};
const tokenizeInline = (line: string) => {
line = line.trim();
const originalLine = line;
const insertMarker = "{^}";
const tokens: InlineToken[] = [];
const tokenMatches = [
{
rx: /\*\*(.*?)\*\*/g,
create: (content: string) =>
tokens.push({
content,
type: "bold",
}),
},
];
for (const token of tokenMatches) {
let match;
let last = 0;
while ((match = token.rx.exec(line)) !== null) {
const tokenStart = match.index;
const tokenEnd = match.index + match[0].length;
console.log(tokenEnd, token.rx.lastIndex);
token.create(line.substring(tokenStart, tokenEnd));
line = line.slice(last, tokenStart) + "{^}" +
line.slice(tokenEnd, line.length);
last = tokenEnd;
}
}
if (tokens.length) {
return zipArrays(
line.split(insertMarker).map((t): InlineToken => ({
content: t,
type: "text",
})),
tokens,
).filter((t) => t.content);
}
return originalLine;
};

226
lib/tcmd/index.ts Normal file
View File

@ -0,0 +1,226 @@
import { zipArrays } from "../zip";
import { inlineTokens } from "./inlineTokens";
import { singleLineTokens } from "./singleLineTokens";
import { tokenizeBlock } from "./tokenizeBlock";
import { tokenizeParagraph } from "./tokenizeParagraph";
export const createElements = (body: string) => {
const tokens = tokenize(body);
return tokens;
};
const tokenize = (body: string) => {
const paragraphs = body.split("\n\n");
const blockTokens: BlockToken[] = [];
const paragraphTokens: ParagraphToken[] = [];
for (const paragraph of paragraphs) {
const block = tokenizeBlock(paragraph);
let openBT = blockTokens.findLast((bt) => !bt.closed);
if (block) {
if (typeof block === "string") {
console.log(block);
if (openBT) {
openBT.closed = true;
}
continue;
}
if (openBT) {
openBT.children.push(block);
block.parent = openBT.type;
}
blockTokens.push(block);
continue;
}
if (!openBT) {
openBT = {
children: [],
closed: false,
metadata: {},
type: "block",
};
blockTokens.push(openBT);
}
const multiline = tokenizeParagraph(paragraph);
let openP = paragraphTokens.findLast((p) => !p.closed);
if (multiline) {
if (Array.isArray(multiline)) {
if (openP) {
openP.closed = true;
openP.content = openP.content.concat(multiline);
}
continue;
}
openBT.children.push(multiline);
paragraphTokens.push(multiline);
continue;
} else if (openP && !openP?.allowsInline) {
openP.content.push({
line: paragraph,
raw: paragraph,
type: "text",
});
}
// I don't think the closed check is necessary, but just in case
// if (openP && !openP.closed && !openP.allowsInline) continue;
if (!openP) {
openP = {
allowsInline: true,
closed: true,
content: [],
metadata: {},
type: "p",
};
openBT.children.push(openP);
paragraphTokens.push(openP);
}
const lines = paragraph.split("\n");
let previous;
for (const line of lines) {
const singleLine = tokenizeLine(line, previous);
if (singleLine) {
if (singleLine !== previous) {
openP.content.push(singleLine);
}
previous = singleLine;
}
}
}
return blockTokens.filter((b) => !b.parent);
};
// const __tokenize = (md: string) => {
// const tokens: (Token)[] = [];
// // md = md.replace(/(?<=[a-z])\n(?=[a-z])/g, " ");
// const lines = md.split("\n");
// let preserveEmpty = false;
// let multilineLines;
// let tokenSettings;
// for (let line of lines) {
// if (!line && !preserveEmpty) continue;
// let foundLine = false;
// if (!multilineLines) {
// token:
// for (const token of multilineTokens) {
// if (!token.rx.test(line)) continue token;
// tokenSettings = token;
// multilineLines = token.create(tokens);
// preserveEmpty = true;
// foundLine = true;
// multilineLines.push({
// type: "text",
// line: token.replace(line),
// });
// }
// } else {
// foundLine = true;
// if (tokenSettings?.closeRx?.test(line) || tokenSettings?.rx.test(line)) {
// tokenSettings = undefined;
// multilineLines = undefined;
// preserveEmpty = false;
// } else {
// multilineLines.push({
// type: "text",
// line,
// });
// }
// }
// if (!multilineLines) {
// token:
// for (const token of singleLineTokens) {
// if (!token.rx.test(line)) continue token;
// foundLine = true;
// line = line.replace(token.replaceRx, "").trim();
// const lineContent = tokenizeInline(line);
// token.create(lineContent, tokens);
// }
// }
// if (foundLine) continue;
// tokens.push({
// type: "text",
// line: tokenizeInline(line),
// });
// }
// return tokens;
// };
const tokenizeLine = (
line: string,
previous?: SingleLineToken,
): SingleLineToken => {
for (const token of singleLineTokens) {
if (!token.rx.test(line)) continue;
const t = token.create(line);
if (t.type === "h2") {
}
t.line = tokenizeInline(line.replace(token.replaceRx, ""));
return t;
}
if (previous?.mends) {
previous.raw += " " + line;
previous.line = tokenizeInline(previous.raw.replace(previous.cfg!.rx, ""));
return previous;
}
return {
line: tokenizeInline(line),
type: "text",
raw: line,
};
};
const tokenizeInline = (line: string) => {
line = line.trim();
const originalLine = line;
const insertMarker = "\u{03A9}";
const tokens: InlineTokenInsert[] = [];
for (const token of inlineTokens) {
token.rx.lastIndex = 0;
let match;
while ((match = token.rx.exec(line)) !== null) {
const tokenStart = match.index;
const tokenEnd = match.index + match[0].length;
token.create(match, tokenStart, tokenEnd, tokens);
}
}
if (tokens.length) {
for (const insert of tokens) {
line = line.slice(0, insert.start) +
"".padStart(insert.end - insert.start, insertMarker) +
line.slice(insert.end, line.length);
}
return zipArrays(
line.split(new RegExp(insertMarker + "{2,}")).map((t): InlineToken => ({
content: t,
type: "text",
})),
tokens,
).filter((t) => t.content);
}
return originalLine;
};

45
lib/tcmd/inlineTokens.ts Normal file
View File

@ -0,0 +1,45 @@
const joiner = "<><>";
export const inlineTokens: {
rx: RegExp;
create: (
content: RegExpExecArray,
start: number,
end: number,
tokens: InlineTokenInsert[],
) => void;
replace: (line: string) => string;
}[] = [
{
rx: /(\*\*)(.*?)(\*\*)/g,
create(content, start, end, tokens) {
tokens.push({
content: this.replace(content[0]),
type: "bold",
end,
start,
});
},
replace(l) {
return l.replace(this.rx, (_, __, val) => val);
},
},
{
rx: /\[(.*?)\]\((.*?)\)/g,
create(content, start, end, tokens) {
const [_, label, href] = content;
tokens.push({
content: label,
type: "anchor",
data: {
href,
},
start,
end,
});
},
replace(l) {
return l.replace(this.rx, (_, label, href) => [label, href].join(joiner));
// return l
},
},
];

View File

@ -0,0 +1,39 @@
export const singleLineTokens: SingleLineCfg[] = [
{
rx: /^#\s/,
create(line) {
return ({ type: "h1", line, raw: line, cfg: this });
},
replaceRx: /^#\s/,
},
{
rx: /^##\s/,
create(line) {
return ({ type: "h2", line, raw: line, cfg: this });
},
replaceRx: /^##\s/,
},
{
rx: /^###\s/,
create(line) {
return ({ type: "h3", line, raw: line, cfg: this });
},
replaceRx: /^###\s/,
},
{
rx: /^-\s/,
create(line) {
return ({ type: "list1", line, raw: line, mends: true, cfg: this });
},
replaceRx: /^-\s/,
shouldMendNextLine: true,
},
{
rx: /^[\t\s]{2}-\s/,
create(line) {
return ({ type: "list2", line, raw: line, mends: true, cfg: this });
},
replaceRx: /^[\t\s]{2}-\s/,
shouldMendNextLine: true,
},
];

44
lib/tcmd/tokenizeBlock.ts Normal file
View File

@ -0,0 +1,44 @@
export const tokenizeBlock = (paragraph: string) => {
for (const block of blockTokens) {
const openTest = block.rx.test(paragraph),
closeTest = block.closeRx.test(paragraph);
if (closeTest) return block.create(paragraph).type;
if (!openTest) continue;
return block.create(paragraph);
}
};
const blockTokens: {
rx: RegExp;
closeRx: RegExp;
create: (line: string) => BlockToken;
}[] = [
// this indicates that this is a grid block, all paragraphs within this block will be placed in a number of columns that match the number of sets of brackets are in this line
{
rx: /^(\[\]){2,}/g,
closeRx: /\/\[\]/,
create(line) {
return {
type: "grid",
metadata: {
columns: line.match(/\[\]/g)?.length,
},
children: [],
closed: false,
};
},
},
{
rx: /^(\[\[)/,
closeRx: /\]\]/,
create() {
return {
type: "card",
metadata: {},
children: [],
closed: false,
};
},
},
];

View File

@ -0,0 +1,42 @@
export const tokenizeParagraph = (paragraph: string) => {
for (const block of blockTokens) {
const openTest = block.rx.test(paragraph),
closeTest = block.closeRx.test(paragraph);
if (openTest && closeTest) {
const p = block.create(paragraph);
p.closed = true;
return p;
}
if (closeTest) return block.create(paragraph).content;
if (openTest) {
return block.create(paragraph);
}
}
};
const blockTokens: {
rx: RegExp;
closeRx: RegExp;
create: (line: string) => ParagraphToken;
}[] = [
{
rx: /^```/g,
closeRx: /\n```/g,
create(line) {
return {
type: "code",
metadata: {
language: line.split("\n").at(0)!.replace(this.rx, ""),
},
closed: false,
content: [{
line: line.replace(/```.*?\n/g, "").replace(/\n```/, ""),
type: "text",
raw: line,
}],
allowsInline: false,
};
},
},
];

View File

@ -39,6 +39,9 @@ const config: Config = {
"600": "#918b93",
},
},
gridTemplateColumns: {
dynamic: "repeat(var(--grid-cols), minmax(0,1fr))",
},
},
},
plugins: [],

63
test.md Normal file
View File

@ -0,0 +1,63 @@
# Hello! Welcome to Tabletop Commander!
[][][]
[[
Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor
incididunt ut labore et dolore magna aliqua. Sollicitudin tempor id eu nisl nunc
mi ipsum faucibus vitae. Lobortis elementum nibh tellus molestie nunc. Purus non
enim praesent elementum facilisis leo vel. Orci nulla pellentesque dignissim
enim sit amet venenatis. Eu feugiat pretium nibh ipsum. Gravida dictum fusce ut
placerat orci nulla pellentesque. Tincidunt vitae semper quis lectus nulla at
volutpat diam ut. Proin sed libero enim sed faucibus turpis in eu mi. Dui sapien
eget mi proin sed libero enim sed faucibus. Felis donec et odio pellentesque
diam volutpat commodo sed egestas. Massa tincidunt dui ut ornare lectus sit amet
est placerat. Auctor urna nunc id cursus metus aliquam eleifend.
- Lorem ipsum dolor sit amet,
- consectetur adipiscing elit, bananana banana ban anana anaba bananananana
bananna anbnao
- sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
- Sollicitudin tempor id eu nisl nunc mi ipsum faucibus vitae. Lobortis
elementum nibh tellus molestie nunc. Purus non enim praesent elementum
facilisis leo vel. Orci nulla pellentesque dignissim enim sit amet
venenatis. Eu feugiat pretium nibh ipsum. Gravida dictum fusce ut placerat
orci nulla pellentesque. Tincidunt vitae semper quis lectus nulla at
volutpat diam ut. Proin sed libero enim sed faucibus turpis in eu mi. Dui
sapien eget mi proin sed libero enim sed faucibus. Felis donec et odio
pellentesque diam volutpat commodo sed egestas. Massa tincidunt dui ut
ornare lectus sit amet est placerat. Auctor urna nunc id cursus metus
aliquam eleifend.
]]
[[
```
const blockTokens: {
rx: RegExp;
closeRx: RegExp;
create: (line: string) => BlockToken;
}[] = [
// this indicates that this is a grid block, all paragraphs within this block will be placed in a number of columns that match the number of sets of brackets are in this line
{
rx: /^(\[\]){2,}/g,
closeRx: /\/\[\]/,
create(line) {
return {
type: "grid",
metadata: {
columns: line.match(/\[\]/g)?.length,
},
children: [],
closed: false,
};
},
},
];
```
]]
/[]

60
types.d.ts vendored Normal file
View File

@ -0,0 +1,60 @@
type InlineToken = {
type: "text" | "bold" | "anchor";
content: string;
data?: any;
};
type InlineTokenInsert = {
start: number;
end: number;
} & InlineToken;
type Line = string | InlineToken[];
type MultilineToken = {
type: "code" | "p";
lines: SingleLineToken[];
};
type SingleLineCfg = {
rx: RegExp;
create: (line: string) => SingleLineToken;
replaceRx: RegExp;
shouldMendNextLine?: boolean;
};
type SingleLineToken = {
type: "h1" | "h2" | "h3" | "text" | `list${number}`;
line: Line;
raw: string;
mends?: boolean;
cfg?: SingleLineCfg;
};
type Token = SingleLineToken | MultilineToken;
type MultilineCfg = {
rx: RegExp;
closeRx?: RegExp;
create: (tokens: Token[]) => SingleLineToken[];
replace: (line: string) => string;
};
// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
type BlockToken = {
type: "block" | "grid" | "card";
metadata: any;
children: BlockChildren[];
parent?: string;
closed: boolean;
};
type BlockChildren = ParagraphToken | BlockToken | SingleLineToken;
type ParagraphToken = {
content: SingleLineToken[];
allowsInline: boolean;
type: "p" | "code";
metadata: any;
closed: boolean;
};