emma/tabletop-commander
1
0
Fork 0
Code Issues 2 Pull Requests Packages Projects 3 Releases Wiki Activity

tcmd: Fixes regex reuse in inline tokenizer,

Browse Source 
tcmd: changes popover syntax to allow for embeddable markdown
This commit is contained in:
Emmaline Autumn 2024-02-29 00:02:04 -07:00
parent ff0a4280e2
commit ce83bdf7af
7 changed files with 147 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -34,3 +34,6 @@ yarn-error.log*
# typescript # typescript
*.tsbuildinfo *.tsbuildinfo
next-env.d.ts next-env.d.ts
# vscode
.vscode

45
app/layout.tsx
View File

@ -7,6 +7,7 @@ import {
Cog8ToothIcon, Cog8ToothIcon,
PuzzlePieceIcon, PuzzlePieceIcon,
} from "@heroicons/react/24/solid"; } from "@heroicons/react/24/solid";
import Link from "next/link";
const inter = Inter({ subsets: ["latin"] }); const inter = Inter({ subsets: ["latin"] });
@ -20,6 +21,29 @@ export default function RootLayout({
}: Readonly<{ }: Readonly<{
children: React.ReactNode; children: React.ReactNode;
}>) { }>) {
const navItems = [
{
to: "/schemas",
icon: CircleStackIcon,
text: "Schemas",
},
{
to: "/game-systems",
icon: PuzzlePieceIcon,
text: "Game Systems",
},
{
to: "/publications",
icon: BookOpenIcon,
text: "Publications",
},
{
to: "/settings",
icon: Cog8ToothIcon,
text: "Settings",
},
];
return ( return (
<html lang="en"> <html lang="en">
<body className={inter.className + " flex min-h-[100vh]"}> <body className={inter.className + " flex min-h-[100vh]"}>
@ -28,18 +52,17 @@ export default function RootLayout({
Tabletop Commander Tabletop Commander
</h1> </h1>
<ul className="my-6 flex flex-col gap-6"> <ul className="my-6 flex flex-col gap-6">
<li className="flex items-center gap-2"> {navItems.map((n) => (
<CircleStackIcon className="w-6 h-6" />Schemas <li key={"nav-item" + n.text}>
</li> <Link
<li className="flex items-center gap-2"> href={n.to}
<PuzzlePieceIcon className="w-6 h-6" />Game Systems className="flex items-center gap-2 group hover:text-purple-300 transition-colors"
</li> >
<li className="flex items-center gap-2"> <n.icon className="w-6 h-6 group-hover:fill-purple-300 transition-colors" />
<BookOpenIcon className="w-6 h-6" />Publications {n.text}
</li> </Link>
<li className="flex items-center gap-2">
<Cog8ToothIcon className="w-6 h-6" />Settings
</li> </li>
))}
</ul> </ul>
</nav> </nav>
<main className="p-8 w-full ml-64"> <main className="p-8 w-full ml-64">

3
components/tcmd/index.tsx
View File

@ -3,6 +3,7 @@
import { Accordion, AccordionContent } from "@/lib/accordion"; import { Accordion, AccordionContent } from "@/lib/accordion";
import { Poppable } from "@/lib/poppables/components/poppable"; import { Poppable } from "@/lib/poppables/components/poppable";
import { createElements } from "@/lib/tcmd"; import { createElements } from "@/lib/tcmd";
import { tokenizeInline } from "@/lib/tcmd/tokenizeInline";
import Link from "next/link"; import Link from "next/link";
import React, { FC, Fragment, ReactNode, use, useMemo } from "react"; import React, { FC, Fragment, ReactNode, use, useMemo } from "react";
@ -192,7 +193,7 @@ const renderInlineToken = (l: Line) => {
case "popover": case "popover":
return ( return (
<Poppable <Poppable
content={token.data.popover} content={renderInlineToken(token.data.popover)}
preferredAlign="centered" preferredAlign="centered"
preferredEdge="bottom" preferredEdge="bottom"
className="cursor-pointer" className="cursor-pointer"

132
lib/tcmd/index.ts
View File

@ -1,7 +1,7 @@
import { zipArrays } from "../zip"; import { zipArrays } from "../zip";
import { inlineTokens } from "./inlineTokens"; import { singleLineTokens, tokenizeLine } from "./tokenizeLine";
import { singleLineTokens } from "./singleLineTokens";
import { tokenizeBlock } from "./tokenizeBlock"; import { tokenizeBlock } from "./tokenizeBlock";
import { tokenizeInline } from "./tokenizeInline";
import { tokenizeParagraph } from "./tokenizeParagraph"; import { tokenizeParagraph } from "./tokenizeParagraph";
export const createElements = (body: string) => { export const createElements = (body: string) => {
@ -11,6 +11,8 @@ export const createElements = (body: string) => {
}; };
const tokenize = (body: string) => { const tokenize = (body: string) => {
body = body.replace(/\n?<!--(.*?)-->\n?/gs, "");
const paragraphs = body.split("\n\n"); const paragraphs = body.split("\n\n");
const blockTokens: BlockToken[] = []; const blockTokens: BlockToken[] = [];
@ -97,129 +99,3 @@ const tokenize = (body: string) => {
return blockTokens.filter((b) => !b.parent); return blockTokens.filter((b) => !b.parent);
}; };
// const __tokenize = (md: string) => {
// const tokens: (Token)[] = [];
// // md = md.replace(/(?<=[a-z])\n(?=[a-z])/g, " ");
// const lines = md.split("\n");
// let preserveEmpty = false;
// let multilineLines;
// let tokenSettings;
// for (let line of lines) {
// if (!line && !preserveEmpty) continue;
// let foundLine = false;
// if (!multilineLines) {
// token:
// for (const token of multilineTokens) {
// if (!token.rx.test(line)) continue token;
// tokenSettings = token;
// multilineLines = token.create(tokens);
// preserveEmpty = true;
// foundLine = true;
// multilineLines.push({
// type: "text",
// line: token.replace(line),
// });
// }
// } else {
// foundLine = true;
// if (tokenSettings?.closeRx?.test(line) || tokenSettings?.rx.test(line)) {
// tokenSettings = undefined;
// multilineLines = undefined;
// preserveEmpty = false;
// } else {
// multilineLines.push({
// type: "text",
// line,
// });
// }
// }
// if (!multilineLines) {
// token:
// for (const token of singleLineTokens) {
// if (!token.rx.test(line)) continue token;
// foundLine = true;
// line = line.replace(token.replaceRx, "").trim();
// const lineContent = tokenizeInline(line);
// token.create(lineContent, tokens);
// }
// }
// if (foundLine) continue;
// tokens.push({
// type: "text",
// line: tokenizeInline(line),
// });
// }
// return tokens;
// };
const tokenizeLine = (
line: string,
previous?: SingleLineToken,
): SingleLineToken => {
for (const token of singleLineTokens) {
if (!token.rx.test(line)) continue;
const t = token.create(line);
if (t.type === "h2") {
}
t.line = tokenizeInline(line.replace(token.replaceRx, ""));
return t;
}
if (previous?.mends) {
previous.raw += " " + line;
previous.line = tokenizeInline(previous.raw.replace(previous.cfg!.rx, ""));
return previous;
}
return {
line: tokenizeInline(line),
type: "text",
raw: line,
};
};
const tokenizeInline = (line: string) => {
line = line.trim();
const originalLine = line;
const insertMarker = "\u{03A9}";
const tokens: InlineTokenInsert[] = [];
for (const token of inlineTokens) {
token.rx.lastIndex = 0;
let match;
while ((match = token.rx.exec(line)) !== null) {
const tokenStart = match.index;
const tokenEnd = match.index + match[0].length;
token.create(match, tokenStart, tokenEnd, tokens);
}
}
if (tokens.length) {
for (const insert of tokens) {
line = line.slice(0, insert.start) +
"".padStart(insert.end - insert.start, insertMarker) +
line.slice(insert.end, line.length);
}
return zipArrays(
line.split(new RegExp(insertMarker + "{2,}")).map((t): InlineToken => ({
content: t,
type: "text",
})),
tokens,
).filter((t) => t.content);
}
return originalLine;
};

57
lib/tcmd/inlineTokens.ts → lib/tcmd/tokenizeInline.ts
View File

@ -1,3 +1,55 @@
import { zipArrays } from "../zip";
export const tokenizeInline = (line: string, recursive?: boolean) => {
if (recursive) console.log("recursive call");
line = line.trim();
const originalLine = line;
const insertMarker = "\u{03A9}";
const tokens: InlineTokenInsert[] = [];
for (const token of inlineTokens) {
const rx = new RegExp(token.rx);
let match;
while ((match = rx.exec(line)) !== null) {
const tokenStart = match.index;
const tokenEnd = match.index + match[0].length;
const wrappingToken = tokens.find((t) =>
t.start < tokenStart && t.end > tokenStart
);
if (wrappingToken) continue;
let wrappedToken;
while (
(wrappedToken = tokens.findIndex((t) =>
t.start > tokenStart && t.start < tokenEnd
)) !== -1
) {
tokens.splice(wrappedToken, 1);
}
token.create(match, tokenStart, tokenEnd, tokens);
}
}
if (tokens.length) {
for (const insert of tokens) {
line = line.slice(0, insert.start) +
"".padStart(insert.end - insert.start, insertMarker) +
line.slice(insert.end, line.length);
}
return zipArrays(
line.split(new RegExp(insertMarker + "{2,}")).map((t): InlineToken => ({
content: t,
type: "text",
})),
tokens,
).filter((t) => t.content);
}
return originalLine;
};
const joiner = "<><>"; const joiner = "<><>";
export const inlineTokens: { export const inlineTokens: {
rx: RegExp; rx: RegExp;
@ -61,16 +113,17 @@ export const inlineTokens: {
}, },
}, },
{ {
rx: /\^\[(.*?)\]\((.*?)\)/g, rx: /\^\[(.*?)\]<<(.*?)>>/gm,
create(content, start, end, tokens) { create(content, start, end, tokens) {
const [_, text, popover] = content; const [_, text, popover] = content;
// tokenizeInline("", true);
tokens.push({ tokens.push({
content: text, content: text,
end, end,
start, start,
type: "popover", type: "popover",
data: { data: {
popover, popover: tokenizeInline(popover),
}, },
}); });
}, },

31
lib/tcmd/singleLineTokens.ts → lib/tcmd/tokenizeLine.ts
View File

@ -1,3 +1,34 @@
import { tokenizeInline } from "./tokenizeInline";
export const tokenizeLine = (
line: string,
previous?: SingleLineToken,
): SingleLineToken => {
for (const token of singleLineTokens) {
if (!token.rx.test(line)) continue;
const t = token.create(line);
if (t.type === "h2") {
}
t.line = tokenizeInline(line.replace(token.replaceRx, ""));
return t;
}
if (previous?.mends) {
previous.raw += " " + line;
previous.line = tokenizeInline(previous.raw.replace(previous.cfg!.rx, ""));
return previous;
}
return {
line: tokenizeInline(line),
type: "text",
raw: line,
};
};
export const singleLineTokens: SingleLineCfg[] = [ export const singleLineTokens: SingleLineCfg[] = [
{ {
rx: /^#\s/, rx: /^#\s/,

28
test.md
View File

@ -1,20 +1,24 @@
# Hello! Welcome to Tabletop Commander! # Hello! Welcome to Tabletop Commander!
<!--
-->
[][][] [][][]
[[ [[
Lorem ^[ipsum](This is a popover test) dolor sit amet, consectetur adipiscing Lorem ^[ipsum]<<This is a popover test: ![Goofy](https://yt3.ggpht.com/a/AATXAJwbIW0TwEhqdT2ZPeSB1AtdtWD2ZXam80oijg=s900-c-k-c0xffffffff-no-rj-mo)>>
elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut
Sollicitudin tempor id eu nisl nunc mi ipsum faucibus vitae. Lobortis elementum labore et dolore magna aliqua. Sollicitudin tempor id eu nisl nunc mi ipsum
nibh tellus molestie nunc. Purus non enim praesent elementum facilisis leo vel. faucibus vitae. Lobortis elementum nibh tellus molestie nunc. Purus non enim
Orci nulla pellentesque dignissim enim sit amet venenatis. Eu feugiat pretium praesent elementum facilisis leo vel. Orci nulla pellentesque dignissim enim sit
nibh ipsum. Gravida dictum fusce ut placerat orci nulla pellentesque. Tincidunt amet venenatis. Eu feugiat pretium nibh ipsum. Gravida dictum fusce ut placerat
vitae semper quis lectus nulla at volutpat diam ut. Proin sed libero enim sed orci nulla pellentesque. Tincidunt vitae semper quis lectus nulla at volutpat
faucibus turpis in eu mi. Dui sapien eget mi proin sed libero enim sed faucibus. diam ut. Proin sed libero enim sed faucibus turpis in eu mi. Dui sapien eget mi
Felis donec et odio pellentesque diam volutpat commodo sed egestas. Massa proin sed libero enim sed faucibus. Felis donec et odio pellentesque diam
tincidunt dui ut ornare lectus sit amet est placerat. Auctor urna nunc id cursus volutpat commodo sed egestas. Massa tincidunt dui ut ornare lectus sit amet est
metus aliquam eleifend. placerat. Auctor urna nunc id cursus metus aliquam eleifend.
- Lorem ipsum dolor sit amet, - Lorem ipsum dolor sit amet,
- consectetur adipiscing elit, bananana banana ban anana anaba bananananana - consectetur adipiscing elit, bananana banana ban anana anaba bananananana
@ -79,6 +83,8 @@ const blockTokens: {
this is the test of a single accordion this is the test of a single accordion
![Goofy](https://yt3.ggpht.com/a/AATXAJwbIW0TwEhqdT2ZPeSB1AtdtWD2ZXam80oijg=s900-c-k-c0xffffffff-no-rj-mo)
[/accordion] [/accordion]
]] ]]