emma/tabletop-commander
1
0
Fork 0
Code Issues 2 Pull Requests Packages Projects 3 Releases Wiki Activity

tcmd: Fixes regex reuse in inline tokenizer,

Browse Source 
tcmd: changes popover syntax to allow for embeddable markdown
This commit is contained in:
Emmaline Autumn 2024-02-29 00:02:04 -07:00
parent ff0a4280e2
commit ce83bdf7af
7 changed files with 147 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -34,3 +34,6 @@ yarn-error.log*
# typescript
*.tsbuildinfo
next-env.d.ts
# vscode
.vscode

45
app/layout.tsx
View File

@ -7,6 +7,7 @@ import {
Cog8ToothIcon,
PuzzlePieceIcon,
} from "@heroicons/react/24/solid";
import Link from "next/link";
const inter = Inter({ subsets: ["latin"] });
@ -20,6 +21,29 @@ export default function RootLayout({
}: Readonly<{
children: React.ReactNode;
}>) {
const navItems = [
{
to: "/schemas",
icon: CircleStackIcon,
text: "Schemas",
},
{
to: "/game-systems",
icon: PuzzlePieceIcon,
text: "Game Systems",
},
{
to: "/publications",
icon: BookOpenIcon,
text: "Publications",
},
{
to: "/settings",
icon: Cog8ToothIcon,
text: "Settings",
},
];
return (
<html lang="en">
<body className={inter.className + " flex min-h-[100vh]"}>
@ -28,18 +52,17 @@ export default function RootLayout({
Tabletop Commander
</h1>
<ul className="my-6 flex flex-col gap-6">
<li className="flex items-center gap-2">
<CircleStackIcon className="w-6 h-6" />Schemas
</li>
<li className="flex items-center gap-2">
<PuzzlePieceIcon className="w-6 h-6" />Game Systems
</li>
<li className="flex items-center gap-2">
<BookOpenIcon className="w-6 h-6" />Publications
</li>
<li className="flex items-center gap-2">
<Cog8ToothIcon className="w-6 h-6" />Settings
{navItems.map((n) => (
<li key={"nav-item" + n.text}>
<Link
href={n.to}
className="flex items-center gap-2 group hover:text-purple-300 transition-colors"
>
<n.icon className="w-6 h-6 group-hover:fill-purple-300 transition-colors" />
{n.text}
</Link>
</li>
))}
</ul>
</nav>
<main className="p-8 w-full ml-64">

3
components/tcmd/index.tsx
View File

@ -3,6 +3,7 @@
import { Accordion, AccordionContent } from "@/lib/accordion";
import { Poppable } from "@/lib/poppables/components/poppable";
import { createElements } from "@/lib/tcmd";
import { tokenizeInline } from "@/lib/tcmd/tokenizeInline";
import Link from "next/link";
import React, { FC, Fragment, ReactNode, use, useMemo } from "react";
@ -192,7 +193,7 @@ const renderInlineToken = (l: Line) => {
case "popover":
return (
<Poppable
content={token.data.popover}
content={renderInlineToken(token.data.popover)}
preferredAlign="centered"
preferredEdge="bottom"
className="cursor-pointer"

132
lib/tcmd/index.ts
View File

@ -1,7 +1,7 @@
import { zipArrays } from "../zip";
import { inlineTokens } from "./inlineTokens";
import { singleLineTokens } from "./singleLineTokens";
import { singleLineTokens, tokenizeLine } from "./tokenizeLine";
import { tokenizeBlock } from "./tokenizeBlock";
import { tokenizeInline } from "./tokenizeInline";
import { tokenizeParagraph } from "./tokenizeParagraph";
export const createElements = (body: string) => {
@ -11,6 +11,8 @@ export const createElements = (body: string) => {
};
const tokenize = (body: string) => {
body = body.replace(/\n?<!--(.*?)-->\n?/gs, "");
const paragraphs = body.split("\n\n");
const blockTokens: BlockToken[] = [];
@ -97,129 +99,3 @@ const tokenize = (body: string) => {
return blockTokens.filter((b) => !b.parent);
};
// const __tokenize = (md: string) => {
// const tokens: (Token)[] = [];
// // md = md.replace(/(?<=[a-z])\n(?=[a-z])/g, " ");
// const lines = md.split("\n");
// let preserveEmpty = false;
// let multilineLines;
// let tokenSettings;
// for (let line of lines) {
// if (!line && !preserveEmpty) continue;
// let foundLine = false;
// if (!multilineLines) {
// token:
// for (const token of multilineTokens) {
// if (!token.rx.test(line)) continue token;
// tokenSettings = token;
// multilineLines = token.create(tokens);
// preserveEmpty = true;
// foundLine = true;
// multilineLines.push({
// type: "text",
// line: token.replace(line),
// });
// }
// } else {
// foundLine = true;
// if (tokenSettings?.closeRx?.test(line) || tokenSettings?.rx.test(line)) {
// tokenSettings = undefined;
// multilineLines = undefined;
// preserveEmpty = false;
// } else {
// multilineLines.push({
// type: "text",
// line,
// });
// }
// }
// if (!multilineLines) {
// token:
// for (const token of singleLineTokens) {
// if (!token.rx.test(line)) continue token;
// foundLine = true;
// line = line.replace(token.replaceRx, "").trim();
// const lineContent = tokenizeInline(line);
// token.create(lineContent, tokens);
// }
// }
// if (foundLine) continue;
// tokens.push({
// type: "text",
// line: tokenizeInline(line),
// });
// }
// return tokens;
// };
const tokenizeLine = (
line: string,
previous?: SingleLineToken,
): SingleLineToken => {
for (const token of singleLineTokens) {
if (!token.rx.test(line)) continue;
const t = token.create(line);
if (t.type === "h2") {
}
t.line = tokenizeInline(line.replace(token.replaceRx, ""));
return t;
}
if (previous?.mends) {
previous.raw += " " + line;
previous.line = tokenizeInline(previous.raw.replace(previous.cfg!.rx, ""));
return previous;
}
return {
line: tokenizeInline(line),
type: "text",
raw: line,
};
};
const tokenizeInline = (line: string) => {
line = line.trim();
const originalLine = line;
const insertMarker = "\u{03A9}";
const tokens: InlineTokenInsert[] = [];
for (const token of inlineTokens) {
token.rx.lastIndex = 0;
let match;
while ((match = token.rx.exec(line)) !== null) {
const tokenStart = match.index;
const tokenEnd = match.index + match[0].length;
token.create(match, tokenStart, tokenEnd, tokens);
}
}
if (tokens.length) {
for (const insert of tokens) {
line = line.slice(0, insert.start) +
"".padStart(insert.end - insert.start, insertMarker) +
line.slice(insert.end, line.length);
}
return zipArrays(
line.split(new RegExp(insertMarker + "{2,}")).map((t): InlineToken => ({
content: t,
type: "text",
})),
tokens,
).filter((t) => t.content);
}
return originalLine;
};

57
lib/tcmd/inlineTokens.ts → lib/tcmd/tokenizeInline.ts
View File

@ -1,3 +1,55 @@
import { zipArrays } from "../zip";
export const tokenizeInline = (line: string, recursive?: boolean) => {
if (recursive) console.log("recursive call");
line = line.trim();
const originalLine = line;
const insertMarker = "\u{03A9}";
const tokens: InlineTokenInsert[] = [];
for (const token of inlineTokens) {
const rx = new RegExp(token.rx);
let match;
while ((match = rx.exec(line)) !== null) {
const tokenStart = match.index;
const tokenEnd = match.index + match[0].length;
const wrappingToken = tokens.find((t) =>
t.start < tokenStart && t.end > tokenStart
);
if (wrappingToken) continue;
let wrappedToken;
while (
(wrappedToken = tokens.findIndex((t) =>
t.start > tokenStart && t.start < tokenEnd
)) !== -1
) {
tokens.splice(wrappedToken, 1);
}
token.create(match, tokenStart, tokenEnd, tokens);
}
}
if (tokens.length) {
for (const insert of tokens) {
line = line.slice(0, insert.start) +
"".padStart(insert.end - insert.start, insertMarker) +
line.slice(insert.end, line.length);
}
return zipArrays(
line.split(new RegExp(insertMarker + "{2,}")).map((t): InlineToken => ({
content: t,
type: "text",
})),
tokens,
).filter((t) => t.content);
}
return originalLine;
};
const joiner = "<><>";
export const inlineTokens: {
rx: RegExp;
@ -61,16 +113,17 @@ export const inlineTokens: {
},
},
{
rx: /\^\[(.*?)\]\((.*?)\)/g,
rx: /\^\[(.*?)\]<<(.*?)>>/gm,
create(content, start, end, tokens) {
const [_, text, popover] = content;
// tokenizeInline("", true);
tokens.push({
content: text,
end,
start,
type: "popover",
data: {
popover,
popover: tokenizeInline(popover),
},
});
},

31
lib/tcmd/singleLineTokens.ts → lib/tcmd/tokenizeLine.ts
View File

@ -1,3 +1,34 @@
import { tokenizeInline } from "./tokenizeInline";
export const tokenizeLine = (
line: string,
previous?: SingleLineToken,
): SingleLineToken => {
for (const token of singleLineTokens) {
if (!token.rx.test(line)) continue;
const t = token.create(line);
if (t.type === "h2") {
}
t.line = tokenizeInline(line.replace(token.replaceRx, ""));
return t;
}
if (previous?.mends) {
previous.raw += " " + line;
previous.line = tokenizeInline(previous.raw.replace(previous.cfg!.rx, ""));
return previous;
}
return {
line: tokenizeInline(line),
type: "text",
raw: line,
};
};
export const singleLineTokens: SingleLineCfg[] = [
{
rx: /^#\s/,

28
test.md
View File

@ -1,20 +1,24 @@
# Hello! Welcome to Tabletop Commander!
<!--
-->
[][][]
[[
Lorem ^[ipsum](This is a popover test) dolor sit amet, consectetur adipiscing
elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
Sollicitudin tempor id eu nisl nunc mi ipsum faucibus vitae. Lobortis elementum
nibh tellus molestie nunc. Purus non enim praesent elementum facilisis leo vel.
Orci nulla pellentesque dignissim enim sit amet venenatis. Eu feugiat pretium
nibh ipsum. Gravida dictum fusce ut placerat orci nulla pellentesque. Tincidunt
vitae semper quis lectus nulla at volutpat diam ut. Proin sed libero enim sed
faucibus turpis in eu mi. Dui sapien eget mi proin sed libero enim sed faucibus.
Felis donec et odio pellentesque diam volutpat commodo sed egestas. Massa
tincidunt dui ut ornare lectus sit amet est placerat. Auctor urna nunc id cursus
metus aliquam eleifend.
Lorem ^[ipsum]<<This is a popover test: ![Goofy](https://yt3.ggpht.com/a/AATXAJwbIW0TwEhqdT2ZPeSB1AtdtWD2ZXam80oijg=s900-c-k-c0xffffffff-no-rj-mo)>>
dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut
labore et dolore magna aliqua. Sollicitudin tempor id eu nisl nunc mi ipsum
faucibus vitae. Lobortis elementum nibh tellus molestie nunc. Purus non enim
praesent elementum facilisis leo vel. Orci nulla pellentesque dignissim enim sit
amet venenatis. Eu feugiat pretium nibh ipsum. Gravida dictum fusce ut placerat
orci nulla pellentesque. Tincidunt vitae semper quis lectus nulla at volutpat
diam ut. Proin sed libero enim sed faucibus turpis in eu mi. Dui sapien eget mi
proin sed libero enim sed faucibus. Felis donec et odio pellentesque diam
volutpat commodo sed egestas. Massa tincidunt dui ut ornare lectus sit amet est
placerat. Auctor urna nunc id cursus metus aliquam eleifend.
- Lorem ipsum dolor sit amet,
- consectetur adipiscing elit, bananana banana ban anana anaba bananananana
@ -79,6 +83,8 @@ const blockTokens: {
this is the test of a single accordion
![Goofy](https://yt3.ggpht.com/a/AATXAJwbIW0TwEhqdT2ZPeSB1AtdtWD2ZXam80oijg=s900-c-k-c0xffffffff-no-rj-mo)
[/accordion]
]]