90 lines
2.1 KiB
TypeScript
90 lines
2.1 KiB
TypeScript
export const tokenizeParagraph = (paragraph: string) => {
|
|
for (const pgraph of paragraphTokens) {
|
|
const openTest = pgraph.rx.test(paragraph),
|
|
closeTest = pgraph.closeRx.test(paragraph);
|
|
if (openTest && closeTest) {
|
|
const p = pgraph.create(paragraph);
|
|
p.closed = true;
|
|
return p;
|
|
}
|
|
if (closeTest) return pgraph.create(paragraph).content;
|
|
|
|
if (openTest) {
|
|
return pgraph.create(paragraph);
|
|
}
|
|
}
|
|
};
|
|
|
|
const paragraphTokens: {
|
|
rx: RegExp;
|
|
closeRx: RegExp;
|
|
create: (line: string) => ParagraphToken;
|
|
}[] = [
|
|
{
|
|
rx: /\n```/g,
|
|
closeRx: /\n```/g,
|
|
create(line) {
|
|
return {
|
|
type: "code",
|
|
metadata: {
|
|
// language: line.split("\n").at(0)!.replace(this.rx, ""),
|
|
},
|
|
closed: false,
|
|
content: [{
|
|
line: line.match(/```(.*?)\n```/g)?.at(1) || line,
|
|
type: "text",
|
|
raw: line,
|
|
uuid: crypto.randomUUID(),
|
|
}],
|
|
allowsInline: false,
|
|
uuid: crypto.randomUUID(),
|
|
};
|
|
},
|
|
},
|
|
];
|
|
|
|
TokenIdentifiers.set("table", {
|
|
rx: /^\|\s[\s\S]*?\|(?=(\n\n)|$)/g,
|
|
parse(s) {
|
|
const rowSections = s.split(/-/gm).map((s) =>
|
|
s.split("\n").map((r) => r.split(/\s?\|\s?/g))
|
|
);
|
|
|
|
let headerRows: string[][] = [],
|
|
bodyRows: string[][] = [],
|
|
footerRows: string[][] = [];
|
|
|
|
switch (rowSections.length) {
|
|
case 1:
|
|
bodyRows = rowSections[0];
|
|
break;
|
|
case 2:
|
|
headerRows = rowSections[0];
|
|
bodyRows = rowSections[1];
|
|
break;
|
|
case 3:
|
|
headerRows = rowSections[0];
|
|
bodyRows = rowSections[1];
|
|
footerRows = rowSections[3];
|
|
break;
|
|
}
|
|
|
|
const maxColumns = Math.max(
|
|
...[...headerRows, ...bodyRows, ...footerRows].map((r) => r.length),
|
|
);
|
|
|
|
return {
|
|
content: s,
|
|
raw: s,
|
|
metadata: {
|
|
headerRows: headerRows.join(" | "),
|
|
bodyRows: bodyRows.join(" | "),
|
|
footerRows: footerRows.join(" | "),
|
|
columns: maxColumns.toString(),
|
|
},
|
|
type: "table",
|
|
uuid: crypto.randomUUID(),
|
|
};
|
|
},
|
|
});
|