ttcMD: added a cleanup step to make sure that line endings are as we expect,

fixes p blocks not replacing line breaks with spaces leading to unintentional linguistic collisions

lib/tcmd/TokenIdentifiers.tsx

@@ -104,7 +104,7 @@ export const buildOnlyDefaultElements = () => {
     debugger;
     return (
       <span className="whitespace-pre-wrap">
-        {t.content.replaceAll("\\n", "\n")}
+        {t.content.replaceAll(/\\n ?/g, "\n")}
       </span>
     );
   });
@@ -509,9 +509,9 @@ export const buildOnlyDefaultElements = () => {
     },
   );
 
-  registerIdentifier("p", /(?<=\n\n)([\s\S]*?)(?=\n\n)/g, (s, rx) => {
+  registerIdentifier("p", /(?<=\n\n)([\s\S]*?)(?=\n\n)/g, (s) => {
     return {
-      content: s,
+      content: s.replace("\n", " "),
       raw: s,
       metadata: {},
       uuid: crypto.randomUUID(),

lib/tcmd/index.ts

@@ -11,6 +11,8 @@ export const createElements = (body: string): Token[] => {
 const tokenize = (body: string) => {
   const tokenizedBody: TokenMarker[] = [];
 
+  body = body.replaceAll(/[ \t]+\n/g, "\n");
+
   const addToken = (thing: TokenMarker) => {
     tokenizedBody.push(thing);
   };
@@ -154,7 +156,7 @@ const contentToChildren = (token: Token) => {
 
   token.children = zipArrays(
     content.split(splitMarker).map((c): Token => ({
-      content: c.replaceAll("\n", ""),
+      content: c.replaceAll("\n", " "),
       metadata: {},
       raw: c,
       type: token.rendersChildrenOnly ? "p" : "text",
@@ -164,7 +166,7 @@ const contentToChildren = (token: Token) => {
       children: token.rendersChildrenOnly && c.replaceAll("\n", "")
         ? [
           {
-            content: c.replaceAll("\n", ""),
+            content: c.replaceAll("\n", " "),
             metadata: {},
             raw: c,
             type: "text",