fixing some chunking issues

.gitignore

@@ -28,3 +28,4 @@ dist-ssr
 BearMetal/
 resources/
 !**/*/resources/
+test.ts

pack_versions/57.json

@@ -0,0 +1,63 @@
+{
+  "version": "48",
+  "mcVersion": "^1.21.3",
+  "schema": {
+    "pack.mcmeta": "json",
+    "pack.png": "image/png",
+    "data": {
+      "<namespace>": {
+        "function": "function",
+        "structure": {
+          "DataVersion": "int",
+          "size": ["int", "int", "int"],
+          "palette": [{
+            "name": "blockId",
+            "properties": ["string"]
+          }],
+          "palettes": [
+            [{
+              "name": "blockId",
+              "properties": ["string"]
+            }]
+          ],
+          "blocks": [{
+            "state": "int",
+            "pos": ["int", "int", "int"],
+            "nbt": "nbt"
+          }],
+          "entities": [{
+            "pos": ["double", "double", "double"],
+            "blockPos": ["int", "int", "int"],
+            "nbt": "nbt"
+          }]
+        },
+        "tags": "tags",
+        "advancment": {
+          "parent": "advancment",
+          "display": {
+            "icon": {
+              "id": "itemId",
+              "count": "int",
+              "components": ["itemComponent"]
+            },
+            "title": "jsonString",
+            "description": "jsonString",
+            "frame": "frame",
+            "background": "resource",
+            "show_toast": "bool",
+            "announce_to_chat": "bool",
+            "hidden": "bool"
+          },
+          "criteria": "criteria",
+          "requirements": ["criterion_name"],
+          "rewards": {
+            "experience": "int",
+            "function": "function",
+            "loot": ["loot_table"],
+            "recipes": ["recipe"]
+          }
+        }
+      }
+    }
+  }
+}

server/main.ts

@@ -6,6 +6,8 @@ import { Router } from "./router.ts";
 import { getPackVersion } from "./util/packVersion.ts";
 import { createTagRoutes } from "./tags/routes.ts";
 import { createResourcesRoutes } from "./resources/routes.ts";
+import { readDirDirs } from "./util/readDir.ts";
+import { unzipResources } from "./resources/unzip.ts";
 
 const installPath = Deno.env.get("BMP_INSTALL_DIR") || "./";
 
@@ -31,6 +33,7 @@ sockpuppet.addHandler((req: Request) => {
 // });
 
 const router = new Router();
+
 router.route("/api/dir")
   .get(async () => {
     using store = new BearMetalStore();
@@ -208,6 +211,15 @@ router.route("/api/pack/version")
         store.get("packlocation") + "/pack.mcmeta",
         JSON.stringify(packMetaJson),
       );
+      const packVersionSchema = Deno.readTextFileSync(
+        installPath + "pack_versions/" + version + ".json",
+      );
+      const packVersionSchemaJson = JSON.parse(packVersionSchema);
+      packVersionSchemaJson.mcVersion = store.get("mcVersion");
+      const versionResourceDir = await readDirDirs(installPath + "resources/");
+      if (!versionResourceDir.includes(version)) {
+        unzipResources();
+      }
     } catch (e: any) {
       return new Response(e, { status: 500 });
     }
@@ -226,6 +238,25 @@ router.route("/api/versions")
 createTagRoutes(router);
 createResourcesRoutes(router);
 
+router.route("/api/stream/test")
+  .get(() => {
+    const stream = new ReadableStream({
+      async start(controller) {
+        const enc = new TextEncoder();
+        controller.enqueue(enc.encode("Hello"));
+        controller.enqueue(enc.encode(", "));
+        controller.enqueue(enc.encode("World!"));
+        controller.close();
+      },
+    });
+    return new Response(stream, {
+      headers: {
+        "content-type": "text/plain",
+        "x-content-type-options": "nosniff",
+      },
+    });
+  })
+
 sockpuppet.addHandler((req: Request) => {
   if (new URL(req.url).pathname.startsWith("/api")) return;
 

server/resources/readers.ts

@@ -2,13 +2,7 @@ import { encodeBase64 } from "@std/encoding/base64";
 import { readDirFiles } from "../util/readDir.ts";
 import { createIsometricCube } from "./renderer.ts";
 
-interface BlockItem {
+export const readBlocks = async (path: string, read?: (b: BlockItem, i: number) => void) => {
-  name: string;
-  resourceLocation: string;
-  images: string[];
-}
-
-export const readBlocks = async (path: string) => {
   const blocks: BlockItem[] =
     (await readDirFiles(path + "/assets/minecraft/blockstates"))
       .map((b) => ({
@@ -300,10 +294,16 @@ export const readBlocks = async (path: string) => {
       block.images.push(await createIsometricCube(b64, b64, b64));
     }
   }
+
+  let i = 0;
+  for (const block of blocks) {
+    read?.(block, i);
+    i++;
+  }
   return blocks;
 };
 
-export const readItems = async (path: string) => {
+export const readItems = async (path: string, read?: (b: BlockItem) => void) => {
   const items: BlockItem[] =
     (await readDirFiles(path + "/assets/minecraft/models/item")).map((i) => ({
       name: i.replace(".json", ""),
@@ -348,6 +348,7 @@ export const readItems = async (path: string) => {
         }
       }
     }
+    read?.(item);
   }
 
   return items;

server/resources/routes.ts

@@ -1,7 +1,8 @@
+import { ensureDir } from "@std/fs/ensure-dir";
 import type { Router } from "../router.ts";
 import { readDirDirs } from "../util/readDir.ts";
 import { versionCompat } from "../util/versionCompat.ts";
-import { readBlocks } from "./readers.ts";
+import { readBlocks, readItems } from "./readers.ts";
 
 export const createResourcesRoutes = (router: Router) => {
   router.route("/api/resources/:path*")
@@ -10,38 +11,92 @@ export const createResourcesRoutes = (router: Router) => {
       if (!path) {
         return new Response("no path provided", { status: 400 });
       }
+
       const format = ctx.url.searchParams.get("format");
       if (!format) {
         return new Response("no format provided", { status: 400 });
       }
+
       const packVersion = await Deno.readTextFile(
         "./pack_versions/" + format + ".json",
       );
       const packVersionJson = JSON.parse(packVersion);
       const mcVersion = packVersionJson.mcVersion;
+
+      await ensureDir("./resources");
       const resourceVersions = await readDirDirs("./resources");
-      console.log("resourceVersions", resourceVersions);
+
       for (const resourceVersion of resourceVersions) {
         if (versionCompat(resourceVersion, mcVersion)) {
           const resourcePath = "./resources/" + resourceVersion;
-          const splitPath = path.split("/");
+          let items: BlockItem[] = [];
-          switch (splitPath[0]) {
+
+          const batch = (
+            controller: ReadableStreamDefaultController,
+            res: BlockItem,
+          ) => {
+            items.push(res);
+            if (items.length > 4) {
+              controller.enqueue(
+                new TextEncoder().encode(JSON.stringify(items) + "\n"),
+              );
+              items = [];
+            }
+          };
+          const body = new ReadableStream({
+            async start(controller) {
+              switch (path) {
                 case "block":
                 case "blocks": {
-              return new Response(
+                  await readBlocks(resourcePath, (res, i) => {
-                JSON.stringify(await readBlocks(resourcePath)),
+                    console.log(i);
-              );
+                    batch(controller, res);
+                  });
+                  controller.close();
+                  break;
                 }
                 case "item":
                 case "items": {
-              return new Response(
+                  await readItems(resourcePath, (res) => {
-                JSON.stringify(await readBlocks(resourcePath)),
+                    batch(controller, res);
-              );
+                  });
+                  controller.close();
+                  break;
                 }
                 default: {
-              return new Response("invalid path", { status: 400 });
+                  controller.close();
+                  break;
                 }
               }
+            },
+          });
+
+          return new Response(body, {
+            headers: {
+              "content-type": "application/json",
+              "access-control-allow-origin": "*",
+            },
+          });
+
+          // const resourcePath = "./resources/" + resourceVersion;
+          // const splitPath = path.split("/");
+          // switch (splitPath[0]) {
+          //   case "block":
+          //   case "blocks": {
+          //     return new Response(
+          //       JSON.stringify(await readBlocks(resourcePath)),
+          //     );
+          //   }
+          //   case "item":
+          //   case "items": {
+          //     return new Response(
+          //       JSON.stringify(await readBlocks(resourcePath)),
+          //     );
+          //   }
+          //   default: {
+          //     return new Response("invalid path", { status: 400 });
+          //   }
+          // }
         }
       }
     });

src/components/editor/tags/editor.tsx

@@ -6,6 +6,7 @@ import { namespaceAtom, useNamespace } from "../../../atoms/namespace.ts";
 import { Loader } from "../../../components/loader.tsx";
 import { useEffect, useState } from "preact/hooks";
 import { NewTagModal } from "./newTagModal.tsx";
+import { ResourceList } from "../../resourceList.tsx";
 
 export const TagRouter = () => {
   return (
@@ -33,7 +34,7 @@ function TagList() {
     <div>
       <ul>
       </ul>
-      <Resources />
+      <ResourceList />
       {false && (
         <ul class="flex flex-col gap-2">
           <li>
@@ -133,24 +134,3 @@ function TagEditor() {
   );
 }
 
-function Resources() {
-  const { data, isLoading } = useSWR<{ name: string; images: string[] }[]>(
-    `/api/resources/blocks?format=48`,
-    fetchJson,
-  );
-  if (isLoading || !data) {
-    return <Loader msg="Your hard drive is full of... interesting things." />;
-  }
-  return (
-    <div>
-      <ul>
-        {data.map((resource) => (
-          <li>
-            {resource.name}
-            <img class="min-w-12 max-h-16" src={resource.images[0]} />
-          </li>
-        ))}
-      </ul>
-    </div>
-  );
-}

src/components/resourceList.tsx

@@ -0,0 +1,33 @@
+import { useState } from "preact/hooks";
+import { useRemoteStream } from "../hooks/useStream.ts";
+
+export const ResourceList = () => {
+  // const [resources, setResources] = useState<BlockItem[]>([]);
+  // // useRemoteStream("/api/stream/test", (e) => {
+  // useRemoteStream("http://localhost:8000/api/resources/blocks?format=57", (e: string) => {
+  //   try {
+  //     JSON.parse(e);
+  //   } catch (err) {
+  //     return console.log(e);
+  //   }
+  //   setResources((resources) => [...resources, ...JSON.parse(e) as BlockItem[]]);
+  // }, (e) => console.log(e));
+
+  const [resources, isLoading, error] = useRemoteStream<BlockItem>('http://localhost:8000/api/resources/blocks?format=57');
+
+  if (isLoading) return <div>Loading...</div>;
+  if (error) return <div>Error: {error}</div>;
+
+  return (
+    <div>
+      <ul>
+        {resources?.map((resource) => (
+          <li>
+            {resource.name}
+            <img class="min-w-6 max-w-8" src={resource.images[0]} />
+          </li>
+        ))}
+      </ul>
+    </div>
+  );
+};

src/hooks/useStream.ts

@@ -1,42 +1,49 @@
-import { useEffect } from "preact/hooks";
+import { useEffect, useState } from "preact/hooks";
 
-export const useStream = <T>(
+export const useRemoteStream = <T = string>(url: string, transformer?: (e: string) => T) : [T[], boolean, string | null] => {
-  stream: ReadableStream<T>,
+  const [items, setItems] = useState<T[]>([]);
-  onData: (data: T) => void,
+  const [isLoading, setIsLoading] = useState(true);
-  onError: (err: Error) => void,
+  const [error, setError] = useState(null);
-) => {
-  const reader = stream.getReader();
-  const read = async () => {
-    const { done, value } = await reader.read();
-    if (done) {
-      reader.releaseLock();
-      return;
-    }
-    onData(value);
-    read();
-  };
-  read();
-};
 
-export const useRemoteStream = <T>(
-  url: string,
-  onData: (data: T) => void,
-  onError: (err: Error) => void,
-) => {
   useEffect(() => {
-    const stream = new ReadableStream({
+    const abortController = new AbortController();
-      async start(controller) {
+
-        const res = await fetch(url);
+    async function fetchStream() {
-        res.body?.pipeThrough(new TextDecoderStream())
+      try {
-          .pipeTo(
+        const response = await fetch(url, {
-            new WritableStream({
+          signal: abortController.signal
-              write(chunk) {
-                controller.enqueue(chunk);
-              },
-            }),
-          );
-      },
         });
-    useStream(stream, onData, onError);
+        
-  }, [url]);
+        if (!response.ok) throw new Error(`HTTP error! status: ${response.status}`);
-};
+        if (!response.body) throw new Error('ReadableStream not supported');
+
+        const reader = response.body.getReader();
+        const decoder = new TextDecoder();
+        
+        while (true) {
+          const { done, value } = await reader.read();
+          if (done) break;
+          
+          const chunk = decoder.decode(value);
+          const batchItems = chunk.trim().split('\n')
+            .filter(line => line.length > 0)
+            .map(line => transformer?.(line) ?? JSON.parse(line))
+            .flat();
+            
+          setItems(prev => [...prev, ...batchItems]);
+        }
+      } catch (err:any) {
+        if (err?.name === 'AbortError') return;
+        setError(err.message);
+      } finally {
+        setIsLoading(false);
+      }
+    }
+
+    fetchStream();
+    return () => abortController.abort();
+  }, [])
+
+  return [items, isLoading, error];
+}
+

types.ts

@@ -3,5 +3,12 @@ declare global {
     url: URL;
     state: Record<string, unknown>;
     params: Record<string, string | undefined>;
+    headers?: Headers,
+  }
+
+  interface BlockItem {
+    name: string;
+    resourceLocation: string;
+    images: string[];
   }
 }

vite.config.ts

@@ -16,6 +16,7 @@ export default defineConfig({
       "/api": {
         target: "http://localhost:8000",
         changeOrigin: true,
+        ws: true,
       },
       "/puppet": {
         target: "http://localhost:8000",
@@ -29,4 +30,11 @@ export default defineConfig({
       },
     },
   },
+  build: {
+    rollupOptions: {
+      external: [
+        "server",
+      ],
+    },
+  },
 });