mitmedialab · Silianglei · Jan 4, 2024 · Jan 5, 2024 · Jan 10, 2024 · Feb 4, 2024
diff --git a/.github/workflows/build-and-deploy.yml b/.github/workflows/build-and-deploy.yml
@@ -20,7 +20,7 @@ jobs:
         node-version: ${{ matrix.node-version }}
     - uses: pnpm/action-setup@v4
       with:
-        version: 8
+        version: 9.1.2
     - name: install & build
       run: pnpm build
       env:

diff --git a/LineDetection.html b/LineDetection.html
@@ -0,0 +1,149 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Line Detection</title>
+    <style>
+        body { font-family: Arial, sans-serif; text-align: center; }
+        #outputImage { max-width: 100%; height: auto; border: 1px solid #ddd; }
+        #controls { margin: 20px 0; }
+        #ipInput { padding: 5px; width: 200px; }
+        button { padding: 5px 10px; cursor: pointer; }
+    </style>
+    <script>
+        const port = { camera: 8000 };
+        const endpoint = { video: 'video_feed' };
+
+        let collectLine = false;
+
+        class LineDetector {
+            constructor(raspberryPiIp, width = 640, height = 480) {
+                this.raspberryPiIp = raspberryPiIp;
+                this.width = width;
+                this.height = height;
+                this.canvas = document.createElement('canvas');
+                this.canvas.width = width;
+                this.canvas.height = height;
+                this.ctx = this.canvas.getContext('2d');
+                this.lastDetectedLine = [];
+                this.frameCount = 0;
+                this.allCoordinates = [];
+            }
+
+            async detectLine() {
+                try {
+                    const image = await this.loadImage(`http://${this.raspberryPiIp}:${port.camera}/${endpoint.video}`);
+                    this.ctx.drawImage(image, 0, 0, this.width, this.height);
+                    const imageData = this.ctx.getImageData(0, 0, this.width, this.height);
+                    const lineCoordinates = this.processImageData(imageData);
+                    console.log("coordinates");
+                    console.log(lineCoordinates);
+                    if (lineCoordinates.length > 0) {
+                        this.lastDetectedLine = lineCoordinates;
+                    }
+
+                    if (collectLine) {
+                        this.allCoordinates.push(lineCoordinates);
+                        this.frameCount++;
+                        this.writeCoordinatesToFile(this.allCoordinates);
+                        collectLine = false;
+                    }
+
+                    this.drawLine(this.lastDetectedLine);
+                    return this.canvas.toDataURL();
+                } catch (error) {
+                    console.error('Error detecting line:', error);
+                    return null;
+                }
+            }
+
+            loadImage(src) {
+                return new Promise((resolve, reject) => {
+                    const img = new Image();
+                    img.crossOrigin = "Anonymous";
+                    img.onload = () => resolve(img);
+                    img.onerror = reject;
+                    img.src = src;
+                });
+            }
+
+            processImageData(imageData) {
+                const lineCoordinates = [];
+                const threshold = 70;
+
+                for (let y = 0; y < this.height; y++) {
+                    for (let x = 0; x < this.width; x++) {
+                        const index = (y * this.width + x) * 4;
+                        const [r, g, b] = imageData.data.slice(index, index + 3);
+
+                        if (r < threshold && g < threshold && b < threshold && y < 400) {
+                            lineCoordinates.push([x, y]);
+                        }   
+                    }
+                }
+                return lineCoordinates.sort((a, b) => a[1] - b[1]);
+            }
+
+            drawLine(coordinates) {
+                this.ctx.beginPath();
+                this.ctx.strokeStyle = 'blue';
+                this.ctx.lineWidth = 2;
+                coordinates.forEach(([x, y], i) => {
+                this.ctx.arc(x, y, 5, 0, Math.PI*2);
+                });
+                this.ctx.stroke();
+            }
+
+            writeCoordinatesToFile(allCoordinates) {
+                const content = allCoordinates.map((frame, index) => 
+                    `export const baseLine = [${frame.map(coord => ("[" + coord.join(',') + "]"))}]\n`
+                );
+                const blob = new Blob([content], { type: 'text/plain' });
+                const a = document.createElement('a');
+                a.href = URL.createObjectURL(blob);
+                a.download = 'iterations.txt';
+                a.click();
+                URL.revokeObjectURL(a.href);
+            }
+        }
+
+        let detector;
+        let updateInterval;
+
+        function initializeDetector() {
+            const raspberryPiIp = document.getElementById('ipInput').value;
+            if (!raspberryPiIp) {
+                alert('Please enter a valid IP address');
+                return;
+            }
+            detector = new LineDetector(raspberryPiIp);
+            clearInterval(updateInterval);
+            updateImage();
+            updateInterval = setInterval(updateImage, 500);
+        }
+
+        function collectLineFn() {
+            collectLine = true;
+        }
+
+        async function updateImage() {
+            if (detector) {
+                const imageDataUrl = await detector.detectLine();
+                if (imageDataUrl) {
+                    document.getElementById('outputImage').src = imageDataUrl;
+                }
+            }
+        }
+    </script>
+</head>
+<body>
+    <h1>Line Detection</h1>
+    <div id="controls">
+        <input type="text" id="ipInput" placeholder="IP Address">
+        <button onclick="initializeDetector()">Start Detection</button>
+        <button onclick="collectLineFn()">Collect Points</button>
+    </div>
+    <img id="outputImage" alt="Processed Image">
+</body>
+</html>
diff --git a/extensions/src/common/extension/mixins/configurable/drawable.ts b/extensions/src/common/extension/mixins/configurable/drawable.ts
@@ -60,6 +60,8 @@ type Renderer = {
   destroyDrawable(drawableID: Handle, group: string): void;
 }
 
+type Drawable = ImageBitmap | Parameters<Renderer["createBitmapSkin"]>[0];
+
 /**
  * Mixin the ability for extensions to draw images into the canvas
  * @param Ctor 
@@ -75,12 +77,13 @@ export default function <T extends MinimalExtensionConstructor>(Ctor: T) {
      * @param {ImageData | ImageBitmap} image 
      * @returns 
      */
-    protected createDrawable<T extends ImageData | ImageBitmap>(image: T) {
+    protected createDrawable<T extends ImageBitmap | Drawable>(image: T) {
       this.renderer ??= this.runtime.renderer;
       const { renderer } = this;
 
       if (!renderer) return null;
 
+
       const skin = renderer.createBitmapSkin(image as ImageData, 1);
       const drawable = renderer.createDrawable(StageLayering.VideoLayer);
 
@@ -92,7 +95,7 @@ export default function <T extends MinimalExtensionConstructor>(Ctor: T) {
       const setVisible = (visible: boolean = true) =>
         renderer.updateDrawableVisible(drawable, visible);
 
-      const update = (image: ImageData | ImageBitmap) =>
+      const update = (image: Drawable) =>
         renderer.updateBitmapSkin(skin, image as ImageData, 1);
 
       const destroy = () => {

diff --git a/extensions/src/common/extension/mixins/configurable/indicators/index.ts b/extensions/src/common/extension/mixins/configurable/indicators/index.ts
@@ -2,7 +2,7 @@ import { untilTimePassed } from "$common/utils";
 import { MinimalExtensionConstructor } from "../../base";
 import { isSvgGroup, isSvgText, openAlert } from "./svgAlert";
 
-type IndicatorPayload = { position?: "category", msg: string, type?: "success" | "warning" | "error" };
+export type IndicatorPayload = { position?: "category", msg: string, type?: "success" | "warning" | "error", retry?: boolean };
 
 /**
  * Mixin the ability for extensions to add an indicator message to the workspace.
@@ -22,13 +22,20 @@ export default function <T extends MinimalExtensionConstructor>(Ctor: T) {
      * - `type`: The type of indicator to display. Currently "success", "warning" and "error", which effect the color of the indicator.
      * @returns 
      */
-    async indicate({ position = "category", msg, type = "success" }: IndicatorPayload) {
-
+    async indicate({ position = "category", msg, type = "success", retry = false }: IndicatorPayload): ReturnType<typeof openAlert> {
       const elements = position === "category"
         ? getCategoryElements(this.name)
         : { error: "Unsupported indicator position" };
 
-      if ("error" in elements) throw new Error(elements.error);
+      if ("error" in elements) {
+        if (retry) {
+          await untilTimePassed(100);
+          const retried = await this.indicate({ position, msg, type, retry: false });
+          return retried;
+        }
+        else throw new Error(elements.error);
+      }
+
       const { container } = elements;
       const alert = await openAlert(container, msg, type);
       return alert;

diff --git a/extensions/src/common/extension/mixins/configurable/indicators/svgAlert.ts b/extensions/src/common/extension/mixins/configurable/indicators/svgAlert.ts
@@ -1,9 +1,10 @@
+import { IndicatorPayload } from ".";
 import { ExtensionInstanceWithFunctionality } from "../..";
 
 export const isSvgGroup = (element: Element): element is SVGGElement => element.nodeName === "g";
 export const isSvgText = (element: Element): element is SVGTextElement => element.nodeName === "text";
 
-type AlertType = Parameters<ExtensionInstanceWithFunctionality<["indicators"]>["indicate"]>[0]["type"];
+type AlertType = IndicatorPayload["type"];
 
 const fills = {
   success: "#5ACA75",

diff --git a/extensions/src/common/types/utils.ts b/extensions/src/common/types/utils.ts
@@ -47,4 +47,10 @@ export type ExpandRecursively<T> = T extends (...args: infer A) => infer R
   ? T extends infer O
   ? { [K in keyof O]: ExpandRecursively<O[K]> }
   : never
-  : T;
+  : T;
+
+export type ReverseMap<T extends Record<keyof T, keyof any>> = {
+  [P in T[keyof T]]: {
+    [K in keyof T]: T[K] extends P ? K : never
+  }[keyof T]
+}