qid as number, number and cleanup fromArrow (#197)

Two related goals in this one.

1. It has turned out to be more useful to refer to points by their quadtree id ("qid"), the tuple of the tile number and row number. This makes mouseover function return that tuple instead, and changes the definition to be [number, number] instead of [number, Some<number>]. Internal Atlas code may have to change to accomodate this.
2. The tile Manifest loading code now works on Arrow tiles as well. We are moving to a world where the existence of a manifest is a hard requirement in deepscatter. I think this is OK.
<!-- ELLIPSIS_HIDDEN -->


----

> [!IMPORTANT]
> This PR updates QID handling to use consistent tuples and enhances manifest loading for Arrow tiles, affecting several core functions and tests.
> 
>   - **Behavior**:
>     - `Qid` is now consistently a tuple `[number, number]` instead of `[number, Some<number>]`.
>     - Mouseover functions return QID tuples.
>     - Manifest loading now supports Arrow tiles, making manifests mandatory.
>   - **Functions**:
>     - `getQids()` in `Deeptable.ts` retrieves `StructRowProxy` for QID tuples.
>     - `color_pick()` in `regl_rendering.ts` returns QID tuples.
>     - `set_highlit_points()` in `interaction.ts` uses QID tuples.
>   - **Misc**:
>     - Update `max_points` and `source_url` in `FourClasses.svelte`.
>     - Replace `manifest` with `metadata` in `SelectPoints.svelte` and `dataset.spec.js`.
> 
> <sup>This description was created by </sup>[<img alt="Ellipsis" src="https://img.shields.io/badge/Ellipsis-blue?color=175173">](https://www.ellipsis.dev?ref=nomic-ai%2Fdeepscatter&utm_source=github&utm_medium=referral)<sup> for 62ea242. You can [customize](https://app.ellipsis.dev/nomic-ai/settings/summaries) this summary. It will automatically update as commits are pushed.</sup>


<!-- ELLIPSIS_HIDDEN -->

Author: bmschmidt

dev/FourClasses.svelte

@@ -8,8 +8,8 @@
   import SelectPoints from './svelte/SelectPoints.svelte';
   const startSize = 2;
   const prefs = {
-    source_url: '/newtiles',
-    max_points: 10,
+    source_url: '/tiles',
+    max_points: 10000,
     alpha: 35, // Target saturation for the full page.
     zoom_balance: 0.22, // Rate at which points increase size. https://observablehq.com/@bmschmidt/zoom-strategies-for-huge-scatterplots-with-three-js
     point_size: startSize, // Default point size before application of size scaling

dev/svelte/SelectPoints.svelte

@@ -8,8 +8,8 @@
     const selection = await scatterplot.deeptable.select_data({
       name: Math.random().toFixed(8),
       tileFunction: async (tile) => {
-        const b = new Bitmask(tile.manifest.nPoints);
-        for (let i = 0; i < tile.manifest.nPoints; i++) {
+        const b = new Bitmask(tile.metadata.nPoints);
+        for (let i = 0; i < tile.metadata.nPoints; i++) {
           if (Math.random() < 0.001) {
             b.set(i);
           }

src/Deeptable.ts

@@ -37,7 +37,7 @@ import type {
 import { DataSelection } from './selection';
 import { Some, TupleMap } from './utilityFunctions';
 import { getNestedVector } from './regl_rendering';
-import { tileKey_to_tix } from './tixrixqid';
+import { Qid, tileKey_to_tix } from './tixrixqid';
 
 type TransformationStatus = 'queued' | 'in progress' | 'complete' | 'failed';
 
@@ -962,6 +962,25 @@ export class Deeptable {
       }
     };
   }
+
+  getQids(qids: Qid[]): StructRowProxy[] {
+    // For a set of tile row tuples, returns underlying
+    // structRowProxies. Note that this makes no concrete guarantee about
+    // **which** fields will be loaded in the underlying arrow arrays.
+    const matches : StructRowProxy[] = []
+    for (const [tix, rix] of qids) {
+      if (this.flatTree[tix] === undefined) {
+        // If we can't immediately find the tile, check if it's hiding
+        const tile = this.map(t => t).find(t => t.tix === tix)
+        this.flatTree[tix] = tile || null;        
+      }
+      if (this.flatTree[tix] === null) {
+        throw new Error("Missing tile index " + tix)
+      }
+      matches.push(this.flatTree[tix].record_batch.get(rix))
+    }
+    return matches
+  }
 }
 
 function area(rect: Rectangle) {

src/interaction.ts

@@ -15,6 +15,7 @@ import type { Deeptable } from './Deeptable';
 import type * as DS from './types';
 import type { Scatterplot } from './scatterplot';
 import { PositionalAesthetic } from './aesthetics/ScaledAesthetic';
+import { Qid } from './tixrixqid';
 type Annotation = {
   x: number;
   y: number;
@@ -181,10 +182,12 @@ export class Zoom {
     this.zoomer = zoomer;
   }
 
-  set_highlit_points(data: StructRowProxy[]) {
+  set_highlit_points(dd: Qid[]) {
     const { x_, y_ } = this.scales();
     const xdim = this.scatterplot.dim('x') as PositionalAesthetic;
     const ydim = this.scatterplot.dim('y') as PositionalAesthetic;
+    
+    const data = this.scatterplot.deeptable.getQids(dd)
     this.scatterplot.highlit_point_change(data, this.scatterplot);
 
     const annotations: Annotation[] = data.map((d) => {
@@ -226,10 +229,6 @@ export class Zoom {
       });
   }
 
-  set_highlit_point(point: StructRowProxy) {
-    this.set_highlit_points([point]);
-  }
-
   add_mouseover() {
     let last_fired = 0;
     const renderer: ReglRenderer = this.renderers.get('regl') as ReglRenderer;

src/regl_rendering.ts

@@ -33,7 +33,6 @@ import {
   Data,
   Dictionary,
   Struct,
-  StructRowProxy,
   Type,
   Utf8,
   Vector,
@@ -43,6 +42,7 @@ import { StatefulAesthetic } from './aesthetics/StatefulAesthetic';
 import { Filter, Foreground } from './aesthetics/BooleanAesthetic';
 import { ZoomTransform } from 'd3-zoom';
 import { Some, TupleMap, TupleSet } from './utilityFunctions';
+import { Qid } from './tixrixqid';
 // eslint-disable-next-line import/prefer-default-export
 export class ReglRenderer extends Renderer {
   public regl: Regl;
@@ -546,7 +546,7 @@ export class ReglRenderer extends Renderer {
     return v;
   }
 
-  color_pick(x: number, y: number): null | StructRowProxy {
+  color_pick(x: number, y: number): null | Qid {
     if (y === 0) {
       // Not sure why, but this makes things complainy.
       // console.warn('that thing again.');
@@ -561,15 +561,7 @@ export class ReglRenderer extends Renderer {
     if (row_number === -1) {
       return null;
     }
-    for (const tile of this.visible_tiles()) {
-      if (tile.tix === tile_number) {
-        return tile.record_batch.get(row_number);
-      }
-    }
-    return null;
-    //    const p = this.tileSet.findPoint(point_as_int);
-    //    if (p.length === 0) { return; }
-    //    return p[0];
+    return [tile_number, row_number]
   }
 
   color_pick_single(

src/tile.ts

@@ -78,7 +78,7 @@ export class Tile {
    * @param key Either the string identifier of the tile,
    * OR a `TileManifest` object including an identifier.   *
    * @param parent The parent tile -- used to navigate through the tree.
-   * @param deeptable The full atlas deeptable of which this tile is a part.
+   * @param deeptable The full deepscatter deeptable of which this tile is a part.
    */
   constructor(key: string, parent: Tile | null, deeptable: Deeptable) {
     // If it's just initiated with a key, build that into a minimal manifest.
@@ -295,16 +295,16 @@ export class Tile {
   }
 
   set metadata(
-    manifest: TileMetadata | (TileMetadata & { extent: Rectangle }),
+    metadata: TileMetadata | (TileMetadata & { extent: Rectangle }),
   ) {
     // Setting the manifest is the thing that spawns children.
-    this.highest_known_ix = manifest.max_ix;
+    this.highest_known_ix = metadata.max_ix;
     this._metadata = {
-      ...manifest,
+      ...metadata,
       extent:
-        typeof manifest.extent === 'string'
-          ? (JSON.parse(manifest.extent) as Rectangle)
-          : manifest.extent,
+        typeof metadata.extent === 'string'
+          ? (JSON.parse(metadata.extent) as Rectangle)
+          : metadata.extent,
     };
   }
 

src/tixrixqid.ts

@@ -8,7 +8,7 @@ import type {
 } from 'apache-arrow';
 
 import type { Tile } from './deepscatter';
-import { Bitmask, DataSelection, Deeptable } from './deepscatter';
+import { DataSelection, Deeptable } from './deepscatter';
 
 // The type below indicates that a Qid is not valid if
 // there are zero rows selected in the tile.

src/types.ts

@@ -11,6 +11,7 @@ import type {
   Timestamp,
   Utf8,
   Vector,
+  TypeMap,
 } from 'apache-arrow';
 import type { Renderer } from './rendering';
 import type { Deeptable } from './Deeptable';
@@ -45,6 +46,9 @@ export type PointFunction<T = number> = (p: StructRowProxy) => T;
  * A proxy class that wraps around tile get calls. Used to avoid
  * putting Nomic login logic in deepscatter while fetching
  * tiles with authentication.
+ * 
+ * An API call type must return Uint8Arrays that represent arrow 
+ * tables of the type produced by quadfeather.
  *
  */
 export interface TileProxy {
@@ -101,7 +105,7 @@ export type DeeptableCreateParams = {
 
   // A manifest listing all the tiles in the deeptable, of the type created by
   // quadfeather v2.0.0 or greater.
-  tileManifest?: Table;
+  tileManifest?: Table<{key: Utf8}>;
 
   // A URL for an arrow file manifest. The schema for this manifest
   // is not yet publically documented: I hope to bundle it into the

src/wrap_arrow.ts

@@ -8,12 +8,15 @@ import {
   vectorFromArray,
   Vector,
   Float,
+  tableFromArrays,
+  Utf8,
 } from 'apache-arrow';
 import { Deeptable } from './Deeptable';
 import { add_or_delete_column } from './Deeptable';
 import type * as DS from './types';
-import { extent } from 'd3-array';
+import { extent, extent, range } from 'd3-array';
 import { Rectangle } from './tile';
+import { tixToZxy } from './tixrixqid';
 
 /**
  * This function is used to wrap an arrow table into a
@@ -31,22 +34,36 @@ export function wrapArrowTable(
 ): Deeptable {
   let tb = tableFromIPC(tbArray);
   let batches = tb.batches;
+  const minIx = []
+  const maxIx = []
+  // Extents of each tile, as JSON.
+  const extents : string[] = []
   if (tb.getChild('ix') === null) {
     let rowNum = 0;
     batches = batches.map((batch) => {
       if (batch.numRows > 2 ** 16) {
         throw new Error(
-          'Arrow record batches temporarily limited to 2^16 rows.',
+          'Arrow record batches limited to 2^16 rows.',
         );
       }
       const array = new Int32Array(batch.numRows);
       for (let i = 0; i < batch.numRows; i++) {
         array[i] = rowNum++;
       }
-      return add_or_delete_column(batch, 'ix', vectorFromArray(array));
+      return add_or_delete_column(batch, 'ix', vectorFromArray(array));      
     });
     tb = new Table(batches);
   }
+  for (const batch of batches) {
+    minIx.push(batch.get(0)['ix'])
+    maxIx.push(batch.get(batch.numRows - 1)['ix'])
+    extents.push(
+      JSON.stringify({
+        x: extent(batch.getChild('x')),
+        y: extent(batch.getChild('y'))
+      })
+    )
+  }
 
   const proxy = new ArrowProxy(batches);
 
@@ -65,12 +82,25 @@ export function wrapArrowTable(
     y: extent([...(y as Iterable<number>)]),
   } as Rectangle;
 
+
+  const tileManifest = tableFromArrays({
+    // @ts-expect-error missing types for tableFromArrays in arrow js
+    key: vectorFromArray(range(batches.length).map(t => tixToZxy(t).join('/')), new Utf8()),
+    min_ix: minIx,
+    max_ix: maxIx,
+    nPoints: batches.map(d => d.numRows),
+    // @ts-expect-error missing types for tableFromArrays in arrow js
+    extent: vectorFromArray(extents, new Utf8())
+  })
+
   return new Deeptable({
     baseUrl: `feather://table`,
     plot,
     tileProxy: proxy,
     tileStructure: 'other',
     extent: dataExtent,
+    // @ts-expect-error missing types for tableFromArrays in arrow js
+    tileManifest
   });
 }
 

tests/dataset.spec.js

@@ -28,7 +28,7 @@ test('Columns can be deleted and replaced', async () => {
 
   dataset.transformations['integers'] = async function (tile) {
     await tile.populateManifest();
-    return new Float32Array(tile.manifest.nPoints);
+    return new Float32Array(tile.metadata.nPoints);
   };
 
   dataset.deleteColumn('integers');