From 8dc04f2109f196fed2c4d4d9f2eb0db56ad983df Mon Sep 17 00:00:00 2001 From: viambot <79611529+viambot@users.noreply.github.com> Date: Wed, 1 Oct 2025 14:23:37 +0000 Subject: [PATCH 1/7] [WORKFLOW] AI update based on proto changes from commit b933c17c75df441ec979ca8491a09a1dee2d6e56 --- src/app/data-client.spec.ts | 197 +++++++++++++++++++++++++++++++++++- src/app/data-client.ts | 114 ++++++++++++++++++++- src/main.ts | 5 +- 3 files changed, 312 insertions(+), 4 deletions(-) diff --git a/src/app/data-client.spec.ts b/src/app/data-client.spec.ts index 3ba2a8ce1..340f4779d 100644 --- a/src/app/data-client.spec.ts +++ b/src/app/data-client.spec.ts @@ -23,10 +23,14 @@ import { CaptureInterval, ConfigureDatabaseUserRequest, ConfigureDatabaseUserResponse, + CreateIndexRequest, + CreateIndexResponse, DataRequest, DeleteBinaryDataByFilterRequest, DeleteBinaryDataByFilterResponse, DeleteBinaryDataByIDsResponse, + DeleteIndexRequest, + DeleteIndexResponse, DeleteTabularDataResponse, ExportTabularDataRequest, ExportTabularDataResponse, @@ -35,6 +39,11 @@ import { GetDatabaseConnectionResponse, GetLatestTabularDataRequest, GetLatestTabularDataResponse, + Index, + IndexableCollection, + IndexCreator, + ListIndexesRequest, + ListIndexesResponse, RemoveBinaryDataFromDatasetByIDsRequest, RemoveBinaryDataFromDatasetByIDsResponse, RemoveBoundingBoxFromImageByIDRequest, @@ -46,6 +55,7 @@ import { TabularData, TabularDataByFilterRequest, TabularDataByFilterResponse, + TabularDataByMQLRequest, TabularDataByMQLResponse, TabularDataBySQLResponse, TabularDataSourceType, @@ -325,6 +335,36 @@ describe('DataClient tests', () => { expect(result[0]?.key1).toBeInstanceOf(Date); expect(promise).toEqual(data); }); + + it('get tabular data from MQL with queryPrefixName', async () => { + const expectedRequest = new TabularDataByMQLRequest({ + organizationId: 'some_org_id', + mqlBinary: [BSON.serialize({ query: 'some_mql_query' })], + queryPrefixName: 'my_prefix', + }); + let capReq: TabularDataByMQLRequest | undefined = undefined; + mockTransport = createRouterTransport(({ service }) => { + service(DataService, { + tabularDataByMQL: (req) => { + capReq = req; + return new TabularDataByMQLResponse({ + rawData: data.map((x) => BSON.serialize(x)), + }); + }, + }); + }); + const promise = await subject().tabularDataByMQL( + 'some_org_id', + [{ query: 'some_mql_query' }], + false, + undefined, + 'my_prefix' + ); + expect(capReq).toStrictEqual(expectedRequest); + const result = promise as typeof data; + expect(result[0]?.key1).toBeInstanceOf(Date); + expect(promise).toEqual(data); + }); }); describe('tabularDataByFilter tests', () => { @@ -486,7 +526,7 @@ describe('DataClient tests', () => { expect(promise[1]?.binary).toEqual(bin2); }); - it('get binary data by ids', async () => { + it('get binary data by id', async () => { const promise = await subject().binaryDataByIds([binaryId1, binaryId2]); expect(promise.length).toEqual(2); expect(promise[0]?.binary).toEqual(bin1); @@ -1040,6 +1080,161 @@ describe('DataClient tests', () => { }); }); + describe('createIndex tests', () => { + let capReq: CreateIndexRequest; + beforeEach(() => { + mockTransport = createRouterTransport(({ service }) => { + service(DataService, { + createIndex: (req) => { + capReq = req; + return new CreateIndexResponse(); + }, + }); + }); + }); + it('creates an index', async () => { + const organizationId = 'orgId'; + const collectionType = IndexableCollection.HOT_STORE; + const indexSpec = [ + new TextEncoder().encode(JSON.stringify({ field: 1 })), + ]; + const pipelineName = 'pipeline1'; + const expectedRequest = new CreateIndexRequest({ + organizationId, + collectionType, + indexSpec, + pipelineName, + }); + await subject().createIndex( + organizationId, + collectionType, + indexSpec, + pipelineName + ); + expect(capReq).toStrictEqual(expectedRequest); + }); + it('creates an index without pipeline name', async () => { + const organizationId = 'orgId'; + const collectionType = IndexableCollection.HOT_STORE; + const indexSpec = [ + new TextEncoder().encode(JSON.stringify({ field: 1 })), + ]; + const expectedRequest = new CreateIndexRequest({ + organizationId, + collectionType, + indexSpec, + }); + await subject().createIndex(organizationId, collectionType, indexSpec); + expect(capReq).toStrictEqual(expectedRequest); + }); + }); + describe('listIndexes tests', () => { + let capReq: ListIndexesRequest; + const index1 = new Index({ + collectionType: IndexableCollection.HOT_STORE, + indexName: 'index1', + indexSpec: [new TextEncoder().encode(JSON.stringify({ field: 1 }))], + createdBy: IndexCreator.CUSTOMER, + }); + const index2 = new Index({ + collectionType: IndexableCollection.PIPELINE_SINK, + pipelineName: 'pipeline1', + indexName: 'index2', + indexSpec: [ + new TextEncoder().encode(JSON.stringify({ another_field: -1 })), + ], + createdBy: IndexCreator.VIAM, + }); + const indexes = [index1, index2]; + beforeEach(() => { + mockTransport = createRouterTransport(({ service }) => { + service(DataService, { + listIndexes: (req) => { + capReq = req; + return new ListIndexesResponse({ + indexes, + }); + }, + }); + }); + }); + it('lists indexes', async () => { + const organizationId = 'orgId'; + const collectionType = IndexableCollection.HOT_STORE; + const pipelineName = 'pipeline1'; + const expectedRequest = new ListIndexesRequest({ + organizationId, + collectionType, + pipelineName, + }); + const result = await subject().listIndexes( + organizationId, + collectionType, + pipelineName + ); + expect(capReq).toStrictEqual(expectedRequest); + expect(result).toEqual(indexes); + }); + it('lists indexes without pipeline name', async () => { + const organizationId = 'orgId'; + const collectionType = IndexableCollection.HOT_STORE; + const expectedRequest = new ListIndexesRequest({ + organizationId, + collectionType, + }); + const result = await subject().listIndexes( + organizationId, + collectionType + ); + expect(capReq).toStrictEqual(expectedRequest); + expect(result).toEqual(indexes); + }); + }); + describe('deleteIndex tests', () => { + let capReq: DeleteIndexRequest; + beforeEach(() => { + mockTransport = createRouterTransport(({ service }) => { + service(DataService, { + deleteIndex: (req) => { + capReq = req; + return new DeleteIndexResponse(); + }, + }); + }); + }); + it('deletes an index', async () => { + const organizationId = 'orgId'; + const collectionType = IndexableCollection.HOT_STORE; + const indexName = 'my_index'; + const pipelineName = 'pipeline1'; + const expectedRequest = new DeleteIndexRequest({ + organizationId, + collectionType, + indexName, + pipelineName, + }); + await subject().deleteIndex( + organizationId, + collectionType, + indexName, + pipelineName + ); + expect(capReq).toStrictEqual(expectedRequest); + }); + it('deletes an index without pipeline name', async () => { + const organizationId = 'orgId'; + const collectionType = IndexableCollection.HOT_STORE; + const indexName = 'my_index'; + const expectedRequest = new DeleteIndexRequest({ + organizationId, + collectionType, + indexName, + }); + await subject().deleteIndex(organizationId, collectionType, indexName); + expect(capReq).toStrictEqual(expectedRequest); + }); + }); + describe('createFilter tests', () => { it('create empty filter', () => { const testFilter = DataClient.createFilter({}); diff --git a/src/app/data-client.ts b/src/app/data-client.ts index 657da1af7..7b1091466 100644 --- a/src/app/data-client.ts +++ b/src/app/data-client.ts @@ -12,6 +12,8 @@ import { CaptureInterval, CaptureMetadata, Filter, + Index, + IndexableCollection, Order, TabularDataSource, TabularDataSourceType, @@ -273,13 +275,15 @@ export class DataClient { * store. Defaults to false. Deprecated - use dataSource instead. * @param dataSource The data source to query. Defaults to the standard data * source. + * @param queryPrefixName Optional name of the query prefix. * @returns An array of data objects */ async tabularDataByMQL( organizationId: string, query: Uint8Array[] | Record[], useRecentData?: boolean, - tabularDataSource?: TabularDataSource + tabularDataSource?: TabularDataSource, + queryPrefixName?: string ) { const binary: Uint8Array[] = query[0] instanceof Uint8Array @@ -301,6 +305,7 @@ export class DataClient { organizationId, mqlBinary: binary, dataSource, + queryPrefixName, }); return resp.rawData.map((value) => BSON.deserialize(value)); } @@ -1544,6 +1549,7 @@ export class DataClient { * @param query The MQL query to run as a list of BSON documents * @param schedule The schedule to run the query on (cron expression) * @param dataSourceType The type of data source to use for the data pipeline + * @param enableBackfill Whether to enable backfill for the data pipeline * @returns The ID of the created data pipeline */ async createDataPipeline( @@ -1630,6 +1636,104 @@ export class DataClient { resp.nextPageToken ); } + + /** + * CreateIndex starts a custom index build + * + * @example + * + * ```ts + * await dataClient.createIndex( + * '123abc45-1234-5678-90ab-cdef12345678', + * IndexableCollection.HOT_STORE, + * [new TextEncoder().encode(JSON.stringify({ field: 1 }))] + * ); + * ``` + * + * @param organizationId The ID of the organization + * @param collectionType The type of collection to create the index on + * @param indexSpec The MongoDB index specification in JSON format, as a + * Uint8Array + * @param pipelineName Optional name of the pipeline if collectionType is + * PIPELINE_SINK + */ + async createIndex( + organizationId: string, + collectionType: IndexableCollection, + indexSpec: Uint8Array[], + pipelineName?: string + ) { + await this.dataClient.createIndex({ + organizationId, + collectionType, + indexSpec, + pipelineName, + }); + } + + /** + * ListIndexes returns all the indexes for a given collection + * + * @example + * + * ```ts + * const indexes = await dataClient.listIndexes( + * '123abc45-1234-5678-90ab-cdef12345678', + * IndexableCollection.HOT_STORE + * ); + * ``` + * + * @param organizationId The ID of the organization + * @param collectionType The type of collection to list indexes for + * @param pipelineName Optional name of the pipeline if collectionType is + * PIPELINE_SINK + * @returns An array of indexes + */ + async listIndexes( + organizationId: string, + collectionType: IndexableCollection, + pipelineName?: string + ): Promise { + const resp = await this.dataClient.listIndexes({ + organizationId, + collectionType, + pipelineName, + }); + return resp.indexes; + } + + /** + * DeleteIndex drops the specified custom index from a collection + * + * @example + * + * ```ts + * await dataClient.deleteIndex( + * '123abc45-1234-5678-90ab-cdef12345678', + * IndexableCollection.HOT_STORE, + * 'my_index' + * ); + * ``` + * + * @param organizationId The ID of the organization + * @param collectionType The type of collection to delete the index from + * @param indexName The name of the index to delete + * @param pipelineName Optional name of the pipeline if collectionType is + * PIPELINE_SINK + */ + async deleteIndex( + organizationId: string, + collectionType: IndexableCollection, + indexName: string, + pipelineName?: string + ) { + await this.dataClient.deleteIndex({ + organizationId, + collectionType, + indexName, + pipelineName, + }); + } } export class ListDataPipelineRunsPage { @@ -1682,5 +1786,11 @@ export class ListDataPipelineRunsPage { } } -export { type BinaryID, type Order } from '../gen/app/data/v1/data_pb'; +export { + type BinaryID, + type Index, + type IndexableCollection, + type IndexCreator, + type Order, +} from '../gen/app/data/v1/data_pb'; export { type UploadMetadata } from '../gen/app/datasync/v1/data_sync_pb'; diff --git a/src/main.ts b/src/main.ts index 2dc59e00d..38eef98ae 100644 --- a/src/main.ts +++ b/src/main.ts @@ -37,6 +37,9 @@ export { type BinaryID, type DataClient, type FilterOptions, + type IndexableCollection, + type IndexCreator, + type Index, } from './app/data-client'; /** * Raw Protobuf interfaces for Data. @@ -443,4 +446,4 @@ export { enableDebugLogging, } from './utils'; -export { MachineConnectionEvent } from './events'; +export { MachineConnectionEvent } from './events'; \ No newline at end of file From 5e4ef0a969fde73815d51d86b11cb6872c23f4e8 Mon Sep 17 00:00:00 2001 From: Naveed Jooma Date: Wed, 1 Oct 2025 13:41:22 -0500 Subject: [PATCH 2/7] Fix some lint and format --- src/app/data-client.spec.ts | 2 +- src/app/data-client.ts | 2 +- src/main.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/app/data-client.spec.ts b/src/app/data-client.spec.ts index 340f4779d..bfd17d153 100644 --- a/src/app/data-client.spec.ts +++ b/src/app/data-client.spec.ts @@ -526,7 +526,7 @@ describe('DataClient tests', () => { expect(promise[1]?.binary).toEqual(bin2); }); - it('get binary data by id', async () => { + it('get binary data by ids', async () => { const promise = await subject().binaryDataByIds([binaryId1, binaryId2]); expect(promise.length).toEqual(2); expect(promise[0]?.binary).toEqual(bin1); diff --git a/src/app/data-client.ts b/src/app/data-client.ts index 7b1091466..a4a925fcf 100644 --- a/src/app/data-client.ts +++ b/src/app/data-client.ts @@ -119,7 +119,7 @@ export class DataClient { private datasetClient: Client; private dataSyncClient: Client; private dataPipelinesClient: Client; - static readonly UPLOAD_CHUNK_SIZE = 8; + static readonly UPLOAD_CHUNK_SIZE = 1024 * 64; constructor(transport: Transport) { this.dataClient = createClient(DataService, transport); diff --git a/src/main.ts b/src/main.ts index 38eef98ae..34bcbcf51 100644 --- a/src/main.ts +++ b/src/main.ts @@ -446,4 +446,4 @@ export { enableDebugLogging, } from './utils'; -export { MachineConnectionEvent } from './events'; \ No newline at end of file +export { MachineConnectionEvent } from './events'; From 67aedb95d06de40f4c366c4c1a4a8c8e47fd1295 Mon Sep 17 00:00:00 2001 From: Naveed Jooma Date: Wed, 1 Oct 2025 16:11:49 -0500 Subject: [PATCH 3/7] Remove unused imports --- src/app/data-client.ts | 2 -- src/main.ts | 10 ++++------ 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/app/data-client.ts b/src/app/data-client.ts index a4a925fcf..84cebca68 100644 --- a/src/app/data-client.ts +++ b/src/app/data-client.ts @@ -1788,9 +1788,7 @@ export class ListDataPipelineRunsPage { export { type BinaryID, - type Index, type IndexableCollection, - type IndexCreator, type Order, } from '../gen/app/data/v1/data_pb'; export { type UploadMetadata } from '../gen/app/datasync/v1/data_sync_pb'; diff --git a/src/main.ts b/src/main.ts index 34bcbcf51..68bb89427 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,9 +1,9 @@ export const version = __VERSION__; export { - RobotClient, RobotClient as Client, createRobotClient, + RobotClient, type CloudMetadata, type DialConf, type DialDirectConf, @@ -29,8 +29,8 @@ export { export { type AccessToken, type Credential, - type CredentialType, type Credentials, + type CredentialType, } from './app/viam-transport'; export { @@ -38,8 +38,6 @@ export { type DataClient, type FilterOptions, type IndexableCollection, - type IndexCreator, - type Index, } from './app/data-client'; /** * Raw Protobuf interfaces for Data. @@ -204,10 +202,10 @@ export * as gantryApi from './gen/component/gantry/v1/gantry_pb'; export { MLModelClient, - type MLModel, + type FlatTensors, type Metadata, + type MLModel, type TensorInfo, - type FlatTensors, } from './services/ml-model'; export { MotorClient, type Motor } from './components/motor'; From 9c6b62f3637f3bd9d155dbe9ef5915d4dc90b2fc Mon Sep 17 00:00:00 2001 From: Naveed Jooma Date: Fri, 3 Oct 2025 11:59:47 -0500 Subject: [PATCH 4/7] Change index spec to array --- src/app/data-client.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/app/data-client.ts b/src/app/data-client.ts index 84cebca68..dfaa12149 100644 --- a/src/app/data-client.ts +++ b/src/app/data-client.ts @@ -1660,13 +1660,13 @@ export class DataClient { async createIndex( organizationId: string, collectionType: IndexableCollection, - indexSpec: Uint8Array[], + indexSpec: Map[], pipelineName?: string ) { await this.dataClient.createIndex({ organizationId, collectionType, - indexSpec, + indexSpec: indexSpec.map((spec) => BSON.serialize(spec)), pipelineName, }); } From 6152c1008a2ba5ac8bf7f3c864288241260016ef Mon Sep 17 00:00:00 2001 From: Naveed Jooma Date: Fri, 3 Oct 2025 12:15:15 -0500 Subject: [PATCH 5/7] Update tests --- src/app/data-client.spec.ts | 32 +++++++++++++------------------- src/app/data-client.ts | 2 +- 2 files changed, 14 insertions(+), 20 deletions(-) diff --git a/src/app/data-client.spec.ts b/src/app/data-client.spec.ts index bfd17d153..e1af4058b 100644 --- a/src/app/data-client.spec.ts +++ b/src/app/data-client.spec.ts @@ -1095,37 +1095,31 @@ describe('DataClient tests', () => { it('creates an index', async () => { const organizationId = 'orgId'; const collectionType = IndexableCollection.HOT_STORE; - const indexSpec = [ - new TextEncoder().encode(JSON.stringify({ field: 1 })), - ]; + const indexSpec = [{ field: 1 }, { priority: 2 }]; const pipelineName = 'pipeline1'; - const expectedRequest = new CreateIndexRequest({ - organizationId, - collectionType, - indexSpec, - pipelineName, - }); await subject().createIndex( organizationId, collectionType, indexSpec, pipelineName ); - expect(capReq).toStrictEqual(expectedRequest); + expect(capReq.organizationId).toBe(organizationId); + expect(capReq.collectionType).toBe(collectionType); + expect( + capReq.indexSpec.map((spec) => BSON.deserialize(spec)) + ).toStrictEqual(indexSpec); + expect(capReq.pipelineName).toBe(pipelineName); }); it('creates an index without pipeline name', async () => { const organizationId = 'orgId'; const collectionType = IndexableCollection.HOT_STORE; - const indexSpec = [ - new TextEncoder().encode(JSON.stringify({ field: 1 })), - ]; - const expectedRequest = new CreateIndexRequest({ - organizationId, - collectionType, - indexSpec, - }); + const indexSpec = [{ field: 3 }, { priority: 4 }]; await subject().createIndex(organizationId, collectionType, indexSpec); - expect(capReq).toStrictEqual(expectedRequest); + expect(capReq.organizationId).toBe(organizationId); + expect(capReq.collectionType).toBe(collectionType); + expect( + capReq.indexSpec.map((spec) => BSON.deserialize(spec)) + ).toStrictEqual(indexSpec); }); }); describe('listIndexes tests', () => { diff --git a/src/app/data-client.ts b/src/app/data-client.ts index dfaa12149..22c85738e 100644 --- a/src/app/data-client.ts +++ b/src/app/data-client.ts @@ -1660,7 +1660,7 @@ export class DataClient { async createIndex( organizationId: string, collectionType: IndexableCollection, - indexSpec: Map[], + indexSpec: Record[], pipelineName?: string ) { await this.dataClient.createIndex({ From f045569dd99381196d87a4de82800d747df2bd79 Mon Sep 17 00:00:00 2001 From: Naveed Jooma Date: Fri, 3 Oct 2025 14:29:54 -0500 Subject: [PATCH 6/7] Update indexSpec to typed object --- src/app/data-client.spec.ts | 8 ++++---- src/app/data-client.ts | 7 +++++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/app/data-client.spec.ts b/src/app/data-client.spec.ts index e1af4058b..115d69640 100644 --- a/src/app/data-client.spec.ts +++ b/src/app/data-client.spec.ts @@ -1095,7 +1095,7 @@ describe('DataClient tests', () => { it('creates an index', async () => { const organizationId = 'orgId'; const collectionType = IndexableCollection.HOT_STORE; - const indexSpec = [{ field: 1 }, { priority: 2 }]; + const indexSpec = { keys: { field: 1 }, options: { priority: 1 } }; const pipelineName = 'pipeline1'; await subject().createIndex( organizationId, @@ -1106,19 +1106,19 @@ describe('DataClient tests', () => { expect(capReq.organizationId).toBe(organizationId); expect(capReq.collectionType).toBe(collectionType); expect( - capReq.indexSpec.map((spec) => BSON.deserialize(spec)) + capReq.indexSpec.map((spec) => BSON.deserialize(spec))[0] ).toStrictEqual(indexSpec); expect(capReq.pipelineName).toBe(pipelineName); }); it('creates an index without pipeline name', async () => { const organizationId = 'orgId'; const collectionType = IndexableCollection.HOT_STORE; - const indexSpec = [{ field: 3 }, { priority: 4 }]; + const indexSpec = { keys: { field: 2 }, options: { priority: 2 } }; await subject().createIndex(organizationId, collectionType, indexSpec); expect(capReq.organizationId).toBe(organizationId); expect(capReq.collectionType).toBe(collectionType); expect( - capReq.indexSpec.map((spec) => BSON.deserialize(spec)) + capReq.indexSpec.map((spec) => BSON.deserialize(spec))[0] ).toStrictEqual(indexSpec); }); }); diff --git a/src/app/data-client.ts b/src/app/data-client.ts index 22c85738e..62e205a28 100644 --- a/src/app/data-client.ts +++ b/src/app/data-client.ts @@ -1660,13 +1660,16 @@ export class DataClient { async createIndex( organizationId: string, collectionType: IndexableCollection, - indexSpec: Record[], + indexSpec: { + keys: Record; + options?: Record; + }, pipelineName?: string ) { await this.dataClient.createIndex({ organizationId, collectionType, - indexSpec: indexSpec.map((spec) => BSON.serialize(spec)), + indexSpec: [BSON.serialize(indexSpec)], pipelineName, }); } From b92b61b0fed8b42646dfd95b14094fd2ff19c3b6 Mon Sep 17 00:00:00 2001 From: Naveed Jooma Date: Fri, 3 Oct 2025 14:54:48 -0500 Subject: [PATCH 7/7] Change collection type in test --- src/app/data-client.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app/data-client.spec.ts b/src/app/data-client.spec.ts index 115d69640..f4c896fd6 100644 --- a/src/app/data-client.spec.ts +++ b/src/app/data-client.spec.ts @@ -1094,7 +1094,7 @@ describe('DataClient tests', () => { }); it('creates an index', async () => { const organizationId = 'orgId'; - const collectionType = IndexableCollection.HOT_STORE; + const collectionType = IndexableCollection.PIPELINE_SINK; const indexSpec = { keys: { field: 1 }, options: { priority: 1 } }; const pipelineName = 'pipeline1'; await subject().createIndex(