From ef8acb0f6222d034d5469322dd4cb6b6c7562522 Mon Sep 17 00:00:00 2001 From: Nick Sanford Date: Tue, 30 Sep 2025 12:10:37 -0400 Subject: [PATCH 1/4] CONSULT-1797-implement-FileUpload-on-the-typescript-SDK --- src/app/data-client.spec.ts | 91 +++++++++++++++++++++++++++++++++++++ src/app/data-client.ts | 91 +++++++++++++++++++++++++++++++++++-- 2 files changed, 178 insertions(+), 4 deletions(-) diff --git a/src/app/data-client.spec.ts b/src/app/data-client.spec.ts index 10bdd17c8..f0fe12dfe 100644 --- a/src/app/data-client.spec.ts +++ b/src/app/data-client.spec.ts @@ -72,6 +72,8 @@ import { DataCaptureUploadRequest, DataCaptureUploadResponse, DataType, + FileUploadRequest, + FileUploadResponse, SensorData, SensorMetadata, UploadMetadata, @@ -1716,3 +1718,92 @@ describe('DataPipelineClient tests', () => { }); }); }); + +describe('fileUpload tests', () => { + const partId = 'testPartId'; + const fileExtension = '.png'; + const tags = ['testTag1', 'testTag2']; + const datasetIds = ['dataset1', 'dataset2']; + const binaryData = new Uint8Array([1, 2, 3, 4, 5]); + const dataRequestTimes: [Date, Date] = [ + new Date('2025-03-19T10:00:00Z'), + new Date('2025-03-19T10:00:01Z'), + ]; + + let capturedRequests: FileUploadRequest[]; + + beforeEach(() => { + capturedRequests = []; + mockTransport = createRouterTransport(({ service }) => { + service(DataSyncService, { + fileUpload: async (requests: AsyncIterable) => { + for await (const request of requests) { + capturedRequests.push(request); + } + return new FileUploadResponse({ + fileId: 'testFileId', + binaryDataId: 'testBinaryDataId', + }); + }, + }); + }); + }); + + it('uploads file with metadata and file contents', async () => { + const result = await subject().fileUpload( + binaryData, + partId, + fileExtension, + dataRequestTimes, + tags, + datasetIds + ); + + expect(result).toBe('testBinaryDataId'); + expect(capturedRequests).toHaveLength(2); + + // Check metadata request + const metadataRequest = capturedRequests[0]!; + expect(metadataRequest.uploadPacket.case).toBe('metadata'); + if (metadataRequest.uploadPacket.case === 'metadata') { + const metadata = metadataRequest.uploadPacket.value; + expect(metadata.partId).toBe(partId); + expect(metadata.type).toBe(DataType.BINARY_SENSOR); + expect(metadata.tags).toEqual(tags); + expect(metadata.fileExtension).toBe(fileExtension); + expect(metadata.datasetIds).toEqual(datasetIds); + } + + // Check file contents request + const fileContentsRequest = capturedRequests[1]!; + expect(fileContentsRequest.uploadPacket.case).toBe('fileContents'); + if (fileContentsRequest.uploadPacket.case === 'fileContents') { + const fileData = fileContentsRequest.uploadPacket.value; + expect(fileData.data).toEqual(binaryData); + } + }); + + it('uploads file without optional parameters', async () => { + const result = await subject().fileUpload( + binaryData, + partId, + fileExtension, + dataRequestTimes + ); + + expect(result).toBe('testBinaryDataId'); + expect(capturedRequests).toHaveLength(2); + + // Check metadata request + const metadataRequest = capturedRequests[0]!; + expect(metadataRequest.uploadPacket.case).toBe('metadata'); + if (metadataRequest.uploadPacket.case === 'metadata') { + const metadata = metadataRequest.uploadPacket.value; + expect(metadata.partId).toBe(partId); + expect(metadata.type).toBe(DataType.BINARY_SENSOR); + expect(metadata.tags).toEqual([]); + expect(metadata.fileExtension).toBe(fileExtension); + expect(metadata.datasetIds).toEqual([]); + } + }); +}); diff --git a/src/app/data-client.ts b/src/app/data-client.ts index 704bfb29b..a6ab030fd 100644 --- a/src/app/data-client.ts +++ b/src/app/data-client.ts @@ -1,5 +1,5 @@ import { BSON } from 'bsonfy'; -import { Struct, Timestamp, type JsonValue } from '@bufbuild/protobuf'; +import { Struct, Timestamp, type JsonValue, type PartialMessage } from '@bufbuild/protobuf'; import { createClient, type Client, type Transport } from '@connectrpc/connect'; import { DataService } from '../gen/app/data/v1/data_connect'; import { @@ -19,6 +19,8 @@ import { DataPipelinesService } from '../gen/app/datapipelines/v1/data_pipelines import { DataCaptureUploadRequest, DataType, + FileData, + FileUploadRequest, SensorData, SensorMetadata, UploadMetadata, @@ -1212,7 +1214,7 @@ export class DataClient { * (for example, "movementSensor") * @param componentName The name of the component used to capture the data * @param methodName The name of the method used to capture the data. - * @param fileExtension The file extension of binary data including the + * @param filename The file extension of binary data including the * period, for example .jpg, .png, or .pcd. The backend will route the * binary to its corresponding mime type based on this extension. Files with * a .jpeg, .jpg, or .png extension will be saved to the images tab. @@ -1229,7 +1231,7 @@ export class DataClient { componentType: string, componentName: string, methodName: string, - fileExtension: string, + filename: string, dataRequestTimes: [Date, Date], tags?: string[], datasetIds?: string[] @@ -1241,7 +1243,7 @@ export class DataClient { methodName, type: DataType.BINARY_SENSOR, tags, - fileExtension, + filename, datasetIds, }); @@ -1265,6 +1267,87 @@ export class DataClient { return resp.binaryDataId; } + /** + * Uploads the contents and metadata for binary (image + file) data using streaming. + * + * Upload binary data collected on a robot along with the relevant metadata to app.viam.com. This + * method uses client streaming to upload the data in chunks, which is more + * efficient for large files. + * + * @example + * + * ```ts + * const binaryDataId = await dataClient.fileUpload( + * binaryData, + * '123abc45-1234-5678-90ab-cdef12345678', + * '.jpg', + * [new Date('2025-03-19'), new Date('2025-03-19')] + * ); + * ``` + * + * For more information, see [Data + * API](https://docs.viam.com/dev/reference/apis/data-client/#fileupload). + * + * @param binaryData The data to be uploaded, represented in bytes + * @param partId The part ID of the component used to capture the data + * @param fileName The filename of binary data including the + * period, for example image.jpg, image.png, or image.pcd. The backend will route the + * binary to its corresponding mime type based on this extension. Files with + * a .jpeg, .jpg, or .png extension will be saved to the images tab. + * @param dataRequestTimes Tuple containing `Date` objects denoting the times + * this data was requested[0] by the robot and received[1] from the + * appropriate sensor. + * @param tags The list of tags to allow for tag-based filtering when + * retrieving data + * @param datasetIds The list of dataset IDs to associate with the uploaded data + * @returns The binary data ID of the uploaded data + */ + async fileUpload( + binaryData: Uint8Array, + partId: string, + fileName: string, + _dataRequestTimes: [Date, Date], + tags?: string[], + datasetIds?: string[] + ) { + const fileExtension = fileName.includes('.') ? fileName.split('.')[1] : ''; + const metadata = new UploadMetadata({ + partId, + type: DataType.BINARY_SENSOR, + tags, + fileName, + fileExtension, + datasetIds, + }); + + const fileData = new FileData({ + data: binaryData, + }); + + // Create async generator for streaming upload + async function* uploadGenerator(): AsyncIterable> { + // Send metadata first + yield new FileUploadRequest({ + uploadPacket: { + case: 'metadata', + value: metadata, + }, + }); + + // Send file contents + yield new FileUploadRequest({ + uploadPacket: { + case: 'fileContents', + value: fileData, + }, + }); + } + + // Call the streaming method + const response = await this.dataSyncClient.fileUpload(uploadGenerator()); + return response.binaryDataId; + } + // eslint-disable-next-line class-methods-use-this createFilter(options: FilterOptions): Filter { const filter = new Filter(options); From 143c0d155abe5b67cbd53c1436f86559d10e5c14 Mon Sep 17 00:00:00 2001 From: Naveed Jooma Date: Tue, 30 Sep 2025 16:43:06 -0500 Subject: [PATCH 2/4] Update streaming file upload --- src/app/data-client.spec.ts | 140 ++++++++++++++++--------------- src/app/data-client.ts | 160 ++++++++++++++++++++++-------------- 2 files changed, 172 insertions(+), 128 deletions(-) diff --git a/src/app/data-client.spec.ts b/src/app/data-client.spec.ts index f0fe12dfe..1b9a2dad2 100644 --- a/src/app/data-client.spec.ts +++ b/src/app/data-client.spec.ts @@ -1,6 +1,6 @@ -import { BSON } from 'bsonfy'; import { Struct, Timestamp, type JsonValue } from '@bufbuild/protobuf'; import { createRouterTransport, type Transport } from '@connectrpc/connect'; +import { BSON } from 'bsonfy'; import { beforeEach, describe, expect, it, vi } from 'vitest'; import { DataService } from '../gen/app/data/v1/data_connect'; import { @@ -33,6 +33,8 @@ import { Filter, GetDatabaseConnectionRequest, GetDatabaseConnectionResponse, + GetLatestTabularDataRequest, + GetLatestTabularDataResponse, RemoveBinaryDataFromDatasetByIDsRequest, RemoveBinaryDataFromDatasetByIDsResponse, RemoveBoundingBoxFromImageByIDRequest, @@ -50,9 +52,23 @@ import { TagsByFilterRequest, TagsByFilterResponse, TagsFilter, - GetLatestTabularDataRequest, - GetLatestTabularDataResponse, } from '../gen/app/data/v1/data_pb'; +import { DataPipelinesService } from '../gen/app/datapipelines/v1/data_pipelines_connect'; +import { + CreateDataPipelineRequest, + CreateDataPipelineResponse, + DataPipeline, + DataPipelineRun, + DataPipelineRunStatus, + DeleteDataPipelineRequest, + DeleteDataPipelineResponse, + GetDataPipelineRequest, + GetDataPipelineResponse, + ListDataPipelineRunsRequest, + ListDataPipelineRunsResponse, + ListDataPipelinesRequest, + ListDataPipelinesResponse, +} from '../gen/app/datapipelines/v1/data_pipelines_pb'; import { DatasetService } from '../gen/app/dataset/v1/dataset_connect'; import { CreateDatasetRequest, @@ -72,36 +88,25 @@ import { DataCaptureUploadRequest, DataCaptureUploadResponse, DataType, + FileData, FileUploadRequest, FileUploadResponse, SensorData, SensorMetadata, UploadMetadata, } from '../gen/app/datasync/v1/data_sync_pb'; -import { DataClient, type FilterOptions } from './data-client'; import { - DataPipeline, - ListDataPipelinesRequest, - ListDataPipelinesResponse, - GetDataPipelineRequest, - GetDataPipelineResponse, - CreateDataPipelineRequest, - CreateDataPipelineResponse, - DeleteDataPipelineRequest, - DeleteDataPipelineResponse, - DataPipelineRun, - DataPipelineRunStatus, - ListDataPipelineRunsRequest, - ListDataPipelineRunsResponse, -} from '../gen/app/datapipelines/v1/data_pipelines_pb'; -import { DataPipelinesService } from '../gen/app/datapipelines/v1/data_pipelines_connect'; + DataClient, + type FileUploadOptions, + type FilterOptions, +} from './data-client'; vi.mock('../gen/app/data/v1/data_pb_service'); let mockTransport: Transport; const subject = () => new DataClient(mockTransport); describe('DataClient tests', () => { - const filter = subject().createFilter({ + const filter = DataClient.createFilter({ componentName: 'testComponentName', componentType: 'testComponentType', }); @@ -1037,13 +1042,13 @@ describe('DataClient tests', () => { describe('createFilter tests', () => { it('create empty filter', () => { - const testFilter = subject().createFilter({}); + const testFilter = DataClient.createFilter({}); expect(testFilter).toEqual(new Filter()); }); it('create filter', () => { const opts = { componentName: 'camera' }; - const testFilter = subject().createFilter(opts); + const testFilter = DataClient.createFilter(opts); const expectedFilter = new Filter({ componentName: 'camera', @@ -1091,7 +1096,7 @@ describe('DataClient tests', () => { endTime, tags: tagsList, }; - const testFilter = subject().createFilter(opts); + const testFilter = DataClient.createFilter(opts); expect(testFilter.componentType).toEqual('testComponentType'); const expectedFilter = new Filter({ @@ -1721,14 +1726,19 @@ describe('DataPipelineClient tests', () => { describe('fileUpload tests', () => { const partId = 'testPartId'; - const fileExtension = '.png'; - const tags = ['testTag1', 'testTag2']; - const datasetIds = ['dataset1', 'dataset2']; const binaryData = new Uint8Array([1, 2, 3, 4, 5]); - const dataRequestTimes: [Date, Date] = [ - new Date('2025-03-19T10:00:00Z'), - new Date('2025-03-19T10:00:01Z'), - ]; + const options: FileUploadOptions = { + componentType: 'componentType', + componentName: 'componentName', + methodName: 'methodName', + fileName: 'fileName', + fileExtension: '.png', + tags: ['testTag1', 'testTag2'], + datasetIds: ['dataset1', 'dataset2'], + }; + + const expectedFileId = 'testFileId'; + const expectedBinaryDataId = 'testBinaryDataId'; let capturedRequests: FileUploadRequest[]; @@ -1741,8 +1751,8 @@ describe('fileUpload tests', () => { capturedRequests.push(request); } return new FileUploadResponse({ - fileId: 'testFileId', - binaryDataId: 'testBinaryDataId', + fileId: expectedFileId, + binaryDataId: expectedBinaryDataId, }); }, }); @@ -1750,46 +1760,34 @@ describe('fileUpload tests', () => { }); it('uploads file with metadata and file contents', async () => { - const result = await subject().fileUpload( - binaryData, - partId, - fileExtension, - dataRequestTimes, - tags, - datasetIds - ); + const result = await subject().fileUpload(binaryData, partId, options); - expect(result).toBe('testBinaryDataId'); + expect(result).toBe(expectedBinaryDataId); expect(capturedRequests).toHaveLength(2); // Check metadata request const metadataRequest = capturedRequests[0]!; expect(metadataRequest.uploadPacket.case).toBe('metadata'); - if (metadataRequest.uploadPacket.case === 'metadata') { - const metadata = metadataRequest.uploadPacket.value; - expect(metadata.partId).toBe(partId); - expect(metadata.type).toBe(DataType.BINARY_SENSOR); - expect(metadata.tags).toEqual(tags); - expect(metadata.fileExtension).toBe(fileExtension); - expect(metadata.datasetIds).toEqual(datasetIds); - } + const metadata = metadataRequest.uploadPacket.value as UploadMetadata; + expect(metadata.partId).toBe(partId); + expect(metadata.type).toBe(DataType.FILE); + expect(metadata.componentType).toBe(options.componentType); + expect(metadata.componentName).toBe(options.componentName); + expect(metadata.methodName).toBe(options.methodName); + expect(metadata.fileName).toBe(options.fileName); + expect(metadata.fileExtension).toBe(options.fileExtension); + expect(metadata.tags).toStrictEqual(options.tags); + expect(metadata.datasetIds).toStrictEqual(options.datasetIds); // Check file contents request const fileContentsRequest = capturedRequests[1]!; expect(fileContentsRequest.uploadPacket.case).toBe('fileContents'); - if (fileContentsRequest.uploadPacket.case === 'fileContents') { - const fileData = fileContentsRequest.uploadPacket.value; - expect(fileData.data).toEqual(binaryData); - } + const fileContents = fileContentsRequest.uploadPacket.value as FileData; + expect(fileContents.data).toEqual(binaryData); }); it('uploads file without optional parameters', async () => { - const result = await subject().fileUpload( - binaryData, - partId, - fileExtension, - dataRequestTimes - ); + const result = await subject().fileUpload(binaryData, partId); expect(result).toBe('testBinaryDataId'); expect(capturedRequests).toHaveLength(2); @@ -1797,13 +1795,21 @@ describe('fileUpload tests', () => { // Check metadata request const metadataRequest = capturedRequests[0]!; expect(metadataRequest.uploadPacket.case).toBe('metadata'); - if (metadataRequest.uploadPacket.case === 'metadata') { - const metadata = metadataRequest.uploadPacket.value; - expect(metadata.partId).toBe(partId); - expect(metadata.type).toBe(DataType.BINARY_SENSOR); - expect(metadata.tags).toEqual([]); - expect(metadata.fileExtension).toBe(fileExtension); - expect(metadata.datasetIds).toEqual([]); - } + const metadata = metadataRequest.uploadPacket.value as UploadMetadata; + expect(metadata.partId).toBe(partId); + expect(metadata.type).toBe(DataType.FILE); + expect(metadata.componentType).toBe(''); + expect(metadata.componentName).toBe(''); + expect(metadata.methodName).toBe(''); + expect(metadata.fileName).toBe(''); + expect(metadata.fileExtension).toBe(''); + expect(metadata.tags).toStrictEqual([]); + expect(metadata.datasetIds).toStrictEqual([]); + + // Check file contents request + const fileContentsRequest = capturedRequests[1]!; + expect(fileContentsRequest.uploadPacket.case).toBe('fileContents'); + const fileContents = fileContentsRequest.uploadPacket.value as FileData; + expect(fileContents.data).toEqual(binaryData); }); }); diff --git a/src/app/data-client.ts b/src/app/data-client.ts index a6ab030fd..22ea4025b 100644 --- a/src/app/data-client.ts +++ b/src/app/data-client.ts @@ -1,6 +1,11 @@ -import { BSON } from 'bsonfy'; -import { Struct, Timestamp, type JsonValue, type PartialMessage } from '@bufbuild/protobuf'; +import { + Struct, + Timestamp, + type JsonValue, + type PartialMessage, +} from '@bufbuild/protobuf'; import { createClient, type Client, type Transport } from '@connectrpc/connect'; +import { BSON } from 'bsonfy'; import { DataService } from '../gen/app/data/v1/data_connect'; import { BinaryID, @@ -8,14 +13,18 @@ import { CaptureMetadata, Filter, Order, - TagsFilter, TabularDataSource, TabularDataSourceType, + TagsFilter, } from '../gen/app/data/v1/data_pb'; +import { DataPipelinesService } from '../gen/app/datapipelines/v1/data_pipelines_connect'; +import { + DataPipeline, + DataPipelineRun, +} from '../gen/app/datapipelines/v1/data_pipelines_pb'; import { DatasetService } from '../gen/app/dataset/v1/dataset_connect'; import type { Dataset as PBDataset } from '../gen/app/dataset/v1/dataset_pb'; import { DataSyncService } from '../gen/app/datasync/v1/data_sync_connect'; -import { DataPipelinesService } from '../gen/app/datapipelines/v1/data_pipelines_connect'; import { DataCaptureUploadRequest, DataType, @@ -25,10 +34,6 @@ import { SensorMetadata, UploadMetadata, } from '../gen/app/datasync/v1/data_sync_pb'; -import { - DataPipeline, - DataPipelineRun, -} from '../gen/app/datapipelines/v1/data_pipelines_pb'; export type FilterOptions = Partial & { endTime?: Date; @@ -59,6 +64,43 @@ interface TabularDataPoint { payload: JsonValue; } +/** Optional parameters for uploading files */ +export interface FileUploadOptions { + /** + * Optional type of the component associated with the file (for example, + * "movement_sensor"). + */ + componentType?: string; + + /** Optional name of the component associated with the file. */ + componentName?: string; + + /** Optional name of the method associated with the file. */ + methodName?: string; + + /** + * Optional name of the file. The empty string `""` will be assigned as the + * file name if one isn't provided. + */ + fileName?: string; + + /** + * Optional file extension. The empty string `""` will be assigned as the file + * extension if one isn't provided. Files with a `.jpeg`, `.jpg`, or `.png` + * extension will be saved to the **Images** tab. + */ + fileExtension?: string; + + /** + * Optional list of tags to allow for tag-based filtering when retrieving + * data. + */ + tags?: string[]; + + /** Optional list of datasets to add the data to. */ + datasetIds?: string[]; +} + export type Dataset = Partial & { created?: Date; }; @@ -1214,7 +1256,7 @@ export class DataClient { * (for example, "movementSensor") * @param componentName The name of the component used to capture the data * @param methodName The name of the method used to capture the data. - * @param filename The file extension of binary data including the + * @param fileExtension The file extension of binary data including the * period, for example .jpg, .png, or .pcd. The backend will route the * binary to its corresponding mime type based on this extension. Files with * a .jpeg, .jpg, or .png extension will be saved to the images tab. @@ -1231,7 +1273,7 @@ export class DataClient { componentType: string, componentName: string, methodName: string, - filename: string, + fileExtension: string, dataRequestTimes: [Date, Date], tags?: string[], datasetIds?: string[] @@ -1243,7 +1285,7 @@ export class DataClient { methodName, type: DataType.BINARY_SENSOR, tags, - filename, + fileExtension, datasetIds, }); @@ -1268,20 +1310,22 @@ export class DataClient { } /** - * Uploads the contents and metadata for binary (image + file) data using streaming. + * Upload arbitrary file data. * - * Upload binary data collected on a robot along with the relevant metadata to app.viam.com. This - * method uses client streaming to upload the data in chunks, which is more - * efficient for large files. + * Upload file data that may be stored on a robot along with the relevant + * metadata. File data can be found in the **Files** tab of the **DATA** + * page. * * @example * * ```ts * const binaryDataId = await dataClient.fileUpload( - * binaryData, - * '123abc45-1234-5678-90ab-cdef12345678', - * '.jpg', - * [new Date('2025-03-19'), new Date('2025-03-19')] + * 'INSERT YOUR PART ID' + * binaryData, + * { + * fileExtension: ".jpeg", + * tags: ["tag_1", "tag_2"], + * } * ); * ``` * @@ -1290,66 +1334,60 @@ export class DataClient { * * @param binaryData The data to be uploaded, represented in bytes * @param partId The part ID of the component used to capture the data - * @param fileName The filename of binary data including the - * period, for example image.jpg, image.png, or image.pcd. The backend will route the - * binary to its corresponding mime type based on this extension. Files with - * a .jpeg, .jpg, or .png extension will be saved to the images tab. - * @param dataRequestTimes Tuple containing `Date` objects denoting the times - * this data was requested[0] by the robot and received[1] from the - * appropriate sensor. - * @param tags The list of tags to allow for tag-based filtering when - * retrieving data - * @param datasetIds The list of dataset IDs to associate with the uploaded data + * @param options Options for the file upload * @returns The binary data ID of the uploaded data */ async fileUpload( binaryData: Uint8Array, partId: string, - fileName: string, - _dataRequestTimes: [Date, Date], - tags?: string[], - datasetIds?: string[] + options?: FileUploadOptions ) { - const fileExtension = fileName.includes('.') ? fileName.split('.')[1] : ''; - const metadata = new UploadMetadata({ + const md = new UploadMetadata({ partId, - type: DataType.BINARY_SENSOR, - tags, - fileName, - fileExtension, - datasetIds, + type: DataType.FILE, + ...options, }); - const fileData = new FileData({ - data: binaryData, - }); - - // Create async generator for streaming upload - async function* uploadGenerator(): AsyncIterable> { - // Send metadata first - yield new FileUploadRequest({ - uploadPacket: { - case: 'metadata', - value: metadata, - }, - }); + const response = await this.dataSyncClient.fileUpload( + DataClient.fileUploadRequests(md, binaryData) + ); + return response.binaryDataId; + } - // Send file contents + /** + * Create an async generator of FileUploadRequests to use with FileUpload + * methods. + * + * @param metadata The file's metadata + * @param data The binary data of the file + */ + private static async *fileUploadRequests( + metadata: UploadMetadata, + data: Uint8Array + ): AsyncGenerator> { + yield new FileUploadRequest({ + uploadPacket: { + case: 'metadata', + value: metadata, + }, + }); + // Awaiting because linter gets mad if not awaiting nor yielding promises + const uploadChunkSize = await Promise.resolve(64 * 1024); + for (let i = 0; i < data.length; i += uploadChunkSize) { + let end = i + uploadChunkSize; + if (end > data.length) { + end = data.length; + } yield new FileUploadRequest({ uploadPacket: { case: 'fileContents', - value: fileData, + value: new FileData({ data: data.slice(i, end) }), }, }); } - - // Call the streaming method - const response = await this.dataSyncClient.fileUpload(uploadGenerator()); - return response.binaryDataId; } - // eslint-disable-next-line class-methods-use-this - createFilter(options: FilterOptions): Filter { + static createFilter(options: FilterOptions): Filter { const filter = new Filter(options); if (options.startTime ?? options.endTime) { From a1794fe40e38da9a5979b46d3fd5227202036b74 Mon Sep 17 00:00:00 2001 From: Naveed Jooma Date: Wed, 1 Oct 2025 10:27:03 -0500 Subject: [PATCH 3/4] Apply PR comments --- src/app/data-client.ts | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/app/data-client.ts b/src/app/data-client.ts index 22ea4025b..66517808e 100644 --- a/src/app/data-client.ts +++ b/src/app/data-client.ts @@ -1320,20 +1320,20 @@ export class DataClient { * * ```ts * const binaryDataId = await dataClient.fileUpload( - * 'INSERT YOUR PART ID' - * binaryData, - * { - * fileExtension: ".jpeg", - * tags: ["tag_1", "tag_2"], - * } + * binaryData, + * 'INSERT YOUR PART ID', + * { + * fileExtension: '.jpeg', + * tags: ['tag_1', 'tag_2'], + * } * ); * ``` * * For more information, see [Data * API](https://docs.viam.com/dev/reference/apis/data-client/#fileupload). * - * @param binaryData The data to be uploaded, represented in bytes - * @param partId The part ID of the component used to capture the data + * @param binaryData The data to be uploaded + * @param partId The part ID of the machine that captured the data * @param options Options for the file upload * @returns The binary data ID of the uploaded data */ @@ -1361,6 +1361,7 @@ export class DataClient { * @param metadata The file's metadata * @param data The binary data of the file */ + // eslint-disable-next-line @typescript-eslint/require-await private static async *fileUploadRequests( metadata: UploadMetadata, data: Uint8Array @@ -1371,8 +1372,7 @@ export class DataClient { value: metadata, }, }); - // Awaiting because linter gets mad if not awaiting nor yielding promises - const uploadChunkSize = await Promise.resolve(64 * 1024); + const uploadChunkSize = 64 * 1024; for (let i = 0; i < data.length; i += uploadChunkSize) { let end = i + uploadChunkSize; if (end > data.length) { From 9fbed6f461da9355076a6c8683b4dbf1f20e930e Mon Sep 17 00:00:00 2001 From: Naveed Jooma Date: Wed, 1 Oct 2025 13:06:21 -0500 Subject: [PATCH 4/4] Test chunking logic --- src/app/data-client.spec.ts | 38 ++++++++++++++++++++++++++++++++++++- src/app/data-client.ts | 6 +++--- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/src/app/data-client.spec.ts b/src/app/data-client.spec.ts index 1b9a2dad2..3ba2a8ce1 100644 --- a/src/app/data-client.spec.ts +++ b/src/app/data-client.spec.ts @@ -1789,7 +1789,7 @@ describe('fileUpload tests', () => { it('uploads file without optional parameters', async () => { const result = await subject().fileUpload(binaryData, partId); - expect(result).toBe('testBinaryDataId'); + expect(result).toBe(expectedBinaryDataId); expect(capturedRequests).toHaveLength(2); // Check metadata request @@ -1812,4 +1812,40 @@ describe('fileUpload tests', () => { const fileContents = fileContentsRequest.uploadPacket.value as FileData; expect(fileContents.data).toEqual(binaryData); }); + + it('chunks file data', async () => { + const numChunks = 3; + const data = Uint8Array.from( + { length: DataClient.UPLOAD_CHUNK_SIZE * numChunks }, + () => Math.floor(Math.random() * 256) + ); + + const result = await subject().fileUpload(data, partId); + expect(result).toBe(expectedBinaryDataId); + expect(capturedRequests).toHaveLength(1 + numChunks); + + const metadataRequest = capturedRequests[0]!; + expect(metadataRequest.uploadPacket.case).toBe('metadata'); + + const contentRequests = capturedRequests.slice(1); + expect(contentRequests).toHaveLength(numChunks); + + const receivedLength = contentRequests.reduce( + (acc, val) => acc + (val.uploadPacket.value as FileData).data.length, + 0 + ); + expect(receivedLength).toEqual(numChunks * DataClient.UPLOAD_CHUNK_SIZE); + + const receivedData = new Uint8Array(receivedLength); + let offset = 0; + for (const req of contentRequests) { + expect(req.uploadPacket.case).toBe('fileContents'); + const fileData = req.uploadPacket.value as FileData; + expect(fileData.data).toHaveLength(DataClient.UPLOAD_CHUNK_SIZE); + receivedData.set(fileData.data, offset); + offset += fileData.data.length; + } + + expect(receivedData).toStrictEqual(data); + }); }); diff --git a/src/app/data-client.ts b/src/app/data-client.ts index 66517808e..657da1af7 100644 --- a/src/app/data-client.ts +++ b/src/app/data-client.ts @@ -117,6 +117,7 @@ export class DataClient { private datasetClient: Client; private dataSyncClient: Client; private dataPipelinesClient: Client; + static readonly UPLOAD_CHUNK_SIZE = 8; constructor(transport: Transport) { this.dataClient = createClient(DataService, transport); @@ -1372,9 +1373,8 @@ export class DataClient { value: metadata, }, }); - const uploadChunkSize = 64 * 1024; - for (let i = 0; i < data.length; i += uploadChunkSize) { - let end = i + uploadChunkSize; + for (let i = 0; i < data.length; i += DataClient.UPLOAD_CHUNK_SIZE) { + let end = i + DataClient.UPLOAD_CHUNK_SIZE; if (end > data.length) { end = data.length; }