Merge branch 'main' into feat/agent.md

Author: evalstate

.devcontainer/devcontainer.json

@@ -0,0 +1,24 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/typescript-node
+{
+    "name": "Node.js & TypeScript",
+    // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
+    "image": "mcr.microsoft.com/devcontainers/typescript-node:1-22-bookworm",
+
+    // Features to add to the dev container. More info: https://containers.dev/features.
+    "features": {
+        "ghcr.io/joshuanianji/devcontainer-features/mount-pnpm-store:1": {}
+    }
+
+    // Use 'forwardPorts' to make a list of ports inside the container available locally.
+    // "forwardPorts": [],
+
+    // Use 'postCreateCommand' to run commands after the container is created.
+    // "postCreateCommand": "yarn install",
+
+    // Configure tool-specific properties.
+    // "customizations": {},
+
+    // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
+    // "remoteUser": "root"
+}

README.md

@@ -99,7 +99,7 @@ You can run our packages with vanilla JS, without any bundler, by using a CDN or
 ```html
 <script type="module">
     import { InferenceClient } from 'https://cdn.jsdelivr.net/npm/@huggingface/inference@4.7.1/+esm';
-    import { createRepo, commit, deleteRepo, listFiles } from "https://cdn.jsdelivr.net/npm/@huggingface/hub@2.5.0/+esm";
+    import { createRepo, commit, deleteRepo, listFiles } from "https://cdn.jsdelivr.net/npm/@huggingface/hub@2.5.2/+esm";
 </script>
 ```
 

packages/hub/.gitignore

@@ -1 +1,3 @@
 xet-core-wasm-build
+shard.bin
+xorb.bin

packages/hub/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@huggingface/hub",
 	"packageManager": "pnpm@10.10.0",
-	"version": "2.5.0",
+	"version": "2.5.2",
 	"description": "Utilities to interact with the Hugging Face hub",
 	"repository": "https://github.com/huggingface/huggingface.js.git",
 	"publishConfig": {

packages/hub/scripts/bench.ts

@@ -8,6 +8,7 @@ import type { RepoId } from "../src/types/public.js";
 import { toRepoId } from "../src/utils/toRepoId.js";
 import { commitIter } from "../src/index.js";
 import { pathToFileURL } from "node:url";
+import { WebBlob } from "../src/utils/WebBlob.js";
 
 /**
  * This script downloads the files from openai-community/gpt2 and simulates an upload to a xet repo.
@@ -23,10 +24,17 @@ const FILES_TO_DOWNLOAD = [
 	{
 		url: "https://huggingface.co/openai-community/gpt2/resolve/main/64-8bits.tflite?download=true",
 		filename: "64-8bits.tflite",
+		sha256: "c966da3b74697803352ca7c6f2f220e7090a557b619de9da0c6b34d89f7825c1",
 	},
 	{
 		url: "https://huggingface.co/openai-community/gpt2/resolve/main/64-fp16.tflite?download=true",
 		filename: "64-fp16.tflite",
+		sha256: "1ceafd82e733dd4b21570b2a86cf27556a983041806c033a55d086e0ed782cd3",
+	},
+	{
+		url: "https://huggingface.co/openai-community/gpt2/resolve/main/64.tflite?download=true",
+		filename: "64.tflite",
+		sha256: "cfcd510b239d90b71ee87d4e57a5a8c2d55b2a941e5d9fe5852298268ddbe61b",
 	},
 ];
 
@@ -68,6 +76,15 @@ async function* createFileSource(
 		const sha256Hash = res.value;
 
 		console.log(`SHA256 for ${file.filename}: ${sha256Hash}`);
+
+		if (sha256Hash !== FILES_TO_DOWNLOAD.find((f) => f.filename === file.filename)?.sha256) {
+			throw new Error(
+				`SHA256 mismatch for ${file.filename}: ${sha256Hash} !== ${FILES_TO_DOWNLOAD.find(
+					(f) => f.filename === file.filename
+				)?.sha256}`
+			);
+		}
+
 		yield {
 			content: blob,
 			path: file.filename,
@@ -92,7 +109,7 @@ function getBodySize(body: RequestInit["body"]): string {
 	return "unknown size";
 }
 
-function createMockFetch(): {
+function createMockFetch(args: { write: boolean }): {
 	fetch: typeof fetch;
 	getStats: () => { xorbCount: number; shardCount: number; xorbBytes: number; shardBytes: number };
 } {
@@ -111,6 +128,11 @@ function createMockFetch(): {
 			xorbBytes += parseInt(bodySize);
 			console.log(`[MOCK] Xorb upload ${xorbCount}: ${init?.method || "GET"} ${url} (${bodySize})`);
 
+			if (args.write) {
+				// Write the body to a file
+				await writeFile("xorb.bin", init?.body as Uint8Array);
+			}
+
 			return new Response(null, {
 				status: 200,
 				statusText: "OK",
@@ -123,6 +145,11 @@ function createMockFetch(): {
 			shardBytes += parseInt(bodySize);
 			console.log(`[MOCK] Shard upload ${shardCount}: ${init?.method || "GET"} ${url} (${bodySize})`);
 
+			if (args.write) {
+				// Write the body to a file
+				await writeFile("shard.bin", init?.body as Uint8Array);
+			}
+
 			return new Response(null, {
 				status: 200,
 				statusText: "OK",
@@ -158,6 +185,11 @@ async function main() {
 				short: "c",
 				default: false,
 			},
+			write: {
+				type: "boolean",
+				short: "w",
+				default: false,
+			},
 		},
 	});
 
@@ -189,7 +221,7 @@ async function main() {
 	const repo: RepoId = toRepoId(repoName);
 
 	// Create mock fetch
-	const mockFetchObj = createMockFetch();
+	const mockFetchObj = createMockFetch({ write: args.write });
 
 	// Setup upload parameters
 	const uploadParams = {
@@ -290,6 +322,19 @@ async function main() {
 		}
 
 		console.log("Done committing");
+
+		console.log("Redownloading files and verifying SHA256 integrity");
+		for (const file of FILES_TO_DOWNLOAD) {
+			const fileBlob = await WebBlob.create(new URL(file.url));
+			const sha256Hash = sha256(fileBlob, { useWebWorker: false });
+			let res: IteratorResult<number, string>;
+			do {
+				res = await sha256Hash.next();
+			} while (!res.done);
+			const finalHash = res.value;
+
+			console.log(`${file.filename}: ${finalHash} === ${file.sha256} ${finalHash === file.sha256 ? "✅" : "❌"}`);
+		}
 	}
 }
 

packages/hub/src/lib/commit.ts

@@ -92,7 +92,7 @@ export type CommitParams = {
 	/**
 	 * @deprecated Not yet ready for production use
 	 */
-	xet?: boolean;
+	useXet?: boolean;
 } & Partial<CredentialsParams>;
 
 export interface CommitOutput {
@@ -298,7 +298,7 @@ export async function* commitIter(params: CommitParams): AsyncGenerator<CommitPr
 
 			const shaToOperation = new Map(operations.map((op, i) => [shas[i], op]));
 
-			if (params.xet) {
+			if (params.useXet) {
 				// First get all the files that are already uploaded out of the way
 				for (const obj of json.objects) {
 					const op = shaToOperation.get(obj.oid);

packages/hub/src/lib/upload-file.ts

@@ -15,6 +15,10 @@ export function uploadFile(
 		fetch?: CommitParams["fetch"];
 		useWebWorkers?: CommitParams["useWebWorkers"];
 		abortSignal?: CommitParams["abortSignal"];
+		/**
+		 * @deprecated Not yet ready for production use
+		 */
+		useXet?: CommitParams["useXet"];
 	} & Partial<CredentialsParams>
 ): Promise<CommitOutput> {
 	const path =
@@ -43,5 +47,6 @@ export function uploadFile(
 		fetch: params.fetch,
 		useWebWorkers: params.useWebWorkers,
 		abortSignal: params.abortSignal,
+		useXet: params.useXet,
 	});
 }