From d5421863e85d1a6ea15bea06dca9007da9b9c792 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mihai=20Capot=C4=83?= Date: Thu, 26 Jun 2025 16:32:54 -0700 Subject: [PATCH 1/3] Add Cohere dataset and configuration The Cohere HDF5 does not have a "distances" dataset. Since this is not used, we can make it optional. --- dataset_reader/ann_h5_reader.py | 6 +- datasets/datasets.json | 16 + .../configurations/cohere-calibration.json | 390 ++++++++++++++++++ 3 files changed, 410 insertions(+), 2 deletions(-) create mode 100644 experiments/configurations/cohere-calibration.json diff --git a/dataset_reader/ann_h5_reader.py b/dataset_reader/ann_h5_reader.py index 1bc984ac..950270c3 100644 --- a/dataset_reader/ann_h5_reader.py +++ b/dataset_reader/ann_h5_reader.py @@ -1,3 +1,4 @@ +import itertools from typing import Iterator import h5py @@ -14,9 +15,10 @@ def __init__(self, path, normalize=False): def read_queries(self) -> Iterator[Query]: data = h5py.File(self.path) + distances = data["distances"] if "distances" in data else itertools.repeat(None) for vector, expected_result, expected_scores in zip( - data["test"], data["neighbors"], data["distances"] + data["test"], data["neighbors"], distances ): if self.normalize: vector /= np.linalg.norm(vector) @@ -24,7 +26,7 @@ def read_queries(self) -> Iterator[Query]: vector=vector.tolist(), meta_conditions=None, expected_result=expected_result.tolist(), - expected_scores=expected_scores.tolist(), + expected_scores=expected_scores.tolist() if expected_scores is not None else None, ) def read_data(self, *args, **kwargs) -> Iterator[Record]: diff --git a/datasets/datasets.json b/datasets/datasets.json index 784cab9b..a18d466e 100644 --- a/datasets/datasets.json +++ b/datasets/datasets.json @@ -1219,5 +1219,21 @@ "type": "tar", "link": "https://storage.googleapis.com/ann-filtered-benchmark/datasets/random_keywords_1m_vocab_10_no_filters.tgz", "path": "random-100-match-kw-small-vocab/random_keywords_1m_vocab_10_no_filters" + }, + { + "name": "cohere-768-1M", + "vector_size": 768, + "distance": "dot", + "type": "h5", + "path": "cohere-768-1M/cohere-768-1M.hdf5", + "link": "https://dbyiw3u3rf9yr.cloudfront.net/corpora/vectorsearch/cohere-wikipedia-22-12-en-embeddings/documents-1m.hdf5.bz2" + }, + { + "name": "cohere-768-10M", + "vector_size": 768, + "distance": "dot", + "type": "h5", + "path": "cohere-768-10M/cohere-768-10M.hdf5", + "link": "https://dbyiw3u3rf9yr.cloudfront.net/corpora/vectorsearch/cohere-wikipedia-22-12-en-embeddings/documents-10m.hdf5.bz2" } ] diff --git a/experiments/configurations/cohere-calibration.json b/experiments/configurations/cohere-calibration.json new file mode 100644 index 00000000..4cb621d9 --- /dev/null +++ b/experiments/configurations/cohere-calibration.json @@ -0,0 +1,390 @@ +[ + { + "name": "cohere-cal-hnsw-float16", + "engine": "redis", + "connection_params": {}, + "collection_params": { + "algorithm": "hnsw", + "data_type": "FLOAT16", + "hnsw_config": { + "M": 32, + "DISTANCE_METRIC": "IP", + "EF_CONSTRUCTION": 200 + } + }, + "search_params": [ + { + "parallel": 100, + "top": 100, + "calibration_param": "ef", + "calibration_precision": 0.95, + "search_params": { + "data_type": "FLOAT16" + } + } + ], + "upload_params": { + "parallel": 100, + "algorithm": "hnsw", + "data_type": "FLOAT16" + } + }, + { + "name": "cohere-cal-hnsw-float32", + "engine": "redis", + "connection_params": {}, + "collection_params": { + "algorithm": "hnsw", + "data_type": "FLOAT32", + "hnsw_config": { + "M": 32, + "DISTANCE_METRIC": "IP", + "EF_CONSTRUCTION": 200 + } + }, + "search_params": [ + { + "parallel": 100, + "top": 100, + "calibration_param": "ef", + "calibration_precision": 0.95, + "search_params": { + "data_type": "FLOAT32" + } + } + ], + "upload_params": { + "parallel": 100, + "algorithm": "hnsw", + "data_type": "FLOAT32" + } + }, +{ + "name": "cohere-cal-svs-noquant-float16", + "engine": "redis", + "connection_params": {}, + "collection_params": { + "algorithm": "svs-vamana", + "data_type": "FLOAT16", + "svs-vamana_config": { + "NUM_THREADS": 16, + "DISTANCE_METRIC": "IP", + "GRAPH_MAX_DEGREE": 64, + "CONSTRUCTION_WINDOW_SIZE": 200 + } + }, + "search_params": [ + { + "algorithm": "svs-vamana", + "parallel": 100, + "top": 100, + "calibration_param": "WS_SEARCH", + "calibration_precision": 0.95, + "search_params": { + "data_type": "FLOAT16" + } + } + ], + "upload_params": { + "parallel": 100, + "data_type": "FLOAT16", + "algorithm": "svs-vamana" + } + }, + { + "name": "cohere-cal-svs-noquant-float32", + "engine": "redis", + "connection_params": {}, + "collection_params": { + "algorithm": "svs-vamana", + "data_type": "FLOAT32", + "svs-vamana_config": { + "NUM_THREADS": 16, + "DISTANCE_METRIC": "IP", + "GRAPH_MAX_DEGREE": 64, + "CONSTRUCTION_WINDOW_SIZE": 200 + } + }, + "search_params": [ + { + "algorithm": "svs-vamana", + "parallel": 100, + "top": 100, + "calibration_param": "WS_SEARCH", + "calibration_precision": 0.95, + "search_params": { + "data_type": "FLOAT32" + } + } + ], + "upload_params": { + "parallel": 100, + "data_type": "FLOAT32", + "algorithm": "svs-vamana" + } + }, + { + "name": "cohere-cal-svs-LVQ8-float16", + "engine": "redis", + "connection_params": {}, + "collection_params": { + "algorithm": "svs-vamana", + "data_type": "FLOAT16", + "svs-vamana_config": { + "NUM_THREADS": 16, + "DISTANCE_METRIC": "IP", + "GRAPH_MAX_DEGREE": 64, + "CONSTRUCTION_WINDOW_SIZE": 200, + "compression": "LVQ8" + } + }, + "search_params": [ + { + "algorithm": "svs-vamana", + "parallel": 100, + "top": 100, + "calibration_param": "WS_SEARCH", + "calibration_precision": 0.95, + "search_params": { + "data_type": "FLOAT16" + } + } + ], + "upload_params": { + "parallel": 100, + "data_type": "FLOAT16", + "algorithm": "svs-vamana" + } + }, + { + "name": "cohere-cal-svs-LVQ8-float32", + "engine": "redis", + "connection_params": {}, + "collection_params": { + "algorithm": "svs-vamana", + "data_type": "FLOAT32", + "svs-vamana_config": { + "NUM_THREADS": 16, + "DISTANCE_METRIC": "IP", + "GRAPH_MAX_DEGREE": 64, + "CONSTRUCTION_WINDOW_SIZE": 200, + "compression": "LVQ8" + } + }, + "search_params": [ + { + "algorithm": "svs-vamana", + "parallel": 100, + "top": 100, + "calibration_param": "WS_SEARCH", + "calibration_precision": 0.95, + "search_params": { + "data_type": "FLOAT32" + } + } + ], + "upload_params": { + "parallel": 100, + "data_type": "FLOAT32", + "algorithm": "svs-vamana" + } + }, + { + "name": "cohere-cal-svs-LVQ4X8-float16", + "engine": "redis", + "connection_params": {}, + "collection_params": { + "algorithm": "svs-vamana", + "data_type": "FLOAT16", + "svs-vamana_config": { + "NUM_THREADS": 16, + "DISTANCE_METRIC": "IP", + "GRAPH_MAX_DEGREE": 64, + "CONSTRUCTION_WINDOW_SIZE": 200, + "compression": "LVQ4X8" + } + }, + "search_params": [ + { + "algorithm": "svs-vamana", + "parallel": 100, + "top": 100, + "calibration_param": "WS_SEARCH", + "calibration_precision": 0.95, + "search_params": { + "data_type": "FLOAT16" + } + } + ], + "upload_params": { + "parallel": 100, + "data_type": "FLOAT16", + "algorithm": "svs-vamana" + } + }, + { + "name": "cohere-cal-svs-LVQ4X8-float32", + "engine": "redis", + "connection_params": {}, + "collection_params": { + "algorithm": "svs-vamana", + "data_type": "FLOAT32", + "svs-vamana_config": { + "NUM_THREADS": 16, + "DISTANCE_METRIC": "IP", + "GRAPH_MAX_DEGREE": 64, + "CONSTRUCTION_WINDOW_SIZE": 200, + "compression": "LVQ4X8" + } + }, + "search_params": [ + { + "algorithm": "svs-vamana", + "parallel": 100, + "top": 100, + "calibration_param": "WS_SEARCH", + "calibration_precision": 0.95, + "search_params": { + "data_type": "FLOAT32" + } + } + ], + "upload_params": { + "parallel": 100, + "data_type": "FLOAT32", + "algorithm": "svs-vamana" + } + }, + { + "name": "cohere-cal-svs-LVQ4X4-float32", + "engine": "redis", + "connection_params": {}, + "collection_params": { + "algorithm": "svs-vamana", + "data_type": "FLOAT32", + "svs-vamana_config": { + "NUM_THREADS": 16, + "DISTANCE_METRIC": "IP", + "GRAPH_MAX_DEGREE": 64, + "CONSTRUCTION_WINDOW_SIZE": 200, + "compression": "LVQ4X4" + } + }, + "search_params": [ + { + "algorithm": "svs-vamana", + "parallel": 100, + "top": 100, + "calibration_param": "WS_SEARCH", + "calibration_precision": 0.95, + "search_params": { + "data_type": "FLOAT32" + } + } + ], + "upload_params": { + "parallel": 100, + "data_type": "FLOAT32", + "algorithm": "svs-vamana" + } + }, + { + "name": "cohere-cal-svs-LVQ4X4-float16", + "engine": "redis", + "connection_params": {}, + "collection_params": { + "algorithm": "svs-vamana", + "data_type": "FLOAT16", + "svs-vamana_config": { + "NUM_THREADS": 16, + "DISTANCE_METRIC": "IP", + "GRAPH_MAX_DEGREE": 64, + "CONSTRUCTION_WINDOW_SIZE": 200, + "compression": "LVQ4X4" + } + }, + "search_params": [ + { + "algorithm": "svs-vamana", + "parallel": 100, + "top": 100, + "calibration_param": "WS_SEARCH", + "calibration_precision": 0.95, + "search_params": { + "data_type": "FLOAT16" + } + } + ], + "upload_params": { + "parallel": 100, + "data_type": "FLOAT16", + "algorithm": "svs-vamana" + } + }, + { + "name": "cohere-cal-svs-LeanVec4x8-float16", + "engine": "redis", + "connection_params": {}, + "collection_params": { + "algorithm": "svs-vamana", + "data_type": "FLOAT16", + "svs-vamana_config": { + "NUM_THREADS": 16, + "DISTANCE_METRIC": "IP", + "GRAPH_MAX_DEGREE": 64, + "CONSTRUCTION_WINDOW_SIZE": 200, + "compression": "LeanVec4x8" + } + }, + "search_params": [ + { + "algorithm": "svs-vamana", + "parallel": 100, + "top": 100, + "calibration_param": "WS_SEARCH", + "calibration_precision": 0.95, + "search_params": { + "data_type": "FLOAT16" + } + } + ], + "upload_params": { + "parallel": 100, + "data_type": "FLOAT16", + "algorithm": "svs-vamana" + } + }, + { + "name": "cohere-cal-svs-LeanVec4x8-float32", + "engine": "redis", + "connection_params": {}, + "collection_params": { + "algorithm": "svs-vamana", + "data_type": "FLOAT32", + "svs-vamana_config": { + "NUM_THREADS": 16, + "DISTANCE_METRIC": "IP", + "GRAPH_MAX_DEGREE": 64, + "CONSTRUCTION_WINDOW_SIZE": 200, + "compression": "LeanVec4x8" + } + }, + "search_params": [ + { + "algorithm": "svs-vamana", + "parallel": 100, + "top": 100, + "calibration_param": "WS_SEARCH", + "calibration_precision": 0.95, + "search_params": { + "data_type": "FLOAT32" + } + } + ], + "upload_params": { + "parallel": 100, + "data_type": "FLOAT32", + "algorithm": "svs-vamana" + } + } +] From c04328f2890b0a816ae09d65d240ab58bc61164e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mihai=20Capot=C4=83?= Date: Mon, 30 Jun 2025 15:33:15 -0700 Subject: [PATCH 2/3] Add SQ8 --- .../configurations/cohere-calibration.json | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/experiments/configurations/cohere-calibration.json b/experiments/configurations/cohere-calibration.json index 4cb621d9..097f0333 100644 --- a/experiments/configurations/cohere-calibration.json +++ b/experiments/configurations/cohere-calibration.json @@ -123,6 +123,72 @@ "algorithm": "svs-vamana" } }, + { + "name": "cohere-cal-svs-SQ8-float16", + "engine": "redis", + "connection_params": {}, + "collection_params": { + "algorithm": "svs-vamana", + "data_type": "FLOAT16", + "svs-vamana_config": { + "NUM_THREADS": 16, + "DISTANCE_METRIC": "L2", + "GRAPH_MAX_DEGREE": 64, + "CONSTRUCTION_WINDOW_SIZE": 200, + "compression": "GlobalSQ8" + } + }, + "search_params": [ + { + "algorithm": "svs-vamana", + "parallel": 100, + "top": 100, + "calibration_param": "WS_SEARCH", + "calibration_precision": 0.95, + "search_params": { + "data_type": "FLOAT16" + } + } + ], + "upload_params": { + "parallel": 100, + "data_type": "FLOAT16", + "algorithm": "svs-vamana" + } + }, + { + "name": "cohere-cal-svs-SQ8-float32", + "engine": "redis", + "connection_params": {}, + "collection_params": { + "algorithm": "svs-vamana", + "data_type": "FLOAT32", + "svs-vamana_config": { + "NUM_THREADS": 16, + "DISTANCE_METRIC": "L2", + "GRAPH_MAX_DEGREE": 64, + "CONSTRUCTION_WINDOW_SIZE": 200, + "compression": "GlobalSQ8" + } + }, + "search_params": [ + { + "algorithm": "svs-vamana", + "parallel": 100, + "top": 100, + "calibration_param": "WS_SEARCH", + "calibration_precision": 0.95, + "search_params": { + "data_type": "FLOAT32" + } + } + ], + "upload_params": { + "parallel": 100, + "data_type": "FLOAT32", + "algorithm": "svs-vamana" + } + }, { "name": "cohere-cal-svs-LVQ8-float16", "engine": "redis", From bd32c62ca7733aa41cff3b6bb04aa5c1213b2de3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mihai=20Capot=C4=83?= Date: Mon, 30 Jun 2025 15:37:16 -0700 Subject: [PATCH 3/3] Fix distance --- experiments/configurations/cohere-calibration.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/experiments/configurations/cohere-calibration.json b/experiments/configurations/cohere-calibration.json index 097f0333..8f3574a0 100644 --- a/experiments/configurations/cohere-calibration.json +++ b/experiments/configurations/cohere-calibration.json @@ -132,7 +132,7 @@ "data_type": "FLOAT16", "svs-vamana_config": { "NUM_THREADS": 16, - "DISTANCE_METRIC": "L2", + "DISTANCE_METRIC": "IP", "GRAPH_MAX_DEGREE": 64, "CONSTRUCTION_WINDOW_SIZE": 200, "compression": "GlobalSQ8" @@ -165,7 +165,7 @@ "data_type": "FLOAT32", "svs-vamana_config": { "NUM_THREADS": 16, - "DISTANCE_METRIC": "L2", + "DISTANCE_METRIC": "IP", "GRAPH_MAX_DEGREE": 64, "CONSTRUCTION_WINDOW_SIZE": 200, "compression": "GlobalSQ8"