From 6ce7c7cec70377ba9c4d7e9d3289d715de2bf3ed Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Sun, 10 Dec 2023 13:27:42 -0500 Subject: [PATCH 01/16] copy core method --- src/hdmf/container.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/hdmf/container.py b/src/hdmf/container.py index 8420805cb..fdc4937df 100644 --- a/src/hdmf/container.py +++ b/src/hdmf/container.py @@ -4,6 +4,7 @@ from copy import deepcopy from uuid import uuid4 from warnings import warn +from typing import Type import h5py import numpy as np @@ -789,6 +790,15 @@ def set_dataio(self, **kwargs): dataio.data = self.__data self.__data = dataio + @docval( + {'name': 'data_io_class', 'type': Type[DataIO], 'doc': 'The DataIO to apply to the data held by this Data.'} + {'name': 'data_io_kwargs', 'type': dict, 'doc': 'The keyword arguments to pass to the DataIO.'} + ) + def set_data_io(self, **kwargs): + """Apply DataIO object to the data held by this Data object.""" + data_io_class, data_io_kwargs = getargs('data_io_class', 'data_io_kwargs', kwargs) + self.__data = data_io_class(data=self.__data, **data_io_kwargs) + @docval({'name': 'func', 'type': types.FunctionType, 'doc': 'a function to transform *data*'}) def transform(self, **kwargs): """ From 3676c5c9e440c6bc0ae13436b15dd1d6a15c2a1b Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Sun, 10 Dec 2023 13:37:23 -0500 Subject: [PATCH 02/16] comma --- src/hdmf/container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hdmf/container.py b/src/hdmf/container.py index fdc4937df..721415214 100644 --- a/src/hdmf/container.py +++ b/src/hdmf/container.py @@ -791,7 +791,7 @@ def set_dataio(self, **kwargs): self.__data = dataio @docval( - {'name': 'data_io_class', 'type': Type[DataIO], 'doc': 'The DataIO to apply to the data held by this Data.'} + {'name': 'data_io_class', 'type': Type[DataIO], 'doc': 'The DataIO to apply to the data held by this Data.'}, {'name': 'data_io_kwargs', 'type': dict, 'doc': 'The keyword arguments to pass to the DataIO.'} ) def set_data_io(self, **kwargs): From 1f93094ccd4184511fc42ca7b0757fab027cd42f Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Sun, 10 Dec 2023 13:41:49 -0500 Subject: [PATCH 03/16] update container method for consistency --- src/hdmf/container.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hdmf/container.py b/src/hdmf/container.py index 721415214..c7b9ffb8a 100644 --- a/src/hdmf/container.py +++ b/src/hdmf/container.py @@ -747,11 +747,11 @@ def __smart_str_dict(d, num_indent): out += '\n' + indent + right_br return out - def set_data_io(self, dataset_name, data_io_class, **kwargs): + def set_data_io(self, dataset_name: str, data_io_class: Type[DataIO], data_io_kwargs: dict): data = self.fields.get(dataset_name) if data is None: raise ValueError(f"{dataset_name} is None and cannot be wrapped in a DataIO class") - self.fields[dataset_name] = data_io_class(data=data, **kwargs) + self.fields[dataset_name] = data_io_class(data=data, **data_io_kwargs) class Data(AbstractContainer): From 823360ea7f79d19bc3b4557205f51e879577c455 Mon Sep 17 00:00:00 2001 From: bendichter Date: Tue, 12 Dec 2023 10:25:27 -0600 Subject: [PATCH 04/16] soft deprecation of using kwargs for set_data_io --- src/hdmf/container.py | 35 +++++++++++++++++++++++++---------- tests/unit/test_container.py | 28 ++++++++++++++++++++++------ 2 files changed, 47 insertions(+), 16 deletions(-) diff --git a/src/hdmf/container.py b/src/hdmf/container.py index 9af06543e..cfac7c800 100644 --- a/src/hdmf/container.py +++ b/src/hdmf/container.py @@ -747,8 +747,9 @@ def __smart_str_dict(d, num_indent): out += '\n' + indent + right_br return out - def set_data_io(self, dataset_name: str, data_io_class: Type[DataIO], **kwargs) -> None: + def set_data_io(self, dataset_name: str, data_io_class: Type[DataIO], data_io_kwargs: dict = None, **kwargs): """ + Apply DataIO object to a dataset field of the Container. Parameters ---------- @@ -756,10 +757,20 @@ def set_data_io(self, dataset_name: str, data_io_class: Type[DataIO], **kwargs) Name of dataset to wrap in DataIO data_io_class: Type[DataIO] Class to use for DataIO, e.g. H5DataIO or ZarrDataIO + data_io_kwargs: dict, optional + kwargs passed to the constructor of the DataIO class. **kwargs: - kwargs are passed to the constructor of the DataIO class + DEPRECATED. Use data_io_kwargs instead. + kwargs are passed to the constructor of the DataIO class. """ - def set_data_io(self, dataset_name: str, data_io_class: Type[DataIO], data_io_kwargs: dict): + if kwargs or (data_io_kwargs is None): + warn( + "Use of **kwargs in Container.set_data_io() is deprecated. Please pass the DataIO kwargs as a dictionary to" + " the `data_io_kwargs` parameter instead.", + DeprecationWarning, + stacklevel=2 + ) + data_io_kwargs = kwargs data = self.fields.get(dataset_name) if data is None: raise ValueError(f"{dataset_name} is None and cannot be wrapped in a DataIO class") @@ -802,13 +813,17 @@ def set_dataio(self, **kwargs): dataio.data = self.__data self.__data = dataio - @docval( - {'name': 'data_io_class', 'type': Type[DataIO], 'doc': 'The DataIO to apply to the data held by this Data.'}, - {'name': 'data_io_kwargs', 'type': dict, 'doc': 'The keyword arguments to pass to the DataIO.'} - ) - def set_data_io(self, **kwargs): - """Apply DataIO object to the data held by this Data object.""" - data_io_class, data_io_kwargs = getargs('data_io_class', 'data_io_kwargs', kwargs) + def set_data_io(self, data_io_class: Type[DataIO], data_io_kwargs: dict) -> None: + """ + Apply DataIO object to the data held by this Data object. + + Parameters + ---------- + data_io_class: Type[DataIO] + The DataIO to apply to the data held by this Data. + data_io_kwargs: dict + The keyword arguments to pass to the DataIO. + """ self.__data = data_io_class(data=self.__data, **data_io_kwargs) @docval({'name': 'func', 'type': types.FunctionType, 'doc': 'a function to transform *data*'}) diff --git a/tests/unit/test_container.py b/tests/unit/test_container.py index 0e3fbc9eb..b5a6938d1 100644 --- a/tests/unit/test_container.py +++ b/tests/unit/test_container.py @@ -397,8 +397,10 @@ def test_get_ancestors(self): self.assertTupleEqual(parent_obj.get_ancestors(), (grandparent_obj, )) self.assertTupleEqual(child_obj.get_ancestors(), (parent_obj, grandparent_obj)) - def test_set_data_io(self): +class TestSetDataIO(TestCase): + + def setUp(self) -> None: class ContainerWithData(Container): __fields__ = ('data1', 'data2') @@ -412,12 +414,26 @@ def __init__(self, **kwargs): self.data1 = kwargs["data1"] self.data2 = kwargs["data2"] - obj = ContainerWithData("name", [1, 2, 3, 4, 5], None) - obj.set_data_io("data1", H5DataIO, chunks=True) - assert isinstance(obj.data1, H5DataIO) + self.obj = ContainerWithData("name", [1, 2, 3, 4, 5], None) - with self.assertRaises(ValueError): - obj.set_data_io("data2", H5DataIO, chunks=True) + def test_set_data_io(self): + self.obj.set_data_io("data1", H5DataIO, data_io_kwargs=dict(chunks=True)) + assert isinstance(self.obj.data1, H5DataIO) + + def test_fail_set_data_io(self): + """Attempt to set a DataIO for a dataset that is missing.""" + with self.assertRaisesWith(ValueError, "data2 is None and cannot be wrapped in a DataIO class"): + self.obj.set_data_io("data2", H5DataIO, data_io_kwargs=dict(chunks=True)) + + def test_set_data_io_old_api(self): + """Test that using the kwargs still works but throws a warning.""" + msg = ( + "Use of **kwargs in Container.set_data_io() is deprecated. Please pass the DataIO kwargs as a dictionary to" + " the `data_io_kwargs` parameter instead." + ) + with self.assertWarnsWith(DeprecationWarning, msg) as cm: + self.obj.set_data_io("data1", H5DataIO, chunks=True) + self.assertIsInstance(self.obj.data1, H5DataIO) class TestHTMLRepr(TestCase): From c0ed0b6f5d930045629ba7c72d1cfc4d3cba01b0 Mon Sep 17 00:00:00 2001 From: bendichter Date: Tue, 12 Dec 2023 10:37:15 -0600 Subject: [PATCH 05/16] move tests to test_io_hdf5tools add test for Data.set_data_io --- tests/unit/test_container.py | 39 ---------------------- tests/unit/test_io_hdf5_h5tools.py | 53 +++++++++++++++++++++++++++++- 2 files changed, 52 insertions(+), 40 deletions(-) diff --git a/tests/unit/test_container.py b/tests/unit/test_container.py index b5a6938d1..b5a2d87e8 100644 --- a/tests/unit/test_container.py +++ b/tests/unit/test_container.py @@ -2,7 +2,6 @@ from uuid import uuid4, UUID import os -from hdmf.backends.hdf5 import H5DataIO from hdmf.container import AbstractContainer, Container, Data, HERDManager from hdmf.common.resources import HERD from hdmf.testing import TestCase @@ -398,44 +397,6 @@ def test_get_ancestors(self): self.assertTupleEqual(child_obj.get_ancestors(), (parent_obj, grandparent_obj)) -class TestSetDataIO(TestCase): - - def setUp(self) -> None: - class ContainerWithData(Container): - __fields__ = ('data1', 'data2') - - @docval( - {"name": "name", "doc": "name", "type": str}, - {'name': 'data1', 'doc': 'field1 doc', 'type': list}, - {'name': 'data2', 'doc': 'field2 doc', 'type': list, 'default': None} - ) - def __init__(self, **kwargs): - super().__init__(name=kwargs["name"]) - self.data1 = kwargs["data1"] - self.data2 = kwargs["data2"] - - self.obj = ContainerWithData("name", [1, 2, 3, 4, 5], None) - - def test_set_data_io(self): - self.obj.set_data_io("data1", H5DataIO, data_io_kwargs=dict(chunks=True)) - assert isinstance(self.obj.data1, H5DataIO) - - def test_fail_set_data_io(self): - """Attempt to set a DataIO for a dataset that is missing.""" - with self.assertRaisesWith(ValueError, "data2 is None and cannot be wrapped in a DataIO class"): - self.obj.set_data_io("data2", H5DataIO, data_io_kwargs=dict(chunks=True)) - - def test_set_data_io_old_api(self): - """Test that using the kwargs still works but throws a warning.""" - msg = ( - "Use of **kwargs in Container.set_data_io() is deprecated. Please pass the DataIO kwargs as a dictionary to" - " the `data_io_kwargs` parameter instead." - ) - with self.assertWarnsWith(DeprecationWarning, msg) as cm: - self.obj.set_data_io("data1", H5DataIO, chunks=True) - self.assertIsInstance(self.obj.data1, H5DataIO) - - class TestHTMLRepr(TestCase): class ContainerWithChildAndData(Container): diff --git a/tests/unit/test_io_hdf5_h5tools.py b/tests/unit/test_io_hdf5_h5tools.py index 90934df94..d19a186a9 100644 --- a/tests/unit/test_io_hdf5_h5tools.py +++ b/tests/unit/test_io_hdf5_h5tools.py @@ -20,7 +20,7 @@ from hdmf.backends.errors import UnsupportedOperation from hdmf.build import GroupBuilder, DatasetBuilder, BuildManager, TypeMap, OrphanContainerBuildError, LinkBuilder from hdmf.container import Container -from hdmf import Data +from hdmf import Data, docval from hdmf.data_utils import DataChunkIterator, GenericDataChunkIterator, InvalidDataIOError from hdmf.spec.catalog import SpecCatalog from hdmf.spec.namespace import NamespaceCatalog, SpecNamespace @@ -3671,3 +3671,54 @@ def test_hdf5io_can_read(): assert not HDF5IO.can_read("not_a_file") assert HDF5IO.can_read("tests/unit/back_compat_tests/1.0.5.h5") assert not HDF5IO.can_read(__file__) # this file is not an HDF5 file + + +class TestContainerSetDataIO(TestCase): + + def setUp(self) -> None: + class ContainerWithData(Container): + __fields__ = ('data1', 'data2') + + @docval( + {"name": "name", "doc": "name", "type": str}, + {'name': 'data1', 'doc': 'field1 doc', 'type': list}, + {'name': 'data2', 'doc': 'field2 doc', 'type': list, 'default': None} + ) + def __init__(self, **kwargs): + super().__init__(name=kwargs["name"]) + self.data1 = kwargs["data1"] + self.data2 = kwargs["data2"] + + self.obj = ContainerWithData("name", [1, 2, 3, 4, 5], None) + + def test_set_data_io(self): + self.obj.set_data_io("data1", H5DataIO, data_io_kwargs=dict(chunks=True)) + assert isinstance(self.obj.data1, H5DataIO) + + def test_fail_set_data_io(self): + """Attempt to set a DataIO for a dataset that is missing.""" + with self.assertRaisesWith(ValueError, "data2 is None and cannot be wrapped in a DataIO class"): + self.obj.set_data_io("data2", H5DataIO, data_io_kwargs=dict(chunks=True)) + + def test_set_data_io_old_api(self): + """Test that using the kwargs still works but throws a warning.""" + msg = ( + "Use of **kwargs in Container.set_data_io() is deprecated. Please pass the DataIO kwargs as a dictionary to" + " the `data_io_kwargs` parameter instead." + ) + with self.assertWarnsWith(DeprecationWarning, msg): + self.obj.set_data_io("data1", H5DataIO, chunks=True) + self.assertIsInstance(self.obj.data1, H5DataIO) + + +class TestDataSetDataIO(TestCase): + + def setUp(self): + class MyData(Data): + pass + + self.data = MyData("my_data", [1, 2, 3]) + + def test_set_data_io(self): + self.data.set_data_io(H5DataIO, dict(chunks=True)) + assert isinstance(self.data.data, H5DataIO) From bbc58f7fe07ffb4eb384d093147d45616d3be735 Mon Sep 17 00:00:00 2001 From: bendichter Date: Tue, 12 Dec 2023 10:38:53 -0600 Subject: [PATCH 06/16] fix ruff --- src/hdmf/container.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hdmf/container.py b/src/hdmf/container.py index cfac7c800..b2b13814e 100644 --- a/src/hdmf/container.py +++ b/src/hdmf/container.py @@ -765,8 +765,8 @@ def set_data_io(self, dataset_name: str, data_io_class: Type[DataIO], data_io_kw """ if kwargs or (data_io_kwargs is None): warn( - "Use of **kwargs in Container.set_data_io() is deprecated. Please pass the DataIO kwargs as a dictionary to" - " the `data_io_kwargs` parameter instead.", + "Use of **kwargs in Container.set_data_io() is deprecated. Please pass the DataIO kwargs as a " + "dictionary to the `data_io_kwargs` parameter instead.", DeprecationWarning, stacklevel=2 ) From f57ce28ed46aa91e3a77596a9566b2b9724dded0 Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Tue, 12 Dec 2023 12:18:26 -0500 Subject: [PATCH 07/16] Update CHANGELOG.md --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1fc3a2995..df6447355 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## HDMF 3.11.1 (Upcoming) +### Enhancements +- Add Data.set_data_io allows(), which allows for setting a DataIO to a data object after-the-fact. @bendichter and @CodyCBakerPhD [#1013](https://github.com/hdmf-dev/hdmf/pull/1013) + ### Minor Improvements - Updated `__gather_columns` to ignore the order of bases when generating columns from the super class. @mavaylon1 [#991](https://github.com/hdmf-dev/hdmf/pull/991) - Update `get_key` to return all the keys if there are multiple within a `HERD` instance. @mavaylon1 [#999](https://github.com/hdmf-dev/hdmf/pull/999) From 5ab39fe7a4518c370a09813f86fda111c32f50f9 Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Tue, 12 Dec 2023 12:19:43 -0500 Subject: [PATCH 08/16] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index df6447355..bcb24991b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # HDMF Changelog -## HDMF 3.11.1 (Upcoming) +## HDMF 3.12.0 (Upcoming) ### Enhancements - Add Data.set_data_io allows(), which allows for setting a DataIO to a data object after-the-fact. @bendichter and @CodyCBakerPhD [#1013](https://github.com/hdmf-dev/hdmf/pull/1013) From efc560ef6c0af6cc23d61659d58af882c777ea35 Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Tue, 12 Dec 2023 14:12:11 -0500 Subject: [PATCH 09/16] Update src/hdmf/container.py --- src/hdmf/container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hdmf/container.py b/src/hdmf/container.py index b2b13814e..96cbbd785 100644 --- a/src/hdmf/container.py +++ b/src/hdmf/container.py @@ -758,7 +758,7 @@ def set_data_io(self, dataset_name: str, data_io_class: Type[DataIO], data_io_kw data_io_class: Type[DataIO] Class to use for DataIO, e.g. H5DataIO or ZarrDataIO data_io_kwargs: dict, optional - kwargs passed to the constructor of the DataIO class. + keyword arguments passed to the constructor of the DataIO class. **kwargs: DEPRECATED. Use data_io_kwargs instead. kwargs are passed to the constructor of the DataIO class. From 07cf784b9bb188b0a55a3051d0156498c50c68a0 Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Tue, 12 Dec 2023 15:33:30 -0800 Subject: [PATCH 10/16] Update tests/unit/test_io_hdf5_h5tools.py --- tests/unit/test_io_hdf5_h5tools.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/test_io_hdf5_h5tools.py b/tests/unit/test_io_hdf5_h5tools.py index d19a186a9..6a1213713 100644 --- a/tests/unit/test_io_hdf5_h5tools.py +++ b/tests/unit/test_io_hdf5_h5tools.py @@ -3709,6 +3709,7 @@ def test_set_data_io_old_api(self): with self.assertWarnsWith(DeprecationWarning, msg): self.obj.set_data_io("data1", H5DataIO, chunks=True) self.assertIsInstance(self.obj.data1, H5DataIO) + self.assertTrue(self.obj.data1.chunks) class TestDataSetDataIO(TestCase): From a215fb1186e4e3d8e9214e3df7bee7b78ddd4d7f Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Tue, 12 Dec 2023 15:34:27 -0800 Subject: [PATCH 11/16] Update tests/unit/test_io_hdf5_h5tools.py --- tests/unit/test_io_hdf5_h5tools.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/test_io_hdf5_h5tools.py b/tests/unit/test_io_hdf5_h5tools.py index 6a1213713..955681a20 100644 --- a/tests/unit/test_io_hdf5_h5tools.py +++ b/tests/unit/test_io_hdf5_h5tools.py @@ -3694,6 +3694,7 @@ def __init__(self, **kwargs): def test_set_data_io(self): self.obj.set_data_io("data1", H5DataIO, data_io_kwargs=dict(chunks=True)) assert isinstance(self.obj.data1, H5DataIO) + assert self.obj.data1.chunks def test_fail_set_data_io(self): """Attempt to set a DataIO for a dataset that is missing.""" From 6e9db902c1790d51dc72fb052a06277a440972ba Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Tue, 12 Dec 2023 15:34:56 -0800 Subject: [PATCH 12/16] Update tests/unit/test_io_hdf5_h5tools.py --- tests/unit/test_io_hdf5_h5tools.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/test_io_hdf5_h5tools.py b/tests/unit/test_io_hdf5_h5tools.py index 955681a20..4cef9013d 100644 --- a/tests/unit/test_io_hdf5_h5tools.py +++ b/tests/unit/test_io_hdf5_h5tools.py @@ -3724,3 +3724,4 @@ class MyData(Data): def test_set_data_io(self): self.data.set_data_io(H5DataIO, dict(chunks=True)) assert isinstance(self.data.data, H5DataIO) + assert self.data.data.chunks From 0c5ccb07e7698bdd580a270ff287b685b5965ea1 Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Tue, 12 Dec 2023 18:38:10 -0500 Subject: [PATCH 13/16] Update src/hdmf/container.py --- src/hdmf/container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hdmf/container.py b/src/hdmf/container.py index 96cbbd785..6498cde4f 100644 --- a/src/hdmf/container.py +++ b/src/hdmf/container.py @@ -757,7 +757,7 @@ def set_data_io(self, dataset_name: str, data_io_class: Type[DataIO], data_io_kw Name of dataset to wrap in DataIO data_io_class: Type[DataIO] Class to use for DataIO, e.g. H5DataIO or ZarrDataIO - data_io_kwargs: dict, optional + data_io_kwargs: dict keyword arguments passed to the constructor of the DataIO class. **kwargs: DEPRECATED. Use data_io_kwargs instead. From d6c60d9e3453314fe09d6e1026b3947a1ae15558 Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Tue, 12 Dec 2023 15:39:14 -0800 Subject: [PATCH 14/16] Update test_io_hdf5_h5tools.py --- tests/unit/test_io_hdf5_h5tools.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/test_io_hdf5_h5tools.py b/tests/unit/test_io_hdf5_h5tools.py index 4cef9013d..5a4fd5a32 100644 --- a/tests/unit/test_io_hdf5_h5tools.py +++ b/tests/unit/test_io_hdf5_h5tools.py @@ -3694,7 +3694,7 @@ def __init__(self, **kwargs): def test_set_data_io(self): self.obj.set_data_io("data1", H5DataIO, data_io_kwargs=dict(chunks=True)) assert isinstance(self.obj.data1, H5DataIO) - assert self.obj.data1.chunks + assert self.obj.data1.io_settings["chunks"] def test_fail_set_data_io(self): """Attempt to set a DataIO for a dataset that is missing.""" @@ -3710,7 +3710,7 @@ def test_set_data_io_old_api(self): with self.assertWarnsWith(DeprecationWarning, msg): self.obj.set_data_io("data1", H5DataIO, chunks=True) self.assertIsInstance(self.obj.data1, H5DataIO) - self.assertTrue(self.obj.data1.chunks) + self.assertTrue(self.obj.data1.io_settings["chunks"]) class TestDataSetDataIO(TestCase): @@ -3724,4 +3724,4 @@ class MyData(Data): def test_set_data_io(self): self.data.set_data_io(H5DataIO, dict(chunks=True)) assert isinstance(self.data.data, H5DataIO) - assert self.data.data.chunks + assert self.data.data.io_settings["chunks"] From eb59d65e5eca169a78dfe0e41eda2f678b9b88dc Mon Sep 17 00:00:00 2001 From: bendichter Date: Tue, 12 Dec 2023 17:45:23 -0600 Subject: [PATCH 15/16] deprecate Data.set_dataio() --- src/hdmf/container.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/hdmf/container.py b/src/hdmf/container.py index 6498cde4f..229e20083 100644 --- a/src/hdmf/container.py +++ b/src/hdmf/container.py @@ -809,6 +809,11 @@ def set_dataio(self, **kwargs): """ Apply DataIO object to the data held by this Data object """ + warn( + "Data.set_dataio() is deprecated. Please use Data.set_data_io() instead.", + DeprecationWarning, + stacklevel=2, + ) dataio = getargs('dataio', kwargs) dataio.data = self.__data self.__data = dataio From 7ea548f069943ad3838621cfd49ac7a1ff2c50ac Mon Sep 17 00:00:00 2001 From: bendichter Date: Tue, 12 Dec 2023 18:04:47 -0600 Subject: [PATCH 16/16] make DynamicTable expandable --- src/hdmf/common/table.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/hdmf/common/table.py b/src/hdmf/common/table.py index 2e2b56979..c7aa3daf2 100644 --- a/src/hdmf/common/table.py +++ b/src/hdmf/common/table.py @@ -11,8 +11,10 @@ import numpy as np import pandas as pd import itertools +from typing import Type from . import register_class, EXP_NAMESPACE +from ..backends.hdf5.h5_utils import H5DataIO from ..container import Container, Data from ..data_utils import DataIO, AbstractDataChunkIterator from ..utils import docval, getargs, ExtenderMeta, popargs, pystr, AllowPositional, check_type @@ -1323,6 +1325,25 @@ def copy(self): colnames=self.colnames) return self.__class__(**kwargs) + def make_expandable(self, data_io_class: Type[DataIO] = H5DataIO, data_io_kwargs: dict = None): + """ + + Parameters + ---------- + data_io_class: Type[DataIO], default=H5DataIO + data_io_kwargs: dict, optional + Keyword arguments to pass to the data_io_class. If None, defaults to dict(shape=(None, *data.shape[1:])) + """ + + if data_io_kwargs is None: + data_io_kwargs = dict() + + for column in self.columns + [self.id]: + shape = column.data.shape + kwargs = dict(shape=(None, *shape[1:])) + kwargs.update(**data_io_kwargs) + column.set_data_io(data_io_class=data_io_class, data_io_kwargs=kwargs) + @register_class('DynamicTableRegion') class DynamicTableRegion(VectorData):