refactor(pydantic): complete migration from Pydantic v1 to v2

This commit completes the migration of all Pydantic v1 legacy syntax to v2 across
the entire DataHub Python codebase.

**Configuration Migration (18 instances):**
- Migrate `class Config:` → `model_config = ConfigDict(...)`
- Updated ConfigModel, PermissiveConfigModel, ConnectionModel base classes
- Migrated 13 additional model classes across multiple files

**Method Migrations (76 instances):**
- `.parse_obj()` → `.model_validate()` (38 instances)
- `.parse_raw()` → `.model_validate_json()` (2 instances)
- `.dict()` → `.model_dump()` (27 instances)
- `.json()` → `.model_dump_json()` (4 instances)
- `.update_forward_refs()` → `.model_rebuild()` (3 instances)
- `.copy()` and `.schema()` - all false positives (dicts/lists/HTTP responses)

**Scope of Changes:**
- metadata-ingestion/src: 50 instances
- metadata-ingestion/tests: 30 instances
- datahub-actions: 12 instances
- smoke-test: 2 instances

**Total: 94 Pydantic v1 calls migrated**

**Key Files Updated:**
- common.py: Base ConfigModel classes
- Multiple source files: sql_queries.py, datahub_classifier.py, schema_assertion.py, etc.
- Multiple CLI files: structuredproperties_cli.py, forms_cli.py, dataset_cli.py, etc.
- Test files: RDS IAM tests, Unity Catalog tests, assertion tests, etc.
- datahub-actions: propagation_action.py, filter tests, consumer offsets
- smoke-test: stateful ingestion tests

All Pydantic v2 deprecation warnings have been resolved. The codebase is now
fully compliant with Pydantic v2 with no remaining v1 syntax.

Author: sgomezvillamor

datahub-actions/src/datahub_actions/plugin/action/propagation/docs/propagation_action.py

@@ -280,7 +280,7 @@ def process_schema_field_documentation(
                     if current_documentation_instance.attribution
                     else {}
                 )
-                source_details_parsed: SourceDetails = SourceDetails.parse_obj(
+                source_details_parsed: SourceDetails = SourceDetails.model_validate(
                     source_details
                 )
                 should_stop_propagation, reason = self.should_stop_propagation(

datahub-actions/src/datahub_actions/plugin/source/acryl/datahub_cloud_events_consumer_offsets_store.py

@@ -46,11 +46,11 @@ def from_serialized_value(cls, value: SerializedValueClass) -> "EventConsumerSta
             return cls()
 
     def to_blob(self) -> bytes:
-        return self.json().encode()
+        return self.model_dump_json().encode()
 
     @staticmethod
     def from_blob(blob: bytes) -> "EventConsumerState":
-        return EventConsumerState.parse_raw(blob.decode())
+        return EventConsumerState.model_validate_json(blob.decode())
 
 
 class DataHubEventsConsumerPlatformResourceOffsetsStore:

datahub-actions/tests/unit/plugin/transform/filter/test_filter_transformer.py

@@ -51,7 +51,7 @@ def as_json(self) -> str:
 
 
 def test_returns_none_when_diff_event_type():
-    filter_transformer_config = FilterTransformerConfig.parse_obj(
+    filter_transformer_config = FilterTransformerConfig.model_validate(
         {"event_type": "EntityChangeEvent_v1", "event": {"field1": "a", "field2": "b"}}
     )
     filter_transformer = FilterTransformer(filter_transformer_config)
@@ -68,7 +68,7 @@ def test_returns_none_when_diff_event_type():
 
 
 def test_does_exact_match():
-    filter_transformer_config = FilterTransformerConfig.parse_obj(
+    filter_transformer_config = FilterTransformerConfig.model_validate(
         {"event_type": "EntityChangeEvent_v1", "event": {"field1": "a", "field2": "b"}}
     )
     filter_transformer = FilterTransformer(filter_transformer_config)
@@ -83,7 +83,7 @@ def test_does_exact_match():
 
 
 def test_returns_none_when_no_match():
-    filter_transformer_config = FilterTransformerConfig.parse_obj(
+    filter_transformer_config = FilterTransformerConfig.model_validate(
         {"event_type": "EntityChangeEvent_v1", "event": {"field1": "a", "field2": "b"}}
     )
     filter_transformer = FilterTransformer(filter_transformer_config)
@@ -97,7 +97,7 @@ def test_returns_none_when_no_match():
 
 
 def test_matches_on_nested_event():
-    filter_transformer_config = FilterTransformerConfig.parse_obj(
+    filter_transformer_config = FilterTransformerConfig.model_validate(
         {
             "event_type": "EntityChangeEvent_v1",
             "event": {"field1": {"nested_1": {"nested_b": "a"}}},
@@ -112,7 +112,7 @@ def test_matches_on_nested_event():
 
 
 def test_returns_none_when_no_match_nested_event():
-    filter_transformer_config = FilterTransformerConfig.parse_obj(
+    filter_transformer_config = FilterTransformerConfig.model_validate(
         {
             "event_type": "EntityChangeEvent_v1",
             "event": {"field1": {"nested_1": {"nested_b": "a"}}},
@@ -127,7 +127,7 @@ def test_returns_none_when_no_match_nested_event():
 
 
 def test_returns_none_when_different_data_type():
-    filter_transformer_config = FilterTransformerConfig.parse_obj(
+    filter_transformer_config = FilterTransformerConfig.model_validate(
         {
             "event_type": "EntityChangeEvent_v1",
             "event": {"field1": {"nested_1": {"nested_b": "a"}}},
@@ -142,7 +142,7 @@ def test_returns_none_when_different_data_type():
 
 
 def test_returns_match_when_either_is_present():
-    filter_transformer_config = FilterTransformerConfig.parse_obj(
+    filter_transformer_config = FilterTransformerConfig.model_validate(
         {
             "event_type": "EntityChangeEvent_v1",
             "event": {"field1": {"nested_1": ["a", "b"]}},
@@ -157,7 +157,7 @@ def test_returns_match_when_either_is_present():
 
 
 def test_returns_none_when_neither_is_present():
-    filter_transformer_config = FilterTransformerConfig.parse_obj(
+    filter_transformer_config = FilterTransformerConfig.model_validate(
         {
             "event_type": "EntityChangeEvent_v1",
             "event": {"field1": {"nested_1": ["a", "b"]}},
@@ -172,7 +172,7 @@ def test_returns_none_when_neither_is_present():
 
 
 def test_no_match_when_list_filter_on_dict_obj():
-    filter_transformer_config = FilterTransformerConfig.parse_obj(
+    filter_transformer_config = FilterTransformerConfig.model_validate(
         {
             "event_type": "EntityChangeEvent_v1",
             "event": {"field1": {"nested_1": ["a", "b"]}},

metadata-ingestion/src/datahub/api/entities/common/serialized_value.py

@@ -3,7 +3,7 @@
 from typing import Dict, Optional, Type, TypeVar, Union
 
 from avrogen.dict_wrapper import DictWrapper
-from pydantic import BaseModel
+from pydantic import BaseModel, ConfigDict
 
 import datahub.metadata.schema_classes as models
 from datahub.metadata.schema_classes import __SCHEMA_TYPES as SCHEMA_TYPES
@@ -17,8 +17,7 @@
 
 
 class SerializedResourceValue(BaseModel):
-    class Config:
-        arbitrary_types_allowed = True
+    model_config = ConfigDict(arbitrary_types_allowed=True)
 
     content_type: str
     blob: bytes

metadata-ingestion/src/datahub/api/entities/datacontract/schema_assertion.py

@@ -3,7 +3,7 @@
 import json
 from typing import List, Union
 
-from pydantic import Field, RootModel
+from pydantic import ConfigDict, Field, RootModel
 from typing_extensions import Literal
 
 from datahub.api.entities.datacontract.assertion import BaseAssertion
@@ -36,8 +36,7 @@ def model_post_init(self, __context: object) -> None:
 
 
 class FieldListSchemaContract(BaseAssertion):
-    class Config:
-        arbitrary_types_allowed = True
+    model_config = ConfigDict(arbitrary_types_allowed=True)
 
     type: Literal["field-list"]
 

metadata-ingestion/src/datahub/api/entities/external/external_entities.py

@@ -137,19 +137,19 @@ def create(self, platform_resource: PlatformResource) -> None:
                             self.entity_class
                         )
                     )
-                    entity = self.entity_class(**entity_obj.dict())
+                    entity = self.entity_class(**entity_obj.model_dump())
 
                     # Create updated entity ID with persisted=True
                     entity_id = entity.get_id()
-                    if hasattr(entity_id, "dict"):
-                        entity_id_data = entity_id.dict()
+                    if hasattr(entity_id, "model_dump"):
+                        entity_id_data = entity_id.model_dump()
                         entity_id_data["persisted"] = True
 
                         # Create new entity ID with updated flags
                         updated_entity_id = type(entity_id)(**entity_id_data)
 
                         # Update the entity with the new ID (immutable update)
-                        entity_data = entity.dict()  # type: ignore[attr-defined]
+                        entity_data = entity.model_dump()  # type: ignore[attr-defined]
                         entity_data["id"] = updated_entity_id
                         updated_entity = type(entity)(**entity_data)
 
@@ -359,13 +359,13 @@ def search_entity_by_urn(self, urn: str) -> Optional[TExternalEntityId]:
                             self.entity_class
                         )
                     )
-                    entity = self.entity_class(**entity_obj.dict())
+                    entity = self.entity_class(**entity_obj.model_dump())
                     # Check if platform instance matches
                     entity_id = entity.get_id()
                     if entity_id.platform_instance == self.platform_instance:
                         # Create a new entity ID with the correct state instead of mutating
-                        # All our entity IDs are Pydantic models, so we can use dict() method
-                        entity_data = entity_id.dict()
+                        # All our entity IDs are Pydantic models, so we can use model_dump() method
+                        entity_data = entity_id.model_dump()
                         entity_data["persisted"] = (
                             True  # This entity was found in DataHub
                         )
@@ -433,7 +433,7 @@ def get_entity_from_datahub(
                 entity_obj = platform_resource.resource_info.value.as_pydantic_object(
                     self.entity_class
                 )
-                result = self.entity_class(**entity_obj.dict())
+                result = self.entity_class(**entity_obj.model_dump())
         elif len(platform_resources) > 1:
             # Handle multiple matches - find the one with matching platform instance
             target_platform_instance = entity_id.platform_instance
@@ -447,7 +447,7 @@ def get_entity_from_datahub(
                             self.entity_class
                         )
                     )
-                    entity = self.entity_class(**entity_obj.dict())
+                    entity = self.entity_class(**entity_obj.model_dump())
                     if entity.get_id().platform_instance == target_platform_instance:
                         result = entity
                         break

metadata-ingestion/src/datahub/cli/migrate.py

@@ -356,7 +356,7 @@ def migrate_containers(
             if mcp.aspectName == "containerProperties":
                 assert isinstance(mcp.aspect, ContainerPropertiesClass)
                 containerProperties: ContainerPropertiesClass = mcp.aspect
-                containerProperties.customProperties = newKey.dict(
+                containerProperties.customProperties = newKey.model_dump(
                     by_alias=True, exclude_none=True
                 )
                 mcp.aspect = containerProperties

metadata-ingestion/src/datahub/cli/specific/dataproduct_cli.py

@@ -245,7 +245,7 @@ def get(urn: str, to_file: str) -> None:
         if graph.exists(urn):
             dataproduct: DataProduct = DataProduct.from_datahub(graph=graph, id=urn)
             click.secho(
-                f"{json.dumps(dataproduct.dict(exclude_unset=True, exclude_none=True), indent=2)}"
+                f"{json.dumps(dataproduct.model_dump(exclude_unset=True, exclude_none=True), indent=2)}"
             )
             if to_file:
                 dataproduct.to_yaml(Path(to_file))

metadata-ingestion/src/datahub/cli/specific/dataset_cli.py

@@ -55,7 +55,7 @@ def get(urn: str, to_file: str) -> None:
         if graph.exists(urn):
             dataset: Dataset = Dataset.from_datahub(graph=graph, urn=urn)
             click.secho(
-                f"{json.dumps(dataset.dict(exclude_unset=True, exclude_none=True), indent=2)}"
+                f"{json.dumps(dataset.model_dump(exclude_unset=True, exclude_none=True), indent=2)}"
             )
             if to_file:
                 dataset.to_yaml(Path(to_file))

metadata-ingestion/src/datahub/cli/specific/forms_cli.py

@@ -41,7 +41,7 @@ def get(urn: str, to_file: str) -> None:
         if graph.exists(urn):
             form: Forms = Forms.from_datahub(graph=graph, urn=urn)
             click.secho(
-                f"{json.dumps(form.dict(exclude_unset=True, exclude_none=True), indent=2)}"
+                f"{json.dumps(form.model_dump(exclude_unset=True, exclude_none=True), indent=2)}"
             )
             if to_file:
                 form.to_yaml(Path(to_file))