Refactor pipeline directory structure and filenames

.github/workflows/test-pipelines.yml

@@ -41,19 +41,19 @@ jobs:
         run: |
           failed=()
           for p in \
-            "pipelines/helloWorld/hello_pipeline.py" \
-            "pipelines/caching/cache_pipeline.py" \
-            "pipelines/fanOut/fan_pipeline.py" \
-            "pipelines/metadata/meta_pipeline.py" \
-            "pipelines/parameters/param_pipeline.py" \
-            "pipelines/retries/robust_pipeline.py" \
-            "pipelines/stepIO/io_pipeline.py" \
-            "pipelines/tagging/tagged_pipeline.py" \
-            "pipelines/visualizations/viz_pipeline.py" \
-            "pipelines/yamlConfig/yaml_pipeline.py"; do
+            "pipelines/hello_pipeline/hello_pipeline.py" \
+            "pipelines/cache_pipeline/cache_pipeline.py" \
+            "pipelines/fan_pipeline/fan_pipeline.py" \
+            "pipelines/meta_pipeline/meta_pipeline.py" \
+            "pipelines/param_pipeline/param_pipeline.py" \
+            "pipelines/robust_pipeline/robust_pipeline.py" \
+            "pipelines/io_pipeline/io_pipeline.py" \
+            "pipelines/tagged_pipeline/tagged_pipeline.py" \
+            "pipelines/viz_pipeline/viz_pipeline.py" \
+            "pipelines/yaml_pipeline/yaml_pipeline.py"; do
 
             echo "Running $p…"
-              if [[ "$p" == *"retries/robust_pipeline.py" ]]; then
+              if [[ "$p" == *"robust_pipeline/robust_pipeline.py" ]]; then
                 PYTHONPATH=$GITHUB_WORKSPACE:$PYTHONPATH python "$p" || echo "⚠  robust_pipeline demo: failure expected"
               else
                 PYTHONPATH=$GITHUB_WORKSPACE:$PYTHONPATH python "$p" || failed+=("$p")

pipelines/tagging/tagged_pipeline.py → pipelines/tagged_pipeline/tagged_pipeline.py

@@ -50,7 +50,7 @@ def process_data(
 
 # Pipeline with cascade tags - these will be applied to all artifacts created during execution
 @pipeline(tags=["tutorial", Tag(name="experiment", cascade=True)])
-def artifact_tagging_pipeline():
+def tagged_pipeline():
     """Pipeline demonstrating various artifact tagging approaches."""
     raw_data = create_raw_data()
     processed_data = process_data(raw_data)
@@ -66,8 +66,8 @@ def artifact_tagging_pipeline():
     logger.info("  4. Filtering and querying artifacts by tags")
 
     # Run the pipeline
-    artifact_tagging_pipeline()
+    tagged_pipeline()
 
     # Log dashboard URLs
-    log_dashboard_urls("artifact_tagging_pipeline")
+    log_dashboard_urls("tagged_pipeline")
     logger.info("Run again to see how tags accumulate across multiple runs")

pipelines/visualizations/visualizations.md → pipelines/viz_pipeline/visualizations.md

@@ -26,7 +26,7 @@ def scatter(df: pd.DataFrame) -> Annotated[HTMLString, "scatter_plot"]:
 - **HTMLString type** lets you embed rich content in the dashboard
 - **Dashboard integration** makes results easily shareable
 
-![Two visualizations generated by this pipeline](./pipelines/visualizations/visualizations-dashboard.png)
+![Two visualizations generated by this pipeline](./pipelines/viz_pipeline/visualizations-dashboard.png)
 
 ## Try it yourself
 

pipelines/yamlConfig/yaml_pipeline.py → pipelines/yaml_pipeline/yaml_pipeline.py

@@ -24,7 +24,7 @@ def yaml_pipeline(name: str = "world"):
     logger.info("Starting YAML-configured pipeline")
     # run exactly with the YAML you wrote
     run = yaml_pipeline.with_options(
-        config_path="pipelines/yamlConfig/my_run.yaml"
+        config_path="pipelines/yaml_pipeline/my_run.yaml"
     )()
 
     # fetch artifact afterwards so users see something in the console

src/tutorialMetadata.json

@@ -17,8 +17,8 @@
       "canRunPipeline": true,
       "steps": [
         {
-          "doc": "pipelines/helloWorld/helloWorld.md",
-          "code": "pipelines/helloWorld/hello_pipeline.py"
+          "doc": "pipelines/hello_pipeline/helloWorld.md",
+          "code": "pipelines/hello_pipeline/hello_pipeline.py"
         }
       ]
     },
@@ -28,8 +28,8 @@
       "canRunPipeline": true,
       "steps": [
         {
-          "doc": "pipelines/stepIO/stepIO.md",
-          "code": "pipelines/stepIO/io_pipeline.py"
+          "doc": "pipelines/io_pipeline/stepIO.md",
+          "code": "pipelines/io_pipeline/io_pipeline.py"
         }
       ]
     },
@@ -39,8 +39,8 @@
       "canRunPipeline": true,
       "steps": [
         {
-          "doc": "pipelines/parameters/parameters.md",
-          "code": "pipelines/parameters/param_pipeline.py"
+          "doc": "pipelines/param_pipeline/parameters.md",
+          "code": "pipelines/param_pipeline/param_pipeline.py"
         }
       ]
     },
@@ -50,8 +50,8 @@
       "canRunPipeline": true,
       "steps": [
         {
-          "doc": "pipelines/tagging/tagging.md",
-          "code": "pipelines/tagging/tagged_pipeline.py"
+          "doc": "pipelines/tagged_pipeline/tagging.md",
+          "code": "pipelines/tagged_pipeline/tagged_pipeline.py"
         }
       ]
     },
@@ -61,8 +61,8 @@
       "canRunPipeline": true,
       "steps": [
         {
-          "doc": "pipelines/metadata/metadata.md",
-          "code": "pipelines/metadata/meta_pipeline.py"
+          "doc": "pipelines/meta_pipeline/metadata.md",
+          "code": "pipelines/meta_pipeline/meta_pipeline.py"
         }
       ]
     },
@@ -72,8 +72,8 @@
       "canRunPipeline": true,
       "steps": [
         {
-          "doc": "pipelines/caching/caching.md",
-          "code": "pipelines/caching/cache_pipeline.py"
+          "doc": "pipelines/cache_pipeline/caching.md",
+          "code": "pipelines/cache_pipeline/cache_pipeline.py"
         }
       ]
     },
@@ -83,8 +83,8 @@
       "canRunPipeline": true,
       "steps": [
         {
-          "doc": "pipelines/visualizations/visualizations.md",
-          "code": "pipelines/visualizations/viz_pipeline.py"
+          "doc": "pipelines/viz_pipeline/visualizations.md",
+          "code": "pipelines/viz_pipeline/viz_pipeline.py"
         }
       ]
     },
@@ -94,8 +94,8 @@
       "canRunPipeline": true,
       "steps": [
         {
-          "doc": "pipelines/fanOut/fanOut.md",
-          "code": "pipelines/fanOut/fan_pipeline.py"
+          "doc": "pipelines/fan_pipeline/fanOut.md",
+          "code": "pipelines/fan_pipeline/fan_pipeline.py"
         }
       ]
     },
@@ -105,8 +105,8 @@
       "canRunPipeline": true,
       "steps": [
         {
-          "doc": "pipelines/retries/retries.md",
-          "code": "pipelines/retries/robust_pipeline.py"
+          "doc": "pipelines/robust_pipeline/retries.md",
+          "code": "pipelines/robust_pipeline/robust_pipeline.py"
         }
       ]
     },
@@ -116,8 +116,8 @@
       "canRunPipeline": true,
       "steps": [
         {
-          "doc": "pipelines/yamlConfig/yamlConfig.md",
-          "code": "pipelines/yamlConfig/yaml_pipeline.py"
+          "doc": "pipelines/yaml_pipeline/yamlConfig.md",
+          "code": "pipelines/yaml_pipeline/yaml_pipeline.py"
         }
       ]
     },