From 90f0c6d521f6ff08e916956ab1b681206f1b31e3 Mon Sep 17 00:00:00 2001 From: NeilBarton-NOAA Date: Thu, 30 Oct 2025 17:28:47 -0400 Subject: [PATCH 01/26] initial commits for reducing size of ocean and sea ice output --- dev/parm/config/gefs/config.oceanice_products | 3 +- dev/parm/config/gfs/config.oceanice_products | 2 + parm/archive/ice_subset.yaml.j2 | 8 ++++ parm/archive/ocean_subset.yaml.j2 | 8 ++++ parm/post/oceanice_products_gfs.yaml | 48 +++++++++++++------ parm/post/oceanice_products_sfs.yaml | 44 +++++++++++------ ush/python/pygfs/task/oceanice_products.py | 40 +++++++++++----- 7 files changed, 112 insertions(+), 41 deletions(-) create mode 100644 parm/archive/ice_subset.yaml.j2 create mode 100644 parm/archive/ocean_subset.yaml.j2 diff --git a/dev/parm/config/gefs/config.oceanice_products b/dev/parm/config/gefs/config.oceanice_products index 48483613f43..7b30de84bc5 100644 --- a/dev/parm/config/gefs/config.oceanice_products +++ b/dev/parm/config/gefs/config.oceanice_products @@ -7,9 +7,10 @@ echo "BEGIN: config.oceanice_products" # Get task specific resources source "${EXPDIR}/config.resources" oceanice_products +export write_grib2=False +export write_netcdf=False export OCEANICEPRODUCTS_CONFIG="${PARMgfs}/post/oceanice_products_${NET}.yaml" # Maximum number of rocoto tasks per member export MAX_TASKS=25 - echo "END: config.oceanice_products" diff --git a/dev/parm/config/gfs/config.oceanice_products b/dev/parm/config/gfs/config.oceanice_products index 6cce26997ce..7f1d94bf0b9 100644 --- a/dev/parm/config/gfs/config.oceanice_products +++ b/dev/parm/config/gfs/config.oceanice_products @@ -10,6 +10,8 @@ source "${EXPDIR}/config.resources" oceanice_products # Maximum number of rocoto tasks per member export MAX_TASKS=25 +export write_grib2=False +export write_netcdf=False export OCEANICEPRODUCTS_CONFIG="${PARMgfs}/post/oceanice_products_gfs.yaml" # No. of forecast hours to process in a single job diff --git a/parm/archive/ice_subset.yaml.j2 b/parm/archive/ice_subset.yaml.j2 new file mode 100644 index 00000000000..5929c02af4e --- /dev/null +++ b/parm/archive/ice_subset.yaml.j2 @@ -0,0 +1,8 @@ +ice_subset: + {% set head = "gfs.ice.t" + cycle_HH + "z." %} + name: "OCEAN_SUBSET" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ice_subset.tar" + required: + {% for fhr in range(6, FHMAX_GFS + 6, 6) %} + - "{{ COMIN_ICE_NATIVE | relpath(ROTDIR) }}/{{ head }}.native.f{{ '%03d' % fhr }}.nc" + {% endfor %} diff --git a/parm/archive/ocean_subset.yaml.j2 b/parm/archive/ocean_subset.yaml.j2 new file mode 100644 index 00000000000..9c33d409a24 --- /dev/null +++ b/parm/archive/ocean_subset.yaml.j2 @@ -0,0 +1,8 @@ +ocean_subset: + {% set head = "gfs.ocean.t" + cycle_HH + "z." %} + name: "OCEAN_SUBSET" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ocean_subset.tar" + required: + {% for fhr in range(6, FHMAX_GFS + 6, 6) %} + - "{{ COMIN_OCEAN_NATIVE | relpath(ROTDIR) }}/{{ head }}.native.f{{ '%03d' % fhr }}.nc" + {% endfor %} diff --git a/parm/post/oceanice_products_gfs.yaml b/parm/post/oceanice_products_gfs.yaml index e961975dd77..291cf395bb9 100644 --- a/parm/post/oceanice_products_gfs.yaml +++ b/parm/post/oceanice_products_gfs.yaml @@ -1,8 +1,8 @@ ocnicepost: executable: "ocnicepost.x" namelist: - write_grib2: True - write_netcdf: False + write_grib2: {{ write_grib2 }} + write_netcdf: {{ write_netcdf }} debug: False fix_data: mkdir: @@ -11,6 +11,7 @@ ocnicepost: - ["{{ EXECgfs }}/ocnicepost.x", "{{ DATA }}/"] - ["{{ PARMgfs }}/post/ocnicepost.nml.jinja2", "{{ DATA }}/"] - ["{{ PARMgfs }}/post/{{ component }}_gfs.csv", "{{ DATA }}/{{ component }}.csv"] + {% if write_grib2 and write_netcdf %} - ["{{ FIXgfs }}/mom6/post/{{ model_grid }}/tripole.{{ model_grid }}.Bu.to.Ct.bilinear.nc", "{{ DATA }}/"] - ["{{ FIXgfs }}/mom6/post/{{ model_grid }}/tripole.{{ model_grid }}.Cu.to.Ct.bilinear.nc", "{{ DATA }}/"] - ["{{ FIXgfs }}/mom6/post/{{ model_grid }}/tripole.{{ model_grid }}.Cv.to.Ct.bilinear.nc", "{{ DATA }}/"] @@ -19,6 +20,7 @@ ocnicepost: - ["{{ FIXgfs }}/mom6/post/{{ model_grid }}/tripole.{{ model_grid }}.Ct.to.rect.{{ grid }}.conserve.nc", "{{ DATA }}/"] - ["{{ FIXgfs }}/mom6/post/template.global.{{ grid }}.gb2", "{{ DATA }}/"] {% endfor %} + {% endif %} ocean: namelist: @@ -28,26 +30,34 @@ ocean: cosvar: "cos_rot" angvar: "" {% if model_grid == 'mx025' or model_grid == 'mx050' or model_grid == 'mx100' %} - ocean_levels: [5, 15, 25, 35, 45, 55, 65, 75, 85, 95, 105, 115, 125, 135, 145, 155, 165, 175, 185, 195, 205, 215, 226, 241, 267, 309, 374, 467, 594, 757, 960, 1204, 1490, 1817, 2184, 2587, 3024, 3489, 3977, 4481] + ocean_levels: [5, 15, 25, 35, 45, 55, 65, 75, 85, 95, 105, 115, 125, 135, 145, 155, 165, 175, 185, 195, 205, 215] {% elif model_grid == 'mx500' %} ocean_levels: [5, 15, 25, 35, 45, 55, 65, 75, 85, 95, 105, 115, 125, 135, 145, 155, 165, 175, 185, 195, 205, 215, 226, 241, 267] {% endif %} - subset: ['SSH', 'SST', 'SSS', 'speed', 'MLD_003', 'latent', 'sensible', 'SW', 'LW', 'LwLatSens', 'Heat_PmE', 'SSU', 'SSV', 'taux', 'tauy', 'temp', 'so', 'uo', 'vo'] + subset: ['SSH', 'SST', 'SSS', 'speed', 'MLD_003', 'SSU', 'SSV', 'temp', 'tob', 'so', 'uo', 'vo'] data_in: copy: - - ["{{ COMIN_OCEAN_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.6hr_avg.f{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/ocean.nc"] + - ["{{ COMIN_OCEAN_HISTORY }}/{{ RUN }}.ocean.t{{ current_cycle | strftime('%H') }}z.{{ interval }}hr_avg.f{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/ocean.nc"] data_out: mkdir: - - "{{ COMOUT_OCEAN_NETCDF }}" + - "{{ COMOUT_OCEAN_NETCDF }}/native" {% for grid in product_grids %} + {% if write_netcdf %} + - "{{ COMOUT_OCEAN_NETCDF }}/{{ grid }}" + {% endif %} + {% if write_grib2 %} - "{{ COMOUT_OCEAN_GRIB }}/{{ grid }}" + {% endif %} {% endfor %} copy: - - ["{{ DATA }}/ocean_subset.nc", "{{ COMOUT_OCEAN_NETCDF }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.tripolar.f{{ '%03d' % forecast_hour }}.nc"] + - ["{{ DATA }}/ocean_subset.nc", "{{ COMOUT_OCEAN_NETCDF }}/native/{{ RUN }}.ocean.t{{ current_cycle | strftime('%H') }}z.native.f{{ '%03d' % forecast_hour }}.nc"] + - ["{{ DATA }}/ocean_compressed.nc", "{{ COMIN_OCEAN_HISTORY }}/{{ RUN }}.ocean.t{{ current_cycle | strftime('%H') }}z.{{ interval }}hr_avg.f{{ '%03d' % forecast_hour }}.nc"] + {% if write_grib2 %} {% for grid in product_grids %} - - ["{{ DATA }}/ocean.{{ grid }}.grib2", "{{ COMOUT_OCEAN_GRIB }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2"] - - ["{{ DATA }}/ocean.{{ grid }}.grib2.idx", "{{ COMOUT_OCEAN_GRIB }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2.idx"] + - ["{{ DATA }}/ocean.{{ grid }}.grib2", "{{ COMOUT_OCEAN_GRIB }}/{{ grid }}/{{ RUN }}.ocean.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2"] + - ["{{ DATA }}/ocean.{{ grid }}.grib2.idx", "{{ COMOUT_OCEAN_GRIB }}/{{ grid }}/{{ RUN }}.ocean.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2.idx"] {% endfor %} + {% endif %} ice: namelist: @@ -56,19 +66,27 @@ ice: sinvar: "" cosvar: "" angvar: "ANGLET" - subset: ['hi_h', 'hs_h', 'aice_h', 'Tsfc_h', 'uvel_h', 'vvel_h', 'frzmlt_h', 'albsni_h', 'mlt_onset_h', 'frz_onset_h'] + subset: ['hi_h', 'hs_h', 'aice_h', 'Tsfc_h', 'uvel_h', 'vvel_h', 'albsni_h'] data_in: copy: - - ["{{ COMIN_ICE_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.6hr_avg.f{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/ice.nc"] + - ["{{ COMIN_ICE_HISTORY }}/{{ RUN }}.ice.t{{ current_cycle | strftime('%H') }}z.{{ interval }}hr_avg.f{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/ice.nc"] data_out: mkdir: - - "{{ COMOUT_ICE_NETCDF }}" + - "{{ COMOUT_ICE_NETCDF }}/native" {% for grid in product_grids %} + {% if write_netcdf %} + - "{{ COMOUT_ICE_NETCDF }}/{{ grid }}" + {% endif %} + {% if write_grib2 %} - "{{ COMOUT_ICE_GRIB }}/{{ grid }}" + {% endif %} {% endfor %} copy: - - ["{{ DATA }}/ice_subset.nc", "{{ COMOUT_ICE_NETCDF }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.tripolar.f{{ '%03d' % forecast_hour }}.nc"] + - ["{{ DATA }}/ice_subset.nc", "{{ COMOUT_ICE_NETCDF }}/native/{{ RUN }}.ice.t{{ current_cycle | strftime('%H') }}z.native.f{{ '%03d' % forecast_hour }}.nc"] + - ["{{ DATA }}/ice_compressed.nc", "{{ COMIN_ICE_HISTORY }}/{{ RUN }}.ice.t{{ current_cycle | strftime('%H') }}z.{{ interval }}hr_avg.f{{ '%03d' % forecast_hour }}.nc"] {% for grid in product_grids %} - - ["{{ DATA }}/ice.{{ grid }}.grib2", "{{ COMOUT_ICE_GRIB }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2"] - - ["{{ DATA }}/ice.{{ grid }}.grib2.idx", "{{ COMOUT_ICE_GRIB }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2.idx"] + {% if write_grib2 %} + - ["{{ DATA }}/ice.{{ grid }}.grib2", "{{ COMOUT_ICE_GRIB }}/{{ grid }}/{{ RUN }}.ice.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2"] + - ["{{ DATA }}/ice.{{ grid }}.grib2.idx", "{{ COMOUT_ICE_GRIB }}/{{ grid }}/{{ RUN }}.ice.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2.idx"] + {% endif %} {% endfor %} diff --git a/parm/post/oceanice_products_sfs.yaml b/parm/post/oceanice_products_sfs.yaml index e7543ee9a3c..c284069abe1 100644 --- a/parm/post/oceanice_products_sfs.yaml +++ b/parm/post/oceanice_products_sfs.yaml @@ -1,16 +1,17 @@ ocnicepost: executable: "ocnicepost.x" namelist: - write_grib2: True - write_netcdf: False + write_grib2: {{ write_grib2 }} + write_netcdf: {{ write_netcdf }} debug: False fix_data: mkdir: - "{{ DATA }}" copy: - - ["{{ EXECgfs }}/ocnicepost.x", "{{ DATA }}/"] - ["{{ PARMgfs }}/post/ocnicepost.nml.jinja2", "{{ DATA }}/"] + {% if write_grib2 and write_netcdf %} - ["{{ PARMgfs }}/post/{{ component }}_gfs.csv", "{{ DATA }}/{{ component }}.csv"] + - ["{{ EXECgfs }}/ocnicepost.x", "{{ DATA }}/"] - ["{{ FIXgfs }}/mom6/post/{{ model_grid }}/tripole.{{ model_grid }}.Bu.to.Ct.bilinear.nc", "{{ DATA }}/"] - ["{{ FIXgfs }}/mom6/post/{{ model_grid }}/tripole.{{ model_grid }}.Cu.to.Ct.bilinear.nc", "{{ DATA }}/"] - ["{{ FIXgfs }}/mom6/post/{{ model_grid }}/tripole.{{ model_grid }}.Cv.to.Ct.bilinear.nc", "{{ DATA }}/"] @@ -19,6 +20,7 @@ ocnicepost: - ["{{ FIXgfs }}/mom6/post/{{ model_grid }}/tripole.{{ model_grid }}.Ct.to.rect.{{ grid }}.conserve.nc", "{{ DATA }}/"] - ["{{ FIXgfs }}/mom6/post/template.global.{{ grid }}.gb2", "{{ DATA }}/"] {% endfor %} + {% endif %} ocean: namelist: @@ -28,27 +30,34 @@ ocean: cosvar: "cos_rot" angvar: "" {% if model_grid == 'mx025' or model_grid == 'mx050' or model_grid == 'mx100' %} - ocean_levels: [5, 15, 25, 35, 45, 55, 65, 75, 85, 95, 105, 115, 125, 135, 145, 155, 165, 175, 185, 195, 205, 215, 226, 241, 267, 309, 374, 467, 594, 757, 960, 1204, 1490, 1817, 2184, 2587, 3024, 3489, 3977, 4481] + ocean_levels: [5, 15, 25, 35, 45, 55, 65, 75, 85, 95, 105, 115, 125, 135, 145, 155, 165, 175, 185, 195, 205, 215] {% elif model_grid == 'mx500' %} ocean_levels: [5, 15, 25, 35, 45, 55, 65, 75, 85, 95, 105, 115, 125, 135, 145, 155, 165, 175, 185, 195, 205, 215, 226, 241, 267] {% endif %} - subset: ['SSH', 'SST', 'SSS', 'speed', 'MLD_003', 'latent', 'sensible', 'SW', 'LW', 'LwLatSens', 'Heat_PmE', 'SSU', 'SSV', 'taux', 'tauy', 'temp', 'tob', 'so', 'uo', 'vo'] + subset: ['SSH', 'SST', 'SSS', 'speed', 'MLD_003', 'SSU', 'SSV', 'temp', 'tob', 'so', 'uo', 'vo'] data_in: copy: - - ["{{ COMIN_OCEAN_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ interval }}hr_avg.f{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/ocean.nc"] + - ["{{ COMIN_OCEAN_HISTORY }}/{{ RUN }}.ocean.t{{ current_cycle | strftime('%H') }}z.{{ interval }}hr_avg.f{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/ocean.nc"] data_out: mkdir: - "{{ COMOUT_OCEAN_NETCDF }}/native" {% for grid in product_grids %} + {% if write_netcdf %} - "{{ COMOUT_OCEAN_NETCDF }}/{{ grid }}" + {% endif %} + {% if write_grib2 %} - "{{ COMOUT_OCEAN_GRIB }}/{{ grid }}" + {% endif %} {% endfor %} copy: - - ["{{ DATA }}/ocean_subset.nc", "{{ COMOUT_OCEAN_NETCDF }}/native/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.tripolar.f{{ '%03d' % forecast_hour }}.nc"] + - ["{{ DATA }}/ocean_subset.nc", "{{ COMOUT_OCEAN_NETCDF }}/native/{{ RUN }}.ocean.t{{ current_cycle | strftime('%H') }}z.native.f{{ '%03d' % forecast_hour }}.nc"] + - ["{{ DATA }}/ocean_compressed.nc", "{{ COMIN_OCEAN_HISTORY }}/{{ RUN }}.ocean.t{{ current_cycle | strftime('%H') }}z.{{ interval }}hr_avg.f{{ '%03d' % forecast_hour }}.nc"] + {% if write_grib2 %} {% for grid in product_grids %} - - ["{{ DATA }}/ocean.{{ grid }}.grib2", "{{ COMOUT_OCEAN_GRIB }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2"] - - ["{{ DATA }}/ocean.{{ grid }}.grib2.idx", "{{ COMOUT_OCEAN_GRIB }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2.idx"] + - ["{{ DATA }}/ocean.{{ grid }}.grib2", "{{ COMOUT_OCEAN_GRIB }}/{{ grid }}/{{ RUN }}.ocean.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2"] + - ["{{ DATA }}/ocean.{{ grid }}.grib2.idx", "{{ COMOUT_OCEAN_GRIB }}/{{ grid }}/{{ RUN }}.ocean.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2.idx"] {% endfor %} + {% endif %} ice: namelist: @@ -57,20 +66,27 @@ ice: sinvar: "" cosvar: "" angvar: "ANGLET" - subset: ['hi_h', 'hs_h', 'aice_h', 'Tsfc_h', 'uvel_h', 'vvel_h', 'frzmlt_h', 'albsni_h', 'mlt_onset_h', 'frz_onset_h'] + subset: ['hi_h', 'hs_h', 'aice_h', 'Tsfc_h', 'uvel_h', 'vvel_h', 'albsni_h'] data_in: copy: - - ["{{ COMIN_ICE_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ interval }}hr_avg.f{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/ice.nc"] + - ["{{ COMIN_ICE_HISTORY }}/{{ RUN }}.ice.t{{ current_cycle | strftime('%H') }}z.{{ interval }}hr_avg.f{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/ice.nc"] data_out: mkdir: - "{{ COMOUT_ICE_NETCDF }}/native" {% for grid in product_grids %} + {% if write_netcdf %} - "{{ COMOUT_ICE_NETCDF }}/{{ grid }}" + {% endif %} + {% if write_grib2 %} - "{{ COMOUT_ICE_GRIB }}/{{ grid }}" + {% endif %} {% endfor %} copy: - - ["{{ DATA }}/ice_subset.nc", "{{ COMOUT_ICE_NETCDF }}/native/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.tripolar.f{{ '%03d' % forecast_hour }}.nc"] + - ["{{ DATA }}/ice_subset.nc", "{{ COMOUT_ICE_NETCDF }}/native/{{ RUN }}.ice.t{{ current_cycle | strftime('%H') }}z.native.f{{ '%03d' % forecast_hour }}.nc"] + - ["{{ DATA }}/ice_compressed.nc", "{{ COMIN_ICE_HISTORY }}/{{ RUN }}.ice.t{{ current_cycle | strftime('%H') }}z.{{ interval }}hr_avg.f{{ '%03d' % forecast_hour }}.nc"] {% for grid in product_grids %} - - ["{{ DATA }}/ice.{{ grid }}.grib2", "{{ COMOUT_ICE_GRIB }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2"] - - ["{{ DATA }}/ice.{{ grid }}.grib2.idx", "{{ COMOUT_ICE_GRIB }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2.idx"] + {% if write_grib2 %} + - ["{{ DATA }}/ice.{{ grid }}.grib2", "{{ COMOUT_ICE_GRIB }}/{{ grid }}/{{ RUN }}.ice.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2"] + - ["{{ DATA }}/ice.{{ grid }}.grib2.idx", "{{ COMOUT_ICE_GRIB }}/{{ grid }}/{{ RUN }}.ice.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2.idx"] + {% endif %} {% endfor %} diff --git a/ush/python/pygfs/task/oceanice_products.py b/ush/python/pygfs/task/oceanice_products.py index d319608ad14..a5139717cc8 100644 --- a/ush/python/pygfs/task/oceanice_products.py +++ b/ush/python/pygfs/task/oceanice_products.py @@ -170,11 +170,12 @@ def execute(config: Dict, product_grid: str) -> None: None """ - # Run the ocnicepost.x executable - OceanIceProducts.interp(config.DATA, config.APRUN_OCNICEPOST, exec_name="ocnicepost.x") + # Run the ocnicepost.x executable if interpolated variables are wanted + if config.oceanice_yaml.ocnicepost.namelist.write_netcdf or config.oceanice_yaml.ocnicepost.namelist.write_grib2: + OceanIceProducts.interp(config.DATA, config.APRUN_OCNICEPOST, exec_name="ocnicepost.x") - # Index the interpolated grib2 file - OceanIceProducts.index(config, product_grid) + # Index the interpolated grib2 file + OceanIceProducts.index(config, product_grid) @staticmethod @logit(logger) @@ -274,6 +275,8 @@ def subset(config: Dict) -> None: input_file = f"{config.component}.nc" output_file = f"{config.component}_subset.nc" + compressed_file = f"{config.component}_compressed.nc" + varlist = config.oceanice_yaml[config.component].subset logger.info(f"Subsetting {varlist} from {input_file} to {output_file}") @@ -281,15 +284,31 @@ def subset(config: Dict) -> None: try: # open the netcdf file ds = xr.open_dataset(input_file) - - # subset the variables - ds_subset = ds[varlist] - + if config.component == 'ice': + varlist.extend(['tarea', 'tmask']) + # subset the variables + # remove coords that were carried from original file but not used + ds_subset = ds_subset.drop_vars('ELON', errors='ignore') + ds_subset = ds_subset.drop_vars('ELAT', errors='ignore') + ds_subset = ds_subset.drop_vars('NLON', errors='ignore') + ds_subset = ds_subset.drop_vars('NLAT', errors='ignore') + + if config.component == 'ocean': + # subset ocean variables for z_levels in products + levels = config.oceanice_yaml.ocean.namelist.ocean_levels + ds_subset = ds[varlist].sel(z_l = levels) + # save global attributes from the old netcdf file into new netcdf file ds_subset.attrs = ds.attrs - # save subsetted variables to a new netcdf file - ds_subset.to_netcdf(output_file) + # save subsetted variables to a new netcdf file and compress + default_compression = {"zlib": True, "complevel": 8} + compress_encoding = {var_name: default_compression for var_name in ds_subset.data_vars} + ds_subset.to_netcdf(output_file, encoding=compress_encoding) + + # save compress original file + compress_encoding = {var_name: default_compression for var_name in ds.data_vars} + ds.to_netcdf(compressed_file, encoding=compress_encoding) except FileNotFoundError: logger.exception(f"FATAL ERROR: Input file not found: {input_file}") @@ -326,6 +345,5 @@ def finalize(config: Dict) -> None: # Copy "component" specific generated data to COM/ directory data_out = config.oceanice_yaml[config.component].data_out - logger.info(f"Copy processed data to COM/ directory") FileHandler(data_out).sync() From c9483c16c1c9e6a53c9697abf5cfda8c317170f0 Mon Sep 17 00:00:00 2001 From: NeilBarton-NOAA Date: Thu, 30 Oct 2025 17:44:37 -0400 Subject: [PATCH 02/26] edits to MOM6 diag_table --- parm/ufs/fv3/diag_table | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/ufs/fv3/diag_table b/parm/ufs/fv3/diag_table index ba4f9c793d5..ffc60816b83 100644 --- a/parm/ufs/fv3/diag_table +++ b/parm/ufs/fv3/diag_table @@ -49,10 +49,10 @@ "ocean_model", "SW", "SW", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 "ocean_model", "LW", "LW", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 "ocean_model", "evap", "evap", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 -"ocean_model", "lprec", "lprec", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 +#"ocean_model", "lprec", "lprec", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 "ocean_model", "lrunoff", "lrunoff", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 #"ocean_model", "frunoff", "frunoff", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 -"ocean_model", "fprec", "fprec", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 +#"ocean_model", "fprec", "fprec", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 "ocean_model", "LwLatSens", "LwLatSens", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 "ocean_model", "Heat_PmE", "Heat_PmE", "@[MOM6_OUTPUT_DIR]/ocn%4yr%2mo%2dy%2hr","all",.true.,"none",2 From 76f89d02acad766ab9a1f736cb829b0d05038249 Mon Sep 17 00:00:00 2001 From: NeilBarton-NOAA Date: Fri, 31 Oct 2025 16:04:34 -0400 Subject: [PATCH 03/26] edits for file name differences --- dev/parm/config/gefs/config.nsst | 2 +- dev/parm/config/gfs/config.ufs | 2 +- dev/parm/config/sfs/config.base.j2 | 19 ++------------ dev/parm/config/sfs/config.nsst.j2 | 30 ++++++++++++++++++++++ jobs/JGLOBAL_ARCHIVE_TARS | 1 + parm/archive/ice_subset.yaml.j2 | 10 +++++--- parm/archive/ocean_subset.yaml.j2 | 8 +++--- parm/post/oceanice_products_gfs.yaml | 24 ++++++++--------- parm/post/oceanice_products_sfs.yaml | 20 +++++++-------- ush/python/pygfs/task/oceanice_products.py | 1 + 10 files changed, 69 insertions(+), 48 deletions(-) create mode 100644 dev/parm/config/sfs/config.nsst.j2 diff --git a/dev/parm/config/gefs/config.nsst b/dev/parm/config/gefs/config.nsst index f9a29d44d3a..04abadc91dc 100644 --- a/dev/parm/config/gefs/config.nsst +++ b/dev/parm/config/gefs/config.nsst @@ -11,7 +11,7 @@ echo "BEGIN: config.nsst" export NST_MODEL=2 # nstf_name(2) : NST_SPINUP : 0 = OFF, 1 = ON, -export NST_SPINUP=0 +export NST_SPINUP="{{ NST_SPINUP }}" # nstf_name(3) : NST_RESV (Reserved, NSST Analysis) : 0 = OFF, 1 = ON export NST_RESV=0 diff --git a/dev/parm/config/gfs/config.ufs b/dev/parm/config/gfs/config.ufs index 6c883d66bf7..fe5c03be37b 100644 --- a/dev/parm/config/gfs/config.ufs +++ b/dev/parm/config/gfs/config.ufs @@ -516,7 +516,7 @@ if [[ "${skip_mom6}" == "false" ]]; then eps_imesh="1.0e-1" case ${RUN} in gfs|gefs|sfs|gcafs) - MOM6_DIAG_COORD_DEF_Z_FILE="interpolate_zgrid_40L.nc" + MOM6_DIAG_COORD_DEF_Z_FILE="interpolate_zgrid_30L.nc" MOM6_DIAG_MISVAL="-1e34" ;; gdas|enkfgdas|enkfgfs) diff --git a/dev/parm/config/sfs/config.base.j2 b/dev/parm/config/sfs/config.base.j2 index 32a26ab5f8a..82f7e75bfee 100644 --- a/dev/parm/config/sfs/config.base.j2 +++ b/dev/parm/config/sfs/config.base.j2 @@ -318,23 +318,8 @@ export DO_METP="NO" # Run METPLUS jobs - set METPLUS settings in config. export DO_FIT2OBS="NO" # Run fit to observations package # Archiving options -export HPSSARCH="{{ HPSSARCH }}" # save data to HPSS archive -export LOCALARCH="{{ LOCALARCH }}" # save data to local archive -if [[ ${HPSSARCH} = "YES" ]] && [[ ${LOCALARCH} = "YES" ]]; then - echo "Both HPSS and local archiving selected. Please choose one or the other." - exit 3 -elif [[ ${HPSSARCH} = "YES" ]] || [[ ${LOCALARCH} = "YES" ]]; then - export DO_ARCHCOM="YES" -else - export DO_ARCHCOM="NO" -fi -export ARCH_CYC=00 # Archive data at this cycle for warm start and/or forecast-only capabilities -export ARCH_WARMICFREQ=4 # Archive frequency in days for warm start capability -export ARCH_FCSTICFREQ=1 # Archive frequency in days for gdas and gfs forecast-only capability -export ARCH_EXPDIR='YES' # Archive the EXPDIR configs, XML, and database -export ARCH_EXPDIR_FREQ=0 # How often to archive the EXPDIR in hours or 0 for first and last cycle only -export ARCH_HASHES='YES' # Archive the hashes of the GW and submodules and 'git status' for each; requires ARCH_EXPDIR -export ARCH_DIFFS='NO' # Archive the output of 'git diff' for the GW; requires ARCH_EXPDIR +export DO_ARCHCOM="{{ DO_ARCHCOM }}" # Tar and archive the COM directories +export ARCHCOM_TO="{{ ARCHCOM_TO }}" # Valid options are hpss, globus_hpss, and local # Number of regional collectives to create soundings for export NUM_SND_COLLECTIVES=${NUM_SND_COLLECTIVES:-9} diff --git a/dev/parm/config/sfs/config.nsst.j2 b/dev/parm/config/sfs/config.nsst.j2 new file mode 100644 index 00000000000..04abadc91dc --- /dev/null +++ b/dev/parm/config/sfs/config.nsst.j2 @@ -0,0 +1,30 @@ +#! /usr/bin/env bash + +########## config.nsst ########## +# NSST specific + +echo "BEGIN: config.nsst" + +# NSST parameters contained within nstf_name + +# nstf_name(1) : NST_MODEL (NSST Model) : 0 = OFF, 1 = ON but uncoupled, 2 = ON and coupled +export NST_MODEL=2 + +# nstf_name(2) : NST_SPINUP : 0 = OFF, 1 = ON, +export NST_SPINUP="{{ NST_SPINUP }}" + +# nstf_name(3) : NST_RESV (Reserved, NSST Analysis) : 0 = OFF, 1 = ON +export NST_RESV=0 + +# nstf_name(4,5) : ZSEA1, ZSEA2 the two depths to apply vertical average (bias correction) +export ZSEA1=0 +export ZSEA2=0 + +export NST_GSI=3 # default 0: No NST info at all; + # 1: Input NST info but not used in GSI; + # 2: Input NST info, used in CRTM simulation, no Tr analysis + # 3: Input NST info, used in both CRTM simulation and Tr analysis +export NSTINFO=0 # number of elements added in obs. data array (default = 0) +if (( NST_GSI > 0 )); then export NSTINFO=4; fi + +echo "END: config.nsst" diff --git a/jobs/JGLOBAL_ARCHIVE_TARS b/jobs/JGLOBAL_ARCHIVE_TARS index f3b95db93cc..ae726ee065f 100755 --- a/jobs/JGLOBAL_ARCHIVE_TARS +++ b/jobs/JGLOBAL_ARCHIVE_TARS @@ -31,6 +31,7 @@ YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ COMIN_ICE_INPUT:COM_ICE_INPUT_TMPL \ COMIN_ICE_RESTART:COM_ICE_RESTART_TMPL \ COMIN_ICE_GRIB:COM_ICE_GRIB_TMPL \ + COMIN_ICE_NETCDF:COM_ICE_NETCDF_TMPL \ COMIN_OBS:COM_OBS_TMPL \ COMIN_TOP:COM_TOP_TMPL \ COMIN_OCEAN_HISTORY:COM_OCEAN_HISTORY_TMPL \ diff --git a/parm/archive/ice_subset.yaml.j2 b/parm/archive/ice_subset.yaml.j2 index 5929c02af4e..ec77c3d9733 100644 --- a/parm/archive/ice_subset.yaml.j2 +++ b/parm/archive/ice_subset.yaml.j2 @@ -1,8 +1,10 @@ ice_subset: - {% set head = "gfs.ice.t" + cycle_HH + "z." %} - name: "OCEAN_SUBSET" + {% set NET = NET %} + {% set FREQ = FHOUT_ICE_GFS %} + {% set head = NET + ".t" + cycle_HH + "z" %} + name: "ICE_SUBSET" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ice_subset.tar" required: - {% for fhr in range(6, FHMAX_GFS + 6, 6) %} - - "{{ COMIN_ICE_NATIVE | relpath(ROTDIR) }}/{{ head }}.native.f{{ '%03d' % fhr }}.nc" + {% for fhr in range(FREQ, FHMAX_GFS + FREQ, FREQ) %} + - "{{ COMIN_ICE_NETCDF | relpath(ROTDIR) }}/native/{{ head }}.native.f{{ '%03d' % fhr }}.nc" {% endfor %} diff --git a/parm/archive/ocean_subset.yaml.j2 b/parm/archive/ocean_subset.yaml.j2 index 9c33d409a24..d87ddf267bf 100644 --- a/parm/archive/ocean_subset.yaml.j2 +++ b/parm/archive/ocean_subset.yaml.j2 @@ -1,8 +1,10 @@ ocean_subset: - {% set head = "gfs.ocean.t" + cycle_HH + "z." %} + {% set NET = NET %} + {% set FREQ = FHOUT_ICE_GFS %} + {% set head = NET + ".t" + cycle_HH + "z" %} name: "OCEAN_SUBSET" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ocean_subset.tar" required: - {% for fhr in range(6, FHMAX_GFS + 6, 6) %} - - "{{ COMIN_OCEAN_NATIVE | relpath(ROTDIR) }}/{{ head }}.native.f{{ '%03d' % fhr }}.nc" + {% for fhr in range(FREQ, FHMAX_GFS + FREQ, FREQ) %} + - "{{ COMIN_OCEAN_NETCDF | relpath(ROTDIR) }}/native/{{ head }}.native.f{{ '%03d' % fhr }}.nc" {% endfor %} diff --git a/parm/post/oceanice_products_gfs.yaml b/parm/post/oceanice_products_gfs.yaml index 291cf395bb9..089a301e4ec 100644 --- a/parm/post/oceanice_products_gfs.yaml +++ b/parm/post/oceanice_products_gfs.yaml @@ -8,10 +8,10 @@ ocnicepost: mkdir: - "{{ DATA }}" copy: - - ["{{ EXECgfs }}/ocnicepost.x", "{{ DATA }}/"] - ["{{ PARMgfs }}/post/ocnicepost.nml.jinja2", "{{ DATA }}/"] - - ["{{ PARMgfs }}/post/{{ component }}_gfs.csv", "{{ DATA }}/{{ component }}.csv"] {% if write_grib2 and write_netcdf %} + - ["{{ PARMgfs }}/post/{{ component }}_gfs.csv", "{{ DATA }}/{{ component }}.csv"] + - ["{{ EXECgfs }}/ocnicepost.x", "{{ DATA }}/"] - ["{{ FIXgfs }}/mom6/post/{{ model_grid }}/tripole.{{ model_grid }}.Bu.to.Ct.bilinear.nc", "{{ DATA }}/"] - ["{{ FIXgfs }}/mom6/post/{{ model_grid }}/tripole.{{ model_grid }}.Cu.to.Ct.bilinear.nc", "{{ DATA }}/"] - ["{{ FIXgfs }}/mom6/post/{{ model_grid }}/tripole.{{ model_grid }}.Cv.to.Ct.bilinear.nc", "{{ DATA }}/"] @@ -37,7 +37,7 @@ ocean: subset: ['SSH', 'SST', 'SSS', 'speed', 'MLD_003', 'SSU', 'SSV', 'temp', 'tob', 'so', 'uo', 'vo'] data_in: copy: - - ["{{ COMIN_OCEAN_HISTORY }}/{{ RUN }}.ocean.t{{ current_cycle | strftime('%H') }}z.{{ interval }}hr_avg.f{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/ocean.nc"] + - ["{{ COMIN_OCEAN_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ interval }}hr_avg.f{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/ocean.nc"] data_out: mkdir: - "{{ COMOUT_OCEAN_NETCDF }}/native" @@ -50,12 +50,12 @@ ocean: {% endif %} {% endfor %} copy: - - ["{{ DATA }}/ocean_subset.nc", "{{ COMOUT_OCEAN_NETCDF }}/native/{{ RUN }}.ocean.t{{ current_cycle | strftime('%H') }}z.native.f{{ '%03d' % forecast_hour }}.nc"] - - ["{{ DATA }}/ocean_compressed.nc", "{{ COMIN_OCEAN_HISTORY }}/{{ RUN }}.ocean.t{{ current_cycle | strftime('%H') }}z.{{ interval }}hr_avg.f{{ '%03d' % forecast_hour }}.nc"] + - ["{{ DATA }}/ocean_subset.nc", "{{ COMOUT_OCEAN_NETCDF }}/native/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.native.f{{ '%03d' % forecast_hour }}.nc"] + - ["{{ DATA }}/ocean_compressed.nc", "{{ COMIN_OCEAN_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ interval }}hr_avg.f{{ '%03d' % forecast_hour }}.nc"] {% if write_grib2 %} {% for grid in product_grids %} - - ["{{ DATA }}/ocean.{{ grid }}.grib2", "{{ COMOUT_OCEAN_GRIB }}/{{ grid }}/{{ RUN }}.ocean.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2"] - - ["{{ DATA }}/ocean.{{ grid }}.grib2.idx", "{{ COMOUT_OCEAN_GRIB }}/{{ grid }}/{{ RUN }}.ocean.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2.idx"] + - ["{{ DATA }}/ocean.{{ grid }}.grib2", "{{ COMOUT_OCEAN_GRIB }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2"] + - ["{{ DATA }}/ocean.{{ grid }}.grib2.idx", "{{ COMOUT_OCEAN_GRIB }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2.idx"] {% endfor %} {% endif %} @@ -69,7 +69,7 @@ ice: subset: ['hi_h', 'hs_h', 'aice_h', 'Tsfc_h', 'uvel_h', 'vvel_h', 'albsni_h'] data_in: copy: - - ["{{ COMIN_ICE_HISTORY }}/{{ RUN }}.ice.t{{ current_cycle | strftime('%H') }}z.{{ interval }}hr_avg.f{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/ice.nc"] + - ["{{ COMIN_ICE_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ interval }}hr_avg.f{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/ice.nc"] data_out: mkdir: - "{{ COMOUT_ICE_NETCDF }}/native" @@ -82,11 +82,11 @@ ice: {% endif %} {% endfor %} copy: - - ["{{ DATA }}/ice_subset.nc", "{{ COMOUT_ICE_NETCDF }}/native/{{ RUN }}.ice.t{{ current_cycle | strftime('%H') }}z.native.f{{ '%03d' % forecast_hour }}.nc"] - - ["{{ DATA }}/ice_compressed.nc", "{{ COMIN_ICE_HISTORY }}/{{ RUN }}.ice.t{{ current_cycle | strftime('%H') }}z.{{ interval }}hr_avg.f{{ '%03d' % forecast_hour }}.nc"] + - ["{{ DATA }}/ice_subset.nc", "{{ COMOUT_ICE_NETCDF }}/native/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.native.f{{ '%03d' % forecast_hour }}.nc"] + - ["{{ DATA }}/ice_compressed.nc", "{{ COMIN_ICE_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ interval }}hr_avg.f{{ '%03d' % forecast_hour }}.nc"] {% for grid in product_grids %} {% if write_grib2 %} - - ["{{ DATA }}/ice.{{ grid }}.grib2", "{{ COMOUT_ICE_GRIB }}/{{ grid }}/{{ RUN }}.ice.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2"] - - ["{{ DATA }}/ice.{{ grid }}.grib2.idx", "{{ COMOUT_ICE_GRIB }}/{{ grid }}/{{ RUN }}.ice.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2.idx"] + - ["{{ DATA }}/ice.{{ grid }}.grib2", "{{ COMOUT_ICE_GRIB }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2"] + - ["{{ DATA }}/ice.{{ grid }}.grib2.idx", "{{ COMOUT_ICE_GRIB }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2.idx"] {% endif %} {% endfor %} diff --git a/parm/post/oceanice_products_sfs.yaml b/parm/post/oceanice_products_sfs.yaml index c284069abe1..089a301e4ec 100644 --- a/parm/post/oceanice_products_sfs.yaml +++ b/parm/post/oceanice_products_sfs.yaml @@ -37,7 +37,7 @@ ocean: subset: ['SSH', 'SST', 'SSS', 'speed', 'MLD_003', 'SSU', 'SSV', 'temp', 'tob', 'so', 'uo', 'vo'] data_in: copy: - - ["{{ COMIN_OCEAN_HISTORY }}/{{ RUN }}.ocean.t{{ current_cycle | strftime('%H') }}z.{{ interval }}hr_avg.f{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/ocean.nc"] + - ["{{ COMIN_OCEAN_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ interval }}hr_avg.f{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/ocean.nc"] data_out: mkdir: - "{{ COMOUT_OCEAN_NETCDF }}/native" @@ -50,12 +50,12 @@ ocean: {% endif %} {% endfor %} copy: - - ["{{ DATA }}/ocean_subset.nc", "{{ COMOUT_OCEAN_NETCDF }}/native/{{ RUN }}.ocean.t{{ current_cycle | strftime('%H') }}z.native.f{{ '%03d' % forecast_hour }}.nc"] - - ["{{ DATA }}/ocean_compressed.nc", "{{ COMIN_OCEAN_HISTORY }}/{{ RUN }}.ocean.t{{ current_cycle | strftime('%H') }}z.{{ interval }}hr_avg.f{{ '%03d' % forecast_hour }}.nc"] + - ["{{ DATA }}/ocean_subset.nc", "{{ COMOUT_OCEAN_NETCDF }}/native/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.native.f{{ '%03d' % forecast_hour }}.nc"] + - ["{{ DATA }}/ocean_compressed.nc", "{{ COMIN_OCEAN_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ interval }}hr_avg.f{{ '%03d' % forecast_hour }}.nc"] {% if write_grib2 %} {% for grid in product_grids %} - - ["{{ DATA }}/ocean.{{ grid }}.grib2", "{{ COMOUT_OCEAN_GRIB }}/{{ grid }}/{{ RUN }}.ocean.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2"] - - ["{{ DATA }}/ocean.{{ grid }}.grib2.idx", "{{ COMOUT_OCEAN_GRIB }}/{{ grid }}/{{ RUN }}.ocean.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2.idx"] + - ["{{ DATA }}/ocean.{{ grid }}.grib2", "{{ COMOUT_OCEAN_GRIB }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2"] + - ["{{ DATA }}/ocean.{{ grid }}.grib2.idx", "{{ COMOUT_OCEAN_GRIB }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2.idx"] {% endfor %} {% endif %} @@ -69,7 +69,7 @@ ice: subset: ['hi_h', 'hs_h', 'aice_h', 'Tsfc_h', 'uvel_h', 'vvel_h', 'albsni_h'] data_in: copy: - - ["{{ COMIN_ICE_HISTORY }}/{{ RUN }}.ice.t{{ current_cycle | strftime('%H') }}z.{{ interval }}hr_avg.f{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/ice.nc"] + - ["{{ COMIN_ICE_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ interval }}hr_avg.f{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/ice.nc"] data_out: mkdir: - "{{ COMOUT_ICE_NETCDF }}/native" @@ -82,11 +82,11 @@ ice: {% endif %} {% endfor %} copy: - - ["{{ DATA }}/ice_subset.nc", "{{ COMOUT_ICE_NETCDF }}/native/{{ RUN }}.ice.t{{ current_cycle | strftime('%H') }}z.native.f{{ '%03d' % forecast_hour }}.nc"] - - ["{{ DATA }}/ice_compressed.nc", "{{ COMIN_ICE_HISTORY }}/{{ RUN }}.ice.t{{ current_cycle | strftime('%H') }}z.{{ interval }}hr_avg.f{{ '%03d' % forecast_hour }}.nc"] + - ["{{ DATA }}/ice_subset.nc", "{{ COMOUT_ICE_NETCDF }}/native/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.native.f{{ '%03d' % forecast_hour }}.nc"] + - ["{{ DATA }}/ice_compressed.nc", "{{ COMIN_ICE_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ interval }}hr_avg.f{{ '%03d' % forecast_hour }}.nc"] {% for grid in product_grids %} {% if write_grib2 %} - - ["{{ DATA }}/ice.{{ grid }}.grib2", "{{ COMOUT_ICE_GRIB }}/{{ grid }}/{{ RUN }}.ice.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2"] - - ["{{ DATA }}/ice.{{ grid }}.grib2.idx", "{{ COMOUT_ICE_GRIB }}/{{ grid }}/{{ RUN }}.ice.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2.idx"] + - ["{{ DATA }}/ice.{{ grid }}.grib2", "{{ COMOUT_ICE_GRIB }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2"] + - ["{{ DATA }}/ice.{{ grid }}.grib2.idx", "{{ COMOUT_ICE_GRIB }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2.idx"] {% endif %} {% endfor %} diff --git a/ush/python/pygfs/task/oceanice_products.py b/ush/python/pygfs/task/oceanice_products.py index a5139717cc8..ffb2a45e087 100644 --- a/ush/python/pygfs/task/oceanice_products.py +++ b/ush/python/pygfs/task/oceanice_products.py @@ -286,6 +286,7 @@ def subset(config: Dict) -> None: ds = xr.open_dataset(input_file) if config.component == 'ice': varlist.extend(['tarea', 'tmask']) + ds_subset = ds[varlist] # subset the variables # remove coords that were carried from original file but not used ds_subset = ds_subset.drop_vars('ELON', errors='ignore') From f8f6fe80e84edf3bde1601968a2c5525e6e71f71 Mon Sep 17 00:00:00 2001 From: NeilBarton-NOAA Date: Fri, 31 Oct 2025 16:31:00 -0400 Subject: [PATCH 04/26] changing to 30L zgrid MOM6 file --- dev/parm/config/gfs/config.ufs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/parm/config/gfs/config.ufs b/dev/parm/config/gfs/config.ufs index fe5c03be37b..411b0461b46 100644 --- a/dev/parm/config/gfs/config.ufs +++ b/dev/parm/config/gfs/config.ufs @@ -451,7 +451,7 @@ if [[ "${skip_mom6}" == "false" ]]; then TOPOEDITS="ufs.topo_edits_011818.nc" case ${RUN} in gfs|gefs|sfs|gcafs) - MOM6_DIAG_COORD_DEF_Z_FILE="interpolate_zgrid_40L.nc" + MOM6_DIAG_COORD_DEF_Z_FILE="interpolate_zgrid_30L.nc" MOM6_DIAG_MISVAL="-1e34" ;; gdas|enkfgdas|enkfgfs) @@ -479,7 +479,7 @@ if [[ "${skip_mom6}" == "false" ]]; then eps_imesh="1.0e-1" case ${RUN} in gfs|gefs|sfs|gcafs) - MOM6_DIAG_COORD_DEF_Z_FILE="interpolate_zgrid_40L.nc" + MOM6_DIAG_COORD_DEF_Z_FILE="interpolate_zgrid_30L.nc" MOM6_DIAG_MISVAL="-1e34" ;; gdas|enkfgdas|enkfgfs) From 5ad905ceb8bece474cdf09d938c8be5a9aafbb2a Mon Sep 17 00:00:00 2001 From: NeilBarton-NOAA Date: Mon, 3 Nov 2025 13:43:13 -0500 Subject: [PATCH 05/26] gfs test and edits --- dev/workflow/applications/sfs.py | 8 ++++- dev/workflow/rocoto/gfs_tasks.py | 4 +-- dev/workflow/rocoto/sfs_tasks.py | 31 +++++++++++++++++++ ...{ice_subset.yaml.j2 => ice_native.yaml.j2} | 6 ++-- ...an_subset.yaml.j2 => ocean_native.yaml.j2} | 6 ++-- parm/post/oceanice_products_gfs.yaml | 2 +- 6 files changed, 47 insertions(+), 10 deletions(-) rename parm/archive/{ice_subset.yaml.j2 => ice_native.yaml.j2} (74%) rename parm/archive/{ocean_subset.yaml.j2 => ocean_native.yaml.j2} (73%) diff --git a/dev/workflow/applications/sfs.py b/dev/workflow/applications/sfs.py index d08e028fdcc..316f74335d8 100644 --- a/dev/workflow/applications/sfs.py +++ b/dev/workflow/applications/sfs.py @@ -93,6 +93,8 @@ def _get_app_configs(self, run): if options['do_archcom']: configs += ['arch_tars'] + if options['do_globusarch']: + configs += ['globus'] configs += ['arch_vrfy', 'cleanup'] @@ -164,7 +166,11 @@ def get_task_names(self): if options['do_extractvars']: tasks += ['extractvars'] - # TODO: Add archive + if options['do_archcom']: + tasks += ['arch_tars'] + if options['do_globusarch']: + tasks += ['globus'] + tasks += ['cleanup'] return {f"{self.run}": tasks} diff --git a/dev/workflow/rocoto/gfs_tasks.py b/dev/workflow/rocoto/gfs_tasks.py index 37e628bc3d3..d38fa7d8505 100644 --- a/dev/workflow/rocoto/gfs_tasks.py +++ b/dev/workflow/rocoto/gfs_tasks.py @@ -2253,12 +2253,12 @@ def arch_tars(self): tarball_types.append('chem') if self.options['do_ocean']: - tarball_types.extend(['ocean_6hravg', 'ocean_grib2', 'gfs_flux_1p00']) + tarball_types.extend(['ocean_6hravg', 'ocean_native', 'gfs_flux_1p00']) if self.options.get('do_jediocnvar', False) and self.app_config.mode == 'cycled': tarball_types.append('gfsocean_analysis') if self.options['do_ice']: - tarball_types.extend(['ice_6hravg', 'ice_grib2']) + tarball_types.extend(['ice_6hravg', 'ice_native']) if self.options['do_bufrsnd']: tarball_types.append('gfs_downstream') diff --git a/dev/workflow/rocoto/sfs_tasks.py b/dev/workflow/rocoto/sfs_tasks.py index 9113a5d9bb4..0182c8ca18b 100644 --- a/dev/workflow/rocoto/sfs_tasks.py +++ b/dev/workflow/rocoto/sfs_tasks.py @@ -611,6 +611,30 @@ def arch_tars(self): task = rocoto.create_task(task_dict) return task + + # Globus transfer for HPSS archiving + def globus(self): + deps = [] + dep_dict = {'type': 'task', 'name': f'{self.run}_arch_tars'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + + resources = self.get_resource('globus') + task_name = f'{self.run}_globus_arch' + task_dict = {'task_name': task_name, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars, + 'cycledef': self.run, + 'command': f'{self.HOMEgfs}/dev/jobs/globus_arch.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + + task = rocoto.create_task(task_dict) + + return task def cleanup(self): deps = [] @@ -637,6 +661,13 @@ def cleanup(self): if self.options['do_extractvars']: dep_dict = {'type': 'metatask', 'name': f'{self.run}_extractvars'} deps.append(rocoto.add_dependency(dep_dict)) + if self.options['do_archcom']: + if self.options['do_globusarch']: + dep_dict = {'type': 'task', 'name': f'{self.run}_globus_arch'} + else: + dep_dict = {'type': 'task', 'name': f'{self.run}_arch_tars'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) dependencies = rocoto.create_dependency(dep=deps, dep_condition='and') resources = self.get_resource('cleanup') task_name = f'{self.run}_cleanup' diff --git a/parm/archive/ice_subset.yaml.j2 b/parm/archive/ice_native.yaml.j2 similarity index 74% rename from parm/archive/ice_subset.yaml.j2 rename to parm/archive/ice_native.yaml.j2 index ec77c3d9733..dde6c3daa99 100644 --- a/parm/archive/ice_subset.yaml.j2 +++ b/parm/archive/ice_native.yaml.j2 @@ -1,9 +1,9 @@ -ice_subset: +ice_native: {% set NET = NET %} {% set FREQ = FHOUT_ICE_GFS %} {% set head = NET + ".t" + cycle_HH + "z" %} - name: "ICE_SUBSET" - target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ice_subset.tar" + name: "ICE_NATIVE" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ice_native_subset.tar" required: {% for fhr in range(FREQ, FHMAX_GFS + FREQ, FREQ) %} - "{{ COMIN_ICE_NETCDF | relpath(ROTDIR) }}/native/{{ head }}.native.f{{ '%03d' % fhr }}.nc" diff --git a/parm/archive/ocean_subset.yaml.j2 b/parm/archive/ocean_native.yaml.j2 similarity index 73% rename from parm/archive/ocean_subset.yaml.j2 rename to parm/archive/ocean_native.yaml.j2 index d87ddf267bf..1bf2c151e9b 100644 --- a/parm/archive/ocean_subset.yaml.j2 +++ b/parm/archive/ocean_native.yaml.j2 @@ -1,9 +1,9 @@ -ocean_subset: +ocean_native: {% set NET = NET %} {% set FREQ = FHOUT_ICE_GFS %} {% set head = NET + ".t" + cycle_HH + "z" %} - name: "OCEAN_SUBSET" - target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ocean_subset.tar" + name: "OCEAN_NATIVE" + target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ocean_native_subset.tar" required: {% for fhr in range(FREQ, FHMAX_GFS + FREQ, FREQ) %} - "{{ COMIN_OCEAN_NETCDF | relpath(ROTDIR) }}/native/{{ head }}.native.f{{ '%03d' % fhr }}.nc" diff --git a/parm/post/oceanice_products_gfs.yaml b/parm/post/oceanice_products_gfs.yaml index 089a301e4ec..d47532d3353 100644 --- a/parm/post/oceanice_products_gfs.yaml +++ b/parm/post/oceanice_products_gfs.yaml @@ -30,7 +30,7 @@ ocean: cosvar: "cos_rot" angvar: "" {% if model_grid == 'mx025' or model_grid == 'mx050' or model_grid == 'mx100' %} - ocean_levels: [5, 15, 25, 35, 45, 55, 65, 75, 85, 95, 105, 115, 125, 135, 145, 155, 165, 175, 185, 195, 205, 215] + ocean_levels: [1, 3, 5, 10, 20, 30, 50, 100, 200, 500, 1000] {% elif model_grid == 'mx500' %} ocean_levels: [5, 15, 25, 35, 45, 55, 65, 75, 85, 95, 105, 115, 125, 135, 145, 155, 165, 175, 185, 195, 205, 215, 226, 241, 267] {% endif %} From d7abd793ffbb051aca6ade53aee04224aea5206f Mon Sep 17 00:00:00 2001 From: NeilBarton-NOAA Date: Mon, 3 Nov 2025 13:54:55 -0500 Subject: [PATCH 06/26] fix python code style --- dev/workflow/rocoto/sfs_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/workflow/rocoto/sfs_tasks.py b/dev/workflow/rocoto/sfs_tasks.py index 0182c8ca18b..fe11bde8699 100644 --- a/dev/workflow/rocoto/sfs_tasks.py +++ b/dev/workflow/rocoto/sfs_tasks.py @@ -611,7 +611,7 @@ def arch_tars(self): task = rocoto.create_task(task_dict) return task - + # Globus transfer for HPSS archiving def globus(self): deps = [] From d2ed37202d31244f24604f0f2246e9b33cbab016 Mon Sep 17 00:00:00 2001 From: NeilBarton-NOAA Date: Mon, 3 Nov 2025 15:30:24 -0500 Subject: [PATCH 07/26] remove trailing white space --- dev/workflow/rocoto/sfs_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/workflow/rocoto/sfs_tasks.py b/dev/workflow/rocoto/sfs_tasks.py index fe11bde8699..2a8906e073e 100644 --- a/dev/workflow/rocoto/sfs_tasks.py +++ b/dev/workflow/rocoto/sfs_tasks.py @@ -612,7 +612,7 @@ def arch_tars(self): return task - # Globus transfer for HPSS archiving + # Globus transfer for HPSS archiving def globus(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.run}_arch_tars'} From 46b0bc2577610487a17a330f5bcd050f90b037ff Mon Sep 17 00:00:00 2001 From: NeilBarton-NOAA Date: Mon, 3 Nov 2025 15:38:02 -0500 Subject: [PATCH 08/26] fixing python errors --- ush/python/pygfs/task/oceanice_products.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ush/python/pygfs/task/oceanice_products.py b/ush/python/pygfs/task/oceanice_products.py index ffb2a45e087..6a1e4df2d6e 100644 --- a/ush/python/pygfs/task/oceanice_products.py +++ b/ush/python/pygfs/task/oceanice_products.py @@ -174,7 +174,7 @@ def execute(config: Dict, product_grid: str) -> None: if config.oceanice_yaml.ocnicepost.namelist.write_netcdf or config.oceanice_yaml.ocnicepost.namelist.write_grib2: OceanIceProducts.interp(config.DATA, config.APRUN_OCNICEPOST, exec_name="ocnicepost.x") - # Index the interpolated grib2 file + # Index the interpolated grib2 file OceanIceProducts.index(config, product_grid) @staticmethod @@ -293,12 +293,12 @@ def subset(config: Dict) -> None: ds_subset = ds_subset.drop_vars('ELAT', errors='ignore') ds_subset = ds_subset.drop_vars('NLON', errors='ignore') ds_subset = ds_subset.drop_vars('NLAT', errors='ignore') - + if config.component == 'ocean': # subset ocean variables for z_levels in products levels = config.oceanice_yaml.ocean.namelist.ocean_levels - ds_subset = ds[varlist].sel(z_l = levels) - + ds_subset = ds[varlist].sel(z_l=levels) + # save global attributes from the old netcdf file into new netcdf file ds_subset.attrs = ds.attrs From b7b220fdb1ae6783d7c2d7e861fc8954d706854b Mon Sep 17 00:00:00 2001 From: NeilBarton-NOAA Date: Mon, 3 Nov 2025 15:41:31 -0500 Subject: [PATCH 09/26] fix python style errors --- ush/python/pygfs/task/oceanice_products.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ush/python/pygfs/task/oceanice_products.py b/ush/python/pygfs/task/oceanice_products.py index 6a1e4df2d6e..665bc5fffd7 100644 --- a/ush/python/pygfs/task/oceanice_products.py +++ b/ush/python/pygfs/task/oceanice_products.py @@ -297,8 +297,8 @@ def subset(config: Dict) -> None: if config.component == 'ocean': # subset ocean variables for z_levels in products levels = config.oceanice_yaml.ocean.namelist.ocean_levels - ds_subset = ds[varlist].sel(z_l=levels) - + ds_subset = ds[varlist].sel(z_l=levels) + # save global attributes from the old netcdf file into new netcdf file ds_subset.attrs = ds.attrs @@ -306,7 +306,7 @@ def subset(config: Dict) -> None: default_compression = {"zlib": True, "complevel": 8} compress_encoding = {var_name: default_compression for var_name in ds_subset.data_vars} ds_subset.to_netcdf(output_file, encoding=compress_encoding) - + # save compress original file compress_encoding = {var_name: default_compression for var_name in ds.data_vars} ds.to_netcdf(compressed_file, encoding=compress_encoding) From 6fcf46f869e50d8273902afde1bc71029cad887e Mon Sep 17 00:00:00 2001 From: NeilBarton-NOAA Date: Mon, 3 Nov 2025 15:45:01 -0500 Subject: [PATCH 10/26] try again --- ush/python/pygfs/task/oceanice_products.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/python/pygfs/task/oceanice_products.py b/ush/python/pygfs/task/oceanice_products.py index 665bc5fffd7..f3286c66a4b 100644 --- a/ush/python/pygfs/task/oceanice_products.py +++ b/ush/python/pygfs/task/oceanice_products.py @@ -276,7 +276,7 @@ def subset(config: Dict) -> None: input_file = f"{config.component}.nc" output_file = f"{config.component}_subset.nc" compressed_file = f"{config.component}_compressed.nc" - + varlist = config.oceanice_yaml[config.component].subset logger.info(f"Subsetting {varlist} from {input_file} to {output_file}") @@ -298,7 +298,7 @@ def subset(config: Dict) -> None: # subset ocean variables for z_levels in products levels = config.oceanice_yaml.ocean.namelist.ocean_levels ds_subset = ds[varlist].sel(z_l=levels) - + # save global attributes from the old netcdf file into new netcdf file ds_subset.attrs = ds.attrs From c9ad5e3ec142ae087719eef38f1ab507837d0cf0 Mon Sep 17 00:00:00 2001 From: NeilBarton-NOAA Date: Mon, 3 Nov 2025 15:51:07 -0500 Subject: [PATCH 11/26] white space --- ush/python/pygfs/task/oceanice_products.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/oceanice_products.py b/ush/python/pygfs/task/oceanice_products.py index f3286c66a4b..ba35a206343 100644 --- a/ush/python/pygfs/task/oceanice_products.py +++ b/ush/python/pygfs/task/oceanice_products.py @@ -276,7 +276,7 @@ def subset(config: Dict) -> None: input_file = f"{config.component}.nc" output_file = f"{config.component}_subset.nc" compressed_file = f"{config.component}_compressed.nc" - + varlist = config.oceanice_yaml[config.component].subset logger.info(f"Subsetting {varlist} from {input_file} to {output_file}") From 156580ad1e92acbbf38e908c71a528ccf160502e Mon Sep 17 00:00:00 2001 From: Neil Barton <103681022+NeilBarton-NOAA@users.noreply.github.com> Date: Mon, 3 Nov 2025 16:34:33 -0500 Subject: [PATCH 12/26] Update parm/archive/ice_native.yaml.j2 Co-authored-by: Rahul Mahajan --- parm/archive/ice_native.yaml.j2 | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/parm/archive/ice_native.yaml.j2 b/parm/archive/ice_native.yaml.j2 index dde6c3daa99..546650a0225 100644 --- a/parm/archive/ice_native.yaml.j2 +++ b/parm/archive/ice_native.yaml.j2 @@ -1,10 +1,7 @@ ice_native: - {% set NET = NET %} - {% set FREQ = FHOUT_ICE_GFS %} - {% set head = NET + ".t" + cycle_HH + "z" %} name: "ICE_NATIVE" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ice_native_subset.tar" required: - {% for fhr in range(FREQ, FHMAX_GFS + FREQ, FREQ) %} - - "{{ COMIN_ICE_NETCDF | relpath(ROTDIR) }}/native/{{ head }}.native.f{{ '%03d' % fhr }}.nc" + {% for fhr in range(FHOUT_ICE_GFS, FHMAX_GFS + FHOUT_ICE_GFS, FHOUT_ICE_GFS) %} + - "{{ COMIN_ICE_NETCDF | relpath(ROTDIR) }}/native/{{ RUN }}.t{{ cycle_HH }}z.native.f{{ '%03d' % fhr }}.nc" {% endfor %} From 486f690f3adf64eef7d9ecdaa8da2f29c5851031 Mon Sep 17 00:00:00 2001 From: Neil Barton <103681022+NeilBarton-NOAA@users.noreply.github.com> Date: Mon, 3 Nov 2025 16:34:55 -0500 Subject: [PATCH 13/26] Update parm/archive/ocean_native.yaml.j2 Co-authored-by: Rahul Mahajan --- parm/archive/ocean_native.yaml.j2 | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/parm/archive/ocean_native.yaml.j2 b/parm/archive/ocean_native.yaml.j2 index 1bf2c151e9b..f0237cdeb02 100644 --- a/parm/archive/ocean_native.yaml.j2 +++ b/parm/archive/ocean_native.yaml.j2 @@ -1,10 +1,7 @@ ocean_native: - {% set NET = NET %} - {% set FREQ = FHOUT_ICE_GFS %} - {% set head = NET + ".t" + cycle_HH + "z" %} name: "OCEAN_NATIVE" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ocean_native_subset.tar" required: - {% for fhr in range(FREQ, FHMAX_GFS + FREQ, FREQ) %} - - "{{ COMIN_OCEAN_NETCDF | relpath(ROTDIR) }}/native/{{ head }}.native.f{{ '%03d' % fhr }}.nc" + {% for fhr in range(FHOUT_ICE_GFS, FHMAX_GFS + FHOUT_ICE_GFS, FHOUT_ICE_GFS) %} + - "{{ COMIN_OCEAN_NETCDF | relpath(ROTDIR) }}/native/{{ RUN }}.t{{ cycle_HH }}z.native.f{{ '%03d' % fhr }}.nc" {% endfor %} From 386a01dd739fde2ce2039bee8633b0f0aed99621 Mon Sep 17 00:00:00 2001 From: NeilBarton-NOAA Date: Mon, 3 Nov 2025 16:39:57 -0500 Subject: [PATCH 14/26] corrected and to or --- parm/post/oceanice_products_gfs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/post/oceanice_products_gfs.yaml b/parm/post/oceanice_products_gfs.yaml index d47532d3353..01093ee7aea 100644 --- a/parm/post/oceanice_products_gfs.yaml +++ b/parm/post/oceanice_products_gfs.yaml @@ -9,7 +9,7 @@ ocnicepost: - "{{ DATA }}" copy: - ["{{ PARMgfs }}/post/ocnicepost.nml.jinja2", "{{ DATA }}/"] - {% if write_grib2 and write_netcdf %} + {% if write_grib2 or write_netcdf %} - ["{{ PARMgfs }}/post/{{ component }}_gfs.csv", "{{ DATA }}/{{ component }}.csv"] - ["{{ EXECgfs }}/ocnicepost.x", "{{ DATA }}/"] - ["{{ FIXgfs }}/mom6/post/{{ model_grid }}/tripole.{{ model_grid }}.Bu.to.Ct.bilinear.nc", "{{ DATA }}/"] From 1943a474a566d9c3ebd092a5aee88e2164185776 Mon Sep 17 00:00:00 2001 From: Neil Barton <103681022+NeilBarton-NOAA@users.noreply.github.com> Date: Mon, 3 Nov 2025 16:52:11 -0500 Subject: [PATCH 15/26] Update parm/post/oceanice_products_gfs.yaml Co-authored-by: Rahul Mahajan --- parm/post/oceanice_products_gfs.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/parm/post/oceanice_products_gfs.yaml b/parm/post/oceanice_products_gfs.yaml index 01093ee7aea..1cf313d2295 100644 --- a/parm/post/oceanice_products_gfs.yaml +++ b/parm/post/oceanice_products_gfs.yaml @@ -42,9 +42,6 @@ ocean: mkdir: - "{{ COMOUT_OCEAN_NETCDF }}/native" {% for grid in product_grids %} - {% if write_netcdf %} - - "{{ COMOUT_OCEAN_NETCDF }}/{{ grid }}" - {% endif %} {% if write_grib2 %} - "{{ COMOUT_OCEAN_GRIB }}/{{ grid }}" {% endif %} From 3091fa1eb5a661eaaf7d54de4c912fb170db9db1 Mon Sep 17 00:00:00 2001 From: Neil Barton <103681022+NeilBarton-NOAA@users.noreply.github.com> Date: Wed, 5 Nov 2025 16:04:51 -0500 Subject: [PATCH 16/26] Update subset list in oceanice_products_gfs.yaml Removed unnecessary variables from the subset list in oceanice_products_gfs.yaml. remove duplicate variables --- parm/post/oceanice_products_gfs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/post/oceanice_products_gfs.yaml b/parm/post/oceanice_products_gfs.yaml index 1cf313d2295..0755947fcbb 100644 --- a/parm/post/oceanice_products_gfs.yaml +++ b/parm/post/oceanice_products_gfs.yaml @@ -34,7 +34,7 @@ ocean: {% elif model_grid == 'mx500' %} ocean_levels: [5, 15, 25, 35, 45, 55, 65, 75, 85, 95, 105, 115, 125, 135, 145, 155, 165, 175, 185, 195, 205, 215, 226, 241, 267] {% endif %} - subset: ['SSH', 'SST', 'SSS', 'speed', 'MLD_003', 'SSU', 'SSV', 'temp', 'tob', 'so', 'uo', 'vo'] + subset: ['SSH', 'MLD_003', 'temp', 'tob', 'so', 'uo', 'vo'] data_in: copy: - ["{{ COMIN_OCEAN_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ interval }}hr_avg.f{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/ocean.nc"] From 93a10fe05956aa7e1a9a87eb6157f7f7183c1b05 Mon Sep 17 00:00:00 2001 From: NeilBarton-NOAA Date: Thu, 6 Nov 2025 14:05:15 -0500 Subject: [PATCH 17/26] removing compressing of COMROT files and adding namelist options --- parm/post/oceanice_products_gfs.yaml | 4 ++-- ush/python/pygfs/task/oceanice_products.py | 14 ++++++-------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/parm/post/oceanice_products_gfs.yaml b/parm/post/oceanice_products_gfs.yaml index 0755947fcbb..e2e668bc2c7 100644 --- a/parm/post/oceanice_products_gfs.yaml +++ b/parm/post/oceanice_products_gfs.yaml @@ -3,6 +3,8 @@ ocnicepost: namelist: write_grib2: {{ write_grib2 }} write_netcdf: {{ write_netcdf }} + compress: True + compress_level: 9 debug: False fix_data: mkdir: @@ -48,7 +50,6 @@ ocean: {% endfor %} copy: - ["{{ DATA }}/ocean_subset.nc", "{{ COMOUT_OCEAN_NETCDF }}/native/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.native.f{{ '%03d' % forecast_hour }}.nc"] - - ["{{ DATA }}/ocean_compressed.nc", "{{ COMIN_OCEAN_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ interval }}hr_avg.f{{ '%03d' % forecast_hour }}.nc"] {% if write_grib2 %} {% for grid in product_grids %} - ["{{ DATA }}/ocean.{{ grid }}.grib2", "{{ COMOUT_OCEAN_GRIB }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2"] @@ -80,7 +81,6 @@ ice: {% endfor %} copy: - ["{{ DATA }}/ice_subset.nc", "{{ COMOUT_ICE_NETCDF }}/native/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.native.f{{ '%03d' % forecast_hour }}.nc"] - - ["{{ DATA }}/ice_compressed.nc", "{{ COMIN_ICE_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ interval }}hr_avg.f{{ '%03d' % forecast_hour }}.nc"] {% for grid in product_grids %} {% if write_grib2 %} - ["{{ DATA }}/ice.{{ grid }}.grib2", "{{ COMOUT_ICE_GRIB }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2"] diff --git a/ush/python/pygfs/task/oceanice_products.py b/ush/python/pygfs/task/oceanice_products.py index ba35a206343..a0ab8073d50 100644 --- a/ush/python/pygfs/task/oceanice_products.py +++ b/ush/python/pygfs/task/oceanice_products.py @@ -275,7 +275,6 @@ def subset(config: Dict) -> None: input_file = f"{config.component}.nc" output_file = f"{config.component}_subset.nc" - compressed_file = f"{config.component}_compressed.nc" varlist = config.oceanice_yaml[config.component].subset @@ -303,13 +302,12 @@ def subset(config: Dict) -> None: ds_subset.attrs = ds.attrs # save subsetted variables to a new netcdf file and compress - default_compression = {"zlib": True, "complevel": 8} - compress_encoding = {var_name: default_compression for var_name in ds_subset.data_vars} - ds_subset.to_netcdf(output_file, encoding=compress_encoding) - - # save compress original file - compress_encoding = {var_name: default_compression for var_name in ds.data_vars} - ds.to_netcdf(compressed_file, encoding=compress_encoding) + if config.oceanice_yaml.ocnicepost.namelist.compress: + default_compression = {"zlib": True, "complevel": int(config.oceanice_yaml.ocnicepost.namelist.compress_level)} + compress_encoding = {var_name: default_compression for var_name in ds_subset.data_vars} + ds_subset.to_netcdf(output_file, encoding=compress_encoding) + else: + ds_subset.to_netcdf(output_file) except FileNotFoundError: logger.exception(f"FATAL ERROR: Input file not found: {input_file}") From fecb2ab764cabf4f4d8078593effc12b8e181775 Mon Sep 17 00:00:00 2001 From: Neil Barton <103681022+NeilBarton-NOAA@users.noreply.github.com> Date: Thu, 6 Nov 2025 15:10:53 -0500 Subject: [PATCH 18/26] Update dev/parm/config/sfs/config.nsst.j2 Co-authored-by: Rahul Mahajan --- dev/parm/config/sfs/config.nsst.j2 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dev/parm/config/sfs/config.nsst.j2 b/dev/parm/config/sfs/config.nsst.j2 index 04abadc91dc..137e785eb65 100644 --- a/dev/parm/config/sfs/config.nsst.j2 +++ b/dev/parm/config/sfs/config.nsst.j2 @@ -25,6 +25,8 @@ export NST_GSI=3 # default 0: No NST info at all; # 2: Input NST info, used in CRTM simulation, no Tr analysis # 3: Input NST info, used in both CRTM simulation and Tr analysis export NSTINFO=0 # number of elements added in obs. data array (default = 0) -if (( NST_GSI > 0 )); then export NSTINFO=4; fi +if [[ ${NST_GSI} -gt 0 ]]; then + export NSTINFO=4 +fi echo "END: config.nsst" From b3f10b3fd2b445f2f5537291961f8258fd1255f1 Mon Sep 17 00:00:00 2001 From: NeilBarton-NOAA Date: Thu, 6 Nov 2025 15:14:25 -0500 Subject: [PATCH 19/26] addressing comments --- parm/archive/ocean_native.yaml.j2 | 2 +- parm/post/oceanice_products_gfs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/archive/ocean_native.yaml.j2 b/parm/archive/ocean_native.yaml.j2 index f0237cdeb02..51c039c06bb 100644 --- a/parm/archive/ocean_native.yaml.j2 +++ b/parm/archive/ocean_native.yaml.j2 @@ -2,6 +2,6 @@ ocean_native: name: "OCEAN_NATIVE" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ocean_native_subset.tar" required: - {% for fhr in range(FHOUT_ICE_GFS, FHMAX_GFS + FHOUT_ICE_GFS, FHOUT_ICE_GFS) %} + {% for fhr in range(FHOUT_OCN_GFS, FHMAX_GFS + FHOUT_OCN_GFS, FHOUT_OCN_GFS) %} - "{{ COMIN_OCEAN_NETCDF | relpath(ROTDIR) }}/native/{{ RUN }}.t{{ cycle_HH }}z.native.f{{ '%03d' % fhr }}.nc" {% endfor %} diff --git a/parm/post/oceanice_products_gfs.yaml b/parm/post/oceanice_products_gfs.yaml index e2e668bc2c7..9e33a481648 100644 --- a/parm/post/oceanice_products_gfs.yaml +++ b/parm/post/oceanice_products_gfs.yaml @@ -11,7 +11,7 @@ ocnicepost: - "{{ DATA }}" copy: - ["{{ PARMgfs }}/post/ocnicepost.nml.jinja2", "{{ DATA }}/"] - {% if write_grib2 or write_netcdf %} + {% if write_grib2 %} - ["{{ PARMgfs }}/post/{{ component }}_gfs.csv", "{{ DATA }}/{{ component }}.csv"] - ["{{ EXECgfs }}/ocnicepost.x", "{{ DATA }}/"] - ["{{ FIXgfs }}/mom6/post/{{ model_grid }}/tripole.{{ model_grid }}.Bu.to.Ct.bilinear.nc", "{{ DATA }}/"] From b7e343a0484d2d154eb2101caab58387763a1bd3 Mon Sep 17 00:00:00 2001 From: Neil Barton <103681022+NeilBarton-NOAA@users.noreply.github.com> Date: Wed, 12 Nov 2025 15:44:32 -0500 Subject: [PATCH 20/26] Update parm/post/oceanice_products_sfs.yaml Co-authored-by: Rahul Mahajan --- parm/post/oceanice_products_sfs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/post/oceanice_products_sfs.yaml b/parm/post/oceanice_products_sfs.yaml index 089a301e4ec..1bb10de9076 100644 --- a/parm/post/oceanice_products_sfs.yaml +++ b/parm/post/oceanice_products_sfs.yaml @@ -2,7 +2,7 @@ ocnicepost: executable: "ocnicepost.x" namelist: write_grib2: {{ write_grib2 }} - write_netcdf: {{ write_netcdf }} + write_netcdf: False debug: False fix_data: mkdir: From 0246dd6236f2c01c062fb962de0b734bce09868c Mon Sep 17 00:00:00 2001 From: NeilBarton-NOAA Date: Wed, 12 Nov 2025 16:01:25 -0500 Subject: [PATCH 21/26] changing to or --- parm/post/oceanice_products_gfs.yaml | 2 +- parm/post/oceanice_products_sfs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/post/oceanice_products_gfs.yaml b/parm/post/oceanice_products_gfs.yaml index 9e33a481648..e2e668bc2c7 100644 --- a/parm/post/oceanice_products_gfs.yaml +++ b/parm/post/oceanice_products_gfs.yaml @@ -11,7 +11,7 @@ ocnicepost: - "{{ DATA }}" copy: - ["{{ PARMgfs }}/post/ocnicepost.nml.jinja2", "{{ DATA }}/"] - {% if write_grib2 %} + {% if write_grib2 or write_netcdf %} - ["{{ PARMgfs }}/post/{{ component }}_gfs.csv", "{{ DATA }}/{{ component }}.csv"] - ["{{ EXECgfs }}/ocnicepost.x", "{{ DATA }}/"] - ["{{ FIXgfs }}/mom6/post/{{ model_grid }}/tripole.{{ model_grid }}.Bu.to.Ct.bilinear.nc", "{{ DATA }}/"] diff --git a/parm/post/oceanice_products_sfs.yaml b/parm/post/oceanice_products_sfs.yaml index 1bb10de9076..6c518891afc 100644 --- a/parm/post/oceanice_products_sfs.yaml +++ b/parm/post/oceanice_products_sfs.yaml @@ -9,7 +9,7 @@ ocnicepost: - "{{ DATA }}" copy: - ["{{ PARMgfs }}/post/ocnicepost.nml.jinja2", "{{ DATA }}/"] - {% if write_grib2 and write_netcdf %} + {% if write_grib2 or write_netcdf %} - ["{{ PARMgfs }}/post/{{ component }}_gfs.csv", "{{ DATA }}/{{ component }}.csv"] - ["{{ EXECgfs }}/ocnicepost.x", "{{ DATA }}/"] - ["{{ FIXgfs }}/mom6/post/{{ model_grid }}/tripole.{{ model_grid }}.Bu.to.Ct.bilinear.nc", "{{ DATA }}/"] From 29f73575d68d5a2975b89e13f705abcaf75356dc Mon Sep 17 00:00:00 2001 From: NeilBarton-NOAA Date: Wed, 12 Nov 2025 16:02:49 -0500 Subject: [PATCH 22/26] remove compressed in COMOUT overwrite --- parm/post/oceanice_products_sfs.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/parm/post/oceanice_products_sfs.yaml b/parm/post/oceanice_products_sfs.yaml index 6c518891afc..bdc3ba0d493 100644 --- a/parm/post/oceanice_products_sfs.yaml +++ b/parm/post/oceanice_products_sfs.yaml @@ -51,7 +51,6 @@ ocean: {% endfor %} copy: - ["{{ DATA }}/ocean_subset.nc", "{{ COMOUT_OCEAN_NETCDF }}/native/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.native.f{{ '%03d' % forecast_hour }}.nc"] - - ["{{ DATA }}/ocean_compressed.nc", "{{ COMIN_OCEAN_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ interval }}hr_avg.f{{ '%03d' % forecast_hour }}.nc"] {% if write_grib2 %} {% for grid in product_grids %} - ["{{ DATA }}/ocean.{{ grid }}.grib2", "{{ COMOUT_OCEAN_GRIB }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2"] @@ -83,7 +82,6 @@ ice: {% endfor %} copy: - ["{{ DATA }}/ice_subset.nc", "{{ COMOUT_ICE_NETCDF }}/native/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.native.f{{ '%03d' % forecast_hour }}.nc"] - - ["{{ DATA }}/ice_compressed.nc", "{{ COMIN_ICE_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ interval }}hr_avg.f{{ '%03d' % forecast_hour }}.nc"] {% for grid in product_grids %} {% if write_grib2 %} - ["{{ DATA }}/ice.{{ grid }}.grib2", "{{ COMOUT_ICE_GRIB }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2"] From 2b705db4e492e5aaf488922943d38dc2240d8e4b Mon Sep 17 00:00:00 2001 From: NeilBarton-NOAA Date: Wed, 12 Nov 2025 16:06:11 -0500 Subject: [PATCH 23/26] defining variable outside of yaml --- parm/post/oceanice_products_sfs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/post/oceanice_products_sfs.yaml b/parm/post/oceanice_products_sfs.yaml index bdc3ba0d493..d43075d28d7 100644 --- a/parm/post/oceanice_products_sfs.yaml +++ b/parm/post/oceanice_products_sfs.yaml @@ -2,7 +2,7 @@ ocnicepost: executable: "ocnicepost.x" namelist: write_grib2: {{ write_grib2 }} - write_netcdf: False + write_netcdf: {{ write_netcdf }} debug: False fix_data: mkdir: From 08f4a5a1dc77467ab2e31d808b16b7f015d70efe Mon Sep 17 00:00:00 2001 From: NeilBarton-NOAA Date: Wed, 12 Nov 2025 16:44:37 -0500 Subject: [PATCH 24/26] add option to copy netcdf lat/lon products to COMROT --- parm/post/oceanice_products_gfs.yaml | 13 +++++++++++-- parm/post/oceanice_products_sfs.yaml | 16 ++++++++++++---- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/parm/post/oceanice_products_gfs.yaml b/parm/post/oceanice_products_gfs.yaml index e2e668bc2c7..a9ffa5c94f8 100644 --- a/parm/post/oceanice_products_gfs.yaml +++ b/parm/post/oceanice_products_gfs.yaml @@ -44,18 +44,24 @@ ocean: mkdir: - "{{ COMOUT_OCEAN_NETCDF }}/native" {% for grid in product_grids %} + {% if write_netcdf %} + - "{{ COMOUT_OCEAN_NETCDF }}/{{ grid }}" + {% endif %} {% if write_grib2 %} - "{{ COMOUT_OCEAN_GRIB }}/{{ grid }}" {% endif %} {% endfor %} copy: - ["{{ DATA }}/ocean_subset.nc", "{{ COMOUT_OCEAN_NETCDF }}/native/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.native.f{{ '%03d' % forecast_hour }}.nc"] - {% if write_grib2 %} {% for grid in product_grids %} + {% if write_netcdf %} + - ["{{ DATA }}/ocean.{{ grid }}.nc", "{{ COMOUT_OCEAN_NETCDF }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.nc"] + {% endif %} + {% if write_grib2 %} - ["{{ DATA }}/ocean.{{ grid }}.grib2", "{{ COMOUT_OCEAN_GRIB }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2"] - ["{{ DATA }}/ocean.{{ grid }}.grib2.idx", "{{ COMOUT_OCEAN_GRIB }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2.idx"] - {% endfor %} {% endif %} + {% endfor %} ice: namelist: @@ -82,6 +88,9 @@ ice: copy: - ["{{ DATA }}/ice_subset.nc", "{{ COMOUT_ICE_NETCDF }}/native/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.native.f{{ '%03d' % forecast_hour }}.nc"] {% for grid in product_grids %} + {% if write_netcdf %} + - ["{{ DATA }}/ice.{{ grid }}.nc", "{{ COMOUT_ICE_NETCDF }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.nc"] + {% endif %} {% if write_grib2 %} - ["{{ DATA }}/ice.{{ grid }}.grib2", "{{ COMOUT_ICE_GRIB }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2"] - ["{{ DATA }}/ice.{{ grid }}.grib2.idx", "{{ COMOUT_ICE_GRIB }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2.idx"] diff --git a/parm/post/oceanice_products_sfs.yaml b/parm/post/oceanice_products_sfs.yaml index d43075d28d7..a9ffa5c94f8 100644 --- a/parm/post/oceanice_products_sfs.yaml +++ b/parm/post/oceanice_products_sfs.yaml @@ -3,6 +3,8 @@ ocnicepost: namelist: write_grib2: {{ write_grib2 }} write_netcdf: {{ write_netcdf }} + compress: True + compress_level: 9 debug: False fix_data: mkdir: @@ -30,11 +32,11 @@ ocean: cosvar: "cos_rot" angvar: "" {% if model_grid == 'mx025' or model_grid == 'mx050' or model_grid == 'mx100' %} - ocean_levels: [5, 15, 25, 35, 45, 55, 65, 75, 85, 95, 105, 115, 125, 135, 145, 155, 165, 175, 185, 195, 205, 215] + ocean_levels: [1, 3, 5, 10, 20, 30, 50, 100, 200, 500, 1000] {% elif model_grid == 'mx500' %} ocean_levels: [5, 15, 25, 35, 45, 55, 65, 75, 85, 95, 105, 115, 125, 135, 145, 155, 165, 175, 185, 195, 205, 215, 226, 241, 267] {% endif %} - subset: ['SSH', 'SST', 'SSS', 'speed', 'MLD_003', 'SSU', 'SSV', 'temp', 'tob', 'so', 'uo', 'vo'] + subset: ['SSH', 'MLD_003', 'temp', 'tob', 'so', 'uo', 'vo'] data_in: copy: - ["{{ COMIN_OCEAN_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ interval }}hr_avg.f{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/ocean.nc"] @@ -51,12 +53,15 @@ ocean: {% endfor %} copy: - ["{{ DATA }}/ocean_subset.nc", "{{ COMOUT_OCEAN_NETCDF }}/native/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.native.f{{ '%03d' % forecast_hour }}.nc"] - {% if write_grib2 %} {% for grid in product_grids %} + {% if write_netcdf %} + - ["{{ DATA }}/ocean.{{ grid }}.nc", "{{ COMOUT_OCEAN_NETCDF }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.nc"] + {% endif %} + {% if write_grib2 %} - ["{{ DATA }}/ocean.{{ grid }}.grib2", "{{ COMOUT_OCEAN_GRIB }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2"] - ["{{ DATA }}/ocean.{{ grid }}.grib2.idx", "{{ COMOUT_OCEAN_GRIB }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2.idx"] - {% endfor %} {% endif %} + {% endfor %} ice: namelist: @@ -83,6 +88,9 @@ ice: copy: - ["{{ DATA }}/ice_subset.nc", "{{ COMOUT_ICE_NETCDF }}/native/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.native.f{{ '%03d' % forecast_hour }}.nc"] {% for grid in product_grids %} + {% if write_netcdf %} + - ["{{ DATA }}/ice.{{ grid }}.nc", "{{ COMOUT_ICE_NETCDF }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.nc"] + {% endif %} {% if write_grib2 %} - ["{{ DATA }}/ice.{{ grid }}.grib2", "{{ COMOUT_ICE_GRIB }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2"] - ["{{ DATA }}/ice.{{ grid }}.grib2.idx", "{{ COMOUT_ICE_GRIB }}/{{ grid }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.{{ grid }}.f{{ '%03d' % forecast_hour }}.grib2.idx"] From 4aa74d518bc1493e085b5b31eee0c2ef3a17bedd Mon Sep 17 00:00:00 2001 From: NeilBarton-NOAA Date: Thu, 13 Nov 2025 14:57:37 -0500 Subject: [PATCH 25/26] update gfs_utils.fd --- sorc/gfs_utils.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gfs_utils.fd b/sorc/gfs_utils.fd index eef8acd56eb..e30ac0a520b 160000 --- a/sorc/gfs_utils.fd +++ b/sorc/gfs_utils.fd @@ -1 +1 @@ -Subproject commit eef8acd56eb5b2f9cdb10d7003e27cd9e965713a +Subproject commit e30ac0a520bf1142123464b50606a80a0334bff2 From fe149cdbe2ee0d9e30462d4acd62fccb6032c9a6 Mon Sep 17 00:00:00 2001 From: NeilBarton-NOAA Date: Thu, 13 Nov 2025 17:01:08 -0500 Subject: [PATCH 26/26] add if statement for indexing grib2 files --- ush/python/pygfs/task/oceanice_products.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ush/python/pygfs/task/oceanice_products.py b/ush/python/pygfs/task/oceanice_products.py index a0ab8073d50..edab120cbd6 100644 --- a/ush/python/pygfs/task/oceanice_products.py +++ b/ush/python/pygfs/task/oceanice_products.py @@ -174,6 +174,7 @@ def execute(config: Dict, product_grid: str) -> None: if config.oceanice_yaml.ocnicepost.namelist.write_netcdf or config.oceanice_yaml.ocnicepost.namelist.write_grib2: OceanIceProducts.interp(config.DATA, config.APRUN_OCNICEPOST, exec_name="ocnicepost.x") + if config.oceanice_yaml.ocnicepost.namelist.write_grib2: # Index the interpolated grib2 file OceanIceProducts.index(config, product_grid)