Merge pull request #780 from rarutter/presim_script

This adds some utility python methods and the start
of a pre-simulation report script to help the user check
settings, file existence, input value validity, etc.

Author: rarutter

scripts/tests/presim_summary.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python
+
+import os
+import util
+import util.input
+import util.param
+import util.runmask
+import util.config
+import commentjson
+
+
+verbose = True
+
+#Print expected config file path
+#Load expected config file
+assumed_config_file = "config/config.js"
+print("Loading " + assumed_config_file)
+with open(assumed_config_file) as config_file:
+  config = commentjson.load(config_file)
+
+
+#Extract and display input file path prefixes (excluding parameter files
+# and output_spec)
+input_filepaths = util.config.get_input_paths(assumed_config_file)
+path_prefixes = []
+for filepath in input_filepaths:
+  prefix = os.path.dirname(filepath)
+  if prefix not in path_prefixes:
+    path_prefixes.append(prefix)
+print("*********************************************************")
+print("|", len(path_prefixes), "input path prefixes specified:")
+for prefix in path_prefixes:
+  print("|", prefix)
+print("*********************************************************")
+
+
+#Check that all input files specified in the config file exist (excluding
+# parameter files and output_spec)
+#We are not using the following method because that checks a whole
+# directory, which would not allow individually different input paths
+#util.input.check_input_set_existence(input directory)
+missing_files = [path for path in input_filepaths if not os.path.exists(path)]
+if len(missing_files) > 0:
+  star_len = len(max(missing_files, key=len)) + 4
+  star_str = '*' * star_len
+
+  print(star_str)
+  print("| Missing input files:")
+  print("|", *missing_files)
+  print(star_str)
+else:
+  print("***********************************")
+  print("| All specified input files exist |")
+  print("***********************************")
+
+
+#Find total count of active cells
+#The call to cell_count will also flag invalid values
+mask_info = util.runmask.cell_count(config['IO']['runmask_file'], verbose)
+invalid_values = {value:count for value,count in mask_info.items() if value not in [0,1]}
+
+print("*********************************************************")
+print("| Active cells in run mask:", mask_info[1])
+print("| Inactive cells in run mask:", mask_info[0])
+print("| Other values:", invalid_values)
+print("*********************************************************")
+
+
+#Check that all CMT values specified in the input vegetation file
+# have a parameterization in the parameter files.
+input_CMTs = util.input.get_vegtypes(config['IO']['veg_class_file'], verbose)
+print("Input veg types:", *input_CMTs.keys(), sep=', ')
+print("input veg counts:", *input_CMTs.values(), sep=', ')
+print(input_CMTs)
+
+param_CMTs = util.param.get_available_CMTs(config['IO']['parameter_dir'])
+print("Param CMTs:", param_CMTs)
+
+unparameterized_CMTs = [x for x in input_CMTs.keys() if x not in param_CMTs]
+
+if len(unparameterized_CMTs) > 0:
+  print("There are CMTs in the veg file that have no parameterization")
+  print(unparameterized_CMTs)
+  print("Check that these are masked out by the run mask or set to\
+        different values before running.")
+else:
+  print("************************************************")
+  print("| All input CMTs exist in the parameter files. |")
+  print("| Please ensure that all are calibrated.       |")
+  print("************************************************")
+
+

scripts/util/config.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python
+
+import sys
+import os
+import argparse
+import textwrap
+
+import commentjson
+
+
+def get_input_paths(config_file="config/config.js", verbose=True):
+  '''
+  '''
+  with open(config_file) as config_fh:
+    config = commentjson.load(config_fh)
+
+  #Should this also handle parameter_dir and output_spec file location?
+  #At least for now, no. The matching 'set_input_paths' method should likely
+  # only apply to this set of files.
+  input_filenames = ['hist_climate_file', 'proj_climate_file',
+                     'veg_class_file', 'drainage_file', 'soil_texture_file',
+                     'co2_file', 'proj_co2_file', 'runmask_file', 'topo_file',
+                     'fri_fire_file', 'hist_exp_fire_file',
+                     'proj_exp_fire_file']
+  if verbose:
+    print("Fetching input paths for filenames:", *input_filenames, sep=', ')
+
+  paths = []
+  for filename in input_filenames:
+#    path = os.path.dirname(config['IO'][filename])
+    path = config['IO'][filename]
+    if path not in paths:
+      paths.append(path)
+
+  return paths
+
+
+def cmdline_define():
+  '''Define the command line interface and return the parser object.'''
+
+  parser = argparse.ArgumentParser(
+    formatter_class=argparse.RawDescriptionHelpFormatter,
+    description=textwrap.dedent('''
+      Helper script for a dvm-dos-tem config file.
+    ''')
+  )
+  parser.add_argument('file', nargs='?', metavar=('FILE'),
+      help=textwrap.dedent('''The config file to operate on.'''))
+
+  parser.add_argument('--verbose', action='store_true',
+      help=textwrap.dedent('''Print info to stdout when script runs.'''))
+
+  parser.add_argument('--get-input-paths', action='store_true',
+      help=textwrap.dedent('''Returns all input file paths'''))
+
+  return parser
+
+def cmdline_parse(argv=None):
+  '''
+  The command line interface specification and parser for util config.py.
+
+  If parse_args(...) is called with argv=None, then parse_args(...) will
+  use sys.argv[1:]. Otherwise argv is parsed according to the specification.
+
+  Parameters
+  ----------
+  argv : None or list of strings
+    arguments that argparse library will parse; if None, then sys.argv[1:] are
+    parsed.
+
+  Returns
+  -------
+  args : Namespace generated by argparse
+
+  '''
+  parser = cmdline_define()
+
+  args = parser.parse_args(argv)
+
+  # print(argv)
+  # print(args)
+
+  # Force vebosity on if user requests showing data
+  #if args.show:
+    #args.verbose = True # ?? Not sure about this...
+  
+  if (args.file is None) or (not os.path.isfile(args.file)):
+    parser.error("'{}' is an invalid path to a config file!".format(args.file))
+
+  return args
+
+
+def cmdline_run(args):
+  '''
+  Executes based on the command line arguments.
+
+  Parameters
+  ----------
+  args : Namespace
+    Should be a Namespace with all the appropriate arguments.
+
+  Returns
+  -------
+  exit_code : int 
+    Non-zero if the program cannot complete successfully.
+
+  '''
+  if args.get_input_paths:
+    get_input_paths(args.file, args.verbose)
+
+  return 0
+
+
+def cmdline_entry(argv=None):
+  '''
+  Wrapper allowing for easier testing of the cmdline run and parse functions.
+  '''
+  args = cmdline_parse(argv)
+  return cmdline_run(args)
+
+
+if __name__ == '__main__':
+  sys.exit(cmdline_entry()) # this makes sure appropriate exit code is passed on.
+

scripts/util/input.py

@@ -5,6 +5,7 @@
 import glob
 import netCDF4 as nc
 import numpy as np
+from collections import Counter
 
 import argparse
 import textwrap
@@ -38,7 +39,7 @@ class MissingInputFilesValueError(ValueError):
 class MustSupplyPixelCoordsError(ValueError):
   '''Raise when user must supply pixel coordinates(i.e. command line flag --yx)'''
 
-def verify_input_files(in_folder):
+def check_input_set_existence(in_folder):
   '''
   Raises various exceptions if there are problems with the files in the in_folder.
 
@@ -73,8 +74,36 @@ def verify_input_files(in_folder):
     msg = "Missing files: {}".format(required_files.difference(files))
     raise MissingInputFilesValueError(msg)
 
-  if 'output' not in dirs:
-    raise MissingInputFilesValueError("'output/' directory not present!")
+
+def get_vegtypes(veg_input, verbose=False):
+  '''
+  Returns a Counter containing the CMT values in a specified vegetation
+  input file, along with a count of the cells for each. Does not check
+  the validity of CMT values, but does include invalid entries for easy
+  error checking from the caller.
+
+  Parameters
+  ----------
+  veg_input : str
+    Filename of the vegetation file to operate on.
+  '''
+
+  with nc.Dataset(veg_input, 'r') as veg_file:
+    if 'veg_class' not in veg_file.variables.keys():
+      raise RuntimeError(f'{veg_input} has no veg_class variable')
+
+    veg_data = veg_file.variables["veg_class"][:,:]
+
+  if verbose:
+    print(veg_data)
+
+  hashable_veg_data = veg_data.flatten()
+
+  veg_counts = Counter(hashable_veg_data)
+  if verbose:
+    print(veg_counts)
+
+  return veg_counts
 
 
 def crop_attr_string(ys='', xs='', yo='', xo='', msg=''):
@@ -630,9 +659,21 @@ def cmdline_define():
     Query one or more dvmdostem inputs for various information.'''))
   query_parser.add_argument('--iyix-from-latlon', default=None, nargs=2, type=float,
       help="Find closest pixel to provided lat and lon arguments.")
-  query_parser.add_argument('--latlon-file', 
-      default='../snap-data/temporary-veg-from-LandCover_TEM_2005.tif', 
-      help="The file to read from for getting lat/lon pixel offsets.")
+
+  query_parser.add_argument('--check-existence', action='store_true',
+    help=textwrap.dedent('''Check the files in the specified input\
+    directory against a hardcoded list of expected files.'''),
+    default=None)
+
+  query_parser.add_argument('--vegtypes', action='store_true',
+    help=textwrap.dedent('''Returns a Counter with the CMT values and the\
+    number of times they appear in a vegetation file'''))
+
+  query_parser.add_argument('input_folder', help="Path to a folder containing a set of dvmdostem inputs.")
+
+#  query_parser.add_argument('--latlon-file', 
+#      default='../snap-data/temporary-veg-from-LandCover_TEM_2005.tif', 
+#      help="The file to read from for getting lat/lon pixel offsets.")
 
   crop_parser = subparsers.add_parser('crop',help=textwrap.dedent('''\
       Crop an input dataset, using offset and size. The reason we need this in
@@ -682,17 +723,23 @@ def cmdline_define():
   print(args)
 
   if args.command == 'crop':
-    verify_input_files(args.input_folder)
+    check_input_set_existence(args.input_folder)
     crop_wrapper(args)
 
   if args.command == 'climate-ts-plot':
-    #verify_input_files(args.input_folder)
+    #check_input_set_existence(args.input_folder)
     climate_ts_plot(args)
 
   if args.command == 'climate-gap-plot':
     climate_gap_count_plot(args)
 
   if args.command == 'query':
+    if args.check_existence:
+      check_input_set_existence(args.input_folder)
+
+    if args.vegtypes:
+      get_vegtypes(args.veg_file)
+
     if args.iyix_from_latlon:
 
       TMP_NC_FILE = '/tmp/WINNING.nc'