Set up beginning CI and add YAML check script (#1)

Adds:
pycodestyle CI test
unit test for YAML checker
YAML checker
basic cmake structure

.github/workflows/pynorms.yaml

@@ -0,0 +1,23 @@
+name: Check Python Coding Norms
+on: [push, pull_request]
+
+jobs:
+  check_pynorms:
+    runs-on: ubuntu-latest
+    name: Check Python coding norms with pycodestyle
+
+    steps:
+
+    - name: Install dependencies
+      run: |
+        pip install --upgrade pip
+        pip install pycodestyle
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+        path: GDASApp
+
+    - name: Run pycodestyle
+      run: |
+        cd $GITHUB_WORKSPACE/GDASApp
+        pycodestyle -v --config ./.pycodestyle .

.github/workflows/unittests.yaml

@@ -0,0 +1,31 @@
+name: Run Unit Tests
+on: [push, pull_request]
+
+jobs:
+  ctests:
+    runs-on: ubuntu-latest
+    name: Run Unit Tests with ctest
+
+    steps:
+
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+        path: GDASApp
+
+    - name: Configure with cmake
+      run: |
+        cd $GITHUB_WORKSPACE/GDASApp
+        mkdir build
+        cd build
+        cmake ../
+
+    - name: Build GDASApp
+      run: |
+        cd $GITHUB_WORKSPACE/GDASApp/build
+        make
+
+    - name: Run ctest
+      run: |
+        cd $GITHUB_WORKSPACE/GDASApp/build
+        ctest

.pycodestyle

@@ -0,0 +1,6 @@
+[pycodestyle]
+count = False
+ignore = E226,E401,E402,W504
+max-line-length = 160
+statistics = True
+exclude = Experimental

CMakeLists.txt

@@ -0,0 +1,10 @@
+cmake_minimum_required(VERSION 3.10)
+
+# set the project name
+project(GDASApp)
+
+find_package(Python3 REQUIRED COMPONENTS Interpreter)
+
+enable_testing()
+
+add_subdirectory(test)

test/CMakeLists.txt

@@ -0,0 +1,3 @@
+add_test(NAME test_check_yaml_keys
+         COMMAND ${Python3_EXECUTABLE} ${PROJECT_SOURCE_DIR}/ush/check_yaml_keys.py ${PROJECT_SOURCE_DIR}/test/testinput/check_yaml_keys_ref.yaml ${PROJECT_SOURCE_DIR}/test/testinput/check_yaml_keys_test.yaml
+         WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/)

test/testinput/check_yaml_keys_ref.yaml

@@ -0,0 +1,12 @@
+app: GDAS
+components:
+- name: atmosphere
+  model: FV3
+- name: ocean
+  model: MOM6
+- name: ice
+  model: CICE6
+- name: aerosols
+  model: GOCART
+- name: land
+  model: noah

test/testinput/check_yaml_keys_test.yaml

@@ -0,0 +1,4 @@
+app: GDAS
+components:
+- name: atmosphere
+  model: FV3

ush/check_yaml_keys.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+# compare keys between two YAML files
+import argparse
+import logging
+import os
+import yaml
+
+
+def check_yaml(YAMLref, YAMLtest, checkValues=False):
+    assert os.path.exists(YAMLref), f"File {YAMLref} not found."
+    assert os.path.exists(YAMLtest), f"File {YAMLtest} not found."
+    logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S')
+    logging.info(f'Comparing {YAMLtest} against {YAMLref}...')
+    # load reference file
+    try:
+        with open(YAMLref, 'r') as YAMLref_opened:
+            ref_dict = yaml.safe_load(YAMLref_opened)
+    except Exception as e:
+        logging.error(f'Error occurred when attempting to load: {YAMLref}, error: {e}')
+    # load file to test
+    try:
+        with open(YAMLtest, 'r') as YAMLtest_opened:
+            test_dict = yaml.safe_load(YAMLtest_opened)
+    except Exception as e:
+        logging.error(f'Error occurred when attempting to load: {YAMLtest}, error: {e}')
+    # loop through top level of YAML
+    compare_dict('', ref_dict, test_dict, checkValues)
+
+
+def compare_dict(rootkey, dict1, dict2, checkValues):
+    for key, value in dict1.items():
+        keypath = f"{rootkey}/{key}"
+        if key not in dict2:
+            logging.error(f"'{keypath}' not in test file.")
+        else:
+            if isinstance(value, dict):
+                compare_dict(keypath, value, dict2[key], checkValues)
+            elif isinstance(value, list):
+                compare_list(keypath, value, dict2[key], checkValues)
+            else:
+                if checkValues:
+                    if value != dict2[key]:
+                        logging.warning(f"{keypath}: {dict2[key]} != {value}")
+
+
+def compare_list(rootkey, list1, list2, checkValues):
+    if len(list2) != len(list1):
+        logging.error(f"{rootkey} len={len(list2)} != {len(list1)}")
+    for i, item in enumerate(list1):
+        newkey = f"{rootkey}[{i}]"
+        if i+1 <= len(list2):
+            if isinstance(item, dict) and i+1 <= len(list2):
+                compare_dict(newkey, item, list2[i], checkValues)
+            elif isinstance(item, list):
+                compare_list(newkey, item, list2[i], checkValues)
+            else:
+                if checkValues:
+                    if item != list2[i]:
+                        logging.warning(f"{rootkey}/{i}: {list2[i]} != {item}")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('YAMLref', type=str, help='Reference YAML file')
+    parser.add_argument('YAMLtest', type=str, help='YAML file to compare to reference')
+    parser.add_argument("--checkvalues", help="Check values in addition to keys",
+                        action="store_true")
+    args = parser.parse_args()
+    check_yaml(args.YAMLref, args.YAMLtest, args.checkvalues)