Skip to content

Commit

Permalink
Merge pull request #546 from climbfuji/feature/merge_feature_capgen_i…
Browse files Browse the repository at this point in the history
…nto_main_20240308

Merge feature/capgen into main as of 2024-03-08
  • Loading branch information
dustinswales authored Mar 18, 2024
2 parents 221788f + 87e6d92 commit f1db415
Show file tree
Hide file tree
Showing 88 changed files with 7,023 additions and 2,008 deletions.
17 changes: 17 additions & 0 deletions .github/workflows/capgen_unit_tests.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
name: Capgen Unit Tests

on:
workflow_dispatch:
pull_request:
branches: [feature/capgen, main]

jobs:
unit_tests:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: update repos and install dependencies
run: sudo apt-get update && sudo apt-get install -y build-essential gfortran cmake python3 git
- name: Run unit tests
run: cd test && ./run_fortran_tests.sh

2 changes: 1 addition & 1 deletion CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ project(ccpp_framework
#------------------------------------------------------------------------------
# Set package definitions
set(PACKAGE "ccpp-framework")
set(AUTHORS "Dom Heinzeller" "Grant Firl" "Mike Kavulich" "Steve Goldhaber")
set(AUTHORS "Dom Heinzeller" "Grant Firl" "Mike Kavulich" "Dustin Swales" "Courtney Peverley")
string(TIMESTAMP YEAR "%Y")

#------------------------------------------------------------------------------
Expand Down
2 changes: 1 addition & 1 deletion CODEOWNERS
Validating CODEOWNERS rules …
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

# These owners will be the default owners for everything in the repo.
#* @defunkt
* @climbfuji @grantfirl @gold2718 @mkavulich
* @climbfuji @gold2718 @dustinswales @mwaxmonsky @peverwhee @grantfirl @mkavulich

# Order is important. The last matching pattern has the most precedence.
# So if a pull request only touches javascript files, only these owners
Expand Down
5 changes: 3 additions & 2 deletions doc/HelloWorld/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
CMAKE_MINIMUM_REQUIRED(VERSION 3.10)
PROJECT(HelloWorld)
ENABLE_LANGUAGE(Fortran)

Expand Down Expand Up @@ -166,7 +166,8 @@ list(APPEND DTABLE_CMD "${CCPP_CAP_FILES}/datatable.xml")
list(APPEND DTABLE_CMD "--ccpp-files")
list(APPEND DTABLE_CMD "--separator=\\;")
string(REPLACE ";" " " DTABLE_STRING "${DTABLE_CMD}")
MESSAGE(STATUS "Running: ${DTABLE_STRING}")
string(STRIP ${DTABLE_STRING} DTABLE_STRING)
MESSAGE(STATUS "Running: ${DTABLE_STRING};")
EXECUTE_PROCESS(COMMAND ${DTABLE_CMD} OUTPUT_VARIABLE CCPP_CAPS
RESULT_VARIABLE RES
OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_STRIP_TRAILING_WHITESPACE)
Expand Down
60 changes: 50 additions & 10 deletions scripts/ccpp_capgen.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import logging
import re
# CCPP framework imports
from ccpp_database_obj import CCPPDatabaseObj
from ccpp_datafile import generate_ccpp_datatable
from ccpp_suite import API
from file_utils import check_for_writeable_file, remove_dir, replace_paths
Expand All @@ -26,11 +27,13 @@
from host_model import HostModel
from metadata_table import parse_metadata_file, SCHEME_HEADER_TYPE
from parse_tools import init_log, set_log_level, context_string
from parse_tools import register_fortran_ddt_name
from parse_tools import CCPPError, ParseInternalError

## Capture the Framework root
__SCRIPT_PATH = os.path.dirname(__file__)
__FRAMEWORK_ROOT = os.path.abspath(os.path.join(__SCRIPT_PATH, os.pardir))
_SCRIPT_PATH = os.path.dirname(__file__)
_FRAMEWORK_ROOT = os.path.abspath(os.path.join(_SCRIPT_PATH, os.pardir))
_SRC_ROOT = os.path.join(_FRAMEWORK_ROOT, "src")
## Init this now so that all Exceptions can be trapped
_LOGGER = init_log(os.path.basename(__file__))

Expand All @@ -43,6 +46,11 @@
## Metadata table types where order is significant
_ORDERED_TABLE_TYPES = [SCHEME_HEADER_TYPE]

## CCPP Framework supported DDT types
_CCPP_FRAMEWORK_DDT_TYPES = ["ccpp_hash_table_t",
"ccpp_hashable_t",
"ccpp_hashable_char_t"]

###############################################################################
def delete_pathnames_from_file(capfile, logger):
###############################################################################
Expand Down Expand Up @@ -306,6 +314,17 @@ def compare_fheader_to_mheader(meta_header, fort_header, logger):
lname = mvar.get_prop_value('local_name')
arrayref = is_arrayspec(lname)
fvar, find = find_var_in_list(lname, flist)
# Check for consistency between optional variables in metadata and
# optional variables in fortran. Error if optional attribute is
# missing from fortran declaration.
mopt = mvar.get_prop_value('optional')
if find and mopt:
fopt = fvar.get_prop_value('optional')
if (not fopt):
errmsg = 'Missing optional attribute in fortran declaration for variable {}, in file {}'
errors_found = add_error(errors_found, errmsg.format(mname,title))
# end if
# end if
if mind >= flen:
if arrayref:
# Array reference, variable not in Fortran table
Expand Down Expand Up @@ -559,7 +578,7 @@ def clean_capgen(cap_output_file, logger):
set_log_level(logger, log_level)

###############################################################################
def capgen(run_env):
def capgen(run_env, return_db=False):
###############################################################################
"""Parse indicated host, scheme, and suite files.
Generate code to allow host model to run indicated CCPP suites."""
Expand All @@ -578,12 +597,22 @@ def capgen(run_env):
# Try to create output_dir (let it crash if it fails)
os.makedirs(run_env.output_dir)
# end if
# Pre-register base CCPP DDT types:
for ddt_name in _CCPP_FRAMEWORK_DDT_TYPES:
register_fortran_ddt_name(ddt_name)
# end for
src_dir = os.path.join(_FRAMEWORK_ROOT, "src")
host_files = run_env.host_files
host_name = run_env.host_name
scheme_files = run_env.scheme_files
# We need to create three lists of files, hosts, schemes, and SDFs
host_files = create_file_list(run_env.host_files, ['meta'], 'Host',
run_env.logger)
# The host model needs to know about the constituents module
const_mod = os.path.join(_SRC_ROOT, "ccpp_constituent_prop_mod.meta")
if const_mod not in host_files:
host_files.append(const_mod)
# end if
scheme_files = create_file_list(run_env.scheme_files, ['meta'],
'Scheme', run_env.logger)
sdfs = create_file_list(run_env.suites, ['xml'], 'Suite', run_env.logger)
Expand All @@ -594,14 +623,21 @@ def capgen(run_env):
# end if
# First up, handle the host files
host_model = parse_host_model_files(host_files, host_name, run_env)
# We always need to parse the ccpp_constituent_prop_ptr_t DDT
const_prop_mod = os.path.join(src_dir, "ccpp_constituent_prop_mod.meta")
if const_prop_mod not in scheme_files:
scheme_files = [const_prop_mod] + scheme_files
# end if
# Next, parse the scheme files
scheme_headers, scheme_tdict = parse_scheme_files(scheme_files, run_env)
ddts = host_model.ddt_lib.keys()
if ddts and run_env.logger and run_env.logger.isEnabledFor(logging.DEBUG):
run_env.logger.debug("DDT definitions = {}".format(ddts))
if run_env.verbose:
ddts = host_model.ddt_lib.keys()
if ddts:
run_env.logger.debug("DDT definitions = {}".format(ddts))
# end if
# end if
plist = host_model.prop_list('local_name')
if run_env.logger and run_env.logger.isEnabledFor(logging.DEBUG):
if run_env.verbose:
run_env.logger.debug("{} variables = {}".format(host_model.name, plist))
run_env.logger.debug("schemes = {}".format([x.title
for x in scheme_headers]))
Expand All @@ -628,7 +664,8 @@ def capgen(run_env):
cap_filenames = ccpp_api.write(outtemp_dir, run_env)
if run_env.generate_host_cap:
# Create a cap file
host_files = [write_host_cap(host_model, ccpp_api,
cap_module = host_model.ccpp_cap_name()
host_files = [write_host_cap(host_model, ccpp_api, cap_module,
outtemp_dir, run_env)]
else:
host_files = list()
Expand All @@ -646,10 +683,13 @@ def capgen(run_env):
# end if
# Finally, create the database of generated files and caps
# This can be directly in output_dir because it will not affect dependencies
src_dir = os.path.join(__FRAMEWORK_ROOT, "src")
generate_ccpp_datatable(run_env, host_model, ccpp_api,
scheme_headers, scheme_tdict, host_files,
cap_filenames, kinds_file, src_dir)
if return_db:
return CCPPDatabaseObj(run_env, host_model=host_model, api=ccpp_api)
# end if
return None

###############################################################################
def _main_func():
Expand All @@ -665,7 +705,7 @@ def _main_func():
if framework_env.clean:
clean_capgen(framework_env.datatable_file, framework_env.logger)
else:
capgen(framework_env)
_ = capgen(framework_env)
# end if (clean)

###############################################################################
Expand Down
87 changes: 87 additions & 0 deletions scripts/ccpp_database_obj.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
#!/usr/bin/env python3

"""
Define the CCPPDatabaseObj object
Object definition and methods to provide information from a run of capgen.
"""

from host_model import HostModel
from ccpp_suite import API

class CCPPDatabaseObjError(ValueError):
"""Error class specific to CCPPDatabaseObj.
All uses of this error should be internal (i.e., programmer error,
not user error)."""

def __init__(self, message):
"""Initialize this exception"""
super().__init__(message)

class CCPPDatabaseObj:
"""Object with data and methods to provide information from a run of capgen.
"""

def __init__(self, run_env, host_model=None, api=None, database_file=None):
"""Initialize this CCPPDatabaseObj.
If <database_file> is not None, all other inputs MUST be None and
the object is created from the database table created by capgen.
To initialize the object from an in-memory capgen run, ALL other
inputs MUST be passed (i.e., not None) and it is an error to pass
a value for <database_file>.
"""

runtime_obj = all([host_model is not None, api is not None])
self.__host_model = None
self.__api = None
self.__database_file = None
if runtime_obj and database_file:
emsg = "Cannot provide both runtime arguments and database_file."
elif (not runtime_obj) and (not database_file):
emsg = "Must provide either database_file or all runtime arguments."
else:
emsg = ""
# end if
if emsg:
raise CCPPDatabaseObjError(f"ERROR: {emsg}")
# end if
if runtime_obj:
self.__host_model = host_model
self.__api = api
else:
self.db_from_file(run_env, database_file)
# end if

def db_from_file(self, run_env, database_file):
"""Create the necessary internal data structures from a CCPP
datatable.xml file created by capgen.
"""
metadata_tables = {}
host_name = "host"
self.__host_model = HostModel(metadata_tables, host_name, run_env)
self.__api = API(sdfs, host_model, scheme_headers, run_env)
raise CCPPDatabaseObjError("ERROR: <database_file> not supported")

def host_model_dict(self):
"""Return the host model dictionary for this CCPP DB object"""
if self.__host_model is not None:
return self.__host_model
# end if
raise CCPPDatabaseObjError("ERROR: <database_file> not supported")

def suite_list(self):
"""Return a list of suites built into the API"""
if self.__api is not None:
return list(self.__api.suites)
# end if
raise CCPPDatabaseObjError("ERROR: <database_file> not supported")

def constituent_dictionary(self, suite):
"""Return the constituent dictionary for <suite>"""
return suite.constituent_dictionary()

def call_list(self, phase):
"""Return the API call list for <phase>"""
if self.__api is not None:
return self.__api.call_list(phase)
# end if
raise CCPPDatabaseObjError("ERROR: <database_file> not supported")
17 changes: 9 additions & 8 deletions scripts/ccpp_datafile.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,6 @@
from parse_tools import read_xml_file, PrettyElementTree
from suite_objects import VerticalLoop, Subcycle

# Find python version
PY3 = sys.version_info[0] > 2
PYSUBVER = sys.version_info[1]

# Global data
_INDENT_STR = " "

Expand Down Expand Up @@ -652,12 +648,13 @@ def _new_var_entry(parent, var, full_entry=True):
"""Create a variable sub-element of <parent> with information from <var>.
If <full_entry> is False, only include standard name and intent.
"""
prop_list = ["intent"]
prop_list = ["intent", "local_name"]
if full_entry:
prop_list.extend(["allocatable", "active", "default_value",
"diagnostic_name", "diagnostic_name_fixed",
"kind", "persistence", "polymorphic", "protected",
"state_variable", "type", "units"])
"state_variable", "type", "units", "molar_mass",
"advected", "top_at_one", "optional"])
prop_list.extend(Var.constituent_property_names())
# end if
ventry = ET.SubElement(parent, "var")
Expand All @@ -671,9 +668,13 @@ def _new_var_entry(parent, var, full_entry=True):
if full_entry:
dims = var.get_dimensions()
if dims:
dim_entry = ET.SubElement(ventry, "dimensions")
dim_entry.text = " ".join(dims)
v_entry = ET.SubElement(ventry, "dimensions")
v_entry.text = " ".join(dims)
# end if
v_entry = ET.SubElement(ventry, "source_type")
v_entry.text = var.source.ptype.lower()
v_entry = ET.SubElement(ventry, "source_name")
v_entry.text = var.source.name.lower()
# end if

###############################################################################
Expand Down
Loading

0 comments on commit f1db415

Please sign in to comment.