OPFLOW with RAJA/HIOP sparse GPU solvers (#8)

* OPFLOW: initial implementation of RAJA/HiOp sparse GPU-based solver

WIP - HIOP Sparse solver with GPU model

OPFLOW: Started work on support for HIOP sparse solver interface for GPUs.

Added a copy of hiop sparse solver interface.

OPFLOW: Added model skeleton for GPU sparse version (copying from pbpolrajahiop)

Fixed build

Did some copy paste to add a test for HIOPSPARSE. This test is not actually
functional yet.

Started updating the hiopsparse model and solver code.

More work on updating the solver and model

Added scalar and vector unit tests for model to be used with HIOP sparse solver on GPU

Apply cmake lint

Fix unit tests.

Set the size of array when using Umpire memset.

Code formatting

Some minor changes to get PBPOLRAJAHIOPSPARSE model code to compile

Separate BUS/LINE/GEN/.../Param structs into reusable module

Minor edit

Rename files

Fix typo

Use BUS/LINE/GEN/.../Param structs in Raja HiOp Sparse model (compiles)

Updating HIOP sparse solver GPU API

Completed bounds kernels

Completed scalar and vector functions

WIP - HIOP Sparse solver with GPU model

OPFLOW: Started work on support for HIOP sparse solver interface for GPUs.

Added a copy of hiop sparse solver interface.

OPFLOW: Added model skeleton for GPU sparse version (copying from pbpolrajahiop)

Fixed build

Did some copy paste to add a test for HIOPSPARSE. This test is not actually
functional yet.

Started updating the hiopsparse model and solver code.

More work on updating the solver and model

Added scalar and vector unit tests for model to be used with HIOP sparse solver on GPU

Apply cmake lint

Fix unit tests.

Set the size of array when using Umpire memset.

Code formatting

Rename files

Use BUS/LINE/GEN/.../Param structs in Raja HiOp Sparse model (compiles)

Updating HIOP sparse solver GPU API

Completed bounds kernels

Jacobian and Hessian for sparse model (CPU --> GPU copy)

Use correct array lengths in Eq. Jacobian

Fix bug in Jacobian.

Fix unused variable/parameter errors

OPFLOW: rework solution callback for RAJA/HIOP GPU-based solver

Formatting changes

* Add unit test for RAJA/HiOp Sparse GPU model (9-bus only)

* Apply pre-commmit fixes

* Add test for 200-bus case

* Apply pre-commmit fixes

---------

Co-authored-by: Abhyankar, Shrirang G <[email protected]>

include/common.h

@@ -7,6 +7,7 @@
 #define COMMON_H
 
 #include <petsc.h>
+#include <exago_config.h>
 
 typedef enum { MATPOWER, CSV, JSON, MINIMAL } OutputFormat;
 

include/opflow.h

@@ -16,11 +16,13 @@
 #define OPFLOWMODEL_PBPOLHIOP "POWER_BALANCE_HIOP"
 #define OPFLOWMODEL_PBPOLRAJAHIOP "PBPOLRAJAHIOP"
 #define OPFLOWMODEL_DCOPF "DCOPF"
+#define OPFLOWMODEL_PBPOLRAJAHIOPSPARSE "PBPOLRAJAHIOPSPARSE"
 
 /* Solvers */
 #define OPFLOWSOLVER_IPOPT "IPOPT"
 #define OPFLOWSOLVER_HIOP "HIOP"
 #define OPFLOWSOLVER_HIOPSPARSE "HIOPSPARSE"
+#define OPFLOWSOLVER_HIOPSPARSEGPU "HIOPSPARSEGPU"
 
 typedef struct _p_OPFLOW *OPFLOW;
 

src/opflow/CMakeLists.txt

@@ -11,13 +11,22 @@ set(OPFLOW_FORM_SRC
 
 # Build RAJA/Umpire modules if RAJA and Umpire are enabled
 if(EXAGO_ENABLE_RAJA)
-  set(OPFLOW_FORM_SRC ${OPFLOW_FORM_SRC} model/power_bal_hiop/pbpolrajahiop.cpp
-                      model/power_bal_hiop/pbpolrajahiopkernels.cpp
+  set(OPFLOW_FORM_SRC
+      ${OPFLOW_FORM_SRC} model/power_bal_hiop/pbpolrajahiop.cpp
+      model/power_bal_hiop/pbpolrajahiopkernels.cpp
+      model/power_bal_hiop/paramsrajahiop.cpp
   )
+  if(EXAGO_ENABLE_HIOP_SPARSE)
+    set(OPFLOW_FORM_SRC
+        ${OPFLOW_FORM_SRC} model/power_bal_hiop/pbpolrajahiopsparse.cpp
+        model/power_bal_hiop/pbpolrajahiopsparsekernels.cpp
+    )
+  endif()
 endif()
 
-set(OPFLOW_SOLVER_SRC solver/ipopt/opflow_ipopt.cpp solver/hiop/opflow_hiop.cpp
-                      solver/hiop/opflow_hiopsparse.cpp
+set(OPFLOW_SOLVER_SRC
+    solver/ipopt/opflow_ipopt.cpp solver/hiop/opflow_hiop.cpp
+    solver/hiop/opflow_hiopsparse.cpp solver/hiop/opflow_hiopsparsegpu.cpp
 )
 
 set(OPFLOW_SRC
@@ -29,7 +38,9 @@ set_source_files_properties(${OPFLOW_SRC} PROPERTIES LANGUAGE CXX)
 
 if(EXAGO_ENABLE_RAJA AND EXAGO_ENABLE_CUDA)
   set_source_files_properties(
-    model/power_bal_hiop/pbpolrajahiopkernels.cpp PROPERTIES LANGUAGE CUDA
+    model/power_bal_hiop/pbpolrajahiopkernels.cpp
+    model/power_bal_hiop/pbpolrajahiopsparsekernels.cpp PROPERTIES LANGUAGE
+                                                                   CUDA
   )
 endif()
 

src/opflow/interface/opflow.cpp

@@ -3008,6 +3008,11 @@ PetscErrorCode OPFLOWCheckModelSolverCompatibility(OPFLOW opflow) {
 #else
   PetscBool rajahiop_pbpol = PETSC_FALSE;
 #endif // RAJA
+  if (hiop && !(hiop_pbpol || rajahiop_pbpol)) {
+    SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP,
+            "OPFLOW solver HIOP incompatible with model %s",
+            (opflow->modelname).c_str());
+  }
 #if defined(EXAGO_ENABLE_HIOP_SPARSE)
   PetscBool hiop_sparse;
   PetscBool hiop_sparse_pbpol;
@@ -3022,14 +3027,22 @@ PetscErrorCode OPFLOWCheckModelSolverCompatibility(OPFLOW opflow) {
             "OPFLOW solver HIOPSPARSE incompatible with model %s",
             (opflow->modelname).c_str());
   }
-#else
-  PetscBool hiop_sparse = PETSC_FALSE;
-#endif // HIOP_SPARSE
-  if ((hiop && !hiop_sparse) && !(hiop_pbpol || rajahiop_pbpol)) {
+#if defined(EXAGO_ENABLE_RAJA)
+  PetscBool hiop_sparsegpu;
+  PetscBool pbpolrajahiopsparse;
+
+  hiop_sparsegpu =
+      static_cast<PetscBool>(opflow->solvername == OPFLOWSOLVER_HIOPSPARSEGPU);
+  pbpolrajahiopsparse = static_cast<PetscBool>(opflow->modelname ==
+                                               OPFLOWMODEL_PBPOLRAJAHIOPSPARSE);
+
+  if (hiop_sparsegpu && !pbpolrajahiopsparse) {
     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP,
-            "OPFLOW solver HIOP incompatible with model %s",
+            "OPFLOW solver HIOPSPARSE incompatible with model %s",
             (opflow->modelname).c_str());
   }
+#endif // EXAGO_ENABLE_RAJA
+#endif // HIOP_SPARSE
   /* FIXME: The PBPOLRAJAHIOP has trouble with individual load
      shedding, so avoid using when load shedding is enabled */
   if (opflow->include_loadloss_variables) {

src/opflow/interface/opflowregi.cpp

@@ -76,6 +76,7 @@ extern PetscErrorCode OPFLOWModelCreate_PBPOLHIOP(OPFLOW);
 
 #if defined(EXAGO_ENABLE_RAJA)
 extern PetscErrorCode OPFLOWModelCreate_PBPOLRAJAHIOP(OPFLOW);
+extern PetscErrorCode OPFLOWModelCreate_PBPOLRAJAHIOPSPARSE(OPFLOW);
 #endif
 
 /*
@@ -114,6 +115,9 @@ PetscErrorCode OPFLOWModelRegisterAll(OPFLOW opflow) {
   ierr = OPFLOWModelRegister(opflow, OPFLOWMODEL_PBPOLRAJAHIOP,
                              OPFLOWModelCreate_PBPOLRAJAHIOP);
   CHKERRQ(ierr);
+  ierr = OPFLOWModelRegister(opflow, OPFLOWMODEL_PBPOLRAJAHIOPSPARSE,
+                             OPFLOWModelCreate_PBPOLRAJAHIOPSPARSE);
+  CHKERRQ(ierr);
 #endif
 
   opflow->OPFLOWModelRegisterAllCalled = PETSC_TRUE;
@@ -128,6 +132,9 @@ extern PetscErrorCode OPFLOWSolverCreate_IPOPT(OPFLOW);
 extern PetscErrorCode OPFLOWSolverCreate_HIOP(OPFLOW);
 #if defined(EXAGO_ENABLE_HIOP_SPARSE)
 extern PetscErrorCode OPFLOWSolverCreate_HIOPSPARSE(OPFLOW);
+#if defined(EXAGO_ENABLE_RAJA)
+extern PetscErrorCode OPFLOWSolverCreate_HIOPSPARSEGPU(OPFLOW);
+#endif
 #endif
 #endif
 
@@ -153,6 +160,11 @@ PetscErrorCode OPFLOWSolverRegisterAll(OPFLOW opflow) {
   ierr = OPFLOWSolverRegister(opflow, OPFLOWSOLVER_HIOPSPARSE,
                               OPFLOWSolverCreate_HIOPSPARSE);
   CHKERRQ(ierr);
+#if defined(EXAGO_ENABLE_RAJA)
+  ierr = OPFLOWSolverRegister(opflow, OPFLOWSOLVER_HIOPSPARSEGPU,
+                              OPFLOWSolverCreate_HIOPSPARSEGPU);
+  CHKERRQ(ierr);
+#endif
 #endif
 #endif
   opflow->OPFLOWSolverRegisterAllCalled = PETSC_TRUE;