Re-balance CI tests so the entire job finishes quicker.

.github/workflows/config.yml

@@ -8,7 +8,7 @@ on:
   workflow_dispatch:
 
 jobs:
-  run-unit-tests:
+  run-ruleset-tests:
     runs-on: ubuntu-latest
     container:
       image: docker://nrel/openstudio:3.7.0
@@ -33,9 +33,9 @@ jobs:
           git config --global --add safe.directory $(realpath .)
           git diff --exit-code --ignore-cr-at-eol workflow/*.xml
 
-      - name: Run measure unit tests
+      - name: Run ruleset tests
         run: |
-          bundle exec rake test_measures
+          bundle exec rake test_rulesets
 
       - name: Store code coverage
         uses: actions/upload-artifact@v3
@@ -54,7 +54,7 @@ jobs:
           name: documentation
           path: docs/_build/html/
 
-  run-eri-tests:
+  run-sample-files1-tests:
     runs-on: ubuntu-latest
     container:
       image: docker://nrel/openstudio:3.7.0
@@ -67,17 +67,17 @@ jobs:
         run: |
           rm -f Gemfile.lock && bundle install
 
-      - name: Run ERI tests
+      - name: Run Sample Files 1 tests
         run: |
-          bundle exec rake test_eri
+          bundle exec rake test_sample_files1
 
       - name: Store results
         uses: actions/upload-artifact@v3
         with:
           path: workflow/tests/test_results
-          name: test_results_eri
+          name: test_results_sample_files1
 
-  run-real-home-tests:
+  run-sample-files2-tests:
     runs-on: ubuntu-latest
     container:
       image: docker://nrel/openstudio:3.7.0
@@ -90,17 +90,17 @@ jobs:
         run: |
           rm -f Gemfile.lock && bundle install
 
-      - name: Run Real Home tests
+      - name: Run Sample Files 2 tests
         run: |
-          bundle exec rake test_real_homes
+          bundle exec rake test_sample_files2
 
       - name: Store results
         uses: actions/upload-artifact@v3
         with:
           path: workflow/tests/test_results
-          name: test_results_real_homes
+          name: test_results_sample_files2
 
-  run-resnet-tests:
+  run-real-home-tests:
     runs-on: ubuntu-latest
     container:
       image: docker://nrel/openstudio:3.7.0
@@ -113,17 +113,17 @@ jobs:
         run: |
           rm -f Gemfile.lock && bundle install
 
-      - name: Run RESNET tests
+      - name: Run Real Home tests
         run: |
-          bundle exec rake test_resnet
+          bundle exec rake test_real_homes
 
       - name: Store results
         uses: actions/upload-artifact@v3
         with:
           path: workflow/tests/test_results
-          name: test_results_resnet
+          name: test_results_real_homes
 
-  run-es-zerh-tests:
+  run-other-tests:
     runs-on: ubuntu-latest
     container:
       image: docker://nrel/openstudio:3.7.0
@@ -136,15 +136,15 @@ jobs:
         run: |
           rm -f Gemfile.lock && bundle install
 
-      - name: Run ES & ZERH tests
+      - name: Run Other tests
         run: |
-          bundle exec rake test_es_zerh
+          bundle exec rake test_other
 
       - name: Store results
         uses: actions/upload-artifact@v3
         with:
           path: workflow/tests/test_results
-          name: test_results_es_zerh
+          name: test_results_other
 
   run-windows-tests:
     runs-on: windows-latest
@@ -164,7 +164,7 @@ jobs:
   compare-results:
     if: github.event_name == 'pull_request'
     runs-on: ubuntu-latest
-    needs: [run-eri-tests, run-real-home-tests, run-resnet-tests, run-es-zerh-tests]
+    needs: [run-sample-files1-tests, run-sample-files2-tests, run-real-home-tests, run-other-tests]
     steps:
       - uses: actions/checkout@v3
         with:
@@ -187,33 +187,33 @@ jobs:
             base_results
           name: base_results
 
-      - name: Download ERI feature results
+      - name: Download Sample Files 1 feature results
         uses: actions/download-artifact@v3
         with:
           path: |
             results
-          name: test_results_eri
+          name: test_results_sample_files1
 
-      - name: Download Real Home feature results
+      - name: Download Sample Files 2 feature results
         uses: actions/download-artifact@v3
         with:
           path: |
             results
-          name: test_results_real_homes
+          name: test_results_sample_files2
 
-      - name: Download RESNET feature results
+      - name: Download Real Home feature results
         uses: actions/download-artifact@v3
         with:
           path: |
             results
-          name: test_results_resnet
+          name: test_results_real_homes
 
-      - name: Download ES & ZERH feature results
+      - name: Download Other feature results
         uses: actions/download-artifact@v3
         with:
           path: |
             results
-          name: test_results_es_zerh
+          name: test_results_other
 
       - name: Compare results
         run: |
@@ -237,39 +237,39 @@ jobs:
 
   update-results:
     runs-on: ubuntu-latest
-    needs: [run-eri-tests, run-real-home-tests, run-resnet-tests, run-es-zerh-tests]
+    needs: [run-sample-files1-tests, run-sample-files2-tests, run-real-home-tests, run-other-tests]
     steps:
       - uses: actions/checkout@v3
         with:
           ref: ${{ github.head_ref }}
 
-      - name: Download ERI feature results
+      - name: Download Sample Files 1 feature results
         uses: actions/download-artifact@v3
         with:
           path: |
             results
-          name: test_results_eri
+          name: test_results_sample_files1
 
-      - name: Download Real Home feature results
+      - name: Download Sample Files 2 feature results
         uses: actions/download-artifact@v3
         with:
           path: |
             results
-          name: test_results_real_homes
+          name: test_results_sample_files2
 
-      - name: Download RESNET feature results
+      - name: Download Real Home feature results
         uses: actions/download-artifact@v3
         with:
           path: |
             results
-          name: test_results_resnet
+          name: test_results_real_homes
 
-      - name: Download ES & ZERH feature results
+      - name: Download Other feature results
         uses: actions/download-artifact@v3
         with:
           path: |
             results
-          name: test_results_es_zerh
+          name: test_results_other
 
       - name: Commit latest results
         shell: bash        

Rakefile

@@ -13,37 +13,37 @@ Rake::TestTask.new('test_all') do |t|
   t.verbose = true
 end
 
-desc 'Run measure unit tests'
-Rake::TestTask.new('test_measures') do |t|
+desc 'Run ruleset tests'
+Rake::TestTask.new('test_rulesets') do |t|
   t.test_files = Dir['rulesets/tests/*.rb']
   t.warning = false
   t.verbose = true
 end
 
-desc 'Run ERI tests'
-Rake::TestTask.new('test_eri') do |t|
-  t.test_files = Dir['workflow/tests/energy_rating_index_test.rb']
+desc 'Run Sample Files 1 tests'
+Rake::TestTask.new('test_sample_files1') do |t|
+  t.test_files = Dir['workflow/tests/sample_files1_test.rb']
   t.warning = false
   t.verbose = true
 end
 
-desc 'Run Real Home tests'
-Rake::TestTask.new('test_real_homes') do |t|
-  t.test_files = Dir['workflow/tests/real_homes_test.rb']
+desc 'Run Sample Files 2 tests'
+Rake::TestTask.new('test_sample_files2') do |t|
+  t.test_files = Dir['workflow/tests/sample_files2_test.rb']
   t.warning = false
   t.verbose = true
 end
 
-desc 'Run RESNET tests'
-Rake::TestTask.new('test_resnet') do |t|
-  t.test_files = Dir['workflow/tests/resnet*test.rb']
+desc 'Run Real Home tests'
+Rake::TestTask.new('test_real_homes') do |t|
+  t.test_files = Dir['workflow/tests/real_homes_test.rb']
   t.warning = false
   t.verbose = true
 end
 
-desc 'Run ES & ZERH tests'
-Rake::TestTask.new('test_es_zerh') do |t|
-  t.test_files = Dir['workflow/tests/es_zerh_test.rb']
+desc 'Run Other tests'
+Rake::TestTask.new('test_other') do |t|
+  t.test_files = Dir['workflow/tests/*test.rb'] - Dir['workflow/tests/real_homes_test.rb'] - Dir['workflow/tests/sample_files*test.rb']
   t.warning = false
   t.verbose = true
 end

workflow/tests/real_homes_test.rb

@@ -12,7 +12,7 @@
 require_relative '../../hpxml-measures/HPXMLtoOpenStudio/resources/hpxml'
 require_relative '../../hpxml-measures/HPXMLtoOpenStudio/resources/xmlhelper'
 
-class EnergyRatingIndexTest < Minitest::Test
+class RealHomesTest < Minitest::Test
   def setup
     @test_results_dir = File.join(File.dirname(__FILE__), 'test_results')
     FileUtils.mkdir_p @test_results_dir

workflow/tests/sample_files1_test.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require_relative '../../hpxml-measures/HPXMLtoOpenStudio/resources/minitest_helper'
+require 'openstudio'
+require 'openstudio/measure/ShowRunnerOutput'
+require 'fileutils'
+require 'csv'
+require 'oga'
+require_relative 'util.rb'
+require_relative '../../rulesets/resources/constants'
+require_relative '../../hpxml-measures/HPXMLtoOpenStudio/resources/constants'
+require_relative '../../hpxml-measures/HPXMLtoOpenStudio/resources/hpxml'
+require_relative '../../hpxml-measures/HPXMLtoOpenStudio/resources/xmlhelper'
+
+class SampleFilesTest1 < Minitest::Test
+  def setup
+    @test_results_dir = File.join(File.dirname(__FILE__), 'test_results')
+    FileUtils.mkdir_p @test_results_dir
+    @test_files_dir = File.join(File.dirname(__FILE__), 'test_files')
+    FileUtils.mkdir_p @test_files_dir
+  end
+
+  def test_sample_files
+    test_name = 'sample_files1'
+    test_results_csv = File.absolute_path(File.join(@test_results_dir, "#{test_name}.csv"))
+    File.delete(test_results_csv) if File.exist? test_results_csv
+
+    # Run simulations
+    files = 'base*.xml'
+    all_results = {}
+    xmldir = "#{File.dirname(__FILE__)}/../sample_files"
+    Dir["#{xmldir}/#{files}"].sort.each do |xml|
+      break if xml.include? 'base-hvac-air-to-air-heat-pump-1-speed.xml' # Run first half of the sample files
+
+      rundir, _hpxmls, csvs = _run_workflow(xml, test_name)
+      all_results[File.basename(xml)] = _get_csv_results([csvs[:eri_results],
+                                                          csvs[:co2e_results],
+                                                          csvs[:es_results],
+                                                          csvs[:zerh_results],
+                                                          csvs[:iecc_eri_results]])
+
+      _rm_path(rundir)
+    end
+    assert(all_results.size > 0)
+
+    # Write results to csv
+    keys = []
+    all_results.values.each do |xml_results|
+      xml_results.keys.each do |key|
+        next if keys.include? key
+
+        keys << key
+      end
+    end
+    CSV.open(test_results_csv, 'w') do |csv|
+      csv << ['XML'] + keys
+      all_results.each do |xml, results|
+        csv_line = [File.basename(xml)]
+        keys.each do |key|
+          csv_line << results[key]
+        end
+        csv << csv_line
+      end
+    end
+    puts "Wrote results to #{test_results_csv}."
+  end
+end