address PR comments

.github/workflows/test.yml

@@ -33,7 +33,7 @@ jobs:
       - run: python3 -m build
       - run: python3 -m pip install dist/crfm_helm-*.whl
       - run: helm-run --run-specs simple1:model=simple/model1 --max-eval-instances 10 --suite test
-      - run: helm-summarize --schema-file schema_classic.yaml --suite test
+      - run: helm-summarize --suite test
       - run: helm-server --help
 
   test:

src/helm-frontend/src/services/getSchema.ts

@@ -1,22 +1,16 @@
-import { parse } from "yaml";
-
 import type Schema from "@/types/Schema";
-import getBenchmarkSuite from "@/utils/getBenchmarkSuite";
-import getBenchmarkRelease from "@/utils/getBenchmarkRelease";
+import getBenchmarkEndpoint from "@/utils/getBenchmarkEndpoint";
+import getVersionBaseUrl from "@/utils/getVersionBaseUrl";
+
+export function getSchemaJsonUrl(): string {
+  return getBenchmarkEndpoint(`${getVersionBaseUrl()}/schema.json`);
+}
 
 export default async function getSchema(signal: AbortSignal): Promise<Schema> {
   try {
-    // TODO: this should not be hard-coded to the main website, want to be able
-    // to still run things locally.
-    const resp = await fetch(
-      `https://crfm.stanford.edu/helm/${
-        getBenchmarkRelease() || getBenchmarkSuite()
-      }/schema.json`,
-      { signal },
-    );
+    const resp = await fetch(getSchemaJsonUrl(), { signal });
     const data = await resp.text();
-    const schema = parse(data) as Schema;
-
+    const schema = JSON.parse(data) as Schema;
     return schema;
   } catch (error) {
     console.log(error);

src/helm/benchmark/presentation/schema.py

@@ -11,6 +11,7 @@
 from helm.benchmark.augmentations.perturbation_description import PERTURBATION_WORST
 
 
+# TODO: change to `helm.benchmark.config`
 SCHEMA_YAML_PACKAGE: str = "helm.benchmark.static"
 
 # TODO: add heim, vhelm, etc.
@@ -248,6 +249,7 @@ def __post_init__(self):
 
 
 def read_schema(filename: str) -> Schema:
+    # TODO: merge in model metadata from `model_metadata.yaml`
     schema_path = resources.files(SCHEMA_YAML_PACKAGE).joinpath(filename)
     hlog(f"Reading schema file {schema_path}...")
     with schema_path.open("r") as f:

src/helm/benchmark/presentation/summarize.py

@@ -46,6 +46,7 @@
     MetricNameMatcher,
     RunGroup,
     read_schema,
+    SCHEMA_CLASSIC_YAML_FILENAME,
     BY_GROUP,
     THIS_GROUP_ONLY,
     NO_GROUPS,
@@ -1300,7 +1301,7 @@ def main():
         "--schema-file",
         type=str,
         help="File name of the schema to read (e.g., schema_classic.yaml).",
-        required=True,
+        default=SCHEMA_CLASSIC_YAML_FILENAME,
     )
     parser.add_argument(
         "--suite",