CSV for "any parking reform" handles reform status (#686)

We've been realizing that the Reform Status is actually quite tricky and
that the modeling was off for "any parking reform". We were including
repealed and proposed policies inside "any parking reform"—despite the
CSV/dataset not supporting "reform status"—which improperly suggested
that some places that were only proposals or repeals actually did have
parking reform.

"Any parking reform" is tricky because it is place-specific rather than
policy record-specific. We cannot put a "reform status" column in the
"any parking reform" sheet because it wouldn't be clear which policy
record the status corresponds to.

Instead, the correct approach is to split out three CSVs:
`overview-passed.csv`, `overview-proposed.csv`, and
`overview-repealed.csv`.

data/generated/README.md

@@ -4,9 +4,11 @@ This folder contains the data used for https://parkingreform.org/resources/manda
 
 ## CSV files
 
-There are four CSV files:
+There are six CSV files:
 
-- `any_parking_reform.csv`: overview of which reforms a place has implemented
+- `overview_passed.csv`: an overview of all places that have passed reforms
+- `overview_proposed.csv`: an overview of all places that have proposed new reforms, but not yet passed them
+- `overview_repealed.csv`: an overview of all places that have repealed prior reforms
 - `add_maximums.csv`: parking maximum policies
 - `reduce_minimums.csv`: parking minimum reductions
 - `remove_minimums.csv`: parking minimum removals

scripts/generateDataSet.ts

@@ -1,5 +1,6 @@
 /* eslint-disable import/no-extraneous-dependencies */
 /* eslint-disable no-console */
+/* eslint-disable @typescript-eslint/no-explicit-any */
 
 import fs from "fs/promises";
 
@@ -12,6 +13,7 @@ import {
   ProcessedCompletePolicy,
   readProcessedCompleteData,
 } from "./lib/data";
+import { ReformStatus } from "../src/js/types";
 
 const DELIMITER = "; ";
 
@@ -50,31 +52,88 @@ function createLegacyCsv(data: ProcessedCompleteEntry[]): string {
   return csv;
 }
 
-export function createAnyPolicyCsv(data: ProcessedCompleteEntry[]): string {
-  const entries = data.map((entry) => ({
-    place: entry.place.name,
-    state: entry.place.state,
-    country: entry.place.country,
-    place_type: entry.place.type,
-    population: entry.place.pop,
-    lat: entry.place.coord[1],
-    long: entry.place.coord[0],
-    all_minimums_repealed: toBoolean(entry.place.repeal),
-    has_minimums_repeal: toBoolean(!!entry.rm_min?.length),
-    has_minimums_reduction: toBoolean(!!entry.reduce_min?.length),
-    has_maximums: toBoolean(!!entry.add_max?.length),
-    prn_url: entry.place.url,
-  }));
-  const csv = Papa.unparse(entries);
+interface AnyPolicySet {
+  hasReforms: boolean;
+  csvValues: {
+    minimums_removal: string;
+    minimums_reduction: string;
+    maximums: string;
+  };
+}
 
-  // Validate expected number of entries.
-  const numJson = data.length;
-  const numCsv = csv.split("\r\n").length - 1;
-  if (numJson !== numCsv) {
-    throw new Error(`CSV has unequal entries to JSON: ${numCsv} vs ${numJson}`);
-  }
+function determineAnyPolicySet(
+  entry: ProcessedCompleteEntry,
+  status: ReformStatus,
+): AnyPolicySet {
+  const hasRm =
+    entry.rm_min?.some((policy) => policy.status === status) ?? false;
+  const hasReduce =
+    entry.reduce_min?.some((policy) => policy.status === status) ?? false;
+  const hasMax =
+    entry.add_max?.some((policy) => policy.status === status) ?? false;
+  return {
+    hasReforms: hasRm || hasReduce || hasMax,
+    csvValues: {
+      minimums_removal: toBoolean(hasRm),
+      minimums_reduction: toBoolean(hasReduce),
+      maximums: toBoolean(hasMax),
+    },
+  };
+}
 
-  return csv;
+export function createAnyPolicyCsvs(data: ProcessedCompleteEntry[]): {
+  passed: string;
+  proposed: string;
+  repealed: string;
+} {
+  const passed: any[] = [];
+  const proposed: any[] = [];
+  const repealed: any[] = [];
+  data.forEach((entry) => {
+    const initialValues = {
+      place: entry.place.name,
+      state: entry.place.state,
+      country: entry.place.country,
+      place_type: entry.place.type,
+      population: entry.place.pop,
+      lat: entry.place.coord[1],
+      long: entry.place.coord[0],
+    };
+    const prnUrl = { prn_url: entry.place.url };
+
+    const passedPolicySet = determineAnyPolicySet(entry, "passed");
+    const proposedPolicySet = determineAnyPolicySet(entry, "proposed");
+    const repealedPolicySet = determineAnyPolicySet(entry, "repealed");
+
+    if (passedPolicySet.hasReforms) {
+      passed.push({
+        ...initialValues,
+        all_minimums_removed: toBoolean(entry.place.repeal),
+        ...passedPolicySet.csvValues,
+        ...prnUrl,
+      });
+    }
+    if (proposedPolicySet.hasReforms) {
+      proposed.push({
+        ...initialValues,
+        ...proposedPolicySet.csvValues,
+        ...prnUrl,
+      });
+    }
+    if (repealedPolicySet.hasReforms) {
+      repealed.push({
+        ...initialValues,
+        ...repealedPolicySet.csvValues,
+        ...prnUrl,
+      });
+    }
+  });
+
+  return {
+    passed: Papa.unparse(passed),
+    proposed: Papa.unparse(proposed),
+    repealed: Papa.unparse(repealed),
+  };
 }
 
 export function createReformCsv(
@@ -94,7 +153,7 @@ export function createReformCsv(
       place_type: entry.place.type,
       lat: entry.place.coord[1],
       long: entry.place.coord[0],
-      all_minimums_repealed: toBoolean(entry.place.repeal),
+      all_minimums_removed: toBoolean(entry.place.repeal),
       status: policy.status,
       reform_date: policy.date?.raw,
       scope: policy.scope.join(DELIMITER),
@@ -140,8 +199,10 @@ async function main(): Promise<void> {
   const legacy = createLegacyCsv(data);
   await writeCsv(legacy, "map/data.csv");
 
-  const anyPolicy = createAnyPolicyCsv(data);
-  await writeCsv(anyPolicy, "data/generated/any_parking_reform.csv");
+  const { passed, proposed, repealed } = createAnyPolicyCsvs(data);
+  await writeCsv(passed, "data/generated/overview_passed.csv");
+  await writeCsv(proposed, "data/generated/overview_proposed.csv");
+  await writeCsv(repealed, "data/generated/overview_repealed.csv");
 
   const addMax = createReformCsv(data, (entry) => entry.add_max);
   await writeCsv(addMax, "data/generated/add_maximums.csv");

tests/scripts/generateDataSet.test.ts

@@ -1,7 +1,7 @@
 import { expect, test } from "@playwright/test";
 
 import {
-  createAnyPolicyCsv,
+  createAnyPolicyCsvs,
   createReformCsv,
 } from "../../scripts/generateDataSet";
 import type {
@@ -14,6 +14,10 @@ import { Date } from "../../src/js/types";
 // This test uses snapshot testing (https://jestjs.io/docs/snapshot-testing#updating-snapshots). If the tests fail and the changes
 // are valid, run `npm test -- --updateSnapshot`.
 
+function normalize(csv: string): string {
+  return csv.replace(/\r\n/g, "\n");
+}
+
 // eslint-disable-next-line no-empty-pattern
 test("generate CSVs", async ({}, testInfo) => {
   // Normally, Playwright saves the operating system name in the snapshot results.
@@ -93,7 +97,7 @@ test("generate CSVs", async ({}, testInfo) => {
       rm_min: [
         {
           summary: "Remove minimums",
-          status: "passed",
+          status: "proposed",
           scope: [],
           land: [],
           requirements: [],
@@ -104,12 +108,11 @@ test("generate CSVs", async ({}, testInfo) => {
       ],
     },
   ];
-  const anyPolicy = createAnyPolicyCsv(entries).replace(/\r\n/g, "\n");
-  expect(anyPolicy).toMatchSnapshot("any-reform.csv");
+  const { passed, proposed, repealed } = createAnyPolicyCsvs(entries);
+  expect(normalize(passed)).toMatchSnapshot("overview-passed.csv");
+  expect(normalize(proposed)).toMatchSnapshot("overview-proposed.csv");
+  expect(normalize(repealed)).toMatchSnapshot("overview-repealed.csv");
 
-  const maximums = createReformCsv(entries, (entry) => entry.add_max).replace(
-    /\r\n/g,
-    "\n",
-  );
-  expect(maximums).toMatchSnapshot("maximums.csv");
+  const maximums = createReformCsv(entries, (entry) => entry.add_max);
+  expect(normalize(maximums)).toMatchSnapshot("maximums.csv");
 });

tests/scripts/generateDataSet.test.ts-snapshots/maximums.csv

@@ -1,3 +1,3 @@
-place,state,country,population,place_type,lat,long,all_minimums_repealed,status,reform_date,scope,land_uses,requirements,summary,num_citations,reporter,prn_url
+place,state,country,population,place_type,lat,long,all_minimums_removed,status,reform_date,scope,land_uses,requirements,summary,num_citations,reporter,prn_url
 My City,NY,US,24104,city,14.23,44.23,TRUE,passed,2022-02-13,citywide,commercial; other,by right,Maximums summary #1,2,Donald Shoup,https://parkingreform.org/my-city-details.html
 My City,NY,US,24104,city,14.23,44.23,TRUE,repealed,,regional,other,,Maximums summary #2,1,Donald Shoup,https://parkingreform.org/my-city-details.html

tests/scripts/generateDataSet.test.ts-snapshots/overview-passed.csv

@@ -0,0 +1,2 @@
+place,state,country,place_type,population,lat,long,all_minimums_removed,minimums_removal,minimums_reduction,maximums,prn_url
+My City,NY,US,city,24104,14.23,44.23,TRUE,FALSE,FALSE,TRUE,https://parkingreform.org/my-city-details.html

tests/scripts/generateDataSet.test.ts-snapshots/overview-proposed.csv

@@ -0,0 +1,2 @@
+place,state,country,place_type,population,lat,long,minimums_removal,minimums_reduction,maximums,prn_url
+Another Place,CA,US,county,414,24.23,80.3,TRUE,FALSE,FALSE,https://parkingreform.org/another-place.html

tests/scripts/generateDataSet.test.ts-snapshots/overview-repealed.csv

@@ -0,0 +1,2 @@
+place,state,country,place_type,population,lat,long,minimums_removal,minimums_reduction,maximums,prn_url
+My City,NY,US,city,24104,14.23,44.23,FALSE,FALSE,TRUE,https://parkingreform.org/my-city-details.html