separate results and dumps

checkpoint/persist.py

@@ -56,15 +56,15 @@ class Loader:
     def __init__(self, module_filename: str | pathlib.Path, env) -> None:
         module_filename = pathlib.Path(module_filename)
         tag = module_filename.parent.name
-        dumps_path = pathlib.Path("dumps")
+        results_path = pathlib.Path("results")
 
         # make sure the cache is invalidated when the module changes
         h = compute_hash(module_filename, env)
         cachedir = pathlib.Path(tempfile.gettempdir()) / f"pythia-{h}"
         print(f"Using cache directory {cachedir}", file=sys.stderr)
         cachedir.mkdir(parents=False, exist_ok=True)
         self.filename = cachedir / "store.pickle"
-        self.tsv_filename = (dumps_path / tag / module_filename.stem).with_suffix(
+        self.tsv_filename = (results_path / tag / module_filename.stem).with_suffix(
             ".tsv"
         )
         print(self.tsv_filename, file=sys.stderr)
@@ -114,7 +114,7 @@ def commit(self, *args) -> None:
             pathlib.Path(temp_filename).rename(self.filename)
 
             with open(self.tsv_filename, "a") as f:
-                size = pickle.dumps((self.i, args, self.iterator)).__sizeof__()
+                size = pickle.results((self.i, args, self.iterator)).__sizeof__()
                 self.printing_index += 1
                 print(
                     self.printing_index,

save_snapshot.py

@@ -96,7 +96,8 @@ def __next__(self) -> int:
         return self.i
 
 
-def count_diff(folder: str, i: int) -> int:
+def count_diff(folder: pathlib.Path, i: int) -> int:
+    folder = folder.as_posix()
     result = subprocess.run(
         [
             "./count_diff",
@@ -112,46 +113,49 @@ def count_diff(folder: str, i: int) -> int:
     return int(result.stdout.strip())
 
 
-async def relay_qmp_dumps(qmp_port: int, server: Server) -> None:
+async def relay_qmp_results(qmp_port: int, server: Server) -> None:
     tag = server.tag
     assert tag.replace("_", "").isalnum()
-    folder = f"{pathlib.Path.cwd().as_posix()}/dumps/{tag}_vm"
-    shutil.rmtree(folder, ignore_errors=True)
-    os.makedirs(folder, exist_ok=False)
+    cwd = pathlib.Path.cwd()
+    dumps_folder = cwd / "dumps" / tag
+    shutil.rmtree(dumps_folder, ignore_errors=True)
+    os.makedirs(dumps_folder, exist_ok=False)
     async with SimpleQmpClient(qmp_port) as vm:
-        await vm.dump(f"{folder}/0.a.dump")
+        await vm.dump(f"{dumps_folder}/0.a.dump")
         with ThreadPoolExecutor() as executor:
             ps: list[Future[int]] = []
             for i, index in enumerate(server):
                 async with vm.pause(server.sleep_duration_ms):
-                    current_next_file = f"{folder}/{i}.b.dump"
-                    next_prev_file = f"{folder}/{i + 1}.a.dump"
+                    current_next_file = dumps_folder / f"{i}.b.dump"
+                    next_prev_file = dumps_folder / f"{i + 1}.a.dump"
                     await vm.dump(current_next_file)
                     os.link(current_next_file, next_prev_file)
-                    p: Future[int] = executor.submit(count_diff, folder, i)
+                    p: Future[int] = executor.submit(count_diff, dumps_folder, i)
                     if p is not None:
                         ps.append(p)
             else:
                 server.finish()
                 os.unlink(next_prev_file)
 
-            with open(f"{folder}.tsv", "w") as f:
+            results_folder = cwd / "results" / tag
+            results_folder.mkdir(exist_ok=True, parents=True)
+            with open(results_folder / f"{tag}_vm.tsv", "w") as f:
                 for i, p in enumerate(ps):
                     print(i, p.result(), sep="\t", file=f, flush=True)
 
-    os.rmdir(folder)
+    os.rmdir(dumps_folder)
     logging.info("Done.")
 
 
 def run_server(qmp_port: int, tcp_port: int):
     with SimpleTcpServer(tcp_port) as server:
-        asyncio.run(relay_qmp_dumps(qmp_port, server))
+        asyncio.run(relay_qmp_results(qmp_port, server))
 
 
 def run_iterator(qmp_port: int, iterations: int, sleep_duration_ms: int, tag: str):
     assert iterations > 0
     server = IteratorServer(iterations, sleep_duration_ms, tag)
-    asyncio.run(relay_qmp_dumps(qmp_port, server))
+    asyncio.run(relay_qmp_results(qmp_port, server))
 
 
 def main():
@@ -179,7 +183,7 @@ def main():
         type=int,
         help="The number of milliseconds between snapshots.",
     )
-    iterator_parser.add_argument("tag", type=str, help="Save as ./dumps/[tag].csv.")
+    iterator_parser.add_argument("tag", type=str, help="Save as ./results/[tag].csv.")
     iterator_parser.set_defaults(func=run_iterator)
 
     args = parser.parse_args()

scripts/vm.sh

@@ -9,6 +9,7 @@ if [ -z "$EXPERIMENT" ]; then
   echo "Usage: $0 <experiment> [QMP_PORT] [TCP_PORT] [STEP]"
   exit 1
 fi
+mkdir -f results/${EXPERIMENT}
 
 STEP=${2:-1}
 QMP_PORT=${3:-4444}
@@ -64,7 +65,7 @@ package_upgrade: false
 mounts:
  - [${EXPERIMENT_TAG}, /mnt/${EXPERIMENT_TAG}, 9p]
  - [${CHECKPOINT_LIB}, /mnt/${CHECKPOINT_LIB}, 9p]
- - [dumps, /mnt/dumps, 9p]
+ - [results, /mnt/results, 9p]
 
 write_files:
   - path: ${GUEST_HOME}/.bashrc
@@ -74,7 +75,7 @@ write_files:
     append: true
     content: |+
       export PYTHONPATH=/mnt
-      export DUMPS=/mnt/dumps
+      export RESULTS=/mnt/results
       export EXPERIMENT=${EXPERIMENT}
       export STEP=${STEP}
       source ${VENV_BIN}/activate
@@ -83,9 +84,9 @@ write_files:
     owner: ubuntu:ubuntu
     defer: true
     content: |+
-      cat ${EXPERIMENT_PATH}/args.txt | xargs python ${EXPERIMENT_PATH}/naive.py
-      cat ${EXPERIMENT_PATH}/args.txt | xargs python ${EXPERIMENT_PATH}/instrumented.py
-      cat ${EXPERIMENT_PATH}/args.txt | xargs python ${EXPERIMENT_PATH}/vm.py
+      cat ${EXPERIMENT}/args.txt | xargs python ${EXPERIMENT}/naive.py
+      cat ${EXPERIMENT}/args.txt | xargs python ${EXPERIMENT}/instrumented.py
+      cat ${EXPERIMENT}/args.txt | xargs python ${EXPERIMENT}/vm.py
 
 runcmd:
   - sudo chown -R ubuntu:ubuntu ${GUEST_HOME}
@@ -105,7 +106,7 @@ args=(
   -drive "file=${user_data},format=qcow2"
   -virtfs local,path=${EXPERIMENT_BASE},mount_tag=${EXPERIMENT_TAG},security_model=none
   -virtfs local,path=./${CHECKPOINT_LIB},mount_tag=${CHECKPOINT_LIB},security_model=none
-  -virtfs local,path="dumps",mount_tag=dumps,security_model=mapped
+  -virtfs local,path="results",mount_tag="results",security_model=mapped
   -enable-kvm
   -m 2G
 #  -serial mon:stdio  # use console for monitor