re-do the testing framework

.github/workflows/ci.yml

@@ -16,11 +16,6 @@ jobs:
         uses: goto-bus-stop/setup-zig@v2
         with:
           version: 0.13.0
-
-      - name: Setup Python3.10
-        uses: actions/setup-python@v2
-        with:
-          python-version: '3.10'
 
       - name: Restore cache
         uses: actions/cache/restore@v3

build.zig

@@ -57,8 +57,8 @@ pub fn build(b: *std.Build) !void {
     exe_options.addOption(usize, "src_file_trimlen", std.fs.path.dirname(std.fs.path.dirname(@src().file).?).?.len);
     exe_options.addOption(bool, "enable_debug_extensions", enable_debug_extensions);
     exe_options.addOption(bool, "build_debug", enable_debug);
-    exe_options.addOption([]const u8, "lib_path", "../python/Lib");
     exe.root_module.addOptions("options", exe_options);
+    exe_options.addOption([]const u8, "lib_path", b.fmt("{s}/python/Lib", .{b.install_path}));
 
     const tracer_dep = b.dependency("tracer", .{ .optimize = optimize, .target = target });
     const libgc_dep = b.dependency("libgc", .{ .optimize = optimize, .target = target });
@@ -74,20 +74,26 @@ pub fn build(b: *std.Build) !void {
         }
     }
 
+    const libpython_install = b.addInstallDirectory(.{
+        .source_dir = cpython_dep.builder.dependency("python", .{}).path("Lib"),
+        .install_dir = .{ .custom = "python" },
+        .install_subdir = "Lib",
+    });
+    exe.step.dependOn(&libpython_install.step);
+
     const run_cmd = b.addRunArtifact(exe);
     run_cmd.step.dependOn(b.getInstallStep());
 
     if (b.args) |args| run_cmd.addArgs(args);
-
     const run_step = b.step("run", "Run the app");
     run_step.dependOn(&run_cmd.step);
 
     const opcode_step = b.step("opcode", "Generate opcodes");
     generateOpCode(b, opcode_step);
 
     const test_step = b.step("test", "Test Osmium");
-    try cases.addCases(b, exe, test_step);
-    // test_step.dependOn(&libpython_install.step);
+    try cases.addCases(b, target, test_step, exe, cpython_dep.artifact("cpython"));
+    test_step.dependOn(&libpython_install.step);
 }
 
 const TraceBackend = enum {

build.zig.zon

@@ -13,9 +13,6 @@
             .url = "https://github.com/Rexicon226/zig-tracer/archive/6400e066bb3849be4906f3b6e124d6c8c5a5083f.tar.gz",
             .hash = "122057453dba0b0cf6f07610d692cc7ab3f1d66defa150c936b398144812564a0c08",
         },
-        .cpython = .{
-            .path = "../zig-cpython",
-        },
         .libgc = .{
             .url = "https://github.com/Rexicon226/zig-libgc/archive/1ea12e140ec1471af5d15dae71b225764c9747ba.tar.gz",
             .hash = "12209ad7282d9422973afe3e468e2f867bb102d04fa446a1d1c94f900bda94b7c68e",
@@ -25,5 +22,9 @@
             .hash = "12206f2265ca2262f919ec125ff7fe15fa6f3be622806e211d7a9bd3055ba51a0908",
             .lazy = true,
         },
+        .cpython = .{
+            .url = "https://github.com/Rexicon226/zig-cpython/archive/969708680122dc4548d64adfae041261e2af480c.tar.gz",
+            .hash = "1220240c0fd3b1b062483c3f9f41b610181cdf70a47e047ae6379247178b20c34024",
+        },
     },
 }

src/frontend/Python.zig

@@ -87,7 +87,7 @@ pub fn Initialize(
     _ = externs.PyConfig_Read(&config);
 
     const utf32_path = try utf8ToUtf32Z(
-        "/home/dr/Zython/osmium/zig-out/python/Lib",
+        build_options.lib_path,
         allocator,
     );
 

tests/cases.zig

@@ -3,7 +3,6 @@
 // SPDX-License-Identifier: GPL-3.0-only
 
 const std = @import("std");
-const matrix = @import("matrix.zig");
 
 const Build = std.Build;
 const Step = Build.Step;
@@ -14,8 +13,81 @@ const test_dirs: []const []const u8 = &.{
     "behaviour",
 };
 
-pub fn addCases(b: *Build, exe: *Step.Compile, parent_step: *Step) !void {
+pub fn addCases(
+    b: *Build,
+    target: std.Build.ResolvedTarget,
+    parent_step: *Step,
+    osmium: *Step.Compile,
+    python: *Step.Compile,
+) !void {
+    parent_step.dependOn(&python.step);
+
+    const compare_tool = b.addExecutable(.{
+        .name = "compare",
+        .root_source_file = b.path("tools/compare.zig"),
+        .target = target,
+        .optimize = .ReleaseSafe,
+        .omit_frame_pointer = false, // we need the stack trace
+    });
+
     for (test_dirs) |dir| {
-        parent_step.dependOn(try matrix.addCases(b, dir, exe));
+        const files = try getPyFilesInDir(b, b.fmt("tests/{s}", .{dir}), b.allocator);
+        for (files) |file| {
+            parent_step.dependOn(addCase(b, file, osmium, python, compare_tool));
+        }
+    }
+}
+
+fn addCase(
+    b: *std.Build,
+    file_path: []const u8,
+    osmium: *std.Build.Step.Compile,
+    python: *std.Build.Step.Compile,
+    compare: *std.Build.Step.Compile,
+) *std.Build.Step {
+    const python_run = b.addRunArtifact(python);
+    python_run.addArg(file_path);
+    const lib_path = b.fmt("{s}/python/Lib", .{b.install_path});
+    python_run.setEnvironmentVariable("PYTHONHOME", lib_path);
+    python_run.setEnvironmentVariable("PYTHONPATH", lib_path);
+    const python_stdout = python_run.captureStdOut();
+
+    const osmium_run = b.addRunArtifact(osmium);
+    osmium_run.addArg(file_path);
+    const osmium_stdout = osmium_run.captureStdOut();
+
+    const compare_run = b.addRunArtifact(compare);
+    compare_run.addFileArg(osmium_stdout);
+    compare_run.addFileArg(python_stdout);
+    compare_run.expectExitCode(0);
+    compare_run.setName(file_path);
+
+    return &compare_run.step;
+}
+
+pub fn getPyFilesInDir(
+    b: *std.Build,
+    dir_path: []const u8,
+    allocator: std.mem.Allocator,
+) ![]const []const u8 {
+    var files = std.ArrayList([]const u8).init(allocator);
+    defer files.deinit();
+
+    var dir = try b.build_root.handle.openDir(dir_path, .{ .iterate = true });
+    var it = dir.iterate();
+    while (try it.next()) |file| {
+        if (file.kind != .file) {
+            continue;
+        }
+        if (!std.mem.endsWith(u8, file.name, ".py")) {
+            continue;
+        }
+        try files.append(try std.mem.concat(allocator, u8, &.{
+            dir_path,
+            &.{std.fs.path.sep},
+            file.name,
+        }));
     }
+
+    return files.toOwnedSlice();
 }

tools/compare.zig

@@ -0,0 +1,24 @@
+const std = @import("std");
+const assert = std.debug.assert;
+
+pub fn main() !void {
+    var gpa = std.heap.GeneralPurposeAllocator(.{}){};
+    defer _ = gpa.deinit();
+    const allocator = gpa.allocator();
+
+    const process_args = try std.process.argsAlloc(allocator);
+    defer std.process.argsFree(allocator, process_args);
+    assert(process_args.len == 3);
+
+    const osmium_path = process_args[1];
+    const python_path = process_args[2];
+
+    const osmium_contents = try std.fs.cwd().readFileAlloc(allocator, osmium_path, 10 * 1024);
+    const python_contents = try std.fs.cwd().readFileAlloc(allocator, python_path, 10 * 1024);
+    defer {
+        allocator.free(osmium_contents);
+        allocator.free(python_contents);
+    }
+
+    try std.testing.expectEqualSlices(u8, python_contents, osmium_contents);
+}