implement input, int, and __import__'s fromlist (#9)

* implement `input`, `int`, `__import__`'s `fromlist`

* fix-up almost all the the leaks

* organize the marshal a bit better

build.zig

@@ -32,9 +32,7 @@ pub fn build(b: *std.Build) !void {
         \\Uses llvm to compile Osmium. Default true.
     ) orelse true;
 
-    const debug_log = b.option(std.log.Level, "debug-log",
-        \\Enable debug logging.
-    ) orelse .info;
+    const enable_logging = b.option(bool, "log", "Enable debug logging.") orelse false;
 
     const enable_debug_extensions = b.option(
         bool,
@@ -48,7 +46,7 @@ pub fn build(b: *std.Build) !void {
     const exe_options = b.addOptions();
     exe_options.addOption(bool, "trace", trace);
     exe_options.addOption(TraceBackend, "backend", backend);
-    exe_options.addOption(std.log.Level, "debug_log", debug_log);
+    exe_options.addOption(bool, "enable_logging", enable_logging);
     exe_options.addOption(usize, "src_file_trimlen", std.fs.path.dirname(std.fs.path.dirname(@src().file).?).?.len);
     exe_options.addOption(bool, "enable_debug_extensions", enable_debug_extensions);
     exe.root_module.addOptions("options", exe_options);

examples/calculator.py

@@ -0,0 +1,8 @@
+from math import add
+
+in1 = input("Input 1st number: ")
+in2 = input("Input 2nd number: ")
+
+result = add(int(in1), int(in2))
+
+print("Result: ", result)

examples/math.py

@@ -0,0 +1,2 @@
+def add(x, y) -> int:
+    return x + y

methods.py

@@ -0,0 +1,4 @@
+def a():
+    print(1)
+a()
+

src/compiler/CodeObject.zig

@@ -51,7 +51,9 @@ pub fn deinit(co: *CodeObject, allocator: std.mem.Allocator) void {
     }
     allocator.free(co.varnames);
 
-    if (co.instructions) |insts| allocator.free(insts);
+    if (co.instructions) |insts| {
+        allocator.free(insts);
+    }
 
     co.* = undefined;
 }
@@ -134,27 +136,21 @@ pub fn process(
     co: *CodeObject,
     allocator: std.mem.Allocator,
 ) !void {
-    var instructions = std.ArrayList(Instruction).init(allocator);
-
     const bytes = co.code;
+    const num_instructions = bytes.len / 2;
+    const instructions = try allocator.alloc(Instruction, num_instructions);
 
-    var cursor: u32 = 0;
-    while (cursor < bytes.len) {
+    var cursor: usize = 0;
+    for (0..num_instructions) |i| {
         const byte = bytes[cursor];
-        const op: OpCode = @enumFromInt(byte);
-
-        const has_arg = byte >= 90;
-
-        const inst: Instruction = .{
-            .op = op,
-            .extra = if (has_arg) bytes[cursor + 1] else undefined,
+        instructions[i] = .{
+            .op = @enumFromInt(byte),
+            .extra = if (byte >= 90) bytes[cursor + 1] else 0,
         };
-        try instructions.append(inst);
         cursor += 2;
-        continue;
     }
 
-    co.instructions = try instructions.toOwnedSlice();
+    co.instructions = instructions;
     co.index = 0;
 }
 

src/compiler/Instruction.zig

@@ -3,6 +3,7 @@
 const Instruction = @This();
 const std = @import("std");
 const CodeObject = @import("CodeObject.zig");
+const Object = @import("../vm/Object.zig");
 
 op: OpCode,
 extra: u8,
@@ -59,6 +60,13 @@ fn format2(
         => try writer.print("{d} ({s})", .{ extra, co.getName(extra) }),
         .CALL_FUNCTION,
         => try writer.print("{d}", .{extra}),
+        .MAKE_FUNCTION,
+        => {
+            const ty: Object.Payload.PythonFunction.ArgType = @enumFromInt(extra);
+            try writer.print("{s}", .{@tagName(ty)});
+        },
+        .BUILD_TUPLE,
+        => try writer.print("({d})", .{extra}),
         else => try writer.print("TODO payload {d}", .{extra}),
     }
 }

src/compiler/Marshal.zig

@@ -13,6 +13,8 @@ const readInt = std.mem.readInt;
 
 const Error = error{} || std.mem.Allocator.Error;
 
+const log = std.log.scoped(.marshal);
+
 const PyLong_SHIFT = 15;
 
 const PythonVersion = struct { major: u8, minor: u8 };
@@ -32,7 +34,10 @@ cursor: usize,
 bytes: []const u8,
 allocator: std.mem.Allocator,
 
-pub fn init(allocator: std.mem.Allocator, input_bytes: []const u8) !Marshal {
+pub fn init(
+    allocator: std.mem.Allocator,
+    input_bytes: []const u8,
+) !Marshal {
     const version = Marshal.getVersion(input_bytes[0..4].*);
     const head_size = switch (version.minor) {
         10 => 16,
@@ -58,8 +63,51 @@ pub fn parse(marshal: *Marshal) !CodeObject {
     const t = tracer.trace(@src(), "", .{});
     defer t.end();
 
-    const obj = try marshal.readObject();
-    return obj.get(.codeobject).*;
+    var co_obj = try marshal.readObject();
+    const co = co_obj.get(.codeobject);
+    defer co_obj.deinit(marshal.allocator);
+    return co.clone(marshal.allocator);
+}
+
+fn readSingleString(marshal: *Marshal) ![]const u8 {
+    var next_byte = marshal.bytes[marshal.cursor];
+    marshal.cursor += 1;
+
+    const allocator = marshal.allocator;
+
+    var ref_id: ?usize = null;
+    if (testBit(next_byte, 7)) {
+        next_byte = clearBit(next_byte, 7);
+        ref_id = marshal.flag_refs.items.len;
+        try marshal.flag_refs.append(allocator, null);
+    }
+
+    const ty: ObjType = @enumFromInt(next_byte);
+    log.debug("readSingleString {s}", .{@tagName(ty)});
+    const string: []u8 = switch (ty) {
+        .TYPE_SHORT_ASCII_INTERNED,
+        .TYPE_SHORT_ASCII,
+        => try marshal.readString(.{ .short = true }),
+        .TYPE_STRING => try marshal.readString(.{}),
+        .TYPE_REF => ref: {
+            const index = marshal.readLong(false);
+            try marshal.references.append(allocator, .{ .byte = marshal.cursor, .index = index });
+            marshal.flag_refs.items[index].?.usages += 1;
+            const ref_obj = marshal.flag_refs.items[index].?.content;
+            const ref_string = ref_obj.get(.string);
+            break :ref try allocator.dupe(u8, ref_string);
+        },
+        else => std.debug.panic("TODO: readSingleString {s}", .{@tagName(ty)}),
+    };
+
+    if (ref_id) |id| {
+        marshal.flag_refs.items[id] = .{
+            .byte = marshal.cursor,
+            .content = try Object.create(.string, allocator, string),
+        };
+    }
+
+    return string;
 }
 
 fn readObject(marshal: *Marshal) Error!Object {
@@ -78,10 +126,14 @@ fn readObject(marshal: *Marshal) Error!Object {
     }
 
     const ty: ObjType = @enumFromInt(next_byte);
+    log.debug("readObject {s}", .{@tagName(ty)});
     const object: Object = switch (ty) {
         .TYPE_NONE => Object.init(.none),
+        .TYPE_CODE => code: {
+            const code = try marshal.readCodeObject();
+            break :code try Object.create(.codeobject, allocator, code);
+        },
 
-        .TYPE_CODE => try Object.create(.codeobject, allocator, try marshal.readCodeObject()),
         .TYPE_STRING => try Object.create(.string, allocator, try marshal.readString(.{})),
 
         .TYPE_SHORT_ASCII_INTERNED,
@@ -110,7 +162,7 @@ fn readObject(marshal: *Marshal) Error!Object {
             const index = marshal.readLong(false);
             try marshal.references.append(allocator, .{ .byte = marshal.cursor, .index = index });
             marshal.flag_refs.items[index].?.usages += 1;
-            break :ref marshal.flag_refs.items[index].?.content;
+            break :ref try marshal.flag_refs.items[index].?.content.clone(allocator);
         },
         else => std.debug.panic("TODO: marshal.readObject {s}", .{@tagName(ty)}),
     };
@@ -135,16 +187,11 @@ fn readCodeObject(marshal: *Marshal) Error!CodeObject {
         .nlocals = marshal.readLong(false),
         .stacksize = marshal.readLong(false),
         .flags = marshal.readLong(false),
-        .code = code: {
-            var code_obj = try marshal.readObject();
-            const string = (try code_obj.getOwnedPayload(.string, allocator));
-            break :code try allocator.dupe(u8, string);
-        },
+        .code = try marshal.readSingleString(),
         .consts = try marshal.readObject(),
         .names = try marshal.readObject(),
         .varnames = varnames: {
-            const varnames_tuple = (try marshal.readObject()).get(.tuple);
-            const varnames = try allocator.dupe(Object, varnames_tuple);
+            const varnames = (try marshal.readObject()).get(.tuple);
             break :varnames varnames;
         },
         .filename = blk: {
@@ -154,15 +201,9 @@ fn readCodeObject(marshal: *Marshal) Error!CodeObject {
             var cellvars = try marshal.readObject();
             cellvars.deinit(allocator);
 
-            var filename_obj = try marshal.readObject();
-            const string = (try filename_obj.getOwnedPayload(.string, allocator));
-            break :blk try allocator.dupe(u8, string);
-        },
-        .name = name: {
-            var name_obj = try marshal.readObject();
-            const string = (try name_obj.getOwnedPayload(.string, allocator));
-            break :name try allocator.dupe(u8, string);
+            break :blk try marshal.readSingleString();
         },
+        .name = try marshal.readSingleString(),
         .firstlineno = marshal.readLong(false),
     };
 

src/crash_report.zig

@@ -119,10 +119,16 @@ pub fn prepVmContext(current_co: CodeObject) VmContext {
     } else .{};
 }
 
+var buffer: [10 * 1024]u8 = undefined;
+
 fn dumpStatusReport() !void {
+    var fba = std.heap.FixedBufferAllocator.init(&buffer);
+    const allocator = fba.allocator();
     const state = vm_state orelse return;
     const stderr = io.getStdErr().writer();
-    const current_co = state.current_co;
+
+    var current_co = state.current_co;
+    try current_co.process(allocator);
 
     try print_co.print_co(stderr, .{
         .co = current_co,