Skip to content

Commit

Permalink
use the build system to build cpython instead of calling it from the …
Browse files Browse the repository at this point in the history
…shell
  • Loading branch information
Rexicon226 committed Jun 12, 2024
1 parent 9c13250 commit 0561b03
Show file tree
Hide file tree
Showing 34 changed files with 1,131 additions and 1,773 deletions.
8 changes: 5 additions & 3 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ jobs:
- name: Setup Zig
uses: goto-bus-stop/setup-zig@v2
with:
version: 0.12.0-dev.3076+6e078883e
version: 0.13.0

- name: Setup Python3.10
uses: actions/setup-python@v2
Expand All @@ -27,7 +27,8 @@ jobs:
with:
path: |
~/.cache/zig
zig-cache
.zig-cache
zig-out/lib
key: osmium-${{hashFiles('build.zig.zon')}}

- name: Run Tests
Expand All @@ -38,5 +39,6 @@ jobs:
with:
path: |
~/.cache/zig
zig-cache
.zig-cache
zig-out/lib
key: osmium-${{hashFiles('build.zig.zon')}}
2 changes: 1 addition & 1 deletion .gitignore
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
zig-cache/
.zig-cache/
zig-out/
temp/
.vscode/
Expand Down
674 changes: 674 additions & 0 deletions LICENSE

Large diffs are not rendered by default.

83 changes: 62 additions & 21 deletions build.zig
Original file line number Diff line number Diff line change
Expand Up @@ -2,33 +2,31 @@ const std = @import("std");

const cases = @import("tests/cases.zig");

var trace: ?bool = false;
var trace: bool = false;
var @"enable-bench": ?bool = false;
var backend: TraceBackend = .None;

pub fn build(b: *std.Build) !void {
const target = b.standardTargetOptions(.{});
const query = b.standardTargetOptionsQueryOnly(.{});
const optimize = b.standardOptimizeOption(.{});

// we don't support building cpython to another platform yet
if (!query.isNative()) {
@panic("cross-compilation isn't allowed");
}

const exe = b.addExecutable(.{
.name = "osmium",
.root_source_file = .{ .path = "src/main.zig" },
.target = target,
.root_source_file = b.path("src/main.zig"),
.target = b.graph.host,
.optimize = optimize,
});

// Deps
const std_extras = b.addModule("std-extras", .{
.root_source_file = .{ .path = "src/std-extra/std.zig" },
});

exe.root_module.addImport("std-extras", std_extras);

trace = b.option(bool, "trace",
\\Enables tracing of the compiler using the default backend (spall)
);
) orelse false;

if (trace) |_| {
if (trace) {
backend = b.option(TraceBackend, "trace-backend",
\\Switch between what backend to use. None is default.
) orelse backend;
Expand All @@ -46,8 +44,7 @@ pub fn build(b: *std.Build) !void {
exe.use_lld = use_llvm;

const exe_options = b.addOptions();

exe_options.addOption(bool, "trace", trace orelse false);
exe_options.addOption(bool, "trace", trace);
exe_options.addOption(TraceBackend, "backend", backend);
exe_options.addOption(std.log.Level, "debug_log", debug_log);
exe_options.addOption(usize, "src_file_trimlen", std.fs.path.dirname(std.fs.path.dirname(@src().file).?).?.len);
Expand All @@ -56,8 +53,16 @@ pub fn build(b: *std.Build) !void {

const tracer_dep = b.dependency("tracer", .{});
exe.root_module.addImport("tracer", tracer_dep.module("tracer"));
// exe.linkLibC(); // Needs libc.

const cpython_step = b.step("cpython", "Builds libcpython for the host");
const cpython_path = try generateLibPython(b, cpython_step, optimize);

exe.step.dependOn(cpython_step);
exe.linkLibC();
exe.addObjectFile(cpython_path);

const cpython_install = b.addInstallFile(cpython_path, "lib/libpython3.10.a");
b.getInstallStep().dependOn(&cpython_install.step);
b.installArtifact(exe);

const run_cmd = b.addRunArtifact(exe);
Expand All @@ -72,7 +77,7 @@ pub fn build(b: *std.Build) !void {

// Generate steps
const opcode_step = b.step("opcode", "Generate opcodes");
generateOpCode(b, opcode_step, target);
generateOpCode(b, opcode_step);

// Test cases
const test_step = b.step("test", "Test Osmium");
Expand All @@ -88,19 +93,55 @@ const TraceBackend = enum {
fn generateOpCode(
b: *std.Build,
step: *std.Build.Step,
target: std.Build.ResolvedTarget,
) void {
const translator = b.addExecutable(.{
.name = "opcode2zig",
.root_source_file = .{ .path = "./tools/opcode2zig.zig" },
.target = target,
.root_source_file = b.path("tools/opcode2zig.zig"),
.target = b.graph.host,
.optimize = .ReleaseFast,
});

const run_cmd = b.addRunArtifact(translator);

run_cmd.addArg("includes/opcode.h");
run_cmd.addArg("vendor/opcode.h");
run_cmd.addArg("src/compiler/opcodes.zig");

step.dependOn(&run_cmd.step);
}

fn generateLibPython(
b: *std.Build,
step: *std.Build.Step,
optimize: std.builtin.OptimizeMode,
) !std.Build.LazyPath {
const source = b.dependency("python", .{});

// TODO: cache properly
const maybe_lib_path = try b.build_root.join(b.allocator, &.{ "zig-out", "lib", "libpython3.10.a" });
const result = if (std.fs.accessAbsolute(maybe_lib_path, .{})) true else |_| false;
if (result) {
return b.path("zig-out/lib/libpython3.10.a");
}

const configure_run = std.Build.Step.Run.create(b, "cpython-configure");
configure_run.setCwd(source.path("."));
configure_run.addFileArg(source.path("configure"));
configure_run.addArgs(&.{
"--disable-shared",
if (optimize == .Debug) "" else "--enable-optimizations",
});

const make_run = std.Build.Step.Run.create(b, "cpython-make");
make_run.setCwd(source.path("."));
make_run.addArgs(&.{
"make", b.fmt("-j{d}", .{cpu: {
const cpu_set = try std.posix.sched_getaffinity(0);
break :cpu std.posix.CPU_COUNT(cpu_set);
}}),
});

make_run.step.dependOn(&configure_run.step);
step.dependOn(&make_run.step);

return source.path("libpython3.10.a");
}
8 changes: 6 additions & 2 deletions build.zig.zon
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,12 @@
.paths = .{""},
.dependencies = .{
.tracer = .{
.url = "https://github.com/Rexicon226/zig-tracer/archive/8c24ad8b1767c874c926417ddccc83981d66aedb.tar.gz",
.hash = "122012ae514d30f7304fd4e9668e6b8a20e0fa5db28c72b32b3f64e5945e5a1cd8a1",
.url = "https://github.com/Rexicon226/zig-tracer/archive/f0c24a3e0ecf232493ab2fadc65f06e48956ccba.tar.gz",
.hash = "1220857ddca6c829d4b68f35e929b32e4eee13c265bc096c634ae8b3d6da7daf34df",
},
.python = .{
.url = "https://github.com/python/cpython/archive/333c7dccd87c637d0b15cf81f9bbec28e39664fd.tar.gz",
.hash = "1220c520b358bd5e0bbcfae04b2e9e963ad3c3d7cc3b5ba24a0721e60a0123bfb1ea",
},
},
}
7 changes: 0 additions & 7 deletions demo/print_ast.py
Original file line number Diff line number Diff line change
@@ -1,7 +0,0 @@

import marshal

filename = './demo/test.py'
with open(filename, 'r') as f:
bytes = marshal.load(f)

190 changes: 18 additions & 172 deletions demo/show_pyc.py
Original file line number Diff line number Diff line change
@@ -1,177 +1,23 @@
# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0
# For details: https://github.com/nedbat/coveragepy/blob/master/NOTICE.txt

"""
Dump the contents of a .pyc file.
The output will only be correct if run with the same version of Python that
produced the .pyc.
"""

import binascii
import dis
import marshal
import struct
import sys
import time
import types


def show_pyc_file(fname):
f = open(fname, "rb")
magic = f.read(4)
print("magic %s" % (binascii.hexlify(magic)))
read_date_and_size = True
flags = struct.unpack('<L', f.read(4))[0]
hash_based = bool(flags & 0x01)
check_source = bool(flags & 0x02)
print(f"flags {flags:#08x}")
if hash_based:
source_hash = f.read(8)
read_date_and_size = False
print(f"hash {binascii.hexlify(source_hash)}")
print(f"check_source {check_source}")
if read_date_and_size:
moddate = f.read(4)
modtime = time.asctime(time.localtime(struct.unpack('<L', moddate)[0]))
print(f"moddate {binascii.hexlify(moddate)} ({modtime})")
size = f.read(4)
print("pysize %s (%d)" % (binascii.hexlify(size), struct.unpack('<L', size)[0]))
code = marshal.load(f)
show_code(code)

def show_py_file(fname):
text = open(fname).read().replace('\r\n', '\n')
show_py_text(text, fname=fname)

def show_py_text(text, fname="<string>"):
code = compile(text, fname, "exec", dont_inherit=True)
show_code(code)

CO_FLAGS = [
('CO_OPTIMIZED', 0x00001),
('CO_NEWLOCALS', 0x00002),
('CO_VARARGS', 0x00004),
('CO_VARKEYWORDS', 0x00008),
('CO_NESTED', 0x00010),
('CO_GENERATOR', 0x00020),
('CO_NOFREE', 0x00040),
('CO_COROUTINE', 0x00080),
('CO_ITERABLE_COROUTINE', 0x00100),
('CO_ASYNC_GENERATOR', 0x00200),
('CO_GENERATOR_ALLOWED', 0x01000),
]

if sys.version_info < (3, 9):
CO_FLAGS += [
('CO_FUTURE_DIVISION', 0x02000),
('CO_FUTURE_ABSOLUTE_IMPORT', 0x04000),
('CO_FUTURE_WITH_STATEMENT', 0x08000),
('CO_FUTURE_PRINT_FUNCTION', 0x10000),
('CO_FUTURE_UNICODE_LITERALS', 0x20000),
('CO_FUTURE_BARRY_AS_BDFL', 0x40000),
('CO_FUTURE_GENERATOR_STOP', 0x80000),
]
else:
CO_FLAGS += [
('CO_FUTURE_DIVISION', 0x0020000),
('CO_FUTURE_ABSOLUTE_IMPORT', 0x0040000),
('CO_FUTURE_WITH_STATEMENT', 0x0080000),
('CO_FUTURE_PRINT_FUNCTION', 0x0100000),
('CO_FUTURE_UNICODE_LITERALS', 0x0200000),
('CO_FUTURE_BARRY_AS_BDFL', 0x0400000),
('CO_FUTURE_GENERATOR_STOP', 0x0800000),
('CO_FUTURE_ANNOTATIONS', 0x1000000),
]


def show_code(code, indent='', number=None):
label = ""
if number is not None:
label = "%d: " % number
print(f"{indent}{label}code")
indent += " "
print(f"{indent}name {code.co_name!r}")
print("%sargcount %d" % (indent, code.co_argcount))
print("%snlocals %d" % (indent, code.co_nlocals))
print("%sstacksize %d" % (indent, code.co_stacksize))
print(f"{indent}flags {code.co_flags:04x}: {flag_words(code.co_flags, CO_FLAGS)}")
show_hex("code", code.co_code, indent=indent)
dis.disassemble(code)
print("%sconsts" % indent)
for i, const in enumerate(code.co_consts):
if type(const) == types.CodeType:
show_code(const, indent+" ", number=i)
else:
print(" %s%d: %r" % (indent, i, const))
print(f"{indent}names {code.co_names!r}")
print(f"{indent}varnames {code.co_varnames!r}")
print(f"{indent}freevars {code.co_freevars!r}")
print(f"{indent}cellvars {code.co_cellvars!r}")
print(f"{indent}filename {code.co_filename!r}")
print("%sfirstlineno %d" % (indent, code.co_firstlineno))
show_hex("lnotab", code.co_lnotab, indent=indent)
print(" {}{}".format(indent, ", ".join(f"{line!r}:{byte!r}" for byte, line in lnotab_interpreted(code))))
if hasattr(code, "co_linetable"):
show_hex("linetable", code.co_linetable, indent=indent)
if hasattr(code, "co_lines"):
print(" {}co_lines {}".format(
indent,
", ".join(f"{line!r}:{start!r}-{end!r}" for start, end, line in code.co_lines())
))

def show_hex(label, h, indent):
h = binascii.hexlify(h)
if len(h) < 60:
print("{}{} {}".format(indent, label, h.decode('ascii')))
else:
print(f"{indent}{label}")
for i in range(0, len(h), 60):
print("{} {}".format(indent, h[i:i+60].decode('ascii')))


def lnotab_interpreted(code):
# Adapted from dis.py in the standard library.
byte_increments = code.co_lnotab[0::2]
line_increments = code.co_lnotab[1::2]

last_line_num = None
line_num = code.co_firstlineno
byte_num = 0
for byte_incr, line_incr in zip(byte_increments, line_increments):
if byte_incr:
if line_num != last_line_num:
yield (byte_num, line_num)
last_line_num = line_num
byte_num += byte_incr
if line_incr >= 0x80:
line_incr -= 0x100
line_num += line_incr
if line_num != last_line_num:
yield (byte_num, line_num)

def flag_words(flags, flag_defs):
words = []
for word, flag in flag_defs:
if flag & flags:
words.append(word)
return ", ".join(words)

def show_file(fname):
if fname.endswith('pyc'):
show_pyc_file(fname)
elif fname.endswith('py'):
show_py_file(fname)
else:
print("Odd file:", fname)

def main(args):
if args[0] == '-c':
show_py_text(" ".join(args[1:]).replace(";", "\n"))
else:
for a in args:
show_file(a)

if __name__ == '__main__':
main(sys.argv[1:])
def disassemble_pyc(filename):
with open(filename, 'rb') as f:
# Read the magic number and timestamp/header
magic = f.read(4)
timestamp = f.read(4)
if sys.version_info >= (3, 7):
# Python 3.7+ includes the size of the source file in the header
size = f.read(4)
code = marshal.load(f)
dis.dis(code)

if __name__ == "__main__":
if len(sys.argv) != 2:
print("Usage: python disassemble_pyc.py <path_to_pyc_file>")
sys.exit(1)

pyc_file = sys.argv[1]
disassemble_pyc(pyc_file)
Loading

0 comments on commit 0561b03

Please sign in to comment.