diff --git a/WORKSPACE b/WORKSPACE index cb756c92..8ac94e63 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -13,9 +13,13 @@ load("//zig:repositories.bzl", "rules_zig_dependencies", "zig_register_toolchain # Fetch dependencies which users need as well rules_zig_dependencies() +# Use the latest known Zig SDK version for testing +# buildifier: disable=bzl-visibility +load("//zig/private:versions.bzl", "TOOL_VERSIONS") + zig_register_toolchains( name = "zig", - zig_version = "0.10.1", + zig_version = TOOL_VERSIONS.keys()[0], ) # For running our own unit tests diff --git a/util/BUILD.bazel b/util/BUILD.bazel index f8d1aa41..11334c29 100644 --- a/util/BUILD.bazel +++ b/util/BUILD.bazel @@ -54,3 +54,13 @@ py_binary( data = ["@buildifier_prebuilt//:buildozer"], deps = ["@rules_python//python/runfiles"], ) + +py_binary( + name = "update_zig_versions", + srcs = ["update_zig_versions.py"], + args = [ + "--output", + "$(rootpath //zig/private:versions.bzl)", + ], + data = ["//zig/private:versions.bzl"], +) diff --git a/util/update_zig_versions.py b/util/update_zig_versions.py new file mode 100755 index 00000000..4fd1aa82 --- /dev/null +++ b/util/update_zig_versions.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 + +import requests +import base64 +import argparse + + +_ZIG_INDEX_URL = "https://ziglang.org/download/index.json" + +_UNSUPPORTED_VERSIONS = [ + "0.9.1", + "0.9.0", + "0.8.1", + "0.8.0", + "0.7.1", + "0.7.0", + "0.6.0", + "0.5.0", + "0.4.0", + "0.3.0", + "0.2.0", + "0.1.1"] + +_SUPPORTED_PLATFORMS = [ + "aarch64-linux", + "aarch64-macos", + "aarch64-windows", + "x86_64-linux", + "x86_64-macos", + "x86_64-windows", + "x86-linux", + "x86-windows"] + + +def fetch_zig_versions(url): + response = requests.get(url) + response.raise_for_status() + return response.json() + + +def convert_sha256(sha256_hex): + return "sha256-" + base64.b64encode(bytes.fromhex(sha256_hex)).decode() + + +_HEADER = '''\ +"""Mirror of Zig release info. + +Generated from {url}. +""" +''' + + +_PLATFORM = '''\ + "{platform}": struct( + url = "{url}", + integrity = "{integrity}", + ),\ +''' + + +def _parse_semver(version_str): + """Split a semantic version into its components. + + Raises an error if the version is malformed. + + If the version contains no pre-release component, then a sentinel of + `0x10FFFF` is returned. The intent is that it sorts higher than any other + code-point, therefore making versions without pre-release component sort + higher than this with. + + If the version is the string `master` then it returns a maximum version + comprising `float("inf")` components and the pre-release sentinel. + + Returns: + (major, minor, patch, pre_release) + """ + max_component = float("inf") + max_prerelease = chr(0x10FFFF) # Highest valid code point in Unicode + + if version_str == "master": + return max_component, max_component, max_component, max_prerelease + + pre_version, *_ = version_str.split("+", maxsplit=1) + main_version, *pre_release = pre_version.split("-", maxsplit=1) + major, minor, patch = map(int, main_version.split(".")) + + pre_release_segment = pre_release[0] if pre_release else max_prerelease + + return major, minor, patch, pre_release_segment + + +def generate_bzl_content(url, data, unsupported_versions, supported_platforms): + content = [_HEADER.format(url = url)] + content.append("TOOL_VERSIONS = {") + + for version, platforms in sorted(data.items(), key=lambda x: _parse_semver(x[0]), reverse=True): + if version in unsupported_versions or version == "master": + continue + + content.append(' "{}": {{'.format(version)) + + for platform, info in sorted(platforms.items()): + if platform not in supported_platforms or not isinstance(info, dict): + continue + content.append(_PLATFORM.format( + platform = platform, + url = info["tarball"], + integrity = convert_sha256(info["shasum"]) + )) + + content.append(' },') + + content.append('}') + + return '\n'.join(content) + + +def main(): + parser = argparse.ArgumentParser(description="Generate Starlark file for Zig compiler versions.") + parser.add_argument("--output", type=argparse.FileType('w'), default='-', help="Output file path or '-' for stdout.") + parser.add_argument("--url", default=_ZIG_INDEX_URL, help="URL to fetch Zig versions JSON") + parser.add_argument("--unsupported-versions", nargs="*", default=_UNSUPPORTED_VERSIONS, help="List of unsupported Zig versions") + parser.add_argument("--supported-platforms", nargs="*", default=_SUPPORTED_PLATFORMS, help="List of supported platforms") + args = parser.parse_args() + + zig_data = fetch_zig_versions(args.url) + bzl_content = generate_bzl_content(args.url, zig_data, set(args.unsupported_versions), set(args.supported_platforms)) + + args.output.write(bzl_content) + + +if __name__ == "__main__": + main() diff --git a/zig/private/BUILD.bazel b/zig/private/BUILD.bazel index edeab06b..0ab8a1aa 100644 --- a/zig/private/BUILD.bazel +++ b/zig/private/BUILD.bazel @@ -1,5 +1,10 @@ load("@bazel_skylib//:bzl_library.bzl", "bzl_library") +exports_files( + ["versions.bzl"], + visibility = ["//util:__pkg__"], +) + bzl_library( name = "zig_package", srcs = ["zig_package.bzl"], diff --git a/zig/private/toolchains_repo.bzl b/zig/private/toolchains_repo.bzl index 48e81e48..a88d8c38 100644 --- a/zig/private/toolchains_repo.bzl +++ b/zig/private/toolchains_repo.bzl @@ -20,18 +20,42 @@ with only the toolchain attribute pointing into the platform-specific repositori # Add more platforms as needed to mirror all the binaries # published by the upstream project. PLATFORMS = { - "linux-x86_64": struct( + "aarch64-linux": struct( + compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:aarch64", + ], + ), + "aarch64-macos": struct( + compatible_with = [ + "@platforms//os:macos", + "@platforms//cpu:aarch64", + ], + ), + "aarch64-windows": struct( + compatible_with = [ + "@platforms//os:windows", + "@platforms//cpu:aarch64", + ], + ), + "x86_64-linux": struct( compatible_with = [ "@platforms//os:linux", "@platforms//cpu:x86_64", ], ), - "macos-x86_64": struct( + "x86_64-macos": struct( compatible_with = [ "@platforms//os:macos", "@platforms//cpu:x86_64", ], ), + "x86_64-windows": struct( + compatible_with = [ + "@platforms//os:windows", + "@platforms//cpu:x86_64", + ], + ), } def _toolchains_repo_impl(repository_ctx): diff --git a/zig/private/versions.bzl b/zig/private/versions.bzl index 1d8a193e..cdd3f94b 100644 --- a/zig/private/versions.bzl +++ b/zig/private/versions.bzl @@ -1,14 +1,59 @@ -"""Mirror of release info +"""Mirror of Zig release info. -TODO[AH]: generate this file from https://ziglang.org/download/index.json""" +Generated from https://ziglang.org/download/index.json. +""" -# The integrity hashes can be computed with -# shasum -b -a 384 [downloaded file] | awk '{ print $1 }' | xxd -r -p | base64 -# Or using the sha256 hashes from ziglang.org converted with Python: -# "sha256-" + base64.b64encode(bytes.fromhex("SHA256")).decode() TOOL_VERSIONS = { "0.10.1": { - "linux-x86_64": "sha256-Zpnw5ykwgbQkKPMsnZyYOFQJS9Ff7lSJ8SxM9FGMw4A=", - "macos-x86_64": "sha256-Akg1ULidKjBwwu0AM1f9bmowWXB7juP7wMZ/g8qJhDc=", + "aarch64-linux": struct( + url = "https://ziglang.org/download/0.10.1/zig-linux-aarch64-0.10.1.tar.xz", + integrity = "sha256-2wdhZk9fIqpbvXRCoWF91pbAdtVxfd78ydi5Unj3H10=", + ), + "aarch64-macos": struct( + url = "https://ziglang.org/download/0.10.1/zig-macos-aarch64-0.10.1.tar.xz", + integrity = "sha256-ubAEd+xfofG4nzWn0qWGiOAZkQq4CmXqwqdBcWJzdlY=", + ), + "aarch64-windows": struct( + url = "https://ziglang.org/download/0.10.1/zig-windows-aarch64-0.10.1.zip", + integrity = "sha256-7Ok7DXeyqwPEDbme98y8Y+C2vWWK8SuXiYlg9iEwVCg=", + ), + "x86_64-linux": struct( + url = "https://ziglang.org/download/0.10.1/zig-linux-x86_64-0.10.1.tar.xz", + integrity = "sha256-Zpnw5ykwgbQkKPMsnZyYOFQJS9Ff7lSJ8SxM9FGMw4A=", + ), + "x86_64-macos": struct( + url = "https://ziglang.org/download/0.10.1/zig-macos-x86_64-0.10.1.tar.xz", + integrity = "sha256-Akg1ULidKjBwwu0AM1f9bmowWXB7juP7wMZ/g8qJhDc=", + ), + "x86_64-windows": struct( + url = "https://ziglang.org/download/0.10.1/zig-windows-x86_64-0.10.1.zip", + integrity = "sha256-V2gATl4nTHlpw4kuiRWW5Rxd8rQi15iGVHHgUEmYgSU=", + ), + }, + "0.10.0": { + "aarch64-linux": struct( + url = "https://ziglang.org/download/0.10.0/zig-linux-aarch64-0.10.0.tar.xz", + integrity = "sha256-Ce9QyL5zOAeZgEFpGXgg7nh2ByOwQw+oI/Vu1CsG6g8=", + ), + "aarch64-macos": struct( + url = "https://ziglang.org/download/0.10.0/zig-macos-aarch64-0.10.0.tar.xz", + integrity = "sha256-Aveng5tqHhJ+6uIupyyHYD+3KYxYvDWCKpUUedU8dVc=", + ), + "aarch64-windows": struct( + url = "https://ziglang.org/download/0.10.0/zig-windows-aarch64-0.10.0.zip", + integrity = "sha256-G72o0SPUTzrk+pDQ2gSx6Qk8P53a40KaSr7OHhwL8Zo=", + ), + "x86_64-linux": struct( + url = "https://ziglang.org/download/0.10.0/zig-linux-x86_64-0.10.0.tar.xz", + integrity = "sha256-Yx7HvLZJzWeVq+QN8ETSRztZtE4QvmicFWMqBFjd6lU=", + ), + "x86_64-macos": struct( + url = "https://ziglang.org/download/0.10.0/zig-macos-x86_64-0.10.0.tar.xz", + integrity = "sha256-OiLLbEdJiEFWqU6ptg86KM9OCYpp8IwY+8qBxzPr/to=", + ), + "x86_64-windows": struct( + url = "https://ziglang.org/download/0.10.0/zig-windows-x86_64-0.10.0.zip", + integrity = "sha256-pm4v9VXG5IeB3hvLBmLvKO5LiK868qV397GVDkMIl+4=", + ), }, } diff --git a/zig/repositories.bzl b/zig/repositories.bzl index 6bf678ae..806821d2 100644 --- a/zig/repositories.bzl +++ b/zig/repositories.bzl @@ -42,22 +42,21 @@ _ATTRS = { } def _zig_repo_impl(repository_ctx): - # TODO[AH] read URLs from https://ziglang.org/download/index.json - basename = "zig-{}-{}".format( - repository_ctx.attr.platform, - repository_ctx.attr.zig_version, - ) - url = "https://ziglang.org/download/{}/{}.tar.xz".format( - repository_ctx.attr.zig_version, - basename, - ) + url = TOOL_VERSIONS[repository_ctx.attr.zig_version][repository_ctx.attr.platform].url + integrity = TOOL_VERSIONS[repository_ctx.attr.zig_version][repository_ctx.attr.platform].integrity + basename = url.rsplit("/", 1)[1] + if basename.endswith(".tar.gz") or basename.endswith(".tar.xz"): + prefix = basename[:-7] + elif basename.endswith(".zip"): + prefix = basename[:-4] + else: + fail("Cannot download Zig SDK at {}. Unsupported file extension.".format(url)) repository_ctx.download_and_extract( url = url, - integrity = TOOL_VERSIONS[repository_ctx.attr.zig_version][repository_ctx.attr.platform], - stripPrefix = basename, + integrity = integrity, + stripPrefix = prefix, ) - # TODO[AH] compiler and lib files build_content = """#Generated by zig/repositories.bzl load("@rules_zig//zig:toolchain.bzl", "zig_toolchain") zig_toolchain( diff --git a/zig/tests/integration_tests/BUILD.bazel b/zig/tests/integration_tests/BUILD.bazel index 9088fcf6..7e4de9bd 100644 --- a/zig/tests/integration_tests/BUILD.bazel +++ b/zig/tests/integration_tests/BUILD.bazel @@ -1,13 +1,18 @@ load("//zig:defs.bzl", "zig_test") +load("//zig/private:versions.bzl", "TOOL_VERSIONS") load("@bazel_binaries//:defs.bzl", "bazel_binaries") load( "@rules_bazel_integration_test//bazel_integration_test:defs.bzl", + "bazel_integration_test", "bazel_integration_tests", "integration_test_utils", ) +# gazelle:exclude workspace + zig_test( name = "integration_tests_runner", + srcs = ["integration_testing.zig"], main = "integration_tests_runner.zig", tags = ["manual"], ) @@ -33,6 +38,27 @@ bazel_integration_tests( workspace_path = "workspace", ) +zig_test( + name = "zig_version_tests_runner", + srcs = ["integration_testing.zig"], + main = "zig_version_tests_runner.zig", + tags = ["manual"], +) + +zig_version_tests = { + "zig_version_test_" + zig_version: bazel_integration_test( + name = "zig_version_test_" + zig_version, + bazel_version = bazel_binaries.versions.current, + env = {"ZIG_VERSION": zig_version}, + test_runner = ":zig_version_tests_runner", + workspace_files = integration_test_utils.glob_workspace_files("workspace") + [ + "//:all_files", + ], + workspace_path = "workspace", + ) + for zig_version in TOOL_VERSIONS.keys() +}.keys() + test_suite( name = "integration_tests", tags = integration_test_utils.DEFAULT_INTEGRATION_TEST_TAGS, @@ -42,5 +68,5 @@ test_suite( ) + [integration_test_utils.bazel_integration_test_name( "bzlmod_test", bazel_binaries.versions.current, - )], + )] + zig_version_tests, ) diff --git a/zig/tests/integration_tests/integration_testing.zig b/zig/tests/integration_tests/integration_testing.zig new file mode 100644 index 00000000..501271dd --- /dev/null +++ b/zig/tests/integration_tests/integration_testing.zig @@ -0,0 +1,90 @@ +const builtin = @import("builtin"); +const std = @import("std"); + +/// Location of the Bazel workspace directory under test. +const BIT_WORKSPACE_DIR = "BIT_WORKSPACE_DIR"; + +/// Location of the Bazel binary. +const BIT_BAZEL_BINARY = "BIT_BAZEL_BINARY"; + +/// Bazel integration testing context. +/// +/// Provides access to the Bazel binary and the workspace directory under test. +pub const BitContext = struct { + workspace_path: []const u8, + bazel_path: []const u8, + bzlmod_enabled: bool, + + pub fn init() !BitContext { + const workspace_path = std.os.getenv(BIT_WORKSPACE_DIR) orelse { + std.log.err("Required environment variable not found: {s}", .{BIT_WORKSPACE_DIR}); + return error.EnvironmentVariableNotFound; + }; + const bazel_path = std.os.getenv(BIT_BAZEL_BINARY) orelse { + std.log.err("Required environment variable not found: {s}", .{BIT_BAZEL_BINARY}); + return error.EnvironmentVariableNotFound; + }; + const bzlmod_enabled = if (std.os.getenv("BZLMOD_ENABLED")) |val| + std.mem.eql(u8, val, "true") + else + false; + return BitContext{ + .workspace_path = workspace_path, + .bazel_path = bazel_path, + .bzlmod_enabled = bzlmod_enabled, + }; + } + + pub const BazelResult = struct { + success: bool, + term: std.ChildProcess.Term, + stdout: []u8, + stderr: []u8, + + pub fn deinit(self: BazelResult) void { + std.testing.allocator.free(self.stdout); + std.testing.allocator.free(self.stderr); + } + }; + + pub fn exec_bazel( + self: BitContext, + args: struct { + argv: []const []const u8, + print_on_error: bool = true, + omit_bzlmod_flag: bool = false, + }, + ) !BazelResult { + var argc = 1 + args.argv.len; + if (self.bzlmod_enabled and !args.omit_bzlmod_flag) { + argc += 1; + } + var argv = try std.testing.allocator.alloc([]const u8, argc); + defer std.testing.allocator.free(argv); + argv[0] = self.bazel_path; + for (args.argv) |arg, i| { + argv[i + 1] = arg; + } + if (self.bzlmod_enabled and !args.omit_bzlmod_flag) { + argv[argc - 1] = "--enable_bzlmod"; + } + const result = try std.ChildProcess.exec(.{ + .allocator = std.testing.allocator, + .argv = argv, + .cwd = self.workspace_path, + }); + const success = switch (result.term) { + .Exited => |code| code == 0, + else => false, + }; + if (args.print_on_error and !success) { + std.debug.print("\n{s}\n{s}\n", .{ result.stdout, result.stderr }); + } + return BazelResult{ + .success = success, + .term = result.term, + .stdout = result.stdout, + .stderr = result.stderr, + }; + } +}; diff --git a/zig/tests/integration_tests/integration_tests_runner.zig b/zig/tests/integration_tests/integration_tests_runner.zig index 3ee3f52f..d8407a32 100644 --- a/zig/tests/integration_tests/integration_tests_runner.zig +++ b/zig/tests/integration_tests/integration_tests_runner.zig @@ -1,93 +1,7 @@ const builtin = @import("builtin"); const std = @import("std"); - -/// Location of the Bazel workspace directory under test. -const BIT_WORKSPACE_DIR = "BIT_WORKSPACE_DIR"; - -/// Location of the Bazel binary. -const BIT_BAZEL_BINARY = "BIT_BAZEL_BINARY"; - -/// Bazel integration testing context. -/// -/// Provides access to the Bazel binary and the workspace directory under test. -const BitContext = struct { - workspace_path: []const u8, - bazel_path: []const u8, - bzlmod_enabled: bool, - - pub fn init() !BitContext { - const workspace_path = std.os.getenv(BIT_WORKSPACE_DIR) orelse { - std.log.err("Required environment variable not found: {s}", .{BIT_WORKSPACE_DIR}); - return error.EnvironmentVariableNotFound; - }; - const bazel_path = std.os.getenv(BIT_BAZEL_BINARY) orelse { - std.log.err("Required environment variable not found: {s}", .{BIT_BAZEL_BINARY}); - return error.EnvironmentVariableNotFound; - }; - const bzlmod_enabled = if (std.os.getenv("BZLMOD_ENABLED")) |val| - std.mem.eql(u8, val, "true") - else - false; - return BitContext{ - .workspace_path = workspace_path, - .bazel_path = bazel_path, - .bzlmod_enabled = bzlmod_enabled, - }; - } - - pub const BazelResult = struct { - success: bool, - term: std.ChildProcess.Term, - stdout: []u8, - stderr: []u8, - - pub fn deinit(self: BazelResult) void { - std.testing.allocator.free(self.stdout); - std.testing.allocator.free(self.stderr); - } - }; - - pub fn exec_bazel( - self: BitContext, - args: struct { - argv: []const []const u8, - print_on_error: bool = true, - omit_bzlmod_flag: bool = false, - }, - ) !BazelResult { - var argc = 1 + args.argv.len; - if (self.bzlmod_enabled and !args.omit_bzlmod_flag) { - argc += 1; - } - var argv = try std.testing.allocator.alloc([]const u8, argc); - defer std.testing.allocator.free(argv); - argv[0] = self.bazel_path; - for (args.argv) |arg, i| { - argv[i + 1] = arg; - } - if (self.bzlmod_enabled and !args.omit_bzlmod_flag) { - argv[argc - 1] = "--enable_bzlmod"; - } - const result = try std.ChildProcess.exec(.{ - .allocator = std.testing.allocator, - .argv = argv, - .cwd = self.workspace_path, - }); - const success = switch (result.term) { - .Exited => |code| code == 0, - else => false, - }; - if (args.print_on_error and !success) { - std.debug.print("\n{s}\n{s}\n", .{ result.stdout, result.stderr }); - } - return BazelResult{ - .success = success, - .term = result.term, - .stdout = result.stdout, - .stderr = result.stderr, - }; - } -}; +const integration_testing = @import("integration_testing.zig"); +const BitContext = integration_testing.BitContext; test "zig_binary prints Hello World!" { const ctx = try BitContext.init(); diff --git a/zig/tests/integration_tests/workspace/WORKSPACE b/zig/tests/integration_tests/workspace/WORKSPACE index c1843e4f..3a82491b 100644 --- a/zig/tests/integration_tests/workspace/WORKSPACE +++ b/zig/tests/integration_tests/workspace/WORKSPACE @@ -7,7 +7,9 @@ load("@rules_zig//zig:repositories.bzl", "rules_zig_dependencies", "zig_register rules_zig_dependencies() +load("//:zig_version.bzl", "ZIG_VERSION") + zig_register_toolchains( name = "zig", - zig_version = "0.10.1", + zig_version = ZIG_VERSION, ) diff --git a/zig/tests/integration_tests/workspace/zig_version.bzl b/zig/tests/integration_tests/workspace/zig_version.bzl new file mode 100644 index 00000000..39ce3e63 --- /dev/null +++ b/zig/tests/integration_tests/workspace/zig_version.bzl @@ -0,0 +1,5 @@ +"""Defines the Zig SDK version to use for integration tests.""" + +load("@rules_zig//zig/private:versions.bzl", "TOOL_VERSIONS") + +ZIG_VERSION = TOOL_VERSIONS.keys()[0] diff --git a/zig/tests/integration_tests/zig_version_tests_runner.zig b/zig/tests/integration_tests/zig_version_tests_runner.zig new file mode 100644 index 00000000..590d5110 --- /dev/null +++ b/zig/tests/integration_tests/zig_version_tests_runner.zig @@ -0,0 +1,31 @@ +const builtin = @import("builtin"); +const std = @import("std"); +const integration_testing = @import("integration_testing.zig"); +const BitContext = integration_testing.BitContext; + +test "zig_binary prints Hello World!" { + const ctx = try BitContext.init(); + + var workspace = try std.fs.cwd().openDir(ctx.workspace_path, .{}); + defer workspace.close(); + + try workspace.rename("zig_version.bzl", "zig_version.bzl.backup"); + defer workspace.rename("zig_version.bzl.backup", "zig_version.bzl") catch {}; + + { + const zig_version = std.os.getenv("ZIG_VERSION").?; + + var version_file = try workspace.createFile("zig_version.bzl", .{}); + defer version_file.close(); + + try version_file.writer().print("ZIG_VERSION = \"{s}\"\n", .{zig_version}); + } + + const result = try ctx.exec_bazel(.{ + .argv = &[_][]const u8{ "run", "//:binary" }, + }); + defer result.deinit(); + + try std.testing.expect(result.success); + try std.testing.expectEqualStrings("Hello World!\n", result.stdout); +}