add install-toolchain command

The command downloads and installs the current Genode toolchain (if unavailable)
and converts it into a squashfs. The squashfs is mounted via squashfuse
and bind mounted into the bubblewrapped build environment.

genodelabs#99

bin/goa

@@ -188,6 +188,10 @@ if {![file exists src]} {
 	set perform(build)     0
 }
 
+if {$perform(install-toolchain)} {
+	goa install-toolchain $args(keep_mounted)
+}
+
 
 if {$perform(build-dir)} {
 

share/bash-completion/goa

@@ -7,19 +7,19 @@ __goa()
     cur="${COMP_WORDS[COMP_CWORD]}"
     prev="${COMP_WORDS[COMP_CWORD-1]}"
 
-    commands="help update-goa --version import diff build-dir build extract-abi-symbols run export publish add-depot-user archive-versions depot-dir run-dir bump-version backtrace"
+    commands="help update-goa --version import diff build-dir build extract-abi-symbols run export publish add-depot-user archive-versions depot-dir run-dir bump-version backtrace install-toolchain"
     help_commands="config api build-systems artifacts add-depot-user import targets runtime index"
     depot_commands="export|publish"
     run_commands="run|run-dir|backtrace"
     depot_opts="--depot-user --depot-overwrite --depot-retain --pkg"
     run_opts="--pkg --run-as --target"
     user_opts="--depot-url --gpg-user-id --pubkey-file"
 
-    globalopts="--arch --rebuild --debug --warn-strict --no-warn-strict --with-backtrace -C --license --versions-from-genode-dir --depot-dir --public-dir --project-name --contrib-dir --build-dir --abi-dir --bin-dir --dbg-dir --run-dir --common-var-dir --verbose -r --jobs --cross-dev-prefix --ld-march --olevel --search-dir"
+    globalopts="--arch --rebuild --debug --warn-strict --no-warn-strict --with-backtrace -C --license --versions-from-genode-dir --depot-dir --public-dir --project-name --contrib-dir --build-dir --abi-dir --bin-dir --dbg-dir --run-dir --common-var-dir --verbose -r --jobs --cross-dev-prefix --ld-march --olevel --search-dir --install-dir"
 
     stropts="--depot-url|--gpg-user-id|--depot-user|--ld-march|--run_as"
     fileopts="--license|--pubkey-file|--cross-dev-prefix"
-    diropts="-C|--versions-from-genode-dir|--depot-dir|--public-dir|--project-name|--contrib-dir|--build-dir|--abi-dir|--bin-dir|--dbg-dir|--run-dir|--common-var-dir|--search-dir"
+    diropts="-C|--versions-from-genode-dir|--depot-dir|--public-dir|--project-name|--contrib-dir|--build-dir|--abi-dir|--bin-dir|--dbg-dir|--run-dir|--common-var-dir|--search-dir|--install-dir"
     pkgopts="--pkg"
     targetvals="linux sculpt"
     archvals="x86_64 arm_v8a"

share/goa/doc/config.txt

@@ -57,7 +57,7 @@ Goa evaluates the following configuration variables.
 
 :cross_dev_prefix:
   The tool-chain prefix for the Genode cross compiler.
-  The default value is "/usr/local/genode/tool/current/bin/genode-x86-".
+  The default value is "/usr/local/genode/tool/<toolchain_version>/bin/genode-x86-".
 
 :rebuild:
   If set to 1 or if the command line argument '--rebuild' is specified,
@@ -110,6 +110,9 @@ Goa evaluates the following configuration variables.
   executing the 'run' command. The default is var/run/ within the project
   directory.
 
+:install_dir: The directory where additional build tools are downloaded and
+  installed.
+
 :common_var_dir:
   If defined, Goa places all generated files in the specified directory
   instead of creating a 'var' directory inside the project directory.
@@ -196,6 +199,9 @@ Goa evaluates the following configuration variables.
   configuration variable in a custom 'goarc' file or via the
   '--sculpt-version' command-line argument.
 
+:toolchain_version:
+  Specifies the to-be-used version of the Genode toolchain.
+
 :target:
   Specifies the run target used for 'goa run'. The default value is 'linux'.
   The value can be overridden by setting this configuration variable or by

share/goa/doc/overview.txt

@@ -116,6 +116,14 @@ USAGE
 :goa diff:
   Show local modifications of the imported 3rd-party software.
 
+:goa install-toolchain [--keep-mounted]:
+  Downloads and installs the Genode toolchain as a squashfs archive into
+  the var/install/ directory if no system-wide installation was found. This
+  command is executed as a prerequisite of the 'build-dir' command. During
+  Goa's execution, the squashfs archive will be temporarily mounted using FUSE.
+  When provided with the --keep-mounted option, the mount is kept persistent
+  after Goa exited.
+
 :goa depot-dir:
   Create a depot directory and install default depot users.
 

share/goa/goarc

@@ -1,6 +1,9 @@
 # Sculpt version is required for publishing index projects
 set  sculpt_version 24.04
 
+# version of Genode toolchain
+set toolchain_version 23.05
+
 # archive versions
 set  version(genodelabs/api/base)                        2024-04-11
 set  version(genodelabs/api/base-linux)                  2024-04-11

share/goa/lib/actions/build.tcl

@@ -6,6 +6,7 @@ namespace eval goa {
 	namespace export build-dir build using_api used_apis check_abis
 	namespace export build extract_artifacts_from_build_dir extract_api_artifacts
 	namespace export extract-abi-symbols
+	namespace export install-toolchain
 
 	proc sandboxed_build_command { } {
 		global config::project_dir config::depot_dir config::var_dir config::build_dir
@@ -70,13 +71,17 @@ namespace eval goa {
 	proc gaol_with_toolchain { {silent 0} } {
 
 		global   gaol verbose allowed_tools
+		variable toolchain_dirs
 
 		set     cmd $gaol
 		lappend cmd --system-usr
 
 		foreach dir [lsearch -all -inline -not $allowed_tools /usr*] {
 			lappend cmd --ro-bind $dir }
 
+		foreach { base_dir at } $toolchain_dirs {
+			lappend cmd --ro-bind-at [file join $base_dir $at] /$at }
+
 		if {$verbose && !$silent} {
 			lappend cmd --verbose }
 
@@ -125,6 +130,17 @@ namespace eval goa {
 	}
 
 
+	proc is_toolchain_path { path } {
+		variable toolchain_dirs
+
+		foreach { dummy dir } $toolchain_dirs {
+			if {[regexp "^/$dir" $path]} {
+				return 1 } }
+
+		return 0
+	}
+
+
 	proc used_apis { } {
 
 		variable _used_apis
@@ -236,6 +252,70 @@ namespace eval goa {
 			exit_with_error "failed to generate ldso_support.lib.a "] }
 	}
 
+	##
+	# Make tool chain available (download and install if necessary)
+	#
+	proc install-toolchain { keep_mounted } {
+
+		global gaol tool_dir
+		global config::cross_dev_prefix
+		global config::install_dir config::toolchain_version
+		variable toolchain_dirs {}
+
+		##
+		# Check for system-wide availability of the Genode tool chain
+		#
+		if {[have_installed ${cross_dev_prefix}gcc]} {
+			return }
+
+		set toolchain genode-toolchain-$toolchain_version
+
+		if {![file exists $install_dir]} {
+			file mkdir $install_dir }
+
+		exit_if_not_installed curl xzcat sqfstar squashfuse_ll
+
+		# trigger (re-)download, integrity check and squashfs creation
+		set     install_cmd $gaol
+		lappend install_cmd --system-usr
+		lappend install_cmd --ro-bind [file join $tool_dir lib]
+		lappend install_cmd --bind $install_dir
+		lappend install_cmd --with-network
+		lappend install_cmd [file join $tool_dir lib install_tool.mk]
+		lappend install_cmd $toolchain
+		lappend install_cmd INSTALL_DIR=$install_dir
+		if {[catch {exec {*}$install_cmd >&@ stdout}]} {
+			exit_with_error "Unable to install $toolchain" }
+
+		# create mountpoint
+		set     mount_dir [file join $install_dir $toolchain]
+		if {![file exists $mount_dir]} {
+			file mkdir $mount_dir
+		} elseif {[glob -nocomplain -dir $mount_dir *] != ""} {
+			exec fusermount -u $mount_dir
+		}
+
+		# mount squashfs
+		set     mount_cmd squashfuse_ll
+		if {!$keep_mounted} {
+			lappend mount_cmd -f }
+		lappend mount_cmd [file join $install_dir download $toolchain.squashfs]
+		lappend mount_cmd $mount_dir
+		spawn -noecho {*}$mount_cmd
+
+		# remember mountpoint and location (used in gaol_with_toolchain)
+		lappend  toolchain_dirs $mount_dir usr/local/genode/tool
+
+		# check mount availability
+		after 100
+		if {![file exists $mount_dir/${cross_dev_prefix}gcc]} {
+			exit_with_error "the tool chain ${cross_dev_prefix}" \
+			                 "is required but not installed." \
+			                 "Please refer to https://genode.org/download/tool-chain" \
+			                 "for more information."
+		}
+	}
+
 	##
 	# Implements 'goa build-dir' command
 	#
@@ -249,16 +329,6 @@ namespace eval goa {
 		global config::with_backtrace config::warn_strict config::depot_user
 		global config::project_name config::project_dir
 
-		#
-		# Check for availability of the Genode tool chain
-		#
-		if {![have_installed ${cross_dev_prefix}gcc]} {
-			exit_with_error "the tool chain ${cross_dev_prefix}" \
-			                 "is required but not installed." \
-			                 "Please refer to https://genode.org/download/tool-chain" \
-			                 "for more information."
-		}
-
 		#
 		# Prepare depot content for the used APIs and generate ABI stubs
 		#
@@ -286,7 +356,7 @@ namespace eval goa {
 
 		# filter out non-existing include directories
 		foreach dir $include_dirs {
-			if {[file exists $dir]} {
+			if {[file exists $dir] || [is_toolchain_path $dir]} {
 				lappend existing_include_dirs $dir } }
 		set include_dirs $existing_include_dirs
 

share/goa/lib/build/qmake.tcl

@@ -58,8 +58,9 @@ proc create_or_update_build_dir { } {
 	global ldlibs_exe ldlibs_so
 	global config::build_dir config::project_dir config::abi_dir
 	global config::cross_dev_prefix config::project_name
+	global config::toolchain_version
 
-	set qt5_tool_dir "/usr/local/genode/tool/23.05/bin"
+	set qt5_tool_dir "/usr/local/genode/tool/$toolchain_version/bin"
 	set qmake_platform [_qmake_platform]
 
 	if {![file exists $build_dir]} {
@@ -73,7 +74,9 @@ proc create_or_update_build_dir { } {
 
 	set qt5_api "qt5_base"
 
-	file link -symbolic qmake_root/bin $qt5_tool_dir
+	# $qt5_tool_dir might only exist in sandbox environment, hence use ln
+	exec ln -sf $qt5_tool_dir qmake_root/bin
+
 	file link -symbolic qmake_root/include [file join [api_archive_dir $qt5_api] include]
 	file link -symbolic qmake_root/lib $abi_dir
 

share/goa/lib/command_line.tcl

@@ -152,7 +152,7 @@ if {[llength $argv] == 0} {
 
 set avail_commands [list update-goa archive-versions backtrace import diff build-dir \
                          build run run-dir export publish add-depot-user bump-version \
-                         extract-abi-symbols help versions depot-dir]
+                         extract-abi-symbols help versions depot-dir install-toolchain]
 
 foreach command $avail_commands {
 	set perform($command) 0 }
@@ -180,6 +180,7 @@ action_dependency backtrace       run
 action_dependency run             run-dir
 action_dependency run-dir         build
 action_dependency build           build-dir
+action_dependency build-dir       install-toolchain
 action_dependency build-dir       depot-dir
 action_dependency add-depot-user  depot-dir
 
@@ -256,6 +257,12 @@ if {$perform(add-depot-user)} {
 		exit_with_error "public-key file $args(pubkey_file) does not exist" }
 }
 
+if {$perform(install-toolchain)} {
+	set args(keep_mounted) 0
+	if {[consume_optional_cmdline_switch "--keep-mounted"]} {
+		set args(keep_mounted) 1 }
+}
+
 # override 'rebuild' variable via optional command-line switch
 if {$perform(build-dir)} {
 	if {[consume_optional_cmdline_switch "--rebuild"]} {

share/goa/lib/config.tcl

@@ -26,6 +26,7 @@ namespace eval ::config {
 	variable run_as                   "genodelabs"
 	variable target                   "linux"
 	variable sculpt_version           ""
+	variable toolchain_version        ""
 	variable cc_cxx_opt_std           "-std=gnu++20"
 	variable binary_name              ""
 	variable with_backtrace           0
@@ -40,6 +41,7 @@ namespace eval ::config {
 	variable run_dir                  ""
 	variable bin_dir                  ""
 	variable dbg_dir                  ""
+	variable install_dir              ""
 	variable target_opt
 	array set target_opt {}
 	variable version
@@ -318,6 +320,7 @@ namespace eval ::config {
 		variable run_as
 		variable binary_name
 		variable var_dir
+		variable toolchain_version
 
 		if {$versions_from_genode_dir == ""} { unset versions_from_genode_dir }
 		if {$license                  == ""} { unset license }
@@ -339,8 +342,8 @@ namespace eval ::config {
 
 		if {![info exists cross_dev_prefix]} {
 			switch $arch {
-			arm_v8a { set cross_dev_prefix "/usr/local/genode/tool/23.05/bin/genode-aarch64-" }
-			x86_64  { set cross_dev_prefix "/usr/local/genode/tool/23.05/bin/genode-x86-"  }
+			arm_v8a { set cross_dev_prefix "/usr/local/genode/tool/$toolchain_version/bin/genode-aarch64-" }
+			x86_64  { set cross_dev_prefix "/usr/local/genode/tool/$toolchain_version/bin/genode-x86-"  }
 			default { exit_with_error "tool-chain prefix is not defined" }
 			}
 		}
@@ -379,6 +382,7 @@ namespace eval ::config {
 		set_if_undefined dbg_dir     [file join $var_dir dbg   $arch]
 		set_if_undefined run_dir     [file join $var_dir run]
 		set_if_undefined api_dir     [file join $var_dir api]
+		set_if_undefined install_dir [file join $var_dir install]
 	}
 
 	# make namespace procs available as subcommands

share/goa/lib/test.expect

@@ -101,7 +101,8 @@ proc run_goa_until {{test} {timeout_value 0}} {
 	variable public var
 	set public [file join [_find_tool_dir] public]
 	set var    [file join [_find_tool_dir] var]
-	eval spawn $goa run -C $test_dir --public-dir $public --common-var-dir $var
+	set tool   [file join $var tool]
+	eval spawn $goa run -C $test_dir --public-dir $public --install-dir $tool --common-var-dir $var
 	set result [wait_for_output $test_output($test) $timeout_value $spawn_id]
 
 	#