Skip to content

Commit

Permalink
allow for use of the wheel tool to resolve to pypi if not found in ur…
Browse files Browse the repository at this point in the history 
…ls provided (#137)

This change introduces the use of the WHEEL_TOOL which can be used to resolve the urls for downloading the wheel from either the urls provided or from pypi if not found in the urls provided.
  • Loading branch information
@goddenrich
goddenrich authored Apr 15, 2024
1 parent 01fb012 commit d9bbe27
Show file tree
Hide file tree
Showing 2 changed files with 58 additions and 35 deletions.
1 change: 0 additions & 1 deletion .plzconfig
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,6 @@ Help = A format string for the wheel repo URL. Format fields url_base, package_n

[PluginConfig "wheel_tool"]
ConfigKey = WheelTool
DefaultValue = //tools:wheel_resolver
Optional = true
Inherit = true
Help = The tool used to resolve wheels with using the pypi API.
Expand Down
92 changes: 58 additions & 34 deletions build_defs/python.build_defs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -494,7 +494,8 @@ def pip_library(name:str, version:str, labels:list=[], hashes:list=None, package
def python_wheel(name:str, version:str, labels:list=[], hashes:list=None, package_name:str=None,
outs:list=None, post_install_commands:list=None, patch:str|list=None, licences:list=None,
test_only:bool&testonly=False, repo:str=None, zip_safe:bool=True, visibility:list=None,
deps:list=[], name_scheme:str=None, strip:list=['*.pyc', 'tests'], binary = False, entry_points={}):
deps:list=[], name_scheme:str=None, strip:list=['*.pyc', 'tests'], binary = False,
entry_points={}, tool:str=CONFIG.PYTHON.WHEEL_TOOL):
"""Downloads a Python wheel and extracts it.

This is a lightweight pip-free alternative to pip_library which supports cross-compiling.
Expand Down Expand Up @@ -533,47 +534,70 @@ def python_wheel(name:str, version:str, labels:list=[], hashes:list=None, packag
This parameter can be a string, in which case the rule can be ran directly, or a
dictionary, for example `{"protoc", "protoc.__main__:main"}. For the latter, each key can
be ran using annotated labels, e.g. `plz run //third_party/python:protobuf|protoc`
tool (str): Tool to locate python wheel file in an index
"""
binary = binary or entry_points
package_name = package_name or name.replace('-', '_')
initial = package_name[0]
url_base = repo or CONFIG.PYTHON.WHEEL_REPO
if not url_base:
if not tool and not url_base:
fail('python.wheel_repo is not set in the config, must pass repo explicitly to python_wheel')
urls = []
if name_scheme:
urls += [name_scheme.format(url_base=url_base,
package_name=package_name,
initial=initial,
version=version)]
elif CONFIG.PYTHON.WHEEL_NAME_SCHEME:
urls += [scheme.format(url_base=url_base, package_name=package_name, initial=initial, version=version)
for scheme in CONFIG.PYTHON.WHEEL_NAME_SCHEME]
if url_base:
if name_scheme:
urls += [name_scheme.format(url_base=url_base,
package_name=package_name,
initial=initial,
version=version)]
elif CONFIG.PYTHON.WHEEL_NAME_SCHEME:
urls += [scheme.format(url_base=url_base, package_name=package_name, initial=initial, version=version)
for scheme in CONFIG.PYTHON.WHEEL_NAME_SCHEME]
else:
# Populate urls using a reasonable set of possible wheel naming schemes.
# Look for an arch-specific wheel first; in some cases there can be both (e.g. protobuf
# has optional arch-specific bits) and we prefer the one with the cool stuff.
urls += ['{url_base}/{package_name}-{version}-${{OS}}-${{ARCH}}.whl'.format(url_base=url_base,
package_name=package_name,
initial=initial,
version=version),
'{url_base}/{package_name}-{version}-${{OS}}_${{ARCH}}.whl'.format(url_base=url_base,
package_name=package_name,
initial=initial,
version=version),
'{url_base}/{package_name}-{version}.whl'.format(url_base=url_base,
package_name=package_name,
initial=initial,
version=version)]

file_rule = None
if tool:
# Try the URLs generated using the wheel name schemes. If those fail, try
# generating a url with the wheel_resolver tool and download that if successful.
cmd = f'$TOOL --package {package_name} --version {version}'
if urls:
cmd += ' --urls ' + ' '.join(['%s' % url for url in urls])
file_rule = build_rule(
name = name,
tag = 'download',
outs = [name + '.whl'],
cmd = cmd,
tools = [tool],
hashes = hashes,
labels = ["python_wheel_download"],
licences = licences if licences else None,
sandbox = False,
)
else:
# Populate urls using a reasonable set of possible wheel naming schemes.
# Look for an arch-specific wheel first; in some cases there can be both (e.g. protobuf
# has optional arch-specific bits) and we prefer the one with the cool stuff.
urls += ['{url_base}/{package_name}-{version}-${{OS}}-${{ARCH}}.whl'.format(url_base=url_base,
package_name=package_name,
initial=initial,
version=version),
'{url_base}/{package_name}-{version}-${{OS}}_${{ARCH}}.whl'.format(url_base=url_base,
package_name=package_name,
initial=initial,
version=version),
'{url_base}/{package_name}-{version}.whl'.format(url_base=url_base,
package_name=package_name,
initial=initial,
version=version)]

file_rule = remote_file(
name = name,
_tag = 'download',
out = name + '.whl',
url = urls,
hashes = hashes,
licences = licences if licences else None,
)
if not urls:
fail('python.wheel_repo is not set in the config, must pass repo explicitly to python_wheel')
file_rule = remote_file(
name = name,
_tag = 'download',
out = name + '.whl',
url = urls,
hashes = hashes,
licences = licences if licences else None,
)

cmd = ['$TOOL x $SRCS_SRC']

Expand Down

0 comments on commit d9bbe27

Please sign in to comment.