Skip to content

Commit

Permalink
more post-merge cleanup for lock_downloader
Browse files Browse the repository at this point in the history
  • Loading branch information
tdyas committed Jan 11, 2025
1 parent 755657e commit 5b8894f
Show file tree
Hide file tree
Showing 2 changed files with 18 additions and 67 deletions.
3 changes: 3 additions & 0 deletions pex/resolve/lock_downloader.py
Original file line number Diff line number Diff line change
Expand Up @@ -220,6 +220,7 @@ def create(
build_configuration=BuildConfiguration(), # type: BuildConfiguration
use_pip_config=False, # type: bool
extra_pip_requirements=(), # type: Tuple[Requirement, ...]
keyring_provider=None, # type: Optional[str]
):
# type: (...) -> LockDownloader

Expand Down Expand Up @@ -248,6 +249,7 @@ def create(
),
use_pip_config=use_pip_config,
extra_pip_requirements=extra_pip_requirements,
keyring_provider=keyring_provider,
),
max_parallel_jobs=max_parallel_jobs,
),
Expand All @@ -269,6 +271,7 @@ def create(
resolver=resolver,
use_pip_config=use_pip_config,
extra_pip_requirements=extra_pip_requirements,
keyring_provider=keyring_provider,
)
for target in targets
}
Expand Down
82 changes: 15 additions & 67 deletions pex/resolve/lock_resolver.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,73 +75,21 @@ def resolve_from_lock(
for downloadable_artifact in resolved_subset.resolved.downloadable_artifacts
)

# Since the download managers are stored to via a thread pool, we need to use BSD style locks.
# These locks are not as portable as POSIX style locks but work with threading unlike POSIX
# locks which are subject to threading-unaware deadlock detection per the standard. Linux, in
# fact, implements deadlock detection for POSIX locks; so we can run afoul of false EDEADLCK
# errors under the right interleaving of processes and threads and download artifact targets.
file_lock_style = FileLockStyle.BSD

file_download_managers_by_target = {
resolved_subset.target: FileArtifactDownloadManager(
file_lock_style=file_lock_style,
downloader=ArtifactDownloader(
resolver=resolver,
lock_configuration=LockConfiguration(
style=lock.style,
requires_python=lock.requires_python,
target_systems=lock.target_systems,
),
target=resolved_subset.target,
package_index_configuration=PackageIndexConfiguration.create(
pip_version=pip_version,
resolver_version=resolver_version,
indexes=indexes,
find_links=find_links,
network_configuration=network_configuration,
password_entries=PasswordDatabase.from_netrc().append(password_entries).entries,
use_pip_config=use_pip_config,
extra_pip_requirements=extra_pip_requirements,
keyring_provider=keyring_provider,
),
max_parallel_jobs=max_parallel_jobs,
),
)
for resolved_subset in subset_result.subsets
}

vcs_download_managers_by_target = {
resolved_subset.target: VCSArtifactDownloadManager(
target=resolved_subset.target,
file_lock_style=file_lock_style,
indexes=indexes,
find_links=find_links,
resolver_version=resolver_version,
network_configuration=network_configuration,
password_entries=password_entries,
build_configuration=build_configuration,
pip_version=pip_version,
resolver=resolver,
use_pip_config=use_pip_config,
extra_pip_requirements=extra_pip_requirements,
keyring_provider=keyring_provider,
)
for resolved_subset in subset_result.subsets
}

local_project_download_managers_by_target = {
resolved_subset.target: LocalProjectDownloadManager(
file_lock_style=file_lock_style,
pip_version=pip_version,
target=resolved_subset.target,
resolver=resolver,
)
for resolved_subset in subset_result.subsets
}

max_threads = min(
len(downloadable_artifacts_and_targets) or 1,
min(MAX_PARALLEL_DOWNLOADS, 4 * (max_parallel_jobs or cpu_count() or 1)),
lock_downloader = LockDownloader.create(
targets=tuple(resolved_subset.target for resolved_subset in subset_result.subsets),
lock=lock,
resolver=resolver,
indexes=indexes,
find_links=find_links,
max_parallel_jobs=max_parallel_jobs,
pip_version=pip_version,
resolver_version=resolver_version,
network_configuration=network_configuration,
password_entries=password_entries,
build_configuration=build_configuration,
use_pip_config=use_pip_config,
extra_pip_requirements=extra_pip_requirements,
keyring_provider=keyring_provider,
)
with TRACER.timed(
"Downloading {url_count} distributions to satisfy {requirement_count} requirements".format(
Expand Down

0 comments on commit 5b8894f

Please sign in to comment.