From a0111178e24e298bc5f8ac851b7a44fd6d659aac Mon Sep 17 00:00:00 2001 From: Zachary Garrett Date: Tue, 30 Jul 2024 14:56:01 -0700 Subject: [PATCH] Update `tensorflow` to version `2.16.1`. PiperOrigin-RevId: 657740192 --- RELEASE.md | 4 + WORKSPACE | 93 ++++++++++++------- pyproject.toml | 2 +- requirements.txt | 4 +- .../cc/core/impl/aggregation/tensorflow/BUILD | 6 +- .../impl/aggregation/tensorflow/converters.cc | 2 +- .../tensorflow_checkpoint_builder_factory.cc | 2 +- .../tensorflow_checkpoint_parser_factory.cc | 2 +- .../cc/core/impl/executors/BUILD | 68 +++++++------- .../cc/core/impl/executors/array_test_utils.h | 2 +- .../core/impl/executors/tensorflow_utils.cc | 2 +- .../impl/executors/tensorflow_utils_test.cc | 2 +- .../cc/core/impl/executors/xla_executor.cc | 22 ++--- .../core/impl/executors/xla_executor_test.cc | 14 +-- .../cc/core/impl/executors/xla_utils.cc | 8 +- .../cc/core/impl/executors/xla_utils.h | 4 +- .../cc/core/impl/executors/xla_utils_test.cc | 16 ++-- .../tensorflow/internal_visibility.patch | 6 +- third_party/tensorflow/python_toolchain.patch | 34 +++++-- .../tensorflow/tf2xla_visibility.patch | 4 +- 20 files changed, 167 insertions(+), 130 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index c7a13c4c78..a47d9cb05c 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -8,6 +8,10 @@ and this project adheres to ## Unreleased +### Changed + +* `tensorflow` verison increased to 2.16.1. + ## Release 0.84.0 ### Added diff --git a/WORKSPACE b/WORKSPACE index 541bb47dfa..a383f3f86c 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -18,64 +18,64 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") http_archive( name = "bazel_skylib", - url = "https://github.com/bazelbuild/bazel-skylib/archive/refs/tags/1.3.0.tar.gz", sha256 = "3b620033ca48fcd6f5ef2ac85e0f6ec5639605fa2f627968490e52fc91a9932f", strip_prefix = "bazel-skylib-1.3.0", + url = "https://github.com/bazelbuild/bazel-skylib/archive/refs/tags/1.3.0.tar.gz", ) http_archive( name = "com_github_grpc_grpc", - url = "https://github.com/grpc/grpc/archive/refs/tags/v1.50.0.tar.gz", - sha256 = "76900ab068da86378395a8e125b5cc43dfae671e09ff6462ddfef18676e2165a", - strip_prefix = "grpc-1.50.0", + sha256 = "c682fc39baefc6e804d735e6b48141157b7213602cc66dbe0bf375b904d8b5f9", + strip_prefix = "grpc-1.64.2", + url = "https://github.com/grpc/grpc/archive/refs/tags/v1.64.2.tar.gz", ) http_archive( name = "com_google_benchmark", - url = "https://github.com/google/benchmark/archive/refs/tags/v1.8.3.tar.gz", sha256 = "6bc180a57d23d4d9515519f92b0c83d61b05b5bab188961f36ac7b06b0d9e9ce", strip_prefix = "benchmark-1.8.3", + url = "https://github.com/google/benchmark/archive/refs/tags/v1.8.3.tar.gz", ) http_archive( - name = "com_google_cc_differential_privacy", - url = "https://github.com/google/differential-privacy/archive/refs/tags/v3.0.0.tar.gz", - sha256 = "6e6e1cd7a819695caae408f4fa938129ab7a86e83fe2410137c85e50131abbe0", - strip_prefix = "differential-privacy-3.0.0/cc", + name = "com_google_cc_differential_privacy", + sha256 = "6e6e1cd7a819695caae408f4fa938129ab7a86e83fe2410137c85e50131abbe0", + strip_prefix = "differential-privacy-3.0.0/cc", + url = "https://github.com/google/differential-privacy/archive/refs/tags/v3.0.0.tar.gz", ) http_archive( - name = "com_google_differential_privacy", - url = "https://github.com/google/differential-privacy/archive/refs/tags/v3.0.0.tar.gz", - sha256 = "6e6e1cd7a819695caae408f4fa938129ab7a86e83fe2410137c85e50131abbe0", - strip_prefix = "differential-privacy-3.0.0", + name = "com_google_differential_privacy", + sha256 = "6e6e1cd7a819695caae408f4fa938129ab7a86e83fe2410137c85e50131abbe0", + strip_prefix = "differential-privacy-3.0.0", + url = "https://github.com/google/differential-privacy/archive/refs/tags/v3.0.0.tar.gz", ) # This commit is determined by # https://github.com/tensorflow/tensorflow/blob/v2.16.1/third_party/absl/workspace.bzl#L10. http_archive( name = "com_google_absl", - url = "https://github.com/abseil/abseil-cpp/archive/fb3621f4f897824c0dbe0615fa94543df6192f30.tar.gz", sha256 = "0320586856674d16b0b7a4d4afb22151bdc798490bb7f295eddd8f6a62b46fea", strip_prefix = "abseil-cpp-fb3621f4f897824c0dbe0615fa94543df6192f30", + url = "https://github.com/abseil/abseil-cpp/archive/fb3621f4f897824c0dbe0615fa94543df6192f30.tar.gz", ) http_archive( name = "com_google_googletest", - url = "https://github.com/google/googletest/archive/refs/tags/release-1.12.1.tar.gz", sha256 = "81964fe578e9bd7c94dfdb09c8e4d6e6759e19967e397dbea48d1c10e45d0df2", strip_prefix = "googletest-release-1.12.1", + url = "https://github.com/google/googletest/archive/refs/tags/release-1.12.1.tar.gz", ) http_archive( name = "com_google_protobuf", - url = "https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.21.9.tar.gz", - sha256 = "1add10f9bd92775b91f326da259f243881e904dd509367d5031d4c782ba82810", - strip_prefix = "protobuf-3.21.9", + sha256 = "27ff8f361e7908b08ab93cb0c05562f6f92b51bca4f0c60638477264e1597125", + strip_prefix = "protobuf-26.1", + url = "https://github.com/protocolbuffers/protobuf/archive/refs/tags/v26.1.tar.gz", ) -# TODO: b/333391041 - Temporarily disable the direct dependency on -# `eigen`, for now we pick this dependency up from the TensorFlow workspace. +# TODO: b/333391041 - Temporarily disable the direct dependency on `eigen`, for +# now we pick this dependency up from the TensorFlow workspace. # http_archive( # name = "eigen", # url = "https://gitlab.com/libeigen/eigen/archive/refs/tags/3.4.0.tar.gz", @@ -91,55 +91,73 @@ http_archive( # https://github.com/google-parfait/tensorflow-federated/blob/main/requirements.txt. http_archive( name = "org_tensorflow", - url = "https://github.com/tensorflow/tensorflow/archive/refs/tags/v2.14.0.tar.gz", patches = [ "//third_party/tensorflow:internal_visibility.patch", "//third_party/tensorflow:python_toolchain.patch", "//third_party/tensorflow:tf2xla_visibility.patch", ], - sha256 = "ce357fd0728f0d1b0831d1653f475591662ec5bca736a94ff789e6b1944df19f", - strip_prefix = "tensorflow-2.14.0", + sha256 = "c729e56efc945c6df08efe5c9f5b8b89329c7c91b8f40ad2bb3e13900bd4876d", + strip_prefix = "tensorflow-2.16.1", + url = "https://github.com/tensorflow/tensorflow/archive/refs/tags/v2.16.1.tar.gz", ) # This commit is determined by # https://github.com/tensorflow/tensorflow/blob/v2.16.1/third_party/pybind11_abseil/workspace.bzl#L11. http_archive( name = "pybind11_abseil", - url = "https://github.com/pybind/pybind11_abseil/archive/2c4932ed6f6204f1656e245838f4f5eae69d2e29.tar.gz", sha256 = "0223b647b8cc817336a51e787980ebc299c8d5e64c069829bf34b69d72337449", strip_prefix = "pybind11_abseil-2c4932ed6f6204f1656e245838f4f5eae69d2e29", + url = "https://github.com/pybind/pybind11_abseil/archive/2c4932ed6f6204f1656e245838f4f5eae69d2e29.tar.gz", ) http_archive( name = "pybind11_bazel", - url = "https://github.com/pybind/pybind11_bazel/archive/refs/tags/v2.11.1.tar.gz", sha256 = "e8355ee56c2ff772334b4bfa22be17c709e5573f6d1d561c7176312156c27bd4", strip_prefix = "pybind11_bazel-2.11.1", + url = "https://github.com/pybind/pybind11_bazel/archive/refs/tags/v2.11.1.tar.gz", ) # This commit is determined by # https://github.com/tensorflow/tensorflow/blob/v2.16.1/tensorflow/workspace2.bzl#L788. http_archive( name = "pybind11_protobuf", - url = "https://github.com/pybind/pybind11_protobuf/archive/80f3440cd8fee124e077e2e47a8a17b78b451363.tar.gz", sha256 = "ba2c54a8b4d1dd0a68c58159e37b1f863c0d9d1dc815558288195493bcc31682", strip_prefix = "pybind11_protobuf-80f3440cd8fee124e077e2e47a8a17b78b451363", + url = "https://github.com/pybind/pybind11_protobuf/archive/80f3440cd8fee124e077e2e47a8a17b78b451363.tar.gz", ) http_archive( name = "rules_license", - url = "https://github.com/bazelbuild/rules_license/archive/refs/tags/0.0.8.tar.gz", sha256 = "8c1155797cb5f5697ea8c6eac6c154cf51aa020e368813d9d9b949558c84f2da", strip_prefix = "rules_license-0.0.8", + url = "https://github.com/bazelbuild/rules_license/archive/refs/tags/0.0.8.tar.gz", ) http_archive( name = "rules_python", - url = "https://github.com/bazelbuild/rules_python/archive/refs/tags/0.31.0.tar.gz", sha256 = "c68bdc4fbec25de5b5493b8819cfc877c4ea299c0dcb15c244c5a00208cde311", strip_prefix = "rules_python-0.31.0", + url = "https://github.com/bazelbuild/rules_python/archive/refs/tags/0.31.0.tar.gz", ) +# TODO: b/347940678 - Temporarily disable the direct dependency on `tsl`, for +# now we pick this dependency up from the TensorFlow workspace. +# http_archive( +# name = "tsl", +# url = "https://github.com/google/tsl/archive/5fdb99d897eadddd136c41156fbda82c6244d60a.tar.gz", +# sha256 = "414f9929d6a631aa225093e24e657e1a28c5e84da8e280f1636db65fe24e4b83", +# strip_prefix = "tsl-5fdb99d897eadddd136c41156fbda82c6244d60a", +# ) + +# TODO: b/347937714 - Temporarily disable the direct dependency on `xla`, for +# now we pick this dependency up from the TensorFlow workspace. +# http_archive( +# name = "xla", +# url = "https://github.com/openxla/xla/archive/940e3a27542b7ce76666173e7b287aa2a9263916.tar.gz", +# sha256 = "bcdc778e5a456839869dea796117b723bdea488075bd9555fe118fd8d6fcf25e", +# strip_prefix = "xla-940e3a27542b7ce76666173e7b287aa2a9263916", +# ) + # # Inlined transitive dependencies, grouped by direct dependency. # @@ -147,20 +165,23 @@ http_archive( # Required by pybind11_abseil and pybind11_protobuf. http_archive( name = "pybind11", - url = "https://github.com/pybind/pybind11/archive/refs/tags/v2.9.2.tar.gz", + build_file = "@pybind11_bazel//:pybind11.BUILD", sha256 = "6bd528c4dbe2276635dc787b6b1f2e5316cf6b49ee3e150264e455a0d68d19c1", strip_prefix = "pybind11-2.9.2", - build_file = "@pybind11_bazel//:pybind11.BUILD", + url = "https://github.com/pybind/pybind11/archive/refs/tags/v2.9.2.tar.gz", ) +# TODO: b/347937669 - Temporarily disable the direct dependency on +# `com_github_grpc_grpc`, for now we pick this dependency up from the +# TensorFlow workspace. # Required by com_github_grpc_grpc. This commit is determined by # https://github.com/grpc/grpc/blob/v1.50.0/bazel/grpc_deps.bzl#L344. -http_archive( - name = "upb", - url = "https://github.com/protocolbuffers/upb/archive/e4635f223e7d36dfbea3b722a4ca4807a7e882e2.tar.gz", - sha256 = "017a7e8e4e842d01dba5dc8aa316323eee080cd1b75986a7d1f94d87220e6502", - strip_prefix = "upb-e4635f223e7d36dfbea3b722a4ca4807a7e882e2", -) +# http_archive( +# name = "upb", +# url = "https://github.com/protocolbuffers/upb/archive/e4635f223e7d36dfbea3b722a4ca4807a7e882e2.tar.gz", +# sha256 = "017a7e8e4e842d01dba5dc8aa316323eee080cd1b75986a7d1f94d87220e6502", +# strip_prefix = "upb-e4635f223e7d36dfbea3b722a4ca4807a7e882e2", +# ) # # Transitive dependencies, grouped by direct dependency. diff --git a/pyproject.toml b/pyproject.toml index bcd14f72b9..c0bb397337 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,7 @@ dependencies = [ 'scipy~=1.9.3', 'tensorflow-model-optimization==0.7.5', 'tensorflow-privacy==0.9.0', - 'tensorflow>=2.14.0,==2.14.*', + 'tensorflow~=2.16.1', 'tqdm~=4.64', 'typing-extensions>=4.5.0,==4.5.*', # TODO: b/315515548 - Required because current dependencies are pulling in diff --git a/requirements.txt b/requirements.txt index cc247f214d..e1a45276e9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -31,7 +31,7 @@ cachetools~=5.3 dm-tree==0.1.8 dp-accounting==0.4.3 google-vizier==0.1.11 -grpcio~=1.46 +grpcio~=1.64 jaxlib==0.4.14 jax==0.4.14 numpy~=1.25 @@ -44,7 +44,7 @@ typing-extensions>=4.5.0,==4.5.* # The version of this dependency should match the version in # https://github.com/google-parfait/tensorflow-federated/blob/main/WORKSPACE. -tensorflow>=2.14.0,==2.14.* +tensorflow~=2.16.1 # TODO: b/315515548 - Required because current dependencies are pulling in later # versions which are causing opaque failures. diff --git a/tensorflow_federated/cc/core/impl/aggregation/tensorflow/BUILD b/tensorflow_federated/cc/core/impl/aggregation/tensorflow/BUILD index d5b3eb12b4..cb271235a4 100644 --- a/tensorflow_federated/cc/core/impl/aggregation/tensorflow/BUILD +++ b/tensorflow_federated/cc/core/impl/aggregation/tensorflow/BUILD @@ -29,10 +29,10 @@ cc_library( "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", + "@local_tsl//tsl/platform:refcount", "@org_tensorflow//tensorflow/core:framework", "@org_tensorflow//tensorflow/core:protos_all_cc", "@org_tensorflow//tensorflow/core/platform:tstring", - "@org_tensorflow//tensorflow/tsl/platform:refcount", ], ) @@ -130,8 +130,8 @@ cc_library( "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:cord", + "@local_tsl//tsl/platform:env", "@org_tensorflow//tensorflow/core/platform:env", - "@org_tensorflow//tensorflow/tsl/platform:env", ], ) @@ -165,8 +165,8 @@ cc_library( "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:cord", + "@local_tsl//tsl/platform:env", "@org_tensorflow//tensorflow/core/platform:env", - "@org_tensorflow//tensorflow/tsl/platform:env", ], ) diff --git a/tensorflow_federated/cc/core/impl/aggregation/tensorflow/converters.cc b/tensorflow_federated/cc/core/impl/aggregation/tensorflow/converters.cc index 3db0cab4f2..098bae0c5c 100644 --- a/tensorflow_federated/cc/core/impl/aggregation/tensorflow/converters.cc +++ b/tensorflow_federated/cc/core/impl/aggregation/tensorflow/converters.cc @@ -29,7 +29,7 @@ #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/framework/tensor_shape.pb.h" #include "tensorflow/core/platform/tstring.h" -#include "tensorflow/tsl/platform/refcount.h" +#include "tsl/platform/refcount.h" #include "tensorflow_federated/cc/core/impl/aggregation/base/monitoring.h" #include "tensorflow_federated/cc/core/impl/aggregation/core/datatype.h" #include "tensorflow_federated/cc/core/impl/aggregation/core/tensor.h" diff --git a/tensorflow_federated/cc/core/impl/aggregation/tensorflow/tensorflow_checkpoint_builder_factory.cc b/tensorflow_federated/cc/core/impl/aggregation/tensorflow/tensorflow_checkpoint_builder_factory.cc index 3903a74d36..1febef4025 100644 --- a/tensorflow_federated/cc/core/impl/aggregation/tensorflow/tensorflow_checkpoint_builder_factory.cc +++ b/tensorflow_federated/cc/core/impl/aggregation/tensorflow/tensorflow_checkpoint_builder_factory.cc @@ -29,7 +29,7 @@ #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "tensorflow/core/platform/file_system.h" -#include "tensorflow/tsl/platform/env.h" +#include "tsl/platform/env.h" #include "tensorflow_federated/cc/core/impl/aggregation/base/monitoring.h" #include "tensorflow_federated/cc/core/impl/aggregation/core/tensor.h" #include "tensorflow_federated/cc/core/impl/aggregation/protocol/checkpoint_builder.h" diff --git a/tensorflow_federated/cc/core/impl/aggregation/tensorflow/tensorflow_checkpoint_parser_factory.cc b/tensorflow_federated/cc/core/impl/aggregation/tensorflow/tensorflow_checkpoint_parser_factory.cc index dd21957730..ce736d63bb 100644 --- a/tensorflow_federated/cc/core/impl/aggregation/tensorflow/tensorflow_checkpoint_parser_factory.cc +++ b/tensorflow_federated/cc/core/impl/aggregation/tensorflow/tensorflow_checkpoint_parser_factory.cc @@ -29,7 +29,7 @@ #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "tensorflow/core/platform/file_system.h" -#include "tensorflow/tsl/platform/env.h" +#include "tsl/platform/env.h" #include "tensorflow_federated/cc/core/impl/aggregation/base/monitoring.h" #include "tensorflow_federated/cc/core/impl/aggregation/core/tensor.h" #include "tensorflow_federated/cc/core/impl/aggregation/protocol/checkpoint_parser.h" diff --git a/tensorflow_federated/cc/core/impl/executors/BUILD b/tensorflow_federated/cc/core/impl/executors/BUILD index b87b798e4b..7021ffd509 100644 --- a/tensorflow_federated/cc/core/impl/executors/BUILD +++ b/tensorflow_federated/cc/core/impl/executors/BUILD @@ -343,7 +343,6 @@ pybind_extension( "@org_tensorflow//tensorflow/core:framework", "@org_tensorflow//tensorflow/core/platform:status", "@org_tensorflow//tensorflow/core/platform:strcat", - "@org_tensorflow//tensorflow/python/lib/core:ndarray_tensor_headers", "@pybind11_abseil//pybind11_abseil:absl_casters", "@pybind11_abseil//pybind11_abseil:status_casters", "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", @@ -554,7 +553,6 @@ cc_library( "//tensorflow_federated/proto/v0:executor_cc_grpc_proto", "//tensorflow_federated/proto/v0:executor_cc_proto", "@com_github_grpc_grpc//:grpc++", - "@com_github_grpc_grpc//:grpc_security_base", "@com_google_absl//absl/strings", "@com_google_googletest//:gtest", ], @@ -1118,20 +1116,20 @@ cc_library( "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_absl//absl/types:span", + "@local_xla//xla:literal", + "@local_xla//xla:shape_util", + "@local_xla//xla:xla_data_proto_cc", + "@local_xla//xla:xla_proto_cc", + "@local_xla//xla/client", + "@local_xla//xla/client:client_library", + "@local_xla//xla/client:xla_computation", + "@local_xla//xla/service", + "@local_xla//xla/service:hlo_proto_cc", + "@local_xla//xla/stream_executor", + "@local_xla//xla/stream_executor:multi_platform_manager", + "@local_xla//xla/stream_executor/host:host_platform", # buildcleaner: keep # Linking in the host platform here ensures that the stream executor can execute on CPU. "@org_tensorflow//tensorflow/compiler/jit:xla_cpu_jit", # buildcleaner: keep # Linking in this dependency ensures that XLA can compile its code for the CPU host. "@org_tensorflow//tensorflow/compiler/tf2xla:common", - "@org_tensorflow//tensorflow/compiler/xla:literal", - "@org_tensorflow//tensorflow/compiler/xla:shape_util", - "@org_tensorflow//tensorflow/compiler/xla:xla_data_proto_cc", - "@org_tensorflow//tensorflow/compiler/xla:xla_proto_cc", - "@org_tensorflow//tensorflow/compiler/xla/client", - "@org_tensorflow//tensorflow/compiler/xla/client:client_library", - "@org_tensorflow//tensorflow/compiler/xla/client:xla_computation", - "@org_tensorflow//tensorflow/compiler/xla/service", - "@org_tensorflow//tensorflow/compiler/xla/service:hlo_proto_cc", - "@org_tensorflow//tensorflow/compiler/xla/stream_executor", - "@org_tensorflow//tensorflow/compiler/xla/stream_executor:multi_platform_manager", - "@org_tensorflow//tensorflow/compiler/xla/stream_executor/host:host_platform", # buildcleaner: keep # Linking in the host platform here ensures that the stream executor can execute on CPU. "@org_tensorflow//tensorflow/core:framework", "@org_tensorflow//tensorflow/core:protos_all_cc", ], @@ -1160,13 +1158,13 @@ tff_cc_cpu_gpu_test( "@com_google_absl//absl/log:check", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", + "@local_xla//xla:shape_util", + "@local_xla//xla:xla_data_proto_cc", + "@local_xla//xla/client:xla_builder", + "@local_xla//xla/client:xla_computation", + "@local_xla//xla/service:platform_util", + "@local_xla//xla/stream_executor", "@org_tensorflow//tensorflow/compiler/tf2xla:common", - "@org_tensorflow//tensorflow/compiler/xla:shape_util", - "@org_tensorflow//tensorflow/compiler/xla:xla_data_proto_cc", - "@org_tensorflow//tensorflow/compiler/xla/client:xla_builder", - "@org_tensorflow//tensorflow/compiler/xla/client:xla_computation", - "@org_tensorflow//tensorflow/compiler/xla/service:platform_util", - "@org_tensorflow//tensorflow/compiler/xla/stream_executor", "@org_tensorflow//tensorflow/core:framework", "@org_tensorflow//tensorflow/core:protos_all_cc", "@org_tensorflow//tensorflow/core/platform:statusor", @@ -1197,15 +1195,15 @@ cc_test( "@com_google_absl//absl/log:check", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", + "@local_xla//xla:shape_util", + "@local_xla//xla:xla_data_proto_cc", + "@local_xla//xla/client:xla_builder", + "@local_xla//xla/client:xla_computation", + "@local_xla//xla/service:platform_util", + "@local_xla//xla/stream_executor", + "@local_xla//xla/stream_executor/cuda:cuda_platform", # buildcleaner: keep # Linking in the host platform here ensures that the stream executor can execute on GPU. "@org_tensorflow//tensorflow/compiler/jit:xla_gpu_jit", # buildcleaner: keep # Linking in this dependency ensures that XLA can compile its code for the GPU. "@org_tensorflow//tensorflow/compiler/tf2xla:common", - "@org_tensorflow//tensorflow/compiler/xla:shape_util", - "@org_tensorflow//tensorflow/compiler/xla:xla_data_proto_cc", - "@org_tensorflow//tensorflow/compiler/xla/client:xla_builder", - "@org_tensorflow//tensorflow/compiler/xla/client:xla_computation", - "@org_tensorflow//tensorflow/compiler/xla/service:platform_util", - "@org_tensorflow//tensorflow/compiler/xla/stream_executor", - "@org_tensorflow//tensorflow/compiler/xla/stream_executor/cuda:cuda_platform", # buildcleaner: keep # Linking in the host platform here ensures that the stream executor can execute on GPU. "@org_tensorflow//tensorflow/core:framework", "@org_tensorflow//tensorflow/core:protos_all_cc", "@org_tensorflow//tensorflow/core/platform:statusor", @@ -1224,9 +1222,9 @@ cc_library( "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", - "@org_tensorflow//tensorflow/compiler/xla:literal", - "@org_tensorflow//tensorflow/compiler/xla:shape_util", - "@org_tensorflow//tensorflow/compiler/xla:types", + "@local_xla//xla:literal", + "@local_xla//xla:shape_util", + "@local_xla//xla:types", ], ) @@ -1245,10 +1243,10 @@ cc_test( "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@eigen_archive//:eigen3", - "@org_tensorflow//tensorflow/compiler/xla:literal", - "@org_tensorflow//tensorflow/compiler/xla:literal_util", - "@org_tensorflow//tensorflow/compiler/xla:shape_util", - "@org_tensorflow//tensorflow/compiler/xla:types", - "@org_tensorflow//tensorflow/compiler/xla:xla_data_proto_cc", + "@local_xla//xla:literal", + "@local_xla//xla:literal_util", + "@local_xla//xla:shape_util", + "@local_xla//xla:types", + "@local_xla//xla:xla_data_proto_cc", ], ) diff --git a/tensorflow_federated/cc/core/impl/executors/array_test_utils.h b/tensorflow_federated/cc/core/impl/executors/array_test_utils.h index 52bdf987d5..c018c2332a 100644 --- a/tensorflow_federated/cc/core/impl/executors/array_test_utils.h +++ b/tensorflow_federated/cc/core/impl/executors/array_test_utils.h @@ -23,7 +23,7 @@ limitations under the License #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" -#include "third_party/eigen3/Eigen/Core" +#include "Eigen/Core" #include "tensorflow_federated/proto/v0/array.pb.h" #include "tensorflow_federated/proto/v0/data_type.pb.h" diff --git a/tensorflow_federated/cc/core/impl/executors/tensorflow_utils.cc b/tensorflow_federated/cc/core/impl/executors/tensorflow_utils.cc index d2f10a7fea..0b128f789d 100644 --- a/tensorflow_federated/cc/core/impl/executors/tensorflow_utils.cc +++ b/tensorflow_federated/cc/core/impl/executors/tensorflow_utils.cc @@ -24,7 +24,7 @@ limitations under the License #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" -#include "third_party/eigen3/Eigen/Core" +#include "Eigen/Core" #include "tensorflow/core/framework/attr_value.pb.h" #include "tensorflow/core/framework/graph.pb.h" #include "tensorflow/core/framework/node_def.pb.h" diff --git a/tensorflow_federated/cc/core/impl/executors/tensorflow_utils_test.cc b/tensorflow_federated/cc/core/impl/executors/tensorflow_utils_test.cc index 73d44bdcff..fd2e0b8465 100644 --- a/tensorflow_federated/cc/core/impl/executors/tensorflow_utils_test.cc +++ b/tensorflow_federated/cc/core/impl/executors/tensorflow_utils_test.cc @@ -22,7 +22,7 @@ limitations under the License #include "googletest/include/gtest/gtest.h" #include "absl/status/status.h" #include "absl/status/statusor.h" -#include "third_party/eigen3/Eigen/Core" +#include "Eigen/Core" #include "tensorflow/core/framework/numeric_types.h" #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/framework/tensor_testutil.h" diff --git a/tensorflow_federated/cc/core/impl/executors/xla_executor.cc b/tensorflow_federated/cc/core/impl/executors/xla_executor.cc index 483b6e25ca..5c70ec8527 100644 --- a/tensorflow_federated/cc/core/impl/executors/xla_executor.cc +++ b/tensorflow_federated/cc/core/impl/executors/xla_executor.cc @@ -33,16 +33,16 @@ limitations under the License #include "absl/types/span.h" #include "tensorflow/compiler/tf2xla/literal_util.h" #include "tensorflow/compiler/tf2xla/type_util.h" -#include "tensorflow/compiler/xla/client/client.h" -#include "tensorflow/compiler/xla/client/client_library.h" -#include "tensorflow/compiler/xla/client/xla_computation.h" -#include "tensorflow/compiler/xla/literal.h" -#include "tensorflow/compiler/xla/service/hlo.pb.h" -#include "tensorflow/compiler/xla/service/service.h" -#include "tensorflow/compiler/xla/shape.h" -#include "tensorflow/compiler/xla/stream_executor/platform.h" -#include "tensorflow/compiler/xla/xla.pb.h" -#include "tensorflow/compiler/xla/xla_data.pb.h" +#include "xla/client/client.h" +#include "xla/client/client_library.h" +#include "xla/client/xla_computation.h" +#include "xla/literal.h" +#include "xla/service/hlo.pb.h" +#include "xla/service/service.h" +#include "xla/shape.h" +#include "xla/stream_executor/platform.h" +#include "xla/xla.pb.h" +#include "xla/xla_data.pb.h" #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor.pb.h" #include "tensorflow/core/framework/tensor_shape.h" @@ -58,7 +58,7 @@ limitations under the License // clang-format off // In TF 2.17 MultiPlatformManager was renamed to PlatformManager. Remove // this code when the OSS build gets updated to TF 2.17+. -#include "tensorflow/compiler/xla/stream_executor/multi_platform_manager.h" +#include "xla/stream_executor/multi_platform_manager.h" namespace stream_executor { using PlatformManager = MultiPlatformManager; } // namespace stream_executor diff --git a/tensorflow_federated/cc/core/impl/executors/xla_executor_test.cc b/tensorflow_federated/cc/core/impl/executors/xla_executor_test.cc index 7150bad17a..9b2cee8fed 100644 --- a/tensorflow_federated/cc/core/impl/executors/xla_executor_test.cc +++ b/tensorflow_federated/cc/core/impl/executors/xla_executor_test.cc @@ -32,13 +32,13 @@ limitations under the License #include "absl/status/statusor.h" #include "tensorflow/compiler/tf2xla/shape_util.h" #include "tensorflow/compiler/tf2xla/type_util.h" -#include "tensorflow/compiler/xla/client/xla_builder.h" -#include "tensorflow/compiler/xla/client/xla_computation.h" -#include "tensorflow/compiler/xla/service/platform_util.h" -#include "tensorflow/compiler/xla/shape.h" -#include "tensorflow/compiler/xla/shape_util.h" -#include "tensorflow/compiler/xla/stream_executor/platform.h" -#include "tensorflow/compiler/xla/xla_data.pb.h" +#include "xla/client/xla_builder.h" +#include "xla/client/xla_computation.h" +#include "xla/service/platform_util.h" +#include "xla/shape.h" +#include "xla/shape_util.h" +#include "xla/stream_executor/platform.h" +#include "xla/xla_data.pb.h" #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/framework/types.pb.h" #include "tensorflow/core/platform/statusor.h" diff --git a/tensorflow_federated/cc/core/impl/executors/xla_utils.cc b/tensorflow_federated/cc/core/impl/executors/xla_utils.cc index ab94b79db9..c421f72bb7 100644 --- a/tensorflow_federated/cc/core/impl/executors/xla_utils.cc +++ b/tensorflow_federated/cc/core/impl/executors/xla_utils.cc @@ -22,10 +22,10 @@ limitations under the License #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" -#include "tensorflow/compiler/xla/literal.h" -#include "tensorflow/compiler/xla/shape.h" -#include "tensorflow/compiler/xla/shape_util.h" -#include "tensorflow/compiler/xla/types.h" +#include "xla/literal.h" +#include "xla/shape.h" +#include "xla/shape_util.h" +#include "xla/types.h" #include "tensorflow_federated/cc/core/impl/executors/status_macros.h" #include "tensorflow_federated/proto/v0/array.pb.h" #include "tensorflow_federated/proto/v0/computation.pb.h" diff --git a/tensorflow_federated/cc/core/impl/executors/xla_utils.h b/tensorflow_federated/cc/core/impl/executors/xla_utils.h index 7a77b90059..80ea387a62 100644 --- a/tensorflow_federated/cc/core/impl/executors/xla_utils.h +++ b/tensorflow_federated/cc/core/impl/executors/xla_utils.h @@ -17,8 +17,8 @@ limitations under the License #define THIRD_PARTY_TENSORFLOW_FEDERATED_CC_CORE_IMPL_EXECUTORS_XLA_UTILS_H_ #include "absl/status/statusor.h" -#include "tensorflow/compiler/xla/literal.h" -#include "tensorflow/compiler/xla/shape.h" +#include "xla/literal.h" +#include "xla/shape.h" #include "tensorflow_federated/proto/v0/array.pb.h" #include "tensorflow_federated/proto/v0/computation.pb.h" #include "tensorflow_federated/proto/v0/data_type.pb.h" diff --git a/tensorflow_federated/cc/core/impl/executors/xla_utils_test.cc b/tensorflow_federated/cc/core/impl/executors/xla_utils_test.cc index bab406f0e8..98c3befc04 100644 --- a/tensorflow_federated/cc/core/impl/executors/xla_utils_test.cc +++ b/tensorflow_federated/cc/core/impl/executors/xla_utils_test.cc @@ -21,14 +21,14 @@ limitations under the License #include "googletest/include/gtest/gtest.h" #include "absl/status/status.h" #include "absl/status/statusor.h" -#include "third_party/eigen3/Eigen/Core" -#include "tensorflow/compiler/xla/literal.h" -#include "tensorflow/compiler/xla/literal_util.h" -#include "tensorflow/compiler/xla/primitive_util.h" -#include "tensorflow/compiler/xla/shape.h" -#include "tensorflow/compiler/xla/shape_util.h" -#include "tensorflow/compiler/xla/types.h" -#include "tensorflow/compiler/xla/xla_data.pb.h" +#include "Eigen/Core" +#include "xla/literal.h" +#include "xla/literal_util.h" +#include "xla/primitive_util.h" +#include "xla/shape.h" +#include "xla/shape_util.h" +#include "xla/types.h" +#include "xla/xla_data.pb.h" #include "tensorflow_federated/cc/core/impl/executors/array_shape_test_utils.h" #include "tensorflow_federated/cc/core/impl/executors/array_test_utils.h" #include "tensorflow_federated/cc/testing/status_matchers.h" diff --git a/third_party/tensorflow/internal_visibility.patch b/third_party/tensorflow/internal_visibility.patch index 7703001ef6..a1cea88be3 100644 --- a/third_party/tensorflow/internal_visibility.patch +++ b/third_party/tensorflow/internal_visibility.patch @@ -1,13 +1,13 @@ diff --git tensorflow/BUILD tensorflow/BUILD -index 202553cd531..171eb04665c 100644 +index 5018c6e642e..cf3a1ca0760 100644 --- tensorflow/BUILD +++ tensorflow/BUILD -@@ -1039,7 +1039,7 @@ package_group( +@@ -1057,7 +1057,7 @@ package_group( "//smartass/brain/configure/...", "//tensorflow/...", "//tensorflow_decision_forests/...", - "//tensorflow_federated/...", + "public", + "//tensorflow_models/google/recml/...", "//third_party/cloud_tpu/convergence_tools/sdc_monitoring/...", "//third_party/cloud_tpu/inference_converter/...", - "//third_party/py/cloud_ml_autoflow/...", diff --git a/third_party/tensorflow/python_toolchain.patch b/third_party/tensorflow/python_toolchain.patch index 4d732d447d..a2d60d1b4a 100644 --- a/third_party/tensorflow/python_toolchain.patch +++ b/third_party/tensorflow/python_toolchain.patch @@ -1,5 +1,5 @@ diff --git tensorflow/tools/toolchains/cpus/aarch64/aarch64_compiler_configure.bzl tensorflow/tools/toolchains/cpus/aarch64/aarch64_compiler_configure.bzl -index a2bdd6a7eed..ec25c23d8d4 100644 +index 9261a652f9c..0557e345ab1 100644 --- tensorflow/tools/toolchains/cpus/aarch64/aarch64_compiler_configure.bzl +++ tensorflow/tools/toolchains/cpus/aarch64/aarch64_compiler_configure.bzl @@ -2,7 +2,7 @@ @@ -24,23 +24,23 @@ index 9f71a414bf7..57f70752323 100644 load("//third_party/nccl:nccl_configure.bzl", "remote_nccl_configure") load("//third_party/gpus:rocm_configure.bzl", "remote_rocm_configure") diff --git tensorflow/workspace2.bzl tensorflow/workspace2.bzl -index 7e9faa558a4..5b18cb0969a 100644 +index 9e15c75c183..facc1de3d73 100644 --- tensorflow/workspace2.bzl +++ tensorflow/workspace2.bzl -@@ -8,7 +8,7 @@ load("//third_party/gpus:rocm_configure.bzl", "rocm_configure") - load("//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") +@@ -37,7 +37,7 @@ load("//third_party/nasm:workspace.bzl", nasm = "repo") load("//third_party/nccl:nccl_configure.bzl", "nccl_configure") - load("//third_party/git:git_configure.bzl", "git_configure") + load("//third_party/opencl_headers:workspace.bzl", opencl_headers = "repo") + load("//third_party/pasta:workspace.bzl", pasta = "repo") -load("//third_party/py:python_configure.bzl", "python_configure") +load("//third_party/py/non_hermetic:python_configure.bzl", "python_configure") - load("//third_party/systemlibs:syslibs_configure.bzl", "syslibs_configure") - load("//tensorflow/tools/toolchains:cpus/aarch64/aarch64_compiler_configure.bzl", "aarch64_compiler_configure") - load("//tensorflow/tools/toolchains:cpus/arm/arm_compiler_configure.bzl", "arm_compiler_configure") + load("//third_party/py/ml_dtypes:workspace.bzl", ml_dtypes = "repo") + load("//third_party/pybind11_abseil:workspace.bzl", pybind11_abseil = "repo") + load("//third_party/pybind11_bazel:workspace.bzl", pybind11_bazel = "repo") diff --git third_party/py/non_hermetic/python_configure.bzl third_party/py/non_hermetic/python_configure.bzl -index 300cbfb6c71..09d98505dd9 100644 +index 89732c3e33d..4ac1c8f5c04 100644 --- third_party/py/non_hermetic/python_configure.bzl +++ third_party/py/non_hermetic/python_configure.bzl -@@ -206,7 +206,7 @@ def _create_local_python_repository(repository_ctx): +@@ -203,7 +203,7 @@ def _create_local_python_repository(repository_ctx): # Resolve all labels before doing any real work. Resolving causes the # function to be restarted with all previous state being lost. This # can easily lead to a O(n^2) runtime in the number of labels. @@ -72,3 +72,17 @@ index 97c7907fc38..c80cc5287bc 100644 ) genrule( +diff --git third_party/py/python_configure.bzl third_party/py/python_configure.bzl +index 3728a91b931..ecb0ecb60e8 100644 +--- third_party/py/python_configure.bzl ++++ third_party/py/python_configure.bzl +@@ -14,7 +14,7 @@ def _create_local_python_repository(repository_ctx): + # Resolve all labels before doing any real work. Resolving causes the + # function to be restarted with all previous state being lost. This + # can easily lead to a O(n^2) runtime in the number of labels. +- build_tpl = repository_ctx.path(Label("//third_party/py:BUILD.tpl")) ++ build_tpl = repository_ctx.path(Label("//third_party/py/non_hermetic:BUILD.tpl")) + platform_constraint = "" + if repository_ctx.attr.platform_constraint: + platform_constraint = "\"%s\"" % repository_ctx.attr.platform_constraint +(END) diff --git a/third_party/tensorflow/tf2xla_visibility.patch b/third_party/tensorflow/tf2xla_visibility.patch index 84b7cdcab3..51f6fdf219 100644 --- a/third_party/tensorflow/tf2xla_visibility.patch +++ b/third_party/tensorflow/tf2xla_visibility.patch @@ -1,8 +1,8 @@ diff --git tensorflow/compiler/tf2xla/BUILD tensorflow/compiler/tf2xla/BUILD -index 22d9877bed9..3f6b421465d 100644 +index c1a2243de20..f4e24260aea 100644 --- tensorflow/compiler/tf2xla/BUILD +++ tensorflow/compiler/tf2xla/BUILD -@@ -46,7 +46,7 @@ package_group( +@@ -47,7 +47,7 @@ package_group( packages = [ "//platforms/performance/automl/...", "//tensorflow/...",