Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix azure logging #6860

Draft
wants to merge 4 commits into
base: master
Choose a base branch
from
Draft
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 31 additions & 20 deletions tensorboard/compat/tensorflow_stub/io/gfile.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
TensorBoard. This allows running TensorBoard without depending on
TensorFlow for file operations.
"""

import dataclasses
import glob as py_glob
import io
Expand Down Expand Up @@ -77,11 +76,12 @@ def get_filesystem(filename):
if index >= 0:
prefix = filename[:index]
fs = _REGISTERED_FILESYSTEMS.get(prefix, None)
fs_class = None
if fs is None:
fs = _get_fsspec_filesystem(filename)
fs, fs_class = _get_fsspec_filesystem(filename)
if fs is None:
raise ValueError("No recognized filesystem for prefix %s" % prefix)
return fs
return fs, fs_class


@dataclasses.dataclass(frozen=True)
Expand Down Expand Up @@ -639,6 +639,12 @@ def stat(self, filename):

_FSSPEC_FILESYSTEM = FSSpecFileSystem()

def _get_fs_class(filename):
"""Returns the filesystem class for the given filename."""
filename = compat.as_str_any(filename)
segment = filename.partition(FSSpecFileSystem.CHAIN_SEPARATOR)[0]
protocol = segment.partition(FSSpecFileSystem.SEPARATOR)[0]
return fsspec.get_filesystem_class(protocol)

def _get_fsspec_filesystem(filename):
"""
Expand All @@ -647,13 +653,9 @@ def _get_fsspec_filesystem(filename):
"""
if not FSSPEC_ENABLED:
return None

segment = filename.partition(FSSpecFileSystem.CHAIN_SEPARATOR)[0]
protocol = segment.partition(FSSpecFileSystem.SEPARATOR)[0]
if fsspec.get_filesystem_class(protocol):
return _FSSPEC_FILESYSTEM
else:
return None
fs_class = _get_fs_class(filename)
fs_wrapper = _FSSPEC_FILESYSTEM if fs_class else None
return fs_wrapper, fs_class


register_filesystem("", LocalFileSystem())
Expand All @@ -670,8 +672,11 @@ def __init__(self, filename, mode):
"mode {} not supported by compat GFile".format(mode)
)
self.filename = compat.as_bytes(filename)
self.fs = get_filesystem(self.filename)
self.fs_supports_append = hasattr(self.fs, "append")
self.fs, self.fs_class = get_filesystem(self.filename)
if "Azure" in str(self.fs_class):
self.fs_supports_append = False
else:
self.fs_supports_append = hasattr(self.fs, "append")
self.buff = None
# The buffer offset and the buffer chunk size are measured in the
# natural units of the underlying stream, i.e. bytes for binary mode,
Expand Down Expand Up @@ -850,7 +855,8 @@ def exists(filename):
Raises:
errors.OpError: Propagates any errors reported by the FileSystem API.
"""
return get_filesystem(filename).exists(filename)
fs, _ = get_filesystem(filename)
return fs.exists(filename)


def glob(filename):
Expand All @@ -865,7 +871,8 @@ def glob(filename):
Raises:
errors.OpError: If there are filesystem / directory listing errors.
"""
return get_filesystem(filename).glob(filename)
fs, _ = get_filesystem(filename)
return fs.glob(filename)


def isdir(dirname):
Expand All @@ -877,7 +884,8 @@ def isdir(dirname):
Returns:
True, if the path is a directory; False otherwise
"""
return get_filesystem(dirname).isdir(dirname)
fs, _ = get_filesystem(dirname)
return fs.isdir(dirname)


def listdir(dirname):
Expand All @@ -895,7 +903,8 @@ def listdir(dirname):
Raises:
errors.NotFoundError if directory doesn't exist
"""
return get_filesystem(dirname).listdir(dirname)
fs, _ = get_filesystem(dirname)
return fs.listdir(dirname)


def makedirs(path):
Expand All @@ -906,7 +915,8 @@ def makedirs(path):
Args:
path: string, name of the directory to be created
"""
return get_filesystem(path).makedirs(path)
fs, _ = get_filesystem(path)
return fs.makedirs(path)


def walk(top, topdown=True, onerror=None):
Expand All @@ -927,7 +937,7 @@ def walk(top, topdown=True, onerror=None):
as strings
"""
top = compat.as_str_any(top)
fs = get_filesystem(top)
fs, _ = get_filesystem(top)
try:
listing = listdir(top)
except errors.NotFoundError as err:
Expand Down Expand Up @@ -971,7 +981,8 @@ def stat(filename):
Raises:
errors.OpError: If the operation fails.
"""
return get_filesystem(filename).stat(filename)
fs, _ = get_filesystem(filename)
return fs.stat(filename)


# Used for tests only
Expand Down Expand Up @@ -1009,4 +1020,4 @@ def _read_file_to_string(filename, binary_mode=False):
f = GFile(filename, mode="rb")
else:
f = GFile(filename, mode="r")
return f.read()
return f.read()