[lint] wip

Makefile

@@ -39,8 +39,7 @@ check-all: check-hail check-services
 
 .PHONY: check-hail-fast
 check-hail-fast:
-	ruff check hail/python/hail
-	ruff check hail/python/hailtop
+	ruff check hail
 	ruff format hail --diff
 	$(PYTHON) -m pyright hail/python/hailtop
 

hail/python/hail/__init__.py

@@ -1,6 +1,7 @@
+import sys
 from typing import Optional
+
 import pkg_resources
-import sys
 
 if sys.version_info < (3, 9):
     raise EnvironmentError(
@@ -31,64 +32,66 @@
 # F403 'from .expr import *' used; unable to detect undefined names
 # F401 '.expr.*' imported but unused
 # E402 module level import not at top of file
-from .expr import *  # noqa: E402, F403
-from .expr import aggregators  # noqa: E402
 from hail.utils import (  # noqa: E402
-    Struct,
+    ANY_REGION,
     Interval,
+    Struct,
+    copy_log,
     hadoop_copy,
-    hadoop_open,
-    hadoop_ls,
-    hadoop_stat,
     hadoop_exists,
-    hadoop_is_file,
     hadoop_is_dir,
+    hadoop_is_file,
+    hadoop_ls,
+    hadoop_open,
     hadoop_scheme_supported,
-    copy_log,
-    ANY_REGION,
+    hadoop_stat,
 )
-from .table import Table, GroupedTable, asc, desc  # noqa: E402
-from .matrixtable import MatrixTable, GroupedMatrixTable  # noqa: E402
-from .genetics import *  # noqa: F403, E402
-from .methods import *  # noqa: F403, E402
-from . import expr  # noqa: E402
-from . import genetics  # noqa: E402
-from . import methods  # noqa: E402
-from . import stats  # noqa: E402
-from . import linalg  # noqa: E402
-from . import plot  # noqa: E402
-from . import ggplot  # noqa: E402
-from . import experimental  # noqa: E402
-from . import ir  # noqa: E402
-from . import backend  # noqa: E402
-from . import nd  # noqa: E402
-from . import utils  # noqa: E402
-from . import vds  # noqa: E402
 
+from . import (  # noqa: E402
+    backend,
+    experimental,
+    expr,
+    genetics,
+    ggplot,
+    ir,
+    linalg,
+    methods,
+    nd,
+    plot,
+    stats,
+    utils,
+    vds,
+)
 from .context import (  # noqa: E402
-    init,
-    init_local,
-    init_batch,
-    stop,
-    spark_context,
-    tmp_dir,
-    default_reference,
-    get_reference,
-    set_global_seed,
-    reset_global_randomness,
-    _set_flags,
+    TemporaryDirectory,
+    TemporaryFilename,
+    _async_current_backend,
     _get_flags,
+    _set_flags,
     _with_flags,
-    _async_current_backend,
-    current_backend,
-    debug_info,
     citation,
     cite_hail,
     cite_hail_bibtex,
+    current_backend,
+    debug_info,
+    default_reference,
+    get_reference,
+    init,
+    init_batch,
+    init_local,
+    reset_global_randomness,
+    set_global_seed,
+    spark_context,
+    stop,
+    tmp_dir,
     version,
-    TemporaryFilename,
-    TemporaryDirectory,
 )
+from .expr import *  # noqa: E402, F403
+from .expr import aggregators  # noqa: E402
+from .genetics import *  # noqa: F403, E402
+from .matrixtable import GroupedMatrixTable, MatrixTable  # noqa: E402
+from .methods import *  # noqa: F403, E402
+from .table import GroupedTable, Table, asc, desc  # noqa: E402
 
 agg = aggregators
 scan = aggregators.aggregators.ScanFunctions({name: getattr(agg, name) for name in agg.__all__})

hail/python/hail/backend/backend.py

@@ -1,20 +1,21 @@
-from typing import Mapping, List, Union, TypeVar, Tuple, Dict, Optional, Any, AbstractSet
 import abc
-from enum import Enum
-from dataclasses import dataclass
 import warnings
+import zipfile
+from dataclasses import dataclass
+from enum import Enum
+from typing import AbstractSet, Any, ClassVar, Dict, List, Mapping, Optional, Tuple, TypeVar, Union
+
 import orjson
 import pkg_resources
-import zipfile
 
 from hailtop.config.user_config import unchecked_configuration_of
 from hailtop.fs.fs import FS
 
 from ..builtin_references import BUILTIN_REFERENCE_RESOURCE_PATHS
 from ..expr import Expression
-from ..expr.table_type import ttable
-from ..expr.matrix_type import tmatrix
 from ..expr.blockmatrix_type import tblockmatrix
+from ..expr.matrix_type import tmatrix
+from ..expr.table_type import ttable
 from ..expr.types import HailType, dtype, tvoid
 from ..ir import BaseIR, finalize_randomness
 from ..ir.renderer import CSERenderer
@@ -23,7 +24,6 @@
 from ..table import Table
 from ..utils.java import FatalError
 
-
 Dataset = TypeVar('Dataset', Table, MatrixTable, BlockMatrix)
 
 
@@ -139,7 +139,7 @@ class FromFASTAFilePayload(ActionPayload):
 
 class Backend(abc.ABC):
     # Must match knownFlags in HailFeatureFlags.scala
-    _flags_env_vars_and_defaults: Dict[str, Tuple[str, Optional[str]]] = {
+    _flags_env_vars_and_defaults: ClassVar[Dict[str, Tuple[str, Optional[str]]]] = {
         "no_whole_stage_codegen": ("HAIL_DEV_NO_WHOLE_STAGE_CODEGEN", None),
         "no_ir_logging": ("HAIL_DEV_NO_IR_LOG", None),
         "lower": ("HAIL_DEV_LOWER", None),

hail/python/hail/backend/local_backend.py

@@ -1,19 +1,19 @@
-from typing import Optional, Union, Tuple, List
 import os
 import sys
+from typing import List, Optional, Tuple, Union
 
-from py4j.java_gateway import JavaGateway, GatewayParameters, launch_gateway
+from py4j.java_gateway import GatewayParameters, JavaGateway, launch_gateway
 
-from hail.ir.renderer import CSERenderer
 from hail.ir import finalize_randomness
-from .py4j_backend import Py4JBackend, uninstall_exception_handler
-from .backend import local_jar_information
+from hail.ir.renderer import CSERenderer
+from hailtop.aiotools.validators import validate_file
+from hailtop.fs.router_fs import RouterFS
+from hailtop.utils import find_spark_home
+
 from ..expr import Expression
 from ..expr.types import HailType
-
-from hailtop.utils import find_spark_home
-from hailtop.fs.router_fs import RouterFS
-from hailtop.aiotools.validators import validate_file
+from .backend import local_jar_information
+from .py4j_backend import Py4JBackend, uninstall_exception_handler
 
 
 class LocalBackend(Py4JBackend):

hail/python/hail/backend/py4j_backend.py

@@ -1,24 +1,23 @@
-from typing import Mapping, Set, Tuple
 import abc
+import http.client
 import socket
 import socketserver
 import sys
 from threading import Thread
+from typing import Mapping, Set, Tuple
 
 import orjson
-import requests
 import py4j
+import requests
 from py4j.java_gateway import JavaObject, JVMView
 
 import hail
 from hail.expr import construct_expr
 from hail.ir import JavaIR
-from hail.utils.java import FatalError, Env, scala_package_object
+from hail.utils.java import Env, FatalError, scala_package_object
 
-from .backend import ActionTag, Backend, fatal_error_from_java_error_triplet
 from ..hail_logging import Logger
-
-import http.client
+from .backend import ActionTag, Backend, fatal_error_from_java_error_triplet
 
 # This defaults to 65536 and fails if a header is longer than _MAXLINE
 # The timing json that we output can exceed 65536 bytes so we raise the limit
@@ -43,7 +42,6 @@ def install_exception_handler():
 
 def uninstall_exception_handler():
     global _installed
-    global _original
     if _installed:
         _installed = False
         py4j.protocol.get_return_value = _original

hail/python/hail/backend/service_backend.py

@@ -1,40 +1,39 @@
-from typing import Dict, Optional, Awaitable, Mapping, Any, List, Union, Tuple, TypeVar, Set
 import abc
 import asyncio
-from dataclasses import dataclass
+import logging
 import math
 import struct
-from hail.expr.expressions.base_expression import Expression
-import orjson
-import logging
-from contextlib import AsyncExitStack
 import warnings
+from contextlib import AsyncExitStack
+from dataclasses import dataclass
+from typing import Any, Awaitable, Dict, List, Mapping, Optional, Set, Tuple, TypeVar, Union
 
-from hail.context import TemporaryDirectory, TemporaryFilename, tmp_dir, revision, version
-from hail.utils import FatalError
-from hail.expr.types import HailType
+import orjson
+
+import hailtop.aiotools.fs as afs
+from hail.context import TemporaryDirectory, TemporaryFilename, revision, tmp_dir, version
 from hail.experimental import read_expression, write_expression
+from hail.expr.expressions.base_expression import Expression
+from hail.expr.types import HailType
 from hail.ir import finalize_randomness
 from hail.ir.renderer import CSERenderer
-
+from hail.utils import FatalError
 from hailtop import yamlx
-from hailtop.config import ConfigVariable, configuration_of, get_remote_tmpdir
-from hailtop.hail_event_loop import hail_event_loop
-from hailtop.utils import async_to_blocking, Timings, am_i_interactive, retry_transient_errors
-from hailtop.utils.rich_progress_bar import BatchProgressBar
-from hailtop.batch_client.aioclient import Batch, BatchClient
-from hailtop.aiotools.router_fs import RouterAsyncFS
 from hailtop.aiocloud.aiogoogle import GCSRequesterPaysConfiguration, get_gcs_requester_pays_configuration
-import hailtop.aiotools.fs as afs
+from hailtop.aiotools.fs.exceptions import UnexpectedEOFError
+from hailtop.aiotools.router_fs import RouterAsyncFS
+from hailtop.aiotools.validators import validate_file
+from hailtop.batch_client.aioclient import Batch, BatchClient
+from hailtop.config import ConfigVariable, configuration_of, get_remote_tmpdir
 from hailtop.fs.fs import FS
 from hailtop.fs.router_fs import RouterFS
-from hailtop.aiotools.fs.exceptions import UnexpectedEOFError
+from hailtop.hail_event_loop import hail_event_loop
+from hailtop.utils import Timings, am_i_interactive, async_to_blocking, retry_transient_errors
+from hailtop.utils.rich_progress_bar import BatchProgressBar
 
-from .backend import Backend, fatal_error_from_java_error_triplet, ActionTag, ActionPayload, ExecutePayload
 from ..builtin_references import BUILTIN_REFERENCES
 from ..utils import ANY_REGION
-from hailtop.aiotools.validators import validate_file
-
+from .backend import ActionPayload, ActionTag, Backend, ExecutePayload, fatal_error_from_java_error_triplet
 
 ReferenceGenomeConfig = Dict[str, Any]
 

hail/python/hail/backend/spark_backend.py

@@ -1,10 +1,10 @@
-import sys
 import os
-import pyspark
-import pyspark.sql
+import sys
+from typing import Optional
 
 import orjson
-from typing import Optional
+import pyspark
+import pyspark.sql
 
 from hail.expr.table_type import ttable
 from hail.fs.hadoop_fs import HadoopFS
@@ -13,8 +13,8 @@
 from hailtop.aiotools.router_fs import RouterAsyncFS
 from hailtop.aiotools.validators import validate_file
 
-from .py4j_backend import Py4JBackend
 from .backend import local_jar_information
+from .py4j_backend import Py4JBackend
 
 
 def append_to_comma_separated_list(conf: pyspark.SparkConf, k: str, *new_values: str):

hail/python/hail/context.py

@@ -1,27 +1,28 @@
-from typing import Optional, Union, Tuple, Type, List, Dict
-from types import TracebackType
-import warnings
-import sys
 import os
+import sys
+import warnings
 from contextlib import contextmanager
-from urllib.parse import urlparse, urlunparse
 from random import Random
+from types import TracebackType
+from typing import Dict, List, Optional, Tuple, Type, Union
+from urllib.parse import urlparse, urlunparse
 
 import pkg_resources
 from pyspark import SparkContext
 
 import hail
+from hail.backend import Backend
 from hail.genetics.reference_genome import ReferenceGenome, reference_genome_type
-from hail.typecheck import nullable, typecheck, typecheck_method, enumeration, dictof, oneof, sized_tupleof, sequenceof
+from hail.typecheck import dictof, enumeration, nullable, oneof, sequenceof, sized_tupleof, typecheck, typecheck_method
 from hail.utils import get_env_or_default
-from hail.utils.java import Env, warning, choose_backend
-from hail.backend import Backend
+from hail.utils.java import Env, choose_backend, warning
+from hailtop.aiocloud.aiogoogle import GCSRequesterPaysConfiguration, get_gcs_requester_pays_configuration
+from hailtop.fs.fs import FS
 from hailtop.hail_event_loop import hail_event_loop
 from hailtop.utils import secret_alnum_string
-from hailtop.fs.fs import FS
-from hailtop.aiocloud.aiogoogle import GCSRequesterPaysConfiguration, get_gcs_requester_pays_configuration
-from .builtin_references import BUILTIN_REFERENCES
+
 from .backend.backend import local_jar_information
+from .builtin_references import BUILTIN_REFERENCES
 
 
 def _get_tmpdir(tmpdir):
@@ -598,8 +599,8 @@ def init_local(
     _optimizer_iterations=None,
     gcs_requester_pays_configuration: Optional[GCSRequesterPaysConfiguration] = None,
 ):
-    from hail.backend.py4j_backend import connect_logger
     from hail.backend.local_backend import LocalBackend
+    from hail.backend.py4j_backend import connect_logger
 
     log = _get_log(log)
     tmpdir = _get_tmpdir(tmpdir)
@@ -961,8 +962,8 @@ def _with_flags(**flags):
 
 
 def debug_info():
-    from hail.backend.spark_backend import SparkBackend
     from hail.backend.backend import local_jar_information
+    from hail.backend.spark_backend import SparkBackend
 
     spark_conf = None
     if isinstance(Env.backend(), SparkBackend):

hail/python/hail/docs/conf.py

@@ -17,10 +17,10 @@
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #
 
+import datetime
 import os
 import re
 import sys
-import datetime
 
 sys.path.insert(0, os.path.abspath('./_ext'))