Rectify nearly all type errors revealed by mypy on current repo

botocore/__init__.py

@@ -61,15 +61,15 @@ def emit(self, record):
 
 
 # Used to specify anonymous (unsigned) request signature
-class UNSIGNED:
+class _UNSIGNED:
     def __copy__(self):
         return self
 
     def __deepcopy__(self, memodict):
         return self
 
 
-UNSIGNED = UNSIGNED()
+UNSIGNED = _UNSIGNED()
 
 
 def xform_name(name, sep='_', _xform_cache=_xform_cache):

botocore/client.py

@@ -11,6 +11,7 @@
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
 import logging
+from typing import Dict
 
 from botocore import waiter, xform_name
 from botocore.args import ClientArgsCreator
@@ -864,7 +865,7 @@ class BaseClient:
     # snake_case name, but we need to know the ListObjects form.
     # xform_name() does the ListObjects->list_objects conversion, but
     # we need the reverse mapping here.
-    _PY_TO_OP_NAME = {}
+    _PY_TO_OP_NAME: Dict[str, str] = {}
 
     def __init__(
         self,

botocore/compat.py

@@ -13,54 +13,70 @@
 
 import copy
 import datetime
-import sys
-import inspect
-import warnings
 import hashlib
-from http.client import HTTPMessage
+import inspect
 import logging
-import shlex
-import re
 import os
+import re
+import shlex
+import sys
+import warnings
 from collections import OrderedDict
 from collections.abc import MutableMapping
+from http.client import HTTPMessage
 from math import floor
 
-from botocore.vendored import six
-from botocore.exceptions import MD5UnavailableError
 from dateutil.tz import tzlocal
 from urllib3 import exceptions
 
+from botocore.exceptions import MD5UnavailableError
+from botocore.vendored import six
+
 logger = logging.getLogger(__name__)
 
 
 class HTTPHeaders(HTTPMessage):
-    pass
+    @classmethod
+    def from_dict(cls, d):
+        new_instance = cls()
+        for key, value in d.items():
+            new_instance[key] = value
+        return new_instance
 
+    @classmethod
+    def from_pairs(cls, pairs):
+        new_instance = cls()
+        for key, value in pairs:
+            new_instance[key] = value
+        return new_instance
+
+
+from base64 import encodebytes
+from email.utils import formatdate
+from http.client import HTTPResponse
+from io import IOBase as _IOBase
+from itertools import zip_longest
 from urllib.parse import (
+    parse_qs,
+    parse_qsl,
     quote,
-    urlencode,
     unquote,
     unquote_plus,
+    urlencode,
+    urljoin,
     urlparse,
     urlsplit,
     urlunsplit,
-    urljoin,
-    parse_qsl,
-    parse_qs,
 )
-from http.client import HTTPResponse
-from io import IOBase as _IOBase
-from base64 import encodebytes
-from email.utils import formatdate
-from itertools import zip_longest
+
 file_type = _IOBase
 zip = zip
 
 # In python3, unquote takes a str() object, url decodes it,
 # then takes the bytestring and decodes it to utf-8.
 unquote_str = unquote_plus
 
+
 def set_socket_timeout(http_response, timeout):
     """Set the timeout of the socket from an HTTPResponse.
 
@@ -69,15 +85,18 @@ def set_socket_timeout(http_response, timeout):
     """
     http_response._fp.fp.raw._sock.settimeout(timeout)
 
+
 def accepts_kwargs(func):
     # In python3.4.1, there's backwards incompatible
     # changes when using getargspec with functools.partials.
     return inspect.getfullargspec(func)[2]
 
+
 def ensure_unicode(s, encoding=None, errors=None):
     # NOOP in Python 3, because every string is already unicode
     return s
 
+
 def ensure_bytes(s, encoding='utf-8', errors='strict'):
     if isinstance(s, str):
         return s.encode(encoding, errors)
@@ -86,9 +105,9 @@ def ensure_bytes(s, encoding='utf-8', errors='strict'):
     raise ValueError(f"Expected str or bytes, received {type(s)}.")
 
 
-try:
+if sys.version_info < (3, 9):
     import xml.etree.cElementTree as ETree
-except ImportError:
+else:
     # cElementTree does not exist from Python3.9+
     import xml.etree.ElementTree as ETree
 XMLParseError = ETree.ParseError
@@ -105,26 +124,6 @@ def filter_ssl_warnings():
     )
 
 
-@classmethod
-def from_dict(cls, d):
-    new_instance = cls()
-    for key, value in d.items():
-        new_instance[key] = value
-    return new_instance
-
-
-@classmethod
-def from_pairs(cls, pairs):
-    new_instance = cls()
-    for key, value in pairs:
-        new_instance[key] = value
-    return new_instance
-
-
-HTTPHeaders.from_dict = from_dict
-HTTPHeaders.from_pairs = from_pairs
-
-
 def copy_kwargs(kwargs):
     """
     This used to be a compat shim for 2.6 but is now just an alias.
@@ -342,6 +341,7 @@ def get_tzinfo_options():
 # Detect if gzip is available for use
 try:
     import gzip
+
     HAS_GZIP = True
 except ImportError:
     HAS_GZIP = False

botocore/configprovider.py

@@ -17,6 +17,7 @@
 import copy
 import logging
 import os
+from typing import Callable, Dict, List, Optional, Tuple, Union
 
 from botocore import utils
 from botocore.exceptions import InvalidConfigError
@@ -50,7 +51,17 @@
 #: found.
 #: NOTE: Fixing the spelling of this variable would be a breaking change.
 #: Please leave as is.
-BOTOCORE_DEFAUT_SESSION_VARIABLES = {
+BOTOCORE_DEFAUT_SESSION_VARIABLES: Dict[
+    str,
+    Tuple[
+        Optional[str],  # Config key (or None)
+        Union[str, List[str], None],  # Env var or list of env vars
+        Union[str, int, dict, None],  # Default value (None, str, int, or dict)
+        Optional[
+            Callable[[str], Union[str, int, None]]
+        ],  # Conversion function (e.g., int) or None
+    ],
+] = {
     # logical:  config_file, env_var,        default_value, conversion_func
     'profile': (None, ['AWS_DEFAULT_PROFILE', 'AWS_PROFILE'], None, None),
     'region': ('region', 'AWS_DEFAULT_REGION', None, None),

botocore/credentials.py

@@ -22,6 +22,7 @@
 from collections import namedtuple
 from copy import deepcopy
 from hashlib import sha1
+from typing import Optional
 
 from dateutil.parser import parse
 from dateutil.tz import tzlocal, tzutc
@@ -939,15 +940,15 @@ def _assume_role_kwargs(self):
 
 class CredentialProvider:
     # A short name to identify the provider within botocore.
-    METHOD = None
+    METHOD: Optional[str] = None
 
     # A name to identify the provider for use in cross-sdk features like
     # assume role's `credential_source` configuration option. These names
     # are to be treated in a case-insensitive way. NOTE: any providers not
     # implemented in botocore MUST prefix their canonical names with
     # 'custom' or we DO NOT guarantee that it will work with any features
     # that this provides.
-    CANONICAL_NAME = None
+    CANONICAL_NAME: Optional[str] = None
 
     def __init__(self, session=None):
         self.session = session

botocore/crt/auth.py

@@ -13,6 +13,7 @@
 
 import datetime
 from io import BytesIO
+from typing import Dict, Type
 
 from botocore.auth import (
     SIGNED_HEADERS_BLACKLIST,
@@ -618,7 +619,7 @@ def _should_add_content_sha256_header(self, explicit_payload):
 
 # Defined at the bottom of module to ensure all Auth
 # classes are defined.
-CRT_AUTH_TYPE_MAPS = {
+CRT_AUTH_TYPE_MAPS: Dict[str, Type[BaseSigner]] = {
     'v4': CrtSigV4Auth,
     'v4-query': CrtSigV4QueryAuth,
     'v4a': CrtSigV4AsymAuth,

botocore/docs/utils.py

@@ -78,7 +78,7 @@ def get_official_service_name(service_model):
 
 
 _DocumentedShape = namedtuple(
-    'DocumentedShape',
+    '_DocumentedShape',
     [
         'name',
         'type_name',

botocore/hooks.py

@@ -20,7 +20,7 @@
 logger = logging.getLogger(__name__)
 
 
-_NodeList = namedtuple('NodeList', ['first', 'middle', 'last'])
+_NodeList = namedtuple('_NodeList', ['first', 'middle', 'last'])
 _FIRST = 0
 _MIDDLE = 1
 _LAST = 2

botocore/httpsession.py

@@ -85,7 +85,7 @@
     from certifi import where
 except ImportError:
 
-    def where():
+    def where() -> str:
         return DEFAULT_CA_BUNDLE
 
 

botocore/loaders.py

@@ -104,13 +104,14 @@
 
 import logging
 import os
+from typing import IO, Callable, Dict
 
 from botocore import BOTOCORE_ROOT
 from botocore.compat import HAS_GZIP, OrderedDict, json
 from botocore.exceptions import DataNotFoundError, UnknownServiceError
 from botocore.utils import deep_merge
 
-_JSON_OPEN_METHODS = {
+_JSON_OPEN_METHODS: Dict[str, Callable[..., IO[str]]] = {
     '.json': open,
 }
 

botocore/parsers.py

@@ -120,6 +120,7 @@
 import json
 import logging
 import re
+from typing import Optional, Type
 
 from botocore.compat import ETree, XMLParseError
 from botocore.eventstream import EventStream, NoInitialResponseError
@@ -200,7 +201,7 @@ class ResponseParser:
     """
 
     DEFAULT_ENCODING = 'utf-8'
-    EVENT_STREAM_PARSER_CLS = None
+    EVENT_STREAM_PARSER_CLS: Optional[Type['BaseEventStreamParser']] = None
 
     def __init__(self, timestamp_parser=None, blob_parser=None):
         if timestamp_parser is None:

botocore/regions.py

@@ -21,6 +21,7 @@
 import logging
 import re
 from enum import Enum
+from typing import Dict
 
 from botocore import UNSIGNED, xform_name
 from botocore.auth import AUTH_TYPE_MAPS, HAS_CRT
@@ -46,7 +47,7 @@
 
 LOG = logging.getLogger(__name__)
 DEFAULT_URI_TEMPLATE = '{service}.{region}.{dnsSuffix}'  # noqa
-DEFAULT_SERVICE_DATA = {'endpoints': {}}
+DEFAULT_SERVICE_DATA: Dict[str, Dict[str, str]] = {'endpoints': {}}
 
 
 class BaseEndpointResolver:

botocore/vendored/requests/__init__.py

@@ -5,6 +5,10 @@
 # / (   (- (/ (/ (- _)  /  _)
 #          /
 from .exceptions import (
-    RequestException, Timeout, URLRequired,
-    TooManyRedirects, HTTPError, ConnectionError
+    RequestException,
+    Timeout,
+    URLRequired,
+    TooManyRedirects,
+    HTTPError,
+    ConnectionError,
 )

botocore/vendored/requests/exceptions.py

@@ -7,6 +7,7 @@
 This module contains the set of Requests' exceptions.
 
 """
+
 from .packages.urllib3.exceptions import HTTPError as BaseHTTPError
 
 
@@ -21,8 +22,11 @@ def __init__(self, *args, **kwargs):
         response = kwargs.pop('response', None)
         self.response = response
         self.request = kwargs.pop('request', None)
-        if (response is not None and not self.request and
-                hasattr(response, 'request')):
+        if (
+            response is not None
+            and not self.request
+            and hasattr(response, 'request')
+        ):
             self.request = self.response.request
         super(RequestException, self).__init__(*args, **kwargs)
 
@@ -80,7 +84,7 @@ class InvalidSchema(RequestException, ValueError):
 
 
 class InvalidURL(RequestException, ValueError):
-    """ The URL provided was somehow invalid. """
+    """The URL provided was somehow invalid."""
 
 
 class ChunkedEncodingError(RequestException):