diff --git a/.gitignore b/.gitignore index edced5128..383cce2a6 100644 --- a/.gitignore +++ b/.gitignore @@ -28,4 +28,4 @@ venv* *wheel-store* Dockerfile -.dockerignore \ No newline at end of file +.dockerignore diff --git a/docs/changelog/1719.bugfix.rst b/docs/changelog/1719.bugfix.rst new file mode 100644 index 000000000..2c47726a6 --- /dev/null +++ b/docs/changelog/1719.bugfix.rst @@ -0,0 +1,3 @@ +Support Python 2 implementations that require the landmark files and ``site.py`` to be in platform standard library +instead of the standard library path of the virtual environment (notably some RHEL ones, such as the Docker +image ``amazonlinux:1``) - by :user:`gaborbernat`. diff --git a/docs/changelog/1728.bugfix.rst b/docs/changelog/1728.bugfix.rst new file mode 100644 index 000000000..feb917c10 --- /dev/null +++ b/docs/changelog/1728.bugfix.rst @@ -0,0 +1 @@ +Automatically create the application data folder if it does not exists - by :user:`gaborbernat`. diff --git a/src/virtualenv/create/describe.py b/src/virtualenv/create/describe.py index 526ec8694..1e59aaeae 100644 --- a/src/virtualenv/create/describe.py +++ b/src/virtualenv/create/describe.py @@ -20,6 +20,7 @@ def __init__(self, dest, interpreter): self.interpreter = interpreter self.dest = dest self._stdlib = None + self._stdlib_platform = None self._system_stdlib = None self._conf_vars = None @@ -49,6 +50,12 @@ def stdlib(self): self._stdlib = Path(self.interpreter.sysconfig_path("stdlib", config_var=self._config_vars)) return self._stdlib + @property + def stdlib_platform(self): + if self._stdlib_platform is None: + self._stdlib_platform = Path(self.interpreter.sysconfig_path("platstdlib", config_var=self._config_vars)) + return self._stdlib_platform + @property def _config_vars(self): if self._conf_vars is None: diff --git a/src/virtualenv/create/via_global_ref/builtin/cpython/cpython2.py b/src/virtualenv/create/via_global_ref/builtin/cpython/cpython2.py index 6a8871b0b..61aa39543 100644 --- a/src/virtualenv/create/via_global_ref/builtin/cpython/cpython2.py +++ b/src/virtualenv/create/via_global_ref/builtin/cpython/cpython2.py @@ -66,8 +66,8 @@ def sources(cls, interpreter): for src in super(CPython2Posix, cls).sources(interpreter): yield src # landmark for exec_prefix - name = "lib-dynload" - yield PathRefToDest(interpreter.stdlib_path(name), dest=cls.to_stdlib) + exec_marker_file, to_path, _ = cls.from_stdlib(cls.mappings(interpreter), "lib-dynload") + yield PathRefToDest(exec_marker_file, dest=to_path) class CPython2Windows(CPython2, CPythonWindows): diff --git a/src/virtualenv/create/via_global_ref/builtin/cpython/mac_os.py b/src/virtualenv/create/via_global_ref/builtin/cpython/mac_os.py index fd7f0e3e6..b7ffaf1ce 100644 --- a/src/virtualenv/create/via_global_ref/builtin/cpython/mac_os.py +++ b/src/virtualenv/create/via_global_ref/builtin/cpython/mac_os.py @@ -77,8 +77,9 @@ def current_mach_o_image_path(self): def sources(cls, interpreter): for src in super(CPython2macOsFramework, cls).sources(interpreter): yield src - name = "lib-dynload" # landmark for exec_prefix - yield PathRefToDest(interpreter.stdlib_path(name), dest=cls.to_stdlib) + # landmark for exec_prefix + exec_marker_file, to_path, _ = cls.from_stdlib(cls.mappings(interpreter), "lib-dynload") + yield PathRefToDest(exec_marker_file, dest=to_path) @property def reload_code(self): diff --git a/src/virtualenv/create/via_global_ref/builtin/python2/python2.py b/src/virtualenv/create/via_global_ref/builtin/python2/python2.py index 22d1da009..dc5ce2a5c 100644 --- a/src/virtualenv/create/via_global_ref/builtin/python2/python2.py +++ b/src/virtualenv/create/via_global_ref/builtin/python2/python2.py @@ -24,8 +24,15 @@ def create(self): """Perform operations needed to make the created environment work on Python 2""" super(Python2, self).create() # install a patched site-package, the default Python 2 site.py is not smart enough to understand pyvenv.cfg, - # so we inject a small shim that can do this - site_py = self.stdlib / "site.py" + # so we inject a small shim that can do this, the location of this depends where it's on host + sys_std_plat = Path(self.interpreter.system_stdlib_platform) + site_py_in = ( + self.stdlib_platform + if ((sys_std_plat / "site.py").exists() or (sys_std_plat / "site.pyc").exists()) + else self.stdlib + ) + site_py = site_py_in / "site.py" + custom_site = get_custom_site() if IS_ZIPAPP: custom_site_text = read_from_zipapp(custom_site) @@ -55,33 +62,45 @@ def skip_rewrite(self): def sources(cls, interpreter): for src in super(Python2, cls).sources(interpreter): yield src - # install files needed to run site.py + # install files needed to run site.py, either from stdlib or stdlib_platform, at least pyc, but both if exists + # if neither exists return the module file to trigger failure + mappings, needs_py_module = ( + cls.mappings(interpreter), + cls.needs_stdlib_py_module(), + ) for req in cls.modules(): - - # the compiled path is optional, but refer to it if exists - module_compiled_path = interpreter.stdlib_path("{}.pyc".format(req)) - has_compile = module_compiled_path.exists() - if has_compile: - yield PathRefToDest(module_compiled_path, dest=cls.to_stdlib) - - # stdlib module src may be missing if the interpreter allows it by falling back to the compiled - module_path = interpreter.stdlib_path("{}.py".format(req)) - add_py_module = cls.needs_stdlib_py_module() - if add_py_module is False: - if module_path.exists(): # if present add it - add_py_module = True - else: - add_py_module = not has_compile # otherwise only add it if the pyc is not present - if add_py_module: - yield PathRefToDest(module_path, dest=cls.to_stdlib) + module_file, to_module, module_exists = cls.from_stdlib(mappings, "{}.py".format(req)) + compiled_file, to_compiled, compiled_exists = cls.from_stdlib(mappings, "{}.pyc".format(req)) + if needs_py_module or module_exists or not compiled_exists: + yield PathRefToDest(module_file, dest=to_module) + if compiled_exists: + yield PathRefToDest(compiled_file, dest=to_compiled) + + @staticmethod + def from_stdlib(mappings, name): + for from_std, to_std in mappings: + src = from_std / name + if src.exists(): + return src, to_std, True + return mappings[0] / name, mappings[1], False @classmethod - def needs_stdlib_py_module(cls): - raise NotImplementedError + def mappings(cls, interpreter): + mappings = [(Path(interpreter.system_stdlib_platform), cls.to_stdlib_platform)] + if interpreter.system_stdlib_platform != interpreter.system_stdlib: + mappings.append((Path(interpreter.system_stdlib), cls.to_stdlib),) + return mappings def to_stdlib(self, src): return self.stdlib / src.name + def to_stdlib_platform(self, src): + return self.stdlib_platform / src.name + + @classmethod + def needs_stdlib_py_module(cls): + raise NotImplementedError + @classmethod def modules(cls): return [] diff --git a/src/virtualenv/discovery/py_info.py b/src/virtualenv/discovery/py_info.py index e94528bcc..c04a31212 100644 --- a/src/virtualenv/discovery/py_info.py +++ b/src/virtualenv/discovery/py_info.py @@ -91,7 +91,6 @@ def abs_path(v): self.system_stdlib_platform = self.sysconfig_path("platstdlib", confs) self.max_size = getattr(sys, "maxsize", getattr(sys, "maxint", None)) self._creators = None - self._stdlib_paths = None def _fast_get_system_executable(self): """Try to get the system executable by just looking at properties""" @@ -278,7 +277,7 @@ def _to_json(self): return json.dumps(self._to_dict(), indent=2) def _to_dict(self): - data = {var: (getattr(self, var) if var not in ("_creators", "_stdlib_paths") else None) for var in vars(self)} + data = {var: (getattr(self, var) if var not in ("_creators",) else None) for var in vars(self)} # noinspection PyProtectedMember data["version_info"] = data["version_info"]._asdict() # namedtuple to dictionary return data @@ -453,19 +452,6 @@ def _possible_base(self): if upper != base: yield upper - def stdlib_path(self, name): - if self._stdlib_paths is None: - from collections import OrderedDict - from virtualenv.util.path import Path - - pat = OrderedDict((Path(i), None) for i in (self.system_stdlib, self.system_stdlib_platform)) - self._stdlib_paths = list(pat.keys()) - for path in self._stdlib_paths: - std_path = path / name - if std_path.exists(): - return std_path - return self._stdlib_paths[0] / name - if __name__ == "__main__": # dump a JSON representation of the current python diff --git a/src/virtualenv/discovery/py_spec.py b/src/virtualenv/discovery/py_spec.py index 71712d1ec..cb63e1516 100644 --- a/src/virtualenv/discovery/py_spec.py +++ b/src/virtualenv/discovery/py_spec.py @@ -69,8 +69,6 @@ def _int_or_none(val): return cls(string_spec, impl, major, minor, micro, arch, path) def generate_names(self): - if self.implementation is None: - return impls = OrderedDict() if self.implementation: # first consider implementation as it is diff --git a/src/virtualenv/run/app_data.py b/src/virtualenv/run/app_data.py index 00a394069..68edb93f1 100644 --- a/src/virtualenv/run/app_data.py +++ b/src/virtualenv/run/app_data.py @@ -48,7 +48,7 @@ def _check_folder(folder): folder = os.path.abspath(folder) if not os.path.exists(folder): try: - os.mkdir(folder) + os.makedirs(folder) logging.debug("created app data folder %s", folder) except OSError as exception: logging.info("could not create app data folder %s due to %r", folder, exception) diff --git a/tests/unit/create/test_creator.py b/tests/unit/create/test_creator.py index 21959becc..7bbff4daa 100644 --- a/tests/unit/create/test_creator.py +++ b/tests/unit/create/test_creator.py @@ -20,6 +20,7 @@ from virtualenv.__main__ import run, run_with_catch from virtualenv.create.creator import DEBUG_SCRIPT, Creator, get_env_debug_info +from virtualenv.create.via_global_ref.builtin.python2.python2 import Python2 from virtualenv.discovery.builtin import get_interpreter from virtualenv.discovery.py_info import PythonInfo from virtualenv.info import IS_PYPY, IS_WIN, PY2, PY3, fs_is_case_sensitive @@ -463,24 +464,18 @@ def _get_sys_path(flag=None): def test_pyc_only(tmp_path, mocker, session_app_data): """Ensure that creation can succeed if os.pyc exists (even if os.py has been deleted)""" interpreter = PythonInfo.from_exe(sys.executable, session_app_data) - host_pyc = interpreter.stdlib_path("os.pyc") - if not host_pyc.exists(): + host_pyc, _, host_pyc_exists = Python2.from_stdlib(Python2.mappings(interpreter), "os.pyc") + if not host_pyc_exists: pytest.skip("missing system os.pyc at {}".format(host_pyc)) - previous = interpreter.stdlib_path + previous = Python2.from_stdlib - def stdlib_path(name): - path = previous(name) + def from_stdlib(mappings, name): + path, to, exists = previous(mappings, name) if name.endswith(".py"): + exists = False + return path, to, exists - class _Path(type(path)): - @staticmethod - def exists(): - return False - - return _Path(path) - return path - - mocker.patch.object(interpreter, "stdlib_path", side_effect=stdlib_path) + mocker.patch.object(Python2, "from_stdlib", side_effect=from_stdlib) result = cli_run([ensure_text(str(tmp_path)), "--without-pip", "--activators", ""])