login page

Lib/site-packages/virtualenv/seed/embed/via_app_data/pip_install/base.py

@@ -0,0 +1,158 @@
+from __future__ import absolute_import, unicode_literals
+
+import logging
+import os
+import re
+import zipfile
+from abc import ABCMeta, abstractmethod
+from tempfile import mkdtemp
+
+from distlib.scripts import ScriptMaker, enquote_executable
+from six import PY3, add_metaclass
+
+from virtualenv.util import ConfigParser
+from virtualenv.util.path import Path, safe_delete
+from virtualenv.util.six import ensure_text
+
+
+@add_metaclass(ABCMeta)
+class PipInstall(object):
+    def __init__(self, wheel, creator, image_folder):
+        self._wheel = wheel
+        self._creator = creator
+        self._image_dir = image_folder
+        self._extracted = False
+        self.__dist_info = None
+        self._console_entry_points = None
+
+    @abstractmethod
+    def _sync(self, src, dst):
+        raise NotImplementedError
+
+    def install(self, version_info):
+        self._extracted = True
+        # sync image
+        for filename in self._image_dir.iterdir():
+            into = self._creator.purelib / filename.name
+            if into.exists():
+                if into.is_dir() and not into.is_symlink():
+                    safe_delete(into)
+                else:
+                    into.unlink()
+            self._sync(filename, into)
+        # generate console executables
+        consoles = set()
+        script_dir = self._creator.script_dir
+        for name, module in self._console_scripts.items():
+            consoles.update(self._create_console_entry_point(name, module, script_dir, version_info))
+        logging.debug("generated console scripts %s", " ".join(i.name for i in consoles))
+
+    def build_image(self):
+        # 1. first extract the wheel
+        logging.debug("build install image for %s to %s", self._wheel.name, self._image_dir)
+        with zipfile.ZipFile(str(self._wheel)) as zip_ref:
+            zip_ref.extractall(str(self._image_dir))
+            self._extracted = True
+        # 2. now add additional files not present in the distribution
+        new_files = self._generate_new_files()
+        # 3. finally fix the records file
+        self._fix_records(new_files)
+
+    def _records_text(self, files):
+        record_data = "\n".join(
+            "{},,".format(os.path.relpath(ensure_text(str(rec)), ensure_text(str(self._image_dir)))) for rec in files
+        )
+        return record_data
+
+    def _generate_new_files(self):
+        new_files = set()
+        installer = self._dist_info / "INSTALLER"
+        installer.write_text("pip\n")
+        new_files.add(installer)
+        # inject a no-op root element, as workaround for bug in https://github.com/pypa/pip/issues/7226
+        marker = self._image_dir / "{}.virtualenv".format(self._dist_info.stem)
+        marker.write_text("")
+        new_files.add(marker)
+        folder = mkdtemp()
+        try:
+            to_folder = Path(folder)
+            rel = os.path.relpath(ensure_text(str(self._creator.script_dir)), ensure_text(str(self._creator.purelib)))
+            version_info = self._creator.interpreter.version_info
+            for name, module in self._console_scripts.items():
+                new_files.update(
+                    Path(os.path.normpath(ensure_text(str(self._image_dir / rel / i.name))))
+                    for i in self._create_console_entry_point(name, module, to_folder, version_info)
+                )
+        finally:
+            safe_delete(folder)
+        return new_files
+
+    @property
+    def _dist_info(self):
+        if self._extracted is False:
+            return None  # pragma: no cover
+        if self.__dist_info is None:
+            files = []
+            for filename in self._image_dir.iterdir():
+                files.append(filename.name)
+                if filename.suffix == ".dist-info":
+                    self.__dist_info = filename
+                    break
+            else:
+                msg = "no .dist-info at {}, has {}".format(self._image_dir, ", ".join(files))  # pragma: no cover
+                raise RuntimeError(msg)  # pragma: no cover
+        return self.__dist_info
+
+    @abstractmethod
+    def _fix_records(self, extra_record_data):
+        raise NotImplementedError
+
+    @property
+    def _console_scripts(self):
+        if self._extracted is False:
+            return None  # pragma: no cover
+        if self._console_entry_points is None:
+            self._console_entry_points = {}
+            entry_points = self._dist_info / "entry_points.txt"
+            if entry_points.exists():
+                parser = ConfigParser.ConfigParser()
+                with entry_points.open() as file_handler:
+                    reader = getattr(parser, "read_file" if PY3 else "readfp")
+                    reader(file_handler)
+                if "console_scripts" in parser.sections():
+                    for name, value in parser.items("console_scripts"):
+                        match = re.match(r"(.*?)-?\d\.?\d*", name)
+                        if match:
+                            name = match.groups(1)[0]
+                        self._console_entry_points[name] = value
+        return self._console_entry_points
+
+    def _create_console_entry_point(self, name, value, to_folder, version_info):
+        result = []
+        maker = ScriptMakerCustom(to_folder, version_info, self._creator.exe, name)
+        specification = "{} = {}".format(name, value)
+        new_files = maker.make(specification)
+        result.extend(Path(i) for i in new_files)
+        return result
+
+    def clear(self):
+        if self._image_dir.exists():
+            safe_delete(self._image_dir)
+
+    def has_image(self):
+        return self._image_dir.exists() and next(self._image_dir.iterdir()) is not None
+
+
+class ScriptMakerCustom(ScriptMaker):
+    def __init__(self, target_dir, version_info, executable, name):
+        super(ScriptMakerCustom, self).__init__(None, str(target_dir))
+        self.clobber = True  # overwrite
+        self.set_mode = True  # ensure they are executable
+        self.executable = enquote_executable(str(executable))
+        self.version_info = version_info.major, version_info.minor
+        self.variants = {"", "X", "X.Y"}
+        self._name = name
+
+    def _write_script(self, names, shebang, script_bytes, filenames, ext):
+        names.add("{}{}.{}".format(self._name, *self.version_info))
+        super(ScriptMakerCustom, self)._write_script(names, shebang, script_bytes, filenames, ext)

Lib/site-packages/virtualenv/seed/embed/via_app_data/pip_install/copy.py

@@ -0,0 +1,35 @@
+from __future__ import absolute_import, unicode_literals
+
+import os
+
+from virtualenv.util.path import Path, copy
+from virtualenv.util.six import ensure_text
+
+from .base import PipInstall
+
+
+class CopyPipInstall(PipInstall):
+    def _sync(self, src, dst):
+        copy(src, dst)
+
+    def _generate_new_files(self):
+        # create the pyc files
+        new_files = super(CopyPipInstall, self)._generate_new_files()
+        new_files.update(self._cache_files())
+        return new_files
+
+    def _cache_files(self):
+        version = self._creator.interpreter.version_info
+        py_c_ext = ".{}-{}{}.pyc".format(self._creator.interpreter.implementation.lower(), version.major, version.minor)
+        for root, dirs, files in os.walk(ensure_text(str(self._image_dir)), topdown=True):
+            root_path = Path(root)
+            for name in files:
+                if name.endswith(".py"):
+                    yield root_path / "{}{}".format(name[:-3], py_c_ext)
+            for name in dirs:
+                yield root_path / name / "__pycache__"
+
+    def _fix_records(self, new_files):
+        extra_record_data_str = self._records_text(new_files)
+        with open(ensure_text(str(self._dist_info / "RECORD")), "ab") as file_handler:
+            file_handler.write(extra_record_data_str.encode("utf-8"))

Lib/site-packages/virtualenv/seed/embed/via_app_data/pip_install/symlink.py

@@ -0,0 +1,61 @@
+from __future__ import absolute_import, unicode_literals
+
+import os
+import subprocess
+from stat import S_IREAD, S_IRGRP, S_IROTH
+
+from virtualenv.util.path import safe_delete, set_tree
+from virtualenv.util.six import ensure_text
+from virtualenv.util.subprocess import Popen
+
+from .base import PipInstall
+
+
+class SymlinkPipInstall(PipInstall):
+    def _sync(self, src, dst):
+        src_str = ensure_text(str(src))
+        dest_str = ensure_text(str(dst))
+        os.symlink(src_str, dest_str)
+
+    def _generate_new_files(self):
+        # create the pyc files, as the build image will be R/O
+        process = Popen(
+            [ensure_text(str(self._creator.exe)), "-m", "compileall", ensure_text(str(self._image_dir))],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+        process.communicate()
+        # the root pyc is shared, so we'll not symlink that - but still add the pyc files to the RECORD for close
+        root_py_cache = self._image_dir / "__pycache__"
+        new_files = set()
+        if root_py_cache.exists():
+            new_files.update(root_py_cache.iterdir())
+            new_files.add(root_py_cache)
+            safe_delete(root_py_cache)
+        core_new_files = super(SymlinkPipInstall, self)._generate_new_files()
+        # remove files that are within the image folder deeper than one level (as these will be not linked directly)
+        for file in core_new_files:
+            try:
+                rel = file.relative_to(self._image_dir)
+                if len(rel.parts) > 1:
+                    continue
+            except ValueError:
+                pass
+            new_files.add(file)
+        return new_files
+
+    def _fix_records(self, new_files):
+        new_files.update(i for i in self._image_dir.iterdir())
+        extra_record_data_str = self._records_text(sorted(new_files, key=str))
+        with open(ensure_text(str(self._dist_info / "RECORD")), "wb") as file_handler:
+            file_handler.write(extra_record_data_str.encode("utf-8"))
+
+    def build_image(self):
+        super(SymlinkPipInstall, self).build_image()
+        # protect the image by making it read only
+        set_tree(self._image_dir, S_IREAD | S_IRGRP | S_IROTH)
+
+    def clear(self):
+        if self._image_dir.exists():
+            safe_delete(self._image_dir)
+        super(SymlinkPipInstall, self).clear()

Lib/site-packages/virtualenv/seed/embed/via_app_data/via_app_data.py

@@ -0,0 +1,140 @@
+"""Bootstrap"""
+from __future__ import absolute_import, unicode_literals
+
+import logging
+import sys
+import traceback
+from contextlib import contextmanager
+from subprocess import CalledProcessError
+from threading import Lock, Thread
+
+from virtualenv.info import fs_supports_symlink
+from virtualenv.seed.embed.base_embed import BaseEmbed
+from virtualenv.seed.wheels import get_wheel
+from virtualenv.util.lock import _CountedFileLock
+from virtualenv.util.path import Path
+from virtualenv.util.six import ensure_text
+
+from .pip_install.copy import CopyPipInstall
+from .pip_install.symlink import SymlinkPipInstall
+
+
+class FromAppData(BaseEmbed):
+    def __init__(self, options):
+        super(FromAppData, self).__init__(options)
+        self.symlinks = options.symlink_app_data
+
+    @classmethod
+    def add_parser_arguments(cls, parser, interpreter, app_data):
+        super(FromAppData, cls).add_parser_arguments(parser, interpreter, app_data)
+        can_symlink = app_data.transient is False and fs_supports_symlink()
+        parser.add_argument(
+            "--symlink-app-data",
+            dest="symlink_app_data",
+            action="store_true" if can_symlink else "store_false",
+            help="{} symlink the python packages from the app-data folder (requires seed pip>=19.3)".format(
+                "" if can_symlink else "not supported - ",
+            ),
+            default=False,
+        )
+
+    def run(self, creator):
+        if not self.enabled:
+            return
+        with self._get_seed_wheels(creator) as name_to_whl:
+            pip_version = name_to_whl["pip"].version_tuple if "pip" in name_to_whl else None
+            installer_class = self.installer_class(pip_version)
+            exceptions = {}
+
+            def _install(name, wheel):
+                try:
+                    logging.debug("install %s from wheel %s via %s", name, wheel, installer_class.__name__)
+                    key = Path(installer_class.__name__) / wheel.path.stem
+                    wheel_img = self.app_data.wheel_image(creator.interpreter.version_release_str, key)
+                    installer = installer_class(wheel.path, creator, wheel_img)
+                    with _CountedFileLock(ensure_text(str(wheel_img.parent / "{}.lock".format(wheel_img.name)))):
+                        if not installer.has_image():
+                            installer.build_image()
+                    installer.install(creator.interpreter.version_info)
+                except Exception:  # noqa
+                    exceptions[name] = sys.exc_info()
+
+            threads = list(Thread(target=_install, args=(n, w)) for n, w in name_to_whl.items())
+            for thread in threads:
+                thread.start()
+            for thread in threads:
+                thread.join()
+            if exceptions:
+                messages = ["failed to build image {} because:".format(", ".join(exceptions.keys()))]
+                for value in exceptions.values():
+                    exc_type, exc_value, exc_traceback = value
+                    messages.append("".join(traceback.format_exception(exc_type, exc_value, exc_traceback)))
+                raise RuntimeError("\n".join(messages))
+
+    @contextmanager
+    def _get_seed_wheels(self, creator):
+        name_to_whl, lock, fail = {}, Lock(), {}
+
+        def _get(distribution, version):
+            for_py_version = creator.interpreter.version_release_str
+            failure, result = None, None
+            # fallback to download in case the exact version is not available
+            for download in [True] if self.download else [False, True]:
+                failure = None
+                try:
+                    result = get_wheel(
+                        distribution=distribution,
+                        version=version,
+                        for_py_version=for_py_version,
+                        search_dirs=self.extra_search_dir,
+                        download=download,
+                        app_data=self.app_data,
+                        do_periodic_update=self.periodic_update,
+                    )
+                    if result is not None:
+                        break
+                except Exception as exception:  # noqa
+                    logging.exception("fail")
+                    failure = exception
+            if failure:
+                if isinstance(failure, CalledProcessError):
+                    msg = "failed to download {}".format(distribution)
+                    if version is not None:
+                        msg += " version {}".format(version)
+                    msg += ", pip download exit code {}".format(failure.returncode)
+                    output = failure.output if sys.version_info < (3, 5) else (failure.output + failure.stderr)
+                    if output:
+                        msg += "\n"
+                        msg += output
+                else:
+                    msg = repr(failure)
+                logging.error(msg)
+                with lock:
+                    fail[distribution] = version
+            else:
+                with lock:
+                    name_to_whl[distribution] = result
+
+        threads = list(
+            Thread(target=_get, args=(distribution, version))
+            for distribution, version in self.distribution_to_versions().items()
+        )
+        for thread in threads:
+            thread.start()
+        for thread in threads:
+            thread.join()
+        if fail:
+            raise RuntimeError("seed failed due to failing to download wheels {}".format(", ".join(fail.keys())))
+        yield name_to_whl
+
+    def installer_class(self, pip_version_tuple):
+        if self.symlinks and pip_version_tuple:
+            # symlink support requires pip 19.3+
+            if pip_version_tuple >= (19, 3):
+                return SymlinkPipInstall
+        return CopyPipInstall
+
+    def __unicode__(self):
+        base = super(FromAppData, self).__unicode__()
+        msg = ", via={}, app_data_dir={}".format("symlink" if self.symlinks else "copy", self.app_data)
+        return base[:-1] + msg + base[-1]