Further venv-related Cleanup (#154)

This also includes some small fixes.

Author: ericsnowcurrently

pyperformance/_benchmark.py

@@ -175,7 +175,7 @@ def run(self, python, runid=None, pyperf_opts=None, *,
             verbose=False,
             ):
         if venv and python == sys.executable:
-            python = venv.get_python_program()
+            python = venv.python
 
         if not runid:
             from ..run import get_run_id
@@ -218,8 +218,18 @@ def _run_perf_script(python, runscript, runid, *,
         else:
             opts, inherit_envvar = _resolve_restricted_opts(opts)
             argv, env = _prep_cmd(python, runscript, opts, runid, inherit_envvar)
-        _utils.run_command(argv, env=env, hide_stderr=not verbose)
-
+        hide_stderr = not verbose
+        ec, _, stderr = _utils.run_cmd(
+            argv,
+            env=env,
+            capture='stderr' if hide_stderr else None,
+        )
+        if ec != 0:
+            if hide_stderr:
+                sys.stderr.flush()
+                sys.stderr.write(stderr)
+                sys.stderr.flush()
+            raise RuntimeError("Benchmark died")
         return pyperf.BenchmarkSuite.load(tmp)
 
 

pyperformance/_pip.py

@@ -0,0 +1,141 @@
+import os
+import os.path
+import shlex
+import subprocess
+import sys
+
+from . import _utils, _pythoninfo
+
+
+GET_PIP_URL = 'https://bootstrap.pypa.io/get-pip.py'
+# pip 6 is the first version supporting environment markers
+MIN_PIP = '6.0'
+OLD_PIP = '7.1.2'
+OLD_SETUPTOOLS = '18.5'
+
+
+def get_best_pip_version(python):
+    """Return the pip to install for the given Python executable."""
+    if not python or isinstance(python, str):
+        info = _pythoninfo.get_info(python)
+    else:
+        info = python
+    # On Python: 3.5a0 <= version < 3.5.0 (final), install pip 7.1.2,
+    # the last version working on Python 3.5a0:
+    # https://sourceforge.net/p/pyparsing/bugs/100/
+    if 0x30500a0 <= info.sys.hexversion < 0x30500f0:
+        return OLD_PIP
+    else:
+        return None
+
+
+def run_pip(cmd, *args, **kwargs):
+    """Return the result of running pip with the given args."""
+    return _utils.run_python('-m', 'pip', cmd, *args, **kwargs)
+
+
+def is_pip_installed(python, *, env=None):
+    """Return True if pip is installed on the given Python executable."""
+    ec, _, _ = run_pip('--version', env=env, capture=True, verbose=False)
+    return ec == 0
+
+
+def install_pip(python=sys.executable, *,
+                info=None,
+                downloaddir=None,
+                env=None,
+                **kwargs
+                ):
+    """Install pip on the given Python executable."""
+    if not python:
+        python = getattr(info, 'executable', None) or sys.executable
+
+    # python -m ensurepip
+    res = _utils.run_python(
+        '-m', 'ensurepip', '--verbose',
+        python=python,
+        **kwargs
+    )
+    ec, _, _ = res
+    if ec == 0 and is_pip_installed(python, env=env):
+        return res
+
+    ##############################
+    # Fall back to get-pip.py.
+
+    if not downloaddir:
+        downloaddir = '.'
+    os.makedirs(downloaddir, exist_ok=True)
+
+    # download get-pip.py
+    filename = os.path.join(downloaddir, 'get-pip.py')
+    if not os.path.exists(filename):
+        print("Download %s into %s" % (GET_PIP_URL, filename))
+        _utils.download(GET_PIP_URL, filename)
+
+    # python get-pip.py
+    argv = [python, '-u', filename]
+    version = get_best_pip_version(info or python)
+    if version:
+        argv.append(version)
+    res = _utils.run_cmd(argv, env=env)
+    ec, _, _ = res
+    if ec != 0:
+        # get-pip.py was maybe not properly downloaded: remove it to
+        # download it again next time
+        os.unlink(filename)
+    return res
+
+
+def upgrade_pip(python=sys.executable, *,
+                info=None,
+                installer=False,
+                **kwargs,
+                ):
+    """Upgrade pip on the given Python to the latest version."""
+    if not python:
+        python = getattr(info, 'executable', None) or sys.executable
+
+    version = get_best_pip_version(info or python)
+    if version:
+        reqs = [f'pip=={version}']
+        if installer:
+            reqs.append(f'setuptools=={OLD_SETUPTOOLS}')
+    else:
+        # pip 6 is the first version supporting environment markers
+        reqs = [f'pip>={MIN_PIP}']
+    res = install_requirements(*reqs, python=python, upgrade=True, **kwargs)
+    ec, _, _ = res
+    if ec != 0:
+        return res
+
+    if installer:
+        # Upgrade installer dependencies (setuptools, ...)
+        reqs = [
+            f'setuptools>={OLD_SETUPTOOLS}',
+            # install wheel so pip can cache binary wheel packages locally,
+            # and install prebuilt wheel packages from PyPI.
+            'wheel',
+        ]
+        res = install_requirements(*reqs, python=python, upgrade=True, **kwargs)
+    return res
+
+
+def install_requirements(reqs, *extra,
+                         upgrade=True,
+                         **kwargs
+                         ):
+    """Install the given packages from PyPI."""
+    args = []
+    if upgrade:
+        args.append('-U')  # --upgrade
+    for reqs in [reqs, *extra]:
+        if os.path.exists(reqs):
+            args.append('-r')  # --requirement
+        args.append(reqs)
+    return run_pip('install', *args, **kwargs)
+
+
+def install_editable(projectroot, **kwargs):
+    """Install the given project as an "editable" install."""
+    return run_pip('install', '-e', projectroot, **kwargs)

pyperformance/_utils.py

@@ -23,7 +23,12 @@
 import errno
 import os
 import os.path
+import shlex
+import shutil
+import subprocess
+import sys
 import tempfile
+import urllib.request
 
 
 @contextlib.contextmanager
@@ -66,6 +71,16 @@ def resolve_file(filename, relroot=None):
     return resolved
 
 
+def safe_rmtree(path):
+    if not os.path.exists(path):
+        return False
+
+    print("Remove directory %s" % path)
+    # XXX Pass onerror to report on any files that could not be deleted?
+    shutil.rmtree(path)
+    return True
+
+
 #######################################
 # platform utils
 
@@ -77,43 +92,86 @@ def resolve_file(filename, relroot=None):
 MS_WINDOWS = (sys.platform == 'win32')
 
 
-def run_command(command, env=None, *, hide_stderr=True):
-    if hide_stderr:
-        kw = {'stderr': subprocess.PIPE}
-    else:
-        kw = {}
-
-    logging.info("Running `%s`",
-                 " ".join(list(map(str, command))))
+def run_cmd(argv, *, env=None, capture=None, verbose=True):
+    try:
+        cmdstr = ' '.join(shlex.quote(a) for a in argv)
+    except TypeError:
+        print(argv)
+        raise  # re-raise
+
+    if capture is True:
+        capture = 'both'
+    kw = dict(
+        env=env,
+    )
+    if capture == 'both':
+        kw.update(dict(
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        ))
+    elif capture == 'combined':
+        kw.update(dict(
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+        ))
+    elif capture == 'stdout':
+        kw.update(dict(
+            stdout=subprocess.PIPE,
+        ))
+    elif capture == 'stderr':
+        kw.update(dict(
+            stderr=subprocess.PIPE,
+        ))
+    elif capture:
+        raise NotImplementedError(repr(capture))
+    if capture:
+        kw.update(dict(
+            encoding='utf-8',
+        ))
+
+    # XXX Use a logger.
+    if verbose:
+        print('#', cmdstr)
 
     # Explicitly flush standard streams, required if streams are buffered
     # (not TTY) to write lines in the expected order
     sys.stdout.flush()
     sys.stderr.flush()
 
-    proc = subprocess.Popen(command,
-                            universal_newlines=True,
-                            env=env,
-                            **kw)
     try:
-        stderr = proc.communicate()[1]
-    except:   # noqa
-        if proc.stderr:
-            proc.stderr.close()
-        try:
-            proc.kill()
-        except OSError:
-            # process already exited
-            pass
-        proc.wait()
+        proc = subprocess.run(argv, **kw)
+    except OSError as exc:
+        if exc.errno == errno.ENOENT:
+            if verbose:
+                print('command failed (not found)')
+            return 127, None, None
         raise
+    if proc.returncode != 0 and verbose:
+        print(f'Command failed with exit code {proc.returncode}')
+    return proc.returncode, proc.stdout, proc.stderr
+
+
+def run_python(*args, python=sys.executable, **kwargs):
+    if not isinstance(python, str) and python is not None:
+        try:
+            # See _pythoninfo.get_info().
+            python = python.sys.executable
+        except AttributeError:
+            raise TypeError(f'expected python str, got {python!r}')
+    return run_cmd([python, *args], **kwargs)
+
+
+#######################################
+# network utils
+
+def download(url, filename):
+    response = urllib.request.urlopen(url)
+    with response:
+        content = response.read()
 
-    if proc.returncode != 0:
-        if hide_stderr:
-            sys.stderr.flush()
-            sys.stderr.write(stderr)
-            sys.stderr.flush()
-        raise RuntimeError("Benchmark died")
+    with open(filename, 'wb') as fp:
+        fp.write(content)
+        fp.flush()
 
 
 #######################################