#25 Update to 20.0.25 and new tests
Merged 3 years ago by churchyard. Opened 3 years ago by lbalhar.
rpms/ lbalhar/python-virtualenv master  into  master

file modified
+15 -12
@@ -1,5 +1,5 @@ 

  Name:           python-virtualenv

- Version:        20.0.23

+ Version:        20.0.25

  Release:        1%{?dist}

  Summary:        Tool to create isolated Python environments

  
@@ -97,9 +97,9 @@ 

  %{__sed} -i -e "1s|#!/usr/bin/env python||" tasks/update_embedded.py

  

  # Remove the wheels provided by RPM packages

- rm src/virtualenv/seed/embed/wheels/pip-*

- rm src/virtualenv/seed/embed/wheels/setuptools-*

- rm src/virtualenv/seed/embed/wheels/wheel-*

+ rm src/virtualenv/seed/wheels/embed/pip-*

+ rm src/virtualenv/seed/wheels/embed/setuptools-*

+ rm src/virtualenv/seed/wheels/embed/wheel-*

  

  test ! -f src/virtualenv/seed/embed/wheels/*.whl

  
@@ -125,17 +125,17 @@ 

  

  # Skip tests which requires internet or some extra dependencies

  # Requires internet:

- # - test_base_bootstrap_via_pip_invoke

- # - test_seed_link_via_app_data

+ # - test_download_*

  # Detects system Python even it should not (problem with mock?):

  # - test_py_info_to_system_raises

  # Uses disabled functionalities around bundled wheels:

- # - test_wheel_support_no_python_requires

- # Requires xonsh (FTBFS with Py 3.9)

- # - test_xonsh

- # Does not work with beta release of Python:

- # - test_version_satisfies_nok

- %pytest -vv -k "not test_base_bootstrap_via_pip_invoke and not test_seed_link_via_app_data and not test_py_info_to_system_raises and not test_wheel_support_no_python_requires and not test_xonsh and not test_version_satisfies_nok"

+ # - test_wheel_*

+ # - test_seed_link_via_app_data

+ # - test_base_bootstrap_via_pip_invoke

+ # - test_acquire_find_wheel.py (whole file)

+ # Uses disabled functionalities around automatic updates:

+ # - test_periodic_update.py (whole file)

+ %pytest -vv -k "not test_acquire_find_wheel and not test_periodic_update and not test_py_info_to_system_raises and not test_wheel_ and not test_download_ and not test_base_bootstrap_via_pip_invoke and not test_seed_link_via_app_data"

  

  rm -r tmp_path

  %endif
@@ -154,6 +154,9 @@ 

  

  

  %changelog

+ * Tue Jun 23 2020 Lumír Balhar <lbalhar@redhat.com> - 20.0.25-1

+ - Update to 20.0.25

+ 

  * Mon Jun 15 2020 Lumír Balhar <lbalhar@redhat.com> - 20.0.23-1

  - Update to 20.0.23 (#1742034)

  

file modified
+62 -121
@@ -1,31 +1,53 @@ 

- From fd3e105dde6f4c14b4d85a325cd70c79d7b452cc Mon Sep 17 00:00:00 2001

+ From 1fd2cb13860a9111c98a0151e1951fdbac49ee16 Mon Sep 17 00:00:00 2001

  From: Lumir Balhar <lbalhar@redhat.com>

- Date: Fri, 22 May 2020 14:25:48 +0200

+ Date: Tue, 23 Jun 2020 09:02:20 +0200

  Subject: [PATCH] rpm wheels

  

  ---

-  src/virtualenv/seed/embed/base_embed.py       |  5 ++++

-  src/virtualenv/seed/embed/pip_invoke.py       |  5 ++--

-  src/virtualenv/seed/embed/wheels/__init__.py  |  3 +++

-  src/virtualenv/seed/embed/wheels/acquire.py   | 24 ++++++++++++-------

-  .../seed/via_app_data/via_app_data.py         |  1 +

-  src/virtualenv/util/path/_system_wheels.py    | 22 +++++++++++++++++

-  6 files changed, 49 insertions(+), 11 deletions(-)

+  src/virtualenv/run/__init__.py                |  5 +++--

+  src/virtualenv/seed/embed/base_embed.py       |  7 +++++-

+  src/virtualenv/seed/embed/pip_invoke.py       |  1 +

+  .../seed/embed/via_app_data/via_app_data.py   |  1 +

+  src/virtualenv/seed/wheels/embed/__init__.py  |  3 +++

+  src/virtualenv/util/path/_system_wheels.py    | 22 +++++++++++++++++++

+  6 files changed, 36 insertions(+), 3 deletions(-)

   create mode 100644 src/virtualenv/util/path/_system_wheels.py

  

+ diff --git a/src/virtualenv/run/__init__.py b/src/virtualenv/run/__init__.py

+ index 8de7962..3df28b6 100644

+ --- a/src/virtualenv/run/__init__.py

+ +++ b/src/virtualenv/run/__init__.py

+ @@ -74,8 +74,9 @@ def build_parser_only(args=None):

+  

+  def handle_extra_commands(options):

+      if options.upgrade_embed_wheels:

+ -        result = manual_upgrade(options.app_data)

+ -        raise SystemExit(result)

+ +        # result = manual_upgrade(options.app_data)

+ +        logging.warning("virtualenv installed from the RPM package uses wheels from RPM packages as well. Updating them via virtualenv is not possible. The RPM packaged wheels are updated together with other RPM packages of the system.")

+ +        raise SystemExit(1)

+  

+  

+  def load_app_data(args, parser, options):

  diff --git a/src/virtualenv/seed/embed/base_embed.py b/src/virtualenv/seed/embed/base_embed.py

- index bffd494..349e5ff 100644

+ index f41b5fc..1b8e8fc 100644

  --- a/src/virtualenv/seed/embed/base_embed.py

  +++ b/src/virtualenv/seed/embed/base_embed.py

- @@ -6,6 +6,7 @@ from six import add_metaclass

+ @@ -6,11 +6,12 @@ from six import add_metaclass

   

   from virtualenv.util.path import Path

   from virtualenv.util.six import ensure_str, ensure_text

  +from virtualenv.util.path._system_wheels import get_system_wheels_paths

   

   from ..seeder import Seeder

+  from ..wheels import Version

+  

+ -PERIODIC_UPDATE_ON_BY_DEFAULT = True

+ +PERIODIC_UPDATE_ON_BY_DEFAULT = False

   

- @@ -101,3 +102,7 @@ class BaseEmbed(Seeder):

+  

+  @add_metaclass(ABCMeta)

+ @@ -115,3 +116,7 @@ class BaseEmbed(Seeder):

   

       def __repr__(self):

           return ensure_str(self.__unicode__())
@@ -34,126 +56,45 @@ 

  +        system_wheels_paths = get_system_wheels_paths(creator.interpreter.executable)

  +        self.extra_search_dir = list(system_wheels_paths) + self.extra_search_dir

  diff --git a/src/virtualenv/seed/embed/pip_invoke.py b/src/virtualenv/seed/embed/pip_invoke.py

- index 25be493..f49a420 100644

+ index 372e140..9bedb00 100644

  --- a/src/virtualenv/seed/embed/pip_invoke.py

  +++ b/src/virtualenv/seed/embed/pip_invoke.py

- @@ -23,8 +23,9 @@ class PipInvoke(BaseEmbed):

+ @@ -17,6 +17,7 @@ class PipInvoke(BaseEmbed):

       def run(self, creator):

           if not self.enabled:

               return

  +        self.insert_system_wheels_paths(creator)

-          with self.get_pip_install_cmd(creator.exe, creator.interpreter.version_release_str) as cmd:

- -            with pip_wheel_env_run(creator.interpreter.version_release_str, self.app_data) as env:

- +            with pip_wheel_env_run(creator.interpreter.version_release_str, creator.interpreter.executable, self.app_data) as env:

-                  self._execute(cmd, env)

-  

-      @staticmethod

- @@ -46,8 +47,6 @@ class PipInvoke(BaseEmbed):

-              cmd.append("{}{}".format(key, "=={}".format(ver) if ver is not None else ""))

-          with ExitStack() as stack:

-              folders = set()

- -            for context in (ensure_file_on_disk(get_bundled_wheel(p, version), self.app_data) for p in pkg_versions):

- -                folders.add(stack.enter_context(context).parent)

-              folders.update(set(self.extra_search_dir))

-              for folder in folders:

-                  cmd.extend(["--find-links", str(folder)])

- diff --git a/src/virtualenv/seed/embed/wheels/__init__.py b/src/virtualenv/seed/embed/wheels/__init__.py

- index 90fea02..99cc251 100644

- --- a/src/virtualenv/seed/embed/wheels/__init__.py

- +++ b/src/virtualenv/seed/embed/wheels/__init__.py

- @@ -38,3 +38,6 @@ BUNDLE_SUPPORT = {

-      },

+          for_py_version = creator.interpreter.version_release_str

+          with self.get_pip_install_cmd(creator.exe, for_py_version) as cmd:

+              env = pip_wheel_env_run(self.extra_search_dir, self.app_data)

+ diff --git a/src/virtualenv/seed/embed/via_app_data/via_app_data.py b/src/virtualenv/seed/embed/via_app_data/via_app_data.py

+ index 779ee18..d6f2157 100644

+ --- a/src/virtualenv/seed/embed/via_app_data/via_app_data.py

+ +++ b/src/virtualenv/seed/embed/via_app_data/via_app_data.py

+ @@ -39,6 +39,7 @@ class FromAppData(BaseEmbed):

+      def run(self, creator):

+          if not self.enabled:

+              return

+ +        self.insert_system_wheels_paths(creator)

+          with self._get_seed_wheels(creator) as name_to_whl:

+              pip_version = name_to_whl["pip"].version_tuple if "pip" in name_to_whl else None

+              installer_class = self.installer_class(pip_version)

+ diff --git a/src/virtualenv/seed/wheels/embed/__init__.py b/src/virtualenv/seed/wheels/embed/__init__.py

+ index d118754..bc747d5 100644

+ --- a/src/virtualenv/seed/wheels/embed/__init__.py

+ +++ b/src/virtualenv/seed/wheels/embed/__init__.py

+ @@ -48,8 +48,11 @@ BUNDLE_SUPPORT = {

   }

-  MAX = "3.9"

- +

+  MAX = "3.10"

+  

  +# Redefined here because bundled wheels are removed in RPM build

  +BUNDLE_SUPPORT = None

- diff --git a/src/virtualenv/seed/embed/wheels/acquire.py b/src/virtualenv/seed/embed/wheels/acquire.py

- index 91b630d..4067f0e 100644

- --- a/src/virtualenv/seed/embed/wheels/acquire.py

- +++ b/src/virtualenv/seed/embed/wheels/acquire.py

- @@ -12,6 +12,7 @@ from zipfile import ZipFile

-  

-  from virtualenv.info import IS_ZIPAPP

-  from virtualenv.util.path import Path

- +from virtualenv.util.path._system_wheels import get_system_wheels_paths

-  from virtualenv.util.six import ensure_str, ensure_text

-  from virtualenv.util.subprocess import Popen, subprocess

-  from virtualenv.util.zipapp import ensure_file_on_disk

- @@ -33,8 +34,9 @@ class WheelDownloadFail(ValueError):

-  def get_wheels(for_py_version, wheel_cache_dir, extra_search_dir, packages, app_data, download):

-      # not all wheels are compatible with all python versions, so we need to py version qualify it

-      processed = copy(packages)

- -    # 1. acquire from bundle

- -    acquire_from_bundle(processed, for_py_version, wheel_cache_dir)

- +    # Do not use bundled wheels, they are removed in rpmbuild anyway

- +    # acquire_from_bundle(processed, for_py_version, wheel_cache_dir)

- +

-      # 2. acquire from extra search dir

-      acquire_from_dir(processed, for_py_version, wheel_cache_dir, extra_search_dir)

-      # 3. download from the internet

- @@ -47,6 +49,7 @@ def get_wheels(for_py_version, wheel_cache_dir, extra_search_dir, packages, app_

-  

-  

-  def acquire_from_bundle(packages, for_py_version, to_folder):

- +    raise NotImplementedError("Bundled wheels are not available")

-      for pkg, version in list(packages.items()):

-          bundle = get_bundled_wheel(pkg, for_py_version)

-          if bundle is not None:

- @@ -67,6 +70,7 @@ def acquire_from_bundle(packages, for_py_version, to_folder):

-  

-  

-  def get_bundled_wheel(package, version_release):

- +    raise NotImplementedError("Bundled wheels are not available")  # and BUNDLE_SUPPORT == None anyway

-      return BUNDLE_FOLDER / (BUNDLE_SUPPORT.get(version_release, {}) or BUNDLE_SUPPORT[MAX]).get(package)

-  

-  

- @@ -156,7 +160,7 @@ def download_wheel(packages, for_py_version, to_folder, app_data):

-      cmd.extend(to_download)

-      # pip has no interface in python - must be a new sub-process

   

- -    with pip_wheel_env_run("{}.{}".format(*sys.version_info[0:2]), app_data) as env:

- +    with pip_wheel_env_run("{}.{}".format(*sys.version_info[0:2]), sys.executable, app_data) as env:

-          process = Popen(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)

-          out, err = process.communicate()

-          if process.returncode != 0:

- @@ -164,7 +168,7 @@ def download_wheel(packages, for_py_version, to_folder, app_data):

+  def get_embed_wheel(distribution, for_py_version):

+ +    return None  # BUNDLE_SUPPORT == None anyway

+      path = BUNDLE_FOLDER / (BUNDLE_SUPPORT.get(for_py_version, {}) or BUNDLE_SUPPORT[MAX]).get(distribution)

+      return Wheel.from_path(path)

   

-  

-  @contextmanager

- -def pip_wheel_env_run(version, app_data):

- +def pip_wheel_env_run(version, executable, app_data):

-      env = os.environ.copy()

-      env.update(

-          {

- @@ -172,7 +176,11 @@ def pip_wheel_env_run(version, app_data):

-              for k, v in {"PIP_USE_WHEEL": "1", "PIP_USER": "0", "PIP_NO_INPUT": "1"}.items()

-          },

-      )

- -    with ensure_file_on_disk(get_bundled_wheel("pip", version), app_data) as pip_wheel_path:

- -        # put the bundled wheel onto the path, and use it to do the bootstrap operation

- -        env[str("PYTHONPATH")] = str(pip_wheel_path)

- -        yield env

- +

- +    paths = list(get_system_wheels_paths(executable))

- +    pip_wheels = []

- +    for path in paths:

- +        pip_wheels.extend([str(wheel) for wheel in path.glob("pip-*")])

- +    env[str("PYTHONPATH")] = pip_wheels[0]  # Use first pip in the list (ensurepip, if exists)

- +

- +    yield env

- diff --git a/src/virtualenv/seed/via_app_data/via_app_data.py b/src/virtualenv/seed/via_app_data/via_app_data.py

- index de3757d..db7c6d9 100644

- --- a/src/virtualenv/seed/via_app_data/via_app_data.py

- +++ b/src/virtualenv/seed/via_app_data/via_app_data.py

- @@ -38,6 +38,7 @@ class FromAppData(BaseEmbed):

-      def run(self, creator):

-          if not self.enabled:

-              return

- +        self.insert_system_wheels_paths(creator)

-          base_cache = self.base_cache / creator.interpreter.version_release_str

-          with self._get_seed_wheels(creator, base_cache) as name_to_whl:

-              pip_version = name_to_whl["pip"].stem.split("-")[1] if "pip" in name_to_whl else None

  diff --git a/src/virtualenv/util/path/_system_wheels.py b/src/virtualenv/util/path/_system_wheels.py

  new file mode 100644

  index 0000000..a968dee

file modified
+1 -1
@@ -1,1 +1,1 @@ 

- SHA512 (virtualenv-20.0.23.tar.gz) = 747ae0031ae587dc2bdc0743dcab7b1673273a5c79fc4bcb539eacd899ddc064b622e567b654fc40bdfdaa90e311d2621856085417b8fd344878546b19cdce84

+ SHA512 (virtualenv-20.0.25.tar.gz) = c143a76ef37ad4b74d37c9dcb01711fe408b5971905fe9eb46183f47de5d99ef275859fec65b6b5edf1fdf3bb0a13c21b8d4bb9ee984b9a5b80e5a4dc8f1e712

file modified
+6
@@ -17,6 +17,9 @@ 

      - smoke34nd:

          dir: python/smoke

          run: VERSION=3.4 METHOD=virtualenv-no-download ./venv.sh

+     - smoke34sp:

+         dir: python/smoke

+         run: VERSION=3.4 METHOD=virtualenv-seeder-pip ./venv.sh

      - smoke35:

          dir: python/smoke

          run: VERSION=3.5 METHOD=virtualenv ./venv.sh
@@ -32,6 +35,9 @@ 

      - smoke38nd:

          dir: python/smoke

          run: VERSION=3.8 METHOD=virtualenv-no-download ./venv.sh

+     - smoke38sp:

+         dir: python/smoke

+         run: VERSION=3.8 METHOD=virtualenv-seeder-pip ./venv.sh

      required_packages:

      - gcc

      - virtualenv

This update makes our downstream patch smaller and it's also documented what to do if you need to do what we do: https://virtualenv.pypa.io/en/stable/user_guide.html#embed-wheels-for-distributions On the other hand, we have to also disable automatic periodic updates of bundled wheels.

Test depends on: https://src.fedoraproject.org/tests/python/pull-request/26

Testing:

Python 3.4

# ll /usr/share/python-wheels/
total 1636
-rwxr-xr-x. 1 root root 1289783 Apr 20 12:08 pip-20.0.2-py2.py3-none-any.whl
-rwxr-xr-x. 1 root root  359270 Apr  1 23:27 setuptools-46.1.3-py3-none-any.whl
-rwxr-xr-x. 1 root root   21726 May 22 13:55 wheel-0.33.6-py2.py3-none-any.whl
# ll /usr/lib64/python3.4/ensurepip/_bundled/
total 1692
-rw-r--r--. 1 root root 1254803 Mar 18  2019 pip-9.0.1-py2.py3-none-any.whl
-rw-r--r--. 1 root root  472830 Mar 18  2019 setuptools-28.8.0-py2.py3-none-any.whl
# virtualenv -p python3.4 test34
created virtual environment CPython3.4.10.final.0-64 in 398ms
  creator CPython3Posix(dest=/tmp/test34, clear=False, global=False)
  seeder FromAppData(extra_search_dir=/usr/lib64/python3.4/ensurepip/_bundled, /usr/share/python-wheels,download=False, pip=bundle, setuptools=bundle, wheel=bundle, via=copy, app_data_dir=/root/.local/share/virtualenv)
    added seed packages: pip==9.0.1, setuptools==28.8.0, wheel==0.33.6
  activators BashActivator,CShellActivator,FishActivator,PowerShellActivator,PythonActivator
# virtualenv -p python3.4 --seeder pip test34pip
created virtual environment CPython3.4.10.final.0-64 in 2071ms
  creator CPython3Posix(dest=/tmp/test34pip, clear=False, global=False)
  seeder PipInvoke(extra_search_dir=/usr/lib64/python3.4/ensurepip/_bundled, /usr/share/python-wheels,download=False, pip=bundle, setuptools=bundle, wheel=bundle)
    added seed packages: pip==9.0.1, setuptools==28.8.0, wheel==0.33.6
  activators BashActivator,CShellActivator,FishActivator,PowerShellActivator,PythonActivator

Python 3.9

# virtualenv -p python3.9 test39
created virtual environment CPython3.9.0.beta.3-64 in 337ms
  creator CPython3Posix(dest=/tmp/test39, clear=False, global=False)
  seeder FromAppData(extra_search_dir=/usr/share/python-wheels,download=False, pip=bundle, setuptools=bundle, wheel=bundle, via=copy, app_data_dir=/root/.local/share/virtualenv)
    added seed packages: pip==20.0.2, setuptools==46.1.3, wheel==0.33.6
  activators BashActivator,CShellActivator,FishActivator,PowerShellActivator,PythonActivator,XonshActivator
# virtualenv -p python3.9 --seeder pip test39pip
created virtual environment CPython3.9.0.beta.3-64 in 2802ms
  creator CPython3Posix(dest=/tmp/test39pip, clear=False, global=False)
  seeder PipInvoke(extra_search_dir=/usr/share/python-wheels,download=False, pip=bundle, setuptools=bundle, wheel=bundle)
    added seed packages: pip==20.0.2, setuptools==46.1.3, wheel==0.33.6
  activators BashActivator,CShellActivator,FishActivator,PowerShellActivator,PythonActivator,XonshActivator

Python 2

# ll /usr/lib64/python2.7/ensurepip/_bundled/
total 1948
-rw-r--r--. 1 root root 1414986 May  7 22:28 pip-19.2.3-py2.py3-none-any.whl
-rw-r--r--. 1 root root  576332 May  7 22:28 setuptools-41.2.0-py2.py3-none-any.whl
# virtualenv -p python2.7 test27
created virtual environment CPython2.7.18.final.0-64 in 371ms
  creator CPython2Posix(dest=/tmp/test27, clear=False, global=False)
  seeder FromAppData(extra_search_dir=/usr/lib64/python2.7/ensurepip/_bundled, /usr/share/python-wheels,download=False, pip=bundle, setuptools=bundle, wheel=bundle, via=copy, app_data_dir=/root/.local/share/virtualenv)
    added seed packages: pip==19.2.3, setuptools==41.2.0, wheel==0.33.6
  activators BashActivator,CShellActivator,FishActivator,PowerShellActivator,PythonActivator
# virtualenv -p python2.7 --seeder pip test27pip
DEPRECATION: Python 2.7 will reach the end of its life on January 1st, 2020. Please upgrade your Python as Python 2.7 won't be maintained after that date. A future version of pip will drop support for Python 2.7. More details about Python 2 support in pip, can be found at https://pip.pypa.io/en/latest/development/release-process/#python-2-support
created virtual environment CPython2.7.18.final.0-64 in 2203ms
  creator CPython2Posix(dest=/tmp/test27pip, clear=False, global=False)
  seeder PipInvoke(extra_search_dir=/usr/lib64/python2.7/ensurepip/_bundled, /usr/share/python-wheels,download=False, pip=bundle, setuptools=bundle, wheel=bundle)
    added seed packages: pip==19.2.3, setuptools==41.2.0, wheel==0.33.6
  activators BashActivator,CShellActivator,FishActivator,PowerShellActivator,PythonActivator

Build failed.

This is not very nice:

<mock-chroot> bash-5.0$ virtualenv --upgrade-embed-wheels
AttributeError: 'NoneType' object has no attribute 'items'

What can we do to provide a better message?

This is not very nice:
<mock-chroot> bash-5.0$ virtualenv --upgrade-embed-wheels
AttributeError: 'NoneType' object has no attribute 'items'

What can we do to provide a better message?

Do we want to support this functionality somehow or just improve the error message to say something like "we are using system wheels provided by RPM package and therefore they cannot be updated this way"? The latter would be much easier IMHO.

rebased onto 02c0e1d

3 years ago

Fixed to:

# virtualenv --upgrade-embed-wheels
In Fedora, virtualenv uses system wheels provided by RPM packages. Check system updates via dnf.
SystemExit: 1

Build failed.

What about:

virtualenv installed from the RPM package uses wheels from RPM packages as well. Updating them via virtualenv is not possible. The RPM packaged wheels are updated together with other RPM packages of the system.

It intentionally avoids the terms Fedora and dnf, so the patch is less platform dependent.
Also, considering that we rarely rebase pip/setuptools in stable Fedoras, I found the "check system updates" a bit too enthusiastic.

rebased onto d78c1f9

3 years ago

Build succeeded.

rebased onto 1de75fd

3 years ago

Build succeeded.

Pull-Request has been merged by churchyard

3 years ago