From c85ce174d148eff79353af94d06ca95c50f60418 Mon Sep 17 00:00:00 2001 From: Tycho Andersen Date: Tue, 10 Dec 2024 12:26:40 -0700 Subject: [PATCH] ci: always use $PATH python Consider the following output form `which python3` and `which pytest-3` in the CI environment: which python3 shell: /usr/bin/bash -e {0} env: pythonLocation: /opt/hostedtoolcache/Python/3.10.15/x64 PKG_CONFIG_PATH: /opt/hostedtoolcache/Python/3.10.15/x64/lib/pkgconfig Python_ROOT_DIR: /opt/hostedtoolcache/Python/3.10.15/x64 Python2_ROOT_DIR: /opt/hostedtoolcache/Python/3.10.15/x64 Python3_ROOT_DIR: /opt/hostedtoolcache/Python/3.10.15/x64 LD_LIBRARY_PATH: /opt/hostedtoolcache/Python/3.10.15/x64/lib /opt/hostedtoolcache/Python/3.10.15/x64/bin/python3 which pytest-3 shell: /usr/bin/bash -e {0} env: pythonLocation: /opt/hostedtoolcache/Python/3.10.15/x64 PKG_CONFIG_PATH: /opt/hostedtoolcache/Python/3.10.15/x64/lib/pkgconfig Python_ROOT_DIR: /opt/hostedtoolcache/Python/3.10.15/x64 Python2_ROOT_DIR: /opt/hostedtoolcache/Python/3.10.15/x64 Python3_ROOT_DIR: /opt/hostedtoolcache/Python/3.10.15/x64 LD_LIBRARY_PATH: /opt/hostedtoolcache/Python/3.10.15/x64/lib /usr/bin/pytest-3 Since we were getting e.g. cffi vi `apt-get install python3-cffi`, we were using the *system* python's cffi for everything. That worked because we invoked the *system* pytest-3. Of course, that means we were not really using the github python action's python in our CI matrix, since we were always using the system python for everything. Instead, let's set up a venv as a traditional project would, and that way we can explicitly call that venv's python everywhere in the makefile, so we always use that venv's packages and get the right python. Then, we need only be careful to use the $PATH python, not /usr/bin/python, to set up this venv, and we always get the right version of python to test with. This likely explains some weirdness that I've seen in the past, should have investigated sooner... Signed-off-by: Tycho Andersen --- .github/workflows/ci.yaml | 4 ++-- .gitignore | 1 + Makefile | 22 +++++++++++++++------- requirements.txt | 3 +++ 4 files changed, 21 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4501b3b..13fd44c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -25,7 +25,7 @@ jobs: with: ghc-version: '9.6' cabal-version: latest - - run: cabal update - - run: sudo apt install x11-apps python3-pytest python3-cffi python3-pytest-xdist flake8 + - run: make xcffib_venv + - run: sudo apt install x11-apps - run: git clone https://gitlab.freedesktop.org/xorg/proto/xcbproto.git proto && cd proto && git checkout ${{ matrix.xcbver }} - run: make XCBDIR=./proto/src check diff --git a/.gitignore b/.gitignore index b49ac5b..5ea3e55 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ dist-newstyle *__pycache__* *.egg-info xcffib +xcffib_venv build/ *egg* *deb diff --git a/Makefile b/Makefile index bb28f6b..1665e00 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,9 @@ NCPUS=$(shell grep -c processor /proc/cpuinfo) PARALLEL=$(shell which parallel) CABAL=cabal --config-file=./cabal.config GEN=$(CABAL) new-run --minimize-conflict-set -j$(NCPUS) exe:xcffibgen -- +VENV=xcffib_venv +PYTHON=$(VENV)/bin/python3 +FLAKE=$(VENV)/bin/flake8 # you should have xcb-proto installed to run this xcffib: module/*.py xcffib.cabal $(shell find . -path ./test -prune -false -o -name \*.hs) @@ -36,7 +39,7 @@ gen: dist-newstyle .PHONY: clean clean: -$(CABAL) new-clean - -rm -rf xcffib + -rm -rf xcffib xcffib_venv -rm -rf module/*pyc module/__pycache__ -rm -rf test/*pyc test/__pycache__ -rm -rf build *egg* *deb .pybuild dist @@ -51,18 +54,23 @@ newtests: # These are all split out so make -j3 check goes as fast as possible. .PHONY: lint -lint: - flake8 --config=./test/flake8.cfg ./module +lint: $(VENV) + $(FLAKE) --config=./test/flake8.cfg ./module .PHONY: htests htests: $(CABAL) new-test -j$(NCPUS) --enable-tests -check: xcffib lint htests +$(VENV): requirements.txt + # the python in $PATH in CI is the python from the matrix, so it is the + # "right" python to start with + python3 -m venv $(VENV) + $(PYTHON) -m pip install -r requirements.txt + +check: xcffib htests $(VENV) lint cabal check - flake8 -j$(NCPUS) --ignore=E128,E231,E251,E301,E302,E305,E501,F401,E402,W503,E741,E999 xcffib/*.py - python3 -m compileall xcffib - pytest-3 -v --durations=3 -n auto + $(PYTHON) -m compileall xcffib + $(PYTHON) -m pytest -v --durations=3 -n $(NCPUS) # make release ver=0.99.99 release: xcffib diff --git a/requirements.txt b/requirements.txt index 13e2b14..b6b7907 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,6 @@ flake8 autopep8 cffi>=0.8.2 +pytest +pytest-xdist +cffi >= 1.6