Script to demonstrate issue with pip-compile pre-commit hook handling of git worktree folders

.gitignore

workspace

bug_demo.sh

#/bin/sh

# This is a demonstration of a strange bug in the pip-compile tool when run as
# a pre-commit hook under a git worktree, where it corrupts the worktree's git
# index.
#
# See [pip-tools issue 1598](https://github.com/jazzband/pip-tools/issues/1598)
# for information.

workspace="workspace"

announce() {
  local divider="\n\n=================================================================\n\n"
  echo "$divider\n$1\n"
}

echo "Making workspace directory: $workspace"
[ -d $workspace ] && echo "  (Wiping first)" && (rm -rf $workspace || exit 1)
mkdir $workspace && cd $workspace

announce "Cloning dependent repo"
git clone https://github.com/gimbo/pip_compile_worktree_bug_demo.git || exit 1

announce "Creating sibling worktree of dependent repo"
git --git-dir=pip_compile_worktree_bug_demo/.git worktree add worktree_dir -b other_branch || exit 1
echo $divider

announce "Entering worktree and setting up python venv"
cd worktree_dir
python -m venv .venv || exit 1
source .venv/bin/activate
pip install pip==22.0.4 setuptools==60.9.3 pip-tools==6.5.1 pre-commit==2.17.0 || exit 1

announce "Installing pre-commit and hook"
pre-commit install --install-hooks

announce "Modifying requirements.in and running pre-commit to change requirements.txt"
echo "Here pre-commit will say 'Failed', but only because it's done what we want...\n"
echo "pipdeptree==2.2.1" >> requirements.in
pre-commit run --all-files

announce "Listing files in worktree's git index with: git ls-files -s"
echo "This should show files from pip_compile_worktree_bug_demo\n"
git ls-files -s

announce "Attempting to commit changes to requirements.*"
git add requirements.* || exit 1
git commit -m "Add pipdeptree dependency"

announce "That should have failed, with e.g. 'error: Error building trees'"

announce "Listing files in worktree's git index with: git ls-files -s"
echo "This 'should' now show files from more-itertools (which is wrong).\n"
git ls-files -s

announce "Trying git status"
git status

announce "Done"

example_run.txt

$ cd /tmp

$ git clone https://gist.github.com/c2d43c9ff2f63771c6d1086422311b2b.git bug_demo
Cloning into 'bug_demo'...
remote: Enumerating objects: 4, done.
remote: Counting objects: 100% (4/4), done.
remote: Compressing objects: 100% (3/3), done.
remote: Total 4 (delta 0), reused 4 (delta 0), pack-reused 0
Receiving objects: 100% (4/4), done.

$ cd bug_demo

$ sh ./bug_demo.sh
Making workspace directory: workspace


=================================================================


Cloning dependent repo

Cloning into 'pip_compile_worktree_bug_demo'...
remote: Enumerating objects: 7, done.
remote: Counting objects: 100% (7/7), done.
remote: Compressing objects: 100% (6/6), done.
remote: Total 7 (delta 0), reused 7 (delta 0), pack-reused 0
Receiving objects: 100% (7/7), done.


=================================================================


Creating sibling worktree of dependent repo

Preparing worktree (new branch 'other_branch')
HEAD is now at facde82 Initial commit



=================================================================


Entering worktree and setting up python venv

Collecting pip==22.0.4
  Using cached pip-22.0.4-py3-none-any.whl (2.1 MB)
Collecting setuptools==60.9.3
  Using cached setuptools-60.9.3-py3-none-any.whl (1.1 MB)
Collecting pip-tools==6.5.1
  Using cached pip_tools-6.5.1-py3-none-any.whl (47 kB)
Collecting pre-commit==2.17.0
  Using cached pre_commit-2.17.0-py2.py3-none-any.whl (195 kB)
Collecting pep517
  Using cached pep517-0.12.0-py2.py3-none-any.whl (19 kB)
Collecting click>=7
  Using cached click-8.0.4-py3-none-any.whl (97 kB)
Collecting wheel
  Using cached wheel-0.37.1-py2.py3-none-any.whl (35 kB)
Collecting nodeenv>=0.11.1
  Using cached nodeenv-1.6.0-py2.py3-none-any.whl (21 kB)
Collecting identify>=1.0.0
  Using cached identify-2.4.11-py2.py3-none-any.whl (98 kB)
Collecting pyyaml>=5.1
  Using cached PyYAML-6.0-cp39-cp39-macosx_11_0_arm64.whl (173 kB)
Collecting virtualenv>=20.0.8
  Using cached virtualenv-20.13.3-py2.py3-none-any.whl (8.7 MB)
Collecting cfgv>=2.0.0
  Using cached cfgv-3.3.1-py2.py3-none-any.whl (7.3 kB)
Collecting toml
  Using cached toml-0.10.2-py2.py3-none-any.whl (16 kB)
Collecting distlib<1,>=0.3.1
  Using cached distlib-0.3.4-py2.py3-none-any.whl (461 kB)
Collecting filelock<4,>=3.2
  Using cached filelock-3.6.0-py3-none-any.whl (10.0 kB)
Collecting six<2,>=1.9.0
  Using cached six-1.16.0-py2.py3-none-any.whl (11 kB)
Collecting platformdirs<3,>=2
  Using cached platformdirs-2.5.1-py3-none-any.whl (14 kB)
Collecting tomli>=1.1.0
  Using cached tomli-2.0.1-py3-none-any.whl (12 kB)
Installing collected packages: tomli, six, platformdirs, filelock, distlib, wheel, virtualenv, toml, setuptools, pyyaml, pip, pep517, nodeenv, identify, click, cfgv, pre-commit, pip-tools
  Attempting uninstall: setuptools
    Found existing installation: setuptools 58.1.0
    Uninstalling setuptools-58.1.0:
      Successfully uninstalled setuptools-58.1.0
  Attempting uninstall: pip
    Found existing installation: pip 21.2.4
    Uninstalling pip-21.2.4:
      Successfully uninstalled pip-21.2.4
Successfully installed cfgv-3.3.1 click-8.0.4 distlib-0.3.4 filelock-3.6.0 identify-2.4.11 nodeenv-1.6.0 pep517-0.12.0 pip-22.0.4 pip-tools-6.5.1 platformdirs-2.5.1 pre-commit-2.17.0 pyyaml-6.0 setuptools-60.9.3 six-1.16.0 toml-0.10.2 tomli-2.0.1 virtualenv-20.13.3 wheel-0.37.1


=================================================================


Installing pre-commit and hook

pre-commit installed at /private/tmp/bug_demo/workspace/pip_compile_worktree_bug_demo/.git/hooks/pre-commit


=================================================================


Modifying requirements.in and running pre-commit to change requirements.txt

Here pre-commit will say 'Failed', but only because it's done what we want...

pip-compile requirements.in..............................................Failed
- hook id: pip-compile
- duration: 7.25s
- files were modified by this hook

Using indexes:
  https://pypi.org/simple

                          ROUND 1
Current constraints:
  more_itertools from git+https://github.com/more-itertools/more-itertools@v8.12.0#egg=more_itertools (from -r requirements.in (line 1))
  pipdeptree==2.2.1 (from -r requirements.in (line 3))

Finding the best candidates:
  found candidate more_itertools @ git+https://github.com/more-itertools/more-itertools@v8.12.0 (constraint was <any>)
  found candidate pipdeptree==2.2.1 (constraint was ==2.2.1)

Finding secondary dependencies:
  pipdeptree==2.2.1         requires pip>=6.0.0

New dependencies found in this round:
  adding ('pip', '>=6.0.0', [])
Removed dependencies in this round:
------------------------------------------------------------
Result of round 1: not stable

                          ROUND 2
Current constraints:
  more_itertools from git+https://github.com/more-itertools/more-itertools@v8.12.0#egg=more_itertools (from -r requirements.in (line 1))
  pip>=6.0.0 (from pipdeptree==2.2.1->-r requirements.in (line 3))
  pipdeptree==2.2.1 (from -r requirements.in (line 3))

Finding the best candidates:
  found candidate more_itertools @ git+https://github.com/more-itertools/more-itertools@v8.12.0 (constraint was <any>)
  found candidate pip==22.0.4 (constraint was >=6.0.0)
  found candidate pipdeptree==2.2.1 (constraint was ==2.2.1)

Finding secondary dependencies:
  pip==22.0.4               requires -
  pipdeptree==2.2.1         requires pip>=6.0.0
------------------------------------------------------------
Result of round 2: stable, done

#
# This file is autogenerated by pip-compile with python 3.9
# To update, run:
#
#    pip-compile --allow-unsafe --pip-args='"--verbose"' requirements.in
#
more_itertools @ git+https://github.com/more-itertools/more-itertools@v8.12.0
    # via -r requirements.in
pipdeptree==2.2.1
    # via -r requirements.in

# The following packages are considered to be unsafe in a requirements file:
pip==22.0.4
    # via pipdeptree



=================================================================


Listing files in worktree's git index with: git ls-files -s

This should show files from pip_compile_worktree_bug_demo

100644 617ef14a4f6486f04eea6440829862ab682aa137 0	.gitignore
100644 012f27ec098959491d0d1a2009741e2c44bac1ce 0	.pre-commit-config.yaml
100644 ffcf9b2d48f818693563687b0b000f2bb3b9b458 0	README.md
100644 f7ea6efc23e54362af50e1c29bc77ca64c2d622f 0	requirements.in
100644 66be85ea884c75192def16238aff63d7ba08d0b2 0	requirements.txt


=================================================================


Attempting to commit changes to requirements.*

pip-compile requirements.in..............................................Passed
- hook id: pip-compile
- duration: 7.19s

Using indexes:
  https://pypi.org/simple

                          ROUND 1
Current constraints:
  more_itertools from git+https://github.com/more-itertools/more-itertools@v8.12.0#egg=more_itertools (from -r requirements.in (line 1))
  pipdeptree==2.2.1 (from -r requirements.in (line 3))

Finding the best candidates:
  found candidate more_itertools @ git+https://github.com/more-itertools/more-itertools@v8.12.0 (constraint was <any>)
  found candidate pipdeptree==2.2.1 (constraint was ==2.2.1)

Finding secondary dependencies:
  pipdeptree==2.2.1         requires pip>=6.0.0

New dependencies found in this round:
  adding ('pip', '>=6.0.0', [])
Removed dependencies in this round:
------------------------------------------------------------
Result of round 1: not stable

                          ROUND 2
Current constraints:
  more_itertools from git+https://github.com/more-itertools/more-itertools@v8.12.0#egg=more_itertools (from -r requirements.in (line 1))
  pip>=6.0.0 (from pipdeptree==2.2.1->-r requirements.in (line 3))
  pipdeptree==2.2.1 (from -r requirements.in (line 3))

Finding the best candidates:
  found candidate more_itertools @ git+https://github.com/more-itertools/more-itertools@v8.12.0 (constraint was <any>)
  found candidate pip==22.0.4 (constraint was >=6.0.0)
  found candidate pipdeptree==2.2.1 (constraint was ==2.2.1)

Finding secondary dependencies:
  pip==22.0.4               requires -
  pipdeptree==2.2.1         requires pip>=6.0.0
------------------------------------------------------------
Result of round 2: stable, done

#
# This file is autogenerated by pip-compile with python 3.9
# To update, run:
#
#    pip-compile --allow-unsafe --pip-args='"--verbose"' requirements.in
#
more_itertools @ git+https://github.com/more-itertools/more-itertools@v8.12.0
    # via -r requirements.in
pipdeptree==2.2.1
    # via -r requirements.in

# The following packages are considered to be unsafe in a requirements file:
pip==22.0.4
    # via pipdeptree

error: invalid object 100644 c80dd09502ddba8e065a01cfe09187a135d12c56 for '.github/ISSUE_TEMPLATE/other-issues.md'
error: Error building trees


=================================================================


That should have failed, with e.g. 'error: Error building trees'



=================================================================


Listing files in worktree's git index with: git ls-files -s

This 'should' now show files from more-itertools (which is wrong).

100644 9aaba7a9fe8ebe94a6354dbd72f8ec3985b4b811 0	.gitattributes
100644 c80dd09502ddba8e065a01cfe09187a135d12c56 0	.github/ISSUE_TEMPLATE/other-issues.md
100644 fa97c780bb321039c21224a0235413f141f8de11 0	.github/ISSUE_TEMPLATE/propose-a-new-itertool.md
100644 27e2985c5bc7fa09a783d072ee4fc397caac7120 0	.github/PULL_REQUEST_TEMPLATE.md
100644 387a2a4d1a34d9cad4f8e4dc88f92a10aeae56f5 0	.github/workflows/python-app.yml
100644 22555cfec7ede29412f141edd04993f43611b9bf 0	.gitignore
100644 0a523bece3e50519653c4d7a38399baa487fefa1 0	LICENSE
100644 6d7fd5bf28c3163682a94db8e93d102ca1400c54 0	MANIFEST.in
100644 4df22091a4fb234eee4abafa1756ecb9ab295a0d 0	README.rst
100644 47888da7b72bcf710f88b72b9dac22749b4d8b74 0	docs/Makefile
100644 3a451aeae168f717d5f0dc7bb4cc1536d675f5a9 0	docs/_static/theme_overrides.css
100644 84890e18839187e9725816f670936d6da8411432 0	docs/api.rst
100644 ecfbfc0b5850fca046553d5abfb8a6b4de63f255 0	docs/conf.py
100644 23e4ac27a075b949a1abe01ea228e404a53c503e 0	docs/index.rst
100644 123c0f54dcaeeaaeda93329a17872b5b128089c6 0	docs/license.rst
100644 8023c0aa67409ee6552f3a73dff3a0522395d678 0	docs/make.bat
100644 6004a7f767d0a40c8577749426160c22648fbd8d 0	docs/requirements.txt
100644 a67c90e00cdbce88a0d4eac272716d5b5b4bae54 0	docs/testing.rst
100644 c9ec0187e7f22fb86190c9e0a1366daffca33cb2 0	docs/versions.rst
100644 ea38bef1f661e62d577b3c2207386d901d851c72 0	more_itertools/__init__.py
100644 96f6e36c7f4ac9ea0aebdcd9e11b8d1ff092d2ef 0	more_itertools/__init__.pyi
100755 630af973f25214d1f34e09a113f69dfb1c944269 0	more_itertools/more.py
100644 fe7d4bdd7a822c713142dca8a3f4289cca89ee07 0	more_itertools/more.pyi
100644 e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 0	more_itertools/py.typed
100644 a2596423a4c3dbd15a357241477a0af0a531f9ec 0	more_itertools/recipes.py
100644 4648a41b5e51155d4db10cd5e221e8a7a9449201 0	more_itertools/recipes.pyi
100644 6c044abedc52a4b720b95f24a3fb8e5ee8b3c195 0	pyproject.toml
100644 26f9d97a3fb64713d15f08ecc8c46f5cdc397165 0	setup.cfg
100644 6e6768441f4261ea19cd38614097e20091840906 0	setup.py
100644 e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 0	tests/__init__.py
100644 f6d51393420f3c36d027dbfecae1018dd1e295a0 0	tests/test_more.py
100644 be409957490b4b32decab34cc8ebaf001deee963 0	tests/test_recipes.py
100644 983031a796a6420fc23a9f10ee4505d3f4d162ce 0	tox.ini


=================================================================


Trying git status

fatal: unable to read 6004a7f767d0a40c8577749426160c22648fbd8d


=================================================================


Done