Splitting Scrapy to Selectors

extract_selectors.bash

#!/bin/bash

cd ~/Development # change it to where you want `selectors` dir
rm -rf selectors

# git clone git@github.com:scrapy/scrapy.git selectors
git clone scrapy-fresh/ selectors
cd selectors
git remote rename origin scrapy
git checkout master
scrapy_sha1=$(git rev-parse HEAD)

git checkout --orphan selectors
git rm --cached -r .

mv scrapy/selector selectors

for f in $(git ls-files -o |\
            grep -v "^.gitignore$" |\
            grep -v "^selectors/" |\
            grep -v "^scrapy/utils/__init__.py$" |\
            grep -v "^scrapy/utils/decorator.py$" |\
            grep -v "^scrapy/utils/misc.py$" |\
            grep -v "^scrapy/utils/python.py$" |\
            grep -v "^tests/test_selector.py$" |\
            grep -v "^tests/test_selector_csstranslator.py$" |\
            grep -v "^docs/utils/" |\
            grep -v "^docs/_ext" |\
            grep -v "^docs/_static" |\
            grep -v "^docs/README$" |\
            grep -v "^docs/Makefile$" |\
            grep -v "^docs/topics/selectors.rst$"); do
    rm $f;
done

mv scrapy/utils selectors/
find . -type d -empty -delete

git add -A

# to store it in commits message
note="A note about history:

This project been split out of scrapy
(https://github.com/scrapy/scrapy).

To resolve the full history in your local copy
first fetch the remaining history:

    git remote add scrapy git@github.com:scrapy/scrapy.git
    git fetch scrapy

Then add this git-replace ref:

    git replace <this-commit-sha1> $scrapy_sha1"

git commit -m "Selectors are dead, long live Selectors" -m "$note"

# following steps are possible because I manually done them first
# although least but there is a chance it will create a merge conflict in future

# git remote add selectors git@github.com:umrashrf/selectors.git
git remote add selectors ~/Development/selectors-fresh/
git fetch selectors
# apply changes
# FIXME: commit hash will always change, so use a rebase from a branch
git cherry-pick b2fe0e8 # removed unnecessary lines from .gitignore
git cherry-pick bfd559c # added selectors readme
git cherry-pick fb2c722 # made selectors independent of scrapy
git cherry-pick 555b75f # added selectors specific python package files
git cherry-pick 3bf1428 # removed scrapy dependent tests and changed code to suit selectors package
git cherry-pick cffe293 # added selectors tests specific support files