heug/Dockerfile

## config.yml
version: 2
jobs:
  build:
    docker:
      - image: heug/triggermail:1.3
        environment:
          CIRCLE_ARTIFACTS: /home/circleci/trigger_mail/artifacts
          CIRCLE_TEST_REPORTS: /home/circleci/trigger_mail/test_reports
          APP_ENGINE_DIR: /home/circleci/google-cloud-sdk/platform/google_appengine
          # SITE_PACKAGES: python -c 'import distutils.sysconfig; print distutils.sysconfig.get_python_lib()'
          PATH: "/home/circleci/google-cloud-sdk/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:~/trigger_mail/google-cloud-sdk/platform/google_appengine"
          BASH_ENV: "~/venv/bin/activate"
    working_directory: ~/trigger_mail
    parallelism: 12

    steps:
      - run: mkdir -p $CIRCLE_ARTIFACTS $CIRCLE_TEST_REPORTS && touch $CIRCLE_ARTIFACTS/
      # source cache
      - restore_cache:
          key: v1-triggermail-source-{{ .Branch }}-{{ .Revision }}
          keys:
            - v1-triggermail-source-{{ .Branch }}-
            - v1-triggermail-source-

      - checkout

      - save_cache:
          key: v1-triggermail-source-{{ .Branch }}-{{ .Revision }}
          paths:
            - ".git"
      # - restore_cache:
      #     keys:
      #       - v3-triggermail-pip-{{ checksum "requirements_app_engine.txt" }}
      #       - v3-triggermail-pip-
      #       - v2-triggermail-pip-

      - run:
          name: virtual env
          command: |
            sudo -H pip install virtualenv
            virtualenv ~/venv
      - run: pip install -r requirements_app_engine.txt

      # add missing pip dependencies
      - run: pip install wheel gunicorn pillow

      - run: ./install_gcloud.sh

      # - save_cache:
      #     key: v3-triggermail-pip-{{ checksum "requirements_app_engine.txt" }}
      #     paths:
      #       - ~/.cache
      #       - ~/google-cloud-sdk
      #       - ~/venv

      - run: mkdir -p $HOME/.go_workspace/src/github.com/$CIRCLE_PROJECT_USERNAME
      - run: ln -fs $HOME/$CIRCLE_PROJECT_REPONAME $HOME/.go_workspace/src/github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME
      # get and compile Go dependencies; compile tests with -race because we run them that way
      - run: cd /home/circleci/.go_workspace/src/github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME && go get -t -d -v ./go/... || true
      # compile and cache all Go test dependencies
      - run: cd $HOME/.go_workspace/src/github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME && go test -v -i -race ./go/...
      - run: if [ $CIRCLE_NODE_INDEX -eq 0 ] ; then yarn install ; npm install ; fi
      - run: touch venv && cd src && ../run_tests_or_ui.sh

      # - run:
      #     name: install gcloud 2
      #     working_directory: ~/google-cloud-sdk/platform/google_appengine
      #     command: |
      #       # symmlink app engine .py's into venv here
      #       cp *.py ~/venv/lib/python2.7/site-packages
      #       cp *.pyc ~/venv/lib/python2.7/site-packages

      #       # ln -sfL ~/google-cloud-sdk/platform/google_appengine ~/venv/lib/python2.7/site-packages
      #       # sudo ln -sfL ~/google-cloud-sdk/platform/google_appengine /usr/local/lib/python2.7/site-packages

      # - run:
      #     name: gcloud stuff 3
      #     command: |
      #       sudo mkdir -p .go_workspace/src/github.com/$CIRCLE_PROJECT_USERNAME
      #       sudo ln -fsL /home/circleci/trigger_mail /home/circleci/trigger_mail/.go_workspace/src/github.com/TriggerMail/trigger_mail

      #       # get and compile Go dependencies; compile tests with -race because we run them that way
      #       # cd .go_workspace/src/github.com/TriggerMail/trigger_mail && go get -t -d -v ./go/...

      #       # compile and cache all Go test dependencies
      #       # cd .go_workspace/src/github.com/TriggerMail/trigger_mail && go test -v -i -race ./go/...

      # - run:
      #     name: test reports & artifacts
      #     command: |
      #       mkdir /home/circleci/trigger_mail/test_reports
      #       mkdir /home/circleci/trigger_mail/artifacts

      #       sudo chown -R circleci /home/circleci/trigger_mail/test_reports
      #       sudo chown -R circleci /home/circleci/trigger_mail/artifacts

      # # in dockerfile now
      # - run:
      #     name: pip permissions
      #     command: |
      #       sudo chown -R circleci /usr/local


      # - run:
      #     name: test
      #     working_directory: ~/trigger_mail/src
      #     command: |
      #       if [ $CIRCLE_NODE_INDEX -eq 1 ]; then
      #           START=$(date +%s)

      #           REPO_IN_GOPATH=$HOME/.go_workspace/src/github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME
      #           # Run tests with race detector and timeout a test if it takes longer than 1 minute
      #           cd $REPO_IN_GOPATH && go test -v -race -test.timeout=1m ./go/...

      #           # require code to pass go vet checks
      #           cd $REPO_IN_GOPATH && go vet ./go/...

      #           # require code to be formatted
      #           NOT_FORMATTED=`cd $REPO_IN_GOPATH && gofmt -l ./go`
      #           if [ ! -z "$NOT_FORMATTED" ]; then
      #               echo "ERROR files must be formatted with gofmt:" > /dev/stderr
      #               echo $NOT_FORMATTED > /dev/stderr
      #               exit 10
      #           fi

      #           END=$(date +%s)
      #           ELAPSED=$((END-START))
      #           echo "run_tests_or_ui.sh: ran go checks in $ELAPSED seconds"
      #       fi

      #       exit_val=0
      #       python $HOME/trigger_mail/should_build_ui.py || exit_val=$?

      #       SKIP_MINIFY=true
      #       if [ $exit_val -eq 102 ]; then
      #           SKIP_MINIFY=false
      #       fi
      #       if [ $exit_val -ge 101 ]; then
      #           if [ $CIRCLE_NODE_INDEX -eq 0 ]; then
      #               $HOME/trigger_mail/build_ui.sh $SKIP_MINIFY;
      #               exit 0
      #           fi
      #           NODE_INDEX=$((CIRCLE_NODE_INDEX - 1));
      #           NODE_TOTAL=$((CIRCLE_NODE_TOTAL - 1));
      #       elif [ $exit_val -eq 0 ]; then
      #           echo "No UI changes detected in the PR or Commit"
      #           NODE_INDEX=$CIRCLE_NODE_INDEX;
      #           NODE_TOTAL=$CIRCLE_NODE_TOTAL;
      #       else
      #           echo "ERROR unknown exit code from should_build_ui.py"
      #           exit 10
      #       fi

      # - run:
      #     name: test part 2
      #     working_directory: ~/trigger_mail/src
      #     command: |
      #       # this is a totally gross hack: Find directories in the current directory, or under packages that
      #       # contain a directory named tests
      #       dirs_with_tests=`find $HOME/trigger_mail $HOME/trigger_mail/packages -maxdepth 2 -type d -name tests | xargs -n1 dirname`
      #       # remove packages that don't actually include tests
      #       packages_with_tests=`echo "${dirs_with_tests}" | grep -v '/trigger_mail$' | grep -v '/trigger_mail/rainforest$' | grep -v '/trigger_mail/spark$'`

      #       # Pre-build the packages and put the wheels into /bluecore-built-packages.
      #       python $HOME/trigger_mail/package_builder.py $HOME/trigger_mail $HOME/trigger_mail/bluecore-built-packages

      #       # CircleCI docs say to put test reports in a subdirectory:
      #       # https://circleci.com/docs/1.0/test-metadata/#metadata-collection-in-custom-test-steps
      #       sudo mkdir -p ${CIRCLE_TEST_REPORTS}/nose
      #       sudo chown -R circleci ${CIRCLE_TEST_REPORTS}/nose

      #       # Setup the services
      #       for package in $packages_with_tests
      #       do
      #           echo "Building and running tests for ${package} ..."
      #           python $HOME/trigger_mail/service_setup.py $package --skip-wheel-builds
      #           cd $package
      #           PACKAGE_WITHOUT_SLASHES=`echo "$package" | sed 's/\//_/g'`
      #           PYTHONPATH=.:service_lib/installed:service_lib/protocol_buffers \
      #               ./venv/bin/nosetests tests \
      #               --with-parallel -v --processes=4 --process-timeout=540 \
      #               --with-xunitmp \
      #               --xunitmp-file=$CIRCLE_TEST_REPORTS/nose/${PACKAGE_WITHOUT_SLASHES}_output.xml \
      #               --process-restartworker \
      #               > >(tee -a $CIRCLE_ARTIFACTS/test.log) 2>&1;

      #           PATH=${HOME}/trigger_mail/src/venv/bin:${PATH} go run \
      #               "${HOME}/trigger_mail/go/pylintparallel/pylintparallel.go" \
      #               "-partitionIndex=${CIRCLE_NODE_INDEX}" \
      #               "-numPartitions=${CIRCLE_NODE_TOTAL}" \
      #               "-srcRoot=${package}" \
      #               -verbose
      #       done


      # - run:
      #     name: test part 3
      #     working_directory: ~/trigger_mail/src
      #     command: |
      #       if [ $CIRCLE_NODE_INDEX -eq 0 ]; then
      #           # run root tests on node 0
      #           # TODO: integrate this with the tests above somehow
      #           cd $HOME/trigger_mail
      #           nosetests -v $HOME/trigger_mail/tests
      #       fi

## Dockerfile
FROM circleci/golang:1.9-browsers

USER root

RUN apt-get update
RUN apt-get install zlib1g-dev \
    # mysql
    default-libmysqlclient-dev mysql-server \
    # lxml
    libxml2 libxml2-dev libxslt1.1 libxslt1-dev \
    # python pre-dependencies
    build-essential checkinstall \
    libreadline-gplv2-dev libncursesw5-dev libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev \
    python-dev \
    libjpeg-dev libjpeg8-dev imagemagick

# install python
RUN wget https://www.python.org/ftp/python/2.7.13/Python-2.7.13.tgz
RUN tar xzf Python-2.7.13.tgz
RUN rm Python-2.7.13.tgz

WORKDIR Python-2.7.13

RUN ./configure
RUN make install

# install pip
RUN wget https://bootstrap.pypa.io/get-pip.py
RUN python get-pip.py

# install node/npm
RUN wget https://nodejs.org/dist/v6.9.5/node-v6.9.5.tar.gz
RUN tar xzf node-v6.9.5.tar.gz
RUN rm node-v6.9.5.tar.gz

WORKDIR node-v6.9.5

RUN ./configure
RUN make -j4
RUN make install

# start mysql
RUN service mysql start

# final steps
RUN ln -s /home/circleci /home/ubuntu
RUN npm install -g yarn
RUN git config --global user.email "CircleCI@example.com"
RUN git config --global user.name "CircleCI"

USER circleci
	version: 2
	jobs:
	build:
	docker:
	- image: heug/triggermail:1.3
	environment:
	CIRCLE_ARTIFACTS: /home/circleci/trigger_mail/artifacts
	CIRCLE_TEST_REPORTS: /home/circleci/trigger_mail/test_reports
	APP_ENGINE_DIR: /home/circleci/google-cloud-sdk/platform/google_appengine
	# SITE_PACKAGES: python -c 'import distutils.sysconfig; print distutils.sysconfig.get_python_lib()'
	PATH: "/home/circleci/google-cloud-sdk/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:~/trigger_mail/google-cloud-sdk/platform/google_appengine"
	BASH_ENV: "~/venv/bin/activate"
	working_directory: ~/trigger_mail
	parallelism: 12

	steps:
	- run: mkdir -p $CIRCLE_ARTIFACTS $CIRCLE_TEST_REPORTS && touch $CIRCLE_ARTIFACTS/
	# source cache
	- restore_cache:
	key: v1-triggermail-source-{{ .Branch }}-{{ .Revision }}
	keys:
	- v1-triggermail-source-{{ .Branch }}-
	- v1-triggermail-source-

	- checkout

	- save_cache:
	key: v1-triggermail-source-{{ .Branch }}-{{ .Revision }}
	paths:
	- ".git"
	# - restore_cache:
	# keys:
	# - v3-triggermail-pip-{{ checksum "requirements_app_engine.txt" }}
	# - v3-triggermail-pip-
	# - v2-triggermail-pip-

	- run:
	name: virtual env
	command: \|
	sudo -H pip install virtualenv
	virtualenv ~/venv
	- run: pip install -r requirements_app_engine.txt

	# add missing pip dependencies
	- run: pip install wheel gunicorn pillow

	- run: ./install_gcloud.sh

	# - save_cache:
	# key: v3-triggermail-pip-{{ checksum "requirements_app_engine.txt" }}
	# paths:
	# - ~/.cache
	# - ~/google-cloud-sdk
	# - ~/venv

	- run: mkdir -p $HOME/.go_workspace/src/github.com/$CIRCLE_PROJECT_USERNAME
	- run: ln -fs $HOME/$CIRCLE_PROJECT_REPONAME $HOME/.go_workspace/src/github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME
	# get and compile Go dependencies; compile tests with -race because we run them that way
	- run: cd /home/circleci/.go_workspace/src/github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME && go get -t -d -v ./go/... \|\| true
	# compile and cache all Go test dependencies
	- run: cd $HOME/.go_workspace/src/github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME && go test -v -i -race ./go/...
	- run: if [ $CIRCLE_NODE_INDEX -eq 0 ] ; then yarn install ; npm install ; fi
	- run: touch venv && cd src && ../run_tests_or_ui.sh

	# - run:
	# name: install gcloud 2
	# working_directory: ~/google-cloud-sdk/platform/google_appengine
	# command: \|
	# # symmlink app engine .py's into venv here
	# cp *.py ~/venv/lib/python2.7/site-packages
	# cp *.pyc ~/venv/lib/python2.7/site-packages

	# # ln -sfL ~/google-cloud-sdk/platform/google_appengine ~/venv/lib/python2.7/site-packages
	# # sudo ln -sfL ~/google-cloud-sdk/platform/google_appengine /usr/local/lib/python2.7/site-packages

	# - run:
	# name: gcloud stuff 3
	# command: \|
	# sudo mkdir -p .go_workspace/src/github.com/$CIRCLE_PROJECT_USERNAME
	# sudo ln -fsL /home/circleci/trigger_mail /home/circleci/trigger_mail/.go_workspace/src/github.com/TriggerMail/trigger_mail

	# # get and compile Go dependencies; compile tests with -race because we run them that way
	# # cd .go_workspace/src/github.com/TriggerMail/trigger_mail && go get -t -d -v ./go/...

	# # compile and cache all Go test dependencies
	# # cd .go_workspace/src/github.com/TriggerMail/trigger_mail && go test -v -i -race ./go/...

	# - run:
	# name: test reports & artifacts
	# command: \|
	# mkdir /home/circleci/trigger_mail/test_reports
	# mkdir /home/circleci/trigger_mail/artifacts

	# sudo chown -R circleci /home/circleci/trigger_mail/test_reports
	# sudo chown -R circleci /home/circleci/trigger_mail/artifacts

	# # in dockerfile now
	# - run:
	# name: pip permissions
	# command: \|
	# sudo chown -R circleci /usr/local




	# - run:
	# name: test
	# working_directory: ~/trigger_mail/src
	# command: \|
	# if [ $CIRCLE_NODE_INDEX -eq 1 ]; then
	# START=$(date +%s)

	# REPO_IN_GOPATH=$HOME/.go_workspace/src/github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME
	# # Run tests with race detector and timeout a test if it takes longer than 1 minute
	# cd $REPO_IN_GOPATH && go test -v -race -test.timeout=1m ./go/...

	# # require code to pass go vet checks
	# cd $REPO_IN_GOPATH && go vet ./go/...

	# # require code to be formatted
	# NOT_FORMATTED=`cd $REPO_IN_GOPATH && gofmt -l ./go`
	# if [ ! -z "$NOT_FORMATTED" ]; then
	# echo "ERROR files must be formatted with gofmt:" > /dev/stderr
	# echo $NOT_FORMATTED > /dev/stderr
	# exit 10
	# fi

	# END=$(date +%s)
	# ELAPSED=$((END-START))
	# echo "run_tests_or_ui.sh: ran go checks in $ELAPSED seconds"
	# fi

	# exit_val=0
	# python $HOME/trigger_mail/should_build_ui.py \|\| exit_val=$?

	# SKIP_MINIFY=true
	# if [ $exit_val -eq 102 ]; then
	# SKIP_MINIFY=false
	# fi
	# if [ $exit_val -ge 101 ]; then
	# if [ $CIRCLE_NODE_INDEX -eq 0 ]; then
	# $HOME/trigger_mail/build_ui.sh $SKIP_MINIFY;
	# exit 0
	# fi
	# NODE_INDEX=$((CIRCLE_NODE_INDEX - 1));
	# NODE_TOTAL=$((CIRCLE_NODE_TOTAL - 1));
	# elif [ $exit_val -eq 0 ]; then
	# echo "No UI changes detected in the PR or Commit"
	# NODE_INDEX=$CIRCLE_NODE_INDEX;
	# NODE_TOTAL=$CIRCLE_NODE_TOTAL;
	# else
	# echo "ERROR unknown exit code from should_build_ui.py"
	# exit 10
	# fi

	# - run:
	# name: test part 2
	# working_directory: ~/trigger_mail/src
	# command: \|
	# # this is a totally gross hack: Find directories in the current directory, or under packages that
	# # contain a directory named tests
	# dirs_with_tests=`find $HOME/trigger_mail $HOME/trigger_mail/packages -maxdepth 2 -type d -name tests \| xargs -n1 dirname`
	# # remove packages that don't actually include tests
	# packages_with_tests=`echo "${dirs_with_tests}" \| grep -v '/trigger_mail$' \| grep -v '/trigger_mail/rainforest$' \| grep -v '/trigger_mail/spark$'`

	# # Pre-build the packages and put the wheels into /bluecore-built-packages.
	# python $HOME/trigger_mail/package_builder.py $HOME/trigger_mail $HOME/trigger_mail/bluecore-built-packages

	# # CircleCI docs say to put test reports in a subdirectory:
	# # https://circleci.com/docs/1.0/test-metadata/#metadata-collection-in-custom-test-steps
	# sudo mkdir -p ${CIRCLE_TEST_REPORTS}/nose
	# sudo chown -R circleci ${CIRCLE_TEST_REPORTS}/nose

	# # Setup the services
	# for package in $packages_with_tests
	# do
	# echo "Building and running tests for ${package} ..."
	# python $HOME/trigger_mail/service_setup.py $package --skip-wheel-builds
	# cd $package
	# PACKAGE_WITHOUT_SLASHES=`echo "$package" \| sed 's/\//_/g'`
	# PYTHONPATH=.:service_lib/installed:service_lib/protocol_buffers \
	# ./venv/bin/nosetests tests \
	# --with-parallel -v --processes=4 --process-timeout=540 \
	# --with-xunitmp \
	# --xunitmp-file=$CIRCLE_TEST_REPORTS/nose/${PACKAGE_WITHOUT_SLASHES}_output.xml \
	# --process-restartworker \
	# > >(tee -a $CIRCLE_ARTIFACTS/test.log) 2>&1;

	# PATH=${HOME}/trigger_mail/src/venv/bin:${PATH} go run \
	# "${HOME}/trigger_mail/go/pylintparallel/pylintparallel.go" \
	# "-partitionIndex=${CIRCLE_NODE_INDEX}" \
	# "-numPartitions=${CIRCLE_NODE_TOTAL}" \
	# "-srcRoot=${package}" \
	# -verbose
	# done


	# - run:
	# name: test part 3
	# working_directory: ~/trigger_mail/src
	# command: \|
	# if [ $CIRCLE_NODE_INDEX -eq 0 ]; then
	# # run root tests on node 0
	# # TODO: integrate this with the tests above somehow
	# cd $HOME/trigger_mail
	# nosetests -v $HOME/trigger_mail/tests
	# fi
	FROM circleci/golang:1.9-browsers

	USER root

	RUN apt-get update
	RUN apt-get install zlib1g-dev \
	# mysql
	default-libmysqlclient-dev mysql-server \
	# lxml
	libxml2 libxml2-dev libxslt1.1 libxslt1-dev \
	# python pre-dependencies
	build-essential checkinstall \
	libreadline-gplv2-dev libncursesw5-dev libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev \
	python-dev \
	libjpeg-dev libjpeg8-dev imagemagick

	# install python
	RUN wget https://www.python.org/ftp/python/2.7.13/Python-2.7.13.tgz
	RUN tar xzf Python-2.7.13.tgz
	RUN rm Python-2.7.13.tgz

	WORKDIR Python-2.7.13

	RUN ./configure
	RUN make install

	# install pip
	RUN wget https://bootstrap.pypa.io/get-pip.py
	RUN python get-pip.py

	# install node/npm
	RUN wget https://nodejs.org/dist/v6.9.5/node-v6.9.5.tar.gz
	RUN tar xzf node-v6.9.5.tar.gz
	RUN rm node-v6.9.5.tar.gz

	WORKDIR node-v6.9.5

	RUN ./configure
	RUN make -j4
	RUN make install

	# start mysql
	RUN service mysql start

	# final steps
	RUN ln -s /home/circleci /home/ubuntu
	RUN npm install -g yarn
	RUN git config --global user.email "CircleCI@example.com"
	RUN git config --global user.name "CircleCI"

	USER circleci