script to build cross compilers and canadian cross compilers

cross.sh

#!/bin/bash
#
# cross.sh
# Building cross compilers and canadian cross compilers
# (i686, x86_64, arm, aarch64, powerpc, powerpc64)
#
# Copyright 2017 knotdevel
# Released under the MIT license
# http://opensource.org/licenses/mit-license.php
#
#
# Prerequisite:
#
# 1. Setup Ubuntu 16.04 64bit desktop machine having the internet connection
# 2. Upgrade the software
#      $ sudo apt update
#      $ sudo apt upgrade
# 3. Install below packages
#      $ sudo apt install git build-essential libtool m4 gawk
# 4. Download this script
#      $ git clone https://gist.github.com/c57086222c83fc8b26bcf94d1aed66eb.git cross
#      $ cd cross
#
# Usage:
#
# 1. Building cross compilers
#      $ ./cross.sh
# 2. Building cross compilers and canadian cross compilers
#      $ BUILD_CANADIAN=1 ./cross.sh
#
# Output:
#
# 1. Cross compilers and sysroot
#      ${PWD}
#      ├── x86_64-pc-linux-gnu
#      │  └── usr
#      │      └── bin
#      │           └── ${TARGET}-gcc
#      └── ${TARGET}
#           └── sysroot
#
# 2. Canadian cross compilers and sysroot
#      ${PWD}
#      ├── ${HOST}
#      │  └── usr
#      │      └── bin
#      │           └── ${TARGET}-gcc
#      └── ${TARGET}
#           └── sysroot
#

# setup script envirnoment
set -e
set -u
set -o pipefail
set -x
set -v
export LC_ALL=C
export LANG=C

SCRIPT_DIR=$(cd $(dirname $0);pwd)
WORK_DIR=`echo $(pwd)`


# if you want to build canadian cross compilers,
# export BUILD_CANADIAN=1
[ -v BUILD_CANADIAN ] || BUILD_CANADIAN=0

# enable multilib
# building x86_64 multilib compiler (m32, mx32, m64) and powerpc64
# multilib compiler (m32, m64)
# if you want to disable multilib, export ENABLE_MULTILIB=0
[ -v ENABLE_MULTILIB ] || ENABLE_MULTILIB=1
echo "ENABLE_MULTILIB=${ENABLE_MULTILIB}"

# versions
[ -v BINUTILS_VER ] || BINUTILS_VER=2.27
BINUTILS_URL=http://ftpmirror.gnu.org/binutils
BINUTILS_EXT=tar.bz2

[ -v GCC_VER ] || GCC_VER=6.3.0
GCC_URL=http://ftpmirror.gnu.org/gcc/gcc-${GCC_VER}
GCC_EXT=tar.bz2

[ -v LINUX_VER ] || LINUX_VER=4.1.38
LINUX_VER_DIR="v${LINUX_VER:0:1}.x" # v4.x
LINUX_URL=https://www.kernel.org/pub/linux/kernel/${LINUX_VER_DIR}
LINUX_EXT=tar.xz

[ -v GLIBC_VER ] || GLIBC_VER=2.24
GLIBC_URL=http://ftpmirror.gnu.org/glibc
GLIBC_EXT=tar.xz

[ -v GMP_VER ] || GMP_VER=6.1.2
GMP_URL=https://ftpmirror.gnu.org/gmp
GMP_EXT=tar.xz
#GMP_VER=4.3.2 # gcc-4.9.4, gcc-6.3.0 prerequisites
#GMP_EXT=tar.bz2

[ -v MPFR_VER ] || MPFR_VER=3.1.5 #OK
#MPFR_VER=2.4.2 # gcc-4.9.4, gcc-6.3.0 prerequisites
MPFR_URL=http://ftpmirror.gnu.org/mpfr
MPFR_EXT=tar.xz

[ -v MPC_VER ] || MPC_VER=1.0.3 #OK
MPC_URL=http://ftpmirror.gnu.org/mpc               # mpc-1.0.x or newer
MPC_EXT=tar.gz
#MPC_VER=0.8.1 # gcc-4.9.4, gcc-6.3.0 prerequisites
#MPC_URL=http://www.multiprecision.org/mpc/download  # mpc-0.8.1

[ -v ISL_VER ] || ISL_VER=0.18
#ISL_VER=0.15 # gcc-6.3.0 prerequisites
#ISL_VER=0.12.2 # gcc-4.9.4 prerequisites
ISL_URL=http://isl.gforge.inria.fr
ISL_EXT=tar.bz2

# paralle make
PROCESSOR_NUM=`echo $(grep -c ^processor /proc/cpuinfo)`
[ -v MAKE_PARALLEL ] || MAKE_PARALLEL=-j${PROCESSOR_NUM}
MAKE="make ${MAKE_PARALLEL}"

# MACHTYPE of cross compilers
[ -v MACHTYPE_X86_32 ] || MACHTYPE_X86_32=i686-my-linux-gnu
[ -v MACHTYPE_X86_64 ] || MACHTYPE_X86_64=x86_64-my-linux-gnu
[ -v MACHTYPE_ARM_32 ] || MACHTYPE_ARM_32=arm-my-linux-gnueabi
[ -v MACHTYPE_ARM_64 ] || MACHTYPE_ARM_64=aarch64-my-linux
[ -v MACHTYPE_PPC_32 ] || MACHTYPE_PPC_32=powerpc-my-linux-gnu
[ -v MACHTYPE_PPC_64 ] || MACHTYPE_PPC_64=powerpc64-my-linux-gnu

CANADIAN_LIST=
if [ "x${MACHTYPE}" != "x${MACHTYPE_X86_32}" ]; then
  CANADIAN_LIST="${CANADIAN_LIST} ${MACHTYPE_X86_32}"
fi
if [ "x${MACHTYPE}" != "x${MACHTYPE_X86_64}" ]; then
  CANADIAN_LIST="${CANADIAN_LIST} ${MACHTYPE_X86_64}"
fi
if [ "x${MACHTYPE}" != "x${MACHTYPE_ARM_32}" ]; then
  CANADIAN_LIST="${CANADIAN_LIST} ${MACHTYPE_ARM_32}"
fi
if [ "x${MACHTYPE}" != "x${MACHTYPE_ARM_64}" ]; then
  CANADIAN_LIST="${CANADIAN_LIST} ${MACHTYPE_ARM_64}"
fi
if [ "x${MACHTYPE}" != "x${MACHTYPE_PPC_32}" ]; then
  CANADIAN_LIST="${CANADIAN_LIST} ${MACHTYPE_PPC_32}"
fi
if [ "x${MACHTYPE}" != "x${MACHTYPE_PPC_64}" ]; then
  CANADIAN_LIST="${CANADIAN_LIST} ${MACHTYPE_PPC_64}"
fi

# linux ARCH
LINUX_ARCH_X86_32=i386
LINUX_ARCH_X86_64=x86_64
LINUX_ARCH_ARM_32=arm
LINUX_ARCH_ARM_64=arm64
LINUX_ARCH_PPC_32=powerpc
LINUX_ARCH_PPC_64=powerpc

function host_cc_dir() {
  echo "${WORK_DIR}/${MACHTYPE}/usr/bin"
}

function host_build_dir() {
  local host=$1
  echo "${WORK_DIR}/${host}/tmp"
}

function tool_build_dir() {
  local tools_host=$1
  local target=$2
  echo "${WORK_DIR}/${tools_host}/tmp/${target}"
}

function prefix_dir() {
  local tools_host=$1
  echo "${WORK_DIR}/${tools_host}/usr"
}

function sysroot_dir() {
  local target=$1
  echo "${WORK_DIR}/${target}/sysroot"
}

function download_src() {
  local name=$1
  local ver=$2
  local ext=$3
  local url=$4
  if [ ! -f ${name}-${ver}.${ext} ]; then
    wget ${url}/${name}-${ver}.${ext}
  fi
}

function build_src() {

  SRC_DIR=${WORK_DIR}/src

  [ -v CACHE_DIR ] || CACHE_DIR=${WORK_DIR}/cache
  mkdir -pv ${CACHE_DIR}
  local cache_dir=$(readlink -f ${CACHE_DIR})

  pushd ${cache_dir}
  download_src binutils ${BINUTILS_VER} ${BINUTILS_EXT} ${BINUTILS_URL}
  download_src gmp ${GMP_VER} ${GMP_EXT} ${GMP_URL}
  download_src mpfr ${MPFR_VER} ${MPFR_EXT} ${MPFR_URL}
  download_src mpc ${MPC_VER} ${MPC_EXT} ${MPC_URL}
  download_src isl ${ISL_VER} ${ISL_EXT} ${ISL_URL}
  download_src linux ${LINUX_VER} ${LINUX_EXT} ${LINUX_URL}
  download_src gcc ${GCC_VER} ${GCC_EXT} ${GCC_URL}
  download_src glibc ${GLIBC_VER} ${GLIBC_EXT} ${GLIBC_URL}
  popd

  mkdir -pv ${SRC_DIR}
  pushd ${SRC_DIR}
  if [ ! -f .done ]; then
    tar xf ${cache_dir}/binutils-${BINUTILS_VER}.${BINUTILS_EXT}
    tar xf ${cache_dir}/gmp-${GMP_VER}.${GMP_EXT}
    tar xf ${cache_dir}/mpfr-${MPFR_VER}.${MPFR_EXT}
    tar xf ${cache_dir}/mpc-${MPC_VER}.${MPC_EXT}
    tar xf ${cache_dir}/isl-${ISL_VER}.${ISL_EXT}
    tar xf ${cache_dir}/linux-${LINUX_VER}.${LINUX_EXT}
    tar xf ${cache_dir}/gcc-${GCC_VER}.${GCC_EXT}
    tar xf ${cache_dir}/glibc-${GLIBC_VER}.${GLIBC_EXT}
    touch .done
  fi
  popd
}

function build_binutils() {
  local tools_host=$1
  local target=$2
  local config_opt=$3

  local prefix=$(prefix_dir ${tools_host})
  local sysroot=$(sysroot_dir ${target})

  local build_dir=$(tool_build_dir ${tools_host} ${target})/binutils
  mkdir -pv ${build_dir}
  pushd ${build_dir}
  if [ ! -f .done ]; then
    ${SRC_DIR}/binutils-${BINUTILS_VER}/configure \
              --host=${tools_host} \
              --target=${target} \
              --prefix=${prefix} \
              --with-sysroot=${sysroot} \
              \
              ${config_opt}
    ${MAKE}
    make install
    touch .done
  fi
  popd
}

function build_gcc_1st() {
  local tools_host=$1
  local target=$2
  local config_opt=$3

  local prefix=$(prefix_dir ${tools_host})

  local build_dir=$(host_build_dir ${tools_host})/gmp
  mkdir -pv ${build_dir}
  pushd ${build_dir}
  if [ ! -f .done ]; then
    ${SRC_DIR}/gmp-${GMP_VER}/configure \
              --host=${tools_host} \
              --prefix=${prefix} \
              --disable-shared
    ${MAKE}
    make install
    touch .done
  fi
  popd

  build_dir=$(host_build_dir ${tools_host})/mpfr
  mkdir -pv ${build_dir}
  pushd ${build_dir}
  if [ ! -f .done ]; then
    ${SRC_DIR}/mpfr-${MPFR_VER}/configure \
              --host=${tools_host} \
              --prefix=${prefix} \
              --with-gmp=${prefix} \
              --disable-shared
    ${MAKE}
    make install
    touch .done
  fi
  popd

  build_dir=$(host_build_dir ${tools_host})/mpc
  mkdir -pv ${build_dir}
  pushd ${build_dir}
  if [ ! -f .done ]; then
    ${SRC_DIR}/mpc-${MPC_VER}/configure \
              --host=${tools_host} \
              --prefix=${prefix} \
              --with-gmp=${prefix} \
              --with-mpfr=${prefix} \
              --disable-shared
    ${MAKE}
    make install
    touch .done
  fi
  popd

  build_dir=$(host_build_dir ${tools_host})/isl
  mkdir -pv ${build_dir}
  pushd ${build_dir}
  if [ ! -f .done ]; then
    ${SRC_DIR}/isl-${ISL_VER}/configure \
              --host=${tools_host} \
              --prefix=${prefix} \
              --with-gmp-prefix=${prefix} \
              --disable-shared
    ${MAKE}
    make install
    touch .done
  fi
  popd

  if [ ${GCC_VER:0:1} == 4 ]; then
    build_dir=$(host_build_dir ${tools_host})/cloog
    mkdir -pv ${build_dir}
    pushd ${build_dir}
    if [ ! -f .done ]; then
      ${SRC_DIR}/cloog-${CLOOG_VER}/configure \
                --host=${tools_host} \
                --prefix=${prefix} \
                --with-gmp-prefix=${prefix} \
                --with-isl-prefix=${prefix} \
                --disable-shared
      ${MAKE}
      make install
      touch .done
    fi
    popd
  fi

  build_dir=$(tool_build_dir ${tools_host} ${target})/gcc-1st
  mkdir -pv ${build_dir}
  pushd ${build_dir}
  if [ ! -f .done ]; then
    local empty_dir=$(tool_build_dir ${tools_host} ${target})/gcc-1st/empty
    mkdir -pv ${empty_dir}
    ${SRC_DIR}/gcc-${GCC_VER}/configure \
              --host=${tools_host} \
              --target=${target} \
              --prefix=${prefix} \
              --with-sysroot=${empty_dir} \
              --with-gmp=${prefix} \
              --with-mpfr=${prefix} \
              --with-mpc=${prefix} \
              --with-isl=${prefix} \
              --disable-isl-version-check \
              --with-newlib \
              --without-headers \
              --with-glibc-version=${GLIBC_VER} \
              --enable-languages=c \
              \
              --disable-shared \
              --disable-threads \
              --disable-plugin \
              --disable-lto \
              \
              --disable-libgomp \
              --disable-libatomic \
              --disable-libmpx \
              --disable-libssp \
              --disable-libquadmath \
              \
              ${config_opt}

    ${MAKE} all-gcc
    ${MAKE} all-target-libgcc
    make install-gcc
    make install-target-libgcc
    touch .done
  fi
  popd
}

function build_linux_headers() {
  local tools_host=$1
  local host=$2
  local linux_arch=$3

  local sysroot=$(sysroot_dir ${host})
  local build_dir=$(host_build_dir ${host})/linux-headers

  mkdir -pv ${build_dir}
  pushd ${build_dir}
  if [ ! -f .done.${tools_host}.${host} ]; then
    pushd ${SRC_DIR}/linux-${LINUX_VER}
    make headers_install ARCH=${linux_arch} INSTALL_HDR_PATH=${sysroot}/usr O=${build_dir}
    popd

    touch .done.${tools_host}.${host}
  fi
  popd
}

function build_glibc_headers() {
  local tools_host=$1
  local host=$2
  local host_cc_opt=$3

  local sysroot=$(sysroot_dir ${host})
  local build_dir=$(host_build_dir ${host})/glibc-headers

  local host_cc="$(host_cc_dir)/${host}-gcc --sysroot=${sysroot} ${host_cc_opt}"
  local MULTILIB_OS_DIR=$(${host_cc} -print-multi-os-directory)

  mkdir -pv ${build_dir}
  pushd ${build_dir}
  if [ ! -f .done.configure ]; then
    BUILD_CC=gcc CC="${host_cc}" \
            ${SRC_DIR}/glibc-${GLIBC_VER}/configure \
            --host=${host} \
            --prefix=/usr \
            --libdir=/usr/lib/${MULTILIB_OS_DIR} \
            libc_cv_slibdir=/lib/${MULTILIB_OS_DIR}
    touch .done.configure
  fi
  if [ ! -f .done.install.${tools_host}.${host} ]; then
    make BUILD_CC=gcc CC="${host_cc}" install_root=${sysroot} install-headers

    touch .done.install.${tools_host}.${host}
  fi
  popd
}

function build_glibc() {
  local tools_host=$1
  local host=$2
  local install_root_host=$3
  local host_cc_opt=$4
  local suffix=$5 # for multilib

  local sysroot=$(sysroot_dir ${host})
  local build_dir=$(host_build_dir ${host})/glibc${suffix}

  local host_cc="$(host_cc_dir)/${host}-gcc --sysroot=${sysroot} ${host_cc_opt}"
  local multilib_os_dir=$(${host_cc} -print-multi-os-directory)

  mkdir -pv ${build_dir}
  pushd ${build_dir}
  if [ ! -f .done.make ]; then

    BUILD_CC=gcc CC="${host_cc}" \
            ${SRC_DIR}/glibc-${GLIBC_VER}/configure \
            --host=${host} \
            --prefix=/usr \
            --libdir=/usr/lib/${multilib_os_dir} \
            libc_cv_slibdir=/lib/${multilib_os_dir}

    ${MAKE} BUILD_CC=gcc CC="${host_cc}"

    touch .done.make
  fi

  if [ ! -f .done.install.${tools_host}.${install_root_host} ]; then
    local install_root=$(sysroot_dir ${install_root_host})
    make BUILD_CC=gcc CC="${host_cc}" install_root=${install_root} install

    touch .done.install.${tools_host}.${install_root_host}
  fi
  popd
}

function build_gcc_2nd() {
  local tools_host=$1
  local target=$2
  local config_opt=$3

  local prefix=$(prefix_dir ${tools_host})
  local sysroot=$(sysroot_dir ${target})
  local build_dir=$(tool_build_dir ${tools_host} ${target})/gcc-2nd
  mkdir -pv ${build_dir}
  pushd ${build_dir}
  if [ ! -f .done ]; then
    ${SRC_DIR}/gcc-${GCC_VER}/configure \
              --host=${tools_host} \
              --target=${target} \
              --prefix=${prefix} \
              --with-sysroot=${sysroot} \
              --with-gmp=${prefix} \
              --with-mpfr=${prefix} \
              --with-mpc=${prefix} \
              --with-isl=${prefix} \
              --disable-isl-version-check \
              --with-glibc-version=${GLIBC_VER} \
              --enable-languages=c,c++ \
              --enable-__cxa_atexit \
              --enable-threads=posix \
              \
              ${config_opt}
    ${MAKE}
    make install

    touch .done
  fi
  popd
}

function dump_sysroot() {
  local tools_host=$1
  local host=$2
  local prefix=$3
  local suffix=$4

  local sysroot=$(sysroot_dir ${host})
  local log_dir=$(host_build_dir ${tools_host})/log

  mkdir -pv ${log_dir}
  pushd ${sysroot}
  find . -type f > ${log_dir}/sysroot_${prefix}${host}${suffix}.txt
  popd
}

function diff_sysroot() {
  local tools_host=$1
  local host1=$2
  local host2=$3

  dump_sysroot ${tools_host} ${host1} "" "-diff-tmp1"
  dump_sysroot ${tools_host} ${host2} "" "-diff-tmp2"

  local log_dir=$(host_build_dir ${tools_host})/log
  mkdir -pv ${log_dir}
  pushd ${log_dir}
  diff sysroot_${host1}-diff-tmp1.txt sysroot_${host2}-diff-tmp2.txt > diff-sysroot_${host1}_${host2}.txt || :
  popd
}

function build_all() {
  local tools_host=$1

  build_binutils ${tools_host} ${MACHTYPE_X86_32} ""
  build_binutils ${tools_host} ${MACHTYPE_X86_64} ""
  build_binutils ${tools_host} ${MACHTYPE_ARM_32} ""
  build_binutils ${tools_host} ${MACHTYPE_ARM_64} ""
  build_binutils ${tools_host} ${MACHTYPE_PPC_32} ""
  build_binutils ${tools_host} ${MACHTYPE_PPC_64} ""

  build_gcc_1st ${tools_host} ${MACHTYPE_X86_32} "--disable-multilib"
  if [ ${ENABLE_MULTILIB} -eq 1 ]; then
    build_gcc_1st ${tools_host} ${MACHTYPE_X86_64} "--with-multilib-list=m32,m64,mx32 --with-arch-32=i686"
  else
    build_gcc_1st ${tools_host} ${MACHTYPE_X86_64} "--disable-multilib"
  fi
  build_gcc_1st ${tools_host} ${MACHTYPE_ARM_32} "--disable-multilib"
  build_gcc_1st ${tools_host} ${MACHTYPE_ARM_64} "--disable-multilib"
  build_gcc_1st ${tools_host} ${MACHTYPE_PPC_32} "--disable-multilib"
  if [ ${ENABLE_MULTILIB} -eq 1 ]; then
    build_gcc_1st ${tools_host} ${MACHTYPE_PPC_64} "--with-multilib-list=m32,m64"
  else
    build_gcc_1st ${tools_host} ${MACHTYPE_PPC_64} "--disable-multilib"
  fi

  build_linux_headers ${tools_host} ${MACHTYPE_X86_32} ${LINUX_ARCH_X86_32}
  build_linux_headers ${tools_host} ${MACHTYPE_X86_64} ${LINUX_ARCH_X86_64}
  build_linux_headers ${tools_host} ${MACHTYPE_ARM_32} ${LINUX_ARCH_ARM_32}
  build_linux_headers ${tools_host} ${MACHTYPE_ARM_64} ${LINUX_ARCH_ARM_64}
  build_linux_headers ${tools_host} ${MACHTYPE_PPC_32} ${LINUX_ARCH_PPC_32}
  build_linux_headers ${tools_host} ${MACHTYPE_PPC_64} ${LINUX_ARCH_PPC_64}

  build_glibc_headers ${tools_host} ${MACHTYPE_X86_32} "-march=i686"
  build_glibc_headers ${tools_host} ${MACHTYPE_X86_64} ""
  build_glibc_headers ${tools_host} ${MACHTYPE_ARM_32} ""
  build_glibc_headers ${tools_host} ${MACHTYPE_ARM_64} ""
  build_glibc_headers ${tools_host} ${MACHTYPE_PPC_32} ""
  build_glibc_headers ${tools_host} ${MACHTYPE_PPC_64} ""

  build_glibc ${tools_host} ${MACHTYPE_X86_32} ${MACHTYPE_X86_32} "-march=i686" ""
  if [ ${ENABLE_MULTILIB} -eq 1 ]; then
    build_glibc ${tools_host} ${MACHTYPE_X86_32} ${MACHTYPE_X86_64} "-march=i686" ""
    build_glibc ${tools_host} ${MACHTYPE_X86_64} ${MACHTYPE_X86_64} "-mx32" "-mx32"
  fi
  build_glibc ${tools_host} ${MACHTYPE_X86_64} ${MACHTYPE_X86_64} "" ""
  build_glibc ${tools_host} ${MACHTYPE_ARM_32} ${MACHTYPE_ARM_32} "" ""
  build_glibc ${tools_host} ${MACHTYPE_ARM_64} ${MACHTYPE_ARM_64} "" ""
  build_glibc ${tools_host} ${MACHTYPE_PPC_32} ${MACHTYPE_PPC_32} "" ""
  if [ ${ENABLE_MULTILIB} -eq 1 ]; then
    build_glibc ${tools_host} ${MACHTYPE_PPC_32} ${MACHTYPE_PPC_64} "" ""
  fi
  build_glibc ${tools_host} ${MACHTYPE_PPC_64} ${MACHTYPE_PPC_64} "" ""

  build_gcc_2nd ${tools_host} ${MACHTYPE_X86_32} "--disable-multilib"
  if [ ${ENABLE_MULTILIB} -eq 1 ]; then
    build_gcc_2nd ${tools_host} ${MACHTYPE_X86_64} "--with-multilib-list=m32,m64,mx32 --with-arch-32=i686"
  else
    build_gcc_2nd ${tools_host} ${MACHTYPE_X86_64} "--disable-multilib"
  fi
  build_gcc_2nd ${tools_host} ${MACHTYPE_ARM_32} "--disable-multilib"
  build_gcc_2nd ${tools_host} ${MACHTYPE_ARM_64} "--disable-multilib"
  build_gcc_2nd ${tools_host} ${MACHTYPE_PPC_32} "--disable-multilib"
  if [ ${ENABLE_MULTILIB} -eq 1 ]; then
    build_gcc_2nd ${tools_host} ${MACHTYPE_PPC_64} "--with-multilib-list=m32,m64"
  else
    build_gcc_2nd ${tools_host} ${MACHTYPE_PPC_64} "--disable-multilib"
  fi
}


function main() {

  # setup source
  build_src

  # build cross compilers
  build_all ${MACHTYPE}

  # build canadian cross compilers
  if [ ${BUILD_CANADIAN} -eq 1 ]; then

    # setup PATH to cross compiler
    export PATH=$(host_cc_dir):${PATH}

    for type in ${CANADIAN_LIST}
    do
      build_all ${type}
    done

  fi
}

main