gistfile1.txt

#!/bin/bash -xe
# see also https://gist.github.com/nickboldt/6afc71826e82518c5e7b7d3c3fdcc4f3
# merge sse repos into a single repo

START_TIME=`date +%s`

# work in a tempdir. The tmp-dir name is something like /home/user/tmp/this_scripts_name/
tmpdir=~/tmp/${0/.sh/.tmp}
tmpdirCache=~/tmp/${0/.sh/.tmp}_cache

#rm -fr ${tmpdir}
mkdir -p ${tmpdir}
cd ${tmpdir}

# in case we have another alias for mv, like 'mv -i'
alias mv=mv
alias rm=rm

# set our local pull command depending on git version
verlte() {
    [  "$1" = "`echo -e "$1\n$2" | sort -V | head -n1`" ]
}

verlt() {
    [ "$1" = "$2" ] && return 1 || verlte $1 $2
}
GIT_VERSION=`git version | head -1 | cut -f 3 -d " "`
verlte $GIT_VERSION 2.9.0 && ALLOW_PULL_FLAG=false || ALLOW_PULL_FLAG=true

if $ALLOW_PULL_FLAG
then
   LOCAL_PULL_CMD="git pull --no-edit --allow-unrelated-histories --no-rebase --rebase=false"
else
   LOCAL_PULL_CMD="git pull --no-edit --no-rebase --rebase=false"
fi

MVN="mvn clean install -fae -e -Dplatform-repo.url=http://download.eclipse.org/eclipse/updates/4.8-I-builds/"

# Make sure different users with different settings will run the script as intended
gitBranchSettings ()
{
    git config branch.autosetuprebase never
    git config branch.autosetupmerge false
    git config branch.master.rebase false
}

# handle changes to parent pom
if [[ ! -d webtools.releng.aggregator ]]; then git clone http://git.eclipse.org/gitroot/webtools/webtools.releng.aggregator.git; fi
pushd webtools.releng.aggregator/wtp-parent
    gitBranchSettings
    git reset --hard origin/master
    ${MVN}
popd


# wst.web and wst.web.ui are in javaee. We need to git filter-branch it 
# So we only get the history for those 2 plugins and NOTHING else. 
# This takes a long time. 5 minutes maybe

# temporary workaround to save time. THe 5 lines below take 10 minutes to run. :| 
if [[ -d ${tmpdirCache}/javaee_tmp_filterbranched ]]; then
   cp -R ${tmpdirCache}/javaee_tmp_filterbranched webtools.javaee
else
   echo "${tmpdirCache}/javaee_tmp_filterbranched does not exist"
   git clone http://git.eclipse.org/gitroot/jeetools/webtools.javaee.git
   cd webtools.javaee
   git filter-branch --index-filter 'git rm --cached -qr --ignore-unmatch -- . && git reset -q $GIT_COMMIT -- plugins/org.eclipse.wst.web plugins/org.eclipse.wst.web.ui' --prune-empty -- --all


   END_TIME=`date +%s`
   EXEC_TIME=$((END_TIME-START_TIME))
   echo $EXEC_TIME " seconds to complete filter branch"

   mkdir ${tmpdirCache}
   mkdir ${tmpdirCache}/javaee_tmp_filterbranched
   cd ../
   cp -R webtools.javaee ${tmpdirCache}/javaee_tmp_filterbranched 
fi


# Now clone the 5 main sse repos
for d in webtools.sourceediting webtools.sourceediting.xpath webtools.sourceediting.xpath.tests webtools.sourceediting.xsl webtools.sourceediting.xsl.tests; do
    if [[ ! -d ${d} ]]; then 
        git clone http://git.eclipse.org/gitroot/sourceediting/${d}.git
    fi
done

# Make sure all branches are at master and have proper git settings
for d in webtools.sourceediting.xpath webtools.sourceediting.xpath.tests webtools.sourceediting.xsl webtools.sourceediting.xsl.tests; do
    pushd ${d}
        gitBranchSettings
        git reset --hard origin/master
    popd
done

# Let's collect all the gitignores into one file to use later
for d in webtools.sourceediting.xpath webtools.sourceediting.xpath.tests webtools.sourceediting.xsl webtools.sourceediting.xsl.tests; do
    pushd ${d}
        rm pom.xml
        cat .gitignore >> ../git_ignore_accumulator
        rm .gitignore
        git commit -a -m "Removing poms and .gitignore files that will be clobbered" --signoff
    popd
done


# Merge the other repos into the main sse repo now
cd webtools.sourceediting
for d in webtools.sourceediting.xpath webtools.sourceediting.xpath.tests webtools.sourceediting.xsl webtools.sourceediting.xsl.tests webtools.javaee; do
    $LOCAL_PULL_CMD ../${d}
done
# git commit -s -m "Merging all repos together" .

# Make the new structure
for d in core xml web xpath xsl json; do
   mkdir -p ${d} ${d}/bundles ${d}/features ${d}/docs ${d}/tests ${d}/itests
done

# Move bundles - NOTE: these will fail if you have an unclean git checkout (from running the top part of the script)
ls -1 bundles/ | grep "json"        | awk '{ print "git mv bundles/" $0 " json/bundles/";}' | sh
ls -1 bundles/ | grep "xsl"         | awk '{ print "git mv bundles/" $0 " xsl/bundles/";}' | sh
ls -1 bundles/ | grep "xpath"       | awk '{ print "git mv bundles/" $0 " xpath/bundles/";}' | sh
ls -1 bundles/ | grep "sse"         | awk '{ print "git mv bundles/" $0 " core/bundles/";}' | sh
ls -1 bundles/ | grep "\.xml\."     | awk '{ print "git mv bundles/" $0 " xml/bundles/";}'  | sh
ls -1 bundles/ | grep "\.dtd\."     | awk '{ print "git mv bundles/" $0 " xml/bundles/";}'  | sh
ls -1 bundles/ | grep "\.xsd\."     | awk '{ print "git mv bundles/" $0 " xml/bundles/";}'  | sh
ls -1 bundles/ | grep "wst.standard.schemas"| awk '{ print "git mv bundles/" $0 " xml/bundles/";}'  | sh
ls -1 bundles/ | grep "org.eclipse" | awk '{ print "git mv bundles/" $0 " web/bundles/";}' | sh
ls -1 plugins/ | grep "org.eclipse" | awk '{ print "git mv plugins/" $0 " web/bundles/";}' | sh
rm -f bundles/pom.xml
rm -f plugins/pom.xml
if [[ ! $(ls bundles) ]]; then rm -rf bundles; else echo "bundles is not empty! Abort merge script"; exit 1; fi
if [[ ! $(ls plugins) ]]; then rm -rf plugins; else echo "plugins is not empty! Abort merge script"; exit 1; fi
git commit -a -s -m "Move bundles"

# Move docs 
rm docs/pom.xml
ls -1 docs/ | grep "json"   | awk '{ print "git mv docs/" $0 " json/docs/";}' | sh
ls -1 docs/ | grep "xsl"    | awk '{ print "git mv docs/" $0 " xsl/docs/";}' | sh
ls -1 docs/ | grep "xpath"  | awk '{ print "git mv docs/" $0 " xpath/docs/";}' | sh
ls -1 docs/ | grep "sse"    | awk '{ print "git mv docs/" $0 " core/docs/";}' | sh
ls -1 docs/ | grep "\.xml." | awk '{ print "git mv docs/" $0 " xml/docs/";}'  | sh
ls -1 docs/ | grep "\.dtd." | awk '{ print "git mv docs/" $0 " xml/docs/";}'  | sh
ls -1 docs/ | grep "\.xsd." | awk '{ print "git mv docs/" $0 " xml/docs/";}'  | sh
ls -1 docs/ | grep "org.eclipse" | awk '{ print "git mv docs/" $0 " web/docs/";}' | sh
rm -f docs/pom.xml
if [[ ! $(ls docs) ]]; then rm -rf docs; else echo "docs is not empty! Abort merge script"; exit 1; fi
git commit -a -s -m "Move docs"

# Move tests
git mv tests/org.w3c.xqts.testsuite xpath/tests/
ls tests | grep "\.sse\."  | awk '{ print "git mv tests/" $0 " core/tests/";}' | sh
git mv tests/org.eclipse.wst.json.core.tests/ json/tests/
git mv tests/org.eclipse.wst.xml.core.tests xml/tests/
git mv tests/org.eclipse.wst.xml.catalog.tests xml/tests/  #deps on o.e.xsd repo
git mv tests/org.eclipse.wst.xml.tests.encoding xml/tests/
git mv tests/org.eclipse.wst.xml.ui.tests xml/tests/
git mv tests/org.eclipse.wst.xml.validation.tests xml/tests/
git mv tests/org.eclipse.wst.xsd.core.tests xml/tests
git mv tests/org.eclipse.wst.xsd.ui.tests xml/tests
git mv tests/org.eclipse.wst.xsd.validation.tests xml/tests
git mv tests/org.eclipse.wst.xml.xpath.core.tests xpath/tests
git mv tests/org.eclipse.wst.xml.xpath2.wtptypes.tests/ xpath/tests
git mv tests/org.eclipse.wst.xml.xpath2.processor.tests/ xpath/tests
git mv tests/org.eclipse.wst.xml.xpath.ui.tests xpath/tests
ls tests/ | grep "xsl" | awk '{ print "git mv tests/" $0 " xsl/tests";}' | sh
git mv tests/org.eclipse.wst.dtd.core.tests xml/tests
git mv tests/org.eclipse.wst.dtd.ui.tests xml/tests
git mv tests/* web/tests/   # all others go into ‘web’
rm -f tests/pom.xml
if [[ ! $(ls tests) ]]; then rm -rf tests; else echo "tests is not empty! Abort merge script"; exit 1; fi

git mv development/org.eclipse.wst.sse.unittests web/tests/
rm -rf tests development
git commit -a -s -m "Move tests"

# Features
ls features | grep org.eclipse.wst.json_ | awk '{ print "git mv features/" $0 " json/features";}'  | sh
ls features/ | grep "xml" | grep "org.eclipse" | grep -v "xpath2" | awk '{ print "git mv features/" $0 " xml/features";}' | sh
ls features/ | grep "xpath2" | awk '{ print "git mv features/" $0 " xpath/features";}'  | sh
ls features | grep xsl | awk '{ print "git mv features/" $0 " xsl/features";}' | sh
git mv features/* web/features/
rm -f features/pom.xml
if [[ ! $(ls features) ]]; then rm -rf features; else echo "features is not empty! Abort merge script"; exit 1; fi
git commit -a -s -m "Move features"

cat ../git_ignore_accumulator >> .gitignore
cat .gitignore | sort | uniq > .gitignore2
mv -f .gitignore2 .gitignore
# WARNING: make sure correct quotes used in next line (stupid google)
git commit -s -m "Accumulate all gitignore files. To be cleaned later" .gitignore

######## generate poms
OVERWRITE_EXISTING_POMS=true # or false
NEWLINES=""
EXCLUDES=".git|site|sourceTemplate|.settings|src*|config|DevTimeSupport|META-INF" #|schema   We have plugins named schema!!!
for td in $(find . -maxdepth 1 -type d | egrep -v "${EXCLUDES}" | sort); do
  TOP_DIR=${td##./}
  # collect top level dirs to add to root pom later
  if [[ ${TOP_DIR} != "." ]]; then
    NEWLINES="${NEWLINES}<module>${TOP_DIR}</module>
"
  fi
  pushd ${td} >/dev/null
    for sd in $(find . -maxdepth 1 -type d | egrep -v "${EXCLUDES}" | sort); do
      SUB_DIR=${sd##./}
      pushd ${sd} >/dev/null
      count_subs=$(find . -maxdepth 1 -type d | egrep -v "${EXCLUDES}" | egrep -v "^\.$" | wc -l)
      # don't overwrite existing files and only process this dir if there are children
      if [[ ${count_subs} -gt 0 ]]; then
          if [[ -f pom.xml ]] || [[ ${OVERWRITE_EXISTING_POMS} == "true" ]]; then 
            if [[ ${TOP_DIR} == "." ]]; then
              artifactId="<artifactId>sourceediting.${SUB_DIR}</artifactId>"
            elif [[ ${SUB_DIR} == "." ]]; then
              artifactId="<artifactId>sourceediting.${TOP_DIR}</artifactId>"
            else
              artifactId="<artifactId>sourceediting.${TOP_DIR}.${SUB_DIR}</artifactId>"
            fi
            echo "Generate ${TOP_DIR}/${SUB_DIR}/pom.xml :: ${artifactId} ..."
            cat <<EOT > pom.xml
<?xml version="1.0" encoding="UTF-8"?>
<!--
  Copyright (c) 2017 Eclipse Foundation and others.
  All rights reserved. This program and the accompanying materials
  are made available under the terms of the Eclipse Distribution License v1.0
  which accompanies this distribution, and is available at
  http://www.eclipse.org/org/documents/edl-v10.php
 
  Contributors:
    Nick Boldt (Red Hat) - initial implementation
    Rob Stryker (Red Hat) - initial implementation
-->
<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
  <modelVersion>4.0.0</modelVersion>

  <parent>
    <groupId>org.eclipse.webtools.sourceediting</groupId>
    <artifactId>sourceediting-parent</artifactId>
    <version>3.6.0-SNAPSHOT</version>
  </parent>

  <groupId>org.eclipse.webtools.sourceediting</groupId>
  ${artifactId}
  <version>3.10.0-SNAPSHOT</version>
  <packaging>pom</packaging>

  <modules>
EOT
            for ssd in $(find . -maxdepth 1 -type d | egrep -v "${EXCLUDES}" | sort); do
              SUBSUB_DIR=${ssd##./}
              if [[ ${SUBSUB_DIR} != "." ]] && [[ $(ls ${SUBSUB_DIR}) ]]; then 
                echo "    <module>${SUBSUB_DIR}</module>" >> pom.xml
                echo "  + ${SUBSUB_DIR}"
              fi
            done
            cat <<EOT >> pom.xml
  </modules>
</project>
EOT
          fi
      fi
      popd >/dev/null
    done
  popd >/dev/null
done

# don't regen the root pom yet
git checkout pom.xml 

# commit OTHER new */pom.xml and */*/pom.xml
git add .
git commit -s -m "Commit generated poms" .


# create missing pom - web/tests/org.eclipse.wst.sse.unittests/pom.xml
cat <<'EOT' > web/tests/org.eclipse.wst.sse.unittests/pom.xml
<?xml version="1.0" encoding="UTF-8"?>
<!--
  Copyright (c) 2017 Eclipse Foundation and others.
  All rights reserved. This program and the accompanying materials
  are made available under the terms of the Eclipse Distribution License v1.0
  which accompanies this distribution, and is available at
  http://www.eclipse.org/org/documents/edl-v10.php
 
  Contributors:
    Nick Boldt (Red Hat) - initial implementation
    Rob Stryker (Red Hat) - initial implementation
-->

<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
  <modelVersion>4.0.0</modelVersion>
  <parent>
    <groupId>org.eclipse.webtools.sourceediting</groupId>
    <artifactId>tests</artifactId>
    <version>3.6.0-SNAPSHOT</version>
  </parent>

  <groupId>org.eclipse.webtools.sourceediting</groupId>
  <artifactId>org.eclipse.wst.sse.unittests</artifactId>
  <version>0.7.0-SNAPSHOT</version>
  <packaging>eclipse-test-plugin</packaging>

  <properties>
    <testSuite>${project.artifactId}</testSuite>
    <testClass>org.eclipse.wst.sse.unittests.MasterListTestSuite</testClass>
  </properties>

  <build>
    <plugins>
      <plugin>
        <groupId>org.eclipse.tycho</groupId>
        <artifactId>tycho-surefire-plugin</artifactId>
        <version>${tycho.version}</version>
        <configuration>
          <useUIHarness>true</useUIHarness>
          <useUIThread>true</useUIThread>
        </configuration>
      </plugin>
      <plugin>
        <groupId>org.eclipse.tycho</groupId>
        <artifactId>target-platform-configuration</artifactId>
        <configuration>
          <dependency-resolution>
             <extraRequirements>
                <requirement>
                  <type>eclipse-plugin</type>
                  <id>org.eclipse.equinox.event</id>
                  <versionRange>0.0.0</versionRange>
                </requirement>
             </extraRequirements>
          </dependency-resolution>
        </configuration>
      </plugin>
    </plugins>
  </build>
</project>
EOT
# set snapshot version in manifest
sed -i -e "s#^Bundle-Version: 0.7.0\$#Bundle-Version: 0.7.0.qualifier#g" web/tests/org.eclipse.wst.sse.unittests/META-INF/MANIFEST.MF
git add web/tests/org.eclipse.wst.sse.unittests/pom.xml
git commit -s -m "add sse.unittests pom.xml; fix manifest.mf" web/tests/org.eclipse.wst.sse.unittests/

# note: these line numbers are subject to change
{ head -n 33 pom.xml; for d in core json xml xpath xsl web site; do echo $d; done  | awk '{ print "    <module>" $0 "</module>";}'; tail -n -43 pom.xml; } > pom2.xml
mv pom2.xml pom.xml
git commit -a -s -m "Update root pom"

 
# Remove patch feature references in the pom.xml for xml specifically
for d in xml web; do
   cd ${d}/features
   cat pom.xml  | grep -v "patch" > pom2.xml
   mv pom2.xml pom.xml
   cd ../../
done
git commit -a -s -m "Remove feature patches from feature pom files"


# TODO remove refs to downstream plugins (wst)



# add <references> in root pom to resolve upstream (core, server tools) plugins/features
wget --no-check-certificate https://raw.githubusercontent.com/jbosstools/jbosstools-build-ci/master/util/insertNode.py -O ../insertNode.py
insertRepository () 
{
  # find the end of the file or the end of the <references> section
  pomfile="$1"
  repoProperty="$2"
  repoURL="$3"
  python -W ignore ../insertNode.py --file ${pomfile} --parent properties --xml "<${repoProperty}>${repoURL}</${repoProperty}>"
  python -W ignore ../insertNode.py --file ${pomfile} --parent repositories --xml "<repository><id>${repoProperty}</id><layout>p2</layout><url>\${${repoProperty}}</url></repository>"
}

# TODO this should be a real URL path to the webtools common CI build site, such as this one:
# insertRepository $(pwd)/pom.xml webtools_common_site https://hudson.eclipse.org/webtools/job/webtools.common_R3_10/lastSuccessfulBuild/artifact/webtools.common.repositories/repository/target/repository/
# for now use a real site 
insertRepository $(pwd)/pom.xml webtools_common_site http://download.eclipse.org/webtools/downloads/drops/R3.9.0/I-latest/repository/
git commit -a -s -m "Insert repository to root pom.xml"


# - Below lines added by Rob due to build fails... still failing though


# At this point, the individual bundles still have incorrect references to parent artifacts
# in webtools.sourceediting repo

SSEARTID=`cat pom.xml  | grep "artifactId" | head -n 2 | tail -n 1  | sed 's/[^s]*//' | sed 's/<.*//'`
SSEVERSID=`cat pom.xml  | grep "version" | head -n 3 | tail -n 1  | sed 's/[^0-9]*//' | sed 's/-SNAPSHOT.*/-SNAPSHOT/'`
for td2 in $(find . -maxdepth 1 -type d  | grep -v "^\.$" | grep -v "\.git" | grep -v "site$"); do
   cd $td2
   COMPARTID=`cat pom.xml  | grep "artifactId" | head -n 2 | tail -n 1  | sed 's/[^s]*//' | sed 's/<.*//'`
   COMPVERSID=`cat pom.xml  | grep "version" | head -n 3 | tail -n 1  | sed 's/[^0-9]*//' | sed 's/-SNAPSHOT.*/-SNAPSHOT/'`
   # in component folder
   for td1 in $(find . -maxdepth 1 -type d  | grep -v "^\.$"); do
      cd $td1
      echo "inside " `pwd`
      BUNDLETYPEARTID=`cat pom.xml  | grep "artifactId" | head -n 2 | tail -n 1  | sed 's/[^s]*//' | sed 's/<.*//'`
      BUNDLETYPEVERSID=`cat pom.xml  | grep "version" | head -n 3 | tail -n 1  | sed 's/[^0-9]*//' | sed 's/-SNAPSHOT.*/-SNAPSHOT/'`

      for td in $(find . -maxdepth 1 -type d  | grep -v "^\.$"); do
         if [[ -f ${td}/pom.xml ]]; then
             echo "Fixing $td pom.xml"
             sed "1,/artifactId.*/s/artifactId.*/artifactId>$BUNDLETYPEARTID<\/artifactId>/" $td/pom.xml > $td/pom2.xml
             sed "1,/groupId.*/s/groupId.*/groupId>org.eclipse.webtools.sourceediting<\/groupId>/" $td/pom2.xml > $td/pom3.xml
             tr '\n' '%' < $td/pom3.xml | sed "s/version>[^>]*/version>$BUNDLETYPEVERSID<\/version/1" | tr '%' '\n' > $td/pom4.xml
             mv $td/pom4.xml $td/pom.xml
             rm $td/pom2.xml $td/pom3.xml
         else
             echo "[WARNING] ${td1} ${td}/pom.xml not found, skipping."
        fi
      done
      if [[ -f pom.xml ]]; then
         echo "Fixing pom.xml"
         sed "1,/artifactId.*/s/artifactId.*/artifactId>$COMPARTID<\/artifactId>/" pom.xml > pom2.xml
         tr '\n' '%' < pom2.xml | sed "s/version>[^>]*/version>$COMPVERSID<\/version/1" | tr '%' '\n' > pom3.xml
         mv pom3.xml pom.xml
         rm pom2.xml
      else
         echo "[WARNING] ${td1}/pom.xml not found, skipping."
      fi   
      cd ../
   done
   cd ../
done

for abc in `grep -R "relativePath>" * | cut -f 1 -d ":" | grep -v "^pom.xml$"`; do 
   if [ -f $FILE ]; then
       cat $abc | grep -v "relativePath>" > $abc.2
       mv $abc.2 $abc
   fi
done

if [[ -f xml/features/org.eclipse.wst.xml_core.feature.patch/pom.xml ]]; then 
  rm xml/features/org.eclipse.wst.xml_core.feature.patch/pom.xml
fi
git commit -a -m "Finalize bundle / test / doc poms and refs to their parent poms" --signoff
git clean -f -d



# A last patch to fix the web features. They include too much and should require instead
git remote add rob https://github.com/robstryker/webtools.sourceediting.git
git fetch rob
git cherry-pick ffa5f40
git cherry-pick e65ac79


END_TIME=`date +%s`
EXEC_TIME=$((END_TIME-START_TIME))
echo $EXEC_TIME " seconds execution time without build"


# Let's build all pre-reqs too.  This is annoying 

cd ../
git clone http://git.eclipse.org/gitroot/webtools/webtools.releng.git
git clone http://git.eclipse.org/gitroot/webtools-common/webtools.common.git
git clone http://git.eclipse.org/gitroot/servertools/webtools.servertools.git
git clone http://git.eclipse.org/gitroot/jsdt/webtools.jsdt.git

cd webtools.releng
cd org.eclipse.wtp.releng.versionchecker
mvn clean install
cd ../tests/org.eclipse.wtp.releng.tests
mvn clean install
cd ../../../
for d in webtools.common webtools.servertools webtools.jsdt; do
  pushd ${d}
    echo ""; echo ""; echo "BUILDING $(pwd) ... "
    mvn clean install -DskipTests
    echo ""; echo ""; echo "BUILDING $(pwd) COMPLETE"
  popd >/dev/null
done

END_TIME=`date +%s`
EXEC_TIME=$((END_TIME-START_TIME))
echo $EXEC_TIME " seconds execution time after building pre-reqs"

cd webtools.sourceediting

#
# run full build without -Pintegration flag
${MVN} 

# DONE!
END_TIME=`date +%s`
EXEC_TIME=$((END_TIME-START_TIME))
echo $EXEC_TIME " seconds execution time"