Some support for parallelizing builds

[wxWidgets.git] / wxPython / distrib / all / build-all

diff --git a/wxPython/distrib/all/build-all b/wxPython/distrib/all/build-all
index 516ddb2a261e7eff93f4df38f5b952ca7f093bbe..736459acee8e6a229ea74a2a692a0d60f1a11168 100755 (executable)
--- a/wxPython/distrib/all/build-all
+++ b/wxPython/distrib/all/build-all
@@ -22,12 +22,12 @@ WIN_BUILD=/c/BUILD
 
 
 # Just like the above
 
 
 # Just like the above

-OSX_HOST=bigmac
-OSX_BUILD=/tmp/BUILD
+OSX_HOST_panther=bigmac
+OSX_HOST_jaguar=whopper
+OSX_BUILD=/BUILD

 
 
 
 

-# Alsmost the same...  See below
-LINUX_HOST=rh9
+# Alsmost the same...  See below for hosts and other info

 LINUX_BUILD=/tmp/BUILD
 
 
 LINUX_BUILD=/tmp/BUILD
 
 


@@ -48,7 +48,7 @@ function usage {
     echo "build types:"
     echo "   dryrun       Do the build, but don't copy anywhere (default)"
     echo "   daily        Do a daily build, copy to starship"
     echo "build types:"
     echo "   dryrun       Do the build, but don't copy anywhere (default)"
     echo "   daily        Do a daily build, copy to starship"

-    echo "   release      Do a normal release build, copy to starship"
+    echo "   release      Do a normal release (cantidate) build, copy to starship"

     echo ""
     echo "optional command flags:"
     echo "   2.2          Build for Python 2.2 (default=off)"
     echo ""
     echo "optional command flags:"
     echo "   2.2          Build for Python 2.2 (default=off)"


@@ -57,14 +57,22 @@ function usage {
     echo ""
     echo "   skipsource   Don't build the source archives, use the ones"
     echo "                already in the staging dir."
     echo ""
     echo "   skipsource   Don't build the source archives, use the ones"
     echo "                already in the staging dir."

-    echo "   onlysource   Exit after building the source archives"
+    echo "   onlysource   Exit after building the source and docs archives"
+    echo "   skipdocs     Don't rebuild the docs"

     echo "   skipwin      Don't do the remote Windows build"
     echo "   skiposx      Don't do the remote OSX build"
     echo "   skiplinux    Don't do the remote Linux build"
     echo "   skipclean    Don't do the cleanup step on the remote builds"
     echo "   skipwin      Don't do the remote Windows build"
     echo "   skiposx      Don't do the remote OSX build"
     echo "   skiplinux    Don't do the remote Linux build"
     echo "   skipclean    Don't do the cleanup step on the remote builds"

+    echo "   skipupload   Don't upload the builds to starship"
+    echo "   parallel     parallelize the builds where possible"

     echo ""
     echo ""

+}
+

 
 
 
 

+function PrefixLines {
+    label=$1
+    tee tmp/$label.log | awk "{ print \"** $label: \" \$0; fflush();  }"

 }
 
 # ---------------------------------------------------------------------------
 }
 
 # ---------------------------------------------------------------------------


@@ -84,10 +92,13 @@ KIND=dryrun
 PYVER=2.3
 skipsource=no
 onlysource=no
 PYVER=2.3
 skipsource=no
 onlysource=no

+skipdocs=no

 skipwin=no
 skiposx=no
 skiplinux=no
 skipclean=no
 skipwin=no
 skiposx=no
 skiplinux=no
 skipclean=no

+skipupload=no
+parallel=no

 
 for flag in $*; do
     case $flag in 
 
 for flag in $*; do
     case $flag in 


@@ -101,10 +112,14 @@ for flag in $*; do
 
        skipsource)  skipsource=yes              ;;
        onlysource)  onlysource=yes              ;;
 
        skipsource)  skipsource=yes              ;;
        onlysource)  onlysource=yes              ;;

+       skipdocs)    skipdocs=yes                ;;

        skipwin)     skipwin=yes                 ;;
        skiposx)     skiposx=yes                 ;;
        skiplinux)   skiplinux=yes               ;;
        skipclean)   skipclean=yes               ;;
        skipwin)     skipwin=yes                 ;;
        skiposx)     skiposx=yes                 ;;
        skiplinux)   skiplinux=yes               ;;
        skipclean)   skipclean=yes               ;;

+       skipupload)  skipupload=yes              ;;
+       parallel)    parallel=yes                ;;
+       noparallel)  parallel=no                 ;;

 
        help) usage; exit 1                      ;;
        *)    echo "Unknown flag \"$flag\""
 
        help) usage; exit 1                      ;;
        *)    echo "Unknown flag \"$flag\""


@@ -130,6 +145,7 @@ VERSION=`python -c "import setup;print setup.VERSION"`
 #echo VERSION=$VERSION
 #exit 0
 
 #echo VERSION=$VERSION
 #exit 0
 

+

 echo "Getting started at " `date`
 
 # ---------------------------------------------------------------------------
 echo "Getting started at " `date`
 
 # ---------------------------------------------------------------------------


@@ -140,42 +156,39 @@ if [ $skipsource != yes -o $onlysource = yes ]; then
     # clean out the local dist dir
     rm -f dist/*
 
     # clean out the local dist dir
     rm -f dist/*
 

-    # Regenerate the reST docs
-    echo "Regenerating the reST docs..."
-    cd docs
-    for x in *.txt; do
-       docutils-html $x `basename $x .txt`.html
-    done
-    cd -
-       
-    # build the doc and demo tarballs
-    distrib/makedemo
-    distrib/makedocs
-
-    # build the new docs too
-    docs/bin/everything
-
-    # make the source tarball
-    distrib/makerpm 2.3 skipclean skiprpm gtk2
-
-    # make the source RPMs
-    for ver in $PYVER; do 
-       distrib/makerpm $ver skipclean skipcopy skiptar srpm
-       distrib/makerpm $ver skipclean skipcopy skiptar srpm gtk2
-    done
+    if [ $skipdocs != yes ]; then 
+        # Regenerate the reST docs
+       echo "Regenerating the reST docs..."
+       cd docs
+       for x in *.txt; do
+           docutils-html $x `basename $x .txt`.html
+       done
+       cd -
+       
+        # build the doc and demo tarballs
+       distrib/makedemo
+       distrib/makedocs
+
+        # build the new docs too
+       docs/bin/everything
+    fi
+
+    # make the source tarball and srpm
+    distrib/makerpm 2.3 srpm

 
     # Copy everything to the staging dir
     echo "Moving stuff to $STAGING_DIR..."
 
     # Copy everything to the staging dir
     echo "Moving stuff to $STAGING_DIR..."

-    rm -f dist/*.spec

     mv dist/* $STAGING_DIR
     mv dist/* $STAGING_DIR

-    for doc in CHANGES BUILD INSTALL MigrationGuide default; do
-       cp docs/$doc.* $STAGING_DIR
-    done
+
+    if [ $skipdocs != yes ]; then 
+       for doc in CHANGES BUILD INSTALL MigrationGuide default; do
+           cp docs/$doc.* $STAGING_DIR
+       done
+    fi

 
     # cleanup
     echo "Cleaning up..."
     rm -f dist/*
 
     # cleanup
     echo "Cleaning up..."
     rm -f dist/*

-    rm -rf _build_rpm

 fi 
 
 if [ $KIND = daily ]; then
 fi 
 
 if [ $KIND = daily ]; then


@@ -189,97 +202,191 @@ fi
 # ---------------------------------------------------------------------------
 # Windows build
 
 # ---------------------------------------------------------------------------
 # Windows build
 

-if [ $skipwin != yes ]; then
+function DoWindowsBuild {
+    set -o errexit
+
+    # test if the target machine is online
+    if ping -q -c1 -w1 $WIN_HOST > /dev/null; then
+       echo "-----------------------------------------------------------------"
+       echo " The $WIN_HOST machine is online, Windows build continuing..."
+       echo "-----------------------------------------------------------------"
+    else
+       echo "-----------------------------------------------------------------"
+       echo "The $WIN_HOST machine is offline, skipping the Windows build."
+       echo "-----------------------------------------------------------------"
+       return 0
+    fi

     echo "-=-=- Starting Windows build..."
 
     echo "Copying source file and build script..."
     echo "-=-=- Starting Windows build..."
 
     echo "Copying source file and build script..."

-    scp $STAGING_DIR/wxPythonSrc-$VERSION.tar.gz \
+    scp $STAGING_DIR/wxPython-src-$VERSION.tar.gz \

        distrib/all/build-windows \
        $WIN_HOST:$WIN_BUILD
     
      echo "Running build script on $WIN_HOST..."
        distrib/all/build-windows \
        $WIN_HOST:$WIN_BUILD
     
      echo "Running build script on $WIN_HOST..."

-     wxdir=$WIN_BUILD/wxPythonSrc-$VERSION
+     wxdir=$WIN_BUILD/wxPython-src-$VERSION

      cmd=./build-windows
      ssh $WIN_HOST "cd $WIN_BUILD && $cmd $wxdir $WIN_BUILD $skipclean $VERSION $PYVER && rm $cmd"
 
      echo "Fetching the results..."
      cmd=./build-windows
      ssh $WIN_HOST "cd $WIN_BUILD && $cmd $wxdir $WIN_BUILD $skipclean $VERSION $PYVER && rm $cmd"
 
      echo "Fetching the results..."

-     scp $WIN_HOST:$WIN_BUILD/wxPythonWIN32*  $STAGING_DIR
-     ssh $WIN_HOST "rm $WIN_BUILD/wxPythonWIN32*"
+     scp "$WIN_HOST:$WIN_BUILD/wxPython*-win32*"  $STAGING_DIR
+     ssh $WIN_HOST "rm $WIN_BUILD/wxPython*-win32*"
+}
+
+if [ $skipwin != yes ]; then
+    if [ $parallel = no ]; then
+       DoWindowsBuild
+    else
+       DoWindowsBuild 2>&1 | PrefixLines $WIN_HOST &
+       winPID=$!
+    fi

 fi
 
 
 # ---------------------------------------------------------------------------
 # OSX build
 
 fi
 
 
 # ---------------------------------------------------------------------------
 # OSX build
 

-if [ $skiposx != yes ]; then
-    echo "-=-=- Starting OSX build..."
+function DoOSXBuild {
+    local host=$1
+    local flavor=$2
+
+    set -o errexit
+
+    # test if the target machine is online
+    if ping -q -c1 -w1 $host > /dev/null; then
+       echo "-----------------------------------------------------------------"
+       echo " The $host machine is online, OSX-$flavor build continuing..."
+       echo "-----------------------------------------------------------------"
+    else
+       echo "-----------------------------------------------------------------"
+       echo "The $host machine is offline, skipping the OSX-$flavor build."
+       echo "-----------------------------------------------------------------"
+       return 0
+    fi
+
+    echo "-=-=- Starting OSX-$flavor build on $host..."

 
     echo "Copying source files and build script..."
 
     echo "Copying source files and build script..."

-    ssh $OSX_HOST "mkdir -p $OSX_BUILD && rm -rf $OSX_BUILD/*"
-    scp $STAGING_DIR/wxPythonSrc-$VERSION.tar.gz \
-       $STAGING_DIR/wxPythonDocs-$VERSION.tar.gz \
-       $STAGING_DIR/wxPythonDemo-$VERSION.tar.gz \
+    ssh root@$host "mkdir -p $OSX_BUILD && rm -rf $OSX_BUILD/* || true"
+    #ssh root@$host "mkdir -p $OSX_BUILD  || true"
+    scp $STAGING_DIR/wxPython-src-$VERSION.tar.gz \
+       $STAGING_DIR/wxPython-docs-$VERSION.tar.gz \
+       $STAGING_DIR/wxPython-demo-$VERSION.tar.gz \

        distrib/all/build-osx \
        distrib/all/build-osx \

-       $OSX_HOST:$OSX_BUILD
+       root@$host:$OSX_BUILD

 
 

-     echo "Running build script on $OSX_HOST..."
-     wxdir=$OSX_BUILD/wxPythonSrc-$VERSION
+     echo "Running build script on $host..."
+     wxdir=$OSX_BUILD/wxPython-src-$VERSION

      cmd=./build-osx
      cmd=./build-osx

-     ssh root@$OSX_HOST "cd $OSX_BUILD && $cmd $wxdir $OSX_BUILD $skipclean $VERSION $PYVER && rm $cmd"
+     ssh root@$host "cd $OSX_BUILD && $cmd $wxdir $OSX_BUILD $skipclean $VERSION $flavor $PYVER && rm $cmd"

 
      echo "Fetching the results..."
 
      echo "Fetching the results..."

-     scp "$OSX_HOST:$OSX_BUILD/wxPythonOSX*"  $STAGING_DIR
-     ssh $OSX_HOST "rm $OSX_BUILD/wxPythonOSX*"
-fi
+     scp "root@$host:$OSX_BUILD/wxPython*-osx*"  $STAGING_DIR
+     ssh root@$host "rm $OSX_BUILD/wxPython*-osx*"

 
 

+}

 
 

-# ---------------------------------------------------------------------------
-# Linux build

 
 

-# This build is optional, check if the target machine is up and
-# running
-if [ $skiplinux != yes ]; then
-    if  ping -q -c1 -w1 $LINUX_HOST > /dev/null; then
-        # the ping succeeded
-       skiplinux=no
+if [ $skiposx != yes ]; then
+    if [ $parallel = no ]; then
+       DoOSXBuild $OSX_HOST_jaguar jaguar
+       DoOSXBuild $OSX_HOST_panther panther

     else
     else

-        # the ping failed, skip the build
-       skiplinux=yes
-       echo "-----------------------------------------------------------------"
-       echo "The $LINUX_HOST machine is offline, skipping the binary RPM build."
-       echo "-----------------------------------------------------------------"
+       DoOSXBuild $OSX_HOST_jaguar jaguar   2>&1 | PrefixLines $OSX_HOST_jaguar  &
+       DoOSXBuild $OSX_HOST_panther panther 2>&1 | PrefixLines $OSX_HOST_panther &

     fi
 fi
 
 
     fi
 fi
 
 

-if [ $skiplinux != yes ]; then
-    echo "-=-=- Starting Linux build..."

 
 

-    # The remote linux build is a bit different than the others.  The
-    # SRPMs will have already been built in the initial source
-    # building steps, the only  thing that the remote build needs to
-    # do is an "rpmbuild --rebuild"  for each package.  So we'll just
-    # copy the build script over and  execute it, there is no need to
-    # unpack the tarball and most of the  other steps...
+# ---------------------------------------------------------------------------
+# Linux build
+
+# The remote Linux builds are different than those above.  The source
+# RPMs were already built in the source step, and so building the
+# binary RPMs is a very simple followup step.  But then add to that
+# the fact that we need to build on more than one distro...

 
 

-    
+
+
+function DoLinuxBuild {
+    local host=$1
+    local reltag=$2
+    shift;shift
+    local pyver=$@
+
+    set -o errexit
+
+    # test if the target machine is online
+    if ping -q -c1 -w1 $host > /dev/null; then
+       echo "-----------------------------------------------------------------"
+       echo " The $host machine is online, build continuing..."
+       echo "-----------------------------------------------------------------"
+    else
+       echo "-----------------------------------------------------------------"
+       echo "The $host machine is offline, skipping the binary RPM build."
+       echo "-----------------------------------------------------------------"
+       return 0
+    fi
+       

     echo "Copying source files and build script..."
     echo "Copying source files and build script..."

-    ssh root@$LINUX_HOST "mkdir -p $LINUX_BUILD && rm -rf $LINUX_BUILD/*"
-    scp $STAGING_DIR/wxPython*.src.rpm \
+    ssh root@$host "mkdir -p $LINUX_BUILD && rm -rf $LINUX_BUILD/*"
+    scp $STAGING_DIR/wxPython-src* $STAGING_DIR/wxPython.spec\

        distrib/all/build-linux \
        distrib/all/build-linux \

-       root@$LINUX_HOST:$LINUX_BUILD
+       root@$host:$LINUX_BUILD

 
 

-    echo "Running build script on $LINUX_HOST..."
+    echo "Running build script on $host..."

     cmd=./build-linux
     cmd=./build-linux

-    ssh root@$LINUX_HOST "cd $LINUX_BUILD && $cmd $LINUX_BUILD $skipclean $VERSION $PYVER"
+    ssh root@$host "cd $LINUX_BUILD && ./build-linux $reltag $skipclean $VERSION $pyver"

     
     echo "Fetching the results..."
     
     echo "Fetching the results..."

-    scp root@$LINUX_HOST:$LINUX_BUILD/wxPythonGTK*.i[0-9]86.rpm  $STAGING_DIR
-    ssh root@$LINUX_HOST "rm $LINUX_BUILD/wxPythonGTK*.i[0-9]86.rpm"
+    scp "root@$host:$LINUX_BUILD/wxPython*.i[0-9]86.rpm"  $STAGING_DIR
+    ssh root@$host "rm $LINUX_BUILD/wxPython*.i[0-9]86.rpm"
+
+}
+
+
+
+
+if [ $skiplinux != yes ]; then
+
+    if [ $parallel = no ]; then
+       DoLinuxBuild co-rh9    rh9    $PYVER
+       DoLinuxBuild co-fc2    fc2    2.3
+       DoLinuxBuild co-mdk92  mdk92  2.3
+       DoLinuxBuild co-mdk101 mdk101 2.3
+    else
+
+        # Since the linux builds are currently done in coLinux
+        # 'machines' running on the WIN_HOST let's wait for the
+        # windows build to be done before launching them
+
+       #if [ -n $winPID ]; then
+       #    wait $winPID
+       #fi
+
+       (DoLinuxBuild co-rh9    rh9    $PYVER 2>&1 | PrefixLines co-rh9 &; \
+        DoLinuxBuild co-fc2    fc2    2.3    2>&1 | PrefixLines co-fc2    ) &
+
+       wait $!  # wait for the previous two to complete before starting the next two
+
+       (DoLinuxBuild co-mdk92  mdk92  2.3    2>&1 | PrefixLines co-mdk92  &; \
+        DoLinuxBuild co-mdk101 mdk101 2.3    2>&1 | PrefixLines co-mdk101    ) &
+    fi

 fi
 
 
 fi
 
 

+# ---------------------------------------------------------------------------
+
+if [ $parallel = yes ]; then
+
+    # TODO:  Figure out how to test if all the builds were successful

 
 

+    echo "***********************************"
+    echo " Waiting for builds to complete... "
+    echo "***********************************"
+    wait
+fi

 
 # ---------------------------------------------------------------------------
 # Final disposition of build results...
 
 # ---------------------------------------------------------------------------
 # Final disposition of build results...


@@ -295,28 +402,30 @@ fi
 
 if [ $KIND = daily ]; then 
 
 
 if [ $KIND = daily ]; then 
 

-    destdir=$UPLOAD_DAILY_ROOT/$DAILY
-    echo "Copying to the starship at $destdir..."
-    ssh $UPLOAD_HOST "mkdir -p $destdir"
-    scp $STAGING_DIR/* $UPLOAD_HOST:/$destdir
-    ssh $UPLOAD_HOST "cd $destdir && ls -al"
+    echo "Copying to the local file server..."
+    destdir=/stuff/temp/$VERSION
+    mkdir -p $destdir
+    cp $STAGING_DIR/* $destdir

 
 

+    if [ $skipupload != yes ]; then 
+       destdir=$UPLOAD_DAILY_ROOT/$DAILY
+       echo "Copying to the starship at $destdir..."
+       ssh $UPLOAD_HOST "mkdir -p $destdir"
+       scp $STAGING_DIR/* $UPLOAD_HOST:/$destdir
+       ssh $UPLOAD_HOST "cd $destdir && ls -al"

 
 

-    echo "Cleaning up staging dir..."
-    rm $STAGING_DIR/*
-    rmdir $STAGING_DIR

 
 

-    # TODO: something to remove old builds from starship, keeping 
-    # only N days worth
+        # TODO: something to remove old builds from starship, keeping 
+        # only N days worth

 
 

-    # TODO:  Send email to wxPython-dev?
-    DATE=`date`
-    TO=wxPython-dev@lists.wxwidgets.org
+        # Send email to wxPython-dev
+       DATE=`date`
+       TO=wxPython-dev@lists.wxwidgets.org

 
 

-    cat <<EOF  | /usr/sbin/sendmail  $TO
+       cat <<EOF  | /usr/sbin/sendmail  $TO

 From: R'bot <rbot@wxpython.org>
 To: $TO
 From: R'bot <rbot@wxpython.org>
 To: $TO

-Subject: New test build uploaded
+Subject: $DAILY test build uploaded

 Date: $DATE
 
 Hi,
 Date: $DATE
 
 Hi,


@@ -324,13 +433,18 @@ Hi,
 A new test build of wxPython has been uploaded to starship.  
 
    Version: $VERSION
 A new test build of wxPython has been uploaded to starship.  
 
    Version: $VERSION

-   Pythons: $PYVER

    URL:     http://starship.python.net/crew/robind/wxPython/daily/$DAILY
    URL:     http://starship.python.net/crew/robind/wxPython/daily/$DAILY

+   Changes: http://starship.python.net/crew/robind/wxPython/daily/$DAILY/CHANGES.html

 
 Have fun!
 R'bot
 
 EOF
 
 Have fun!
 R'bot
 
 EOF

+    fi
+
+    echo "Cleaning up staging dir..."
+    rm $STAGING_DIR/*
+    rmdir $STAGING_DIR

 
     echo "Finished at " `date`
     exit 0
 
     echo "Finished at " `date`
     exit 0


@@ -344,10 +458,36 @@ if [ $KIND = release ]; then
     mkdir -p $destdir
     cp $STAGING_DIR/* $destdir
 
     mkdir -p $destdir
     cp $STAGING_DIR/* $destdir
 

-    echo "Copying to the starship..."
-    destdir=$UPLOAD_PREVIEW_ROOT/$VERSION
-    ssh $UPLOAD_HOST "mkdir -p $destdir"
-    scp $STAGING_DIR/* $UPLOAD_HOST:/$destdir
+    if [ $skipupload != yes ]; then 
+       echo "Copying to the starship..."
+       destdir=$UPLOAD_PREVIEW_ROOT/$VERSION
+       ssh $UPLOAD_HOST "mkdir -p $destdir"
+       scp $STAGING_DIR/* $UPLOAD_HOST:/$destdir
+
+        # Send email to wxPython-dev
+       DATE=`date`
+       TO=wxPython-dev@lists.wxwidgets.org
+
+       cat <<EOF  | /usr/sbin/sendmail  $TO
+From: R'bot <rbot@wxpython.org>
+To: $TO
+Subject: $VERSION release candidate build uploaded
+Date: $DATE
+
+Hi,
+
+A new RC build of wxPython has been uploaded to starship.  
+
+   Version: $VERSION
+   URL:     http://starship.python.net/crew/robind/wxPython/preview/$VERSION
+   Changes: http://starship.python.net/crew/robind/wxPython/preview/$VERSION/CHANGES.html
+
+Have fun!
+R'bot
+
+EOF
+
+    fi

 
     echo "Cleaning up staging dir..."
     rm $STAGING_DIR/*
 
     echo "Cleaning up staging dir..."
     rm $STAGING_DIR/*