wxPython/distrib/all/build-all

   1 #!/bin/bash
   2 # ---------------------------------------------------------------------------
   3 # Master build script for building all the installers and such on all the
   4 # build machines in my lab, and then distributing the results as needed.
   5 # ---------------------------------------------------------------------------
   6
   7 set -o errexit
   8 #set -o xtrace
   9
  10 # ---------------------------------------------------------------------------
  11 # Some control variables...
  12
  13 # the local spot that we put everything when done, before possibly copying
  14 # to remote hosts
  15 STAGING_DIR=./BUILD
  16
  17
  18 # host name of the machine to use for windows builds
  19 WIN_HOST=beast
  20 # Where is the build dir from the remote machine's perspective?
  21 WIN_BUILD=/c/BUILD
  22
  23
  24 # Just like the above
  25 OSX_HOST_panther=bigmac
  26 OSX_HOST_jaguar=whopper
  27 OSX_BUILD=/BUILD
  28
  29
  30 # Alsmost the same...  See below for hosts and other info
  31 LINUX_BUILD=/tmp/BUILD
  32
  33
  34 # Upload server locations
  35 UPLOAD_HOST=starship.python.net
  36 UPLOAD_DAILY_ROOT=/home/crew/robind/public_html/wxPython/daily
  37 UPLOAD_PREVIEW_ROOT=/home/crew/robind/public_html/wxPython/preview
  38
  39
  40
  41 # ---------------------------------------------------------------------------
  42 # functions
  43
  44 function usage {
  45     echo ""
  46     echo "Usage: $0 [command flags...]"
  47     echo ""
  48     echo "build types:"
  49     echo "   dryrun       Do the build, but don't copy anywhere (default)"
  50     echo "   daily        Do a daily build, copy to starship"
  51     echo "   release      Do a normal release (cantidate) build, copy to starship"
  52     echo ""
  53     echo "optional command flags:"
  54     echo "   2.2          Build for Python 2.2 (default=off)"
  55     echo "   2.3          Build for Python 2.3 (default=on)"
  56     echo "   all          Build for all supported Python versions"
  57     echo ""
  58     echo "   skipsource   Don't build the source archives, use the ones"
  59     echo "                already in the staging dir."
  60     echo "   onlysource   Exit after building the source and docs archives"
  61     echo "   skipdocs     Don't rebuild the docs"
  62     echo "   skipwin      Don't do the remote Windows build"
  63     echo "   skiposx      Don't do the remote OSX build"
  64     echo "   skiplinux    Don't do the remote Linux build"
  65     echo "   skipclean    Don't do the cleanup step on the remote builds"
  66     echo "   skipupload   Don't upload the builds to starship"
  67     echo "   parallel     parallelize the builds where possible"
  68     echo ""
  69 }
  70
  71
  72
  73 function PrefixLines {
  74     label=$1
  75     tee tmp/$label.log | awk "{ print \"** $label: \" \$0; fflush();  }"
  76 }
  77
  78 # ---------------------------------------------------------------------------
  79
  80 # Make sure we are running in the right directory.  TODO: make this
  81 # test more robust.  Currenly we just test for the presence of
  82 # 'wxPython' and 'wx' subdirs.
  83 if [ ! -d wxPython -o ! -d wx ]; then
  84     echo "Please run this script from the root wxPython directory."
  85     exit 1
  86 fi
  87
  88
  89
  90 # Set defaults and check the command line options
  91 KIND=dryrun
  92 PYVER=2.3
  93 skipsource=no
  94 onlysource=no
  95 skipdocs=no
  96 skipwin=no
  97 skiposx=no
  98 skiplinux=no
  99 skipclean=no
 100 skipupload=no
 101 parallel=no
 102
 103 for flag in $*; do
 104     case $flag in
 105         dryrun)   KIND=dryrun                    ;;
 106         daily)    KIND=daily                     ;;
 107         release)  KIND=release                   ;;
 108
 109         2.2)      PYVER=2.2                      ;;
 110         2.3)      PYVER=2.3                      ;;
 111         all)      PYVER="2.2 2.3"                ;;
 112
 113         skipsource)  skipsource=yes              ;;
 114         onlysource)  onlysource=yes              ;;
 115         skipdocs)    skipdocs=yes                ;;
 116         skipwin)     skipwin=yes                 ;;
 117         skiposx)     skiposx=yes                 ;;
 118         skiplinux)   skiplinux=yes               ;;
 119         skipclean)   skipclean=yes               ;;
 120         skipupload)  skipupload=yes              ;;
 121         parallel)    parallel=yes                ;;
 122         noparallel)  parallel=no                 ;;
 123
 124         help) usage; exit 1                      ;;
 125         *)    echo "Unknown flag \"$flag\""
 126               usage
 127               exit 1
 128     esac
 129 done
 130
 131
 132 # ensure the staging area exists
 133 if [ ! -d $STAGING_DIR ]; then
 134     mkdir -p $STAGING_DIR
 135 fi
 136
 137 # Figure out the wxPython version number, possibly adjusted for being a daily build
 138 if [ $KIND = daily ]; then
 139     DAILY=`date +%Y%m%d`   # should it include the hour too?  2-digit year?
 140     echo $DAILY > DAILY_BUILD
 141 fi
 142 VERSION=`python -c "import setup;print setup.VERSION"`
 143
 144
 145 #echo VERSION=$VERSION
 146 #exit 0
 147
 148
 149 echo "Getting started at " `date`
 150
 151 # ---------------------------------------------------------------------------
 152 # Make the sources and other basic stuff.
 153
 154 if [ $skipsource != yes -o $onlysource = yes ]; then
 155
 156     # clean out the local dist dir
 157     rm -f dist/*
 158
 159     if [ $skipdocs != yes ]; then
 160         # Regenerate the reST docs
 161         echo "Regenerating the reST docs..."
 162         cd docs
 163         for x in *.txt; do
 164             docutils-html $x `basename $x .txt`.html
 165         done
 166         cd -
 167
 168         # build the doc and demo tarballs
 169         distrib/makedemo
 170         distrib/makedocs
 171
 172         # build the new docs too
 173         docs/bin/everything
 174     fi
 175
 176     # make the source tarball and srpm
 177     distrib/makerpm 2.3 srpm
 178
 179     # Copy everything to the staging dir
 180     echo "Moving stuff to $STAGING_DIR..."
 181     mv dist/* $STAGING_DIR
 182
 183     if [ $skipdocs != yes ]; then
 184         for doc in CHANGES BUILD INSTALL MigrationGuide default; do
 185             cp docs/$doc.* $STAGING_DIR
 186         done
 187     fi
 188
 189     # cleanup
 190     echo "Cleaning up..."
 191     rm -f dist/*
 192 fi
 193
 194 if [ $KIND = daily ]; then
 195     rm DAILY_BUILD
 196 fi
 197
 198 if [ $onlysource = yes ]; then
 199     exit 0
 200 fi
 201
 202 # ---------------------------------------------------------------------------
 203 # Windows build
 204
 205 function DoWindowsBuild {
 206     set -o errexit
 207
 208     # test if the target machine is online
 209     if ping -q -c1 -w1 $WIN_HOST > /dev/null; then
 210         echo "-----------------------------------------------------------------"
 211         echo " The $WIN_HOST machine is online, Windows build continuing..."
 212         echo "-----------------------------------------------------------------"
 213     else
 214         echo "-----------------------------------------------------------------"
 215         echo "The $WIN_HOST machine is offline, skipping the Windows build."
 216         echo "-----------------------------------------------------------------"
 217         return 0
 218     fi
 219     echo "-=-=- Starting Windows build..."
 220
 221     echo "Copying source file and build script..."
 222     scp $STAGING_DIR/wxPython-src-$VERSION.tar.gz \
 223         distrib/all/build-windows \
 224         $WIN_HOST:$WIN_BUILD
 225
 226      echo "Running build script on $WIN_HOST..."
 227      wxdir=$WIN_BUILD/wxPython-src-$VERSION
 228      cmd=./build-windows
 229      ssh $WIN_HOST "cd $WIN_BUILD && $cmd $wxdir $WIN_BUILD $skipclean $VERSION $PYVER && rm $cmd"
 230
 231      echo "Fetching the results..."
 232      scp "$WIN_HOST:$WIN_BUILD/wxPython*-win32*"  $STAGING_DIR
 233      ssh $WIN_HOST "rm $WIN_BUILD/wxPython*-win32*"
 234 }
 235
 236 if [ $skipwin != yes ]; then
 237     if [ $parallel = no ]; then
 238         DoWindowsBuild
 239     else
 240         DoWindowsBuild 2>&1 | PrefixLines $WIN_HOST &
 241         winPID=$!
 242     fi
 243 fi
 244
 245
 246 # ---------------------------------------------------------------------------
 247 # OSX build
 248
 249 function DoOSXBuild {
 250     local host=$1
 251     local flavor=$2
 252
 253     set -o errexit
 254
 255     # test if the target machine is online
 256     if ping -q -c1 -w1 $host > /dev/null; then
 257         echo "-----------------------------------------------------------------"
 258         echo " The $host machine is online, OSX-$flavor build continuing..."
 259         echo "-----------------------------------------------------------------"
 260     else
 261         echo "-----------------------------------------------------------------"
 262         echo "The $host machine is offline, skipping the OSX-$flavor build."
 263         echo "-----------------------------------------------------------------"
 264         return 0
 265     fi
 266
 267     echo "-=-=- Starting OSX-$flavor build on $host..."
 268
 269     echo "Copying source files and build script..."
 270     ssh root@$host "mkdir -p $OSX_BUILD && rm -rf $OSX_BUILD/* || true"
 271     #ssh root@$host "mkdir -p $OSX_BUILD  || true"
 272     scp $STAGING_DIR/wxPython-src-$VERSION.tar.gz \
 273         $STAGING_DIR/wxPython-docs-$VERSION.tar.gz \
 274         $STAGING_DIR/wxPython-demo-$VERSION.tar.gz \
 275         distrib/all/build-osx \
 276         root@$host:$OSX_BUILD
 277
 278      echo "Running build script on $host..."
 279      wxdir=$OSX_BUILD/wxPython-src-$VERSION
 280      cmd=./build-osx
 281      ssh root@$host "cd $OSX_BUILD && $cmd $wxdir $OSX_BUILD $skipclean $VERSION $flavor $PYVER && rm $cmd"
 282
 283      echo "Fetching the results..."
 284      scp "root@$host:$OSX_BUILD/wxPython*-osx*"  $STAGING_DIR
 285      ssh root@$host "rm $OSX_BUILD/wxPython*-osx*"
 286
 287 }
 288
 289
 290 if [ $skiposx != yes ]; then
 291     if [ $parallel = no ]; then
 292         DoOSXBuild $OSX_HOST_jaguar jaguar
 293         DoOSXBuild $OSX_HOST_panther panther
 294     else
 295         DoOSXBuild $OSX_HOST_jaguar jaguar   2>&1 | PrefixLines $OSX_HOST_jaguar  &
 296         DoOSXBuild $OSX_HOST_panther panther 2>&1 | PrefixLines $OSX_HOST_panther &
 297     fi
 298 fi
 299
 300
 301
 302 # ---------------------------------------------------------------------------
 303 # Linux build
 304
 305 # The remote Linux builds are different than those above.  The source
 306 # RPMs were already built in the source step, and so building the
 307 # binary RPMs is a very simple followup step.  But then add to that
 308 # the fact that we need to build on more than one distro...
 309
 310
 311
 312 function DoLinuxBuild {
 313     local host=$1
 314     local reltag=$2
 315     shift;shift
 316     local pyver=$@
 317
 318     set -o errexit
 319
 320     # test if the target machine is online
 321     if ping -q -c1 -w1 $host > /dev/null; then
 322         echo "-----------------------------------------------------------------"
 323         echo " The $host machine is online, build continuing..."
 324         echo "-----------------------------------------------------------------"
 325     else
 326         echo "-----------------------------------------------------------------"
 327         echo "The $host machine is offline, skipping the binary RPM build."
 328         echo "-----------------------------------------------------------------"
 329         return 0
 330     fi
 331
 332     echo "Copying source files and build script..."
 333     ssh root@$host "mkdir -p $LINUX_BUILD && rm -rf $LINUX_BUILD/*"
 334     scp $STAGING_DIR/wxPython-src* $STAGING_DIR/wxPython.spec\
 335         distrib/all/build-linux \
 336         root@$host:$LINUX_BUILD
 337
 338     echo "Running build script on $host..."
 339     cmd=./build-linux
 340     ssh root@$host "cd $LINUX_BUILD && ./build-linux $reltag $skipclean $VERSION $pyver"
 341
 342     echo "Fetching the results..."
 343     scp "root@$host:$LINUX_BUILD/wxPython*.i[0-9]86.rpm"  $STAGING_DIR
 344     ssh root@$host "rm $LINUX_BUILD/wxPython*.i[0-9]86.rpm"
 345
 346 }
 347
 348
 349
 350
 351 if [ $skiplinux != yes ]; then
 352
 353     if [ $parallel = no ]; then
 354         DoLinuxBuild co-rh9    rh9    $PYVER
 355         DoLinuxBuild co-fc2    fc2    2.3
 356         DoLinuxBuild co-mdk92  mdk92  2.3
 357         DoLinuxBuild co-mdk101 mdk101 2.3
 358     else
 359
 360         # Since the linux builds are currently done in coLinux
 361         # 'machines' running on the WIN_HOST let's wait for the
 362         # windows build to be done before launching them
 363
 364         #if [ -n $winPID ]; then
 365         #    wait $winPID
 366         #fi
 367
 368         (DoLinuxBuild co-rh9    rh9    $PYVER 2>&1 | PrefixLines co-rh9 &; \
 369          DoLinuxBuild co-fc2    fc2    2.3    2>&1 | PrefixLines co-fc2    ) &
 370
 371         wait $!  # wait for the previous two to complete before starting the next two
 372
 373         (DoLinuxBuild co-mdk92  mdk92  2.3    2>&1 | PrefixLines co-mdk92  &; \
 374          DoLinuxBuild co-mdk101 mdk101 2.3    2>&1 | PrefixLines co-mdk101    ) &
 375     fi
 376 fi
 377
 378
 379 # ---------------------------------------------------------------------------
 380
 381 if [ $parallel = yes ]; then
 382
 383     # TODO:  Figure out how to test if all the builds were successful
 384
 385     echo "***********************************"
 386     echo " Waiting for builds to complete... "
 387     echo "***********************************"
 388     wait
 389 fi
 390
 391 # ---------------------------------------------------------------------------
 392 # Final disposition of build results...
 393
 394 chmod a+r $STAGING_DIR/*
 395
 396 if [ $KIND = dryrun ]; then
 397     # we're done
 398     echo "Finished at " `date`
 399     exit 0
 400 fi
 401
 402
 403 if [ $KIND = daily ]; then
 404
 405     echo "Copying to the local file server..."
 406     destdir=/stuff/temp/$VERSION
 407     mkdir -p $destdir
 408     cp $STAGING_DIR/* $destdir
 409
 410     if [ $skipupload != yes ]; then
 411         destdir=$UPLOAD_DAILY_ROOT/$DAILY
 412         echo "Copying to the starship at $destdir..."
 413         ssh $UPLOAD_HOST "mkdir -p $destdir"
 414         scp $STAGING_DIR/* $UPLOAD_HOST:/$destdir
 415         ssh $UPLOAD_HOST "cd $destdir && ls -al"
 416
 417
 418         # TODO: something to remove old builds from starship, keeping
 419         # only N days worth
 420
 421         # Send email to wxPython-dev
 422         DATE=`date`
 423         TO=wxPython-dev@lists.wxwidgets.org
 424
 425         cat <<EOF  | /usr/sbin/sendmail  $TO
 426 From: R'bot <rbot@wxpython.org>
 427 To: $TO
 428 Subject: $DAILY test build uploaded
 429 Date: $DATE
 430
 431 Hi,
 432
 433 A new test build of wxPython has been uploaded to starship.
 434
 435    Version: $VERSION
 436    URL:     http://starship.python.net/crew/robind/wxPython/daily/$DAILY
 437    Changes: http://starship.python.net/crew/robind/wxPython/daily/$DAILY/CHANGES.html
 438
 439 Have fun!
 440 R'bot
 441
 442 EOF
 443     fi
 444
 445     echo "Cleaning up staging dir..."
 446     rm $STAGING_DIR/*
 447     rmdir $STAGING_DIR
 448
 449     echo "Finished at " `date`
 450     exit 0
 451 fi
 452
 453
 454 if [ $KIND = release ]; then
 455
 456     echo "Copying to the local file server..."
 457     destdir=/stuff/Development/wxPython/dist/$VERSION
 458     mkdir -p $destdir
 459     cp $STAGING_DIR/* $destdir
 460
 461     if [ $skipupload != yes ]; then
 462         echo "Copying to the starship..."
 463         destdir=$UPLOAD_PREVIEW_ROOT/$VERSION
 464         ssh $UPLOAD_HOST "mkdir -p $destdir"
 465         scp $STAGING_DIR/* $UPLOAD_HOST:/$destdir
 466
 467         # Send email to wxPython-dev
 468         DATE=`date`
 469         TO=wxPython-dev@lists.wxwidgets.org
 470
 471         cat <<EOF  | /usr/sbin/sendmail  $TO
 472 From: R'bot <rbot@wxpython.org>
 473 To: $TO
 474 Subject: $VERSION release candidate build uploaded
 475 Date: $DATE
 476
 477 Hi,
 478
 479 A new RC build of wxPython has been uploaded to starship.
 480
 481    Version: $VERSION
 482    URL:     http://starship.python.net/crew/robind/wxPython/preview/$VERSION
 483    Changes: http://starship.python.net/crew/robind/wxPython/preview/$VERSION/CHANGES.html
 484
 485 Have fun!
 486 R'bot
 487
 488 EOF
 489
 490     fi
 491
 492     echo "Cleaning up staging dir..."
 493     rm $STAGING_DIR/*
 494     rmdir $STAGING_DIR
 495
 496     echo "Finished at " `date`
 497     exit 0
 498 fi
 499
 500
 501 # ---------------------------------------------------------------------------