Final gasp of cloud image version.

util/vm/build.py

@@ -36,41 +36,42 @@ Notes du jour:
  
 - and use pexpect to interact with it on the serial console
 
-Something to think about:
-
-Maybe download the cloud image and customize it so that
-it is an actual usable/bootable image???
-
 More notes:
     
-We really want a full, partitioned disk image!
+- We use Ubuntu's cloud images, which means that we need to 
+  adapt them for our own (evil) purposes. This isn't ideal
+  but is the easiest way to get official Ubuntu images until
+  they start building official non-cloud images.
 
-This means we want to use the disk1.image file ???
+- We could install grub into a raw ext4 partition rather than
+  partitioning everything. This would save time and it might
+  also confuse people who might be expecting a "normal" volume
+  and who might want to expand it and add more partitions.
+  On the other hand it makes the file system a lot easier to mount
+  and modify!! But vmware might not be able to boot it.
 
-However, this means that we will need to change the grub2
+- grub-install fails miserably unless you load part_msdos !!
-configuratin to use a serial console.
 
-/etc/default/grub:
+- Installing TexLive is just painful - I would like to avoid it
-    GRUB_TERMINAL=serial
+  if we could... wireshark plugin build is also slow and painful...
-    GRUB_SERIAL_COMMAND="serial --unit=0 --speed=38400 --word=8 --parity=no --stop=1"
-    BOOT_IMAGE="console=ttyS0"
 
-# grub2-mkconfig -o /boot/grub2/grub.cfg
+- Maybe we want to install our own packages for these things...
-
+  that would make the whole installation process a lot easier,
-by the way, we should use wget -c
+  but it would mean that we don't automatically get upstream
+  updates
 
 """
 
 import os
 from os import stat
 from stat import ST_MODE
-from os.path import exists, splitext, abspath, realpath
+from os.path import exists, splitext, abspath
 from sys import exit, argv
 from glob import glob
 from urllib import urlretrieve
 from subprocess import check_output, call, Popen, PIPE
 from tempfile import mkdtemp
-from time import time
+from time import time, strftime, localtime
 import argparse
 
 pexpect = None  # For code check - imported dynamically
@@ -90,10 +91,25 @@ ImageURLBase = {
     'raring-server-cloudimg-amd64'
 }
 
+logStartTime = time()
+
+def log( *args, **kwargs ):
+    """Simple log function: log( message along with local and elapsed time
+       cr: False/0 for no CR"""
+    cr = kwargs.get( 'cr', True )
+    elapsed = time() - logStartTime
+    clocktime = strftime( '%H:%M:%S', localtime() )
+    msg = ' '.join( str( arg ) for arg in args )
+    output = '%s [ %.3f ] %s' % ( clocktime, elapsed, msg )
+    if cr:
+        print output
+    else:
+        print output,
+
 
 def run( cmd, **kwargs ):
     "Convenient interface to check_output"
-    print cmd
+    log( '-', cmd )
     cmd = cmd.split()
     return check_output( cmd, **kwargs )
 
@@ -105,7 +121,7 @@ def srun( cmd, **kwargs ):
 
 def depend():
     "Install packagedependencies"
-    print '* Installing package dependencies'
+    log( '* Installing package dependencies' )
     run( 'sudo apt-get -y update' )
     run( 'sudo apt-get install -y'
          ' kvm cloud-utils genisoimage qemu-kvm qemu-utils'
@@ -117,7 +133,7 @@ def depend():
 
 def popen( cmd ):
     "Convenient interface to popen"
-    print cmd
+    log( cmd )
     cmd = cmd.split()
     return Popen( cmd )
 
@@ -148,8 +164,8 @@ def fetchImage( image, path=None ):
     tgz = path + '.disk1.img'
     disk = path + '.img'
     kernel = path + '-vmlinuz-generic'
-    if exists( disk ) and exists( kernel ):
+    if exists( disk ):
-        print '* Found', disk, 'and', kernel
+        log( '* Found', disk )
         # Detect race condition with multiple builds
         perms = stat( disk )[ ST_MODE ] & 0777
         if perms != 0444:
@@ -160,13 +176,13 @@ def fetchImage( image, path=None ):
         run( 'mkdir -p %s' % dir )
         if not os.path.exists( tgz ):
             url = imageURL( image ) + '.tar.gz'
-            print '* Retrieving', url
+            log( '* Retrieving', url )
             urlretrieve( url, tgz )
-        print '* Extracting', tgz
+        log( '* Extracting', tgz )
         run( 'tar -C %s -xzf %s' % ( dir, tgz ) )
         # Write-protect disk image so it remains pristine;
         # We will not use it directly but will use a COW disk
-        print '* Write-protecting disk image', disk
+        log( '* Write-protecting disk image', disk )
         os.chmod( disk, 0444 )
     return disk, kernel
 
@@ -174,22 +190,23 @@ def fetchImage( image, path=None ):
 def addTo( file, line ):
     "Add line to file if it's not there already"
     if call( [ 'sudo', 'grep', line, file ] ) != 0:
-        call( 'echo "%s" | sudo tee -a %s' % ( line, file ), shell=True )
+        call( 'echo "%s" | sudo tee -a %s > /dev/null' % ( line, file ),
+              shell=True )
 
 
 def disableCloud( bind ):
     "Disable cloud junk for disk mounted at bind"
-    print '* Disabling cloud startup scripts'
+    log( '* Disabling cloud startup scripts' )
     modules = glob( '%s/etc/init/cloud*.conf' % bind )
     for module in modules:
         path, ext = splitext( module )
-        override = path + '.override'
+        call( 'echo manual | sudo tee %s.override > /dev/null' % path,
-        call( 'echo manual | sudo tee ' + override, shell=True )
+              shell=True )
 
 
 def addMininetUser( nbd ):
     "Add mininet user/group to filesystem"
-    print '* Adding mininet user to filesystem on device', nbd
+    log( '* Adding mininet user to filesystem on device', nbd )
     # 1. We bind-mount / into a temporary directory, and
     # then mount the volume's /etc and /home on top of it!
     mnt = mkdtemp()
@@ -205,8 +222,8 @@ def addMininetUser( nbd ):
     addTo( bind + '/etc/hosts', '127.0.1.1 mininet-vm' )
     # 2. Next, we delete any old mininet user and add a new one
     chroot( 'deluser mininet' )
-    chroot( 'useradd --create-home mininet' )
+    chroot( 'useradd --create-home --shell /bin/bash mininet' )
-    print '* Setting password'
+    log( '* Setting password' )
     call( 'echo mininet:mininet | sudo chroot %s chpasswd -c SHA512'
           % bind, shell=True )
     # 2a. Add mininet to sudoers
@@ -215,7 +232,7 @@ def addMininetUser( nbd ):
     disableCloud( bind )
     chroot( 'sudo update-rc.d landscape-client disable' )
     # 2c. Add serial getty
-    print '* Adding getty on ttyS0'
+    log( '* Adding getty on ttyS0' )
     chroot( 'cp /etc/init/tty1.conf /etc/init/ttyS0.conf' )
     chroot( 'sed -i "s/tty1/ttyS0/g" /etc/init/ttyS0.conf' )
     # 3. Lastly, we umount and clean up everything
@@ -226,8 +243,6 @@ def addMininetUser( nbd ):
     srun( 'umount ' + mnt )
     run( 'rmdir ' + bind )
     run( 'rmdir ' + mnt )
-    # 4. Just to make sure, we check the filesystem
-    srun( 'e2fsck -y ' + nbd )
 
 
 def attachNBD( cow, flags='' ):
@@ -235,16 +250,15 @@ def attachNBD( cow, flags='' ):
        flags: additional flags for qemu-nbd (e.g. -r for readonly)"""
     # qemu-nbd requires an absolute path
     cow = abspath( cow )
-    print '* Checking for unused /dev/nbdX device ',
+    log( '* Checking for unused /dev/nbdX device ' )
     for i in range ( 0, 63 ):
         nbd = '/dev/nbd%d' % i
-        print i,
         # Check whether someone's already messing with that device
         if call( [ 'pgrep', '-f', nbd ] ) == 0:
             continue
-        # Fails without -v for some annoying reason...
+        srun( 'modprobe nbd max-part=64' )
-        print
         srun( 'qemu-nbd %s -c %s %s' % ( flags, nbd, cow ) )
+        print
         return nbd
     raise Exception( "Error: could not find unused /dev/nbdX device" )
 
@@ -258,11 +272,10 @@ def makeCOWDisk( image, dir='.' ):
     "Create new COW disk for image"
     disk, kernel = fetchImage( image )
     cow = '%s/%s.qcow2' % ( dir, image )
-    print '* Creating COW disk', cow
+    log( '* Creating COW disk', cow )
     run( 'qemu-img create -f qcow2 -b %s %s' % ( disk, cow ) )
-    print '* Resizing COW disk and file system'
+    log( '* Resizing COW disk and file system' )
     run( 'qemu-img resize %s +8G' % cow )
-    srun( 'modprobe nbd max-part=64')
     nbd = attachNBD( cow )
     srun( 'e2fsck -y ' + nbd )
     srun( 'resize2fs ' + nbd )
@@ -271,39 +284,58 @@ def makeCOWDisk( image, dir='.' ):
     return cow, kernel
 
 
-def makeVolume( volume, cylinders=1000  ):
+def makeVolume( volume, size='8G'  ):
-    """Create volume as a qcow2 and add a single boot partition
+    """Create volume as a qcow2 and add a single boot partition"""
-       cylinders: number of ~8MB (255*63*512) cylinders in volume"""
+    log( '* Creating volume of size', size )
-    heads, sectors, bytes = 255, 63, 512
-    size = cylinders * heads * sectors * bytes
-    print '* Creating volume of size', size
     run( 'qemu-img create -f qcow2 %s %s' % ( volume, size ) )
-    print '* Partitioning volume'
+    log( '* Partitioning volume' )
     # We need to mount it using qemu-nbd!!
     nbd = attachNBD( volume )
-    # A bit hacky - we may change this to use parted(8) later
+    parted = Popen( [ 'sudo', 'parted', nbd ], stdin=PIPE )
-    fdisk = Popen( [ 'sudo', 'fdisk', nbd ], stdin=PIPE )
+    cmds = [ 'mklabel msdos',
-    cmds = 'x\nc\n%d\nr\no\nn\np\n1\n\n\na\n1\nw\n' % cylinders
+             'mkpart primary ext4 1 %s' % size,
-    fdisk.stdin.write( cmds )
+             'set 1 boot on',
-    fdisk.wait()
+             'quit' ]
-    print '* Volume partition table:'
+    parted.stdin.write( '\n'.join( cmds ) + '\n' )
-    print srun( 'fdisk -l ' + nbd )
+    parted.wait()
+    log( '* Volume partition table:' )
+    log( srun( 'fdisk -l ' + nbd ) )
     detachNBD( nbd )
 
 
+def installGrub( voldev, partnum=1 ):
+    "Install grub2 on voldev to boot from partition partnum"
+    mnt = mkdtemp()
+    # Find partitions and make sure we have partition 1
+    assert ( '# %d:' % partnum ) in srun( 'partx ' + voldev )
+    partdev = voldev + 'p%d' % partnum
+    srun( 'mount %s %s' % ( partdev, mnt ) )
+    # Make sure we have a boot directory
+    bootdir = mnt + '/boot'
+    run( 'ls ' + bootdir )
+    # Install grub - make sure we preload part_msdos !!
+    srun( 'grub-install --boot-directory=%s --modules=part_msdos %s' % (
+          bootdir, voldev ) )
+    srun( 'umount ' + mnt )
+    run( 'rmdir ' + mnt )
+
+
 def initPartition( partition, volume ):
-    """Copy partition to volume-p1 and call addMininetUser"""
+    """Copy partition to volume-p1 and initialize everything"""
     srcdev = attachNBD( partition, flags='-r' )
     voldev = attachNBD( volume )
-    print srun( 'fdisk -l ' + voldev )
+    log( srun( 'fdisk -l ' + voldev ) )
-    print srun( 'partx ' + voldev )
+    log( srun( 'partx ' + voldev ) )
     dstdev = voldev + 'p1'
-    print "* Copying partition from", srcdev, "to", dstdev
+    log( "* Copying partition from", srcdev, "to", dstdev )
-    print srun( 'time dd if=%s of=%s bs=1M' % ( srcdev, dstdev ) )
+    log( srun( 'dd if=%s of=%s bs=1M' % ( srcdev, dstdev ) ) )
-    print '* Resizing and adding Mininet user'
+    log( '* Resizing file system' )
     srun( 'resize2fs ' + dstdev )
     srun( 'e2fsck -y ' + dstdev )
+    log( '* Adding mininet user' )
     addMininetUser( dstdev )
+    log( '* Installing grub2' )
+    installGrub( voldev, partnum=1 )
     detachNBD( voldev )
     detachNBD( srcdev )
 
@@ -322,7 +354,7 @@ def boot( cow, kernel, tap ):
     elif 'i386' in kernel:
         kvm = 'qemu-system-i386'
     else:
-        print "Error: can't discern CPU for image", cow
+        log( "Error: can't discern CPU for image", cow )
         exit( 1 )
     cmd = [ 'sudo', kvm,
             '-machine accel=kvm',
@@ -335,8 +367,8 @@ def boot( cow, kernel, tap ):
             '-drive file=%s,if=virtio' % cow,
             '-append "root=/dev/vda1 init=/sbin/init console=ttyS0" ' ]
     cmd = ' '.join( cmd )
-    print '* STARTING VM'
+    log( '* STARTING VM' )
-    print cmd
+    log( cmd )
     vm = pexpect.spawn( cmd, timeout=TIMEOUT )
     return vm
 
@@ -344,64 +376,64 @@ def boot( cow, kernel, tap ):
 def interact( vm ):
     "Interact with vm, which is a pexpect object"
     prompt = '\$ '
-    print '* Waiting for login prompt'
+    log( '* Waiting for login prompt' )
     vm.expect( 'login: ' )
-    print '* Logging in'
+    log( '* Logging in' )
     vm.sendline( 'mininet' )
-    print '* Waiting for password prompt'
+    log( '* Waiting for password prompt' )
     vm.expect( 'Password: ' )
-    print '* Sending password'
+    log( '* Sending password' )
     vm.sendline( 'mininet' )
-    print '* Waiting for login...'
+    log( '* Waiting for login...' )
     vm.expect( prompt )
-    print '* Sending hostname command'
+    log( '* Sending hostname command' )
     vm.sendline( 'hostname' )
-    print '* Waiting for output'
+    log( '* Waiting for output' )
     vm.expect( prompt )
-    print '* Fetching Mininet VM install script'
+    log( '* Fetching Mininet VM install script' )
     vm.sendline( 'wget '
                  'https://raw.github.com/mininet/mininet/master/util/vm/'
                  'install-mininet-vm.sh' )
     vm.expect( prompt )
-    print '* Running VM install script'
+    log( '* Running VM install script' )
     vm.sendline( 'bash install-mininet-vm.sh' )
-    print '* Waiting for script to complete... '
+    log( '* Waiting for script to complete... ' )
     # Gigantic timeout for now ;-(
     vm.expect( 'Done preparing Mininet', timeout=3600 )
-    print '* Completed successfully'
+    log( '* Completed successfully' )
     vm.expect( prompt )
-    print '* Testing Mininet'
+    log( '* Testing Mininet' )
     vm.sendline( 'sudo mn --test pingall' )
-    if vm.expect( [ ' 0% dropped', pexpect.TIMEOUT ], timeout=45 ):
+    if vm.expect( [ ' 0% dropped', pexpect.TIMEOUT ], timeout=45 ) == 0:
-        print '* Sanity check succeeded'
+        log( '* Sanity check succeeded' )
     else:
-        print '* Sanity check FAILED'
+        log( '* Sanity check FAILED' )
     vm.expect( prompt )
-    print '* Making sure cgroups are mounted'
+    log( '* Making sure cgroups are mounted' )
     vm.sendline( 'sudo service cgroup-lite restart' )
     vm.expect( prompt )
     vm.sendline( 'sudo cgroups-mount' )
     vm.expect( prompt )
-    print '* Running make test'
+    log( '* Running make test' )
     vm.sendline( 'cd ~/mininet; sudo make test' )
     vm.expect( prompt )
-    print '* Shutting down'
+    log( '* Shutting down' )
     vm.sendline( 'sync; sudo shutdown -h now' )
-    print '* Waiting for EOF/shutdown'
+    log( '* Waiting for EOF/shutdown' )
     vm.read()
-    print '* Interaction complete'
+    log( '* Interaction complete' )
 
 
 def cleanup():
     "Clean up leftover qemu-nbd processes and other junk"
-    call( 'sudo pkill -9 qemu-nbd', shell=True )
+    call( [ 'sudo', 'pkill', '-9', 'qemu-nbd' ] )
 
 
 def convert( cow, basename ):
     """Convert a qcow2 disk to a vmdk and put it a new directory
        basename: base name for output vmdk file"""
     vmdk = basename + '.vmdk'
-    print '* Converting qcow2 to vmdk'
+    log( '* Converting qcow2 to vmdk' )
     run( 'qemu-img convert -f qcow2 -O vmdk %s %s' % ( cow, vmdk ) )
     return vmdk
 
@@ -411,28 +443,32 @@ def build( flavor='raring32server' ):
     start = time()
     dir = mkdtemp( prefix=flavor + '-result-', dir='.' )
     os.chdir( dir )
-    print '* Created working directory', dir
+    log( '* Created working directory', dir )
     image, kernel = fetchImage( flavor )
     volume = flavor + '.qcow2'
     makeVolume( volume )
     initPartition( image, volume )
-    print '* VM image for', flavor, 'created as', volume
+    log( '* VM image for', flavor, 'created as', volume )
     logfile = open( flavor + '.log', 'w+' )
-    print '* Logging results to', abspath( logfile.name )
+    log( '* Logging results to', abspath( logfile.name ) )
     vm = boot( volume, kernel, logfile )
     vm.logfile_read = logfile
     interact( vm )
     vmdk = convert( volume, basename=flavor )
-    print '* Converted VM image stored as', vmdk
+    log( '* Converted VM image stored as', vmdk )
     end = time()
     elapsed = end - start
-    print '* Results logged to', abspath( logfile.name )
+    log( '* Results logged to', abspath( logfile.name ) )
-    print '* Completed in %.2f seconds' % elapsed
+    log( '* Completed in %.2f seconds' % elapsed )
-    print '* %s VM build DONE!!!!! :D' % flavor
+    log( '* %s VM build DONE!!!!! :D' % flavor )
-    print
+    log( '* ' )
     os.chdir( '..' )
 
 
+def listFlavors():
+    "List valid build flavors"
+    print '\nvalid build flavors:', ' '.join( ImageURLBase ), '\n'
+
 def parseArgs():
     "Parse command line arguments and run"
     parser = argparse.ArgumentParser( description='Mininet VM build script' )
@@ -448,17 +484,19 @@ def parseArgs():
     if args.depend:
         depend()
     if args.list:
-        print 'valid build flavors:', ' '.join( ImageURLBase )
+        listFlavors()
     if args.clean:
         cleanup()
     flavors = args.flavor[ 1: ]
     for flavor in flavors:
         if flavor not in ImageURLBase:
             parser.print_help()
+            listFlavors()
+            break
         # try:
         build( flavor )
         # except Exception as e:
-        # print '* BUILD FAILED with exception: ', e
+        # log( '* BUILD FAILED with exception: ', e )
         # exit( 1 )
     if not ( args.depend or args.list or args.clean or flavors ):
         parser.print_help()