From 94954177e5fc524aa35c7986acc402ae01087e09 Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Wed, 21 Aug 2013 21:18:11 -0700 Subject: [PATCH 001/109] Added support for creating a volume rather than a raw partition. --- util/vm/build.py | 124 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 92 insertions(+), 32 deletions(-) diff --git a/util/vm/build.py b/util/vm/build.py index eb54c94..9accde1 100755 --- a/util/vm/build.py +++ b/util/vm/build.py @@ -41,22 +41,41 @@ Something to think about: Maybe download the cloud image and customize it so that it is an actual usable/bootable image??? +More notes: + +We really want a full, partitioned disk image! + +This means we want to use the disk1.image file ??? + +However, this means that we will need to change the grub2 +configuratin to use a serial console. + +/etc/default/grub: + GRUB_TERMINAL=serial + GRUB_SERIAL_COMMAND="serial --unit=0 --speed=38400 --word=8 --parity=no --stop=1" + BOOT_IMAGE="console=ttyS0" + +# grub2-mkconfig -o /boot/grub2/grub.cfg + +by the way, we should use wget -c """ - import os from os import stat from stat import ST_MODE -from os.path import exists, splitext +from os.path import exists, splitext, abspath, realpath from sys import exit, argv from glob import glob from urllib import urlretrieve from subprocess import check_output, call, Popen, PIPE -from tempfile import mkdtemp, NamedTemporaryFile +from tempfile import mkdtemp from time import time import argparse +pexpect = None # For code check - imported dynamically + + # boot can be slooooow!!!! need to debug/optimize somehow TIMEOUT=600 @@ -87,7 +106,6 @@ def srun( cmd, **kwargs ): def depend(): "Install packagedependencies" print '* Installing package dependencies' - packages = ( ) run( 'sudo apt-get -y update' ) run( 'sudo apt-get install -y' ' kvm cloud-utils genisoimage qemu-kvm qemu-utils' @@ -127,10 +145,9 @@ def fetchImage( image, path=None ): "Fetch base VM image if it's not there already" if not path: path = imagePath( image ) - tgz = path + '.tar.gz' + tgz = path + '.disk1.img' disk = path + '.img' kernel = path + '-vmlinuz-generic' - floppy = path + '-floppy' if exists( disk ) and exists( kernel ): print '* Found', disk, 'and', kernel # Detect race condition with multiple builds @@ -213,8 +230,11 @@ def addMininetUser( nbd ): srun( 'e2fsck -y ' + nbd ) -def connectCOWdevice( cow ): - "Attempt to connect a COW disk and return its nbd device" +def attachNBD( cow, flags='' ): + """Attempt to attach a COW disk image and return its nbd device + flags: additional flags for qemu-nbd (e.g. -r for readonly)""" + # qemu-nbd requires an absolute path + cow = abspath( cow ) print '* Checking for unused /dev/nbdX device ', for i in range ( 0, 63 ): nbd = '/dev/nbd%d' % i @@ -224,33 +244,70 @@ def connectCOWdevice( cow ): continue # Fails without -v for some annoying reason... print - srun( 'qemu-nbd -c %s %s' % ( nbd, cow ) ) + srun( 'qemu-nbd %s -c %s %s' % ( flags, nbd, cow ) ) return nbd raise Exception( "Error: could not find unused /dev/nbdX device" ) -def disconnectCOWdevice( nbd ): +def detachNBD( nbd ): + "Detatch an nbd device" srun( 'qemu-nbd -d ' + nbd ) -def makeCOWDisk( image ): +def makeCOWDisk( image, dir='.' ): "Create new COW disk for image" disk, kernel = fetchImage( image ) - cow = NamedTemporaryFile( prefix=image + '-', suffix='.qcow2', - dir='.' ).name + cow = '%s/%s.qcow2' % ( dir, image ) print '* Creating COW disk', cow run( 'qemu-img create -f qcow2 -b %s %s' % ( disk, cow ) ) print '* Resizing COW disk and file system' run( 'qemu-img resize %s +8G' % cow ) srun( 'modprobe nbd max-part=64') - nbd = connectCOWdevice( cow ) + nbd = attachNBD( cow ) srun( 'e2fsck -y ' + nbd ) srun( 'resize2fs ' + nbd ) addMininetUser( nbd ) - disconnectCOWdevice( nbd ) + detachNBD( nbd ) return cow, kernel +def makeVolume( volume, cylinders=1000 ): + """Create volume as a qcow2 and add a single boot partition + cylinders: number of ~8MB (255*63*512) cylinders in volume""" + heads, sectors, bytes = 255, 63, 512 + size = cylinders * heads * sectors * bytes + print '* Creating volume of size', size + run( 'qemu-img create -f qcow2 %s %s' % ( volume, size ) ) + print '* Partitioning volume' + # We need to mount it using qemu-nbd!! + nbd = attachNBD( volume ) + # A bit hacky - we may change this to use parted(8) later + fdisk = Popen( [ 'sudo', 'fdisk', nbd ], stdin=PIPE ) + cmds = 'x\nc\n%d\nr\no\nn\np\n1\n\n\na\n1\nw\n' % cylinders + fdisk.stdin.write( cmds ) + fdisk.wait() + print '* Volume partition table:' + print srun( 'fdisk -l ' + nbd ) + detachNBD( nbd ) + + +def initPartition( partition, volume ): + """Copy partition to volume-p1 and call addMininetUser""" + srcdev = attachNBD( partition, flags='-r' ) + voldev = attachNBD( volume ) + print srun( 'fdisk -l ' + voldev ) + print srun( 'partx ' + voldev ) + dstdev = voldev + 'p1' + print "* Copying partition from", srcdev, "to", dstdev + print srun( 'time dd if=%s of=%s bs=1M' % ( srcdev, dstdev ) ) + print '* Resizing and adding Mininet user' + srun( 'resize2fs ' + dstdev ) + srun( 'e2fsck -y ' + dstdev ) + addMininetUser( dstdev ) + detachNBD( voldev ) + detachNBD( srcdev ) + + def boot( cow, kernel, tap ): """Boot qemu/kvm with a COW disk and local/user data store cow: COW disk path @@ -276,7 +333,7 @@ def boot( cow, kernel, tap ): '-k en-us', '-kernel', kernel, '-drive file=%s,if=virtio' % cow, - '-append "root=/dev/vda init=/sbin/init console=ttyS0" ' ] + '-append "root=/dev/vda1 init=/sbin/init console=ttyS0" ' ] cmd = ' '.join( cmd ) print '* STARTING VM' print cmd @@ -315,7 +372,7 @@ def interact( vm ): vm.expect( prompt ) print '* Testing Mininet' vm.sendline( 'sudo mn --test pingall' ) - if vm.expect( [ ' 0% dropped', pexpect.TIMEOUT ], timeout=30 ): + if vm.expect( [ ' 0% dropped', pexpect.TIMEOUT ], timeout=45 ): print '* Sanity check succeeded' else: print '* Sanity check FAILED' @@ -343,34 +400,37 @@ def cleanup(): def convert( cow, basename ): """Convert a qcow2 disk to a vmdk and put it a new directory basename: base name for output vmdk file""" - dir = mkdtemp( prefix=basename, dir='.' ) - vmdk = '%s/%s.vmdk' % ( dir, basename ) + vmdk = basename + '.vmdk' print '* Converting qcow2 to vmdk' run( 'qemu-img convert -f qcow2 -O vmdk %s %s' % ( cow, vmdk ) ) return vmdk -def build( flavor='raring-server-amd64' ): +def build( flavor='raring32server' ): "Build a Mininet VM" start = time() - cow, kernel = makeCOWDisk( flavor ) - print '* VM image for', flavor, 'created as', cow - with NamedTemporaryFile( - prefix='mn-build-%s-' % flavor, suffix='.log', dir='.' ) as logfile: - print '* Logging results to', logfile.name - vm = boot( cow, kernel, logfile ) - vm.logfile_read = logfile - interact( vm ) - # cow is a temporary file and will go away when we quit! - # We convert it to a .vmdk which can be used in most VMMs - vmdk = convert( cow, basename=flavor ) + dir = mkdtemp( prefix=flavor + '-result-', dir='.' ) + os.chdir( dir ) + print '* Created working directory', dir + image, kernel = fetchImage( flavor ) + volume = flavor + '.qcow2' + makeVolume( volume ) + initPartition( image, volume ) + print '* VM image for', flavor, 'created as', volume + logfile = open( flavor + '.log', 'w+' ) + print '* Logging results to', abspath( logfile.name ) + vm = boot( volume, kernel, logfile ) + vm.logfile_read = logfile + interact( vm ) + vmdk = convert( volume, basename=flavor ) print '* Converted VM image stored as', vmdk end = time() elapsed = end - start - print '* Results logged to', logfile.name + print '* Results logged to', abspath( logfile.name ) print '* Completed in %.2f seconds' % elapsed print '* %s VM build DONE!!!!! :D' % flavor print + os.chdir( '..' ) def parseArgs(): From 14903d6a053022772494d6fb1a40b8cc213d5d56 Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Thu, 22 Aug 2013 18:40:10 -0700 Subject: [PATCH 002/109] Final gasp of cloud image version. --- util/vm/build.py | 240 +++++++++++++++++++++++++++-------------------- 1 file changed, 139 insertions(+), 101 deletions(-) diff --git a/util/vm/build.py b/util/vm/build.py index 9accde1..9b02da0 100755 --- a/util/vm/build.py +++ b/util/vm/build.py @@ -36,41 +36,42 @@ Notes du jour: - and use pexpect to interact with it on the serial console -Something to think about: - -Maybe download the cloud image and customize it so that -it is an actual usable/bootable image??? - More notes: - -We really want a full, partitioned disk image! - -This means we want to use the disk1.image file ??? - -However, this means that we will need to change the grub2 -configuratin to use a serial console. - -/etc/default/grub: - GRUB_TERMINAL=serial - GRUB_SERIAL_COMMAND="serial --unit=0 --speed=38400 --word=8 --parity=no --stop=1" - BOOT_IMAGE="console=ttyS0" -# grub2-mkconfig -o /boot/grub2/grub.cfg +- We use Ubuntu's cloud images, which means that we need to + adapt them for our own (evil) purposes. This isn't ideal + but is the easiest way to get official Ubuntu images until + they start building official non-cloud images. -by the way, we should use wget -c +- We could install grub into a raw ext4 partition rather than + partitioning everything. This would save time and it might + also confuse people who might be expecting a "normal" volume + and who might want to expand it and add more partitions. + On the other hand it makes the file system a lot easier to mount + and modify!! But vmware might not be able to boot it. + +- grub-install fails miserably unless you load part_msdos !! + +- Installing TexLive is just painful - I would like to avoid it + if we could... wireshark plugin build is also slow and painful... + +- Maybe we want to install our own packages for these things... + that would make the whole installation process a lot easier, + but it would mean that we don't automatically get upstream + updates """ import os from os import stat from stat import ST_MODE -from os.path import exists, splitext, abspath, realpath +from os.path import exists, splitext, abspath from sys import exit, argv from glob import glob from urllib import urlretrieve from subprocess import check_output, call, Popen, PIPE from tempfile import mkdtemp -from time import time +from time import time, strftime, localtime import argparse pexpect = None # For code check - imported dynamically @@ -90,10 +91,25 @@ ImageURLBase = { 'raring-server-cloudimg-amd64' } +logStartTime = time() + +def log( *args, **kwargs ): + """Simple log function: log( message along with local and elapsed time + cr: False/0 for no CR""" + cr = kwargs.get( 'cr', True ) + elapsed = time() - logStartTime + clocktime = strftime( '%H:%M:%S', localtime() ) + msg = ' '.join( str( arg ) for arg in args ) + output = '%s [ %.3f ] %s' % ( clocktime, elapsed, msg ) + if cr: + print output + else: + print output, + def run( cmd, **kwargs ): "Convenient interface to check_output" - print cmd + log( '-', cmd ) cmd = cmd.split() return check_output( cmd, **kwargs ) @@ -105,7 +121,7 @@ def srun( cmd, **kwargs ): def depend(): "Install packagedependencies" - print '* Installing package dependencies' + log( '* Installing package dependencies' ) run( 'sudo apt-get -y update' ) run( 'sudo apt-get install -y' ' kvm cloud-utils genisoimage qemu-kvm qemu-utils' @@ -117,7 +133,7 @@ def depend(): def popen( cmd ): "Convenient interface to popen" - print cmd + log( cmd ) cmd = cmd.split() return Popen( cmd ) @@ -148,8 +164,8 @@ def fetchImage( image, path=None ): tgz = path + '.disk1.img' disk = path + '.img' kernel = path + '-vmlinuz-generic' - if exists( disk ) and exists( kernel ): - print '* Found', disk, 'and', kernel + if exists( disk ): + log( '* Found', disk ) # Detect race condition with multiple builds perms = stat( disk )[ ST_MODE ] & 0777 if perms != 0444: @@ -160,13 +176,13 @@ def fetchImage( image, path=None ): run( 'mkdir -p %s' % dir ) if not os.path.exists( tgz ): url = imageURL( image ) + '.tar.gz' - print '* Retrieving', url + log( '* Retrieving', url ) urlretrieve( url, tgz ) - print '* Extracting', tgz + log( '* Extracting', tgz ) run( 'tar -C %s -xzf %s' % ( dir, tgz ) ) # Write-protect disk image so it remains pristine; # We will not use it directly but will use a COW disk - print '* Write-protecting disk image', disk + log( '* Write-protecting disk image', disk ) os.chmod( disk, 0444 ) return disk, kernel @@ -174,22 +190,23 @@ def fetchImage( image, path=None ): def addTo( file, line ): "Add line to file if it's not there already" if call( [ 'sudo', 'grep', line, file ] ) != 0: - call( 'echo "%s" | sudo tee -a %s' % ( line, file ), shell=True ) + call( 'echo "%s" | sudo tee -a %s > /dev/null' % ( line, file ), + shell=True ) def disableCloud( bind ): "Disable cloud junk for disk mounted at bind" - print '* Disabling cloud startup scripts' + log( '* Disabling cloud startup scripts' ) modules = glob( '%s/etc/init/cloud*.conf' % bind ) for module in modules: path, ext = splitext( module ) - override = path + '.override' - call( 'echo manual | sudo tee ' + override, shell=True ) + call( 'echo manual | sudo tee %s.override > /dev/null' % path, + shell=True ) def addMininetUser( nbd ): "Add mininet user/group to filesystem" - print '* Adding mininet user to filesystem on device', nbd + log( '* Adding mininet user to filesystem on device', nbd ) # 1. We bind-mount / into a temporary directory, and # then mount the volume's /etc and /home on top of it! mnt = mkdtemp() @@ -205,8 +222,8 @@ def addMininetUser( nbd ): addTo( bind + '/etc/hosts', '127.0.1.1 mininet-vm' ) # 2. Next, we delete any old mininet user and add a new one chroot( 'deluser mininet' ) - chroot( 'useradd --create-home mininet' ) - print '* Setting password' + chroot( 'useradd --create-home --shell /bin/bash mininet' ) + log( '* Setting password' ) call( 'echo mininet:mininet | sudo chroot %s chpasswd -c SHA512' % bind, shell=True ) # 2a. Add mininet to sudoers @@ -215,7 +232,7 @@ def addMininetUser( nbd ): disableCloud( bind ) chroot( 'sudo update-rc.d landscape-client disable' ) # 2c. Add serial getty - print '* Adding getty on ttyS0' + log( '* Adding getty on ttyS0' ) chroot( 'cp /etc/init/tty1.conf /etc/init/ttyS0.conf' ) chroot( 'sed -i "s/tty1/ttyS0/g" /etc/init/ttyS0.conf' ) # 3. Lastly, we umount and clean up everything @@ -226,8 +243,6 @@ def addMininetUser( nbd ): srun( 'umount ' + mnt ) run( 'rmdir ' + bind ) run( 'rmdir ' + mnt ) - # 4. Just to make sure, we check the filesystem - srun( 'e2fsck -y ' + nbd ) def attachNBD( cow, flags='' ): @@ -235,16 +250,15 @@ def attachNBD( cow, flags='' ): flags: additional flags for qemu-nbd (e.g. -r for readonly)""" # qemu-nbd requires an absolute path cow = abspath( cow ) - print '* Checking for unused /dev/nbdX device ', + log( '* Checking for unused /dev/nbdX device ' ) for i in range ( 0, 63 ): nbd = '/dev/nbd%d' % i - print i, # Check whether someone's already messing with that device if call( [ 'pgrep', '-f', nbd ] ) == 0: continue - # Fails without -v for some annoying reason... - print + srun( 'modprobe nbd max-part=64' ) srun( 'qemu-nbd %s -c %s %s' % ( flags, nbd, cow ) ) + print return nbd raise Exception( "Error: could not find unused /dev/nbdX device" ) @@ -258,11 +272,10 @@ def makeCOWDisk( image, dir='.' ): "Create new COW disk for image" disk, kernel = fetchImage( image ) cow = '%s/%s.qcow2' % ( dir, image ) - print '* Creating COW disk', cow + log( '* Creating COW disk', cow ) run( 'qemu-img create -f qcow2 -b %s %s' % ( disk, cow ) ) - print '* Resizing COW disk and file system' + log( '* Resizing COW disk and file system' ) run( 'qemu-img resize %s +8G' % cow ) - srun( 'modprobe nbd max-part=64') nbd = attachNBD( cow ) srun( 'e2fsck -y ' + nbd ) srun( 'resize2fs ' + nbd ) @@ -271,39 +284,58 @@ def makeCOWDisk( image, dir='.' ): return cow, kernel -def makeVolume( volume, cylinders=1000 ): - """Create volume as a qcow2 and add a single boot partition - cylinders: number of ~8MB (255*63*512) cylinders in volume""" - heads, sectors, bytes = 255, 63, 512 - size = cylinders * heads * sectors * bytes - print '* Creating volume of size', size +def makeVolume( volume, size='8G' ): + """Create volume as a qcow2 and add a single boot partition""" + log( '* Creating volume of size', size ) run( 'qemu-img create -f qcow2 %s %s' % ( volume, size ) ) - print '* Partitioning volume' + log( '* Partitioning volume' ) # We need to mount it using qemu-nbd!! nbd = attachNBD( volume ) - # A bit hacky - we may change this to use parted(8) later - fdisk = Popen( [ 'sudo', 'fdisk', nbd ], stdin=PIPE ) - cmds = 'x\nc\n%d\nr\no\nn\np\n1\n\n\na\n1\nw\n' % cylinders - fdisk.stdin.write( cmds ) - fdisk.wait() - print '* Volume partition table:' - print srun( 'fdisk -l ' + nbd ) + parted = Popen( [ 'sudo', 'parted', nbd ], stdin=PIPE ) + cmds = [ 'mklabel msdos', + 'mkpart primary ext4 1 %s' % size, + 'set 1 boot on', + 'quit' ] + parted.stdin.write( '\n'.join( cmds ) + '\n' ) + parted.wait() + log( '* Volume partition table:' ) + log( srun( 'fdisk -l ' + nbd ) ) detachNBD( nbd ) +def installGrub( voldev, partnum=1 ): + "Install grub2 on voldev to boot from partition partnum" + mnt = mkdtemp() + # Find partitions and make sure we have partition 1 + assert ( '# %d:' % partnum ) in srun( 'partx ' + voldev ) + partdev = voldev + 'p%d' % partnum + srun( 'mount %s %s' % ( partdev, mnt ) ) + # Make sure we have a boot directory + bootdir = mnt + '/boot' + run( 'ls ' + bootdir ) + # Install grub - make sure we preload part_msdos !! + srun( 'grub-install --boot-directory=%s --modules=part_msdos %s' % ( + bootdir, voldev ) ) + srun( 'umount ' + mnt ) + run( 'rmdir ' + mnt ) + + def initPartition( partition, volume ): - """Copy partition to volume-p1 and call addMininetUser""" + """Copy partition to volume-p1 and initialize everything""" srcdev = attachNBD( partition, flags='-r' ) voldev = attachNBD( volume ) - print srun( 'fdisk -l ' + voldev ) - print srun( 'partx ' + voldev ) + log( srun( 'fdisk -l ' + voldev ) ) + log( srun( 'partx ' + voldev ) ) dstdev = voldev + 'p1' - print "* Copying partition from", srcdev, "to", dstdev - print srun( 'time dd if=%s of=%s bs=1M' % ( srcdev, dstdev ) ) - print '* Resizing and adding Mininet user' + log( "* Copying partition from", srcdev, "to", dstdev ) + log( srun( 'dd if=%s of=%s bs=1M' % ( srcdev, dstdev ) ) ) + log( '* Resizing file system' ) srun( 'resize2fs ' + dstdev ) srun( 'e2fsck -y ' + dstdev ) + log( '* Adding mininet user' ) addMininetUser( dstdev ) + log( '* Installing grub2' ) + installGrub( voldev, partnum=1 ) detachNBD( voldev ) detachNBD( srcdev ) @@ -322,7 +354,7 @@ def boot( cow, kernel, tap ): elif 'i386' in kernel: kvm = 'qemu-system-i386' else: - print "Error: can't discern CPU for image", cow + log( "Error: can't discern CPU for image", cow ) exit( 1 ) cmd = [ 'sudo', kvm, '-machine accel=kvm', @@ -335,8 +367,8 @@ def boot( cow, kernel, tap ): '-drive file=%s,if=virtio' % cow, '-append "root=/dev/vda1 init=/sbin/init console=ttyS0" ' ] cmd = ' '.join( cmd ) - print '* STARTING VM' - print cmd + log( '* STARTING VM' ) + log( cmd ) vm = pexpect.spawn( cmd, timeout=TIMEOUT ) return vm @@ -344,64 +376,64 @@ def boot( cow, kernel, tap ): def interact( vm ): "Interact with vm, which is a pexpect object" prompt = '\$ ' - print '* Waiting for login prompt' + log( '* Waiting for login prompt' ) vm.expect( 'login: ' ) - print '* Logging in' + log( '* Logging in' ) vm.sendline( 'mininet' ) - print '* Waiting for password prompt' + log( '* Waiting for password prompt' ) vm.expect( 'Password: ' ) - print '* Sending password' + log( '* Sending password' ) vm.sendline( 'mininet' ) - print '* Waiting for login...' + log( '* Waiting for login...' ) vm.expect( prompt ) - print '* Sending hostname command' + log( '* Sending hostname command' ) vm.sendline( 'hostname' ) - print '* Waiting for output' + log( '* Waiting for output' ) vm.expect( prompt ) - print '* Fetching Mininet VM install script' + log( '* Fetching Mininet VM install script' ) vm.sendline( 'wget ' 'https://raw.github.com/mininet/mininet/master/util/vm/' 'install-mininet-vm.sh' ) vm.expect( prompt ) - print '* Running VM install script' + log( '* Running VM install script' ) vm.sendline( 'bash install-mininet-vm.sh' ) - print '* Waiting for script to complete... ' + log( '* Waiting for script to complete... ' ) # Gigantic timeout for now ;-( vm.expect( 'Done preparing Mininet', timeout=3600 ) - print '* Completed successfully' + log( '* Completed successfully' ) vm.expect( prompt ) - print '* Testing Mininet' + log( '* Testing Mininet' ) vm.sendline( 'sudo mn --test pingall' ) - if vm.expect( [ ' 0% dropped', pexpect.TIMEOUT ], timeout=45 ): - print '* Sanity check succeeded' + if vm.expect( [ ' 0% dropped', pexpect.TIMEOUT ], timeout=45 ) == 0: + log( '* Sanity check succeeded' ) else: - print '* Sanity check FAILED' + log( '* Sanity check FAILED' ) vm.expect( prompt ) - print '* Making sure cgroups are mounted' + log( '* Making sure cgroups are mounted' ) vm.sendline( 'sudo service cgroup-lite restart' ) vm.expect( prompt ) vm.sendline( 'sudo cgroups-mount' ) vm.expect( prompt ) - print '* Running make test' + log( '* Running make test' ) vm.sendline( 'cd ~/mininet; sudo make test' ) vm.expect( prompt ) - print '* Shutting down' + log( '* Shutting down' ) vm.sendline( 'sync; sudo shutdown -h now' ) - print '* Waiting for EOF/shutdown' + log( '* Waiting for EOF/shutdown' ) vm.read() - print '* Interaction complete' + log( '* Interaction complete' ) def cleanup(): "Clean up leftover qemu-nbd processes and other junk" - call( 'sudo pkill -9 qemu-nbd', shell=True ) + call( [ 'sudo', 'pkill', '-9', 'qemu-nbd' ] ) def convert( cow, basename ): """Convert a qcow2 disk to a vmdk and put it a new directory basename: base name for output vmdk file""" vmdk = basename + '.vmdk' - print '* Converting qcow2 to vmdk' + log( '* Converting qcow2 to vmdk' ) run( 'qemu-img convert -f qcow2 -O vmdk %s %s' % ( cow, vmdk ) ) return vmdk @@ -411,28 +443,32 @@ def build( flavor='raring32server' ): start = time() dir = mkdtemp( prefix=flavor + '-result-', dir='.' ) os.chdir( dir ) - print '* Created working directory', dir + log( '* Created working directory', dir ) image, kernel = fetchImage( flavor ) volume = flavor + '.qcow2' makeVolume( volume ) initPartition( image, volume ) - print '* VM image for', flavor, 'created as', volume + log( '* VM image for', flavor, 'created as', volume ) logfile = open( flavor + '.log', 'w+' ) - print '* Logging results to', abspath( logfile.name ) + log( '* Logging results to', abspath( logfile.name ) ) vm = boot( volume, kernel, logfile ) vm.logfile_read = logfile interact( vm ) vmdk = convert( volume, basename=flavor ) - print '* Converted VM image stored as', vmdk + log( '* Converted VM image stored as', vmdk ) end = time() elapsed = end - start - print '* Results logged to', abspath( logfile.name ) - print '* Completed in %.2f seconds' % elapsed - print '* %s VM build DONE!!!!! :D' % flavor - print + log( '* Results logged to', abspath( logfile.name ) ) + log( '* Completed in %.2f seconds' % elapsed ) + log( '* %s VM build DONE!!!!! :D' % flavor ) + log( '* ' ) os.chdir( '..' ) +def listFlavors(): + "List valid build flavors" + print '\nvalid build flavors:', ' '.join( ImageURLBase ), '\n' + def parseArgs(): "Parse command line arguments and run" parser = argparse.ArgumentParser( description='Mininet VM build script' ) @@ -448,17 +484,19 @@ def parseArgs(): if args.depend: depend() if args.list: - print 'valid build flavors:', ' '.join( ImageURLBase ) + listFlavors() if args.clean: cleanup() flavors = args.flavor[ 1: ] for flavor in flavors: if flavor not in ImageURLBase: parser.print_help() + listFlavors() + break # try: build( flavor ) # except Exception as e: - # print '* BUILD FAILED with exception: ', e + # log( '* BUILD FAILED with exception: ', e ) # exit( 1 ) if not ( args.depend or args.list or args.clean or flavors ): parser.print_help() From fa1758b950e229bd61cb08705a74ee0906cd0069 Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Fri, 23 Aug 2013 17:26:46 -0700 Subject: [PATCH 003/109] First draft of new world order (create build image from iso) --- util/vm/build.py | 414 +++++++++++++++++++++-------------------------- 1 file changed, 186 insertions(+), 228 deletions(-) diff --git a/util/vm/build.py b/util/vm/build.py index 9b02da0..26bf746 100755 --- a/util/vm/build.py +++ b/util/vm/build.py @@ -6,11 +6,12 @@ build.py: build a Mininet VM Basic idea: prepare - - download cloud image if it's missing - - write-protect it - + -> create base install image if it's missing + - download iso if it's missing + - install from iso onto image + build - -> create cow disk for vm + -> create cow disk for new VM, based on base image -> boot it in qemu/kvm with text /serial console -> install Mininet @@ -23,72 +24,40 @@ Basic idea: -> shrink-wrap VM -> upload to storage - -Notes du jour: - -- our infrastructure is currently based on 12.04 LTS, so we - can't rely on cloud-localds which is only in 12.10+ - -- as a result, we should download the tar image, extract it, - and boot with tty0 as the console (I think) - -- and we'll manually add the mininet user to it - -- and use pexpect to interact with it on the serial console - -More notes: - -- We use Ubuntu's cloud images, which means that we need to - adapt them for our own (evil) purposes. This isn't ideal - but is the easiest way to get official Ubuntu images until - they start building official non-cloud images. - -- We could install grub into a raw ext4 partition rather than - partitioning everything. This would save time and it might - also confuse people who might be expecting a "normal" volume - and who might want to expand it and add more partitions. - On the other hand it makes the file system a lot easier to mount - and modify!! But vmware might not be able to boot it. - -- grub-install fails miserably unless you load part_msdos !! - -- Installing TexLive is just painful - I would like to avoid it - if we could... wireshark plugin build is also slow and painful... - -- Maybe we want to install our own packages for these things... - that would make the whole installation process a lot easier, - but it would mean that we don't automatically get upstream - updates - """ import os -from os import stat +from os import stat, path from stat import ST_MODE -from os.path import exists, splitext, abspath +from os.path import abspath from sys import exit, argv from glob import glob from urllib import urlretrieve -from subprocess import check_output, call, Popen, PIPE +from subprocess import check_output, call, Popen from tempfile import mkdtemp from time import time, strftime, localtime import argparse pexpect = None # For code check - imported dynamically - # boot can be slooooow!!!! need to debug/optimize somehow TIMEOUT=600 VMImageDir = os.environ[ 'HOME' ] + '/vm-images' -ImageURLBase = { +isoURLs = { + 'quetzal32server': + 'http://mirrors.kernel.org/ubuntu-releases/12.10/' + 'ubuntu-12.10-server-i386.iso', + 'quetzal64server': + 'http://mirrors.kernel.org/ubuntu-releases/13.04/' + 'ubuntu-12.04-server-amd64.iso', 'raring32server': - 'http://cloud-images.ubuntu.com/raring/current/' - 'raring-server-cloudimg-i386', + 'http://mirrors.kernel.org/ubuntu-releases/13.04/' + 'ubuntu-13.04-server-i386.iso', 'raring64server': - 'http://cloud-images.ubuntu.com/raring/current/' - 'raring-server-cloudimg-amd64' + 'http://mirrors.kernel.org/ubuntu-releases/13.04/' + 'ubuntu-13.04-server-amd64.iso', } logStartTime = time() @@ -120,7 +89,7 @@ def srun( cmd, **kwargs ): def depend(): - "Install packagedependencies" + "Install package dependencies" log( '* Installing package dependencies' ) run( 'sudo apt-get -y update' ) run( 'sudo apt-get install -y' @@ -143,111 +112,30 @@ def remove( fname ): return run( 'rm -f %s' % fname ) - -def imageURL( image ): - "Return base URL for VM image" - return ImageURLBase[ image ] - - -def imagePath( image ): - "Return base pathname for VM image files" - url = imageURL( image ) - fname = url.split( '/' )[ -1 ] - path = os.path.join( VMImageDir, fname ) - return path - - -def fetchImage( image, path=None ): - "Fetch base VM image if it's not there already" - if not path: - path = imagePath( image ) - tgz = path + '.disk1.img' - disk = path + '.img' - kernel = path + '-vmlinuz-generic' - if exists( disk ): - log( '* Found', disk ) +def findiso( flavor ): + "Find iso, fetching it if it's not there already" + url = isoURLs[ flavor ] + name = path.basename( url ) + iso = path.join( VMImageDir, name ) + if path.exists( iso ): # Detect race condition with multiple builds - perms = stat( disk )[ ST_MODE ] & 0777 + perms = stat( iso )[ ST_MODE ] & 0777 if perms != 0444: - raise Exception( 'Error - %s is writable ' % disk + + raise Exception( 'Error - %s is writable ' % iso + '; are multiple builds running?' ) else: - dir = os.path.dirname( path ) - run( 'mkdir -p %s' % dir ) - if not os.path.exists( tgz ): - url = imageURL( image ) + '.tar.gz' - log( '* Retrieving', url ) - urlretrieve( url, tgz ) - log( '* Extracting', tgz ) - run( 'tar -C %s -xzf %s' % ( dir, tgz ) ) - # Write-protect disk image so it remains pristine; - # We will not use it directly but will use a COW disk - log( '* Write-protecting disk image', disk ) - os.chmod( disk, 0444 ) - return disk, kernel - - -def addTo( file, line ): - "Add line to file if it's not there already" - if call( [ 'sudo', 'grep', line, file ] ) != 0: - call( 'echo "%s" | sudo tee -a %s > /dev/null' % ( line, file ), - shell=True ) - - -def disableCloud( bind ): - "Disable cloud junk for disk mounted at bind" - log( '* Disabling cloud startup scripts' ) - modules = glob( '%s/etc/init/cloud*.conf' % bind ) - for module in modules: - path, ext = splitext( module ) - call( 'echo manual | sudo tee %s.override > /dev/null' % path, - shell=True ) - - -def addMininetUser( nbd ): - "Add mininet user/group to filesystem" - log( '* Adding mininet user to filesystem on device', nbd ) - # 1. We bind-mount / into a temporary directory, and - # then mount the volume's /etc and /home on top of it! - mnt = mkdtemp() - bind = mkdtemp() - srun( 'mount %s %s' % ( nbd, mnt ) ) - srun( 'mount -B / ' + bind ) - srun( 'mount -B %s/etc %s/etc' % ( mnt, bind ) ) - srun( 'mount -B %s/home %s/home' % ( mnt, bind ) ) - def chroot( cmd ): - "Chroot into bind mount and run command" - call( 'sudo chroot %s ' % bind + cmd, shell=True ) - # 1a. Add hostname entry in /etc/hosts - addTo( bind + '/etc/hosts', '127.0.1.1 mininet-vm' ) - # 2. Next, we delete any old mininet user and add a new one - chroot( 'deluser mininet' ) - chroot( 'useradd --create-home --shell /bin/bash mininet' ) - log( '* Setting password' ) - call( 'echo mininet:mininet | sudo chroot %s chpasswd -c SHA512' - % bind, shell=True ) - # 2a. Add mininet to sudoers - addTo( bind + '/etc/sudoers', 'mininet ALL=NOPASSWD: ALL' ) - # 2b. Disable cloud junk - disableCloud( bind ) - chroot( 'sudo update-rc.d landscape-client disable' ) - # 2c. Add serial getty - log( '* Adding getty on ttyS0' ) - chroot( 'cp /etc/init/tty1.conf /etc/init/ttyS0.conf' ) - chroot( 'sed -i "s/tty1/ttyS0/g" /etc/init/ttyS0.conf' ) - # 3. Lastly, we umount and clean up everything - run( 'sync' ) - srun( 'umount %s/home ' % bind ) - srun( 'umount %s/etc ' % bind ) - srun( 'umount %s' % bind ) - srun( 'umount ' + mnt ) - run( 'rmdir ' + bind ) - run( 'rmdir ' + mnt ) + log( '* Retrieving', url ) + urlretrieve( url, iso ) + # Write-protect iso, signaling it is complete + log( '* Write-protecting iso', iso) + os.chmod( iso, 0444 ) + log( '* Using iso', iso ) + return iso def attachNBD( cow, flags='' ): """Attempt to attach a COW disk image and return its nbd device - flags: additional flags for qemu-nbd (e.g. -r for readonly)""" + flags: additional flags for qemu-nbd (e.g. -r for readonly)""" # qemu-nbd requires an absolute path cow = abspath( cow ) log( '* Checking for unused /dev/nbdX device ' ) @@ -268,94 +156,165 @@ def detachNBD( nbd ): srun( 'qemu-nbd -d ' + nbd ) -def makeCOWDisk( image, dir='.' ): - "Create new COW disk for image" - disk, kernel = fetchImage( image ) - cow = '%s/%s.qcow2' % ( dir, image ) - log( '* Creating COW disk', cow ) - run( 'qemu-img create -f qcow2 -b %s %s' % ( disk, cow ) ) - log( '* Resizing COW disk and file system' ) - run( 'qemu-img resize %s +8G' % cow ) - nbd = attachNBD( cow ) - srun( 'e2fsck -y ' + nbd ) - srun( 'resize2fs ' + nbd ) - addMininetUser( nbd ) - detachNBD( nbd ) - return cow, kernel +def kernelpath( flavor ): + "Return kernel path for flavor" + return path.join( VMImageDir, flavor + '-vmlinuz' ) -def makeVolume( volume, size='8G' ): - """Create volume as a qcow2 and add a single boot partition""" - log( '* Creating volume of size', size ) - run( 'qemu-img create -f qcow2 %s %s' % ( volume, size ) ) - log( '* Partitioning volume' ) - # We need to mount it using qemu-nbd!! - nbd = attachNBD( volume ) - parted = Popen( [ 'sudo', 'parted', nbd ], stdin=PIPE ) - cmds = [ 'mklabel msdos', - 'mkpart primary ext4 1 %s' % size, - 'set 1 boot on', - 'quit' ] - parted.stdin.write( '\n'.join( cmds ) + '\n' ) - parted.wait() - log( '* Volume partition table:' ) - log( srun( 'fdisk -l ' + nbd ) ) - detachNBD( nbd ) - - -def installGrub( voldev, partnum=1 ): - "Install grub2 on voldev to boot from partition partnum" +def extractKernel( image, kernel ): + "Extract kernel from base image" + nbd = attachNBD( image ) + print srun( 'partx ' + nbd ) + # Assume kernel is in partition 1/boot/vmlinuz*generic for now + part = nbd + 'p1' mnt = mkdtemp() - # Find partitions and make sure we have partition 1 - assert ( '# %d:' % partnum ) in srun( 'partx ' + voldev ) - partdev = voldev + 'p%d' % partnum - srun( 'mount %s %s' % ( partdev, mnt ) ) - # Make sure we have a boot directory - bootdir = mnt + '/boot' - run( 'ls ' + bootdir ) - # Install grub - make sure we preload part_msdos !! - srun( 'grub-install --boot-directory=%s --modules=part_msdos %s' % ( - bootdir, voldev ) ) + srun( 'mount %s %s' % ( part, mnt ) ) + kernsrc = glob( '%s/boot/vmlinuz*generic' % mnt )[ 0 ] + run( 'cp %s %s' % ( kernsrc, kernel ) ) srun( 'umount ' + mnt ) run( 'rmdir ' + mnt ) + detachNBD( image ) -def initPartition( partition, volume ): - """Copy partition to volume-p1 and initialize everything""" - srcdev = attachNBD( partition, flags='-r' ) - voldev = attachNBD( volume ) - log( srun( 'fdisk -l ' + voldev ) ) - log( srun( 'partx ' + voldev ) ) - dstdev = voldev + 'p1' - log( "* Copying partition from", srcdev, "to", dstdev ) - log( srun( 'dd if=%s of=%s bs=1M' % ( srcdev, dstdev ) ) ) - log( '* Resizing file system' ) - srun( 'resize2fs ' + dstdev ) - srun( 'e2fsck -y ' + dstdev ) - log( '* Adding mininet user' ) - addMininetUser( dstdev ) - log( '* Installing grub2' ) - installGrub( voldev, partnum=1 ) - detachNBD( voldev ) - detachNBD( srcdev ) +def findBaseImage( flavor, size='8G' ): + "Return base VM image and kernel, creating them if needed" + image = path.join( VMImageDir, flavor + '-base.img' ) + kernel = path.join( VMImageDir, flavor + '-vmlinuz' ) + if path.exists( image ): + # Detect race condition with multiple builds + perms = stat( image )[ ST_MODE ] & 0777 + if perms != 0444: + raise Exception( 'Error - %s is writable ' % image + + '; are multiple builds running?' ) + else: + # We create VMImageDir here since we are called first + run( 'mkdir -p %s' % VMImageDir ) + iso = findiso( flavor ) + log( '* Creating image file', image ) + run( 'qemu-img create %s %s' % ( image, size ) ) + installUbuntu( iso, image ) + log( '* Extracting kernel to', kernel ) + extractKernel( image, kernel ) + # Write-protect image, also signaling it is complete + log( '* Write-protecting image', image) + os.chmod( image, 0444 ) + log( '* Using base image', image ) + return image, kernel -def boot( cow, kernel, tap ): +def makeKickstartFloppy(): + "Create and return kickstart floppy, kickstart, preseed" + kickstart = 'ks.cfg' + kstext = '\n'.join( [ '#Generated by Kickstart Configurator', + '#platform=x86', + '#System language', + 'lang en_US', + '#Language modules to install', + 'langsupport en_US', + '#System keyboard', + 'keyboard us', + '#System mouse', + 'mouse', + '#System timezone', + 'timezone America/Los_Angeles', + '#Root password', + 'rootpw --disabled', + '#Initial user' + 'user mininet --fullname "mininet" --password "mininet"', + '#Use text mode install', + 'text', + '#Install OS instead of upgrade', + 'install', + '#Use CDROM installation media', + 'cdrom', + '#System bootloader configuration', + 'bootloader --location=mbr', + '#Clear the Master Boot Record', + 'zerombr yes', + '#Partition clearing information', + 'clearpart --all --initlabel', + '#Automatic partitioning', + 'autopart', + '#System authorization infomation', + 'auth --useshadow --enablemd5', + '#Firewall configuration', + 'firewall --disabled', + '#Do not configure the X Window System', + 'skipx', '' ] ) + with open( kickstart, 'w' ) as f: + f.write( kstext ) + preseed = 'ks.preseed' + pstext = '\n'.join( [ 'd-i partman/confirm_write_new_label boolean true', + 'd-i partman/choose_partition select finish', + 'd-i partman/confirm boolean true', + 'd-i partman/confirm_nooverwrite boolean true', + 'd-i user-setup/allow-password-weak boolean true' ] ) + with open( preseed, 'w' ) as f: + f.write( pstext ) + # Create floppy and copy files to it + floppy = 'ksfloppy.img' + run( 'qemu-img create %s 1M' % floppy ) + run( 'mcopy -i %s %s ::/' % ( floppy, kickstart ) ) + run( 'mcopy -i %s %s ::/' % ( floppy, preseed ) ) + log( '* Created floppy image %s containing %s and %s' % + ( floppy, kickstart, preseed ) ) + return floppy, kickstart, preseed + + +def kvmFor( name ): + "Guess kvm version for file name" + if 'amd64' in name: + kvm = 'qemu-system-x86_64' + elif 'i386' in name: + kvm = 'qemu-system-i386' + else: + log( "Error: can't discern CPU for file name", name ) + exit( 1 ) + return kvm + + +def installUbuntu( iso, image ): + "Install Ubuntu from iso onto image" + kvm = kvmFor( iso ) + floppy, kickstart, preseed = makeKickstartFloppy() + # Mount iso so we can use its kernel + mnt = mkdtemp() + srun( 'mount %s %s' % ( iso, mnt ) ) + kernel = mnt + 'install/vmlinuz' + cmd = [ 'sudo', kvm, + '-machine accel=kvm', + '-nographic', + '-netdev user,id=mnbuild', + '-device virtio-net,netdev=mnbuild', + '-m 1024', + '-k en-us', + '-cdrom', iso, + '-drive file=%s,if=virtio' % image, + '-fda', floppy, + '-kernel', kernel, + '-append "root=/dev/vda1 init=/sbin/init console=ttyS0' + + 'ks=floppy:/' + kickstart + + 'preseed/file=floppy://' + preseed + '"' ] + cmd = ' '.join( cmd ) + log( '* INSTALLING UBUNTU FROM', iso, 'ONTO', image ) + log( cmd ) + run( cmd ) + # Unmount iso and clean up + srun( 'umount ' + mnt ) + run( 'rmdir ' + mnt ) + log( '* UBUNTU INSTALLATION COMPLETED FOR', image ) + + +def boot( cow, kernel, logfile ): """Boot qemu/kvm with a COW disk and local/user data store cow: COW disk path kernel: kernel path - tap: tap device to connect to VM + logfile: log file for pexpect object returns: pexpect object to qemu process""" # pexpect might not be installed until after depend() is called global pexpect import pexpect - if 'amd64' in kernel: - kvm = 'qemu-system-x86_64' - elif 'i386' in kernel: - kvm = 'qemu-system-i386' - else: - log( "Error: can't discern CPU for image", cow ) - exit( 1 ) + kvm = kvmFor( kernel ) cmd = [ 'sudo', kvm, '-machine accel=kvm', '-nographic', @@ -367,9 +326,9 @@ def boot( cow, kernel, tap ): '-drive file=%s,if=virtio' % cow, '-append "root=/dev/vda1 init=/sbin/init console=ttyS0" ' ] cmd = ' '.join( cmd ) - log( '* STARTING VM' ) + log( '* BOOTING VM FROM', cow ) log( cmd ) - vm = pexpect.spawn( cmd, timeout=TIMEOUT ) + vm = pexpect.spawn( cmd, timeout=TIMEOUT, logfile=logfile ) return vm @@ -444,15 +403,13 @@ def build( flavor='raring32server' ): dir = mkdtemp( prefix=flavor + '-result-', dir='.' ) os.chdir( dir ) log( '* Created working directory', dir ) - image, kernel = fetchImage( flavor ) + image, kernel = findBaseImage( flavor ) volume = flavor + '.qcow2' - makeVolume( volume ) - initPartition( image, volume ) + run( 'qemu-img create -f qcow2 -b %s %s' % ( image, volume ) ) log( '* VM image for', flavor, 'created as', volume ) logfile = open( flavor + '.log', 'w+' ) log( '* Logging results to', abspath( logfile.name ) ) vm = boot( volume, kernel, logfile ) - vm.logfile_read = logfile interact( vm ) vmdk = convert( volume, basename=flavor ) log( '* Converted VM image stored as', vmdk ) @@ -467,7 +424,8 @@ def build( flavor='raring32server' ): def listFlavors(): "List valid build flavors" - print '\nvalid build flavors:', ' '.join( ImageURLBase ), '\n' + print '\nvalid build flavors:', ' '.join( isoURLs ), '\n' + def parseArgs(): "Parse command line arguments and run" @@ -489,7 +447,7 @@ def parseArgs(): cleanup() flavors = args.flavor[ 1: ] for flavor in flavors: - if flavor not in ImageURLBase: + if flavor not in isoURLs: parser.print_help() listFlavors() break From f605a4e430fcc4188aa1dd55fd642c80cc49f516 Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Fri, 23 Aug 2013 20:33:25 -0700 Subject: [PATCH 004/109] Works, more or less. --- util/vm/build.py | 210 +++++++++++++++++++++++++++-------------------- 1 file changed, 121 insertions(+), 89 deletions(-) diff --git a/util/vm/build.py b/util/vm/build.py index 26bf746..fd7325d 100755 --- a/util/vm/build.py +++ b/util/vm/build.py @@ -32,7 +32,6 @@ from stat import ST_MODE from os.path import abspath from sys import exit, argv from glob import glob -from urllib import urlretrieve from subprocess import check_output, call, Popen from tempfile import mkdtemp from time import time, strftime, localtime @@ -117,15 +116,9 @@ def findiso( flavor ): url = isoURLs[ flavor ] name = path.basename( url ) iso = path.join( VMImageDir, name ) - if path.exists( iso ): - # Detect race condition with multiple builds - perms = stat( iso )[ ST_MODE ] & 0777 - if perms != 0444: - raise Exception( 'Error - %s is writable ' % iso + - '; are multiple builds running?' ) - else: + if not path.exists( iso ) or ( stat( iso )[ ST_MODE ] & 0777 != 0444 ): log( '* Retrieving', url ) - urlretrieve( url, iso ) + run( 'curl -C - -o %s %s' % ( iso, url ) ) # Write-protect iso, signaling it is complete log( '* Write-protecting iso', iso) os.chmod( iso, 0444 ) @@ -156,30 +149,30 @@ def detachNBD( nbd ): srun( 'qemu-nbd -d ' + nbd ) -def kernelpath( flavor ): - "Return kernel path for flavor" - return path.join( VMImageDir, flavor + '-vmlinuz' ) - - -def extractKernel( image, kernel ): +def extractKernel( image, flavor ): "Extract kernel from base image" - nbd = attachNBD( image ) + kernel = path.join( VMImageDir, flavor + '-vmlinuz' ) + if path.exists( kernel ) and ( stat( image )[ ST_MODE ] & 0777 ) == 0444: + return kernel + log( '* Extracting kernel to', kernel ) + nbd = attachNBD( image, flags='-r' ) print srun( 'partx ' + nbd ) # Assume kernel is in partition 1/boot/vmlinuz*generic for now part = nbd + 'p1' mnt = mkdtemp() srun( 'mount %s %s' % ( part, mnt ) ) kernsrc = glob( '%s/boot/vmlinuz*generic' % mnt )[ 0 ] - run( 'cp %s %s' % ( kernsrc, kernel ) ) + run( 'sudo cp %s %s' % ( kernsrc, kernel ) ) + run( 'sudo chmod 0444 ' + kernel ) srun( 'umount ' + mnt ) run( 'rmdir ' + mnt ) - detachNBD( image ) + detachNBD( nbd ) + return kernel def findBaseImage( flavor, size='8G' ): "Return base VM image and kernel, creating them if needed" image = path.join( VMImageDir, flavor + '-base.img' ) - kernel = path.join( VMImageDir, flavor + '-vmlinuz' ) if path.exists( image ): # Detect race condition with multiple builds perms = stat( image )[ ST_MODE ] & 0777 @@ -193,79 +186,104 @@ def findBaseImage( flavor, size='8G' ): log( '* Creating image file', image ) run( 'qemu-img create %s %s' % ( image, size ) ) installUbuntu( iso, image ) - log( '* Extracting kernel to', kernel ) - extractKernel( image, kernel ) # Write-protect image, also signaling it is complete log( '* Write-protecting image', image) os.chmod( image, 0444 ) - log( '* Using base image', image ) + kernel = extractKernel( image, flavor ) + log( '* Using base image', image, 'and kernel', kernel ) return image, kernel +# Kickstart and Preseed files for Ubuntu/Debian installer +# +# Comments: this is really clunky and painful. If Ubuntu +# gets their act together and supports kickstart a bit better +# then we can get rid of preseed and even use this as a +# Fedora installer as well. +# +# Another annoying thing about Ubuntu is that it can't just +# install a normal system from the iso - it has to download +# junk from the internet, making this house of cards even +# more precarious. + +KickstartText =""" +#Generated by Kickstart Configurator +#platform=x86 + +#System language +lang en_US +#Language modules to install +langsupport en_US +#System keyboard +keyboard us +#System mouse +mouse +#System timezone +timezone America/Los_Angeles +#Root password +rootpw --disabled +#Initial user +user mininet --fullname "mininet" --password "mininet" +#Use text mode install +text +#Install OS instead of upgrade +install +#Use CDROM installation media +cdrom +#System bootloader configuration +bootloader --location=mbr +#Clear the Master Boot Record +zerombr yes +#Partition clearing information +clearpart --all --initlabel +#Automatic partitioning +autopart +#System authorization infomation +auth --useshadow --enablemd5 +#Firewall configuration +firewall --disabled +#Do not configure the X Window System +skipx +""" + +# Tell the Ubuntu/Debian installer to stop asking stupid questions + +PreseedText = """ +d-i mirror/country string manual +d-i mirror/http/hostname string mirrors.kernel.org +d-i mirror/http/directory string /ubuntu +d-i mirror/http/proxy string +d-i partman/confirm_write_new_label boolean true +d-i partman/choose_partition select finish +d-i partman/confirm boolean true +d-i partman/confirm_nooverwrite boolean true +d-i user-setup/allow-password-weak boolean true +d-i finish-install/reboot_in_progress note +d-i debian-installer/exit/poweroff boolean true +""" + def makeKickstartFloppy(): "Create and return kickstart floppy, kickstart, preseed" kickstart = 'ks.cfg' - kstext = '\n'.join( [ '#Generated by Kickstart Configurator', - '#platform=x86', - '#System language', - 'lang en_US', - '#Language modules to install', - 'langsupport en_US', - '#System keyboard', - 'keyboard us', - '#System mouse', - 'mouse', - '#System timezone', - 'timezone America/Los_Angeles', - '#Root password', - 'rootpw --disabled', - '#Initial user' - 'user mininet --fullname "mininet" --password "mininet"', - '#Use text mode install', - 'text', - '#Install OS instead of upgrade', - 'install', - '#Use CDROM installation media', - 'cdrom', - '#System bootloader configuration', - 'bootloader --location=mbr', - '#Clear the Master Boot Record', - 'zerombr yes', - '#Partition clearing information', - 'clearpart --all --initlabel', - '#Automatic partitioning', - 'autopart', - '#System authorization infomation', - 'auth --useshadow --enablemd5', - '#Firewall configuration', - 'firewall --disabled', - '#Do not configure the X Window System', - 'skipx', '' ] ) with open( kickstart, 'w' ) as f: - f.write( kstext ) + f.write( KickstartText ) preseed = 'ks.preseed' - pstext = '\n'.join( [ 'd-i partman/confirm_write_new_label boolean true', - 'd-i partman/choose_partition select finish', - 'd-i partman/confirm boolean true', - 'd-i partman/confirm_nooverwrite boolean true', - 'd-i user-setup/allow-password-weak boolean true' ] ) with open( preseed, 'w' ) as f: - f.write( pstext ) + f.write( PreseedText ) # Create floppy and copy files to it floppy = 'ksfloppy.img' - run( 'qemu-img create %s 1M' % floppy ) + run( 'qemu-img create %s 1440k' % floppy ) + run( 'mkfs -t msdos ' + floppy ) run( 'mcopy -i %s %s ::/' % ( floppy, kickstart ) ) run( 'mcopy -i %s %s ::/' % ( floppy, preseed ) ) - log( '* Created floppy image %s containing %s and %s' % - ( floppy, kickstart, preseed ) ) return floppy, kickstart, preseed def kvmFor( name ): "Guess kvm version for file name" - if 'amd64' in name: + if '64' in name: kvm = 'qemu-system-x86_64' - elif 'i386' in name: + elif 'i386' in name or '32' in name: kvm = 'qemu-system-i386' else: log( "Error: can't discern CPU for file name", name ) @@ -273,36 +291,49 @@ def kvmFor( name ): return kvm -def installUbuntu( iso, image ): +def installUbuntu( iso, image, logfilename='install.log' ): "Install Ubuntu from iso onto image" + global pexpect + import pexpect kvm = kvmFor( iso ) floppy, kickstart, preseed = makeKickstartFloppy() # Mount iso so we can use its kernel mnt = mkdtemp() srun( 'mount %s %s' % ( iso, mnt ) ) - kernel = mnt + 'install/vmlinuz' + srun( 'ls ' + mnt ) + kernel = path.join( mnt, 'install/vmlinuz' ) + initrd = path.join( mnt, 'install/initrd.gz' ) cmd = [ 'sudo', kvm, - '-machine accel=kvm', + '-machine', 'accel=kvm', '-nographic', - '-netdev user,id=mnbuild', - '-device virtio-net,netdev=mnbuild', - '-m 1024', - '-k en-us', - '-cdrom', iso, - '-drive file=%s,if=virtio' % image, + '-netdev', 'user,id=mnbuild', + '-device', 'virtio-net,netdev=mnbuild', + '-m', '1024', + '-k', 'en-us', '-fda', floppy, + '-drive', 'file=%s,if=virtio' % image, + '-cdrom', iso, '-kernel', kernel, - '-append "root=/dev/vda1 init=/sbin/init console=ttyS0' + - 'ks=floppy:/' + kickstart + - 'preseed/file=floppy://' + preseed + '"' ] - cmd = ' '.join( cmd ) + '-initrd', initrd, + '-append', + ' ks=floppy:/' + kickstart + + ' preseed/file=floppy://' + preseed + + ' console=ttyS0' ] + ubuntuStart = time() log( '* INSTALLING UBUNTU FROM', iso, 'ONTO', image ) - log( cmd ) - run( cmd ) + log( ' '.join( cmd ) ) + log( '* logging to', abspath( logfilename ) ) + logfile = open( logfilename, 'w' ) + vm = Popen( cmd, stdout=logfile, stderr=logfile ) + log( '* Waiting for installation to complete') + vm.wait() + logfile.close() + elapsed = time() - ubuntuStart # Unmount iso and clean up srun( 'umount ' + mnt ) run( 'rmdir ' + mnt ) log( '* UBUNTU INSTALLATION COMPLETED FOR', image ) + log( '* Ubuntu installation completed in %.2f seconds ' % elapsed ) def boot( cow, kernel, logfile ): @@ -356,6 +387,8 @@ def interact( vm ): vm.expect( prompt ) log( '* Running VM install script' ) vm.sendline( 'bash install-mininet-vm.sh' ) + vm.expect ( 'password for mininet: ' ) + vm.sendline( 'mininet' ) log( '* Waiting for script to complete... ' ) # Gigantic timeout for now ;-( vm.expect( 'Done preparing Mininet', timeout=3600 ) @@ -380,7 +413,7 @@ def interact( vm ): vm.sendline( 'sync; sudo shutdown -h now' ) log( '* Waiting for EOF/shutdown' ) vm.read() - log( '* Interaction complete' ) + log( '* Interaction complete' ) def cleanup(): @@ -412,14 +445,13 @@ def build( flavor='raring32server' ): vm = boot( volume, kernel, logfile ) interact( vm ) vmdk = convert( volume, basename=flavor ) - log( '* Converted VM image stored as', vmdk ) + log( '* Converted VM image stored as', abspath( vmdk ) ) end = time() elapsed = end - start log( '* Results logged to', abspath( logfile.name ) ) log( '* Completed in %.2f seconds' % elapsed ) log( '* %s VM build DONE!!!!! :D' % flavor ) - log( '* ' ) - os.chdir( '..' ) + os.chdir( '..' ) def listFlavors(): From 4556e06fcf40d5ee5374cf7e4427960923109625 Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Fri, 23 Aug 2013 21:25:07 -0700 Subject: [PATCH 005/109] Fix erroneous tab hit before commit. --- util/vm/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/vm/build.py b/util/vm/build.py index fd7325d..b613eac 100755 --- a/util/vm/build.py +++ b/util/vm/build.py @@ -413,7 +413,7 @@ def interact( vm ): vm.sendline( 'sync; sudo shutdown -h now' ) log( '* Waiting for EOF/shutdown' ) vm.read() - log( '* Interaction complete' ) + log( '* Interaction complete' ) def cleanup(): From bbf808c347de96829115b1ae50d82d973afcf99e Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Fri, 23 Aug 2013 21:27:01 -0700 Subject: [PATCH 006/109] Get rid of unused pexpect import. --- util/vm/build.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/util/vm/build.py b/util/vm/build.py index b613eac..21ff92e 100755 --- a/util/vm/build.py +++ b/util/vm/build.py @@ -293,8 +293,6 @@ def kvmFor( name ): def installUbuntu( iso, image, logfilename='install.log' ): "Install Ubuntu from iso onto image" - global pexpect - import pexpect kvm = kvmFor( iso ) floppy, kickstart, preseed = makeKickstartFloppy() # Mount iso so we can use its kernel @@ -451,7 +449,7 @@ def build( flavor='raring32server' ): log( '* Results logged to', abspath( logfile.name ) ) log( '* Completed in %.2f seconds' % elapsed ) log( '* %s VM build DONE!!!!! :D' % flavor ) - os.chdir( '..' ) + os.chdir( '..' ) def listFlavors(): From 3dc3e066aa7cfdec68f9fcbedcc5275660ab0545 Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Sat, 24 Aug 2013 13:46:00 -0700 Subject: [PATCH 007/109] Update build directory name to include date. --- util/vm/build.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/util/vm/build.py b/util/vm/build.py index 21ff92e..2296518 100755 --- a/util/vm/build.py +++ b/util/vm/build.py @@ -279,8 +279,9 @@ def makeKickstartFloppy(): return floppy, kickstart, preseed -def kvmFor( name ): - "Guess kvm version for file name" +def kvmFor( path ): + "Guess kvm version for file path" + name = path.basename( path ) if '64' in name: kvm = 'qemu-system-x86_64' elif 'i386' in name or '32' in name: @@ -431,7 +432,8 @@ def convert( cow, basename ): def build( flavor='raring32server' ): "Build a Mininet VM" start = time() - dir = mkdtemp( prefix=flavor + '-result-', dir='.' ) + date = time.strftime( '%y%m%d-%H:%M:%S', time.localtime()) + dir = os.mkdir( 'mn-' + flavor + date ) os.chdir( dir ) log( '* Created working directory', dir ) image, kernel = findBaseImage( flavor ) From 40a9c153453d781634f37951e3ba5083b5353b2a Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Sun, 25 Aug 2013 13:55:50 -0700 Subject: [PATCH 008/109] Remove gigantic doxypy/texlive/fonts from install.sh -a fixes #192 --- util/install.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/util/install.sh b/util/install.sh index 4873a8e..5ffbc92 100755 --- a/util/install.sh +++ b/util/install.sh @@ -607,13 +607,16 @@ function modprobe { } function all { - echo "Running all commands..." + echo "Installing all packages except for -eix (doxypy, ivs, nox-classic)..." kernel mn_deps - mn_dev + # Skip mn_dev (doxypy/texlive/fonts/etc.) because it's huge + # mn_dev of wireshark ovs + # We may add ivs once it's more mature + # ivs # NOX-classic is deprecated, but you can install it manually if desired. # nox pox From 1dfa7776e1b43649a71844cd3c64af5baf9a3934 Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Sun, 25 Aug 2013 15:25:04 -0700 Subject: [PATCH 009/109] Change to extract kernel and initrd --- util/vm/build.py | 42 ++++++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/util/vm/build.py b/util/vm/build.py index 2296518..d56b8ef 100755 --- a/util/vm/build.py +++ b/util/vm/build.py @@ -150,24 +150,29 @@ def detachNBD( nbd ): def extractKernel( image, flavor ): - "Extract kernel from base image" + "Extract kernel and initrd from base image" kernel = path.join( VMImageDir, flavor + '-vmlinuz' ) + initrd = path.join( VMImageDir, flavor + '-initrd' ) if path.exists( kernel ) and ( stat( image )[ ST_MODE ] & 0777 ) == 0444: - return kernel + # If kernel is there, then initrd should also be there + return kernel, initrd log( '* Extracting kernel to', kernel ) nbd = attachNBD( image, flags='-r' ) print srun( 'partx ' + nbd ) # Assume kernel is in partition 1/boot/vmlinuz*generic for now part = nbd + 'p1' mnt = mkdtemp() - srun( 'mount %s %s' % ( part, mnt ) ) + srun( 'mount -o ro %s %s' % ( part, mnt ) ) kernsrc = glob( '%s/boot/vmlinuz*generic' % mnt )[ 0 ] - run( 'sudo cp %s %s' % ( kernsrc, kernel ) ) - run( 'sudo chmod 0444 ' + kernel ) + initrdsrc = glob( '%s/boot/initrd*generic' % mnt )[ 0 ] + srun( 'cp %s %s' % ( initrdsrc, initrd ) ) + srun( 'chmod 0444 ' + initrd ) + srun( 'cp %s %s' % ( kernsrc, kernel ) ) + srun( 'chmod 0444 ' + kernel ) srun( 'umount ' + mnt ) run( 'rmdir ' + mnt ) detachNBD( nbd ) - return kernel + return kernel, initrd def findBaseImage( flavor, size='8G' ): @@ -189,9 +194,9 @@ def findBaseImage( flavor, size='8G' ): # Write-protect image, also signaling it is complete log( '* Write-protecting image', image) os.chmod( image, 0444 ) - kernel = extractKernel( image, flavor ) + kernel, initrd = extractKernel( image, flavor ) log( '* Using base image', image, 'and kernel', kernel ) - return image, kernel + return image, kernel, initrd # Kickstart and Preseed files for Ubuntu/Debian installer @@ -279,9 +284,9 @@ def makeKickstartFloppy(): return floppy, kickstart, preseed -def kvmFor( path ): +def kvmFor( filepath ): "Guess kvm version for file path" - name = path.basename( path ) + name = path.basename( filepath ) if '64' in name: kvm = 'qemu-system-x86_64' elif 'i386' in name or '32' in name: @@ -335,7 +340,7 @@ def installUbuntu( iso, image, logfilename='install.log' ): log( '* Ubuntu installation completed in %.2f seconds ' % elapsed ) -def boot( cow, kernel, logfile ): +def boot( cow, kernel, initrd, logfile ): """Boot qemu/kvm with a COW disk and local/user data store cow: COW disk path kernel: kernel path @@ -353,6 +358,7 @@ def boot( cow, kernel, logfile ): '-m 1024', '-k en-us', '-kernel', kernel, + '-initrd', initrd, '-drive file=%s,if=virtio' % cow, '-append "root=/dev/vda1 init=/sbin/init console=ttyS0" ' ] cmd = ' '.join( cmd ) @@ -432,17 +438,21 @@ def convert( cow, basename ): def build( flavor='raring32server' ): "Build a Mininet VM" start = time() - date = time.strftime( '%y%m%d-%H:%M:%S', time.localtime()) - dir = os.mkdir( 'mn-' + flavor + date ) + date = strftime( '%y%m%d-%H-%M-%S', localtime()) + dir = 'mn-%s-%s' % ( flavor, date ) + try: + os.mkdir( dir ) + except: + raise Exception( "Failed to create build directory %s" % dir ) os.chdir( dir ) log( '* Created working directory', dir ) - image, kernel = findBaseImage( flavor ) + image, kernel, initrd = findBaseImage( flavor ) volume = flavor + '.qcow2' run( 'qemu-img create -f qcow2 -b %s %s' % ( image, volume ) ) log( '* VM image for', flavor, 'created as', volume ) logfile = open( flavor + '.log', 'w+' ) log( '* Logging results to', abspath( logfile.name ) ) - vm = boot( volume, kernel, logfile ) + vm = boot( volume, kernel, initrd, logfile ) interact( vm ) vmdk = convert( volume, basename=flavor ) log( '* Converted VM image stored as', abspath( vmdk ) ) @@ -463,7 +473,7 @@ def parseArgs(): "Parse command line arguments and run" parser = argparse.ArgumentParser( description='Mininet VM build script' ) parser.add_argument( '--depend', action='store_true', - help='Install dependencies for this script' ) + help='install dependencies for this script' ) parser.add_argument( '--list', action='store_true', help='list valid build flavors' ) parser.add_argument( '--clean', action='store_true', From dbcfda77d9a6c1bba4ac85f3bd3a56894be9fa9c Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Sun, 25 Aug 2013 15:51:56 -0700 Subject: [PATCH 010/109] Update release URLS - should probably clean this up. --- util/vm/build.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/util/vm/build.py b/util/vm/build.py index d56b8ef..07c99e8 100755 --- a/util/vm/build.py +++ b/util/vm/build.py @@ -45,12 +45,18 @@ TIMEOUT=600 VMImageDir = os.environ[ 'HOME' ] + '/vm-images' isoURLs = { + 'precise32server': + 'http://mirrors.kernel.org/ubuntu-releases/12.04/' + 'ubuntu-12.04-server-i386.iso', + 'precise64server': + 'http://mirrors.kernel.org/ubuntu-releases/12.04/' + 'ubuntu-12.04-server-amd64.iso', 'quetzal32server': 'http://mirrors.kernel.org/ubuntu-releases/12.10/' 'ubuntu-12.10-server-i386.iso', 'quetzal64server': - 'http://mirrors.kernel.org/ubuntu-releases/13.04/' - 'ubuntu-12.04-server-amd64.iso', + 'http://mirrors.kernel.org/ubuntu-releases/12.10/' + 'ubuntu-12.10-server-amd64.iso', 'raring32server': 'http://mirrors.kernel.org/ubuntu-releases/13.04/' 'ubuntu-13.04-server-i386.iso', @@ -500,6 +506,7 @@ def parseArgs(): # exit( 1 ) if not ( args.depend or args.list or args.clean or flavors ): parser.print_help() + listFlavors() if __name__ == '__main__': parseArgs() From 28165f7b4e54a8b4d8e1139d317bf06961515e84 Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Sun, 25 Aug 2013 16:27:43 -0700 Subject: [PATCH 011/109] Check `make test` results --- util/vm/build.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/util/vm/build.py b/util/vm/build.py index 07c99e8..4cd257f 100755 --- a/util/vm/build.py +++ b/util/vm/build.py @@ -408,7 +408,7 @@ def interact( vm ): log( '* Testing Mininet' ) vm.sendline( 'sudo mn --test pingall' ) if vm.expect( [ ' 0% dropped', pexpect.TIMEOUT ], timeout=45 ) == 0: - log( '* Sanity check succeeded' ) + log( '* Sanity check OK' ) else: log( '* Sanity check FAILED' ) vm.expect( prompt ) @@ -419,6 +419,15 @@ def interact( vm ): vm.expect( prompt ) log( '* Running make test' ) vm.sendline( 'cd ~/mininet; sudo make test' ) + # We should change "make test" to report the number of + # successful and failed tests. For now, we have to + # know the time for each test, which means that this + # script will have to change as we add more tests. + for test in range( 0, 2 ): + if vm.expect( [ 'OK', 'FAILED', pexpect.timeout ], timeout=60 ) == 0: + log( '* Test', test, 'OK' ) + else: + log( '* Test', test, 'FAILED' ) vm.expect( prompt ) log( '* Shutting down' ) vm.sendline( 'sync; sudo shutdown -h now' ) From 662fb712bc07cf6586a558a9e61c1780facde35c Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Sun, 25 Aug 2013 17:42:15 -0700 Subject: [PATCH 012/109] Detect failed iso download; begin virt-image support --- util/vm/build.py | 50 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/util/vm/build.py b/util/vm/build.py index 4cd257f..b874e05 100755 --- a/util/vm/build.py +++ b/util/vm/build.py @@ -35,6 +35,7 @@ from glob import glob from subprocess import check_output, call, Popen from tempfile import mkdtemp from time import time, strftime, localtime +from lxml import etree import argparse pexpect = None # For code check - imported dynamically @@ -47,10 +48,10 @@ VMImageDir = os.environ[ 'HOME' ] + '/vm-images' isoURLs = { 'precise32server': 'http://mirrors.kernel.org/ubuntu-releases/12.04/' - 'ubuntu-12.04-server-i386.iso', + 'ubuntu-12.04.3-server-i386.iso', 'precise64server': 'http://mirrors.kernel.org/ubuntu-releases/12.04/' - 'ubuntu-12.04-server-amd64.iso', + 'ubuntu-12.04.3-server-amd64.iso', 'quetzal32server': 'http://mirrors.kernel.org/ubuntu-releases/12.10/' 'ubuntu-12.10-server-i386.iso', @@ -125,6 +126,9 @@ def findiso( flavor ): if not path.exists( iso ) or ( stat( iso )[ ST_MODE ] & 0777 != 0444 ): log( '* Retrieving', url ) run( 'curl -C - -o %s %s' % ( iso, url ) ) + if 'ISO' not in run( 'file ' + iso ): + os.remove( iso ) + raise Exception( 'findiso: could not download iso from ' + url ) # Write-protect iso, signaling it is complete log( '* Write-protecting iso', iso) os.chmod( iso, 0444 ) @@ -310,7 +314,6 @@ def installUbuntu( iso, image, logfilename='install.log' ): # Mount iso so we can use its kernel mnt = mkdtemp() srun( 'mount %s %s' % ( iso, mnt ) ) - srun( 'ls ' + mnt ) kernel = path.join( mnt, 'install/vmlinuz' ) initrd = path.join( mnt, 'install/initrd.gz' ) cmd = [ 'sudo', kvm, @@ -450,6 +453,47 @@ def convert( cow, basename ): return vmdk +# Template for virt-image(5) file + +VirtImageXML = """ + + + %s + + + + %s/arch> + + + + + + + + 1 + %s + + + + + + + + +""" + +def genVirtImage( name, mem, diskname, disksize ): + "Generate and return virt-image file name.xml" + # Our strategy is going to be: create a + # virt-image file and then use virt-convert to convert + # it to an .ovf file + xmlfile = name + '.xml' + xmltext = VirtImageXML % ( name, mem, diskname, disksize ) + with open( xmlfile, 'w+' ) as f: + f.write( xmltext ) + return xmlfile + + def build( flavor='raring32server' ): "Build a Mininet VM" start = time() From c353e6091380d48c0388565e3b08a327b3146a92 Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Sun, 25 Aug 2013 18:06:22 -0700 Subject: [PATCH 013/109] correction: pexpect.timeout -> TIMEOUT --- util/vm/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/vm/build.py b/util/vm/build.py index b874e05..22683c9 100755 --- a/util/vm/build.py +++ b/util/vm/build.py @@ -427,7 +427,7 @@ def interact( vm ): # know the time for each test, which means that this # script will have to change as we add more tests. for test in range( 0, 2 ): - if vm.expect( [ 'OK', 'FAILED', pexpect.timeout ], timeout=60 ) == 0: + if vm.expect( [ 'OK', 'FAILED', pexpect.TIMEOUT ], timeout=60 ) == 0: log( '* Test', test, 'OK' ) else: log( '* Test', test, 'FAILED' ) From 67f9d8f655db542a719a16de7d4d324820860b35 Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Mon, 26 Aug 2013 11:46:25 -0700 Subject: [PATCH 014/109] Remove qcow2 post conversion; drop unused etree dep --- util/vm/build.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/util/vm/build.py b/util/vm/build.py index 22683c9..d27d129 100755 --- a/util/vm/build.py +++ b/util/vm/build.py @@ -35,7 +35,6 @@ from glob import glob from subprocess import check_output, call, Popen from tempfile import mkdtemp from time import time, strftime, localtime -from lxml import etree import argparse pexpect = None # For code check - imported dynamically @@ -514,6 +513,8 @@ def build( flavor='raring32server' ): vm = boot( volume, kernel, initrd, logfile ) interact( vm ) vmdk = convert( volume, basename=flavor ) + log( '* Removing qcow2 volume', volume ) + os.remove( volume ) log( '* Converted VM image stored as', abspath( vmdk ) ) end = time() elapsed = end - start From d13505b6c6ef5b5db87b4d850e819b38c6a98881 Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Mon, 26 Aug 2013 15:29:05 -0700 Subject: [PATCH 015/109] updating setup to include examples --- mininet/examples | 1 + setup.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 120000 mininet/examples diff --git a/mininet/examples b/mininet/examples new file mode 120000 index 0000000..a6573af --- /dev/null +++ b/mininet/examples @@ -0,0 +1 @@ +../examples \ No newline at end of file diff --git a/setup.py b/setup.py index 9cee655..c65f5c3 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ setup( description='Process-based OpenFlow emulator', author='Bob Lantz', author_email='rlantz@cs.stanford.edu', - packages=find_packages(exclude='test'), + packages=[ 'mininet', 'mininet.examples' ], long_description=""" Mininet is a network emulator which uses lightweight virtualization to create virtual networks for rapid From b26f38a6aadd359068693d898cddb9a5cb12609c Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Mon, 26 Aug 2013 13:39:05 -0700 Subject: [PATCH 016/109] Added CONTRIBUTORS file --- CONTRIBUTORS | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 CONTRIBUTORS diff --git a/CONTRIBUTORS b/CONTRIBUTORS new file mode 100644 index 0000000..ab20638 --- /dev/null +++ b/CONTRIBUTORS @@ -0,0 +1,39 @@ +Mininet Contributors + +Mininet is an open source project and we gratefully acknowledge +the many contributions to the project! If you have contributed +code to the project and are not on this list, please let us know +or send a pull request. + +Contributors include: + +Mininet Core Team + +Bob Lantz +Brandon Heller +Nikhil Handigol +Vimal Jeyakumar +Brian O'Connor + +Additional Mininet Contributors + +Gustavo Pantuza Coelho Pinto +Ryan Cox +Suaun Crampton +David Erickson +Glen Gibb +Andrew Ferguson +Eder Leao Fernandes +Vitaly Ivanov +Rich Lane +Murphy McCauley +James Page +Angad Singh +Piyush Srivastava +Ed Swierk +Isaku Yamahata + +Thanks also to everyone who has submitted issues and pull +requests on github, and to our friendly mininet-discuss +mailing list! + From 325074981ca6d39cbab2369e09c57aa4cf2c9887 Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Mon, 26 Aug 2013 13:48:55 -0700 Subject: [PATCH 017/109] Initial text and version updates for 2.1.0 --- LICENSE | 4 ++-- README.md | 58 ++++++++++++++++++++++++-------------------------- mininet/net.py | 2 +- 3 files changed, 31 insertions(+), 33 deletions(-) diff --git a/LICENSE b/LICENSE index 704d157..de9b391 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ -Mininet 2.0.0 License +Mininet 2.1.0 License -Copyright (c) 2012 Open Networking Laboratory +Copyright (c) 2013 Open Networking Laboratory Copyright (c) 2009-2012 Bob Lantz and The Board of Trustees of The Leland Stanford Junior University diff --git a/README.md b/README.md index 724d5fa..fce180f 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Mininet: Rapid Prototyping for Software Defined Networks *The best way to emulate almost any network on your laptop!* -Version 2.0.0 +Version 2.1.0 ### What is Mininet? @@ -67,32 +67,30 @@ Mininet includes: `mn -c` -### New features in 2.0.0 +### New features in 2.1.0 -Mininet 2.0.0 is a major upgrade and provides -a number of enhancements and new features, including: +Mininet 2.1.0 provides a number of bug fixes as well as a +number of new features, including: -* "Mininet-HiFi" functionality: +* Convenient access to Mininet() as a dict +* X11 tunneling (wireshark in Mininet hosts!) +* Accurate reflection of the Mininet() object in the CLI +* Automatically detecting and adjusting resource limits +* Automatic cleanup on failure of the `mn` command +* Support for running OVS in user space mode +* Preliminary support for the Indigo Virtual Switch (IVSSwitch) +* The ability to import examples as modules - * Link bandwidth limits using `tc` (`TCIntf` and `TCLink` classes) +We have provided several new examples which can also be +imported to provide useful functionality, including: - * CPU isolation and bandwidth limits (`CPULimitedHost` class) +* A simple NAT script to attach Mininet networks to your LAN +* An example of modeling control and data networks +* An example of per-host custom directories using bind mounts -* Support for Open vSwitch 1.4+ (including Ubuntu OVS packages) - -* Debian packaging (and `apt-get install mininet` in Ubuntu 12.10) - -* First-class Interface (`Intf`) and Link (`Link`) classes for easier - extensibility - -* An upgraded Topology (`Topo`) class which supports node and link - customization - -* Man pages for the `mn` and `mnexec` utilities. - -[Since the API (most notably the topology) has changed, existing code -that runs in Mininet 1.0 will need to be changed to run with Mininet -2.0. This is the primary reason for the major version number change.] +Note that these are experimental features which may "graduate" +into mainline Mininet in the future, so they should not be +considered a stable part of the Mininet API! ### Installation @@ -116,19 +114,19 @@ Mininet mailing list, `mininet-discuss` at: ### Contributing -Mininet is an open-source project and is currently hosted at -. You are encouraged to download the code, -examine it, modify it, and submit bug reports, bug fixes, feature -requests, and enhancements! +Mininet is an open source project and is currently hosted +at . You are encouraged to download +the code, examine it, modify it, and submit bug reports, bug fixes, +feature requests, new features and other issues and pull requests. +Thanks to everyone who has contributed to the project +(see CONTRIBUTORS for more info!) Best wishes, and we look forward to seeing what you can do with Mininet to change the networking world! ### Credits -The Mininet Team: +The Mininet 2.1.0 Team: * Bob Lantz -* Brandon Heller -* Nikhil Handigol -* Vimal Jeyakumar +* Brian O'Connor diff --git a/mininet/net.py b/mininet/net.py index beaf1bd..91ec40e 100755 --- a/mininet/net.py +++ b/mininet/net.py @@ -102,7 +102,7 @@ from mininet.util import macColonHex, ipStr, ipParse, netParse, ipAdd from mininet.term import cleanUpScreens, makeTerms # Mininet version: should be consistent with README and LICENSE -VERSION = "2.0.0" +VERSION = "2.1.0" class Mininet( object ): "Network emulation with hosts spawned in network namespaces." From 896c4cbccc0b756aab11ceeb9e0359c417cc02e1 Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Mon, 26 Aug 2013 14:15:16 -0700 Subject: [PATCH 018/109] Edits for 2.1.0 --- CONTRIBUTORS | 4 ++-- INSTALL | 6 +++--- setup.py | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/CONTRIBUTORS b/CONTRIBUTORS index ab20638..5c6120b 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -2,7 +2,7 @@ Mininet Contributors Mininet is an open source project and we gratefully acknowledge the many contributions to the project! If you have contributed -code to the project and are not on this list, please let us know +code into the project and are not on this list, please let us know or send a pull request. Contributors include: @@ -19,7 +19,7 @@ Additional Mininet Contributors Gustavo Pantuza Coelho Pinto Ryan Cox -Suaun Crampton +Shaun Crampton David Erickson Glen Gibb Andrew Ferguson diff --git a/INSTALL b/INSTALL index d3e4695..0b0732a 100644 --- a/INSTALL +++ b/INSTALL @@ -2,7 +2,7 @@ Mininet Installation/Configuration Notes ---------------------------------------- -Mininet 2.0.0 +Mininet 2.1.0 --- The supported installation methods for Mininet are 1) using a @@ -42,7 +42,7 @@ like to contribute an installation script, we would welcome it!) sudo rm /usr/local/bin/ovs* sudo rm /usr/local/sbin/ovs* -3. Native installation from source on Ubuntu 11.10+ +3. Native installation from source on Ubuntu 12.04+ If you're reading this, you've probably already done so, but the command to download the Mininet source code is: @@ -56,7 +56,7 @@ like to contribute an installation script, we would welcome it!) `install.sh` is a bit intrusive and may possibly damage your OS and/or home directory, by creating/modifying several directories - such as `mininet`, `openflow`, `oftest`, `pox`, or `noxcosre`. + such as `mininet`, `openflow`, `oftest`, `pox`, etc.. Although we hope it won't do anything completely terrible, you may want to look at the script before you run it, and you should make sure your system and home directory are backed up just in case! diff --git a/setup.py b/setup.py index c65f5c3..346f525 100644 --- a/setup.py +++ b/setup.py @@ -25,14 +25,14 @@ setup( Mininet is a network emulator which uses lightweight virtualization to create virtual networks for rapid prototyping of Software-Defined Network (SDN) designs - using OpenFlow. http://openflow.org/mininet + using OpenFlow. http://mininet.org """, classifiers=[ "License :: OSI Approved :: BSD License", "Programming Language :: Python", - "Development Status :: 2 - Pre-Alpha", + "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", - "Topic :: Internet", + "Topic :: System :: Emulators", ], keywords='networking emulator protocol Internet OpenFlow SDN', license='BSD', From 765d126ee942235f580b567fe66f3af1946fcdec Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Mon, 26 Aug 2013 15:05:51 -0700 Subject: [PATCH 019/109] Delete leftover TAP interface from OVS with datapath=user fixes #199 --- mininet/node.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mininet/node.py b/mininet/node.py index 3185746..951b870 100644 --- a/mininet/node.py +++ b/mininet/node.py @@ -1050,6 +1050,8 @@ class OVSSwitch( Switch ): def stop( self ): "Terminate OVS switch." self.cmd( 'ovs-vsctl del-br', self ) + if self.datapath == 'user': + self.cmd( 'ip link del', self ) self.deleteIntfs() OVSKernelSwitch = OVSSwitch From d2762938b7fb5b0cc0f8a7a4c7be4eecde4aae8e Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Mon, 26 Aug 2013 15:07:45 -0700 Subject: [PATCH 020/109] Increase timeout (for lengthy hifi test) --- util/vm/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/vm/build.py b/util/vm/build.py index d27d129..f281504 100755 --- a/util/vm/build.py +++ b/util/vm/build.py @@ -426,7 +426,7 @@ def interact( vm ): # know the time for each test, which means that this # script will have to change as we add more tests. for test in range( 0, 2 ): - if vm.expect( [ 'OK', 'FAILED', pexpect.TIMEOUT ], timeout=60 ) == 0: + if vm.expect( [ 'OK', 'FAILED', pexpect.TIMEOUT ], timeout=180 ) == 0: log( '* Test', test, 'OK' ) else: log( '* Test', test, 'FAILED' ) From 94324e3f46bf7673e9892f2ad82eae1666af93b4 Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Mon, 26 Aug 2013 15:08:11 -0700 Subject: [PATCH 021/109] Skip IVS and UserSwitch tests if they are not installed --- mininet/test/test_hifi.py | 4 ++++ mininet/test/test_nets.py | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/mininet/test/test_hifi.py b/mininet/test/test_hifi.py index e881f3a..542e009 100755 --- a/mininet/test/test_hifi.py +++ b/mininet/test/test_hifi.py @@ -11,6 +11,7 @@ from mininet.node import CPULimitedHost from mininet.link import TCLink from mininet.topo import Topo from mininet.log import setLogLevel +from mininet.util import quietRun # Number of hosts for each test N = 2 @@ -125,10 +126,13 @@ class testOptionsTopoOVSKernel( testOptionsTopoCommon, unittest.TestCase ): "Verify ability to create networks with host and link options (OVS kernel switch)." switchClass = OVSKernelSwitch +@unittest.skipUnless( quietRun( 'which ivs-ctl' ), 'IVS is not installed' ) class testOptionsTopoIVS( testOptionsTopoCommon, unittest.TestCase ): "Verify ability to create networks with host and link options (IVS switch)." switchClass = IVSSwitch +@unittest.skipUnless( quietRun( 'which ofprotocol' ), + 'Reference user switch is not installed' ) class testOptionsTopoUserspace( testOptionsTopoCommon, unittest.TestCase ): "Verify ability to create networks with host and link options (Userspace switch)." switchClass = UserSwitch diff --git a/mininet/test/test_nets.py b/mininet/test/test_nets.py index 027bdd4..45fc3eb 100755 --- a/mininet/test/test_nets.py +++ b/mininet/test/test_nets.py @@ -10,6 +10,7 @@ from mininet.node import Host, Controller from mininet.node import UserSwitch, OVSKernelSwitch, IVSSwitch from mininet.topo import SingleSwitchTopo, LinearTopo from mininet.log import setLogLevel +from mininet.util import quietRun class testSingleSwitchCommon( object ): @@ -53,14 +54,18 @@ class testLinearCommon( object ): dropped = mn.run( mn.ping ) self.assertEqual( dropped, 0 ) + class testLinearOVSKernel( testLinearCommon, unittest.TestCase ): "Test all-pairs ping with LinearNet (OVS kernel switch)." switchClass = OVSKernelSwitch +@unittest.skipUnless( quietRun( 'which ivs-ctl' ), 'IVS is not installed' ) class testLinearIVS( testLinearCommon, unittest.TestCase ): "Test all-pairs ping with LinearNet (IVS switch)." switchClass = IVSSwitch +@unittest.skipUnless( quietRun( 'which ofprotocol' ), + 'Reference user switch is not installed' ) class testLinearUserspace( testLinearCommon, unittest.TestCase ): "Test all-pairs ping with LinearNet (Userspace switch)." switchClass = UserSwitch From 45d365f98b6b2f7e2cb5c03a39e2de213c860a8b Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Mon, 26 Aug 2013 15:29:00 -0700 Subject: [PATCH 022/109] Need a few more skipUnless() checks. --- mininet/test/test_nets.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mininet/test/test_nets.py b/mininet/test/test_nets.py index 45fc3eb..f378ce8 100755 --- a/mininet/test/test_nets.py +++ b/mininet/test/test_nets.py @@ -34,10 +34,14 @@ class testSingleSwitchOVSKernel( testSingleSwitchCommon, unittest.TestCase ): "Test ping with single switch topology (OVS kernel switch)." switchClass = OVSKernelSwitch + +@unittest.skipUnless( quietRun( 'which ivs-ctl' ), 'IVS is not installed' ) class testSingleSwitchIVS( testSingleSwitchCommon, unittest.TestCase ): "Test ping with single switch topology (IVS switch)." switchClass = IVSSwitch +@unittest.skipUnless( quietRun( 'which ofprotocol' ), + 'Reference user switch is not installed' ) class testSingleSwitchUserspace( testSingleSwitchCommon, unittest.TestCase ): "Test ping with single switch topology (Userspace switch)." switchClass = UserSwitch From 549321257806d8e07520f673d5e35d2e522303e8 Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Mon, 26 Aug 2013 15:44:49 -0700 Subject: [PATCH 023/109] Add tests for OVS user switch (skipping hifi test for now) test_hifi.py currently fails for OVS when datapath=user - we should look at this and fix it. --- mininet/test/test_hifi.py | 10 ++++++++-- mininet/test/test_nets.py | 14 +++++++++++--- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/mininet/test/test_hifi.py b/mininet/test/test_hifi.py index 542e009..f6ebb71 100755 --- a/mininet/test/test_hifi.py +++ b/mininet/test/test_hifi.py @@ -4,9 +4,10 @@ Test creation and pings for topologies with link and/or CPU options.""" import unittest +from functools import partial from mininet.net import Mininet -from mininet.node import OVSKernelSwitch, UserSwitch, IVSSwitch +from mininet.node import OVSSwitch, UserSwitch, IVSSwitch from mininet.node import CPULimitedHost from mininet.link import TCLink from mininet.topo import Topo @@ -124,7 +125,12 @@ class testOptionsTopoCommon( object ): class testOptionsTopoOVSKernel( testOptionsTopoCommon, unittest.TestCase ): "Verify ability to create networks with host and link options (OVS kernel switch)." - switchClass = OVSKernelSwitch + switchClass = OVSSwitch + +@unittest.skip( 'Skipping OVS user switch test for now' ) +class testOptionsTopoOVSUser( testOptionsTopoCommon, unittest.TestCase ): + "Verify ability to create networks with host and link options (OVS user switch)." + switchClass = partial( OVSSwitch, datapath='user' ) @unittest.skipUnless( quietRun( 'which ivs-ctl' ), 'IVS is not installed' ) class testOptionsTopoIVS( testOptionsTopoCommon, unittest.TestCase ): diff --git a/mininet/test/test_nets.py b/mininet/test/test_nets.py index f378ce8..a56061b 100755 --- a/mininet/test/test_nets.py +++ b/mininet/test/test_nets.py @@ -4,10 +4,11 @@ Test creation and all-pairs ping for each included mininet topo type.""" import unittest +from functools import partial from mininet.net import Mininet from mininet.node import Host, Controller -from mininet.node import UserSwitch, OVSKernelSwitch, IVSSwitch +from mininet.node import UserSwitch, OVSSwitch, IVSSwitch from mininet.topo import SingleSwitchTopo, LinearTopo from mininet.log import setLogLevel from mininet.util import quietRun @@ -32,8 +33,11 @@ class testSingleSwitchCommon( object ): class testSingleSwitchOVSKernel( testSingleSwitchCommon, unittest.TestCase ): "Test ping with single switch topology (OVS kernel switch)." - switchClass = OVSKernelSwitch + switchClass = OVSSwitch +class testSingleSwitchOVSUser( testSingleSwitchCommon, unittest.TestCase ): + "Test ping with single switch topology (OVS user switch)." + switchClass = partial( OVSSwitch, datapath='user' ) @unittest.skipUnless( quietRun( 'which ivs-ctl' ), 'IVS is not installed' ) class testSingleSwitchIVS( testSingleSwitchCommon, unittest.TestCase ): @@ -61,7 +65,11 @@ class testLinearCommon( object ): class testLinearOVSKernel( testLinearCommon, unittest.TestCase ): "Test all-pairs ping with LinearNet (OVS kernel switch)." - switchClass = OVSKernelSwitch + switchClass = OVSSwitch + +class testLinearOVSUser( testLinearCommon, unittest.TestCase ): + "Test all-pairs ping with LinearNet (OVS user switch)." + switchClass = partial( OVSSwitch, datapath='user' ) @unittest.skipUnless( quietRun( 'which ivs-ctl' ), 'IVS is not installed' ) class testLinearIVS( testLinearCommon, unittest.TestCase ): From ec810dd6db989fa158152a1f7cd611fad00e22fc Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Mon, 26 Aug 2013 15:48:05 -0700 Subject: [PATCH 024/109] minor edits --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index fce180f..c728a9c 100644 --- a/README.md +++ b/README.md @@ -72,14 +72,14 @@ Mininet includes: Mininet 2.1.0 provides a number of bug fixes as well as a number of new features, including: -* Convenient access to Mininet() as a dict -* X11 tunneling (wireshark in Mininet hosts!) +* Convenient access to Mininet() as a dict of nodes +* X11 tunneling (wireshark in Mininet hosts, finally!) * Accurate reflection of the Mininet() object in the CLI * Automatically detecting and adjusting resource limits * Automatic cleanup on failure of the `mn` command -* Support for running OVS in user space mode +* Preliminary support for running OVS in user space mode * Preliminary support for the Indigo Virtual Switch (IVSSwitch) -* The ability to import examples as modules +* The ability to import modules from mininet.examples We have provided several new examples which can also be imported to provide useful functionality, including: From 226a1dc3918703fb4c3bff7b88bd007cd8c635d9 Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Mon, 26 Aug 2013 15:53:34 -0700 Subject: [PATCH 025/109] Minor edits --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index c728a9c..90ead38 100644 --- a/README.md +++ b/README.md @@ -69,8 +69,8 @@ Mininet includes: ### New features in 2.1.0 -Mininet 2.1.0 provides a number of bug fixes as well as a -number of new features, including: +Mininet 2.1.0 provides a number of bug fixes as well as +several new features, including: * Convenient access to Mininet() as a dict of nodes * X11 tunneling (wireshark in Mininet hosts, finally!) @@ -84,9 +84,9 @@ number of new features, including: We have provided several new examples which can also be imported to provide useful functionality, including: -* A simple NAT script to attach Mininet networks to your LAN -* An example of modeling control and data networks -* An example of per-host custom directories using bind mounts +* Connecting Mininet hosts the internet (or a LAN) using NAT +* Modeling control as well as data networks +* Creating per-host custom directories using bind mounts Note that these are experimental features which may "graduate" into mainline Mininet in the future, so they should not be From 5413d2e5a37a6ea0d504145bca3fcb5ad7fd8860 Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Fri, 5 Jul 2013 20:06:21 -0700 Subject: [PATCH 026/109] Check for chroot dir and chroot if necessary. --- util/m | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/util/m b/util/m index 786dcce..4ec11f3 100755 --- a/util/m +++ b/util/m @@ -33,4 +33,12 @@ if [ -d "$cgroup" ]; then cg="-g $host" fi -exec sudo mnexec -a $pid $cg $cmd +# Check whether host should be running in a chroot dir +rootdir="/var/run/mn/$host/root" +if [ -d $rootdir ]; then + cmd="'cd `pwd`; exec $cmd'" + cmd="chroot $rootdir bash -c $cmd" +fi + +cmd="exec sudo mnexec -a $pid $cg $cmd" +eval $cmd From 5ae8c936e74a6f11f639ced87a818286e8024ea2 Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Fri, 5 Jul 2013 20:07:02 -0700 Subject: [PATCH 027/109] Prototype implementation of bind mounts. --- examples/bind.py | 164 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100755 examples/bind.py diff --git a/examples/bind.py b/examples/bind.py new file mode 100755 index 0000000..142eec4 --- /dev/null +++ b/examples/bind.py @@ -0,0 +1,164 @@ +#!/usr/bin/python + +""" +bind.py: Bind mount prototype + +This creates hosts with private directories as desired. +""" + +from mininet.net import Mininet +from mininet.node import Host, Switch, Controller +from mininet.cli import CLI +from mininet.util import errFail, quietRun, errRun +from mininet.topo import SingleSwitchTopo +from mininet.log import setLogLevel, info, debug + +from os.path import join, realpath +from functools import partial + +# Utility functions for unmounting a tree + +def mountPoints(): + "Return list of mounted file systems" + mtab, _err, _ret = errFail( 'cat /proc/mounts' ) + lines = mtab.split( '\n' ) + mounts = [] + for line in lines: + if not line: + continue + fields = line.split( ' ') + mount = fields[ 1 ] + mounts.append( mount ) + return mounts + +def unmountAll( dir='/var/run/mn' ): + "Unmount all mounts under a directory tree" + dir = realpath( dir ) + # Find all mounts below dir + # This is subtle because /foo is not + # a parent of /foot + dirslash = dir + '/' + mounts = [ m for m in mountPoints() + if m == dir or m.find( dirslash ) == 0 ] + # Unmount them from bottom to top + mounts.sort( reverse=True ) + for mount in mounts: + debug( 'Unmounting', mount, '\n' ) + out, err, code = errRun( 'umount', mount ) + if code != 0: + info( '*** Warning: failed to umount', mount, '\n' ) + info( err ) + + +class HostWithPrivateDirs( Host ): + "Host with private directories" + + mnRunDir = realpath( '/var/run/mn' ) + + def __init__(self, name, *args, **kwargs ): + "privateDirs: list of private directories" + self.privateDirs = kwargs.pop( 'privateDirs', [] ) + Host.__init__( self, name, *args, **kwargs ) + self.rundir = '%s/%s' % ( self.mnRunDir, name ) + if self.privateDirs: + self.privateDirs = [ realpath( d ) for d in self.privateDirs ] + self.createBindMounts() + # These should run in the namespace before we chroot, + # in order to put the right entries in /etc/mtab + # Eventually this will allow a local pid space + # Now we chroot and cd to wherever we were before. + pwd = self.cmd( 'pwd' ).strip() + self.sendCmd( 'exec chroot', self.root, 'bash -ms mininet:' + + self.name ) + self.waiting = False + self.cmd( 'cd', pwd ) + # In order for many utilities to work, + # we need to remount /proc and /sys + self.cmd( 'mount /proc' ) + self.cmd( 'mount /sys' ) + + def mountPrivateDirs( self ): + "Create and bind mount private dirs" + for dir in self.privateDirs: + privateDir = self.private + dir + errFail( 'mkdir -p ' + privateDir ) + mountPoint = self.root + dir + errFail( 'mount -B %s %s' % + ( privateDir, mountPoint) ) + + def remountDirs( self, fstypes=[ 'nfs' ] ): + "Remount mounted file systems" + dirs = self.cmd( 'cat /proc/mounts' ).strip().split( '\n' ) + for dir in dirs: + line = dir.split() + mountpoint, fstype = line[ 1 ], line[ 2 ] + # Don't re-remount directories!!! + if mountpoint.find( self.mnRunDir ) == 0: + continue + if fstype in fstypes: + print "remounting:", mountpoint + errFail( 'mount -B %s %s' % ( + mountpoint, self.root + mountpoint ) ) + + def createBindMounts( self ): + """Create a chroot directory structure, + with self.privateDirs as private dirs""" + errFail( 'mkdir -p '+ self.rundir ) + unmountAll( self.rundir ) + # Create /root and /private directories + self.root = self.rundir + '/root' + self.private = self.rundir + '/private' + errFail( 'mkdir -p ' + self.root ) + errFail( 'mkdir -p ' + self.private ) + # Recursively mount / in private doort + # note we'll remount /sys and /proc later + errFail( 'mount -B / ' + self.root ) + self.remountDirs() + self.mountPrivateDirs() + + def unmountBindMounts( self ): + "Unmount all of our bind mounts" + unmountAll( self.rundir ) + + def popen( self, *args, **kwargs ): + "Popen with chroot support" + chroot = kwargs.pop( 'chroot', True ) + mncmd = kwargs.get( 'mncmd', + [ 'mnexec', '-a', str( self.pid ) ] ) + if chroot: + mncmd = [ 'chroot', self.root ] + mncmd + kwargs[ 'mncmd' ] = mncmd + return Host.popen( self, *args, **kwargs ) + + def cleanup( self ): + "Clean up, then unmount bind mounts" + # Wait for process to actually terminate + self.shell.wait() + Host.cleanup( self ) + self.unmountBindMounts() + errFail( 'rmdir ' + self.root ) + +# Sample usage + +def testHostWithPrivateDirs(): + "Test bind mounts" + topo = SingleSwitchTopo( 2 ) + privateDirs = [ '/var/log', '/var/run' ] + host = partial( HostWithPrivateDirs, privateDirs=privateDirs ) + net = Mininet( topo=topo, host=host ) + net.start() + print 'Private Directories:', privateDirs + CLI( net ) + net.stop() + + +if __name__ == '__main__': + unmountAll() + setLogLevel( 'info' ) + testHostWithPrivateDirs() + unmountAll() + + + + + From f34429036800574328097bb957d6c493bd5d7260 Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Sun, 7 Jul 2013 15:27:49 -0700 Subject: [PATCH 028/109] Change API for more efficient remount and unmount. --- examples/bind.py | 72 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 51 insertions(+), 21 deletions(-) diff --git a/examples/bind.py b/examples/bind.py index 142eec4..d4d28a4 100755 --- a/examples/bind.py +++ b/examples/bind.py @@ -7,17 +7,20 @@ This creates hosts with private directories as desired. """ from mininet.net import Mininet -from mininet.node import Host, Switch, Controller +from mininet.node import Host from mininet.cli import CLI from mininet.util import errFail, quietRun, errRun from mininet.topo import SingleSwitchTopo from mininet.log import setLogLevel, info, debug -from os.path import join, realpath +from os.path import realpath from functools import partial + # Utility functions for unmounting a tree +MNRUNDIR = realpath( '/var/run/mn' ) + def mountPoints(): "Return list of mounted file systems" mtab, _err, _ret = errFail( 'cat /proc/mounts' ) @@ -31,7 +34,7 @@ def mountPoints(): mounts.append( mount ) return mounts -def unmountAll( dir='/var/run/mn' ): +def unmountAll( dir=MNRUNDIR ): "Unmount all mounts under a directory tree" dir = realpath( dir ) # Find all mounts below dir @@ -53,11 +56,17 @@ def unmountAll( dir='/var/run/mn' ): class HostWithPrivateDirs( Host ): "Host with private directories" - mnRunDir = realpath( '/var/run/mn' ) + mnRunDir = MNRUNDIR def __init__(self, name, *args, **kwargs ): - "privateDirs: list of private directories" + """privateDirs: list of private directories + remounts: dirs to remount + unmount: unmount dirs in cleanup? (True) + Note: if unmount is False, you must call unmountAll() + manually.""" self.privateDirs = kwargs.pop( 'privateDirs', [] ) + self.remounts = kwargs.pop( 'remounts', [] ) + self.unmount = kwargs.pop( 'unmount', True ) Host.__init__( self, name, *args, **kwargs ) self.rundir = '%s/%s' % ( self.mnRunDir, name ) if self.privateDirs: @@ -86,19 +95,28 @@ class HostWithPrivateDirs( Host ): errFail( 'mount -B %s %s' % ( privateDir, mountPoint) ) - def remountDirs( self, fstypes=[ 'nfs' ] ): - "Remount mounted file systems" - dirs = self.cmd( 'cat /proc/mounts' ).strip().split( '\n' ) + def mountDirs( self, dirs ): + "Mount a list of directories" + for dir in dirs: + mountpoint = self.root + dir + errFail( 'mount -B %s %s' % + ( dir, mountpoint ) ) + + @classmethod + def findRemounts( cls, fstypes=[ 'nfs' ] ): + """Identify mount points in /proc/mounts to remount + fstypes: file system types to match""" + dirs = quietRun( 'cat /proc/mounts' ).strip().split( '\n' ) + remounts = [] for dir in dirs: line = dir.split() mountpoint, fstype = line[ 1 ], line[ 2 ] # Don't re-remount directories!!! - if mountpoint.find( self.mnRunDir ) == 0: + if mountpoint.find( cls.mnRunDir ) == 0: continue if fstype in fstypes: - print "remounting:", mountpoint - errFail( 'mount -B %s %s' % ( - mountpoint, self.root + mountpoint ) ) + remounts.append( mountpoint ) + return remounts def createBindMounts( self ): """Create a chroot directory structure, @@ -113,7 +131,7 @@ class HostWithPrivateDirs( Host ): # Recursively mount / in private doort # note we'll remount /sys and /proc later errFail( 'mount -B / ' + self.root ) - self.remountDirs() + self.mountDirs( self.remounts ) self.mountPrivateDirs() def unmountBindMounts( self ): @@ -131,33 +149,45 @@ class HostWithPrivateDirs( Host ): return Host.popen( self, *args, **kwargs ) def cleanup( self ): - "Clean up, then unmount bind mounts" + """Clean up, then unmount bind mounts + unmount: actually unmount bind mounts?""" # Wait for process to actually terminate self.shell.wait() Host.cleanup( self ) - self.unmountBindMounts() - errFail( 'rmdir ' + self.root ) + if self.unmount: + self.unmountBindMounts() + errFail( 'rmdir ' + self.root ) + + +# Convenience aliases + +findRemounts = HostWithPrivateDirs.findRemounts + # Sample usage def testHostWithPrivateDirs(): "Test bind mounts" - topo = SingleSwitchTopo( 2 ) + topo = SingleSwitchTopo( 10 ) + remounts = findRemounts( fstypes=[ 'nfs' ] ) privateDirs = [ '/var/log', '/var/run' ] - host = partial( HostWithPrivateDirs, privateDirs=privateDirs ) + host = partial( HostWithPrivateDirs, remounts=remounts, + privateDirs=privateDirs, unmount=False ) net = Mininet( topo=topo, host=host ) net.start() - print 'Private Directories:', privateDirs + info( 'Private Directories:', privateDirs, '\n' ) CLI( net ) net.stop() + # We do this all at once to save a bit of time + info( 'Unmounting host bind mounts...\n' ) + unmountAll() if __name__ == '__main__': unmountAll() setLogLevel( 'info' ) testHostWithPrivateDirs() - unmountAll() - + info( 'Done.\n') From a56e29704e0cbdef719462feff2db85ee3237076 Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Mon, 26 Aug 2013 17:45:02 -0700 Subject: [PATCH 029/109] Make sure that /bin/bash exists before attempting to chroot. --- util/m | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/util/m b/util/m index 4ec11f3..860eeb0 100755 --- a/util/m +++ b/util/m @@ -35,9 +35,9 @@ fi # Check whether host should be running in a chroot dir rootdir="/var/run/mn/$host/root" -if [ -d $rootdir ]; then +if [ -d $rootdir -a -x $rootdir/bin/bash ]; then cmd="'cd `pwd`; exec $cmd'" - cmd="chroot $rootdir bash -c $cmd" + cmd="chroot $rootdir /bin/bash -c $cmd" fi cmd="exec sudo mnexec -a $pid $cg $cmd" From ad5a0e42d0903fba096323a6e1717eb15a33f340 Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Mon, 26 Aug 2013 22:19:17 -0700 Subject: [PATCH 030/109] Explicitly create a qcow2 image --- util/vm/build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/util/vm/build.py b/util/vm/build.py index f281504..1537548 100755 --- a/util/vm/build.py +++ b/util/vm/build.py @@ -186,7 +186,7 @@ def extractKernel( image, flavor ): def findBaseImage( flavor, size='8G' ): "Return base VM image and kernel, creating them if needed" - image = path.join( VMImageDir, flavor + '-base.img' ) + image = path.join( VMImageDir, flavor + '-base.qcow2' ) if path.exists( image ): # Detect race condition with multiple builds perms = stat( image )[ ST_MODE ] & 0777 @@ -198,7 +198,7 @@ def findBaseImage( flavor, size='8G' ): run( 'mkdir -p %s' % VMImageDir ) iso = findiso( flavor ) log( '* Creating image file', image ) - run( 'qemu-img create %s %s' % ( image, size ) ) + run( 'qemu-img create -f qcow2 %s %s' % ( image, size ) ) installUbuntu( iso, image ) # Write-protect image, also signaling it is complete log( '* Write-protecting image', image) From 15146d900c2798a27e744e44f94d96e416b1b66e Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Mon, 19 Aug 2013 23:12:42 -0700 Subject: [PATCH 031/109] changed CLI to MininetFacade; a great deal of logic also changed --- examples/controlnet.py | 76 +++++++++++++++++++++++++----------------- mininet/net.py | 23 +++++++++---- mininet/node.py | 4 ++- 3 files changed, 64 insertions(+), 39 deletions(-) diff --git a/examples/controlnet.py b/examples/controlnet.py index 3d5dd9d..0b36dd7 100755 --- a/examples/controlnet.py +++ b/examples/controlnet.py @@ -31,30 +31,43 @@ class DataController( Controller ): pass -class CLI2( CLI ): - "CLI that can talk to two networks" +class MininetFacade: + "TODO: CLI that can talk to two or more networks" - def __init__( self, *args, **kwargs ): - "cnet: second network" - self.cnet = kwargs.pop( 'cnet' ) - CLI.__init__( self, *args, **kwargs ) + def __init__( self, *args ): + self.nets = args - def updateVars( self ): - "Update variables to include cnet" - cnet = self.cnet - nodes2 = cnet.controllers + cnet.switches + cnet.hosts - self.nodelist += nodes2 - for node in nodes2: - self.nodemap[ node.name ] = node - self.locals[ 'cnet' ] = cnet - self.locals.update( self.nodemap ) - - def cmdloop( self, *args, **kwargs ): - "Patch to add cnet if needed" - if 'cnet' not in self.locals: - self.updateVars() - CLI.cmdloop( self, *args, **kwargs ) + # default is first net + def __getattr__( self, name ): + return getattr( self.nets[ 0 ], name ) + def __getitem__( self, key ): + for net in self.nets: + if key in net: + return net[ key ] + + def __iter__( self ): + for net in self.nets: + for node in net: + yield node + + def __len__( self ): + count = 0 + for net in self.nets: + count += len(net) + return count + + def __contains__( self, key ): + return key in self.keys() + + def keys( self ): + return list( self ) + + def values( self ): + return [ self[ key ] for key in self ] + + def items( self ): + return zip( self.keys(), self.values() ) # A real control network! @@ -83,32 +96,33 @@ setLogLevel( 'info' ) info( '* Creating Control Network\n' ) ctopo = ControlNetwork( n=4, dataController=DataController ) -cnet = Mininet( topo=ctopo, ipBase='192.168.123.0/24', build=False ) +cnet = Mininet( topo=ctopo, ipBase='192.168.123.0/24', controller=None ) info( '* Adding Control Network Controller\n') -cnet.addController( 'cc0' ) +cnet.addController( 'cc0', controller=Controller ) info( '* Starting Control Network\n') -cnet.build() cnet.start() -dataControllers = cnet.hosts[ : -1 ] # ignore 'root' node info( '* Creating Data Network\n' ) topo = TreeTopo( depth=2, fanout=2 ) # UserSwitch so we can easily test failover -net = Mininet( topo=topo, switch=UserSwitch, build=False ) +net = Mininet( topo=topo, switch=UserSwitch, controller=None ) info( '* Adding Controllers to Data Network\n' ) -net.controllers = dataControllers -net.build() +for host in cnet.hosts: + if isinstance(host, Controller): + net.addController( host ) info( '* Starting Data Network\n') net.start() -CLI2( net, cnet=cnet ) +mn = MininetFacade( net, cnet ) +mn.keys() +CLI( mn ) info( '* Stopping Data Network\n' ) net.stop() info( '* Stopping Control Network\n' ) -# dataControllers have already been stopped -cnet.hosts = list( set( cnet.hosts ) - set( dataControllers ) ) +# dataControllers have already been stopped -- now terminate is idempotent +#cnet.hosts = list( set( cnet.hosts ) - set( dataControllers ) ) cnet.stop() diff --git a/mininet/net.py b/mininet/net.py index 91ec40e..37b1f4f 100755 --- a/mininet/net.py +++ b/mininet/net.py @@ -209,9 +209,16 @@ class Mininet( object ): def addController( self, name='c0', controller=None, **params ): """Add controller. controller: Controller class""" + #Get controller class if not controller: controller = self.controller - controller_new = controller( name, **params ) + #Construct new controller if one is not given + if isinstance(name, Controller): + controller_new = name + name = controller_new.name + else: + controller_new = controller( name, **params ) + #Add new controller to net if controller_new: # allow controller-less setups self.controllers.append( controller_new ) self.nameToNode[ name ] = controller_new @@ -230,9 +237,9 @@ class Mininet( object ): return self.getNodeByName( *args ) # Even more convenient syntax for node lookup and iteration - def __getitem__( self, *args ): + def __getitem__( self, key ): """net [ name ] operator: Return node(s) with given name(s)""" - return self.getNodeByName( *args ) + return self.nameToNode[ key ] def __iter__( self ): "return iterator over nodes" @@ -246,15 +253,17 @@ class Mininet( object ): def __contains__( self, item ): "returns True if net contains named node" - return item in self.keys() + return item in self.nameToNode def keys( self ): "return a list of all node names or net's keys" - return list( self.__iter__() ) + #TODO: fix this + return list( self ) def values( self ): "return a list of all nodes or net's values" - return [ self[name] for name in self.__iter__() ] + #TODO: fix this + return [ self[name] for name in self ] def items( self ): "return (key,value) tuple list for every node in net" @@ -307,7 +316,7 @@ class Mininet( object ): info( '*** Creating network\n' ) - if not self.controllers: + if not self.controllers and self.controller: # Add a default controller info( '*** Adding controller\n' ) classes = self.controller diff --git a/mininet/node.py b/mininet/node.py index 951b870..895c731 100644 --- a/mininet/node.py +++ b/mininet/node.py @@ -183,7 +183,8 @@ class Node( object ): def terminate( self ): "Send kill signal to Node and clean up after it." - os.kill( self.pid, signal.SIGKILL ) + if self.shell: + os.kill( self.pid, signal.SIGKILL ) self.cleanup() def stop( self ): @@ -1240,3 +1241,4 @@ class RemoteController( Controller ): if 'Unable' in listening: warn( "Unable to contact the remote controller" " at %s:%d\n" % ( self.ip, self.port ) ) + From dc882d69051bce7845fcf5f95afb7b501db3d2e7 Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Mon, 19 Aug 2013 23:15:01 -0700 Subject: [PATCH 032/109] clean up controlnet --- examples/controlnet.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/controlnet.py b/examples/controlnet.py index 0b36dd7..4118f5f 100755 --- a/examples/controlnet.py +++ b/examples/controlnet.py @@ -114,7 +114,6 @@ info( '* Starting Data Network\n') net.start() mn = MininetFacade( net, cnet ) -mn.keys() CLI( mn ) info( '* Stopping Data Network\n' ) From 7c962d2f61eb1e424457cda4e1f142cfa1ad33fa Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Tue, 20 Aug 2013 18:46:34 -0700 Subject: [PATCH 033/109] Fixed MininetFacade and moved main logic into run. First shot at "test" function --- examples/controlnet.py | 83 +++++++++++++++++++++++++----------------- 1 file changed, 50 insertions(+), 33 deletions(-) diff --git a/examples/controlnet.py b/examples/controlnet.py index 4118f5f..c4d2268 100755 --- a/examples/controlnet.py +++ b/examples/controlnet.py @@ -30,18 +30,24 @@ class DataController( Controller ): "Ignore spurious error" pass - -class MininetFacade: +class MininetFacade( object ): "TODO: CLI that can talk to two or more networks" - def __init__( self, *args ): - self.nets = args + def __init__( self, net, *args, **kwargs ): + self.net = net + self.nets = [ net ] + list( args ) + kwargs.values() + self.nameToNet = kwargs + self.nameToNet['net'] = net # default is first net def __getattr__( self, name ): - return getattr( self.nets[ 0 ], name ) + return getattr( self.net, name ) def __getitem__( self, key ): + #search kwargs for net named key + if key in self.nameToNet: + return self.nameToNet[ key ] + #search each net for node named key for net in self.nets: if key in net: return net[ key ] @@ -91,41 +97,52 @@ class ControlNetwork( Topo ): # Make it Happen!! +def run( func=CLI ): + info( '* Creating Control Network\n' ) + ctopo = ControlNetwork( n=4, dataController=DataController ) + cnet = Mininet( topo=ctopo, ipBase='192.168.123.0/24', controller=None ) + info( '* Adding Control Network Controller\n') + cnet.addController( 'cc0', controller=Controller ) + info( '* Starting Control Network\n') + cnet.start() -setLogLevel( 'info' ) + info( '* Creating Data Network\n' ) + topo = TreeTopo( depth=2, fanout=2 ) + # UserSwitch so we can easily test failover + net = Mininet( topo=topo, switch=UserSwitch, controller=None ) + info( '* Adding Controllers to Data Network\n' ) + for host in cnet.hosts: + if isinstance(host, Controller): + net.addController( host ) + info( '* Starting Data Network\n') + net.start() -info( '* Creating Control Network\n' ) -ctopo = ControlNetwork( n=4, dataController=DataController ) -cnet = Mininet( topo=ctopo, ipBase='192.168.123.0/24', controller=None ) -info( '* Adding Control Network Controller\n') -cnet.addController( 'cc0', controller=Controller ) -info( '* Starting Control Network\n') -cnet.start() + mn = MininetFacade( net, cnet=cnet ) -info( '* Creating Data Network\n' ) -topo = TreeTopo( depth=2, fanout=2 ) -# UserSwitch so we can easily test failover -net = Mininet( topo=topo, switch=UserSwitch, controller=None ) -info( '* Adding Controllers to Data Network\n' ) -for host in cnet.hosts: - if isinstance(host, Controller): - net.addController( host ) -info( '* Starting Data Network\n') -net.start() + # run the function passed as an argument + func( mn ) -mn = MininetFacade( net, cnet ) -CLI( mn ) - -info( '* Stopping Data Network\n' ) -net.stop() - -info( '* Stopping Control Network\n' ) -# dataControllers have already been stopped -- now terminate is idempotent -#cnet.hosts = list( set( cnet.hosts ) - set( dataControllers ) ) -cnet.stop() + info( '* Stopping Data Network\n' ) + net.stop() + info( '* Stopping Control Network\n' ) + # dataControllers have already been stopped -- now terminate is idempotent + #cnet.hosts = list( set( cnet.hosts ) - set( dataControllers ) ) + cnet.stop() +def test( net ): + netLoss = net.pingAll() + cnetLoss = net['cnet'].pingAll() +if __name__ == '__main__': + setLogLevel( 'info' ) + import argparse + parser = argparse.ArgumentParser(description='TODO:description') + parser.add_argument('--test', dest='func', action='store_const', + const=test, default=CLI, + help='TODO: test help') + args = parser.parse_args() + run( func=args.func ) From 3a35480c7a22bb6fcb59f3bc5297b63478446b49 Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Mon, 26 Aug 2013 17:43:31 -0700 Subject: [PATCH 034/109] removing test from controlnet --- examples/controlnet.py | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/examples/controlnet.py b/examples/controlnet.py index c4d2268..d75d1d0 100755 --- a/examples/controlnet.py +++ b/examples/controlnet.py @@ -97,7 +97,7 @@ class ControlNetwork( Topo ): # Make it Happen!! -def run( func=CLI ): +def run(): info( '* Creating Control Network\n' ) ctopo = ControlNetwork( n=4, dataController=DataController ) cnet = Mininet( topo=ctopo, ipBase='192.168.123.0/24', controller=None ) @@ -120,7 +120,7 @@ def run( func=CLI ): mn = MininetFacade( net, cnet=cnet ) # run the function passed as an argument - func( mn ) + CLI( mn ) info( '* Stopping Data Network\n' ) net.stop() @@ -130,19 +130,7 @@ def run( func=CLI ): #cnet.hosts = list( set( cnet.hosts ) - set( dataControllers ) ) cnet.stop() -def test( net ): - netLoss = net.pingAll() - cnetLoss = net['cnet'].pingAll() - if __name__ == '__main__': setLogLevel( 'info' ) - import argparse - parser = argparse.ArgumentParser(description='TODO:description') - parser.add_argument('--test', dest='func', action='store_const', - const=test, default=CLI, - help='TODO: test help') - - args = parser.parse_args() - - run( func=args.func ) + run() From aacf7c4613d019889241976be8eaa2558fbb5c60 Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Mon, 26 Aug 2013 17:43:53 -0700 Subject: [PATCH 035/109] fixing controllers.py to use api --- examples/controllers.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/examples/controllers.py b/examples/controllers.py index ac86429..1fcefb1 100755 --- a/examples/controllers.py +++ b/examples/controllers.py @@ -15,8 +15,8 @@ setLogLevel( 'info' ) # Two local and one "external" controller (which is actually c0) # Ignore the warning message that the remote isn't (yet) running -c0 = Controller( 'c0' ) -c1 = Controller( 'c1' ) +c0 = Controller( 'c0', port=6633 ) +c1 = Controller( 'c1', port=6634 ) c2 = RemoteController( 'c2', ip='127.0.0.1' ) cmap = { 's1': c0, 's2': c1, 's3': c2 } @@ -28,7 +28,9 @@ class MultiSwitch( OVSSwitch ): topo = TreeTopo( depth=2, fanout=2 ) net = Mininet( topo=topo, switch=MultiSwitch, build=False ) -net.controllers = [ c0, c1 ] +for c in [ c0, c1 ]: + net.addController(c) + net.build() net.start() CLI( net ) From ecddbcf240f8acacbd103942cc7af00fd8938d83 Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Mon, 26 Aug 2013 17:51:41 -0700 Subject: [PATCH 036/109] updated emptynet to use addLink --- examples/emptynet.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/emptynet.py b/examples/emptynet.py index 9f57855..afcf6a8 100755 --- a/examples/emptynet.py +++ b/examples/emptynet.py @@ -27,8 +27,8 @@ def emptyNet(): s3 = net.addSwitch( 's3' ) info( '*** Creating links\n' ) - h1.linkTo( s3 ) - h2.linkTo( s3 ) + net.addLink( h1, s3 ) + net.addLink( h2, s3 ) info( '*** Starting network\n') net.start() From 891d80713758aaabc98c721b721a9cf69702a361 Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Mon, 26 Aug 2013 19:12:18 -0700 Subject: [PATCH 037/109] fixed multiping example --- examples/multiping.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/examples/multiping.py b/examples/multiping.py index 3bd231c..88667d4 100755 --- a/examples/multiping.py +++ b/examples/multiping.py @@ -23,13 +23,8 @@ def chunks( l, n ): def startpings( host, targetips ): "Tell host to repeatedly ping targets" - targetips.append( '10.0.0.200' ) - targetips = ' '.join( targetips ) - # BL: Not sure why loopback intf isn't up! - host.cmd( 'ifconfig lo up' ) - # Simple ping loop cmd = ( 'while true; do ' ' for ip in %s; do ' % targetips + @@ -63,6 +58,8 @@ def multiping( netsize, chunksize, seconds): # Start pings for subnet in subnets: ips = [ host.IP() for host in subnet ] + #adding bogus to generate packet loss + ips.append( '10.0.0.200' ) for host in subnet: startpings( host, ips ) From 967614f64a371a4c0ad90b2b1511e670882c306a Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Mon, 26 Aug 2013 23:09:14 -0700 Subject: [PATCH 038/109] adding examples/__init__.py --- examples/__init__.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 examples/__init__.py diff --git a/examples/__init__.py b/examples/__init__.py new file mode 100644 index 0000000..fedbfea --- /dev/null +++ b/examples/__init__.py @@ -0,0 +1 @@ +# Mininet Examples From 0840af5277f49e9b1e32e66bfbc7ebb912b9fc46 Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Mon, 26 Aug 2013 23:23:31 -0700 Subject: [PATCH 039/109] removing todos in net.py --- mininet/net.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/mininet/net.py b/mininet/net.py index 37b1f4f..6eaac20 100755 --- a/mininet/net.py +++ b/mininet/net.py @@ -257,12 +257,10 @@ class Mininet( object ): def keys( self ): "return a list of all node names or net's keys" - #TODO: fix this return list( self ) def values( self ): "return a list of all nodes or net's values" - #TODO: fix this return [ self[name] for name in self ] def items( self ): From 92bf2cf10574489d2677538474df1408884a418d Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Tue, 27 Aug 2013 11:51:01 -0700 Subject: [PATCH 040/109] codecheck: removed unused variable in topo.py --- mininet/topo.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/mininet/topo.py b/mininet/topo.py index 148011d..784095f 100644 --- a/mininet/topo.py +++ b/mininet/topo.py @@ -261,8 +261,6 @@ class LinearTopo(Topo): switch = self.addSwitch('s%s' % i) # Add hosts to switch for j in irange(1, n): - hostNum = (i-1)*n + j - #host = self.addHost('h%s' % hostNum) host = self.addHost(genHostName(i, j)) self.addLink(host, switch) # Connect switch to previous From e935da461ae72993ce300bc2c53f70aa85928183 Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Tue, 27 Aug 2013 12:27:27 -0700 Subject: [PATCH 041/109] added comments and cleaned up controlnet.py --- examples/controlnet.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/examples/controlnet.py b/examples/controlnet.py index d75d1d0..dba8756 100755 --- a/examples/controlnet.py +++ b/examples/controlnet.py @@ -10,8 +10,8 @@ control network to control the data network. Since we're using UserSwitch on the data network, it should correctly fail over to a backup controller. -We also hack/subclass the CLI slightly so it can talk to -both the control and data networks. +We also use a Mininet Facade to talk to both the +control and data networks from a single CLI. """ from mininet.net import Mininet @@ -31,19 +31,25 @@ class DataController( Controller ): pass class MininetFacade( object ): - "TODO: CLI that can talk to two or more networks" + """Mininet object facade that allows a single CLI to + talk to one or more networks""" def __init__( self, net, *args, **kwargs ): + """Create MininetFacade object. + net: Primary Mininet object + args: unnamed networks passed as arguments + kwargs: named networks passed as arguments""" self.net = net self.nets = [ net ] + list( args ) + kwargs.values() self.nameToNet = kwargs self.nameToNet['net'] = net - # default is first net def __getattr__( self, name ): + "returns attribute from Primary Mininet object" return getattr( self.net, name ) def __getitem__( self, key ): + "returns primary/named networks or node from any net" #search kwargs for net named key if key in self.nameToNet: return self.nameToNet[ key ] @@ -53,26 +59,32 @@ class MininetFacade( object ): return net[ key ] def __iter__( self ): + "Iterate through all nodes in all Mininet objects" for net in self.nets: for node in net: yield node def __len__( self ): + "returns aggregate number of nodes in all nets" count = 0 for net in self.nets: count += len(net) return count def __contains__( self, key ): + "returns True if node is a member of any net" return key in self.keys() def keys( self ): + "returns a list of all node names in all networks" return list( self ) def values( self ): + "returns a list of all nodes in all networks" return [ self[ key ] for key in self ] def items( self ): + "returns (key,value) tuple list for every node in all networks" return zip( self.keys(), self.values() ) # A real control network! @@ -126,8 +138,6 @@ def run(): net.stop() info( '* Stopping Control Network\n' ) - # dataControllers have already been stopped -- now terminate is idempotent - #cnet.hosts = list( set( cnet.hosts ) - set( dataControllers ) ) cnet.stop() if __name__ == '__main__': From a155795837e07d9d9e71f06e66a2278da7637870 Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Mon, 26 Aug 2013 23:36:42 -0700 Subject: [PATCH 042/109] quetzal -> quantal --- util/vm/build.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/util/vm/build.py b/util/vm/build.py index 1537548..153a159 100755 --- a/util/vm/build.py +++ b/util/vm/build.py @@ -9,7 +9,7 @@ Basic idea: -> create base install image if it's missing - download iso if it's missing - install from iso onto image - + build -> create cow disk for new VM, based on base image -> boot it in qemu/kvm with text /serial console @@ -51,10 +51,10 @@ isoURLs = { 'precise64server': 'http://mirrors.kernel.org/ubuntu-releases/12.04/' 'ubuntu-12.04.3-server-amd64.iso', - 'quetzal32server': + 'quantal32server': 'http://mirrors.kernel.org/ubuntu-releases/12.10/' 'ubuntu-12.10-server-i386.iso', - 'quetzal64server': + 'quantal64server': 'http://mirrors.kernel.org/ubuntu-releases/12.10/' 'ubuntu-12.10-server-amd64.iso', 'raring32server': From b79ce2a5498fd6abaded187fc9af4093d8b3ad87 Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Mon, 26 Aug 2013 23:40:30 -0700 Subject: [PATCH 043/109] Clarify actual testing in module comment --- util/vm/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/vm/build.py b/util/vm/build.py index 153a159..beeb9fc 100755 --- a/util/vm/build.py +++ b/util/vm/build.py @@ -16,7 +16,7 @@ Basic idea: -> install Mininet test - -> make codecheck + -> sudo mn --test pingall -> make test release From b55806017ab45768b2e8f6945b70bfb6fdb580ad Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Mon, 26 Aug 2013 23:57:11 -0700 Subject: [PATCH 044/109] Check for `Connected` in checkListening() --- mininet/node.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mininet/node.py b/mininet/node.py index 895c731..3d6dc0e 100644 --- a/mininet/node.py +++ b/mininet/node.py @@ -1143,7 +1143,7 @@ class Controller( Node ): "installed." ) listening = self.cmd( "echo A | telnet -e A %s %d" % ( self.ip, self.port ) ) - if 'Unable' not in listening: + if 'Connected' in listening: servers = self.cmd( 'netstat -atp' ).split( '\n' ) pstr = ':%d ' % self.port clist = servers[ 0:1 ] + [ s for s in servers if pstr in s ] @@ -1238,7 +1238,7 @@ class RemoteController( Controller ): "Warn if remote controller is not accessible" listening = self.cmd( "echo A | telnet -e A %s %d" % ( self.ip, self.port ) ) - if 'Unable' in listening: + if 'Connected' not in listening: warn( "Unable to contact the remote controller" " at %s:%d\n" % ( self.ip, self.port ) ) From e9a835ac55d19f59ad61ee95e23ea18b74b2410b Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Tue, 27 Aug 2013 13:58:23 -0700 Subject: [PATCH 045/109] print usage message for unknown command --- mininet/cli.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/mininet/cli.py b/mininet/cli.py index 5de9beb..3ca190f 100644 --- a/mininet/cli.py +++ b/mininet/cli.py @@ -323,14 +323,13 @@ class CLI( Cmd ): corresponding IP addrs.""" first, args, line = self.parseline( line ) - if not args: - return - if args and len(args) > 0 and args[ -1 ] == '\n': - args = args[ :-1 ] - rest = args.split( ' ' ) if first in self.mn: + if not args: + print "*** Enter a command for node: %s " % first + return node = self.mn[ first ] + rest = args.split( ' ' ) # Substitute IP addresses for node names in command rest = [ self.mn[ arg ].defaultIntf().updateIP() if arg in self.mn else arg @@ -341,7 +340,7 @@ class CLI( Cmd ): node.sendCmd( rest, printPid=( not builtin ) ) self.waitForNode( node ) else: - error( '*** Unknown command: %s\n' % first ) + error( '*** Unknown command: %s\n' % line ) # pylint: enable-msg=R0201 From 2e7d0d4934a216353f86ccba802f3a546fd64092 Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Tue, 27 Aug 2013 14:01:39 -0700 Subject: [PATCH 046/109] fixed indent issue with examples/controllers.py --- examples/controllers.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/controllers.py b/examples/controllers.py index 1fcefb1..d3694bc 100755 --- a/examples/controllers.py +++ b/examples/controllers.py @@ -29,8 +29,7 @@ class MultiSwitch( OVSSwitch ): topo = TreeTopo( depth=2, fanout=2 ) net = Mininet( topo=topo, switch=MultiSwitch, build=False ) for c in [ c0, c1 ]: - net.addController(c) - + net.addController(c) net.build() net.start() CLI( net ) From 0c5aae157a85c7e3feecb00e0d7d862bbebe4f74 Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Tue, 27 Aug 2013 13:59:06 -0700 Subject: [PATCH 047/109] examples -> mininet/examples for code check --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c0c53f2..c989e61 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ MININET = mininet/*.py TEST = mininet/test/*.py -EXAMPLES = examples/*.py +EXAMPLES = mininet/examples/*.py MN = bin/mn BIN = $(MN) PYSRC = $(MININET) $(TEST) $(EXAMPLES) $(BIN) From 445c0959b59450cf712160efe55128df5c817791 Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Tue, 27 Aug 2013 14:23:05 -0700 Subject: [PATCH 048/109] Pass code check (except bogus Popen error) --- examples/bind.py | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/examples/bind.py b/examples/bind.py index d4d28a4..2c317cf 100755 --- a/examples/bind.py +++ b/examples/bind.py @@ -34,20 +34,20 @@ def mountPoints(): mounts.append( mount ) return mounts -def unmountAll( dir=MNRUNDIR ): +def unmountAll( rootdir=MNRUNDIR ): "Unmount all mounts under a directory tree" - dir = realpath( dir ) - # Find all mounts below dir + rootdir = realpath( rootdir ) + # Find all mounts below rootdir # This is subtle because /foo is not # a parent of /foot - dirslash = dir + '/' + dirslash = rootdir + '/' mounts = [ m for m in mountPoints() if m == dir or m.find( dirslash ) == 0 ] # Unmount them from bottom to top mounts.sort( reverse=True ) for mount in mounts: debug( 'Unmounting', mount, '\n' ) - out, err, code = errRun( 'umount', mount ) + _out, err, code = errRun( 'umount', mount ) if code != 0: info( '*** Warning: failed to umount', mount, '\n' ) info( err ) @@ -69,6 +69,7 @@ class HostWithPrivateDirs( Host ): self.unmount = kwargs.pop( 'unmount', True ) Host.__init__( self, name, *args, **kwargs ) self.rundir = '%s/%s' % ( self.mnRunDir, name ) + self.root, self.private = None, None # set in createBindMounts if self.privateDirs: self.privateDirs = [ realpath( d ) for d in self.privateDirs ] self.createBindMounts() @@ -88,28 +89,30 @@ class HostWithPrivateDirs( Host ): def mountPrivateDirs( self ): "Create and bind mount private dirs" - for dir in self.privateDirs: - privateDir = self.private + dir + for dir_ in self.privateDirs: + privateDir = self.private + dir_ errFail( 'mkdir -p ' + privateDir ) - mountPoint = self.root + dir + mountPoint = self.root + dir_ errFail( 'mount -B %s %s' % ( privateDir, mountPoint) ) def mountDirs( self, dirs ): "Mount a list of directories" - for dir in dirs: - mountpoint = self.root + dir + for dir_ in dirs: + mountpoint = self.root + dir_ errFail( 'mount -B %s %s' % - ( dir, mountpoint ) ) + ( dir_, mountpoint ) ) @classmethod - def findRemounts( cls, fstypes=[ 'nfs' ] ): + def findRemounts( cls, fstypes=None ): """Identify mount points in /proc/mounts to remount fstypes: file system types to match""" + if fstypes is None: + fstypes = [ 'nfs' ] dirs = quietRun( 'cat /proc/mounts' ).strip().split( '\n' ) remounts = [] - for dir in dirs: - line = dir.split() + for dir_ in dirs: + line = dir_.split() mountpoint, fstype = line[ 1 ], line[ 2 ] # Don't re-remount directories!!! if mountpoint.find( cls.mnRunDir ) == 0: From 350299786dfa2a5d6ce88e41e4c8f09f1fd3ce24 Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Tue, 27 Aug 2013 14:30:52 -0700 Subject: [PATCH 049/109] code check fixes & add comment spaces --- mininet/net.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/mininet/net.py b/mininet/net.py index 6eaac20..ec068c8 100755 --- a/mininet/net.py +++ b/mininet/net.py @@ -209,16 +209,19 @@ class Mininet( object ): def addController( self, name='c0', controller=None, **params ): """Add controller. controller: Controller class""" - #Get controller class + # Get controller class if not controller: controller = self.controller - #Construct new controller if one is not given + # Construct new controller if one is not given if isinstance(name, Controller): controller_new = name + # Pylint thinks controller is a str() + # pylint: disable=E1103 name = controller_new.name + # pylint: enable=E1103 else: controller_new = controller( name, **params ) - #Add new controller to net + # Add new controller to net if controller_new: # allow controller-less setups self.controllers.append( controller_new ) self.nameToNode[ name ] = controller_new @@ -249,7 +252,8 @@ class Mininet( object ): def __len__( self ): "returns number of nodes in net" - return len( self.hosts ) + len( self.switches ) + len( self.controllers ) + return ( len( self.hosts ) + len( self.switches ) + + len( self.controllers ) ) def __contains__( self, item ): "returns True if net contains named node" From f796f01f3859326ee156b8dfff01f9ccbae1ffff Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Tue, 27 Aug 2013 14:36:10 -0700 Subject: [PATCH 050/109] add spaces to satisfy pylint ;-p --- mininet/topo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mininet/topo.py b/mininet/topo.py index 784095f..2ab455c 100644 --- a/mininet/topo.py +++ b/mininet/topo.py @@ -250,9 +250,9 @@ class LinearTopo(Topo): self.n = n if n == 1: - genHostName = lambda i,j: 'h%s' % i + genHostName = lambda i, j: 'h%s' % i else: - genHostName = lambda i,j: 'h%ss%d' % (j,i) + genHostName = lambda i, j: 'h%ss%d' % (j, i) lastSwitch = None From 9d14c841d7e675818bdacff66ae24dc355516ecd Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Tue, 27 Aug 2013 15:16:03 -0700 Subject: [PATCH 051/109] Pass code check --- mininet/test/test_hifi.py | 23 ++++++++++++++++------- mininet/test/test_nets.py | 9 +++++++++ 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/mininet/test/test_hifi.py b/mininet/test/test_hifi.py index f6ebb71..c9d288b 100755 --- a/mininet/test/test_hifi.py +++ b/mininet/test/test_hifi.py @@ -31,9 +31,12 @@ class SingleSwitchOptionsTopo(Topo): host = self.addHost('h%s' % (h + 1)) self.addLink(host, switch) +# Tell pylint not to complain about calls to other class +# pylint: disable=E1101 class testOptionsTopoCommon( object ): - "Verify ability to create networks with host and link options (common code)." + """Verify ability to create networks with host and link options + (common code).""" switchClass = None # overridden in subclasses @@ -41,7 +44,8 @@ class testOptionsTopoCommon( object ): "Generic topology-with-options test runner." mn = Mininet( topo=SingleSwitchOptionsTopo( n=n, hopts=hopts, lopts=lopts ), - host=CPULimitedHost, link=TCLink, switch=self.switchClass ) + host=CPULimitedHost, link=TCLink, + switch=self.switchClass ) dropped = mn.run( mn.ping ) self.assertEqual( dropped, 0 ) @@ -107,7 +111,8 @@ class testOptionsTopoCommon( object ): REPS = 1 lopts = { 'loss': LOSS_PERCENT, 'use_htb': True } mn = Mininet( topo=SingleSwitchOptionsTopo( n=N, lopts=lopts ), - host=CPULimitedHost, link=TCLink, switch=self.switchClass ) + host=CPULimitedHost, link=TCLink, + switch=self.switchClass ) # Drops are probabilistic, but the chance of no dropped packets is # 1 in 100 million with 4 hops for a link w/99% loss. dropped_total = 0 @@ -123,24 +128,28 @@ class testOptionsTopoCommon( object ): hopts = { 'cpu': 0.5 / N } self.runOptionsTopoTest( N, hopts=hopts, lopts=lopts ) +# pylint: enable=E1101 + class testOptionsTopoOVSKernel( testOptionsTopoCommon, unittest.TestCase ): - "Verify ability to create networks with host and link options (OVS kernel switch)." + """Verify ability to create networks with host and link options + (OVS kernel switch).""" switchClass = OVSSwitch @unittest.skip( 'Skipping OVS user switch test for now' ) class testOptionsTopoOVSUser( testOptionsTopoCommon, unittest.TestCase ): - "Verify ability to create networks with host and link options (OVS user switch)." + """Verify ability to create networks with host and link options + (OVS user switch).""" switchClass = partial( OVSSwitch, datapath='user' ) @unittest.skipUnless( quietRun( 'which ivs-ctl' ), 'IVS is not installed' ) class testOptionsTopoIVS( testOptionsTopoCommon, unittest.TestCase ): - "Verify ability to create networks with host and link options (IVS switch)." + "Verify ability to create networks with host and link options (IVS)." switchClass = IVSSwitch @unittest.skipUnless( quietRun( 'which ofprotocol' ), 'Reference user switch is not installed' ) class testOptionsTopoUserspace( testOptionsTopoCommon, unittest.TestCase ): - "Verify ability to create networks with host and link options (Userspace switch)." + "Verify ability to create networks with host and link options (UserSwitch)." switchClass = UserSwitch if __name__ == '__main__': diff --git a/mininet/test/test_nets.py b/mininet/test/test_nets.py index a56061b..9176646 100755 --- a/mininet/test/test_nets.py +++ b/mininet/test/test_nets.py @@ -13,6 +13,8 @@ from mininet.topo import SingleSwitchTopo, LinearTopo from mininet.log import setLogLevel from mininet.util import quietRun +# Tell pylint not to complain about calls to other class +# pylint: disable=E1101 class testSingleSwitchCommon( object ): "Test ping with single switch topology (common code)." @@ -31,6 +33,8 @@ class testSingleSwitchCommon( object ): dropped = mn.run( mn.ping ) self.assertEqual( dropped, 0 ) +# pylint: enable=E1101 + class testSingleSwitchOVSKernel( testSingleSwitchCommon, unittest.TestCase ): "Test ping with single switch topology (OVS kernel switch)." switchClass = OVSSwitch @@ -51,6 +55,9 @@ class testSingleSwitchUserspace( testSingleSwitchCommon, unittest.TestCase ): switchClass = UserSwitch +# Tell pylint not to complain about calls to other class +# pylint: disable=E1101 + class testLinearCommon( object ): "Test all-pairs ping with LinearNet (common code)." @@ -62,6 +69,8 @@ class testLinearCommon( object ): dropped = mn.run( mn.ping ) self.assertEqual( dropped, 0 ) +# pylint: enable=E1101 + class testLinearOVSKernel( testLinearCommon, unittest.TestCase ): "Test all-pairs ping with LinearNet (OVS kernel switch)." From e69355f78fd0b933ce59d30d222f959cd5ee3742 Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Tue, 27 Aug 2013 15:16:38 -0700 Subject: [PATCH 052/109] One last code check fix: line too long --- mininet/test/test_nets.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mininet/test/test_nets.py b/mininet/test/test_nets.py index 9176646..159ba34 100755 --- a/mininet/test/test_nets.py +++ b/mininet/test/test_nets.py @@ -29,7 +29,8 @@ class testSingleSwitchCommon( object ): def testSingle5( self ): "Ping test on 5-host single-switch topology" - mn = Mininet( SingleSwitchTopo( k=5 ), self.switchClass, Host, Controller ) + mn = Mininet( SingleSwitchTopo( k=5 ), self.switchClass, Host, + Controller ) dropped = mn.run( mn.ping ) self.assertEqual( dropped, 0 ) From 045ef7b801e99a07b179694285aa104881f307bf Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Tue, 27 Aug 2013 15:20:11 -0700 Subject: [PATCH 053/109] Add docstring to satisfy pylint --- examples/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/__init__.py b/examples/__init__.py index fedbfea..4b2e73c 100644 --- a/examples/__init__.py +++ b/examples/__init__.py @@ -1 +1,4 @@ -# Mininet Examples +""" +Mininet Examples +See README for details +""" From 9bfc7c77687a1108f4cc105617fed0f9515a615b Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Tue, 27 Aug 2013 15:21:06 -0700 Subject: [PATCH 054/109] Satisfy pylint --- examples/consoles.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/consoles.py b/examples/consoles.py index 53fc400..3197ca1 100755 --- a/examples/consoles.py +++ b/examples/consoles.py @@ -418,7 +418,8 @@ class ConsoleApp( Frame ): count = len( consoles ) self.setOutputHook( self.updateGraph ) for console in consoles: - #sometimes iperf -sD doesn't return, so we run it in the background instead + # Sometimes iperf -sD doesn't return, + # so we run it in the background instead console.node.cmd( 'iperf -s &' ) i = 0 for console in consoles: From 13554a3d836b8706fb2f366225eba2280d260821 Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Tue, 27 Aug 2013 15:23:46 -0700 Subject: [PATCH 055/109] Minor cleanup --- examples/controlnet.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/controlnet.py b/examples/controlnet.py index dba8756..f9454c7 100755 --- a/examples/controlnet.py +++ b/examples/controlnet.py @@ -109,7 +109,10 @@ class ControlNetwork( Topo ): # Make it Happen!! + def run(): + "Create control and data networks, and invoke the CLI" + info( '* Creating Control Network\n' ) ctopo = ControlNetwork( n=4, dataController=DataController ) cnet = Mininet( topo=ctopo, ipBase='192.168.123.0/24', controller=None ) @@ -131,7 +134,6 @@ def run(): mn = MininetFacade( net, cnet=cnet ) - # run the function passed as an argument CLI( mn ) info( '* Stopping Data Network\n' ) @@ -140,7 +142,7 @@ def run(): info( '* Stopping Control Network\n' ) cnet.stop() + if __name__ == '__main__': setLogLevel( 'info' ) - run() From 20ba29590a3657da5101533d8e7b0b969cd2451d Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Tue, 27 Aug 2013 15:32:03 -0700 Subject: [PATCH 056/109] Add 13.10 (won't work until final) --- util/vm/build.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/util/vm/build.py b/util/vm/build.py index beeb9fc..f5be017 100755 --- a/util/vm/build.py +++ b/util/vm/build.py @@ -63,6 +63,12 @@ isoURLs = { 'raring64server': 'http://mirrors.kernel.org/ubuntu-releases/13.04/' 'ubuntu-13.04-server-amd64.iso', + 'saucy32server': + 'http://mirrors.kernel.org/ubuntu-releases/13.10/' + 'ubuntu-13.10-server-i386.iso', + 'saucy64server': + 'http://mirrors.kernel.org/ubuntu-releases/13.10/' + 'ubuntu-13.10-server-amd64.iso', } logStartTime = time() From 803a1a5489f8a7518db2942359c5a60c2b44e32f Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Tue, 27 Aug 2013 15:59:20 -0700 Subject: [PATCH 057/109] Write build log to file, and detect installation failure --- util/vm/build.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/util/vm/build.py b/util/vm/build.py index f5be017..a2787f3 100755 --- a/util/vm/build.py +++ b/util/vm/build.py @@ -71,13 +71,14 @@ isoURLs = { 'ubuntu-13.10-server-amd64.iso', } -logStartTime = time() +LogStartTime = time() +LogFile = None def log( *args, **kwargs ): """Simple log function: log( message along with local and elapsed time cr: False/0 for no CR""" cr = kwargs.get( 'cr', True ) - elapsed = time() - logStartTime + elapsed = time() - LogStartTime clocktime = strftime( '%H:%M:%S', localtime() ) msg = ' '.join( str( arg ) for arg in args ) output = '%s [ %.3f ] %s' % ( clocktime, elapsed, msg ) @@ -85,6 +86,11 @@ def log( *args, **kwargs ): print output else: print output, + # Optionally mirror to LogFile + if type( LogFile ) is file: + if cr: + output += '\n' + LogFile.write( output ) def run( cmd, **kwargs ): @@ -350,6 +356,9 @@ def installUbuntu( iso, image, logfilename='install.log' ): # Unmount iso and clean up srun( 'umount ' + mnt ) run( 'rmdir ' + mnt ) + if vm.returncode != 0: + raise Exception( 'Ubuntu installation returned error %d' % + vm.returncode ) log( '* UBUNTU INSTALLATION COMPLETED FOR', image ) log( '* Ubuntu installation completed in %.2f seconds ' % elapsed ) @@ -501,6 +510,7 @@ def genVirtImage( name, mem, diskname, disksize ): def build( flavor='raring32server' ): "Build a Mininet VM" + global LogFile start = time() date = strftime( '%y%m%d-%H-%M-%S', localtime()) dir = 'mn-%s-%s' % ( flavor, date ) @@ -509,6 +519,8 @@ def build( flavor='raring32server' ): except: raise Exception( "Failed to create build directory %s" % dir ) os.chdir( dir ) + LogFile = open( 'build.log', 'w' ) + log( '* Logging to ', abspath( LogFile.name ) ) log( '* Created working directory', dir ) image, kernel, initrd = findBaseImage( flavor ) volume = flavor + '.qcow2' From 20005f5bbc5b0595dbadd6d125c7db2ee4bbb2fb Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Tue, 27 Aug 2013 16:00:34 -0700 Subject: [PATCH 058/109] Add a space --- util/vm/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/vm/build.py b/util/vm/build.py index a2787f3..7a22c13 100755 --- a/util/vm/build.py +++ b/util/vm/build.py @@ -520,7 +520,7 @@ def build( flavor='raring32server' ): raise Exception( "Failed to create build directory %s" % dir ) os.chdir( dir ) LogFile = open( 'build.log', 'w' ) - log( '* Logging to ', abspath( LogFile.name ) ) + log( '* Logging to', abspath( LogFile.name ) ) log( '* Created working directory', dir ) image, kernel, initrd = findBaseImage( flavor ) volume = flavor + '.qcow2' From 662f2447e3bcd2f393f2b89f3d703479aa9cb90c Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Tue, 27 Aug 2013 16:13:34 -0700 Subject: [PATCH 059/109] Flush log file output to avoid slow buffering --- util/vm/build.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/util/vm/build.py b/util/vm/build.py index 7a22c13..d3ed1aa 100755 --- a/util/vm/build.py +++ b/util/vm/build.py @@ -71,6 +71,7 @@ isoURLs = { 'ubuntu-13.10-server-amd64.iso', } + LogStartTime = time() LogFile = None @@ -91,6 +92,7 @@ def log( *args, **kwargs ): if cr: output += '\n' LogFile.write( output ) + LogFile.flush() def run( cmd, **kwargs ): From 09b12391313224a89308e1d23936bfd4453a03d6 Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Tue, 27 Aug 2013 17:41:34 -0700 Subject: [PATCH 060/109] fixing comment --- mininet/net.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mininet/net.py b/mininet/net.py index ec068c8..4994f9e 100755 --- a/mininet/net.py +++ b/mininet/net.py @@ -245,8 +245,7 @@ class Mininet( object ): return self.nameToNode[ key ] def __iter__( self ): - "return iterator over nodes" - #or dow we want to iterate of the keys i.e. node.name like a dict + "return iterator over node names" for node in chain( self.hosts, self.switches, self.controllers ): yield node.name From bda54a9aed232ebdb18bee1d51da80e3b8853cf8 Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Tue, 27 Aug 2013 17:41:51 -0700 Subject: [PATCH 061/109] updating INSTALL --- INSTALL | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/INSTALL b/INSTALL index 0b0732a..52980bf 100644 --- a/INSTALL +++ b/INSTALL @@ -15,7 +15,7 @@ like to contribute an installation script, we would welcome it!) 1. Easiest "installation" - use our pre-built VM image! The easiest way to get Mininet running is to start with one of our - pre-built virtual machine images from + pre-built virtual machine images from Boot up the VM image, log in, and follow the instructions on the Mininet web site. @@ -81,6 +81,11 @@ like to contribute an installation script, we would welcome it!) This takes about 4 minutes on our test system. + You can change the directory where the dependencies are installed using + the -s flag. + + mininet/util/install.sh -s -a + 4. Creating your own Mininet/OpenFlow tutorial VM Creating your own Ubuntu Mininet VM for use with the OpenFlow tutorial From 389c7aa5af5a45cf6b982da8186a58910b5b8d75 Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Tue, 27 Aug 2013 17:42:46 -0700 Subject: [PATCH 062/109] install.sh: making BUILD_DIR more robust --- util/install.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/util/install.sh b/util/install.sh index 5ffbc92..cf74dc6 100755 --- a/util/install.sh +++ b/util/install.sh @@ -10,15 +10,15 @@ set -e set -o nounset # Get directory containing mininet folder -MININET_DIR="$( cd -P "$( dirname "${BASH_SOURCE[0]}" )/../.." && pwd )" +MININET_DIR="$( cd -P "$( dirname "${BASH_SOURCE[0]}" )/../.." && pwd -P )" # Set up build directory, which by default is the working directory # unless the working directory is a subdirectory of mininet, # in which case we use the directory containing mininet -BUILD_DIR=$PWD -case $PWD in +BUILD_DIR="$(pwd -P)" +case $BUILD_DIR in $MININET_DIR/*) BUILD_DIR=$MININET_DIR;; # currect directory is a subdirectory - *) BUILD_DIR=$PWD;; + *) BUILD_DIR=$BUILD_DIR;; esac # Location of CONFIG_NET_NS-enabled kernel(s) From 92b51563b55653b6557baf6457833bda593f10d4 Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Tue, 27 Aug 2013 16:26:56 -0700 Subject: [PATCH 063/109] Remove extra ` --- INSTALL | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/INSTALL b/INSTALL index 52980bf..8771678 100644 --- a/INSTALL +++ b/INSTALL @@ -113,7 +113,7 @@ like to contribute an installation script, we would welcome it!) modules (e.g. tun and ofdatapath for the reference kernel implementation) must be loaded. - * Python, `bash`, `ping`, `iperf`, etc.` + * Python, `bash`, `ping`, `iperf`, etc. * Root privileges (required for network device access) From 9de7bd666d6387d17123c6b0ac5399c64bcb01c3 Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Tue, 27 Aug 2013 16:34:42 -0700 Subject: [PATCH 064/109] Minor additions and edits --- README.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 90ead38..3c965a9 100644 --- a/README.md +++ b/README.md @@ -78,19 +78,21 @@ several new features, including: * Automatically detecting and adjusting resource limits * Automatic cleanup on failure of the `mn` command * Preliminary support for running OVS in user space mode -* Preliminary support for the Indigo Virtual Switch (IVSSwitch) +* Preliminary support (`IVSSwitch()`) for the Indigo Virtual Switch +* support for installing the OpenFlow 1.3 versions of the reference + user switch and NOX from CPqD * The ability to import modules from mininet.examples -We have provided several new examples which can also be -imported to provide useful functionality, including: +We have provided several new examples (which can easily be +imported to provide useful functionality) including: +* Modeling separate control and data networks * Connecting Mininet hosts the internet (or a LAN) using NAT -* Modeling control as well as data networks * Creating per-host custom directories using bind mounts -Note that these are experimental features which may "graduate" -into mainline Mininet in the future, so they should not be -considered a stable part of the Mininet API! +Note that examples contain experimental features which might +"graduate" into mainline Mininet in the future, but they should +not be considered a stable part of the Mininet API! ### Installation From 3027856c7bd1bb70b2a7d00960cb79ca72eb5abc Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Tue, 27 Aug 2013 17:18:58 -0700 Subject: [PATCH 065/109] Find wireshark dir using find (fix for 13.10) --- util/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/install.sh b/util/install.sh index cf74dc6..e39f174 100755 --- a/util/install.sh +++ b/util/install.sh @@ -255,7 +255,7 @@ function wireshark { export WIRESHARK=/usr/include/wireshark scons # libwireshark0/ on 11.04; libwireshark1/ on later - WSDIR=`ls -d /usr/lib/wireshark/libwireshark* | head -1` + WSDIR=`find /usr/lib -type d -name 'libwireshark*' | head -1` WSPLUGDIR=$WSDIR/plugins/ sudo cp openflow.so $WSPLUGDIR echo "Copied openflow plugin to $WSPLUGDIR" From d82e0ef5c29e769c92e9b0a142106016574f2935 Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Tue, 27 Aug 2013 18:29:45 -0700 Subject: [PATCH 066/109] Add mtools to dependencies --- util/vm/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/vm/build.py b/util/vm/build.py index d3ed1aa..3366cff 100755 --- a/util/vm/build.py +++ b/util/vm/build.py @@ -115,7 +115,7 @@ def depend(): ' kvm cloud-utils genisoimage qemu-kvm qemu-utils' ' e2fsprogs ' ' landscape-client' - ' python-setuptools' ) + ' python-setuptools mtools' ) run( 'sudo easy_install pexpect' ) From f7abd084c699f5daf5eb4eb74bdef6da5985a6ff Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Tue, 27 Aug 2013 18:40:54 -0700 Subject: [PATCH 067/109] Added socat, iperf, cgroup-bin to dependencies --- debian/control | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/debian/control b/debian/control index e2297ae..b89b3b3 100644 --- a/debian/control +++ b/debian/control @@ -17,10 +17,13 @@ Architecture: any Depends: openvswitch-switch, telnet, + socat, + iperf, + cgroup-bin, ${misc:Depends}, ${python:Depends}, ${shlibs:Depends} -Recommends: iperf, openvswitch-controller, socat +Recommends: openvswitch-controller Description: Process-based network emulator Mininet is a network emulator which uses lightweight virtualization to create virtual networks for rapid From 1ea9d7d4dee7fa383f48a46c01f88e2db8ab7a89 Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Wed, 4 Sep 2013 13:22:22 -0700 Subject: [PATCH 068/109] Update copyright for 2013 --- debian/copyright | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/copyright b/debian/copyright index 4c9fd63..5b43112 100644 --- a/debian/copyright +++ b/debian/copyright @@ -3,7 +3,7 @@ Upstream-Name: mininet Source: https://github.com/mininet/mininet Files: * -Copyright: 2012 Open Networking Laboratory, +Copyright: 2012-2013 Open Networking Laboratory, 2009-2012 Bob Lantz, 2009-2012 The Board of Trustees of the Leland Stanford Junior University From d4279559fa5cb2aa00adb2dec4c51848db9d5294 Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Thu, 5 Sep 2013 23:33:30 -0700 Subject: [PATCH 069/109] Add options; generate virtimage file (in progress) --- util/vm/build.py | 102 +++++++++++++++++++++++++++++++---------------- 1 file changed, 67 insertions(+), 35 deletions(-) diff --git a/util/vm/build.py b/util/vm/build.py index 3366cff..1cac35a 100755 --- a/util/vm/build.py +++ b/util/vm/build.py @@ -30,7 +30,8 @@ import os from os import stat, path from stat import ST_MODE from os.path import abspath -from sys import exit, argv +from sys import exit, stdout +import re from glob import glob from subprocess import check_output, call, Popen from tempfile import mkdtemp @@ -42,6 +43,10 @@ pexpect = None # For code check - imported dynamically # boot can be slooooow!!!! need to debug/optimize somehow TIMEOUT=600 +# Some configuration +LogToConsole = False # VM output to console rather than log file +SaveQCOW2 = False # Save QCOW2 image rather than deleting it + VMImageDir = os.environ[ 'HOME' ] + '/vm-images' isoURLs = { @@ -307,22 +312,22 @@ def makeKickstartFloppy(): return floppy, kickstart, preseed -def kvmFor( filepath ): - "Guess kvm version for file path" +def archFor( filepath ): + "Guess architecture for file path" name = path.basename( filepath ) if '64' in name: - kvm = 'qemu-system-x86_64' + arch = 'x86_64' elif 'i386' in name or '32' in name: - kvm = 'qemu-system-i386' + arch = 'i386' else: log( "Error: can't discern CPU for file name", name ) exit( 1 ) - return kvm + return arch def installUbuntu( iso, image, logfilename='install.log' ): "Install Ubuntu from iso onto image" - kvm = kvmFor( iso ) + kvm = 'qemu-system-' + archFor( iso ) floppy, kickstart, preseed = makeKickstartFloppy() # Mount iso so we can use its kernel mnt = mkdtemp() @@ -349,11 +354,15 @@ def installUbuntu( iso, image, logfilename='install.log' ): log( '* INSTALLING UBUNTU FROM', iso, 'ONTO', image ) log( ' '.join( cmd ) ) log( '* logging to', abspath( logfilename ) ) - logfile = open( logfilename, 'w' ) - vm = Popen( cmd, stdout=logfile, stderr=logfile ) + params = {} + if not LogToConsole: + logfile = open( logfilename, 'w' ) + params = { 'stdout': logfile, 'stderr': logfile } + vm = Popen( cmd, **params ) log( '* Waiting for installation to complete') vm.wait() - logfile.close() + if not LogToConsole: + logfile.close() elapsed = time() - ubuntuStart # Unmount iso and clean up srun( 'umount ' + mnt ) @@ -374,8 +383,8 @@ def boot( cow, kernel, initrd, logfile ): # pexpect might not be installed until after depend() is called global pexpect import pexpect - kvm = kvmFor( kernel ) - cmd = [ 'sudo', kvm, + arch = archFor( kernel ) + cmd = [ 'sudo', 'qemu-system-' + arch, '-machine accel=kvm', '-nographic', '-netdev user,id=mnbuild', @@ -478,7 +487,7 @@ VirtImageXML = """ - %s/arch> + %s @@ -498,20 +507,30 @@ VirtImageXML = """ """ + def genVirtImage( name, mem, diskname, disksize ): "Generate and return virt-image file name.xml" # Our strategy is going to be: create a # virt-image file and then use virt-convert to convert # it to an .ovf file xmlfile = name + '.xml' - xmltext = VirtImageXML % ( name, mem, diskname, disksize ) + arch = archFor( name ) + xmltext = VirtImageXML % ( name, arch, mem, diskname, disksize ) with open( xmlfile, 'w+' ) as f: f.write( xmltext ) return xmlfile +def qcow2size( qcow2 ): + "Return virtual disk size (in bytes) of qcow2 image" + output = check_output( [ 'file', qcow2 ] ) + assert 'QCOW' in output + bytes = int( re.findall( '(\d+) bytes', output )[ 0 ] ) + return bytes + + def build( flavor='raring32server' ): - "Build a Mininet VM" + "Build a Mininet VM; return vmdk and vdisk size" global LogFile start = time() date = strftime( '%y%m%d-%H-%M-%S', localtime()) @@ -528,14 +547,21 @@ def build( flavor='raring32server' ): volume = flavor + '.qcow2' run( 'qemu-img create -f qcow2 -b %s %s' % ( image, volume ) ) log( '* VM image for', flavor, 'created as', volume ) - logfile = open( flavor + '.log', 'w+' ) + if LogToConsole: + logfile = stdout + else: + logfile = open( flavor + '.log', 'w+' ) log( '* Logging results to', abspath( logfile.name ) ) vm = boot( volume, kernel, initrd, logfile ) interact( vm ) + size = qcow2size( volume ) vmdk = convert( volume, basename=flavor ) - log( '* Removing qcow2 volume', volume ) - os.remove( volume ) + if not SaveQCOW2: + log( '* Removing qcow2 volume', volume ) + os.remove( volume ) log( '* Converted VM image stored as', abspath( vmdk ) ) + vimage = genVirtImage( flavor, mem=512, diskname=vmdk, disksize=size ) + log( '* Generated virtimage file as', vimage ) end = time() elapsed = end - start log( '* Results logged to', abspath( logfile.name ) ) @@ -543,44 +569,50 @@ def build( flavor='raring32server' ): log( '* %s VM build DONE!!!!! :D' % flavor ) os.chdir( '..' ) - -def listFlavors(): - "List valid build flavors" - print '\nvalid build flavors:', ' '.join( isoURLs ), '\n' +def buildFlavorString(): + "Return string listing valid build flavors" + return 'valid build flavors: %s' % ' '.join( sorted( isoURLs ) ) def parseArgs(): "Parse command line arguments and run" - parser = argparse.ArgumentParser( description='Mininet VM build script' ) - parser.add_argument( '--depend', action='store_true', + global LogToConsole + parser = argparse.ArgumentParser( description='Mininet VM build script', + epilog=buildFlavorString() ) + parser.add_argument( '-v', '--verbose', action='store_true', + help='send VM output to console rather than log file' ) + parser.add_argument( '-d', '--depend', action='store_true', help='install dependencies for this script' ) - parser.add_argument( '--list', action='store_true', + parser.add_argument( '-l', '--list', action='store_true', help='list valid build flavors' ) - parser.add_argument( '--clean', action='store_true', + parser.add_argument( '-c', '--clean', action='store_true', help='clean up leftover build junk (e.g. qemu-nbd)' ) + parser.add_argument( '-q', '--qcow2', action='store_true', + help='save qcow2 image rather than deleting it' ) parser.add_argument( 'flavor', nargs='*', - help='VM flavor to build (e.g. raring32server)' ) - args = parser.parse_args( argv ) + help='VM flavor(s) to build (e.g. raring32server)' ) + args = parser.parse_args() if args.depend: depend() if args.list: - listFlavors() + print buildFlavorString() if args.clean: cleanup() - flavors = args.flavor[ 1: ] - for flavor in flavors: + if args.verbose: + LogToConsole = True + for flavor in args.flavor: if flavor not in isoURLs: - parser.print_help() - listFlavors() + print "Unknown build flavor:", flavor + print buildFlavorString() break # try: build( flavor ) # except Exception as e: # log( '* BUILD FAILED with exception: ', e ) # exit( 1 ) - if not ( args.depend or args.list or args.clean or flavors ): + if not ( args.depend or args.list or args.clean or args.flavor ): parser.print_help() - listFlavors() + if __name__ == '__main__': parseArgs() From 0038720c01d8f2257910d82f4d20533f39d4a461 Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Sat, 7 Sep 2013 16:53:48 -0700 Subject: [PATCH 070/109] Add generateOVF to finally create the OVF descriptor file! --- util/vm/build.py | 169 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 126 insertions(+), 43 deletions(-) diff --git a/util/vm/build.py b/util/vm/build.py index 1cac35a..bcd8726 100755 --- a/util/vm/build.py +++ b/util/vm/build.py @@ -28,15 +28,16 @@ Basic idea: import os from os import stat, path -from stat import ST_MODE +from stat import ST_MODE, ST_SIZE from os.path import abspath -from sys import exit, stdout +from sys import exit, stdout, argv import re from glob import glob from subprocess import check_output, call, Popen from tempfile import mkdtemp from time import time, strftime, localtime import argparse +from distutils.spawn import find_executable pexpect = None # For code check - imported dynamically @@ -46,6 +47,7 @@ TIMEOUT=600 # Some configuration LogToConsole = False # VM output to console rather than log file SaveQCOW2 = False # Save QCOW2 image rather than deleting it +NoKVM = False # Don't use kvm and use emulation instead VMImageDir = os.environ[ 'HOME' ] + '/vm-images' @@ -104,6 +106,10 @@ def run( cmd, **kwargs ): "Convenient interface to check_output" log( '-', cmd ) cmd = cmd.split() + arg0 = cmd[ 0 ] + if not find_executable( arg0 ): + raise Exception( 'Cannot find executable "%s";' % arg0 + + 'you might try %s --depend' % argv[ 0 ] ) return check_output( cmd, **kwargs ) @@ -112,6 +118,9 @@ def srun( cmd, **kwargs ): return run( 'sudo ' + cmd, **kwargs ) +# BL: we should probably have a "checkDepend()" which +# checks to make sure all dependencies are satisfied! + def depend(): "Install package dependencies" log( '* Installing package dependencies' ) @@ -334,8 +343,12 @@ def installUbuntu( iso, image, logfilename='install.log' ): srun( 'mount %s %s' % ( iso, mnt ) ) kernel = path.join( mnt, 'install/vmlinuz' ) initrd = path.join( mnt, 'install/initrd.gz' ) + if NoKVM: + accel = 'tcg' + else: + accel = 'kvm' cmd = [ 'sudo', kvm, - '-machine', 'accel=kvm', + '-machine', 'accel=%s' % accel, '-nographic', '-netdev', 'user,id=mnbuild', '-device', 'virtio-net,netdev=mnbuild', @@ -384,8 +397,12 @@ def boot( cow, kernel, initrd, logfile ): global pexpect import pexpect arch = archFor( kernel ) + if NoKVM: + accel = 'tcg' + else: + accel = 'kvm' cmd = [ 'sudo', 'qemu-system-' + arch, - '-machine accel=kvm', + '-machine accel=%s' % accel, '-nographic', '-netdev user,id=mnbuild', '-device virtio-net,netdev=mnbuild', @@ -478,47 +495,109 @@ def convert( cow, basename ): return vmdk -# Template for virt-image(5) file +# Template for OVF - a very verbose format! +# In the best of all possible worlds, we might use an XML +# library to generate this, but a template is easier and +# possibly more concise! -VirtImageXML = """ - - - %s - - - - %s - - - - - - - - 1 - %s - - - - - - - - +OVFTemplate = """ + + + + + + +Virtual disk information + + + +The list of logical networks + +The nat network + + + +A Mininet Virtual Machine (%s) +mininet-vm + +Virtual hardware requirements + +hertz * 10^6 +Number of Virtual CPUs +1 virtual CPU(s) +1 +3 +1 + + +byte * 2^20 +Memory Size +%dMB of memory +2 +4 +%d + + +0 +scsiController0 +SCSI Controller +scsiController0 +4 +lsilogic +6 + + +0 +disk1 +ovf:/disk/vmdisk1 +11 +4 +17 + + +2 +true +nat +E1000 ethernet adapter on nat +ethernet0 +12 +E1000 +10 + + +0 +usb +USB Controller +usb +9 +23 + + + + """ -def genVirtImage( name, mem, diskname, disksize ): - "Generate and return virt-image file name.xml" - # Our strategy is going to be: create a - # virt-image file and then use virt-convert to convert - # it to an .ovf file - xmlfile = name + '.xml' - arch = archFor( name ) - xmltext = VirtImageXML % ( name, arch, mem, diskname, disksize ) - with open( xmlfile, 'w+' ) as f: +def generateOVF( name, diskname, disksize, mem=1024 ): + """Generate (and return) OVF file "name.ovf" + name: root name of OVF file to generate + diskname: name of disk file + disksize: size of virtual disk in bytes + mem: VM memory size in MB""" + ovf = name + '.ovf' + filesize = stat( diskname )[ ST_SIZE ] + # OVFTemplate uses the memory size twice in a row + xmltext = OVFTemplate % ( diskname, filesize, disksize, name, mem, mem ) + with open( ovf, 'w+' ) as f: f.write( xmltext ) - return xmlfile + return ovf def qcow2size( qcow2 ): @@ -560,8 +639,8 @@ def build( flavor='raring32server' ): log( '* Removing qcow2 volume', volume ) os.remove( volume ) log( '* Converted VM image stored as', abspath( vmdk ) ) - vimage = genVirtImage( flavor, mem=512, diskname=vmdk, disksize=size ) - log( '* Generated virtimage file as', vimage ) + ovf = generateOVF( diskname=vmdk, disksize=size, name=flavor, mem=1024 ) + log( '* Generated OVF descriptor file', ovf ) end = time() elapsed = end - start log( '* Results logged to', abspath( logfile.name ) ) @@ -576,7 +655,7 @@ def buildFlavorString(): def parseArgs(): "Parse command line arguments and run" - global LogToConsole + global LogToConsole, NoKVM parser = argparse.ArgumentParser( description='Mininet VM build script', epilog=buildFlavorString() ) parser.add_argument( '-v', '--verbose', action='store_true', @@ -589,6 +668,8 @@ def parseArgs(): help='clean up leftover build junk (e.g. qemu-nbd)' ) parser.add_argument( '-q', '--qcow2', action='store_true', help='save qcow2 image rather than deleting it' ) + parser.add_argument( '-n', '--nokvm', action='store_true', + help="Don't use kvm - use tcg emulation instead" ) parser.add_argument( 'flavor', nargs='*', help='VM flavor(s) to build (e.g. raring32server)' ) args = parser.parse_args() @@ -600,6 +681,8 @@ def parseArgs(): cleanup() if args.verbose: LogToConsole = True + if args.nokvm: + NoKVM = True for flavor in args.flavor: if flavor not in isoURLs: print "Unknown build flavor:", flavor From e02cdc0c54c12ac783de552ba853e191edae34ed Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Sat, 7 Sep 2013 17:11:17 -0700 Subject: [PATCH 071/109] XML file cannot begin with a newline :( --- util/vm/build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/util/vm/build.py b/util/vm/build.py index bcd8726..bd04151 100755 --- a/util/vm/build.py +++ b/util/vm/build.py @@ -499,9 +499,9 @@ def convert( cow, basename ): # In the best of all possible worlds, we might use an XML # library to generate this, but a template is easier and # possibly more concise! +# Warning: XML file cannot begin with a newline! -OVFTemplate = """ - +OVFTemplate = """ Date: Sat, 7 Sep 2013 17:19:25 -0700 Subject: [PATCH 072/109] Add 'mininet' prefix to output files --- util/vm/build.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/util/vm/build.py b/util/vm/build.py index bd04151..8f1610d 100755 --- a/util/vm/build.py +++ b/util/vm/build.py @@ -502,7 +502,7 @@ def convert( cow, basename ): # Warning: XML file cannot begin with a newline! OVFTemplate = """ - Date: Tue, 10 Sep 2013 16:00:47 -0700 Subject: [PATCH 073/109] Added --test option to boot and test a VM --- util/vm/build.py | 127 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 95 insertions(+), 32 deletions(-) diff --git a/util/vm/build.py b/util/vm/build.py index 8f1610d..b7db867 100755 --- a/util/vm/build.py +++ b/util/vm/build.py @@ -34,7 +34,7 @@ from sys import exit, stdout, argv import re from glob import glob from subprocess import check_output, call, Popen -from tempfile import mkdtemp +from tempfile import mkdtemp, NamedTemporaryFile from time import time, strftime, localtime import argparse from distutils.spawn import find_executable @@ -51,6 +51,8 @@ NoKVM = False # Don't use kvm and use emulation instead VMImageDir = os.environ[ 'HOME' ] + '/vm-images' +Prompt = '\$ ' # Shell prompt that pexpect will wait for + isoURLs = { 'precise32server': 'http://mirrors.kernel.org/ubuntu-releases/12.04/' @@ -141,8 +143,11 @@ def popen( cmd ): def remove( fname ): - "rm -f fname" - return run( 'rm -f %s' % fname ) + "Remove a file, ignoring errors" + try: + os.remove( fname ) + except OSError: + pass def findiso( flavor ): @@ -186,10 +191,10 @@ def detachNBD( nbd ): srun( 'qemu-nbd -d ' + nbd ) -def extractKernel( image, flavor ): +def extractKernel( image, flavor, imageDir=VMImageDir ): "Extract kernel and initrd from base image" - kernel = path.join( VMImageDir, flavor + '-vmlinuz' ) - initrd = path.join( VMImageDir, flavor + '-initrd' ) + kernel = path.join( imageDir, flavor + '-vmlinuz' ) + initrd = path.join( imageDir, flavor + '-initrd' ) if path.exists( kernel ) and ( stat( image )[ ST_MODE ] & 0777 ) == 0444: # If kernel is there, then initrd should also be there return kernel, initrd @@ -384,7 +389,7 @@ def installUbuntu( iso, image, logfilename='install.log' ): raise Exception( 'Ubuntu installation returned error %d' % vm.returncode ) log( '* UBUNTU INSTALLATION COMPLETED FOR', image ) - log( '* Ubuntu installation completed in %.2f seconds ' % elapsed ) + log( '* Ubuntu installation completed in %.2f seconds' % elapsed ) def boot( cow, kernel, initrd, logfile ): @@ -419,9 +424,8 @@ def boot( cow, kernel, initrd, logfile ): return vm -def interact( vm ): - "Interact with vm, which is a pexpect object" - prompt = '\$ ' +def login( vm ): + "Log in to vm (pexpect object)" log( '* Waiting for login prompt' ) vm.expect( 'login: ' ) log( '* Logging in' ) @@ -431,6 +435,41 @@ def interact( vm ): log( '* Sending password' ) vm.sendline( 'mininet' ) log( '* Waiting for login...' ) + + +def sanityTest( vm ): + "Run Mininet sanity test (pingall) in vm" + vm.sendline( 'sudo mn --test pingall' ) + if vm.expect( [ ' 0% dropped', pexpect.TIMEOUT ], timeout=45 ) == 0: + log( '* Sanity check OK' ) + else: + log( '* Sanity check FAILED' ) + + +def coreTest( vm, prompt=Prompt ): + "Run core tests (make test) in VM" + log( '* Making sure cgroups are mounted' ) + vm.sendline( 'sudo service cgroup-lite restart' ) + vm.expect( prompt ) + vm.sendline( 'sudo cgroups-mount' ) + vm.expect( prompt ) + log( '* Running make test' ) + vm.sendline( 'cd ~/mininet; sudo make test' ) + # We should change "make test" to report the number of + # successful and failed tests. For now, we have to + # know the time for each test, which means that this + # script will have to change as we add more tests. + for test in range( 0, 2 ): + if vm.expect( [ 'OK', 'FAILED', pexpect.TIMEOUT ], timeout=180 ) == 0: + log( '* Test', test, 'OK' ) + else: + log( '* Test', test, 'FAILED' ) + + +def interact( vm, prompt=Prompt ): + "Interact with vm, which is a pexpect object" + login( vm ) + log( '* Waiting for login...' ) vm.expect( prompt ) log( '* Sending hostname command' ) vm.sendline( 'hostname' ) @@ -451,28 +490,9 @@ def interact( vm ): log( '* Completed successfully' ) vm.expect( prompt ) log( '* Testing Mininet' ) - vm.sendline( 'sudo mn --test pingall' ) - if vm.expect( [ ' 0% dropped', pexpect.TIMEOUT ], timeout=45 ) == 0: - log( '* Sanity check OK' ) - else: - log( '* Sanity check FAILED' ) + sanityTest( vm ) vm.expect( prompt ) - log( '* Making sure cgroups are mounted' ) - vm.sendline( 'sudo service cgroup-lite restart' ) - vm.expect( prompt ) - vm.sendline( 'sudo cgroups-mount' ) - vm.expect( prompt ) - log( '* Running make test' ) - vm.sendline( 'cd ~/mininet; sudo make test' ) - # We should change "make test" to report the number of - # successful and failed tests. For now, we have to - # know the time for each test, which means that this - # script will have to change as we add more tests. - for test in range( 0, 2 ): - if vm.expect( [ 'OK', 'FAILED', pexpect.TIMEOUT ], timeout=180 ) == 0: - log( '* Test', test, 'OK' ) - else: - log( '* Test', test, 'FAILED' ) + coreTest( vm ) vm.expect( prompt ) log( '* Shutting down' ) vm.sendline( 'sync; sudo shutdown -h now' ) @@ -649,6 +669,44 @@ def build( flavor='raring32server' ): log( '* %s VM build DONE!!!!! :D' % flavor ) os.chdir( '..' ) + +def bootAndTest( image, tests=None ): + """Boot and test VM + tests: list of tests (default: sanityTest, coreTest)""" + bootTestStart = time() + if tests is None: + tests = [ sanityTest, coreTest ] + basename = path.basename( image ) + image = abspath( image ) + tmpdir = mkdtemp( prefix='test-' + basename ) + cow = path.join( tmpdir, image + '-cow.qcow2' ) + log( '* Creating COW disk' ) + run( 'qemu-img create -f qcow2 -b %s %s' % ( image, cow ) ) + log( '* Extracting kernel and initrd' ) + kernel, initrd = extractKernel( image, flavor=basename, imageDir=tmpdir ) + if LogToConsole: + logfile = stdout + else: + logfile = NamedTemporaryFile( prefix=image, delete=False ) + log( '* Logging VM output to', logfile.name ) + vm = boot( cow=cow, kernel=kernel, initrd=initrd, logfile=logfile ) + prompt = '\$ ' + login( vm ) + log( '* Waiting for VM boot and login' ) + vm.expect( prompt ) + for test in tests: + test( vm ) + vm.expect( prompt ) + log( '* Shutting down' ) + vm.sendline( 'sudo shutdown -h now ' ) + log( '* Waiting for shutdown' ) + vm.wait() + log( '* Removing temporary dir', tmpdir ) + srun( 'rm -rf ' + tmpdir ) + elapsed = time() - bootTestStart + log( '* Boot and test completed in %.2f seconds' % elapsed ) + + def buildFlavorString(): "Return string listing valid build flavors" return 'valid build flavors: %s' % ' '.join( sorted( isoURLs ) ) @@ -671,6 +729,8 @@ def parseArgs(): help='save qcow2 image rather than deleting it' ) parser.add_argument( '-n', '--nokvm', action='store_true', help="Don't use kvm - use tcg emulation instead" ) + parser.add_argument( '-t', '--test', metavar='image', action='append', + help='Boot and test a VM image' ) parser.add_argument( 'flavor', nargs='*', help='VM flavor(s) to build (e.g. raring32server)' ) args = parser.parse_args() @@ -694,7 +754,10 @@ def parseArgs(): # except Exception as e: # log( '* BUILD FAILED with exception: ', e ) # exit( 1 ) - if not ( args.depend or args.list or args.clean or args.flavor ): + for image in args.test: + bootAndTest( image ) + if not ( args.depend or args.list or args.clean or args.flavor + or args.test ): parser.print_help() From 39058432570246a4c7d4cbe2e3575cfb0d30f678 Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Mon, 9 Sep 2013 19:21:56 -0700 Subject: [PATCH 074/109] accept command line args in baresshd.py --- examples/baresshd.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/examples/baresshd.py b/examples/baresshd.py index a714edb..081a812 100755 --- a/examples/baresshd.py +++ b/examples/baresshd.py @@ -2,6 +2,7 @@ "This example doesn't use OpenFlow, but attempts to run sshd in a namespace." +import sys from mininet.node import Host from mininet.util import ensureRoot @@ -27,6 +28,10 @@ f.write( 'Welcome to %s at %s\n' % ( h1.name, h1.IP() ) ) f.close() print "*** Running sshd" -h1.cmd( '/usr/sbin/sshd -o "Banner /tmp/%s.banner"' % h1.name ) +cmd = '/usr/sbin/sshd -o "Banner /tmp/%s.banner"' % h1.name +# add arguments from the command line +if len( sys.argv ) > 1: + cmd += ' ' + ' '.join( sys.argv[ 1: ] ) +h1.cmd( cmd ) print "*** You may now ssh into", h1.name, "at", h1.IP() From cfb6bf95a367d4984d101bc3a6c98bab96374db1 Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Mon, 9 Sep 2013 19:22:41 -0700 Subject: [PATCH 075/109] adding commandline args to UserSwitch in controlnet, examples of partial --- examples/controlnet.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/controlnet.py b/examples/controlnet.py index f9454c7..d5a8781 100755 --- a/examples/controlnet.py +++ b/examples/controlnet.py @@ -14,6 +14,8 @@ We also use a Mininet Facade to talk to both the control and data networks from a single CLI. """ +from functools import partial + from mininet.net import Mininet from mininet.node import Controller, UserSwitch from mininet.cli import CLI @@ -124,7 +126,8 @@ def run(): info( '* Creating Data Network\n' ) topo = TreeTopo( depth=2, fanout=2 ) # UserSwitch so we can easily test failover - net = Mininet( topo=topo, switch=UserSwitch, controller=None ) + sw = partial( UserSwitch, opts='--inactivity-probe=1 --max-backoff=1' ) + net = Mininet( topo=topo, switch=sw, controller=None ) info( '* Adding Controllers to Data Network\n' ) for host in cnet.hosts: if isinstance(host, Controller): From 220376b6e214e11ef8aa2cd98e9f4ebf22057ebc Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Mon, 9 Sep 2013 19:23:14 -0700 Subject: [PATCH 076/109] hwintf.py: allow intf to be specified in cmd line --- examples/hwintf.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/examples/hwintf.py b/examples/hwintf.py index e5960d9..1e010fd 100755 --- a/examples/hwintf.py +++ b/examples/hwintf.py @@ -5,7 +5,7 @@ This example shows how to add an interface (for example a real hardware interface) to a network after the network is created. """ -import re +import re, sys from mininet.cli import CLI from mininet.log import setLogLevel, info, error @@ -28,7 +28,10 @@ def checkIntf( intf ): if __name__ == '__main__': setLogLevel( 'info' ) - intfName = 'eth1' + # try to get hw intf from the command line; by default, use eth1 + intfName = sys.argv[ 1 ] if len( sys.argv ) > 1 else 'eth1' + info( '*** Connecting to hw intf: %s' % intfName ) + info( '*** Checking', intfName, '\n' ) checkIntf( intfName ) From b605cf74d287126857cd10803b444efaba8c1532 Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Mon, 9 Sep 2013 19:24:11 -0700 Subject: [PATCH 077/109] style in multitest --- examples/multitest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/multitest.py b/examples/multitest.py index bcb40f7..b50acb2 100755 --- a/examples/multitest.py +++ b/examples/multitest.py @@ -22,7 +22,7 @@ if __name__ == '__main__': info( "*** Initializing Mininet and kernel modules\n" ) OVSKernelSwitch.setup() info( "*** Creating network\n" ) - network = Mininet( TreeTopo( depth=2, fanout=2 ), switch=OVSKernelSwitch) + network = Mininet( TreeTopo( depth=2, fanout=2 ), switch=OVSKernelSwitch ) info( "*** Starting network\n" ) network.start() info( "*** Running ping test\n" ) From 9a73dcad535845519ae0fb6c8ce0f69cb394c751 Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Mon, 9 Sep 2013 19:25:18 -0700 Subject: [PATCH 078/109] fixed print format in popenpoll --- examples/popenpoll.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/popenpoll.py b/examples/popenpoll.py index c581c27..33478ae 100755 --- a/examples/popenpoll.py +++ b/examples/popenpoll.py @@ -23,7 +23,7 @@ def pmonitorTest( N=3, seconds=10 ): endTime = time() + seconds for h, line in pmonitor( popens, timeoutms=500 ): if h: - print '%s: %s' % ( h.name, line ), + print '<%s>: %s' % ( h.name, line ), if time() >= endTime: for p in popens.values(): p.send_signal( SIGINT ) From 5a646a0d20c50d61d942c63745cc2b844c5135a2 Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Mon, 9 Sep 2013 19:27:02 -0700 Subject: [PATCH 079/109] sshd.py: allow sshd args to be passed via commandline --- examples/sshd.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/examples/sshd.py b/examples/sshd.py index 2bedb9c..3c94359 100755 --- a/examples/sshd.py +++ b/examples/sshd.py @@ -16,6 +16,8 @@ demonstrates: - running server processes (sshd in this case) on hosts """ +import sys + from mininet.net import Mininet from mininet.cli import CLI from mininet.log import lg @@ -68,4 +70,6 @@ def sshd( network, cmd='/usr/sbin/sshd', opts='-D' ): if __name__ == '__main__': lg.setLogLevel( 'info') net = TreeNet( depth=1, fanout=4, switch=OVSKernelSwitch ) - sshd( net ) + # get sshd args from the command line; default: -D + opts = ' '.join( sys.argv[ 1: ] ) if len( sys.argv ) > 1 else '-D' + sshd( net, opts=opts ) From a46fae06872ef756f70d8f4a9e131c6b9e8c3264 Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Mon, 9 Sep 2013 19:28:12 -0700 Subject: [PATCH 080/109] adding first draft of tests for all examples, they need comments and clean up, some could be made more rebust --- examples/test/runner.py | 10 +++ examples/test/test_baresshd.py | 60 ++++++++++++++ examples/test/test_bind.py | 59 ++++++++++++++ examples/test/test_controllers.py | 55 +++++++++++++ examples/test/test_controlnet.py | 51 ++++++++++++ examples/test/test_cpu.py | 36 ++++++++ examples/test/test_emptynet.py | 39 +++++++++ examples/test/test_hwintf.py | 113 ++++++++++++++++++++++++++ examples/test/test_limit.py | 38 +++++++++ examples/test/test_linearbandwidth.py | 48 +++++++++++ examples/test/test_multiping.py | 48 +++++++++++ examples/test/test_multipoll.py | 40 +++++++++ examples/test/test_multitest.py | 32 ++++++++ examples/test/test_nat.py | 44 ++++++++++ examples/test/test_popen.py | 45 ++++++++++ examples/test/test_scratchnet.py | 29 +++++++ examples/test/test_simpleperf.py | 37 +++++++++ examples/test/test_sshd.py | 57 +++++++++++++ examples/test/test_tree1024.py | 37 +++++++++ examples/test/test_treeping64.py | 36 ++++++++ 20 files changed, 914 insertions(+) create mode 100644 examples/test/runner.py create mode 100755 examples/test/test_baresshd.py create mode 100755 examples/test/test_bind.py create mode 100755 examples/test/test_controllers.py create mode 100755 examples/test/test_controlnet.py create mode 100755 examples/test/test_cpu.py create mode 100755 examples/test/test_emptynet.py create mode 100755 examples/test/test_hwintf.py create mode 100755 examples/test/test_limit.py create mode 100755 examples/test/test_linearbandwidth.py create mode 100755 examples/test/test_multiping.py create mode 100755 examples/test/test_multipoll.py create mode 100755 examples/test/test_multitest.py create mode 100755 examples/test/test_nat.py create mode 100755 examples/test/test_popen.py create mode 100755 examples/test/test_scratchnet.py create mode 100755 examples/test/test_simpleperf.py create mode 100755 examples/test/test_sshd.py create mode 100755 examples/test/test_tree1024.py create mode 100755 examples/test/test_treeping64.py diff --git a/examples/test/runner.py b/examples/test/runner.py new file mode 100644 index 0000000..dfa2702 --- /dev/null +++ b/examples/test/runner.py @@ -0,0 +1,10 @@ +import glob +import unittest + +test_file_strings = glob.glob('test_*.py') +module_strings = [str[0:len(str)-3] for str in test_file_strings] +print module_strings +suites = [unittest.defaultTestLoader.loadTestsFromName(str) for str + in module_strings] +testSuite = unittest.TestSuite(suites) +text_runner = unittest.TextTestRunner().run(testSuite) \ No newline at end of file diff --git a/examples/test/test_baresshd.py b/examples/test/test_baresshd.py new file mode 100755 index 0000000..d57436e --- /dev/null +++ b/examples/test/test_baresshd.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python + +"""TEST""" + +import unittest +import pexpect +from time import sleep +from mininet.log import setLogLevel +from mininet.clean import cleanup, sh + +class testBareSSHD( unittest.TestCase ): + "Test ping with single switch topology (common code)." + + opts = [ '\(yes/no\)\?', 'Welcome to h1', 'refused', pexpect.EOF, pexpect.TIMEOUT ] + + def connected( self ): + "check connected" + p = pexpect.spawn( 'ssh 10.0.0.1 -i /tmp/ssh/test_rsa ' ) + while True: + index = p.expect( self.opts ) + if index == 0: + p.sendline( 'yes' ) + elif index == 1: + return True + else: + return False + + def setUp( self ): + self.assertFalse( self.connected() ) + # create public key pair for testing + sh( 'mkdir /tmp/ssh' ) + sh( "ssh-keygen -t rsa -P '' -f /tmp/ssh/test_rsa" ) + sh( 'cat /tmp/ssh/test_rsa.pub >> /tmp/ssh/authorized_keys' ) + cmd = ( 'python -m mininet.examples.baresshd ' + '-o AuthorizedKeysFile=/tmp/ssh/authorized_keys ' + '-o StrictModes=no' ) + sh( cmd ) + + def testSSH( self ): + result = False + # try to connect up to 3 times + for _ in range( 3 ): + result = self.connected() + if result: + break + else: + sleep( 1 ) + self.assertTrue( result ) + + def tearDown( self ): + # kill the ssh process + sh( "ps aux | grep 'ssh.*Banner' | awk '{ print $2 }' | xargs kill" ) + cleanup() + # remove public key pair + sh( 'rm -rf /tmp/ssh' ) + + +if __name__ == '__main__': + setLogLevel( 'warning' ) + unittest.main() diff --git a/examples/test/test_bind.py b/examples/test/test_bind.py new file mode 100755 index 0000000..cc27ca9 --- /dev/null +++ b/examples/test/test_bind.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python + +"""TEST""" + +import unittest +import pexpect +from time import sleep +from mininet.log import setLogLevel +from mininet.clean import cleanup, sh + +class testBind( unittest.TestCase ): + "Test ping with single switch topology (common code)." + + prompt = 'mininet>' + + def connected( self, ip ): + "check connected" + p = pexpect.spawn( 'ssh -i /tmp/ssh/test_rsa %s' % ip ) + while True: + index = p.expect( self.opts ) + if index == 0: + print p.match.group(0) + p.sendline( 'yes' ) + elif index == 1: + return False + elif index == 2: + p.sendline( 'exit' ) + p.wait() + return True + else: + return False + + def setUp( self ): + self.net = pexpect.spawn( 'python -m mininet.examples.bind' ) + self.net.expect( "Private Directories: \[([\w\s,'/]+)\]" ) + self.directories = [] + # parse directories from mn output + for d in self.net.match.group(1).split(', '): + self.directories.append( d.strip("'") ) + self.net.expect( self.prompt ) + self.assertTrue( len( self.directories ) > 0 ) + + def testCreateFile( self ): + fileName = 'a.txt' + directory = self.directories[ 0 ] + self.net.sendline( 'h1 touch %s/%s; ls %s' % ( directory, fileName, directory ) ) + index = self.net.expect( [ fileName, self.prompt ] ) + self.assertTrue( index == 0 ) + + # TODO: need more tests + + def tearDown( self ): + self.net.sendline( 'exit' ) + self.net.wait() + +if __name__ == '__main__': + setLogLevel( 'warning' ) + unittest.main() + diff --git a/examples/test/test_controllers.py b/examples/test/test_controllers.py new file mode 100755 index 0000000..e178db5 --- /dev/null +++ b/examples/test/test_controllers.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python + +"""TEST""" + +import unittest +import pexpect +#from time import sleep +from mininet.log import setLogLevel +#from mininet.net import Mininet +#from mininet.node import CPULimitedHost +#from mininet.link import TCLink + +#from mininet.examples.simpleperf import SingleSwitchTopo + +class testControllers( unittest.TestCase ): + "Test ping with single switch topology (common code)." + + prompt = 'mininet>' + + def connectedTest( self, name, cmap ): + p = pexpect.spawn( 'python -m %s' % name ) + p.expect( self.prompt ) + p.sendline( 'pingall' ) + p.expect ( '(\d+)% dropped' ) + percent = int( p.match.group( 1 ) ) if p.match else -1 + self.assertEqual( percent, 0 ) # or this + p.expect( self.prompt ) + for switch in cmap: + p.sendline( 'sh ovs-vsctl get-controller %s' % switch ) + p.expect( 'tcp:([\d.:]+)') + actual = p.match.group(1) + expected = cmap[ switch ] + self.assertEqual( actual, expected) + p.expect( self.prompt ) + p.sendline( 'exit' ) + p.wait() + #TODO remove this + self.assertEqual( percent, 0 ) + + def testControllers( self ): + c0 = '127.0.0.1:6633' + c1 = '127.0.0.1:6634' + cmap = { 's1': c0, 's2': c1, 's3': c0 } + self.connectedTest( 'mininet.examples.controllers', cmap ) + + def testControllers2( self ): + c0 = '127.0.0.1:6633' + c1 = '127.0.0.1:6634' + cmap = { 's1': c0, 's2': c1 } + self.connectedTest( 'mininet.examples.controllers2', cmap ) + + +if __name__ == '__main__': + setLogLevel( 'warning' ) + unittest.main() diff --git a/examples/test/test_controlnet.py b/examples/test/test_controlnet.py new file mode 100755 index 0000000..73ffa3e --- /dev/null +++ b/examples/test/test_controlnet.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python + +"""TEST""" + +import unittest +import pexpect +from mininet.log import setLogLevel +#from mininet.net import Mininet +#from mininet.node import CPULimitedHost +#from mininet.link import TCLink + +#from mininet.examples.simpleperf import SingleSwitchTopo + +class testControlNet( unittest.TestCase ): + "Test ping with single switch topology (common code)." + + prompt = 'mininet>' + + def testPingall( self ): + p = pexpect.spawn( 'python -m mininet.examples.controlnet' ) + p.expect( self.prompt ) + p.sendline( 'pingall' ) + p.expect ( '(\d+)% dropped' ) + percent = int( p.match.group( 1 ) ) if p.match else -1 + self.assertEqual( percent, 0 ) + p.expect( self.prompt ) + p.sendline( 'exit' ) + p.wait() + + def testFailover( self ): + count = 1 + p = pexpect.spawn( 'python -m mininet.examples.controlnet' ) + p.expect( self.prompt ) + lp = pexpect.spawn( 'tail -f /tmp/s1-ofp.log' ) + lp.expect( 'tcp:\d+\.\d+\.\d+\.(\d+):\d+: connected' ) + ip = int( lp.match.group( 1 ) ) + self.assertEqual( count, ip ) + count += 1 + for c in [ 'c0', 'c1' ]: + p.sendline( '%s ifconfig %s-eth0 down' % ( c, c) ) + p.expect( self.prompt ) + lp.expect( 'tcp:\d+\.\d+\.\d+\.(\d+):\d+: connected' ) + ip = int( lp.match.group( 1 ) ) + self.assertEqual( count, ip ) + count += 1 + p.sendline( 'exit' ) + p.wait() + +if __name__ == '__main__': + setLogLevel( 'warning' ) + unittest.main() diff --git a/examples/test/test_cpu.py b/examples/test/test_cpu.py new file mode 100755 index 0000000..a0598cf --- /dev/null +++ b/examples/test/test_cpu.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python + +"""TEST""" + +import unittest +import pexpect +from mininet.log import setLogLevel + +class testCPU( unittest.TestCase ): + "Test ping with single switch topology (common code)." + + prompt = 'mininet>' + + def testCPU( self ): + opts = [ '([a-z]+)\t([\d\.]+)%\t([\d\.]+)', pexpect.EOF ] + p = pexpect.spawn( 'python -m mininet.examples.cpu' ) + scheds = [] + while True: + index = p.expect( opts, timeout=600 ) + if index == 0: + sched = p.match.group( 1 ) + cpu = float( p.match.group( 2 ) ) + bw = float( p.match.group( 3 ) ) + if sched not in scheds: + scheds.append( sched ) + previous_bw = 10 ** 4 # 10 GB/s + self.assertTrue( bw < previous_bw ) + previous_bw = bw + else: + break + + self.assertTrue( len( scheds ) > 0 ) + +if __name__ == '__main__': + setLogLevel( 'warning' ) + unittest.main() diff --git a/examples/test/test_emptynet.py b/examples/test/test_emptynet.py new file mode 100755 index 0000000..c571ecb --- /dev/null +++ b/examples/test/test_emptynet.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python + +"""TEST""" + +import unittest +import pexpect +#from time import sleep +from mininet.log import setLogLevel +#from mininet.net import Mininet +#from mininet.node import CPULimitedHost +#from mininet.link import TCLink + +#from mininet.examples.simpleperf import SingleSwitchTopo + +class testEmptyNet( unittest.TestCase ): + "Test ping with single switch topology (common code)." + + prompt = 'mininet>' + + def testEmptyNet( self ): + p = pexpect.spawn( 'python -m mininet.examples.emptynet' ) + p.expect( self.prompt ) + p.sendline( 'pingall' ) + p.expect ( '(\d+)% dropped' ) + percent = int( p.match.group( 1 ) ) if p.match else -1 + self.assertEqual( percent, 0 ) # or this + p.expect( self.prompt ) + p.sendline( 'iperf' ) + p.expect( "Results: \['[\d.]+ .bits/sec', '[\d.]+ .bits/sec'\]" ) + #TODO check the results? maybe we dont care + p.expect( self.prompt ) + p.sendline( 'exit' ) + p.wait() + #TODO remove this + self.assertEqual( percent, 0 ) + +if __name__ == '__main__': + setLogLevel( 'warning' ) + unittest.main() diff --git a/examples/test/test_hwintf.py b/examples/test/test_hwintf.py new file mode 100755 index 0000000..ec0ffdd --- /dev/null +++ b/examples/test/test_hwintf.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python + +"""TEST""" + +import unittest +import pexpect +import re +from time import sleep +from mininet.log import setLogLevel +from mininet.net import Mininet +from mininet.node import Node +from mininet.link import Link, Intf + +class testHwintf( unittest.TestCase ): + "Test ping with single switch topology (common code)." + + prompt = 'mininet>' + + def _testE2E( self ): + results = [ "Results:", pexpect.EOF, pexpect.TIMEOUT ] + p = pexpect.spawn( 'python -m mininet.examples.simpleperf' ) + index = p.expect( results, timeout=600 ) + self.assertEqual( index, 0 ) + p.wait() + + def setUp( self ): + self.h3 = Node( 't0', ip='10.0.0.3/8' ) + self.n0 = Node( 't1', inNamespace=False) + Link( self.h3, self.n0 ) + self.h3.configDefault() + + def testLocalPing( self ): + p = pexpect.spawn( 'python -m mininet.examples.hwintf %s' % self.n0.intf() ) + p.expect( self.prompt ) + p.sendline( 'pingall' ) + p.expect ( '(\d+)% dropped' ) + percent = int( p.match.group( 1 ) ) if p.match else -1 + self.assertEqual( percent, 0 ) + p.expect( self.prompt ) + p.sendline( 'exit' ) + p.wait() + + def testExternalPing( self ): + expectStr = '(\d+) packets transmitted, (\d+) received' + p = pexpect.spawn( 'python -m mininet.examples.hwintf %s' % self.n0.intf() ) + p.expect( self.prompt ) + + m = re.search( expectStr, self.h3.cmd( 'ping -v -c 1 10.0.0.1' ) ) + tx = m.group( 1 ) + rx = m.group( 2 ) + self.assertEqual( tx, rx ) + + p.sendline( 'h1 ping -c 1 10.0.0.3') + p.expect( expectStr ) + tx = p.match.group( 1 ) + rx = p.match.group( 2 ) + self.assertEqual( tx, rx ) + p.expect( self.prompt ) + p.sendline( 'exit' ) + p.wait() + + def tearDown( self ): + self.h3.terminate() + self.n0.terminate() + + + + ''' TAP garbage + def testHwintf( self ): + ifname = 'br3' + #sudo ip tuntap add mode tap br0 + #sudo ip tuntap del mode tap br0 + #sudo ip link add name test0 type veth peer name test1 + #sudo ip link del test0 + t0 = Node( 't0', inNamespace=False, ip='10.0.0.3/8' ) + + t1 = Node( 't1', inNamespace=False) + #t0.cmd( 'ip tuntap add mode tap %s' % ifname ) + #Intf( ifname, t0 ) + print Link( t0, t1 ) + t0.configDefault() + + print t0.cmd( 'ifconfig' ) + ifname = t1.intf() + + + try: + foo = pexpect.spawn( 'wireshark' ) + p = pexpect.spawn( 'python -m mininet.examples.hwintf %s' % ifname ) + p.expect( self.prompt ) + #t0.cmd( 'ip link set dev %s up' % ifname ) + #t0.cmd( "bash -c 'echo 1 > /proc/sys/net/ipv4/ip_forward'" ) + #t0.cmd( "bash -c 'echo 1 > /proc/sys/net/ipv4/conf/%s/proxy_arp'" % ifname) + #t0.cmd( 'arp -Ds 10.0.0.3 s1 pub' ) + + #p.sendline( 'x s1 wireshark' ) + print t0.cmd( 'ifconfig %s' % ifname ) + print t0.cmd( 'ip route' ) + print t0.cmd( 'ping -v -c 1 10.0.0.3' ) + print t0.cmd( 'ping -v -c 1 10.0.0.1' ) + + p.interact() + #p.wait() + finally: + #t0.cmd( 'ip tuntap del mode tap %s' % ifname ) + t0.terminate() + t1.terminate() + #t0.configDefault() + ''' + +if __name__ == '__main__': + setLogLevel( 'warning' ) + unittest.main() diff --git a/examples/test/test_limit.py b/examples/test/test_limit.py new file mode 100755 index 0000000..2629d1f --- /dev/null +++ b/examples/test/test_limit.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python + +"""TEST""" + +import unittest +import pexpect +from mininet.log import setLogLevel + +class testLimit( unittest.TestCase ): + "Test ping with single switch topology (common code)." + + def testLimit( self ): + opts = [ '\*\*\* Testing network ([\d\.]+) Mbps', + '\*\*\* Results: \[([\d\., ]+)\]', + pexpect.EOF ] + p = pexpect.spawn( 'python -m mininet.examples.limit' ) + count = 0 + bw = 0 + tolerance = 1 + while True: + index = p.expect( opts ) + if index == 0: + bw = float( p.match.group( 1 ) ) + count += 1 + elif index == 1: + results = p.match.group( 1 ) + for x in results.split(','): + result = float( x ) + self.assertTrue( result < bw + tolerance ) + self.assertTrue( result > bw - tolerance) + else: + break + + self.assertTrue( count > 0 ) + +if __name__ == '__main__': + setLogLevel( 'warning' ) + unittest.main() diff --git a/examples/test/test_linearbandwidth.py b/examples/test/test_linearbandwidth.py new file mode 100755 index 0000000..2a01edc --- /dev/null +++ b/examples/test/test_linearbandwidth.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python + +"""TEST""" + +import unittest +import pexpect +from mininet.log import setLogLevel +#from mininet.net import Mininet +#from mininet.node import CPULimitedHost +#from mininet.link import TCLink + +#from mininet.examples.simpleperf import SingleSwitchTopo + +class testLinearBandwidth( unittest.TestCase ): + "Test ping with single switch topology (common code)." + + prompt = 'mininet>' + + def testLinearBandwidth( self ): + count = 0 + tolerance = 0.5 + opts = [ '\*\*\* Linear network results', '(\d+)\s+([\d\.]+) (.bits)', pexpect.EOF ] + p = pexpect.spawn( 'python -m mininet.examples.linearbandwidth' ) + while True: + index = p.expect( opts, timeout=600 ) + if index == 0: + previous_bw = 10 ** 10 # 10 Gbits + count += 1 + elif index == 1: + n = int( p.match.group( 1 ) ) + bw = float( p.match.group( 2 ) ) + unit = p.match.group( 3 ) + if unit[ 0 ] == 'K': + bw *= 10 ** 3 + elif unit[ 0 ] == 'M': + bw *= 10 ** 6 + elif unit[ 0 ] == 'G': + bw *= 10 ** 9 + self.assertTrue( bw < previous_bw ) + previous_bw = bw + else: + break + + self.assertTrue( count > 0 ) + +if __name__ == '__main__': + setLogLevel( 'warning' ) + unittest.main() diff --git a/examples/test/test_multiping.py b/examples/test/test_multiping.py new file mode 100755 index 0000000..1bd9f94 --- /dev/null +++ b/examples/test/test_multiping.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python + +"""TEST""" + +import unittest +import pexpect +from collections import defaultdict +from mininet.log import setLogLevel + +class testMultiPing( unittest.TestCase ): + "Test ping with single switch topology (common code)." + + def testMultiPing( self ): + p = pexpect.spawn( 'python -m mininet.examples.multiping' ) + opts = [] + opts.append( "Host (h\d+) \(([\d.]+)\) will be pinging ips: ([\d. ]+)" ) + opts.append( "(h\d+): ([\d.]+) -> ([\d.]+) \d packets transmitted, (\d) received" ) + opts.append( pexpect.EOF ) + pings = defaultdict( list ) + while True: + index = p.expect( opts ) + if index == 0: + name = p.match.group(1) + ip = p.match.group(2) + targets = p.match.group(3).split() + pings[ name ] += targets + elif index == 1: + name = p.match.group(1) + ip = p.match.group(2) + target = p.match.group(3) + received = int( p.match.group(4) ) + if target == '10.0.0.200': + self.assertEqual( received, 0 ) + else: + self.assertEqual( received, 1 ) + try: + pings[ name ].remove( target ) + except: + pass + else: + break + self.assertTrue( len(pings) > 0 ) + for t in pings.values(): + self.assertEqual( len( t ), 0 ) + +if __name__ == '__main__': + setLogLevel( 'warning' ) + unittest.main() diff --git a/examples/test/test_multipoll.py b/examples/test/test_multipoll.py new file mode 100755 index 0000000..5c13f34 --- /dev/null +++ b/examples/test/test_multipoll.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python + +"""TEST""" + +import unittest +import pexpect +from collections import defaultdict +from mininet.log import setLogLevel + +class testMultiPoll( unittest.TestCase ): + "Test ping with single switch topology (common code)." + + def testMultiPoll( self ): + p = pexpect.spawn( 'python -m mininet.examples.multipoll' ) + opts = [] + opts.append( "\*\*\* (h\d) :" ) + opts.append( "(h\d+): \d+ bytes from" ) + opts.append( "Monitoring output for (\d+) seconds" ) + opts.append( pexpect.EOF ) + pings = {} + while True: + index = p.expect( opts ) + if index == 0: + name = p.match.group( 1 ) + pings[ name ] = 0 + elif index == 1: + name = p.match.group( 1 ) + pings[ name ] += 1 + elif index == 2: + seconds = int( p.match.group( 1 ) ) + else: + break + self.assertTrue( len(pings) > 0 ) + # make sure we have received at least one ping per second + for count in pings.values(): + self.assertTrue( count >= seconds ) + +if __name__ == '__main__': + setLogLevel( 'warning' ) + unittest.main() diff --git a/examples/test/test_multitest.py b/examples/test/test_multitest.py new file mode 100755 index 0000000..80887ef --- /dev/null +++ b/examples/test/test_multitest.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python + +"""TEST""" + +import unittest +import pexpect +from mininet.log import setLogLevel + +class testMultiTest( unittest.TestCase ): + "Test ping with single switch topology (common code)." + + prompt = 'mininet>' + + def testMultiTest( self ): + p = pexpect.spawn( 'python -m mininet.examples.multitest' ) + p.expect( '(\d+)% dropped' ) + dropped = int( p.match.group(1) ) + self.assertEqual( dropped, 0 ) + ifCount = 0 + while True: + index = p.expect( [ 'h\d-eth0', self.prompt ] ) + if index == 0: + ifCount += 1 + elif index == 1: + p.sendline( 'exit' ) + break + p.wait() + self.assertEqual( ifCount, 4 ) + +if __name__ == '__main__': + setLogLevel( 'warning' ) + unittest.main() diff --git a/examples/test/test_nat.py b/examples/test/test_nat.py new file mode 100755 index 0000000..8e2e8a2 --- /dev/null +++ b/examples/test/test_nat.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python + +"""TEST""" + +import unittest +import pexpect +#from time import sleep +from mininet.log import setLogLevel +#from mininet.net import Mininet +#from mininet.node import CPULimitedHost +#from mininet.link import TCLink + +#from mininet.examples.simpleperf import SingleSwitchTopo + +class testNAT( unittest.TestCase ): + "Test ping with single switch topology (common code)." + + prompt = 'mininet>' + + # skip if 8.8.8.8 unreachable + def testNAT( self ): + p = pexpect.spawn( 'python -m mininet.examples.nat' ) + p.expect( self.prompt ) + p.sendline( 'h1 ping -c 1 8.8.8.8' ) + p.expect ( '(\d+)% packet loss' ) + percent = int( p.match.group( 1 ) ) if p.match else -1 + p.expect( self.prompt ) + p.sendline( 'exit' ) + p.wait() + self.assertEqual( percent, 0 ) + ''' + def testTopo( self ): + topo = SingleSwitchTopo(n=4) + net = Mininet(topo=topo, + host=CPULimitedHost, link=TCLink) + net.start() + h1, h4 = net.get('h1', 'h4') + h1.cmd( 'ping -c 1 %s' % h4.IP() ) + net.stop() + ''' + +if __name__ == '__main__': + setLogLevel( 'warning' ) + unittest.main() diff --git a/examples/test/test_popen.py b/examples/test/test_popen.py new file mode 100755 index 0000000..f9b8ab3 --- /dev/null +++ b/examples/test/test_popen.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python + +"""TEST""" + +import unittest +import pexpect +from collections import defaultdict +from mininet.log import setLogLevel + +class testPopen( unittest.TestCase ): + "Test ping with single switch topology (common code)." + + def pingTest( self, name ): + p = pexpect.spawn( 'python -m %s' % name ) + opts = [] + opts.append( "<(h\d+)>: PING " ) + opts.append( "<(h\d+)>: (\d+) packets transmitted, (\d+) received" ) + opts.append( pexpect.EOF ) + pings = {} + while True: + index = p.expect( opts ) + if index == 0: + name = p.match.group(1) + pings[ name ] = 0 + elif index == 1: + name = p.match.group(1) + transmitted = p.match.group(2) + received = p.match.group(3) + self.assertEqual( received, transmitted ) + pings[ name ] += 1 + else: + break + self.assertTrue( len(pings) > 0 ) + for count in pings.values(): + self.assertEqual( count, 1 ) + + def testPopen( self ): + self.pingTest( 'mininet.examples.popen' ) + + def testPopenPoll( self ): + self.pingTest( 'mininet.examples.popenpoll') + +if __name__ == '__main__': + setLogLevel( 'warning' ) + unittest.main() diff --git a/examples/test/test_scratchnet.py b/examples/test/test_scratchnet.py new file mode 100755 index 0000000..941aaba --- /dev/null +++ b/examples/test/test_scratchnet.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python + +"""TEST""" + +import unittest +import pexpect +from mininet.log import setLogLevel + +class testScratchNet( unittest.TestCase ): + "Test ping with single switch topology (common code)." + + results = [ "1 packets transmitted, 1 received, 0% packet loss", pexpect.EOF ] + + def pingTest( self, name ): + p = pexpect.spawn( 'python -m %s' % name ) + index = p.expect( self.results ) + self.assertEqual( index, 0 ) + + + def testPingKernel( self ): + self.pingTest( 'mininet.examples.scratchnet' ) + + + def testPingUser( self ): + self.pingTest( 'mininet.examples.scratchnetuser' ) + +if __name__ == '__main__': + setLogLevel( 'warning' ) + unittest.main() diff --git a/examples/test/test_simpleperf.py b/examples/test/test_simpleperf.py new file mode 100755 index 0000000..2e589d7 --- /dev/null +++ b/examples/test/test_simpleperf.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python + +"""TEST""" + +import unittest +import pexpect +from time import sleep +from mininet.log import setLogLevel +from mininet.net import Mininet +from mininet.node import CPULimitedHost +from mininet.link import TCLink + +from mininet.examples.simpleperf import SingleSwitchTopo + +class testSimplePerf( unittest.TestCase ): + "Test ping with single switch topology (common code)." + + + def testE2E( self ): + results = [ "Results:", pexpect.EOF, pexpect.TIMEOUT ] + p = pexpect.spawn( 'python -m mininet.examples.simpleperf' ) + index = p.expect( results, timeout=600 ) + self.assertEqual( index, 0 ) + p.wait() + + def testTopo( self ): + topo = SingleSwitchTopo(n=4) + net = Mininet(topo=topo, + host=CPULimitedHost, link=TCLink) + net.start() + h1, h4 = net.get('h1', 'h4') + h1.cmd( 'ping -c 1 %s' % h4.IP() ) + net.stop() + +if __name__ == '__main__': + setLogLevel( 'warning' ) + unittest.main() diff --git a/examples/test/test_sshd.py b/examples/test/test_sshd.py new file mode 100755 index 0000000..35fd632 --- /dev/null +++ b/examples/test/test_sshd.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python + +"""TEST""" + +import unittest +import pexpect +from time import sleep +from mininet.log import setLogLevel +from mininet.clean import cleanup, sh + +class testBareSSHD( unittest.TestCase ): + "Test ping with single switch topology (common code)." + + opts = [ '\(yes/no\)\?', 'refused', 'Welcome', pexpect.EOF, pexpect.TIMEOUT ] + + def connected( self, ip ): + "check connected" + p = pexpect.spawn( 'ssh -i /tmp/ssh/test_rsa %s' % ip ) + while True: + index = p.expect( self.opts ) + if index == 0: + print p.match.group(0) + p.sendline( 'yes' ) + elif index == 1: + return False + elif index == 2: + p.sendline( 'exit' ) + p.wait() + return True + else: + return False + + def setUp( self ): + # create public key pair for testing + sh( 'mkdir /tmp/ssh' ) + sh( "ssh-keygen -t rsa -P '' -f /tmp/ssh/test_rsa" ) + sh( 'cat /tmp/ssh/test_rsa.pub >> /tmp/ssh/authorized_keys' ) + cmd = ( 'python -m mininet.examples.sshd -D ' + '-o AuthorizedKeysFile=/tmp/ssh/authorized_keys ' + '-o StrictModes=no' ) + self.net = pexpect.spawn( cmd ) + self.net.expect( 'mininet>' ) + + def testSSH( self ): + for h in range( 1, 5 ): + self.assertTrue( self.connected( '10.0.0.%d' % h ) ) + + def tearDown( self ): + self.net.sendline( 'exit' ) + self.net.wait() + # remove public key pair + sh( 'rm -rf /tmp/ssh' ) + +if __name__ == '__main__': + setLogLevel( 'warning' ) + unittest.main() + diff --git a/examples/test/test_tree1024.py b/examples/test/test_tree1024.py new file mode 100755 index 0000000..adacb90 --- /dev/null +++ b/examples/test/test_tree1024.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python + +"""TEST""" + +import unittest +import pexpect +#from time import sleep +from mininet.log import setLogLevel +#from mininet.net import Mininet +#from mininet.node import CPULimitedHost +#from mininet.link import TCLink + +#from mininet.examples.simpleperf import SingleSwitchTopo + +class testTree1024( unittest.TestCase ): + "Test ping with single switch topology (common code)." + + prompt = 'mininet>' + + def testTree1024( self ): + p = pexpect.spawn( 'python -m mininet.examples.tree1024' ) + p.expect( self.prompt, timeout=6000 ) # it takes awhile to set up + p.sendline( 'h1 ping -c 1 h1024' ) + p.expect ( '(\d+)% packet loss' ) + percent = int( p.match.group( 1 ) ) if p.match else -1 + #self.assertEqual( percent, 0 ) + #p.expect( self.prompt ) + #p.sendline( 'iperf' ) + #p.expect( "Results: \['\d+ .bits/sec', '\d+ .bits/sec'\]" ) + p.expect( self.prompt ) + p.sendline( 'exit' ) + p.wait() + self.assertEqual( percent, 0 ) + +if __name__ == '__main__': + setLogLevel( 'warning' ) + unittest.main() diff --git a/examples/test/test_treeping64.py b/examples/test/test_treeping64.py new file mode 100755 index 0000000..8328c35 --- /dev/null +++ b/examples/test/test_treeping64.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python + +"""TEST""" + +import unittest +import pexpect +#from time import sleep +from mininet.log import setLogLevel +#from mininet.net import Mininet +#from mininet.node import CPULimitedHost +#from mininet.link import TCLink + +#from mininet.examples.simpleperf import SingleSwitchTopo + +class testTreePing64( unittest.TestCase ): + "Test ping with single switch topology (common code)." + + prompt = 'mininet>' + + def testTreePing64( self ): + p = pexpect.spawn( 'python -m mininet.examples.treeping64' ) + p.expect( 'Tree network ping results:', timeout=6000 ) + count = 0 + while True: + index = p.expect( [ '(\d+)% packet loss', pexpect.EOF ] ) + if index == 0: + percent = int( p.match.group( 1 ) ) if p.match else -1 + self.assertEqual( percent, 0 ) + count += 1 + else: + break + self.assertTrue( count > 0 ) + +if __name__ == '__main__': + setLogLevel( 'warning' ) + unittest.main() From 01c0ef001332fb89892cbbde573dca195f3380d3 Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Tue, 10 Sep 2013 14:07:46 -0700 Subject: [PATCH 081/109] added comments to test_baresshd.py --- examples/test/test_baresshd.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/examples/test/test_baresshd.py b/examples/test/test_baresshd.py index d57436e..cec2c81 100755 --- a/examples/test/test_baresshd.py +++ b/examples/test/test_baresshd.py @@ -1,6 +1,8 @@ #!/usr/bin/env python -"""TEST""" +""" +Tests for baresshd.py +""" import unittest import pexpect @@ -9,13 +11,12 @@ from mininet.log import setLogLevel from mininet.clean import cleanup, sh class testBareSSHD( unittest.TestCase ): - "Test ping with single switch topology (common code)." opts = [ '\(yes/no\)\?', 'Welcome to h1', 'refused', pexpect.EOF, pexpect.TIMEOUT ] def connected( self ): - "check connected" - p = pexpect.spawn( 'ssh 10.0.0.1 -i /tmp/ssh/test_rsa ' ) + "Log into ssh server, check banner, then exit" + p = pexpect.spawn( 'ssh 10.0.0.1 -i /tmp/ssh/test_rsa exit' ) while True: index = p.expect( self.opts ) if index == 0: @@ -26,19 +27,22 @@ class testBareSSHD( unittest.TestCase ): return False def setUp( self ): + # verify that sshd is not running self.assertFalse( self.connected() ) # create public key pair for testing sh( 'mkdir /tmp/ssh' ) sh( "ssh-keygen -t rsa -P '' -f /tmp/ssh/test_rsa" ) sh( 'cat /tmp/ssh/test_rsa.pub >> /tmp/ssh/authorized_keys' ) + # run example with custom sshd args cmd = ( 'python -m mininet.examples.baresshd ' '-o AuthorizedKeysFile=/tmp/ssh/authorized_keys ' '-o StrictModes=no' ) sh( cmd ) def testSSH( self ): + "Simple test to verify that we can ssh into h1" result = False - # try to connect up to 3 times + # try to connect up to 3 times; sshd can take a while to start for _ in range( 3 ): result = self.connected() if result: @@ -54,7 +58,6 @@ class testBareSSHD( unittest.TestCase ): # remove public key pair sh( 'rm -rf /tmp/ssh' ) - if __name__ == '__main__': setLogLevel( 'warning' ) unittest.main() From 43f058df6db7d33f5cb52f35b941e89effff6aab Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Tue, 10 Sep 2013 14:31:15 -0700 Subject: [PATCH 082/109] cleaned up and commented test_bind.py; added one new test --- examples/test/test_bind.py | 56 ++++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 24 deletions(-) diff --git a/examples/test/test_bind.py b/examples/test/test_bind.py index cc27ca9..d8df0b3 100755 --- a/examples/test/test_bind.py +++ b/examples/test/test_bind.py @@ -1,35 +1,16 @@ #!/usr/bin/env python -"""TEST""" +""" +Tests for bind.py +""" import unittest import pexpect -from time import sleep -from mininet.log import setLogLevel -from mininet.clean import cleanup, sh class testBind( unittest.TestCase ): - "Test ping with single switch topology (common code)." prompt = 'mininet>' - def connected( self, ip ): - "check connected" - p = pexpect.spawn( 'ssh -i /tmp/ssh/test_rsa %s' % ip ) - while True: - index = p.expect( self.opts ) - if index == 0: - print p.match.group(0) - p.sendline( 'yes' ) - elif index == 1: - return False - elif index == 2: - p.sendline( 'exit' ) - p.wait() - return True - else: - return False - def setUp( self ): self.net = pexpect.spawn( 'python -m mininet.examples.bind' ) self.net.expect( "Private Directories: \[([\w\s,'/]+)\]" ) @@ -41,11 +22,39 @@ class testBind( unittest.TestCase ): self.assertTrue( len( self.directories ) > 0 ) def testCreateFile( self ): + "Create a file, a.txt, in the first private directory and verify" fileName = 'a.txt' directory = self.directories[ 0 ] - self.net.sendline( 'h1 touch %s/%s; ls %s' % ( directory, fileName, directory ) ) + path = directory + '/' + fileName + self.net.sendline( 'h1 touch %s; ls %s' % ( path, directory ) ) index = self.net.expect( [ fileName, self.prompt ] ) self.assertTrue( index == 0 ) + self.net.expect( self.prompt ) + self.net.sendline( 'h1 rm %s' % path ) + self.net.expect( self.prompt ) + + def testIsolation( self ): + "Create a file in two hosts and verify that contents are different" + fileName = 'b.txt' + directory = self.directories[ 0 ] + path = directory + '/' + fileName + contents = { 'h1' : '1', 'h2' : '2' } + # Verify file doesn't exist, then write private copy of file + for host in contents: + value = contents[ host ] + self.net.sendline( '%s cat %s' % ( host, path ) ) + self.net.expect( 'No such file' ) + self.net.expect( self.prompt ) + self.net.sendline( '%s echo %s > %s' % ( host, value, path ) ) + self.net.expect( self.prompt ) + # Verify file contents + for host in contents: + value = contents[ host ] + self.net.sendline( '%s cat %s' % ( host, path ) ) + self.net.expect( value ) + self.net.expect( self.prompt ) + self.net.sendline( '%s rm %s' % ( host, path ) ) + self.net.expect( self.prompt ) # TODO: need more tests @@ -54,6 +63,5 @@ class testBind( unittest.TestCase ): self.net.wait() if __name__ == '__main__': - setLogLevel( 'warning' ) unittest.main() From e875c0de2660f132139c14ebe2a2f5932edf8350 Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Tue, 10 Sep 2013 14:34:50 -0700 Subject: [PATCH 083/109] minor test cleanup --- examples/test/test_baresshd.py | 2 -- examples/test/test_bind.py | 1 - 2 files changed, 3 deletions(-) diff --git a/examples/test/test_baresshd.py b/examples/test/test_baresshd.py index cec2c81..b76fe20 100755 --- a/examples/test/test_baresshd.py +++ b/examples/test/test_baresshd.py @@ -7,7 +7,6 @@ Tests for baresshd.py import unittest import pexpect from time import sleep -from mininet.log import setLogLevel from mininet.clean import cleanup, sh class testBareSSHD( unittest.TestCase ): @@ -59,5 +58,4 @@ class testBareSSHD( unittest.TestCase ): sh( 'rm -rf /tmp/ssh' ) if __name__ == '__main__': - setLogLevel( 'warning' ) unittest.main() diff --git a/examples/test/test_bind.py b/examples/test/test_bind.py index d8df0b3..dcc9fbf 100755 --- a/examples/test/test_bind.py +++ b/examples/test/test_bind.py @@ -64,4 +64,3 @@ class testBind( unittest.TestCase ): if __name__ == '__main__': unittest.main() - From fba3fd81fa21877aed1eda9ee0e98655d96b404a Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Tue, 10 Sep 2013 14:37:01 -0700 Subject: [PATCH 084/109] cleaned up and commented test_controllers.py --- examples/test/test_controllers.py | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/examples/test/test_controllers.py b/examples/test/test_controllers.py index e178db5..b9b93d9 100755 --- a/examples/test/test_controllers.py +++ b/examples/test/test_controllers.py @@ -1,41 +1,36 @@ #!/usr/bin/env python -"""TEST""" +""" +Tests for controllers.py and controllers2.py +""" import unittest import pexpect -#from time import sleep -from mininet.log import setLogLevel -#from mininet.net import Mininet -#from mininet.node import CPULimitedHost -#from mininet.link import TCLink - -#from mininet.examples.simpleperf import SingleSwitchTopo class testControllers( unittest.TestCase ): - "Test ping with single switch topology (common code)." prompt = 'mininet>' def connectedTest( self, name, cmap ): + "Verify that switches are connected to the controller specified by cmap" p = pexpect.spawn( 'python -m %s' % name ) p.expect( self.prompt ) + # but first a simple ping test p.sendline( 'pingall' ) p.expect ( '(\d+)% dropped' ) percent = int( p.match.group( 1 ) ) if p.match else -1 - self.assertEqual( percent, 0 ) # or this + self.assertEqual( percent, 0 ) p.expect( self.prompt ) + # verify connected controller for switch in cmap: p.sendline( 'sh ovs-vsctl get-controller %s' % switch ) p.expect( 'tcp:([\d.:]+)') actual = p.match.group(1) expected = cmap[ switch ] - self.assertEqual( actual, expected) + self.assertEqual( actual, expected ) p.expect( self.prompt ) p.sendline( 'exit' ) p.wait() - #TODO remove this - self.assertEqual( percent, 0 ) def testControllers( self ): c0 = '127.0.0.1:6633' @@ -49,7 +44,5 @@ class testControllers( unittest.TestCase ): cmap = { 's1': c0, 's2': c1 } self.connectedTest( 'mininet.examples.controllers2', cmap ) - if __name__ == '__main__': - setLogLevel( 'warning' ) unittest.main() From b7e506341f94f6e098818c2fa0e127467c55046b Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Tue, 10 Sep 2013 14:45:19 -0700 Subject: [PATCH 085/109] cleaned up and commented test_controlnet.py --- examples/test/test_controlnet.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/examples/test/test_controlnet.py b/examples/test/test_controlnet.py index 73ffa3e..91631f2 100755 --- a/examples/test/test_controlnet.py +++ b/examples/test/test_controlnet.py @@ -1,22 +1,18 @@ #!/usr/bin/env python -"""TEST""" +""" +Test for controlnet.py +""" import unittest import pexpect -from mininet.log import setLogLevel -#from mininet.net import Mininet -#from mininet.node import CPULimitedHost -#from mininet.link import TCLink - -#from mininet.examples.simpleperf import SingleSwitchTopo class testControlNet( unittest.TestCase ): - "Test ping with single switch topology (common code)." prompt = 'mininet>' def testPingall( self ): + "Simple pingall test that verifies 0% packet drop in data network" p = pexpect.spawn( 'python -m mininet.examples.controlnet' ) p.expect( self.prompt ) p.sendline( 'pingall' ) @@ -28,6 +24,7 @@ class testControlNet( unittest.TestCase ): p.wait() def testFailover( self ): + "Kill controllers and verity that switch, s1, fails over properly" count = 1 p = pexpect.spawn( 'python -m mininet.examples.controlnet' ) p.expect( self.prompt ) @@ -47,5 +44,4 @@ class testControlNet( unittest.TestCase ): p.wait() if __name__ == '__main__': - setLogLevel( 'warning' ) unittest.main() From 94abeeabb98c0c32e5aaf70896f58ee39edbdc10 Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Tue, 10 Sep 2013 14:54:02 -0700 Subject: [PATCH 086/109] cleaned up and commented test_cpu.py --- examples/test/test_cpu.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/test/test_cpu.py b/examples/test/test_cpu.py index a0598cf..dd35490 100755 --- a/examples/test/test_cpu.py +++ b/examples/test/test_cpu.py @@ -1,19 +1,20 @@ #!/usr/bin/env python -"""TEST""" +""" +Test for cpu.py +""" import unittest import pexpect -from mininet.log import setLogLevel class testCPU( unittest.TestCase ): - "Test ping with single switch topology (common code)." prompt = 'mininet>' def testCPU( self ): - opts = [ '([a-z]+)\t([\d\.]+)%\t([\d\.]+)', pexpect.EOF ] + "Verify that CPU utilization is monotonically decreasing for each scheduler" p = pexpect.spawn( 'python -m mininet.examples.cpu' ) + opts = [ '([a-z]+)\t([\d\.]+)%\t([\d\.]+)', pexpect.EOF ] scheds = [] while True: index = p.expect( opts, timeout=600 ) @@ -32,5 +33,4 @@ class testCPU( unittest.TestCase ): self.assertTrue( len( scheds ) > 0 ) if __name__ == '__main__': - setLogLevel( 'warning' ) unittest.main() From 49fc496c122e2fe49272dddbf490d328a6641ae3 Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Tue, 10 Sep 2013 14:56:51 -0700 Subject: [PATCH 087/109] cleaned up and commented test_emptynet.py --- examples/test/test_emptynet.py | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/examples/test/test_emptynet.py b/examples/test/test_emptynet.py index c571ecb..0d4d01d 100755 --- a/examples/test/test_emptynet.py +++ b/examples/test/test_emptynet.py @@ -1,39 +1,32 @@ #!/usr/bin/env python -"""TEST""" +""" +Test for emptynet.py +""" import unittest import pexpect -#from time import sleep -from mininet.log import setLogLevel -#from mininet.net import Mininet -#from mininet.node import CPULimitedHost -#from mininet.link import TCLink - -#from mininet.examples.simpleperf import SingleSwitchTopo class testEmptyNet( unittest.TestCase ): - "Test ping with single switch topology (common code)." prompt = 'mininet>' def testEmptyNet( self ): + "Run simple CLI tests: pingall (verify 0% drop) and iperf (sanity)" p = pexpect.spawn( 'python -m mininet.examples.emptynet' ) p.expect( self.prompt ) + # pingall test p.sendline( 'pingall' ) p.expect ( '(\d+)% dropped' ) percent = int( p.match.group( 1 ) ) if p.match else -1 - self.assertEqual( percent, 0 ) # or this + self.assertEqual( percent, 0 ) p.expect( self.prompt ) + # iperf test p.sendline( 'iperf' ) p.expect( "Results: \['[\d.]+ .bits/sec', '[\d.]+ .bits/sec'\]" ) - #TODO check the results? maybe we dont care p.expect( self.prompt ) p.sendline( 'exit' ) p.wait() - #TODO remove this - self.assertEqual( percent, 0 ) if __name__ == '__main__': - setLogLevel( 'warning' ) unittest.main() From 91a06063b4c380d482146b3dcfbb4840ab8e6e99 Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Tue, 10 Sep 2013 15:24:27 -0700 Subject: [PATCH 088/109] cleaned up and commented test_hwintf.py --- examples/test/test_hwintf.py | 68 +++++------------------------------- 1 file changed, 9 insertions(+), 59 deletions(-) diff --git a/examples/test/test_hwintf.py b/examples/test/test_hwintf.py index ec0ffdd..20d08d3 100755 --- a/examples/test/test_hwintf.py +++ b/examples/test/test_hwintf.py @@ -1,35 +1,29 @@ #!/usr/bin/env python -"""TEST""" +""" +Test for hwintf.py +""" import unittest import pexpect import re -from time import sleep from mininet.log import setLogLevel from mininet.net import Mininet from mininet.node import Node from mininet.link import Link, Intf class testHwintf( unittest.TestCase ): - "Test ping with single switch topology (common code)." prompt = 'mininet>' - def _testE2E( self ): - results = [ "Results:", pexpect.EOF, pexpect.TIMEOUT ] - p = pexpect.spawn( 'python -m mininet.examples.simpleperf' ) - index = p.expect( results, timeout=600 ) - self.assertEqual( index, 0 ) - p.wait() - def setUp( self ): self.h3 = Node( 't0', ip='10.0.0.3/8' ) - self.n0 = Node( 't1', inNamespace=False) + self.n0 = Node( 't1', inNamespace=False ) Link( self.h3, self.n0 ) self.h3.configDefault() def testLocalPing( self ): + "Verify connectivity between virtual hosts using pingall" p = pexpect.spawn( 'python -m mininet.examples.hwintf %s' % self.n0.intf() ) p.expect( self.prompt ) p.sendline( 'pingall' ) @@ -41,15 +35,16 @@ class testHwintf( unittest.TestCase ): p.wait() def testExternalPing( self ): - expectStr = '(\d+) packets transmitted, (\d+) received' + "Verify connnectivity between virtual host and virtual-physical 'external' host " p = pexpect.spawn( 'python -m mininet.examples.hwintf %s' % self.n0.intf() ) p.expect( self.prompt ) - + # test ping external to internal + expectStr = '(\d+) packets transmitted, (\d+) received' m = re.search( expectStr, self.h3.cmd( 'ping -v -c 1 10.0.0.1' ) ) tx = m.group( 1 ) rx = m.group( 2 ) self.assertEqual( tx, rx ) - + # test ping internal to external p.sendline( 'h1 ping -c 1 10.0.0.3') p.expect( expectStr ) tx = p.match.group( 1 ) @@ -63,51 +58,6 @@ class testHwintf( unittest.TestCase ): self.h3.terminate() self.n0.terminate() - - - ''' TAP garbage - def testHwintf( self ): - ifname = 'br3' - #sudo ip tuntap add mode tap br0 - #sudo ip tuntap del mode tap br0 - #sudo ip link add name test0 type veth peer name test1 - #sudo ip link del test0 - t0 = Node( 't0', inNamespace=False, ip='10.0.0.3/8' ) - - t1 = Node( 't1', inNamespace=False) - #t0.cmd( 'ip tuntap add mode tap %s' % ifname ) - #Intf( ifname, t0 ) - print Link( t0, t1 ) - t0.configDefault() - - print t0.cmd( 'ifconfig' ) - ifname = t1.intf() - - - try: - foo = pexpect.spawn( 'wireshark' ) - p = pexpect.spawn( 'python -m mininet.examples.hwintf %s' % ifname ) - p.expect( self.prompt ) - #t0.cmd( 'ip link set dev %s up' % ifname ) - #t0.cmd( "bash -c 'echo 1 > /proc/sys/net/ipv4/ip_forward'" ) - #t0.cmd( "bash -c 'echo 1 > /proc/sys/net/ipv4/conf/%s/proxy_arp'" % ifname) - #t0.cmd( 'arp -Ds 10.0.0.3 s1 pub' ) - - #p.sendline( 'x s1 wireshark' ) - print t0.cmd( 'ifconfig %s' % ifname ) - print t0.cmd( 'ip route' ) - print t0.cmd( 'ping -v -c 1 10.0.0.3' ) - print t0.cmd( 'ping -v -c 1 10.0.0.1' ) - - p.interact() - #p.wait() - finally: - #t0.cmd( 'ip tuntap del mode tap %s' % ifname ) - t0.terminate() - t1.terminate() - #t0.configDefault() - ''' - if __name__ == '__main__': setLogLevel( 'warning' ) unittest.main() From e6fe480a306a875d94609e04954c3cb04f0e0dd9 Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Tue, 10 Sep 2013 15:32:03 -0700 Subject: [PATCH 089/109] cleaned up and commented test_limit.py --- examples/test/test_limit.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/examples/test/test_limit.py b/examples/test/test_limit.py index 2629d1f..ea5ce35 100755 --- a/examples/test/test_limit.py +++ b/examples/test/test_limit.py @@ -1,19 +1,20 @@ #!/usr/bin/env python -"""TEST""" +""" +Test for limit.py +""" import unittest import pexpect -from mininet.log import setLogLevel class testLimit( unittest.TestCase ): - "Test ping with single switch topology (common code)." def testLimit( self ): + "Verify that CPU limits are within a 1% tolerance of limit for each scheduler" + p = pexpect.spawn( 'python -m mininet.examples.limit' ) opts = [ '\*\*\* Testing network ([\d\.]+) Mbps', '\*\*\* Results: \[([\d\., ]+)\]', pexpect.EOF ] - p = pexpect.spawn( 'python -m mininet.examples.limit' ) count = 0 bw = 0 tolerance = 1 @@ -24,15 +25,14 @@ class testLimit( unittest.TestCase ): count += 1 elif index == 1: results = p.match.group( 1 ) - for x in results.split(','): + for x in results.split( ',' ): result = float( x ) self.assertTrue( result < bw + tolerance ) - self.assertTrue( result > bw - tolerance) + self.assertTrue( result > bw - tolerance ) else: break self.assertTrue( count > 0 ) if __name__ == '__main__': - setLogLevel( 'warning' ) unittest.main() From 1d555e724b5380c905c412b6b0eb593950d19ccc Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Tue, 10 Sep 2013 15:38:32 -0700 Subject: [PATCH 090/109] cleaned up and commented test_linearbandwidth.py --- examples/test/test_linearbandwidth.py | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/examples/test/test_linearbandwidth.py b/examples/test/test_linearbandwidth.py index 2a01edc..b6c47fd 100755 --- a/examples/test/test_linearbandwidth.py +++ b/examples/test/test_linearbandwidth.py @@ -1,26 +1,21 @@ #!/usr/bin/env python -"""TEST""" +""" +Test for linearbandwidth.py +""" import unittest import pexpect -from mininet.log import setLogLevel -#from mininet.net import Mininet -#from mininet.node import CPULimitedHost -#from mininet.link import TCLink - -#from mininet.examples.simpleperf import SingleSwitchTopo class testLinearBandwidth( unittest.TestCase ): - "Test ping with single switch topology (common code)." - - prompt = 'mininet>' def testLinearBandwidth( self ): - count = 0 - tolerance = 0.5 - opts = [ '\*\*\* Linear network results', '(\d+)\s+([\d\.]+) (.bits)', pexpect.EOF ] + "Verify that bandwidth is monotonically decreasing as # of hops increases" p = pexpect.spawn( 'python -m mininet.examples.linearbandwidth' ) + count = 0 + opts = [ '\*\*\* Linear network results', + '(\d+)\s+([\d\.]+) (.bits)', + pexpect.EOF ] while True: index = p.expect( opts, timeout=600 ) if index == 0: @@ -44,5 +39,4 @@ class testLinearBandwidth( unittest.TestCase ): self.assertTrue( count > 0 ) if __name__ == '__main__': - setLogLevel( 'warning' ) unittest.main() From e6e1260bc236b5261ae67566ee3453093fbdf5d0 Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Tue, 10 Sep 2013 15:50:43 -0700 Subject: [PATCH 091/109] cleaned up and commented test_multiping.py --- examples/test/test_multiping.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/examples/test/test_multiping.py b/examples/test/test_multiping.py index 1bd9f94..ff0571c 100755 --- a/examples/test/test_multiping.py +++ b/examples/test/test_multiping.py @@ -1,21 +1,22 @@ #!/usr/bin/env python -"""TEST""" +""" +Test for multiping.py +""" import unittest import pexpect from collections import defaultdict -from mininet.log import setLogLevel class testMultiPing( unittest.TestCase ): - "Test ping with single switch topology (common code)." def testMultiPing( self ): + """Verify that each target is pinged at least once, and + that pings to 'real' targets are successful and unknown targets fail""" p = pexpect.spawn( 'python -m mininet.examples.multiping' ) - opts = [] - opts.append( "Host (h\d+) \(([\d.]+)\) will be pinging ips: ([\d. ]+)" ) - opts.append( "(h\d+): ([\d.]+) -> ([\d.]+) \d packets transmitted, (\d) received" ) - opts.append( pexpect.EOF ) + opts = [ "Host (h\d+) \(([\d.]+)\) will be pinging ips: ([\d\. ]+)", + "(h\d+): ([\d.]+) -> ([\d.]+) \d packets transmitted, (\d) received", + pexpect.EOF ] pings = defaultdict( list ) while True: index = p.expect( opts ) @@ -39,10 +40,9 @@ class testMultiPing( unittest.TestCase ): pass else: break - self.assertTrue( len(pings) > 0 ) + self.assertTrue( len( pings ) > 0 ) for t in pings.values(): self.assertEqual( len( t ), 0 ) if __name__ == '__main__': - setLogLevel( 'warning' ) unittest.main() From bc90a7958104b1f2823c82c5a832eeb222d0d7e5 Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Tue, 10 Sep 2013 15:56:27 -0700 Subject: [PATCH 092/109] cleaned up and commented test_multipoll.py --- examples/test/test_multipoll.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/examples/test/test_multipoll.py b/examples/test/test_multipoll.py index 5c13f34..12321ea 100755 --- a/examples/test/test_multipoll.py +++ b/examples/test/test_multipoll.py @@ -1,22 +1,21 @@ #!/usr/bin/env python -"""TEST""" +""" +Test for multipoll.py +""" import unittest import pexpect -from collections import defaultdict -from mininet.log import setLogLevel class testMultiPoll( unittest.TestCase ): - "Test ping with single switch topology (common code)." def testMultiPoll( self ): + "Verify that we receive one ping per second per host" p = pexpect.spawn( 'python -m mininet.examples.multipoll' ) - opts = [] - opts.append( "\*\*\* (h\d) :" ) - opts.append( "(h\d+): \d+ bytes from" ) - opts.append( "Monitoring output for (\d+) seconds" ) - opts.append( pexpect.EOF ) + opts = [ "\*\*\* (h\d) :" , + "(h\d+): \d+ bytes from", + "Monitoring output for (\d+) seconds", + pexpect.EOF ] pings = {} while True: index = p.expect( opts ) @@ -30,11 +29,10 @@ class testMultiPoll( unittest.TestCase ): seconds = int( p.match.group( 1 ) ) else: break - self.assertTrue( len(pings) > 0 ) + self.assertTrue( len( pings ) > 0 ) # make sure we have received at least one ping per second for count in pings.values(): self.assertTrue( count >= seconds ) if __name__ == '__main__': - setLogLevel( 'warning' ) unittest.main() From cdd5210bb70300499e9cc9f94055a6253352a966 Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Tue, 10 Sep 2013 15:59:39 -0700 Subject: [PATCH 093/109] cleaned up and commented test_multitest.py --- examples/test/test_multitest.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/test/test_multitest.py b/examples/test/test_multitest.py index 80887ef..a2bb10b 100755 --- a/examples/test/test_multitest.py +++ b/examples/test/test_multitest.py @@ -1,17 +1,18 @@ #!/usr/bin/env python -"""TEST""" +""" +Test for multitest.py +""" import unittest import pexpect -from mininet.log import setLogLevel class testMultiTest( unittest.TestCase ): - "Test ping with single switch topology (common code)." prompt = 'mininet>' def testMultiTest( self ): + "Verify pingall (0% dropped) and hX-eth0 interface for each host (ifconfig)" p = pexpect.spawn( 'python -m mininet.examples.multitest' ) p.expect( '(\d+)% dropped' ) dropped = int( p.match.group(1) ) @@ -28,5 +29,4 @@ class testMultiTest( unittest.TestCase ): self.assertEqual( ifCount, 4 ) if __name__ == '__main__': - setLogLevel( 'warning' ) unittest.main() From b9b1f2e7f06d2e80929a9d1df3301b6af4fc30bf Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Tue, 10 Sep 2013 16:10:23 -0700 Subject: [PATCH 094/109] cleaned up and commented test_nat.py; added check for connectivity before running test --- examples/test/test_multitest.py | 2 +- examples/test/test_nat.py | 30 +++++++++--------------------- 2 files changed, 10 insertions(+), 22 deletions(-) diff --git a/examples/test/test_multitest.py b/examples/test/test_multitest.py index a2bb10b..09172c5 100755 --- a/examples/test/test_multitest.py +++ b/examples/test/test_multitest.py @@ -15,7 +15,7 @@ class testMultiTest( unittest.TestCase ): "Verify pingall (0% dropped) and hX-eth0 interface for each host (ifconfig)" p = pexpect.spawn( 'python -m mininet.examples.multitest' ) p.expect( '(\d+)% dropped' ) - dropped = int( p.match.group(1) ) + dropped = int( p.match.group( 1 ) ) self.assertEqual( dropped, 0 ) ifCount = 0 while True: diff --git a/examples/test/test_nat.py b/examples/test/test_nat.py index 8e2e8a2..8e49ed6 100755 --- a/examples/test/test_nat.py +++ b/examples/test/test_nat.py @@ -1,44 +1,32 @@ #!/usr/bin/env python -"""TEST""" +""" +Test for nat.py +""" import unittest import pexpect -#from time import sleep -from mininet.log import setLogLevel -#from mininet.net import Mininet -#from mininet.node import CPULimitedHost -#from mininet.link import TCLink +from mininet.util import quietRun -#from mininet.examples.simpleperf import SingleSwitchTopo +destIP = '8.8.8.8' # Google DNS class testNAT( unittest.TestCase ): - "Test ping with single switch topology (common code)." prompt = 'mininet>' - # skip if 8.8.8.8 unreachable + @unittest.skipIf( '0 received' in quietRun( 'ping -c 1 %s' % destIP ), + 'Destination IP is not reachable' ) def testNAT( self ): + "Attempt to ping an IP on the Internet and verify 0% packet loss" p = pexpect.spawn( 'python -m mininet.examples.nat' ) p.expect( self.prompt ) - p.sendline( 'h1 ping -c 1 8.8.8.8' ) + p.sendline( 'h1 ping -c 1 %s' % destIP ) p.expect ( '(\d+)% packet loss' ) percent = int( p.match.group( 1 ) ) if p.match else -1 p.expect( self.prompt ) p.sendline( 'exit' ) p.wait() self.assertEqual( percent, 0 ) - ''' - def testTopo( self ): - topo = SingleSwitchTopo(n=4) - net = Mininet(topo=topo, - host=CPULimitedHost, link=TCLink) - net.start() - h1, h4 = net.get('h1', 'h4') - h1.cmd( 'ping -c 1 %s' % h4.IP() ) - net.stop() - ''' if __name__ == '__main__': - setLogLevel( 'warning' ) unittest.main() From 213b7c57ee9170a800754195746dfab11e9e5dd5 Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Tue, 10 Sep 2013 16:17:56 -0700 Subject: [PATCH 095/109] cleaned up and commented test_popen.py --- examples/test/test_popen.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/examples/test/test_popen.py b/examples/test/test_popen.py index f9b8ab3..c7f83f0 100755 --- a/examples/test/test_popen.py +++ b/examples/test/test_popen.py @@ -1,21 +1,20 @@ #!/usr/bin/env python -"""TEST""" +""" +Test for popen.py and popenpoll.py +""" import unittest import pexpect -from collections import defaultdict -from mininet.log import setLogLevel class testPopen( unittest.TestCase ): - "Test ping with single switch topology (common code)." def pingTest( self, name ): + "Verify that there are no dropped packets for each host" p = pexpect.spawn( 'python -m %s' % name ) - opts = [] - opts.append( "<(h\d+)>: PING " ) - opts.append( "<(h\d+)>: (\d+) packets transmitted, (\d+) received" ) - opts.append( pexpect.EOF ) + opts = [ "<(h\d+)>: PING ", + "<(h\d+)>: (\d+) packets transmitted, (\d+) received", + pexpect.EOF ] pings = {} while True: index = p.expect( opts ) @@ -26,11 +25,13 @@ class testPopen( unittest.TestCase ): name = p.match.group(1) transmitted = p.match.group(2) received = p.match.group(3) + # verify no dropped packets self.assertEqual( received, transmitted ) pings[ name ] += 1 else: break self.assertTrue( len(pings) > 0 ) + # verify that each host has gotten results for count in pings.values(): self.assertEqual( count, 1 ) @@ -38,8 +39,7 @@ class testPopen( unittest.TestCase ): self.pingTest( 'mininet.examples.popen' ) def testPopenPoll( self ): - self.pingTest( 'mininet.examples.popenpoll') + self.pingTest( 'mininet.examples.popenpoll' ) if __name__ == '__main__': - setLogLevel( 'warning' ) unittest.main() From d4993c0ba4c8232aa9d37903da1c95f4ab897330 Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Tue, 10 Sep 2013 16:20:44 -0700 Subject: [PATCH 096/109] cleaned up and commented test_scratchnet.py --- examples/test/test_scratchnet.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/examples/test/test_scratchnet.py b/examples/test/test_scratchnet.py index 941aaba..31739b8 100755 --- a/examples/test/test_scratchnet.py +++ b/examples/test/test_scratchnet.py @@ -1,29 +1,27 @@ #!/usr/bin/env python -"""TEST""" +""" +Test for scratchnet.py +""" import unittest import pexpect -from mininet.log import setLogLevel class testScratchNet( unittest.TestCase ): - "Test ping with single switch topology (common code)." - results = [ "1 packets transmitted, 1 received, 0% packet loss", pexpect.EOF ] + opts = [ "1 packets transmitted, 1 received, 0% packet loss", pexpect.EOF ] def pingTest( self, name ): + "Verify that no ping packets were dropped" p = pexpect.spawn( 'python -m %s' % name ) - index = p.expect( self.results ) + index = p.expect( self.opts ) self.assertEqual( index, 0 ) - def testPingKernel( self ): self.pingTest( 'mininet.examples.scratchnet' ) - def testPingUser( self ): self.pingTest( 'mininet.examples.scratchnetuser' ) if __name__ == '__main__': - setLogLevel( 'warning' ) unittest.main() From 48c49c54e2299df27219987359e6ba58d92f30dd Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Tue, 10 Sep 2013 16:57:31 -0700 Subject: [PATCH 097/109] cleaned up and commented test_sshd.py --- examples/test/test_sshd.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/examples/test/test_sshd.py b/examples/test/test_sshd.py index 35fd632..9d4ef1f 100755 --- a/examples/test/test_sshd.py +++ b/examples/test/test_sshd.py @@ -1,21 +1,23 @@ #!/usr/bin/env python -"""TEST""" +""" +Test for sshd.py +""" import unittest import pexpect from time import sleep -from mininet.log import setLogLevel -from mininet.clean import cleanup, sh +from mininet.clean import sh -class testBareSSHD( unittest.TestCase ): - "Test ping with single switch topology (common code)." +class testSSHD( unittest.TestCase ): - opts = [ '\(yes/no\)\?', 'refused', 'Welcome', pexpect.EOF, pexpect.TIMEOUT ] + opts = [ '\(yes/no\)\?', 'refused', 'Welcome|\$|#', pexpect.EOF, pexpect.TIMEOUT ] def connected( self, ip ): - "check connected" - p = pexpect.spawn( 'ssh -i /tmp/ssh/test_rsa %s' % ip ) + "Log into ssh server, check banner, then exit" + # Note: this test will fail if "Welcome" is not in the sshd banner + # and '#'' or '$'' are not in the prompt + p = pexpect.spawn( 'ssh -i /tmp/ssh/test_rsa %s' % ip, timeout=10 ) while True: index = p.expect( self.opts ) if index == 0: @@ -38,10 +40,12 @@ class testBareSSHD( unittest.TestCase ): cmd = ( 'python -m mininet.examples.sshd -D ' '-o AuthorizedKeysFile=/tmp/ssh/authorized_keys ' '-o StrictModes=no' ) + # run example with custom sshd args self.net = pexpect.spawn( cmd ) self.net.expect( 'mininet>' ) def testSSH( self ): + "Verify that we can ssh into all hosts (h1 to h4)" for h in range( 1, 5 ): self.assertTrue( self.connected( '10.0.0.%d' % h ) ) @@ -52,6 +56,5 @@ class testBareSSHD( unittest.TestCase ): sh( 'rm -rf /tmp/ssh' ) if __name__ == '__main__': - setLogLevel( 'warning' ) unittest.main() From 3577a6989d2bbb0e5c7664e693968267a63ac0a4 Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Tue, 10 Sep 2013 17:14:31 -0700 Subject: [PATCH 098/109] cleaned up and commented test_tree1024.py and test_treeping64.py --- examples/test/test_tree1024.py | 18 ++++-------------- examples/test/test_treeping64.py | 14 ++++---------- 2 files changed, 8 insertions(+), 24 deletions(-) diff --git a/examples/test/test_tree1024.py b/examples/test/test_tree1024.py index adacb90..2d180cb 100755 --- a/examples/test/test_tree1024.py +++ b/examples/test/test_tree1024.py @@ -1,37 +1,27 @@ #!/usr/bin/env python -"""TEST""" +""" +Test for tree1024.py +""" import unittest import pexpect -#from time import sleep -from mininet.log import setLogLevel -#from mininet.net import Mininet -#from mininet.node import CPULimitedHost -#from mininet.link import TCLink - -#from mininet.examples.simpleperf import SingleSwitchTopo class testTree1024( unittest.TestCase ): - "Test ping with single switch topology (common code)." prompt = 'mininet>' def testTree1024( self ): + "Run the example and do a simple ping test from h1 to h1024" p = pexpect.spawn( 'python -m mininet.examples.tree1024' ) p.expect( self.prompt, timeout=6000 ) # it takes awhile to set up p.sendline( 'h1 ping -c 1 h1024' ) p.expect ( '(\d+)% packet loss' ) percent = int( p.match.group( 1 ) ) if p.match else -1 - #self.assertEqual( percent, 0 ) - #p.expect( self.prompt ) - #p.sendline( 'iperf' ) - #p.expect( "Results: \['\d+ .bits/sec', '\d+ .bits/sec'\]" ) p.expect( self.prompt ) p.sendline( 'exit' ) p.wait() self.assertEqual( percent, 0 ) if __name__ == '__main__': - setLogLevel( 'warning' ) unittest.main() diff --git a/examples/test/test_treeping64.py b/examples/test/test_treeping64.py index 8328c35..8154314 100755 --- a/examples/test/test_treeping64.py +++ b/examples/test/test_treeping64.py @@ -1,23 +1,18 @@ #!/usr/bin/env python -"""TEST""" +""" +Test for treeping64.py +""" import unittest import pexpect -#from time import sleep -from mininet.log import setLogLevel -#from mininet.net import Mininet -#from mininet.node import CPULimitedHost -#from mininet.link import TCLink - -#from mininet.examples.simpleperf import SingleSwitchTopo class testTreePing64( unittest.TestCase ): - "Test ping with single switch topology (common code)." prompt = 'mininet>' def testTreePing64( self ): + "Run the example and verify ping results" p = pexpect.spawn( 'python -m mininet.examples.treeping64' ) p.expect( 'Tree network ping results:', timeout=6000 ) count = 0 @@ -32,5 +27,4 @@ class testTreePing64( unittest.TestCase ): self.assertTrue( count > 0 ) if __name__ == '__main__': - setLogLevel( 'warning' ) unittest.main() From 24b38126ececb3d7fed101640f33b67709598e74 Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Tue, 10 Sep 2013 17:21:08 -0700 Subject: [PATCH 099/109] cleaned up and commented test_simpleperf.py --- examples/test/test_simpleperf.py | 38 ++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/examples/test/test_simpleperf.py b/examples/test/test_simpleperf.py index 2e589d7..319ebc5 100755 --- a/examples/test/test_simpleperf.py +++ b/examples/test/test_simpleperf.py @@ -1,10 +1,12 @@ #!/usr/bin/env python -"""TEST""" +""" +Test for simpleperf.py +""" import unittest import pexpect -from time import sleep +import re from mininet.log import setLogLevel from mininet.net import Mininet from mininet.node import CPULimitedHost @@ -13,24 +15,36 @@ from mininet.link import TCLink from mininet.examples.simpleperf import SingleSwitchTopo class testSimplePerf( unittest.TestCase ): - "Test ping with single switch topology (common code)." - def testE2E( self ): - results = [ "Results:", pexpect.EOF, pexpect.TIMEOUT ] + "Run the example and verify ping and iperf results" p = pexpect.spawn( 'python -m mininet.examples.simpleperf' ) - index = p.expect( results, timeout=600 ) - self.assertEqual( index, 0 ) + # check ping results + p.expect( "Results: (\d+)% dropped", timeout=120 ) + loss = int( p.match.group( 1 ) ) + self.assertTrue( loss > 0 and loss < 100 ) + # check iperf results + p.expect( "Results: \['([\d\.]+) .bits/sec", timeout=480 ) + bw = float( p.match.group( 1 ) ) + self.assertTrue( bw > 0 ) p.wait() def testTopo( self ): - topo = SingleSwitchTopo(n=4) - net = Mininet(topo=topo, - host=CPULimitedHost, link=TCLink) + """Import SingleSwitchTopo from example and test connectivity between two hosts + Note: this test may fail very rarely because it is non-deterministic + i.e. links are configured with 10% packet loss, but if we get unlucky and + none or all of the packets are dropped, the test will fail""" + topo = SingleSwitchTopo( n=4 ) + net = Mininet( topo=topo, host=CPULimitedHost, link=TCLink ) net.start() - h1, h4 = net.get('h1', 'h4') - h1.cmd( 'ping -c 1 %s' % h4.IP() ) + h1, h4 = net.get( 'h1', 'h4' ) + # have h1 ping h4 ten times + expectStr = '(\d+) packets transmitted, (\d+) received, (\d+)% packet loss' + output = h1.cmd( 'ping -c 10 %s' % h4.IP() ) + m = re.search( expectStr, output ) + loss = int( m.group( 3 ) ) net.stop() + self.assertTrue( loss > 0 and loss < 100 ) if __name__ == '__main__': setLogLevel( 'warning' ) From 1e9e781c127c280e20dc806b9d53c34ed1c43c0c Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Tue, 10 Sep 2013 18:15:28 -0700 Subject: [PATCH 100/109] changed 1% to 2% in test_limit.py --- examples/test/test_limit.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/test/test_limit.py b/examples/test/test_limit.py index ea5ce35..99a8c25 100755 --- a/examples/test/test_limit.py +++ b/examples/test/test_limit.py @@ -10,14 +10,14 @@ import pexpect class testLimit( unittest.TestCase ): def testLimit( self ): - "Verify that CPU limits are within a 1% tolerance of limit for each scheduler" + "Verify that CPU limits are within a 2% tolerance of limit for each scheduler" p = pexpect.spawn( 'python -m mininet.examples.limit' ) opts = [ '\*\*\* Testing network ([\d\.]+) Mbps', '\*\*\* Results: \[([\d\., ]+)\]', pexpect.EOF ] count = 0 bw = 0 - tolerance = 1 + tolerance = 2 while True: index = p.expect( opts ) if index == 0: From 10fdd01dc8ac58d2beb25e4eb689e4a90d7362ac Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Tue, 10 Sep 2013 18:46:07 -0700 Subject: [PATCH 101/109] fixed runner.py and added -v and -quick options --- examples/test/runner.py | 37 ++++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) mode change 100644 => 100755 examples/test/runner.py diff --git a/examples/test/runner.py b/examples/test/runner.py old mode 100644 new mode 100755 index dfa2702..4939e08 --- a/examples/test/runner.py +++ b/examples/test/runner.py @@ -1,10 +1,29 @@ -import glob -import unittest +#!/usr/bin/env python -test_file_strings = glob.glob('test_*.py') -module_strings = [str[0:len(str)-3] for str in test_file_strings] -print module_strings -suites = [unittest.defaultTestLoader.loadTestsFromName(str) for str - in module_strings] -testSuite = unittest.TestSuite(suites) -text_runner = unittest.TextTestRunner().run(testSuite) \ No newline at end of file +""" +Run all mininet.examples tests + -v : verbose output + -quick : skip tests that take more than ~30 seconds +""" + +import unittest +import os +import sys +from mininet.util import ensureRoot +from mininet.clean import cleanup + +def runTests( testDir, verbosity=1 ): + "discover and run all tests in testDir" + # ensure root and cleanup before starting tests + ensureRoot() + cleanup() + # discover all tests in testDir + testSuite = unittest.defaultTestLoader.discover( testDir ) + # run tests + unittest.TextTestRunner( verbosity=verbosity ).run( testSuite ) + +if __name__ == '__main__': + # get the directory containing example tests + testDir = os.path.dirname( os.path.realpath( __file__ ) ) + verbosity = 2 if '-v' in sys.argv else 1 + runTests( testDir, verbosity ) From c5da46f1255111a3c163f7bfa42dd5e4034526ce Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Tue, 10 Sep 2013 18:49:13 -0700 Subject: [PATCH 102/109] added -quick options to skip long tests --- examples/test/test_cpu.py | 2 ++ examples/test/test_limit.py | 2 ++ examples/test/test_linearbandwidth.py | 2 ++ examples/test/test_simpleperf.py | 2 ++ examples/test/test_tree1024.py | 2 ++ examples/test/test_treeping64.py | 2 ++ 6 files changed, 12 insertions(+) diff --git a/examples/test/test_cpu.py b/examples/test/test_cpu.py index dd35490..2547784 100755 --- a/examples/test/test_cpu.py +++ b/examples/test/test_cpu.py @@ -6,11 +6,13 @@ Test for cpu.py import unittest import pexpect +import sys class testCPU( unittest.TestCase ): prompt = 'mininet>' + @unittest.skipIf( '-quick' in sys.argv, 'long test' ) def testCPU( self ): "Verify that CPU utilization is monotonically decreasing for each scheduler" p = pexpect.spawn( 'python -m mininet.examples.cpu' ) diff --git a/examples/test/test_limit.py b/examples/test/test_limit.py index 99a8c25..db9eb33 100755 --- a/examples/test/test_limit.py +++ b/examples/test/test_limit.py @@ -6,9 +6,11 @@ Test for limit.py import unittest import pexpect +import sys class testLimit( unittest.TestCase ): + @unittest.skipIf( '-quick' in sys.argv, 'long test' ) def testLimit( self ): "Verify that CPU limits are within a 2% tolerance of limit for each scheduler" p = pexpect.spawn( 'python -m mininet.examples.limit' ) diff --git a/examples/test/test_linearbandwidth.py b/examples/test/test_linearbandwidth.py index b6c47fd..d3c1144 100755 --- a/examples/test/test_linearbandwidth.py +++ b/examples/test/test_linearbandwidth.py @@ -6,9 +6,11 @@ Test for linearbandwidth.py import unittest import pexpect +import sys class testLinearBandwidth( unittest.TestCase ): + @unittest.skipIf( '-quick' in sys.argv, 'long test' ) def testLinearBandwidth( self ): "Verify that bandwidth is monotonically decreasing as # of hops increases" p = pexpect.spawn( 'python -m mininet.examples.linearbandwidth' ) diff --git a/examples/test/test_simpleperf.py b/examples/test/test_simpleperf.py index 319ebc5..7d44c9c 100755 --- a/examples/test/test_simpleperf.py +++ b/examples/test/test_simpleperf.py @@ -7,6 +7,7 @@ Test for simpleperf.py import unittest import pexpect import re +import sys from mininet.log import setLogLevel from mininet.net import Mininet from mininet.node import CPULimitedHost @@ -16,6 +17,7 @@ from mininet.examples.simpleperf import SingleSwitchTopo class testSimplePerf( unittest.TestCase ): + @unittest.skipIf( '-quick' in sys.argv, 'long test' ) def testE2E( self ): "Run the example and verify ping and iperf results" p = pexpect.spawn( 'python -m mininet.examples.simpleperf' ) diff --git a/examples/test/test_tree1024.py b/examples/test/test_tree1024.py index 2d180cb..fbc82c5 100755 --- a/examples/test/test_tree1024.py +++ b/examples/test/test_tree1024.py @@ -6,11 +6,13 @@ Test for tree1024.py import unittest import pexpect +import sys class testTree1024( unittest.TestCase ): prompt = 'mininet>' + @unittest.skipIf( '-quick' in sys.argv, 'long test' ) def testTree1024( self ): "Run the example and do a simple ping test from h1 to h1024" p = pexpect.spawn( 'python -m mininet.examples.tree1024' ) diff --git a/examples/test/test_treeping64.py b/examples/test/test_treeping64.py index 8154314..ae02afc 100755 --- a/examples/test/test_treeping64.py +++ b/examples/test/test_treeping64.py @@ -6,11 +6,13 @@ Test for treeping64.py import unittest import pexpect +import sys class testTreePing64( unittest.TestCase ): prompt = 'mininet>' + @unittest.skipIf( '-quick' in sys.argv, 'long test' ) def testTreePing64( self ): "Run the example and verify ping results" p = pexpect.spawn( 'python -m mininet.examples.treeping64' ) From bfb560045c2c2fad9fc155b8e729db714411f970 Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Tue, 10 Sep 2013 18:55:21 -0700 Subject: [PATCH 103/109] add rm to sshd tests --- examples/test/test_baresshd.py | 1 + examples/test/test_sshd.py | 1 + 2 files changed, 2 insertions(+) diff --git a/examples/test/test_baresshd.py b/examples/test/test_baresshd.py index b76fe20..d708761 100755 --- a/examples/test/test_baresshd.py +++ b/examples/test/test_baresshd.py @@ -29,6 +29,7 @@ class testBareSSHD( unittest.TestCase ): # verify that sshd is not running self.assertFalse( self.connected() ) # create public key pair for testing + sh( 'rm -rf /tmp/ssh' ) sh( 'mkdir /tmp/ssh' ) sh( "ssh-keygen -t rsa -P '' -f /tmp/ssh/test_rsa" ) sh( 'cat /tmp/ssh/test_rsa.pub >> /tmp/ssh/authorized_keys' ) diff --git a/examples/test/test_sshd.py b/examples/test/test_sshd.py index 9d4ef1f..04bd184 100755 --- a/examples/test/test_sshd.py +++ b/examples/test/test_sshd.py @@ -34,6 +34,7 @@ class testSSHD( unittest.TestCase ): def setUp( self ): # create public key pair for testing + sh( 'rm -rf /tmp/ssh' ) sh( 'mkdir /tmp/ssh' ) sh( "ssh-keygen -t rsa -P '' -f /tmp/ssh/test_rsa" ) sh( 'cat /tmp/ssh/test_rsa.pub >> /tmp/ssh/authorized_keys' ) From 7bd9a79b12f2867d18824b049edefd6fcedfecae Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Wed, 11 Sep 2013 22:59:50 -0700 Subject: [PATCH 104/109] Add --test {test} and --branch {branch} options, and exampletest --- util/vm/build.py | 113 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 89 insertions(+), 24 deletions(-) diff --git a/util/vm/build.py b/util/vm/build.py index b7db867..adefd9f 100755 --- a/util/vm/build.py +++ b/util/vm/build.py @@ -30,7 +30,7 @@ import os from os import stat, path from stat import ST_MODE, ST_SIZE from os.path import abspath -from sys import exit, stdout, argv +from sys import exit, stdout, argv, modules import re from glob import glob from subprocess import check_output, call, Popen @@ -38,16 +38,20 @@ from tempfile import mkdtemp, NamedTemporaryFile from time import time, strftime, localtime import argparse from distutils.spawn import find_executable +import inspect pexpect = None # For code check - imported dynamically # boot can be slooooow!!!! need to debug/optimize somehow TIMEOUT=600 -# Some configuration +# Some configuration options +# Possibly change this to use the parsed arguments instead! + LogToConsole = False # VM output to console rather than log file SaveQCOW2 = False # Save QCOW2 image rather than deleting it NoKVM = False # Don't use kvm and use emulation instead +Branch = None # Branch to update and check out before testing VMImageDir = os.environ[ 'HOME' ] + '/vm-images' @@ -466,6 +470,27 @@ def coreTest( vm, prompt=Prompt ): log( '* Test', test, 'FAILED' ) +def examplesquickTest( vm, prompt=Prompt ): + "Quick test of mininet examples" + vm.sendline( 'sudo apt-get install python-pexpect' ) + vm.expect( prompt ) + vm.sendline( 'sudo python ~/mininet/examples/test/runner.py -quick' ) + + +def examplesfullTest( vm, prompt=Prompt ): + "Full (slow) test of mininet examples" + vm.sendline( 'sudo apt-get install python-pexpect' ) + vm.expect( prompt ) + vm.sendline( 'sudo python ~/mininet/examples/test/runner.py' ) + + +def checkOutBranch( vm, branch, prompt=Prompt ): + vm.sendline( 'cd ~/mininet; git fetch; git pull --rebase; git checkout ' + + branch ) + vm.expect( prompt ) + vm.sendline( 'sudo make install' ) + + def interact( vm, prompt=Prompt ): "Interact with vm, which is a pexpect object" login( vm ) @@ -490,10 +515,7 @@ def interact( vm, prompt=Prompt ): log( '* Completed successfully' ) vm.expect( prompt ) log( '* Testing Mininet' ) - sanityTest( vm ) - vm.expect( prompt ) - coreTest( vm ) - vm.expect( prompt ) + runTests( vm ) log( '* Shutting down' ) vm.sendline( 'sync; sudo shutdown -h now' ) log( '* Waiting for EOF/shutdown' ) @@ -670,33 +692,50 @@ def build( flavor='raring32server' ): os.chdir( '..' ) -def bootAndTest( image, tests=None ): +def runTests( vm, tests=None, prompt=Prompt ): + "Run tests (list) in vm (pexpect object)" + if not tests: + tests = [ 'sanity', 'core' ] + testfns = testDict() + for test in tests: + if test not in testfns: + raise Exception( 'Unknown test: ' + test ) + log( '* Running test', test ) + fn = testfns[ test ] + fn( vm ) + vm.expect( prompt ) + + +def bootAndRunTests( image, tests=None ): """Boot and test VM - tests: list of tests (default: sanityTest, coreTest)""" + tests: list of tests (default: sanity, core)""" bootTestStart = time() - if tests is None: - tests = [ sanityTest, coreTest ] basename = path.basename( image ) image = abspath( image ) tmpdir = mkdtemp( prefix='test-' + basename ) - cow = path.join( tmpdir, image + '-cow.qcow2' ) - log( '* Creating COW disk' ) + log( '* Using tmpdir', tmpdir ) + cow = path.join( tmpdir, basename + '.qcow2' ) + log( '* Creating COW disk', cow ) run( 'qemu-img create -f qcow2 -b %s %s' % ( image, cow ) ) log( '* Extracting kernel and initrd' ) kernel, initrd = extractKernel( image, flavor=basename, imageDir=tmpdir ) if LogToConsole: logfile = stdout else: - logfile = NamedTemporaryFile( prefix=image, delete=False ) + logfile = NamedTemporaryFile( prefix=basename, + suffix='.testlog', delete=False ) log( '* Logging VM output to', logfile.name ) vm = boot( cow=cow, kernel=kernel, initrd=initrd, logfile=logfile ) prompt = '\$ ' login( vm ) log( '* Waiting for VM boot and login' ) vm.expect( prompt ) - for test in tests: - test( vm ) + if Branch: + checkOutBranch( vm, branch=Branch ) vm.expect( prompt ) + log( '* Running tests' ) + runTests( vm, tests=tests ) + # runTests eats its last prompt, but maybe it shouldn't... log( '* Shutting down' ) vm.sendline( 'sudo shutdown -h now ' ) log( '* Waiting for shutdown' ) @@ -709,28 +748,52 @@ def bootAndTest( image, tests=None ): def buildFlavorString(): "Return string listing valid build flavors" - return 'valid build flavors: %s' % ' '.join( sorted( isoURLs ) ) + return 'valid build flavors: ( %s )' % ' '.join( sorted( isoURLs ) ) + + +def testDict(): + "Return dict of tests in this module" + suffix = 'Test' + trim = len( suffix ) + fdict = dict( [ ( fname[ : -trim ], f ) for fname, f in + inspect.getmembers( modules[ __name__ ], + inspect.isfunction ) + if fname.endswith( suffix ) ] ) + return fdict + + +def testString(): + "Return string listing valid tests" + return 'valid tests: ( %s )' % ' '.join( testDict().keys() ) def parseArgs(): "Parse command line arguments and run" - global LogToConsole, NoKVM + global LogToConsole, NoKVM, Branch parser = argparse.ArgumentParser( description='Mininet VM build script', - epilog=buildFlavorString() ) + epilog=buildFlavorString() + ' ' + + testString() ) parser.add_argument( '-v', '--verbose', action='store_true', help='send VM output to console rather than log file' ) parser.add_argument( '-d', '--depend', action='store_true', help='install dependencies for this script' ) parser.add_argument( '-l', '--list', action='store_true', - help='list valid build flavors' ) + help='list valid build flavors and tests' ) parser.add_argument( '-c', '--clean', action='store_true', help='clean up leftover build junk (e.g. qemu-nbd)' ) parser.add_argument( '-q', '--qcow2', action='store_true', help='save qcow2 image rather than deleting it' ) parser.add_argument( '-n', '--nokvm', action='store_true', help="Don't use kvm - use tcg emulation instead" ) - parser.add_argument( '-t', '--test', metavar='image', action='append', - help='Boot and test a VM image' ) + parser.add_argument( '-i', '--image', metavar='image', default=[], + action='append', + help='Boot and test an existing VM image' ) + parser.add_argument( '-t', '--test', metavar='test', default=[], + action='append', + help='specify a test to run' ) + parser.add_argument( '-b', '--branch', metavar='branch', + help='For an existing VM image, check out and install' + ' this branch before testing' ) parser.add_argument( 'flavor', nargs='*', help='VM flavor(s) to build (e.g. raring32server)' ) args = parser.parse_args() @@ -744,6 +807,8 @@ def parseArgs(): LogToConsole = True if args.nokvm: NoKVM = True + if args.branch: + Branch = args.branch for flavor in args.flavor: if flavor not in isoURLs: print "Unknown build flavor:", flavor @@ -754,10 +819,10 @@ def parseArgs(): # except Exception as e: # log( '* BUILD FAILED with exception: ', e ) # exit( 1 ) - for image in args.test: - bootAndTest( image ) + for image in args.image: + bootAndRunTests( image, tests=args.test ) if not ( args.depend or args.list or args.clean or args.flavor - or args.test ): + or args.image ): parser.print_help() From 4ea0c0936d6c7b2efa95ce135df79c3c208061ec Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Thu, 12 Sep 2013 13:49:40 -0700 Subject: [PATCH 105/109] Updated mininet/util.py to support better resource setting semantics and protected with try block --- mininet/util.py | 65 +++++++++++++++++++++++++++---------------------- 1 file changed, 36 insertions(+), 29 deletions(-) diff --git a/mininet/util.py b/mininet/util.py index 05df136..385cdb7 100644 --- a/mininet/util.py +++ b/mininet/util.py @@ -1,6 +1,6 @@ "Utility functions for Mininet." -from mininet.log import output, info, error, warn +from mininet.log import output, info, error, warn, debug from time import sleep from resource import getrlimit, setrlimit, RLIMIT_NPROC, RLIMIT_NOFILE @@ -361,16 +361,17 @@ def sysctlTestAndSet( name, limit ): if '/' not in name: name = '/proc/sys/' + name.replace( '.', '/' ) #read limit - f = open( name, 'r+' ) - oldLimit = f.readline() - if type( limit ) is int: - #compare integer limits before overriding - if int( oldLimit ) < limit: - f.write( "%d" % limit ) - else: - #overwrite non-integer limits - f.write( limit ) - f.close() + with open( name, 'r' ) as readFile: + oldLimit = readFile.readline() + if type( limit ) is int: + #compare integer limits before overriding + if int( oldLimit ) < limit: + with open( name, 'w' ) as writeFile: + writeFile.write( "%d" % limit ) + else: + #overwrite non-integer limits + with open( name, 'w' ) as writeFile: + writeFile.write( limit ) def rlimitTestAndSet( name, limit ): "Helper function to set rlimits" @@ -381,24 +382,30 @@ def rlimitTestAndSet( name, limit ): def fixLimits(): "Fix ridiculously small resource limits." - rlimitTestAndSet( RLIMIT_NPROC, 8192 ) - rlimitTestAndSet( RLIMIT_NOFILE, 16384 ) - #Increase open file limit - sysctlTestAndSet( 'fs.file-max', 10000 ) - #Increase network buffer space - sysctlTestAndSet( 'net.core.wmem_max', 16777216 ) - sysctlTestAndSet( 'net.core.rmem_max', 16777216 ) - sysctlTestAndSet( 'net.ipv4.tcp_rmem', '10240 87380 16777216' ) - sysctlTestAndSet( 'net.ipv4.tcp_wmem', '10240 87380 16777216' ) - sysctlTestAndSet( 'net.core.netdev_max_backlog', 5000 ) - #Increase arp cache size - sysctlTestAndSet( 'net.ipv4.neigh.default.gc_thresh1', 4096 ) - sysctlTestAndSet( 'net.ipv4.neigh.default.gc_thresh2', 8192 ) - sysctlTestAndSet( 'net.ipv4.neigh.default.gc_thresh3', 16384 ) - #Increase routing table size - sysctlTestAndSet( 'net.ipv4.route.max_size', 32768 ) - #Increase number of PTYs for nodes - sysctlTestAndSet( 'kernel.pty.max', 20000 ) + debug( "*** Setting resource limits\n" ) + try: + rlimitTestAndSet( RLIMIT_NPROC, 8192 ) + rlimitTestAndSet( RLIMIT_NOFILE, 16384 ) + #Increase open file limit + sysctlTestAndSet( 'fs.file-max', 10000 ) + #Increase network buffer space + sysctlTestAndSet( 'net.core.wmem_max', 16777216 ) + sysctlTestAndSet( 'net.core.rmem_max', 16777216 ) + sysctlTestAndSet( 'net.ipv4.tcp_rmem', '10240 87380 16777216' ) + sysctlTestAndSet( 'net.ipv4.tcp_wmem', '10240 87380 16777216' ) + sysctlTestAndSet( 'net.core.netdev_max_backlog', 5000 ) + #Increase arp cache size + sysctlTestAndSet( 'net.ipv4.neigh.default.gc_thresh1', 4096 ) + sysctlTestAndSet( 'net.ipv4.neigh.default.gc_thresh2', 8192 ) + sysctlTestAndSet( 'net.ipv4.neigh.default.gc_thresh3', 16384 ) + #Increase routing table size + sysctlTestAndSet( 'net.ipv4.route.max_size', 32768 ) + #Increase number of PTYs for nodes + sysctlTestAndSet( 'kernel.pty.max', 20000 ) + assert False + except: + warn( "*** Error setting resource limits. " + "Mininet's performance may be affected.\n" ) def mountCgroups(): "Make sure cgroups file system is mounted" From 4e242e921130db7449793d1e5b4ded28ff9c6464 Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Thu, 12 Sep 2013 14:14:55 -0700 Subject: [PATCH 106/109] Add -v so that we can see exampletest results --- util/vm/build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/util/vm/build.py b/util/vm/build.py index adefd9f..79ec1dd 100755 --- a/util/vm/build.py +++ b/util/vm/build.py @@ -474,14 +474,14 @@ def examplesquickTest( vm, prompt=Prompt ): "Quick test of mininet examples" vm.sendline( 'sudo apt-get install python-pexpect' ) vm.expect( prompt ) - vm.sendline( 'sudo python ~/mininet/examples/test/runner.py -quick' ) + vm.sendline( 'sudo python ~/mininet/examples/test/runner.py -v -quick' ) def examplesfullTest( vm, prompt=Prompt ): "Full (slow) test of mininet examples" vm.sendline( 'sudo apt-get install python-pexpect' ) vm.expect( prompt ) - vm.sendline( 'sudo python ~/mininet/examples/test/runner.py' ) + vm.sendline( 'sudo python ~/mininet/examples/test/runner.py -v' ) def checkOutBranch( vm, branch, prompt=Prompt ): From 5b9f6b219282f77e95dbb0707559d716a74b69b0 Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Thu, 12 Sep 2013 14:16:58 -0700 Subject: [PATCH 107/109] Added .md to README so that it displays nicely on github. --- examples/{README => README.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename examples/{README => README.md} (100%) diff --git a/examples/README b/examples/README.md similarity index 100% rename from examples/README rename to examples/README.md From 4b719d74438d18ff18c66960df05f2138cd2d2c9 Mon Sep 17 00:00:00 2001 From: Bob Date: Thu, 12 Sep 2013 14:21:20 -0700 Subject: [PATCH 108/109] Minor cosmetic edits --- examples/README.md | 49 +++++++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/examples/README.md b/examples/README.md index 7b3f795..5463101 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,4 +1,3 @@ - Mininet Examples ======================================================== @@ -7,114 +6,114 @@ Mininet's Python API. ======================================================== -### baresshd.py: +#### baresshd.py: This example uses Mininet's medium-level API to create an sshd process running in a namespace. Doesn't use OpenFlow. -### consoles.py: +#### consoles.py: This example creates a grid of console windows, one for each node, and allows interaction with and monitoring of each console, including graphical monitoring. -### controllers.py: +#### controllers.py: This example creates a network with multiple controllers, by -using a custom Switch() subclass. +using a custom `Switch()` subclass. -### controllers2.py: +#### controllers2.py: This example creates a network with multiple controllers by creating an empty network, adding nodes to it, and manually starting the switches. -### controlnet.py: +#### controlnet.py: This examples shows how you can model the control network as well as the data network, by actually creating two Mininet objects. -### cpu.py: +#### cpu.py: This example tests iperf bandwidth for varying CPU limits. -### emptynet.py: +#### emptynet.py: This example demonstrates creating an empty network (i.e. with no topology object) and adding nodes to it. -### hwintf.py: +#### hwintf.py: This example shows how to add an interface (for example a real hardware interface) to a network after the network is created. -### limit.py: +#### limit.py: This example shows how to use link and CPU limits. -### linearbandwidth.py: +#### linearbandwidth.py: This example shows how to create a custom topology programatically by subclassing Topo, and how to run a series of tests on it. -### miniedit.py: +#### miniedit.py: This example demonstrates creating a network via a graphical editor. -### multiping.py: +#### multiping.py: This example demonstrates one method for monitoring output from multiple hosts, using `node.monitor()`. -### multipoll.py: +#### multipoll.py: This example demonstrates monitoring output files from multiple hosts. -### multitest.py: +#### multitest.py: This example creates a network and runs multiple tests on it. -### nat.py: +#### nat.py: This example shows how to connect a Mininet network to the Internet using NAT. It also answers the eternal question "why can't I ping google?" -### popen.py: +#### popen.py: This example monitors a number of hosts using `host.popen()` and `pmonitor()`. -### popenpoll.py: +#### popenpoll.py: This example demonstrates monitoring output from multiple hosts using the `node.popen()` interface (which returns Popen objects) and `pmonitor()`. -### scratchnet.py, scratchnetuser.py: +#### scratchnet.py, scratchnetuser.py: These two examples demonstrate how to create a network by using the lowest- level Mininet functions. Generally the higher-level API is easier to use, but scratchnet shows what is going on behind the scenes. -### simpleperf.py: +#### simpleperf.py: A simple example of configuring network and CPU bandwidth limits. -### sshd.py: +#### sshd.py: -This example shows how to run an sshd process in each host, allowing +This example shows how to run an `sshd` process in each host, allowing you to log in via ssh. This requires connecting the Mininet data network to an interface in the root namespace (generaly the control network already lives in the root namespace, so it does not need to be explicitly connected.) -### tree1024.py: +#### tree1024.py: This example attempts to create a 1024-host network, and then runs the CLI on it. It may run into scalability limits, depending on available memory and sysctl configuration (see `INSTALL`.) -### treeping64.py: +#### treeping64.py: This example creates a 64-host tree network, and attempts to check full connectivity using ping, for different switch/datapath types. From d70ca981c983d015f22419591b744e1fde489a65 Mon Sep 17 00:00:00 2001 From: Bob Date: Thu, 12 Sep 2013 14:23:19 -0700 Subject: [PATCH 109/109] Update README.md --- examples/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/README.md b/examples/README.md index 5463101..4be5564 100644 --- a/examples/README.md +++ b/examples/README.md @@ -77,7 +77,7 @@ This example creates a network and runs multiple tests on it. This example shows how to connect a Mininet network to the Internet using NAT. It also answers the eternal question "why can't I ping -google?" +`google.com`?" #### popen.py: @@ -87,7 +87,7 @@ This example monitors a number of hosts using `host.popen()` and #### popenpoll.py: This example demonstrates monitoring output from multiple hosts using -the `node.popen()` interface (which returns Popen objects) and `pmonitor()`. +the `node.popen()` interface (which returns `Popen` objects) and `pmonitor()`. #### scratchnet.py, scratchnetuser.py: @@ -102,7 +102,7 @@ A simple example of configuring network and CPU bandwidth limits. #### sshd.py: This example shows how to run an `sshd` process in each host, allowing -you to log in via ssh. This requires connecting the Mininet data network +you to log in via `ssh`. This requires connecting the Mininet data network to an interface in the root namespace (generaly the control network already lives in the root namespace, so it does not need to be explicitly connected.) @@ -111,9 +111,9 @@ connected.) This example attempts to create a 1024-host network, and then runs the CLI on it. It may run into scalability limits, depending on available -memory and sysctl configuration (see `INSTALL`.) +memory and `sysctl` configuration (see `INSTALL`.) #### treeping64.py: This example creates a 64-host tree network, and attempts to check full -connectivity using ping, for different switch/datapath types. +connectivity using `ping`, for different switch/datapath types.