Build script for 20.04 (#1004)

Should be able to build 16.04, 18.04 and 20.04,
as well as branch builds.

util/install.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 
 # Mininet install script for Ubuntu and Debian
 # Original author: Brandon Heller
@@ -181,7 +181,13 @@ function mn_deps {
                  ethtool help2man $pf pylint pep8 \
                  net-tools \
                  ${PYPKG}-pexpect ${PYPKG}-tk
+        # Install pip
         $install ${PYPKG}-pip || $install ${PYPKG}-pip-whl
+        if ! ${PYTHON} -m pip; then
+            curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
+            sudo ${PYTHON} get-pip.py
+            rm get-pip.py
+        fi
         $install iproute2 || $install iproute
         $install cgroup-tools || $install cgroup-bin
     fi
@@ -192,9 +198,9 @@ function mn_deps {
     popd
 }
 
-# Install Mininet developer dependencies
+# Install Mininet documentation dependencies
-function mn_dev {
+function mn_doc {
-    echo "Installing Mininet developer dependencies"
+    echo "Installing Mininet documentation dependencies"
     $install doxygen doxypy texlive-fonts-recommended
     if ! $install doxygen-latex; then
         echo "doxygen-latex not needed"
@@ -437,6 +443,12 @@ function ovs {
             sudo update-rc.d $OVSC disable
         fi
     fi
+    # This service seems to hang on 20.04
+    if systemctl list-units | \
+            grep status netplan-ovs-cleanup.service>&/dev/null; then
+        echo 'TimeoutSec=10' | sudo EDITOR='tee -a' \
+        sudo systemctl edit --full netplan-ovs-cleanup.service
+    fi
 }
 
 function remove_ovs {
@@ -602,7 +614,8 @@ function oftest {
     echo "Installing oftest..."
 
     # Install deps:
-    $install tcpdump ${PYPKG}-scapy
+    $install tcpdump
+    $install ${PYPKG}-scapy || sudo $PYTHON -m pip install scapy
 
     # Install oftest:
     cd $BUILD_DIR/
@@ -660,9 +673,10 @@ net.ipv6.conf.default.disable_ipv6 = 1
 net.ipv6.conf.lo.disable_ipv6 = 1' | sudo tee -a /etc/sysctl.conf > /dev/null
     fi
     # Since the above doesn't disable neighbor discovery, also do this:
+    # disable via boot line, and also restore eth0 naming for VM use
     if ! grep 'ipv6.disable' /etc/default/grub; then
         sudo sed -i -e \
-        's/GRUB_CMDLINE_LINUX_DEFAULT="/GRUB_CMDLINE_LINUX_DEFAULT="ipv6.disable=1 /' \
+        's/GRUB_CMDLINE_LINUX_DEFAULT="/GRUB_CMDLINE_LINUX_DEFAULT="ipv6.disable=1 net.ifnames=0 /' \
         /etc/default/grub
         sudo update-grub
     fi
@@ -740,8 +754,8 @@ function all {
     echo "Installing all packages except for -eix (doxypy, ivs, nox-classic)..."
     kernel
     mn_deps
-    # Skip mn_dev (doxypy/texlive/fonts/etc.) because it's huge
+    # Skip mn_doc (doxypy/texlive/fonts/etc.) because it's huge
-    # mn_dev
+    # mn_doc
     of
     install_wireshark
     ovs
@@ -772,8 +786,12 @@ function vm_clean {
     # Remove SSH keys and regenerate on boot
     echo 'Removing SSH keys from /etc/ssh/'
     sudo rm -f /etc/ssh/*key*
+    if [ ! -e /etc/rc.local ]; then
+        echo '#!/usr/bin/bash' | sudo tee /etc/rc.local
+        sudo chmod +x /etc/rc.local
+    fi
     if ! grep mininet /etc/rc.local >& /dev/null; then
-        sudo sed -i -e "s/exit 0//" /etc/rc.local
+        sudo sed -i -e "s/exit 0//" /etc/rc.local || true
         echo '
 # mininet: regenerate ssh keys if we deleted them
 if ! stat -t /etc/ssh/*key* >/dev/null 2>&1; then
@@ -781,15 +799,15 @@ if ! stat -t /etc/ssh/*key* >/dev/null 2>&1; then
 fi
 exit 0
 ' | sudo tee -a /etc/rc.local > /dev/null
+        sudo chmod +x /etc/rc.local
     fi
 
-    # Remove Mininet files
-    #sudo rm -f /lib/modules/python2.5/site-packages/mininet*
-    #sudo rm -f /usr/bin/mnexec
-
     # Clear optional dev script for SSH keychain load on boot
     rm -f ~/.bash_profile
 
+    # Remove leftover install script if any
+    rm -f install-mininet-vm.sh
+
     # Clear git changes
     git config --global user.name "None"
     git config --global user.email "None"
@@ -816,7 +834,7 @@ function usage {
     printf -- ' -b: install controller (B)enchmark (oflops)\n' >&2
     printf -- ' -c: (C)lean up after kernel install\n' >&2
     printf -- ' -d: (D)elete some sensitive files from a VM image\n' >&2
-    printf -- ' -e: install Mininet d(E)veloper dependencies\n' >&2
+    printf -- ' -e: install Mininet documentation/LaT(e)X dependencies\n' >&2
     printf -- ' -f: install Open(F)low\n' >&2
     printf -- ' -h: print this (H)elp message\n' >&2
     printf -- ' -i: install (I)ndigo Virtual Switch\n' >&2
@@ -850,7 +868,7 @@ else
       b)    cbench;;
       c)    kernel_clean;;
       d)    vm_clean;;
-      e)    mn_dev;;
+      e)    mn_doc;;
       f)    case $OF_VERSION in
             1.0) of;;
             1.3) of13;;

util/vm/build.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python2.7
 
 """
 build.py: build a Mininet VM
@@ -31,6 +31,7 @@ from os import stat, path
 from stat import ST_MODE, ST_SIZE
 from os.path import abspath
 from sys import exit, stdout, argv, modules
+import sys
 import re
 from glob import glob
 from subprocess import check_output, call, Popen
@@ -39,11 +40,13 @@ from time import time, strftime, localtime
 import argparse
 from distutils.spawn import find_executable
 import inspect
+from traceback import print_exc
+
 
 pexpect = None  # For code check - imported dynamically
 
 # boot can be slooooow!!!! need to debug/optimize somehow
-TIMEOUT=600
+TIMEOUT = 600
 
 # Some configuration options
 # Possibly change this to use the parsed arguments instead!
@@ -60,10 +63,23 @@ VMImageDir = os.environ[ 'HOME' ] + '/vm-images'
 
 Prompt = '\$ '              # Shell prompt that pexpect will wait for
 
+
+# URLs for Ubunto .iso images
+
+def serverURL( version, arch ):
+    "Return .iso URL for Ubuntu version and arch"
+    server = 'http://cdimage.ubuntu.com/ubuntu/releases/%s/release/'
+    iso = 'ubuntu-%s-server-%s.iso'
+    return (server + iso )  % ( version, version, arch )
+
+def legacyURL( version, arch ):
+    "Return .iso URL for Ubuntu version"
+    server = ( 'http://cdimage.ubuntu.com/ubuntu-legacy-server/'
+               'releases/%s/release/' )
+    iso = 'ubuntu-%s-legacy-server-%s.iso'
+    return (server + iso ) % ( version, version, arch )
+
 isoURLs = {
-    'precise32server':
-    'http://mirrors.kernel.org/ubuntu-releases/12.04/'
-    'ubuntu-12.04.5-server-i386.iso',
     'precise64server':
     'http://mirrors.kernel.org/ubuntu-releases/12.04/'
     'ubuntu-12.04.5-server-amd64.iso',
@@ -73,18 +89,14 @@ isoURLs = {
     'trusty64server':
     'http://mirrors.kernel.org/ubuntu-releases/14.04/'
     'ubuntu-14.04.4-server-amd64.iso',
-    'wily32server':
-    'http://mirrors.kernel.org/ubuntu-releases/15.10/'
-    'ubuntu-15.10-server-i386.iso',
-    'wily64server':
-    'http://mirrors.kernel.org/ubuntu-releases/15.10/'
-    'ubuntu-15.10-server-amd64.iso',
     'xenial32server':
     'http://mirrors.kernel.org/ubuntu-releases/16.04/'
-    'ubuntu-16.04.1-server-i386.iso',
+    'ubuntu-16.04.6-server-i386.iso',
     'xenial64server':
     'http://mirrors.kernel.org/ubuntu-releases/16.04/'
-    'ubuntu-16.04.1-server-amd64.iso',
+    'ubuntu-16.04.7-server-amd64.iso',
+    'bionic64server': serverURL( '18.04.5', 'amd64' ),
+    'focal64server':  legacyURL( '20.04.1', 'amd64' ),
 }
 
 
@@ -122,7 +134,7 @@ def log( *args, **kwargs ):
     else:
         print( output, )
     # Optionally mirror to LogFile
-    if type( LogFile ) is file:
+    if LogFile:
         if cr:
             output += '\n'
         LogFile.write( output )
@@ -131,7 +143,7 @@ def log( *args, **kwargs ):
 
 def run( cmd, **kwargs ):
     "Convenient interface to check_output"
-    log( '-', cmd )
+    log( '+', cmd )
     cmd = cmd.split()
     arg0 = cmd[ 0 ]
     if not find_executable( arg0 ):
@@ -153,7 +165,7 @@ def depend():
     log( '* Installing package dependencies' )
     run( 'sudo apt-get -qy update' )
     run( 'sudo apt-get -qy install'
-         ' kvm cloud-utils genisoimage qemu-kvm qemu-utils'
+         ' kvmtool cloud-utils genisoimage qemu-kvm qemu-utils'
          ' e2fsprogs curl'
          ' python-setuptools mtools zip' )
     run( 'sudo easy_install pexpect' )
@@ -181,7 +193,7 @@ def findiso( flavor ):
     iso = path.join( VMImageDir, name )
     if not path.exists( iso ) or ( stat( iso )[ ST_MODE ] & 0o777 != 0o444 ):
         log( '* Retrieving', url )
-        run( 'curl -C - -o %s %s' % ( iso, url ) )
+        run( 'curl -L -C - -o %s %s' % ( iso, url ) )
         # Make sure the file header/type is something reasonable like
         # 'ISO' or 'x86 boot sector', and not random html or text
         result = run( 'file ' + iso )
@@ -201,10 +213,11 @@ def attachNBD( cow, flags='' ):
     # qemu-nbd requires an absolute path
     cow = abspath( cow )
     log( '* Checking for unused /dev/nbdX device ' )
-    for i in range ( 0, 63 ):
+    for i in range ( 1, 63 ):
-        nbd = '/dev/nbd%d' % i
+        entry = 'nbd%d' % i
+        nbd = '/dev/' + entry
         # Check whether someone's already messing with that device
-        if call( [ 'pgrep', '-f', nbd ] ) == 0:
+        if call( [ 'pgrep', '-f', entry ] ) == 0:
             continue
         srun( 'modprobe nbd max-part=64' )
         srun( 'qemu-nbd %s -c %s %s' % ( flags, nbd, cow ) )
@@ -222,17 +235,26 @@ def extractKernel( image, flavor, imageDir=VMImageDir ):
     "Extract kernel and initrd from base image"
     kernel = path.join( imageDir, flavor + '-vmlinuz' )
     initrd = path.join( imageDir, flavor + '-initrd' )
-    if path.exists( kernel ) and ( stat( image )[ ST_MODE ] & 0o777 ) == 0o444:
-        # If kernel is there, then initrd should also be there
-        return kernel, initrd
     log( '* Extracting kernel to', kernel )
     nbd = attachNBD( image, flags='-r' )
     try:
         print( srun( 'partx ' + nbd ) )
     except:
         log( 'Warning - partx failed with error' )
-    # Assume kernel is in partition 1/boot/vmlinuz*generic for now
+    # Guess that  kernel is in partition 1/boot/vmlinuz*generic
+    # ...but look for other Linux partitions just in case
     part = nbd + 'p1'
+    partitions = srun( 'fdisk -l ' + nbd )
+    for line in partitions.split( '\n' ):
+        line = line.strip()
+        if line.endswith( 'Linux' ):
+            part = line.split()[ 0 ]
+            break
+    partnum = int( part.split( 'p' )[ -1 ] )
+    if path.exists( kernel ) and ( stat( image )[ ST_MODE ] & 0o777 ) == 0o444:
+        # If kernel is there, then initrd should also be there
+        detachNBD( nbd )
+        return kernel, initrd, partnum
     mnt = mkdtemp()
     srun( 'mount -o ro,noload %s %s' % ( part, mnt  ) )
     kernsrc = glob( '%s/boot/vmlinuz*generic' % mnt )[ 0 ]
@@ -244,7 +266,7 @@ def extractKernel( image, flavor, imageDir=VMImageDir ):
     srun( 'umount ' + mnt )
     run( 'rmdir ' + mnt )
     detachNBD( nbd )
-    return kernel, initrd
+    return kernel, initrd, partnum
 
 
 def findBaseImage( flavor, size='8G' ):
@@ -267,9 +289,10 @@ def findBaseImage( flavor, size='8G' ):
         # Write-protect image, also signaling it is complete
         log( '* Write-protecting image', image)
         os.chmod( image, 0o444 )
-    kernel, initrd = extractKernel( image, flavor )
+    kernel, initrd, partnum = extractKernel( image, flavor )
-    log( '* Using base image', image, 'and kernel', kernel )
+    log( '* Using base image', image, 'and kernel', kernel,
-    return image, kernel, initrd
+         'and partition #', partnum )
+    return image, kernel, initrd, partnum
 
 
 # Kickstart and Preseed files for Ubuntu/Debian installer
@@ -382,8 +405,20 @@ def installUbuntu( iso, image, logfilename='install.log', memory=1024 ):
     # Mount iso so we can use its kernel
     mnt = mkdtemp()
     srun( 'mount %s %s' % ( iso, mnt ) )
-    kernel = path.join( mnt, 'install/vmlinuz' )
+    for kdir in 'install', 'casper':
-    initrd = path.join( mnt, 'install/initrd.gz' )
+        kernel = path.join( mnt, kdir, 'vmlinuz' )
+        if not path.exists( kernel ):
+            kernel = ''
+        for initrd in 'initrd.gz', 'initrd':
+            initrd = path.join( mnt, kdir, initrd )
+            if path.exists( initrd ):
+                break
+            else:
+                initrd = ''
+        if kernel and initrd:
+            break
+    if not kernel or not initrd:
+        raise Exception( 'unable to locate kernel and initrd in iso image' )
     if NoKVM:
         accel = 'tcg'
     else:
@@ -407,6 +442,7 @@ def installUbuntu( iso, image, logfilename='install.log', memory=1024 ):
            '-append',
            ' ks=floppy:/' + kickstart +
            ' preseed/file=floppy://' + preseed +
+           ' net.ifnames=0' +
            ' console=ttyS0' ]
     ubuntuStart = time()
     log( '* INSTALLING UBUNTU FROM', iso, 'ONTO', image )
@@ -423,6 +459,8 @@ def installUbuntu( iso, image, logfilename='install.log', memory=1024 ):
         logfile.close()
     elapsed = time() - ubuntuStart
     # Unmount iso and clean up
+    ### DEBUGGING
+    srun( 'ls -l ' + mnt )
     srun( 'umount ' + mnt )
     run( 'rmdir ' + mnt )
     if vm.returncode != 0:
@@ -432,7 +470,7 @@ def installUbuntu( iso, image, logfilename='install.log', memory=1024 ):
     log( '* Ubuntu installation completed in %.2f seconds' % elapsed )
 
 
-def boot( cow, kernel, initrd, logfile, memory=1024, cpuCores=1 ):
+def boot( cow, kernel, initrd, logfile, memory=1024, cpuCores=1, partnum=1 ):
     """Boot qemu/kvm with a COW disk and local/user data store
        cow: COW disk path
        kernel: kernel path
@@ -464,7 +502,9 @@ def boot( cow, kernel, initrd, logfile, memory=1024, cpuCores=1 ):
             '-kernel', kernel,
             '-initrd', initrd,
             '-drive file=%s,if=virtio' % cow,
-            '-append "root=/dev/vda1 init=/sbin/init console=ttyS0" ' ]
+            '-append "root=/dev/vda%d init=/sbin/init'
+            ' net.ifnames=0 console=ttyS0" ' % partnum ]
+    log( cmd )
     if Forward:
         cmd += sum( [ [ '-redir', f ] for f in Forward ], [] )
     if cpuCores > 1:
@@ -590,10 +630,12 @@ def checkOutBranch( vm, branch, prompt=Prompt ):
     # The branch will be rebased to its parent on origin.
     # This probably doesn't matter since we're running on a COW disk
     # anyway.
-    vm.sendline( 'cd ~/mininet; git fetch --all; git checkout '
+    vm.sendline( 'cd ~/mininet; git fetch origin ' + branch +
-                 + branch + '; git pull --rebase origin ' + branch )
+                 '; git checkout ' + branch +
+                 '; git pull --rebase origin ' + branch )
     vm.expect( prompt )
-    vm.sendline( 'sudo -n make install' )
+    # Use install.sh since we may need to identify python version?
+    vm.sendline( 'util/install.sh -n' )
 
 
 def interact( vm, tests, pre='', post='', prompt=Prompt ):
@@ -799,7 +841,7 @@ def build( flavor='raring32server', tests=None, pre='', post='', memory=1024 ):
     ovfdate = strftime( '%y%m%d', lstart )
     dir = 'mn-%s-%s' % ( flavor, date )
     if Branch:
-        dirname = 'mn-%s-%s-%s' % ( Branch, flavor, date )
+        dir = 'mn-%s-%s-%s' % ( Branch, flavor, date )
     try:
         os.mkdir( dir)
     except:
@@ -810,7 +852,7 @@ def build( flavor='raring32server', tests=None, pre='', post='', memory=1024 ):
     LogFile = open( 'build.log', 'w' )
     log( '* Logging to', abspath( LogFile.name ) )
     log( '* Created working directory', dir )
-    image, kernel, initrd = findBaseImage( flavor )
+    image, kernel, initrd, partnum = findBaseImage( flavor )
     basename = 'mininet-' + flavor
     volume = basename + '.qcow2'
     run( 'qemu-img create -f qcow2 -b %s %s' % ( image, volume ) )
@@ -820,7 +862,7 @@ def build( flavor='raring32server', tests=None, pre='', post='', memory=1024 ):
     else:
         logfile = open( flavor + '.log', 'w+' )
     log( '* Logging results to', abspath( logfile.name ) )
-    vm = boot( volume, kernel, initrd, logfile, memory=memory )
+    vm = boot( volume, kernel, initrd, logfile, memory=memory, partnum=partnum )
     version = interact( vm, tests=tests, pre=pre, post=post )
     size = qcow2size( volume )
     arch = archFor( flavor )
@@ -879,7 +921,7 @@ def runTests( vm, tests=None, pre='', post='', prompt=Prompt, uninstallNtpd=Fals
 
 def getMininetVersion( vm ):
     "Run mn to find Mininet version in VM"
-    vm.sendline( '~/mininet/bin/mn --version' )
+    vm.sendline( '(cd ~/mininet; PYTHONPATH=. bin/mn --version)' )
     # Eat command line echo, then read output line
     vm.readline()
     version = vm.readline().strip()
@@ -904,7 +946,7 @@ def bootAndRun( image, prompt=Prompt, memory=1024, cpuCores=1, outputFile=None,
     log( '* Creating COW disk', cow )
     run( 'qemu-img create -f qcow2 -b %s %s' % ( image, cow ) )
     log( '* Extracting kernel and initrd' )
-    kernel, initrd = extractKernel( image, flavor=basename, imageDir=tmpdir )
+    kernel, initrd, part = extractKernel( image, flavor=basename, imageDir=tmpdir )
     if LogToConsole:
         logfile = stdout
     else:
@@ -912,7 +954,7 @@ def bootAndRun( image, prompt=Prompt, memory=1024, cpuCores=1, outputFile=None,
                                       suffix='.testlog', delete=False )
     log( '* Logging VM output to', logfile.name )
     vm = boot( cow=cow, kernel=kernel, initrd=initrd, logfile=logfile,
-               memory=memory, cpuCores=cpuCores )
+               memory=memory, cpuCores=cpuCores, part=part )
     login( vm )
     log( '* Waiting for prompt after login' )
     vm.expect( prompt )
@@ -951,7 +993,7 @@ def testDict():
 def testString():
     "Return string listing valid tests"
     tests = [ '%s <%s>' % ( name, func.__doc__ )
-              for name, func in testDict().iteritems() ]
+              for name, func in testDict().items() ]
     return 'valid tests: %s' % ', '.join( tests )
 
 
@@ -1031,6 +1073,7 @@ def parseArgs():
                    memory=args.memory )
         except Exception as e:
             log( '* BUILD FAILED with exception: ', e )
+            print_exc( e )
             exit( 1 )
     for image in args.image:
         bootAndRun( image, runFunction=runTests, tests=args.test, pre=args.run,

util/vm/install-mininet-vm.sh

@@ -15,13 +15,7 @@ sudo sed -i -e 's/splash//' /etc/default/grub
 sudo sed -i -e 's/quiet/text/' /etc/default/grub
 sudo update-grub
 # Update from official archive
-sudo apt-get update
+sudo apt-get -qq update
-# 12.10 and earlier
-#sudo sed -i -e 's/us.archive.ubuntu.com/mirrors.kernel.org/' \
-#	/etc/apt/sources.list
-# 13.04 and later
-#sudo sed -i -e 's/\/archive.ubuntu.com/\/mirrors.kernel.org/' \
-#	/etc/apt/sources.list
 # Clean up vmware easy install junk if present
 if [ -e /etc/issue.backup ]; then
     sudo mv /etc/issue.backup /etc/issue
@@ -34,11 +28,10 @@ sudo apt-get -y install git-core openssh-server
 git clone git://github.com/mininet/mininet
 # Optionally check out branch
 if [ "$1" != "" ]; then
-  pushd mininet
+    pushd mininet
-  #git checkout -b $1 $1
+    git fetch origin $1
-  # TODO branch will in detached HEAD state if it is not master
+    git checkout $1
-  git checkout $1
+    popd
-  popd
 fi
 # Install Mininet
 time mininet/util/install.sh