From 9bda98486d7d4569554949fa6338148b99ec7c8b Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Tue, 20 Jan 2015 18:10:59 -0800 Subject: [PATCH 01/20] Add OVSBatch class (experimental) This implements batch startup for OVS switches. --- bin/mn | 3 +- mininet/net.py | 7 ++++ mininet/node.py | 107 +++++++++++++++++++++++++++++++++++------------- 3 files changed, 87 insertions(+), 30 deletions(-) diff --git a/bin/mn b/bin/mn index bdb5ea3..d3a9a80 100755 --- a/bin/mn +++ b/bin/mn @@ -27,7 +27,7 @@ from mininet.net import Mininet, MininetWithControlNet, VERSION from mininet.node import ( Host, CPULimitedHost, Controller, OVSController, RYU, NOX, RemoteController, findController, DefaultController, - UserSwitch, OVSSwitch, OVSBridge, + UserSwitch, OVSSwitch, OVSBridge, OVSBatch, OVSLegacyKernelSwitch, IVSSwitch ) from mininet.nodelib import LinuxBridge from mininet.link import Link, TCLink, OVSLink @@ -62,6 +62,7 @@ SWITCHES = { 'user': UserSwitch, # Keep ovsk for compatibility with 2.0 'ovsk': OVSSwitch, 'ovsl': OVSLegacyKernelSwitch, + 'ovsbatch': OVSBatch, # experimental!!' 'ivs': IVSSwitch, 'lxbr': LinuxBridge, 'default': OVSSwitch } diff --git a/mininet/net.py b/mininet/net.py index bb7b66d..2697046 100755 --- a/mininet/net.py +++ b/mininet/net.py @@ -481,6 +481,13 @@ class Mininet( object ): for switch in self.switches: info( switch.name + ' ') switch.start( self.controllers ) + started = {} + for swclass, switches in groupby( + sorted( self.switches, key=type ), type ): + switches = tuple( switches ) + if ( hasattr( swclass, 'batchStartup' ) and + swclass.batchStartup( switches ) ): + started.update( { s: s for s in switches } ) info( '\n' ) if self.waitConn: self.waitConnected() diff --git a/mininet/node.py b/mininet/node.py index 1e0e121..6912f71 100644 --- a/mininet/node.py +++ b/mininet/node.py @@ -1106,16 +1106,26 @@ class OVSSwitch( Switch ): @classmethod def batchShutdown( cls, switches ): - "Call ovs-vsctl del-br on all OVSSwitches in a list" + "Shut down a list of OVS switches" + # First, delete them all from ovsdb quietRun( 'ovs-vsctl ' + ' -- '.join( '--if-exists del-br %s' % s for s in switches ) ) + # Next, shut down all of the processes + pids = ' '.join( str( switch.pid ) for switch in switches ) + quietRun( 'kill -HUP ' + pids ) + for switch in switches: + switch.shell = None return True def dpctl( self, *args ): "Run ovs-ofctl command" return self.cmd( 'ovs-ofctl', args[ 0 ], self, *args[ 1: ] ) + def vsctl( self, *args, **kwargs ): + "Run ovs-vsctl command" + return self.cmd( 'ovs-vsctl', *args, **kwargs ) + @staticmethod def TCReapply( intf ): """Unfortunately OVS and Mininet are fighting @@ -1126,19 +1136,19 @@ class OVSSwitch( Switch ): def attach( self, intf ): "Connect a data port" - self.cmd( 'ovs-vsctl add-port', self, intf ) + self.vsctl( 'add-port', self, intf ) self.cmd( 'ifconfig', intf, 'up' ) self.TCReapply( intf ) def detach( self, intf ): "Disconnect a data port" - self.cmd( 'ovs-vsctl del-port', self, intf ) + self.vsctl( 'del-port', self, intf ) def controllerUUIDs( self, update=False ): """Return ovsdb UUIDs for our controllers update: update cached value""" if not self._uuids or update: - controllers = self.cmd( 'ovs-vsctl -- get Bridge', self, + controllers = self.vsctl( '-- get Bridge', self, 'Controller' ).strip() if controllers.startswith( '[' ) and controllers.endswith( ']' ): controllers = controllers[ 1 : -1 ] @@ -1150,8 +1160,8 @@ class OVSSwitch( Switch ): def connected( self ): "Are we connected to at least one of our controllers?" for uuid in self.controllerUUIDs(): - if 'true' in self.cmd( 'ovs-vsctl -- get Controller', - uuid, 'is_connected' ): + if 'true' in self.vsctl( '-- get Controller', + uuid, 'is_connected' ): return True return self.failMode == 'standalone' @@ -1163,8 +1173,8 @@ class OVSSwitch( Switch ): return '' intf1, intf2 = intf.link.intf1, intf.link.intf2 peer = intf1 if intf1 != intf else intf2 - return ( '-- set Interface %s type=patch ' - '-- set Interface %s options:peer=%s ' % + return ( ' -- set Interface %s type=patch' + ' -- set Interface %s options:peer=%s ' % ( intf, intf, peer ) ) # pylint: disable=too-many-branches @@ -1177,7 +1187,7 @@ class OVSSwitch( Switch ): # Interfaces and controllers intfs = ' '.join( '-- add-port %s %s ' % ( self, intf ) + '-- set Interface %s ' % intf + - 'ofport_request=%s ' % self.ports[ intf ] + 'ofport_request=%s' % self.ports[ intf ] + self.patchOpts( intf ) for intf in self.intfList() if self.ports[ intf ] and not intf.IP() ) @@ -1187,43 +1197,43 @@ class OVSSwitch( Switch ): clist += ' ptcp:%s' % self.listenPort # Construct big ovs-vsctl command for new versions of OVS if not self.isOldOVS(): - cmd = ( 'ovs-vsctl --if-exists del-br %s ' % self + - '-- add-br %s ' % self + - '-- set Bridge %s ' % self + - 'other_config:datapath-id=%s ' % self.dpid + - '-- set-fail-mode %s %s ' % ( self, self.failMode ) + + cmd = ( '-- --if-exists del-br %s' % self + + ' -- add-br %s' % self + + ' -- set Bridge %s' % self + + ' other_config:datapath-id=%s' % self.dpid + + ' -- set-fail-mode %s %s ' % ( self, self.failMode ) + intfs + - '-- set-controller %s %s ' % ( self, clist ) ) + ' -- set-controller %s %s' % ( self, clist ) ) # Construct ovs-vsctl commands for old versions of OVS else: # Annoyingly, --if-exists option seems not to work - self.cmd( 'ovs-vsctl del-br', self ) - self.cmd( 'ovs-vsctl add-br', self ) + self.vsctl( 'del-br', self ) + self.vsctl( 'add-br', self ) for intf in self.intfList(): if not intf.IP(): - self.cmd( 'ovs-vsctl add-port', self, intf ) - cmd = ( 'ovs-vsctl set Bridge %s ' % self + - 'other_config:datapath-id=%s ' % self.dpid + - '-- set-fail-mode %s %s ' % ( self, self.failMode ) + - '-- set-controller %s %s ' % ( self, clist ) ) + self.vsctl( 'add-port', self, intf ) + cmd = ( 'set Bridge %s' % self + + ' other_config:datapath-id=%s' % self.dpid + + ' -- set-fail-mode %s %s ' % ( self, self.failMode ) + + ' -- set-controller %s %s ' % ( self, clist ) ) if not self.inband: - cmd += ( '-- set bridge %s ' - 'other-config:disable-in-band=true ' % self ) + cmd += ( ' -- set bridge %s ' + 'other-config:disable-in-band=true' % self ) if self.datapath == 'user': - cmd += '-- set bridge %s datapath_type=netdev ' % self + cmd += ' -- set bridge %s datapath_type=netdev' % self if self.protocols and not self.isOldOVS(): - cmd += '-- set bridge %s protocols=%s ' % ( self, self.protocols ) + cmd += ' -- set bridge %s protocols=%s' % ( self, self.protocols ) if self.stp and self.failMode == 'standalone': - cmd += '-- set bridge %s stp_enable=true ' % self + cmd += ' -- set bridge %s stp_enable=true' % self # Do it!! - self.cmd( cmd ) + self.vsctl( cmd ) # Reconnect quickly to controllers (1s vs. 15s max_backoff) if self.reconnectms: uuids = [ '-- set Controller %s max_backoff=%d' % ( uuid, self.reconnectms ) for uuid in self.controllerUUIDs() ] if uuids: - self.cmd( 'ovs-vsctl', *uuids ) + self.vsctl( *uuids ) # If necessary, restore TC config overwritten by OVS for intf in self.intfList(): self.TCReapply( intf ) @@ -1260,6 +1270,45 @@ class OVSBridge( OVSSwitch ): return True +class OVSBatch( OVSSwitch ): + "Experiment: batch startup of OVS switches" + + def __init__( self, *args, **kwargs ): + kwargs.update( reconnectms=None ) + self.commands = [] + self.started = False + super( OVSBatch, self ).__init__( *args, **kwargs ) + + @classmethod + def batchStartup( cls, switches ): + "Batch startup for OVS" + if cls.isOldOVS(): + return False + info( '...' ) + cmds = '' + for switch in switches: + for cmd in switch.commands: + cmds += ' ' + cmd.strip() + # Split into 1 MB blocks + if len( cmds ) > 1000000: + print quietRun( 'ovs-vsctl' + cmds ) + cmds = '' + switch.started = True + if cmds: + quietRun( 'ovs-vsctl' + cmds ) + return True + + def vsctl( self, *args, **kwargs ): + "Append ovs-vsctl command to list for later execution" + if self.started: + return OVSSwitch.vsctl( self, *args, **kwargs ) + cmd = ' '.join( str( arg ) for arg in args ).strip() + self.commands.append( cmd ) + + def cleanup( self): + "Don't bother to clean up" + return + class IVSSwitch( Switch ): "Indigo Virtual Switch" From 959586bc8f2aecbbf5cad58bce2c77869bc6596d Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Tue, 20 Jan 2015 22:54:07 -0800 Subject: [PATCH 02/20] Add debug(cmd) to errRun() --- mininet/util.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mininet/util.py b/mininet/util.py index a655f7d..a84148e 100644 --- a/mininet/util.py +++ b/mininet/util.py @@ -77,6 +77,7 @@ def errRun( *cmd, **kwargs ): cmd = [ str( arg ) for arg in cmd ] elif isinstance( cmd, list ) and shell: cmd = " ".join( arg for arg in cmd ) + debug( '*** errRun:', cmd, '\n' ) popen = Popen( cmd, stdout=PIPE, stderr=stderr, shell=shell ) # We use poll() because select() doesn't work with large fd numbers, # and thus communicate() doesn't work either From 30ebb852a395d13d9bbc682283c34d5444592be4 Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Thu, 22 Jan 2015 06:15:57 -0800 Subject: [PATCH 03/20] errRun: add debug( results ) --- mininet/util.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mininet/util.py b/mininet/util.py index a84148e..4b0e209 100644 --- a/mininet/util.py +++ b/mininet/util.py @@ -114,6 +114,7 @@ def errRun( *cmd, **kwargs ): poller.unregister( fd ) returncode = popen.wait() + debug( out, err, returncode ) return out, err, returncode def errFail( *cmd, **kwargs ): From 3b4738c2ca9c10e80df7265cc90052ed06c96143 Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Thu, 22 Jan 2015 06:16:41 -0800 Subject: [PATCH 04/20] First crack at setting controller backoff in single command --- mininet/node.py | 44 ++++++++++++++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/mininet/node.py b/mininet/node.py index 6912f71..8f59baf 100644 --- a/mininet/node.py +++ b/mininet/node.py @@ -1174,7 +1174,7 @@ class OVSSwitch( Switch ): intf1, intf2 = intf.link.intf1, intf.link.intf2 peer = intf1 if intf1 != intf else intf2 return ( ' -- set Interface %s type=patch' - ' -- set Interface %s options:peer=%s ' % + ' -- set Interface %s options:peer=%s' % ( intf, intf, peer ) ) # pylint: disable=too-many-branches @@ -1191,19 +1191,27 @@ class OVSSwitch( Switch ): + self.patchOpts( intf ) for intf in self.intfList() if self.ports[ intf ] and not intf.IP() ) - clist = ' '.join( '%s:%s:%d' % ( c.protocol, c.IP(), c.port ) - for c in controllers ) - if self.listenPort: - clist += ' ptcp:%s' % self.listenPort # Construct big ovs-vsctl command for new versions of OVS + clist = [ ( self.name + c.name, '%s:%s:%d' % + ( c.protocol, c.IP(), c.port ) ) + for c in controllers ] + if self.listenPort: + clist.append( ( self.name + '-listen', + 'ptcp:%s' % self.listenPort ) ) if not self.isOldOVS(): + ccmd = ' -- --id=@%s create Controller target=\\"%s\\"' + if self.reconnectms: + ccmd += ' max_backoff=%d' % self.reconnectms + cargs = ''.join( ccmd % ( name, target ) + for name, target in clist ) + clist = ','.join( '@%s' % name for name, _target in clist ) cmd = ( '-- --if-exists del-br %s' % self + ' -- add-br %s' % self + - ' -- set Bridge %s' % self + + ' -- set bridge %s' % self + ' other_config:datapath-id=%s' % self.dpid + - ' -- set-fail-mode %s %s ' % ( self, self.failMode ) + - intfs + - ' -- set-controller %s %s' % ( self, clist ) ) + ' fail_mode=%s' % self.failMode + + ' controller=[%s] ' % clist + + intfs + cargs ) # Construct ovs-vsctl commands for old versions of OVS else: # Annoyingly, --if-exists option seems not to work @@ -1215,7 +1223,7 @@ class OVSSwitch( Switch ): cmd = ( 'set Bridge %s' % self + ' other_config:datapath-id=%s' % self.dpid + ' -- set-fail-mode %s %s ' % ( self, self.failMode ) + - ' -- set-controller %s %s ' % ( self, clist ) ) + ' -- set-controller %s %s ' % ( self, ' '.join( clist ) ) ) if not self.inband: cmd += ( ' -- set bridge %s ' 'other-config:disable-in-band=true' % self ) @@ -1228,7 +1236,7 @@ class OVSSwitch( Switch ): # Do it!! self.vsctl( cmd ) # Reconnect quickly to controllers (1s vs. 15s max_backoff) - if self.reconnectms: + if self.isOldOVS() and self.reconnectms: uuids = [ '-- set Controller %s max_backoff=%d' % ( uuid, self.reconnectms ) for uuid in self.controllerUUIDs() ] @@ -1273,12 +1281,19 @@ class OVSBridge( OVSSwitch ): class OVSBatch( OVSSwitch ): "Experiment: batch startup of OVS switches" + reconnectms = 1000 # shared for all switches + def __init__( self, *args, **kwargs ): - kwargs.update( reconnectms=None ) self.commands = [] self.started = False + # Use global rather than local reconnectms + reconnectms = kwargs.pop( 'reconnectms', 1000 ) + self.__class__.reconnectms = max( reconnectms, + self.__class__.reconnectms ) + kwargs.update( reconnectms=None ) super( OVSBatch, self ).__init__( *args, **kwargs ) + @classmethod def batchStartup( cls, switches ): "Batch startup for OVS" @@ -1291,11 +1306,11 @@ class OVSBatch( OVSSwitch ): cmds += ' ' + cmd.strip() # Split into 1 MB blocks if len( cmds ) > 1000000: - print quietRun( 'ovs-vsctl' + cmds ) + errRun( 'ovs-vsctl' + cmds, shell=True ) cmds = '' switch.started = True if cmds: - quietRun( 'ovs-vsctl' + cmds ) + quietRun( 'ovs-vsctl' + cmds, shell=True ) return True def vsctl( self, *args, **kwargs ): @@ -1428,6 +1443,7 @@ class Controller( Node ): "Stop controller." self.cmd( 'kill %' + self.command ) self.cmd( 'wait %' + self.command ) + kwargs.update( deleteIntfs=False ) super( Controller, self ).stop( *args, **kwargs ) def IP( self, intf=None ): From 957fe1db93f2af9edaf5d7cb55a69f1b6d9ea2b9 Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Thu, 22 Jan 2015 20:15:26 -0800 Subject: [PATCH 05/20] Refactor for compatibility with isOldOVS() == True --- mininet/node.py | 127 +++++++++++++++++++++++------------------------- 1 file changed, 62 insertions(+), 65 deletions(-) diff --git a/mininet/node.py b/mininet/node.py index 8f59baf..8e3b2e0 100644 --- a/mininet/node.py +++ b/mininet/node.py @@ -1107,10 +1107,12 @@ class OVSSwitch( Switch ): @classmethod def batchShutdown( cls, switches ): "Shut down a list of OVS switches" + delcmd = 'del-br %s' + if not cls.isOldOVS(): + delcmd = '--if-exists ' + delcmd # First, delete them all from ovsdb quietRun( 'ovs-vsctl ' + - ' -- '.join( '--if-exists del-br %s' % s - for s in switches ) ) + ' -- '.join( delcmd % s for s in switches ) ) # Next, shut down all of the processes pids = ' '.join( str( switch.pid ) for switch in switches ) quietRun( 'kill -HUP ' + pids ) @@ -1165,17 +1167,31 @@ class OVSSwitch( Switch ): return True return self.failMode == 'standalone' - @staticmethod - def patchOpts( intf ): - "Return OVS patch port options (if any) for intf" - if not isinstance( intf, OVSIntf ): - # Ignore if it's not a patch link - return '' - intf1, intf2 = intf.link.intf1, intf.link.intf2 - peer = intf1 if intf1 != intf else intf2 - return ( ' -- set Interface %s type=patch' - ' -- set Interface %s options:peer=%s' % - ( intf, intf, peer ) ) + def intfOpts( self, intf ): + "Return OVS interface options for intf" + opts = '' + if not self.isOldOVS(): + # ofport_request is not supported + opts += ' ofport_request=%s' % self.ports[ intf ] + # Patch ports don't work well with old OVS + if isinstance( intf, OVSIntf ): + intf1, intf2 = intf.link.intf1, intf.link.intf2 + peer = intf1 if intf1 != intf else intf2 + opts += ' type=patch options:peer=%s' % peer + return '' if not opts else ' -- set Interface %s' % intf + opts + + def bridgeOpts( self ): + "Return OVS bridge options" + opts = '' + if not self.inband: + opts += ' other-config:disable-in-band=true' + if self.datapath == 'user': + opts += ' datapath_type=netdev' % self + if self.protocols and not self.isOldOVS(): + opts += ' protocols=%s' % ( self, self.protocols ) + if self.stp and self.failMode == 'standalone': + opts += ' stp_enable=true' % self + return opts # pylint: disable=too-many-branches def start( self, controllers ): @@ -1185,54 +1201,30 @@ class OVSSwitch( Switch ): 'OVS kernel switch does not work in a namespace' ) int( self.dpid, 16 ) # DPID must be a hex string # Interfaces and controllers - intfs = ' '.join( '-- add-port %s %s ' % ( self, intf ) + - '-- set Interface %s ' % intf + - 'ofport_request=%s' % self.ports[ intf ] - + self.patchOpts( intf ) - for intf in self.intfList() - if self.ports[ intf ] and not intf.IP() ) - # Construct big ovs-vsctl command for new versions of OVS + intfs = ''.join( ' -- add-port %s %s' % ( self, intf ) + + self.intfOpts( intf ) + for intf in self.intfList() + if self.ports[ intf ] and not intf.IP() ) + # Construct big ovs-vsctl command clist = [ ( self.name + c.name, '%s:%s:%d' % ( c.protocol, c.IP(), c.port ) ) for c in controllers ] if self.listenPort: clist.append( ( self.name + '-listen', 'ptcp:%s' % self.listenPort ) ) + ccmd = '-- --id=@%s create Controller target=\\"%s\\"' + if self.reconnectms: + ccmd += ' max_backoff=%d' % self.reconnectms + cargs = ' '.join( ccmd % ( name, target ) + for name, target in clist ) + cids = ','.join( '@%s' % name for name, _target in clist ) if not self.isOldOVS(): - ccmd = ' -- --id=@%s create Controller target=\\"%s\\"' - if self.reconnectms: - ccmd += ' max_backoff=%d' % self.reconnectms - cargs = ''.join( ccmd % ( name, target ) - for name, target in clist ) - clist = ','.join( '@%s' % name for name, _target in clist ) - cmd = ( '-- --if-exists del-br %s' % self + - ' -- add-br %s' % self + - ' -- set bridge %s' % self + - ' other_config:datapath-id=%s' % self.dpid + - ' fail_mode=%s' % self.failMode + - ' controller=[%s] ' % clist + - intfs + cargs ) - # Construct ovs-vsctl commands for old versions of OVS - else: - # Annoyingly, --if-exists option seems not to work - self.vsctl( 'del-br', self ) - self.vsctl( 'add-br', self ) - for intf in self.intfList(): - if not intf.IP(): - self.vsctl( 'add-port', self, intf ) - cmd = ( 'set Bridge %s' % self + - ' other_config:datapath-id=%s' % self.dpid + - ' -- set-fail-mode %s %s ' % ( self, self.failMode ) + - ' -- set-controller %s %s ' % ( self, ' '.join( clist ) ) ) - if not self.inband: - cmd += ( ' -- set bridge %s ' - 'other-config:disable-in-band=true' % self ) - if self.datapath == 'user': - cmd += ' -- set bridge %s datapath_type=netdev' % self - if self.protocols and not self.isOldOVS(): - cmd += ' -- set bridge %s protocols=%s' % ( self, self.protocols ) - if self.stp and self.failMode == 'standalone': - cmd += ' -- set bridge %s stp_enable=true' % self + cargs += ' -- --if-exists del-br %s' % self + cmd = ( cargs + + ' -- add-br %s' % self + + ' -- set bridge %s controller=[%s]' % ( self, cids ) + + self.bridgeOpts() + + intfs ) # Do it!! self.vsctl( cmd ) # Reconnect quickly to controllers (1s vs. 15s max_backoff) @@ -1283,6 +1275,10 @@ class OVSBatch( OVSSwitch ): reconnectms = 1000 # shared for all switches + # This should be ~ int( quietRun( 'getconf ARG_MAX' ) ), + # but the real limit seems to be much lower + argmax = 128000 + def __init__( self, *args, **kwargs ): self.commands = [] self.started = False @@ -1293,37 +1289,38 @@ class OVSBatch( OVSSwitch ): kwargs.update( reconnectms=None ) super( OVSBatch, self ).__init__( *args, **kwargs ) - @classmethod def batchStartup( cls, switches ): "Batch startup for OVS" - if cls.isOldOVS(): - return False info( '...' ) - cmds = '' + cmds = 'ovs-vsctl ' for switch in switches: + if cls.isOldOVS(): + quietRun( 'ovs-vsctl del-br %s' % switch ) for cmd in switch.commands: - cmds += ' ' + cmd.strip() - # Split into 1 MB blocks - if len( cmds ) > 1000000: - errRun( 'ovs-vsctl' + cmds, shell=True ) - cmds = '' + cmd = cmd.strip() + # Don't exceed ARG_MAX + if len( cmds ) + len( cmd ) >= cls.argmax: + errRun( cmds, shell=True ) + cmds = 'ovs-vsctl' + cmds += ' ' + cmd switch.started = True if cmds: - quietRun( 'ovs-vsctl' + cmds, shell=True ) + errRun( cmds, shell=True ) return True def vsctl( self, *args, **kwargs ): "Append ovs-vsctl command to list for later execution" if self.started: return OVSSwitch.vsctl( self, *args, **kwargs ) - cmd = ' '.join( str( arg ) for arg in args ).strip() + cmd = ' '.join( str( arg ).strip() for arg in args ) self.commands.append( cmd ) def cleanup( self): "Don't bother to clean up" return + class IVSSwitch( Switch ): "Indigo Virtual Switch" From 9ca6322603f5682c4bf2e77505f8bd09348814c7 Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Thu, 22 Jan 2015 09:18:24 -0800 Subject: [PATCH 06/20] Remove shared reconnectms, improve self.started We still need to set it in batchShutdown() --- mininet/node.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/mininet/node.py b/mininet/node.py index 8e3b2e0..c65a9c0 100644 --- a/mininet/node.py +++ b/mininet/node.py @@ -1273,8 +1273,6 @@ class OVSBridge( OVSSwitch ): class OVSBatch( OVSSwitch ): "Experiment: batch startup of OVS switches" - reconnectms = 1000 # shared for all switches - # This should be ~ int( quietRun( 'getconf ARG_MAX' ) ), # but the real limit seems to be much lower argmax = 128000 @@ -1282,11 +1280,6 @@ class OVSBatch( OVSSwitch ): def __init__( self, *args, **kwargs ): self.commands = [] self.started = False - # Use global rather than local reconnectms - reconnectms = kwargs.pop( 'reconnectms', 1000 ) - self.__class__.reconnectms = max( reconnectms, - self.__class__.reconnectms ) - kwargs.update( reconnectms=None ) super( OVSBatch, self ).__init__( *args, **kwargs ) @classmethod @@ -1316,6 +1309,14 @@ class OVSBatch( OVSSwitch ): cmd = ' '.join( str( arg ).strip() for arg in args ) self.commands.append( cmd ) + def start( self, *args, **kwargs ): + super( OVSBatch, self ).start( *args, **kwargs ) + self.started = True + + def stop( self, *args, **kwargs ): + super( OVSBatch, self ).stop( *args, **kwargs ) + self.started = False + def cleanup( self): "Don't bother to clean up" return From bec34e722756dc111adf3ff0370bd77b3cca281d Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Thu, 22 Jan 2015 21:27:02 -0800 Subject: [PATCH 07/20] Clean up - TCReapply still broken! --- mininet/node.py | 42 ++++++++++++++++++------------------------ 1 file changed, 18 insertions(+), 24 deletions(-) diff --git a/mininet/node.py b/mininet/node.py index c65a9c0..e08739c 100644 --- a/mininet/node.py +++ b/mininet/node.py @@ -1150,7 +1150,7 @@ class OVSSwitch( Switch ): """Return ovsdb UUIDs for our controllers update: update cached value""" if not self._uuids or update: - controllers = self.vsctl( '-- get Bridge', self, + controllers = self.cmd( 'ovs-vsctl -- get Bridge', self, 'Controller' ).strip() if controllers.startswith( '[' ) and controllers.endswith( ']' ): controllers = controllers[ 1 : -1 ] @@ -1171,7 +1171,7 @@ class OVSSwitch( Switch ): "Return OVS interface options for intf" opts = '' if not self.isOldOVS(): - # ofport_request is not supported + # ofport_request is not supported on old OVS opts += ' ofport_request=%s' % self.ports[ intf ] # Patch ports don't work well with old OVS if isinstance( intf, OVSIntf ): @@ -1182,7 +1182,8 @@ class OVSSwitch( Switch ): def bridgeOpts( self ): "Return OVS bridge options" - opts = '' + opts = ( ' other_config:datapath-id=%s' % self.dpid + + ' fail_mode=%s' % self.failMode ) if not self.inband: opts += ' other-config:disable-in-band=true' if self.datapath == 'user': @@ -1193,19 +1194,18 @@ class OVSSwitch( Switch ): opts += ' stp_enable=true' % self return opts - # pylint: disable=too-many-branches def start( self, controllers ): "Start up a new OVS OpenFlow switch using ovs-vsctl" if self.inNamespace: raise Exception( 'OVS kernel switch does not work in a namespace' ) int( self.dpid, 16 ) # DPID must be a hex string - # Interfaces and controllers + # Command to add interfaces intfs = ''.join( ' -- add-port %s %s' % ( self, intf ) + self.intfOpts( intf ) for intf in self.intfList() if self.ports[ intf ] and not intf.IP() ) - # Construct big ovs-vsctl command + # Command to create controller entries clist = [ ( self.name + c.name, '%s:%s:%d' % ( c.protocol, c.IP(), c.port ) ) for c in controllers ] @@ -1217,27 +1217,21 @@ class OVSSwitch( Switch ): ccmd += ' max_backoff=%d' % self.reconnectms cargs = ' '.join( ccmd % ( name, target ) for name, target in clist ) + # Controller ID list cids = ','.join( '@%s' % name for name, _target in clist ) + # Try to delete any existing bridges with the same name if not self.isOldOVS(): cargs += ' -- --if-exists del-br %s' % self - cmd = ( cargs + - ' -- add-br %s' % self + - ' -- set bridge %s controller=[%s]' % ( self, cids ) + - self.bridgeOpts() + - intfs ) - # Do it!! - self.vsctl( cmd ) - # Reconnect quickly to controllers (1s vs. 15s max_backoff) - if self.isOldOVS() and self.reconnectms: - uuids = [ '-- set Controller %s max_backoff=%d' % - ( uuid, self.reconnectms ) - for uuid in self.controllerUUIDs() ] - if uuids: - self.vsctl( *uuids ) + # One ovs-vsctl command to rule them all! + self.vsctl( cargs + + ' -- add-br %s' % self + + ' -- set bridge %s controller=[%s]' % ( self, cids ) + + self.bridgeOpts() + + intfs ) + # XXX BROKEN - need to fix this!! # If necessary, restore TC config overwritten by OVS - for intf in self.intfList(): - self.TCReapply( intf ) - # pylint: enable=too-many-branches + # for intf in self.intfList(): + # self.TCReapply( intf ) def stop( self, deleteIntfs=True ): """Terminate OVS switch. @@ -1305,7 +1299,7 @@ class OVSBatch( OVSSwitch ): def vsctl( self, *args, **kwargs ): "Append ovs-vsctl command to list for later execution" if self.started: - return OVSSwitch.vsctl( self, *args, **kwargs ) + return super( OVSBridge, self).vsctl( *args, **kwargs ) cmd = ' '.join( str( arg ).strip() for arg in args ) self.commands.append( cmd ) From 8014a7023c3385ef6d4f89cde27b358fb390f349 Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Thu, 22 Jan 2015 10:01:17 -0800 Subject: [PATCH 08/20] Fix super() typo --- mininet/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mininet/node.py b/mininet/node.py index e08739c..dfc0b44 100644 --- a/mininet/node.py +++ b/mininet/node.py @@ -1299,7 +1299,7 @@ class OVSBatch( OVSSwitch ): def vsctl( self, *args, **kwargs ): "Append ovs-vsctl command to list for later execution" if self.started: - return super( OVSBridge, self).vsctl( *args, **kwargs ) + return super( OVSBatch, self).vsctl( *args, **kwargs ) cmd = ' '.join( str( arg ).strip() for arg in args ) self.commands.append( cmd ) From 7485b035af5b255471ed4437c78f6600d2f2913c Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Thu, 22 Jan 2015 10:04:09 -0800 Subject: [PATCH 09/20] make 'ovs-vsctl' string symmetric --- mininet/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mininet/node.py b/mininet/node.py index dfc0b44..e416261 100644 --- a/mininet/node.py +++ b/mininet/node.py @@ -1280,7 +1280,7 @@ class OVSBatch( OVSSwitch ): def batchStartup( cls, switches ): "Batch startup for OVS" info( '...' ) - cmds = 'ovs-vsctl ' + cmds = 'ovs-vsctl' for switch in switches: if cls.isOldOVS(): quietRun( 'ovs-vsctl del-br %s' % switch ) From eafbd2a597298d4844cbe3c45855cc0ed9ad6c4b Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Thu, 22 Jan 2015 12:05:03 -0800 Subject: [PATCH 10/20] Change to OVSSwitch --- examples/tree1024.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/tree1024.py b/examples/tree1024.py index 9397131..d5c25c2 100755 --- a/examples/tree1024.py +++ b/examples/tree1024.py @@ -9,10 +9,10 @@ and running sysctl -p. Check util/sysctl_addon. from mininet.cli import CLI from mininet.log import setLogLevel -from mininet.node import OVSKernelSwitch +from mininet.node import OVSSwitch from mininet.topolib import TreeNet if __name__ == '__main__': setLogLevel( 'info' ) - network = TreeNet( depth=2, fanout=32, switch=OVSKernelSwitch ) + network = TreeNet( depth=2, fanout=32, switch=OVSSwitch ) network.run( CLI, network ) From 574d634fc293a10dc4bd0dcce5588e14da756751 Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Thu, 22 Jan 2015 12:08:43 -0800 Subject: [PATCH 11/20] Don't clean up links that may have been dumped into root NS. This should rarely happen - in the usual case, either the links will be shut down by Mininet.stop(), or the interfaces will be deleted by node.stop( deleteIntfs=True ), or the links or interfaces will be explicitly deleted or stopped using the low-level API. Cases that are relying on links being automatically deleted in cleanup() will potentially find that they are now no longer deleted, but these cases should be rare. --- mininet/link.py | 7 ++++--- mininet/node.py | 11 ++++------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/mininet/link.py b/mininet/link.py index af895a1..4a1a683 100644 --- a/mininet/link.py +++ b/mininet/link.py @@ -197,9 +197,10 @@ class Intf( object ): def delete( self ): "Delete interface" self.cmd( 'ip link del ' + self.name ) - if self.node.inNamespace: - # Link may have been dumped into root NS - quietRun( 'ip link del ' + self.name ) + # We used to do this, but it slows us down: + # if self.node.inNamespace: + # Link may have been dumped into root NS + # quietRun( 'ip link del ' + self.name ) def status( self ): "Return intf status as a string" diff --git a/mininet/node.py b/mininet/node.py index e416261..ec473e0 100644 --- a/mininet/node.py +++ b/mininet/node.py @@ -194,10 +194,11 @@ class Node( object ): def cleanup( self ): "Help python collect its garbage." + # We used to do this, but it slows us down: # Intfs may end up in root NS - for intfName in self.intfNames(): - if self.name in intfName: - quietRun( 'ip link del ' + intfName ) + # for intfName in self.intfNames(): + # if self.name in intfName: + # quietRun( 'ip link del ' + intfName ) self.shell = None # Subshell I/O, commands and control @@ -1311,10 +1312,6 @@ class OVSBatch( OVSSwitch ): super( OVSBatch, self ).stop( *args, **kwargs ) self.started = False - def cleanup( self): - "Don't bother to clean up" - return - class IVSSwitch( Switch ): "Indigo Virtual Switch" From bdad3e8c8e34d28dda5fb4534cb85e15a688f722 Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Mon, 26 Jan 2015 18:01:20 -0800 Subject: [PATCH 12/20] Merge OVSBatch into OVSSwitch Note that we are changing the interface of batchStartup/Shutdown slightly so that the method can choose not to start some of the switches. We might wish to refine this a bit... --- bin/mn | 3 +- examples/cluster.py | 12 +++- mininet/net.py | 14 +++-- mininet/node.py | 139 +++++++++++++++++++++----------------------- 4 files changed, 86 insertions(+), 82 deletions(-) diff --git a/bin/mn b/bin/mn index d3a9a80..bdb5ea3 100755 --- a/bin/mn +++ b/bin/mn @@ -27,7 +27,7 @@ from mininet.net import Mininet, MininetWithControlNet, VERSION from mininet.node import ( Host, CPULimitedHost, Controller, OVSController, RYU, NOX, RemoteController, findController, DefaultController, - UserSwitch, OVSSwitch, OVSBridge, OVSBatch, + UserSwitch, OVSSwitch, OVSBridge, OVSLegacyKernelSwitch, IVSSwitch ) from mininet.nodelib import LinuxBridge from mininet.link import Link, TCLink, OVSLink @@ -62,7 +62,6 @@ SWITCHES = { 'user': UserSwitch, # Keep ovsk for compatibility with 2.0 'ovsk': OVSSwitch, 'ovsl': OVSLegacyKernelSwitch, - 'ovsbatch': OVSBatch, # experimental!!' 'ivs': IVSSwitch, 'lxbr': LinuxBridge, 'default': OVSSwitch } diff --git a/examples/cluster.py b/examples/cluster.py index 7f3c09b..387c8b0 100755 --- a/examples/cluster.py +++ b/examples/cluster.py @@ -280,6 +280,11 @@ class RemoteOVSSwitch( RemoteMixin, OVSSwitch ): OVSVersions = {} + def __init__( self, *args, **kwargs ): + # No batch startup yet + kwargs.update( batch=False ) + super( RemoteOVSSwitch, self ).__init__( *args, **kwargs ) + def isOldOVS( self ): "Is remote switch using an old OVS version?" cls = type( self ) @@ -292,10 +297,15 @@ class RemoteOVSSwitch( RemoteMixin, OVSSwitch ): return ( StrictVersion( cls.OVSVersions[ self.server ] ) < StrictVersion( '1.10' ) ) + @classmethod + def batchStartup( cls, *_args, **_kwargs ): + "Not implemented yet" + return [] # no switches started + @classmethod def batchShutdown( cls, *_args, **_kwargs ): "Not implemented yet" - return False + return [] # no switchest stopped class RemoteLink( Link ): diff --git a/mininet/net.py b/mininet/net.py index 2697046..85fdafc 100755 --- a/mininet/net.py +++ b/mininet/net.py @@ -485,9 +485,11 @@ class Mininet( object ): for swclass, switches in groupby( sorted( self.switches, key=type ), type ): switches = tuple( switches ) - if ( hasattr( swclass, 'batchStartup' ) and - swclass.batchStartup( switches ) ): - started.update( { s: s for s in switches } ) + if hasattr( swclass, 'batchStartup' ): + print "STARTING", switches + success = swclass.batchStartup( switches ) + print "STARTED", success + started.update( { s: s for s in success } ) info( '\n' ) if self.waitConn: self.waitConnected() @@ -512,9 +514,9 @@ class Mininet( object ): for swclass, switches in groupby( sorted( self.switches, key=type ), type ): switches = tuple( switches ) - if ( hasattr( swclass, 'batchShutdown' ) and - swclass.batchShutdown( switches ) ): - stopped.update( { s: s for s in switches } ) + if hasattr( swclass, 'batchShutdown' ): + success = swclass.batchShutdown( switches ) + stopped.update( { s: s for s in success } ) for switch in self.switches: info( switch.name + ' ' ) if switch not in stopped: diff --git a/mininet/node.py b/mininet/node.py index ec473e0..0ccddff 100644 --- a/mininet/node.py +++ b/mininet/node.py @@ -1059,7 +1059,7 @@ class OVSSwitch( Switch ): def __init__( self, name, failMode='secure', datapath='kernel', inband=False, protocols=None, - reconnectms=1000, stp=False, **params ): + reconnectms=1000, stp=False, batch=True, **params ): """name: name for switch failMode: controller loss behavior (secure|open) datapath: userspace or kernel mode (kernel|user) @@ -1067,7 +1067,8 @@ class OVSSwitch( Switch ): protocols: use specific OpenFlow version(s) (e.g. OpenFlow13) Unspecified (or old OVS version) uses OVS default reconnectms: max reconnect timeout in ms (0/None for default) - stp: enable STP (False, requires failMode=standalone)""" + stp: enable STP (False, requires failMode=standalone) + batch: enable batch startup (True)""" Switch.__init__( self, name, **params ) self.failMode = failMode self.datapath = datapath @@ -1076,6 +1077,8 @@ class OVSSwitch( Switch ): self.reconnectms = reconnectms self.stp = stp self._uuids = [] # controller UUIDs + self.batch = batch + self.commands = [] # saved commands for batch startup @classmethod def setup( cls ): @@ -1105,29 +1108,17 @@ class OVSSwitch( Switch ): return ( StrictVersion( cls.OVSVersion ) < StrictVersion( '1.10' ) ) - @classmethod - def batchShutdown( cls, switches ): - "Shut down a list of OVS switches" - delcmd = 'del-br %s' - if not cls.isOldOVS(): - delcmd = '--if-exists ' + delcmd - # First, delete them all from ovsdb - quietRun( 'ovs-vsctl ' + - ' -- '.join( delcmd % s for s in switches ) ) - # Next, shut down all of the processes - pids = ' '.join( str( switch.pid ) for switch in switches ) - quietRun( 'kill -HUP ' + pids ) - for switch in switches: - switch.shell = None - return True - def dpctl( self, *args ): "Run ovs-ofctl command" return self.cmd( 'ovs-ofctl', args[ 0 ], self, *args[ 1: ] ) def vsctl( self, *args, **kwargs ): - "Run ovs-vsctl command" - return self.cmd( 'ovs-vsctl', *args, **kwargs ) + "Run ovs-vsctl command (or queue for later execution)" + if self.batch: + cmd = ' '.join( str( arg ).strip() for arg in args ) + self.commands.append( cmd ) + else: + return self.cmd( 'ovs-vsctl', *args, **kwargs ) @staticmethod def TCReapply( intf ): @@ -1217,7 +1208,7 @@ class OVSSwitch( Switch ): if self.reconnectms: ccmd += ' max_backoff=%d' % self.reconnectms cargs = ' '.join( ccmd % ( name, target ) - for name, target in clist ) + for name, target in clist ) # Controller ID list cids = ','.join( '@%s' % name for name, _target in clist ) # Try to delete any existing bridges with the same name @@ -1229,10 +1220,43 @@ class OVSSwitch( Switch ): ' -- set bridge %s controller=[%s]' % ( self, cids ) + self.bridgeOpts() + intfs ) - # XXX BROKEN - need to fix this!! # If necessary, restore TC config overwritten by OVS - # for intf in self.intfList(): - # self.TCReapply( intf ) + if not self.batch: + for intf in self.intfList(): + self.TCReapply( intf ) + + # This should be ~ int( quietRun( 'getconf ARG_MAX' ) ), + # but the real limit seems to be much lower + argmax = 128000 + + @classmethod + def batchStartup( cls, switches, run=errRun ): + """Batch startup for OVS + switches: switches to start up + run: function to run commands (errRun)""" + info( '...' ) + cmds = 'ovs-vsctl' + for switch in switches: + if switch.isOldOVS(): + # Ideally we'd optimize this also + run( 'ovs-vsctl del-br %s' % switch ) + for cmd in switch.commands: + cmd = cmd.strip() + # Don't exceed ARG_MAX + if len( cmds ) + len( cmd ) >= cls.argmax: + run( cmds, shell=True ) + cmds = 'ovs-vsctl' + cmds += ' ' + cmd + switch.cmds = [] + switch.batch = False + if cmds: + run( cmds, shell=True ) + # Reapply link config if necessary... + for switch in switches: + for intf in switch.intfs.itervalues(): + if isinstance( intf, TCIntf ): + intf.config( **intf.params ) + return switches def stop( self, deleteIntfs=True ): """Terminate OVS switch. @@ -1242,6 +1266,22 @@ class OVSSwitch( Switch ): self.cmd( 'ip link del', self ) super( OVSSwitch, self ).stop( deleteIntfs ) + @classmethod + def batchShutdown( cls, switches, run=errRun ): + "Shut down a list of OVS switches" + delcmd = 'del-br %s' + if not cls.isOldOVS(): + delcmd = '--if-exists ' + delcmd + # First, delete them all from ovsdb + run( 'ovs-vsctl ' + + ' -- '.join( delcmd % s for s in switches ) ) + # Next, shut down all of the processes + pids = ' '.join( str( switch.pid ) for switch in switches ) + run( 'kill -HUP ' + pids ) + for switch in switches: + switch.shell = None + return switches + OVSKernelSwitch = OVSSwitch @@ -1265,54 +1305,6 @@ class OVSBridge( OVSSwitch ): return True -class OVSBatch( OVSSwitch ): - "Experiment: batch startup of OVS switches" - - # This should be ~ int( quietRun( 'getconf ARG_MAX' ) ), - # but the real limit seems to be much lower - argmax = 128000 - - def __init__( self, *args, **kwargs ): - self.commands = [] - self.started = False - super( OVSBatch, self ).__init__( *args, **kwargs ) - - @classmethod - def batchStartup( cls, switches ): - "Batch startup for OVS" - info( '...' ) - cmds = 'ovs-vsctl' - for switch in switches: - if cls.isOldOVS(): - quietRun( 'ovs-vsctl del-br %s' % switch ) - for cmd in switch.commands: - cmd = cmd.strip() - # Don't exceed ARG_MAX - if len( cmds ) + len( cmd ) >= cls.argmax: - errRun( cmds, shell=True ) - cmds = 'ovs-vsctl' - cmds += ' ' + cmd - switch.started = True - if cmds: - errRun( cmds, shell=True ) - return True - - def vsctl( self, *args, **kwargs ): - "Append ovs-vsctl command to list for later execution" - if self.started: - return super( OVSBatch, self).vsctl( *args, **kwargs ) - cmd = ' '.join( str( arg ).strip() for arg in args ) - self.commands.append( cmd ) - - def start( self, *args, **kwargs ): - super( OVSBatch, self ).start( *args, **kwargs ) - self.started = True - - def stop( self, *args, **kwargs ): - super( OVSBatch, self ).stop( *args, **kwargs ) - self.started = False - - class IVSSwitch( Switch ): "Indigo Virtual Switch" @@ -1338,6 +1330,7 @@ class IVSSwitch( Switch ): "Kill each IVS switch, to be waited on later in stop()" for switch in switches: switch.cmd( 'kill %ivs' ) + return switches def start( self, controllers ): "Start up a new IVS switch" @@ -1556,4 +1549,4 @@ def DefaultController( name, controllers=DefaultControllers, **kwargs ): controller = findController( controllers ) if not controller: raise Exception( 'Could not find a default OpenFlow controller' ) - return controller( name, **kwargs ) + return contr \ No newline at end of file From 254fae2dc90875a7261970a9930e5cc5bda93c09 Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Tue, 27 Jan 2015 15:23:48 -0800 Subject: [PATCH 13/20] Clarify which intf pair failed and raise exception --- mininet/util.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mininet/util.py b/mininet/util.py index 4b0e209..1780632 100644 --- a/mininet/util.py +++ b/mininet/util.py @@ -190,7 +190,8 @@ def makeIntfPair( intf1, intf2, addr1=None, addr2=None, node1=None, node2=None, if cmdOutput == '': return True else: - error( "Error creating interface pair: %s " % cmdOutput ) + raise Exception( "Error creating interface pair (%s,%s): %s " % + ( intf1, intf2, cmdOutput ) ) return False def retry( retries, delaySecs, fn, *args, **keywords ): From c702840a0a24c6a67ee9e67a2f9a8725b0e714ad Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Tue, 27 Jan 2015 15:24:29 -0800 Subject: [PATCH 14/20] Remove debug print lines --- mininet/net.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/mininet/net.py b/mininet/net.py index 85fdafc..b4263bd 100755 --- a/mininet/net.py +++ b/mininet/net.py @@ -486,9 +486,7 @@ class Mininet( object ): sorted( self.switches, key=type ), type ): switches = tuple( switches ) if hasattr( swclass, 'batchStartup' ): - print "STARTING", switches success = swclass.batchStartup( switches ) - print "STARTED", success started.update( { s: s for s in success } ) info( '\n' ) if self.waitConn: From acdcf9b6ae2ae46badcf04206acc6171b3f994af Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Tue, 27 Jan 2015 15:27:26 -0800 Subject: [PATCH 15/20] cluster: add batchStartup/Shutdown, cleanup --- bin/mn | 10 +++- examples/cluster.py | 97 +++++++++++++++++++++++++------------ mininet/clean.py | 114 ++++++++++++++++++++++++++------------------ mininet/node.py | 7 ++- 4 files changed, 145 insertions(+), 83 deletions(-) diff --git a/bin/mn b/bin/mn index bdb5ea3..d69b6b1 100755 --- a/bin/mn +++ b/bin/mn @@ -41,7 +41,8 @@ from functools import partial # Experimental! cluster edition prototype from mininet.examples.cluster import ( MininetCluster, RemoteHost, RemoteOVSSwitch, RemoteLink, - SwitchBinPlacer, RandomPlacer ) + SwitchBinPlacer, RandomPlacer, + ClusterCleanup ) from mininet.examples.clustercli import ClusterCLI PLACEMENT = { 'block': SwitchBinPlacer, 'random': RandomPlacer } @@ -281,6 +282,11 @@ class MininetRunner( object ): def begin( self ): "Create and run mininet." + if self.options.cluster: + servers = self.options.cluster.split( ',' ) + for server in servers: + ClusterCleanup.add( server ) + if self.options.clean: cleanup() exit() @@ -334,7 +340,7 @@ class MininetRunner( object ): warn( '*** WARNING: Experimental cluster mode!\n' '*** Using RemoteHost, RemoteOVSSwitch, RemoteLink\n' ) host, switch, link = RemoteHost, RemoteOVSSwitch, RemoteLink - Net = partial( MininetCluster, servers=cluster.split( ',' ), + Net = partial( MininetCluster, servers=servers, placement=PLACEMENT[ self.options.placement ] ) mn = Net( topo=topo, diff --git a/examples/cluster.py b/examples/cluster.py index 387c8b0..aa7ddfd 100755 --- a/examples/cluster.py +++ b/examples/cluster.py @@ -82,6 +82,7 @@ from mininet.topolib import TreeTopo from mininet.util import quietRun, errRun, retry from mininet.examples.clustercli import CLI from mininet.log import setLogLevel, debug, info, error +from mininet.clean import addCleanupCallback from signal import signal, SIGINT, SIG_IGN from subprocess import Popen, PIPE, STDOUT @@ -89,9 +90,51 @@ import os from random import randrange import sys import re - +from itertools import groupby +from operator import attrgetter from distutils.version import StrictVersion + +def findUser(): + "Try to return logged-in (usually non-root) user" + return ( + # If we're running sudo + os.environ.get( 'SUDO_USER', False ) or + # Logged-in user (if we have a tty) + ( quietRun( 'who am i' ).split() or [ False ] )[ 0 ] or + # Give up and return effective user + quietRun( 'whoami' ) ) + + +class ClusterCleanup( object ): + "Cleanup callback" + + inited = False + serveruser = {} + + @classmethod + def add( cls, server, user='' ): + "Add an entry to server: user dict" + if not cls.inited: + addCleanupCallback( cls.cleanup ) + if not user: + user = findUser() + cls.serveruser[ server ] = user + + @classmethod + def cleanup( cls ): + "Clean up" + info( '*** Cleaning up cluster\n' ) + for server, user in cls.serveruser.iteritems(): + if server == 'localhost': + # Handled by mininet.clean.cleanup() + continue + else: + cmd = [ 'su', user, '-c', + 'ssh %s@%s sudo mn -c' % ( user, server ) ] + info( cmd, '\n' ) + info( quietRun( cmd ) ) + # BL note: so little code is required for remote nodes, # we will probably just want to update the main Node() # class to enable it for remote access! However, there @@ -125,7 +168,8 @@ class RemoteMixin( object ): self.server = server if server else 'localhost' self.serverIP = ( serverIP if serverIP else self.findServerIP( self.server ) ) - self.user = user if user else self.findUser() + self.user = user if user else findUser() + ClusterCleanup.add( server=server, user=user ) if controlPath is True: # Set a default control path for shared SSH connections controlPath = '/tmp/mn-%r@%h:%p' @@ -148,17 +192,6 @@ class RemoteMixin( object ): self.shell, self.pid = None, None super( RemoteMixin, self ).__init__( name, **kwargs ) - @staticmethod - def findUser(): - "Try to return logged-in (usually non-root) user" - return ( - # If we're running sudo - os.environ.get( 'SUDO_USER', False ) or - # Logged-in user (if we have a tty) - ( quietRun( 'who am i' ).split() or [ False ] )[ 0 ] or - # Give up and return effective user - quietRun( 'whoami' ) ) - # Determine IP address of local host _ipMatchRegex = re.compile( r'\d+\.\d+\.\d+\.\d+' ) @@ -244,7 +277,7 @@ class RemoteMixin( object ): # Drop privileges cmd = [ 'sudo', '-E', '-u', self.user ] + cmd params.update( preexec_fn=self._ignoreSignal ) - debug( '_popen', ' '.join(cmd), params ) + debug( '_popen', cmd, '\n' ) popen = super( RemoteMixin, self )._popen( cmd, **params ) return popen @@ -257,13 +290,6 @@ class RemoteMixin( object ): kwargs.update( moveIntfFn=RemoteLink.moveIntf ) return super( RemoteMixin, self).addIntf( *args, **kwargs ) - def cleanup( self ): - "Help python collect its garbage." - # Intfs may end up in root NS - for intfName in self.intfNames(): - if self.name in intfName: - self.rcmd( 'ip link del ' + intfName ) - self.shell = None class RemoteNode( RemoteMixin, Node ): "A node on a remote server" @@ -282,7 +308,7 @@ class RemoteOVSSwitch( RemoteMixin, OVSSwitch ): def __init__( self, *args, **kwargs ): # No batch startup yet - kwargs.update( batch=False ) + kwargs.update( batch=True ) super( RemoteOVSSwitch, self ).__init__( *args, **kwargs ) def isOldOVS( self ): @@ -298,14 +324,24 @@ class RemoteOVSSwitch( RemoteMixin, OVSSwitch ): StrictVersion( '1.10' ) ) @classmethod - def batchStartup( cls, *_args, **_kwargs ): - "Not implemented yet" - return [] # no switches started - + def batchStartup( cls, switches, **_kwargs ): + "Start up switches in per-server batches" + for server, switchGroup in groupby( switches, attrgetter( 'server' ) ): + info( '(%s)' % server ) + group = tuple( switchGroup ) + switch = group[ 0 ] + OVSSwitch.batchStartup( group, run=switch.cmd ) + return switches + @classmethod - def batchShutdown( cls, *_args, **_kwargs ): - "Not implemented yet" - return [] # no switchest stopped + def batchShutdown( cls, switches, **_kwargs ): + "Stop switches in per-server batches" + for server, switchGroup in groupby( switches, attrgetter( 'server' ) ): + info( '(%s)' % server ) + group = tuple( switchGroup ) + switch = group[ 0 ] + OVSSwitch.batchShutdown( group, run=switch.rcmd ) + return switches class RemoteLink( Link ): @@ -325,6 +361,7 @@ class RemoteLink( Link ): def stop( self ): "Stop this link" + Link.stop( self ) if self.tunnel: self.tunnel.terminate() self.tunnel = None @@ -636,7 +673,7 @@ class MininetCluster( Mininet ): if not self.serverIP: self.serverIP = { server: RemoteMixin.findServerIP( server ) for server in self.servers } - self.user = params.pop( 'user', RemoteMixin.findUser() ) + self.user = params.pop( 'user', findUser() ) if params.pop( 'precheck' ): self.precheck() self.connections = {} diff --git a/mininet/clean.py b/mininet/clean.py index 4761721..707452e 100755 --- a/mininet/clean.py +++ b/mininet/clean.py @@ -38,60 +38,80 @@ def killprocs( pattern ): else: break -def cleanup(): - """Clean up junk which might be left over from old runs; - do fast stuff before slow dp and link removal!""" +class Cleanup( object ): + "Wrapper for cleanup()" - info("*** Removing excess controllers/ofprotocols/ofdatapaths/pings/noxes" - "\n") - zombies = 'controller ofprotocol ofdatapath ping nox_core lt-nox_core ' - zombies += 'ovs-openflowd ovs-controller udpbwtest mnexec ivs' - # Note: real zombie processes can't actually be killed, since they - # are already (un)dead. Then again, - # you can't connect to them either, so they're mostly harmless. - # Send SIGTERM first to give processes a chance to shutdown cleanly. - sh( 'killall ' + zombies + ' 2> /dev/null' ) - time.sleep( 1 ) - sh( 'killall -9 ' + zombies + ' 2> /dev/null' ) + callbacks = [] - # And kill off sudo mnexec - sh( 'pkill -9 -f "sudo mnexec"') + @classmethod + def cleanup( cls): + """Clean up junk which might be left over from old runs; + do fast stuff before slow dp and link removal!""" - info( "*** Removing junk from /tmp\n" ) - sh( 'rm -f /tmp/vconn* /tmp/vlogs* /tmp/*.out /tmp/*.log' ) + info("*** Removing excess controllers/ofprotocols/ofdatapaths/pings/noxes" + "\n") + zombies = 'controller ofprotocol ofdatapath ping nox_core lt-nox_core ' + zombies += 'ovs-openflowd ovs-controller udpbwtest mnexec ivs' + # Note: real zombie processes can't actually be killed, since they + # are already (un)dead. Then again, + # you can't connect to them either, so they're mostly harmless. + # Send SIGTERM first to give processes a chance to shutdown cleanly. + sh( 'killall ' + zombies + ' 2> /dev/null' ) + time.sleep( 1 ) + sh( 'killall -9 ' + zombies + ' 2> /dev/null' ) - info( "*** Removing old X11 tunnels\n" ) - cleanUpScreens() + # And kill off sudo mnexec + sh( 'pkill -9 -f "sudo mnexec"') - info( "*** Removing excess kernel datapaths\n" ) - dps = sh( "ps ax | egrep -o 'dp[0-9]+' | sed 's/dp/nl:/'" ).splitlines() - for dp in dps: - if dp: - sh( 'dpctl deldp ' + dp ) + info( "*** Removing junk from /tmp\n" ) + sh( 'rm -f /tmp/vconn* /tmp/vlogs* /tmp/*.out /tmp/*.log' ) - info( "*** Removing OVS datapaths" ) - dps = sh("ovs-vsctl --timeout=1 list-br").strip().splitlines() - if dps: - sh( "ovs-vsctl " + " -- ".join( "--if-exists del-br " + dp - for dp in dps if dp ) ) - # And in case the above didn't work... - dps = sh("ovs-vsctl --timeout=1 list-br").strip().splitlines() - for dp in dps: - sh( 'ovs-vsctl del-br ' + dp ) + info( "*** Removing old X11 tunnels\n" ) + cleanUpScreens() - info( "*** Removing all links of the pattern foo-ethX\n" ) - links = sh( "ip link show | " - "egrep -o '([-_.[:alnum:]]+-eth[[:digit:]]+)'" ).splitlines() - for link in links: - if link: - sh( "ip link del " + link ) + info( "*** Removing excess kernel datapaths\n" ) + dps = sh( "ps ax | egrep -o 'dp[0-9]+' | sed 's/dp/nl:/'" ).splitlines() + for dp in dps: + if dp: + sh( 'dpctl deldp ' + dp ) - info( "*** Killing stale mininet node processes\n" ) - killprocs( 'mininet:' ) + info( "*** Removing OVS datapaths" ) + dps = sh("ovs-vsctl --timeout=1 list-br").strip().splitlines() + if dps: + sh( "ovs-vsctl " + " -- ".join( "--if-exists del-br " + dp + for dp in dps if dp ) ) + # And in case the above didn't work... + dps = sh("ovs-vsctl --timeout=1 list-br").strip().splitlines() + for dp in dps: + sh( 'ovs-vsctl del-br ' + dp ) - info( "*** Shutting down stale tunnels\n" ) - killprocs( 'Tunnel=Ethernet' ) - killprocs( '.ssh/mn') - sh( 'rm -f ~/.ssh/mn/*' ) + info( "*** Removing all links of the pattern foo-ethX\n" ) + links = sh( "ip link show | " + "egrep -o '([-_.[:alnum:]]+-eth[[:digit:]]+)'" ).splitlines() + for link in links: + if link: + sh( "ip link del " + link ) - info( "*** Cleanup complete.\n" ) + info( "*** Killing stale mininet node processes\n" ) + killprocs( 'mininet:' ) + + info( "*** Shutting down stale tunnels\n" ) + killprocs( 'Tunnel=Ethernet' ) + killprocs( '.ssh/mn') + sh( 'rm -f ~/.ssh/mn/*' ) + + # Call any additional cleanup code if necessary + for callback in cls.callbacks: + callback() + + info( "*** Cleanup complete.\n" ) + + @classmethod + def addCleanupCallback( cls, callback ): + "Add cleanup callback" + if callback not in cls.callbacks: + cls.callbacks.append( callback ) + + +cleanup = Cleanup.cleanup +addCleanupCallback = Cleanup.addCleanupCallback diff --git a/mininet/node.py b/mininet/node.py index 0ccddff..3d459a3 100644 --- a/mininet/node.py +++ b/mininet/node.py @@ -157,8 +157,7 @@ class Node( object ): break self.pollOut.poll() self.waiting = False - self.cmd( 'stty -echo' ) - self.cmd( 'set +m' ) + self.cmd( 'stty -echo; set +m' ) def mountPrivateDirs( self ): "mount private directories" @@ -1270,7 +1269,7 @@ class OVSSwitch( Switch ): def batchShutdown( cls, switches, run=errRun ): "Shut down a list of OVS switches" delcmd = 'del-br %s' - if not cls.isOldOVS(): + if switches and not switches[ 0 ].isOldOVS(): delcmd = '--if-exists ' + delcmd # First, delete them all from ovsdb run( 'ovs-vsctl ' + @@ -1549,4 +1548,4 @@ def DefaultController( name, controllers=DefaultControllers, **kwargs ): controller = findController( controllers ) if not controller: raise Exception( 'Could not find a default OpenFlow controller' ) - return contr \ No newline at end of file + return controller( name, **kwargs ) From c11e9f3316145a9849713d8e317dd028e794945b Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Wed, 28 Jan 2015 17:04:58 -0800 Subject: [PATCH 16/20] Fix OVS user switch (remove unnecessary % parameter) --- mininet/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mininet/node.py b/mininet/node.py index 3d459a3..d3f4183 100644 --- a/mininet/node.py +++ b/mininet/node.py @@ -1178,7 +1178,7 @@ class OVSSwitch( Switch ): if not self.inband: opts += ' other-config:disable-in-band=true' if self.datapath == 'user': - opts += ' datapath_type=netdev' % self + opts += ' datapath_type=netdev' if self.protocols and not self.isOldOVS(): opts += ' protocols=%s' % ( self, self.protocols ) if self.stp and self.failMode == 'standalone': From a4e933688aba170dd50c11e2b90c43633447e86b Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Thu, 29 Jan 2015 00:26:15 -0800 Subject: [PATCH 17/20] Set batch=False in OVSSwitch for low-level API If you try to use the low-level API, you are probably not going to call batchStartup()! So, we set batch=False by default. This means that buildFromTopo() needs to set it to True, so we add a bit of irritatingly complex machinery to allow this to happen. The good fallout of this is that now customConstructor() returns a real subclass, not simply a constructor function! We also detect errors where people are incorrectly attempting to give parameters to a lambda function - since none of our lambdas accept parameters!! Note that this is a bit like functools.partial for classes - it would be nice if functools had a true subclassing function. --- mininet/net.py | 7 ++++++- mininet/node.py | 4 ++-- mininet/util.py | 33 +++++++++++++++++++++------------ 3 files changed, 29 insertions(+), 15 deletions(-) diff --git a/mininet/net.py b/mininet/net.py index b4263bd..7a01ecb 100755 --- a/mininet/net.py +++ b/mininet/net.py @@ -414,7 +414,12 @@ class Mininet( object ): info( '\n*** Adding switches:\n' ) for switchName in topo.switches(): - self.addSwitch( switchName, **topo.nodeInfo( switchName) ) + # A bit ugly: add batch parameter if appropriate + params = topo.nodeInfo( switchName) + cls = params.get( 'cls', self.switch ) + if hasattr( cls, 'batchStartup' ): + params.setdefault( 'batch', True ) + self.addSwitch( switchName, **params ) info( switchName + ' ' ) info( '\n*** Adding links:\n' ) diff --git a/mininet/node.py b/mininet/node.py index d3f4183..768ff68 100644 --- a/mininet/node.py +++ b/mininet/node.py @@ -1058,7 +1058,7 @@ class OVSSwitch( Switch ): def __init__( self, name, failMode='secure', datapath='kernel', inband=False, protocols=None, - reconnectms=1000, stp=False, batch=True, **params ): + reconnectms=1000, stp=False, batch=False, **params ): """name: name for switch failMode: controller loss behavior (secure|open) datapath: userspace or kernel mode (kernel|user) @@ -1067,7 +1067,7 @@ class OVSSwitch( Switch ): Unspecified (or old OVS version) uses OVS default reconnectms: max reconnect timeout in ms (0/None for default) stp: enable STP (False, requires failMode=standalone) - batch: enable batch startup (True)""" + batch: enable batch startup (False)""" Switch.__init__( self, name, **params ) self.failMode = failMode self.datapath = datapath diff --git a/mininet/util.py b/mininet/util.py index 1780632..1f5f64d 100644 --- a/mininet/util.py +++ b/mininet/util.py @@ -536,19 +536,28 @@ def customConstructor( constructors, argStr ): raise Exception( "error: %s is unknown - please specify one of %s" % ( cname, constructors.keys() ) ) - def customized( name, *args, **params ): - "Customized constructor, useful for Node, Link, and other classes" - params = params.copy() - params.update( kwargs ) - if not newargs: - return constructor( name, *args, **params ) - if args: - warn( 'warning: %s replacing %s with %s\n' % ( - constructor, args, newargs ) ) - return constructor( name, *newargs, **params ) + if not newargs and not kwargs: + return constructor - customized.__name__ = 'customConstructor(%s)' % argStr - return customized + if not isinstance( constructor, type ): + raise Exception( "error: invalid arguments %s" % argStr ) + + # Return a customized subclass + cls = constructor + class CustomClass( cls ): + "Customized subclass, useful for Node, Link, and other classes" + def __init__( self, name, *args, **params ): + params = params.copy() + params.update( kwargs ) + if not newargs: + return cls.__init__( self, name, *args, **params ) + if args: + warn( 'warning: %s replacing %s with %s\n' % + ( constructor, args, newargs ) ) + return cls.__init__( self, name, *newargs, **params ) + + CustomClass.__name__ = '%s%s' % ( cls.__name__, kwargs ) + return CustomClass def buildTopo( topos, topoStr ): """Create topology from string with format (object, arg1, arg2,...). From 9483f6378f7fb4167f918fd30ede20d24d765093 Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Tue, 3 Feb 2015 15:30:16 -0800 Subject: [PATCH 18/20] Make sure DataController's interfaces are deleted --- examples/controlnet.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/examples/controlnet.py b/examples/controlnet.py index 9397188..50f989c 100755 --- a/examples/controlnet.py +++ b/examples/controlnet.py @@ -27,11 +27,17 @@ from mininet.log import setLogLevel, info class DataController( Controller ): """Data Network Controller. - patched to avoid checkListening error""" + patched to avoid checkListening error and to delete intfs""" + def checkListening( self ): "Ignore spurious error" pass + def stop( self, *args, **kwargs ): + "Make sure intfs are deleted" + kwargs.update( deleteIntfs=True ) + super( Controller, self ).stop( *args, **kwargs ) + class MininetFacade( object ): """Mininet object facade that allows a single CLI to talk to one or more networks""" From 19331ca287c50bb739dd95ce771513f4b1e3c2db Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Tue, 3 Feb 2015 16:02:07 -0800 Subject: [PATCH 19/20] use net.addLink() so that link is cleaned up --- examples/sshd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/sshd.py b/examples/sshd.py index 39107ec..3fbd182 100755 --- a/examples/sshd.py +++ b/examples/sshd.py @@ -39,7 +39,7 @@ def connectToRootNS( network, switch, ip, routes ): routes: host networks to route to""" # Create a node in root namespace and link to switch 0 root = Node( 'root', inNamespace=False ) - intf = Link( root, switch ).intf1 + intf = network.addLink( root, switch ).intf1 root.setIP( ip, intf=intf ) # Start network that now includes link to root namespace network.start() From 2e4dd134823545b4e9e4a440e38e2f8e6dad9dd5 Mon Sep 17 00:00:00 2001 From: Bob Lantz Date: Tue, 3 Feb 2015 17:40:20 -0800 Subject: [PATCH 20/20] Turn off printPid by default to avoid mnexec fork/exec --- mininet/node.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mininet/node.py b/mininet/node.py index 768ff68..c602919 100644 --- a/mininet/node.py +++ b/mininet/node.py @@ -258,9 +258,9 @@ class Node( object ): """Send a command, followed by a command to echo a sentinel, and return without waiting for the command to complete. args: command and arguments, or string - printPid: print command's PID?""" + printPid: print command's PID? (False)""" assert self.shell and not self.waiting - printPid = kwargs.get( 'printPid', True ) + printPid = kwargs.get( 'printPid', False ) # Allow sendCmd( [ list ] ) if len( args ) == 1 and isinstance( args[ 0 ], list ): cmd = args[ 0 ]