Add cgroup2 support (#1166)

util.py: replace mountCgroups with checkCgroups, which
         returns the cgroup version (cgroup, cgroup2)

node.py: handle cpu.max='quota period' for cgroup2
         vs. cpu.{cfs_quota_us,cfs_period_us} for cgroup

net.py: make _iperfVals more robust in the case of a missing
        UDP ACK

examples/cpu.py: user _iperfVals to handle new iperf behavior
                 of no output from telnet

examples/cpu.py

@@ -68,11 +68,16 @@ def bwtest( cpuLimits, period_us=100000, seconds=10 ):
             # the client's buffer fill rate
             popen = server.popen( 'iperf -yc -s -p 5001' )
             waitListening( client, server, 5001 )
-            # ignore empty result from waitListening/telnet
-            popen.stdout.readline()
             client.cmd( 'iperf -yc -t %s -c %s' % ( seconds, server.IP() ) )
-            result = decode( popen.stdout.readline() ).split( ',' )
+            # ignore empty result from waitListening/telnet for old iperf
-            bps = float( result[ -1 ] )
+            svals = {}
+            while not svals or int( svals[ 'rate' ] ) == 0:
+                line = decode( popen.stdout.readline() )
+                # Probably shouldn't depend on an internal method, but
+                # this is the easiest way
+                svals = Mininet._iperfVals(  # pylint: disable=protected-access
+                    line, server.IP() )
+            bps = float( svals[ 'rate' ] )
             popen.terminate()
             net.stop()
             updated = results.get( sched, [] )

mininet/net.py

@@ -789,8 +789,11 @@ class Mininet( object ):
         """
         fields = 'date cip cport sip sport ipver interval sent rate'
         lines = iperfcsv.strip().split('\n')
-        if lines:
+        svals = {}
-            line = lines[ -1 ].split( ',' )
+        for line in lines:
+            if ',' not in line:
+                continue
+            line = line.split( ',' )
             svals = dict( zip( fields.split(), line ) )
             # Return client in cip:cport, server in sip:sport
             if svals[ 'cip' ] == serverip:
@@ -799,7 +802,6 @@ class Mininet( object ):
                 svals[ 'cport' ], svals[ 'sport' ] = (
                     svals[ 'sport' ], svals[ 'cport' ] )
         return svals
-        return {}
 
     # XXX This should be cleaned up
 

mininet/node.py

@@ -708,20 +708,20 @@ class CPULimitedHost( Host ):
 
     def cgroupSet( self, param, value, resource='cpu' ):
         "Set a cgroup parameter and return its value"
-        cmd = 'cgset -r %s.%s=%s /%s' % (
+        cmd = [ 'cgset', '-r', "%s.%s=%s" % (
-            resource, param, value, self.name )
+            resource, param, value), '/' + self.name ]
-        quietRun( cmd )
+        errFail( cmd )
-        nvalue = int( self.cgroupGet( param, resource ) )
+        nvalue = self.cgroupGet( param, resource )
-        if nvalue != value:
+        if nvalue != str( value ):
             error( '*** error: cgroupSet: %s set to %s instead of %s\n'
                    % ( param, nvalue, value ) )
         return nvalue
 
     def cgroupGet( self, param, resource='cpu' ):
         "Return value of cgroup parameter"
-        cmd = 'cgget -r %s.%s /%s' % (
+        pname = '%s.%s' % ( resource, param )
-            resource, param, self.name )
+        cmd = 'cgget -n -r %s /%s' % ( pname, self.name )
-        return int( quietRun( cmd ).split()[ -1 ] )
+        return quietRun( cmd )[len(pname)+1:].strip()
 
     def cgroupDel( self ):
         "Clean up our cgroup"
@@ -788,6 +788,8 @@ class CPULimitedHost( Host ):
     def cfsInfo( self, f ):
         "Internal method: return parameters for CFS bandwidth"
         pstr, qstr = 'cfs_period_us', 'cfs_quota_us'
+        if self.cgversion == 'cgroup2':
+            pstr, qstr = 'max', ''
         # CFS uses wall clock time for period and CPU time for quota.
         quota = int( self.period_us * f * numCores() )
         period = self.period_us
@@ -797,7 +799,7 @@ class CPULimitedHost( Host ):
             period = int( quota / f / numCores() )
         # Reset to unlimited on negative quota
         if quota < 0:
-            quota = -1
+            quota = 'max' if self.cgversion == 'cgroup2' else -1
         return pstr, qstr, period, quota
 
     # BL comment:
@@ -827,12 +829,16 @@ class CPULimitedHost( Host ):
         else:
             return
         # Set cgroup's period and quota
+        if self.cgversion == 'cgroup':
             setPeriod = self.cgroupSet( pstr, period )
             setQuota = self.cgroupSet( qstr, quota )
+        else:
+            setQuota, setPeriod = self.cgroupSet(
+                    pstr, '%s %s' % (quota, period) ).split()
         if sched == 'rt':
             # Set RT priority if necessary
             sched = self.chrt()
-        info( '(%s %d/%dus) ' % ( sched, setQuota, setPeriod ) )
+        info( '(%s %s/%dus) ' % ( sched, setQuota, int( setPeriod ) ) )
 
     def setCPUs( self, cores, mems=0 ):
         "Specify (real) cores that our cgroup can run on"
@@ -864,11 +870,12 @@ class CPULimitedHost( Host ):
         return r
 
     inited = False
+    cgversion = 'cgroup2'
 
     @classmethod
     def init( cls ):
         "Initialization for CPULimitedHost class"
-        mountCgroups()
+        cls.cgversion = mountCgroups()
         cls.inited = True
 
 

mininet/util.py

@@ -5,6 +5,7 @@ import os
 import re
 import sys
 
+from collections import namedtuple
 from fcntl import fcntl, F_GETFL, F_SETFL
 from functools import partial
 from os import O_NONBLOCK
@@ -126,6 +127,8 @@ def oldQuietRun( *cmd ):
 # This is a bit complicated, but it enables us to
 # monitor command output as it is happening
 
+CmdResult = namedtuple( 'CmdResult', 'out err ret' )
+
 # pylint: disable=too-many-branches,too-many-statements
 def errRun( *cmd, **kwargs ):
     """Run a command and return stdout, stderr and return code
@@ -194,7 +197,8 @@ def errRun( *cmd, **kwargs ):
     if stderr == PIPE:
         popen.stderr.close()
     debug( out, err, returncode )
-    return out, err, returncode
+    return CmdResult( out, err, returncode )
+
 # pylint: enable=too-many-branches
 
 def errFail( *cmd, **kwargs ):
@@ -203,11 +207,11 @@ def errFail( *cmd, **kwargs ):
     if ret:
         raise Exception( "errFail: %s failed with return code %s: %s"
                          % ( cmd, ret, err ) )
-    return out, err, ret
+    return CmdResult( out, err, ret )
 
 def quietRun( cmd, **kwargs ):
     "Run a command and return merged stdout and stderr"
-    return errRun( cmd, stderr=STDOUT, **kwargs )[ 0 ]
+    return errRun( cmd, stderr=STDOUT, **kwargs ).out
 
 def which(cmd, **kwargs ):
     "Run a command and return merged stdout and stderr"
@@ -545,18 +549,25 @@ def fixLimits():
               "Mininet's performance may be affected.\n" )
     # pylint: enable=broad-except
 
-
+def mountCgroups( cgcontrol='cpu cpuacct cpuset' ):
-def mountCgroups():
+    """Mount cgroupfs if needed and return cgroup version
-    "Make sure cgroups file system is mounted"
+       cgcontrol: cgroup controllers to check ('cpu cpuacct cpuset')
-    mounts = quietRun( 'grep cgroup /proc/mounts' )
+       Returns: 'cgroup' | 'cgroup2' """
-    cgdir = '/sys/fs/cgroup'
+    # Try to read the cgroup controllers in cgcontrol
-    csdir = cgdir + '/cpuset'
+    cglist = cgcontrol.split()
-    if ('cgroup %s' % cgdir not in mounts and
+    paths = ' '.join( '-g ' + c for c in cglist )
-            'cgroups %s' % cgdir not in mounts):
+    cmd = 'cgget -n %s /' % paths
-        raise Exception( "cgroups not mounted on " + cgdir )
+    result = errRun( cmd )
-    if 'cpuset %s' % csdir not in mounts:
+    # If it failed, mount cgroupfs and retry
-        errRun( 'mkdir -p ' + csdir )
+    if result.ret or result.err or any(
-        errRun( 'mount -t cgroup -ocpuset cpuset ' + csdir )
+            c not in result.out for c in cglist ):
+        errFail( 'cgroupfs-mount' )
+        result = errRun( cmd )
+        errFail( cmd )
+    # cpu.cfs_period_us is used for cgroup but not cgroup2
+    if 'cpu.cfs_period_us' in result.out:
+        return 'cgroup'
+    return 'cgroup2'
 
 def natural( text ):
     "To sort sanely/alphabetically: sorted( l, key=natural )"