From b20c9470c7f78caebdd139622c98255fad860c34 Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Fri, 2 Aug 2013 11:08:38 -0700 Subject: [PATCH 1/3] add sysctl test and set fixes #184 --- mininet/util.py | 70 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 66 insertions(+), 4 deletions(-) diff --git a/mininet/util.py b/mininet/util.py index 21e71ce..8c7e9ba 100644 --- a/mininet/util.py +++ b/mininet/util.py @@ -2,8 +2,8 @@ from mininet.log import output, info, error, warn -from time import sleep -from resource import setrlimit, RLIMIT_NPROC, RLIMIT_NOFILE +from time import sleep, time +from resource import getrlimit, setrlimit, RLIMIT_NPROC, RLIMIT_NOFILE from select import poll, POLLIN, POLLHUP from subprocess import call, check_call, Popen, PIPE, STDOUT import re @@ -355,11 +355,73 @@ def pmonitor(popens, timeoutms=500, readline=True, yield None, '' # Other stuff we use +#PTY_LIMIT=8192 +# Mininet: Increase open file limit +#fs.file-max = 100000 + +# Mininet: increase network buffer space +#net.core.wmem_max = 16777216 +#net.core.rmem_max = 16777216 +#net.ipv4.tcp_rmem = 10240 87380 16777216 +#net.ipv4.tcp_rmem = 10240 87380 16777216 +#net.core.netdev_max_backlog = 5000 + +# Mininet: increase arp cache size +#net.ipv4.neigh.default.gc_thresh1 = 4096 +#net.ipv4.neigh.default.gc_thresh2 = 8192 +#net.ipv4.neigh.default.gc_thresh3 = 16384 + +# Mininet: increase routing table size +#net.ipv4.route.max_size=32768 + +def sysctlTestAndSet( name, limit ): + #read limit + oldLimit = quietRun( 'sysctl -n %s' % name ) + #print name, type(limit), limit + if type( limit ) is int: + if int( oldLimit ) < limit: # or True: + #print 'set new limit:', limit + output = errRun( 'sysctl -w %s=%s' % ( name, limit) ) + #else: + # print 'keep old limit' + else: + #print 'cmd:', 'sysctl', '-w', '%s=%s' % ( name, limit ) + arg = '%s="%s"' % ( name, limit ) + output = errRun( 'sysctl', '-w', '%s=%s' % ( name, limit ) ) + #print output + #compare to limit + # set new limit if necessary + #print 'new limit:', quietRun( 'sysctl -n %s' % name ) + +def rlimitTestAndSet( name, limit ): + #print 'old: ', name, getrlimit( name ) + #read limit + soft, hard = getrlimit( name ) + if soft < limit: + hardLimit = hard if limit < hard else limit + setrlimit( name, ( limit, hardLimit ) ) + #compare to limit + # set new limit if necessary + #print 'new: ', name, getrlimit( name ) def fixLimits(): "Fix ridiculously small resource limits." - setrlimit( RLIMIT_NPROC, ( 8192, 8192 ) ) - setrlimit( RLIMIT_NOFILE, ( 16384, 16384 ) ) + start = time() + rlimitTestAndSet( RLIMIT_NPROC, 8192 ) + rlimitTestAndSet( RLIMIT_NOFILE, 16384 ) + sysctlTestAndSet( 'fs.file-max', 10000 ) + sysctlTestAndSet( 'net.core.wmem_max', 16777216 ) + sysctlTestAndSet( 'net.core.rmem_max', 16777216 ) + sysctlTestAndSet( 'net.ipv4.tcp_rmem', '10240 87380 16777216' ) + sysctlTestAndSet( 'net.ipv4.tcp_wmem', '10240 87380 16777216' ) + sysctlTestAndSet( 'net.core.netdev_max_backlog', 5000 ) + sysctlTestAndSet( 'net.ipv4.neigh.default.gc_thresh1', 4096 ) + sysctlTestAndSet( 'net.ipv4.neigh.default.gc_thresh2', 8192 ) + sysctlTestAndSet( 'net.ipv4.neigh.default.gc_thresh3', 16384 ) + sysctlTestAndSet( 'net.ipv4.route.max_size', 32768 ) + sysctlTestAndSet( 'kernel.pty.max', 20000 ) + end = time() + print 'limits:', start, end, end-start def mountCgroups(): "Make sure cgroups file system is mounted" From 867a6d67317b50f4a1c67807f1ad7cd58265f12b Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Wed, 14 Aug 2013 15:04:11 -0700 Subject: [PATCH 2/3] Cleaned up sysctl and rlimit test and set Use files instead of sysctl to set limits --- mininet/util.py | 61 +++++++++++++++---------------------------------- util/install.sh | 17 +++++++------- 2 files changed, 28 insertions(+), 50 deletions(-) diff --git a/mininet/util.py b/mininet/util.py index 8c7e9ba..0c2b19b 100644 --- a/mininet/util.py +++ b/mininet/util.py @@ -2,7 +2,7 @@ from mininet.log import output, info, error, warn -from time import sleep, time +from time import sleep from resource import getrlimit, setrlimit, RLIMIT_NPROC, RLIMIT_NOFILE from select import poll, POLLIN, POLLHUP from subprocess import call, check_call, Popen, PIPE, STDOUT @@ -355,73 +355,50 @@ def pmonitor(popens, timeoutms=500, readline=True, yield None, '' # Other stuff we use -#PTY_LIMIT=8192 -# Mininet: Increase open file limit -#fs.file-max = 100000 - -# Mininet: increase network buffer space -#net.core.wmem_max = 16777216 -#net.core.rmem_max = 16777216 -#net.ipv4.tcp_rmem = 10240 87380 16777216 -#net.ipv4.tcp_rmem = 10240 87380 16777216 -#net.core.netdev_max_backlog = 5000 - -# Mininet: increase arp cache size -#net.ipv4.neigh.default.gc_thresh1 = 4096 -#net.ipv4.neigh.default.gc_thresh2 = 8192 -#net.ipv4.neigh.default.gc_thresh3 = 16384 - -# Mininet: increase routing table size -#net.ipv4.route.max_size=32768 - def sysctlTestAndSet( name, limit ): + "Helper function to set sysctl limits" + #convert non-directory names into directory names + if '/' not in name: + name = '/proc/sys/' + name.replace('.','/') #read limit - oldLimit = quietRun( 'sysctl -n %s' % name ) - #print name, type(limit), limit + f = open(name, 'r+') + oldLimit = f.readline() if type( limit ) is int: - if int( oldLimit ) < limit: # or True: - #print 'set new limit:', limit - output = errRun( 'sysctl -w %s=%s' % ( name, limit) ) - #else: - # print 'keep old limit' + #compare integer limits before overriding + if int( oldLimit ) < limit: + f.write("%d" % limit) else: - #print 'cmd:', 'sysctl', '-w', '%s=%s' % ( name, limit ) - arg = '%s="%s"' % ( name, limit ) - output = errRun( 'sysctl', '-w', '%s=%s' % ( name, limit ) ) - #print output - #compare to limit - # set new limit if necessary - #print 'new limit:', quietRun( 'sysctl -n %s' % name ) + #overwrite non-integer limits + f.write(limit) + f.close() def rlimitTestAndSet( name, limit ): - #print 'old: ', name, getrlimit( name ) - #read limit + "Helper function to set rlimits" soft, hard = getrlimit( name ) if soft < limit: hardLimit = hard if limit < hard else limit setrlimit( name, ( limit, hardLimit ) ) - #compare to limit - # set new limit if necessary - #print 'new: ', name, getrlimit( name ) def fixLimits(): "Fix ridiculously small resource limits." - start = time() rlimitTestAndSet( RLIMIT_NPROC, 8192 ) rlimitTestAndSet( RLIMIT_NOFILE, 16384 ) + #Increase open file limit sysctlTestAndSet( 'fs.file-max', 10000 ) + #Increase network buffer space sysctlTestAndSet( 'net.core.wmem_max', 16777216 ) sysctlTestAndSet( 'net.core.rmem_max', 16777216 ) sysctlTestAndSet( 'net.ipv4.tcp_rmem', '10240 87380 16777216' ) sysctlTestAndSet( 'net.ipv4.tcp_wmem', '10240 87380 16777216' ) sysctlTestAndSet( 'net.core.netdev_max_backlog', 5000 ) + #Increase arp cache size sysctlTestAndSet( 'net.ipv4.neigh.default.gc_thresh1', 4096 ) sysctlTestAndSet( 'net.ipv4.neigh.default.gc_thresh2', 8192 ) sysctlTestAndSet( 'net.ipv4.neigh.default.gc_thresh3', 16384 ) + #Increase routing table size sysctlTestAndSet( 'net.ipv4.route.max_size', 32768 ) + #Increase number of PTYs for nodes sysctlTestAndSet( 'kernel.pty.max', 20000 ) - end = time() - print 'limits:', start, end, end-start def mountCgroups(): "Make sure cgroups file system is mounted" diff --git a/util/install.sh b/util/install.sh index e6fdf92..5ecfc16 100755 --- a/util/install.sh +++ b/util/install.sh @@ -144,14 +144,15 @@ function mn_deps { python-setuptools cgroup-bin ethtool help2man \ pyflakes pylint pep8 - # Add sysctl parameters as noted in the INSTALL file to increase kernel - # limits to support larger setups: - if ! grep Mininet /etc/sysctl.conf; then - echo "Adding Mininet sysctl settings" - sudo su -c "cat $MININET_DIR/mininet/util/sysctl_addon >> /etc/sysctl.conf" - fi - # Load new sysctl settings: - sudo sysctl -p + # TODO: remove this; it's no longer necessary as these are check when mn starts + ## Add sysctl parameters as noted in the INSTALL file to increase kernel + ## limits to support larger setups: + #if ! grep Mininet /etc/sysctl.conf; then + # echo "Adding Mininet sysctl settings" + # sudo su -c "cat $MININET_DIR/mininet/util/sysctl_addon >> /etc/sysctl.conf" + #fi + ## Load new sysctl settings: + #sudo sysctl -p echo "Installing Mininet core" pushd $MININET_DIR/mininet From b635fd9edd5a1ead381b5068bea2ccba56e6faa8 Mon Sep 17 00:00:00 2001 From: Brian O'Connor Date: Wed, 14 Aug 2013 17:29:23 -0700 Subject: [PATCH 3/3] Some cleanup and style fixes for fixlimits --- mininet/util.py | 8 ++++---- util/install.sh | 10 ---------- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/mininet/util.py b/mininet/util.py index 0c2b19b..05df136 100644 --- a/mininet/util.py +++ b/mininet/util.py @@ -359,17 +359,17 @@ def sysctlTestAndSet( name, limit ): "Helper function to set sysctl limits" #convert non-directory names into directory names if '/' not in name: - name = '/proc/sys/' + name.replace('.','/') + name = '/proc/sys/' + name.replace( '.', '/' ) #read limit - f = open(name, 'r+') + f = open( name, 'r+' ) oldLimit = f.readline() if type( limit ) is int: #compare integer limits before overriding if int( oldLimit ) < limit: - f.write("%d" % limit) + f.write( "%d" % limit ) else: #overwrite non-integer limits - f.write(limit) + f.write( limit ) f.close() def rlimitTestAndSet( name, limit ): diff --git a/util/install.sh b/util/install.sh index 5ecfc16..4873a8e 100755 --- a/util/install.sh +++ b/util/install.sh @@ -144,16 +144,6 @@ function mn_deps { python-setuptools cgroup-bin ethtool help2man \ pyflakes pylint pep8 - # TODO: remove this; it's no longer necessary as these are check when mn starts - ## Add sysctl parameters as noted in the INSTALL file to increase kernel - ## limits to support larger setups: - #if ! grep Mininet /etc/sysctl.conf; then - # echo "Adding Mininet sysctl settings" - # sudo su -c "cat $MININET_DIR/mininet/util/sysctl_addon >> /etc/sysctl.conf" - #fi - ## Load new sysctl settings: - #sudo sysctl -p - echo "Installing Mininet core" pushd $MININET_DIR/mininet sudo make install