Add support for native Kubernetes CPU Manager

For Kubernetes environment core ids in the config and test files are relative and automatically remapped to the allowed cores allocated for container. There is no change to the baremetal or VM environment. Signed-off-by: Yury Kylulin <[email protected]> Change-Id: I63e499723e8213de1b05d4175eb3eddc4492ccf5
opnfv · Feb 11, 2021 · 849357b · 849357b
1 parent 4ada3cb
commit 849357b
Show file tree

Hide file tree

Showing 33 changed files with 132 additions and 21 deletions.
diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/configs/gen.cfg b/VNFs/DPPD-PROX/helper-scripts/rapid/configs/gen.cfg
@@ -41,7 +41,7 @@ mempool size=8K
 name=${name}
 heartbeat timeout=${heartbeat}
 
-[core 0]
+[core $mcore]
 mode=master
 
 [core $gencores]

diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/configs/gen_gw.cfg b/VNFs/DPPD-PROX/helper-scripts/rapid/configs/gen_gw.cfg
@@ -40,7 +40,7 @@ mempool size=8K
 name=${name}
 heartbeat timeout=${heartbeat}
 
-[core 0]
+[core $mcore]
 mode=master
 
 [core $gencores]

diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/configs/genv6.cfg b/VNFs/DPPD-PROX/helper-scripts/rapid/configs/genv6.cfg
@@ -38,7 +38,7 @@ mempool size=8K
 name=${name}
 heartbeat timeout=${heartbeat}
 
-[core 0]
+[core $mcore]
 mode=master
 
 [core $gencores]

diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/configs/impair.cfg b/VNFs/DPPD-PROX/helper-scripts/rapid/configs/impair.cfg
@@ -31,14 +31,13 @@ vlan=yes
 vdev=impair_tap
 local ipv4=${local_ip1}
 
-
 [defaults]
 mempool size=8K
 
 [global]
 name=${name}
 
-[core 0]
+[core $mcore]
 mode=master
 
 [core $cores]

diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/configs/irq.cfg b/VNFs/DPPD-PROX/helper-scripts/rapid/configs/irq.cfg
@@ -34,7 +34,7 @@ mempool size=8K
 [global]
 name=${name}
 
-[core 0]
+[core $mcore]
 mode=master
 
 [core $cores]

diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/configs/l2gen.cfg b/VNFs/DPPD-PROX/helper-scripts/rapid/configs/l2gen.cfg
@@ -37,7 +37,7 @@ mempool size=8K
 [global]
 name=${name}
 
-[core 0]
+[core $mcore]
 mode=master
 
 [core $gencores]

diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/configs/l2gen_bare.cfg b/VNFs/DPPD-PROX/helper-scripts/rapid/configs/l2gen_bare.cfg
@@ -37,7 +37,7 @@ mempool size=8K
 [global]
 name=${name}
 
-[core 0]
+[core $mcore]
 mode=master
 
 [core $gencores]

diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/configs/l2swap.cfg b/VNFs/DPPD-PROX/helper-scripts/rapid/configs/l2swap.cfg
@@ -35,7 +35,7 @@ mempool size=8K
 [global]
 name=${name}
 
-[core 0]
+[core $mcore]
 mode=master
 
 [core $cores]

diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/configs/secgw1.cfg b/VNFs/DPPD-PROX/helper-scripts/rapid/configs/secgw1.cfg
@@ -41,7 +41,7 @@ mempool size=16K
 start time=20
 name=${name}
 
-[core 0]
+[core $mcore]
 mode=master
 
 [core $cores]

diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/configs/secgw2.cfg b/VNFs/DPPD-PROX/helper-scripts/rapid/configs/secgw2.cfg
@@ -41,7 +41,7 @@ mempool size=16K
 start time=20
 name=${name}
 
-[core 0]
+[core $mcore]
 mode=master
 
 [core $cores]

diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/configs/swap.cfg b/VNFs/DPPD-PROX/helper-scripts/rapid/configs/swap.cfg
@@ -35,7 +35,7 @@ mempool size=8K
 [global]
 name=${name}
 
-[core 0]
+[core $mcore]
 mode=master
 
 [core $cores]

diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/configs/swapv6.cfg b/VNFs/DPPD-PROX/helper-scripts/rapid/configs/swapv6.cfg
@@ -33,7 +33,7 @@ mempool size=8K
 [global]
 name=${name}
 
-[core 0]
+[core $mcore]
 mode=master
 
 [core $cores]

diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/pod-rapid.yaml b/VNFs/DPPD-PROX/helper-scripts/rapid/pod-rapid.yaml
@@ -7,7 +7,7 @@ metadata:
 spec:
   containers:
   - name: pod-rapid
-    image: localhost:5000/prox_slim:latest
+    image: opnfv/rapid:latest
     imagePullPolicy: Always
     securityContext:
       capabilities:
@@ -19,9 +19,12 @@ spec:
       requests:
         hugepages-2Mi: 512Mi
         memory: 1Gi
+        cpu: 8
         intel.com/intel_sriov_vfio: '1'
       limits:
         hugepages-2Mi: 512Mi
+        memory: 1Gi
+        cpu: 8
         intel.com/intel_sriov_vfio: '1'
   volumes:
   - name: hugepages

diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid.pods b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid.pods
@@ -15,7 +15,7 @@
 ##
 
 [DEFAULT]
-total_number_of_pods=3
+total_number_of_pods=2
 
 [POD1]
 nodeSelector_hostname=k8s-node1
@@ -26,8 +26,3 @@ dp_subnet=24
 nodeSelector_hostname=k8s-node2
 dp_ip=192.168.30.12
 dp_subnet=24
-
-[POD3]
-nodeSelector_hostname=k8s-node2
-dp_ip=192.168.30.13
-dp_subnet=24
diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_generator_machine.py b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_generator_machine.py
@@ -81,6 +81,24 @@ def get_cores(self):
         return (self.machine_params['gencores'] +
                 self.machine_params['latcores'])
 
+    def remap_all_cpus(self):
+        """Convert relative cpu ids for different parameters (gencores, latcores)
+        """
+        super().remap_all_cpus()
+
+        if self.cpu_mapping is None:
+            return
+
+        if 'gencores' in self.machine_params.keys():
+            cpus_remapped = super().remap_cpus(self.machine_params['gencores'])
+            RapidLog.debug('{} ({}): gencores {} remapped to {}'.format(self.name, self.ip, self.machine_params['gencores'], cpus_remapped))
+            self.machine_params['gencores'] = cpus_remapped
+
+        if 'latcores' in self.machine_params.keys():
+            cpus_remapped = super().remap_cpus(self.machine_params['latcores'])
+            RapidLog.debug('{} ({}): latcores {} remapped to {}'.format(self.name, self.ip, self.machine_params['latcores'], cpus_remapped))
+            self.machine_params['latcores'] = cpus_remapped
+
     def generate_lua(self, vim, prox_config_file):
         appendix = 'gencores="%s"\n'% ','.join(map(str,
             self.machine_params['gencores']))
@@ -109,6 +127,8 @@ def start_prox(self):
         # Start the generator with the -e option so that the cores don't
         # start automatically
         super().start_prox('-e')
+        if self.vim in ['kubernetes']:
+            self.remap_all_cpus()
 
     def set_generator_speed(self, speed):
         # The assumption is that we only use task 0 for generating

diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_machine.py b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_machine.py
@@ -49,6 +49,7 @@ def __init__(self, key, user, vim, rundir, resultsdir, machine_params,
                 break
         self.machine_params = machine_params
         self.vim = vim
+        self.cpu_mapping = None
 
     def __del__(self):
         if ((not self.configonly) and self.machine_params['prox_socket']):
@@ -58,6 +59,62 @@ def __del__(self):
     def get_cores(self):
         return (self.machine_params['cores'])
 
+    def expand_cpuset(self, cpuset):
+        """Expand cpuset provided as comma-separated list of CPU numbers and
+        CPU ranges of numbers. For more information please see
+        https://man7.org/linux/man-pages/man7/cpuset.7.html
+        """
+        cpuset_expanded = []
+        for cpu in cpuset.split(','):
+            if '-' in cpu:
+                cpu_range = cpu.split('-')
+                cpuset_expanded += range(int(cpu_range[0]), int(cpu_range[1]) + 1)
+            else:
+                cpuset_expanded.append(int(cpu))
+        return cpuset_expanded
+
+    def read_cpuset(self):
+        """Read list of cpus on which we allowed to execute
+        """
+        cmd = 'cat /sys/fs/cgroup/cpuset/cpuset.cpus'
+        cpuset_cpus = self._client.run_cmd(cmd).decode().rstrip()
+        RapidLog.debug('{} ({}): Allocated cpuset: {}'.format(self.name, self.ip, cpuset_cpus))
+        self.cpu_mapping = self.expand_cpuset(cpuset_cpus)
+        RapidLog.debug('{} ({}): Expanded cpuset: {}'.format(self.name, self.ip, self.cpu_mapping))
+
+        # Log CPU core mapping for user information
+        cpu_mapping_str = ''
+        for i in range(len(self.cpu_mapping)):
+            cpu_mapping_str = cpu_mapping_str + '[' + str(i) + '->' + str(self.cpu_mapping[i]) + '], '
+        cpu_mapping_str = cpu_mapping_str[:-2]
+        RapidLog.debug('{} ({}): CPU mapping: {}'.format(self.name, self.ip, cpu_mapping_str))
+
+    def remap_cpus(self, cpus):
+        """Convert relative cpu ids provided as function parameter to match
+        cpu ids from allocated list
+        """
+        cpus_remapped = []
+        for cpu in cpus:
+            cpus_remapped.append(self.cpu_mapping[cpu])
+        return cpus_remapped
+
+    def remap_all_cpus(self):
+        """Convert relative cpu ids for different parameters (mcore, cores)
+        """
+        if self.cpu_mapping is None:
+            RapidLog.debug('{} ({}): cpu mapping is not defined! Please check the configuration!'.format(self.name, self.ip))
+            return
+
+        if 'mcore' in self.machine_params.keys():
+            cpus_remapped = self.remap_cpus(self.machine_params['mcore'])
+            RapidLog.debug('{} ({}): mcore {} remapped to {}'.format(self.name, self.ip, self.machine_params['mcore'], cpus_remapped))
+            self.machine_params['mcore'] = cpus_remapped
+
+        if 'cores' in self.machine_params.keys():
+            cpus_remapped = self.remap_cpus(self.machine_params['cores'])
+            RapidLog.debug('{} ({}): cores {} remapped to {}'.format(self.name, self.ip, self.machine_params['cores'], cpus_remapped))
+            self.machine_params['cores'] = cpus_remapped
+
     def devbind(self):
         # Script to bind the right network interface to the poll mode driver
         for index, dp_port in enumerate(self.dp_ports, start = 1):
@@ -86,6 +143,8 @@ def generate_lua(self, vim, prox_config_file, appendix = ''):
                 LuaFile.write("eal=\"--socket-mem=512,0 --file-prefix %s --pci-whitelist %s\"\n" % (self.name, self.machine_params['dp_pci_dev']))
             else:
                 LuaFile.write("eal=\"\"\n")
+            if 'mcore' in self.machine_params.keys():
+                LuaFile.write('mcore="%s"\n'% ','.join(map(str, self.machine_params['mcore'])))
             if 'cores' in self.machine_params.keys():
                 LuaFile.write('cores="%s"\n'% ','.join(map(str, self.machine_params['cores'])))
             if 'ports' in self.machine_params.keys():
@@ -105,6 +164,9 @@ def start_prox(self, autostart=''):
             self._client.connect()
             if self.vim in ['OpenStack']:
                 self.devbind()
+            if self.vim in ['kubernetes']:
+                self.read_cpuset()
+                self.remap_all_cpus()
             _, prox_config_file_name = os.path.split(self.machine_params['config_file'])
             self.generate_lua(self.vim, self.machine_params['config_file'])
             self._client.scp_put(self.machine_params['config_file'], '{}/{}'.format(self.rundir, prox_config_file_name))

diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_parser.py b/VNFs/DPPD-PROX/helper-scripts/rapid/rapid_parser.py
@@ -120,7 +120,7 @@ def parse_config(test_params):
             for option in options:
                 if option in ['prox_socket','prox_launch_exit','monitor']:
                     machine[option] = testconfig.getboolean(section, option)
-                elif option in ['cores', 'gencores','latcores']:
+                elif option in ['mcore', 'cores', 'gencores','latcores']:
                     machine[option] = ast.literal_eval(testconfig.get(
                         section, option))
                 elif option in ['bucket_size_exp']:

diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/tests/TST009_Throughput.test b/VNFs/DPPD-PROX/helper-scripts/rapid/tests/TST009_Throughput.test
@@ -24,13 +24,15 @@ lat_percentile = 99
 name = Generator
 config_file = configs/gen.cfg
 dest_vm = 2
+mcore = [0]
 gencores = [1]
 latcores = [3]
 #bucket_size_exp = 12
 
 [TestM2]
 name = Swap
 config_file = configs/swap.cfg
+mcore = [0]
 cores = [1]
 
 [test1]

diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/tests/TST009_Throughput_64B_64F.test b/VNFs/DPPD-PROX/helper-scripts/rapid/tests/TST009_Throughput_64B_64F.test
@@ -24,13 +24,15 @@ lat_percentile = 99
 name = Generator
 config_file = configs/gen.cfg
 dest_vm = 2
+mcore = [0]
 gencores = [1]
 latcores = [3]
 #bucket_size_exp = 12
 
 [TestM2]
 name = Swap
 config_file = configs/swap.cfg
+mcore = [0]
 cores = [1]
 
 [test1]

diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/tests/TST009_Throughput_acaeab_16384F.test b/VNFs/DPPD-PROX/helper-scripts/rapid/tests/TST009_Throughput_acaeab_16384F.test
@@ -24,13 +24,15 @@ lat_percentile = 99
 name = Generator
 config_file = configs/gen.cfg
 dest_vm = 2
+mcore = [0]
 gencores = [1]
 latcores = [3]
 #bucket_size_exp = 12
 
 [TestM2]
 name = Swap
 config_file = configs/swap.cfg
+mcore = [0]
 cores = [1]
 
 [test1]

diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/tests/TST009ipV6.test b/VNFs/DPPD-PROX/helper-scripts/rapid/tests/TST009ipV6.test
@@ -24,13 +24,15 @@ lat_percentile = 99
 name = Generator
 config_file = configs/genv6.cfg
 dest_vm = 2
+mcore = [0]
 gencores = [1]
 latcores = [3]
 #bucket_size_exp = 12
 
 [TestM2]
 name = Swap
 config_file = configs/swapv6.cfg
+mcore = [0]
 cores = [1]
 #prox_socket = true
 #prox_launch_exit = true

diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/tests/bare.test b/VNFs/DPPD-PROX/helper-scripts/rapid/tests/bare.test
@@ -23,12 +23,14 @@ total_number_of_test_machines = 2
 name = Generator
 config_file = configs/l2gen_bare.cfg
 dest_vm = 2
+mcore = [0]
 gencores = [1]
 latcores = [3]
 
 [TestM2]
 name = Swap
 config_file = configs/l2swap.cfg
+mcore = [0]
 cores = [1]
 
 [test1]

diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/tests/basicrapid.test b/VNFs/DPPD-PROX/helper-scripts/rapid/tests/basicrapid.test
@@ -24,13 +24,15 @@ lat_percentile = 99
 name = Generator
 config_file = configs/gen.cfg
 dest_vm = 2
+mcore = [0]
 gencores = [1]
 latcores = [3]
 #bucket_size_exp = 12
 
 [TestM2]
 name = Swap
 config_file = configs/swap.cfg
+mcore = [0]
 cores = [1]
 #prox_socket = true
 #prox_launch_exit = true

diff --git a/VNFs/DPPD-PROX/helper-scripts/rapid/tests/corestats.test b/VNFs/DPPD-PROX/helper-scripts/rapid/tests/corestats.test
@@ -22,6 +22,7 @@ total_number_of_test_machines = 1
 [TestM1]
 name = Swap
 config_file = configs/swap.cfg
+mcore = [0]
 cores = [1]
 
 [test1]