Merge pull request kubernetes#1470 from vamossagar12/perf-tests-1458

Migrate dns benchmarks from python2 to python3

clusterloader2/pkg/prometheus/manifests/dashboards/__init__.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # Copyright 2019 The Kubernetes Authors.
 #

clusterloader2/pkg/prometheus/manifests/dashboards/defaults.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # Copyright 2019 The Kubernetes Authors.
 #

clusterloader2/pkg/prometheus/manifests/dashboards/dns.dashboard.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # Copyright 2019 The Kubernetes Authors.
 #

clusterloader2/pkg/prometheus/manifests/dashboards/master-dashboard.dashboard.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # Copyright 2019 The Kubernetes Authors.
 #

clusterloader2/pkg/prometheus/manifests/dashboards/network.dashboard.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # Copyright 2019 The Kubernetes Authors.
 #

clusterloader2/pkg/prometheus/manifests/dashboards/slo.dashboard.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # Copyright 2019 The Kubernetes Authors.
 #

dns/py/data.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # Copyright 2016 The Kubernetes Authors.
 #
@@ -70,7 +70,7 @@ class Parser(object):
   Parses dnsperf output file.
   """
   def __init__(self, out):
-    self.lines = [x.strip() for x in out.split('\n')]
+    self.lines = [x.strip() for x in out.decode().split('\n')]
     self.results = {}
     self.histogram = []
 
@@ -192,19 +192,19 @@ def put(self, results, ignore_if_dup=True):
           'rtt_ms_count': count,
       }
 
-      columns = ','.join(data.keys())
+      columns = ','.join(list(data.keys()))
       qs = ','.join(['?'] * len(data))
       stmt = 'INSERT INTO histograms (' + columns + ') VALUES (' + qs + ')'
       _log.debug('histogram sql -- %s', stmt)
-      self.c.execute(stmt, data.values())
+      self.c.execute(stmt, list(data.values()))
 
   def get_results(self, run_id, run_subid):
     sql = ('SELECT ' + ','.join([r.name for r in RESULTS])
            + ' FROM results WHERE run_id = ? and run_subid = ? and pod_name = ?')
     _log.debug('%s', sql)
     self.c.execute(sql, (run_id, run_subid, pod_name))
     rows = self.c.fetchall()
-    return dict(zip([r.name for r in RESULTS], rows[0])) if rows else None
+    return dict(list(zip([r.name for r in RESULTS], rows[0]))) if rows else None
 
   def _exists(self, key):
     self.c.execute(

dns/py/data_test.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # Copyright 2016 The Kubernetes Authors.
 #
@@ -29,14 +29,14 @@ def test_parser(self):
     parser = Parser(raw)
     parser.parse()
     # Check a subset of fields
-    self.assertEquals(1000, parser.results['queries_sent'])
+    self.assertEqual(1000, parser.results['queries_sent'])
 
   def test_result_db(self):
     results = yaml.load(open('fixtures/results.yaml'))
 
     self.db.put(results)
     res = self.db.get_results(1234, 0)
-    self.assertEquals(727354, res['queries_sent'])
+    self.assertEqual(727354, res['queries_sent'])
     self.db.put(results) # dup
 
     self.assertRaises(

dns/py/ingest.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # Copyright 2016 The Kubernetes Authors.
 #
@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import print_function
+
 
 
 import argparse

dns/py/params.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # Copyright 2016 The Kubernetes Authors.
 #
@@ -22,7 +22,7 @@
 variable.
 """
 
-from __future__ import print_function
+
 
 import copy
 import logging
@@ -160,7 +160,7 @@ def is_relevant(self, attributes):
   def set(self, inputs, value):
     if value > 0:
       cf = inputs.configmap_yaml['data']['Corefile']
-      cfList = cf.split("\n")
+      cfList = cf.decode().split("\n")
       cfList.insert(1,
                     "  cache {\n"
                     "    success " + repr(value) + "\n"

dns/py/params_test.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # Copyright 2016 The Kubernetes Authors.
 #
@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import print_function
+
 
 import unittest
 import yaml
@@ -93,19 +93,19 @@ def test_params(self):
         continue
       param.set(inputs, values[param.name])
 
-    self.assertEquals(
+    self.assertEqual(
         '100m',
         inputs.deployment_yaml['spec']['template']['spec']['containers']\
             [1]['resources']['limits']['cpu'])
     self.assertTrue(
         '--cache-size=200' in
         inputs.deployment_yaml['spec']['template']['spec']['containers']\
             [1]['args'])
-    self.assertEquals(
+    self.assertEqual(
         '300m',
         inputs.deployment_yaml['spec']['template']['spec']['containers']\
             [0]['resources']['limits']['cpu'])
-    self.assertEquals(
+    self.assertEqual(
         '-l,120,-Q,400,-d,/queries/abc',
         ','.join(inputs.dnsperf_cmdline))
 
@@ -127,7 +127,7 @@ def test_coredns_params(self):
                     in inputs.configmap_yaml['data']['Corefile'])
     self.assertTrue("denial 200"
                     in inputs.configmap_yaml['data']['Corefile'])
-    self.assertEquals(
+    self.assertEqual(
         '100m',
         inputs.deployment_yaml['spec']['template']['spec']['containers']
         [0]['resources']['limits']['cpu'])
@@ -155,7 +155,7 @@ def test_null_params(self):
     self.assertTrue(
         'cpu' not in inputs.deployment_yaml\
         ['spec']['template']['spec']['containers'][1]['resources']['limits'])
-    self.assertEquals(
+    self.assertEqual(
         '-l,120,-d,/queries/abc',
         ','.join(inputs.dnsperf_cmdline))
 
@@ -166,24 +166,24 @@ def test_TestCases(self):
         'query_file': ['a', 'b'],
         })
     tc = tp.generate(set())
-    self.assertEquals(4, len(tc))
+    self.assertEqual(4, len(tc))
 
-    self.assertEquals(0, tc[0].run_subid)
-    self.assertEquals(
+    self.assertEqual(0, tc[0].run_subid)
+    self.assertEqual(
         "[(<dnsmasq_cpu>, 200), (<kubedns_cpu>, 100), (<query_file>, 'a')]",
         str(tc[0].pv))
-    self.assertEquals(
+    self.assertEqual(
         "[(<dnsmasq_cpu>, 200), (<kubedns_cpu>, 100), (<query_file>, 'b')]",
         str(tc[1].pv))
-    self.assertEquals(1, tc[1].run_subid)
-    self.assertEquals(
+    self.assertEqual(1, tc[1].run_subid)
+    self.assertEqual(
         "[(<dnsmasq_cpu>, 300), (<kubedns_cpu>, 100), (<query_file>, 'a')]",
         str(tc[2].pv))
-    self.assertEquals(2, tc[2].run_subid)
-    self.assertEquals(
+    self.assertEqual(2, tc[2].run_subid)
+    self.assertEqual(
         "[(<dnsmasq_cpu>, 300), (<kubedns_cpu>, 100), (<query_file>, 'b')]",
         str(tc[3].pv))
-    self.assertEquals(3, tc[3].run_subid)
+    self.assertEqual(3, tc[3].run_subid)
 
   def test_TestCases_attributes(self):
     tp = TestCases({
@@ -192,13 +192,13 @@ def test_TestCases_attributes(self):
         'query_file': ['a', 'b'],
         })
     tc = tp.generate(set([ATTRIBUTE_CLUSTER_DNS]))
-    self.assertEquals(2, len(tc))
+    self.assertEqual(2, len(tc))
 
-    self.assertEquals(0, tc[0].run_subid)
-    self.assertEquals(
+    self.assertEqual(0, tc[0].run_subid)
+    self.assertEqual(
         "[(<query_file>, 'a')]",
         str(tc[0].pv))
-    self.assertEquals(
+    self.assertEqual(
         "[(<query_file>, 'b')]",
         str(tc[1].pv))
-    self.assertEquals(1, tc[1].run_subid)
+    self.assertEqual(1, tc[1].run_subid)

dns/py/run_perf.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # Copyright 2016 The Kubernetes Authors.
 #
@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import print_function
+
 
 import argparse
 import logging

dns/py/runner.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # Copyright 2016 The Kubernetes Authors.
 #
@@ -23,7 +23,7 @@
 import yaml
 import threading
 import re
-import Queue
+import queue
 from subprocess import PIPE
 
 from data import Parser, ResultDb
@@ -41,7 +41,7 @@
 MAX_TEST_SVC = 20
 
 def add_prefix(prefix, text):
-  return '\n'.join([prefix + l for l in text.split('\n')])
+  return '\n'.join([prefix + l for l in text.decode().split('\n')])
 
 
 class Runner(object):
@@ -232,7 +232,7 @@ def _run_perf(self, test_case, inputs, podname):
     output_file = '%s/run-%s/result-%s-%s.out' % \
       (self.args.out_dir, test_case.run_id, test_case.run_subid, test_case.pod_name)
     _log.info('Writing to output file %s', output_file)
-    res_usage = Queue.Queue()
+    res_usage = queue.Queue()
     dt = threading.Thread(target=self._run_top,args=[res_usage])
     dt.start()
     header = '''### run_id {run_id}:{run_subid}
@@ -260,8 +260,8 @@ def _run_perf(self, test_case, inputs, podname):
       results = {}
       results['params'] = test_case.to_yaml()
       results['code'] = code
-      results['stdout'] = out.split('\n')
-      results['stderr'] = err.split('\n')
+      results['stdout'] = out.decode().split('\n')
+      results['stderr'] = err.decode().split('\n')
       results['data'] = {}
 
       try:
@@ -273,7 +273,7 @@ def _run_perf(self, test_case, inputs, podname):
         results['data']['ok'] = True
         results['data']['msg'] = None
 
-        for key, value in parser.results.items():
+        for key, value in list(parser.results.items()):
           results['data'][key] = value
         results['data']['max_perfserver_cpu'] = res_usage.get()
         results['data']['max_perfserver_memory'] = res_usage.get()
@@ -370,7 +370,7 @@ def _reset_client(self):
       if code != 0:
         _log.error('Error: stderr\n%s', add_prefix('err | ', err))
         raise Exception('error getting pod information: %d', code)
-      client_pods=client_pods.rstrip().split('\n')
+      client_pods=client_pods.rstrip().decode().split('\n')
       _log.info('got client pods "%s"', client_pods)
       if len(client_pods) > 0:
         break
@@ -392,7 +392,7 @@ def _copy_query_files(self, podname):
       try:
         _log.info('Downloading large query file')
         subprocess.check_call(['wget', _dnsperf_qfile_path])
-        subprocess.check_call(['gunzip', _dnsperf_qfile_path.split('/')[-1]])
+        subprocess.check_call(['gunzip', _dnsperf_qfile_path.decode().split('/')[-1]])
         _log.info('Removing hostnames matching specified patterns')
         for pattern in _remove_query_pattern:
           subprocess.check_call(['sed', '-i', '-e', '/%s/d' %(pattern), _dnsperf_qfile_name])

dns/run

@@ -22,17 +22,17 @@ outDir=${2:-"out"}
 nodeLocalIP=${4:-"169.254.20.10"}
 mkdir -p ${outDir}
 
-pip2 install numpy
+pip3 install numpy
 
 case "$1" in
   kube-dns )
-    python2 py/run_perf.py --params params/kubedns/default.yaml --out-dir ${outDir} --use-cluster-dns
+    python3 py/run_perf.py --params params/kubedns/default.yaml --out-dir ${outDir} --use-cluster-dns
     ;;
   core-dns )
-    python2 py/run_perf.py --params params/coredns/default.yaml --out-dir ${outDir} --use-cluster-dns
+    python3 py/run_perf.py --params params/coredns/default.yaml --out-dir ${outDir} --use-cluster-dns
     ;;
   node-local-dns )
-    python2 py/run_perf.py --params params/nodelocaldns/default.yaml --out-dir ${outDir} --nodecache-ip ${nodeLocalIP}
+    python3 py/run_perf.py --params params/nodelocaldns/default.yaml --out-dir ${outDir} --nodecache-ip ${nodeLocalIP}
     ;;
 esac
 

verify/boilerplate/boilerplate.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # Copyright 2015 The Kubernetes Authors.
 #

verify/boilerplate/boilerplate.py.txt

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # Copyright YEAR The Kubernetes Authors.
 #

verify/verify-flags-underscore.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # Copyright 2015 The Kubernetes Authors.
 #