forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2023-12-11 6778948f9de86c3cfaf36725a7c87dcff9ba247f 
 kernel/tools/testing/selftests/tc-testing/tdc.py
....@@ -23,7 +23,11 @@
2323 from tdc_helper import *
2424 
2525 import TdcPlugin
26
+from TdcResults import *
2627 
28
+class PluginDependencyException(Exception):
29
+    def __init__(self, missing_pg):
30
+        self.missing_pg = missing_pg
2731 
2832 class PluginMgrTestFail(Exception):
2933     def __init__(self, stage, output, message):
....@@ -36,7 +40,7 @@
3640         super().__init__()
3741         self.plugins = {}
3842         self.plugin_instances = []
39
-        self.args = []
43
+        self.failed_plugins = {}
4044         self.argparser = argparser
4145 
4246         # TODO, put plugins in order
....@@ -52,6 +56,64 @@
5256                     self.plugins[mn] = foo
5357                     self.plugin_instances.append(foo.SubPlugin())
5458 
59
+    def load_plugin(self, pgdir, pgname):
60
+        pgname = pgname[0:-3]
61
+        foo = importlib.import_module('{}.{}'.format(pgdir, pgname))
62
+        self.plugins[pgname] = foo
63
+        self.plugin_instances.append(foo.SubPlugin())
64
+        self.plugin_instances[-1].check_args(self.args, None)
65
+
66
+    def get_required_plugins(self, testlist):
67
+        '''
68
+        Get all required plugins from the list of test cases and return
69
+        all unique items.
70
+        '''
71
+        reqs = []
72
+        for t in testlist:
73
+            try:
74
+                if 'requires' in t['plugins']:
75
+                    if isinstance(t['plugins']['requires'], list):
76
+                        reqs.extend(t['plugins']['requires'])
77
+                    else:
78
+                        reqs.append(t['plugins']['requires'])
79
+            except KeyError:
80
+                continue
81
+        reqs = get_unique_item(reqs)
82
+        return reqs
83
+
84
+    def load_required_plugins(self, reqs, parser, args, remaining):
85
+        '''
86
+        Get all required plugins from the list of test cases and load any plugin
87
+        that is not already enabled.
88
+        '''
89
+        pgd = ['plugin-lib', 'plugin-lib-custom']
90
+        pnf = []
91
+
92
+        for r in reqs:
93
+            if r not in self.plugins:
94
+                fname = '{}.py'.format(r)
95
+                source_path = []
96
+                for d in pgd:
97
+                    pgpath = '{}/{}'.format(d, fname)
98
+                    if os.path.isfile(pgpath):
99
+                        source_path.append(pgpath)
100
+                if len(source_path) == 0:
101
+                    print('ERROR: unable to find required plugin {}'.format(r))
102
+                    pnf.append(fname)
103
+                    continue
104
+                elif len(source_path) > 1:
105
+                    print('WARNING: multiple copies of plugin {} found, using version found')
106
+                    print('at {}'.format(source_path[0]))
107
+                pgdir = source_path[0]
108
+                pgdir = pgdir.split('/')[0]
109
+                self.load_plugin(pgdir, fname)
110
+        if len(pnf) > 0:
111
+            raise PluginDependencyException(pnf)
112
+
113
+        parser = self.call_add_args(parser)
114
+        (args, remaining) = parser.parse_known_args(args=remaining, namespace=args)
115
+        return args
116
+
55117     def call_pre_suite(self, testcount, testidlist):
56118         for pgn_inst in self.plugin_instances:
57119             pgn_inst.pre_suite(testcount, testidlist)
....@@ -60,15 +122,15 @@
60122         for pgn_inst in reversed(self.plugin_instances):
61123             pgn_inst.post_suite(index)
62124 
63
-    def call_pre_case(self, test_ordinal, testid):
125
+    def call_pre_case(self, caseinfo, *, test_skip=False):
64126         for pgn_inst in self.plugin_instances:
65127             try:
66
-                pgn_inst.pre_case(test_ordinal, testid)
128
+                pgn_inst.pre_case(caseinfo, test_skip)
67129             except Exception as ee:
68130                 print('exception {} in call to pre_case for {} plugin'.
69131                       format(ee, pgn_inst.__class__))
70132                 print('test_ordinal is {}'.format(test_ordinal))
71
-                print('testid is {}'.format(testid))
133
+                print('testid is {}'.format(caseinfo['id']))
72134                 raise
73135 
74136     def call_post_case(self):
....@@ -97,11 +159,13 @@
97159             command = pgn_inst.adjust_command(stage, command)
98160         return command
99161 
162
+    def set_args(self, args):
163
+        self.args = args
164
+
100165     @staticmethod
101166     def _make_argparser(args):
102167         self.argparser = argparse.ArgumentParser(
103168             description='Linux TC unit tests')
104
-
105169 
106170 def replace_keywords(cmd):
107171     """
....@@ -131,12 +195,16 @@
131195         stdout=subprocess.PIPE,
132196         stderr=subprocess.PIPE,
133197         env=ENVIR)
134
-    (rawout, serr) = proc.communicate()
135198 
136
-    if proc.returncode != 0 and len(serr) > 0:
137
-        foutput = serr.decode("utf-8", errors="ignore")
138
-    else:
139
-        foutput = rawout.decode("utf-8", errors="ignore")
199
+    try:
200
+        (rawout, serr) = proc.communicate(timeout=NAMES['TIMEOUT'])
201
+        if proc.returncode != 0 and len(serr) > 0:
202
+            foutput = serr.decode("utf-8", errors="ignore")
203
+        else:
204
+            foutput = rawout.decode("utf-8", errors="ignore")
205
+    except subprocess.TimeoutExpired:
206
+        foutput = "Command \"{}\" timed out\n".format(command)
207
+        proc.returncode = 255
140208 
141209     proc.stdout.close()
142210     proc.stderr.close()
....@@ -183,14 +251,24 @@
183251     result = True
184252     tresult = ""
185253     tap = ""
254
+    res = TestResult(tidx['id'], tidx['name'])
186255     if args.verbose > 0:
187256         print("\t====================\n=====> ", end="")
188257     print("Test " + tidx["id"] + ": " + tidx["name"])
189258 
259
+    if 'skip' in tidx:
260
+        if tidx['skip'] == 'yes':
261
+            res = TestResult(tidx['id'], tidx['name'])
262
+            res.set_result(ResultState.skip)
263
+            res.set_errormsg('Test case designated as skipped.')
264
+            pm.call_pre_case(tidx, test_skip=True)
265
+            pm.call_post_execute()
266
+            return res
267
+
190268     # populate NAMES with TESTID for this test
191269     NAMES['TESTID'] = tidx['id']
192270 
193
-    pm.call_pre_case(index, tidx['id'])
271
+    pm.call_pre_case(tidx)
194272     prepare_env(args, pm, 'setup', "-----> prepare stage", tidx["setup"])
195273 
196274     if (args.verbose > 0):
....@@ -205,10 +283,11 @@
205283     pm.call_post_execute()
206284 
207285     if (exit_code is None or exit_code != int(tidx["expExitCode"])):
208
-        result = False
209286         print("exit: {!r}".format(exit_code))
210287         print("exit: {}".format(int(tidx["expExitCode"])))
211288         #print("exit: {!r} {}".format(exit_code, int(tidx["expExitCode"])))
289
+        res.set_result(ResultState.fail)
290
+        res.set_failmsg('Command exited with {}, expected {}\n{}'.format(exit_code, tidx["expExitCode"], procout))
212291         print(procout)
213292     else:
214293         if args.verbose > 0:
....@@ -219,20 +298,15 @@
219298         if procout:
220299             match_index = re.findall(match_pattern, procout)
221300             if len(match_index) != int(tidx["matchCount"]):
222
-                result = False
301
+                res.set_result(ResultState.fail)
302
+                res.set_failmsg('Could not match regex pattern. Verify command output:\n{}'.format(procout))
303
+            else:
304
+                res.set_result(ResultState.success)
223305         elif int(tidx["matchCount"]) != 0:
224
-            result = False
225
-
226
-    if not result:
227
-        tresult += 'not '
228
-    tresult += 'ok {} - {} # {}\n'.format(str(index), tidx['id'], tidx['name'])
229
-    tap += tresult
230
-
231
-    if result == False:
232
-        if procout:
233
-            tap += procout
306
+            res.set_result(ResultState.fail)
307
+            res.set_failmsg('No output generated by verify command.')
234308         else:
235
-            tap += 'No output!\n'
309
+            res.set_result(ResultState.success)
236310 
237311     prepare_env(args, pm, 'teardown', '-----> teardown stage', tidx['teardown'], procout)
238312     pm.call_post_case()
....@@ -241,7 +315,7 @@
241315 
242316     # remove TESTID from NAMES
243317     del(NAMES['TESTID'])
244
-    return tap
318
+    return res
245319 
246320 def test_runner(pm, args, filtered_tests):
247321     """
....@@ -261,25 +335,15 @@
261335     emergency_exit = False
262336     emergency_exit_message = ''
263337 
264
-    if args.notap:
265
-        if args.verbose:
266
-            tap = 'notap requested:  omitting test plan\n'
267
-    else:
268
-        tap = str(index) + ".." + str(tcount) + "\n"
338
+    tsr = TestSuiteReport()
339
+
269340     try:
270341         pm.call_pre_suite(tcount, [tidx['id'] for tidx in testlist])
271342     except Exception as ee:
272343         ex_type, ex, ex_tb = sys.exc_info()
273344         print('Exception {} {} (caught in pre_suite).'.
274345               format(ex_type, ex))
275
-        # when the extra print statements are uncommented,
276
-        # the traceback does not appear between them
277
-        # (it appears way earlier in the tdc.py output)
278
-        # so don't bother ...
279
-        # print('--------------------(')
280
-        # print('traceback')
281346         traceback.print_tb(ex_tb)
282
-        # print('--------------------)')
283347         emergency_exit_message = 'EMERGENCY EXIT, call_pre_suite failed with exception {} {}\n'.format(ex_type, ex)
284348         emergency_exit = True
285349         stage = 'pre-SUITE'
....@@ -292,18 +356,31 @@
292356     time.sleep(2)
293357     for tidx in testlist:
294358         if "flower" in tidx["category"] and args.device == None:
359
+            errmsg = "Tests using the DEV2 variable must define the name of a "
360
+            errmsg += "physical NIC with the -d option when running tdc.\n"
361
+            errmsg += "Test has been skipped."
295362             if args.verbose > 1:
296
-                print('Not executing test {} {} because DEV2 not defined'.
297
-                      format(tidx['id'], tidx['name']))
363
+                print(errmsg)
364
+            res = TestResult(tidx['id'], tidx['name'])
365
+            res.set_result(ResultState.skip)
366
+            res.set_errormsg(errmsg)
367
+            tsr.add_resultdata(res)
298368             continue
299369         try:
300370             badtest = tidx  # in case it goes bad
301
-            tap += run_one_test(pm, args, index, tidx)
371
+            res = run_one_test(pm, args, index, tidx)
372
+            tsr.add_resultdata(res)
302373         except PluginMgrTestFail as pmtf:
303374             ex_type, ex, ex_tb = sys.exc_info()
304375             stage = pmtf.stage
305376             message = pmtf.message
306377             output = pmtf.output
378
+            res = TestResult(tidx['id'], tidx['name'])
379
+            res.set_result(ResultState.skip)
380
+            res.set_errormsg(pmtf.message)
381
+            res.set_failmsg(pmtf.output)
382
+            tsr.add_resultdata(res)
383
+            index += 1
307384             print(message)
308385             print('Exception {} {} (caught in test_runner, running test {} {} {} stage {})'.
309386                   format(ex_type, ex, index, tidx['id'], tidx['name'], stage))
....@@ -322,16 +399,16 @@
322399     # if we failed in setup or teardown,
323400     # fill in the remaining tests with ok-skipped
324401     count = index
325
-    if not args.notap:
326
-        tap += 'about to flush the tap output if tests need to be skipped\n'
327
-        if tcount + 1 != index:
328
-            for tidx in testlist[index - 1:]:
329
-                msg = 'skipped - previous {} failed'.format(stage)
330
-                tap += 'ok {} - {} # {} {} {}\n'.format(
331
-                    count, tidx['id'], msg, index, badtest.get('id', '--Unknown--'))
332
-                count += 1
333402 
334
-        tap += 'done flushing skipped test tap output\n'
403
+    if tcount + 1 != count:
404
+        for tidx in testlist[count - 1:]:
405
+            res = TestResult(tidx['id'], tidx['name'])
406
+            res.set_result(ResultState.skip)
407
+            msg = 'skipped - previous {} failed {} {}'.format(stage,
408
+                index, badtest.get('id', '--Unknown--'))
409
+            res.set_errormsg(msg)
410
+            tsr.add_resultdata(res)
411
+            count += 1
335412 
336413     if args.pause:
337414         print('Want to pause\nPress enter to continue ...')
....@@ -340,7 +417,7 @@
340417 
341418     pm.call_post_suite(index)
342419 
343
-    return tap
420
+    return tsr
344421 
345422 def has_blank_ids(idlist):
346423     """
....@@ -381,6 +458,10 @@
381458     Set the command line arguments for tdc.
382459     """
383460     parser.add_argument(
461
+        '--outfile', type=str,
462
+        help='Path to the file in which results should be saved. ' +
463
+        'Default target is the current directory.')
464
+    parser.add_argument(
384465         '-p', '--path', type=str,
385466         help='The full path to the tc executable to use')
386467     sg = parser.add_argument_group(
....@@ -416,10 +497,13 @@
416497         '-v', '--verbose', action='count', default=0,
417498         help='Show the commands that are being run')
418499     parser.add_argument(
419
-        '-N', '--notap', action='store_true',
420
-        help='Suppress tap results for command under test')
500
+        '--format', default='tap', const='tap', nargs='?',
501
+        choices=['none', 'xunit', 'tap'],
502
+        help='Specify the format for test results. (Default: TAP)')
421503     parser.add_argument('-d', '--device',
422
-                        help='Execute the test case in flower category')
504
+                        help='Execute test cases that use a physical device, ' +
505
+                        'where DEVICE is its name. (If not defined, tests ' +
506
+                        'that require a physical device will be skipped)')
423507     parser.add_argument(
424508         '-P', '--pause', action='store_true',
425509         help='Pause execution just before post-suite stage')
....@@ -438,6 +522,8 @@
438522         NAMES['TC'] = args.path
439523     if args.device != None:
440524         NAMES['DEV2'] = args.device
525
+    if 'TIMEOUT' not in NAMES:
526
+        NAMES['TIMEOUT'] = None
441527     if not os.path.isfile(NAMES['TC']):
442528         print("The specified tc path " + NAMES['TC'] + " does not exist.")
443529         exit(1)
....@@ -532,6 +618,7 @@
532618 
533619     return answer
534620 
621
+
535622 def get_test_cases(args):
536623     """
537624     If a test case file is specified, retrieve tests from that file.
....@@ -593,7 +680,7 @@
593680     return allcatlist, allidlist, testcases_by_cats, alltestcases
594681 
595682 
596
-def set_operation_mode(pm, args):
683
+def set_operation_mode(pm, parser, args, remaining):
597684     """
598685     Load the test case data and process remaining arguments to determine
599686     what the script should do for this run, and call the appropriate
....@@ -626,18 +713,41 @@
626713         exit(0)
627714 
628715     if args.list:
629
-        if args.list:
630
-            list_test_cases(alltests)
631
-            exit(0)
716
+        list_test_cases(alltests)
717
+        exit(0)
632718 
633719     if len(alltests):
720
+        req_plugins = pm.get_required_plugins(alltests)
721
+        try:
722
+            args = pm.load_required_plugins(req_plugins, parser, args, remaining)
723
+        except PluginDependencyException as pde:
724
+            print('The following plugins were not found:')
725
+            print('{}'.format(pde.missing_pg))
634726         catresults = test_runner(pm, args, alltests)
727
+        if args.format == 'none':
728
+            print('Test results output suppression requested\n')
729
+        else:
730
+            print('\nAll test results: \n')
731
+            if args.format == 'xunit':
732
+                suffix = 'xml'
733
+                res = catresults.format_xunit()
734
+            elif args.format == 'tap':
735
+                suffix = 'tap'
736
+                res = catresults.format_tap()
737
+            print(res)
738
+            print('\n\n')
739
+            if not args.outfile:
740
+                fname = 'test-results.{}'.format(suffix)
741
+            else:
742
+                fname = args.outfile
743
+            with open(fname, 'w') as fh:
744
+                fh.write(res)
745
+                fh.close()
746
+                if os.getenv('SUDO_UID') is not None:
747
+                    os.chown(fname, uid=int(os.getenv('SUDO_UID')),
748
+                        gid=int(os.getenv('SUDO_GID')))
635749     else:
636
-        catresults = 'No tests found\n'
637
-    if args.notap:
638
-        print('Tap output suppression requested\n')
639
-    else:
640
-        print('All test results: \n\n{}'.format(catresults))
750
+        print('No tests found\n')
641751 
642752 def main():
643753     """
....@@ -650,11 +760,12 @@
650760     parser = pm.call_add_args(parser)
651761     (args, remaining) = parser.parse_known_args()
652762     args.NAMES = NAMES
763
+    pm.set_args(args)
653764     check_default_settings(args, remaining, pm)
654765     if args.verbose > 2:
655766         print('args is {}'.format(args))
656767 
657
-    set_operation_mode(pm, args)
768
+    set_operation_mode(pm, parser, args, remaining)
658769 
659770     exit(0)
660771