Skip to content

Commit 578a715

Browse files
torvaldstorvalds
committed
Merge tag 'linux-kselftest-kunit-fixes-5.10-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest
Pull Kunit updates from Shuah Khan: "Several kunit tool bug fixes in flag handling, run outside kernel tree, make errors, and generating results" * tag 'linux-kselftest-kunit-fixes-5.10-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest: kunit: tool: fix display of make errors kunit: tool: handle when .kunit exists but .kunitconfig does not kunit: tool: fix --alltests flag kunit: tool: allow generating test results in JSON kunit: tool: fix running kunit_tool from outside kernel tree
2 parents 0674324 + 1abdd39 commit 578a715

5 files changed

Lines changed: 154 additions & 28 deletions

File tree

‎tools/testing/kunit/configs/broken_on_uml.config‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,3 +39,4 @@
3939
# CONFIG_QCOM_CPR is not set
4040
# CONFIG_RESET_BRCMSTB_RESCAL is not set
4141
# CONFIG_RESET_INTEL_GW is not set
42+
# CONFIG_ADI_AXI_ADC is not set

‎tools/testing/kunit/kunit.py‎

Lines changed: 41 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
from enum import Enum, auto
1818

1919
import kunit_config
20+
import kunit_json
2021
import kunit_kernel
2122
import kunit_parser
2223

@@ -30,9 +31,9 @@
3031
KunitExecRequest = namedtuple('KunitExecRequest',
3132
['timeout', 'build_dir', 'alltests'])
3233
KunitParseRequest = namedtuple('KunitParseRequest',
33-
['raw_output', 'input_data'])
34+
['raw_output', 'input_data', 'build_dir', 'json'])
3435
KunitRequest = namedtuple('KunitRequest', ['raw_output','timeout', 'jobs',
35-
'build_dir', 'alltests',
36+
'build_dir', 'alltests', 'json',
3637
'make_options'])
3738

3839
KernelDirectoryPath = sys.argv[0].split('tools/testing/kunit/')[0]
@@ -113,12 +114,22 @@ def parse_tests(request: KunitParseRequest) -> KunitResult:
113114
test_result = kunit_parser.TestResult(kunit_parser.TestStatus.SUCCESS,
114115
[],
115116
'Tests not Parsed.')
117+
116118
if request.raw_output:
117119
kunit_parser.raw_output(request.input_data)
118120
else:
119121
test_result = kunit_parser.parse_run_tests(request.input_data)
120122
parse_end = time.time()
121123

124+
if request.json:
125+
json_obj = kunit_json.get_json_result(
126+
test_result=test_result,
127+
def_config='kunit_defconfig',
128+
build_dir=request.build_dir,
129+
json_path=request.json)
130+
if request.json == 'stdout':
131+
print(json_obj)
132+
122133
if test_result.status != kunit_parser.TestStatus.SUCCESS:
123134
return KunitResult(KunitStatus.TEST_FAILURE, test_result,
124135
parse_end - parse_start)
@@ -151,7 +162,9 @@ def run_tests(linux: kunit_kernel.LinuxSourceTree,
151162
return exec_result
152163

153164
parse_request = KunitParseRequest(request.raw_output,
154-
exec_result.result)
165+
exec_result.result,
166+
request.build_dir,
167+
request.json)
155168
parse_result = parse_tests(parse_request)
156169

157170
run_end = time.time()
@@ -195,7 +208,12 @@ def add_exec_opts(parser):
195208
def add_parse_opts(parser):
196209
parser.add_argument('--raw_output', help='don\'t format output from kernel',
197210
action='store_true')
198-
211+
parser.add_argument('--json',
212+
nargs='?',
213+
help='Stores test results in a JSON, and either '
214+
'prints to stdout or saves to file if a '
215+
'filename is specified',
216+
type=str, const='stdout', default=None)
199217

200218
def main(argv, linux=None):
201219
parser = argparse.ArgumentParser(
@@ -237,10 +255,16 @@ def main(argv, linux=None):
237255

238256
cli_args = parser.parse_args(argv)
239257

258+
if get_kernel_root_path():
259+
os.chdir(get_kernel_root_path())
260+
240261
if cli_args.subcommand == 'run':
241262
if not os.path.exists(cli_args.build_dir):
242263
os.mkdir(cli_args.build_dir)
243264

265+
if not os.path.exists(kunit_kernel.kunitconfig_path):
266+
create_default_kunitconfig()
267+
244268
if not linux:
245269
linux = kunit_kernel.LinuxSourceTree()
246270

@@ -249,14 +273,18 @@ def main(argv, linux=None):
249273
cli_args.jobs,
250274
cli_args.build_dir,
251275
cli_args.alltests,
276+
cli_args.json,
252277
cli_args.make_options)
253278
result = run_tests(linux, request)
254279
if result.status != KunitStatus.SUCCESS:
255280
sys.exit(1)
256281
elif cli_args.subcommand == 'config':
257-
if cli_args.build_dir:
258-
if not os.path.exists(cli_args.build_dir):
259-
os.mkdir(cli_args.build_dir)
282+
if cli_args.build_dir and (
283+
not os.path.exists(cli_args.build_dir)):
284+
os.mkdir(cli_args.build_dir)
285+
286+
if not os.path.exists(kunit_kernel.kunitconfig_path):
287+
create_default_kunitconfig()
260288

261289
if not linux:
262290
linux = kunit_kernel.LinuxSourceTree()
@@ -270,10 +298,6 @@ def main(argv, linux=None):
270298
if result.status != KunitStatus.SUCCESS:
271299
sys.exit(1)
272300
elif cli_args.subcommand == 'build':
273-
if cli_args.build_dir:
274-
if not os.path.exists(cli_args.build_dir):
275-
os.mkdir(cli_args.build_dir)
276-
277301
if not linux:
278302
linux = kunit_kernel.LinuxSourceTree()
279303

@@ -288,10 +312,6 @@ def main(argv, linux=None):
288312
if result.status != KunitStatus.SUCCESS:
289313
sys.exit(1)
290314
elif cli_args.subcommand == 'exec':
291-
if cli_args.build_dir:
292-
if not os.path.exists(cli_args.build_dir):
293-
os.mkdir(cli_args.build_dir)
294-
295315
if not linux:
296316
linux = kunit_kernel.LinuxSourceTree()
297317

@@ -300,7 +320,9 @@ def main(argv, linux=None):
300320
cli_args.alltests)
301321
exec_result = exec_tests(linux, exec_request)
302322
parse_request = KunitParseRequest(cli_args.raw_output,
303-
exec_result.result)
323+
exec_result.result,
324+
cli_args.build_dir,
325+
cli_args.json)
304326
result = parse_tests(parse_request)
305327
kunit_parser.print_with_timestamp((
306328
'Elapsed time: %.3fs\n') % (
@@ -314,7 +336,9 @@ def main(argv, linux=None):
314336
with open(cli_args.file, 'r') as f:
315337
kunit_output = f.read().splitlines()
316338
request = KunitParseRequest(cli_args.raw_output,
317-
kunit_output)
339+
kunit_output,
340+
cli_args.build_dir,
341+
cli_args.json)
318342
result = parse_tests(request)
319343
if result.status != KunitStatus.SUCCESS:
320344
sys.exit(1)

‎tools/testing/kunit/kunit_json.py‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
# SPDX-License-Identifier: GPL-2.0
2+
#
3+
# Generates JSON from KUnit results according to
4+
# KernelCI spec: https://github.com/kernelci/kernelci-doc/wiki/Test-API
5+
#
6+
# Copyright (C) 2020, Google LLC.
7+
# Author: Heidi Fahim <heidifahim@google.com>
8+
9+
import json
10+
import os
11+
12+
import kunit_parser
13+
14+
from kunit_parser import TestStatus
15+
16+
def get_json_result(test_result, def_config, build_dir, json_path):
17+
sub_groups = []
18+
19+
# Each test suite is mapped to a KernelCI sub_group
20+
for test_suite in test_result.suites:
21+
sub_group = {
22+
"name": test_suite.name,
23+
"arch": "UM",
24+
"defconfig": def_config,
25+
"build_environment": build_dir,
26+
"test_cases": [],
27+
"lab_name": None,
28+
"kernel": None,
29+
"job": None,
30+
"git_branch": "kselftest",
31+
}
32+
test_cases = []
33+
# TODO: Add attachments attribute in test_case with detailed
34+
# failure message, see https://api.kernelci.org/schema-test-case.html#get
35+
for case in test_suite.cases:
36+
test_case = {"name": case.name, "status": "FAIL"}
37+
if case.status == TestStatus.SUCCESS:
38+
test_case["status"] = "PASS"
39+
elif case.status == TestStatus.TEST_CRASHED:
40+
test_case["status"] = "ERROR"
41+
test_cases.append(test_case)
42+
sub_group["test_cases"] = test_cases
43+
sub_groups.append(sub_group)
44+
test_group = {
45+
"name": "KUnit Test Group",
46+
"arch": "UM",
47+
"defconfig": def_config,
48+
"build_environment": build_dir,
49+
"sub_groups": sub_groups,
50+
"lab_name": None,
51+
"kernel": None,
52+
"job": None,
53+
"git_branch": "kselftest",
54+
}
55+
json_obj = json.dumps(test_group, indent=4)
56+
if json_path != 'stdout':
57+
with open(json_path, 'w') as result_path:
58+
result_path.write(json_obj)
59+
root = __file__.split('tools/testing/kunit/')[0]
60+
kunit_parser.print_with_timestamp(
61+
"Test results stored in %s" %
62+
os.path.join(root, result_path.name))
63+
return json_obj

‎tools/testing/kunit/kunit_kernel.py‎

Lines changed: 16 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -36,9 +36,9 @@ def make_mrproper(self):
3636
try:
3737
subprocess.check_output(['make', 'mrproper'], stderr=subprocess.STDOUT)
3838
except OSError as e:
39-
raise ConfigError('Could not call make command: ' + e)
39+
raise ConfigError('Could not call make command: ' + str(e))
4040
except subprocess.CalledProcessError as e:
41-
raise ConfigError(e.output)
41+
raise ConfigError(e.output.decode())
4242

4343
def make_olddefconfig(self, build_dir, make_options):
4444
command = ['make', 'ARCH=um', 'olddefconfig']
@@ -49,22 +49,27 @@ def make_olddefconfig(self, build_dir, make_options):
4949
try:
5050
subprocess.check_output(command, stderr=subprocess.STDOUT)
5151
except OSError as e:
52-
raise ConfigError('Could not call make command: ' + e)
52+
raise ConfigError('Could not call make command: ' + str(e))
5353
except subprocess.CalledProcessError as e:
54-
raise ConfigError(e.output)
54+
raise ConfigError(e.output.decode())
5555

56-
def make_allyesconfig(self):
56+
def make_allyesconfig(self, build_dir, make_options):
5757
kunit_parser.print_with_timestamp(
5858
'Enabling all CONFIGs for UML...')
59+
command = ['make', 'ARCH=um', 'allyesconfig']
60+
if make_options:
61+
command.extend(make_options)
62+
if build_dir:
63+
command += ['O=' + build_dir]
5964
process = subprocess.Popen(
60-
['make', 'ARCH=um', 'allyesconfig'],
65+
command,
6166
stdout=subprocess.DEVNULL,
6267
stderr=subprocess.STDOUT)
6368
process.wait()
6469
kunit_parser.print_with_timestamp(
6570
'Disabling broken configs to run KUnit tests...')
6671
with ExitStack() as es:
67-
config = open(KCONFIG_PATH, 'a')
72+
config = open(get_kconfig_path(build_dir), 'a')
6873
disable = open(BROKEN_ALLCONFIG_PATH, 'r').read()
6974
config.write(disable)
7075
kunit_parser.print_with_timestamp(
@@ -79,9 +84,9 @@ def make(self, jobs, build_dir, make_options):
7984
try:
8085
subprocess.check_output(command, stderr=subprocess.STDOUT)
8186
except OSError as e:
82-
raise BuildError('Could not call execute make: ' + e)
87+
raise BuildError('Could not call execute make: ' + str(e))
8388
except subprocess.CalledProcessError as e:
84-
raise BuildError(e.output)
89+
raise BuildError(e.output.decode())
8590

8691
def linux_bin(self, params, timeout, build_dir, outfile):
8792
"""Runs the Linux UML binary. Must be named 'linux'."""
@@ -161,9 +166,9 @@ def build_reconfig(self, build_dir, make_options):
161166
return self.build_config(build_dir, make_options)
162167

163168
def build_um_kernel(self, alltests, jobs, build_dir, make_options):
164-
if alltests:
165-
self._ops.make_allyesconfig()
166169
try:
170+
if alltests:
171+
self._ops.make_allyesconfig(build_dir, make_options)
167172
self._ops.make_olddefconfig(build_dir, make_options)
168173
self._ops.make(jobs, build_dir, make_options)
169174
except (ConfigError, BuildError) as e:

‎tools/testing/kunit/kunit_tool_test.py‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,13 @@
1111

1212
import tempfile, shutil # Handling test_tmpdir
1313

14+
import json
1415
import os
1516

1617
import kunit_config
1718
import kunit_parser
1819
import kunit_kernel
20+
import kunit_json
1921
import kunit
2022

2123
test_tmpdir = ''
@@ -230,6 +232,37 @@ def test_pound_no_prefix(self):
230232
result = kunit_parser.parse_run_tests(file.readlines())
231233
self.assertEqual('kunit-resource-test', result.suites[0].name)
232234

235+
class KUnitJsonTest(unittest.TestCase):
236+
237+
def _json_for(self, log_file):
238+
with(open(get_absolute_path(log_file))) as file:
239+
test_result = kunit_parser.parse_run_tests(file)
240+
json_obj = kunit_json.get_json_result(
241+
test_result=test_result,
242+
def_config='kunit_defconfig',
243+
build_dir=None,
244+
json_path='stdout')
245+
return json.loads(json_obj)
246+
247+
def test_failed_test_json(self):
248+
result = self._json_for(
249+
'test_data/test_is_test_passed-failure.log')
250+
self.assertEqual(
251+
{'name': 'example_simple_test', 'status': 'FAIL'},
252+
result["sub_groups"][1]["test_cases"][0])
253+
254+
def test_crashed_test_json(self):
255+
result = self._json_for(
256+
'test_data/test_is_test_passed-crash.log')
257+
self.assertEqual(
258+
{'name': 'example_simple_test', 'status': 'ERROR'},
259+
result["sub_groups"][1]["test_cases"][0])
260+
261+
def test_no_tests_json(self):
262+
result = self._json_for(
263+
'test_data/test_is_test_passed-no_tests_run.log')
264+
self.assertEqual(0, len(result['sub_groups']))
265+
233266
class StrContains(str):
234267
def __eq__(self, other):
235268
return self in other

0 commit comments

Comments
 (0)