From 94f378ebc7d2843799dd28a5a3060e24f615f1cd Mon Sep 17 00:00:00 2001 From: Tejas Manhas Date: Fri, 7 Jun 2024 11:45:13 +0530 Subject: [PATCH 1/3] OpTestKernelTest: Kernel build test with bisection This is a new testcase for kernel build with build bisection support having email support, storing the build status in json and proper error handling and also support for linux-next bisection and a flag to skip bisection Signed-off-by: Tejas Manhas --- OpTestConfiguration.py | 2 + op-test | 1 + testcases/OpTestKernelTest.py | 205 ++++++++++++++++++++++++++++++++++ 3 files changed, 208 insertions(+) create mode 100644 testcases/OpTestKernelTest.py diff --git a/OpTestConfiguration.py b/OpTestConfiguration.py index 3c5ac4a1..9ea342a1 100644 --- a/OpTestConfiguration.py +++ b/OpTestConfiguration.py @@ -460,6 +460,8 @@ def get_parser(): help="Don't exit if we find unknown command line arguments") misc_group.add_argument("--secvar-payload-url", help="Specify a URL for the secvar test data payload") + misc_group.add_argument("--bisect-flag", + help="Specify if bisection is to be done or not") return parser diff --git a/op-test b/op-test index 629f4a0f..6cf475f4 100755 --- a/op-test +++ b/op-test @@ -110,6 +110,7 @@ from testcases import GcovSetup from testcases import Lcov from testcases import BisectKernel from testcases import OpTestHtxBootme +from testcases import OpTestKernelTest import OpTestConfiguration import sys import time diff --git a/testcases/OpTestKernelTest.py b/testcases/OpTestKernelTest.py new file mode 100644 index 00000000..d35a5698 --- /dev/null +++ b/testcases/OpTestKernelTest.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +# OpenPOWER Automated Test Project +# +# Contributors Listed Below - COPYRIGHT 2024 +# [+] International Business Machines Corp. +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# Author : Tejas Manhas +# Co-Author : Abdul Haleem + +import json +import OpTestConfiguration +import OpTestLogger +import os +import unittest +from urllib.parse import urlparse +import re +import subprocess +import sys +import time + +from common.OpTestSystem import OpSystemState +from common.OpTestSOL import OpSOLMonitorThread +from common.Exceptions import CommandFailed +from common.OpTestUtil import OpTestUtil + +log = OpTestLogger.optest_logger_glob.get_logger(__name__) + + +class KernelTest(unittest.TestCase): + + def setUp(self): + """ + Set up the test environment. + Initializes test parameters and checks required configurations. + """ + self.conf = OpTestConfiguration.conf + self.cv_HOST = self.conf.host() + self.cv_SYSTEM = self.conf.system() + self.con = self.cv_SYSTEM.cv_HOST.get_ssh_connection() + self.host_cmd_timeout = self.conf.args.host_cmd_timeout + self.repo = self.conf.args.git_repo + self.repo_reference = self.conf.args.git_repo_reference + self.branch = self.conf.args.git_branch + self.home = self.conf.args.git_home + self.config_path = self.conf.args.git_repoconfigpath + self.config = self.conf.args.git_repoconfig + self.good_commit = self.conf.args.good_commit + self.bad_commit = self.conf.args.bad_commit + self.bisect_script = self.conf.args.bisect_script + self.bisect_category = self.conf.args.bisect_category + self.append_kernel_cmdline = self.conf.args.append_kernel_cmdline + self.linux_path = os.path.join(self.home, "linux") + self.bisect_flag = self.conf.args.bisect_flag + self.util = OpTestUtil(OpTestConfiguration.conf) + self.host_distro_name = self.util.distro_name() + self.console_thread = OpSOLMonitorThread(1, "console") + # in case bisection see if we need powercycle not for build, but for boot + self.cv_SYSTEM.goto_state(OpSystemState.OFF) + self.cv_SYSTEM.goto_state(OpSystemState.OS) + self.console_thread.start() + if not self.repo: + self.fail("Provide git repo of kernel to install") + if not (self.conf.args.host_ip and self.conf.args.host_user and self.conf.args.host_password): + self.fail( + "Provide host ip user details refer, --host-{ip,user,password}") + + def wait_for(self, func, timeout, first=0.0, step=1.0, text=None, args=None, kwargs=None): + args = args or [] + kwargs = kwargs or {} + + start_time = time.monotonic() + end_time = start_time + timeout + + time.sleep(first) + + while time.monotonic() < end_time: + if text: + log.debug("%s (%.9f secs)", text, (time.monotonic() - start_time)) + + output = func(*args, **kwargs) + if output: + return output + + time.sleep(step) + + return None + + def build_kernel(self): + """ + Build and install the Linux kernel. + """ + self.config_path = self.conf.args.git_repoconfigpath + + def is_url(path): + ''' + param path: path to download + return: boolean True if given path is url False Otherwise + ''' + valid_schemes = ['http', 'https', 'git', 'ftp'] + if urlparse(path).scheme in valid_schemes: + return True + return False + + if self.config_path: + if is_url(self.config_path): + self.con.run_command("wget %s -O linux/.config" % self.config_path) + else: + self.cv_HOST.copy_test_file_to_host(self.config_path, sourcedir="", dstdir=os.path.join(linux_path, ".config")) + self.con.run_command("cd linux && make olddefconfig") + # the below part of the code is needed for only first run and will be decided bisect flag false + ker_ver = self.con.run_command("make kernelrelease")[-1] + sha = self.con.run_command("git rev-parse HEAD") + tcommit = self.con.run_command("export 'TERM=xterm-256color';git show -s --format=%ci") + tcommit = re.sub(r"\x1b\[[0-9;]*[mGKHF]", "", tcommit[1]) + log.info("Upstream kernel version: %s", ker_ver) + log.info("Upstream kernel commit-id: %s", sha[-1]) + log.info("Upstream kernel commit-time: %s", tcommit) + log.debug("Compile the upstream kernel") + try: + cpu= self.con.run_command("lscpu | grep '^CPU(s):' | awk '{print $2}'") + err=self.con.run_command("make -j {} -s vmlinux".format(cpu[-1]), timeout=self.host_cmd_timeout) + log.info("Kernel build successful") + return 0,err + except CommandFailed as e: + log.error("Kernel build failed: {}".format(e)) + return 4,e + + def Store_loc(self, er) : + """ + To get location of file in which error is introduced + """ + pattern = r"([\w\d_]+\/(?:(?:[\w\d_]+\/)*[\w\d_]+\b))" + matches = [match.group(1) for match in re.finditer(pattern,er)] + return matches + + +class KernelBuild(KernelTest): + """ + Does the build for any Linux repo and in case of build failure, calls build bisector + from OpTestUtils to give first bad commit and related information along with email template. + """ + + def setUp(self): + """ + Does setup for KernelBUild from parent KernelTest + """ + super(KernelBuild,self).setUp() + + def runTest(self): + """ + Clones git repo and builds to check for failure and do bisection + """ + self.con.run_command("if [ -d {} ]; then rm -rf {}; fi".format(self.home,self.home)) + self.con.run_command("if [ ! -d {} ]; then mkdir -p {}; fi".format(self.home,self.home)) + self.con.run_command("cd {}".format(self.home)) + if not self.branch: + self.branch='master' + self.con.run_command("git clone --depth 1 -b {} {} linux".format( self.branch, self.repo),timeout=3000) + self.con.run_command("cd linux") + commit = self.con.run_command(" git log -1 --format=%H | sed -r 's/\x1B\[[0-9:]*[JKsu]//g'") + self.con.run_command("cd ..") + error = self.build_kernel() + exit_code = error[0] + errVal = str(error[1]) + log.info("printing the exit code '{}'".format(exit_code)) + entry=[] + err_msg=[] + if exit_code != 0: + entry = self.Store_loc(errVal)[-1] + err_msg= self.util.err_message(error) + badCommit = commit[-1] + if self.bisect_flag == '1': + log.info("STARTING BUILD_BISECTION") + res = self.util.build_bisector(self.linux_path, self.good_commit, self.repo) + log.info("BUILD_BISECTION ENDED") + emaili=res[0] + commiti=res[1] + log.info("revert commit check is manual for now") + else : + emaili="" + commiti=commit[-1] + else : + emaili="" + commiti=commit[-1] + with open('output.json','w') as f: + json.dump({"exit_code":exit_code,"email":emaili,"commit": commiti,"error":entry,"err_msg":err_msg,"flag":self.bisect_flag},f) + if exit_code != 0: + self.util.format_email(self.linux_path, self.repo) + + def tearDown(self): + self.console_thread.console_terminate() + self.con.close() From 0f6ca830fb80e3b4346bc50044e872527ceba3e2 Mon Sep 17 00:00:00 2001 From: Tejas Manhas Date: Fri, 7 Jun 2024 11:50:31 +0530 Subject: [PATCH 2/3] APIs added to support kernel build test and bisection Libraries added for error message parsing, build bisection email format, get git details Signed-off-by: Tejas Manhas --- common/OpTestUtil.py | 116 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) diff --git a/common/OpTestUtil.py b/common/OpTestUtil.py index ca23d4c7..97453811 100644 --- a/common/OpTestUtil.py +++ b/common/OpTestUtil.py @@ -2353,6 +2353,122 @@ def prepare_source(self,spec_file, host, dest_path, package, build_option=None, except OpTestError: return "" + def err_message(self, er): + """ + To get entire error msg + """ + error_message = [str(item) for item in er] + combined_error = '\n'.join(error_message) + combined_error = re.sub(r'\x1b\[[0-9;]*[mK]|,', '', combined_error) + parts = combined_error.split('\x1b[') + cleaned_error = ''.join(char for char in combined_error if ord(char) < 128) + pattern = r'in file.*?compilation terminated' + match = re.search(pattern, cleaned_error, re.IGNORECASE | re.DOTALL) + if match: + relevant_part = match.group(0) + return relevant_part + return " " + + def get_email(self, commit_id): + """ + To get email of Author from the identified bad commit + """ + connection = self.conf.host() + try : + connection.host_run_command("git config --global color.ui true") + result = connection.host_run_command("git show --format=%ce {} | sed -r 's/\x1B\[[0-9:]*[JKsu]//g'".format(commit_id)) + return result[0] + except subprocess.CalledProcessError as e: + log.info(e) + return None + + def build_bisector(self, linux_path, good_commit, repo): + connection = self.conf.host() + connection.host_run_command(" if [ '$(pwd)' != {} ]; then cd {} || exit 1 ; fi ".format(linux_path,linux_path)) + shallow = connection.host_run_command("git rev-parse --is-shallow-repository") + if shallow[-1] in [True,'true']: + connection.host_run_command("git fetch --unshallow",timeout=3000) + makefile_path = os.path.join(self.conf.basedir, "test_binaries/make.sh") + connection.copy_test_file_to_host(makefile_path, dstdir=linux_path) + connection.host_run_command("git bisect start") + folder_type=re.split(r'[\/\\.]',str(repo))[-2] + if folder_type == 'linux-next': + connection.host_run_command("git fetch --tags") + good_tag=connection.host_run_command("git tag -l 'v[0-9]*' | sort -V | tail -n 1") + connection.host_run_command("git bisect good {} ".format(good_tag)) + else: + connection.host_run_command("git bisect good {} ".format(good_commit)) + connection.host_run_command(" git bisect bad ") + connection.host_run_command("chmod +x ./make.sh ") + commit = connection.host_run_command(" git bisect run ./make.sh") + badCommit = [word for word in commit if word.endswith("is the first bad commit")] + badCommit= badCommit[0].split()[0] + email = self.get_email(badCommit) + connection.host_run_command("git bisect log") + connection.host_run_command("git bisect reset") + return email, badCommit + + def get_commit_message(self, linux_path, commit_sha): + connection = self.conf.host() + try: + connection.host_run_command(" if [ '$(pwd)' != {} ]; then cd {} || exit 1 ; fi ".format(linux_path,linux_path)) + commit_message = connection.host_run_command("git log -n 1 --pretty=format:%s {} | sed -r 's/\x1B\[[0-9:]*[JKsu]//g'".format(commit_sha)) + except subprocess.CalledProcessError as e: + log.info(e) + commit_message = None + return commit_message[0].strip() + + def format_email(self, linux_path , repo): + connection = self.conf.host() + machine_type = connection.host_run_command("uname -m") + gcc_version = connection.host_run_command("gcc --version")[0] + kernel_version = connection.host_run_command("uname -r") + try: + with open("output.json", "r") as file: + data = json.load(file) + error_message = data.get("error", "") + err_long = data.get("err_msg","") + commit = str(data.get("commit", ""))[:7] + except FileNotFoundError: + log.error("Error: output.json not found.") + error_message = "" + commit = "" + fix_description = self.get_commit_message(linux_path,commit) + if "netdev/net" in repo: + linux = "netdev/net" + elif "netdev/net-next" in repo: + linux = "netdev/net-next" + elif "mkp/scsi" in repo : + linux = "scsi/scsi-queue" + elif "torvalds/linux" in repo: + linux = "mainline/master" + elif "next/linux-next" in repo: + linux = "linux-next/master" + else: + linux = "linux" + subject = "[{}][bisected {}] [{}] build fail with error: {}".format(linux,commit, machine_type, error_message) + body = """ + Greetings, + + Today's {} kernel fails to build on {} machine. + + Kernel build fail at error: {} + {} + + Kernel Version: {} + Machine Type: {} + gcc: {} + Bisected Commit: {} + + kernel builds fine when the bad commit ({}) is reverted + {} - {} + -- + Regards + Linux CI + """.format(linux, machine_type, error_message,err_long,kernel_version, machine_type, gcc_version, commit,commit, commit, fix_description) + with open("email.json","w") as email: + json.dump({"subject":subject,"body":body},email) + class Server(object): ''' From 6cc6b007f7e7cb2337f9c2f63682235a19c258b4 Mon Sep 17 00:00:00 2001 From: Tejas Manhas Date: Fri, 7 Jun 2024 11:53:46 +0530 Subject: [PATCH 3/3] Test_binaries for build bisection Added make.sh for kernel build bisection Signed-off-by: Tejas Manhas --- test_binaries/make.sh | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 test_binaries/make.sh diff --git a/test_binaries/make.sh b/test_binaries/make.sh new file mode 100644 index 00000000..dc8e2795 --- /dev/null +++ b/test_binaries/make.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# OpenPOWER Automated Test Project +# +# Contributors Listed Below - COPYRIGHT 2024 +# [+] International Business Machines Corp. +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +nproc=`nproc` +yes "" | make olddefconfig /dev/null 2>&1 +make -j$nproc -S vmlinux > /dev/null 2>&1