Get Gentoo!
gentoo.org sites
gentoo.org Wiki Bugs Forums Packages
Planet Archives Sources
Infra Status
aboutsummaryrefslogtreecommitdiff
path: root/buildbot_gentoo_ci
diff options
context:
space:
mode:
authorMagnus Granberg <zorry@gentoo.org>2022-07-13 21:49:23 +0200
committerMagnus Granberg <zorry@gentoo.org>2022-07-13 21:49:23 +0200
commit1250899a4c17be77c99048576090ae1b32e6d227 (patch)
tree50a5d71ef0ab9218e02979aeb04c24edfdcb5e34 /buildbot_gentoo_ci
parentAdd support log docker worker (diff)
downloadtinderbox-cluster-1250899a4c17be77c99048576090ae1b32e6d227.tar.gz
tinderbox-cluster-1250899a4c17be77c99048576090ae1b32e6d227.tar.bz2
tinderbox-cluster-1250899a4c17be77c99048576090ae1b32e6d227.zip
Use log docker for log parser
Signed-off-by: Magnus Granberg <zorry@gentoo.org>
Diffstat (limited to 'buildbot_gentoo_ci')
-rw-r--r--buildbot_gentoo_ci/logs/log_parser.py180
-rw-r--r--buildbot_gentoo_ci/steps/logs.py88
2 files changed, 53 insertions, 215 deletions
diff --git a/buildbot_gentoo_ci/logs/log_parser.py b/buildbot_gentoo_ci/logs/log_parser.py
deleted file mode 100644
index b890c12..0000000
--- a/buildbot_gentoo_ci/logs/log_parser.py
+++ /dev/null
@@ -1,180 +0,0 @@
-# Copyright 2021 Gentoo Authors
-# Distributed under the terms of the GNU General Public License v2
-
-import sys
-from multiprocessing import Pool, cpu_count
-import re
-import io
-import gzip
-import json
-import os
-from sqlalchemy.ext.declarative import declarative_base
-import sqlalchemy as sa
-
-Base = declarative_base()
-
-class ProjectsPattern(Base):
- __tablename__ = "projects_pattern"
- id = sa.Column(sa.Integer, primary_key=True)
- project_uuid = sa.Column(sa.String(36), nullable=False)
- search = sa.Column(sa.String(50), nullable=False)
- start = sa.Column(sa.Integer, default=0)
- end = sa.Column(sa.Integer, default=0)
- status = sa.Column(sa.Enum('info', 'warning', 'ignore', 'error'), default='info')
- type = sa.Column(sa.Enum('info', 'qa', 'compile', 'configure', 'install', 'postinst', 'prepare', 'pretend', 'setup', 'test', 'unpack', 'ignore', 'issues', 'misc', 'elog'), default='info')
- search_type = sa.Column(sa.Enum('in', 'startswith', 'endswith', 'search'), default='in')
-
-def getDBSession(config):
- #FIXME: Read the user/pass from file
- engine = sa.create_engine(config['database'])
- Session = sa.orm.sessionmaker(bind = engine)
- return Session()
-
-def getMultiprocessingPool(config):
- return Pool(processes = int(config['core']))
-
-def addPatternToList(Session, pattern_list, uuid):
- for project_pattern in Session.query(ProjectsPattern).filter_by(project_uuid=uuid).all():
- # check if the search pattern is vaild
- project_pattern_search = project_pattern.search
- try:
- re.compile(project_pattern_search)
- except re.error:
- print("Non valid regex pattern")
- print(project_pattern.search)
- print(project_pattern.id)
- else:
- patten_dict = {}
- patten_dict['id'] = project_pattern.id
- patten_dict['project_uuid'] = project_pattern.project_uuid
- patten_dict['search'] = project_pattern_search
- patten_dict['start'] = project_pattern.start
- patten_dict['end'] = project_pattern.end
- patten_dict['status'] = project_pattern.status
- patten_dict['type'] = project_pattern.type
- patten_dict['search_type'] = project_pattern.search_type
- pattern_list.append(patten_dict)
- return pattern_list
-
-def get_log_search_pattern(Session, uuid, default_uuid):
- # get pattern from the projects
- # add that to log_search_pattern_list
- log_search_pattern_list = []
- log_search_pattern_list = addPatternToList(Session, log_search_pattern_list, uuid)
- log_search_pattern_list = addPatternToList(Session, log_search_pattern_list, default_uuid)
- return log_search_pattern_list
-
-def search_buildlog(log_search_pattern_list, logfile_text_dict, tmp_index, max_text_lines):
- # get text line to search
- text_line = logfile_text_dict[tmp_index]
- summery_dict = {}
- # loop true the pattern list for match
- for search_pattern in log_search_pattern_list:
- search_hit = False
- ignore_line = False
- # check if should ignore the line
- #FIXME take the ignore line pattern from db
- if re.search('^>>> /', text_line):
- ignore_line = True
- #if else re.search('./\w+/'):
- # pass
- else:
- # search for match
- if search_pattern['search_type'] == 'in':
- if search_pattern['search'] in text_line:
- search_hit = True
- if search_pattern['search_type'] == 'startswith':
- if text_line.startswith(search_pattern['search']):
- search_hit = True
- if search_pattern['search_type'] == 'endswith':
- if text_line.endswith(search_pattern['search']):
- search_hit = True
- if search_pattern['search_type'] == 'search':
- if re.search(search_pattern['search'], text_line):
- search_hit = True
- # add the line if the pattern match
- if search_hit:
- summery_dict[tmp_index] = {}
- summery_dict[tmp_index]['text'] = text_line
- summery_dict[tmp_index]['type'] = search_pattern['type']
- summery_dict[tmp_index]['status'] = search_pattern['status']
- summery_dict[tmp_index]['id'] = search_pattern['id']
- summery_dict[tmp_index]['search_pattern'] = search_pattern['search']
- # add upper text lines if requested
- # max 5
- if search_pattern['start'] != 0:
- i = tmp_index - search_pattern['start'] - 1
- match = True
- while match:
- i = i + 1
- if i < (tmp_index - 9) or i == tmp_index:
- match = False
- else:
- if not i in summery_dict:
- summery_dict[i] = {}
- summery_dict[i]['text'] = logfile_text_dict[i]
- summery_dict[i]['type'] = 'info'
- summery_dict[i]['status'] = 'info'
- summery_dict[i]['id'] = 0
- summery_dict[i]['search_pattern'] = 'auto'
- # add lower text lines if requested
- # max 5
- if search_pattern['end'] != 0:
- i = tmp_index
- end = tmp_index + search_pattern['end']
- match = True
- while match:
- i = i + 1
- if i > max_text_lines or i > end:
- match = False
- else:
- if not i in summery_dict:
- summery_dict[i] = {}
- summery_dict[i]['text'] = logfile_text_dict[i]
- summery_dict[i]['type'] = 'info'
- summery_dict[i]['status'] = 'info'
- summery_dict[i]['id'] = 0
- summery_dict[i]['search_pattern'] = 'auto'
- if not ignore_line or not search_hit:
- # we add all line that start with ' * ' as info
- # we add all line that start with '>>>' as info
- if text_line.startswith(' * ') or text_line.startswith('>>>'):
- if not tmp_index in summery_dict:
- summery_dict[tmp_index] = {}
- summery_dict[tmp_index]['text'] = text_line
- summery_dict[tmp_index]['type'] = 'info'
- summery_dict[tmp_index]['status'] = 'info'
- summery_dict[tmp_index]['id'] = 0
- summery_dict[tmp_index]['search_pattern'] = 'auto'
- if summery_dict == {}:
- return False
- return summery_dict
-
-def getConfigSettings():
- configpath = os.getcwd().split('workers/')[0]
- with open(configpath + 'logparser.json') as f:
- config = json.load(f)
- return config
-
-def runLogParser(args):
- index = 1
- max_text_lines = 0
- logfile_text_dict = {}
- config = getConfigSettings()
- Session = getDBSession(config)
- mp_pool = getMultiprocessingPool(config)
- #NOTE: The patten is from https://github.com/toralf/tinderbox/tree/master/data files.
- # Is stored in a db instead of files.
- log_search_pattern_list = get_log_search_pattern(Session, args.uuid, config['default_uuid'])
- Session.close()
- for text_line in io.TextIOWrapper(io.BufferedReader(gzip.open(args.file)), encoding='utf8', errors='ignore'):
- logfile_text_dict[index] = text_line.strip('\n')
- index = index + 1
- max_text_lines = index
- # run the parse patten on the line
- for tmp_index, text in logfile_text_dict.items():
- res = mp_pool.apply_async(search_buildlog, (log_search_pattern_list, logfile_text_dict, tmp_index, max_text_lines,))
- if res.get():
- print(json.dumps(res.get()))
- mp_pool.close()
- mp_pool.join()
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 6bd3279..e139cf9 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -86,6 +86,12 @@ class SetupPropertys(BuildStep):
self.setProperty("default_project_data", default_project_data, 'default_project_data')
self.setProperty("version_data", version_data, 'version_data')
self.setProperty("status", 'completed', 'status')
+ if self.getProperty('faild_cpv'):
+ log_cpv = self.getProperty('faild_cpv')
+ else:
+ log_cpv = self.getProperty('cpv')
+ self.setProperty("log_cpv", log_cpv, 'log_cpv')
+ self.descriptionDone = 'Runing log checker on ' + log_cpv
return SUCCESS
class SetupParserBuildLoger(BuildStep):
@@ -102,24 +108,42 @@ class SetupParserBuildLoger(BuildStep):
@defer.inlineCallbacks
def run(self):
+ self.aftersteps_list = []
workdir = yield os.path.join(self.master.basedir, 'workers', self.getProperty('build_workername'), str(self.getProperty("project_build_data")['buildbot_build_id']))
- if self.getProperty('faild_cpv'):
- log_cpv = self.getProperty('log_build_data')[self.getProperty('faild_cpv')]
- else:
- log_cpv = self.getProperty('log_build_data')[self.getProperty('cpv')]
+ log_cpv = self.getProperty('log_build_data')[self.getProperty('log_cpv')]
+ mastersrc_log = yield os.path.join(workdir, log_cpv['full_logname'])
+ log_py = 'log_parser.py'
+ config_log_py = 'logparser.json'
+ mastersrc_py = yield os.path.join(self.master.basedir, log_py)
+ mastersrc_config = yield os.path.join(self.master.basedir, config_log_py)
+ # Upload logfile to worker
+ self.aftersteps_list.append(steps.FileDownload(
+ mastersrc=mastersrc_log,
+ workerdest=log_cpv['full_logname']
+ ))
+ # Upload log parser py code
+ self.aftersteps_list.append(steps.FileDownload(
+ mastersrc=mastersrc_py,
+ workerdest=log_py
+ ))
+ # Upload log parser py config
+ self.aftersteps_list.append(steps.FileDownload(
+ mastersrc=mastersrc_config,
+ workerdest=config_log_py
+ ))
+ # Run the log parser code
command = []
- command.append('ci_log_parser')
+ command.append('python3')
+ command.append(log_py)
command.append('-f')
command.append(log_cpv['full_logname'])
command.append('-u')
command.append(self.getProperty('project_data')['uuid'])
- self.aftersteps_list = []
- self.aftersteps_list.append(master_steps.MasterSetPropertyFromCommand(
+ self.aftersteps_list.append(steps.SetPropertyFromCommand(
name = 'RunBuildLogParser',
haltOnFailure = True,
flunkOnFailure = True,
command=command,
- workdir=workdir,
strip=False,
extract_fn=PersOutputOfLogParser
))
@@ -296,19 +320,15 @@ class MakeIssue(BuildStep):
separator1 = '\n'
separator2 = ' '
log = yield self.addLog('issue')
- if self.getProperty('faild_cpv'):
- cpv = self.getProperty('faild_cpv')
- else:
- cpv = self.getProperty('cpv')
- self.error_dict['cpv'] = cpv
+ self.error_dict['cpv'] = self.getProperty('log_cpv')
yield log.addStdout('Title:' + '\n')
- yield log.addStdout(separator2.join([cpv, '-', self.error_dict['title']]) + separator1)
+ yield log.addStdout(separator2.join([self.getProperty('log_cpv'), '-', self.error_dict['title']]) + separator1)
yield log.addStdout('Summary:' + '\n')
for line in self.summary_log_list:
yield log.addStdout(line + '\n')
yield log.addStdout('Attachments:' + '\n')
yield log.addStdout('emerge_info.log' + '\n')
- log_cpv = self.getProperty('log_build_data')[cpv]
+ log_cpv = self.getProperty('log_build_data')[self.getProperty('log_cpv')]
yield log.addStdout(log_cpv['full_logname'] + '\n')
yield log.addStdout('world.log' + '\n')
@@ -389,9 +409,9 @@ class setBuildbotLog(BuildStep):
yield log.addStdout(line + '\n')
return SUCCESS
-class SetupParserEmergeInfoLog(BuildStep):
+class ReadEmergeInfoLog(BuildStep):
- name = 'SetupParserEmergeInfoLog'
+ name = 'ReadEmergeInfoLog'
description = 'Running'
descriptionDone = 'Ran'
descriptionSuffix = None
@@ -404,21 +424,22 @@ class SetupParserEmergeInfoLog(BuildStep):
@defer.inlineCallbacks
def run(self):
+ emerge_info_output = {}
+ emerge_info_list = []
+ emerge_package_info = []
+ # Read the file and add it to a property
workdir = yield os.path.join(self.master.basedir, 'workers', self.getProperty('build_workername'), str(self.getProperty("project_build_data")['buildbot_build_id']))
- command = []
- command.append('cat')
- command.append('emerge_info.txt')
- self.aftersteps_list = []
- self.aftersteps_list.append(master_steps.MasterSetPropertyFromCommand(
- name = 'RunEmergeInfoLogParser',
- haltOnFailure = True,
- flunkOnFailure = True,
- command=command,
- workdir=workdir,
- strip=False,
- extract_fn=PersOutputOfEmergeInfo
- ))
- yield self.build.addStepsAfterCurrentStep(self.aftersteps_list)
+ with open(os.path.join(workdir, 'emerge_info.txt'), encoding='utf-8') as source:
+ emerge_info = source.read()
+ # set emerge_info_output Property
+ for line in emerge_info.split('\n'):
+ if line.startswith('['):
+ emerge_package_info.append(line)
+ else:
+ emerge_info_list.append(line)
+ emerge_info_output['emerge_info'] = emerge_info_list
+ emerge_info_output['emerge_package_info'] = emerge_package_info
+ self.setProperty("emerge_info_output", emerge_info_output, 'emerge_info_output')
return SUCCESS
class setEmergeInfoLog(BuildStep):
@@ -483,10 +504,7 @@ class Upload(BuildStep):
@defer.inlineCallbacks
def run(self):
- if self.getProperty('faild_cpv'):
- log_cpv = self.getProperty('log_build_data')[self.getProperty('faild_cpv')]
- else:
- log_cpv = self.getProperty('log_build_data')[self.getProperty('cpv')]
+ log_cpv = self.getProperty('log_build_data')[self.getProperty('log_cpv')]
bucket = self.getProperty('project_data')['uuid'] + '-' + 'logs'
file_path = yield os.path.join(self.master.basedir, 'workers', self.getProperty('build_workername'), str(self.getProperty("project_build_data")['buildbot_build_id']) ,log_cpv['full_logname'])
aftersteps_list = []