From 222e2a7620e6520ffaf4fc4e69d79c18da31542e Mon Sep 17 00:00:00 2001 From: "Zancanaro; Carlo" Date: Mon, 24 Sep 2012 09:58:17 +1000 Subject: Add the clang library to the repo (with some of my changes, too). --- clang/tools/scan-view/ScanView.py | 770 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 770 insertions(+) create mode 100644 clang/tools/scan-view/ScanView.py (limited to 'clang/tools/scan-view/ScanView.py') diff --git a/clang/tools/scan-view/ScanView.py b/clang/tools/scan-view/ScanView.py new file mode 100644 index 0000000..c6dddba --- /dev/null +++ b/clang/tools/scan-view/ScanView.py @@ -0,0 +1,770 @@ +import BaseHTTPServer +import SimpleHTTPServer +import os +import sys +import urllib, urlparse +import posixpath +import StringIO +import re +import shutil +import threading +import time +import socket +import itertools + +import Reporter +import ConfigParser + +### +# Various patterns matched or replaced by server. + +kReportFileRE = re.compile('(.*/)?report-(.*)\\.html') + +kBugKeyValueRE = re.compile('') + +# + +kReportCrashEntryRE = re.compile('') +kReportCrashEntryKeyValueRE = re.compile(' ?([^=]+)="(.*?)"') + +kReportReplacements = [] + +# Add custom javascript. +kReportReplacements.append((re.compile(''), """\ +""")) + +# Insert additional columns. +kReportReplacements.append((re.compile(''), + '')) + +# Insert report bug and open file links. +kReportReplacements.append((re.compile(''), + ('Report Bug' + + 'Open File'))) + +kReportReplacements.append((re.compile(''), + '

Summary > Report %(report)s

')) + +kReportReplacements.append((re.compile(''), + 'Report Bug')) + +# Insert report crashes link. + +# Disabled for the time being until we decide exactly when this should +# be enabled. Also the radar reporter needs to be fixed to report +# multiple files. + +#kReportReplacements.append((re.compile(''), +# '
These files will automatically be attached to ' + +# 'reports filed here: Report Crashes.')) + +### +# Other simple parameters + +kResources = posixpath.join(posixpath.dirname(__file__), 'Resources') +kConfigPath = os.path.expanduser('~/.scanview.cfg') + +### + +__version__ = "0.1" + +__all__ = ["create_server"] + +class ReporterThread(threading.Thread): + def __init__(self, report, reporter, parameters, server): + threading.Thread.__init__(self) + self.report = report + self.server = server + self.reporter = reporter + self.parameters = parameters + self.success = False + self.status = None + + def run(self): + result = None + try: + if self.server.options.debug: + print >>sys.stderr, "%s: SERVER: submitting bug."%(sys.argv[0],) + self.status = self.reporter.fileReport(self.report, self.parameters) + self.success = True + time.sleep(3) + if self.server.options.debug: + print >>sys.stderr, "%s: SERVER: submission complete."%(sys.argv[0],) + except Reporter.ReportFailure,e: + self.status = e.value + except Exception,e: + s = StringIO.StringIO() + import traceback + print >>s,'Unhandled Exception
'
+            traceback.print_exc(e,file=s)
+            print >>s,'
' + self.status = s.getvalue() + +class ScanViewServer(BaseHTTPServer.HTTPServer): + def __init__(self, address, handler, root, reporters, options): + BaseHTTPServer.HTTPServer.__init__(self, address, handler) + self.root = root + self.reporters = reporters + self.options = options + self.halted = False + self.config = None + self.load_config() + + def load_config(self): + self.config = ConfigParser.RawConfigParser() + + # Add defaults + self.config.add_section('ScanView') + for r in self.reporters: + self.config.add_section(r.getName()) + for p in r.getParameters(): + if p.saveConfigValue(): + self.config.set(r.getName(), p.getName(), '') + + # Ignore parse errors + try: + self.config.read([kConfigPath]) + except: + pass + + # Save on exit + import atexit + atexit.register(lambda: self.save_config()) + + def save_config(self): + # Ignore errors (only called on exit). + try: + f = open(kConfigPath,'w') + self.config.write(f) + f.close() + except: + pass + + def halt(self): + self.halted = True + if self.options.debug: + print >>sys.stderr, "%s: SERVER: halting." % (sys.argv[0],) + + def serve_forever(self): + while not self.halted: + if self.options.debug > 1: + print >>sys.stderr, "%s: SERVER: waiting..." % (sys.argv[0],) + try: + self.handle_request() + except OSError,e: + print 'OSError',e.errno + + def finish_request(self, request, client_address): + if self.options.autoReload: + import ScanView + self.RequestHandlerClass = reload(ScanView).ScanViewRequestHandler + BaseHTTPServer.HTTPServer.finish_request(self, request, client_address) + + def handle_error(self, request, client_address): + # Ignore socket errors + info = sys.exc_info() + if info and isinstance(info[1], socket.error): + if self.options.debug > 1: + print >>sys.stderr, "%s: SERVER: ignored socket error." % (sys.argv[0],) + return + BaseHTTPServer.HTTPServer.handle_error(self, request, client_address) + +# Borrowed from Quixote, with simplifications. +def parse_query(qs, fields=None): + if fields is None: + fields = {} + for chunk in filter(None, qs.split('&')): + if '=' not in chunk: + name = chunk + value = '' + else: + name, value = chunk.split('=', 1) + name = urllib.unquote(name.replace('+', ' ')) + value = urllib.unquote(value.replace('+', ' ')) + item = fields.get(name) + if item is None: + fields[name] = [value] + else: + item.append(value) + return fields + +class ScanViewRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): + server_version = "ScanViewServer/" + __version__ + dynamic_mtime = time.time() + + def do_HEAD(self): + try: + SimpleHTTPServer.SimpleHTTPRequestHandler.do_HEAD(self) + except Exception,e: + self.handle_exception(e) + + def do_GET(self): + try: + SimpleHTTPServer.SimpleHTTPRequestHandler.do_GET(self) + except Exception,e: + self.handle_exception(e) + + def do_POST(self): + """Serve a POST request.""" + try: + length = self.headers.getheader('content-length') or "0" + try: + length = int(length) + except: + length = 0 + content = self.rfile.read(length) + fields = parse_query(content) + f = self.send_head(fields) + if f: + self.copyfile(f, self.wfile) + f.close() + except Exception,e: + self.handle_exception(e) + + def log_message(self, format, *args): + if self.server.options.debug: + sys.stderr.write("%s: SERVER: %s - - [%s] %s\n" % + (sys.argv[0], + self.address_string(), + self.log_date_time_string(), + format%args)) + + def load_report(self, report): + path = os.path.join(self.server.root, 'report-%s.html'%report) + data = open(path).read() + keys = {} + for item in kBugKeyValueRE.finditer(data): + k,v = item.groups() + keys[k] = v + return keys + + def load_crashes(self): + path = posixpath.join(self.server.root, 'index.html') + data = open(path).read() + problems = [] + for item in kReportCrashEntryRE.finditer(data): + fieldData = item.group(1) + fields = dict([i.groups() for i in + kReportCrashEntryKeyValueRE.finditer(fieldData)]) + problems.append(fields) + return problems + + def handle_exception(self, exc): + import traceback + s = StringIO.StringIO() + print >>s, "INTERNAL ERROR\n" + traceback.print_exc(exc, s) + f = self.send_string(s.getvalue(), 'text/plain') + if f: + self.copyfile(f, self.wfile) + f.close() + + def get_scalar_field(self, name): + if name in self.fields: + return self.fields[name][0] + else: + return None + + def submit_bug(self, c): + title = self.get_scalar_field('title') + description = self.get_scalar_field('description') + report = self.get_scalar_field('report') + reporterIndex = self.get_scalar_field('reporter') + files = [] + for fileID in self.fields.get('files',[]): + try: + i = int(fileID) + except: + i = None + if i is None or i<0 or i>=len(c.files): + return (False, 'Invalid file ID') + files.append(c.files[i]) + + if not title: + return (False, "Missing title.") + if not description: + return (False, "Missing description.") + try: + reporterIndex = int(reporterIndex) + except: + return (False, "Invalid report method.") + + # Get the reporter and parameters. + reporter = self.server.reporters[reporterIndex] + parameters = {} + for o in reporter.getParameters(): + name = '%s_%s'%(reporter.getName(),o.getName()) + if name not in self.fields: + return (False, + 'Missing field "%s" for %s report method.'%(name, + reporter.getName())) + parameters[o.getName()] = self.get_scalar_field(name) + + # Update config defaults. + if report != 'None': + self.server.config.set('ScanView', 'reporter', reporterIndex) + for o in reporter.getParameters(): + if o.saveConfigValue(): + name = o.getName() + self.server.config.set(reporter.getName(), name, parameters[name]) + + # Create the report. + bug = Reporter.BugReport(title, description, files) + + # Kick off a reporting thread. + t = ReporterThread(bug, reporter, parameters, self.server) + t.start() + + # Wait for thread to die... + while t.isAlive(): + time.sleep(.25) + submitStatus = t.status + + return (t.success, t.status) + + def send_report_submit(self): + report = self.get_scalar_field('report') + c = self.get_report_context(report) + if c.reportSource is None: + reportingFor = "Report Crashes > " + fileBug = """\ +File Bug > """%locals() + else: + reportingFor = 'Report %s > ' % (c.reportSource, + report) + fileBug = 'File Bug > ' % report + title = self.get_scalar_field('title') + description = self.get_scalar_field('description') + + res,message = self.submit_bug(c) + + if res: + statusClass = 'SubmitOk' + statusName = 'Succeeded' + else: + statusClass = 'SubmitFail' + statusName = 'Failed' + + result = """ + + Bug Submission + + + +

+Summary > +%(reportingFor)s +%(fileBug)s +Submit

+
+ + +
+ + + + + + + + +
Title: + +
Description: + +
+
+
+

Submission %(statusName)s

+%(message)s +

+


+Return to Summary + +"""%locals() + return self.send_string(result) + + def send_open_report(self, report): + try: + keys = self.load_report(report) + except IOError: + return self.send_error(400, 'Invalid report.') + + file = keys.get('FILE') + if not file or not posixpath.exists(file): + return self.send_error(400, 'File does not exist: "%s"' % file) + + import startfile + if self.server.options.debug: + print >>sys.stderr, '%s: SERVER: opening "%s"'%(sys.argv[0], + file) + + status = startfile.open(file) + if status: + res = 'Opened: "%s"' % file + else: + res = 'Open failed: "%s"' % file + + return self.send_string(res, 'text/plain') + + def get_report_context(self, report): + class Context: + pass + if report is None or report == 'None': + data = self.load_crashes() + # Don't allow empty reports. + if not data: + raise ValueError, 'No crashes detected!' + c = Context() + c.title = 'clang static analyzer failures' + + stderrSummary = "" + for item in data: + if 'stderr' in item: + path = posixpath.join(self.server.root, item['stderr']) + if os.path.exists(path): + lns = itertools.islice(open(path), 0, 10) + stderrSummary += '%s\n--\n%s' % (item.get('src', + ''), + ''.join(lns)) + + c.description = """\ +The clang static analyzer failed on these inputs: +%s + +STDERR Summary +-------------- +%s +""" % ('\n'.join([item.get('src','') for item in data]), + stderrSummary) + c.reportSource = None + c.navMarkup = "Report Crashes > " + c.files = [] + for item in data: + c.files.append(item.get('src','')) + c.files.append(posixpath.join(self.server.root, + item.get('file',''))) + c.files.append(posixpath.join(self.server.root, + item.get('clangfile',''))) + c.files.append(posixpath.join(self.server.root, + item.get('stderr',''))) + c.files.append(posixpath.join(self.server.root, + item.get('info',''))) + # Just in case something failed, ignore files which don't + # exist. + c.files = [f for f in c.files + if os.path.exists(f) and os.path.isfile(f)] + else: + # Check that this is a valid report. + path = posixpath.join(self.server.root, 'report-%s.html' % report) + if not posixpath.exists(path): + raise ValueError, 'Invalid report ID' + keys = self.load_report(report) + c = Context() + c.title = keys.get('DESC','clang error (unrecognized') + c.description = """\ +Bug reported by the clang static analyzer. + +Description: %s +File: %s +Line: %s +"""%(c.title, keys.get('FILE',''), keys.get('LINE', '')) + c.reportSource = 'report-%s.html' % report + c.navMarkup = """Report %s > """ % (c.reportSource, + report) + + c.files = [path] + return c + + def send_report(self, report, configOverrides=None): + def getConfigOption(section, field): + if (configOverrides is not None and + section in configOverrides and + field in configOverrides[section]): + return configOverrides[section][field] + return self.server.config.get(section, field) + + # report is None is used for crashes + try: + c = self.get_report_context(report) + except ValueError, e: + return self.send_error(400, e.message) + + title = c.title + description= c.description + reportingFor = c.navMarkup + if c.reportSource is None: + extraIFrame = "" + else: + extraIFrame = """\ +""" % (c.reportSource, c.reportSource) + + reporterSelections = [] + reporterOptions = [] + + try: + active = int(getConfigOption('ScanView','reporter')) + except: + active = 0 + for i,r in enumerate(self.server.reporters): + selected = (i == active) + if selected: + selectedStr = ' selected' + else: + selectedStr = '' + reporterSelections.append(''%(i,selectedStr,r.getName())) + options = '\n'.join([ o.getHTML(r,title,getConfigOption) for o in r.getParameters()]) + display = ('none','')[selected] + reporterOptions.append("""\ + + %s Options + + +%s +
+ + +"""%(r.getName(),display,r.getName(),options)) + reporterSelections = '\n'.join(reporterSelections) + reporterOptionsDivs = '\n'.join(reporterOptions) + reportersArray = '[%s]'%(','.join([`r.getName()` for r in self.server.reporters])) + + if c.files: + fieldSize = min(5, len(c.files)) + attachFileOptions = '\n'.join(["""\ +""" % (i,v) for i,v in enumerate(c.files)]) + attachFileRow = """\ + + Attach: + + + + +""" % (min(5, len(c.files)), attachFileOptions) + else: + attachFileRow = "" + + result = """ + + File Bug + + + + +

+Summary > +%(reportingFor)s +File Bug

+
+ + + + + +
+ + + + + + + + + + +%(attachFileRow)s + +
Title: + +
Description: + +
+
+ + + + + +%(reporterOptionsDivs)s +
Method: + +
+
+
+ +
+
+ +%(extraIFrame)s + + +"""%locals() + + return self.send_string(result) + + def send_head(self, fields=None): + if (self.server.options.onlyServeLocal and + self.client_address[0] != '127.0.0.1'): + return self.send_error(401, 'Unauthorized host.') + + if fields is None: + fields = {} + self.fields = fields + + o = urlparse.urlparse(self.path) + self.fields = parse_query(o.query, fields) + path = posixpath.normpath(urllib.unquote(o.path)) + + # Split the components and strip the root prefix. + components = path.split('/')[1:] + + # Special case some top-level entries. + if components: + name = components[0] + if len(components)==2: + if name=='report': + return self.send_report(components[1]) + elif name=='open': + return self.send_open_report(components[1]) + elif len(components)==1: + if name=='quit': + self.server.halt() + return self.send_string('Goodbye.', 'text/plain') + elif name=='report_submit': + return self.send_report_submit() + elif name=='report_crashes': + overrides = { 'ScanView' : {}, + 'Radar' : {}, + 'Email' : {} } + for i,r in enumerate(self.server.reporters): + if r.getName() == 'Radar': + overrides['ScanView']['reporter'] = i + break + overrides['Radar']['Component'] = 'llvm - checker' + overrides['Radar']['Component Version'] = 'X' + return self.send_report(None, overrides) + elif name=='favicon.ico': + return self.send_path(posixpath.join(kResources,'bugcatcher.ico')) + + # Match directory entries. + if components[-1] == '': + components[-1] = 'index.html' + + suffix = '/'.join(components) + + # The summary may reference source files on disk using rooted + # paths. Make sure these resolve correctly for now. + # FIXME: This isn't a very good idea... we should probably + # mark rooted paths somehow. + if os.path.exists(posixpath.join('/', suffix)): + path = posixpath.join('/', suffix) + else: + path = posixpath.join(self.server.root, suffix) + + if self.server.options.debug > 1: + print >>sys.stderr, '%s: SERVER: sending path "%s"'%(sys.argv[0], + path) + return self.send_path(path) + + def send_404(self): + self.send_error(404, "File not found") + return None + + def send_path(self, path): + ctype = self.guess_type(path) + if ctype.startswith('text/'): + # Patch file instead + return self.send_patched_file(path, ctype) + else: + mode = 'rb' + try: + f = open(path, mode) + except IOError: + return self.send_404() + return self.send_file(f, ctype) + + def send_file(self, f, ctype): + # Patch files to add links, but skip binary files. + self.send_response(200) + self.send_header("Content-type", ctype) + fs = os.fstat(f.fileno()) + self.send_header("Content-Length", str(fs[6])) + self.send_header("Last-Modified", self.date_time_string(fs.st_mtime)) + self.end_headers() + return f + + def send_string(self, s, ctype='text/html', headers=True, mtime=None): + if headers: + self.send_response(200) + self.send_header("Content-type", ctype) + self.send_header("Content-Length", str(len(s))) + if mtime is None: + mtime = self.dynamic_mtime + self.send_header("Last-Modified", self.date_time_string(mtime)) + self.end_headers() + return StringIO.StringIO(s) + + def send_patched_file(self, path, ctype): + # Allow a very limited set of variables. This is pretty gross. + variables = {} + variables['report'] = '' + m = kReportFileRE.match(path) + if m: + variables['report'] = m.group(2) + + try: + f = open(path,'r') + except IOError: + return self.send_404() + fs = os.fstat(f.fileno()) + data = f.read() + for a,b in kReportReplacements: + data = a.sub(b % variables, data) + return self.send_string(data, ctype, mtime=fs.st_mtime) + + +def create_server(address, options, root): + import Reporter + + reporters = Reporter.getReporters() + + return ScanViewServer(address, ScanViewRequestHandler, + root, + reporters, + options) -- cgit v1.2.3