Make parallelized run_corpus_tests.py handle ctrl-c.
[pdfium.git] / testing / tools / run_corpus_tests.py
1 #!/usr/bin/env python
2 # Copyright 2015 The PDFium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
5
6 import cStringIO
7 import functools
8 import multiprocessing
9 import optparse
10 import os
11 import re
12 import shutil
13 import subprocess
14 import sys
15
16 import common
17 import pngdiffer
18 import suppressor
19
20 class KeyboardInterruptError(Exception): pass
21
22 # Nomenclature:
23 #   x_root - "x"
24 #   x_filename - "x.ext"
25 #   x_path - "path/to/a/b/c/x.ext"
26 #   c_dir - "path/to/a/b/c"
27
28 def test_one_file(input_filename, source_dir, working_dir,
29                   pdfium_test_path, image_differ, redirect_output=False):
30   input_path = os.path.join(source_dir, input_filename)
31   pdf_path = os.path.join(working_dir, input_filename)
32
33   # Remove any existing generated images from previous runs.
34   actual_images = image_differ.GetActualFiles(
35       input_filename, source_dir, working_dir)
36   for image in actual_images:
37     if os.path.exists(image):
38       os.remove(image)
39
40   shutil.copyfile(input_path, pdf_path)
41   sys.stdout.flush()
42   error = common.RunCommand([pdfium_test_path, '--png', pdf_path],
43                             redirect_output)
44   if error:
45     print "FAILURE: " + input_filename + "; " + str(error)
46     return False
47   return not image_differ.HasDifferences(input_filename, source_dir,
48                                          working_dir, redirect_output)
49
50
51 def test_one_file_parallel(working_dir, pdfium_test_path, image_differ,
52                            test_case):
53   """Wrapper function to call test_one_file() and redirect output to stdout."""
54   try:
55     old_stdout = sys.stdout
56     old_stderr = sys.stderr
57     sys.stdout = cStringIO.StringIO()
58     sys.stderr = sys.stdout
59     input_filename, source_dir = test_case
60     result = test_one_file(input_filename, source_dir, working_dir,
61                            pdfium_test_path, image_differ, True);
62     output = sys.stdout
63     sys.stdout = old_stdout
64     sys.stderr = old_stderr
65     return (result, output.getvalue(), input_filename, source_dir)
66   except KeyboardInterrupt:
67     raise KeyboardInterruptError()
68
69
70 def handle_result(test_suppressor, input_filename, input_path, result,
71                   surprises, failures):
72   if test_suppressor.IsSuppressed(input_filename):
73     if result:
74       surprises.append(input_path)
75   else:
76     if not result:
77       failures.append(input_path)
78
79
80 def main():
81   parser = optparse.OptionParser()
82   parser.add_option('--build-dir', default=os.path.join('out', 'Debug'),
83                     help='relative path from the base source directory')
84   parser.add_option('-j', default=multiprocessing.cpu_count(),
85                     dest='num_workers', type='int',
86                     help='run NUM_WORKERS jobs in parallel')
87   options, args = parser.parse_args()
88   finder = common.DirectoryFinder(options.build_dir)
89   pdfium_test_path = finder.ExecutablePath('pdfium_test')
90   if not os.path.exists(pdfium_test_path):
91     print "FAILURE: Can't find test executable '%s'" % pdfium_test_path
92     print "Use --build-dir to specify its location."
93     return 1
94   working_dir = finder.WorkingDir(os.path.join('testing', 'corpus'))
95   if not os.path.exists(working_dir):
96     os.makedirs(working_dir)
97
98   test_suppressor = suppressor.Suppressor(finder)
99   image_differ = pngdiffer.PNGDiffer(finder)
100
101   # test files are under .../pdfium/testing/corpus.
102   failures = []
103   surprises = []
104   walk_from_dir = finder.TestingDir('corpus');
105   input_file_re = re.compile('^[a-zA-Z0-9_.]+[.]pdf$')
106   test_cases = []
107   for source_dir, _, filename_list in os.walk(walk_from_dir):
108     for input_filename in filename_list:
109       if input_file_re.match(input_filename):
110         input_path = os.path.join(source_dir, input_filename)
111         if os.path.isfile(input_path):
112           test_cases.append((input_filename, source_dir))
113
114   if options.num_workers > 1:
115     try:
116       pool = multiprocessing.Pool(options.num_workers)
117       worker_func = functools.partial(test_one_file_parallel, working_dir,
118                                       pdfium_test_path, image_differ)
119       worker_results = pool.imap(worker_func, test_cases)
120       for worker_result in worker_results:
121         result, output, input_filename, source_dir = worker_result
122         input_path = os.path.join(source_dir, input_filename)
123         sys.stdout.write(output)
124         handle_result(test_suppressor, input_filename, input_path, result,
125                       surprises, failures)
126       pool.close()
127     except KeyboardInterrupt:
128       pool.terminate()
129     finally:
130       pool.join()
131   else:
132     for test_case in test_cases:
133       input_filename, source_dir = test_case
134       result = test_one_file(input_filename, source_dir, working_dir,
135                              pdfium_test_path, image_differ)
136       handle_result(test_suppressor, input_filename, input_path, result,
137                     surprises, failures)
138
139   if surprises:
140     surprises.sort()
141     print '\n\nUnexpected Successes:'
142     for surprise in surprises:
143       print surprise;
144
145   if failures:
146     failures.sort()
147     print '\n\nSummary of Failures:'
148     for failure in failures:
149       print failure
150     return 1
151
152   return 0
153
154
155 if __name__ == '__main__':
156   sys.exit(main())