Clean up CPDF_AnnotList.
[pdfium.git] / testing / tools / run_corpus_tests.py
1 #!/usr/bin/env python
2 # Copyright 2015 The PDFium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
5
6 import cStringIO
7 import functools
8 import multiprocessing
9 import optparse
10 import os
11 import re
12 import shutil
13 import subprocess
14 import sys
15
16 import common
17 import pngdiffer
18 import suppressor
19
20 class KeyboardInterruptError(Exception): pass
21
22 # Nomenclature:
23 #   x_root - "x"
24 #   x_filename - "x.ext"
25 #   x_path - "path/to/a/b/c/x.ext"
26 #   c_dir - "path/to/a/b/c"
27
28 def test_one_file(input_filename, source_dir, working_dir,
29                   pdfium_test_path, image_differ, redirect_output=False):
30   input_path = os.path.join(source_dir, input_filename)
31   pdf_path = os.path.join(working_dir, input_filename)
32
33   # Remove any existing generated images from previous runs.
34   actual_images = image_differ.GetActualFiles(
35       input_filename, source_dir, working_dir)
36   for image in actual_images:
37     if os.path.exists(image):
38       os.remove(image)
39
40   shutil.copyfile(input_path, pdf_path)
41   sys.stdout.flush()
42   error = common.RunCommand([pdfium_test_path, '--png', pdf_path],
43                             redirect_output)
44   if error:
45     print "FAILURE: " + input_filename + "; " + str(error)
46     return False
47   return not image_differ.HasDifferences(input_filename, source_dir,
48                                          working_dir, redirect_output)
49
50
51 def test_one_file_parallel(working_dir, pdfium_test_path, image_differ,
52                            test_case):
53   """Wrapper function to call test_one_file() and redirect output to stdout."""
54   try:
55     old_stdout = sys.stdout
56     old_stderr = sys.stderr
57     sys.stdout = cStringIO.StringIO()
58     sys.stderr = sys.stdout
59     input_filename, source_dir = test_case
60     result = test_one_file(input_filename, source_dir, working_dir,
61                            pdfium_test_path, image_differ, True);
62     output = sys.stdout
63     sys.stdout = old_stdout
64     sys.stderr = old_stderr
65     return (result, output.getvalue(), input_filename, source_dir)
66   except KeyboardInterrupt:
67     raise KeyboardInterruptError()
68
69
70 def handle_result(test_suppressor, input_filename, input_path, result,
71                   surprises, failures):
72   if test_suppressor.IsSuppressed(input_filename):
73     if result:
74       surprises.append(input_path)
75   else:
76     if not result:
77       failures.append(input_path)
78
79
80 def main():
81   parser = optparse.OptionParser()
82   parser.add_option('--build-dir', default=os.path.join('out', 'Debug'),
83                     help='relative path from the base source directory')
84   parser.add_option('-j', default=multiprocessing.cpu_count(),
85                     dest='num_workers', type='int',
86                     help='run NUM_WORKERS jobs in parallel')
87   options, args = parser.parse_args()
88   finder = common.DirectoryFinder(options.build_dir)
89   pdfium_test_path = finder.ExecutablePath('pdfium_test')
90   if not os.path.exists(pdfium_test_path):
91     print "FAILURE: Can't find test executable '%s'" % pdfium_test_path
92     print "Use --build-dir to specify its location."
93     return 1
94   working_dir = finder.WorkingDir(os.path.join('testing', 'corpus'))
95   if not os.path.exists(working_dir):
96     os.makedirs(working_dir)
97
98   test_suppressor = suppressor.Suppressor(finder)
99   image_differ = pngdiffer.PNGDiffer(finder)
100
101   # test files are under .../pdfium/testing/corpus.
102   failures = []
103   surprises = []
104   walk_from_dir = finder.TestingDir('corpus');
105   input_file_re = re.compile('^[a-zA-Z0-9_.]+[.]pdf$')
106   test_cases = []
107
108   if len(args):
109     for file_name in args:
110       input_path = os.path.join(walk_from_dir, file_name)
111       if not os.path.isfile(input_path):
112         print "Can't find test file '%s'" % file_name
113         return 1
114
115       test_cases.append((os.path.basename(input_path),
116                          os.path.dirname(input_path)))
117   else:
118     for source_dir, _, filename_list in os.walk(walk_from_dir):
119       for input_filename in filename_list:
120         if input_file_re.match(input_filename):
121           input_path = os.path.join(source_dir, input_filename)
122           if os.path.isfile(input_path):
123             test_cases.append((input_filename, source_dir))
124
125   if options.num_workers > 1 and len(test_cases) > 1:
126     try:
127       pool = multiprocessing.Pool(options.num_workers)
128       worker_func = functools.partial(test_one_file_parallel, working_dir,
129                                       pdfium_test_path, image_differ)
130       worker_results = pool.imap(worker_func, test_cases)
131       for worker_result in worker_results:
132         result, output, input_filename, source_dir = worker_result
133         input_path = os.path.join(source_dir, input_filename)
134         sys.stdout.write(output)
135         handle_result(test_suppressor, input_filename, input_path, result,
136                       surprises, failures)
137       pool.close()
138     except KeyboardInterrupt:
139       pool.terminate()
140     finally:
141       pool.join()
142   else:
143     for test_case in test_cases:
144       input_filename, source_dir = test_case
145       result = test_one_file(input_filename, source_dir, working_dir,
146                              pdfium_test_path, image_differ)
147       handle_result(test_suppressor, input_filename, input_path, result,
148                     surprises, failures)
149
150   if surprises:
151     surprises.sort()
152     print '\n\nUnexpected Successes:'
153     for surprise in surprises:
154       print surprise;
155
156   if failures:
157     failures.sort()
158     print '\n\nSummary of Failures:'
159     for failure in failures:
160       print failure
161     return 1
162
163   return 0
164
165
166 if __name__ == '__main__':
167   sys.exit(main())