source: pyutilib.testdriver/trunk/pyutilib/testdriver/driver.py @ 1838

Revision 1838, 10.7 KB checked in by wehart, 4 years ago (diff)

Adding initial tests.

Line 
1#  _________________________________________________________________________
2#
3#  PyUtilib: A Python utility library.
4#  Copyright (c) 2008 Sandia Corporation.
5#  This software is distributed under the BSD License.
6#  Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
7#  the U.S. Government retains certain rights in this software.
8#  _________________________________________________________________________
9#
10
11__all__ = ['run', 'main', 'create_test_suites']
12
13import os
14from os.path import dirname, abspath
15import sys
16import optparse
17import plugins
18from pyutilib.misc import Options
19from pyutilib.component.core import ExtensionPoint, PluginGlobals
20import pyutilib.th as unittest
21import copy
22import types
23
24
25def validate_test_config(suite):
26    if suite is None:
27        raise IOError, "Empty suite indicates problem processing suite configuration"
28    #
29    tmp = set(suite.keys())
30    if not tmp.issubset(set(['python','solvers','problems','suites', 'driver'])):
31        raise IOError, "Unexpected test sections: "+str(suite.keys())
32    #
33    if 'python' in suite:
34        if not type(suite['python']) is list:
35            raise IOError, "Expected list of Python expressions"
36    #
37    if 'solvers' in suite:
38        if not type(suite['solvers']) is dict:
39            raise IOError, "Expected dictionary of solvers"
40        for key in suite['solvers']:
41            if suite['solvers'][key] is None:
42                suite['solvers'][key] = {}
43            elif not type(suite['solvers'][key]) is dict:
44                raise IOError, "Expected solvers to have a dictionary of options: %s" % str(suite)
45    #
46    if 'problems' in suite:
47        if not type(suite['problems']) is dict:
48            raise IOError, "Expected dictionary of problems"
49        for key in suite['problems']:
50            if suite['problems'][key] is None:
51                suite['problems'][key] = {}
52            elif not type(suite['problems'][key]) is dict:
53                raise IOError, "Expected problems to have a dictionary of options"
54    #
55    if 'suites' in suite:
56        if not type(suite['suites']) is dict:
57            raise IOError, "Expected dictionary of suites"
58        for key in suite['suites']:
59            if suite['suites'][key] is None:
60                suite['suites'][key] = {}
61            elif not type(suite['suites'][key]) is dict:
62                raise IOError, "Expected suites to have a dictionary of options"
63
64
65
66@unittest.nottest
67def create_test_suites(filename=None, config=None, _globals=None, options=None):
68    if options is None:
69        options = Options()
70    if not filename is None:
71        if options.currdir is None:
72            options.currdir = dirname(abspath(filename))+os.sep
73        #
74        ep = ExtensionPoint(plugins.ITestParser)
75        ftype = os.path.splitext(filename)[1]
76        if not ftype == '':
77            ftype = ftype[1:]
78        service = ep.service(ftype)
79        if service is None:
80            raise IOError, "Unknown file type.  Cannot load test configuration from file '%s'" % filename
81        config = service.load_test_config(filename)
82    #service.print_test_config(config)
83    validate_test_config(config)
84    #
85    # Create test driver, which is put in the global namespace
86    #
87    driver = plugins.TestDriverFactory(config['driver'])
88    if driver is None:
89        raise IOError, "Unexpected test driver '%s'" % config['driver']
90    _globals["test_driver"] = driver
91    #
92    # Evaluate Python expressions
93    #
94    for item in config.get('python',[]):
95        try:
96            eval(item, _globals)
97        except Exception, err:
98            print "ERROR evaluating '%s'" % item
99    #
100    # Generate suite
101    #
102    for suite in config.get('suites',{}):
103        create_test_suite(suite, config, _globals, options)
104
105
106@unittest.nottest
107def create_test_suite(suite, config, _globals, options):
108    #
109    # Skip suite creation if the options category is not in the list of test suite categories
110    #
111    if not (options.category is None or options.category in config['suites'][suite].get('categories',[])):
112        return
113    #
114    # Create test driver
115    #
116    if suite in _globals:
117        raise IOError, "Cannot create suite %s since there is another symbol with that name in the global namespace!" % suite
118    def setUpClassFn(cls):
119        options = cls._options[None]
120        cls._test_driver.setUpClass(cls,options)
121    _globals[suite] = type(suite,(unittest.TestCase,),{'setUpClass': classmethod(setUpClassFn)})
122    _globals[suite]._options[None] = options
123    setattr(_globals[suite],'_test_driver', _globals['test_driver'])
124    setattr(_globals[suite],'suite_categories', config['suites'][suite].get('categories',[]))
125    #
126    # Create test functions
127    #
128    for solver in config['suites'][suite]['solvers']:
129        sname = solver['solver']
130        for problem in config['suites'][suite]['problems']:
131            test_name = sname+"_"+problem
132            #
133            def fn(testcase, name):
134                options = testcase._options[name]
135                fn.test_driver.setUp(testcase, options)
136                ans = fn.test_driver.run_test(testcase, name, options)
137                fn.test_driver.tearDown(testcase, options)
138                return ans
139            fn.test_driver = _globals['test_driver']
140            #
141            _options = Options()
142            for attr,value in config['problems'].get(problem,{}).items():
143                _options[attr] = value
144            _name = solver.get('name', solver['solver'])
145            for attr,value in config['solvers'].get(sname,{}).items():
146                _options[attr] = value
147                if attr == 'name':
148                    _name = value
149            for attr,value in solver.items():
150                _options[attr] = value
151            problem_options = config['suites'][suite]['problems'][problem]
152            if not problem_options is None:
153                for attr,value in problem_options.items():
154                    _options[attr] = value
155            _options.solver = _name
156            _options.problem = problem
157            _options.currdir = options.currdir
158            #
159            _globals[suite].add_fn_test(name=test_name, fn=fn, options=_options)
160
161
162
163def run(argv, _globals=None):
164
165    parser = optparse.OptionParser()
166    parser.remove_option('-h')
167
168    parser.add_option('-h','--help',
169        action='store_true',
170        dest='help',
171        default=False,
172        help='Print command options')
173
174    parser.add_option('-d','--debug',
175        action='store_true',
176        dest='debug',
177        default=False,
178        help='Set debugging flag')
179
180    parser.add_option('-v','--verbose',
181        action='store_true',
182        dest='verbose',
183        default=False,
184        help='Verbose output')
185
186    parser.add_option('-q','--quiet',
187        action='store_true',
188        dest='quiet',
189        default=False,
190        help='Minimal output')
191
192    parser.add_option('-f','--failfast',
193        action='store_true',
194        dest='failfast',
195        default=False,
196        help='Stop on first failure')
197
198    parser.add_option('-c','--catch',
199        action='store_true',
200        dest='catch',
201        default=False,
202        help='Catch control-C and display results')
203
204    parser.add_option('-b','--buffer',
205        action='store_true',
206        dest='buffer',
207        default=False,
208        help='Buffer stdout and stderr durring test runs')
209
210    parser.add_option('--cat', '--category',
211        action='store',
212        dest='category',
213        default=None,
214        help='Define the category of the test suite that is executed')
215   
216    parser.add_option('--help-suites',
217        action='store_true',
218        dest='help_suites',
219        default=False,
220        help='Print the test suites that can be executed')
221   
222    parser.add_option('--help-tests',
223        action='store',
224        dest='help_tests',
225        default=None,
226        help='Print the tests in the specified test suite')
227   
228    parser.add_option('--help-categories',
229        action='store_true',
230        dest='help_categories',
231        default=False,
232        help='Print the test suite categories that can be specified')
233   
234    sys.argv = argv
235    _options, args = parser.parse_args(sys.argv)
236    if _options.help:
237        parser.print_help()
238
239        print """
240Examples:
241  %s                               - run all test suites
242  %s MyTestCase.testSomething      - run MyTestCase.testSomething
243  %s MyTestCase                    - run all 'test*' test methods
244                                               in MyTestCase
245""" % (args[0],args[0],args[0])
246        return
247    if _globals is None:
248        _globals=globals()
249
250    if _options.help_tests and not _globals is None:
251        suite = _globals.get(_options.help_tests, None)
252        if not type(suite) is types.TypeType:
253            print "Test suite '%s' not found!" % str(_options.help_tests)
254            return
255        tests = []
256        for item in dir(suite):
257            if item.startswith('test'):
258                tests.append(item)
259        print ""
260        if len(tests) > 0:
261            print "Tests defined in test suite %s:" % _options.help_tests
262            for tmp in sorted(tests):
263                print "    "+tmp
264        else:
265            print "No tests defined in test suite %s:" % _options.help_tests
266        print ""
267        return
268
269    if (_options.help_suites or _options.help_categories) and not _globals is None:
270        suites = []
271        categories = set()
272        for key in _globals.keys():
273            if type(_globals[key]) is types.TypeType:
274                suites.append(key)
275                for c in _globals[key].suite_categories:
276                    categories.add(c)
277        if _options.help_suites:
278            print ""
279            if len(suites) > 0:
280                print "Test suites defined in %s:" % os.path.basename(argv[0])
281                for suite in suites:
282                    print "    "+suite
283            else:
284                print "No test suites defined in %s!" % os.path.basename(argv[0])
285            print ""
286        if _options.help_categories:
287            tmp = list(categories)
288            print ""
289            if len(tmp) > 0:
290                print "Test suite categories defined in %s:" % os.path.basename(argv[0])
291                for c in sorted(tmp):
292                    print "    "+c
293            else:
294                print "No test suite categories defined in %s:" % os.path.basename(argv[0])
295            print ""
296        return
297
298    tmp = [args[0]]
299    if _options.quiet:
300        tmp.append('-q')
301    if _options.verbose or _options.debug:
302        tmp.append('-v')
303    if _options.failfast:
304        tmp.append('-f')
305    if _options.catch:
306        tmp.append('-c')
307    if _options.buffer:
308        tmp.append('-b')
309    tmp += args[1:]
310    sys.argv = tmp
311    #
312    # Execute the unittest main function to run tests
313    #
314    unittest.main(module=_globals['__name__'])
315       
316       
317def main(_globals=None):
318    run(sys.argv, _globals=_globals)
Note: See TracBrowser for help on using the repository browser.