From 8e54687138041772c5d1d82d3fd8d0e743041ada Mon Sep 17 00:00:00 2001 From: Satish Suradkar Date: Tue, 2 Aug 2016 18:01:43 +0530 Subject: [PATCH 1/8] main changes in resttest.py & tests.py, others are about import only --- pyresttest/benchmarks.py | 14 +-- pyresttest/contenthandling.py | 4 +- pyresttest/generators.py | 4 +- pyresttest/parsing.py | 6 +- pyresttest/resttest.py | 160 +++++++++++++++++++++++----------- pyresttest/tests.py | 40 ++++++--- pyresttest/validators.py | 8 +- 7 files changed, 154 insertions(+), 82 deletions(-) mode change 100644 => 100755 pyresttest/resttest.py mode change 100644 => 100755 pyresttest/tests.py diff --git a/pyresttest/benchmarks.py b/pyresttest/benchmarks.py index f81a91fb..e440026a 100644 --- a/pyresttest/benchmarks.py +++ b/pyresttest/benchmarks.py @@ -3,19 +3,19 @@ import pycurl import sys -from . import tests -from .tests import Test -from . import parsing -from .parsing import * +import tests +from tests import Test +import parsing +from parsing import * # Python 2/3 switches if sys.version_info[0] > 2: from past.builtins import basestring # Python 3 compatibility shims -from . import six -from .six import binary_type -from .six import text_type +import six +from six import binary_type +from six import text_type """ Encapsulates logic related to benchmarking diff --git a/pyresttest/contenthandling.py b/pyresttest/contenthandling.py index 52e35598..ab041a11 100644 --- a/pyresttest/contenthandling.py +++ b/pyresttest/contenthandling.py @@ -1,8 +1,8 @@ import os import sys -from . import parsing -from .parsing import * +import parsing +from parsing import * # Python 2/3 switches PYTHON_MAJOR_VERSION = sys.version_info[0] diff --git a/pyresttest/generators.py b/pyresttest/generators.py index 70fd2233..bea717e8 100644 --- a/pyresttest/generators.py +++ b/pyresttest/generators.py @@ -4,8 +4,8 @@ import logging import sys -from . import parsing -from .parsing import flatten_dictionaries, lowercase_keys, safe_to_bool +import parsing +from parsing import flatten_dictionaries, lowercase_keys, safe_to_bool # Python 3 compatibility if sys.version_info[0] > 2: diff --git a/pyresttest/parsing.py b/pyresttest/parsing.py index 9e20f29d..14650dae 100644 --- a/pyresttest/parsing.py +++ b/pyresttest/parsing.py @@ -4,9 +4,9 @@ # Python 3 compatibility shims -from . import six -from .six import binary_type -from .six import text_type +import six +from six import binary_type +from six import text_type # Python 2/3 switches PYTHON_MAJOR_VERSION = sys.version_info[0] diff --git a/pyresttest/resttest.py b/pyresttest/resttest.py old mode 100644 new mode 100755 index 31d42288..94cccb69 --- a/pyresttest/resttest.py +++ b/pyresttest/resttest.py @@ -12,6 +12,7 @@ from optparse import OptionParser from email import message_from_string # For headers handling import time +#import pdb try: from cStringIO import StringIO as MyIO @@ -32,34 +33,34 @@ if __name__ == '__main__': sys.path.append(os.path.dirname(os.path.dirname( os.path.realpath(__file__)))) - from pyresttest.six import text_type - from pyresttest.binding import Context - from pyresttest import generators - from pyresttest import validators - from pyresttest import tests - from pyresttest.generators import parse_generator - from pyresttest.parsing import flatten_dictionaries, lowercase_keys, safe_to_bool, safe_to_json - - from pyresttest.validators import Failure - from pyresttest.tests import Test, DEFAULT_TIMEOUT - from pyresttest.benchmarks import Benchmark, AGGREGATES, METRICS, parse_benchmark + from six import text_type + from binding import Context + import generators + import validators + import tests + from generators import parse_generator + from parsing import flatten_dictionaries, lowercase_keys, safe_to_bool, safe_to_json + + from validators import Failure + from tests import Test, DEFAULT_TIMEOUT + from benchmarks import Benchmark, AGGREGATES, METRICS, parse_benchmark else: # Normal imports - from . import six - from .six import text_type + import six + from six import text_type # Pyresttest internals - from . import binding - from .binding import Context - from . import generators - from .generators import parse_generator - from . import parsing - from .parsing import flatten_dictionaries, lowercase_keys, safe_to_bool, safe_to_json - from . import validators - from .validators import Failure - from . import tests - from .tests import Test, DEFAULT_TIMEOUT - from . import benchmarks - from .benchmarks import Benchmark, AGGREGATES, METRICS, parse_benchmark + import binding + from binding import Context + import generators + from generators import parse_generator + import parsing + from parsing import flatten_dictionaries, lowercase_keys, safe_to_bool, safe_to_json + import validators + from validators import Failure + import tests + from tests import Test, DEFAULT_TIMEOUT + import benchmarks + from benchmarks import Benchmark, AGGREGATES, METRICS, parse_benchmark """ Executable class, ties everything together into the framework. @@ -81,6 +82,9 @@ logger = logging.getLogger('pyresttest') DIR_LOCK = threading.RLock() # Guards operations changing the working directory + +test_result = dict() +test_result_list = [] class cd: """Context manager for changing the current working directory""" # http://stackoverflow.com/questions/431684/how-do-i-cd-in-python/13197763#13197763 @@ -95,7 +99,7 @@ def __enter__(self): os.chdir(self.newPath) def __exit__(self, etype, value, traceback): - if self.newPath: # Don't CD to nothingness + if self.newPath: # Don't CD to nothingness os.chdir(self.savedPath) DIR_LOCK.release() @@ -114,7 +118,7 @@ class TestConfig: # Binding and creation of generators variable_binds = None - generators = None # Map of generator name to generator function + generators = None # Map of generator name to generator functionOB def __str__(self): return json.dumps(self, default=safe_to_json) @@ -193,7 +197,7 @@ def parse_headers(header_string): return list() # Python 2.6 message header parsing fails for Unicode strings, 2.7 is fine. Go figure. - if sys.version_info < (2,7): + if sys.version_info < (2, 7): header_msg = message_from_string(headers.encode(HEADER_ENCODING)) return [(text_type(k.lower(), HEADER_ENCODING), text_type(v, HEADER_ENCODING)) for k, v in header_msg.items()] @@ -260,6 +264,7 @@ def parse_testsets(base_url, test_structure, test_files=set(), working_directory elif key == u'config' or key == u'configuration': test_config = parse_configuration( node[key], base_config=test_config) + testset = TestSet() testset.tests = tests_out testset.config = test_config @@ -283,6 +288,8 @@ def parse_configuration(node, base_config=None): test_config.print_bodies = safe_to_bool(value) elif key == u'retries': test_config.retries = int(value) + elif key == u'delay': + test_config.delay = int(value) elif key == u'variable_binds': if not test_config.variable_binds: test_config.variable_binds = dict() @@ -305,6 +312,8 @@ def read_file(path): f.close() return string +#flag to check test is already retried +is_retried = False def run_test(mytest, test_config=TestConfig(), context=None, curl_handle=None, *args, **kwargs): """ Put together test pieces: configure & run actual test, return results """ @@ -346,9 +355,13 @@ def run_test(mytest, test_config=TestConfig(), context=None, curl_handle=None, * print("\n%s" % templated_test.body) raw_input("Press ENTER when ready (%d): " % (mytest.delay)) - if mytest.delay > 0: - print("Delaying for %ds" % mytest.delay) - time.sleep(mytest.delay) +# if mytest.delay > 0: +# print("Delaying for %ds" % mytest.delay) +# time.sleep(mytest.delay) + for test_need in mytest.depends_on: + if not test_result[test_need] or test_result[test_need] == 'skip': + print "\n\033[1;31m 'test: {0}' depends on 'test: {1}' \033[0m".format(mytest.name, test_need) + return try: curl.perform() # Run the actual call @@ -374,15 +387,50 @@ def run_test(mytest, test_config=TestConfig(), context=None, curl_handle=None, * logger.debug("Initial Test Result, based on expected response code: " + str(response_code in mytest.expected_status)) + flag = 0 + global is_retried + retry = 0 + + if mytest.retries >= 0: + retry = mytest.retries + flag = 1 + is_retried = True + elif not is_retried: + retry = test_config.retries + flag = 2 + if response_code in mytest.expected_status: result.passed = True else: - # Invalid response code - result.passed = False - failure_message = "Invalid HTTP response code: response code {0} not in expected codes [{1}]".format( - response_code, mytest.expected_status) - result.failures.append(Failure( - message=failure_message, details=None, failure_type=validators.FAILURE_INVALID_RESPONSE)) + # rerty not define + if retry == 0: + # Invalid response code + result.passed = False + failure_message = "Invalid HTTP response code: response code {0} not in expected codes [{1}]".format( + response_code, mytest.expected_status) + result.failures.append(Failure( + message=failure_message, details=None, failure_type=validators.FAILURE_INVALID_RESPONSE)) + + # retry test + else: + retry -= 1 + #retry from test + if flag == 1: + #print "########## Retrying test :",mytest.name + mytest.retries = retry + #print "Delaying : ",mytest.delay + time.sleep(mytest.delay) + return run_test(mytest, test_config, context, curl_handle) + + #retry from config + if flag == 2: + #print "########## Retrying :",mytest.name + test_config.retries = retry + #print "Delaying : ",test_config.dalay + time.sleep(test_config.delay) + #mytest.expected_status = [204] + return run_test(mytest, test_config, context, curl_handle) + # Parse HTTP headers try: @@ -399,7 +447,6 @@ def run_test(mytest, test_config=TestConfig(), context=None, curl_handle=None, * # ' + str(test_config.print_bodies or not result.passed) head = result.response_headers - # execute validator on body if result.passed is True: body = result.body @@ -647,27 +694,36 @@ def run_testsets(testsets): myinteractive = True if myinteractive or myconfig.interactive else False + skip = 0 # Run tests, collecting statistics as needed for test in mytests: # Initialize the dictionaries to store test fail counts and results if test.group not in group_results: group_results[test.group] = list() group_failure_counts[test.group] = 0 - + + global is_retried + is_retried = False + print "\n ==================================================== \n" result = run_test(test, test_config=myconfig, context=context, curl_handle=curl_handle) - result.body = None # Remove the body, save some memory! + + if result is None: + skip += 1 + test_result[test.name] = "skip" + continue + + test_result[test.name] = result.passed + result.body = None # Remove the body, save some memory! if not result.passed: # Print failure, increase failure counts for that test group - # Use result test URL to allow for templating + # Use result test URL to allow for templating logger.error('Test Failed: ' + test.name + " URL=" + result.test.url + - " Group=" + test.group + " HTTP Status Code: " + str(result.response_code)) - + " Group=" + test.group + " HTTP Status Code: " + str(result.response_code)) # Print test failure reasons if result.failures: for failure in result.failures: log_failure(failure, context=context, test_config=myconfig) - # Increment test failure counts for that group (adding an entry # if not present) failures = group_failure_counts[test.group] @@ -676,7 +732,7 @@ def run_testsets(testsets): else: # Test passed, print results logger.info('Test Succeeded: ' + test.name + - " URL=" + test.url + " Group=" + test.group) + " URL=" + test.url + " Group=" + test.group) # Add results for this test group to the resultset group_results[test.group].append(result) @@ -710,7 +766,7 @@ def run_testsets(testsets): write_method(my_file, benchmark_result, benchmark, test_config=myconfig) my_file.close() - + print "\n ==================================================== \n" if myinteractive: # a break for when interactive bits are complete, before summary data print("===================================") @@ -722,13 +778,18 @@ def run_testsets(testsets): total_failures = total_failures + failures passfail = {True: u'SUCCEEDED: ', False: u'FAILED: '} - output_string = "Test Group {0} {1}: {2}/{3} Tests Passed!".format(group, passfail[failures == 0], str(test_count - failures), str(test_count)) - + output_string = "Test Group {0} {1}: {2}/{3} Tests Passed!\nTest Group {0} SKIPPED: {4}"\ + .format(group, passfail[failures == 0], str(test_count - failures), str(test_count), str(skip)) + if myconfig.skip_term_colors: - print(output_string) + print(output_string) + print test_result else: if failures > 0: print('\033[91m' + output_string + '\033[0m') + str1 = json.dumps(test_result, indent=4, sort_keys=True) + with open('test_result.json', 'w') as out: + json.dump(test_result, out) else: print('\033[92m' + output_string + '\033[0m') @@ -816,7 +877,6 @@ def main(args): test_file = args['test'] test_structure = read_test_file(test_file) - my_vars = None if 'vars' in args and args['vars'] is not None: my_vars = yaml.safe_load(args['vars']) diff --git a/pyresttest/tests.py b/pyresttest/tests.py old mode 100644 new mode 100755 index 56365fce..d9f2e88f --- a/pyresttest/tests.py +++ b/pyresttest/tests.py @@ -5,12 +5,11 @@ import pycurl import sys - -from . import contenthandling -from .contenthandling import ContentHandler -from . import validators -from . import parsing -from .parsing import * +import contenthandling +from contenthandling import ContentHandler +import validators +import parsing +from parsing import * # Find the best implementation available on this platform try: @@ -30,11 +29,11 @@ import urlparse # Python 3 compatibility shims -from . import six -from .six import binary_type -from .six import text_type -from .six import iteritems -from .six.moves import filter as ifilter +import six +from six import binary_type +from six import text_type +from six import iteritems +from six.moves import filter as ifilter """ Pull out the Test objects and logic associated with them @@ -90,8 +89,15 @@ def coerce_list_of_ints(val): else: return [int(val)] +def coerce_list_of_strings(val): + """ If single value, try to parse as string, else try to parse as list of string """ + if isinstance(val, list): + return [x for x in val] + else: + return [val] + class Test(object): - """ Describes a REST test """ + """ Describes a REST test "/delay""" _url = None expected_status = [200] # expected HTTP status code or codes _body = None @@ -106,6 +112,8 @@ class Test(object): auth_password = None auth_type = pycurl.HTTPAUTH_BASIC delay = 0 + retries = 0 + depends_on = [] curl_options = None templates = None # Dictionary of template to compiled template @@ -406,7 +414,8 @@ def parse_test(cls, base_url, node, input_test=None, test_path=None): This is to say: list(dict(),dict()) or dict(key,value) --> dict() for some elements Accepted structure must be a single dictionary of key-value pairs for test configuration """ - + + mytest = input_test if not mytest: mytest = Test() @@ -428,6 +437,8 @@ def parse_test(cls, base_url, node, input_test=None, test_path=None): u'expected_status': [coerce_list_of_ints], u'delay': [lambda x: int(x)], u'stop_on_failure': [safe_to_bool], + u'retries': [lambda x: int(x)], + u'depends_on': [coerce_list_of_strings], # Templated / special handling #u'url': [coerce_templatable, set_templated), # TODO: special handling for templated content, sigh @@ -458,6 +469,7 @@ def use_config_parser(configobject, configelement, configvalue): # Copy/convert input elements into appropriate form for a test object for configelement, configvalue in node.items(): + if use_config_parser(mytest, configelement, configvalue): continue @@ -492,7 +504,7 @@ def use_config_parser(configobject, configelement, configvalue): # Safe because length can only be 1 for extractor_type, extractor_config in extractor.items(): mytest.extract_binds[variable_name] = validators.parse_extractor(extractor_type, extractor_config) - + elif configelement == u'validators': # Add a list of validators diff --git a/pyresttest/validators.py b/pyresttest/validators.py index d59cbbea..3238e70c 100644 --- a/pyresttest/validators.py +++ b/pyresttest/validators.py @@ -8,12 +8,12 @@ import sys # Local module imports -from . import parsing +import parsing # Python 3 compatibility shims -from . import six -from .six import binary_type -from .six import text_type +import six +from six import binary_type +from six import text_type # Python 3 compatibility PYTHON_MAJOR_VERSION = sys.version_info[0] From cf747148ce32ee060c0ef4df5e4a1bc444623e64 Mon Sep 17 00:00:00 2001 From: Satish Suradkar Date: Tue, 2 Aug 2016 18:07:08 +0530 Subject: [PATCH 2/8] Reverted chmod +x --- pyresttest/resttest.py | 0 pyresttest/tests.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 pyresttest/resttest.py mode change 100755 => 100644 pyresttest/tests.py diff --git a/pyresttest/resttest.py b/pyresttest/resttest.py old mode 100755 new mode 100644 diff --git a/pyresttest/tests.py b/pyresttest/tests.py old mode 100755 new mode 100644 From 0c23aada0523f56f3e37d338824712a94171436b Mon Sep 17 00:00:00 2001 From: Satish Suradkar Date: Wed, 3 Aug 2016 14:11:13 +0530 Subject: [PATCH 3/8] Generate test_result.json file --- pyresttest/resttest.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/pyresttest/resttest.py b/pyresttest/resttest.py index 94cccb69..4806ef2d 100644 --- a/pyresttest/resttest.py +++ b/pyresttest/resttest.py @@ -402,7 +402,7 @@ def run_test(mytest, test_config=TestConfig(), context=None, curl_handle=None, * if response_code in mytest.expected_status: result.passed = True else: - # rerty not define + # if retry not define if retry == 0: # Invalid response code result.passed = False @@ -416,19 +416,16 @@ def run_test(mytest, test_config=TestConfig(), context=None, curl_handle=None, * retry -= 1 #retry from test if flag == 1: - #print "########## Retrying test :",mytest.name + print "########## Retrying test : ",mytest.name mytest.retries = retry - #print "Delaying : ",mytest.delay time.sleep(mytest.delay) return run_test(mytest, test_config, context, curl_handle) #retry from config if flag == 2: - #print "########## Retrying :",mytest.name + print "########## Retrying : ",mytest.name test_config.retries = retry - #print "Delaying : ",test_config.dalay time.sleep(test_config.delay) - #mytest.expected_status = [204] return run_test(mytest, test_config, context, curl_handle) @@ -781,15 +778,15 @@ def run_testsets(testsets): output_string = "Test Group {0} {1}: {2}/{3} Tests Passed!\nTest Group {0} SKIPPED: {4}"\ .format(group, passfail[failures == 0], str(test_count - failures), str(test_count), str(skip)) + with open('test_result.json', 'w') as out: + json.dump(test_result, out) + if myconfig.skip_term_colors: print(output_string) print test_result else: if failures > 0: print('\033[91m' + output_string + '\033[0m') - str1 = json.dumps(test_result, indent=4, sort_keys=True) - with open('test_result.json', 'w') as out: - json.dump(test_result, out) else: print('\033[92m' + output_string + '\033[0m') From 19b780b2befb176409856415df6357c111c9fa81 Mon Sep 17 00:00:00 2001 From: Satish Suradkar Date: Wed, 3 Aug 2016 16:24:53 +0530 Subject: [PATCH 4/8] Generate test_result.json file. --- pyresttest/resttest.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pyresttest/resttest.py b/pyresttest/resttest.py index 4806ef2d..a2783a77 100644 --- a/pyresttest/resttest.py +++ b/pyresttest/resttest.py @@ -12,6 +12,7 @@ from optparse import OptionParser from email import message_from_string # For headers handling import time +from json2html import * #import pdb try: @@ -779,7 +780,10 @@ def run_testsets(testsets): .format(group, passfail[failures == 0], str(test_count - failures), str(test_count), str(skip)) with open('test_result.json', 'w') as out: - json.dump(test_result, out) + json.dump(test_result, out, indent=4) + + #with open('test_result.html', 'w') as out: + # json.dump(json2html.convert(json = test_result), out) if myconfig.skip_term_colors: print(output_string) From 5f71436918067f99fdf0c3a94a9e789bd3bfbe8a Mon Sep 17 00:00:00 2001 From: Satish Suradkar Date: Wed, 3 Aug 2016 16:28:19 +0530 Subject: [PATCH 5/8] Generate test_result.json file. --- pyresttest/resttest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pyresttest/resttest.py b/pyresttest/resttest.py index a2783a77..095382ef 100644 --- a/pyresttest/resttest.py +++ b/pyresttest/resttest.py @@ -12,7 +12,6 @@ from optparse import OptionParser from email import message_from_string # For headers handling import time -from json2html import * #import pdb try: From 9b4d4eaadcc37f9335412eeb3b140e2be83507a0 Mon Sep 17 00:00:00 2001 From: Satish Suradkar Date: Tue, 9 Aug 2016 11:17:15 +0530 Subject: [PATCH 6/8] able to generate test_result.json with multiple field --- pyresttest/resttest.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/pyresttest/resttest.py b/pyresttest/resttest.py index 095382ef..dfe6f875 100644 --- a/pyresttest/resttest.py +++ b/pyresttest/resttest.py @@ -12,7 +12,7 @@ from optparse import OptionParser from email import message_from_string # For headers handling import time -#import pdb +from collections import defaultdict try: from cStringIO import StringIO as MyIO @@ -83,8 +83,9 @@ DIR_LOCK = threading.RLock() # Guards operations changing the working directory -test_result = dict() +test_result = defaultdict(dict) test_result_list = [] + class cd: """Context manager for changing the current working directory""" # http://stackoverflow.com/questions/431684/how-do-i-cd-in-python/13197763#13197763 @@ -359,7 +360,7 @@ def run_test(mytest, test_config=TestConfig(), context=None, curl_handle=None, * # print("Delaying for %ds" % mytest.delay) # time.sleep(mytest.delay) for test_need in mytest.depends_on: - if not test_result[test_need] or test_result[test_need] == 'skip': + if not test_result[test_need]['result'] or test_result[test_need]['result'] == 'skip': print "\n\033[1;31m 'test: {0}' depends on 'test: {1}' \033[0m".format(mytest.name, test_need) return @@ -698,19 +699,24 @@ def run_testsets(testsets): if test.group not in group_results: group_results[test.group] = list() group_failure_counts[test.group] = 0 - + global is_retried is_retried = False print "\n ==================================================== \n" result = run_test(test, test_config=myconfig, context=context, curl_handle=curl_handle) - + + if result is not None: + test_result[test.name]['result'] = result.passed + if not result.passed: + test_result[test.name]['status'] = result.response_headers[3][1] + test_result[test.name]['expected_status'] = test.expected_status + if result is None: skip += 1 - test_result[test.name] = "skip" + test_result[test.name]['result'] = "skip" + test_result[test.name]['depends_on'] = test.depends_on continue - test_result[test.name] = result.passed - result.body = None # Remove the body, save some memory! if not result.passed: # Print failure, increase failure counts for that test group # Use result test URL to allow for templating @@ -781,12 +787,9 @@ def run_testsets(testsets): with open('test_result.json', 'w') as out: json.dump(test_result, out, indent=4) - #with open('test_result.html', 'w') as out: - # json.dump(json2html.convert(json = test_result), out) if myconfig.skip_term_colors: print(output_string) - print test_result else: if failures > 0: print('\033[91m' + output_string + '\033[0m') From 8174c7498f823aed5bf692b5070206ea4198d0b8 Mon Sep 17 00:00:00 2001 From: Randhir Kishor Bhosale Date: Mon, 19 Dec 2016 16:13:35 +0530 Subject: [PATCH 7/8] check --- pyresttest/resttest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyresttest/resttest.py b/pyresttest/resttest.py index dfe6f875..42605862 100644 --- a/pyresttest/resttest.py +++ b/pyresttest/resttest.py @@ -14,6 +14,7 @@ import time from collections import defaultdict + try: from cStringIO import StringIO as MyIO except: From 4d76662d154933a9118a37d1104d183a4aab07ee Mon Sep 17 00:00:00 2001 From: Randhir Kishor Bhosale Date: Mon, 19 Dec 2016 16:23:56 +0530 Subject: [PATCH 8/8] my repo check --- pyresttest/resttest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pyresttest/resttest.py b/pyresttest/resttest.py index 42605862..dfe6f875 100644 --- a/pyresttest/resttest.py +++ b/pyresttest/resttest.py @@ -14,7 +14,6 @@ import time from collections import defaultdict - try: from cStringIO import StringIO as MyIO except: