Update to tlsfuzzer 20200828
jsing has upstreamed his zero content type test, which in turn found bugs in GnuTLS and NSS - thanks!
This commit is contained in:
tb
parent
098eba1653
commit
902a90e4b9
@ -1,15 +1,15 @@
|
||||
# $OpenBSD: Makefile,v 1.19 2020/08/27 16:10:12 tb Exp $
|
||||
# $OpenBSD: Makefile,v 1.20 2020/08/28 18:32:02 tb Exp $
|
||||
|
||||
COMMENT = fuzzer and test suite for TLS implementations.
|
||||
|
||||
MODPY_EGG_VERSION = 20200825
|
||||
MODPY_EGG_VERSION = 20200828
|
||||
DISTNAME = tlsfuzzer-${MODPY_EGG_VERSION}
|
||||
PKGNAME = py-${DISTNAME}
|
||||
CATEGORIES = security
|
||||
|
||||
GH_ACCOUNT = tomato42
|
||||
GH_PROJECT = tlsfuzzer
|
||||
GH_COMMIT = 92cb1c4ffce3d592d1be46a59551aed66704156e
|
||||
GH_COMMIT = ac1b278c68ed96f1fed8f8d59310c0d2d1e1bd32
|
||||
|
||||
MAINTAINER = Theo Buehler <tb@openbsd.org>
|
||||
|
||||
@ -37,7 +37,6 @@ pre-install:
|
||||
SCRIPTS_DIR = ${PREFIX}/share/tlsfuzzer/scripts
|
||||
post-install:
|
||||
${INSTALL_DATA_DIR} ${SCRIPTS_DIR}
|
||||
${INSTALL_DATA} ${FILESDIR}/*.py ${SCRIPTS_DIR}
|
||||
cd ${WRKSRC}/scripts && pax -rw *.py ${SCRIPTS_DIR}
|
||||
${MODPY_BIN} -m compileall ${SCRIPTS_DIR}
|
||||
|
||||
|
||||
@ -1,2 +1,2 @@
|
||||
SHA256 (tlsfuzzer-20200825-92cb1c4f.tar.gz) = icjC2yWexkbn4LkNE5wQp9FmTZVK/nT2riSmWP/c40s=
|
||||
SIZE (tlsfuzzer-20200825-92cb1c4f.tar.gz) = 5012982
|
||||
SHA256 (tlsfuzzer-20200828-ac1b278c.tar.gz) = 7g5SZ/YZOMPK3A/efXzq36HHlGmUDcBj879Hm4B0Z5w=
|
||||
SIZE (tlsfuzzer-20200828-ac1b278c.tar.gz) = 5014891
|
||||
|
||||
@ -1,464 +0,0 @@
|
||||
# Author: Anderson Toshiyuki Sasaki, (c) 2018
|
||||
# Author: Joel Sing, (c) 2020
|
||||
# Released under Gnu GPL v2.0, see LICENSE file for details
|
||||
|
||||
from __future__ import print_function
|
||||
import traceback
|
||||
import sys
|
||||
import getopt
|
||||
from itertools import chain
|
||||
from random import sample
|
||||
|
||||
from tlsfuzzer.runner import Runner
|
||||
from tlsfuzzer.messages import Connect, ClientHelloGenerator, \
|
||||
ClientKeyExchangeGenerator, ChangeCipherSpecGenerator, \
|
||||
FinishedGenerator, ApplicationDataGenerator, AlertGenerator, \
|
||||
split_message, PopMessageFromList, SetPaddingCallback, \
|
||||
RawMessageGenerator
|
||||
from tlsfuzzer.expect import ExpectServerHello, ExpectCertificate, \
|
||||
ExpectServerHelloDone, ExpectChangeCipherSpec, ExpectFinished, \
|
||||
ExpectAlert, ExpectApplicationData, ExpectClose, \
|
||||
ExpectEncryptedExtensions, ExpectCertificateVerify, \
|
||||
ExpectNewSessionTicket
|
||||
|
||||
from tlslite.constants import CipherSuite, AlertLevel, AlertDescription, \
|
||||
TLS_1_3_DRAFT, GroupName, ExtensionType, SignatureScheme
|
||||
from tlslite.keyexchange import ECDHKeyExchange
|
||||
from tlsfuzzer.utils.lists import natural_sort_keys
|
||||
from tlslite.extensions import KeyShareEntry, ClientKeyShareExtension, \
|
||||
SupportedVersionsExtension, SupportedGroupsExtension, \
|
||||
SignatureAlgorithmsExtension, SignatureAlgorithmsCertExtension
|
||||
from tlsfuzzer.helpers import key_share_gen, RSA_SIG_ALL
|
||||
|
||||
|
||||
version = 1
|
||||
|
||||
|
||||
def help_msg():
|
||||
print("Usage: <script-name> [-h hostname] [-p port] [[probe-name] ...]")
|
||||
print(" -h hostname name of the host to run the test against")
|
||||
print(" localhost by default")
|
||||
print(" -p port port number to use for connection, 4433 by default")
|
||||
print(" probe-name if present, will run only the probes with given")
|
||||
print(" names and not all of them, e.g \"sanity\"")
|
||||
print(" -e probe-name exclude the probe from the list of the ones run")
|
||||
print(" may be specified multiple times")
|
||||
print(" -x probe-name expect the probe to fail. When such probe passes despite being marked like this")
|
||||
print(" it will be reported in the test summary and the whole script will fail.")
|
||||
print(" May be specified multiple times.")
|
||||
print(" -X message expect the `message` substring in exception raised during")
|
||||
print(" execution of preceding expected failure probe")
|
||||
print(" usage: [-x probe-name] [-X exception], order is compulsory!")
|
||||
print(" -n num only run `num` random tests instead of a full set")
|
||||
print(" (excluding \"sanity\" tests)")
|
||||
print(" --help this message")
|
||||
|
||||
|
||||
def main():
|
||||
host = "localhost"
|
||||
port = 4433
|
||||
num_limit = None
|
||||
run_exclude = set()
|
||||
expected_failures = {}
|
||||
last_exp_tmp = None
|
||||
|
||||
argv = sys.argv[1:]
|
||||
opts, args = getopt.getopt(argv, "h:p:e:x:X:n:", ["help"])
|
||||
for opt, arg in opts:
|
||||
if opt == '-h':
|
||||
host = arg
|
||||
elif opt == '-p':
|
||||
port = int(arg)
|
||||
elif opt == '-e':
|
||||
run_exclude.add(arg)
|
||||
elif opt == '-x':
|
||||
expected_failures[arg] = None
|
||||
last_exp_tmp = str(arg)
|
||||
elif opt == '-X':
|
||||
if not last_exp_tmp:
|
||||
raise ValueError("-x has to be specified before -X")
|
||||
expected_failures[last_exp_tmp] = str(arg)
|
||||
elif opt == '-n':
|
||||
num_limit = int(arg)
|
||||
elif opt == '--help':
|
||||
help_msg()
|
||||
sys.exit(0)
|
||||
else:
|
||||
raise ValueError("Unknown option: {0}".format(opt))
|
||||
|
||||
if args:
|
||||
run_only = set(args)
|
||||
else:
|
||||
run_only = None
|
||||
|
||||
conversations = {}
|
||||
|
||||
conversation = Connect(host, port)
|
||||
node = conversation
|
||||
ciphers = [CipherSuite.TLS_AES_128_GCM_SHA256,
|
||||
CipherSuite.TLS_EMPTY_RENEGOTIATION_INFO_SCSV]
|
||||
ext = {}
|
||||
groups = [GroupName.secp256r1]
|
||||
key_shares = []
|
||||
for group in groups:
|
||||
key_shares.append(key_share_gen(group))
|
||||
ext[ExtensionType.key_share] = ClientKeyShareExtension().create(key_shares)
|
||||
ext[ExtensionType.supported_versions] = SupportedVersionsExtension()\
|
||||
.create([TLS_1_3_DRAFT, (3, 3)])
|
||||
ext[ExtensionType.supported_groups] = SupportedGroupsExtension()\
|
||||
.create(groups)
|
||||
sig_algs = [SignatureScheme.rsa_pss_rsae_sha256,
|
||||
SignatureScheme.rsa_pss_pss_sha256]
|
||||
ext[ExtensionType.signature_algorithms] = SignatureAlgorithmsExtension()\
|
||||
.create(sig_algs)
|
||||
ext[ExtensionType.signature_algorithms_cert] = SignatureAlgorithmsCertExtension()\
|
||||
.create(RSA_SIG_ALL)
|
||||
node = node.add_child(ClientHelloGenerator(ciphers, extensions=ext))
|
||||
node = node.add_child(ExpectServerHello())
|
||||
node = node.add_child(ExpectChangeCipherSpec())
|
||||
node = node.add_child(ExpectEncryptedExtensions())
|
||||
node = node.add_child(ExpectCertificate())
|
||||
node = node.add_child(ExpectCertificateVerify())
|
||||
node = node.add_child(ExpectFinished())
|
||||
node = node.add_child(FinishedGenerator())
|
||||
node = node.add_child(ApplicationDataGenerator(
|
||||
bytearray(b"GET / HTTP/1.0\r\n\r\n")))
|
||||
|
||||
# This message is optional and may show up 0 to many times
|
||||
cycle = ExpectNewSessionTicket()
|
||||
node = node.add_child(cycle)
|
||||
node.add_child(cycle)
|
||||
|
||||
node.next_sibling = ExpectApplicationData()
|
||||
node = node.next_sibling.add_child(AlertGenerator(AlertLevel.warning,
|
||||
AlertDescription.close_notify))
|
||||
|
||||
node = node.add_child(ExpectAlert())
|
||||
node.next_sibling = ExpectClose()
|
||||
conversations["sanity"] = conversation
|
||||
|
||||
# Record with zero content type after handshake.
|
||||
conversation = Connect(host, port)
|
||||
node = conversation
|
||||
ciphers = [CipherSuite.TLS_AES_128_GCM_SHA256,
|
||||
CipherSuite.TLS_EMPTY_RENEGOTIATION_INFO_SCSV]
|
||||
ext = {}
|
||||
groups = [GroupName.secp256r1]
|
||||
key_shares = []
|
||||
for group in groups:
|
||||
key_shares.append(key_share_gen(group))
|
||||
ext[ExtensionType.key_share] = ClientKeyShareExtension().create(key_shares)
|
||||
ext[ExtensionType.supported_versions] = SupportedVersionsExtension()\
|
||||
.create([TLS_1_3_DRAFT, (3, 3)])
|
||||
ext[ExtensionType.supported_groups] = SupportedGroupsExtension()\
|
||||
.create(groups)
|
||||
sig_algs = [SignatureScheme.rsa_pss_rsae_sha256,
|
||||
SignatureScheme.rsa_pss_pss_sha256]
|
||||
ext[ExtensionType.signature_algorithms] = SignatureAlgorithmsExtension()\
|
||||
.create(sig_algs)
|
||||
ext[ExtensionType.signature_algorithms_cert] = SignatureAlgorithmsCertExtension()\
|
||||
.create(RSA_SIG_ALL)
|
||||
node = node.add_child(ClientHelloGenerator(ciphers, extensions=ext))
|
||||
node = node.add_child(ExpectServerHello())
|
||||
node = node.add_child(ExpectChangeCipherSpec())
|
||||
node = node.add_child(ExpectEncryptedExtensions())
|
||||
node = node.add_child(ExpectCertificate())
|
||||
node = node.add_child(ExpectCertificateVerify())
|
||||
node = node.add_child(ExpectFinished())
|
||||
node = node.add_child(FinishedGenerator())
|
||||
node = node.add_child(RawMessageGenerator(content_type=0, data=bytearray(0)))
|
||||
|
||||
# This message is optional and may show up 0 to many times
|
||||
cycle = ExpectNewSessionTicket()
|
||||
node = node.add_child(cycle)
|
||||
node.add_child(cycle)
|
||||
|
||||
node.next_sibling = ExpectAlert(AlertLevel.fatal,
|
||||
AlertDescription.unexpected_message)
|
||||
node.next_sibling.add_child(ExpectClose())
|
||||
|
||||
conversations["zero content type after handshake"] = conversation
|
||||
|
||||
# Record with zero content type after handshake.
|
||||
conversation = Connect(host, port)
|
||||
node = conversation
|
||||
ciphers = [CipherSuite.TLS_AES_128_GCM_SHA256,
|
||||
CipherSuite.TLS_EMPTY_RENEGOTIATION_INFO_SCSV]
|
||||
ext = {}
|
||||
groups = [GroupName.secp256r1]
|
||||
key_shares = []
|
||||
for group in groups:
|
||||
key_shares.append(key_share_gen(group))
|
||||
ext[ExtensionType.key_share] = ClientKeyShareExtension().create(key_shares)
|
||||
ext[ExtensionType.supported_versions] = SupportedVersionsExtension()\
|
||||
.create([TLS_1_3_DRAFT, (3, 3)])
|
||||
ext[ExtensionType.supported_groups] = SupportedGroupsExtension()\
|
||||
.create(groups)
|
||||
sig_algs = [SignatureScheme.rsa_pss_rsae_sha256,
|
||||
SignatureScheme.rsa_pss_pss_sha256]
|
||||
ext[ExtensionType.signature_algorithms] = SignatureAlgorithmsExtension()\
|
||||
.create(sig_algs)
|
||||
ext[ExtensionType.signature_algorithms_cert] = SignatureAlgorithmsCertExtension()\
|
||||
.create(RSA_SIG_ALL)
|
||||
node = node.add_child(ClientHelloGenerator(ciphers, extensions=ext))
|
||||
node = node.add_child(ExpectServerHello())
|
||||
node = node.add_child(ExpectChangeCipherSpec())
|
||||
node = node.add_child(ExpectEncryptedExtensions())
|
||||
node = node.add_child(ExpectCertificate())
|
||||
node = node.add_child(ExpectCertificateVerify())
|
||||
node = node.add_child(ExpectFinished())
|
||||
node = node.add_child(FinishedGenerator())
|
||||
node = node.add_child(RawMessageGenerator(content_type=0, data=bytearray(42)))
|
||||
|
||||
# This message is optional and may show up 0 to many times
|
||||
cycle = ExpectNewSessionTicket()
|
||||
node = node.add_child(cycle)
|
||||
node.add_child(cycle)
|
||||
|
||||
node.next_sibling = ExpectAlert(AlertLevel.fatal,
|
||||
AlertDescription.unexpected_message)
|
||||
node.next_sibling.add_child(ExpectClose())
|
||||
|
||||
conversations["zero content type with padding after handshake"] = conversation
|
||||
|
||||
# Record with zero content type during handshake.
|
||||
conversation = Connect(host, port)
|
||||
node = conversation
|
||||
ciphers = [CipherSuite.TLS_AES_128_GCM_SHA256,
|
||||
CipherSuite.TLS_EMPTY_RENEGOTIATION_INFO_SCSV]
|
||||
ext = {}
|
||||
groups = [GroupName.secp256r1]
|
||||
key_shares = []
|
||||
for group in groups:
|
||||
key_shares.append(key_share_gen(group))
|
||||
ext[ExtensionType.key_share] = ClientKeyShareExtension().create(key_shares)
|
||||
ext[ExtensionType.supported_versions] = SupportedVersionsExtension()\
|
||||
.create([TLS_1_3_DRAFT, (3, 3)])
|
||||
ext[ExtensionType.supported_groups] = SupportedGroupsExtension()\
|
||||
.create(groups)
|
||||
sig_algs = [SignatureScheme.rsa_pss_rsae_sha256,
|
||||
SignatureScheme.rsa_pss_pss_sha256]
|
||||
ext[ExtensionType.signature_algorithms] = SignatureAlgorithmsExtension()\
|
||||
.create(sig_algs)
|
||||
ext[ExtensionType.signature_algorithms_cert] = SignatureAlgorithmsCertExtension()\
|
||||
.create(RSA_SIG_ALL)
|
||||
node = node.add_child(ClientHelloGenerator(ciphers, extensions=ext))
|
||||
node = node.add_child(ExpectServerHello())
|
||||
node = node.add_child(ExpectChangeCipherSpec())
|
||||
node = node.add_child(ExpectEncryptedExtensions())
|
||||
node = node.add_child(ExpectCertificate())
|
||||
node = node.add_child(ExpectCertificateVerify())
|
||||
node = node.add_child(ExpectFinished())
|
||||
node = node.add_child(RawMessageGenerator(content_type=0, data=bytearray(0)))
|
||||
|
||||
node = node.add_child(ExpectAlert(AlertLevel.fatal,
|
||||
AlertDescription.unexpected_message))
|
||||
node = node.add_child(ExpectClose())
|
||||
|
||||
conversations["zero content type during handshake"] = conversation
|
||||
|
||||
# Record with zero content type and padding during handshake.
|
||||
conversation = Connect(host, port)
|
||||
node = conversation
|
||||
ciphers = [CipherSuite.TLS_AES_128_GCM_SHA256,
|
||||
CipherSuite.TLS_EMPTY_RENEGOTIATION_INFO_SCSV]
|
||||
ext = {}
|
||||
groups = [GroupName.secp256r1]
|
||||
key_shares = []
|
||||
for group in groups:
|
||||
key_shares.append(key_share_gen(group))
|
||||
ext[ExtensionType.key_share] = ClientKeyShareExtension().create(key_shares)
|
||||
ext[ExtensionType.supported_versions] = SupportedVersionsExtension()\
|
||||
.create([TLS_1_3_DRAFT, (3, 3)])
|
||||
ext[ExtensionType.supported_groups] = SupportedGroupsExtension()\
|
||||
.create(groups)
|
||||
sig_algs = [SignatureScheme.rsa_pss_rsae_sha256,
|
||||
SignatureScheme.rsa_pss_pss_sha256]
|
||||
ext[ExtensionType.signature_algorithms] = SignatureAlgorithmsExtension()\
|
||||
.create(sig_algs)
|
||||
ext[ExtensionType.signature_algorithms_cert] = SignatureAlgorithmsCertExtension()\
|
||||
.create(RSA_SIG_ALL)
|
||||
node = node.add_child(ClientHelloGenerator(ciphers, extensions=ext))
|
||||
node = node.add_child(ExpectServerHello())
|
||||
node = node.add_child(ExpectChangeCipherSpec())
|
||||
node = node.add_child(ExpectEncryptedExtensions())
|
||||
node = node.add_child(ExpectCertificate())
|
||||
node = node.add_child(ExpectCertificateVerify())
|
||||
node = node.add_child(ExpectFinished())
|
||||
node = node.add_child(RawMessageGenerator(content_type=0, data=bytearray(42)))
|
||||
|
||||
node = node.add_child(ExpectAlert(AlertLevel.fatal,
|
||||
AlertDescription.unexpected_message))
|
||||
node = node.add_child(ExpectClose())
|
||||
|
||||
conversations["zero content type with padding during handshake"] = conversation
|
||||
|
||||
# Record with zero content type during application data.
|
||||
conversation = Connect(host, port)
|
||||
node = conversation
|
||||
ciphers = [CipherSuite.TLS_AES_128_GCM_SHA256,
|
||||
CipherSuite.TLS_EMPTY_RENEGOTIATION_INFO_SCSV]
|
||||
ext = {}
|
||||
groups = [GroupName.secp256r1]
|
||||
key_shares = []
|
||||
for group in groups:
|
||||
key_shares.append(key_share_gen(group))
|
||||
ext[ExtensionType.key_share] = ClientKeyShareExtension().create(key_shares)
|
||||
ext[ExtensionType.supported_versions] = SupportedVersionsExtension()\
|
||||
.create([TLS_1_3_DRAFT, (3, 3)])
|
||||
ext[ExtensionType.supported_groups] = SupportedGroupsExtension()\
|
||||
.create(groups)
|
||||
sig_algs = [SignatureScheme.rsa_pss_rsae_sha256,
|
||||
SignatureScheme.rsa_pss_pss_sha256]
|
||||
ext[ExtensionType.signature_algorithms] = SignatureAlgorithmsExtension()\
|
||||
.create(sig_algs)
|
||||
ext[ExtensionType.signature_algorithms_cert] = SignatureAlgorithmsCertExtension()\
|
||||
.create(RSA_SIG_ALL)
|
||||
node = node.add_child(ClientHelloGenerator(ciphers, extensions=ext))
|
||||
node = node.add_child(ExpectServerHello())
|
||||
node = node.add_child(ExpectChangeCipherSpec())
|
||||
node = node.add_child(ExpectEncryptedExtensions())
|
||||
node = node.add_child(ExpectCertificate())
|
||||
node = node.add_child(ExpectCertificateVerify())
|
||||
node = node.add_child(ExpectFinished())
|
||||
node = node.add_child(FinishedGenerator())
|
||||
node = node.add_child(ApplicationDataGenerator(bytearray(b"GET /")))
|
||||
node = node.add_child(RawMessageGenerator(content_type=0, data=bytearray(0)))
|
||||
|
||||
# This message is optional and may show up 0 to many times
|
||||
cycle = ExpectNewSessionTicket()
|
||||
node = node.add_child(cycle)
|
||||
node.add_child(cycle)
|
||||
|
||||
node.next_sibling = ExpectAlert(AlertLevel.fatal,
|
||||
AlertDescription.unexpected_message)
|
||||
node.next_sibling.add_child(ExpectClose())
|
||||
|
||||
conversations["zero content type during application data"] = conversation
|
||||
|
||||
# Record with zero content type and padding during application data.
|
||||
conversation = Connect(host, port)
|
||||
node = conversation
|
||||
ciphers = [CipherSuite.TLS_AES_128_GCM_SHA256,
|
||||
CipherSuite.TLS_EMPTY_RENEGOTIATION_INFO_SCSV]
|
||||
ext = {}
|
||||
groups = [GroupName.secp256r1]
|
||||
key_shares = []
|
||||
for group in groups:
|
||||
key_shares.append(key_share_gen(group))
|
||||
ext[ExtensionType.key_share] = ClientKeyShareExtension().create(key_shares)
|
||||
ext[ExtensionType.supported_versions] = SupportedVersionsExtension()\
|
||||
.create([TLS_1_3_DRAFT, (3, 3)])
|
||||
ext[ExtensionType.supported_groups] = SupportedGroupsExtension()\
|
||||
.create(groups)
|
||||
sig_algs = [SignatureScheme.rsa_pss_rsae_sha256,
|
||||
SignatureScheme.rsa_pss_pss_sha256]
|
||||
ext[ExtensionType.signature_algorithms] = SignatureAlgorithmsExtension()\
|
||||
.create(sig_algs)
|
||||
ext[ExtensionType.signature_algorithms_cert] = SignatureAlgorithmsCertExtension()\
|
||||
.create(RSA_SIG_ALL)
|
||||
node = node.add_child(ClientHelloGenerator(ciphers, extensions=ext))
|
||||
node = node.add_child(ExpectServerHello())
|
||||
node = node.add_child(ExpectChangeCipherSpec())
|
||||
node = node.add_child(ExpectEncryptedExtensions())
|
||||
node = node.add_child(ExpectCertificate())
|
||||
node = node.add_child(ExpectCertificateVerify())
|
||||
node = node.add_child(ExpectFinished())
|
||||
node = node.add_child(FinishedGenerator())
|
||||
node = node.add_child(ApplicationDataGenerator(bytearray(b"GET /")))
|
||||
node = node.add_child(RawMessageGenerator(content_type=0, data=bytearray(42)))
|
||||
|
||||
# This message is optional and may show up 0 to many times
|
||||
cycle = ExpectNewSessionTicket()
|
||||
node = node.add_child(cycle)
|
||||
node.add_child(cycle)
|
||||
|
||||
node.next_sibling = ExpectAlert(AlertLevel.fatal,
|
||||
AlertDescription.unexpected_message)
|
||||
node.next_sibling.add_child(ExpectClose())
|
||||
|
||||
conversations["zero content type and padding during application data"] = conversation
|
||||
|
||||
# run the conversation
|
||||
good = 0
|
||||
bad = 0
|
||||
xfail = 0
|
||||
xpass = 0
|
||||
failed = []
|
||||
xpassed = []
|
||||
if not num_limit:
|
||||
num_limit = len(conversations)
|
||||
|
||||
# make sure that sanity test is run first and last
|
||||
# to verify that server was running and kept running throughout
|
||||
sanity_tests = [('sanity', conversations['sanity'])]
|
||||
regular_tests = [(k, v) for k, v in conversations.items() if k != 'sanity']
|
||||
sampled_tests = sample(regular_tests, min(num_limit, len(regular_tests)))
|
||||
ordered_tests = chain(sanity_tests, sampled_tests, sanity_tests)
|
||||
|
||||
for c_name, c_test in ordered_tests:
|
||||
if run_only and c_name not in run_only or c_name in run_exclude:
|
||||
continue
|
||||
print("{0} ...".format(c_name))
|
||||
|
||||
runner = Runner(c_test)
|
||||
|
||||
res = True
|
||||
exception = None
|
||||
try:
|
||||
runner.run()
|
||||
except Exception as exp:
|
||||
exception = exp
|
||||
print("Error while processing")
|
||||
print(traceback.format_exc())
|
||||
res = False
|
||||
|
||||
if c_name in expected_failures:
|
||||
if res:
|
||||
xpass += 1
|
||||
xpassed.append(c_name)
|
||||
print("XPASS: expected failure but test passed\n")
|
||||
else:
|
||||
if expected_failures[c_name] is not None and \
|
||||
expected_failures[c_name] not in str(exception):
|
||||
bad += 1
|
||||
failed.append(c_name)
|
||||
print("Expected error message: {0}\n"
|
||||
.format(expected_failures[c_name]))
|
||||
else:
|
||||
xfail += 1
|
||||
print("OK-expected failure\n")
|
||||
else:
|
||||
if res:
|
||||
good += 1
|
||||
print("OK\n")
|
||||
else:
|
||||
bad += 1
|
||||
failed.append(c_name)
|
||||
|
||||
print("TLS 1.3 zero-value content type")
|
||||
print("Check if handling of records with an internal content type of zero is ")
|
||||
print("correct.\n")
|
||||
print("version: {0}\n".format(version))
|
||||
|
||||
print("Test end")
|
||||
print(20 * '=')
|
||||
print("TOTAL: {0}".format(len(sampled_tests) + 2*len(sanity_tests)))
|
||||
print("SKIP: {0}".format(len(run_exclude.intersection(conversations.keys()))))
|
||||
print("PASS: {0}".format(good))
|
||||
print("XFAIL: {0}".format(xfail))
|
||||
print("FAIL: {0}".format(bad))
|
||||
print("XPASS: {0}".format(xpass))
|
||||
print(20 * '=')
|
||||
sort = sorted(xpassed ,key=natural_sort_keys)
|
||||
if len(sort):
|
||||
print("XPASSED:\n\t{0}".format('\n\t'.join(repr(i) for i in sort)))
|
||||
sort = sorted(failed, key=natural_sort_keys)
|
||||
if len(sort):
|
||||
print("FAILED:\n\t{0}".format('\n\t'.join(repr(i) for i in sort)))
|
||||
|
||||
if bad > 0:
|
||||
sys.exit(1)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Reference in New Issue
Block a user