upstream: add some knobs:

UNITTEST_FAST?= no     # Skip slow tests (e.g. less intensive fuzzing).
UNITTEST_SLOW?= no     # Include slower tests (e.g. more intensive fuzzing).
UNITTEST_VERBOSE?= no  # Verbose test output (inc. per-test names).

useful if you want to run the tests as a smoke test to exercise the
functionality without waiting for all the fuzzers to run.

OpenBSD-Regress-ID: e04d82ebec86068198cd903acf1c67563c57315e

regress/unittests/Makefile.inc

@@ -1,10 +1,15 @@
-#	$OpenBSD: Makefile.inc,v 1.12 2017/12/21 00:41:22 djm Exp $
+#	$OpenBSD: Makefile.inc,v 1.13 2018/10/17 23:28:05 djm Exp $
 
 REGRESS_FAIL_EARLY?=	yes
 
 .include <bsd.own.mk>
 .include <bsd.obj.mk>
 
+# User-settable options
+UNITTEST_FAST?= no	# Skip slow tests (e.g. less intensive fuzzing).
+UNITTEST_SLOW?= no	# Include slower tests (e.g. more intensive fuzzing).
+UNITTEST_VERBOSE?= no	# Verbose test output (inc. per-test names).
+
 MALLOC_OPTIONS?=	CFGJRSUX
 TEST_ENV?=		MALLOC_OPTIONS=${MALLOC_OPTIONS}
 
@@ -68,3 +73,14 @@ DPADD+= ${LIBUTIL}
 LDADD+= -lcrypto
 DPADD+= ${LIBCRYPTO}
 .endif
+
+UNITTEST_ARGS?=
+
+.if (${UNITTEST_VERBOSE:L} != "no")
+UNITTEST_ARGS+= -v
+.endif
+.if (${UNITTEST_FAST:L} != "no")
+UNITTEST_ARGS+= -f
+.elif (${UNITTEST_SLOW:L} != "no")
+UNITTEST_ARGS+= -F
+.endif

regress/unittests/sshbuf/Makefile

@@ -1,4 +1,4 @@
-#	$OpenBSD: Makefile,v 1.6 2017/12/21 00:41:22 djm Exp $
+#	$OpenBSD: Makefile,v 1.7 2018/10/17 23:28:05 djm Exp $
 
 .include <bsd.regress.mk>
 
@@ -17,6 +17,5 @@ SRCS+=sshbuf-getput-basic.c sshbuf-getput-crypto.c sshbuf-misc.c sshbuf.c
 SRCS+=atomicio.c
 
 run-regress-${PROG}: ${PROG}
-	env ${TEST_ENV} ./${PROG}
+	env ${TEST_ENV} ./${PROG} ${UNITTEST_ARGS}
- 
 

regress/unittests/sshbuf/test_sshbuf_fuzz.c

@@ -1,4 +1,4 @@
-/* 	$OpenBSD: test_sshbuf_fuzz.c,v 1.1 2014/04/30 05:32:00 djm Exp $ */
+/* 	$OpenBSD: test_sshbuf_fuzz.c,v 1.2 2018/10/17 23:28:05 djm Exp $ */
 /*
  * Regress test for sshbuf.h buffer API
  *
@@ -30,10 +30,15 @@ sshbuf_fuzz_tests(void)
 {
 	struct sshbuf *p1;
 	u_char *dp;
-	size_t sz, sz2, i;
+	size_t sz, sz2, i, ntests = NUM_FUZZ_TESTS;
 	u_int32_t r;
 	int ret;
 
+	if (test_is_fast())
+		ntests >>= 2;
+	if (test_is_slow())
+		ntests <<= 2;
+
 	/* NB. uses sshbuf internals */
 	TEST_START("fuzz alloc/dealloc");
 	p1 = sshbuf_new();

regress/unittests/sshbuf/test_sshbuf_getput_fuzz.c

@@ -1,4 +1,4 @@
-/* 	$OpenBSD: test_sshbuf_getput_fuzz.c,v 1.2 2014/05/02 02:54:00 djm Exp $ */
+/* 	$OpenBSD: test_sshbuf_getput_fuzz.c,v 1.3 2018/10/17 23:28:05 djm Exp $ */
 /*
  * Regress test for sshbuf.h buffer API
  *
@@ -115,11 +115,15 @@ sshbuf_getput_fuzz_tests(void)
 		0x55, 0x0f, 0x69, 0xd8, 0x0e, 0xc2, 0x3c, 0xd4,
 	};
 	struct fuzz *fuzz;
+	u_int fuzzers = FUZZ_1_BIT_FLIP | FUZZ_2_BIT_FLIP |
+	    FUZZ_1_BYTE_FLIP | FUZZ_2_BYTE_FLIP |
+	    FUZZ_TRUNCATE_START | FUZZ_TRUNCATE_END;
+
+	if (test_is_fast())
+		fuzzers &= ~(FUZZ_2_BYTE_FLIP|FUZZ_2_BIT_FLIP);
 
 	TEST_START("fuzz blob parsing");
-	fuzz = fuzz_begin(FUZZ_1_BIT_FLIP | FUZZ_2_BIT_FLIP |
+	fuzz = fuzz_begin(fuzzers, blob, sizeof(blob));
-	    FUZZ_1_BYTE_FLIP | FUZZ_2_BYTE_FLIP |
-	    FUZZ_TRUNCATE_START | FUZZ_TRUNCATE_END, blob, sizeof(blob));
 	TEST_ONERROR(onerror, fuzz);
 	for(; !fuzz_done(fuzz); fuzz_next(fuzz))
 		attempt_parse_blob(blob, sizeof(blob));

regress/unittests/sshkey/Makefile

@@ -1,4 +1,4 @@
-#	$OpenBSD: Makefile,v 1.5 2017/12/21 00:41:22 djm Exp $
+#	$OpenBSD: Makefile,v 1.6 2018/10/17 23:28:05 djm Exp $
 
 PROG=test_sshkey
 SRCS=tests.c test_sshkey.c test_file.c test_fuzz.c common.c
@@ -18,7 +18,7 @@ SRCS+=digest-openssl.c
 REGRESS_TARGETS=run-regress-${PROG}
 
 run-regress-${PROG}: ${PROG}
-	env ${TEST_ENV} ./${PROG} -d ${.CURDIR}/testdata
+	env ${TEST_ENV} ./${PROG} ${UNITTEST_ARGS} -d ${.CURDIR}/testdata
 
 .include <bsd.regress.mk>
 

regress/unittests/sshkey/test_fuzz.c

@@ -1,4 +1,4 @@
-/* 	$OpenBSD: test_fuzz.c,v 1.8 2017/12/21 00:41:22 djm Exp $ */
+/* 	$OpenBSD: test_fuzz.c,v 1.9 2018/10/17 23:28:05 djm Exp $ */
 /*
  * Fuzz tests for key parsing
  *
@@ -51,14 +51,16 @@ public_fuzz(struct sshkey *k)
 	struct sshkey *k1;
 	struct sshbuf *buf;
 	struct fuzz *fuzz;
+	u_int fuzzers = FUZZ_1_BIT_FLIP | FUZZ_1_BYTE_FLIP |
+	    FUZZ_TRUNCATE_START | FUZZ_TRUNCATE_END;
 
+	if (test_is_fast())
+		fuzzers &= ~FUZZ_1_BIT_FLIP;
+	if (test_is_slow())
+		fuzzers |= FUZZ_2_BIT_FLIP | FUZZ_2_BYTE_FLIP;
 	ASSERT_PTR_NE(buf = sshbuf_new(), NULL);
 	ASSERT_INT_EQ(sshkey_putb(k, buf), 0);
-	/* XXX need a way to run the tests in "slow, but complete" mode */
+	fuzz = fuzz_begin(fuzzers, sshbuf_mutable_ptr(buf), sshbuf_len(buf));
-	fuzz = fuzz_begin(FUZZ_1_BIT_FLIP | /* XXX too slow FUZZ_2_BIT_FLIP | */
-	    FUZZ_1_BYTE_FLIP | /* XXX too slow FUZZ_2_BYTE_FLIP | */
-	    FUZZ_TRUNCATE_START | FUZZ_TRUNCATE_END,
-	    sshbuf_mutable_ptr(buf), sshbuf_len(buf));
 	ASSERT_INT_EQ(sshkey_from_blob(sshbuf_ptr(buf), sshbuf_len(buf),
 	    &k1), 0);
 	sshkey_free(k1);
@@ -77,12 +79,17 @@ sig_fuzz(struct sshkey *k, const char *sig_alg)
 	struct fuzz *fuzz;
 	u_char *sig, c[] = "some junk to be signed";
 	size_t l;
+	u_int fuzzers = FUZZ_1_BIT_FLIP | FUZZ_1_BYTE_FLIP | FUZZ_2_BYTE_FLIP |
+	    FUZZ_TRUNCATE_START | FUZZ_TRUNCATE_END;
+
+	if (test_is_fast())
+		fuzzers &= ~FUZZ_2_BYTE_FLIP;
+	if (test_is_slow())
+		fuzzers |= FUZZ_2_BIT_FLIP;
 
 	ASSERT_INT_EQ(sshkey_sign(k, &sig, &l, c, sizeof(c), sig_alg, 0), 0);
 	ASSERT_SIZE_T_GT(l, 0);
-	fuzz = fuzz_begin(FUZZ_1_BIT_FLIP | /* too slow FUZZ_2_BIT_FLIP | */
+	fuzz = fuzz_begin(fuzzers, sig, l);
-	    FUZZ_1_BYTE_FLIP | FUZZ_2_BYTE_FLIP |
-	    FUZZ_TRUNCATE_START | FUZZ_TRUNCATE_END, sig, l);
 	ASSERT_INT_EQ(sshkey_verify(k, sig, l, c, sizeof(c), NULL, 0), 0);
 	free(sig);
 	TEST_ONERROR(onerror, fuzz);
@@ -96,13 +103,15 @@ sig_fuzz(struct sshkey *k, const char *sig_alg)
 	fuzz_cleanup(fuzz);
 }
 
+#define NUM_FAST_BASE64_TESTS	1024
+
 void
 sshkey_fuzz_tests(void)
 {
 	struct sshkey *k1;
 	struct sshbuf *buf, *fuzzed;
 	struct fuzz *fuzz;
-	int r;
+	int r, i;
 
 
 	TEST_START("fuzz RSA private");
@@ -114,12 +123,14 @@ sshkey_fuzz_tests(void)
 	sshbuf_free(buf);
 	ASSERT_PTR_NE(fuzzed = sshbuf_new(), NULL);
 	TEST_ONERROR(onerror, fuzz);
-	for(; !fuzz_done(fuzz); fuzz_next(fuzz)) {
+	for(i = 0; !fuzz_done(fuzz); i++, fuzz_next(fuzz)) {
 		r = sshbuf_put(fuzzed, fuzz_ptr(fuzz), fuzz_len(fuzz));
 		ASSERT_INT_EQ(r, 0);
 		if (sshkey_parse_private_fileblob(fuzzed, "", &k1, NULL) == 0)
 			sshkey_free(k1);
 		sshbuf_reset(fuzzed);
+		if (test_is_fast() && i >= NUM_FAST_BASE64_TESTS)
+			break;
 	}
 	sshbuf_free(fuzzed);
 	fuzz_cleanup(fuzz);
@@ -134,12 +145,14 @@ sshkey_fuzz_tests(void)
 	sshbuf_free(buf);
 	ASSERT_PTR_NE(fuzzed = sshbuf_new(), NULL);
 	TEST_ONERROR(onerror, fuzz);
-	for(; !fuzz_done(fuzz); fuzz_next(fuzz)) {
+	for(i = 0; !fuzz_done(fuzz); i++, fuzz_next(fuzz)) {
 		r = sshbuf_put(fuzzed, fuzz_ptr(fuzz), fuzz_len(fuzz));
 		ASSERT_INT_EQ(r, 0);
 		if (sshkey_parse_private_fileblob(fuzzed, "", &k1, NULL) == 0)
 			sshkey_free(k1);
 		sshbuf_reset(fuzzed);
+		if (test_is_fast() && i >= NUM_FAST_BASE64_TESTS)
+			break;
 	}
 	sshbuf_free(fuzzed);
 	fuzz_cleanup(fuzz);
@@ -154,12 +167,14 @@ sshkey_fuzz_tests(void)
 	sshbuf_free(buf);
 	ASSERT_PTR_NE(fuzzed = sshbuf_new(), NULL);
 	TEST_ONERROR(onerror, fuzz);
-	for(; !fuzz_done(fuzz); fuzz_next(fuzz)) {
+	for(i = 0; !fuzz_done(fuzz); i++, fuzz_next(fuzz)) {
 		r = sshbuf_put(fuzzed, fuzz_ptr(fuzz), fuzz_len(fuzz));
 		ASSERT_INT_EQ(r, 0);
 		if (sshkey_parse_private_fileblob(fuzzed, "", &k1, NULL) == 0)
 			sshkey_free(k1);
 		sshbuf_reset(fuzzed);
+		if (test_is_fast() && i >= NUM_FAST_BASE64_TESTS)
+			break;
 	}
 	sshbuf_free(fuzzed);
 	fuzz_cleanup(fuzz);
@@ -174,12 +189,14 @@ sshkey_fuzz_tests(void)
 	sshbuf_free(buf);
 	ASSERT_PTR_NE(fuzzed = sshbuf_new(), NULL);
 	TEST_ONERROR(onerror, fuzz);
-	for(; !fuzz_done(fuzz); fuzz_next(fuzz)) {
+	for(i = 0; !fuzz_done(fuzz); i++, fuzz_next(fuzz)) {
 		r = sshbuf_put(fuzzed, fuzz_ptr(fuzz), fuzz_len(fuzz));
 		ASSERT_INT_EQ(r, 0);
 		if (sshkey_parse_private_fileblob(fuzzed, "", &k1, NULL) == 0)
 			sshkey_free(k1);
 		sshbuf_reset(fuzzed);
+		if (test_is_fast() && i >= NUM_FAST_BASE64_TESTS)
+			break;
 	}
 	sshbuf_free(fuzzed);
 	fuzz_cleanup(fuzz);
@@ -195,12 +212,14 @@ sshkey_fuzz_tests(void)
 	sshbuf_free(buf);
 	ASSERT_PTR_NE(fuzzed = sshbuf_new(), NULL);
 	TEST_ONERROR(onerror, fuzz);
-	for(; !fuzz_done(fuzz); fuzz_next(fuzz)) {
+	for(i = 0; !fuzz_done(fuzz); i++, fuzz_next(fuzz)) {
 		r = sshbuf_put(fuzzed, fuzz_ptr(fuzz), fuzz_len(fuzz));
 		ASSERT_INT_EQ(r, 0);
 		if (sshkey_parse_private_fileblob(fuzzed, "", &k1, NULL) == 0)
 			sshkey_free(k1);
 		sshbuf_reset(fuzzed);
+		if (test_is_fast() && i >= NUM_FAST_BASE64_TESTS)
+			break;
 	}
 	sshbuf_free(fuzzed);
 	fuzz_cleanup(fuzz);
@@ -215,12 +234,14 @@ sshkey_fuzz_tests(void)
 	sshbuf_free(buf);
 	ASSERT_PTR_NE(fuzzed = sshbuf_new(), NULL);
 	TEST_ONERROR(onerror, fuzz);
-	for(; !fuzz_done(fuzz); fuzz_next(fuzz)) {
+	for(i = 0; !fuzz_done(fuzz); i++, fuzz_next(fuzz)) {
 		r = sshbuf_put(fuzzed, fuzz_ptr(fuzz), fuzz_len(fuzz));
 		ASSERT_INT_EQ(r, 0);
 		if (sshkey_parse_private_fileblob(fuzzed, "", &k1, NULL) == 0)
 			sshkey_free(k1);
 		sshbuf_reset(fuzzed);
+		if (test_is_fast() && i >= NUM_FAST_BASE64_TESTS)
+			break;
 	}
 	sshbuf_free(fuzzed);
 	fuzz_cleanup(fuzz);
@@ -236,12 +257,14 @@ sshkey_fuzz_tests(void)
 	sshbuf_free(buf);
 	ASSERT_PTR_NE(fuzzed = sshbuf_new(), NULL);
 	TEST_ONERROR(onerror, fuzz);
-	for(; !fuzz_done(fuzz); fuzz_next(fuzz)) {
+	for(i = 0; !fuzz_done(fuzz); i++, fuzz_next(fuzz)) {
 		r = sshbuf_put(fuzzed, fuzz_ptr(fuzz), fuzz_len(fuzz));
 		ASSERT_INT_EQ(r, 0);
 		if (sshkey_parse_private_fileblob(fuzzed, "", &k1, NULL) == 0)
 			sshkey_free(k1);
 		sshbuf_reset(fuzzed);
+		if (test_is_fast() && i >= NUM_FAST_BASE64_TESTS)
+			break;
 	}
 	sshbuf_free(fuzzed);
 	fuzz_cleanup(fuzz);

regress/unittests/test_helper/test_helper.c

@@ -1,4 +1,4 @@
-/*	$OpenBSD: test_helper.c,v 1.8 2018/02/08 08:46:20 djm Exp $	*/
+/*	$OpenBSD: test_helper.c,v 1.9 2018/10/17 23:28:05 djm Exp $	*/
 /*
  * Copyright (c) 2011 Damien Miller <djm@mindrot.org>
  *
@@ -115,6 +115,8 @@ static test_onerror_func_t *test_onerror = NULL;
 static void *onerror_ctx = NULL;
 static const char *data_dir = NULL;
 static char subtest_info[512];
+static int fast = 0;
+static int slow = 0;
 
 int
 main(int argc, char **argv)
@@ -134,8 +136,14 @@ main(int argc, char **argv)
 		}
 	}
 
-	while ((ch = getopt(argc, argv, "vqd:")) != -1) {
+	while ((ch = getopt(argc, argv, "Ffvqd:")) != -1) {
 		switch (ch) {
+		case 'F':
+			slow = 1;
+			break;
+		case 'f':
+			fast = 1;
+			break;
 		case 'd':
 			data_dir = optarg;
 			break;
@@ -167,17 +175,29 @@ main(int argc, char **argv)
 }
 
 int
-test_is_verbose()
+test_is_verbose(void)
 {
 	return verbose_mode;
 }
 
 int
-test_is_quiet()
+test_is_quiet(void)
 {
 	return quiet_mode;
 }
 
+int
+test_is_fast(void)
+{
+	return fast;
+}
+
+int
+test_is_slow(void)
+{
+	return slow;
+}
+
 const char *
 test_data_file(const char *name)
 {

regress/unittests/test_helper/test_helper.h

@@ -1,4 +1,4 @@
-/*	$OpenBSD: test_helper.h,v 1.8 2018/02/08 08:46:20 djm Exp $	*/
+/*	$OpenBSD: test_helper.h,v 1.9 2018/10/17 23:28:05 djm Exp $	*/
 /*
  * Copyright (c) 2011 Damien Miller <djm@mindrot.org>
  *
@@ -45,6 +45,8 @@ void set_onerror_func(test_onerror_func_t *f, void *ctx);
 void test_done(void);
 int test_is_verbose(void);
 int test_is_quiet(void);
+int test_is_fast(void);
+int test_is_slow(void);
 void test_subtest_info(const char *fmt, ...)
     __attribute__((format(printf, 1, 2)));
 void ssl_err_check(const char *file, int line);