Disable sntrup761 if compiler doesn't support VLAs.

The sntrup761 code sourced from supercop uses variable length
arrays.  Although widely supported, they are not part of the ANSI
C89 spec so if the compiler does not support VLAs, disable the
sntrup761x25519-sha512@openssh.com KEX method by replacing the kex
functions with no-op ones similar to what we do in kexecdh.c.

This should allow OpenSSH to build with a plain C89 compiler again.
Spotted by tim@, ok djm@.

configure.ac

@@ -297,6 +297,16 @@ typedef void foo(const char *, ...) __attribute__((format(printf, 1, 2)));]],
 	 [compiler does not accept __attribute__ on prototype args]) ]
 )
 
+AC_MSG_CHECKING([if compiler supports variable length arrays])
+AC_COMPILE_IFELSE(
+    [AC_LANG_PROGRAM([[#include <stdlib.h>]],
+    [[ int i; for (i=0; i<3; i++){int a[i]; a[i-1]=0;} exit(0); ]])],
+    [ AC_MSG_RESULT([yes])
+      AC_DEFINE(VARIABLE_LENGTH_ARRAYS, [1],
+	 [compiler supports variable length arrays]) ],
+    [ AC_MSG_RESULT([no]) ]
+)
+
 if test "x$no_attrib_nonnull" != "x1" ; then
 	AC_DEFINE([HAVE_ATTRIBUTE__NONNULL__], [1], [Have attribute nonnull])
 fi

defines.h

@@ -894,4 +894,11 @@ struct winsize {
 # define USE_SYSTEM_GLOB
 #endif
 
+/*
+ * sntrup761 uses variable length arrays, only enable if the compiler
+ * supports them.
+ */
+#ifdef VARIABLE_LENGTH_ARRAYS
+# define USE_SNTRUP761X25519 1
+#endif
 #endif /* _DEFINES_H */

kex.c

@@ -110,8 +110,10 @@ static const struct kexalg kexalgs[] = {
 #if defined(HAVE_EVP_SHA256) || !defined(WITH_OPENSSL)
 	{ KEX_CURVE25519_SHA256, KEX_C25519_SHA256, 0, SSH_DIGEST_SHA256 },
 	{ KEX_CURVE25519_SHA256_OLD, KEX_C25519_SHA256, 0, SSH_DIGEST_SHA256 },
+#ifdef USE_SNTRUP761X25519
 	{ KEX_SNTRUP761X25519_SHA512, KEX_KEM_SNTRUP761X25519_SHA512, 0,
 	    SSH_DIGEST_SHA512 },
+#endif
 #endif /* HAVE_EVP_SHA256 || !WITH_OPENSSL */
 	{ NULL, 0, -1, -1},
 };

kexsntrup761x25519.c

@@ -25,6 +25,8 @@
 
 #include "includes.h"
 
+#ifdef USE_SNTRUP761X25519
+
 #include <sys/types.h>
 
 #include <stdio.h>
@@ -217,3 +219,33 @@ kex_kem_sntrup761x25519_dec(struct kex *kex,
 	sshbuf_free(buf);
 	return r;
 }
+
+#else
+
+#include "ssherr.h"
+
+struct kex;
+struct sshbuf;
+struct sshkey;
+
+int
+kex_kem_sntrup761x25519_keypair(struct kex *kex)
+{
+	return SSH_ERR_SIGN_ALG_UNSUPPORTED;
+}
+
+int
+kex_kem_sntrup761x25519_enc(struct kex *kex,
+   const struct sshbuf *client_blob, struct sshbuf **server_blobp,
+   struct sshbuf **shared_secretp)
+{
+	return SSH_ERR_SIGN_ALG_UNSUPPORTED;
+}
+
+int
+kex_kem_sntrup761x25519_dec(struct kex *kex,
+    const struct sshbuf *server_blob, struct sshbuf **shared_secretp)
+{
+	return SSH_ERR_SIGN_ALG_UNSUPPORTED;
+}
+#endif /* USE_SNTRUP761X25519 */

sntrup761.c

@@ -10,6 +10,8 @@
 
 #include "includes.h"
 
+#ifdef USE_SNTRUP761X25519
+
 #include <string.h>
 #include "crypto_api.h"
 
@@ -1268,4 +1270,4 @@ int crypto_kem_sntrup761_dec(unsigned char *k,const unsigned char *c,const unsig
   Decap(k,c,sk);
   return 0;
 }
-
+#endif /* USE_SNTRUP761X25519 */