kex: Add sntrup761x25519-sha512@openssh.com.

All of the initial work was done by Simon. Jakub cleaned up the
formatting issues, resolved the padding of bignum to match specs
and be interoperable with OpenSSH (and few more minor details).

Closes: #194.

Signed-off-by: Simon Josefsson <simon@josefsson.org>
Signed-off-by: Jakub Jelen <jjelen@redhat.com>
Reviewed-by: Andreas Schneider <asn@cryptomilk.org>
Reviewed-by: Sahana Prasad <sahana@redhat.com>

include/libssh/crypto.h

@@ -45,10 +45,11 @@
 #ifdef HAVE_OPENSSL_ECDH_H
 #include <openssl/ecdh.h>
 #endif
+#include "libssh/curve25519.h"
 #include "libssh/dh.h"
 #include "libssh/ecdh.h"
 #include "libssh/kex.h"
-#include "libssh/curve25519.h"
+#include "libssh/sntrup761.h"
 
 #define DIGEST_MAX_LEN 64
 
@@ -82,6 +83,8 @@ enum ssh_key_exchange_e {
     SSH_KEX_DH_GROUP18_SHA512,
     /* diffie-hellman-group14-sha256 */
     SSH_KEX_DH_GROUP14_SHA256,
+    /* sntrup761x25519-sha512@openssh.com */
+    SSH_KEX_SNTRUP761X25519_SHA512_OPENSSH_COM,
 };
 
 enum ssh_cipher_e {
@@ -132,6 +135,11 @@ struct ssh_crypto_struct {
 #endif
     ssh_curve25519_pubkey curve25519_client_pubkey;
     ssh_curve25519_pubkey curve25519_server_pubkey;
+#endif
+#ifdef HAVE_SNTRUP761
+    ssh_sntrup761_privkey sntrup761_privkey;
+    ssh_sntrup761_pubkey sntrup761_client_pubkey;
+    ssh_sntrup761_ciphertext sntrup761_ciphertext;
 #endif
     ssh_string dh_server_signature; /* information used by dh_handshake. */
     size_t session_id_len;

include/libssh/sntrup761.h

@@ -0,0 +1,86 @@
+/*
+ * This file is part of the SSH Library
+ *
+ * Copyright (c) 2013 by Aris Adamantiadis <aris@badcode.be>
+ * Copyright (c) 2023 Simon Josefsson <simon@josefsson.org>
+ * Copyright (c) 2025 Jakub Jelen <jjelen@redhat.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation,
+ * version 2.1 of the License.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#ifndef SNTRUP761_H_
+#define SNTRUP761_H_
+
+#include "config.h"
+#include "curve25519.h"
+#include "libssh.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifdef HAVE_CURVE25519
+#define HAVE_SNTRUP761 1
+#endif
+
+extern void crypto_hash_sha512(unsigned char *out,
+                               const unsigned char *in,
+                               unsigned long long inlen);
+
+/*
+ * Derived from public domain source, written by (in alphabetical order):
+ * - Daniel J. Bernstein
+ * - Chitchanok Chuengsatiansup
+ * - Tanja Lange
+ * - Christine van Vredendaal
+ */
+
+#include <stdint.h>
+#include <string.h>
+
+#define SNTRUP761_SECRETKEY_SIZE 1763
+#define SNTRUP761_PUBLICKEY_SIZE 1158
+#define SNTRUP761_CIPHERTEXT_SIZE 1039
+#define SNTRUP761_SIZE 32
+
+typedef void sntrup761_random_func(void *ctx, size_t length, uint8_t *dst);
+
+void sntrup761_keypair(uint8_t *pk,
+                       uint8_t *sk,
+                       void *random_ctx,
+                       sntrup761_random_func *random);
+void sntrup761_enc(uint8_t *c,
+                   uint8_t *k,
+                   const uint8_t *pk,
+                   void *random_ctx,
+                   sntrup761_random_func *random);
+void sntrup761_dec(uint8_t *k, const uint8_t *c, const uint8_t *sk);
+
+typedef unsigned char ssh_sntrup761_pubkey[SNTRUP761_PUBLICKEY_SIZE];
+typedef unsigned char ssh_sntrup761_privkey[SNTRUP761_SECRETKEY_SIZE];
+typedef unsigned char ssh_sntrup761_ciphertext[SNTRUP761_CIPHERTEXT_SIZE];
+
+int ssh_client_sntrup761x25519_init(ssh_session session);
+void ssh_client_sntrup761x25519_remove_callbacks(ssh_session session);
+
+#ifdef WITH_SERVER
+void ssh_server_sntrup761x25519_init(ssh_session session);
+#endif /* WITH_SERVER */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* SNTRUP761_H_ */