kex: do not ignore failure of libssh2_sha1_init()

Based upon 43b730ce56f010e9d33573fcb020df49798c1ed8.
Fixes ticket 290. Thanks for the suggestion, mstrsn.

src/hostkey.c

@@ -500,7 +500,9 @@ libssh2_hostkey_hash(LIBSSH2_SESSION * session, int hash_type)
         break;
 #endif /* LIBSSH2_MD5 */
     case LIBSSH2_HOSTKEY_HASH_SHA1:
-        return (char *) session->server_hostkey_sha1;
+        return (session->server_hostkey_sha1_valid)
+          ? (char *) session->server_hostkey_sha1
+          : NULL;
         break;
     default:
         return NULL;

src/kex.c

@@ -221,7 +221,8 @@ static int diffie_hellman_sha1(LIBSSH2_SESSION *session,
             if (libssh2_md5_init(&fingerprint_ctx)) {
                 libssh2_md5_update(fingerprint_ctx, session->server_hostkey,
                                    session->server_hostkey_len);
-                libssh2_md5_final(fingerprint_ctx, session->server_hostkey_md5);
+                libssh2_md5_final(fingerprint_ctx,
+                                  session->server_hostkey_md5);
                 session->server_hostkey_md5_valid = TRUE;
             }
             else {
@@ -245,10 +246,16 @@ static int diffie_hellman_sha1(LIBSSH2_SESSION *session,
         {
             libssh2_sha1_ctx fingerprint_ctx;
 
-            libssh2_sha1_init(&fingerprint_ctx);
+            if (libssh2_sha1_init(&fingerprint_ctx)) {
                 libssh2_sha1_update(fingerprint_ctx, session->server_hostkey,
                                     session->server_hostkey_len);
-            libssh2_sha1_final(fingerprint_ctx, session->server_hostkey_sha1);
+                libssh2_sha1_final(fingerprint_ctx,
+                                   session->server_hostkey_sha1);
+                session->server_hostkey_sha1_valid = TRUE;
+            }
+            else {
+                session->server_hostkey_sha1_valid = FALSE;
+            }
         }
 #ifdef LIBSSH2DEBUG
         {

src/libgcrypt.h

@@ -60,7 +60,10 @@
   (gcry_randomize ((buf), (len), GCRY_STRONG_RANDOM), 1)
 
 #define libssh2_sha1_ctx gcry_md_hd_t
-#define libssh2_sha1_init(ctx) gcry_md_open (ctx,  GCRY_MD_SHA1, 0);
+
+/* returns 0 in case of failure */
+#define libssh2_sha1_init(ctx) \
+  (GPG_ERR_NO_ERROR == gcry_md_open (ctx,  GCRY_MD_SHA1, 0))
 #define libssh2_sha1_update(ctx, data, len) \
   gcry_md_write (ctx, (unsigned char *) data, len)
 #define libssh2_sha1_final(ctx, out) \

src/libssh2_priv.h

@@ -600,6 +600,7 @@ struct _LIBSSH2_SESSION
     int server_hostkey_md5_valid;
 #endif                          /* ! LIBSSH2_MD5 */
     unsigned char server_hostkey_sha1[SHA_DIGEST_LENGTH];
+    int server_hostkey_sha1_valid;
 
     /* (remote as source of data -- packet_read ) */
     libssh2_endpoint_data remote;

src/openssl.h

@@ -107,6 +107,8 @@
 #define _libssh2_random(buf, len) RAND_bytes ((buf), (len))
 
 #define libssh2_sha1_ctx EVP_MD_CTX
+
+/* returns 0 in case of failure */
 int libssh2_sha1_init(libssh2_sha1_ctx *ctx);
 #define libssh2_sha1_update(ctx, data, len) EVP_DigestUpdate(&(ctx), data, len)
 #define libssh2_sha1_final(ctx, out) EVP_DigestFinal(&(ctx), out, NULL)

src/wincng.h

@@ -123,7 +123,7 @@ typedef struct __libssh2_wincng_hash_ctx {
 #define libssh2_sha1_ctx _libssh2_wincng_hash_ctx
 #define libssh2_sha1_init(ctx) \
   _libssh2_wincng_hash_init(ctx, _libssh2_wincng.hAlgHashSHA1, \
-                            SHA_DIGEST_LENGTH, NULL, 0)
+                            SHA_DIGEST_LENGTH, NULL, 0) == 0
 #define libssh2_sha1_update(ctx, data, datalen) \
   _libssh2_wincng_hash_update(&ctx, (unsigned char *) data, datalen)
 #define libssh2_sha1_final(ctx, hash) \