Add return values to set_encrypt_key and set_decrypt_key.

git-svn-id: svn+ssh://svn.berlios.de/svnroot/repos/libssh/trunk@519 7dcaeef0-15fb-0310-b436-a5af3365683c

include/libssh/crypto.h

@@ -54,15 +54,21 @@ struct crypto_struct {
 #endif
     unsigned int keysize; /* bytes of key used. != keylen */
 #ifdef HAVE_LIBGCRYPT
-    void (*set_encrypt_key)(struct crypto_struct *cipher, void *key, void *IV); /* sets the new key for immediate use */
+    /* sets the new key for immediate use */
-    void (*set_decrypt_key)(struct crypto_struct *cipher, void *key, void *IV);
+    int (*set_encrypt_key)(struct crypto_struct *cipher, void *key, void *IV);
-    void (*cbc_encrypt)(struct crypto_struct *cipher, void *in, void *out,unsigned long len);
+    int (*set_decrypt_key)(struct crypto_struct *cipher, void *key, void *IV);
-    void (*cbc_decrypt)(struct crypto_struct *cipher, void *in, void *out,unsigned long len);
+    void (*cbc_encrypt)(struct crypto_struct *cipher, void *in, void *out,
+        unsigned long len);
+    void (*cbc_decrypt)(struct crypto_struct *cipher, void *in, void *out,
+        unsigned long len);
 #elif defined HAVE_LIBCRYPTO
-    void (*set_encrypt_key)(struct crypto_struct *cipher, void *key); /* sets the new key for immediate use */
+    /* sets the new key for immediate use */
-    void (*set_decrypt_key)(struct crypto_struct *cipher, void *key);
+    int (*set_encrypt_key)(struct crypto_struct *cipher, void *key);
-    void (*cbc_encrypt)(struct crypto_struct *cipher, void *in, void *out,unsigned long len,void *IV);
+    int (*set_decrypt_key)(struct crypto_struct *cipher, void *key);
-    void (*cbc_decrypt)(struct crypto_struct *cipher, void *in, void *out,unsigned long len,void *IV);
+    void (*cbc_encrypt)(struct crypto_struct *cipher, void *in, void *out,
+        unsigned long len, void *IV);
+    void (*cbc_decrypt)(struct crypto_struct *cipher, void *in, void *out,
+        unsigned long len, void *IV);
 #endif
 };
 

libssh/crypt.c

@@ -67,10 +67,17 @@ int packet_decrypt(SSH_SESSION *session, void *data,u32 len) {
   ssh_log(session,SSH_LOG_PACKET, "Decrypting %d bytes", len);
 
 #ifdef HAVE_LIBGCRYPT
-  crypto->set_decrypt_key(crypto,session->current_crypto->decryptkey,session->current_crypto->decryptIV);
+  if (crypto->set_decrypt_key(crypto, session->current_crypto->decryptkey,
+        session->current_crypto->decryptIV) < 0) {
+    SAFE_FREE(out);
+    return -1;
+  }
   crypto->cbc_decrypt(crypto,data,out,len);
 #elif defined HAVE_LIBCRYPTO
-  crypto->set_decrypt_key(crypto,session->current_crypto->decryptkey);
+  if (crypto->set_decrypt_key(crypto, session->current_crypto->decryptkey) < 0) {
+    SAFE_FREE(out);
+    return -1;
+  }
   crypto->cbc_decrypt(crypto,data,out,len,session->current_crypto->decryptIV);
 #endif
 
@@ -105,10 +112,16 @@ unsigned char *packet_encrypt(SSH_SESSION *session, void *data, u32 len) {
       session->send_seq,len);
 
 #ifdef HAVE_LIBGCRYPT
-  crypto->set_encrypt_key(crypto, session->current_crypto->encryptkey,
+  if (crypto->set_encrypt_key(crypto, session->current_crypto->encryptkey,
-      session->current_crypto->encryptIV);
+      session->current_crypto->encryptIV) < 0) {
+    SAFE_FREE(out);
+    return NULL;
+  }
 #elif defined HAVE_LIBCRYPTO
-  crypto->set_encrypt_key(crypto, session->current_crypto->encryptkey);
+  if (crypto->set_encrypt_key(crypto, session->current_crypto->encryptkey) < 0) {
+    SAFE_FREE(out);
+    return NULL;
+  }
 #endif
 
   if (session->version == 2) {

libssh/wrapper.c

@@ -39,7 +39,6 @@
 #include <stdio.h>
 #include <string.h>
 
-/* TODO FIXME */
 static int alloc_key(struct crypto_struct *cipher) {
     cipher->key = malloc(cipher->keylen);
     if (cipher->key == NULL) {
@@ -120,105 +119,185 @@ void hmac_final(HMACCTX c, unsigned char *hashmacbuf, unsigned int *len) {
 }
 
 /* the wrapper functions for blowfish */
-static void blowfish_set_key(struct crypto_struct *cipher, void *key, void *IV){
+static int blowfish_set_key(struct crypto_struct *cipher, void *key, void *IV){
   if (cipher->key == NULL) {
-      /* TODO FIXME */
     if (alloc_key(cipher) < 0) {
-        return;
+      return -1;
     }
-      gcry_cipher_open(&cipher->key[0],GCRY_CIPHER_BLOWFISH,GCRY_CIPHER_MODE_CBC,0);
+
-      gcry_cipher_setkey(cipher->key[0],key,16);
+    if (gcry_cipher_open(&cipher->key[0], GCRY_CIPHER_BLOWFISH,
-      gcry_cipher_setiv(cipher->key[0],IV,8);
+        GCRY_CIPHER_MODE_CBC, 0)) {
+      SAFE_FREE(cipher->key);
+      return -1;
+    }
+    if (gcry_cipher_setkey(cipher->key[0], key, 16)) {
+      SAFE_FREE(cipher->key);
+      return -1;
+    }
+    if (gcry_cipher_setiv(cipher->key[0], IV, 8)) {
+      SAFE_FREE(cipher->key);
+      return -1;
     }
   }
 
-static void blowfish_encrypt(struct crypto_struct *cipher, void *in, void *out,unsigned long len){
+  return 0;
+}
+
+static void blowfish_encrypt(struct crypto_struct *cipher, void *in,
+    void *out, unsigned long len) {
   gcry_cipher_encrypt(cipher->key[0], out, len, in, len);
 }
 
-static void blowfish_decrypt(struct crypto_struct *cipher, void *in, void *out, unsigned long len){
+static void blowfish_decrypt(struct crypto_struct *cipher, void *in,
+    void *out, unsigned long len) {
   gcry_cipher_decrypt(cipher->key[0], out, len, in, len);
 }
 
-static void aes_set_key(struct crypto_struct *cipher, void *key, void *IV){
+static int aes_set_key(struct crypto_struct *cipher, void *key, void *IV) {
-    if(!cipher->key){
+  if (cipher->key == NULL) {
-        /* TODO FIXME */
     if (alloc_key(cipher) < 0) {
-          return;
+      return -1;
     }
+
     switch (cipher->keysize) {
       case 128:
-            gcry_cipher_open(&cipher->key[0],GCRY_CIPHER_AES128,GCRY_CIPHER_MODE_CBC,0);
+        if (gcry_cipher_open(&cipher->key[0], GCRY_CIPHER_AES128,
+              GCRY_CIPHER_MODE_CBC, 0)) {
+          SAFE_FREE(cipher->key);
+          return -1;
+        }
         break;
       case 192:
-            gcry_cipher_open(&cipher->key[0],GCRY_CIPHER_AES192,GCRY_CIPHER_MODE_CBC,0);
+        if (gcry_cipher_open(&cipher->key[0], GCRY_CIPHER_AES192,
+              GCRY_CIPHER_MODE_CBC, 0)) {
+          SAFE_FREE(cipher->key);
+          return -1;
+        }
         break;
       case 256:
-            gcry_cipher_open(&cipher->key[0],GCRY_CIPHER_AES256,GCRY_CIPHER_MODE_CBC,0);
+        if (gcry_cipher_open(&cipher->key[0], GCRY_CIPHER_AES256,
+              GCRY_CIPHER_MODE_CBC, 0)) {
+          SAFE_FREE(cipher->key);
+          return -1;
+        }
         break;
     }
-        gcry_cipher_setkey(cipher->key[0],key,cipher->keysize/8);
+    if (gcry_cipher_setkey(cipher->key[0], key, cipher->keysize / 8)) {
-        gcry_cipher_setiv(cipher->key[0], IV, 16);
+      SAFE_FREE(cipher->key);
+      return -1;
+    }
+    if (gcry_cipher_setiv(cipher->key[0], IV, 16)) {
+      SAFE_FREE(cipher->key);
+      return -1;
     }
   }
 
-static void aes_encrypt(struct crypto_struct *cipher, void *in, void *out, unsigned long len){
+  return 0;
-    gcry_cipher_encrypt(cipher->key[0],out,len,in,len);
 }
 
-static void aes_decrypt(struct crypto_struct *cipher, void *in, void *out,unsigned long len){
+static void aes_encrypt(struct crypto_struct *cipher, void *in, void *out,
-    gcry_cipher_decrypt(cipher->key[0],out,len,in,len);
-}
-
-static void des3_set_key(struct crypto_struct *cipher, void *key, void *IV){
-    if(!cipher->key){
-        /* TODO FIXME */
-        if (alloc_key(cipher) < 0) {
-          return;
-        }
-        gcry_cipher_open(&cipher->key[0],GCRY_CIPHER_3DES,GCRY_CIPHER_MODE_CBC,0);
-        gcry_cipher_setkey(cipher->key[0],key,24);
-        gcry_cipher_setiv(cipher->key[0],IV,8);
-    }
-}
-
-static void des3_encrypt(struct crypto_struct *cipher, void *in, void *out, 
     unsigned long len) {
   gcry_cipher_encrypt(cipher->key[0], out, len, in, len);
 }
 
-static void des3_decrypt(struct crypto_struct *cipher, void *in, void *out, 
+static void aes_decrypt(struct crypto_struct *cipher, void *in, void *out,
     unsigned long len) {
   gcry_cipher_decrypt(cipher->key[0], out, len, in, len);
 }
 
-static void des3_1_set_key(struct crypto_struct *cipher, void *key, void *IV){
+static int des3_set_key(struct crypto_struct *cipher, void *key, void *IV) {
-    if(!cipher->key){
+  if (cipher->key == NULL) {
-        /* TODO FIXME */
     if (alloc_key(cipher) < 0) {
-          return;
+      return -1;
     }
-        gcry_cipher_open(&cipher->key[0],GCRY_CIPHER_DES,GCRY_CIPHER_MODE_CBC,0);
+    if (gcry_cipher_open(&cipher->key[0], GCRY_CIPHER_3DES,
-        gcry_cipher_setkey(cipher->key[0],key,8);
+          GCRY_CIPHER_MODE_CBC, 0)) {
-        gcry_cipher_setiv(cipher->key[0],IV,8);
+      SAFE_FREE(cipher->key);
-        gcry_cipher_open(&cipher->key[1],GCRY_CIPHER_DES,GCRY_CIPHER_MODE_CBC,0);
+      return -1;
-        gcry_cipher_setkey(cipher->key[1],key+8,8);
+    }
-        gcry_cipher_setiv(cipher->key[1],IV+8,8);
+    if (gcry_cipher_setkey(cipher->key[0], key, 24)) {
-        gcry_cipher_open(&cipher->key[2],GCRY_CIPHER_DES,GCRY_CIPHER_MODE_CBC,0);
+      SAFE_FREE(cipher->key);
-        gcry_cipher_setkey(cipher->key[2],key+16,8);
+      return -1;
-        gcry_cipher_setiv(cipher->key[2],IV+16,8);
+    }
+    if (gcry_cipher_setiv(cipher->key[0], IV, 8)) {
+      SAFE_FREE(cipher->key);
+      return -1;
     }
   }
 
-static void des3_1_encrypt(struct crypto_struct *cipher, void *in, void *out, 
+  return 0;
-        unsigned long len){
+}
+
+static void des3_encrypt(struct crypto_struct *cipher, void *in,
+    void *out, unsigned long len) {
+  gcry_cipher_encrypt(cipher->key[0], out, len, in, len);
+}
+
+static void des3_decrypt(struct crypto_struct *cipher, void *in,
+    void *out, unsigned long len) {
+  gcry_cipher_decrypt(cipher->key[0], out, len, in, len);
+}
+
+static int des3_1_set_key(struct crypto_struct *cipher, void *key, void *IV) {
+  if (cipher->key == NULL) {
+    if (alloc_key(cipher) < 0) {
+      return -1;
+    }
+    if (gcry_cipher_open(&cipher->key[0], GCRY_CIPHER_DES,
+          GCRY_CIPHER_MODE_CBC, 0)) {
+      SAFE_FREE(cipher->key);
+      return -1;
+    }
+    if (gcry_cipher_setkey(cipher->key[0], key, 8)) {
+      SAFE_FREE(cipher->key);
+      return -1;
+    }
+    if (gcry_cipher_setiv(cipher->key[0], IV, 8)) {
+      SAFE_FREE(cipher->key);
+      return -1;
+    }
+
+    if (gcry_cipher_open(&cipher->key[1], GCRY_CIPHER_DES,
+          GCRY_CIPHER_MODE_CBC, 0)) {
+      SAFE_FREE(cipher->key);
+      return -1;
+    }
+    if (gcry_cipher_setkey(cipher->key[1], key + 8, 8)) {
+      SAFE_FREE(cipher->key);
+      return -1;
+    }
+    if (gcry_cipher_setiv(cipher->key[1], IV + 8, 8)) {
+      SAFE_FREE(cipher->key);
+      return -1;
+    }
+
+    if (gcry_cipher_open(&cipher->key[2], GCRY_CIPHER_DES,
+          GCRY_CIPHER_MODE_CBC, 0)) {
+      SAFE_FREE(cipher->key);
+      return -1;
+    }
+    if (gcry_cipher_setkey(cipher->key[2], key + 16, 8)) {
+      SAFE_FREE(cipher->key);
+      return -1;
+    }
+    if (gcry_cipher_setiv(cipher->key[2], IV + 16, 8)) {
+      SAFE_FREE(cipher->key);
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+static void des3_1_encrypt(struct crypto_struct *cipher, void *in,
+    void *out, unsigned long len) {
   gcry_cipher_encrypt(cipher->key[0], out, len, in, len);
   gcry_cipher_decrypt(cipher->key[1], in, len, out, len);
   gcry_cipher_encrypt(cipher->key[2], out, len, in, len);
 }
 
-static void des3_1_decrypt(struct crypto_struct *cipher, void *in, void *out, 
+static void des3_1_decrypt(struct crypto_struct *cipher, void *in,
-        unsigned long len){
+    void *out, unsigned long len) {
   gcry_cipher_decrypt(cipher->key[2], out, len, in, len);
   gcry_cipher_encrypt(cipher->key[1], in, len, out, len);
   gcry_cipher_decrypt(cipher->key[0], out, len, in, len);
@@ -357,47 +436,61 @@ static void blowfish_set_key(struct crypto_struct *cipher, void *key){
     }
 }
 
-static void blowfish_encrypt(struct crypto_struct *cipher, void *in, void *out,unsigned long len,void *IV){
+static void blowfish_encrypt(struct crypto_struct *cipher, void *in,
+    void *out, unsigned long len, void *IV) {
   BF_cbc_encrypt(in,out,len,cipher->key,IV,BF_ENCRYPT);
 }
 
-static void blowfish_decrypt(struct crypto_struct *cipher, void *in, void *out,unsigned long len,void *IV){
+static void blowfish_decrypt(struct crypto_struct *cipher, void *in,
+    void *out, unsigned long len, void *IV) {
     BF_cbc_encrypt(in,out,len,cipher->key,IV,BF_DECRYPT);
 }
-#endif
+#endif /* HAS_BLOWFISH */
+
 #ifdef HAS_AES
-static void aes_set_encrypt_key(struct crypto_struct *cipher, void *key){
+static int aes_set_encrypt_key(struct crypto_struct *cipher, void *key) {
-    if(!cipher->key){
+  if (cipher->key == NULL) {
-        /* TODO FIXME */
     if (alloc_key(cipher) < 0) {
-          return;
+      return -1;
     }
-        AES_set_encrypt_key(key,cipher->keysize,cipher->key);
+    if (AES_set_encrypt_key(key,cipher->keysize,cipher->key) < 0) {
+      SAFE_FREE(cipher->key);
+      return -1;
     }
   }
-static void aes_set_decrypt_key(struct crypto_struct *cipher, void *key){
+
-    if(!cipher->key){
+  return 0;
-        /* TODO FIXME */
+}
+static int aes_set_decrypt_key(struct crypto_struct *cipher, void *key) {
+  if (cipher->key == NULL) {
     if (alloc_key(cipher) < 0) {
-          return;
+      return -1;
     }
-        AES_set_decrypt_key(key,cipher->keysize,cipher->key);
+    if (AES_set_decrypt_key(key,cipher->keysize,cipher->key) < 0) {
+      SAFE_FREE(cipher->key);
+      return -1;
     }
   }
-static void aes_encrypt(struct crypto_struct *cipher, void *in, void *out, unsigned long len, void *IV){
+}
+
+static void aes_encrypt(struct crypto_struct *cipher, void *in, void *out,
+    unsigned long len, void *IV) {
   AES_cbc_encrypt(in, out, len, cipher->key, IV, AES_ENCRYPT);
 }
-static void aes_decrypt(struct crypto_struct *cipher, void *in, void *out, unsigned long len, void *IV){
+
+static void aes_decrypt(struct crypto_struct *cipher, void *in, void *out,
+    unsigned long len, void *IV) {
   AES_cbc_encrypt(in, out, len, cipher->key, IV, AES_DECRYPT);
 }
-#endif
+#endif /* HAS_AES */
+
 #ifdef HAS_DES
-static void des3_set_key(struct crypto_struct *cipher, void *key){
+static int des3_set_key(struct crypto_struct *cipher, void *key) {
-    if(!cipher->key){
+  if (cipher->key == NULL) {
-        /* TODO FIXME */
     if (alloc_key(cipher) < 0) {
-          return;
+      return -1;
     }
+
     DES_set_odd_parity(key);
     DES_set_odd_parity(key + 8);
     DES_set_odd_parity(key + 16);
@@ -405,22 +498,30 @@ static void des3_set_key(struct crypto_struct *cipher, void *key){
     DES_set_key_unchecked(key + 8, cipher->key + sizeof(DES_key_schedule));
     DES_set_key_unchecked(key + 16, cipher->key + 2 * sizeof(DES_key_schedule));
   }
+
+  return 0;
 }
 
-static void des3_encrypt(struct crypto_struct *cipher, void *in, void *out, 
+static void des3_encrypt(struct crypto_struct *cipher, void *in,
-        unsigned long len, void *IV){
+    void *out, unsigned long len, void *IV) {
-    DES_ede3_cbc_encrypt(in,out,len,cipher->key,cipher->key+sizeof(DES_key_schedule),cipher->key+2*sizeof(DES_key_schedule),IV,1);
+  DES_ede3_cbc_encrypt(in, out, len, cipher->key,
+      cipher->key + sizeof(DES_key_schedule),
+      cipher->key + 2 * sizeof(DES_key_schedule),
+      IV, 1);
 }
 
-static void des3_decrypt(struct crypto_struct *cipher, void *in, void *out, 
+static void des3_decrypt(struct crypto_struct *cipher, void *in,
-        unsigned long len, void *IV){
+    void *out, unsigned long len, void *IV) {
-    DES_ede3_cbc_encrypt(in,out,len,cipher->key,cipher->key+sizeof(DES_key_schedule),cipher->key+2*sizeof(DES_key_schedule),IV,0);
+  DES_ede3_cbc_encrypt(in, out, len, cipher->key,
+      cipher->key + sizeof(DES_key_schedule),
+      cipher->key + 2 * sizeof(DES_key_schedule),
+      IV, 0);
 }
 
-static void des3_1_encrypt(struct crypto_struct *cipher, void *in, void *out, 
+static void des3_1_encrypt(struct crypto_struct *cipher, void *in,
-        unsigned long len, void *IV){
+    void *out, unsigned long len, void *IV) {
 #ifdef DEBUG_CRYPTO
-    ssh_print_hexa("encrypt IV before",IV,24);
+  ssh_print_hexa("Encrypt IV before", IV, 24);
 #endif
   DES_ncbc_encrypt(in, out, len, cipher->key, IV, 1);
   DES_ncbc_encrypt(out, in, len, cipher->key + sizeof(DES_key_schedule),
@@ -428,26 +529,27 @@ static void des3_1_encrypt(struct crypto_struct *cipher, void *in, void *out,
   DES_ncbc_encrypt(in, out, len, cipher->key + 2 * sizeof(DES_key_schedule),
       IV + 16, 1);
 #ifdef DEBUG_CRYPTO
-    ssh_print_hexa("encrypt IV after",IV,24);
+  ssh_print_hexa("Encrypt IV after", IV, 24);
 #endif
 }
 
-static void des3_1_decrypt(struct crypto_struct *cipher, void *in, void *out, 
+static void des3_1_decrypt(struct crypto_struct *cipher, void *in,
-        unsigned long len, void *IV){
+    void *out, unsigned long len, void *IV) {
 #ifdef DEBUG_CRYPTO
-    ssh_print_hexa("decrypt IV before",IV,24);
+  ssh_print_hexa("Decrypt IV before", IV, 24);
 #endif
+
   DES_ncbc_encrypt(in, out, len, cipher->key + 2 * sizeof(DES_key_schedule),
       IV, 0);
   DES_ncbc_encrypt(out, in, len, cipher->key + sizeof(DES_key_schedule),
       IV + 8, 1);
-    DES_ncbc_encrypt(in,out,len, cipher->key,
+  DES_ncbc_encrypt(in, out, len, cipher->key, IV + 16, 0);
-            IV+16,0);
+
 #ifdef DEBUG_CRYPTO
-    ssh_print_hexa("decrypt IV after",IV,24);
+  ssh_print_hexa("Decrypt IV after", IV, 24);
 #endif
 }
-#endif
+#endif /* HAS_DES */
 
 /* the table of supported ciphers */
 static struct crypto_struct ssh_ciphertab[] = {