--- sourceforge.net/trunk/rdesktop/secure.c	2001/06/20 13:54:48	28
+++ sourceforge.net/trunk/rdesktop/secure.c	2002/07/13 02:30:49	57
@@ -1,7 +1,7 @@
 /*
    rdesktop: A Remote Desktop Protocol client.
    Protocol services - RDP encryption and licensing
-   Copyright (C) Matthew Chapman 1999-2000
+   Copyright (C) Matthew Chapman 1999-2001
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -19,28 +19,36 @@
 */
 
 #include "rdesktop.h"
+
+#ifdef WITH_OPENSSL
+#include <openssl/rc4.h>
+#include <openssl/md5.h>
+#include <openssl/sha.h>
+#include <openssl/bn.h>
+#else
 #include "crypto/rc4.h"
 #include "crypto/md5.h"
 #include "crypto/sha.h"
-#include "crypto/arith.h"
+#include "crypto/bn.h"
+#endif
 
 extern char hostname[16];
 extern int width;
 extern int height;
 extern int keylayout;
-extern BOOL use_encryption;
+extern BOOL encryption;
 extern BOOL licence_issued;
 
 static int rc4_key_len;
 static RC4_KEY rc4_decrypt_key;
 static RC4_KEY rc4_encrypt_key;
 
-static uint8 sec_sign_key[8];
+static uint8 sec_sign_key[16];
 static uint8 sec_decrypt_key[16];
 static uint8 sec_encrypt_key[16];
-static uint8 sec_decrypt_update_key[8];
-static uint8 sec_encrypt_update_key[8];
-static uint8 sec_crypted_random[64];
+static uint8 sec_decrypt_update_key[16];
+static uint8 sec_encrypt_update_key[16];
+static uint8 sec_crypted_random[SEC_MODULUS_SIZE];
 
 /*
  * General purpose 48-byte transformation, using two 32-byte salts (generally,
@@ -115,8 +123,8 @@
 	sec_hash_48(temp_hash, input, client_key, server_key, 65);
 	sec_hash_48(session_key, temp_hash, client_key, server_key, 88);
 
-	/* Store first 8 bytes of session key, for generating signatures */
-	memcpy(sec_sign_key, session_key, 8);
+	/* Store first 16 bytes of session key, for generating signatures */
+	memcpy(sec_sign_key, session_key, 16);
 
 	/* Generate RC4 keys */
 	sec_hash_16(sec_decrypt_key, &session_key[16], client_key,
@@ -126,7 +134,7 @@
 
 	if (rc4_key_size == 1)
 	{
-		DEBUG("40-bit encryption enabled\n");
+		DEBUG(("40-bit encryption enabled\n"));
 		sec_make_40bit(sec_sign_key);
 		sec_make_40bit(sec_decrypt_key);
 		sec_make_40bit(sec_encrypt_key);
@@ -134,13 +142,13 @@
 	}
 	else
 	{
-		DEBUG("128-bit encryption enabled\n");
+		DEBUG(("128-bit encryption enabled\n"));
 		rc4_key_len = 16;
 	}
 
-	/* Store first 8 bytes of RC4 keys as update keys */
-	memcpy(sec_decrypt_update_key, sec_decrypt_key, 8);
-	memcpy(sec_encrypt_update_key, sec_encrypt_key, 8);
+	/* Save initial RC4 keys as update keys */
+	memcpy(sec_decrypt_update_key, sec_decrypt_key, 16);
+	memcpy(sec_encrypt_update_key, sec_encrypt_key, 16);
 
 	/* Initialise RC4 state arrays */
 	RC4_set_key(&rc4_decrypt_key, rc4_key_len, sec_decrypt_key);
@@ -185,14 +193,14 @@
 	buf_out_uint32(lenhdr, datalen);
 
 	SHA1_Init(&sha);
-	SHA1_Update(&sha, session_key, length);
+	SHA1_Update(&sha, session_key, rc4_key_len);
 	SHA1_Update(&sha, pad_54, 40);
 	SHA1_Update(&sha, lenhdr, 4);
 	SHA1_Update(&sha, data, datalen);
 	SHA1_Final(shasig, &sha);
 
 	MD5_Init(&md5);
-	MD5_Update(&md5, session_key, length);
+	MD5_Update(&md5, session_key, rc4_key_len);
 	MD5_Update(&md5, pad_92, 48);
 	MD5_Update(&md5, shasig, 20);
 	MD5_Final(md5sig, &md5);
@@ -210,13 +218,13 @@
 	RC4_KEY update;
 
 	SHA1_Init(&sha);
-	SHA1_Update(&sha, update_key, 8);
+	SHA1_Update(&sha, update_key, rc4_key_len);
 	SHA1_Update(&sha, pad_54, 40);
-	SHA1_Update(&sha, key, 8);
+	SHA1_Update(&sha, key, rc4_key_len);
 	SHA1_Final(shasig, &sha);
 
 	MD5_Init(&md5);
-	MD5_Update(&md5, update_key, 8);
+	MD5_Update(&md5, update_key, rc4_key_len);
 	MD5_Update(&md5, pad_92, 48);
 	MD5_Update(&md5, shasig, 20);
 	MD5_Final(key, &md5);
@@ -262,30 +270,17 @@
 	use_count++;
 }
 
-/* Read in a NUMBER from a buffer */
-static void
-sec_read_number(NUMBER * num, uint8 *buffer, int len)
-{
-	INT *data = num->n_part;
-	int i, j;
-
-	for (i = 0, j = 0; j < len; i++, j += 2)
-		data[i] = buffer[j] | (buffer[j + 1] << 8);
-
-	num->n_len = i;
-}
-
-/* Write a NUMBER to a buffer */
 static void
-sec_write_number(NUMBER * num, uint8 *buffer, int len)
+reverse(uint8 *p, int len)
 {
-	INT *data = num->n_part;
 	int i, j;
+	uint8 temp;
 
-	for (i = 0, j = 0; j < len; i++, j += 2)
+	for (i = 0, j = len-1; i < j; i++, j--)
 	{
-		buffer[j] = data[i] & 0xff;
-		buffer[j + 1] = data[i] >> 8;
+		temp = p[i];
+		p[i] = p[j];
+		p[j] = temp;
 	}
 }
 
@@ -294,17 +289,36 @@
 sec_rsa_encrypt(uint8 *out, uint8 *in, int len,
 		uint8 *modulus, uint8 *exponent)
 {
-	NUMBER data, key;
-
-	/* Set modulus for arithmetic */
-	sec_read_number(&key, modulus, SEC_MODULUS_SIZE);
-	m_init(&key, NULL);
-
-	/* Exponentiate */
-	sec_read_number(&data, in, len);
-	sec_read_number(&key, exponent, SEC_EXPONENT_SIZE);
-	m_exp(&data, &key, &data);
-	sec_write_number(&data, out, SEC_MODULUS_SIZE);
+	BN_CTX ctx;
+	BIGNUM mod, exp, x, y;
+	uint8 inr[SEC_MODULUS_SIZE];
+	int outlen;
+
+	reverse(modulus, SEC_MODULUS_SIZE);
+	reverse(exponent, SEC_EXPONENT_SIZE);
+	memcpy(inr, in, len);
+	reverse(inr, len);
+
+	BN_CTX_init(&ctx);
+	BN_init(&mod);
+	BN_init(&exp);
+	BN_init(&x);
+	BN_init(&y);
+
+	BN_bin2bn(modulus, SEC_MODULUS_SIZE, &mod);
+	BN_bin2bn(exponent, SEC_EXPONENT_SIZE, &exp);
+	BN_bin2bn(inr, len, &x);
+	BN_mod_exp(&y, &x, &exp, &mod, &ctx);
+	outlen = BN_bn2bin(&y, out);
+	reverse(out, outlen);
+	if (outlen < SEC_MODULUS_SIZE)
+		memset(out+outlen, 0, SEC_MODULUS_SIZE-outlen);
+
+	BN_free(&y);
+	BN_clear_free(&x);
+	BN_free(&exp);
+	BN_free(&mod);
+	BN_CTX_free(&ctx);
 }
 
 /* Initialise secure transport packet */
@@ -339,8 +353,8 @@
 		flags &= ~SEC_ENCRYPT;
 		datalen = s->end - s->p - 8;
 
-#if RDP_DEBUG
-		DEBUG("Sending encrypted packet:\n");
+#if WITH_DEBUG
+		DEBUG(("Sending encrypted packet:\n"));
 		hexdump(s->p + 8, datalen);
 #endif
 
@@ -418,7 +432,7 @@
 	/* Client encryption settings */
 	out_uint16_le(s, SEC_TAG_CLI_CRYPT);
 	out_uint16(s, 8);	/* length */
-	out_uint32_le(s, use_encryption ? 1 : 0);	/* encryption enabled */
+	out_uint32_le(s, encryption ? 0x3 : 0);	/* encryption supported, 128-bit supported */
 	s_mark_end(s);
 }
 
@@ -431,14 +445,14 @@
 	in_uint32_le(s, magic);
 	if (magic != SEC_RSA_MAGIC)
 	{
-		ERROR("RSA magic 0x%x\n", magic);
+		error("RSA magic 0x%x\n", magic);
 		return False;
 	}
 
 	in_uint32_le(s, modulus_len);
 	if (modulus_len != SEC_MODULUS_SIZE + SEC_PADDING_SIZE)
 	{
-		ERROR("modulus len 0x%x\n", modulus_len);
+		error("modulus len 0x%x\n", modulus_len);
 		return False;
 	}
 
@@ -466,7 +480,7 @@
 
 	if (random_len != SEC_RANDOM_SIZE)
 	{
-		ERROR("random len %d\n", random_len);
+		error("random len %d\n", random_len);
 		return False;
 	}
 
@@ -501,7 +515,7 @@
 				break;
 
 			default:
-				NOTIMP("crypt tag 0x%x\n", tag);
+				unimpl("crypt tag 0x%x\n", tag);
 		}
 
 		s->p = next_tag;
@@ -559,7 +573,7 @@
 				break;
 
 			default:
-				NOTIMP("response tag 0x%x\n", tag);
+				unimpl("response tag 0x%x\n", tag);
 		}
 
 		s->p = next_tag;
@@ -575,7 +589,7 @@
 
 	while ((s = mcs_recv()) != NULL)
 	{
-		if (use_encryption || !licence_issued)
+		if (encryption || !licence_issued)
 		{
 			in_uint32_le(s, sec_flags);
 
@@ -613,7 +627,7 @@
 		return False;
 
 	sec_process_mcs_data(&mcs_data);
-	if (use_encryption)
+	if (encryption)
 		sec_establish_key();
 	return True;
 }