/[rdesktop]/sourceforge.net/trunk/rdesktop/secure.c
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Diff of /sourceforge.net/trunk/rdesktop/secure.c

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 30 by matty, Fri Sep 14 13:51:38 2001 UTC revision 309 by jsorg71, Tue Feb 4 05:32:13 2003 UTC
# Line 1  Line 1 
1  /*  /*
2     rdesktop: A Remote Desktop Protocol client.     rdesktop: A Remote Desktop Protocol client.
3     Protocol services - RDP encryption and licensing     Protocol services - RDP encryption and licensing
4     Copyright (C) Matthew Chapman 1999-2001     Copyright (C) Matthew Chapman 1999-2002
5    
6     This program is free software; you can redistribute it and/or modify     This program is free software; you can redistribute it and/or modify
7     it under the terms of the GNU General Public License as published by     it under the terms of the GNU General Public License as published by
# Line 19  Line 19 
19  */  */
20    
21  #include "rdesktop.h"  #include "rdesktop.h"
22    
23    #ifdef WITH_OPENSSL
24    #include <openssl/rc4.h>
25    #include <openssl/md5.h>
26    #include <openssl/sha.h>
27    #include <openssl/bn.h>
28    #else
29  #include "crypto/rc4.h"  #include "crypto/rc4.h"
30  #include "crypto/md5.h"  #include "crypto/md5.h"
31  #include "crypto/sha.h"  #include "crypto/sha.h"
32  #include "crypto/arith.h"  #include "crypto/bn.h"
33    #endif
34    
35  extern char hostname[16];  extern char hostname[16];
36  extern int width;  extern int width;
# Line 30  extern int height; Line 38  extern int height;
38  extern int keylayout;  extern int keylayout;
39  extern BOOL encryption;  extern BOOL encryption;
40  extern BOOL licence_issued;  extern BOOL licence_issued;
41    extern int server_bpp;
42    
43  static int rc4_key_len;  static int rc4_key_len;
44  static RC4_KEY rc4_decrypt_key;  static RC4_KEY rc4_decrypt_key;
45  static RC4_KEY rc4_encrypt_key;  static RC4_KEY rc4_encrypt_key;
46    
47  static uint8 sec_sign_key[8];  static uint8 sec_sign_key[16];
48  static uint8 sec_decrypt_key[16];  static uint8 sec_decrypt_key[16];
49  static uint8 sec_encrypt_key[16];  static uint8 sec_encrypt_key[16];
50  static uint8 sec_decrypt_update_key[8];  static uint8 sec_decrypt_update_key[16];
51  static uint8 sec_encrypt_update_key[8];  static uint8 sec_encrypt_update_key[16];
52  static uint8 sec_crypted_random[64];  static uint8 sec_crypted_random[SEC_MODULUS_SIZE];
53    
54  /*  /*
55   * General purpose 48-byte transformation, using two 32-byte salts (generally,   * General purpose 48-byte transformation, using two 32-byte salts (generally,
# Line 48  static uint8 sec_crypted_random[64]; Line 57  static uint8 sec_crypted_random[64];
57   * Both SHA1 and MD5 algorithms are used.   * Both SHA1 and MD5 algorithms are used.
58   */   */
59  void  void
60  sec_hash_48(uint8 *out, uint8 *in, uint8 *salt1, uint8 *salt2, uint8 salt)  sec_hash_48(uint8 * out, uint8 * in, uint8 * salt1, uint8 * salt2, uint8 salt)
61  {  {
62          uint8 shasig[20];          uint8 shasig[20];
63          uint8 pad[4];          uint8 pad[4];
# Line 79  sec_hash_48(uint8 *out, uint8 *in, uint8 Line 88  sec_hash_48(uint8 *out, uint8 *in, uint8
88   * only using a single round of MD5.   * only using a single round of MD5.
89   */   */
90  void  void
91  sec_hash_16(uint8 *out, uint8 *in, uint8 *salt1, uint8 *salt2)  sec_hash_16(uint8 * out, uint8 * in, uint8 * salt1, uint8 * salt2)
92  {  {
93          MD5_CTX md5;          MD5_CTX md5;
94    
# Line 92  sec_hash_16(uint8 *out, uint8 *in, uint8 Line 101  sec_hash_16(uint8 *out, uint8 *in, uint8
101    
102  /* Reduce key entropy from 64 to 40 bits */  /* Reduce key entropy from 64 to 40 bits */
103  static void  static void
104  sec_make_40bit(uint8 *key)  sec_make_40bit(uint8 * key)
105  {  {
106          key[0] = 0xd1;          key[0] = 0xd1;
107          key[1] = 0x26;          key[1] = 0x26;
# Line 101  sec_make_40bit(uint8 *key) Line 110  sec_make_40bit(uint8 *key)
110    
111  /* Generate a session key and RC4 keys, given client and server randoms */  /* Generate a session key and RC4 keys, given client and server randoms */
112  static void  static void
113  sec_generate_keys(uint8 *client_key, uint8 *server_key, int rc4_key_size)  sec_generate_keys(uint8 * client_key, uint8 * server_key, int rc4_key_size)
114  {  {
115          uint8 session_key[48];          uint8 session_key[48];
116          uint8 temp_hash[48];          uint8 temp_hash[48];
# Line 115  sec_generate_keys(uint8 *client_key, uin Line 124  sec_generate_keys(uint8 *client_key, uin
124          sec_hash_48(temp_hash, input, client_key, server_key, 65);          sec_hash_48(temp_hash, input, client_key, server_key, 65);
125          sec_hash_48(session_key, temp_hash, client_key, server_key, 88);          sec_hash_48(session_key, temp_hash, client_key, server_key, 88);
126    
127          /* Store first 8 bytes of session key, for generating signatures */          /* Store first 16 bytes of session key, for generating signatures */
128          memcpy(sec_sign_key, session_key, 8);          memcpy(sec_sign_key, session_key, 16);
129    
130          /* Generate RC4 keys */          /* Generate RC4 keys */
131          sec_hash_16(sec_decrypt_key, &session_key[16], client_key,          sec_hash_16(sec_decrypt_key, &session_key[16], client_key, server_key);
132                      server_key);          sec_hash_16(sec_encrypt_key, &session_key[32], client_key, server_key);
         sec_hash_16(sec_encrypt_key, &session_key[32], client_key,  
                     server_key);  
133    
134          if (rc4_key_size == 1)          if (rc4_key_size == 1)
135          {          {
# Line 138  sec_generate_keys(uint8 *client_key, uin Line 145  sec_generate_keys(uint8 *client_key, uin
145                  rc4_key_len = 16;                  rc4_key_len = 16;
146          }          }
147    
148          /* Store first 8 bytes of RC4 keys as update keys */          /* Save initial RC4 keys as update keys */
149          memcpy(sec_decrypt_update_key, sec_decrypt_key, 8);          memcpy(sec_decrypt_update_key, sec_decrypt_key, 16);
150          memcpy(sec_encrypt_update_key, sec_encrypt_key, 8);          memcpy(sec_encrypt_update_key, sec_encrypt_key, 16);
151    
152          /* Initialise RC4 state arrays */          /* Initialise RC4 state arrays */
153          RC4_set_key(&rc4_decrypt_key, rc4_key_len, sec_decrypt_key);          RC4_set_key(&rc4_decrypt_key, rc4_key_len, sec_decrypt_key);
# Line 163  static uint8 pad_92[48] = { Line 170  static uint8 pad_92[48] = {
170    
171  /* Output a uint32 into a buffer (little-endian) */  /* Output a uint32 into a buffer (little-endian) */
172  void  void
173  buf_out_uint32(uint8 *buffer, uint32 value)  buf_out_uint32(uint8 * buffer, uint32 value)
174  {  {
175          buffer[0] = (value) & 0xff;          buffer[0] = (value) & 0xff;
176          buffer[1] = (value >> 8) & 0xff;          buffer[1] = (value >> 8) & 0xff;
# Line 173  buf_out_uint32(uint8 *buffer, uint32 val Line 180  buf_out_uint32(uint8 *buffer, uint32 val
180    
181  /* Generate a signature hash, using a combination of SHA1 and MD5 */  /* Generate a signature hash, using a combination of SHA1 and MD5 */
182  void  void
183  sec_sign(uint8 *signature, uint8 *session_key, int length,  sec_sign(uint8 * signature, int siglen, uint8 * session_key, int keylen, uint8 * data, int datalen)
          uint8 *data, int datalen)  
184  {  {
185          uint8 shasig[20];          uint8 shasig[20];
186          uint8 md5sig[16];          uint8 md5sig[16];
# Line 185  sec_sign(uint8 *signature, uint8 *sessio Line 191  sec_sign(uint8 *signature, uint8 *sessio
191          buf_out_uint32(lenhdr, datalen);          buf_out_uint32(lenhdr, datalen);
192    
193          SHA1_Init(&sha);          SHA1_Init(&sha);
194          SHA1_Update(&sha, session_key, length);          SHA1_Update(&sha, session_key, keylen);
195          SHA1_Update(&sha, pad_54, 40);          SHA1_Update(&sha, pad_54, 40);
196          SHA1_Update(&sha, lenhdr, 4);          SHA1_Update(&sha, lenhdr, 4);
197          SHA1_Update(&sha, data, datalen);          SHA1_Update(&sha, data, datalen);
198          SHA1_Final(shasig, &sha);          SHA1_Final(shasig, &sha);
199    
200          MD5_Init(&md5);          MD5_Init(&md5);
201          MD5_Update(&md5, session_key, length);          MD5_Update(&md5, session_key, keylen);
202          MD5_Update(&md5, pad_92, 48);          MD5_Update(&md5, pad_92, 48);
203          MD5_Update(&md5, shasig, 20);          MD5_Update(&md5, shasig, 20);
204          MD5_Final(md5sig, &md5);          MD5_Final(md5sig, &md5);
205    
206          memcpy(signature, md5sig, length);          memcpy(signature, md5sig, siglen);
207  }  }
208    
209  /* Update an encryption key - similar to the signing process */  /* Update an encryption key - similar to the signing process */
210  static void  static void
211  sec_update(uint8 *key, uint8 *update_key)  sec_update(uint8 * key, uint8 * update_key)
212  {  {
213          uint8 shasig[20];          uint8 shasig[20];
214          SHA_CTX sha;          SHA_CTX sha;
# Line 210  sec_update(uint8 *key, uint8 *update_key Line 216  sec_update(uint8 *key, uint8 *update_key
216          RC4_KEY update;          RC4_KEY update;
217    
218          SHA1_Init(&sha);          SHA1_Init(&sha);
219          SHA1_Update(&sha, update_key, 8);          SHA1_Update(&sha, update_key, rc4_key_len);
220          SHA1_Update(&sha, pad_54, 40);          SHA1_Update(&sha, pad_54, 40);
221          SHA1_Update(&sha, key, 8);          SHA1_Update(&sha, key, rc4_key_len);
222          SHA1_Final(shasig, &sha);          SHA1_Final(shasig, &sha);
223    
224          MD5_Init(&md5);          MD5_Init(&md5);
225          MD5_Update(&md5, update_key, 8);          MD5_Update(&md5, update_key, rc4_key_len);
226          MD5_Update(&md5, pad_92, 48);          MD5_Update(&md5, pad_92, 48);
227          MD5_Update(&md5, shasig, 20);          MD5_Update(&md5, shasig, 20);
228          MD5_Final(key, &md5);          MD5_Final(key, &md5);
# Line 230  sec_update(uint8 *key, uint8 *update_key Line 236  sec_update(uint8 *key, uint8 *update_key
236    
237  /* Encrypt data using RC4 */  /* Encrypt data using RC4 */
238  static void  static void
239  sec_encrypt(uint8 *data, int length)  sec_encrypt(uint8 * data, int length)
240  {  {
241          static int use_count;          static int use_count;
242    
# Line 247  sec_encrypt(uint8 *data, int length) Line 253  sec_encrypt(uint8 *data, int length)
253    
254  /* Decrypt data using RC4 */  /* Decrypt data using RC4 */
255  static void  static void
256  sec_decrypt(uint8 *data, int length)  sec_decrypt(uint8 * data, int length)
257  {  {
258          static int use_count;          static int use_count;
259    
# Line 262  sec_decrypt(uint8 *data, int length) Line 268  sec_decrypt(uint8 *data, int length)
268          use_count++;          use_count++;
269  }  }
270    
 /* Read in a NUMBER from a buffer */  
 static void  
 sec_read_number(NUMBER * num, uint8 *buffer, int len)  
 {  
         INT *data = num->n_part;  
         int i, j;  
   
         for (i = 0, j = 0; j < len; i++, j += 2)  
                 data[i] = buffer[j] | (buffer[j + 1] << 8);  
   
         num->n_len = i;  
 }  
   
 /* Write a NUMBER to a buffer */  
271  static void  static void
272  sec_write_number(NUMBER * num, uint8 *buffer, int len)  reverse(uint8 * p, int len)
273  {  {
         INT *data = num->n_part;  
274          int i, j;          int i, j;
275            uint8 temp;
276    
277          for (i = 0, j = 0; j < len; i++, j += 2)          for (i = 0, j = len - 1; i < j; i++, j--)
278          {          {
279                  buffer[j] = data[i] & 0xff;                  temp = p[i];
280                  buffer[j + 1] = data[i] >> 8;                  p[i] = p[j];
281                    p[j] = temp;
282          }          }
283  }  }
284    
285  /* Perform an RSA public key encryption operation */  /* Perform an RSA public key encryption operation */
286  static void  static void
287  sec_rsa_encrypt(uint8 *out, uint8 *in, int len,  sec_rsa_encrypt(uint8 * out, uint8 * in, int len, uint8 * modulus, uint8 * exponent)
                 uint8 *modulus, uint8 *exponent)  
288  {  {
289          NUMBER data, key;          BN_CTX *ctx;
290            BIGNUM mod, exp, x, y;
291          /* Set modulus for arithmetic */          uint8 inr[SEC_MODULUS_SIZE];
292          sec_read_number(&key, modulus, SEC_MODULUS_SIZE);          int outlen;
293          m_init(&key, NULL);  
294            reverse(modulus, SEC_MODULUS_SIZE);
295          /* Exponentiate */          reverse(exponent, SEC_EXPONENT_SIZE);
296          sec_read_number(&data, in, len);          memcpy(inr, in, len);
297          sec_read_number(&key, exponent, SEC_EXPONENT_SIZE);          reverse(inr, len);
298          m_exp(&data, &key, &data);  
299          sec_write_number(&data, out, SEC_MODULUS_SIZE);          ctx = BN_CTX_new();
300            BN_init(&mod);
301            BN_init(&exp);
302            BN_init(&x);
303            BN_init(&y);
304    
305            BN_bin2bn(modulus, SEC_MODULUS_SIZE, &mod);
306            BN_bin2bn(exponent, SEC_EXPONENT_SIZE, &exp);
307            BN_bin2bn(inr, len, &x);
308            BN_mod_exp(&y, &x, &exp, &mod, ctx);
309            outlen = BN_bn2bin(&y, out);
310            reverse(out, outlen);
311            if (outlen < SEC_MODULUS_SIZE)
312                    memset(out + outlen, 0, SEC_MODULUS_SIZE - outlen);
313    
314            BN_free(&y);
315            BN_clear_free(&x);
316            BN_free(&exp);
317            BN_free(&mod);
318            BN_CTX_free(ctx);
319  }  }
320    
321  /* Initialise secure transport packet */  /* Initialise secure transport packet */
# Line 344  sec_send(STREAM s, uint32 flags) Line 355  sec_send(STREAM s, uint32 flags)
355                  hexdump(s->p + 8, datalen);                  hexdump(s->p + 8, datalen);
356  #endif  #endif
357    
358                  sec_sign(s->p, sec_sign_key, 8, s->p + 8, datalen);                  sec_sign(s->p, 8, sec_sign_key, rc4_key_len, s->p + 8, datalen);
359                  sec_encrypt(s->p + 8, datalen);                  sec_encrypt(s->p + 8, datalen);
360          }          }
361    
# Line 353  sec_send(STREAM s, uint32 flags) Line 364  sec_send(STREAM s, uint32 flags)
364    
365  /* Transfer the client random to the server */  /* Transfer the client random to the server */
366  static void  static void
367  sec_establish_key()  sec_establish_key(void)
368  {  {
369          uint32 length = SEC_MODULUS_SIZE + SEC_PADDING_SIZE;          uint32 length = SEC_MODULUS_SIZE + SEC_PADDING_SIZE;
370          uint32 flags = SEC_CLIENT_RANDOM;          uint32 flags = SEC_CLIENT_RANDOM;
# Line 375  sec_out_mcs_data(STREAM s) Line 386  sec_out_mcs_data(STREAM s)
386  {  {
387          int hostlen = 2 * strlen(hostname);          int hostlen = 2 * strlen(hostname);
388    
389            if (hostlen > 30)
390                    hostlen = 30;
391    
392          out_uint16_be(s, 5);    /* unknown */          out_uint16_be(s, 5);    /* unknown */
393          out_uint16_be(s, 0x14);          out_uint16_be(s, 0x14);
394          out_uint8(s, 0x7c);          out_uint8(s, 0x7c);
# Line 412  sec_out_mcs_data(STREAM s) Line 426  sec_out_mcs_data(STREAM s)
426          out_uint32_le(s, 12);          out_uint32_le(s, 12);
427          out_uint8s(s, 64);      /* reserved? 4 + 12 doublewords */          out_uint8s(s, 64);      /* reserved? 4 + 12 doublewords */
428    
429          out_uint16(s, 0xca01);          if (server_bpp == 16)
430            {
431                    out_uint16_le(s, 0xca03); /* 16 bit */
432            }
433            else
434            {
435                    out_uint16_le(s, 0xca01); /* 8 bit */
436            }
437          out_uint16(s, 0);          out_uint16(s, 0);
438    
439          /* Client encryption settings */          /* Client encryption settings */
440          out_uint16_le(s, SEC_TAG_CLI_CRYPT);          out_uint16_le(s, SEC_TAG_CLI_CRYPT);
441          out_uint16(s, 8);       /* length */          out_uint16_le(s, 8);    /* length */
442          out_uint32_le(s, encryption ? 1 : 0);   /* encryption enabled */          out_uint32_le(s, encryption ? 0x3 : 0); /* encryption supported, 128-bit supported */
443          s_mark_end(s);          s_mark_end(s);
444  }  }
445    
446  /* Parse a public key structure */  /* Parse a public key structure */
447  static BOOL  static BOOL
448  sec_parse_public_key(STREAM s, uint8 **modulus, uint8 **exponent)  sec_parse_public_key(STREAM s, uint8 ** modulus, uint8 ** exponent)
449  {  {
450          uint32 magic, modulus_len;          uint32 magic, modulus_len;
451    
# Line 452  sec_parse_public_key(STREAM s, uint8 **m Line 473  sec_parse_public_key(STREAM s, uint8 **m
473    
474  /* Parse a crypto information structure */  /* Parse a crypto information structure */
475  static BOOL  static BOOL
476  sec_parse_crypt_info(STREAM s, uint32 *rc4_key_size,  sec_parse_crypt_info(STREAM s, uint32 * rc4_key_size,
477                       uint8 **server_random, uint8 **modulus, uint8 **exponent)                       uint8 ** server_random, uint8 ** modulus, uint8 ** exponent)
478  {  {
479          uint32 crypt_level, random_len, rsa_info_len;          uint32 crypt_level, random_len, rsa_info_len;
480          uint16 tag, length;          uint16 tag, length;
# Line 461  sec_parse_crypt_info(STREAM s, uint32 *r Line 482  sec_parse_crypt_info(STREAM s, uint32 *r
482    
483          in_uint32_le(s, *rc4_key_size); /* 1 = 40-bit, 2 = 128-bit */          in_uint32_le(s, *rc4_key_size); /* 1 = 40-bit, 2 = 128-bit */
484          in_uint32_le(s, crypt_level);   /* 1 = low, 2 = medium, 3 = high */          in_uint32_le(s, crypt_level);   /* 1 = low, 2 = medium, 3 = high */
485            if (crypt_level == 0)   /* no encryptation */
486                    return False;
487          in_uint32_le(s, random_len);          in_uint32_le(s, random_len);
488          in_uint32_le(s, rsa_info_len);          in_uint32_le(s, rsa_info_len);
489    
# Line 489  sec_parse_crypt_info(STREAM s, uint32 *r Line 512  sec_parse_crypt_info(STREAM s, uint32 *r
512                  switch (tag)                  switch (tag)
513                  {                  {
514                          case SEC_TAG_PUBKEY:                          case SEC_TAG_PUBKEY:
515                                  if (!sec_parse_public_key                                  if (!sec_parse_public_key(s, modulus, exponent))
                                     (s, modulus, exponent))  
516                                          return False;                                          return False;
517    
518                                  break;                                  break;
# Line 518  sec_process_crypt_info(STREAM s) Line 540  sec_process_crypt_info(STREAM s)
540          uint8 client_random[SEC_RANDOM_SIZE];          uint8 client_random[SEC_RANDOM_SIZE];
541          uint32 rc4_key_size;          uint32 rc4_key_size;
542    
543          if (!sec_parse_crypt_info(s, &rc4_key_size, &server_random,          if (!sec_parse_crypt_info(s, &rc4_key_size, &server_random, &modulus, &exponent))
                                   &modulus, &exponent))  
544                  return;                  return;
545    
546          /* Generate a client random, and hence determine encryption keys */          /* Generate a client random, and hence determine encryption keys */
547          generate_random(client_random);          generate_random(client_random);
548          sec_rsa_encrypt(sec_crypted_random, client_random,          sec_rsa_encrypt(sec_crypted_random, client_random, SEC_RANDOM_SIZE, modulus, exponent);
                         SEC_RANDOM_SIZE, modulus, exponent);  
549          sec_generate_keys(client_random, server_random, rc4_key_size);          sec_generate_keys(client_random, server_random, rc4_key_size);
550  }  }
551    
# Line 535  sec_process_mcs_data(STREAM s) Line 555  sec_process_mcs_data(STREAM s)
555  {  {
556          uint16 tag, length;          uint16 tag, length;
557          uint8 *next_tag;          uint8 *next_tag;
558            uint8 len;
559    
560          in_uint8s(s, 23);       /* header */          in_uint8s(s, 21);       /* header */
561            in_uint8(s, len);
562            if (len & 0x80)
563                    in_uint8(s, len);
564    
565          while (s->p < s->end)          while (s->p < s->end)
566          {          {
# Line 568  sec_process_mcs_data(STREAM s) Line 592  sec_process_mcs_data(STREAM s)
592    
593  /* Receive secure transport packet */  /* Receive secure transport packet */
594  STREAM  STREAM
595  sec_recv()  sec_recv(void)
596  {  {
597          uint32 sec_flags;          uint32 sec_flags;
598          STREAM s;          STREAM s;
# Line 615  sec_connect(char *server) Line 639  sec_connect(char *server)
639          sec_process_mcs_data(&mcs_data);          sec_process_mcs_data(&mcs_data);
640          if (encryption)          if (encryption)
641                  sec_establish_key();                  sec_establish_key();
642            xfree(mcs_data.data);
643          return True;          return True;
644  }  }
645    
646  /* Disconnect a connection */  /* Disconnect a connection */
647  void  void
648  sec_disconnect()  sec_disconnect(void)
649  {  {
650          mcs_disconnect();          mcs_disconnect();
651  }  }
Legend:
Removed from v.30  
changed lines
  Added in v.309
  ViewVC Help
Powered by ViewVC 1.1.26