/[rdesktop]/sourceforge.net/trunk/rdesktop/secure.c
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Annotation of /sourceforge.net/trunk/rdesktop/secure.c

Parent Directory Parent Directory | Revision Log Revision Log


Revision 82 - (hide annotations)
Tue Jul 30 07:18:48 2002 UTC (21 years, 9 months ago) by astrand
File MIME type: text/plain
File size: 14016 byte(s)
Changed max line length to 100

1 matty 10 /*
2     rdesktop: A Remote Desktop Protocol client.
3     Protocol services - RDP encryption and licensing
4 matty 30 Copyright (C) Matthew Chapman 1999-2001
5 matty 10
6     This program is free software; you can redistribute it and/or modify
7     it under the terms of the GNU General Public License as published by
8     the Free Software Foundation; either version 2 of the License, or
9     (at your option) any later version.
10    
11     This program is distributed in the hope that it will be useful,
12     but WITHOUT ANY WARRANTY; without even the implied warranty of
13     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14     GNU General Public License for more details.
15    
16     You should have received a copy of the GNU General Public License
17     along with this program; if not, write to the Free Software
18     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
19     */
20    
21     #include "rdesktop.h"
22 matty 32
23     #ifdef WITH_OPENSSL
24     #include <openssl/rc4.h>
25     #include <openssl/md5.h>
26     #include <openssl/sha.h>
27     #include <openssl/bn.h>
28     #else
29 matty 10 #include "crypto/rc4.h"
30     #include "crypto/md5.h"
31     #include "crypto/sha.h"
32 matty 32 #include "crypto/bn.h"
33     #endif
34 matty 10
35     extern char hostname[16];
36     extern int width;
37     extern int height;
38     extern int keylayout;
39 matty 30 extern BOOL encryption;
40 matty 28 extern BOOL licence_issued;
41 matty 10
42     static int rc4_key_len;
43     static RC4_KEY rc4_decrypt_key;
44     static RC4_KEY rc4_encrypt_key;
45    
46 matthewc 57 static uint8 sec_sign_key[16];
47 matty 10 static uint8 sec_decrypt_key[16];
48     static uint8 sec_encrypt_key[16];
49 matthewc 57 static uint8 sec_decrypt_update_key[16];
50     static uint8 sec_encrypt_update_key[16];
51     static uint8 sec_crypted_random[SEC_MODULUS_SIZE];
52 matty 10
53     /*
54     * General purpose 48-byte transformation, using two 32-byte salts (generally,
55     * a client and server salt) and a global salt value used for padding.
56     * Both SHA1 and MD5 algorithms are used.
57     */
58 matty 25 void
59 astrand 64 sec_hash_48(uint8 * out, uint8 * in, uint8 * salt1, uint8 * salt2, uint8 salt)
60 matty 10 {
61     uint8 shasig[20];
62     uint8 pad[4];
63     SHA_CTX sha;
64     MD5_CTX md5;
65     int i;
66    
67     for (i = 0; i < 3; i++)
68     {
69 matty 24 memset(pad, salt + i, i + 1);
70 matty 10
71     SHA1_Init(&sha);
72 matty 24 SHA1_Update(&sha, pad, i + 1);
73 matty 10 SHA1_Update(&sha, in, 48);
74     SHA1_Update(&sha, salt1, 32);
75     SHA1_Update(&sha, salt2, 32);
76     SHA1_Final(shasig, &sha);
77    
78     MD5_Init(&md5);
79     MD5_Update(&md5, in, 48);
80     MD5_Update(&md5, shasig, 20);
81 matty 24 MD5_Final(&out[i * 16], &md5);
82 matty 10 }
83     }
84    
85     /*
86     * Weaker 16-byte transformation, also using two 32-byte salts, but
87     * only using a single round of MD5.
88     */
89 matty 25 void
90 astrand 64 sec_hash_16(uint8 * out, uint8 * in, uint8 * salt1, uint8 * salt2)
91 matty 10 {
92     MD5_CTX md5;
93    
94     MD5_Init(&md5);
95     MD5_Update(&md5, in, 16);
96     MD5_Update(&md5, salt1, 32);
97     MD5_Update(&md5, salt2, 32);
98     MD5_Final(out, &md5);
99     }
100    
101     /* Reduce key entropy from 64 to 40 bits */
102 matty 25 static void
103 astrand 64 sec_make_40bit(uint8 * key)
104 matty 10 {
105     key[0] = 0xd1;
106     key[1] = 0x26;
107     key[2] = 0x9e;
108     }
109    
110     /* Generate a session key and RC4 keys, given client and server randoms */
111 matty 25 static void
112 astrand 64 sec_generate_keys(uint8 * client_key, uint8 * server_key, int rc4_key_size)
113 matty 10 {
114     uint8 session_key[48];
115     uint8 temp_hash[48];
116     uint8 input[48];
117    
118     /* Construct input data to hash */
119 matty 24 memcpy(input, client_key, 24);
120     memcpy(input + 24, server_key, 24);
121 matty 10
122     /* Generate session key - two rounds of sec_hash_48 */
123 matty 24 sec_hash_48(temp_hash, input, client_key, server_key, 65);
124 matty 10 sec_hash_48(session_key, temp_hash, client_key, server_key, 88);
125    
126 matthewc 57 /* Store first 16 bytes of session key, for generating signatures */
127     memcpy(sec_sign_key, session_key, 16);
128 matty 10
129     /* Generate RC4 keys */
130 astrand 82 sec_hash_16(sec_decrypt_key, &session_key[16], client_key, server_key);
131     sec_hash_16(sec_encrypt_key, &session_key[32], client_key, server_key);
132 matty 10
133     if (rc4_key_size == 1)
134     {
135 matty 30 DEBUG(("40-bit encryption enabled\n"));
136 matty 10 sec_make_40bit(sec_sign_key);
137     sec_make_40bit(sec_decrypt_key);
138     sec_make_40bit(sec_encrypt_key);
139     rc4_key_len = 8;
140     }
141     else
142     {
143 matty 30 DEBUG(("128-bit encryption enabled\n"));
144 matty 10 rc4_key_len = 16;
145     }
146    
147 matthewc 57 /* Save initial RC4 keys as update keys */
148     memcpy(sec_decrypt_update_key, sec_decrypt_key, 16);
149     memcpy(sec_encrypt_update_key, sec_encrypt_key, 16);
150 matty 10
151     /* Initialise RC4 state arrays */
152     RC4_set_key(&rc4_decrypt_key, rc4_key_len, sec_decrypt_key);
153     RC4_set_key(&rc4_encrypt_key, rc4_key_len, sec_encrypt_key);
154     }
155    
156 matty 24 static uint8 pad_54[40] = {
157     54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54,
158 matty 25 54, 54, 54,
159 matty 24 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54,
160 matty 25 54, 54, 54
161 matty 10 };
162    
163 matty 24 static uint8 pad_92[48] = {
164     92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92,
165 matty 25 92, 92, 92, 92, 92, 92, 92,
166 matty 24 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92,
167 matty 25 92, 92, 92, 92, 92, 92, 92
168 matty 10 };
169    
170     /* Output a uint32 into a buffer (little-endian) */
171 matty 25 void
172 astrand 64 buf_out_uint32(uint8 * buffer, uint32 value)
173 matty 10 {
174     buffer[0] = (value) & 0xff;
175     buffer[1] = (value >> 8) & 0xff;
176     buffer[2] = (value >> 16) & 0xff;
177     buffer[3] = (value >> 24) & 0xff;
178     }
179    
180     /* Generate a signature hash, using a combination of SHA1 and MD5 */
181 matty 25 void
182 astrand 82 sec_sign(uint8 * signature, int siglen, uint8 * session_key, int keylen, uint8 * data, int datalen)
183 matty 10 {
184     uint8 shasig[20];
185     uint8 md5sig[16];
186     uint8 lenhdr[4];
187     SHA_CTX sha;
188     MD5_CTX md5;
189    
190 matty 24 buf_out_uint32(lenhdr, datalen);
191 matty 10
192     SHA1_Init(&sha);
193 matthewc 61 SHA1_Update(&sha, session_key, keylen);
194 matty 10 SHA1_Update(&sha, pad_54, 40);
195     SHA1_Update(&sha, lenhdr, 4);
196     SHA1_Update(&sha, data, datalen);
197     SHA1_Final(shasig, &sha);
198    
199     MD5_Init(&md5);
200 matthewc 61 MD5_Update(&md5, session_key, keylen);
201 matty 10 MD5_Update(&md5, pad_92, 48);
202     MD5_Update(&md5, shasig, 20);
203     MD5_Final(md5sig, &md5);
204    
205 matthewc 61 memcpy(signature, md5sig, siglen);
206 matty 10 }
207    
208     /* Update an encryption key - similar to the signing process */
209 matty 25 static void
210 astrand 64 sec_update(uint8 * key, uint8 * update_key)
211 matty 10 {
212     uint8 shasig[20];
213     SHA_CTX sha;
214     MD5_CTX md5;
215     RC4_KEY update;
216    
217     SHA1_Init(&sha);
218 matthewc 57 SHA1_Update(&sha, update_key, rc4_key_len);
219 matty 10 SHA1_Update(&sha, pad_54, 40);
220 matthewc 57 SHA1_Update(&sha, key, rc4_key_len);
221 matty 10 SHA1_Final(shasig, &sha);
222    
223     MD5_Init(&md5);
224 matthewc 57 MD5_Update(&md5, update_key, rc4_key_len);
225 matty 10 MD5_Update(&md5, pad_92, 48);
226     MD5_Update(&md5, shasig, 20);
227     MD5_Final(key, &md5);
228    
229     RC4_set_key(&update, rc4_key_len, key);
230     RC4(&update, rc4_key_len, key, key);
231    
232     if (rc4_key_len == 8)
233     sec_make_40bit(key);
234     }
235    
236     /* Encrypt data using RC4 */
237 matty 25 static void
238 astrand 64 sec_encrypt(uint8 * data, int length)
239 matty 10 {
240     static int use_count;
241    
242     if (use_count == 4096)
243     {
244     sec_update(sec_encrypt_key, sec_encrypt_update_key);
245     RC4_set_key(&rc4_encrypt_key, rc4_key_len, sec_encrypt_key);
246     use_count = 0;
247     }
248    
249     RC4(&rc4_encrypt_key, length, data, data);
250     use_count++;
251     }
252    
253     /* Decrypt data using RC4 */
254 matty 25 static void
255 astrand 64 sec_decrypt(uint8 * data, int length)
256 matty 10 {
257     static int use_count;
258    
259     if (use_count == 4096)
260     {
261     sec_update(sec_decrypt_key, sec_decrypt_update_key);
262     RC4_set_key(&rc4_decrypt_key, rc4_key_len, sec_decrypt_key);
263     use_count = 0;
264     }
265    
266     RC4(&rc4_decrypt_key, length, data, data);
267     use_count++;
268     }
269    
270 matty 25 static void
271 astrand 64 reverse(uint8 * p, int len)
272 matty 10 {
273     int i, j;
274 matty 32 uint8 temp;
275 matty 10
276 astrand 64 for (i = 0, j = len - 1; i < j; i++, j--)
277 matty 10 {
278 matty 32 temp = p[i];
279     p[i] = p[j];
280     p[j] = temp;
281 matty 10 }
282     }
283    
284     /* Perform an RSA public key encryption operation */
285 matty 25 static void
286 astrand 82 sec_rsa_encrypt(uint8 * out, uint8 * in, int len, uint8 * modulus, uint8 * exponent)
287 matty 10 {
288 matty 32 BN_CTX ctx;
289     BIGNUM mod, exp, x, y;
290     uint8 inr[SEC_MODULUS_SIZE];
291     int outlen;
292 matty 10
293 matty 32 reverse(modulus, SEC_MODULUS_SIZE);
294     reverse(exponent, SEC_EXPONENT_SIZE);
295     memcpy(inr, in, len);
296     reverse(inr, len);
297 matty 10
298 matty 32 BN_CTX_init(&ctx);
299     BN_init(&mod);
300     BN_init(&exp);
301     BN_init(&x);
302     BN_init(&y);
303    
304     BN_bin2bn(modulus, SEC_MODULUS_SIZE, &mod);
305     BN_bin2bn(exponent, SEC_EXPONENT_SIZE, &exp);
306     BN_bin2bn(inr, len, &x);
307     BN_mod_exp(&y, &x, &exp, &mod, &ctx);
308     outlen = BN_bn2bin(&y, out);
309     reverse(out, outlen);
310     if (outlen < SEC_MODULUS_SIZE)
311 astrand 64 memset(out + outlen, 0, SEC_MODULUS_SIZE - outlen);
312 matty 32
313     BN_free(&y);
314     BN_clear_free(&x);
315     BN_free(&exp);
316     BN_free(&mod);
317     BN_CTX_free(&ctx);
318 matty 10 }
319    
320     /* Initialise secure transport packet */
321 matty 25 STREAM
322     sec_init(uint32 flags, int maxlen)
323 matty 10 {
324     int hdrlen;
325     STREAM s;
326    
327 matty 28 if (!licence_issued)
328     hdrlen = (flags & SEC_ENCRYPT) ? 12 : 4;
329     else
330     hdrlen = (flags & SEC_ENCRYPT) ? 12 : 0;
331 matty 24 s = mcs_init(maxlen + hdrlen);
332 matty 10 s_push_layer(s, sec_hdr, hdrlen);
333    
334     return s;
335     }
336    
337     /* Transmit secure transport packet */
338 matty 25 void
339     sec_send(STREAM s, uint32 flags)
340 matty 10 {
341     int datalen;
342    
343     s_pop_layer(s, sec_hdr);
344 matty 28 if (!licence_issued || (flags & SEC_ENCRYPT))
345     out_uint32_le(s, flags);
346 matty 10
347     if (flags & SEC_ENCRYPT)
348     {
349     flags &= ~SEC_ENCRYPT;
350     datalen = s->end - s->p - 8;
351    
352 matty 30 #if WITH_DEBUG
353     DEBUG(("Sending encrypted packet:\n"));
354 matty 24 hexdump(s->p + 8, datalen);
355 matty 10 #endif
356    
357 astrand 82 sec_sign(s->p, 8, sec_sign_key, rc4_key_len, s->p + 8, datalen);
358 matty 24 sec_encrypt(s->p + 8, datalen);
359 matty 10 }
360    
361     mcs_send(s);
362     }
363    
364     /* Transfer the client random to the server */
365 matty 25 static void
366     sec_establish_key()
367 matty 10 {
368     uint32 length = SEC_MODULUS_SIZE + SEC_PADDING_SIZE;
369     uint32 flags = SEC_CLIENT_RANDOM;
370     STREAM s;
371    
372     s = sec_init(flags, 76);
373    
374     out_uint32_le(s, length);
375     out_uint8p(s, sec_crypted_random, SEC_MODULUS_SIZE);
376     out_uint8s(s, SEC_PADDING_SIZE);
377    
378     s_mark_end(s);
379     sec_send(s, flags);
380     }
381    
382     /* Output connect initial data blob */
383 matty 25 static void
384     sec_out_mcs_data(STREAM s)
385 matty 10 {
386     int hostlen = 2 * strlen(hostname);
387    
388     out_uint16_be(s, 5); /* unknown */
389     out_uint16_be(s, 0x14);
390     out_uint8(s, 0x7c);
391     out_uint16_be(s, 1);
392    
393 matty 24 out_uint16_be(s, (158 | 0x8000)); /* remaining length */
394 matty 10
395     out_uint16_be(s, 8); /* length? */
396     out_uint16_be(s, 16);
397     out_uint8(s, 0);
398     out_uint16_le(s, 0xc001);
399     out_uint8(s, 0);
400    
401 matty 24 out_uint32_le(s, 0x61637544); /* "Duca" ?! */
402     out_uint16_be(s, (144 | 0x8000)); /* remaining length */
403 matty 10
404     /* Client information */
405     out_uint16_le(s, SEC_TAG_CLI_INFO);
406     out_uint16_le(s, 136); /* length */
407     out_uint16_le(s, 1);
408     out_uint16_le(s, 8);
409     out_uint16_le(s, width);
410     out_uint16_le(s, height);
411     out_uint16_le(s, 0xca01);
412     out_uint16_le(s, 0xaa03);
413     out_uint32_le(s, keylayout);
414 matty 24 out_uint32_le(s, 419); /* client build? we are 419 compatible :-) */
415 matty 10
416     /* Unicode name of client, padded to 32 bytes */
417     rdp_out_unistr(s, hostname, hostlen);
418 matty 24 out_uint8s(s, 30 - hostlen);
419 matty 10
420     out_uint32_le(s, 4);
421     out_uint32(s, 0);
422     out_uint32_le(s, 12);
423 matty 24 out_uint8s(s, 64); /* reserved? 4 + 12 doublewords */
424 matty 10
425     out_uint16(s, 0xca01);
426     out_uint16(s, 0);
427    
428     /* Client encryption settings */
429     out_uint16_le(s, SEC_TAG_CLI_CRYPT);
430     out_uint16(s, 8); /* length */
431 matthewc 57 out_uint32_le(s, encryption ? 0x3 : 0); /* encryption supported, 128-bit supported */
432 matty 10 s_mark_end(s);
433     }
434    
435     /* Parse a public key structure */
436 matty 25 static BOOL
437 astrand 64 sec_parse_public_key(STREAM s, uint8 ** modulus, uint8 ** exponent)
438 matty 10 {
439     uint32 magic, modulus_len;
440    
441     in_uint32_le(s, magic);
442     if (magic != SEC_RSA_MAGIC)
443     {
444 matty 30 error("RSA magic 0x%x\n", magic);
445 matty 10 return False;
446     }
447    
448     in_uint32_le(s, modulus_len);
449     if (modulus_len != SEC_MODULUS_SIZE + SEC_PADDING_SIZE)
450     {
451 matty 30 error("modulus len 0x%x\n", modulus_len);
452 matty 10 return False;
453     }
454    
455 matty 24 in_uint8s(s, 8); /* modulus_bits, unknown */
456 matty 10 in_uint8p(s, *exponent, SEC_EXPONENT_SIZE);
457     in_uint8p(s, *modulus, SEC_MODULUS_SIZE);
458     in_uint8s(s, SEC_PADDING_SIZE);
459    
460     return s_check(s);
461     }
462    
463     /* Parse a crypto information structure */
464 matty 25 static BOOL
465 astrand 64 sec_parse_crypt_info(STREAM s, uint32 * rc4_key_size,
466 astrand 82 uint8 ** server_random, uint8 ** modulus, uint8 ** exponent)
467 matty 10 {
468     uint32 crypt_level, random_len, rsa_info_len;
469     uint16 tag, length;
470     uint8 *next_tag, *end;
471    
472 matty 24 in_uint32_le(s, *rc4_key_size); /* 1 = 40-bit, 2 = 128-bit */
473     in_uint32_le(s, crypt_level); /* 1 = low, 2 = medium, 3 = high */
474 matty 10 in_uint32_le(s, random_len);
475     in_uint32_le(s, rsa_info_len);
476    
477     if (random_len != SEC_RANDOM_SIZE)
478     {
479 matty 30 error("random len %d\n", random_len);
480 matty 10 return False;
481     }
482    
483     in_uint8p(s, *server_random, random_len);
484    
485     /* RSA info */
486     end = s->p + rsa_info_len;
487     if (end > s->end)
488     return False;
489    
490 matty 24 in_uint8s(s, 12); /* unknown */
491 matty 10
492     while (s->p < end)
493     {
494     in_uint16_le(s, tag);
495     in_uint16_le(s, length);
496    
497     next_tag = s->p + length;
498    
499     switch (tag)
500     {
501     case SEC_TAG_PUBKEY:
502 astrand 82 if (!sec_parse_public_key(s, modulus, exponent))
503 matty 10 return False;
504    
505     break;
506    
507     case SEC_TAG_KEYSIG:
508     /* Is this a Microsoft key that we just got? */
509     /* Care factor: zero! */
510     break;
511    
512     default:
513 matty 30 unimpl("crypt tag 0x%x\n", tag);
514 matty 10 }
515    
516     s->p = next_tag;
517     }
518    
519     return s_check_end(s);
520     }
521    
522     /* Process crypto information blob */
523 matty 25 static void
524     sec_process_crypt_info(STREAM s)
525 matty 10 {
526     uint8 *server_random, *modulus, *exponent;
527     uint8 client_random[SEC_RANDOM_SIZE];
528     uint32 rc4_key_size;
529    
530 astrand 82 if (!sec_parse_crypt_info(s, &rc4_key_size, &server_random, &modulus, &exponent))
531 matty 10 return;
532    
533     /* Generate a client random, and hence determine encryption keys */
534     generate_random(client_random);
535 astrand 82 sec_rsa_encrypt(sec_crypted_random, client_random, SEC_RANDOM_SIZE, modulus, exponent);
536 matty 10 sec_generate_keys(client_random, server_random, rc4_key_size);
537     }
538    
539     /* Process connect response data blob */
540 matty 25 static void
541     sec_process_mcs_data(STREAM s)
542 matty 10 {
543     uint16 tag, length;
544     uint8 *next_tag;
545    
546 matty 24 in_uint8s(s, 23); /* header */
547 matty 10
548     while (s->p < s->end)
549     {
550     in_uint16_le(s, tag);
551     in_uint16_le(s, length);
552    
553     if (length <= 4)
554     return;
555    
556     next_tag = s->p + length - 4;
557    
558     switch (tag)
559     {
560     case SEC_TAG_SRV_INFO:
561     case SEC_TAG_SRV_3:
562     break;
563    
564     case SEC_TAG_SRV_CRYPT:
565     sec_process_crypt_info(s);
566     break;
567    
568     default:
569 matty 30 unimpl("response tag 0x%x\n", tag);
570 matty 10 }
571    
572     s->p = next_tag;
573     }
574     }
575    
576     /* Receive secure transport packet */
577 matty 25 STREAM
578     sec_recv()
579 matty 10 {
580     uint32 sec_flags;
581     STREAM s;
582    
583     while ((s = mcs_recv()) != NULL)
584     {
585 matty 30 if (encryption || !licence_issued)
586 matty 10 {
587 matty 28 in_uint32_le(s, sec_flags);
588 matty 10
589 matty 28 if (sec_flags & SEC_LICENCE_NEG)
590     {
591     licence_process(s);
592     continue;
593     }
594    
595     if (sec_flags & SEC_ENCRYPT)
596     {
597     in_uint8s(s, 8); /* signature */
598     sec_decrypt(s->p, s->end - s->p);
599     }
600 matty 10 }
601    
602     return s;
603     }
604    
605     return NULL;
606     }
607    
608     /* Establish a secure connection */
609 matty 25 BOOL
610     sec_connect(char *server)
611 matty 10 {
612     struct stream mcs_data;
613    
614     /* We exchange some RDP data during the MCS-Connect */
615     mcs_data.size = 512;
616     mcs_data.p = mcs_data.data = xmalloc(mcs_data.size);
617     sec_out_mcs_data(&mcs_data);
618    
619     if (!mcs_connect(server, &mcs_data))
620     return False;
621    
622     sec_process_mcs_data(&mcs_data);
623 matty 30 if (encryption)
624 matty 28 sec_establish_key();
625 matty 10 return True;
626     }
627    
628     /* Disconnect a connection */
629 matty 25 void
630     sec_disconnect()
631 matty 10 {
632     mcs_disconnect();
633     }

  ViewVC Help
Powered by ViewVC 1.1.26