/[rdesktop]/sourceforge.net/trunk/rdesktop/secure.c
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Annotation of /sourceforge.net/trunk/rdesktop/secure.c

Parent Directory Parent Directory | Revision Log Revision Log


Revision 64 - (hide annotations)
Thu Jul 18 16:38:31 2002 UTC (21 years, 10 months ago) by astrand
File MIME type: text/plain
File size: 14061 byte(s)
Fixed indentation with indent

1 matty 10 /*
2     rdesktop: A Remote Desktop Protocol client.
3     Protocol services - RDP encryption and licensing
4 matty 30 Copyright (C) Matthew Chapman 1999-2001
5 matty 10
6     This program is free software; you can redistribute it and/or modify
7     it under the terms of the GNU General Public License as published by
8     the Free Software Foundation; either version 2 of the License, or
9     (at your option) any later version.
10    
11     This program is distributed in the hope that it will be useful,
12     but WITHOUT ANY WARRANTY; without even the implied warranty of
13     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14     GNU General Public License for more details.
15    
16     You should have received a copy of the GNU General Public License
17     along with this program; if not, write to the Free Software
18     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
19     */
20    
21     #include "rdesktop.h"
22 matty 32
23     #ifdef WITH_OPENSSL
24     #include <openssl/rc4.h>
25     #include <openssl/md5.h>
26     #include <openssl/sha.h>
27     #include <openssl/bn.h>
28     #else
29 matty 10 #include "crypto/rc4.h"
30     #include "crypto/md5.h"
31     #include "crypto/sha.h"
32 matty 32 #include "crypto/bn.h"
33     #endif
34 matty 10
35     extern char hostname[16];
36     extern int width;
37     extern int height;
38     extern int keylayout;
39 matty 30 extern BOOL encryption;
40 matty 28 extern BOOL licence_issued;
41 matty 10
42     static int rc4_key_len;
43     static RC4_KEY rc4_decrypt_key;
44     static RC4_KEY rc4_encrypt_key;
45    
46 matthewc 57 static uint8 sec_sign_key[16];
47 matty 10 static uint8 sec_decrypt_key[16];
48     static uint8 sec_encrypt_key[16];
49 matthewc 57 static uint8 sec_decrypt_update_key[16];
50     static uint8 sec_encrypt_update_key[16];
51     static uint8 sec_crypted_random[SEC_MODULUS_SIZE];
52 matty 10
53     /*
54     * General purpose 48-byte transformation, using two 32-byte salts (generally,
55     * a client and server salt) and a global salt value used for padding.
56     * Both SHA1 and MD5 algorithms are used.
57     */
58 matty 25 void
59 astrand 64 sec_hash_48(uint8 * out, uint8 * in, uint8 * salt1, uint8 * salt2, uint8 salt)
60 matty 10 {
61     uint8 shasig[20];
62     uint8 pad[4];
63     SHA_CTX sha;
64     MD5_CTX md5;
65     int i;
66    
67     for (i = 0; i < 3; i++)
68     {
69 matty 24 memset(pad, salt + i, i + 1);
70 matty 10
71     SHA1_Init(&sha);
72 matty 24 SHA1_Update(&sha, pad, i + 1);
73 matty 10 SHA1_Update(&sha, in, 48);
74     SHA1_Update(&sha, salt1, 32);
75     SHA1_Update(&sha, salt2, 32);
76     SHA1_Final(shasig, &sha);
77    
78     MD5_Init(&md5);
79     MD5_Update(&md5, in, 48);
80     MD5_Update(&md5, shasig, 20);
81 matty 24 MD5_Final(&out[i * 16], &md5);
82 matty 10 }
83     }
84    
85     /*
86     * Weaker 16-byte transformation, also using two 32-byte salts, but
87     * only using a single round of MD5.
88     */
89 matty 25 void
90 astrand 64 sec_hash_16(uint8 * out, uint8 * in, uint8 * salt1, uint8 * salt2)
91 matty 10 {
92     MD5_CTX md5;
93    
94     MD5_Init(&md5);
95     MD5_Update(&md5, in, 16);
96     MD5_Update(&md5, salt1, 32);
97     MD5_Update(&md5, salt2, 32);
98     MD5_Final(out, &md5);
99     }
100    
101     /* Reduce key entropy from 64 to 40 bits */
102 matty 25 static void
103 astrand 64 sec_make_40bit(uint8 * key)
104 matty 10 {
105     key[0] = 0xd1;
106     key[1] = 0x26;
107     key[2] = 0x9e;
108     }
109    
110     /* Generate a session key and RC4 keys, given client and server randoms */
111 matty 25 static void
112 astrand 64 sec_generate_keys(uint8 * client_key, uint8 * server_key, int rc4_key_size)
113 matty 10 {
114     uint8 session_key[48];
115     uint8 temp_hash[48];
116     uint8 input[48];
117    
118     /* Construct input data to hash */
119 matty 24 memcpy(input, client_key, 24);
120     memcpy(input + 24, server_key, 24);
121 matty 10
122     /* Generate session key - two rounds of sec_hash_48 */
123 matty 24 sec_hash_48(temp_hash, input, client_key, server_key, 65);
124 matty 10 sec_hash_48(session_key, temp_hash, client_key, server_key, 88);
125    
126 matthewc 57 /* Store first 16 bytes of session key, for generating signatures */
127     memcpy(sec_sign_key, session_key, 16);
128 matty 10
129     /* Generate RC4 keys */
130 matty 24 sec_hash_16(sec_decrypt_key, &session_key[16], client_key,
131     server_key);
132     sec_hash_16(sec_encrypt_key, &session_key[32], client_key,
133     server_key);
134 matty 10
135     if (rc4_key_size == 1)
136     {
137 matty 30 DEBUG(("40-bit encryption enabled\n"));
138 matty 10 sec_make_40bit(sec_sign_key);
139     sec_make_40bit(sec_decrypt_key);
140     sec_make_40bit(sec_encrypt_key);
141     rc4_key_len = 8;
142     }
143     else
144     {
145 matty 30 DEBUG(("128-bit encryption enabled\n"));
146 matty 10 rc4_key_len = 16;
147     }
148    
149 matthewc 57 /* Save initial RC4 keys as update keys */
150     memcpy(sec_decrypt_update_key, sec_decrypt_key, 16);
151     memcpy(sec_encrypt_update_key, sec_encrypt_key, 16);
152 matty 10
153     /* Initialise RC4 state arrays */
154     RC4_set_key(&rc4_decrypt_key, rc4_key_len, sec_decrypt_key);
155     RC4_set_key(&rc4_encrypt_key, rc4_key_len, sec_encrypt_key);
156     }
157    
158 matty 24 static uint8 pad_54[40] = {
159     54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54,
160 matty 25 54, 54, 54,
161 matty 24 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54,
162 matty 25 54, 54, 54
163 matty 10 };
164    
165 matty 24 static uint8 pad_92[48] = {
166     92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92,
167 matty 25 92, 92, 92, 92, 92, 92, 92,
168 matty 24 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92,
169 matty 25 92, 92, 92, 92, 92, 92, 92
170 matty 10 };
171    
172     /* Output a uint32 into a buffer (little-endian) */
173 matty 25 void
174 astrand 64 buf_out_uint32(uint8 * buffer, uint32 value)
175 matty 10 {
176     buffer[0] = (value) & 0xff;
177     buffer[1] = (value >> 8) & 0xff;
178     buffer[2] = (value >> 16) & 0xff;
179     buffer[3] = (value >> 24) & 0xff;
180     }
181    
182     /* Generate a signature hash, using a combination of SHA1 and MD5 */
183 matty 25 void
184 astrand 64 sec_sign(uint8 * signature, int siglen, uint8 * session_key, int keylen,
185     uint8 * data, int datalen)
186 matty 10 {
187     uint8 shasig[20];
188     uint8 md5sig[16];
189     uint8 lenhdr[4];
190     SHA_CTX sha;
191     MD5_CTX md5;
192    
193 matty 24 buf_out_uint32(lenhdr, datalen);
194 matty 10
195     SHA1_Init(&sha);
196 matthewc 61 SHA1_Update(&sha, session_key, keylen);
197 matty 10 SHA1_Update(&sha, pad_54, 40);
198     SHA1_Update(&sha, lenhdr, 4);
199     SHA1_Update(&sha, data, datalen);
200     SHA1_Final(shasig, &sha);
201    
202     MD5_Init(&md5);
203 matthewc 61 MD5_Update(&md5, session_key, keylen);
204 matty 10 MD5_Update(&md5, pad_92, 48);
205     MD5_Update(&md5, shasig, 20);
206     MD5_Final(md5sig, &md5);
207    
208 matthewc 61 memcpy(signature, md5sig, siglen);
209 matty 10 }
210    
211     /* Update an encryption key - similar to the signing process */
212 matty 25 static void
213 astrand 64 sec_update(uint8 * key, uint8 * update_key)
214 matty 10 {
215     uint8 shasig[20];
216     SHA_CTX sha;
217     MD5_CTX md5;
218     RC4_KEY update;
219    
220     SHA1_Init(&sha);
221 matthewc 57 SHA1_Update(&sha, update_key, rc4_key_len);
222 matty 10 SHA1_Update(&sha, pad_54, 40);
223 matthewc 57 SHA1_Update(&sha, key, rc4_key_len);
224 matty 10 SHA1_Final(shasig, &sha);
225    
226     MD5_Init(&md5);
227 matthewc 57 MD5_Update(&md5, update_key, rc4_key_len);
228 matty 10 MD5_Update(&md5, pad_92, 48);
229     MD5_Update(&md5, shasig, 20);
230     MD5_Final(key, &md5);
231    
232     RC4_set_key(&update, rc4_key_len, key);
233     RC4(&update, rc4_key_len, key, key);
234    
235     if (rc4_key_len == 8)
236     sec_make_40bit(key);
237     }
238    
239     /* Encrypt data using RC4 */
240 matty 25 static void
241 astrand 64 sec_encrypt(uint8 * data, int length)
242 matty 10 {
243     static int use_count;
244    
245     if (use_count == 4096)
246     {
247     sec_update(sec_encrypt_key, sec_encrypt_update_key);
248     RC4_set_key(&rc4_encrypt_key, rc4_key_len, sec_encrypt_key);
249     use_count = 0;
250     }
251    
252     RC4(&rc4_encrypt_key, length, data, data);
253     use_count++;
254     }
255    
256     /* Decrypt data using RC4 */
257 matty 25 static void
258 astrand 64 sec_decrypt(uint8 * data, int length)
259 matty 10 {
260     static int use_count;
261    
262     if (use_count == 4096)
263     {
264     sec_update(sec_decrypt_key, sec_decrypt_update_key);
265     RC4_set_key(&rc4_decrypt_key, rc4_key_len, sec_decrypt_key);
266     use_count = 0;
267     }
268    
269     RC4(&rc4_decrypt_key, length, data, data);
270     use_count++;
271     }
272    
273 matty 25 static void
274 astrand 64 reverse(uint8 * p, int len)
275 matty 10 {
276     int i, j;
277 matty 32 uint8 temp;
278 matty 10
279 astrand 64 for (i = 0, j = len - 1; i < j; i++, j--)
280 matty 10 {
281 matty 32 temp = p[i];
282     p[i] = p[j];
283     p[j] = temp;
284 matty 10 }
285     }
286    
287     /* Perform an RSA public key encryption operation */
288 matty 25 static void
289 astrand 64 sec_rsa_encrypt(uint8 * out, uint8 * in, int len,
290     uint8 * modulus, uint8 * exponent)
291 matty 10 {
292 matty 32 BN_CTX ctx;
293     BIGNUM mod, exp, x, y;
294     uint8 inr[SEC_MODULUS_SIZE];
295     int outlen;
296 matty 10
297 matty 32 reverse(modulus, SEC_MODULUS_SIZE);
298     reverse(exponent, SEC_EXPONENT_SIZE);
299     memcpy(inr, in, len);
300     reverse(inr, len);
301 matty 10
302 matty 32 BN_CTX_init(&ctx);
303     BN_init(&mod);
304     BN_init(&exp);
305     BN_init(&x);
306     BN_init(&y);
307    
308     BN_bin2bn(modulus, SEC_MODULUS_SIZE, &mod);
309     BN_bin2bn(exponent, SEC_EXPONENT_SIZE, &exp);
310     BN_bin2bn(inr, len, &x);
311     BN_mod_exp(&y, &x, &exp, &mod, &ctx);
312     outlen = BN_bn2bin(&y, out);
313     reverse(out, outlen);
314     if (outlen < SEC_MODULUS_SIZE)
315 astrand 64 memset(out + outlen, 0, SEC_MODULUS_SIZE - outlen);
316 matty 32
317     BN_free(&y);
318     BN_clear_free(&x);
319     BN_free(&exp);
320     BN_free(&mod);
321     BN_CTX_free(&ctx);
322 matty 10 }
323    
324     /* Initialise secure transport packet */
325 matty 25 STREAM
326     sec_init(uint32 flags, int maxlen)
327 matty 10 {
328     int hdrlen;
329     STREAM s;
330    
331 matty 28 if (!licence_issued)
332     hdrlen = (flags & SEC_ENCRYPT) ? 12 : 4;
333     else
334     hdrlen = (flags & SEC_ENCRYPT) ? 12 : 0;
335 matty 24 s = mcs_init(maxlen + hdrlen);
336 matty 10 s_push_layer(s, sec_hdr, hdrlen);
337    
338     return s;
339     }
340    
341     /* Transmit secure transport packet */
342 matty 25 void
343     sec_send(STREAM s, uint32 flags)
344 matty 10 {
345     int datalen;
346    
347     s_pop_layer(s, sec_hdr);
348 matty 28 if (!licence_issued || (flags & SEC_ENCRYPT))
349     out_uint32_le(s, flags);
350 matty 10
351     if (flags & SEC_ENCRYPT)
352     {
353     flags &= ~SEC_ENCRYPT;
354     datalen = s->end - s->p - 8;
355    
356 matty 30 #if WITH_DEBUG
357     DEBUG(("Sending encrypted packet:\n"));
358 matty 24 hexdump(s->p + 8, datalen);
359 matty 10 #endif
360    
361 astrand 64 sec_sign(s->p, 8, sec_sign_key, rc4_key_len, s->p + 8,
362     datalen);
363 matty 24 sec_encrypt(s->p + 8, datalen);
364 matty 10 }
365    
366     mcs_send(s);
367     }
368    
369     /* Transfer the client random to the server */
370 matty 25 static void
371     sec_establish_key()
372 matty 10 {
373     uint32 length = SEC_MODULUS_SIZE + SEC_PADDING_SIZE;
374     uint32 flags = SEC_CLIENT_RANDOM;
375     STREAM s;
376    
377     s = sec_init(flags, 76);
378    
379     out_uint32_le(s, length);
380     out_uint8p(s, sec_crypted_random, SEC_MODULUS_SIZE);
381     out_uint8s(s, SEC_PADDING_SIZE);
382    
383     s_mark_end(s);
384     sec_send(s, flags);
385     }
386    
387     /* Output connect initial data blob */
388 matty 25 static void
389     sec_out_mcs_data(STREAM s)
390 matty 10 {
391     int hostlen = 2 * strlen(hostname);
392    
393     out_uint16_be(s, 5); /* unknown */
394     out_uint16_be(s, 0x14);
395     out_uint8(s, 0x7c);
396     out_uint16_be(s, 1);
397    
398 matty 24 out_uint16_be(s, (158 | 0x8000)); /* remaining length */
399 matty 10
400     out_uint16_be(s, 8); /* length? */
401     out_uint16_be(s, 16);
402     out_uint8(s, 0);
403     out_uint16_le(s, 0xc001);
404     out_uint8(s, 0);
405    
406 matty 24 out_uint32_le(s, 0x61637544); /* "Duca" ?! */
407     out_uint16_be(s, (144 | 0x8000)); /* remaining length */
408 matty 10
409     /* Client information */
410     out_uint16_le(s, SEC_TAG_CLI_INFO);
411     out_uint16_le(s, 136); /* length */
412     out_uint16_le(s, 1);
413     out_uint16_le(s, 8);
414     out_uint16_le(s, width);
415     out_uint16_le(s, height);
416     out_uint16_le(s, 0xca01);
417     out_uint16_le(s, 0xaa03);
418     out_uint32_le(s, keylayout);
419 matty 24 out_uint32_le(s, 419); /* client build? we are 419 compatible :-) */
420 matty 10
421     /* Unicode name of client, padded to 32 bytes */
422     rdp_out_unistr(s, hostname, hostlen);
423 matty 24 out_uint8s(s, 30 - hostlen);
424 matty 10
425     out_uint32_le(s, 4);
426     out_uint32(s, 0);
427     out_uint32_le(s, 12);
428 matty 24 out_uint8s(s, 64); /* reserved? 4 + 12 doublewords */
429 matty 10
430     out_uint16(s, 0xca01);
431     out_uint16(s, 0);
432    
433     /* Client encryption settings */
434     out_uint16_le(s, SEC_TAG_CLI_CRYPT);
435     out_uint16(s, 8); /* length */
436 matthewc 57 out_uint32_le(s, encryption ? 0x3 : 0); /* encryption supported, 128-bit supported */
437 matty 10 s_mark_end(s);
438     }
439    
440     /* Parse a public key structure */
441 matty 25 static BOOL
442 astrand 64 sec_parse_public_key(STREAM s, uint8 ** modulus, uint8 ** exponent)
443 matty 10 {
444     uint32 magic, modulus_len;
445    
446     in_uint32_le(s, magic);
447     if (magic != SEC_RSA_MAGIC)
448     {
449 matty 30 error("RSA magic 0x%x\n", magic);
450 matty 10 return False;
451     }
452    
453     in_uint32_le(s, modulus_len);
454     if (modulus_len != SEC_MODULUS_SIZE + SEC_PADDING_SIZE)
455     {
456 matty 30 error("modulus len 0x%x\n", modulus_len);
457 matty 10 return False;
458     }
459    
460 matty 24 in_uint8s(s, 8); /* modulus_bits, unknown */
461 matty 10 in_uint8p(s, *exponent, SEC_EXPONENT_SIZE);
462     in_uint8p(s, *modulus, SEC_MODULUS_SIZE);
463     in_uint8s(s, SEC_PADDING_SIZE);
464    
465     return s_check(s);
466     }
467    
468     /* Parse a crypto information structure */
469 matty 25 static BOOL
470 astrand 64 sec_parse_crypt_info(STREAM s, uint32 * rc4_key_size,
471     uint8 ** server_random, uint8 ** modulus,
472     uint8 ** exponent)
473 matty 10 {
474     uint32 crypt_level, random_len, rsa_info_len;
475     uint16 tag, length;
476     uint8 *next_tag, *end;
477    
478 matty 24 in_uint32_le(s, *rc4_key_size); /* 1 = 40-bit, 2 = 128-bit */
479     in_uint32_le(s, crypt_level); /* 1 = low, 2 = medium, 3 = high */
480 matty 10 in_uint32_le(s, random_len);
481     in_uint32_le(s, rsa_info_len);
482    
483     if (random_len != SEC_RANDOM_SIZE)
484     {
485 matty 30 error("random len %d\n", random_len);
486 matty 10 return False;
487     }
488    
489     in_uint8p(s, *server_random, random_len);
490    
491     /* RSA info */
492     end = s->p + rsa_info_len;
493     if (end > s->end)
494     return False;
495    
496 matty 24 in_uint8s(s, 12); /* unknown */
497 matty 10
498     while (s->p < end)
499     {
500     in_uint16_le(s, tag);
501     in_uint16_le(s, length);
502    
503     next_tag = s->p + length;
504    
505     switch (tag)
506     {
507     case SEC_TAG_PUBKEY:
508 matty 24 if (!sec_parse_public_key
509     (s, modulus, exponent))
510 matty 10 return False;
511    
512     break;
513    
514     case SEC_TAG_KEYSIG:
515     /* Is this a Microsoft key that we just got? */
516     /* Care factor: zero! */
517     break;
518    
519     default:
520 matty 30 unimpl("crypt tag 0x%x\n", tag);
521 matty 10 }
522    
523     s->p = next_tag;
524     }
525    
526     return s_check_end(s);
527     }
528    
529     /* Process crypto information blob */
530 matty 25 static void
531     sec_process_crypt_info(STREAM s)
532 matty 10 {
533     uint8 *server_random, *modulus, *exponent;
534     uint8 client_random[SEC_RANDOM_SIZE];
535     uint32 rc4_key_size;
536    
537     if (!sec_parse_crypt_info(s, &rc4_key_size, &server_random,
538 matty 24 &modulus, &exponent))
539 matty 10 return;
540    
541     /* Generate a client random, and hence determine encryption keys */
542     generate_random(client_random);
543     sec_rsa_encrypt(sec_crypted_random, client_random,
544 matty 24 SEC_RANDOM_SIZE, modulus, exponent);
545 matty 10 sec_generate_keys(client_random, server_random, rc4_key_size);
546     }
547    
548     /* Process connect response data blob */
549 matty 25 static void
550     sec_process_mcs_data(STREAM s)
551 matty 10 {
552     uint16 tag, length;
553     uint8 *next_tag;
554    
555 matty 24 in_uint8s(s, 23); /* header */
556 matty 10
557     while (s->p < s->end)
558     {
559     in_uint16_le(s, tag);
560     in_uint16_le(s, length);
561    
562     if (length <= 4)
563     return;
564    
565     next_tag = s->p + length - 4;
566    
567     switch (tag)
568     {
569     case SEC_TAG_SRV_INFO:
570     case SEC_TAG_SRV_3:
571     break;
572    
573     case SEC_TAG_SRV_CRYPT:
574     sec_process_crypt_info(s);
575     break;
576    
577     default:
578 matty 30 unimpl("response tag 0x%x\n", tag);
579 matty 10 }
580    
581     s->p = next_tag;
582     }
583     }
584    
585     /* Receive secure transport packet */
586 matty 25 STREAM
587     sec_recv()
588 matty 10 {
589     uint32 sec_flags;
590     STREAM s;
591    
592     while ((s = mcs_recv()) != NULL)
593     {
594 matty 30 if (encryption || !licence_issued)
595 matty 10 {
596 matty 28 in_uint32_le(s, sec_flags);
597 matty 10
598 matty 28 if (sec_flags & SEC_LICENCE_NEG)
599     {
600     licence_process(s);
601     continue;
602     }
603    
604     if (sec_flags & SEC_ENCRYPT)
605     {
606     in_uint8s(s, 8); /* signature */
607     sec_decrypt(s->p, s->end - s->p);
608     }
609 matty 10 }
610    
611     return s;
612     }
613    
614     return NULL;
615     }
616    
617     /* Establish a secure connection */
618 matty 25 BOOL
619     sec_connect(char *server)
620 matty 10 {
621     struct stream mcs_data;
622    
623     /* We exchange some RDP data during the MCS-Connect */
624     mcs_data.size = 512;
625     mcs_data.p = mcs_data.data = xmalloc(mcs_data.size);
626     sec_out_mcs_data(&mcs_data);
627    
628     if (!mcs_connect(server, &mcs_data))
629     return False;
630    
631     sec_process_mcs_data(&mcs_data);
632 matty 30 if (encryption)
633 matty 28 sec_establish_key();
634 matty 10 return True;
635     }
636    
637     /* Disconnect a connection */
638 matty 25 void
639     sec_disconnect()
640 matty 10 {
641     mcs_disconnect();
642     }

  ViewVC Help
Powered by ViewVC 1.1.26