/[rdesktop]/sourceforge.net/trunk/rdesktop/secure.c
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Contents of /sourceforge.net/trunk/rdesktop/secure.c

Parent Directory Parent Directory | Revision Log Revision Log


Revision 315 - (show annotations)
Sun Feb 9 01:38:07 2003 UTC (21 years, 3 months ago) by jsorg71
File MIME type: text/plain
File size: 14444 byte(s)
added 15 and 24 bit server options

1 /*
2 rdesktop: A Remote Desktop Protocol client.
3 Protocol services - RDP encryption and licensing
4 Copyright (C) Matthew Chapman 1999-2002
5
6 This program is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
10
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
15
16 You should have received a copy of the GNU General Public License
17 along with this program; if not, write to the Free Software
18 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
19 */
20
21 #include "rdesktop.h"
22
23 #ifdef WITH_OPENSSL
24 #include <openssl/rc4.h>
25 #include <openssl/md5.h>
26 #include <openssl/sha.h>
27 #include <openssl/bn.h>
28 #else
29 #include "crypto/rc4.h"
30 #include "crypto/md5.h"
31 #include "crypto/sha.h"
32 #include "crypto/bn.h"
33 #endif
34
35 extern char hostname[16];
36 extern int width;
37 extern int height;
38 extern int keylayout;
39 extern BOOL encryption;
40 extern BOOL licence_issued;
41 extern int server_bpp;
42
43 static int rc4_key_len;
44 static RC4_KEY rc4_decrypt_key;
45 static RC4_KEY rc4_encrypt_key;
46
47 static uint8 sec_sign_key[16];
48 static uint8 sec_decrypt_key[16];
49 static uint8 sec_encrypt_key[16];
50 static uint8 sec_decrypt_update_key[16];
51 static uint8 sec_encrypt_update_key[16];
52 static uint8 sec_crypted_random[SEC_MODULUS_SIZE];
53
54 /*
55 * General purpose 48-byte transformation, using two 32-byte salts (generally,
56 * a client and server salt) and a global salt value used for padding.
57 * Both SHA1 and MD5 algorithms are used.
58 */
59 void
60 sec_hash_48(uint8 * out, uint8 * in, uint8 * salt1, uint8 * salt2, uint8 salt)
61 {
62 uint8 shasig[20];
63 uint8 pad[4];
64 SHA_CTX sha;
65 MD5_CTX md5;
66 int i;
67
68 for (i = 0; i < 3; i++)
69 {
70 memset(pad, salt + i, i + 1);
71
72 SHA1_Init(&sha);
73 SHA1_Update(&sha, pad, i + 1);
74 SHA1_Update(&sha, in, 48);
75 SHA1_Update(&sha, salt1, 32);
76 SHA1_Update(&sha, salt2, 32);
77 SHA1_Final(shasig, &sha);
78
79 MD5_Init(&md5);
80 MD5_Update(&md5, in, 48);
81 MD5_Update(&md5, shasig, 20);
82 MD5_Final(&out[i * 16], &md5);
83 }
84 }
85
86 /*
87 * Weaker 16-byte transformation, also using two 32-byte salts, but
88 * only using a single round of MD5.
89 */
90 void
91 sec_hash_16(uint8 * out, uint8 * in, uint8 * salt1, uint8 * salt2)
92 {
93 MD5_CTX md5;
94
95 MD5_Init(&md5);
96 MD5_Update(&md5, in, 16);
97 MD5_Update(&md5, salt1, 32);
98 MD5_Update(&md5, salt2, 32);
99 MD5_Final(out, &md5);
100 }
101
102 /* Reduce key entropy from 64 to 40 bits */
103 static void
104 sec_make_40bit(uint8 * key)
105 {
106 key[0] = 0xd1;
107 key[1] = 0x26;
108 key[2] = 0x9e;
109 }
110
111 /* Generate a session key and RC4 keys, given client and server randoms */
112 static void
113 sec_generate_keys(uint8 * client_key, uint8 * server_key, int rc4_key_size)
114 {
115 uint8 session_key[48];
116 uint8 temp_hash[48];
117 uint8 input[48];
118
119 /* Construct input data to hash */
120 memcpy(input, client_key, 24);
121 memcpy(input + 24, server_key, 24);
122
123 /* Generate session key - two rounds of sec_hash_48 */
124 sec_hash_48(temp_hash, input, client_key, server_key, 65);
125 sec_hash_48(session_key, temp_hash, client_key, server_key, 88);
126
127 /* Store first 16 bytes of session key, for generating signatures */
128 memcpy(sec_sign_key, session_key, 16);
129
130 /* Generate RC4 keys */
131 sec_hash_16(sec_decrypt_key, &session_key[16], client_key, server_key);
132 sec_hash_16(sec_encrypt_key, &session_key[32], client_key, server_key);
133
134 if (rc4_key_size == 1)
135 {
136 DEBUG(("40-bit encryption enabled\n"));
137 sec_make_40bit(sec_sign_key);
138 sec_make_40bit(sec_decrypt_key);
139 sec_make_40bit(sec_encrypt_key);
140 rc4_key_len = 8;
141 }
142 else
143 {
144 DEBUG(("128-bit encryption enabled\n"));
145 rc4_key_len = 16;
146 }
147
148 /* Save initial RC4 keys as update keys */
149 memcpy(sec_decrypt_update_key, sec_decrypt_key, 16);
150 memcpy(sec_encrypt_update_key, sec_encrypt_key, 16);
151
152 /* Initialise RC4 state arrays */
153 RC4_set_key(&rc4_decrypt_key, rc4_key_len, sec_decrypt_key);
154 RC4_set_key(&rc4_encrypt_key, rc4_key_len, sec_encrypt_key);
155 }
156
157 static uint8 pad_54[40] = {
158 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54,
159 54, 54, 54,
160 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54,
161 54, 54, 54
162 };
163
164 static uint8 pad_92[48] = {
165 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92,
166 92, 92, 92, 92, 92, 92, 92,
167 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92,
168 92, 92, 92, 92, 92, 92, 92
169 };
170
171 /* Output a uint32 into a buffer (little-endian) */
172 void
173 buf_out_uint32(uint8 * buffer, uint32 value)
174 {
175 buffer[0] = (value) & 0xff;
176 buffer[1] = (value >> 8) & 0xff;
177 buffer[2] = (value >> 16) & 0xff;
178 buffer[3] = (value >> 24) & 0xff;
179 }
180
181 /* Generate a signature hash, using a combination of SHA1 and MD5 */
182 void
183 sec_sign(uint8 * signature, int siglen, uint8 * session_key, int keylen, uint8 * data, int datalen)
184 {
185 uint8 shasig[20];
186 uint8 md5sig[16];
187 uint8 lenhdr[4];
188 SHA_CTX sha;
189 MD5_CTX md5;
190
191 buf_out_uint32(lenhdr, datalen);
192
193 SHA1_Init(&sha);
194 SHA1_Update(&sha, session_key, keylen);
195 SHA1_Update(&sha, pad_54, 40);
196 SHA1_Update(&sha, lenhdr, 4);
197 SHA1_Update(&sha, data, datalen);
198 SHA1_Final(shasig, &sha);
199
200 MD5_Init(&md5);
201 MD5_Update(&md5, session_key, keylen);
202 MD5_Update(&md5, pad_92, 48);
203 MD5_Update(&md5, shasig, 20);
204 MD5_Final(md5sig, &md5);
205
206 memcpy(signature, md5sig, siglen);
207 }
208
209 /* Update an encryption key - similar to the signing process */
210 static void
211 sec_update(uint8 * key, uint8 * update_key)
212 {
213 uint8 shasig[20];
214 SHA_CTX sha;
215 MD5_CTX md5;
216 RC4_KEY update;
217
218 SHA1_Init(&sha);
219 SHA1_Update(&sha, update_key, rc4_key_len);
220 SHA1_Update(&sha, pad_54, 40);
221 SHA1_Update(&sha, key, rc4_key_len);
222 SHA1_Final(shasig, &sha);
223
224 MD5_Init(&md5);
225 MD5_Update(&md5, update_key, rc4_key_len);
226 MD5_Update(&md5, pad_92, 48);
227 MD5_Update(&md5, shasig, 20);
228 MD5_Final(key, &md5);
229
230 RC4_set_key(&update, rc4_key_len, key);
231 RC4(&update, rc4_key_len, key, key);
232
233 if (rc4_key_len == 8)
234 sec_make_40bit(key);
235 }
236
237 /* Encrypt data using RC4 */
238 static void
239 sec_encrypt(uint8 * data, int length)
240 {
241 static int use_count;
242
243 if (use_count == 4096)
244 {
245 sec_update(sec_encrypt_key, sec_encrypt_update_key);
246 RC4_set_key(&rc4_encrypt_key, rc4_key_len, sec_encrypt_key);
247 use_count = 0;
248 }
249
250 RC4(&rc4_encrypt_key, length, data, data);
251 use_count++;
252 }
253
254 /* Decrypt data using RC4 */
255 static void
256 sec_decrypt(uint8 * data, int length)
257 {
258 static int use_count;
259
260 if (use_count == 4096)
261 {
262 sec_update(sec_decrypt_key, sec_decrypt_update_key);
263 RC4_set_key(&rc4_decrypt_key, rc4_key_len, sec_decrypt_key);
264 use_count = 0;
265 }
266
267 RC4(&rc4_decrypt_key, length, data, data);
268 use_count++;
269 }
270
271 static void
272 reverse(uint8 * p, int len)
273 {
274 int i, j;
275 uint8 temp;
276
277 for (i = 0, j = len - 1; i < j; i++, j--)
278 {
279 temp = p[i];
280 p[i] = p[j];
281 p[j] = temp;
282 }
283 }
284
285 /* Perform an RSA public key encryption operation */
286 static void
287 sec_rsa_encrypt(uint8 * out, uint8 * in, int len, uint8 * modulus, uint8 * exponent)
288 {
289 BN_CTX *ctx;
290 BIGNUM mod, exp, x, y;
291 uint8 inr[SEC_MODULUS_SIZE];
292 int outlen;
293
294 reverse(modulus, SEC_MODULUS_SIZE);
295 reverse(exponent, SEC_EXPONENT_SIZE);
296 memcpy(inr, in, len);
297 reverse(inr, len);
298
299 ctx = BN_CTX_new();
300 BN_init(&mod);
301 BN_init(&exp);
302 BN_init(&x);
303 BN_init(&y);
304
305 BN_bin2bn(modulus, SEC_MODULUS_SIZE, &mod);
306 BN_bin2bn(exponent, SEC_EXPONENT_SIZE, &exp);
307 BN_bin2bn(inr, len, &x);
308 BN_mod_exp(&y, &x, &exp, &mod, ctx);
309 outlen = BN_bn2bin(&y, out);
310 reverse(out, outlen);
311 if (outlen < SEC_MODULUS_SIZE)
312 memset(out + outlen, 0, SEC_MODULUS_SIZE - outlen);
313
314 BN_free(&y);
315 BN_clear_free(&x);
316 BN_free(&exp);
317 BN_free(&mod);
318 BN_CTX_free(ctx);
319 }
320
321 /* Initialise secure transport packet */
322 STREAM
323 sec_init(uint32 flags, int maxlen)
324 {
325 int hdrlen;
326 STREAM s;
327
328 if (!licence_issued)
329 hdrlen = (flags & SEC_ENCRYPT) ? 12 : 4;
330 else
331 hdrlen = (flags & SEC_ENCRYPT) ? 12 : 0;
332 s = mcs_init(maxlen + hdrlen);
333 s_push_layer(s, sec_hdr, hdrlen);
334
335 return s;
336 }
337
338 /* Transmit secure transport packet */
339 void
340 sec_send(STREAM s, uint32 flags)
341 {
342 int datalen;
343
344 s_pop_layer(s, sec_hdr);
345 if (!licence_issued || (flags & SEC_ENCRYPT))
346 out_uint32_le(s, flags);
347
348 if (flags & SEC_ENCRYPT)
349 {
350 flags &= ~SEC_ENCRYPT;
351 datalen = s->end - s->p - 8;
352
353 #if WITH_DEBUG
354 DEBUG(("Sending encrypted packet:\n"));
355 hexdump(s->p + 8, datalen);
356 #endif
357
358 sec_sign(s->p, 8, sec_sign_key, rc4_key_len, s->p + 8, datalen);
359 sec_encrypt(s->p + 8, datalen);
360 }
361
362 mcs_send(s);
363 }
364
365 /* Transfer the client random to the server */
366 static void
367 sec_establish_key(void)
368 {
369 uint32 length = SEC_MODULUS_SIZE + SEC_PADDING_SIZE;
370 uint32 flags = SEC_CLIENT_RANDOM;
371 STREAM s;
372
373 s = sec_init(flags, 76);
374
375 out_uint32_le(s, length);
376 out_uint8p(s, sec_crypted_random, SEC_MODULUS_SIZE);
377 out_uint8s(s, SEC_PADDING_SIZE);
378
379 s_mark_end(s);
380 sec_send(s, flags);
381 }
382
383 /* Output connect initial data blob */
384 static void
385 sec_out_mcs_data(STREAM s)
386 {
387 int hostlen = 2 * strlen(hostname);
388
389 if (hostlen > 30)
390 hostlen = 30;
391
392 out_uint16_be(s, 5); /* unknown */
393 out_uint16_be(s, 0x14);
394 out_uint8(s, 0x7c);
395 out_uint16_be(s, 1);
396
397 out_uint16_be(s, (158 | 0x8000)); /* remaining length */
398
399 out_uint16_be(s, 8); /* length? */
400 out_uint16_be(s, 16);
401 out_uint8(s, 0);
402 out_uint16_le(s, 0xc001);
403 out_uint8(s, 0);
404
405 out_uint32_le(s, 0x61637544); /* "Duca" ?! */
406 out_uint16_be(s, (144 | 0x8000)); /* remaining length */
407
408 /* Client information */
409 out_uint16_le(s, SEC_TAG_CLI_INFO);
410 out_uint16_le(s, 136); /* length */
411 out_uint16_le(s, 1);
412 out_uint16_le(s, 8);
413 out_uint16_le(s, width);
414 out_uint16_le(s, height);
415 out_uint16_le(s, 0xca01);
416 out_uint16_le(s, 0xaa03);
417 out_uint32_le(s, keylayout);
418 out_uint32_le(s, 419); /* client build? we are 419 compatible :-) */
419
420 /* Unicode name of client, padded to 32 bytes */
421 rdp_out_unistr(s, hostname, hostlen);
422 out_uint8s(s, 30 - hostlen);
423
424 out_uint32_le(s, 4);
425 out_uint32(s, 0);
426 out_uint32_le(s, 12);
427 out_uint8s(s, 64); /* reserved? 4 + 12 doublewords */
428
429 switch (server_bpp)
430 {
431 case 8:
432 out_uint16_le(s, 0xca01);
433 break;
434 case 15:
435 out_uint16_le(s, 0xca02);
436 break;
437 case 16:
438 out_uint16_le(s, 0xca03);
439 break;
440 case 24:
441 out_uint16_le(s, 0xca04);
442 break;
443 }
444 out_uint16(s, 0);
445
446 /* Client encryption settings */
447 out_uint16_le(s, SEC_TAG_CLI_CRYPT);
448 out_uint16_le(s, 8); /* length */
449 out_uint32_le(s, encryption ? 0x3 : 0); /* encryption supported, 128-bit supported */
450 s_mark_end(s);
451 }
452
453 /* Parse a public key structure */
454 static BOOL
455 sec_parse_public_key(STREAM s, uint8 ** modulus, uint8 ** exponent)
456 {
457 uint32 magic, modulus_len;
458
459 in_uint32_le(s, magic);
460 if (magic != SEC_RSA_MAGIC)
461 {
462 error("RSA magic 0x%x\n", magic);
463 return False;
464 }
465
466 in_uint32_le(s, modulus_len);
467 if (modulus_len != SEC_MODULUS_SIZE + SEC_PADDING_SIZE)
468 {
469 error("modulus len 0x%x\n", modulus_len);
470 return False;
471 }
472
473 in_uint8s(s, 8); /* modulus_bits, unknown */
474 in_uint8p(s, *exponent, SEC_EXPONENT_SIZE);
475 in_uint8p(s, *modulus, SEC_MODULUS_SIZE);
476 in_uint8s(s, SEC_PADDING_SIZE);
477
478 return s_check(s);
479 }
480
481 /* Parse a crypto information structure */
482 static BOOL
483 sec_parse_crypt_info(STREAM s, uint32 * rc4_key_size,
484 uint8 ** server_random, uint8 ** modulus, uint8 ** exponent)
485 {
486 uint32 crypt_level, random_len, rsa_info_len;
487 uint16 tag, length;
488 uint8 *next_tag, *end;
489
490 in_uint32_le(s, *rc4_key_size); /* 1 = 40-bit, 2 = 128-bit */
491 in_uint32_le(s, crypt_level); /* 1 = low, 2 = medium, 3 = high */
492 if (crypt_level == 0) /* no encryptation */
493 return False;
494 in_uint32_le(s, random_len);
495 in_uint32_le(s, rsa_info_len);
496
497 if (random_len != SEC_RANDOM_SIZE)
498 {
499 error("random len %d\n", random_len);
500 return False;
501 }
502
503 in_uint8p(s, *server_random, random_len);
504
505 /* RSA info */
506 end = s->p + rsa_info_len;
507 if (end > s->end)
508 return False;
509
510 in_uint8s(s, 12); /* unknown */
511
512 while (s->p < end)
513 {
514 in_uint16_le(s, tag);
515 in_uint16_le(s, length);
516
517 next_tag = s->p + length;
518
519 switch (tag)
520 {
521 case SEC_TAG_PUBKEY:
522 if (!sec_parse_public_key(s, modulus, exponent))
523 return False;
524
525 break;
526
527 case SEC_TAG_KEYSIG:
528 /* Is this a Microsoft key that we just got? */
529 /* Care factor: zero! */
530 break;
531
532 default:
533 unimpl("crypt tag 0x%x\n", tag);
534 }
535
536 s->p = next_tag;
537 }
538
539 return s_check_end(s);
540 }
541
542 /* Process crypto information blob */
543 static void
544 sec_process_crypt_info(STREAM s)
545 {
546 uint8 *server_random, *modulus, *exponent;
547 uint8 client_random[SEC_RANDOM_SIZE];
548 uint32 rc4_key_size;
549
550 if (!sec_parse_crypt_info(s, &rc4_key_size, &server_random, &modulus, &exponent))
551 return;
552
553 /* Generate a client random, and hence determine encryption keys */
554 generate_random(client_random);
555 sec_rsa_encrypt(sec_crypted_random, client_random, SEC_RANDOM_SIZE, modulus, exponent);
556 sec_generate_keys(client_random, server_random, rc4_key_size);
557 }
558
559 /* Process connect response data blob */
560 static void
561 sec_process_mcs_data(STREAM s)
562 {
563 uint16 tag, length;
564 uint8 *next_tag;
565 uint8 len;
566
567 in_uint8s(s, 21); /* header */
568 in_uint8(s, len);
569 if (len & 0x80)
570 in_uint8(s, len);
571
572 while (s->p < s->end)
573 {
574 in_uint16_le(s, tag);
575 in_uint16_le(s, length);
576
577 if (length <= 4)
578 return;
579
580 next_tag = s->p + length - 4;
581
582 switch (tag)
583 {
584 case SEC_TAG_SRV_INFO:
585 case SEC_TAG_SRV_3:
586 break;
587
588 case SEC_TAG_SRV_CRYPT:
589 sec_process_crypt_info(s);
590 break;
591
592 default:
593 unimpl("response tag 0x%x\n", tag);
594 }
595
596 s->p = next_tag;
597 }
598 }
599
600 /* Receive secure transport packet */
601 STREAM
602 sec_recv(void)
603 {
604 uint32 sec_flags;
605 STREAM s;
606
607 while ((s = mcs_recv()) != NULL)
608 {
609 if (encryption || !licence_issued)
610 {
611 in_uint32_le(s, sec_flags);
612
613 if (sec_flags & SEC_LICENCE_NEG)
614 {
615 licence_process(s);
616 continue;
617 }
618
619 if (sec_flags & SEC_ENCRYPT)
620 {
621 in_uint8s(s, 8); /* signature */
622 sec_decrypt(s->p, s->end - s->p);
623 }
624 }
625
626 return s;
627 }
628
629 return NULL;
630 }
631
632 /* Establish a secure connection */
633 BOOL
634 sec_connect(char *server)
635 {
636 struct stream mcs_data;
637
638 /* We exchange some RDP data during the MCS-Connect */
639 mcs_data.size = 512;
640 mcs_data.p = mcs_data.data = xmalloc(mcs_data.size);
641 sec_out_mcs_data(&mcs_data);
642
643 if (!mcs_connect(server, &mcs_data))
644 return False;
645
646 sec_process_mcs_data(&mcs_data);
647 if (encryption)
648 sec_establish_key();
649 xfree(mcs_data.data);
650 return True;
651 }
652
653 /* Disconnect a connection */
654 void
655 sec_disconnect(void)
656 {
657 mcs_disconnect();
658 }

  ViewVC Help
Powered by ViewVC 1.1.26