/[rdesktop]/sourceforge.net/trunk/rdesktop/secure.c
This is repository of my old source code which isn't updated any more. Go to git.rot13.org for current projects!
ViewVC logotype

Annotation of /sourceforge.net/trunk/rdesktop/secure.c

Parent Directory Parent Directory | Revision Log Revision Log


Revision 61 - (hide annotations)
Sun Jul 14 12:06:03 2002 UTC (21 years, 10 months ago) by matthewc
File MIME type: text/plain
File size: 14014 byte(s)
The 128-bit change broke licensing when used with 40-bit encryption
(sec_sign is also used from licence.c).  Fix from Lars Heete.

1 matty 10 /*
2     rdesktop: A Remote Desktop Protocol client.
3     Protocol services - RDP encryption and licensing
4 matty 30 Copyright (C) Matthew Chapman 1999-2001
5 matty 10
6     This program is free software; you can redistribute it and/or modify
7     it under the terms of the GNU General Public License as published by
8     the Free Software Foundation; either version 2 of the License, or
9     (at your option) any later version.
10    
11     This program is distributed in the hope that it will be useful,
12     but WITHOUT ANY WARRANTY; without even the implied warranty of
13     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14     GNU General Public License for more details.
15    
16     You should have received a copy of the GNU General Public License
17     along with this program; if not, write to the Free Software
18     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
19     */
20    
21     #include "rdesktop.h"
22 matty 32
23     #ifdef WITH_OPENSSL
24     #include <openssl/rc4.h>
25     #include <openssl/md5.h>
26     #include <openssl/sha.h>
27     #include <openssl/bn.h>
28     #else
29 matty 10 #include "crypto/rc4.h"
30     #include "crypto/md5.h"
31     #include "crypto/sha.h"
32 matty 32 #include "crypto/bn.h"
33     #endif
34 matty 10
35     extern char hostname[16];
36     extern int width;
37     extern int height;
38     extern int keylayout;
39 matty 30 extern BOOL encryption;
40 matty 28 extern BOOL licence_issued;
41 matty 10
42     static int rc4_key_len;
43     static RC4_KEY rc4_decrypt_key;
44     static RC4_KEY rc4_encrypt_key;
45    
46 matthewc 57 static uint8 sec_sign_key[16];
47 matty 10 static uint8 sec_decrypt_key[16];
48     static uint8 sec_encrypt_key[16];
49 matthewc 57 static uint8 sec_decrypt_update_key[16];
50     static uint8 sec_encrypt_update_key[16];
51     static uint8 sec_crypted_random[SEC_MODULUS_SIZE];
52 matty 10
53     /*
54     * General purpose 48-byte transformation, using two 32-byte salts (generally,
55     * a client and server salt) and a global salt value used for padding.
56     * Both SHA1 and MD5 algorithms are used.
57     */
58 matty 25 void
59     sec_hash_48(uint8 *out, uint8 *in, uint8 *salt1, uint8 *salt2, uint8 salt)
60 matty 10 {
61     uint8 shasig[20];
62     uint8 pad[4];
63     SHA_CTX sha;
64     MD5_CTX md5;
65     int i;
66    
67     for (i = 0; i < 3; i++)
68     {
69 matty 24 memset(pad, salt + i, i + 1);
70 matty 10
71     SHA1_Init(&sha);
72 matty 24 SHA1_Update(&sha, pad, i + 1);
73 matty 10 SHA1_Update(&sha, in, 48);
74     SHA1_Update(&sha, salt1, 32);
75     SHA1_Update(&sha, salt2, 32);
76     SHA1_Final(shasig, &sha);
77    
78     MD5_Init(&md5);
79     MD5_Update(&md5, in, 48);
80     MD5_Update(&md5, shasig, 20);
81 matty 24 MD5_Final(&out[i * 16], &md5);
82 matty 10 }
83     }
84    
85     /*
86     * Weaker 16-byte transformation, also using two 32-byte salts, but
87     * only using a single round of MD5.
88     */
89 matty 25 void
90     sec_hash_16(uint8 *out, uint8 *in, uint8 *salt1, uint8 *salt2)
91 matty 10 {
92     MD5_CTX md5;
93    
94     MD5_Init(&md5);
95     MD5_Update(&md5, in, 16);
96     MD5_Update(&md5, salt1, 32);
97     MD5_Update(&md5, salt2, 32);
98     MD5_Final(out, &md5);
99     }
100    
101     /* Reduce key entropy from 64 to 40 bits */
102 matty 25 static void
103     sec_make_40bit(uint8 *key)
104 matty 10 {
105     key[0] = 0xd1;
106     key[1] = 0x26;
107     key[2] = 0x9e;
108     }
109    
110     /* Generate a session key and RC4 keys, given client and server randoms */
111 matty 25 static void
112     sec_generate_keys(uint8 *client_key, uint8 *server_key, int rc4_key_size)
113 matty 10 {
114     uint8 session_key[48];
115     uint8 temp_hash[48];
116     uint8 input[48];
117    
118     /* Construct input data to hash */
119 matty 24 memcpy(input, client_key, 24);
120     memcpy(input + 24, server_key, 24);
121 matty 10
122     /* Generate session key - two rounds of sec_hash_48 */
123 matty 24 sec_hash_48(temp_hash, input, client_key, server_key, 65);
124 matty 10 sec_hash_48(session_key, temp_hash, client_key, server_key, 88);
125    
126 matthewc 57 /* Store first 16 bytes of session key, for generating signatures */
127     memcpy(sec_sign_key, session_key, 16);
128 matty 10
129     /* Generate RC4 keys */
130 matty 24 sec_hash_16(sec_decrypt_key, &session_key[16], client_key,
131     server_key);
132     sec_hash_16(sec_encrypt_key, &session_key[32], client_key,
133     server_key);
134 matty 10
135     if (rc4_key_size == 1)
136     {
137 matty 30 DEBUG(("40-bit encryption enabled\n"));
138 matty 10 sec_make_40bit(sec_sign_key);
139     sec_make_40bit(sec_decrypt_key);
140     sec_make_40bit(sec_encrypt_key);
141     rc4_key_len = 8;
142     }
143     else
144     {
145 matty 30 DEBUG(("128-bit encryption enabled\n"));
146 matty 10 rc4_key_len = 16;
147     }
148    
149 matthewc 57 /* Save initial RC4 keys as update keys */
150     memcpy(sec_decrypt_update_key, sec_decrypt_key, 16);
151     memcpy(sec_encrypt_update_key, sec_encrypt_key, 16);
152 matty 10
153     /* Initialise RC4 state arrays */
154     RC4_set_key(&rc4_decrypt_key, rc4_key_len, sec_decrypt_key);
155     RC4_set_key(&rc4_encrypt_key, rc4_key_len, sec_encrypt_key);
156     }
157    
158 matty 24 static uint8 pad_54[40] = {
159     54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54,
160 matty 25 54, 54, 54,
161 matty 24 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54,
162 matty 25 54, 54, 54
163 matty 10 };
164    
165 matty 24 static uint8 pad_92[48] = {
166     92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92,
167 matty 25 92, 92, 92, 92, 92, 92, 92,
168 matty 24 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92, 92,
169 matty 25 92, 92, 92, 92, 92, 92, 92
170 matty 10 };
171    
172     /* Output a uint32 into a buffer (little-endian) */
173 matty 25 void
174     buf_out_uint32(uint8 *buffer, uint32 value)
175 matty 10 {
176     buffer[0] = (value) & 0xff;
177     buffer[1] = (value >> 8) & 0xff;
178     buffer[2] = (value >> 16) & 0xff;
179     buffer[3] = (value >> 24) & 0xff;
180     }
181    
182     /* Generate a signature hash, using a combination of SHA1 and MD5 */
183 matty 25 void
184 matthewc 61 sec_sign(uint8 *signature, int siglen, uint8 *session_key, int keylen,
185 matty 25 uint8 *data, int datalen)
186 matty 10 {
187     uint8 shasig[20];
188     uint8 md5sig[16];
189     uint8 lenhdr[4];
190     SHA_CTX sha;
191     MD5_CTX md5;
192    
193 matty 24 buf_out_uint32(lenhdr, datalen);
194 matty 10
195     SHA1_Init(&sha);
196 matthewc 61 SHA1_Update(&sha, session_key, keylen);
197 matty 10 SHA1_Update(&sha, pad_54, 40);
198     SHA1_Update(&sha, lenhdr, 4);
199     SHA1_Update(&sha, data, datalen);
200     SHA1_Final(shasig, &sha);
201    
202     MD5_Init(&md5);
203 matthewc 61 MD5_Update(&md5, session_key, keylen);
204 matty 10 MD5_Update(&md5, pad_92, 48);
205     MD5_Update(&md5, shasig, 20);
206     MD5_Final(md5sig, &md5);
207    
208 matthewc 61 memcpy(signature, md5sig, siglen);
209 matty 10 }
210    
211     /* Update an encryption key - similar to the signing process */
212 matty 25 static void
213     sec_update(uint8 *key, uint8 *update_key)
214 matty 10 {
215     uint8 shasig[20];
216     SHA_CTX sha;
217     MD5_CTX md5;
218     RC4_KEY update;
219    
220     SHA1_Init(&sha);
221 matthewc 57 SHA1_Update(&sha, update_key, rc4_key_len);
222 matty 10 SHA1_Update(&sha, pad_54, 40);
223 matthewc 57 SHA1_Update(&sha, key, rc4_key_len);
224 matty 10 SHA1_Final(shasig, &sha);
225    
226     MD5_Init(&md5);
227 matthewc 57 MD5_Update(&md5, update_key, rc4_key_len);
228 matty 10 MD5_Update(&md5, pad_92, 48);
229     MD5_Update(&md5, shasig, 20);
230     MD5_Final(key, &md5);
231    
232     RC4_set_key(&update, rc4_key_len, key);
233     RC4(&update, rc4_key_len, key, key);
234    
235     if (rc4_key_len == 8)
236     sec_make_40bit(key);
237     }
238    
239     /* Encrypt data using RC4 */
240 matty 25 static void
241     sec_encrypt(uint8 *data, int length)
242 matty 10 {
243     static int use_count;
244    
245     if (use_count == 4096)
246     {
247     sec_update(sec_encrypt_key, sec_encrypt_update_key);
248     RC4_set_key(&rc4_encrypt_key, rc4_key_len, sec_encrypt_key);
249     use_count = 0;
250     }
251    
252     RC4(&rc4_encrypt_key, length, data, data);
253     use_count++;
254     }
255    
256     /* Decrypt data using RC4 */
257 matty 25 static void
258     sec_decrypt(uint8 *data, int length)
259 matty 10 {
260     static int use_count;
261    
262     if (use_count == 4096)
263     {
264     sec_update(sec_decrypt_key, sec_decrypt_update_key);
265     RC4_set_key(&rc4_decrypt_key, rc4_key_len, sec_decrypt_key);
266     use_count = 0;
267     }
268    
269     RC4(&rc4_decrypt_key, length, data, data);
270     use_count++;
271     }
272    
273 matty 25 static void
274 matty 32 reverse(uint8 *p, int len)
275 matty 10 {
276     int i, j;
277 matty 32 uint8 temp;
278 matty 10
279 matty 32 for (i = 0, j = len-1; i < j; i++, j--)
280 matty 10 {
281 matty 32 temp = p[i];
282     p[i] = p[j];
283     p[j] = temp;
284 matty 10 }
285     }
286    
287     /* Perform an RSA public key encryption operation */
288 matty 25 static void
289     sec_rsa_encrypt(uint8 *out, uint8 *in, int len,
290     uint8 *modulus, uint8 *exponent)
291 matty 10 {
292 matty 32 BN_CTX ctx;
293     BIGNUM mod, exp, x, y;
294     uint8 inr[SEC_MODULUS_SIZE];
295     int outlen;
296 matty 10
297 matty 32 reverse(modulus, SEC_MODULUS_SIZE);
298     reverse(exponent, SEC_EXPONENT_SIZE);
299     memcpy(inr, in, len);
300     reverse(inr, len);
301 matty 10
302 matty 32 BN_CTX_init(&ctx);
303     BN_init(&mod);
304     BN_init(&exp);
305     BN_init(&x);
306     BN_init(&y);
307    
308     BN_bin2bn(modulus, SEC_MODULUS_SIZE, &mod);
309     BN_bin2bn(exponent, SEC_EXPONENT_SIZE, &exp);
310     BN_bin2bn(inr, len, &x);
311     BN_mod_exp(&y, &x, &exp, &mod, &ctx);
312     outlen = BN_bn2bin(&y, out);
313     reverse(out, outlen);
314     if (outlen < SEC_MODULUS_SIZE)
315     memset(out+outlen, 0, SEC_MODULUS_SIZE-outlen);
316    
317     BN_free(&y);
318     BN_clear_free(&x);
319     BN_free(&exp);
320     BN_free(&mod);
321     BN_CTX_free(&ctx);
322 matty 10 }
323    
324     /* Initialise secure transport packet */
325 matty 25 STREAM
326     sec_init(uint32 flags, int maxlen)
327 matty 10 {
328     int hdrlen;
329     STREAM s;
330    
331 matty 28 if (!licence_issued)
332     hdrlen = (flags & SEC_ENCRYPT) ? 12 : 4;
333     else
334     hdrlen = (flags & SEC_ENCRYPT) ? 12 : 0;
335 matty 24 s = mcs_init(maxlen + hdrlen);
336 matty 10 s_push_layer(s, sec_hdr, hdrlen);
337    
338     return s;
339     }
340    
341     /* Transmit secure transport packet */
342 matty 25 void
343     sec_send(STREAM s, uint32 flags)
344 matty 10 {
345     int datalen;
346    
347     s_pop_layer(s, sec_hdr);
348 matty 28 if (!licence_issued || (flags & SEC_ENCRYPT))
349     out_uint32_le(s, flags);
350 matty 10
351     if (flags & SEC_ENCRYPT)
352     {
353     flags &= ~SEC_ENCRYPT;
354     datalen = s->end - s->p - 8;
355    
356 matty 30 #if WITH_DEBUG
357     DEBUG(("Sending encrypted packet:\n"));
358 matty 24 hexdump(s->p + 8, datalen);
359 matty 10 #endif
360    
361 matthewc 61 sec_sign(s->p, 8, sec_sign_key, rc4_key_len, s->p + 8, datalen);
362 matty 24 sec_encrypt(s->p + 8, datalen);
363 matty 10 }
364    
365     mcs_send(s);
366     }
367    
368     /* Transfer the client random to the server */
369 matty 25 static void
370     sec_establish_key()
371 matty 10 {
372     uint32 length = SEC_MODULUS_SIZE + SEC_PADDING_SIZE;
373     uint32 flags = SEC_CLIENT_RANDOM;
374     STREAM s;
375    
376     s = sec_init(flags, 76);
377    
378     out_uint32_le(s, length);
379     out_uint8p(s, sec_crypted_random, SEC_MODULUS_SIZE);
380     out_uint8s(s, SEC_PADDING_SIZE);
381    
382     s_mark_end(s);
383     sec_send(s, flags);
384     }
385    
386     /* Output connect initial data blob */
387 matty 25 static void
388     sec_out_mcs_data(STREAM s)
389 matty 10 {
390     int hostlen = 2 * strlen(hostname);
391    
392     out_uint16_be(s, 5); /* unknown */
393     out_uint16_be(s, 0x14);
394     out_uint8(s, 0x7c);
395     out_uint16_be(s, 1);
396    
397 matty 24 out_uint16_be(s, (158 | 0x8000)); /* remaining length */
398 matty 10
399     out_uint16_be(s, 8); /* length? */
400     out_uint16_be(s, 16);
401     out_uint8(s, 0);
402     out_uint16_le(s, 0xc001);
403     out_uint8(s, 0);
404    
405 matty 24 out_uint32_le(s, 0x61637544); /* "Duca" ?! */
406     out_uint16_be(s, (144 | 0x8000)); /* remaining length */
407 matty 10
408     /* Client information */
409     out_uint16_le(s, SEC_TAG_CLI_INFO);
410     out_uint16_le(s, 136); /* length */
411     out_uint16_le(s, 1);
412     out_uint16_le(s, 8);
413     out_uint16_le(s, width);
414     out_uint16_le(s, height);
415     out_uint16_le(s, 0xca01);
416     out_uint16_le(s, 0xaa03);
417     out_uint32_le(s, keylayout);
418 matty 24 out_uint32_le(s, 419); /* client build? we are 419 compatible :-) */
419 matty 10
420     /* Unicode name of client, padded to 32 bytes */
421     rdp_out_unistr(s, hostname, hostlen);
422 matty 24 out_uint8s(s, 30 - hostlen);
423 matty 10
424     out_uint32_le(s, 4);
425     out_uint32(s, 0);
426     out_uint32_le(s, 12);
427 matty 24 out_uint8s(s, 64); /* reserved? 4 + 12 doublewords */
428 matty 10
429     out_uint16(s, 0xca01);
430     out_uint16(s, 0);
431    
432     /* Client encryption settings */
433     out_uint16_le(s, SEC_TAG_CLI_CRYPT);
434     out_uint16(s, 8); /* length */
435 matthewc 57 out_uint32_le(s, encryption ? 0x3 : 0); /* encryption supported, 128-bit supported */
436 matty 10 s_mark_end(s);
437     }
438    
439     /* Parse a public key structure */
440 matty 25 static BOOL
441     sec_parse_public_key(STREAM s, uint8 **modulus, uint8 **exponent)
442 matty 10 {
443     uint32 magic, modulus_len;
444    
445     in_uint32_le(s, magic);
446     if (magic != SEC_RSA_MAGIC)
447     {
448 matty 30 error("RSA magic 0x%x\n", magic);
449 matty 10 return False;
450     }
451    
452     in_uint32_le(s, modulus_len);
453     if (modulus_len != SEC_MODULUS_SIZE + SEC_PADDING_SIZE)
454     {
455 matty 30 error("modulus len 0x%x\n", modulus_len);
456 matty 10 return False;
457     }
458    
459 matty 24 in_uint8s(s, 8); /* modulus_bits, unknown */
460 matty 10 in_uint8p(s, *exponent, SEC_EXPONENT_SIZE);
461     in_uint8p(s, *modulus, SEC_MODULUS_SIZE);
462     in_uint8s(s, SEC_PADDING_SIZE);
463    
464     return s_check(s);
465     }
466    
467     /* Parse a crypto information structure */
468 matty 25 static BOOL
469     sec_parse_crypt_info(STREAM s, uint32 *rc4_key_size,
470     uint8 **server_random, uint8 **modulus, uint8 **exponent)
471 matty 10 {
472     uint32 crypt_level, random_len, rsa_info_len;
473     uint16 tag, length;
474     uint8 *next_tag, *end;
475    
476 matty 24 in_uint32_le(s, *rc4_key_size); /* 1 = 40-bit, 2 = 128-bit */
477     in_uint32_le(s, crypt_level); /* 1 = low, 2 = medium, 3 = high */
478 matty 10 in_uint32_le(s, random_len);
479     in_uint32_le(s, rsa_info_len);
480    
481     if (random_len != SEC_RANDOM_SIZE)
482     {
483 matty 30 error("random len %d\n", random_len);
484 matty 10 return False;
485     }
486    
487     in_uint8p(s, *server_random, random_len);
488    
489     /* RSA info */
490     end = s->p + rsa_info_len;
491     if (end > s->end)
492     return False;
493    
494 matty 24 in_uint8s(s, 12); /* unknown */
495 matty 10
496     while (s->p < end)
497     {
498     in_uint16_le(s, tag);
499     in_uint16_le(s, length);
500    
501     next_tag = s->p + length;
502    
503     switch (tag)
504     {
505     case SEC_TAG_PUBKEY:
506 matty 24 if (!sec_parse_public_key
507     (s, modulus, exponent))
508 matty 10 return False;
509    
510     break;
511    
512     case SEC_TAG_KEYSIG:
513     /* Is this a Microsoft key that we just got? */
514     /* Care factor: zero! */
515     break;
516    
517     default:
518 matty 30 unimpl("crypt tag 0x%x\n", tag);
519 matty 10 }
520    
521     s->p = next_tag;
522     }
523    
524     return s_check_end(s);
525     }
526    
527     /* Process crypto information blob */
528 matty 25 static void
529     sec_process_crypt_info(STREAM s)
530 matty 10 {
531     uint8 *server_random, *modulus, *exponent;
532     uint8 client_random[SEC_RANDOM_SIZE];
533     uint32 rc4_key_size;
534    
535     if (!sec_parse_crypt_info(s, &rc4_key_size, &server_random,
536 matty 24 &modulus, &exponent))
537 matty 10 return;
538    
539     /* Generate a client random, and hence determine encryption keys */
540     generate_random(client_random);
541     sec_rsa_encrypt(sec_crypted_random, client_random,
542 matty 24 SEC_RANDOM_SIZE, modulus, exponent);
543 matty 10 sec_generate_keys(client_random, server_random, rc4_key_size);
544     }
545    
546     /* Process connect response data blob */
547 matty 25 static void
548     sec_process_mcs_data(STREAM s)
549 matty 10 {
550     uint16 tag, length;
551     uint8 *next_tag;
552    
553 matty 24 in_uint8s(s, 23); /* header */
554 matty 10
555     while (s->p < s->end)
556     {
557     in_uint16_le(s, tag);
558     in_uint16_le(s, length);
559    
560     if (length <= 4)
561     return;
562    
563     next_tag = s->p + length - 4;
564    
565     switch (tag)
566     {
567     case SEC_TAG_SRV_INFO:
568     case SEC_TAG_SRV_3:
569     break;
570    
571     case SEC_TAG_SRV_CRYPT:
572     sec_process_crypt_info(s);
573     break;
574    
575     default:
576 matty 30 unimpl("response tag 0x%x\n", tag);
577 matty 10 }
578    
579     s->p = next_tag;
580     }
581     }
582    
583     /* Receive secure transport packet */
584 matty 25 STREAM
585     sec_recv()
586 matty 10 {
587     uint32 sec_flags;
588     STREAM s;
589    
590     while ((s = mcs_recv()) != NULL)
591     {
592 matty 30 if (encryption || !licence_issued)
593 matty 10 {
594 matty 28 in_uint32_le(s, sec_flags);
595 matty 10
596 matty 28 if (sec_flags & SEC_LICENCE_NEG)
597     {
598     licence_process(s);
599     continue;
600     }
601    
602     if (sec_flags & SEC_ENCRYPT)
603     {
604     in_uint8s(s, 8); /* signature */
605     sec_decrypt(s->p, s->end - s->p);
606     }
607 matty 10 }
608    
609     return s;
610     }
611    
612     return NULL;
613     }
614    
615     /* Establish a secure connection */
616 matty 25 BOOL
617     sec_connect(char *server)
618 matty 10 {
619     struct stream mcs_data;
620    
621     /* We exchange some RDP data during the MCS-Connect */
622     mcs_data.size = 512;
623     mcs_data.p = mcs_data.data = xmalloc(mcs_data.size);
624     sec_out_mcs_data(&mcs_data);
625    
626     if (!mcs_connect(server, &mcs_data))
627     return False;
628    
629     sec_process_mcs_data(&mcs_data);
630 matty 30 if (encryption)
631 matty 28 sec_establish_key();
632 matty 10 return True;
633     }
634    
635     /* Disconnect a connection */
636 matty 25 void
637     sec_disconnect()
638 matty 10 {
639     mcs_disconnect();
640     }

  ViewVC Help
Powered by ViewVC 1.1.26