Fix various stuff: that VC++ 5.0 chokes on:
[openssl.git] / ssl / s3_both.c
1 /* ssl/s3_both.c */
2 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
3  * All rights reserved.
4  *
5  * This package is an SSL implementation written
6  * by Eric Young (eay@cryptsoft.com).
7  * The implementation was written so as to conform with Netscapes SSL.
8  * 
9  * This library is free for commercial and non-commercial use as long as
10  * the following conditions are aheared to.  The following conditions
11  * apply to all code found in this distribution, be it the RC4, RSA,
12  * lhash, DES, etc., code; not just the SSL code.  The SSL documentation
13  * included with this distribution is covered by the same copyright terms
14  * except that the holder is Tim Hudson (tjh@cryptsoft.com).
15  * 
16  * Copyright remains Eric Young's, and as such any Copyright notices in
17  * the code are not to be removed.
18  * If this package is used in a product, Eric Young should be given attribution
19  * as the author of the parts of the library used.
20  * This can be in the form of a textual message at program startup or
21  * in documentation (online or textual) provided with the package.
22  * 
23  * Redistribution and use in source and binary forms, with or without
24  * modification, are permitted provided that the following conditions
25  * are met:
26  * 1. Redistributions of source code must retain the copyright
27  *    notice, this list of conditions and the following disclaimer.
28  * 2. Redistributions in binary form must reproduce the above copyright
29  *    notice, this list of conditions and the following disclaimer in the
30  *    documentation and/or other materials provided with the distribution.
31  * 3. All advertising materials mentioning features or use of this software
32  *    must display the following acknowledgement:
33  *    "This product includes cryptographic software written by
34  *     Eric Young (eay@cryptsoft.com)"
35  *    The word 'cryptographic' can be left out if the rouines from the library
36  *    being used are not cryptographic related :-).
37  * 4. If you include any Windows specific code (or a derivative thereof) from 
38  *    the apps directory (application code) you must include an acknowledgement:
39  *    "This product includes software written by Tim Hudson (tjh@cryptsoft.com)"
40  * 
41  * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND
42  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
43  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
44  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
45  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
46  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
47  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
48  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
49  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
50  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
51  * SUCH DAMAGE.
52  * 
53  * The licence and distribution terms for any publically available version or
54  * derivative of this code cannot be changed.  i.e. this code cannot simply be
55  * copied and put under another distribution licence
56  * [including the GNU Public Licence.]
57  */
58
59 #include <stdio.h>
60 #include "buffer.h"
61 #include "rand.h"
62 #include "objects.h"
63 #include "evp.h"
64 #include "x509.h"
65 #include "ssl_locl.h"
66
67 #define BREAK   break
68
69 /* SSL3err(SSL_F_SSL3_GET_FINISHED,SSL_R_EXCESSIVE_MESSAGE_SIZE);
70  */
71
72 int ssl3_send_finished(s,a,b,sender,slen)
73 SSL *s;
74 int a;
75 int b;
76 unsigned char *sender;
77 int slen;
78         {
79         unsigned char *p,*d;
80         int i;
81         unsigned long l;
82
83         if (s->state == a)
84                 {
85                 d=(unsigned char *)s->init_buf->data;
86                 p= &(d[4]);
87
88                 i=s->method->ssl3_enc->final_finish_mac(s,
89                         &(s->s3->finish_dgst1),
90                         &(s->s3->finish_dgst2),
91                         sender,slen,p);
92                 p+=i;
93                 l=i;
94
95 #ifdef WIN16
96                 /* MSVC 1.5 does not clear the top bytes of the word unless
97                  * I do this.
98                  */
99                 l&=0xffff;
100 #endif
101
102                 *(d++)=SSL3_MT_FINISHED;
103                 l2n3(l,d);
104                 s->init_num=(int)l+4;
105                 s->init_off=0;
106
107                 s->state=b;
108                 }
109
110         /* SSL3_ST_SEND_xxxxxx_HELLO_B */
111         return(ssl3_do_write(s,SSL3_RT_HANDSHAKE));
112         }
113
114 int ssl3_get_finished(s,a,b)
115 SSL *s;
116 int a;
117 int b;
118         {
119         int al,i,ok;
120         long n;
121         unsigned char *p;
122
123         /* the mac has already been generated when we received the
124          * change cipher spec message and is in s->s3->tmp.in_dgst[12]
125          */ 
126
127         n=ssl3_get_message(s,
128                 a,
129                 b,
130                 SSL3_MT_FINISHED,
131                 64, /* should actually be 36+4 :-) */
132                 &ok);
133
134         if (!ok) return((int)n);
135
136         /* If this occurs if we has missed a message */
137         if (!s->s3->change_cipher_spec)
138                 {
139                 al=SSL_AD_UNEXPECTED_MESSAGE;
140                 SSLerr(SSL_F_SSL3_GET_FINISHED,SSL_R_GOT_A_FIN_BEFORE_A_CCS);
141                 goto f_err;
142                 }
143         s->s3->change_cipher_spec=0;
144
145         p=(unsigned char *)s->init_buf->data;
146
147         i=s->method->ssl3_enc->finish_mac_length;
148
149         if (i != n)
150                 {
151                 al=SSL_AD_DECODE_ERROR;
152                 SSLerr(SSL_F_SSL3_GET_FINISHED,SSL_R_BAD_DIGEST_LENGTH);
153                 goto f_err;
154                 }
155
156         if (memcmp(  p,    (char *)&(s->s3->tmp.finish_md[0]),i) != 0)
157                 {
158                 al=SSL_AD_DECRYPT_ERROR;
159                 SSLerr(SSL_F_SSL3_GET_FINISHED,SSL_R_DIGEST_CHECK_FAILED);
160                 goto f_err;
161                 }
162
163         return(1);
164 f_err:
165         ssl3_send_alert(s,SSL3_AL_FATAL,al);
166         return(0);
167         }
168
169 /* for these 2 messages, we need to
170  * ssl->enc_read_ctx                    re-init
171  * ssl->s3->read_sequence               zero
172  * ssl->s3->read_mac_secret             re-init
173  * ssl->session->read_sym_enc           assign
174  * ssl->session->read_compression       assign
175  * ssl->session->read_hash              assign
176  */
177 int ssl3_send_change_cipher_spec(s,a,b)
178 SSL *s;
179 int a,b;
180         { 
181         unsigned char *p;
182
183         if (s->state == a)
184                 {
185                 p=(unsigned char *)s->init_buf->data;
186                 *p=SSL3_MT_CCS;
187                 s->init_num=1;
188                 s->init_off=0;
189
190                 s->state=b;
191                 }
192
193         /* SSL3_ST_CW_CHANGE_B */
194         return(ssl3_do_write(s,SSL3_RT_CHANGE_CIPHER_SPEC));
195         }
196
197 unsigned long ssl3_output_cert_chain(s,x)
198 SSL *s;
199 X509 *x;
200         {
201         unsigned char *p;
202         int n,i;
203         unsigned long l=7;
204         BUF_MEM *buf;
205         X509_STORE_CTX xs_ctx;
206         X509_OBJECT obj;
207
208         /* TLSv1 sends a chain with nothing in it, instead of an alert */
209         buf=s->init_buf;
210         if (!BUF_MEM_grow(buf,(int)(10)))
211                 {
212                 SSLerr(SSL_F_SSL3_OUTPUT_CERT_CHAIN,ERR_R_BUF_LIB);
213                 return(0);
214                 }
215         if (x != NULL)
216                 {
217                 X509_STORE_CTX_init(&xs_ctx,s->ctx->cert_store,NULL,NULL);
218
219                 for (;;)
220                         {
221                         n=i2d_X509(x,NULL);
222                         if (!BUF_MEM_grow(buf,(int)(n+l+3)))
223                                 {
224                                 SSLerr(SSL_F_SSL3_OUTPUT_CERT_CHAIN,ERR_R_BUF_LIB);
225                                 return(0);
226                                 }
227                         p=(unsigned char *)&(buf->data[l]);
228                         l2n3(n,p);
229                         i2d_X509(x,&p);
230                         l+=n+3;
231                         if (X509_NAME_cmp(X509_get_subject_name(x),
232                                 X509_get_issuer_name(x)) == 0) break;
233
234                         i=X509_STORE_get_by_subject(&xs_ctx,X509_LU_X509,
235                                 X509_get_issuer_name(x),&obj);
236                         if (i <= 0) break;
237                         x=obj.data.x509;
238                         /* Count is one too high since the X509_STORE_get uped the
239                          * ref count */
240                         X509_free(x);
241                         }
242
243                 X509_STORE_CTX_cleanup(&xs_ctx);
244                 }
245
246         /* Thawte special :-) */
247         if (s->ctx->extra_certs != NULL)
248         for (i=0; i<sk_num(s->ctx->extra_certs); i++)
249                 {
250                 x=(X509 *)sk_value(s->ctx->extra_certs,i);
251                 n=i2d_X509(x,NULL);
252                 if (!BUF_MEM_grow(buf,(int)(n+l+3)))
253                         {
254                         SSLerr(SSL_F_SSL3_OUTPUT_CERT_CHAIN,ERR_R_BUF_LIB);
255                         return(0);
256                         }
257                 p=(unsigned char *)&(buf->data[l]);
258                 l2n3(n,p);
259                 i2d_X509(x,&p);
260                 l+=n+3;
261                 }
262
263         l-=7;
264         p=(unsigned char *)&(buf->data[4]);
265         l2n3(l,p);
266         l+=3;
267         p=(unsigned char *)&(buf->data[0]);
268         *(p++)=SSL3_MT_CERTIFICATE;
269         l2n3(l,p);
270         l+=4;
271         return(l);
272         }
273
274 long ssl3_get_message(s,st1,stn,mt,max,ok)
275 SSL *s;
276 int st1,stn,mt;
277 long max;
278 int *ok;
279         {
280         unsigned char *p;
281         unsigned long l;
282         long n;
283         int i,al;
284
285         if (s->s3->tmp.reuse_message)
286                 {
287                 s->s3->tmp.reuse_message=0;
288                 if ((mt >= 0) && (s->s3->tmp.message_type != mt))
289                         {
290                         al=SSL_AD_UNEXPECTED_MESSAGE;
291                         SSLerr(SSL_F_SSL3_GET_MESSAGE,SSL_R_UNEXPECTED_MESSAGE);
292                         goto f_err;
293                         }
294                 *ok=1;
295                 return((int)s->s3->tmp.message_size);
296                 }
297
298         p=(unsigned char *)s->init_buf->data;
299
300         if (s->state == st1)
301                 {
302                 i=ssl3_read_bytes(s,SSL3_RT_HANDSHAKE,
303                         (char *)&(p[s->init_num]),
304                         4-s->init_num);
305                 if (i < (4-s->init_num))
306                         {
307                         *ok=0;
308                         return(ssl3_part_read(s,i));
309                         }
310
311                 if ((mt >= 0) && (*p != mt))
312                         {
313                         al=SSL_AD_UNEXPECTED_MESSAGE;
314                         SSLerr(SSL_F_SSL3_GET_MESSAGE,SSL_R_UNEXPECTED_MESSAGE);
315                         goto f_err;
316                         }
317                 s->s3->tmp.message_type= *(p++);
318
319                 n2l3(p,l);
320                 if (l > (unsigned long)max)
321                         {
322                         al=SSL_AD_ILLEGAL_PARAMETER;
323                         SSLerr(SSL_F_SSL3_GET_MESSAGE,SSL_R_EXCESSIVE_MESSAGE_SIZE);
324                         goto f_err;
325                         }
326                 if (l && !BUF_MEM_grow(s->init_buf,(int)l))
327                         {
328                         SSLerr(SSL_F_SSL3_GET_MESSAGE,ERR_R_BUF_LIB);
329                         goto err;
330                         }
331                 s->s3->tmp.message_size=l;
332                 s->state=stn;
333
334                 s->init_num=0;
335                 }
336
337         /* next state (stn) */
338         p=(unsigned char *)s->init_buf->data;
339         n=s->s3->tmp.message_size;
340         if (n > 0)
341                 {
342                 i=ssl3_read_bytes(s,SSL3_RT_HANDSHAKE,
343                         (char *)&(p[s->init_num]),(int)n);
344                 if (i != (int)n)
345                         {
346                         *ok=0;
347                         return(ssl3_part_read(s,i));
348                         }
349                 }
350         *ok=1;
351         return(n);
352 f_err:
353         ssl3_send_alert(s,SSL3_AL_FATAL,al);
354 err:
355         *ok=0;
356         return(-1);
357         }
358
359 int ssl_cert_type(x,pkey)
360 X509 *x;
361 EVP_PKEY *pkey;
362         {
363         EVP_PKEY *pk;
364         int ret= -1,i,j;
365
366         if (pkey == NULL)
367                 pk=X509_get_pubkey(x);
368         else
369                 pk=pkey;
370         if (pk == NULL) goto err;
371
372         i=pk->type;
373         if (i == EVP_PKEY_RSA)
374                 {
375                 ret=SSL_PKEY_RSA_ENC;
376                 if (x != NULL)
377                         {
378                         j=X509_get_ext_count(x);
379                         /* check to see if this is a signing only certificate */
380                         /* EAY EAY EAY EAY */
381                         }
382                 }
383         else if (i == EVP_PKEY_DSA)
384                 {
385                 ret=SSL_PKEY_DSA_SIGN;
386                 }
387         else if (i == EVP_PKEY_DH)
388                 {
389                 /* if we just have a key, we needs to be guess */
390
391                 if (x == NULL)
392                         ret=SSL_PKEY_DH_DSA;
393                 else
394                         {
395                         j=X509_get_signature_type(x);
396                         if (j == EVP_PKEY_RSA)
397                                 ret=SSL_PKEY_DH_RSA;
398                         else if (j== EVP_PKEY_DSA)
399                                 ret=SSL_PKEY_DH_DSA;
400                         else ret= -1;
401                         }
402                 }
403         else
404                 ret= -1;
405
406 err:
407         return(ret);
408         }
409
410 int ssl_verify_alarm_type(type)
411 long type;
412         {
413         int al;
414
415         switch(type)
416                 {
417         case X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT:
418         case X509_V_ERR_UNABLE_TO_GET_CRL:
419                 al=SSL_AD_UNKNOWN_CA;
420                 break;
421         case X509_V_ERR_UNABLE_TO_DECRYPT_CERT_SIGNATURE:
422         case X509_V_ERR_UNABLE_TO_DECRYPT_CRL_SIGNATURE:
423         case X509_V_ERR_UNABLE_TO_DECODE_ISSUER_PUBLIC_KEY:
424         case X509_V_ERR_ERROR_IN_CERT_NOT_BEFORE_FIELD:
425         case X509_V_ERR_ERROR_IN_CERT_NOT_AFTER_FIELD:
426         case X509_V_ERR_ERROR_IN_CRL_LAST_UPDATE_FIELD:
427         case X509_V_ERR_ERROR_IN_CRL_NEXT_UPDATE_FIELD:
428         case X509_V_ERR_CERT_NOT_YET_VALID:
429         case X509_V_ERR_CRL_NOT_YET_VALID:
430                 al=SSL_AD_BAD_CERTIFICATE;
431                 break;
432         case X509_V_ERR_CERT_SIGNATURE_FAILURE:
433         case X509_V_ERR_CRL_SIGNATURE_FAILURE:
434                 al=SSL_AD_DECRYPT_ERROR;
435                 break;
436         case X509_V_ERR_CERT_HAS_EXPIRED:
437         case X509_V_ERR_CRL_HAS_EXPIRED:
438                 al=SSL_AD_CERTIFICATE_EXPIRED;
439                 break;
440         case X509_V_ERR_CERT_REVOKED:
441                 al=SSL_AD_CERTIFICATE_REVOKED;
442                 break;
443         case X509_V_ERR_OUT_OF_MEM:
444                 al=SSL_AD_INTERNAL_ERROR;
445                 break;
446         case X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT:
447         case X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN:
448         case X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY:
449         case X509_V_ERR_UNABLE_TO_VERIFY_LEAF_SIGNATURE:
450         case X509_V_ERR_CERT_CHAIN_TOO_LONG:
451                 al=SSL_AD_UNKNOWN_CA;
452                 break;
453         case X509_V_ERR_APPLICATION_VERIFICATION:
454                 al=SSL_AD_HANDSHAKE_FAILURE;
455                 break;
456         default:
457                 al=SSL_AD_CERTIFICATE_UNKNOWN;
458                 break;
459                 }
460         return(al);
461         }
462
463 int ssl3_setup_buffers(s)
464 SSL *s;
465         {
466         unsigned char *p;
467         unsigned int extra;
468
469         if (s->s3->rbuf.buf == NULL)
470                 {
471                 if (s->options & SSL_OP_MICROSOFT_BIG_SSLV3_BUFFER)
472                         extra=SSL3_RT_MAX_EXTRA;
473                 else
474                         extra=0;
475                 if ((p=(unsigned char *)Malloc(SSL3_RT_MAX_PACKET_SIZE+extra))
476                         == NULL)
477                         goto err;
478                 s->s3->rbuf.buf=p;
479                 }
480
481         if (s->s3->wbuf.buf == NULL)
482                 {
483                 if ((p=(unsigned char *)Malloc(SSL3_RT_MAX_PACKET_SIZE))
484                         == NULL)
485                         goto err;
486                 s->s3->wbuf.buf=p;
487                 }
488         s->packet= &(s->s3->rbuf.buf[0]);
489         return(1);
490 err:
491         SSLerr(SSL_F_SSL3_SETUP_BUFFERS,ERR_R_MALLOC_FAILURE);
492         return(0);
493         }