make scrypt ASN.1 parameter functions public

[openssl.git] / crypto / asn1 / a_int.c

diff --git a/crypto/asn1/a_int.c b/crypto/asn1/a_int.c
index fe700b22f7ed0f10d040b1dc326c0384e11c7399..217650a036c98b3452f3cb29f594bcd8b6c59080 100644 (file)
--- a/crypto/asn1/a_int.c
+++ b/crypto/asn1/a_int.c
@@ -167,10 +167,21 @@ static size_t c2i_ibuf(unsigned char *b, int *pneg,
         }
         return 1;
     }
-    if (p[0] == 0 || p[0] == 0xFF)
+
+    pad = 0;
+    if (p[0] == 0) {
         pad = 1;
-    else
-        pad = 0;
+    } else if (p[0] == 0xFF) {
+        size_t i;
+
+        /*
+         * Special case [of "one less minimal negative" for given length]:
+         * if any other bytes non zero it was padded, otherwise not.
+         */
+        for (pad = 0, i = 1; i < plen; i++)
+            pad |= p[i];
+        pad = pad != 0 ? 1 : 0;
+    }
     /* reject illegal padding: first two octets MSB can't match */
     if (pad && (neg == (p[1] & 0x80))) {
         ASN1err(ASN1_F_C2I_IBUF, ASN1_R_ILLEGAL_PADDING);
@@ -229,12 +240,10 @@ static size_t asn1_put_uint64(unsigned char b[sizeof(uint64_t)], uint64_t r)
 }
 
 /*
- * Absolute value of INT64_MIN: we can't just use -INT64_MIN as it produces
+ * Absolute value of INT64_MIN: we can't just use -INT64_MIN as gcc produces
  * overflow warnings.
  */
-
-#define ABS_INT64_MIN \
-    ((uint64_t)INT64_MAX + (uint64_t)(-(INT64_MIN + INT64_MAX)))
+#define ABS_INT64_MIN ((uint64_t)INT64_MAX + (-(INT64_MIN + INT64_MAX)))
 
 /* signed version of asn1_get_uint64 */
 static int asn1_get_int64(int64_t *pr, const unsigned char *b, size_t blen,
@@ -244,17 +253,25 @@ static int asn1_get_int64(int64_t *pr, const unsigned char *b, size_t blen,
     if (asn1_get_uint64(&r, b, blen) == 0)
         return 0;
     if (neg) {
-        if (r > ABS_INT64_MIN) {
+        if (r <= INT64_MAX) {
+            /* Most significant bit is guaranteed to be clear, negation
+             * is guaranteed to be meaningful in platform-neutral sense. */
+            *pr = -(int64_t)r;
+        } else if (r == ABS_INT64_MIN) {
+            /* This never happens if INT64_MAX == ABS_INT64_MIN, e.g.
+             * on ones'-complement system. */
+            *pr = (int64_t)(0 - r);
+        } else {
             ASN1err(ASN1_F_ASN1_GET_INT64, ASN1_R_TOO_SMALL);
             return 0;
         }
-        *pr = 0 - (uint64_t)r;
     } else {
-        if (r > INT64_MAX) {
+        if (r <= INT64_MAX) {
+            *pr = (int64_t)r;
+        } else {
             ASN1err(ASN1_F_ASN1_GET_INT64, ASN1_R_TOO_LARGE);
             return 0;
         }
-        *pr = (int64_t)r;
     }
     return 1;
 }
@@ -319,7 +336,12 @@ static int asn1_string_set_int64(ASN1_STRING *a, int64_t r, int itype)
 
     a->type = itype;
     if (r < 0) {
-        off = asn1_put_uint64(tbuf, -r);
+        /* Most obvious '-r' triggers undefined behaviour for most
+         * common INT64_MIN. Even though below '0 - (uint64_t)r' can
+         * appear two's-complement centric, it does produce correct/
+         * expected result even on one's-complement. This is because
+         * cast to unsigned has to change bit pattern... */
+        off = asn1_put_uint64(tbuf, 0 - (uint64_t)r);
         a->type |= V_ASN1_NEG;
     } else {
         off = asn1_put_uint64(tbuf, r);