Clean up the VMS hacks in crypto/rand/randfile.c

[openssl.git] / crypto / rand / randfile.c

diff --git a/crypto/rand/randfile.c b/crypto/rand/randfile.c
index 43547646c46238b574b51e59b710b24088e0d657..49f5405dbcbc3532f1a11a4c62ec2f6ea891baa6 100644 (file)
--- a/crypto/rand/randfile.c
+++ b/crypto/rand/randfile.c
@@ -64,13 +64,46 @@
 #define RAND_DATA 1024
 
 #ifdef OPENSSL_SYS_VMS
+/*
+ * Misc hacks needed for specific cases.
+ *
+ * __FILE_ptr32 is a type provided by DEC C headers (types.h specifically)
+ * to make sure the FILE* is a 32-bit pointer no matter what.  We know that
+ * stdio function return this type (a study of stdio.h proves it).
+ * Additionally, we create a similar char pointer type for the sake of
+ * vms_setbuf below.
+ */
+# if __INITIAL_POINTER_SIZE == 64
+#  pragma pointer_size save
+#  pragma pointer_size 32
+# endif
+typedef char *char_ptr32;
+# if __INITIAL_POINTER_SIZE == 64
+#  pragma pointer_size restore
+# endif
+
+/*
+ * On VMS, setbuf() will only take 32-bit pointers, and a compilation
+ * with /POINTER_SIZE=64 will give off a MAYLOSEDATA2 warning here.
+ * Since we know that the FILE* really is a 32-bit pointer expanded to
+ * 64 bits, we also know it's safe to convert it back to a 32-bit pointer.
+ * As for the buffer parameter, we only use NULL here, so that passes as
+ * well...
+ */
+static void vms_setbuf(FILE *fp, char *buf)
+{
+    setbuf((__FILE_ptr32)fp, (char_ptr32)buf);
+}
 /*
  * This declaration is a nasty hack to get around vms' extension to fopen for
- * passing in sharing options being disabled by our /STANDARD=ANSI89
+ * passing in sharing options being disabled by /STANDARD=ANSI89
  */
-static FILE *(*const vms_fopen)(const char *, const char *, ...) =
-    (FILE *(*)(const char *, const char *, ...))fopen;
+static __FILE_ptr32 (*const vms_fopen)(const char *, const char *, ...) =
+    (__FILE_ptr32 (*)(const char *, const char *, ...))fopen;
 # define VMS_OPEN_ATTRS "shr=get,put,upd,del","ctx=bin,stm","rfm=stm","rat=none","mrs=0"
+
+# define fopen(fname,mode) vms_fopen((fname), (mode), VMS_OPEN_ATTRS)
+# define setbuf(fp,buf) vms_setbuf((fp), (buf))
 #endif
 
 #define RFILE ".rnd"
@@ -112,11 +145,7 @@ int RAND_load_file(const char *file, long bytes)
     if (bytes == 0)
         return (ret);
 
-#ifdef OPENSSL_SYS_VMS
-    in = vms_fopen(file, "rb", VMS_OPEN_ATTRS);
-#else
     in = fopen(file, "rb");
-#endif
     if (in == NULL)
         goto err;
 #if defined(S_ISBLK) && defined(S_ISCHR) && !defined(OPENSSL_NO_POSIX_IO)
@@ -212,13 +241,10 @@ int RAND_write_file(const char *file)
      * rand file in a concurrent use situation.
      */
 
-    out = vms_fopen(file, "rb+", VMS_OPEN_ATTRS);
-    if (out == NULL)
-        out = vms_fopen(file, "wb", VMS_OPEN_ATTRS);
-#else
+    out = fopen(file, "rb+");
+#endif
     if (out == NULL)
         out = fopen(file, "wb");
-#endif
     if (out == NULL)
         goto err;