Add atexit configuration option to using atexit() in libcrypto at build-time.

This fixes an issue with a mix of atexit() usage in DLL and statically linked
libcrypto that came out in the test suite on NonStop, which has slightly
different DLL unload processing semantics compared to Linux. The change
allows a build configuration to select whether to register OPENSSL_cleanup()
with atexit() or not, so avoid situations where atexit() registration causes
SIGSEGV.

INSTALL.md and CHANGES.md have been modified to include and describe this
option.

The no-atexit option has been added to .github/workflows/run-checker-daily.yml.

Fixes: #23135
Signed-of-by: Randall S. Becker <randall.becker@nexbridge.ca>
Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Neil Horman <nhorman@openssl.org>
Reviewed-by: Todd Short <todd.short@me.com>
(Merged from https://github.com/openssl/openssl/pull/23394)

diff --git a/.github/workflows/run-checker-daily.yml b/.github/workflows/run-checker-daily.yml
index eda5005f68f9e577be377d83a0f40833a10cad54..6d400781222446a5b30235228628e6e38ed14f69 100644 (file)
--- a/.github/workflows/run-checker-daily.yml
+++ b/.github/workflows/run-checker-daily.yml
@@ -27,6 +27,7 @@ jobs:
           no-asan,
           no-asm,
           no-async,
+          no-atexit,
           no-autoalginit,
           no-autoerrinit,
           no-autoload-config,

diff --git a/CHANGES.md b/CHANGES.md
index 06821648649a6f893787e882212011640856c231..e8ac02d12fc5e8dbc02b7ce4f1456586f569d9b3 100644 (file)
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -87,6 +87,13 @@ OpenSSL 3.3
 
    *Hugo Landau*
 
+ * New atexit configuration switch, which controls whether the OPENSSL_cleanup
+   is registered when libcrypto is unloaded. This is turned off on NonStop
+   configurations because of loader differences on that platform compared to
+   Linux.
+
+   *Randall S. Becker*
+
 OpenSSL 3.2
 -----------
 

diff --git a/Configurations/50-nonstop.conf b/Configurations/50-nonstop.conf
index 873e398169bb7c0b820d60ac8203971029dffc90..7a5c5dcd454ad7d05a258cccfc81512cb408036f 100644 (file)
--- a/Configurations/50-nonstop.conf
+++ b/Configurations/50-nonstop.conf
@@ -21,6 +21,8 @@
         shared_extension => ".so",
         ex_libs          => add('-lrld'),
         enable           => ['egd'],
+       # Not currently inherited
+        disable          => ['atexit'],
         dso_scheme       => 'DLFCN',
         sys_id           => 'TANDEM',
     },
@@ -178,7 +180,7 @@
                               'nonstop-archenv-x86_64-oss',
                               'nonstop-ilp32',
                               'nonstop-efloat-x86_64' ],
-        disable          => ['threads'],
+        disable          => ['threads','atexit'],
     },
     'nonstop-nsx_put' => {
         inherit_from     => [ 'nonstop-common',
@@ -188,6 +190,7 @@
                               'nonstop-model-put' ],
         multilib         => '-put',
         multibin         => '-put',
+        disable          => ['atexit'],
     },
     'nonstop-nsx_64' => {
         inherit_from     => [ 'nonstop-common',
@@ -196,7 +199,7 @@
                               'nonstop-efloat-x86_64' ],
         multilib         => '64',
         multibin         => '64',
-        disable          => ['threads'],
+        disable          => ['threads','atexit'],
     },
     'nonstop-nsx_64_put' => {
         inherit_from     => [ 'nonstop-common',
@@ -206,18 +209,19 @@
                               'nonstop-model-put' ],
         multilib         => '64-put',
         multibin         => '64-put',
+        disable          => ['atexit'],
     },
     'nonstop-nsx_g' => {
         inherit_from     => [ 'nonstop-common',
                               'nonstop-archenv-x86_64-guardian',
                               'nonstop-ilp32', 'nonstop-nfloat-x86_64' ],
-        disable          => ['threads'],
+        disable          => ['threads','atexit'],
     },
     'nonstop-nsx_g_tandem' => {
         inherit_from     => [ 'nonstop-common',
                               'nonstop-archenv-x86_64-guardian',
                               'nonstop-ilp32', 'nonstop-tfloat-x86_64' ],
-        disable          => ['threads'],
+        disable          => ['threads','atexit'],
     },
     'nonstop-nsv' => {
         inherit_from     => [ 'nonstop-nsx' ],
@@ -227,7 +231,7 @@
                               'nonstop-archenv-itanium-oss',
                               'nonstop-ilp32',
                               'nonstop-efloat-itanium' ],
-        disable          => ['threads'],
+        disable          => ['threads','atexit'],
     },
     'nonstop-nse_put' => {
         inherit_from     => [ 'nonstop-common',
@@ -237,6 +241,7 @@
                               'nonstop-model-put' ],
         multilib         => '-put',
         multibin         => '-put',
+        disable          => ['atexit'],
     },
     'nonstop-nse_64' => {
         inherit_from     => [ 'nonstop-common',
@@ -245,7 +250,7 @@
                               'nonstop-efloat-itanium' ],
         multilib         => '64',
         multibin         => '64',
-        disable          => ['threads'],
+        disable          => ['threads','atexit'],
     },
     'nonstop-nse_64_put' => {
         inherit_from     => [ 'nonstop-common',
@@ -255,17 +260,18 @@
                               'nonstop-model-put' ],
         multilib         => '64-put',
         multibin         => '64-put',
+        disable          => ['atexit'],
     },
     'nonstop-nse_g' => {
         inherit_from     => [ 'nonstop-common',
                               'nonstop-archenv-itanium-guardian',
                               'nonstop-ilp32', 'nonstop-nfloat-itanium' ],
-        disable          => ['threads'],
+        disable          => ['threads','atexit'],
     },
 
     'nonstop-nse_g_tandem' => {
         inherit_from     => [ 'nonstop-common',
                               'nonstop-archenv-itanium-guardian',
                               'nonstop-ilp32', 'nonstop-tfloat-itanium' ],
-        disable          => ['threads'],
+        disable          => ['threads','atexit'],
     },

diff --git a/Configure b/Configure
index 846a427cff8a17837fdd0de438cac361b760ff03..2aa6af83a2cc674a03dd48891992b99e1ae8df1e 100755 (executable)
--- a/Configure
+++ b/Configure
@@ -420,6 +420,7 @@ my @disablables = (
     "asan",
     "asm",
     "async",
+    "atexit",
     "autoalginit",
     "autoerrinit",
     "autoload-config",
@@ -596,7 +597,7 @@ our %disabled = ( # "what"         => "comment"
 my @disable_cascades = (
     # "what"            => [ "cascade", ... ]
     "bulk"              => [ "shared", "dso",
-                             "aria", "async", "autoload-config",
+                             "aria", "async", "atexit", "autoload-config",
                              "blake2", "bf", "camellia", "cast", "chacha",
                              "cmac", "cms", "cmp", "comp", "ct",
                              "des", "dgram", "dh", "dsa",

diff --git a/INSTALL.md b/INSTALL.md
index 37b57027f4f123ddd06311f469f8edd988b688c2..18bfe59c571710c31b0c332de0ff742ce2f73bfb 100644 (file)
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -64,6 +64,7 @@ issues and other details, please read one of these:
  * [Notes for Windows platforms](NOTES-WINDOWS.md)
  * [Notes for the DOS platform with DJGPP](NOTES-DJGPP.md)
  * [Notes for the OpenVMS platform](NOTES-VMS.md)
+ * [Notes for the HPE NonStop platform](NOTES-NONSTOP.md)
  * [Notes on Perl](NOTES-PERL.md)
  * [Notes on Valgrind](NOTES-VALGRIND.md)
 
@@ -141,7 +142,7 @@ Use the following commands to configure, build and test OpenSSL.
 The testing is optional, but recommended if you intend to install
 OpenSSL for production use.
 
-### Unix / Linux / macOS
+### Unix / Linux / macOS / NonStop
 
     $ ./Configure
     $ make
@@ -198,7 +199,7 @@ the global search path for system libraries.
 Finally, if you plan on using the FIPS module, you need to read the
 [Post-installation Notes](#post-installation-notes) further down.
 
-### Unix / Linux / macOS
+### Unix / Linux / macOS / NonStop
 
 Depending on your distribution, you need to run the following command as
 root user or prepend `sudo` to the command:
@@ -600,6 +601,14 @@ be used even with this option.
 
 Do not build support for async operations.
 
+### no-atexit
+
+Do not use `atexit()` in libcrypto builds.
+
+`atexit()` has varied semantics between platforms and can cause SIGSEGV in some
+circumstances. This options disables the atexit registration of OPENSSL_cleanup.
+By default, NonStop configurations use `no-atexit`.
+
 ### no-autoalginit
 
 Don't automatically load all supported ciphers and digests.

diff --git a/NOTES-NONSTOP.md b/NOTES-NONSTOP.md
index 65bfc1087dcabfaf6f941f68c513af0b810c1859..6cf12042d53af9bcf5e13580cdb53eb79586738d 100644 (file)
--- a/NOTES-NONSTOP.md
+++ b/NOTES-NONSTOP.md
@@ -57,7 +57,13 @@ relating to `atexit()` processing when a shared library is unloaded and when
 the program terminates. This limitation applies to all OpenSSL shared library
 components.
 
-A resolution to this situation is under investigation.
+A control has been added as of 3.3.x to disable calls to `atexit()` within the
+`libcrypto` builds (specifically in `crypto/init.c`). This switch can be
+controlled using `disable-atexit` or `enable-atexit`, and is disabled by default
+for NonStop builds. If you need to have `atexit()` functionality, set
+`enabled-atexit` when configuring OpenSSL to enable the `atexit()` call to
+register `OPENSSL_cleanup()` automatically. Preferably, you can explicitly call
+`OPENSSL_cleanup()` from your application.
 
 About Prefix and OpenSSLDir
 ---------------------------

diff --git a/crypto/init.c b/crypto/init.c
index 33c739c30e6c78292793b19bbce781fe5e31dbee..40be312b8a6fd0a943a02c97b0caf352d5a73897 100644 (file)
--- a/crypto/init.c
+++ b/crypto/init.c
@@ -97,17 +97,19 @@ static int win32atexit(void)
 
 DEFINE_RUN_ONCE_STATIC(ossl_init_register_atexit)
 {
-#ifdef OPENSSL_INIT_DEBUG
+#ifndef OPENSSL_NO_ATEXIT
+# ifdef OPENSSL_INIT_DEBUG
     fprintf(stderr, "OPENSSL_INIT: ossl_init_register_atexit()\n");
-#endif
-#ifndef OPENSSL_SYS_UEFI
-# if defined(_WIN32) && !defined(__BORLANDC__)
+# endif
+# ifndef OPENSSL_SYS_UEFI
+#  if defined(_WIN32) && !defined(__BORLANDC__)
     /* We use _onexit() in preference because it gets called on DLL unload */
     if (_onexit(win32atexit) == NULL)
         return 0;
-# else
+#  else
     if (atexit(OPENSSL_cleanup) != 0)
         return 0;
+#  endif
 # endif
 #endif