1 # Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
3 # Licensed under the OpenSSL license (the "License"). You may not use
4 # this file except in compliance with the License. You can obtain a copy
5 # in the file LICENSE in the source distribution or at
6 # https://www.openssl.org/source/license.html
16 use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS);
19 @EXPORT = (@Test::More::EXPORT, qw(setup indir app perlapp test perltest run));
20 @EXPORT_OK = (@Test::More::EXPORT_OK, qw(bldtop_dir bldtop_file
21 srctop_dir srctop_file
22 pipe with cmdstr quotify));
26 OpenSSL::Test - a private extension of Test::More
32 setup("my_test_name");
34 ok(run(app(["openssl", "version"])), "check for openssl presence");
36 indir "subdir" => sub {
37 ok(run(test(["sometest", "arg1"], stdout => "foo.txt")),
38 "run sometest with output to foo.txt");
43 This module is a private extension of L<Test::More> for testing OpenSSL.
44 In addition to the Test::More functions, it also provides functions that
45 easily find the diverse programs within a OpenSSL build tree, as well as
46 some other useful functions.
48 This module I<depends> on the environment variables C<$TOP> or C<$SRCTOP>
49 and C<$BLDTOP>. Without one of the combinations it refuses to work.
50 See L</ENVIRONMENT> below.
55 use File::Spec::Functions qw/file_name_is_absolute curdir canonpath splitdir
56 catdir catfile splitpath catpath devnull abs2rel
58 use File::Path 2.00 qw/rmtree mkpath/;
61 # The name of the test. This is set by setup() and is used in the other
62 # functions to verify that setup() has been used.
63 my $test_name = undef;
65 # Directories we want to keep track of TOP, APPS, TEST and RESULTS are the
66 # ones we're interested in, corresponding to the environment variables TOP
67 # (mandatory), BIN_D, TEST_D, UTIL_D and RESULT_D.
70 # The environment variables that gave us the contents in %directories. These
71 # get modified whenever we change directories, so that subprocesses can use
72 # the values of those environment variables as well
75 # A bool saying if we shall stop all testing if the current recipe has failing
76 # tests or not. This is set by setup() if the environment variable STOPTEST
77 # is defined with a non-empty value.
78 my $end_with_bailout = 0;
80 # A set of hooks that is affected by with() and may be used in diverse places.
81 # All hooks are expected to be CODE references.
84 # exit_checker is used by run() directly after completion of a command.
85 # it receives the exit code from that command and is expected to return
86 # 1 (for success) or 0 (for failure). This is the value that will be
88 # NOTE: When run() gets the option 'capture => 1', this hook is ignored.
89 exit_checker => sub { return shift == 0 ? 1 : 0 },
93 # Debug flag, to be set manually when needed
96 # Declare some utility functions that are defined at the end
103 # Declare some private functions that are defined at the end
111 =head2 Main functions
113 The following functions are exported by default when using C<OpenSSL::Test>.
119 =item B<setup "NAME">
121 C<setup> is used for initial setup, and it is mandatory that it's used.
122 If it's not used in a OpenSSL test recipe, the rest of the recipe will
123 most likely refuse to run.
125 C<setup> checks for environment variables (see L</ENVIRONMENT> below),
126 checks that C<$TOP/Configure> or C<$SRCTOP/Configure> exists, C<chdir>
127 into the results directory (defined by the C<$RESULT_D> environment
128 variable if defined, otherwise C<$BLDTOP/test> or C<$TOP/test>, whichever
136 my $old_test_name = $test_name;
139 BAIL_OUT("setup() must receive a name") unless $test_name;
140 warn "setup() detected test name change. Innocuous, so we continue...\n"
141 if $old_test_name && $old_test_name ne $test_name;
143 return if $old_test_name;
145 BAIL_OUT("setup() needs \$TOP or \$SRCTOP and \$BLDTOP to be defined")
146 unless $ENV{TOP} || ($ENV{SRCTOP} && $ENV{BLDTOP});
147 BAIL_OUT("setup() found both \$TOP and \$SRCTOP or \$BLDTOP...")
148 if $ENV{TOP} && ($ENV{SRCTOP} || $ENV{BLDTOP});
152 BAIL_OUT("setup() expects the file Configure in the source top directory")
153 unless -f srctop_file("Configure");
155 __cwd($directories{RESULTS});
160 =item B<indir "SUBDIR" =E<gt> sub BLOCK, OPTS>
162 C<indir> is used to run a part of the recipe in a different directory than
163 the one C<setup> moved into, usually a subdirectory, given by SUBDIR.
164 The part of the recipe that's run there is given by the codeblock BLOCK.
166 C<indir> takes some additional options OPTS that affect the subdirectory:
170 =item B<create =E<gt> 0|1>
172 When set to 1 (or any value that perl preceives as true), the subdirectory
173 will be created if it doesn't already exist. This happens before BLOCK
176 =item B<cleanup =E<gt> 0|1>
178 When set to 1 (or any value that perl preceives as true), the subdirectory
179 will be cleaned out and removed. This happens both before and after BLOCK
187 ok(run(app(["openssl", "version"]), stdout => "foo.txt"));
188 if (ok(open(RESULT, "foo.txt"), "reading foo.txt")) {
191 is($line, qr/^OpenSSL 1\./,
192 "check that we're using OpenSSL 1.x.x");
194 }, create => 1, cleanup => 1;
202 my $codeblock = shift;
205 my $reverse = __cwd($subdir,%opts);
206 BAIL_OUT("FAILURE: indir, \"$subdir\" wasn't possible to move into")
213 if ($opts{cleanup}) {
214 rmtree($subdir, { safe => 0 });
220 =item B<app ARRAYREF, OPTS>
222 =item B<test ARRAYREF, OPTS>
224 Both of these functions take a reference to a list that is a command and
225 its arguments, and some additional options (described further on).
227 C<app> expects to find the given command (the first item in the given list
228 reference) as an executable in C<$BIN_D> (if defined, otherwise C<$TOP/apps>
231 C<test> expects to find the given command (the first item in the given list
232 reference) as an executable in C<$TEST_D> (if defined, otherwise C<$TOP/test>
235 Both return a CODEREF to be used by C<run>, C<pipe> or C<cmdstr>.
237 The options that both C<app> and C<test> can take are in the form of hash
242 =item B<stdin =E<gt> PATH>
244 =item B<stdout =E<gt> PATH>
246 =item B<stderr =E<gt> PATH>
248 In all three cases, the corresponding standard input, output or error is
249 redirected from (for stdin) or to (for the others) a file given by the
250 string PATH, I<or>, if the value is C<undef>, C</dev/null> or similar.
254 =item B<perlapp ARRAYREF, OPTS>
256 =item B<perltest ARRAYREF, OPTS>
258 Both these functions function the same way as B<app> and B<test>, except
259 that they expect the command to be a perl script. Also, they support one
264 =item B<interpreter_args =E<gt> ARRAYref>
266 The array reference is a set of arguments for perl rather than the script.
267 Take care so that none of them can be seen as a script! Flags and their
268 eventual arguments only!
274 ok(run(perlapp(["foo.pl", "arg1"],
275 interpreter_args => [ "-I", srctop_dir("test") ])));
284 return sub { my $num = shift;
285 return __build_cmd($num, \&__apps_file, $cmd, %opts); }
291 return sub { my $num = shift;
292 return __build_cmd($num, \&__test_file, $cmd, %opts); }
298 return sub { my $num = shift;
299 return __build_cmd($num, \&__perlapps_file, $cmd, %opts); }
305 return sub { my $num = shift;
306 return __build_cmd($num, \&__perltest_file, $cmd, %opts); }
311 =item B<run CODEREF, OPTS>
313 This CODEREF is expected to be the value return by C<app> or C<test>,
314 anything else will most likely cause an error unless you know what you're
317 C<run> executes the command returned by CODEREF and return either the
318 resulting output (if the option C<capture> is set true) or a boolean indicating
319 if the command succeeded or not.
321 The options that C<run> can take are in the form of hash values:
325 =item B<capture =E<gt> 0|1>
327 If true, the command will be executed with a perl backtick, and C<run> will
328 return the resulting output as an array of lines. If false or not given,
329 the command will be executed with C<system()>, and C<run> will return 1 if
330 the command was successful or 0 if it wasn't.
334 For further discussion on what is considered a successful command or not, see
335 the function C<with> further down.
342 my ($cmd, $display_cmd) = shift->(0);
348 if ( $^O eq "VMS" ) { # VMS
356 # In non-verbose, we want to shut up the command interpreter, in case
357 # it has something to complain about. On VMS, it might complain both
358 # on stdout and stderr
359 *save_STDOUT = *STDOUT;
360 *save_STDERR = *STDERR;
361 if ($ENV{HARNESS_ACTIVE} && !$ENV{HARNESS_VERBOSE}) {
362 open STDOUT, ">", devnull();
363 open STDERR, ">", devnull();
366 # The dance we do with $? is the same dance the Unix shells appear to
367 # do. For example, a program that gets aborted (and therefore signals
368 # SIGABRT = 6) will appear to exit with the code 134. We mimic this
369 # to make it easier to compare with a manual run of the command.
370 if ($opts{capture}) {
372 $e = ($? & 0x7f) ? ($? & 0x7f)|0x80 : ($? >> 8);
374 system("$prefix$cmd");
375 $e = ($? & 0x7f) ? ($? & 0x7f)|0x80 : ($? >> 8);
376 $r = $hooks{exit_checker}->($e);
379 if ($ENV{HARNESS_ACTIVE} && !$ENV{HARNESS_VERBOSE}) {
383 *STDOUT = *save_STDOUT;
384 *STDERR = *save_STDERR;
386 print STDERR "$prefix$display_cmd => $e\n"
387 if !$ENV{HARNESS_ACTIVE} || $ENV{HARNESS_VERBOSE};
389 # At this point, $? stops being interesting, and unfortunately,
390 # there are Test::More versions that get picky if we leave it
394 if ($opts{capture}) {
402 my $tb = Test::More->builder;
403 my $failure = scalar(grep { $_ == 0; } $tb->summary);
404 if ($failure && $end_with_bailout) {
405 BAIL_OUT("Stoptest!");
409 =head2 Utility functions
411 The following functions are exported on request when using C<OpenSSL::Test>.
413 # To only get the bldtop_file and srctop_file functions.
414 use OpenSSL::Test qw/bldtop_file srctop_file/;
416 # To only get the bldtop_file function in addition to the default ones.
417 use OpenSSL::Test qw/:DEFAULT bldtop_file/;
421 # Utility functions, exported on request
425 =item B<bldtop_dir LIST>
427 LIST is a list of directories that make up a path from the top of the OpenSSL
428 build directory (as indicated by the environment variable C<$TOP> or
430 C<bldtop_dir> returns the resulting directory as a string, adapted to the local
438 return __bldtop_dir(@_); # This caters for operating systems that have
439 # a very distinct syntax for directories.
444 =item B<bldtop_file LIST, FILENAME>
446 LIST is a list of directories that make up a path from the top of the OpenSSL
447 build directory (as indicated by the environment variable C<$TOP> or
448 C<$BLDTOP>) and FILENAME is the name of a file located in that directory path.
449 C<bldtop_file> returns the resulting file path as a string, adapted to the local
457 return __bldtop_file(@_);
462 =item B<srctop_dir LIST>
464 LIST is a list of directories that make up a path from the top of the OpenSSL
465 source directory (as indicated by the environment variable C<$TOP> or
467 C<srctop_dir> returns the resulting directory as a string, adapted to the local
475 return __srctop_dir(@_); # This caters for operating systems that have
476 # a very distinct syntax for directories.
481 =item B<srctop_file LIST, FILENAME>
483 LIST is a list of directories that make up a path from the top of the OpenSSL
484 source directory (as indicated by the environment variable C<$TOP> or
485 C<$SRCTOP>) and FILENAME is the name of a file located in that directory path.
486 C<srctop_file> returns the resulting file path as a string, adapted to the local
494 return __srctop_file(@_);
501 LIST is a list of CODEREFs returned by C<app> or C<test>, from which C<pipe>
502 creates a new command composed of all the given commands put together in a
503 pipe. C<pipe> returns a new CODEREF in the same manner as C<app> or C<test>,
504 to be passed to C<run> for execution.
519 my ($c, $dc, @el) = $_->(++$counter);
537 =item B<with HASHREF, CODEREF>
539 C<with> will temporarly install hooks given by the HASHREF and then execute
540 the given CODEREF. Hooks are usually expected to have a coderef as value.
542 The currently available hoosk are:
546 =item B<exit_checker =E<gt> CODEREF>
548 This hook is executed after C<run> has performed its given command. The
549 CODEREF receives the exit code as only argument and is expected to return
550 1 (if the exit code indicated success) or 0 (if the exit code indicated
562 my $codeblock = shift;
564 my %saved_hooks = ();
566 foreach (keys %opts) {
567 $saved_hooks{$_} = $hooks{$_} if exists($hooks{$_});
568 $hooks{$_} = $opts{$_};
573 foreach (keys %saved_hooks) {
574 $hooks{$_} = $saved_hooks{$_};
580 =item B<cmdstr CODEREF, OPTS>
582 C<cmdstr> takes a CODEREF from C<app> or C<test> and simply returns the
585 C<cmdstr> takes some additiona options OPTS that affect the string returned:
589 =item B<display =E<gt> 0|1>
591 When set to 0, the returned string will be with all decorations, such as a
592 possible redirect of stderr to the null device. This is suitable if the
593 string is to be used directly in a recipe.
595 When set to 1, the returned string will be without extra decorations. This
596 is suitable for display if that is desired (doesn't confuse people with all
597 internal stuff), or if it's used to pass a command down to a subprocess.
608 my ($cmd, $display_cmd) = shift->(0);
611 if ($opts{display}) {
620 =item B<quotify LIST>
622 LIST is a list of strings that are going to be used as arguments for a
623 command, and makes sure to inject quotes and escapes as necessary depending
624 on the content of each string.
626 This can also be used to put quotes around the executable of a command.
627 I<This must never ever be done on VMS.>
634 # Unix setup (default if nothing else is mentioned)
636 sub { $_ = shift; /\s|[\{\}\\\$\[\]\*\?\|\&:;<>]/ ? "'$_'" : $_ };
638 if ( $^O eq "VMS") { # VMS setup
639 $arg_formatter = sub {
641 if (/\s|["[:upper:]]/) {
648 } elsif ( $^O eq "MSWin32") { # MSWin setup
649 $arg_formatter = sub {
651 if (/\s|["\|\&\*\;<>]/) {
660 return map { $arg_formatter->($_) } @_;
663 ######################################################################
664 # private functions. These are never exported.
668 OpenSSL::Test depends on some environment variables.
674 This environment variable is mandatory. C<setup> will check that it's
675 defined and that it's a directory that contains the file C<Configure>.
676 If this isn't so, C<setup> will C<BAIL_OUT>.
680 If defined, its value should be the directory where the openssl application
681 is located. Defaults to C<$TOP/apps> (adapted to the operating system).
685 If defined, its value should be the directory where the test applications
686 are located. Defaults to C<$TOP/test> (adapted to the operating system).
690 If defined, it puts testing in a different mode, where a recipe with
691 failures will result in a C<BAIL_OUT> at the end of its run.
698 $directories{SRCTOP} = $ENV{SRCTOP} || $ENV{TOP};
699 $directories{BLDTOP} = $ENV{BLDTOP} || $ENV{TOP};
700 $directories{BLDAPPS} = $ENV{BIN_D} || __bldtop_dir("apps");
701 $directories{SRCAPPS} = __srctop_dir("apps");
702 $directories{BLDTEST} = $ENV{TEST_D} || __bldtop_dir("test");
703 $directories{SRCTEST} = __srctop_dir("test");
704 $directories{RESULTS} = $ENV{RESULT_D} || $directories{BLDTEST};
706 push @direnv, "TOP" if $ENV{TOP};
707 push @direnv, "SRCTOP" if $ENV{SRCTOP};
708 push @direnv, "BLDTOP" if $ENV{BLDTOP};
709 push @direnv, "BIN_D" if $ENV{BIN_D};
710 push @direnv, "TEST_D" if $ENV{TEST_D};
711 push @direnv, "RESULT_D" if $ENV{RESULT_D};
713 $end_with_bailout = $ENV{STOPTEST} ? 1 : 0;
717 BAIL_OUT("Must run setup() first") if (! $test_name);
720 return catfile($directories{SRCTOP},@_,$f);
724 BAIL_OUT("Must run setup() first") if (! $test_name);
726 return catdir($directories{SRCTOP},@_);
730 BAIL_OUT("Must run setup() first") if (! $test_name);
733 return catfile($directories{BLDTOP},@_,$f);
737 BAIL_OUT("Must run setup() first") if (! $test_name);
739 return catdir($directories{BLDTOP},@_);
744 if ($^O eq "VMS" ) { # VMS
746 } elsif ($^O eq "MSWin32") { # Windows
749 return $ENV{"EXE_EXT"} || $ext;
753 BAIL_OUT("Must run setup() first") if (! $test_name);
756 $f = catfile($directories{BLDTEST},@_,$f . __exeext());
757 $f = catfile($directories{SRCTEST},@_,$f) unless -x $f;
761 sub __perltest_file {
762 BAIL_OUT("Must run setup() first") if (! $test_name);
765 $f = catfile($directories{BLDTEST},@_,$f);
766 $f = catfile($directories{SRCTEST},@_,$f) unless -f $f;
771 BAIL_OUT("Must run setup() first") if (! $test_name);
774 $f = catfile($directories{BLDAPPS},@_,$f . __exeext());
775 $f = catfile($directories{SRCAPPS},@_,$f) unless -x $f;
779 sub __perlapps_file {
780 BAIL_OUT("Must run setup() first") if (! $test_name);
783 $f = catfile($directories{BLDAPPS},@_,$f);
784 $f = catfile($directories{SRCAPPS},@_,$f) unless -f $f;
789 BAIL_OUT("Must run setup() first") if (! $test_name);
792 return catfile($directories{RESULTS},@_,$f);
796 my $dir = catdir(shift);
798 my $abscurdir = rel2abs(curdir());
799 my $absdir = rel2abs($dir);
800 my $reverse = abs2rel($abscurdir, $absdir);
802 # PARANOIA: if we're not moving anywhere, we do nothing more
803 if ($abscurdir eq $absdir) {
807 # Do not support a move to a different volume for now. Maybe later.
808 BAIL_OUT("FAILURE: \"$dir\" moves to a different volume, not supported")
809 if $reverse eq $abscurdir;
811 # If someone happened to give a directory that leads back to the current,
812 # it's extremely silly to do anything more, so just simulate that we did
814 # In this case, we won't even clean it out, for safety's sake.
815 return "." if $reverse eq "";
817 $dir = canonpath($dir);
822 # Should we just bail out here as well? I'm unsure.
823 return undef unless chdir($dir);
825 if ($opts{cleanup}) {
826 rmtree(".", { safe => 0, keep_root => 1 });
829 # For each of these directory variables, figure out where they are relative
830 # to the directory we want to move to if they aren't absolute (if they are,
831 # they don't change!)
832 my @dirtags = sort keys %directories;
834 if (!file_name_is_absolute($directories{$_})) {
835 my $newpath = abs2rel(rel2abs($directories{$_}), rel2abs($dir));
836 $directories{$_} = $newpath;
840 # Treat each environment variable that was used to get us the values in
841 # %directories the same was as the paths in %directories, so any sub
842 # process can use their values properly as well
844 if (!file_name_is_absolute($ENV{$_})) {
845 my $newpath = abs2rel(rel2abs($ENV{$_}), rel2abs($dir));
851 print STDERR "DEBUG: __cwd(), directories and files:\n";
852 print STDERR " \$directories{BLDTEST} = \"$directories{BLDTEST}\"\n";
853 print STDERR " \$directories{SRCTEST} = \"$directories{SRCTEST}\"\n";
854 print STDERR " \$directories{RESULTS} = \"$directories{RESULTS}\"\n";
855 print STDERR " \$directories{BLDAPPS} = \"$directories{BLDAPPS}\"\n";
856 print STDERR " \$directories{SRCAPPS} = \"$directories{SRCAPPS}\"\n";
857 print STDERR " \$directories{SRCTOP} = \"$directories{SRCTOP}\"\n";
858 print STDERR " \$directories{BLDTOP} = \"$directories{BLDTOP}\"\n";
860 print STDERR " current directory is \"",curdir(),"\"\n";
861 print STDERR " the way back is \"$reverse\"\n";
869 my $exe_shell = shift;
871 my $prefix = __bldtop_file("util", "shlib_wrap.sh")." ";
873 if (defined($exe_shell)) {
874 $prefix = "$exe_shell ";
875 } elsif ($^O eq "VMS" ) { # VMS
876 $prefix = ($prog =~ /^(?:[\$a-z0-9_]+:)?[<\[]/i ? "mcr " : "mcr []");
877 } elsif ($^O eq "MSWin32") { # Windows
881 # We test both with and without extension. The reason
882 # is that we might be passed a complete file spec, with
891 if (defined($prog)) {
892 # Make sure to quotify the program file on platforms that may
893 # have spaces or similar in their path name.
894 # To our knowledge, VMS is the exception where quotifying should
896 ($prog) = quotify($prog) unless $^O eq "VMS";
897 return $prefix.$prog;
900 print STDERR "$prog not found\n";
905 BAIL_OUT("Must run setup() first") if (! $test_name);
908 my $path_builder = shift;
909 # Make a copy to not destroy the caller's array
910 my @cmdarray = ( @{$_[0]} ); shift;
913 # We do a little dance, as $path_builder might return a list of
914 # more than one. If so, only the first is to be considered a
915 # program to fix up, the rest is part of the arguments. This
916 # happens for perl scripts, where $path_builder will return
917 # a list of two, $^X and the script name.
918 # Also, if $path_builder returned more than one, we don't apply
919 # the EXE_SHELL environment variable.
920 my @prog = ($path_builder->(shift @cmdarray));
921 my $first = shift @prog;
922 my $exe_shell = @prog ? undef : $ENV{EXE_SHELL};
923 my $cmd = __fixup_cmd($first, $exe_shell);
925 if ( ! -f $prog[0] ) {
926 print STDERR "$prog[0] not found\n";
930 my @args = (@prog, @cmdarray);
931 if (defined($opts{interpreter_args})) {
932 unshift @args, @{$opts{interpreter_args}};
938 my $null = devnull();
941 $arg_str = " ".join(" ", quotify @args) if @args;
943 my $fileornull = sub { $_[0] ? $_[0] : $null; };
947 my $saved_stderr = undef;
948 $stdin = " < ".$fileornull->($opts{stdin}) if exists($opts{stdin});
949 $stdout= " > ".$fileornull->($opts{stdout}) if exists($opts{stdout});
950 $stderr=" 2> ".$fileornull->($opts{stderr}) if exists($opts{stderr});
952 my $display_cmd = "$cmd$arg_str$stdin$stdout$stderr";
955 unless $stderr || !$ENV{HARNESS_ACTIVE} || $ENV{HARNESS_VERBOSE};
957 $cmd .= "$arg_str$stdin$stdout$stderr";
960 print STDERR "DEBUG[__build_cmd]: \$cmd = \"$cmd\"\n";
961 print STDERR "DEBUG[__build_cmd]: \$display_cmd = \"$display_cmd\"\n";
964 return ($cmd, $display_cmd);
969 L<Test::More>, L<Test::Harness>
973 Richard Levitte E<lt>levitte@openssl.orgE<gt> with assitance and
974 inspiration from Andy Polyakov E<lt>appro@openssl.org<gt>.