9 use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS);
12 @EXPORT = (@Test::More::EXPORT, qw(setup indir app perlapp test perltest run));
13 @EXPORT_OK = (@Test::More::EXPORT_OK, qw(bldtop_dir bldtop_file
14 srctop_dir srctop_file
15 pipe with cmdstr quotify));
19 OpenSSL::Test - a private extension of Test::More
25 setup("my_test_name");
27 ok(run(app(["openssl", "version"])), "check for openssl presence");
29 indir "subdir" => sub {
30 ok(run(test(["sometest", "arg1"], stdout => "foo.txt")),
31 "run sometest with output to foo.txt");
36 This module is a private extension of L<Test::More> for testing OpenSSL.
37 In addition to the Test::More functions, it also provides functions that
38 easily find the diverse programs within a OpenSSL build tree, as well as
39 some other useful functions.
41 This module I<depends> on the environment variables C<$TOP> or C<$SRCTOP>
42 and C<$BLDTOP>. Without one of the combinations it refuses to work.
43 See L</ENVIRONMENT> below.
48 use File::Spec::Functions qw/file_name_is_absolute curdir canonpath splitdir
49 catdir catfile splitpath catpath devnull abs2rel
51 use File::Path 2.00 qw/rmtree mkpath/;
54 # The name of the test. This is set by setup() and is used in the other
55 # functions to verify that setup() has been used.
56 my $test_name = undef;
58 # Directories we want to keep track of TOP, APPS, TEST and RESULTS are the
59 # ones we're interested in, corresponding to the environment variables TOP
60 # (mandatory), BIN_D, TEST_D, UTIL_D and RESULT_D.
63 # A bool saying if we shall stop all testing if the current recipe has failing
64 # tests or not. This is set by setup() if the environment variable STOPTEST
65 # is defined with a non-empty value.
66 my $end_with_bailout = 0;
68 # A set of hooks that is affected by with() and may be used in diverse places.
69 # All hooks are expected to be CODE references.
72 # exit_checker is used by run() directly after completion of a command.
73 # it receives the exit code from that command and is expected to return
74 # 1 (for success) or 0 (for failure). This is the value that will be
76 # NOTE: When run() gets the option 'capture => 1', this hook is ignored.
77 exit_checker => sub { return shift == 0 ? 1 : 0 },
81 # Debug flag, to be set manually when needed
84 # Declare some utility functions that are defined at the end
91 # Declare some private functions that are defined at the end
101 The following functions are exported by default when using C<OpenSSL::Test>.
107 =item B<setup "NAME">
109 C<setup> is used for initial setup, and it is mandatory that it's used.
110 If it's not used in a OpenSSL test recipe, the rest of the recipe will
111 most likely refuse to run.
113 C<setup> checks for environment variables (see L</ENVIRONMENT> below),
114 checks that C<$TOP/Configure> or C<$SRCTOP/Configure> exists, C<chdir>
115 into the results directory (defined by the C<$RESULT_D> environment
116 variable if defined, otherwise C<$BLDTOP/test> or C<$TOP/test>, whichever
124 my $old_test_name = $test_name;
127 BAIL_OUT("setup() must receive a name") unless $test_name;
128 warn "setup() detected test name change. Innocuous, so we continue...\n"
129 if $old_test_name && $old_test_name ne $test_name;
131 return if $old_test_name;
133 BAIL_OUT("setup() needs \$TOP or \$SRCTOP and \$BLDTOP to be defined")
134 unless $ENV{TOP} || ($ENV{SRCTOP} && $ENV{BLDTOP});
135 BAIL_OUT("setup() found both \$TOP and \$SRCTOP or \$BLDTOP...")
136 if $ENV{TOP} && ($ENV{SRCTOP} || $ENV{BLDTOP});
140 BAIL_OUT("setup() expects the file Configure in the source top directory")
141 unless -f srctop_file("Configure");
143 __cwd($directories{RESULTS});
148 =item B<indir "SUBDIR" =E<gt> sub BLOCK, OPTS>
150 C<indir> is used to run a part of the recipe in a different directory than
151 the one C<setup> moved into, usually a subdirectory, given by SUBDIR.
152 The part of the recipe that's run there is given by the codeblock BLOCK.
154 C<indir> takes some additional options OPTS that affect the subdirectory:
158 =item B<create =E<gt> 0|1>
160 When set to 1 (or any value that perl preceives as true), the subdirectory
161 will be created if it doesn't already exist. This happens before BLOCK
164 =item B<cleanup =E<gt> 0|1>
166 When set to 1 (or any value that perl preceives as true), the subdirectory
167 will be cleaned out and removed. This happens both before and after BLOCK
175 ok(run(app(["openssl", "version"]), stdout => "foo.txt"));
176 if (ok(open(RESULT, "foo.txt"), "reading foo.txt")) {
179 is($line, qr/^OpenSSL 1\./,
180 "check that we're using OpenSSL 1.x.x");
182 }, create => 1, cleanup => 1;
190 my $codeblock = shift;
193 my $reverse = __cwd($subdir,%opts);
194 BAIL_OUT("FAILURE: indir, \"$subdir\" wasn't possible to move into")
201 if ($opts{cleanup}) {
202 rmtree($subdir, { safe => 0 });
208 =item B<app ARRAYREF, OPTS>
210 =item B<test ARRAYREF, OPTS>
212 Both of these functions take a reference to a list that is a command and
213 its arguments, and some additional options (described further on).
215 C<app> expects to find the given command (the first item in the given list
216 reference) as an executable in C<$BIN_D> (if defined, otherwise C<$TOP/apps>
219 C<test> expects to find the given command (the first item in the given list
220 reference) as an executable in C<$TEST_D> (if defined, otherwise C<$TOP/test>
223 Both return a CODEREF to be used by C<run>, C<pipe> or C<cmdstr>.
225 The options that both C<app> and C<test> can take are in the form of hash
230 =item B<stdin =E<gt> PATH>
232 =item B<stdout =E<gt> PATH>
234 =item B<stderr =E<gt> PATH>
236 In all three cases, the corresponding standard input, output or error is
237 redirected from (for stdin) or to (for the others) a file given by the
238 string PATH, I<or>, if the value is C<undef>, C</dev/null> or similar.
242 =item B<perlapp ARRAYREF, OPTS>
244 =item B<perltest ARRAYREF, OPTS>
246 Both these functions function the same way as B<app> and B<test>, except
247 that they expect the command to be a perl script. Also, they support one
252 =item B<interpreter_args =E<gt> ARRAYref>
254 The array reference is a set of arguments for perl rather than the script.
255 Take care so that none of them can be seen as a script! Flags and their
256 eventual arguments only!
262 ok(run(perlapp(["foo.pl", "arg1"],
263 interpreter_args => [ "-I", srctop_dir("test") ])));
272 return sub { my $num = shift;
273 return __build_cmd($num, \&__apps_file, $cmd, %opts); }
279 return sub { my $num = shift;
280 return __build_cmd($num, \&__test_file, $cmd, %opts); }
286 return sub { my $num = shift;
287 return __build_cmd($num, \&__perlapps_file, $cmd, %opts); }
293 return sub { my $num = shift;
294 return __build_cmd($num, \&__perltest_file, $cmd, %opts); }
299 =item B<run CODEREF, OPTS>
301 This CODEREF is expected to be the value return by C<app> or C<test>,
302 anything else will most likely cause an error unless you know what you're
305 C<run> executes the command returned by CODEREF and return either the
306 resulting output (if the option C<capture> is set true) or a boolean indicating
307 if the command succeeded or not.
309 The options that C<run> can take are in the form of hash values:
313 =item B<capture =E<gt> 0|1>
315 If true, the command will be executed with a perl backtick, and C<run> will
316 return the resulting output as an array of lines. If false or not given,
317 the command will be executed with C<system()>, and C<run> will return 1 if
318 the command was successful or 0 if it wasn't.
322 For further discussion on what is considered a successful command or not, see
323 the function C<with> further down.
330 my ($cmd, $display_cmd) = shift->(0);
336 if ( $^O eq "VMS" ) { # VMS
344 # The dance we do with $? is the same dance the Unix shells appear to
345 # do. For example, a program that gets aborted (and therefore signals
346 # SIGABRT = 6) will appear to exit with the code 134. We mimic this
347 # to make it easier to compare with a manual run of the command.
348 if ($opts{capture}) {
350 $e = ($? & 0x7f) ? ($? & 0x7f)|0x80 : ($? >> 8);
352 system("$prefix$cmd");
353 $e = ($? & 0x7f) ? ($? & 0x7f)|0x80 : ($? >> 8);
354 $r = $hooks{exit_checker}->($e);
357 print STDERR "$prefix$display_cmd => $e\n"
358 if !$ENV{HARNESS_ACTIVE} || $ENV{HARNESS_VERBOSE};
360 # At this point, $? stops being interesting, and unfortunately,
361 # there are Test::More versions that get picky if we leave it
365 if ($opts{capture}) {
373 my $tb = Test::More->builder;
374 my $failure = scalar(grep { $_ == 0; } $tb->summary);
375 if ($failure && $end_with_bailout) {
376 BAIL_OUT("Stoptest!");
380 =head2 Utility functions
382 The following functions are exported on request when using C<OpenSSL::Test>.
384 # To only get the bldtop_file and srctop_file functions.
385 use OpenSSL::Test qw/bldtop_file srctop_file/;
387 # To only get the bldtop_file function in addition to the default ones.
388 use OpenSSL::Test qw/:DEFAULT bldtop_file/;
392 # Utility functions, exported on request
396 =item B<bldtop_dir LIST>
398 LIST is a list of directories that make up a path from the top of the OpenSSL
399 build directory (as indicated by the environment variable C<$TOP> or
401 C<bldtop_dir> returns the resulting directory as a string, adapted to the local
409 return __bldtop_dir(@_); # This caters for operating systems that have
410 # a very distinct syntax for directories.
415 =item B<bldtop_file LIST, FILENAME>
417 LIST is a list of directories that make up a path from the top of the OpenSSL
418 build directory (as indicated by the environment variable C<$TOP> or
419 C<$BLDTOP>) and FILENAME is the name of a file located in that directory path.
420 C<bldtop_file> returns the resulting file path as a string, adapted to the local
428 return __bldtop_file(@_);
433 =item B<srctop_dir LIST>
435 LIST is a list of directories that make up a path from the top of the OpenSSL
436 source directory (as indicated by the environment variable C<$TOP> or
438 C<srctop_dir> returns the resulting directory as a string, adapted to the local
446 return __srctop_dir(@_); # This caters for operating systems that have
447 # a very distinct syntax for directories.
452 =item B<srctop_file LIST, FILENAME>
454 LIST is a list of directories that make up a path from the top of the OpenSSL
455 source directory (as indicated by the environment variable C<$TOP> or
456 C<$SRCTOP>) and FILENAME is the name of a file located in that directory path.
457 C<srctop_file> returns the resulting file path as a string, adapted to the local
465 return __srctop_file(@_);
472 LIST is a list of CODEREFs returned by C<app> or C<test>, from which C<pipe>
473 creates a new command composed of all the given commands put together in a
474 pipe. C<pipe> returns a new CODEREF in the same manner as C<app> or C<test>,
475 to be passed to C<run> for execution.
490 my ($c, $dc, @el) = $_->(++$counter);
508 =item B<with HASHREF, CODEREF>
510 C<with> will temporarly install hooks given by the HASHREF and then execute
511 the given CODEREF. Hooks are usually expected to have a coderef as value.
513 The currently available hoosk are:
517 =item B<exit_checker =E<gt> CODEREF>
519 This hook is executed after C<run> has performed its given command. The
520 CODEREF receives the exit code as only argument and is expected to return
521 1 (if the exit code indicated success) or 0 (if the exit code indicated
533 my $codeblock = shift;
535 my %saved_hooks = ();
537 foreach (keys %opts) {
538 $saved_hooks{$_} = $hooks{$_} if exists($hooks{$_});
539 $hooks{$_} = $opts{$_};
544 foreach (keys %saved_hooks) {
545 $hooks{$_} = $saved_hooks{$_};
551 =item B<cmdstr CODEREF>
553 C<cmdstr> takes a CODEREF from C<app> or C<test> and simply returns the
561 my ($cmd, $display_cmd) = shift->(0);
568 =item B<quotify LIST>
570 LIST is a list of strings that are going to be used as arguments for a
571 command, and makes sure to inject quotes and escapes as necessary depending
572 on the content of each string.
574 This can also be used to put quotes around the executable of a command.
575 I<This must never ever be done on VMS.>
582 # Unix setup (default if nothing else is mentioned)
584 sub { $_ = shift; /\s|[\{\}\\\$\[\]\*\?\|\&:;<>]/ ? "'$_'" : $_ };
586 if ( $^O eq "VMS") { # VMS setup
587 $arg_formatter = sub {
589 if (/\s|["[:upper:]]/) {
596 } elsif ( $^O eq "MSWin32") { # MSWin setup
597 $arg_formatter = sub {
599 if (/\s|["\|\&\*\;<>]/) {
608 return map { $arg_formatter->($_) } @_;
611 ######################################################################
612 # private functions. These are never exported.
616 OpenSSL::Test depends on some environment variables.
622 This environment variable is mandatory. C<setup> will check that it's
623 defined and that it's a directory that contains the file C<Configure>.
624 If this isn't so, C<setup> will C<BAIL_OUT>.
628 If defined, its value should be the directory where the openssl application
629 is located. Defaults to C<$TOP/apps> (adapted to the operating system).
633 If defined, its value should be the directory where the test applications
634 are located. Defaults to C<$TOP/test> (adapted to the operating system).
638 If defined, it puts testing in a different mode, where a recipe with
639 failures will result in a C<BAIL_OUT> at the end of its run.
646 $directories{SRCTOP} = $ENV{SRCTOP} || $ENV{TOP};
647 $directories{BLDTOP} = $ENV{BLDTOP} || $ENV{TOP};
648 $directories{BLDAPPS} = $ENV{BIN_D} || __bldtop_dir("apps");
649 $directories{SRCAPPS} = __srctop_dir("apps");
650 $directories{BLDTEST} = $ENV{TEST_D} || __bldtop_dir("test");
651 $directories{SRCTEST} = __srctop_dir("test");
652 $directories{RESULTS} = $ENV{RESULT_D} || $directories{BLDTEST};
654 $end_with_bailout = $ENV{STOPTEST} ? 1 : 0;
658 BAIL_OUT("Must run setup() first") if (! $test_name);
661 return catfile($directories{SRCTOP},@_,$f);
665 BAIL_OUT("Must run setup() first") if (! $test_name);
667 return catdir($directories{SRCTOP},@_);
671 BAIL_OUT("Must run setup() first") if (! $test_name);
674 return catfile($directories{BLDTOP},@_,$f);
678 BAIL_OUT("Must run setup() first") if (! $test_name);
680 return catdir($directories{BLDTOP},@_);
685 if ($^O eq "VMS" ) { # VMS
687 } elsif ($^O eq "MSWin32") { # Windows
690 return $ENV{"EXE_EXT"} || $ext;
694 BAIL_OUT("Must run setup() first") if (! $test_name);
696 my $f = pop . __exeext();
697 $f = catfile($directories{BLDTEST},@_,$f);
698 $f = catfile($directories{SRCTEST},@_,$f) unless -x $f;
702 sub __perltest_file {
703 BAIL_OUT("Must run setup() first") if (! $test_name);
706 $f = catfile($directories{BLDTEST},@_,$f);
707 $f = catfile($directories{SRCTEST},@_,$f) unless -f $f;
712 BAIL_OUT("Must run setup() first") if (! $test_name);
714 my $f = pop . __exeext();
715 $f = catfile($directories{BLDAPPS},@_,$f);
716 $f = catfile($directories{SRCAPPS},@_,$f) unless -x $f;
720 sub __perlapps_file {
721 BAIL_OUT("Must run setup() first") if (! $test_name);
724 $f = catfile($directories{BLDAPPS},@_,$f);
725 $f = catfile($directories{SRCAPPS},@_,$f) unless -f $f;
730 BAIL_OUT("Must run setup() first") if (! $test_name);
733 return catfile($directories{RESULTS},@_,$f);
737 my $dir = catdir(shift);
739 my $abscurdir = rel2abs(curdir());
740 my $absdir = rel2abs($dir);
741 my $reverse = abs2rel($abscurdir, $absdir);
743 # PARANOIA: if we're not moving anywhere, we do nothing more
744 if ($abscurdir eq $absdir) {
748 # Do not support a move to a different volume for now. Maybe later.
749 BAIL_OUT("FAILURE: \"$dir\" moves to a different volume, not supported")
750 if $reverse eq $abscurdir;
752 # If someone happened to give a directory that leads back to the current,
753 # it's extremely silly to do anything more, so just simulate that we did
755 # In this case, we won't even clean it out, for safety's sake.
756 return "." if $reverse eq "";
758 $dir = canonpath($dir);
763 # Should we just bail out here as well? I'm unsure.
764 return undef unless chdir($dir);
766 if ($opts{cleanup}) {
767 rmtree(".", { safe => 0, keep_root => 1 });
770 # For each of these directory variables, figure out where they are relative
771 # to the directory we want to move to if they aren't absolute (if they are,
772 # they don't change!)
773 my @dirtags = sort keys %directories;
775 if (!file_name_is_absolute($directories{$_})) {
776 my $newpath = abs2rel(rel2abs($directories{$_}), rel2abs($dir));
777 $directories{$_} = $newpath;
782 print STDERR "DEBUG: __cwd(), directories and files:\n";
783 print STDERR " \$directories{BLDTEST} = \"$directories{BLDTEST}\"\n";
784 print STDERR " \$directories{SRCTEST} = \"$directories{SRCTEST}\"\n";
785 print STDERR " \$directories{RESULTS} = \"$directories{RESULTS}\"\n";
786 print STDERR " \$directories{BLDAPPS} = \"$directories{BLDAPPS}\"\n";
787 print STDERR " \$directories{SRCAPPS} = \"$directories{SRCAPPS}\"\n";
788 print STDERR " \$directories{SRCTOP} = \"$directories{SRCTOP}\"\n";
789 print STDERR " \$directories{BLDTOP} = \"$directories{BLDTOP}\"\n";
791 print STDERR " current directory is \"",curdir(),"\"\n";
792 print STDERR " the way back is \"$reverse\"\n";
800 my $exe_shell = shift;
802 my $prefix = __bldtop_file("util", "shlib_wrap.sh")." ";
804 if (defined($exe_shell)) {
805 $prefix = "$exe_shell ";
806 } elsif ($^O eq "VMS" ) { # VMS
807 $prefix = ($prog =~ /^(?:[\$a-z0-9_]+:)?[<\[]/i ? "mcr " : "mcr []");
808 } elsif ($^O eq "MSWin32") { # Windows
812 # We test both with and without extension. The reason
813 # is that we might be passed a complete file spec, with
822 if (defined($prog)) {
823 # Make sure to quotify the program file on platforms that may
824 # have spaces or similar in their path name.
825 # To our knowledge, VMS is the exception where quotifying should
827 ($prog) = quotify($prog) unless $^O eq "VMS";
828 return $prefix.$prog;
831 print STDERR "$prog not found\n";
836 BAIL_OUT("Must run setup() first") if (! $test_name);
839 my $path_builder = shift;
840 # Make a copy to not destroy the caller's array
841 my @cmdarray = ( @{$_[0]} ); shift;
844 # We do a little dance, as $path_builder might return a list of
845 # more than one. If so, only the first is to be considered a
846 # program to fix up, the rest is part of the arguments. This
847 # happens for perl scripts, where $path_builder will return
848 # a list of two, $^X and the script name.
849 # Also, if $path_builder returned more than one, we don't apply
850 # the EXE_SHELL environment variable.
851 my @prog = ($path_builder->(shift @cmdarray));
852 my $first = shift @prog;
853 my $exe_shell = @prog ? undef : $ENV{EXE_SHELL};
854 my $cmd = __fixup_cmd($first, $exe_shell);
856 if ( ! -f $prog[0] ) {
857 print STDERR "$prog[0] not found\n";
861 my @args = (@prog, @cmdarray);
862 if (defined($opts{interpreter_args})) {
863 unshift @args, @{$opts{interpreter_args}};
869 my $null = devnull();
872 $arg_str = " ".join(" ", quotify @args) if @args;
874 my $fileornull = sub { $_[0] ? $_[0] : $null; };
878 my $saved_stderr = undef;
879 $stdin = " < ".$fileornull->($opts{stdin}) if exists($opts{stdin});
880 $stdout= " > ".$fileornull->($opts{stdout}) if exists($opts{stdout});
881 $stderr=" 2> ".$fileornull->($opts{stderr}) if exists($opts{stderr});
883 my $display_cmd = "$cmd$arg_str$stdin$stdout$stderr";
886 unless $stderr || !$ENV{HARNESS_ACTIVE} || $ENV{HARNESS_VERBOSE};
888 $cmd .= "$arg_str$stdin$stdout$stderr";
891 print STDERR "DEBUG[__build_cmd]: \$cmd = \"$cmd\"\n";
892 print STDERR "DEBUG[__build_cmd]: \$display_cmd = \"$display_cmd\"\n";
895 return ($cmd, $display_cmd);
900 L<Test::More>, L<Test::Harness>
904 Richard Levitte E<lt>levitte@openssl.orgE<gt> with assitance and
905 inspiration from Andy Polyakov E<lt>appro@openssl.org<gt>.