2 # Copyright 2018 The OpenSSL Project Authors. All Rights Reserved.
4 # Licensed under the Apache License 2.0 (the "License"). You may not use
5 # this file except in compliance with the License. You can obtain a copy
6 # in the file LICENSE in the source distribution or at
7 # https://www.openssl.org/source/license.html
9 package OpenSSL::Ordinals;
14 use Scalar::Util qw(blessed);
18 # "magic" filters, see the filters at the end of the file
25 OpenSSL::Ordinals - a private module to read and walk through ordinals
29 use OpenSSL::Ordinals;
31 my $ordinals = OpenSSL::Ordinals->new(from => "foo.num");
33 my $ordinals = OpenSSL::Ordinals->new();
34 $ordinals->load("foo.num");
36 foreach ($ordinals->items(comparator => by_name()) {
37 print $_->name(), "\n";
42 This is a OpenSSL private module to load an ordinals (F<.num>) file and
43 write out the data you want, sorted and filtered according to your rules.
45 An ordinals file is a file that enumerates all the symbols that a shared
46 library or loadable module must export. Each of them have a unique
47 assigned number as well as other attributes to indicate if they only exist
48 on a subset of the supported platforms, or if they are specific to certain
51 The unique numbers each symbol gets assigned needs to be maintained for a
52 shared library or module to stay compatible with previous versions on
53 platforms that maintain a transfer vector indexed by position rather than
54 by name. They also help keep information on certain symbols that are
55 aliases for others for certain platforms, or that have different forms
56 on different platforms.
64 =item B<new> I<%options>
66 Creates a new instance of the C<OpenSSL::Ordinals> class. It takes options
67 in keyed pair form, i.e. a series of C<key =E<gt> value> pairs. Available
72 =item B<from =E<gt> FILENAME>
74 Not only create a new instance, but immediately load it with data from the
75 ordinals file FILENAME.
86 filename => undef, # File name registered when loading
87 loaded_maxnum => 0, # Highest allocated item number when loading
88 loaded_contents => [], # Loaded items, if loading there was
89 maxnum => 0, # Current highest allocated item number
90 contents => [], # Items, indexed by number
91 name2num => {}, # Name to number dictionary
92 aliases => {}, # Aliases cache.
93 stats => {}, # Statistics, see 'sub validate'
94 debug => $opts{debug},
96 bless $instance, $class;
98 $instance->set_version($opts{version});
99 $instance->load($opts{from}) if defined($opts{from});
104 =item B<$ordinals-E<gt>load FILENAME>
106 Loads the data from FILENAME into the instance. Any previously loaded data
109 Two internal databases are created. One database is simply a copy of the file
110 contents and is treated as read-only. The other database is an exact copy of
111 the first, but is treated as a work database, i.e. it can be modified and added
118 my $filename = shift;
120 croak "Undefined filename" unless defined($filename);
122 my @tmp_contents = ();
123 my %tmp_name2num = ();
125 open F, '<', $filename or croak "Unable to open $filename";
127 s|\R$||; # Better chomp
131 my $item = OpenSSL::Ordinals::Item->new(from => $_);
133 my $num = $item->number();
134 croak "Disordered ordinals, $num < $max_num"
138 push @{$tmp_contents[$item->number()]}, $item;
139 $tmp_name2num{$item->name()} = $item->number();
143 $self->{contents} = [ @tmp_contents ];
144 $self->{name2num} = { %tmp_name2num };
145 $self->{maxnum} = $max_num;
146 $self->{filename} = $filename;
148 # Make a deep copy, allowing {contents} to be an independent work array
149 foreach my $i (1..$max_num) {
150 if ($tmp_contents[$i]) {
151 $self->{loaded_contents}->[$i] =
152 [ map { OpenSSL::Ordinals::Item->new($_) }
153 @{$tmp_contents[$i]} ];
156 $self->{loaded_maxnum} = $max_num;
160 =item B<$ordinals-E<gt>rewrite>
162 If an ordinals file has been loaded, it gets rewritten with the data from
163 the current work database.
170 $self->write($self->{filename});
173 =item B<$ordinals-E<gt>write FILENAME>
175 Writes the current work database data to the ordinals file FILENAME.
176 This also validates the data, see B<$ordinals-E<gt>validate> below.
182 my $filename = shift;
184 croak "Undefined filename" unless defined($filename);
188 open F, '>', $filename or croak "Unable to open $filename";
189 foreach ($self->items(by => by_number())) {
190 print F $_->to_string(),"\n";
193 $self->{filename} = $filename;
194 $self->{loaded_maxnum} = $self->{maxnum};
198 =item B<$ordinals-E<gt>items> I<%options>
200 Returns a list of items according to a set of criteria. The criteria is
201 given in form keyed pair form, i.e. a series of C<key =E<gt> value> pairs.
202 Available options are:
206 =item B<sort =E<gt> SORTFUNCTION>
208 SORTFUNCTION is a reference to a function that takes two arguments, which
209 correspond to the classic C<$a> and C<$b> that are available in a C<sort>
212 =item B<filter =E<gt> FILTERFUNCTION>
214 FILTERFUNTION is a reference to a function that takes one argument, which
215 is every OpenSSL::Ordinals::Item element available.
225 my $comparator = $opts{sort};
226 my $filter = $opts{filter} // sub { 1; };
229 if (ref($filter) eq 'ARRAY') {
230 # run a "magic" filter
231 if ($filter->[0] == F_NUMBER) {
232 my $index = $filter->[1];
233 @l = $index ? @{$self->{contents}->[$index] // []} : ();
234 } elsif ($filter->[0] == F_NAME) {
235 my $index = $self->{name2num}->{$filter->[1]};
236 @l = $index ? @{$self->{contents}->[$index] // []} : ();
238 croak __PACKAGE__."->items called with invalid filter";
240 } elsif (ref($filter) eq 'CODE') {
241 @l = grep { $filter->($_) }
243 @{$self->{contents}};
245 croak __PACKAGE__."->items called with invalid filter";
248 return sort { $comparator->($a, $b); } @l
249 if (defined $comparator);
253 # Put an array of items back into the object after having checked consistency
254 # If there are exactly two items:
255 # - They MUST have the same number
256 # - For platforms, both MUST hold the same ones, but with opposite values
257 # - For features, both MUST hold the same ones.
258 # If there's just one item, just put it in the slot of its number
259 # In all other cases, something is wrong
264 if (scalar @items < 1 || scalar @items > 2) {
265 croak "Wrong number of items: ", scalar @items, " : ",
266 join(", ", map { $_->name() } @items), "\n";
268 if (scalar @items == 2) {
274 $numbers{$_->number()} = 1;
275 $versions{$_->version()} = 1;
276 foreach ($_->features()) {
281 # Check that all items we're trying to put back have the same number
282 croak "Items don't have the same numeral: ",
283 join(", ", map { $_->name()." => ".$_->number() } @items), "\n"
284 if (scalar keys %numbers > 1);
285 croak "Items don't have the same version: ",
286 join(", ", map { $_->name()." => ".$_->version() } @items), "\n"
287 if (scalar keys %versions > 1);
289 # Check that both items run with the same features
292 foreach (keys %features) {
293 delete $features{$_} if $features{$_} == 2;
295 croak "Features not in common between ",
296 $items[0]->name(), " and ", $items[1]->name(), ":",
297 join(", ", sort keys %features), "\n"
300 # Check that all platforms exist in both items, and have opposite values
301 my @platforms = ( { $items[0]->platforms() },
302 { $items[1]->platforms() } );
303 foreach my $platform (keys %{$platforms[0]}) {
304 if (exists $platforms[1]->{$platform}) {
305 if ($platforms[0]->{$platform} != !$platforms[1]->{$platform}) {
306 croak "Platforms aren't opposite: ",
308 map { my %tmp_h = $_->platforms();
309 $_->name().":".$platform
311 .$tmp_h{$platform} } @items),
315 # We're done with these
316 delete $platforms[0]->{$platform};
317 delete $platforms[1]->{$platform};
320 # If there are any remaining platforms, something's wrong
321 if (%{$platforms[0]} || %{$platforms[0]}) {
322 croak "There are platforms not in common between ",
323 $items[0]->name(), " and ", $items[1]->name(), "\n";
326 $self->{contents}->[$items[0]->number()] = [ @items ];
329 sub _parse_platforms {
336 my $op = !(defined $1 && $1 eq '!');
339 if ($def =~ m{^_?WIN32$}) { $platforms{$&} = $op; }
340 if ($def =~ m{^__FreeBSD__$}) { $platforms{$&} = $op; }
342 # if ($def =~ m{^__DragonFly__$}) { $platforms{$&} = $op; }
343 # if ($def =~ m{^__OpenBSD__$}) { $platforms{$&} = $op; }
344 # if ($def =~ m{^__NetBSD__$}) { $platforms{$&} = $op; }
346 m{^OPENSSL_(EXPORT_VAR_AS_FUNCTION)$}) { $platforms{$1} = $op; }
347 if ($def =~ m{^OPENSSL_SYS_}) { $platforms{$'} = $op; }
353 sub _parse_features {
360 my $op = !(defined $1 && $1 eq '!');
363 if ($def =~ m{^ZLIB$}) { $features{$&} = $op; }
364 if ($def =~ m{^OPENSSL_USE_}) { $features{$'} = $op; }
365 if ($def =~ m{^OPENSSL_NO_}) { $features{$'} = !$op; }
366 if ($def =~ m{^DEPRECATEDIN_(.*)$}) { $features{$&} = !$op; }
372 sub _adjust_version {
375 my $baseversion = $self->{baseversion};
377 $version = $baseversion
378 if ($baseversion ne '*' && $version ne '*'
379 && cmp_versions($baseversion, $version) > 0);
384 =item B<$ordinals-E<gt>add NAME, TYPE, LIST>
386 Adds a new item named NAME with the type TYPE, and a set of C macros in
387 LIST that are expected to be defined or undefined to use this symbol, if
388 any. For undefined macros, they each must be prefixed with a C<!>.
390 If this symbol already exists in loaded data, it will be rewritten using
391 the new input data, but will keep the same ordinal number and version.
392 If it's entirely new, it will get a new number and the current default
393 version. The new ordinal number is a simple increment from the last
401 my $type = shift; # FUNCTION or VARIABLE
402 my @defs = @_; # Macros from #ifdef and #ifndef
403 # (the latter prefixed with a '!')
405 # call signature for debug output
406 my $verbsig = "add('$name' , '$type' , [ " . join(', ', @defs) . " ])";
408 croak __PACKAGE__."->add got a bad type '$type'"
409 unless $type eq 'FUNCTION' || $type eq 'VARIABLE';
411 my %platforms = _parse_platforms(@defs);
412 my %features = _parse_features(@defs);
414 my @items = $self->items(filter => f_name($name));
415 my $version = @items ? $items[0]->version() : $self->{currversion};
416 my $number = @items ? $items[0]->number() : ++$self->{maxnum};
417 print STDERR "DEBUG[",__PACKAGE__,":add] $verbsig\n",
418 @items ? map { "\t".$_->to_string()."\n" } @items : "No previous items\n",
420 @items = grep { $_->exists() } @items;
423 OpenSSL::Ordinals::Item->new( name => $name,
427 $self->_adjust_version($version),
429 platforms => { %platforms },
431 grep { $features{$_} } keys %features
434 push @items, $new_item;
435 print STDERR "DEBUG[",__PACKAGE__,"::add] $verbsig\n", map { "\t".$_->to_string()."\n" } @items
437 $self->_putback(@items);
439 # If an alias was defined beforehand, add an item for it now
440 my $alias = $self->{aliases}->{$name};
441 delete $self->{aliases}->{$name};
443 # For the caller to show
444 my @returns = ( $new_item );
445 push @returns, $self->add_alias($alias->{name}, $name, @{$alias->{defs}})
450 =item B<$ordinals-E<gt>add_alias ALIAS, NAME, LIST>
452 Adds an alias ALIAS for the symbol NAME, and a set of C macros in LIST
453 that are expected to be defined or undefined to use this symbol, if any.
454 For undefined macros, they each must be prefixed with a C<!>.
456 If this symbol already exists in loaded data, it will be rewritten using
457 the new input data. Otherwise, the data will just be store away, to wait
458 that the symbol NAME shows up.
464 my $alias = shift; # This is the alias being added
465 my $name = shift; # For this name (assuming it exists)
466 my @defs = @_; # Platform attributes for the alias
468 # call signature for debug output
470 "add_alias('$alias' , '$name' , [ " . join(', ', @defs) . " ])";
472 croak "You're kidding me..." if $alias eq $name;
474 my %platforms = _parse_platforms(@defs);
475 my %features = _parse_features(@defs);
477 croak "Alias with associated features is forbidden\n"
480 my $f_byalias = f_name($alias);
481 my $f_byname = f_name($name);
482 my @items = $self->items(filter => $f_byalias);
483 foreach my $item ($self->items(filter => $f_byname)) {
484 push @items, $item unless grep { $_ == $item } @items;
486 @items = grep { $_->exists() } @items;
488 croak "Alias already exists ($alias => $name)"
489 if scalar @items > 1;
490 if (scalar @items == 0) {
491 # The item we want to alias for doesn't exist yet, so we cache the
492 # alias and hope the item we're making an alias of shows up later
493 $self->{aliases}->{$name} = { name => $alias, defs => [ @defs ] };
495 print STDERR "DEBUG[",__PACKAGE__,":add_alias] $verbsig\n",
496 "\tSet future alias $alias => $name\n"
499 } elsif (scalar @items == 1) {
500 # The rule is that an alias is more or less a copy of the original
501 # item, just with another name. Also, the platforms given here are
502 # given to the original item as well, with opposite values.
503 my %alias_platforms = $items[0]->platforms();
504 foreach (keys %platforms) {
505 $alias_platforms{$_} = !$platforms{$_};
507 # We supposedly do now know how to do this... *ahem*
508 $items[0]->{platforms} = { %alias_platforms };
510 my $alias_item = OpenSSL::Ordinals::Item->new(
512 type => $items[0]->type(),
513 number => $items[0]->number(),
514 version => $self->_adjust_version($items[0]->version()),
515 exists => $items[0]->exists(),
516 platforms => { %platforms },
517 features => [ $items[0]->features() ]
519 push @items, $alias_item;
521 print STDERR "DEBUG[",__PACKAGE__,":add_alias] $verbsig\n",
522 map { "\t".$_->to_string()."\n" } @items
524 $self->_putback(@items);
526 # For the caller to show
527 return ( $alias_item->to_string() );
529 croak "$name has an alias already (trying to add alias $alias)\n",
530 "\t", join(", ", map { $_->name() } @items), "\n";
533 =item B<$ordinals-E<gt>set_version VERSION>
535 =item B<$ordinals-E<gt>set_version VERSION BASEVERSION>
537 Sets the default version for new symbol to VERSION.
539 If given, BASEVERSION sets the base version, i.e. the minimum version
540 for all symbols. If not given, it will be calculated as follows:
544 If the given version is '*', then the base version will also be '*'.
546 If the given version starts with '0.', the base version will be '0.0.0'.
548 If the given version starts with '1.0.', the base version will be '1.0.0'.
550 If the given version starts with '1.1.', the base version will be '1.1.0'.
552 If the given version has a first number C<N> that's greater than 1, the
553 base version will be formed from C<N>: 'N.0.0'.
561 # '*' is for "we don't care"
562 my $version = shift // '*';
563 my $baseversion = shift // '*';
565 $version =~ s|-.*||g;
567 if ($baseversion eq '*') {
568 $baseversion = $version;
569 if ($baseversion ne '*') {
570 if ($baseversion =~ m|^(\d+)\.|, $1 > 1) {
571 $baseversion = "$1.0.0";
573 $baseversion =~ s|^0\..*$|0.0.0|;
574 $baseversion =~ s|^1\.0\..*$|1.0.0|;
575 $baseversion =~ s|^1\.1\..*$|1.1.0|;
577 die 'Invalid version'
578 if ($baseversion ne '0.0.0'
579 && $baseversion !~ m|^1\.[01]\.0$|);
584 die 'Invalid base version'
585 if ($baseversion ne '*' && $version ne '*'
586 && cmp_versions($baseversion, $version) > 0);
588 $self->{currversion} = $version;
589 $self->{baseversion} = $baseversion;
590 foreach ($self->items(filter => sub { $_[0] eq '*' })) {
591 $_->{version} = $self->{currversion};
596 =item B<$ordinals-E<gt>invalidate>
598 Invalidates the whole working database. The practical effect is that all
599 symbols are set to not exist, but are kept around in the database to retain
600 ordinal numbers and versions.
607 foreach (@{$self->{contents}}) {
608 foreach (@{$_ // []}) {
615 =item B<$ordinals-E<gt>validate>
617 Validates the current working database by collection statistics on how many
618 symbols were added and how many were changed. These numbers can be retrieved
619 with B<$ordinals-E<gt>stats>.
627 for my $i (1..$self->{maxnum}) {
628 if ($i > $self->{loaded_maxnum}
629 || (!@{$self->{loaded_contents}->[$i] // []}
630 && @{$self->{contents}->[$i] // []})) {
631 $self->{stats}->{new}++;
633 next if ($i > $self->{loaded_maxnum});
636 map { $_->to_string() } @{$self->{loaded_contents}->[$i] // []};
637 my @current_strings =
638 map { $_->to_string() } @{$self->{contents}->[$i] // []};
640 foreach my $str (@current_strings) {
641 @loaded_strings = grep { $str ne $_ } @loaded_strings;
643 if (@loaded_strings) {
644 $self->{stats}->{modified}++;
649 =item B<$ordinals-E<gt>stats>
651 Returns the statistics that B<validate> calculate.
658 return %{$self->{stats}};
665 Data elements, which is each line in an ordinals file, are instances
666 of a separate class, OpenSSL::Ordinals::Item, with its own methods:
672 package OpenSSL::Ordinals::Item;
678 =item B<new> I<%options>
680 Creates a new instance of the C<OpenSSL::Ordinals::Item> class. It takes
681 options in keyed pair form, i.e. a series of C<key =E<gt> value> pairs.
682 Available options are:
686 =item B<from =E<gt> STRING>
688 This will create a new item, filled with data coming from STRING.
690 STRING must conform to the following EBNF description:
692 ordinal string = symbol, spaces, ordinal, spaces, version, spaces,
693 exist, ":", platforms, ":", type, ":", features;
694 spaces = space, { space };
696 symbol = ( letter | "_"), { letter | digit | "_" };
698 version = number, "_", number, "_", number, [ letter, [ letter ] ];
699 exist = "EXIST" | "NOEXIST";
700 platforms = platform, { ",", platform };
701 platform = ( letter | "_" ) { letter | digit | "_" };
702 type = "FUNCTION" | "VARIABLE";
703 features = feature, { ",", feature };
704 feature = ( letter | "_" ) { letter | digit | "_" };
705 number = digit, { digit };
707 (C<letter> and C<digit> are assumed self evident)
709 =item B<name =E<gt> STRING>, B<number =E<gt> NUMBER>, B<version =E<gt> STRING>,
710 B<exists =E<gt> BOOLEAN>, B<type =E<gt> STRING>,
711 B<platforms =E<gt> HASHref>, B<features =E<gt> LISTref>
713 This will create a new item with data coming from the arguments.
722 if (ref($_[0]) eq $class) {
723 return $class->new( map { $_ => $_[0]->{$_} } keys %{$_[0]} );
728 croak "No argument given" unless %opts;
730 my $instance = undef;
732 my @a = split /\s+/, $opts{from};
734 croak "Badly formatted ordinals string: $opts{from}"
735 unless ( scalar @a == 4
736 && $a[0] =~ /^[A-Za-z_][A-Za-z_0-9]*$/
738 && $a[2] =~ /^(?:\*|\d+_\d+_\d+[a-z]{0,2})$/
742 (?:FUNCTION|VARIABLE):
747 my @b = split /:/, $a[3];
748 %opts = ( name => $a[0],
751 exists => $b[0] eq 'EXIST',
752 platforms => { map { m|^(!)?|; $' => !$1 }
755 features => [ split /,/,$b[3] // '' ] );
758 if ($opts{name} && $opts{version} && defined $opts{exists} && $opts{type}
759 && ref($opts{platforms} // {}) eq 'HASH'
760 && ref($opts{features} // []) eq 'ARRAY') {
761 my $version = $opts{version};
764 $instance = { name => $opts{name},
766 number => $opts{number},
768 exists => !!$opts{exists},
769 platforms => { %{$opts{platforms} // {}} },
770 features => [ sort @{$opts{features} // []} ] };
772 croak __PACKAGE__."->new() called with bad arguments\n".
773 join("", map { " $_\t=> ".$opts{$_}."\n" } sort keys %opts);
776 return bless $instance, $class;
782 =item B<$item-E<gt>name>
784 The symbol name for this item.
786 =item B<$item-E<gt>number>
788 The positional number for this item.
790 =item B<$item-E<gt>version>
792 The version number for this item. Please note that these version numbers
793 have underscore (C<_>) as a separator the the version parts.
795 =item B<$item-E<gt>exists>
797 A boolean that tells if this symbol exists in code or not.
799 =item B<$item-E<gt>platforms>
801 A hash table reference. The keys of the hash table are the names of
802 the specified platforms, with a value of 0 to indicate that this symbol
803 isn't available on that platform, and 1 to indicate that it is. Platforms
804 that aren't mentioned default to 1.
806 =item B<$item-E<gt>type>
808 C<FUNCTION> or C<VARIABLE>, depending on what the symbol represents.
809 Some platforms do not care about this, others do.
811 =item B<$item-E<gt>features>
813 An array reference, where every item indicates a feature where this symbol
814 is available. If no features are mentioned, the symbol is always available.
815 If any feature is mentioned, this symbol is I<only> available when those
816 features are enabled.
825 my $funcname = $AUTOLOAD;
826 (my $item = $funcname) =~ s|.*::||g;
828 croak "$funcname called as setter" if @_;
829 croak "$funcname invalid" unless exists $self->{$item};
830 return $self->{$item} if ref($self->{$item}) eq '';
831 return @{$self->{$item}} if ref($self->{$item}) eq 'ARRAY';
832 return %{$self->{$item}} if ref($self->{$item}) eq 'HASH';
835 =item B<$item-E<gt>to_string>
837 Converts the item to a string that can be saved in an ordinals file.
844 croak "Too many arguments" if @_;
845 my %platforms = $self->platforms();
846 my @features = $self->features();
847 my $version = $self->version();
848 $version =~ s|\.|_|g;
849 return sprintf "%-39s %d\t%s\t%s:%s:%s:%s",
853 $self->exists() ? 'EXIST' : 'NOEXIST',
854 join(',', (map { ($platforms{$_} ? '' : '!') . $_ }
855 sort keys %platforms)),
857 join(',', @features);
862 =head2 Comparators and filters
864 For the B<$ordinals-E<gt>items> method, there are a few functions to create
865 comparators based on specific data:
871 # Go back to the main package to create comparators and filters
872 package OpenSSL::Ordinals;
878 Returns a comparator that will compare the names of two OpenSSL::Ordinals::Item
884 return sub { $_[0]->name() cmp $_[1]->name() };
889 Returns a comparator that will compare the ordinal numbers of two
890 OpenSSL::Ordinals::Item objects.
895 return sub { $_[0]->number() <=> $_[1]->number() };
900 Returns a comparator that will compare the version of two
901 OpenSSL::Ordinals::Item objects.
907 # cmp_versions comes from OpenSSL::Util
908 return cmp_versions($_[0]->version(), $_[1]->version());
914 There are also the following filters:
920 # Filters... these are called by grep, the return sub must use $_ for
923 =item B<f_version VERSION>
925 Returns a filter that only lets through symbols with a version number
933 croak "No version specified"
934 unless $version && $version =~ /^\d+\.\d+\.\d+[a-z]{0,2}$/;
936 return sub { $_[0]->version() eq $version };
939 =item B<f_number NUMBER>
941 Returns a filter that only lets through symbols with the ordinal number
944 NOTE that this returns a "magic" value that can not be used as a function.
945 It's only useful when passed directly as a filter to B<items>.
952 croak "No number specified"
953 unless $number && $number =~ /^\d+$/;
955 return [ F_NUMBER, $number ];
961 Returns a filter that only lets through symbols with the symbol name
964 NOTE that this returns a "magic" value that can not be used as a function.
965 It's only useful when passed directly as a filter to B<items>.
972 croak "No name specified"
975 return [ F_NAME, $name ];
982 Richard Levitte E<lt>levitte@openssl.orgE<gt>.