2 # Copyright 2018 The OpenSSL Project Authors. All Rights Reserved.
4 # Licensed under the OpenSSL license (the "License"). You may not use
5 # this file except in compliance with the License. You can obtain a copy
6 # in the file LICENSE in the source distribution or at
7 # https://www.openssl.org/source/license.html
9 package OpenSSL::Ordinals;
14 use Scalar::Util qw(blessed);
17 # "magic" filters, see the filters at the end of the file
24 OpenSSL::Ordinals - a private module to read and walk through ordinals
28 use OpenSSL::Ordinals;
30 my $ordinals = OpenSSL::Ordinals->new(from => "foo.num");
32 my $ordinals = OpenSSL::Ordinals->new();
33 $ordinals->load("foo.num");
35 foreach ($ordinals->items(comparator => by_name()) {
36 print $_->name(), "\n";
41 This is a OpenSSL private module to load an ordinals (F<.num>) file and
42 write out the data you want, sorted and filtered according to your rules.
44 An ordinals file is a file that enumerates all the symbols that a shared
45 library or loadable module must export. Each of them have a unique
46 assigned number as well as other attributes to indicate if they only exist
47 on a subset of the supported platforms, or if they are specific to certain
50 The unique numbers each symbol gets assigned needs to be maintained for a
51 shared library or module to stay compatible with previous versions on
52 platforms that maintain a transfer vector indexed by position rather than
53 by name. They also help keep information on certain symbols that are
54 aliases for others for certain platforms, or that have different forms
55 on different platforms.
63 =item B<new> I<%options>
65 Creates a new instance of the C<OpenSSL::Ordinals> class. It takes options
66 in keyed pair form, i.e. a series of C<key =E<gt> value> pairs. Available
71 =item B<from =E<gt> FILENAME>
73 Not only create a new instance, but immediately load it with data from the
74 ordinals file FILENAME.
85 filename => undef, # File name registered when loading
86 loaded_maxnum => 0, # Highest allocated item number when loading
87 loaded_contents => [], # Loaded items, if loading there was
88 maxnum => 0, # Current highest allocated item number
89 contents => [], # Items, indexed by number
90 name2num => {}, # Name to number dictionary
91 aliases => {}, # Aliases cache.
92 stats => {}, # Statistics, see 'sub validate'
93 currversion => $opts{version} // '*', # '*' is for "we don't care"
94 debug => $opts{debug},
96 bless $instance, $class;
98 $instance->load($opts{from}) if defined($opts{from});
103 =item B<$ordinals-E<gt>load FILENAME>
105 Loads the data from FILENAME into the instance. Any previously loaded data
108 Two internal databases are created. One database is simply a copy of the file
109 contents and is treated as read-only. The other database is an exact copy of
110 the first, but is treated as a work database, i.e. it can be modified and added
117 my $filename = shift;
119 croak "Undefined filename" unless defined($filename);
121 my @tmp_contents = ();
122 my %tmp_name2num = ();
124 open F, '<', $filename or croak "Unable to open $filename";
126 s|\R$||; # Better chomp
130 my $item = OpenSSL::Ordinals::Item->new(from => $_);
132 my $num = $item->number();
133 croak "Disordered ordinals, $num < $max_num"
137 push @{$tmp_contents[$item->number()]}, $item;
138 $tmp_name2num{$item->name()} = $item->number();
142 $self->{contents} = [ @tmp_contents ];
143 $self->{name2num} = { %tmp_name2num };
144 $self->{maxnum} = $max_num;
145 $self->{filename} = $filename;
147 # Make a deep copy, allowing {contents} to be an independent work array
148 foreach my $i (1..$max_num) {
149 if ($tmp_contents[$i]) {
150 $self->{loaded_contents}->[$i] =
151 [ map { OpenSSL::Ordinals::Item->new($_) }
152 @{$tmp_contents[$i]} ];
155 $self->{loaded_maxnum} = $max_num;
159 =item B<$ordinals-E<gt>rewrite>
161 If an ordinals file has been loaded, it gets rewritten with the data from
162 the current work database.
169 $self->write($self->{filename});
172 =item B<$ordinals-E<gt>write FILENAME>
174 Writes the current work database data to the ordinals file FILENAME.
175 This also validates the data, see B<$ordinals-E<gt>validate> below.
181 my $filename = shift;
183 croak "Undefined filename" unless defined($filename);
187 open F, '>', $filename or croak "Unable to open $filename";
188 foreach ($self->items(by => by_number())) {
189 print F $_->to_string(),"\n";
192 $self->{filename} = $filename;
193 $self->{loaded_maxnum} = $self->{maxnum};
197 =item B<$ordinals-E<gt>items> I<%options>
199 Returns a list of items according to a set of criteria. The criteria is
200 given in form keyed pair form, i.e. a series of C<key =E<gt> value> pairs.
201 Available options are:
205 =item B<sort =E<gt> SORTFUNCTION>
207 SORTFUNCTION is a reference to a function that takes two arguments, which
208 correspond to the classic C<$a> and C<$b> that are available in a C<sort>
211 =item B<filter =E<gt> FILTERFUNCTION>
213 FILTERFUNTION is a reference to a function that takes one argument, which
214 is every OpenSSL::Ordinals::Item element available.
224 my $comparator = $opts{sort};
225 my $filter = $opts{filter} // sub { 1; };
228 if (ref($filter) eq 'ARRAY') {
229 # run a "magic" filter
230 if ($filter->[0] == F_NUMBER) {
231 my $index = $filter->[1];
232 @l = $index ? @{$self->{contents}->[$index] // []} : ();
233 } elsif ($filter->[0] == F_NAME) {
234 my $index = $self->{name2num}->{$filter->[1]};
235 @l = $index ? @{$self->{contents}->[$index] // []} : ();
237 croak __PACKAGE__."->items called with invalid filter";
239 } elsif (ref($filter) eq 'CODE') {
240 @l = grep { $filter->($_) }
242 @{$self->{contents}};
244 croak __PACKAGE__."->items called with invalid filter";
247 return sort { $comparator->($a, $b); } @l
248 if (defined $comparator);
252 # Put an array of items back into the object after having checked consistency
253 # If there are exactly two items:
254 # - They MUST have the same number
255 # - For platforms, both MUST hold the same ones, but with opposite values
256 # - For features, both MUST hold the same ones.
257 # If there's just one item, just put it in the slot of its number
258 # In all other cases, something is wrong
263 if (scalar @items < 1 || scalar @items > 2) {
264 croak "Wrong number of items: ", scalar @items, " : ",
265 join(", ", map { $_->name() } @items), "\n";
267 if (scalar @items == 2) {
273 $numbers{$_->number()} = 1;
274 $versions{$_->version()} = 1;
275 foreach ($_->features()) {
280 # Check that all items we're trying to put back have the same number
281 croak "Items don't have the same numeral: ",
282 join(", ", map { $_->name()." => ".$_->number() } @items), "\n"
283 if (scalar keys %numbers > 1);
284 croak "Items don't have the same version: ",
285 join(", ", map { $_->name()." => ".$_->version() } @items), "\n"
286 if (scalar keys %versions > 1);
288 # Check that both items run with the same features
291 foreach (keys %features) {
292 delete $features{$_} if $features{$_} == 2;
294 croak "Features not in common between ",
295 $items[0]->name(), " and ", $items[1]->name(), ":",
296 join(", ", sort keys %features), "\n"
299 # Check that all platforms exist in both items, and have opposite values
300 my @platforms = ( { $items[0]->platforms() },
301 { $items[1]->platforms() } );
302 foreach my $platform (keys %{$platforms[0]}) {
303 if (exists $platforms[1]->{$platform}) {
304 if ($platforms[0]->{$platform} != !$platforms[1]->{$platform}) {
305 croak "Platforms aren't opposite: ",
307 map { my %tmp_h = $_->platforms();
308 $_->name().":".$platform
310 .$tmp_h{$platform} } @items),
314 # We're done with these
315 delete $platforms[0]->{$platform};
316 delete $platforms[1]->{$platform};
319 # If there are any remaining platforms, something's wrong
320 if (%{$platforms[0]} || %{$platforms[0]}) {
321 croak "There are platforms not in common between ",
322 $items[0]->name(), " and ", $items[1]->name(), "\n";
325 $self->{contents}->[$items[0]->number()] = [ @items ];
328 sub _parse_platforms {
335 my $op = !(defined $1 && $1 eq '!');
338 if ($def =~ m{^_?WIN32$}) { $platforms{$&} = $op; }
339 if ($def =~ m{^__FreeBSD__$}) { $platforms{$&} = $op; }
341 # if ($def =~ m{^__DragonFly__$}) { $platforms{$&} = $op; }
342 # if ($def =~ m{^__OpenBSD__$}) { $platforms{$&} = $op; }
343 # if ($def =~ m{^__NetBSD__$}) { $platforms{$&} = $op; }
345 m{^OPENSSL_(EXPORT_VAR_AS_FUNCTION)$}) { $platforms{$1} = $op; }
346 if ($def =~ m{^OPENSSL_SYS_}) { $platforms{$'} = $op; }
352 sub _parse_features {
359 my $op = !(defined $1 && $1 eq '!');
362 if ($def =~ m{^ZLIB$}) { $features{$&} = $op; }
363 if ($def =~ m{^OPENSSL_USE_}) { $features{$'} = $op; }
364 if ($def =~ m{^OPENSSL_NO_}) { $features{$'} = !$op; }
365 if ($def =~ m{^DEPRECATEDIN_(.*)$}) { $features{$&} = !$op; }
371 =item B<$ordinals-E<gt>add NAME, TYPE, LIST>
373 Adds a new item named NAME with the type TYPE, and a set of C macros in
374 LIST that are expected to be defined or undefined to use this symbol, if
375 any. For undefined macros, they each must be prefixed with a C<!>.
377 If this symbol already exists in loaded data, it will be rewritten using
378 the new input data, but will keep the same ordinal number and version.
379 If it's entirely new, it will get a new number and the current default
380 version. The new ordinal number is a simple increment from the last
388 my $type = shift; # FUNCTION or VARIABLE
389 my @defs = @_; # Macros from #ifdef and #ifndef
390 # (the latter prefixed with a '!')
392 # call signature for debug output
393 my $verbsig = "add('$name' , '$type' , [ " . join(', ', @defs) . " ])";
395 croak __PACKAGE__."->add got a bad type '$type'"
396 unless $type eq 'FUNCTION' || $type eq 'VARIABLE';
398 my %platforms = _parse_platforms(@defs);
399 my %features = _parse_features(@defs);
401 my @items = $self->items(filter => f_name($name));
402 my $version = @items ? $items[0]->version() : $self->{currversion};
403 my $number = @items ? $items[0]->number() : ++$self->{maxnum};
404 print STDERR "DEBUG[",__PACKAGE__,":add] $verbsig\n",
405 @items ? map { "\t".$_->to_string()."\n" } @items : "No previous items\n",
407 @items = grep { $_->exists() } @items;
410 OpenSSL::Ordinals::Item->new( name => $name,
415 platforms => { %platforms },
417 grep { $features{$_} } keys %features
420 push @items, $new_item;
421 print STDERR "DEBUG[",__PACKAGE__,"::add] $verbsig\n", map { "\t".$_->to_string()."\n" } @items
423 $self->_putback(@items);
425 # If an alias was defined beforehand, add an item for it now
426 my $alias = $self->{aliases}->{$name};
427 delete $self->{aliases}->{$name};
429 # For the caller to show
430 my @returns = ( $new_item );
431 push @returns, $self->add_alias($alias->{name}, $name, @{$alias->{defs}})
436 =item B<$ordinals-E<gt>add_alias ALIAS, NAME, LIST>
438 Adds an alias ALIAS for the symbol NAME, and a set of C macros in LIST
439 that are expected to be defined or undefined to use this symbol, if any.
440 For undefined macros, they each must be prefixed with a C<!>.
442 If this symbol already exists in loaded data, it will be rewritten using
443 the new input data. Otherwise, the data will just be store away, to wait
444 that the symbol NAME shows up.
450 my $alias = shift; # This is the alias being added
451 my $name = shift; # For this name (assuming it exists)
452 my @defs = @_; # Platform attributes for the alias
454 # call signature for debug output
456 "add_alias('$alias' , '$name' , [ " . join(', ', @defs) . " ])";
458 croak "You're kidding me..." if $alias eq $name;
460 my %platforms = _parse_platforms(@defs);
461 my %features = _parse_features(@defs);
463 croak "Alias with associated features is forbidden\n"
466 my $f_byalias = f_name($alias);
467 my $f_byname = f_name($name);
468 my @items = $self->items(filter => $f_byalias);
469 foreach my $item ($self->items(filter => $f_byname)) {
470 push @items, $item unless grep { $_ == $item } @items;
472 @items = grep { $_->exists() } @items;
474 croak "Alias already exists ($alias => $name)"
475 if scalar @items > 1;
476 if (scalar @items == 0) {
477 # The item we want to alias for doesn't exist yet, so we cache the
478 # alias and hope the item we're making an alias of shows up later
479 $self->{aliases}->{$name} = { name => $alias, defs => [ @defs ] };
481 print STDERR "DEBUG[",__PACKAGE__,":add_alias] $verbsig\n",
482 "\tSet future alias $alias => $name\n"
485 } elsif (scalar @items == 1) {
486 # The rule is that an alias is more or less a copy of the original
487 # item, just with another name. Also, the platforms given here are
488 # given to the original item as well, with opposite values.
489 my %alias_platforms = $items[0]->platforms();
490 foreach (keys %platforms) {
491 $alias_platforms{$_} = !$platforms{$_};
493 # We supposedly do now know how to do this... *ahem*
494 $items[0]->{platforms} = { %alias_platforms };
496 my $alias_item = OpenSSL::Ordinals::Item->new(
498 type => $items[0]->type(),
499 number => $items[0]->number(),
500 version => $items[0]->version(),
501 exists => $items[0]->exists(),
502 platforms => { %platforms },
503 features => [ $items[0]->features() ]
505 push @items, $alias_item;
507 print STDERR "DEBUG[",__PACKAGE__,":add_alias] $verbsig\n",
508 map { "\t".$_->to_string()."\n" } @items
510 $self->_putback(@items);
512 # For the caller to show
513 return ( $alias_item->to_string() );
515 croak "$name has an alias already (trying to add alias $alias)\n",
516 "\t", join(", ", map { $_->name() } @items), "\n";
519 =item B<$ordinals-E<gt>set_version VERSION>
521 Sets the default version for new symbol to VERSION.
530 $version =~ s|-.*||g;
531 $version =~ s|\.|_|g;
532 $self->{currversion} = $version;
533 foreach ($self->items(filter => sub { $_[0] eq '*' })) {
534 $_->{version} = $self->{currversion};
539 =item B<$ordinals-E<gt>invalidate>
541 Invalidates the whole working database. The practical effect is that all
542 symbols are set to not exist, but are kept around in the database to retain
543 ordinal numbers and versions.
550 foreach (@{$self->{contents}}) {
551 foreach (@{$_ // []}) {
558 =item B<$ordinals-E<gt>validate>
560 Validates the current working database by collection statistics on how many
561 symbols were added and how many were changed. These numbers can be retrieved
562 with B<$ordinals-E<gt>stats>.
570 for my $i (1..$self->{maxnum}) {
571 if ($i > $self->{loaded_maxnum}
572 || (!@{$self->{loaded_contents}->[$i] // []}
573 && @{$self->{contents}->[$i] // []})) {
574 $self->{stats}->{new}++;
576 next if ($i > $self->{loaded_maxnum});
579 map { $_->to_string() } @{$self->{loaded_contents}->[$i] // []};
580 my @current_strings =
581 map { $_->to_string() } @{$self->{contents}->[$i] // []};
583 foreach my $str (@current_strings) {
584 @loaded_strings = grep { $str ne $_ } @loaded_strings;
586 if (@loaded_strings) {
587 $self->{stats}->{modified}++;
592 =item B<$ordinals-E<gt>stats>
594 Returns the statistics that B<validate> calculate.
601 return %{$self->{stats}};
608 Data elements, which is each line in an ordinals file, are instances
609 of a separate class, OpenSSL::Ordinals::Item, with its own methods:
615 package OpenSSL::Ordinals::Item;
621 =item B<new> I<%options>
623 Creates a new instance of the C<OpenSSL::Ordinals::Item> class. It takes
624 options in keyed pair form, i.e. a series of C<key =E<gt> value> pairs.
625 Available options are:
629 =item B<from =E<gt> STRING>
631 This will create a new item, filled with data coming from STRING.
633 STRING must conform to the following EBNF description:
635 ordinal string = symbol, spaces, ordinal, spaces, version, spaces,
636 exist, ":", platforms, ":", type, ":", features;
637 spaces = space, { space };
639 symbol = ( letter | "_"), { letter | digit | "_" };
641 version = number, "_", number, "_", number, letter, [ letter ];
642 exist = "EXIST" | "NOEXIST";
643 platforms = platform, { ",", platform };
644 platform = ( letter | "_" ) { letter | digit | "_" };
645 type = "FUNCTION" | "VARIABLE";
646 features = feature, { ",", feature };
647 feature = ( letter | "_" ) { letter | digit | "_" };
648 number = digit, { digit };
650 (C<letter> and C<digit> are assumed self evident)
652 =item B<name =E<gt> STRING>, B<number =E<gt> NUMBER>, B<version =E<gt> STRING>,
653 B<exists =E<gt> BOOLEAN>, B<type =E<gt> STRING>,
654 B<platforms =E<gt> HASHref>, B<features =E<gt> LISTref>
656 This will create a new item with data coming from the arguments.
665 if (ref($_[0]) eq $class) {
666 return $class->new( map { $_ => $_[0]->{$_} } keys %{$_[0]} );
671 croak "No argument given" unless %opts;
673 my $instance = undef;
675 my @a = split /\s+/, $opts{from};
677 croak "Badly formatted ordinals string: $opts{from}"
678 unless ( scalar @a == 4
679 && $a[0] =~ /^[A-Za-z_][A-Za-z_0-9]*$/
681 && $a[2] =~ /^(?:\*|\d+_\d+_\d+(?:[a-z]{0,2}))$/
685 (?:FUNCTION|VARIABLE):
690 my @b = split /:/, $a[3];
691 %opts = ( name => $a[0],
694 exists => $b[0] eq 'EXIST',
695 platforms => { map { m|^(!)?|; $' => !$1 }
698 features => [ split /,/,$b[3] // '' ] );
701 if ($opts{name} && $opts{version} && defined $opts{exists} && $opts{type}
702 && ref($opts{platforms} // {}) eq 'HASH'
703 && ref($opts{features} // []) eq 'ARRAY') {
704 $instance = { name => $opts{name},
706 number => $opts{number},
707 version => $opts{version},
708 exists => !!$opts{exists},
709 platforms => { %{$opts{platforms} // {}} },
710 features => [ sort @{$opts{features} // []} ] };
712 croak __PACKAGE__."->new() called with bad arguments\n".
713 join("", map { " $_\t=> ".$opts{$_}."\n" } sort keys %opts);
716 return bless $instance, $class;
722 =item B<$item-E<gt>name>
724 The symbol name for this item.
726 =item B<$item-E<gt>number>
728 The positional number for this item.
730 =item B<$item-E<gt>version>
732 The version number for this item. Please note that these version numbers
733 have underscore (C<_>) as a separator the the version parts.
735 =item B<$item-E<gt>exists>
737 A boolean that tells if this symbol exists in code or not.
739 =item B<$item-E<gt>platforms>
741 A hash table reference. The keys of the hash table are the names of
742 the specified platforms, with a value of 0 to indicate that this symbol
743 isn't available on that platform, and 1 to indicate that it is. Platforms
744 that aren't mentioned default to 1.
746 =item B<$item-E<gt>type>
748 C<FUNCTION> or C<VARIABLE>, depending on what the symbol represents.
749 Some platforms do not care about this, others do.
751 =item B<$item-E<gt>features>
753 An array reference, where every item indicates a feature where this symbol
754 is available. If no features are mentioned, the symbol is always available.
755 If any feature is mentioned, this symbol is I<only> available when those
756 features are enabled.
765 my $funcname = $AUTOLOAD;
766 (my $item = $funcname) =~ s|.*::||g;
768 croak "$funcname called as setter" if @_;
769 croak "$funcname invalid" unless exists $self->{$item};
770 return $self->{$item} if ref($self->{$item}) eq '';
771 return @{$self->{$item}} if ref($self->{$item}) eq 'ARRAY';
772 return %{$self->{$item}} if ref($self->{$item}) eq 'HASH';
775 =item B<$item-E<gt>to_string>
777 Converts the item to a string that can be saved in an ordinals file.
784 croak "Too many arguments" if @_;
785 my %platforms = $self->platforms();
786 my @features = $self->features();
787 return sprintf "%-39s %d\t%s\t%s:%s:%s:%s",
791 $self->exists() ? 'EXIST' : 'NOEXIST',
792 join(',', (map { ($platforms{$_} ? '' : '!') . $_ }
793 sort keys %platforms)),
795 join(',', @features);
800 =head2 Comparators and filters
802 For the B<$ordinals-E<gt>items> method, there are a few functions to create
803 comparators based on specific data:
809 # Go back to the main package to create comparators and filters
810 package OpenSSL::Ordinals;
816 Returns a comparator that will compare the names of two OpenSSL::Ordinals::Item
822 return sub { $_[0]->name() cmp $_[1]->name() };
827 Returns a comparator that will compare the ordinal numbers of two
828 OpenSSL::Ordinals::Item objects.
833 return sub { $_[0]->number() <=> $_[1]->number() };
838 Returns a comparator that will compare the version of two
839 OpenSSL::Ordinals::Item objects.
844 sub _ossl_versionsplit {
845 my $textversion = shift;
846 return $textversion if $textversion eq '*';
847 my ($major,$minor,$edit,$patch) =
848 $textversion =~ /^(\d+)_(\d+)_(\d+)([a-z]{0,2})$/;
849 return ($major,$minor,$edit,$patch);
853 my @a_split = _ossl_versionsplit($_[0]->version());
854 my @b_split = _ossl_versionsplit($_[1]->version());
857 # The last part is a letter sequence (or a '*')
858 if (scalar @a_split == 1) {
859 $verdict = $a_split[0] cmp $b_split[0];
861 $verdict = $a_split[0] <=> $b_split[0];
865 last unless $verdict == 0;
873 There are also the following filters:
879 # Filters... these are called by grep, the return sub must use $_ for
882 =item B<f_version VERSION>
884 Returns a filter that only lets through symbols with a version number
892 $version =~ s|\.|_|g if $version;
893 croak "No version specified"
894 unless $version && $version =~ /^\d_\d_\d[a-z]{0,2}$/;
896 return sub { $_[0]->version() eq $version };
899 =item B<f_number NUMBER>
901 Returns a filter that only lets through symbols with the ordinal number
904 NOTE that this returns a "magic" value that can not be used as a function.
905 It's only useful when passed directly as a filter to B<items>.
912 croak "No number specified"
913 unless $number && $number =~ /^\d+$/;
915 return [ F_NUMBER, $number ];
921 Returns a filter that only lets through symbols with the symbol name
924 NOTE that this returns a "magic" value that can not be used as a function.
925 It's only useful when passed directly as a filter to B<items>.
932 croak "No name specified"
935 return [ F_NAME, $name ];
942 Richard Levitte E<lt>levitte@openssl.orgE<gt>.