2 # Copyright 2018 The OpenSSL Project Authors. All Rights Reserved.
4 # Licensed under the Apache License 2.0 (the "License"). You may not use
5 # this file except in compliance with the License. You can obtain a copy
6 # in the file LICENSE in the source distribution or at
7 # https://www.openssl.org/source/license.html
9 package OpenSSL::Ordinals;
14 use Scalar::Util qw(blessed);
17 # "magic" filters, see the filters at the end of the file
24 OpenSSL::Ordinals - a private module to read and walk through ordinals
28 use OpenSSL::Ordinals;
30 my $ordinals = OpenSSL::Ordinals->new(from => "foo.num");
32 my $ordinals = OpenSSL::Ordinals->new();
33 $ordinals->load("foo.num");
35 foreach ($ordinals->items(comparator => by_name()) {
36 print $_->name(), "\n";
41 This is a OpenSSL private module to load an ordinals (F<.num>) file and
42 write out the data you want, sorted and filtered according to your rules.
44 An ordinals file is a file that enumerates all the symbols that a shared
45 library or loadable module must export. Each of them have a unique
46 assigned number as well as other attributes to indicate if they only exist
47 on a subset of the supported platforms, or if they are specific to certain
50 The unique numbers each symbol gets assigned needs to be maintained for a
51 shared library or module to stay compatible with previous versions on
52 platforms that maintain a transfer vector indexed by position rather than
53 by name. They also help keep information on certain symbols that are
54 aliases for others for certain platforms, or that have different forms
55 on different platforms.
63 =item B<new> I<%options>
65 Creates a new instance of the C<OpenSSL::Ordinals> class. It takes options
66 in keyed pair form, i.e. a series of C<key =E<gt> value> pairs. Available
71 =item B<from =E<gt> FILENAME>
73 Not only create a new instance, but immediately load it with data from the
74 ordinals file FILENAME.
85 filename => undef, # File name registered when loading
86 loaded_maxnum => 0, # Highest allocated item number when loading
87 loaded_contents => [], # Loaded items, if loading there was
88 maxnum => 0, # Current highest allocated item number
89 contents => [], # Items, indexed by number
90 name2num => {}, # Name to number dictionary
91 aliases => {}, # Aliases cache.
92 stats => {}, # Statistics, see 'sub validate'
93 currversion => $opts{version} // '*', # '*' is for "we don't care"
94 debug => $opts{debug},
96 bless $instance, $class;
98 $instance->load($opts{from}) if defined($opts{from});
103 =item B<$ordinals-E<gt>load FILENAME>
105 Loads the data from FILENAME into the instance. Any previously loaded data
108 Two internal databases are created. One database is simply a copy of the file
109 contents and is treated as read-only. The other database is an exact copy of
110 the first, but is treated as a work database, i.e. it can be modified and added
117 my $filename = shift;
119 croak "Undefined filename" unless defined($filename);
121 my @tmp_contents = ();
122 my %tmp_name2num = ();
124 open F, '<', $filename or croak "Unable to open $filename";
126 s|\R$||; # Better chomp
130 my $item = OpenSSL::Ordinals::Item->new(from => $_);
132 my $num = $item->number();
133 croak "Disordered ordinals, $num < $max_num"
137 push @{$tmp_contents[$item->number()]}, $item;
138 $tmp_name2num{$item->name()} = $item->number();
142 $self->{contents} = [ @tmp_contents ];
143 $self->{name2num} = { %tmp_name2num };
144 $self->{maxnum} = $max_num;
145 $self->{filename} = $filename;
147 # Make a deep copy, allowing {contents} to be an independent work array
148 foreach my $i (1..$max_num) {
149 if ($tmp_contents[$i]) {
150 $self->{loaded_contents}->[$i] =
151 [ map { OpenSSL::Ordinals::Item->new($_) }
152 @{$tmp_contents[$i]} ];
155 $self->{loaded_maxnum} = $max_num;
159 =item B<$ordinals-E<gt>rewrite>
161 If an ordinals file has been loaded, it gets rewritten with the data from
162 the current work database.
169 $self->write($self->{filename});
172 =item B<$ordinals-E<gt>write FILENAME>
174 Writes the current work database data to the ordinals file FILENAME.
175 This also validates the data, see B<$ordinals-E<gt>validate> below.
181 my $filename = shift;
183 croak "Undefined filename" unless defined($filename);
187 open F, '>', $filename or croak "Unable to open $filename";
188 foreach ($self->items(by => by_number())) {
189 print F $_->to_string(),"\n";
192 $self->{filename} = $filename;
193 $self->{loaded_maxnum} = $self->{maxnum};
197 =item B<$ordinals-E<gt>items> I<%options>
199 Returns a list of items according to a set of criteria. The criteria is
200 given in form keyed pair form, i.e. a series of C<key =E<gt> value> pairs.
201 Available options are:
205 =item B<sort =E<gt> SORTFUNCTION>
207 SORTFUNCTION is a reference to a function that takes two arguments, which
208 correspond to the classic C<$a> and C<$b> that are available in a C<sort>
211 =item B<filter =E<gt> FILTERFUNCTION>
213 FILTERFUNTION is a reference to a function that takes one argument, which
214 is every OpenSSL::Ordinals::Item element available.
224 my $comparator = $opts{sort};
225 my $filter = $opts{filter} // sub { 1; };
228 if (ref($filter) eq 'ARRAY') {
229 # run a "magic" filter
230 if ($filter->[0] == F_NUMBER) {
231 my $index = $filter->[1];
232 @l = $index ? @{$self->{contents}->[$index] // []} : ();
233 } elsif ($filter->[0] == F_NAME) {
234 my $index = $self->{name2num}->{$filter->[1]};
235 @l = $index ? @{$self->{contents}->[$index] // []} : ();
237 croak __PACKAGE__."->items called with invalid filter";
239 } elsif (ref($filter) eq 'CODE') {
240 @l = grep { $filter->($_) }
242 @{$self->{contents}};
244 croak __PACKAGE__."->items called with invalid filter";
247 return sort { $comparator->($a, $b); } @l
248 if (defined $comparator);
252 # Put an array of items back into the object after having checked consistency
253 # If there are exactly two items:
254 # - They MUST have the same number
255 # - For platforms, both MUST hold the same ones, but with opposite values
256 # - For features, both MUST hold the same ones.
257 # If there's just one item, just put it in the slot of its number
258 # In all other cases, something is wrong
263 if (scalar @items < 1 || scalar @items > 2) {
264 croak "Wrong number of items: ", scalar @items, " : ",
265 join(", ", map { $_->name() } @items), "\n";
267 if (scalar @items == 2) {
273 $numbers{$_->number()} = 1;
274 $versions{$_->version()} = 1;
275 foreach ($_->features()) {
280 # Check that all items we're trying to put back have the same number
281 croak "Items don't have the same numeral: ",
282 join(", ", map { $_->name()." => ".$_->number() } @items), "\n"
283 if (scalar keys %numbers > 1);
284 croak "Items don't have the same version: ",
285 join(", ", map { $_->name()." => ".$_->version() } @items), "\n"
286 if (scalar keys %versions > 1);
288 # Check that both items run with the same features
291 foreach (keys %features) {
292 delete $features{$_} if $features{$_} == 2;
294 croak "Features not in common between ",
295 $items[0]->name(), " and ", $items[1]->name(), ":",
296 join(", ", sort keys %features), "\n"
299 # Check that all platforms exist in both items, and have opposite values
300 my @platforms = ( { $items[0]->platforms() },
301 { $items[1]->platforms() } );
302 foreach my $platform (keys %{$platforms[0]}) {
303 if (exists $platforms[1]->{$platform}) {
304 if ($platforms[0]->{$platform} != !$platforms[1]->{$platform}) {
305 croak "Platforms aren't opposite: ",
307 map { my %tmp_h = $_->platforms();
308 $_->name().":".$platform
310 .$tmp_h{$platform} } @items),
314 # We're done with these
315 delete $platforms[0]->{$platform};
316 delete $platforms[1]->{$platform};
319 # If there are any remaining platforms, something's wrong
320 if (%{$platforms[0]} || %{$platforms[0]}) {
321 croak "There are platforms not in common between ",
322 $items[0]->name(), " and ", $items[1]->name(), "\n";
325 $self->{contents}->[$items[0]->number()] = [ @items ];
328 sub _parse_platforms {
335 my $op = !(defined $1 && $1 eq '!');
338 if ($def =~ m{^_?WIN32$}) { $platforms{$&} = $op; }
339 if ($def =~ m{^__FreeBSD__$}) { $platforms{$&} = $op; }
341 # if ($def =~ m{^__DragonFly__$}) { $platforms{$&} = $op; }
342 # if ($def =~ m{^__OpenBSD__$}) { $platforms{$&} = $op; }
343 # if ($def =~ m{^__NetBSD__$}) { $platforms{$&} = $op; }
345 m{^OPENSSL_(EXPORT_VAR_AS_FUNCTION)$}) { $platforms{$1} = $op; }
346 if ($def =~ m{^OPENSSL_SYS_}) { $platforms{$'} = $op; }
352 sub _parse_features {
359 my $op = !(defined $1 && $1 eq '!');
362 if ($def =~ m{^ZLIB$}) { $features{$&} = $op; }
363 if ($def =~ m{^OPENSSL_USE_}) { $features{$'} = $op; }
364 if ($def =~ m{^OPENSSL_NO_}) { $features{$'} = !$op; }
365 if ($def =~ m{^DEPRECATEDIN_(.*)$}) { $features{$&} = !$op; }
371 =item B<$ordinals-E<gt>add NAME, TYPE, LIST>
373 Adds a new item named NAME with the type TYPE, and a set of C macros in
374 LIST that are expected to be defined or undefined to use this symbol, if
375 any. For undefined macros, they each must be prefixed with a C<!>.
377 If this symbol already exists in loaded data, it will be rewritten using
378 the new input data, but will keep the same ordinal number and version.
379 If it's entirely new, it will get a new number and the current default
380 version. The new ordinal number is a simple increment from the last
388 my $type = shift; # FUNCTION or VARIABLE
389 my @defs = @_; # Macros from #ifdef and #ifndef
390 # (the latter prefixed with a '!')
392 # call signature for debug output
393 my $verbsig = "add('$name' , '$type' , [ " . join(', ', @defs) . " ])";
395 croak __PACKAGE__."->add got a bad type '$type'"
396 unless $type eq 'FUNCTION' || $type eq 'VARIABLE';
398 my %platforms = _parse_platforms(@defs);
399 my %features = _parse_features(@defs);
401 my @items = $self->items(filter => f_name($name));
402 my $version = @items ? $items[0]->version() : $self->{currversion};
403 my $number = @items ? $items[0]->number() : ++$self->{maxnum};
404 print STDERR "DEBUG[",__PACKAGE__,":add] $verbsig\n",
405 @items ? map { "\t".$_->to_string()."\n" } @items : "No previous items\n",
407 @items = grep { $_->exists() } @items;
410 OpenSSL::Ordinals::Item->new( name => $name,
415 platforms => { %platforms },
417 grep { $features{$_} } keys %features
420 push @items, $new_item;
421 print STDERR "DEBUG[",__PACKAGE__,"::add] $verbsig\n", map { "\t".$_->to_string()."\n" } @items
423 $self->_putback(@items);
425 # If an alias was defined beforehand, add an item for it now
426 my $alias = $self->{aliases}->{$name};
427 delete $self->{aliases}->{$name};
429 # For the caller to show
430 my @returns = ( $new_item );
431 push @returns, $self->add_alias($alias->{name}, $name, @{$alias->{defs}})
436 =item B<$ordinals-E<gt>add_alias ALIAS, NAME, LIST>
438 Adds an alias ALIAS for the symbol NAME, and a set of C macros in LIST
439 that are expected to be defined or undefined to use this symbol, if any.
440 For undefined macros, they each must be prefixed with a C<!>.
442 If this symbol already exists in loaded data, it will be rewritten using
443 the new input data. Otherwise, the data will just be store away, to wait
444 that the symbol NAME shows up.
450 my $alias = shift; # This is the alias being added
451 my $name = shift; # For this name (assuming it exists)
452 my @defs = @_; # Platform attributes for the alias
454 # call signature for debug output
456 "add_alias('$alias' , '$name' , [ " . join(', ', @defs) . " ])";
458 croak "You're kidding me..." if $alias eq $name;
460 my %platforms = _parse_platforms(@defs);
461 my %features = _parse_features(@defs);
463 croak "Alias with associated features is forbidden\n"
466 my $f_byalias = f_name($alias);
467 my $f_byname = f_name($name);
468 my @items = $self->items(filter => $f_byalias);
469 foreach my $item ($self->items(filter => $f_byname)) {
470 push @items, $item unless grep { $_ == $item } @items;
472 @items = grep { $_->exists() } @items;
474 croak "Alias already exists ($alias => $name)"
475 if scalar @items > 1;
476 if (scalar @items == 0) {
477 # The item we want to alias for doesn't exist yet, so we cache the
478 # alias and hope the item we're making an alias of shows up later
479 $self->{aliases}->{$name} = { name => $alias, defs => [ @defs ] };
481 print STDERR "DEBUG[",__PACKAGE__,":add_alias] $verbsig\n",
482 "\tSet future alias $alias => $name\n"
485 } elsif (scalar @items == 1) {
486 # The rule is that an alias is more or less a copy of the original
487 # item, just with another name. Also, the platforms given here are
488 # given to the original item as well, with opposite values.
489 my %alias_platforms = $items[0]->platforms();
490 foreach (keys %platforms) {
491 $alias_platforms{$_} = !$platforms{$_};
493 # We supposedly do now know how to do this... *ahem*
494 $items[0]->{platforms} = { %alias_platforms };
496 my $alias_item = OpenSSL::Ordinals::Item->new(
498 type => $items[0]->type(),
499 number => $items[0]->number(),
500 version => $items[0]->version(),
501 exists => $items[0]->exists(),
502 platforms => { %platforms },
503 features => [ $items[0]->features() ]
505 push @items, $alias_item;
507 print STDERR "DEBUG[",__PACKAGE__,":add_alias] $verbsig\n",
508 map { "\t".$_->to_string()."\n" } @items
510 $self->_putback(@items);
512 # For the caller to show
513 return ( $alias_item->to_string() );
515 croak "$name has an alias already (trying to add alias $alias)\n",
516 "\t", join(", ", map { $_->name() } @items), "\n";
519 =item B<$ordinals-E<gt>set_version VERSION>
521 Sets the default version for new symbol to VERSION.
530 $version =~ s|-.*||g;
531 $self->{currversion} = $version;
532 foreach ($self->items(filter => sub { $_[0] eq '*' })) {
533 $_->{version} = $self->{currversion};
538 =item B<$ordinals-E<gt>invalidate>
540 Invalidates the whole working database. The practical effect is that all
541 symbols are set to not exist, but are kept around in the database to retain
542 ordinal numbers and versions.
549 foreach (@{$self->{contents}}) {
550 foreach (@{$_ // []}) {
557 =item B<$ordinals-E<gt>validate>
559 Validates the current working database by collection statistics on how many
560 symbols were added and how many were changed. These numbers can be retrieved
561 with B<$ordinals-E<gt>stats>.
569 for my $i (1..$self->{maxnum}) {
570 if ($i > $self->{loaded_maxnum}
571 || (!@{$self->{loaded_contents}->[$i] // []}
572 && @{$self->{contents}->[$i] // []})) {
573 $self->{stats}->{new}++;
575 next if ($i > $self->{loaded_maxnum});
578 map { $_->to_string() } @{$self->{loaded_contents}->[$i] // []};
579 my @current_strings =
580 map { $_->to_string() } @{$self->{contents}->[$i] // []};
582 foreach my $str (@current_strings) {
583 @loaded_strings = grep { $str ne $_ } @loaded_strings;
585 if (@loaded_strings) {
586 $self->{stats}->{modified}++;
591 =item B<$ordinals-E<gt>stats>
593 Returns the statistics that B<validate> calculate.
600 return %{$self->{stats}};
607 Data elements, which is each line in an ordinals file, are instances
608 of a separate class, OpenSSL::Ordinals::Item, with its own methods:
614 package OpenSSL::Ordinals::Item;
620 =item B<new> I<%options>
622 Creates a new instance of the C<OpenSSL::Ordinals::Item> class. It takes
623 options in keyed pair form, i.e. a series of C<key =E<gt> value> pairs.
624 Available options are:
628 =item B<from =E<gt> STRING>
630 This will create a new item, filled with data coming from STRING.
632 STRING must conform to the following EBNF description:
634 ordinal string = symbol, spaces, ordinal, spaces, version, spaces,
635 exist, ":", platforms, ":", type, ":", features;
636 spaces = space, { space };
638 symbol = ( letter | "_"), { letter | digit | "_" };
640 version = number, "_", number, "_", number, [ letter, [ letter ] ];
641 exist = "EXIST" | "NOEXIST";
642 platforms = platform, { ",", platform };
643 platform = ( letter | "_" ) { letter | digit | "_" };
644 type = "FUNCTION" | "VARIABLE";
645 features = feature, { ",", feature };
646 feature = ( letter | "_" ) { letter | digit | "_" };
647 number = digit, { digit };
649 (C<letter> and C<digit> are assumed self evident)
651 =item B<name =E<gt> STRING>, B<number =E<gt> NUMBER>, B<version =E<gt> STRING>,
652 B<exists =E<gt> BOOLEAN>, B<type =E<gt> STRING>,
653 B<platforms =E<gt> HASHref>, B<features =E<gt> LISTref>
655 This will create a new item with data coming from the arguments.
664 if (ref($_[0]) eq $class) {
665 return $class->new( map { $_ => $_[0]->{$_} } keys %{$_[0]} );
670 croak "No argument given" unless %opts;
672 my $instance = undef;
674 my @a = split /\s+/, $opts{from};
676 croak "Badly formatted ordinals string: $opts{from}"
677 unless ( scalar @a == 4
678 && $a[0] =~ /^[A-Za-z_][A-Za-z_0-9]*$/
680 && $a[2] =~ /^(?:\*|\d+_\d+_\d+[a-z]{0,2})$/
684 (?:FUNCTION|VARIABLE):
689 my @b = split /:/, $a[3];
690 %opts = ( name => $a[0],
693 exists => $b[0] eq 'EXIST',
694 platforms => { map { m|^(!)?|; $' => !$1 }
697 features => [ split /,/,$b[3] // '' ] );
700 if ($opts{name} && $opts{version} && defined $opts{exists} && $opts{type}
701 && ref($opts{platforms} // {}) eq 'HASH'
702 && ref($opts{features} // []) eq 'ARRAY') {
703 my $version = $opts{version};
706 $instance = { name => $opts{name},
708 number => $opts{number},
710 exists => !!$opts{exists},
711 platforms => { %{$opts{platforms} // {}} },
712 features => [ sort @{$opts{features} // []} ] };
714 croak __PACKAGE__."->new() called with bad arguments\n".
715 join("", map { " $_\t=> ".$opts{$_}."\n" } sort keys %opts);
718 return bless $instance, $class;
724 =item B<$item-E<gt>name>
726 The symbol name for this item.
728 =item B<$item-E<gt>number>
730 The positional number for this item.
732 =item B<$item-E<gt>version>
734 The version number for this item. Please note that these version numbers
735 have underscore (C<_>) as a separator the the version parts.
737 =item B<$item-E<gt>exists>
739 A boolean that tells if this symbol exists in code or not.
741 =item B<$item-E<gt>platforms>
743 A hash table reference. The keys of the hash table are the names of
744 the specified platforms, with a value of 0 to indicate that this symbol
745 isn't available on that platform, and 1 to indicate that it is. Platforms
746 that aren't mentioned default to 1.
748 =item B<$item-E<gt>type>
750 C<FUNCTION> or C<VARIABLE>, depending on what the symbol represents.
751 Some platforms do not care about this, others do.
753 =item B<$item-E<gt>features>
755 An array reference, where every item indicates a feature where this symbol
756 is available. If no features are mentioned, the symbol is always available.
757 If any feature is mentioned, this symbol is I<only> available when those
758 features are enabled.
767 my $funcname = $AUTOLOAD;
768 (my $item = $funcname) =~ s|.*::||g;
770 croak "$funcname called as setter" if @_;
771 croak "$funcname invalid" unless exists $self->{$item};
772 return $self->{$item} if ref($self->{$item}) eq '';
773 return @{$self->{$item}} if ref($self->{$item}) eq 'ARRAY';
774 return %{$self->{$item}} if ref($self->{$item}) eq 'HASH';
777 =item B<$item-E<gt>to_string>
779 Converts the item to a string that can be saved in an ordinals file.
786 croak "Too many arguments" if @_;
787 my %platforms = $self->platforms();
788 my @features = $self->features();
789 my $version = $self->version();
790 $version =~ s|\.|_|g;
791 return sprintf "%-39s %d\t%s\t%s:%s:%s:%s",
795 $self->exists() ? 'EXIST' : 'NOEXIST',
796 join(',', (map { ($platforms{$_} ? '' : '!') . $_ }
797 sort keys %platforms)),
799 join(',', @features);
804 =head2 Comparators and filters
806 For the B<$ordinals-E<gt>items> method, there are a few functions to create
807 comparators based on specific data:
813 # Go back to the main package to create comparators and filters
814 package OpenSSL::Ordinals;
820 Returns a comparator that will compare the names of two OpenSSL::Ordinals::Item
826 return sub { $_[0]->name() cmp $_[1]->name() };
831 Returns a comparator that will compare the ordinal numbers of two
832 OpenSSL::Ordinals::Item objects.
837 return sub { $_[0]->number() <=> $_[1]->number() };
842 Returns a comparator that will compare the version of two
843 OpenSSL::Ordinals::Item objects.
849 # cmp_versions comes from OpenSSL::Util
850 return cmp_versions($_[0]->version(), $_[1]->version());
856 There are also the following filters:
862 # Filters... these are called by grep, the return sub must use $_ for
865 =item B<f_version VERSION>
867 Returns a filter that only lets through symbols with a version number
875 croak "No version specified"
876 unless $version && $version =~ /^\d+\.\d+\.\d+[a-z]{0,2}$/;
878 return sub { $_[0]->version() eq $version };
881 =item B<f_number NUMBER>
883 Returns a filter that only lets through symbols with the ordinal number
886 NOTE that this returns a "magic" value that can not be used as a function.
887 It's only useful when passed directly as a filter to B<items>.
894 croak "No number specified"
895 unless $number && $number =~ /^\d+$/;
897 return [ F_NUMBER, $number ];
903 Returns a filter that only lets through symbols with the symbol name
906 NOTE that this returns a "magic" value that can not be used as a function.
907 It's only useful when passed directly as a filter to B<items>.
914 croak "No name specified"
917 return [ F_NAME, $name ];
924 Richard Levitte E<lt>levitte@openssl.orgE<gt>.