Differences

This shows you the differences between two versions of the page.

--- bufr.pm:bufrencode_source [2023-02-05 10:16:41]
pals
+++ bufr.pm:bufrencode_source [2025-11-05 09:27:00] (current)
pals
@@ Line 2: / Line 2: @@
 #!/usr/bin/perl
-# (C) Copyright 2010-2023 MET Norway
+# (C) Copyright 2010-2025 MET Norway
 #
 # This program is free software; you can redistribute it and/or modify
@@ Line 34: / Line 34: @@
 use constant DEFAULT_TABLE_PATH_BUFRDC => '/usr/local/lib/bufrtables';
 use constant DEFAULT_TABLE_PATH_ECCODES => '/usr/local/share/eccodes/definitions/bufr/tables';
-# Ought to be your most up-to-date B table
-use constant DEFAULT_TABLE_BUFRDC => 'B0000000000000037000';
-use constant DEFAULT_TABLE_ECCODES => '0/wmo/37';
 # Parse command line options
@@ Line 43: / Line 40: @@
 GetOptions(
            \%option,
-           'bufrtable=s',# Set BUFR tables
+           'data=s',
-           'code=s',     # Print the contents of code table
+           'help',
-           'flag=i',     # Resolve the flag value given
+           'metadata=s',
-           'help',       # Print help information and exit
+           'outfile=s',
-           'noexpand',   # Don't expand D descriptors
+           'strict_checking=i',
-           'partial',    # Expand D descriptors only once, ignoring
+           'tableformat=s',
-                         # replication
+           'tablepath=s',
-           'simple',     # Like 'partial', but displaying the resulting
+           'verbose=i',
-                         # descriptors on one line
-           'tableformat=s',  # Set BUFR table format
-           'tablepath=s',# Set BUFR table path
-           'verbose=i',  # Display path and tables used
        ) or pod2usage(-verbose => 0);
 # User asked for help
 pod2usage(-verbose => 1) if $option{help};
-# No arguments if --code or --flag, else there should be at least one argument
+# Data or metadata file not provided
-if (defined $option{code} or defined $option{flag}) {
+pod2usage(-verbose => 0) if not $option{data} or not $option{metadata};
-    pod2usage(-verbose => 0) if @ARGV;
-} else {
-    pod2usage(-verbose => 0) if not @ARGV;
-}
-# If --flag is set, user must also provide code table
+my $data_file     =  $option{data};
-pod2usage(-verbose => 0) if defined $option{flag} and !defined $option{code};
+my $metadata_file =  $option{metadata};
-# All arguments must be integers
+# Default is croak if (recoverable) error found in encoded BUFR format
-foreach (@ARGV) {
+my $strict_checking = defined $option{strict_checking}
-    pod2usage("All arguments must be integers!") unless /^\d+$/;
+    ? $option{strict_checking} : 2;
-}
+Geo::BUFR->set_strict_checking($strict_checking);
-if (defined $option{code} && $option{code} !~ /^\d+$/) {
-    pod2usage("Code table is not a (positive) integer!");
-}
-if (defined $option{flag} && $option{flag} !~ /^\d+$/) {
-    pod2usage("Flag value is not a (positive) integer!");
-}
-# Set verbosity level for the BUFR module
-my $verbose = $option{verbose} ? 1 : 0;
-Geo::BUFR->set_verbose($verbose);
-# From version 1.32 a descriptor sequence ending in e.g. 106000 031001
+# Set verbosity level
-# will be allowed unless strict checking is set, and we really want
+Geo::BUFR->set_verbose($option{verbose}) if $option{verbose};
-# bufrresolve.pl to complain in this case
-Geo::BUFR->set_strict_checking(2);
 # Set BUFR table format
@@ Line 111: / Line 86: @@
     }
 }
-# BUFR table file to use
-my $table = $option{bufrtable} ||
-    ($tableformat eq 'BUFRDC' ? DEFAULT_TABLE_BUFRDC : DEFAULT_TABLE_ECCODES);
 my $bufr = Geo::BUFR->new();
-if (defined $option{code}) {
+# Read metadata into $bufr
-    # Resolve flag value or dump code table
+read_metadata($metadata_file, $bufr);
-    my $code_table = $option{code};
-    if (defined $option{flag}) {
+# Load B and D tables (table version inferred from metadata)
-        if ($option{flag} == 0) {
+$bufr->load_BDtables();
-            print "No bits are set\n";
-        } else {
+# Get the data
-            print $bufr->resolve_flagvalue($option{flag}, $code_table, $table);
+my ($data_refs, $desc_refs, $num_subsets) = readdata($data_file);
-        }
+$bufr->set_number_of_subsets($num_subsets);
+# Print the encoded BUFR message
+my $buffer = $bufr->encode_message($data_refs, $desc_refs);
+if ($option{outfile}) {
+    my $outfile = $option{outfile};
+    open my $fh, '>', $outfile or die "Can't open $outfile: $!";
+    binmode($fh);
+    print $fh $buffer;
+} else {
+    binmode(STDOUT);
+    print $buffer;
+}
+# See OPTIONS section in pod for format of metadata file
+sub read_metadata {
+    my ($file, $bufr) = @_;
+    # Read metadata from file into a hash
+    my %metadata;
+    open (my $fh, '<', $file) or die "Cannot open $file: $!";
+    while ( <$fh> ) {
+        chomp;
+        next if /^\s*$/;
+        s/^\s+//;
+        my ($key, $value) = split /\s+/, $_, 2;
+        $metadata{$key} = $value;
+    }
+    close $fh or die "Cannot close $file: $!";
+    # Load the metadata into the BUFR object
+    my $m = \%metadata;
+    my $bufr_edition = $m->{BUFR_EDITION};
+    $bufr->set_bufr_edition($bufr_edition);
+    $bufr->set_master_table($m->{MASTER_TABLE});
+    $bufr->set_centre($m->{CENTRE});
+    $bufr->set_subcentre($m->{SUBCENTRE});
+    $bufr->set_update_sequence_number($m->{UPDATE_SEQUENCE_NUMBER});
+    $bufr->set_optional_section($m->{OPTIONAL_SECTION});
+    $bufr->set_data_category($m->{DATA_CATEGORY});
+    if ( $bufr_edition < 4 ) {
+        $bufr->set_data_subcategory($m->{DATA_SUBCATEGORY});
     } else {
-        print $bufr->dump_codetable($code_table, $table);
+        $bufr->set_int_data_subcategory($m->{INT_DATA_SUBCATEGORY});
+        $bufr->set_loc_data_subcategory($m->{LOC_DATA_SUBCATEGORY});
     }
-} else {
+    $bufr->set_master_table_version($m->{MASTER_TABLE_VERSION});
-    # Resolve descriptor(s)
+    $bufr->set_local_table_version($m->{LOCAL_TABLE_VERSION});
-    $bufr->load_BDtables($table);
+    if ( $bufr_edition < 4 ) {
-    if ($option{simple}) {
+        $bufr->set_year_of_century($m->{YEAR_OF_CENTURY});
-        print $bufr->resolve_descriptor('simply', @ARGV);
-    } elsif ($option{partial}) {
-        print $bufr->resolve_descriptor('partially', @ARGV);
-    } elsif ($option{noexpand}) {
-        print $bufr->resolve_descriptor('noexpand', @ARGV);
     } else {
-        print $bufr->resolve_descriptor('fully', @ARGV);
+        $bufr->set_year($m->{YEAR});
     }
+    $bufr->set_month($m->{MONTH});
+    $bufr->set_day($m->{DAY});
+    $bufr->set_hour($m->{HOUR});
+    $bufr->set_minute($m->{MINUTE});
+    $bufr->set_second($m->{SECOND}) if $bufr_edition >= 4;
+    $bufr->set_observed_data($m->{OBSERVED_DATA});
+    $bufr->set_compressed_data($m->{COMPRESSED_DATA});
+    $bufr->set_descriptors_unexpanded($m->{DESCRIPTORS_UNEXPANDED});
+    $bufr->set_local_use($m->{LOCAL_USE}) if exists $m->{LOCAL_USE};
+    return;
+}
+# See OPTIONS section in pod for format of data file
+sub readdata {
+    my $file = shift;
+    open (my $fh, '<', $file) or die "Cannot open $file: $!";
+    my ($data_refs, $desc_refs);
+    my $subset = 0;
+    while ( <$fh> ) {
+        s/^\s+//;
+        # Lines not starting with a number are ignored
+        next if not /^\d/;
+        my ($n, $desc, $value) = split /\s+/, $_, 3;
+        $subset++ if $n == 1;
+        # Some operator descriptors are written on unnumbered lines
+        # without a value
+        if (!defined $desc || $desc !~ /^\d/) {
+            next unless $n >= 200000 && $n < 300000; # Better to die here?
+            $desc = $n;
+            $value = undef;
+        } else {
+            $value =~ s/\s+$//;
+            $value = undef if $value eq '' or $value eq 'missing';
+        }
+        push @{$data_refs->[$subset]}, $value;
+        push @{$desc_refs->[$subset]}, $desc;
+    }
+    close $fh or die "Cannot close $file: $!";
+    return ($data_refs, $desc_refs, $subset);
 }
@@ Line 150: / Line 203: @@
 =head1 SYNOPSIS
-) bufrresolve.pl <descriptor(s)>
+  bufrencode.pl --data <data file> --metadata <metadata file>
-     [--partial]
+      [--outfile <file to print encoded BUFR message to>]
-     [--simple]
+      [--strict_checking n]
-     [--noexpand]
+      [--tableformat <BUFRDC|ECCODES>]
-     [--bufrtable <name of BUFR table]
+      [--tablepath <path to BUFR tables>]
-     [--tableformat <BUFRDC|ECCODES>]
+      [--verbose n]
-     [--tablepath <path to BUFR tables>]
+      [--help]
-     [--verbose n]
-     [--help]
-) bufrresolve.pl --code <code or flag table>
-     [--bufrtable <name of BUFR table>]
-     [--tableformat <BUFRDC|ECCODES>]
-     [--tablepath <path to BUFR tables>]
-     [--verbose n]
-) bufrresolve.pl --flag <value> --code <flag table>
-     [--bufrtable <name of BUFR table]
-     [--tableformat <BUFRDC|ECCODES>]
-     [--tablepath <path to BUFR tables>]
-     [--verbose n]
 =head1 DESCRIPTION
-Utility program for fetching info from BUFR tables.
+Encode a BUFR message, reading data and metadata from files. The
+resulting BUFR message will be printed to STDOUT unless option
+C<--outfile> is set.
-Execute without arguments for Usage, with option C<--help> for some
+Execute without arguments for Usage, with option --help for some
 additional info. See also L<https://wiki.met.no/bufr.pm/start> for
 examples of use.
-The tables used can be selected by the user with options
+=head1 OPTIONS
-C<--bufrtable>, C<--tablepath> and C<--tableformat>. Default
-tableformat in Geo::BUFR is BUFRDC, while default tablepath in
-bufrresolve.pl will be overridden if the environment variable
-BUFR_TABLES is set. You should consider edit the source code of
-bufrresolve.pl if you are not satisfied with the defaults chosen for
-tablepath and bufrtable (search for 'DEFAULT').
-For tableformat ECCODES, see
+   --help               Display Usage and explain the options. Almost
-L<http://search.cpan.org/dist/Geo-BUFR/lib/Geo/BUFR.pm#BUFR-TABLE-FILES>
+                        the same as consulting perldoc bufrencode.pl
-for more info on how to set C<--tablepath>.
+   --outfile <filename> Will print the encoded BUFR message to <filename>
+                        instead of STDOUT
+   --strict_checking n  n=0 Disable strict checking of BUFR format
+                        n=1 Issue warning if (recoverable) error in
+                            BUFR format
+                        n=2 (default) Croak if (recoverable) error in BUFR format.
+                            Nothing more in this message will be encoded.
+   --tableformat        Currently supported are BUFRDC and ECCODES (default is BUFRDC)
+   --tablepath <path to BUFR tables>
+                        If used, will set path to BUFR tables. If not
+                        set, will fetch tables from the environment
+                        variable BUFR_TABLES, or if this is not set:
+                        will use DEFAULT_TABLE_PATH_<tableformat>
+                        hard coded in source code.
+   --verbose n          Set verbose level to n, 0<=n<=6 (default 0).
+                        Verbose output is sent to STDOUT, so ought to
+                        be combined with option --outfile
-For the table name in C<--bufrtable> in BUFRDC, use basename of B
+=head2 Required options
-table, e.g.  B0000000000098013001.TXT. Replacing B with D or C, or
-omitting this prefix altogether, or even omitting the trailing '.TXT'
-(i.e. 0000000000098013001) will also work.
-For the table name in C<--bufrtable> in ECCODES, use last significant part
+=head4 --metadata <metadata file>
-of table location, e.g. '0/wmo/29' for WMO master tables or
-'0/local/8/78/236' for local tables on Unix-like systems. For looking
-up local sequence descriptors, you might need to provide both a master
-and the local table to get the full expansion, e.g.
-'0/wmo/29,0/local/8/78/236'.
-See also L</"CAVEAT"> below for more about the C<--bufrtable> option.
+For the metadata file, use this as a prototype and change the values
+as desired:
-=head1 OPTIONS
+  BUFR_EDITION  4
+  MASTER_TABLE  0
+  CENTRE  88
+  SUBCENTRE  0
+  UPDATE_SEQUENCE_NUMBER  0
+  OPTIONAL_SECTION  0
+  DATA_CATEGORY  0
+  INT_DATA_SUBCATEGORY  2
+  LOC_DATA_SUBCATEGORY  255
+  MASTER_TABLE_VERSION  14
+  LOCAL_TABLE_VERSION  0
+  YEAR  2008
+  MONTH  9
+  DAY  1
+  HOUR  6
+  MINUTE  0
+  SECOND  0
+  OBSERVED_DATA  1
+  COMPRESSED_DATA  0
+  DESCRIPTORS_UNEXPANDED  308004 012005 002002
-   --partial    Expand D descriptors only once, ignoring replication
+For BUFR edition < 4, replace the lines INT_DATA_SUBCATEGORY,
-   --simple     Like --partial, but displaying the resulting
+LOC_DATA_SUBCATEGORY, YEAR and SECOND with new lines DATA_SUBCATEGORY
-                descriptors on one line
+and YEAR_OF_CENTURY (the order of lines doesn't matter).
-   --noexpand   Don't expand D descriptors at all
-   --bufrtable <name of BUFR B or D table>  Set BUFR tables
-   --tableformat Currently supported are BUFRDC and ECCODES (default is BUFRDC)
-   --tablepath <path to BUFR tables>  Set BUFR table path
-   --verbose n  Display path and tables used if n > 0
-   --help       Display Usage and explain the options used. Almost
-                the same as consulting perldoc bufrresolve.pl
-Usage 1): Resolves the given descriptor(s) fully into table B
+=head4 --data <data file>
-descriptors, with name, unit, scale, reference value and width (in
-bits) written on each line (except for --simple). --partial, --simple
-and --noexpand are mutually exclusive (full expansion is default).
-Usage 2): Prints the contents of the requested code or flag table
+For the data file, use the same format as would result if you did run
-(named by the table B descriptor).
+on the generated BUFR message
-Usage 3): Displays the bits set when the data value for the requested
+    bufrread.pl <bufr file> --data_only | cut -c -31
-flag table is <value>.
-Options may be abbreviated, e.g. C<--h> or C<-h> for C<--help>
+or if you use bufrread.pl with C<--width n>, replace 31 with n+16.
+For example, the file might begin with
-=head1 CAVEAT
+  001195          Newport
+  005002            51.55
+  006002            -2.99
+  004001             2008
+...
-The C<--bufrtable> option could be considered mandatory, since there
+Every time a new line starting with the number 1 is met, a new subset
-is no guarantee that the same BUFR descriptor resolves the same way
+will be generated in the BUFR message. Lines not starting with a
-for different BUFR tables. However, as soon as a new BUFR descriptor
+number are ignored.
-is introduced in a BUFR table, it is extremely rare that the
-descriptor is redefined in later versions. So for convenience,
+For missing values, use 'missing' or stop the line after the BUFR
-bufrresolve.pl uses a default table (adding option C<--verbose 1> will
+descriptor.
-show you the tables used). If this is the wrong table for your purpose
-(most common case will be if the descriptor was added in a higher
+Associated values should use BUFR descriptor 999999, and operator
-version than that of the default table), you should definitely use
+descriptors 22[2345]000 and 23[2567]000 should not have a value,
-C<--bufrtable> with the appropriate table.
+neither should this line be numbered, e.g.
+  011002          missing
+
+  031002              160
+  031031                0
+...
+To encode a NIL subset, all delayed replication factors should be
+nonzero, and all other values set to missing except for the
+descriptors defining the station.
+Options may be abbreviated, e.g. C<--h> or C<-h> for C<--help>
 =head1 AUTHOR
@@ Line 251: / Line 320: @@
 =head1 COPYRIGHT
-Copyright (C) 2010-2023 MET Norway
+Copyright (C) 2010-2025 MET Norway
 =cut
 </code>