Differences
This shows you the differences between two versions of the page.
Both sides previous revision Previous revision Next revision | Previous revision | ||
bufr.pm:bufrread.pl [2013-09-13 11:08:46] pals |
bufr.pm:bufrread.pl [2023-02-05 10:14:41] (current) pals |
||
---|---|---|---|
Line 1: | Line 1: | ||
<code perl> | <code perl> | ||
- | # | + | # |
- | # (C) Copyright 2010, met.no | + | # (C) Copyright 2010-2023 MET Norway |
# | # | ||
# This program is free software; you can redistribute it and/or modify | # This program is free software; you can redistribute it and/or modify | ||
Line 22: | Line 22: | ||
use strict; | use strict; | ||
+ | use warnings; | ||
use Getopt:: | use Getopt:: | ||
use Pod::Usage qw(pod2usage); | use Pod::Usage qw(pod2usage); | ||
use Geo::BUFR; | use Geo::BUFR; | ||
+ | |||
+ | # This is actually default in BUFR.pm, but provided here to make it | ||
+ | # easier for users to change to ' | ||
+ | use constant DEFAULT_TABLE_FORMAT => ' | ||
# Will be used if neither --tablepath nor $ENV{BUFR_TABLES} is set | # Will be used if neither --tablepath nor $ENV{BUFR_TABLES} is set | ||
- | use constant | + | use constant |
- | # Ought to be your most up-to-date | + | use constant DEFAULT_TABLE_PATH_ECCODES => '/ |
- | use constant | + | # Ought to be your most up-to-date |
+ | use constant | ||
+ | use constant DEFAULT_CTABLE_ECCODES => ' | ||
# Parse command line options | # Parse command line options | ||
Line 35: | Line 42: | ||
GetOptions( | GetOptions( | ||
| | ||
- | ' | + | ' |
+ | ' | ||
+ | # | ||
' | ' | ||
' | ' | ||
Line 41: | Line 50: | ||
' | ' | ||
' | ' | ||
+ | ' | ||
' | ' | ||
' | ' | ||
Line 47: | Line 57: | ||
' | ' | ||
' | ' | ||
+ | ' | ||
' | ' | ||
- | ' | + | ' |
' | ' | ||
) or pod2usage(-verbose => 0); | ) or pod2usage(-verbose => 0); | ||
Line 60: | Line 71: | ||
# Set verbosity level | # Set verbosity level | ||
Geo:: | Geo:: | ||
+ | |||
+ | # Set whether section 4 should be decoded for the BUFR module | ||
+ | Geo:: | ||
# Set whether quality information should be decoded for the BUFR module | # Set whether quality information should be decoded for the BUFR module | ||
Line 67: | Line 81: | ||
Geo:: | Geo:: | ||
+ | |||
+ | # Set BUFR table format | ||
+ | my $tableformat = (defined $option{tableformat}) ? uc $option{tableformat} : DEFAULT_TABLE_FORMAT; | ||
+ | Geo:: | ||
# Set BUFR table path | # Set BUFR table path | ||
Line 76: | Line 94: | ||
Geo:: | Geo:: | ||
} else { | } else { | ||
- | # If all else fails, use the libbufr bufrtables | + | # If all else fails, use the default tablepath in BUFRDC/ |
- | Geo:: | + | |
+ | | ||
+ | } elsif ($tableformat eq ' | ||
+ | Geo:: | ||
+ | } | ||
+ | } | ||
+ | |||
+ | my $ahl_regexp; | ||
+ | if ($option{ahl}) { | ||
+ | eval { $ahl_regexp = qr/ | ||
+ | die " | ||
} | } | ||
Line 95: | Line 123: | ||
# Arrays over filter criteria, used if option --filter is set | # Arrays over filter criteria, used if option --filter is set | ||
- | my @fid; # Filter descriptors, | + | my @fid; # Filter descriptors, |
my @fiv; # Filter values, e.g. $fiv[1] = [ [ 3, 895 ], [ 6 252 ] ] | my @fiv; # Filter values, e.g. $fiv[1] = [ [ 3, 895 ], [ 6 252 ] ] | ||
my @num_desc; # Number of filter descriptors for each criterion, e.g. $num_desc[1] = 2 | my @num_desc; # Number of filter descriptors for each criterion, e.g. $num_desc[1] = 2 | ||
Line 114: | Line 142: | ||
foreach my $inputfname ( @ARGV ) { | foreach my $inputfname ( @ARGV ) { | ||
my $bufr = Geo:: | my $bufr = Geo:: | ||
+ | $bufr-> | ||
# Open BUFR file | # Open BUFR file | ||
Line 124: | Line 153: | ||
- | # Extract data from BUFR file. Print WMO ahl for first message in | + | # Extract data from BUFR file. Print AHL for first message in each GTS |
- | # each WMO bulletin, print message number for each new message, print | + | # bulletin, print message number for each new message, print subset |
- | # subset | + | # number for each subset. |
sub decode { | sub decode { | ||
my $bufr = shift; | my $bufr = shift; | ||
Line 139: | Line 168: | ||
# Read next observation. If an error is encountered during | # Read next observation. If an error is encountered during | ||
# decoding, skip this observation while printing the error | # decoding, skip this observation while printing the error | ||
- | # message to STDERR, also displaying ahl of bulletin if found. | + | # message to STDERR, also displaying ahl of bulletin if found |
+ | # (but skip error message if the message should be skipped on | ||
+ | # --ahl anyway). | ||
my ($data, $descriptors); | my ($data, $descriptors); | ||
eval { | eval { | ||
Line 145: | Line 176: | ||
}; | }; | ||
if ($@) { | if ($@) { | ||
+ | $current_ahl = $bufr-> | ||
+ | next READLOOP if $option{ahl} && $current_ahl !~ $ahl_regexp; | ||
+ | |||
warn $@; | warn $@; | ||
# Try to extract message number and ahl of the bulletin | # Try to extract message number and ahl of the bulletin | ||
# where the error occurred | # where the error occurred | ||
- | | + | |
- | | + | if (defined |
- | my $error_msg = "In message $current_message_number"; | + | my $error_msg = "In message $current_message_number"; |
- | $current_ahl = $bufr-> | + | $error_msg .= " contained in bulletin with ahl $current_ahl\n" |
- | $error_msg .= " contained in bulletin with ahl $current_ahl\n" | + | if $current_ahl; |
- | | + | warn $error_msg if $error_msg; |
- | warn $error_msg if $error_msg; | + | } |
- | | + | |
exit(1) if $option{on_error_stop}; | exit(1) if $option{on_error_stop}; | ||
next READLOOP; | next READLOOP; | ||
} | } | ||
- | if ($option{codetables}) { | + | |
+ | |||
+ | | ||
# Load C table, trying first to use same table version as | # Load C table, trying first to use same table version as | ||
# the B and D tables loaded in next_observation, | # the B and D tables loaded in next_observation, | ||
Line 166: | Line 201: | ||
# instead. | # instead. | ||
my $table_version = $bufr-> | my $table_version = $bufr-> | ||
- | $bufr-> | + | |
+ | if ($tableformat eq ' | ||
+ | | ||
+ | } elsif ($tableformat eq ' | ||
+ | $bufr-> | ||
+ | } | ||
} | } | ||
my $current_subset_number = $bufr-> | my $current_subset_number = $bufr-> | ||
- | | + | |
+ | # should have been set to at least 1 (even in a 0 subset message) | ||
+ | last READLOOP if $current_subset_number == 0; | ||
- | if ($current_subset_number == 1) { | + | if ($current_subset_number == 1 || $option{nodata}) { |
$current_message_number = $bufr-> | $current_message_number = $bufr-> | ||
$current_ahl = $bufr-> | $current_ahl = $bufr-> | ||
Line 193: | Line 235: | ||
$section013_dumped = 1; | $section013_dumped = 1; | ||
} | } | ||
+ | next READLOOP if $option{nodata}; | ||
} else { # subset number > 1 | } else { # subset number > 1 | ||
next READLOOP if ($option{filter} | next READLOOP if ($option{filter} | ||
Line 216: | Line 259: | ||
} | } | ||
- | | + | |
- | # error, we might end up here with current subset number 0 | + | |
- | last READLOOP if $current_subset_number | + | |
- | printf $OUT " | + | |
- | + | # this subset while printing the error message to STDERR, also | |
- | # If an error is encountered during dumping of section 4, skip | + | # displaying ahl of bulletin if found. |
- | # this subset while printing the error message to STDERR, also | + | my $dump; |
- | # displaying ahl of bulletin if found. | + | eval { |
- | my $dump; | + | $dump = ( $option{bitmap} ) |
- | eval { | + | ? $bufr-> |
- | | + | |
- | ? $bufr-> | + | : $bufr-> |
- | $current_subset_number, | + | }; |
- | : $bufr-> | + | if ($@) { |
- | }; | + | warn $@; |
- | if ($@) { | + | my $error_msg = "In message $current_message_number" |
- | | + | . " and subset $current_subset_number"; |
- | | + | $error_msg .= " contained in bulletin with ahl $current_ahl\n" |
- | . " and subset $current_subset_number"; | + | if $current_ahl; |
- | | + | warn $error_msg; |
- | if $current_ahl; | + | exit(1) if $option{on_error_stop}; |
- | | + | next READLOOP; |
- | | + | } else { |
- | | + | print $OUT $dump; |
- | } else { | + | } |
- | | + | |
- | } | + | |
} | } | ||
} | } | ||
Line 283: | Line 322: | ||
# Filter routines | # Filter routines | ||
- | # Read in content | + | sub filter_on_ahl { |
+ | my $obj = shift; | ||
+ | my $ahl_regexp = shift; | ||
+ | my $ahl = $obj-> | ||
+ | return $ahl =~ $ahl_regexp ? 0 : 1; | ||
+ | } | ||
+ | |||
+ | # Read in contents | ||
# @num_desc, @num_val and $num_criteria, | # @num_desc, @num_val and $num_criteria, | ||
+ | # Note that index 0 of the arrays is not used. | ||
sub read_filter_file { | sub read_filter_file { | ||
my $filter_file = shift; | my $filter_file = shift; | ||
Line 310: | Line 357: | ||
} else { | } else { | ||
my @values = split; | my @values = split; | ||
+ | # Check that value line contains correct number of values | ||
+ | die " | ||
+ | . " for line $. in filter file ' | ||
+ | if scalar @values != scalar @{$fid[$num_criteria]}; | ||
# Remove leading 0's in numerical values (to prepare for string comparison) | # Remove leading 0's in numerical values (to prepare for string comparison) | ||
for $_ (@values) { s/ | for $_ (@values) { s/ | ||
Line 328: | Line 379: | ||
my ($data, $descriptors) = @_; | my ($data, $descriptors) = @_; | ||
- | my $num_ordinary_criteria = @fid - $num_required_criteria; | + | my $num_ordinary_criteria = $#fid - $num_required_criteria; |
my $num_success_req_criteria = 0; # Number of required criteria successfully fulfilled | my $num_success_req_criteria = 0; # Number of required criteria successfully fulfilled | ||
my $num_success_ord_criteria = 0; # Number of ordinary criteria successfully fulfilled | my $num_success_ord_criteria = 0; # Number of ordinary criteria successfully fulfilled | ||
Line 386: | Line 437: | ||
or $num_success_ord_criteria > 0)) { | or $num_success_ord_criteria > 0)) { | ||
return 0; # Don't filter this observation | return 0; # Don't filter this observation | ||
+ | } else { | ||
+ | next DESC; | ||
} | } | ||
} else { | } else { | ||
Line 414: | Line 467: | ||
=pod | =pod | ||
+ | |||
+ | =encoding utf8 | ||
=head1 SYNOPSIS | =head1 SYNOPSIS | ||
bufrread.pl <bufr file(s)> | bufrread.pl <bufr file(s)> | ||
+ | [--ahl < | ||
+ | [--all_operators] | ||
+ | [--bitmap] | ||
[--codetables] | [--codetables] | ||
[--data_only] | [--data_only] | ||
- | [--param < | ||
[--filter <filter file>] | [--filter <filter file>] | ||
- | [--bitmap] | + | [--help] |
+ | [--nodata] | ||
[--noqc] | [--noqc] | ||
- | [--outfile < | + | [--on_error_stop] |
[--optional_section] | [--optional_section] | ||
- | [--width n] | + | [--outfile < |
+ | [--param < | ||
[--strict_checking n] | [--strict_checking n] | ||
- | [--on_error_stop] | + | [--tableformat < |
- | [--all_operators] | + | |
[--tablepath <path to BUFR tables>] | [--tablepath <path to BUFR tables>] | ||
[--verbose n] | [--verbose n] | ||
- | [--help] | + | [--width n] |
=head1 DESCRIPTION | =head1 DESCRIPTION | ||
Extract BUFR messages from BUFR file(s) and print the decoded content | Extract BUFR messages from BUFR file(s) and print the decoded content | ||
- | to screen. Will include WMO ahl if the BUFR message is part of a WMO | + | to screen, including AHL (Abbreviated Header Line) if present. |
- | bulletin. | + | |
Execute without arguments for Usage, with option C< | Execute without arguments for Usage, with option C< | ||
- | additional info. See also L</https:// | + | additional info. See also L< |
examples of use. | examples of use. | ||
Line 447: | Line 504: | ||
=head1 OPTIONS | =head1 OPTIONS | ||
+ | --ahl < | ||
+ | | ||
+ | | ||
+ | when printing section 4 | ||
+ | | ||
| | ||
is [CODE TABLE] or [FLAG TABLE] | is [CODE TABLE] or [FLAG TABLE] | ||
| | ||
- | | ||
- | | ||
| | ||
| | ||
- | --bitmap | + | --help |
+ | more info you might prefer to consult perldoc bufrread.pl | ||
+ | --nodata | ||
| | ||
(or any descriptors following 222000) | (or any descriptors following 222000) | ||
+ | | ||
| | ||
Will print to < | Will print to < | ||
| | ||
| | ||
- | --width n Set width of field used for data values to n characters | + | --param < |
- | (default is 15) | + | Display parameters with descriptors in < |
| | ||
n=1 Issue warning if (recoverable) error in | n=1 Issue warning if (recoverable) error in | ||
Line 468: | Line 531: | ||
n=2 Croak if (recoverable) error in BUFR format. | n=2 Croak if (recoverable) error in BUFR format. | ||
| | ||
- | --on_error_stop Stop processing as soon as an error occurs during decoding | + | --tableformat |
- | | + | |
| | ||
Set path to BUFR tables (overrides ENV{BUFR_TABLES}) | Set path to BUFR tables (overrides ENV{BUFR_TABLES}) | ||
- | | + | |
show the tables loaded. | show the tables loaded. | ||
- | --help Display Usage and explain the options | + | --width n Set width of field used for data values to n characters |
- | more info you might prefer to consult perldoc bufrread.pl | + | (default is 15) |
Options may be abbreviated, | Options may be abbreviated, | ||
To avoid having to use the C< | To avoid having to use the C< | ||
- | set the invironment | + | set the environment |
BUFR tables are located (unless the default path provided by | BUFR tables are located (unless the default path provided by | ||
- | bufrread.pl works for you). | + | bufrread.pl works for you). For tableformat ECCODES, se |
+ | L< | ||
+ | for more info on how to set C< | ||
+ | |||
+ | For option C< | ||
+ | expression. E.g. C<--ahl " | ||
+ | (ISS) from CCCC=ENMI. This is the only case where a little knowledge | ||
+ | of Perl might possibly be required when using the utility programs | ||
+ | included in Geo::BUFR. | ||
- | Each line in < | + | For option C< |
- | digits). | + | a BUFR descriptor (6 digits). |
- | for these descriptors only. | + | bufrread.pl will display values for these descriptors only. |
Using C< | Using C< | ||
- | the criteria in <filter file> | + | the criteria in <filter file> |
- | D!). Comments (starting with #) are ignored. An example of a filter | + | marked |
- | file is | + | filter |
D: 001001 | D: 001001 | ||
Line 506: | Line 576: | ||
which decodes all observations with block number 01, two other | which decodes all observations with block number 01, two other | ||
specific WMO stations and one specific ship, all of which having hour | specific WMO stations and one specific ship, all of which having hour | ||
- | (004004) equal to 6 or 7. If there is no value line after a | + | (004004) equal to 6 or 7. If there is no value line after a |
descriptor line, it is enough that the observation contains the | descriptor line, it is enough that the observation contains the | ||
descriptor(s), | descriptor(s), | ||
- | messages from a BUFR SYNOP file, the filter file should contain this | + | messages from a BUFR file, the filter file should contain this single |
- | single line only: | + | line only: |
D: 001011 | D: 001011 | ||
If an error occurs during decoding (typically because the required | If an error occurs during decoding (typically because the required | ||
- | BUFR table is missing or message is corrupt) the message is skipped, | + | BUFR table is missing or message is corrupt), the BUFR message is |
- | and the number of errors is reported at end of output. You can change | + | skipped |
- | this default behaviour, however, by setting C< | + | continues with the next BUFR message. You can change this default |
+ | behaviour, however, by setting C< | ||
=head1 CAVEAT | =head1 CAVEAT | ||
- | Option --bitmap may not work properly for complicated BUFR messages. | + | Option |
Namely, when the first bit-map is encountered, | Namely, when the first bit-map is encountered, | ||
their descriptors) will be displayed unless they refer to the | their descriptors) will be displayed unless they refer to the | ||
Line 534: | Line 605: | ||
=head1 COPYRIGHT | =head1 COPYRIGHT | ||
- | Copyright (C) 2010 met.no | + | Copyright (C) 2010-2023 MET Norway |
=cut | =cut | ||
</ | </ |