Differences
This shows you the differences between two versions of the page.
Both sides previous revision Previous revision Next revision | Previous revision | ||
bufr.pm:bufrread.pl [2010-09-21 08:08:50] pals old revision restored |
bufr.pm:bufrread.pl [2023-02-05 10:14:41] (current) pals |
||
---|---|---|---|
Line 1: | Line 1: | ||
- | < | + | < |
- | # | + | # |
- | # (C) Copyright 2010, met.no | + | # (C) Copyright 2010-2023 MET Norway |
# | # | ||
# This program is free software; you can redistribute it and/or modify | # This program is free software; you can redistribute it and/or modify | ||
Line 22: | Line 22: | ||
use strict; | use strict; | ||
+ | use warnings; | ||
use Getopt:: | use Getopt:: | ||
use Pod::Usage qw(pod2usage); | use Pod::Usage qw(pod2usage); | ||
use Geo::BUFR; | use Geo::BUFR; | ||
+ | |||
+ | # This is actually default in BUFR.pm, but provided here to make it | ||
+ | # easier for users to change to ' | ||
+ | use constant DEFAULT_TABLE_FORMAT => ' | ||
# Will be used if neither --tablepath nor $ENV{BUFR_TABLES} is set | # Will be used if neither --tablepath nor $ENV{BUFR_TABLES} is set | ||
- | use constant | + | use constant |
- | # Ought to be your most up-to-date | + | use constant DEFAULT_TABLE_PATH_ECCODES => '/ |
- | use constant | + | # Ought to be your most up-to-date |
+ | use constant | ||
+ | use constant DEFAULT_CTABLE_ECCODES => ' | ||
# Parse command line options | # Parse command line options | ||
Line 35: | Line 42: | ||
GetOptions( | GetOptions( | ||
| | ||
- | ' | + | ' |
- | ' | + | ' |
+ | # | ||
+ | ' | ||
' | ' | ||
' | ' | ||
' | ' | ||
' | ' | ||
+ | ' | ||
' | ' | ||
+ | ' | ||
' | ' | ||
' | ' | ||
' | ' | ||
' | ' | ||
+ | ' | ||
' | ' | ||
- | ' | + | ' |
' | ' | ||
) or pod2usage(-verbose => 0); | ) or pod2usage(-verbose => 0); | ||
Line 57: | Line 69: | ||
pod2usage(-verbose => 0) unless @ARGV; | pod2usage(-verbose => 0) unless @ARGV; | ||
- | # Set verbosity level for the BUFR module. Must be set also for each | + | # Set verbosity level |
- | # BUFR object generated | + | Geo::BUFR-> |
- | my $verbose = $option{verbose} | + | |
- | Geo:: | + | # Set whether section 4 should be decoded for the BUFR module |
+ | Geo:: | ||
# Set whether quality information should be decoded for the BUFR module | # Set whether quality information should be decoded for the BUFR module | ||
Line 68: | Line 81: | ||
Geo:: | Geo:: | ||
+ | |||
+ | # Set BUFR table format | ||
+ | my $tableformat = (defined $option{tableformat}) ? uc $option{tableformat} : DEFAULT_TABLE_FORMAT; | ||
+ | Geo:: | ||
# Set BUFR table path | # Set BUFR table path | ||
Line 77: | Line 94: | ||
Geo:: | Geo:: | ||
} else { | } else { | ||
- | # If all else fails, use the libemos bufrtables | + | # If all else fails, use the default tablepath in BUFRDC/ |
- | Geo:: | + | |
+ | | ||
+ | } elsif ($tableformat eq ' | ||
+ | Geo:: | ||
+ | } | ||
+ | } | ||
+ | |||
+ | my $ahl_regexp; | ||
+ | if ($option{ahl}) { | ||
+ | eval { $ahl_regexp = qr/ | ||
+ | die " | ||
} | } | ||
Line 96: | Line 123: | ||
# Arrays over filter criteria, used if option --filter is set | # Arrays over filter criteria, used if option --filter is set | ||
- | my @fid; # Filter descriptors, | + | my @fid; # Filter descriptors, |
my @fiv; # Filter values, e.g. $fiv[1] = [ [ 3, 895 ], [ 6 252 ] ] | my @fiv; # Filter values, e.g. $fiv[1] = [ [ 3, 895 ], [ 6 252 ] ] | ||
my @num_desc; # Number of filter descriptors for each criterion, e.g. $num_desc[1] = 2 | my @num_desc; # Number of filter descriptors for each criterion, e.g. $num_desc[1] = 2 | ||
Line 115: | Line 142: | ||
foreach my $inputfname ( @ARGV ) { | foreach my $inputfname ( @ARGV ) { | ||
my $bufr = Geo:: | my $bufr = Geo:: | ||
- | + | | |
- | # This sets object verbose level equal to class verbose level | + | |
- | | + | |
# Open BUFR file | # Open BUFR file | ||
Line 128: | Line 153: | ||
- | # Extract data from BUFR file. Print WMO ahl for first message in | + | # Extract data from BUFR file. Print AHL for first message in each GTS |
- | # each WMO bulletin, print message number for each new message, print | + | # bulletin, print message number for each new message, print subset |
- | # subset | + | # number for each subset. |
sub decode { | sub decode { | ||
my $bufr = shift; | my $bufr = shift; | ||
Line 143: | Line 168: | ||
# Read next observation. If an error is encountered during | # Read next observation. If an error is encountered during | ||
# decoding, skip this observation while printing the error | # decoding, skip this observation while printing the error | ||
- | # message to STDERR, also displaying ahl of bulletin if found. | + | # message to STDERR, also displaying ahl of bulletin if found |
+ | # (but skip error message if the message should be skipped on | ||
+ | # --ahl anyway). | ||
my ($data, $descriptors); | my ($data, $descriptors); | ||
eval { | eval { | ||
Line 149: | Line 176: | ||
}; | }; | ||
if ($@) { | if ($@) { | ||
+ | $current_ahl = $bufr-> | ||
+ | next READLOOP if $option{ahl} && $current_ahl !~ $ahl_regexp; | ||
+ | |||
warn $@; | warn $@; | ||
# Try to extract message number and ahl of the bulletin | # Try to extract message number and ahl of the bulletin | ||
# where the error occurred | # where the error occurred | ||
- | | + | $current_message_number = $bufr-> |
- | | + | if (defined |
- | $current_ahl = $bufr-> | + | my $error_msg = "In message $current_message_number"; |
- | my $error_msg; | + | |
- | | + | |
- | if $current_message_number; | + | |
$error_msg .= " contained in bulletin with ahl $current_ahl\n" | $error_msg .= " contained in bulletin with ahl $current_ahl\n" | ||
if $current_ahl; | if $current_ahl; | ||
warn $error_msg if $error_msg; | warn $error_msg if $error_msg; | ||
- | }; | + | |
+ | exit(1) if $option{on_error_stop}; | ||
next READLOOP; | next READLOOP; | ||
} | } | ||
- | if ($option{codetables}) { | + | |
+ | |||
+ | | ||
# Load C table, trying first to use same table version as | # Load C table, trying first to use same table version as | ||
# the B and D tables loaded in next_observation, | # the B and D tables loaded in next_observation, | ||
Line 171: | Line 201: | ||
# instead. | # instead. | ||
my $table_version = $bufr-> | my $table_version = $bufr-> | ||
- | $bufr-> | + | |
+ | if ($tableformat eq ' | ||
+ | | ||
+ | } elsif ($tableformat eq ' | ||
+ | $bufr-> | ||
+ | } | ||
} | } | ||
my $current_subset_number = $bufr-> | my $current_subset_number = $bufr-> | ||
- | | + | |
+ | # should have been set to at least 1 (even in a 0 subset message) | ||
+ | last READLOOP if $current_subset_number == 0; | ||
- | if ($current_subset_number == 1) { | + | if ($current_subset_number == 1 || $option{nodata}) { |
$current_message_number = $bufr-> | $current_message_number = $bufr-> | ||
$current_ahl = $bufr-> | $current_ahl = $bufr-> | ||
Line 198: | Line 235: | ||
$section013_dumped = 1; | $section013_dumped = 1; | ||
} | } | ||
+ | next READLOOP if $option{nodata}; | ||
} else { # subset number > 1 | } else { # subset number > 1 | ||
next READLOOP if ($option{filter} | next READLOOP if ($option{filter} | ||
Line 221: | Line 259: | ||
} | } | ||
- | | + | printf $OUT " |
- | # error, we might end up here with current subset number 0 | + | |
- | if ($current_subset_number > 0) { | + | # If an error is encountered during dumping of section 4, skip |
- | | + | # this subset while printing the error message to STDERR, also |
- | if ($option{bitmap}) | + | # displaying ahl of bulletin |
- | | + | my $dump; |
- | } else { | + | eval { |
- | | + | $dump = ( $option{bitmap} ) |
- | } | + | |
+ | | ||
+ | | ||
+ | }; | ||
+ | if ($@) { | ||
+ | | ||
+ | my $error_msg = "In message $current_message_number" | ||
+ | . " and subset $current_subset_number"; | ||
+ | $error_msg .= " contained in bulletin with ahl $current_ahl\n" | ||
+ | if $current_ahl; | ||
+ | warn $error_msg; | ||
+ | exit(1) if $option{on_error_stop}; | ||
+ | next READLOOP; | ||
+ | } else { | ||
+ | print $OUT $dump; | ||
} | } | ||
} | } | ||
Line 270: | Line 322: | ||
# Filter routines | # Filter routines | ||
- | # Read in content | + | sub filter_on_ahl { |
+ | my $obj = shift; | ||
+ | my $ahl_regexp = shift; | ||
+ | my $ahl = $obj-> | ||
+ | return $ahl =~ $ahl_regexp ? 0 : 1; | ||
+ | } | ||
+ | |||
+ | # Read in contents | ||
# @num_desc, @num_val and $num_criteria, | # @num_desc, @num_val and $num_criteria, | ||
+ | # Note that index 0 of the arrays is not used. | ||
sub read_filter_file { | sub read_filter_file { | ||
my $filter_file = shift; | my $filter_file = shift; | ||
Line 297: | Line 357: | ||
} else { | } else { | ||
my @values = split; | my @values = split; | ||
+ | # Check that value line contains correct number of values | ||
+ | die " | ||
+ | . " for line $. in filter file ' | ||
+ | if scalar @values != scalar @{$fid[$num_criteria]}; | ||
+ | # Remove leading 0's in numerical values (to prepare for string comparison) | ||
+ | for $_ (@values) { s/ | ||
$fiv[$num_criteria]-> | $fiv[$num_criteria]-> | ||
} | } | ||
Line 313: | Line 379: | ||
my ($data, $descriptors) = @_; | my ($data, $descriptors) = @_; | ||
- | my $num_ordinary_criteria = @fid - $num_required_criteria; | + | my $num_ordinary_criteria = $#fid - $num_required_criteria; |
my $num_success_req_criteria = 0; # Number of required criteria successfully fulfilled | my $num_success_req_criteria = 0; # Number of required criteria successfully fulfilled | ||
my $num_success_ord_criteria = 0; # Number of ordinary criteria successfully fulfilled | my $num_success_ord_criteria = 0; # Number of ordinary criteria successfully fulfilled | ||
Line 358: | Line 424: | ||
for (my $j = 0; $j < @{$descriptors}; | for (my $j = 0; $j < @{$descriptors}; | ||
if ($descriptors-> | if ($descriptors-> | ||
+ | next DESC if !defined $data-> | ||
(my $val = $data-> | (my $val = $data-> | ||
if ($val eq $fiv[$filter_criterion]-> | if ($val eq $fiv[$filter_criterion]-> | ||
Line 370: | Line 437: | ||
or $num_success_ord_criteria > 0)) { | or $num_success_ord_criteria > 0)) { | ||
return 0; # Don't filter this observation | return 0; # Don't filter this observation | ||
+ | } else { | ||
+ | next DESC; | ||
} | } | ||
} else { | } else { | ||
Line 398: | Line 467: | ||
=pod | =pod | ||
+ | |||
+ | =encoding utf8 | ||
=head1 SYNOPSIS | =head1 SYNOPSIS | ||
bufrread.pl <bufr file(s)> | bufrread.pl <bufr file(s)> | ||
+ | [--ahl < | ||
+ | [--all_operators] | ||
+ | [--bitmap] | ||
[--codetables] | [--codetables] | ||
[--data_only] | [--data_only] | ||
- | [--param < | ||
[--filter <filter file>] | [--filter <filter file>] | ||
- | [--bitmap] | + | [--help] |
+ | [--nodata] | ||
[--noqc] | [--noqc] | ||
- | [--outfile < | + | [--on_error_stop] |
[--optional_section] | [--optional_section] | ||
- | [--width n] | + | [--outfile < |
+ | [--param < | ||
[--strict_checking n] | [--strict_checking n] | ||
- | [--all_operators] | + | [--tableformat < |
[--tablepath <path to BUFR tables>] | [--tablepath <path to BUFR tables>] | ||
[--verbose n] | [--verbose n] | ||
- | [--help] | + | [--width n] |
=head1 DESCRIPTION | =head1 DESCRIPTION | ||
Extract BUFR messages from BUFR file(s) and print the decoded content | Extract BUFR messages from BUFR file(s) and print the decoded content | ||
- | to screen. Will include WMO ahl if the BUFR message is part of a WMO | + | to screen, including AHL (Abbreviated Header Line) if present. |
- | bulletin. | + | |
Execute without arguments for Usage, with option C< | Execute without arguments for Usage, with option C< | ||
- | additional info. See also L</https:// | + | additional info. See also L< |
examples of use. | examples of use. | ||
Line 430: | Line 504: | ||
=head1 OPTIONS | =head1 OPTIONS | ||
+ | --ahl < | ||
+ | | ||
+ | | ||
+ | when printing section 4 | ||
+ | | ||
| | ||
is [CODE TABLE] or [FLAG TABLE] | is [CODE TABLE] or [FLAG TABLE] | ||
| | ||
- | | ||
- | | ||
| | ||
| | ||
- | --bitmap | + | --help |
+ | more info you might prefer to consult perldoc bufrread.pl | ||
+ | | ||
| | ||
(or any descriptors following 222000) | (or any descriptors following 222000) | ||
+ | | ||
| | ||
Will print to < | Will print to < | ||
| | ||
| | ||
- | --width n Set width of field used for data values to n characters | + | --param < |
- | (default is 15) | + | Display parameters with descriptors in < |
| | ||
n=1 Issue warning if (recoverable) error in | n=1 Issue warning if (recoverable) error in | ||
BUFR format | BUFR format | ||
n=2 Croak if (recoverable) error in BUFR format. | n=2 Croak if (recoverable) error in BUFR format. | ||
- | | + | |
- | --all_operators Show all operator descriptors when printing section 4 | + | --tableformat |
| | ||
Set path to BUFR tables (overrides ENV{BUFR_TABLES}) | Set path to BUFR tables (overrides ENV{BUFR_TABLES}) | ||
- | | + | |
show the tables loaded. | show the tables loaded. | ||
- | --help Display Usage and explain the options | + | --width n Set width of field used for data values to n characters |
- | more info you might prefer to consult perldoc bufrread.pl | + | (default is 15) |
Options may be abbreviated, | Options may be abbreviated, | ||
To avoid having to use the C< | To avoid having to use the C< | ||
- | set the invironment | + | set the environment |
BUFR tables are located (unless the default path provided by | BUFR tables are located (unless the default path provided by | ||
- | bufralter.pl works for you). | + | bufrread.pl works for you). For tableformat ECCODES, se |
+ | L< | ||
+ | for more info on how to set C< | ||
- | Each line in < | + | For option C< |
- | digits). | + | expression. E.g. C<--ahl " |
- | for these descriptors only. | + | (ISS) from CCCC=ENMI. This is the only case where a little knowledge |
+ | of Perl might possibly be required when using the utility programs | ||
+ | included in Geo:: | ||
+ | |||
+ | For option C< | ||
+ | a BUFR descriptor (6 digits). | ||
+ | bufrread.pl will display values for these descriptors only. | ||
Using C< | Using C< | ||
- | the criteria in <filter file> | + | the criteria in <filter file> |
- | D!). Comments (starting with #) are ignored. An example of a filter | + | marked |
- | file is | + | filter |
D: 001001 | D: 001001 | ||
Line 488: | Line 576: | ||
which decodes all observations with block number 01, two other | which decodes all observations with block number 01, two other | ||
specific WMO stations and one specific ship, all of which having hour | specific WMO stations and one specific ship, all of which having hour | ||
- | (004004) equal to 6 or 7. If there is no value line after a | + | (004004) equal to 6 or 7. If there is no value line after a |
descriptor line, it is enough that the observation contains the | descriptor line, it is enough that the observation contains the | ||
descriptor(s), | descriptor(s), | ||
- | messages from a BUFR SYNOP file, the filter file should contain this | + | messages from a BUFR file, the filter file should contain this single |
- | single line only: | + | line only: |
D: 001011 | D: 001011 | ||
If an error occurs during decoding (typically because the required | If an error occurs during decoding (typically because the required | ||
- | BUFR table is missing or message is corrupt) the message is skipped, | + | BUFR table is missing or message is corrupt), the BUFR message is |
- | and the number of errors | + | skipped |
+ | continues with the next BUFR message. You can change this default | ||
+ | behaviour, however, by setting C< | ||
+ | |||
+ | =head1 CAVEAT | ||
+ | |||
+ | Option C< | ||
+ | Namely, when the first bit-map | ||
+ | their descriptors) will be displayed unless they refer to the | ||
+ | preceding data values by a bit-map. And output | ||
+ | if a bit-map refers to another bit-map or the bit-mapped values are | ||
+ | combined with 204YYY (add associated field operator). | ||
=head1 AUTHOR | =head1 AUTHOR | ||
Line 506: | Line 605: | ||
=head1 COPYRIGHT | =head1 COPYRIGHT | ||
- | Copyright (C) 2010 met.no | + | Copyright (C) 2010-2023 MET Norway |
=cut | =cut | ||
</ | </ |