Differences
This shows you the differences between two versions of the page.
Both sides previous revision Previous revision Next revision | Previous revision Next revision Both sides next revision | ||
bufr.pm:bufrdump.pl_source [2010-05-27 08:53:05] pals |
bufr.pm:bufrdump.pl_source [2018-04-25 08:41:39] pals |
||
---|---|---|---|
Line 1: | Line 1: | ||
- | < | + | < |
# | # | ||
- | # (C) Copyright 2010, met.no | + | # (C) Copyright 2010-2018, met.no |
# | # | ||
# This program is free software; you can redistribute it and/or modify | # This program is free software; you can redistribute it and/or modify | ||
Line 26: | Line 26: | ||
use constant DEFAULT_TABLE_PATH => '/ | use constant DEFAULT_TABLE_PATH => '/ | ||
- | my $BUFRDUMP = '/ | + | my $BUFRDUMP = ' |
+ | # installed in a non-standard place | ||
# Parse command line options | # Parse command line options | ||
Line 34: | Line 35: | ||
' | ' | ||
' | ' | ||
- | ' | + | ' |
- | ' | + | ' |
' | ' | ||
+ | ' | ||
+ | ' | ||
+ | ' | ||
+ | ' | ||
+ | ' | ||
' | ' | ||
' | ' | ||
' | ' | ||
' | ' | ||
+ | ' | ||
) or pod2usage(-verbose => 0); | ) or pod2usage(-verbose => 0); | ||
Line 65: | Line 72: | ||
$ENV{BUFR_TABLES} .= '/' | $ENV{BUFR_TABLES} .= '/' | ||
- | my $filter | + | die " |
+ | if ! -d $ENV{BUFR_TABLES}; | ||
+ | |||
+ | my $obstype | ||
+ | my $filt = $option{filter} ? "$option{filter}" | ||
my $lon1 = $option{lon1} ? " | my $lon1 = $option{lon1} ? " | ||
my $lat1 = $option{lat1} ? " | my $lat1 = $option{lat1} ? " | ||
my $lon2 = $option{lon2} ? " | my $lon2 = $option{lon2} ? " | ||
my $lat2 = $option{lat2} ? " | my $lat2 = $option{lat2} ? " | ||
+ | my $del = $option{delimiter} ? " | ||
+ | my $delimiter = $option{delimiter} ? " | ||
- | my $criteria_ref = []; | + | # Any filter criteria provided? |
- | if ($filter) { | + | my ($filter, $criteria_ref, $num_alt_ref) |
- | | + | |
- | } | + | |
- | my $param_file | + | # Any specific stations requested? |
- | my ($forced_params_ref, $params_ref); | + | my ($req_id, $req_stn_ref) |
- | ($forced_params_ref, $params_ref) | + | |
- | | + | # Any specific parameters specified? |
+ | my ($params_ref, $forced_params_ref) = get_params($option{param}); | ||
my $csv = $option{csv} ? 1 : 0; | my $csv = $option{csv} ? 1 : 0; | ||
# First line in CSV should be the parameters | # First line in CSV should be the parameters | ||
- | print join(',' | + | print join($del, @$params_ref) . " |
+ | |||
+ | # Any transformations of units specified? | ||
+ | my $transform_file = $option{transform} ? $option{transform} : 0; | ||
+ | my $transform_ref = read_transformation_file($transform_file); | ||
+ | |||
+ | # Any sorting requested? | ||
+ | my $sort = $option{sort} ? 1 : 0; | ||
+ | my $sort_on = $option{sort_on} ? $option{sort_on} : ''; | ||
+ | |||
+ | # What kind of sorting is required (if any)? | ||
+ | ($sort_on, my $by) = get_sort_method($sort_on, | ||
# Loop for processing of BUFR input files | # Loop for processing of BUFR input files | ||
+ | my %data_of = (); | ||
foreach my $inputfname (@ARGV) { | foreach my $inputfname (@ARGV) { | ||
# Dump the content of the BUFR file using the Fortran program $BUFRDUMP | # Dump the content of the BUFR file using the Fortran program $BUFRDUMP | ||
- | my $dump = `$BUFRDUMP | + | my $fortran_options |
+ | my $dump = `$BUFRDUMP $fortran_options | ||
die if $?; # Reason for bufrdump failing should have been printed to STDERR | die if $?; # Reason for bufrdump failing should have been printed to STDERR | ||
# Then process the output from the dump | # Then process the output from the dump | ||
my @lines = split /\n/, $dump; | my @lines = split /\n/, $dump; | ||
+ | # Add an empty line to simplify processing | ||
+ | push @lines, ''; | ||
+ | my $stnid = ''; | ||
- | | + | |
- | # Same output as from bufrdump, except that spaces after ' | + | |
- | my $txt = ''; | + | |
- | foreach my $line (@lines) { | + | |
- | $line =~ s/=\s+/=/; | + | |
- | if ($line) { | + | |
- | $txt .= $line . " | + | |
- | } else { | + | |
- | # Blank line, marks start of data from new station or eof | + | |
- | print $txt .= " | + | |
- | $txt = ''; | + | |
- | } | + | |
- | } | + | |
- | } else { # Options has been used which the Fortran program doesn' | + | |
- | # handle, so special massaging is necessary | + | |
- | # Skip first(blank) line | + | my @lines_to_print; |
- | shift @lines; | + | my %msg; # Hash with parameter name as key, parameter value as value |
- | my @lines_to_print; | + | |
- | my %msg; # Hash with parameter name as key, parameter value as value | + | |
- | + | # Each new message starts with a blank line | |
- | | + | if ($line !~ /^\s*$/) { |
- | # Each new message starts with a blank line | + | # Skip error messages from libbufr, which should start with space(s) |
- | if ($line !~ /^\s*$/) { | + | next LINE if $line =~ /^\s+/; |
- | # Build up the message to be (possibly) printed | + | |
- | push @lines_to_print, | + | my ($param, $value) = ($line =~ / |
- | | + | # Know only of one case where next check is necessary: if |
- | $msg{$name} = $value; | + | # a CCITT IA5 value contains new line (\n) |
+ | next LINE if !defined $value; | ||
+ | if ($transform_file && $transform_ref-> | ||
+ | | ||
+ | my $transform = $transform_ref-> | ||
+ | $transform | ||
+ | $value = eval $transform; | ||
+ | die " | ||
+ | . $transform . " | ||
+ | $line =~ s/ | ||
} | } | ||
+ | $msg{$param} = $value; | ||
+ | push @lines_to_print, | ||
+ | } | ||
- | | + | |
- | # A full message has been completed. Should it be printed? | + | # A full message has been completed. Should it be printed? |
- | if ($filter | + | if ($filt && filter_obs(\%msg, |
- | # Skip this message | + | # Skip this message |
+ | } elsif ($req_id && filter_station(\%msg, | ||
+ | # Skip this station | ||
+ | } else { | ||
+ | # Print the message (or if --sort or --sort_on: save the message) | ||
+ | my $txt = ''; | ||
+ | if ($params_ref) { | ||
+ | # Print the params in @$params_ref if exists in | ||
+ | # message, in same order as in @$params_ref | ||
+ | foreach my $name (@$params_ref) { | ||
+ | if (exists $msg{$name}) { | ||
+ | $txt .= $csv ? $msg{$name} . $del : " | ||
+ | } elsif ($forced_params_ref-> | ||
+ | $txt .= $csv ? ' | ||
+ | } elsif ($csv) { | ||
+ | $txt .= $del; | ||
+ | } | ||
+ | } | ||
} else { | } else { | ||
- | | + | |
- | | + | $line2 =~ s/ |
- | if ($param_file) { | + | |
- | # Print the params in @$params_ref if exists in | + | |
- | # message, in same order as in @$params_ref | + | } |
- | | + | |
- | if (exists | + | # Remove last $del |
- | $txt .= $csv ? $msg{$name} . ',' | + | for (1 .. length($del)) { |
- | } elsif ($forced_params_ref-> | + | |
- | $txt .= $csv ? '-32767,' | + | } |
- | } elsif ($csv) { | + | } |
- | $txt .= ','; | + | if ($txt) { |
- | } | + | if ($sort) { |
+ | # Sort wmonr before nationalnr before call | ||
+ | # sign before buoy_id before aircraft before wigosid | ||
+ | if ($msg{wmonr}) { | ||
+ | $stnid = ' | ||
+ | } elsif ($msg{nationalnr}) { | ||
+ | $stnid = '10_' | ||
+ | } elsif ($msg{call_sign}) { | ||
+ | | ||
+ | } elsif ($msg{buoy_id}) { | ||
+ | $stnid = '30_' | ||
+ | } elsif ($msg{aircraft}) { | ||
+ | $stnid = '40_' | ||
+ | } elsif ($msg{wigosid}) { | ||
+ | | ||
+ | } else { | ||
+ | # Skip observation if no station identification found | ||
+ | next LINE; | ||
} | } | ||
- | } else { | + | if ($sort_on) |
- | | + | my $val = exists $msg{$sort_on} ? $msg{$sort_on} : ''; |
- | $line2 =~ s/=\s+/=/; | + | |
- | $txt .= $line2 . " | + | $data_of{$key} |
+ | ? $data_of{$key} . "$txt\n" : " | ||
+ | } else { | ||
+ | $data_of{$stnid} | ||
+ | ? $data_of{$stnid} | ||
} | } | ||
+ | } elsif ($sort_on) { | ||
+ | my $val = exists $msg{$sort_on} ? $msg{$sort_on} : ''; | ||
+ | $data_of{$val} = exists $data_of{$val} | ||
+ | ? $data_of{$val} . " | ||
+ | } else { | ||
+ | # No sorting. We can print the line immediately | ||
+ | print $txt .= " | ||
} | } | ||
- | chop $txt if $csv; # removes last ',' | + | } |
- | print $txt .= " | + | $txt = ''; |
- | | + | } |
- | } | + | @lines_to_print = (); |
- | @lines_to_print = (); | + | %msg = (); |
- | %msg = (); | + | |
- | } | + | |
} | } | ||
} | } | ||
} | } | ||
+ | # If sorting requested, we cannot print before now | ||
+ | if ($sort && $sort_on) { | ||
+ | for (sort $by keys %data_of) { | ||
+ | print $data_of{$_}; | ||
+ | } | ||
+ | } elsif ($sort) { | ||
+ | for (sort keys %data_of) { | ||
+ | print $data_of{$_}; | ||
+ | } | ||
+ | } elsif ($sort_on) { | ||
+ | # Print observations with missing value for the sort parameter lastly | ||
+ | my $data_of_missing_value = $data_of{'' | ||
+ | delete $data_of{'' | ||
+ | for (sort $by keys %data_of) { | ||
+ | print $data_of{$_}; | ||
+ | } | ||
+ | print $data_of_missing_value if $data_of_missing_value; | ||
+ | } | ||
- | sub read_param_file { | ||
- | my $parameter_file = shift; | ||
- | open my $PARAM, | + | # Read the filter conditions (if any). Return the filter option to be |
- | or die " | + | # used by bufrdump, the found criteria (if any) as well as the number |
+ | # of succeeding alternatives for each criterium | ||
+ | sub get_filter_conditions { | ||
+ | | ||
+ | return ('' | ||
- | my %forced_params; | + | my $fortran_filter = '' |
- | my @params; | + | my @f; |
- | | + | |
- | | + | |
- | | + | |
- | $name = substr | + | } else { |
- | $forced_params{$name} = 1; | + | # Argument to --filter is a file |
+ | | ||
+ | open my $FILTER, '<', | ||
+ | | ||
+ | # Skip the criteria meant for Fortran parsing, i.e. proceed to | ||
+ | # first line following a blank line | ||
+ | while (< | ||
+ | last if $_ =~ /^\s*$/; | ||
} | } | ||
- | | + | @f = <$FILTER>; |
+ | close $FILTER or die " | ||
} | } | ||
- | | + | |
- | return \%forced_params, | ||
- | } | ||
- | |||
- | sub read_filter_file { | ||
- | my $filter_file = shift; | ||
my @allowed_operators = | my @allowed_operators = | ||
(' | (' | ||
Line 193: | Line 286: | ||
'> | '> | ||
' | ' | ||
- | ); | + | ' |
+ | ' | ||
+ | | ||
my @criteria; | my @criteria; | ||
- | + | | |
- | open my $FILTER, '<', | + | # |
- | or die " | + | # corresponding values in @num_alt will be 2,1,0 |
- | | + | |
- | # first line following a blank line | + | |
- | while (<$FILTER>) { | + | |
- | last if $_ =~ /^\s*$/; | + | |
- | } | + | |
# Read the filter criteria meant for Perl parsing, skipping blank | # Read the filter criteria meant for Perl parsing, skipping blank | ||
# lines and comment lines | # lines and comment lines | ||
- | if (not eof) { | + | FILTERLINE: |
- | | + | foreach my $line (@f) { |
- | push @criteria, | + | $line =~ s/^\s+//; |
- | if $line !~ /^\s*$/ && | + | $line =~ s/\s+$//; |
+ | next FILTERLINE | ||
+ | my @crit = split /\|/, $line; | ||
+ | my $num = scalar @crit; | ||
+ | foreach my $criterium (@crit) { | ||
+ | $criterium =~ s/^\s+//; | ||
+ | $criterium =~ s/\s+$//; | ||
+ | if ($criterium ne '' | ||
+ | push @criteria, $criterium; | ||
+ | push @num_alt, --$num; | ||
+ | } | ||
} | } | ||
} | } | ||
+ | return ($fortran_filter) if !@criteria; | ||
# Check that the criteria are properly formatted | # Check that the criteria are properly formatted | ||
foreach my $criterium (@criteria) { | foreach my $criterium (@criteria) { | ||
+ | # Naked parameter possibly preceded by ' | ||
+ | next if $criterium =~ /^!?\w+$/; | ||
+ | |||
my $op = (split / +/, $criterium)[1]; | my $op = (split / +/, $criterium)[1]; | ||
- | if (!defined($op) or not grep(/ | + | if (!defined($op) or grep(/ |
- | print "Error in $filter_file, | + | or !grep(/ |
+ | print "Error in $filt: | ||
+ | . " or operator not supported: | ||
exit 1; | exit 1; | ||
} | } | ||
} | } | ||
- | return \@criteria; | + | return |
+ | } | ||
+ | |||
+ | |||
+ | # Read the parameters into @params, those preceded by an exclamation | ||
+ | # mark also into %forced_params, | ||
+ | sub get_params { | ||
+ | my $params = shift; | ||
+ | return if ! $params; | ||
+ | |||
+ | my @params; | ||
+ | my %forced_params; | ||
+ | |||
+ | if ($params =~ /,/) { | ||
+ | # Argument to --params is a comma separated list | ||
+ | my @p = split /,/, $params; | ||
+ | foreach my $name (@p) { | ||
+ | $name =~ s/^\s+//; | ||
+ | $name =~ s/\s+$//; | ||
+ | if ($name =~ /^!/) { | ||
+ | $name = substr $name, 1; | ||
+ | $forced_params{$name} = 1; | ||
+ | } | ||
+ | push @params, $name; | ||
+ | } | ||
+ | } else { | ||
+ | # Argument to --params is a file | ||
+ | open my $PARAM, '<', | ||
+ | or die " | ||
+ | while (my $name = < | ||
+ | # Skip blank lines and comment lines | ||
+ | $name =~ s/^\s+//; | ||
+ | $name =~ s/\s+$//; | ||
+ | next if !$name || $name =~ /^#/; | ||
+ | if ($name =~ /^!/) { | ||
+ | $name = substr $name, 1; | ||
+ | $forced_params{$name} = 1; | ||
+ | } | ||
+ | push @params, $name; | ||
+ | } | ||
+ | close $PARAM or die " | ||
+ | } | ||
+ | return \@params, \%forced_params; | ||
+ | } | ||
+ | |||
+ | |||
+ | sub read_transformation_file { | ||
+ | my $transform_file = shift; | ||
+ | return if !$transform_file; | ||
+ | |||
+ | open my $TRANSFORM, '<', | ||
+ | or die " | ||
+ | |||
+ | # Read in the transformations, | ||
+ | # lines | ||
+ | my %transform_of; | ||
+ | while (my $line = < | ||
+ | $line =~ s/^\s+//; | ||
+ | $line =~ s/\s+$//; | ||
+ | next if !$line || $line =~ /^#/; | ||
+ | my ($param, $transform) = split /=/, $line, 2; | ||
+ | die " | ||
+ | unless $transform; | ||
+ | $param =~ s/\s+$//; | ||
+ | $transform =~ s/^\s+//; | ||
+ | $transform_of{$param} = $transform; | ||
+ | } | ||
+ | close $TRANSFORM or die " | ||
+ | return \%transform_of; | ||
} | } | ||
# Return true (1) if observation is to be filtered, i.e. does not | # Return true (1) if observation is to be filtered, i.e. does not | ||
- | # comply with at least one of the < | + | # comply with at least one line in filter file, where each line is one |
- | # criteria in filter file | + | # or more alternatives < |
sub filter_obs { | sub filter_obs { | ||
my $msg_ref = shift; | my $msg_ref = shift; | ||
my $criteria_ref = shift; | my $criteria_ref = shift; | ||
+ | my $num_alt_ref = shift; # gives the number of alternative | ||
+ | # criteria still to be checked | ||
+ | return unless $criteria_ref; | ||
- | my @ascii_params = qw(call_sign icao_id name obstime type); | + | my @ascii_params = qw(aircraft |
- | | + | |
+ | for (my $i=0; $i < @{$criteria_ref}; $i++) { | ||
+ | my $num_alt = $num_alt_ref-> | ||
+ | my $criterium = $criteria_ref-> | ||
my ($f_param, $f_operator, | my ($f_param, $f_operator, | ||
- | chomp $f_value; | + | |
- | if ($f_operator | + | # First check for !$par, meaning $par should not be in the observation |
- | | + | if (substr($f_param, |
- | if (grep {$_ eq $f_param} | + | $f_param = substr($f_param,1); |
- | $msg_ref-> | + | # If parameter is present, criterium is not fullfilled |
- | return 1 unless $msg_ref-> | + | if (exists $msg_ref-> |
+ | | ||
+ | |||
+ | # Criterium not fulfilled and no more alternatives to | ||
+ | # check. This observation should be filtered away | ||
+ | return 1; | ||
} else { | } else { | ||
- | | + | |
+ | | ||
+ | next; | ||
} | } | ||
- | } elsif ($f_operator eq '<' | + | } |
- | | + | |
- | and $msg_ref-> | + | # If parameter not present, criterium is obviously not fullfilled |
- | | + | if (not exists $msg_ref-> |
- | | + | next if $num_alt; |
- | and $msg_ref-> | + | return 1; |
- | | + | } |
- | | + | |
- | and $msg_ref->{$f_param} | + | my $msg_value = $msg_ref-> |
- | } elsif ($f_operator eq '>=') { | + | |
- | | + | # present |
- | and $msg_ref-> | + | if (not defined |
- | } elsif ($f_operator eq ' | + | |
- | return 1 unless exists $msg_ref-> | + | # No need to check the alternative criteria |
+ | | ||
+ | | ||
+ | next; | ||
+ | } | ||
+ | |||
+ | chomp $f_value; | ||
+ | | ||
+ | if ($f_operator eq '<' | ||
+ | | ||
+ | | ||
+ | || $f_operator eq '>=' | ||
+ | | ||
+ | | ||
+ | $op = $f_operator; | ||
+ | } elsif ($f_operator eq ' | ||
+ | | ||
if (grep {$_ eq $f_param} @ascii_params) { | if (grep {$_ eq $f_param} @ascii_params) { | ||
- | $msg_ref-> | + | $msg_value |
- | | + | $op = ($f_operator eq ' |
} else { | } else { | ||
- | | + | $op = ($f_operator eq ' |
} | } | ||
+ | } else { | ||
+ | die " | ||
} | } | ||
+ | |||
+ | # Some parameters might need special massaging | ||
+ | if ($f_operator !~ /~/) { | ||
+ | if ($f_param eq ' | ||
+ | # Make non octal by removing leading 0 | ||
+ | $msg_value =~ s/^0+//; | ||
+ | $f_value =~ s/^0+// if $f_value != 0; | ||
+ | } elsif ($f_param eq ' | ||
+ | # Convert to a pure numerical value (float). For 001101 | ||
+ | # State id only numbers between 100 and 699 are operational | ||
+ | $msg_value =~ s/_0*/./; | ||
+ | $f_value =~ s/_0*/./; | ||
+ | } elsif ($f_param eq ' | ||
+ | # Convert to a pure numerical value (float) | ||
+ | $msg_value =~ s/[-:]//g; | ||
+ | $msg_value =~ s/ /./; | ||
+ | $f_value =~ s/ | ||
+ | $f_value =~ s/^ +//; | ||
+ | $f_value =~ s/ +$//; | ||
+ | $f_value =~ s/ /./; | ||
+ | } elsif ($f_param eq ' | ||
+ | # Add or correct quoting to "" | ||
+ | $msg_value = '"' | ||
+ | $f_value = lc $f_value; | ||
+ | if ($f_value =~ / | ||
+ | $f_value =~ s/ | ||
+ | $f_value =~ s/' | ||
+ | } | ||
+ | if ($f_value !~ / | ||
+ | $f_value = '"' | ||
+ | } | ||
+ | } | ||
+ | } | ||
+ | |||
+ | my $condition = " | ||
+ | # Some values should be string values | ||
+ | if ($f_operator =~ /~/) { | ||
+ | $condition = " | ||
+ | } elsif (grep {$_ eq $f_param} @ascii_params) { | ||
+ | $condition = " | ||
+ | } | ||
+ | |||
+ | # Finally, do the criterium check | ||
+ | if (eval $condition) { | ||
+ | # No need to check the remaining alternative criteria | ||
+ | $i += $num_alt if $num_alt; | ||
+ | next; | ||
+ | } else { | ||
+ | next if $num_alt; | ||
+ | return 1; | ||
+ | } | ||
} | } | ||
# All filter conditions have been fullfilled | # All filter conditions have been fullfilled | ||
return 0; | return 0; | ||
+ | } | ||
+ | |||
+ | # Return the type of station requested, and the station | ||
+ | # identifications. Leave some leeway for how to list wmonr and | ||
+ | # nationalnr (leading 0's might be omitted - added here) | ||
+ | sub get_requested_stations { | ||
+ | my $req_stations = shift; | ||
+ | return if !$req_stations; | ||
+ | die " | ||
+ | . "' | ||
+ | unless $req_stations | ||
+ | =~ / | ||
+ | |||
+ | my ($id, $rest) = split /=/, $req_stations; | ||
+ | my @stations = split /,/, $rest; | ||
+ | if ($id eq ' | ||
+ | my @req_stn; | ||
+ | foreach my $station (@stations) { | ||
+ | # Turn $station into a 5 digit wmonr | ||
+ | $station =~ s/^0+//; | ||
+ | $station += 1000 if $station < 1000; | ||
+ | $station = sprintf(" | ||
+ | push @req_stn, $station; | ||
+ | } | ||
+ | return ($id, \@req_stn); | ||
+ | } elsif ($id eq ' | ||
+ | my @req_stn; | ||
+ | foreach my $station (@stations) { | ||
+ | # Turn national station number into 10 digits | ||
+ | my ($state_id, $national_id) = split /_/, $station; | ||
+ | die " | ||
+ | if !defined $national_id or $national_id eq ''; | ||
+ | $station = $state_id . ' | ||
+ | push @req_stn, $station; | ||
+ | } | ||
+ | return ($id, \@req_stn); | ||
+ | } else { | ||
+ | return ($id, \@stations); | ||
+ | } | ||
+ | } | ||
+ | |||
+ | # Return true (1) if observation does not contain one of the stations | ||
+ | # listed in @$req_stn_ref (of type $req_id), i.e. if this observation | ||
+ | # should be filtered away | ||
+ | sub filter_station { | ||
+ | my ($msg_ref, $req_id, $req_stn_ref) = @_; | ||
+ | |||
+ | return 1 unless $msg_ref-> | ||
+ | my $stn = $msg_ref-> | ||
+ | |||
+ | return !grep { $_ eq $stn } @$req_stn_ref; | ||
+ | } | ||
+ | |||
+ | |||
+ | # When --sort_on is used, need to supply the sort method. Also returns | ||
+ | # input parameter $sort_on with possible trailing ' | ||
+ | # stripped off | ||
+ | sub get_sort_method { | ||
+ | my ($sort_on, $sort) = @_; | ||
+ | return if !$sort_on; | ||
+ | |||
+ | my $ascending_sort = 1; | ||
+ | # A minus sign appended to the sort parameter means descending sort | ||
+ | if ($sort_on =~ /-$/) { | ||
+ | $ascending_sort = 0; | ||
+ | chop $sort_on; | ||
+ | } | ||
+ | # Just in case someone adds a ' | ||
+ | if ($sort_on =~ /[+]$/) { | ||
+ | chop $sort_on; | ||
+ | } | ||
+ | |||
+ | my @ascii_params = qw(aircraft call_sign icao_id obstime name type wigosid); | ||
+ | my $lexical_sort = grep {$_ eq $sort_on} @ascii_params; | ||
+ | |||
+ | my $sort_sub; | ||
+ | if ($sort) { | ||
+ | $sort_sub = sub { | ||
+ | my ($stn_a, $val_a) = split /\|/, $a; | ||
+ | my ($stn_b, $val_b) = split /\|/, $b; | ||
+ | # Sort on stationid is always lexical and ascending | ||
+ | my $cmp = $stn_a cmp $stn_b; | ||
+ | if ($cmp) { | ||
+ | return $cmp; | ||
+ | } else { | ||
+ | # Stationids are equal. Sort on value of sort_on parameter | ||
+ | if ($lexical_sort) { | ||
+ | if ($ascending_sort) { | ||
+ | return $val_a cmp $val_b; | ||
+ | } else { | ||
+ | return $val_b cmp $val_a; | ||
+ | } | ||
+ | } else { | ||
+ | # Numerical sort | ||
+ | if ($ascending_sort) { | ||
+ | return $val_a <=> $val_b; | ||
+ | } else { | ||
+ | return $val_b <=> $val_a; | ||
+ | } | ||
+ | } | ||
+ | } | ||
+ | } | ||
+ | } else { | ||
+ | # Sort restricted to sort_on parameter | ||
+ | $sort_sub = sub { | ||
+ | if ($lexical_sort) { | ||
+ | if ($ascending_sort) { | ||
+ | return $a cmp $b; | ||
+ | } else { | ||
+ | return $b cmp $a; | ||
+ | } | ||
+ | } else { | ||
+ | # Numerical sort | ||
+ | if ($ascending_sort) { | ||
+ | return $a <=> $b; | ||
+ | } else { | ||
+ | return $b <=> $a; | ||
+ | } | ||
+ | } | ||
+ | } | ||
+ | } | ||
+ | |||
+ | return ($sort_on, $sort_sub); | ||
} | } | ||
Line 276: | Line 655: | ||
bufrdump.pl <bufr file(s)> | bufrdump.pl <bufr file(s)> | ||
- | [--filter <filter file>] | + | [--filter <filter file | filter list>] |
- | [--param < | + | [--param < |
- | [--lon1 x1] | + | [--sort] |
- | [--lat1 y1] | + | [--sort_on < |
- | [--lon2 x2] | + | [--station <station list>] |
- | [--lat2 | + | [--transform < |
+ | [--lon1 | ||
+ | [--lat1 | ||
+ | [--lon2 | ||
+ | [--lat2 | ||
+ | [--obstype < | ||
[--tablepath <path to BUFR tables>] | [--tablepath <path to BUFR tables>] | ||
[--help] | [--help] | ||
Line 300: | Line 684: | ||
- | --filter <filter file> | + | --filter <filter file | filter list> |
- | Decode observations meeting criteria in <filter file> only | + | Decode observations meeting criteria in filter file or |
- | --param < | + | filter list only |
- | Print parameters in <parameter file> only, in same order | + | --param < |
- | as they occur in < | + | Print parameters in parameter file or comma |
- | | + | separated list (e.g. wmonr, |
- | --lon1 x1 | + | as they occur there. If using --csv possibly |
- | --lat1 y1 | + | followed by --delimiter <del>, the parameters |
- | --lon2 x2 | + | be printed using the CSV (comma-separated values) |
- | --lat2 y2 | + | |
+ | --sort | ||
+ | first stations with wmonr, then stations with nationalnr, | ||
+ | call sign, buoy_id, aircraft or wigosid (others left out) | ||
+ | --sort_on < | ||
+ | values of parameter, or decreasing values if a ' | ||
+ | follows the parameter name. E.g. --sort_on TA- will | ||
+ | sort on decreasing temperatures. Observations not | ||
+ | containing the parameter at all will be printed lastly, | ||
+ | except when --sort_on is combined with --sort (in which | ||
+ | case sorting is done firstly on station identification, | ||
+ | secondly on parameter with missing values printed first) | ||
+ | --station <station list> | ||
+ | Print observations for stations in station list only, | ||
+ | e.g. wmonr=01384, | ||
+ | --transform < | ||
+ | Do the transformations of parameter values listed in | ||
+ | transformation file | ||
+ | --lon1 | ||
+ | --lat1 | ||
+ | --lon2 | ||
+ | --lat2 | ||
x1,y1,x2,y2 should be decimal degrees | x1,y1,x2,y2 should be decimal degrees | ||
+ | --obstype < | ||
+ | Force observation type. If this option is not set, | ||
+ | will make an educated guess of observation type | ||
+ | based on metadata in section 1 of each BUFR message | ||
--tablepath <path to BUFR tables> | --tablepath <path to BUFR tables> | ||
Set path to BUFR tables (overrides ENV{BUFR_TABLES}) | Set path to BUFR tables (overrides ENV{BUFR_TABLES}) | ||
- | --help | + | --help |
+ | perldoc bufrdump.pl) | ||
- | Options may be abbreviated, | + | Options may be abbreviated, |
To avoid having to use the C< | To avoid having to use the C< | ||
- | set the invironment | + | set the environment |
BUFR tables are located (unless the default path provided by | BUFR tables are located (unless the default path provided by | ||
bufrdump.pl works for you). | bufrdump.pl works for you). | ||
- | The lines in < | + | The lines in < |
- | want to be printed. For example, if you want only station | + | < |
- | identification and temperature to be printed for a BUFR SYNOP file, | + | printed. For example, if you want only station identification and |
- | the < | + | temperature to be printed for a BUFR SYNOP file, either supply |
+ | |||
+ | wmonr, | ||
+ | |||
+ | as argument to --params, or supply a < | ||
+ | look like this: | ||
wmonr | wmonr | ||
+ | nationalnr | ||
call_sign | call_sign | ||
TA | TA | ||
Line 333: | Line 749: | ||
If you want " | If you want " | ||
in BUFR message, precede the parameter name with an exclamation mark | in BUFR message, precede the parameter name with an exclamation mark | ||
- | (e.g. ' | + | (e.g. ' |
+ | argument to --param is a parameter list, you must prevent the shell | ||
+ | from attaching special meaning to the exclamation mark by enclosing the | ||
+ | list in single quotes. | ||
+ | |||
+ | If the parameter list consists of one parameter only, a comma must be | ||
+ | appended (e.g. ' | ||
+ | comma to signal that this is not a filename but parameter name(s). | ||
If --csv is used in conjunction with --param, all values will be | If --csv is used in conjunction with --param, all values will be | ||
printed using the CSV format, with first line listing the parameters, | printed using the CSV format, with first line listing the parameters, | ||
and with missing fields printed as -32767 if the parameter is marked | and with missing fields printed as -32767 if the parameter is marked | ||
- | with ' | + | with ' |
- | may for example start like | + | listing |
- | wmonr,call_sign,TA | + | wmonr; |
- | 01001,,-1.5 | + | 01001;;;-1.5 |
- | | + | |
+ | |||
+ | You can choose another delimiter than semicolon by use of option | ||
+ | --delimiter < | ||
Using --filter will decode only those observations that meet at least | Using --filter will decode only those observations that meet at least | ||
Line 349: | Line 775: | ||
in <filter file>, where the BUFR descriptor criteria should come first | in <filter file>, where the BUFR descriptor criteria should come first | ||
in filter file followed by a blank line, then comes the parameter | in filter file followed by a blank line, then comes the parameter | ||
- | criteria which should match < | + | criteria which should match < |
- | is one of =, !=, <, <=, > and >=. An example filter file is | + | < |
+ | follows =~ and !~ should be a Perl match regular expression. The parameter | ||
+ | criteria may be phrased as alternatives by separating them with ' | ||
+ | a single line. An example filter file is | ||
D: 001001 I2.2 | D: 001001 I2.2 | ||
Line 358: | Line 787: | ||
06 252 | 06 252 | ||
D: 001011 A9 | D: 001011 A9 | ||
- | | + | |
- | NN != 0 | + | |
+ | | ||
TA >= 5 | TA >= 5 | ||
TA < 9.5 | TA < 9.5 | ||
+ | RR_24 | ||
which decodes all observations with block number 01, two other | which decodes all observations with block number 01, two other | ||
- | specific wmo stations and one specific ship, having | + | specific wmo stations and one specific ship, where stations should be |
- | different from 0 (but NN must be part of the message) | + | manned and have cloud cover with a value different from 8, and have |
- | between 5 and 9.5 degrees Celsius. Comment lines starting with # | + | temperature |
- | will be ignored. | + | precipitation for last 24 hours. Comment lines starting with # will be |
+ | ignored. | ||
+ | |||
+ | Another example: the filter file (starting with a blank line!) | ||
+ | |||
+ | |||
+ | call_sign =~ / | ||
+ | obstime >= ' | ||
+ | HW | HWA | PW | PWA | ||
+ | FF > 10 | FG_010 > 10 | ||
+ | |||
+ | will print only those ship observations for which the 4 character | ||
+ | call_sign starts with 2 letters in the interval LA-LN, and having | ||
+ | obstime larger or equal to the datetime given, and containing wave | ||
+ | data (specifically: | ||
+ | automatically measured), and with wind or 10 minutes gust more than 10 | ||
+ | m/s. | ||
+ | |||
+ | For convenience, | ||
+ | provide the filter criteria on the command line. Example: | ||
+ | |||
+ | --filter ' | ||
+ | |||
+ | will decode only observations with wmonr, having positive temperature | ||
+ | and containing precipitation for 12 or 24 hours and not reporting | ||
+ | wind. If (like for --param) the filter list consists of one criterium | ||
+ | only, a comma must be appended. | ||
+ | |||
+ | To avoid the need of creating a filter file when observations for some | ||
+ | few stations are requested, you can provide the stations in a comma | ||
+ | separated list after option --station. Some examples: | ||
+ | |||
+ | --station wmonr=01001, | ||
+ | --station nationalnr=614_0050410003, | ||
+ | --station call_sign=LF5U | ||
+ | --station buoyid=64607, | ||
+ | --station aircraft=EU3421, | ||
+ | --station wigosid=0-376-0-511, | ||
+ | |||
+ | You cannot mix different kinds of stations this way (before ' | ||
+ | must choose either wmonr, nationalnr, call_sign, buoy_id, aircraft or | ||
+ | wigosid). Note also that providing the stations in the BUFR descriptor | ||
+ | part (first part) of the filter file will speed up execution time | ||
+ | considerably, | ||
+ | combine --filter with --station if done with some care, | ||
+ | e.g. specifying WMO block 01 and the required parameters in filter | ||
+ | file, then the requested stations in station list. | ||
+ | |||
+ | The --transform option is provided mainly to be able to use other | ||
+ | units than what is default in bufrdump.pl. The transformation file | ||
+ | should list the transformations wanted, one per line as | ||
+ | |||
+ | < | ||
+ | |||
+ | where $x is original value of the parameter. | ||
- | Another | + | For example, the following transformation |
+ | FF and wind gust FG in knots instead of m/s, rounded to one decimal, | ||
+ | and cloud cover NN in % (instead of the default which is using WMO | ||
+ | code table 2700, roughly counting octas): | ||
+ | FF = sprintf(" | ||
+ | FG = sprintf(" | ||
+ | NN = int($x*12.5 + .5) | ||
- | wmonr > 0 | + | If --transform is combined with --filter, the filter criteria should |
+ | refer to the transformed values. E.g. if the above NN transform to % | ||
+ | is to be applied for sky not all covered by clouds, you should use NN | ||
+ | != 100 instead of NN != 8 in filter file. | ||
- | will print only those observations containing a wmonr (skipping | + | The --obstype option might be handy in some special cases, like when |
- | ships). | + | you are interested |
+ | (then use ' | ||
+ | with vss>0 in high resolution radiosonde data (then use ' | ||
+ | sounding-' | ||
+ | BUFR messages have unusual values. | ||
=head1 AUTHOR | =head1 AUTHOR |