test data, e.g.: | – | – |

select * from data where obstime>'2027-01-01' and obstime<'2028-01-01' and cfailed like '%TEST%Qc2%' \g | cat >> ALG001.dat

006	Download the following processing and plotting scripts alg001_plot.bash, binplot.R and plot_xy_analysis.r	–	–
007	./alg001_plot.bash ALG001.dat		–
008	–	–	–
009	–	–	–
010	–	–	–

Alternative configuration file for performing triangulation interpolation: alg001-qcd2027_triangle.cfg.gz

Prerequisites

Set up the test data (these are prepatory notes for now for set up on pak}

• $ psql kvalobs
• psql⇒ DELETE FROM data WHERE obstime>'2027-01-01' and obstime<'2028-01-01';
• psql⇒ COPY data FROM '/metno/kvalobs/TESTDATA/2027.dat' USING DELIMTERS ';';

• Run generator.pl commands (current set in /metno/kvalobs/TESTDATA/gencom2027.sh)

kvalobs@pak:~/TESTDATA$ psql kvalobs
Welcome to psql 8.3.8, the PostgreSQL interactive terminal.

Type:  \copyright for distribution terms
       \h for help with SQL commands
       \? for help with psql commands
       \g or terminate with semicolon to execute query
       \q to quit

kvalobs=# DELETE FROM data WHERE obstime>'2027-01-01' and obstime<'2028-01-01';
DELETE 124515
kvalobs=# COPY data FROM '/metno/kvalobs/TESTDATA/2027.dat' USING DELIMITERS ';';
COPY 124515
kvalobs=# \q

kvalobs@pak:~/TESTDATA$ ls
2027.dat              gencom2027.sh  generator.pl  RA24-RESULTS-20091230.dat  results20091230.dat
2047_paramid_110.dat  gencom.sh      plotXY.R      RA_RA24_ScriptFile.sql     Rplots.ps
kvalobs@pak:~/TESTDATA$ ./gencom2027.sh 

Saved the results of the TEST data into a file and made ready for reinsert

 kvalobs=# select * from data where obstime>'2027-01-01' and obstime<'2028-01-01' \g | cat >> ./TEST-DATA-2027-SIMULATED.dat;
 $ ./MakeDataForInsert.sh TEST-DATA-2027-SIMULATED.dat 
(NB afterwards replace "NULL" with "")
 $ mv TEST-DATA-2027-SIMULATED.dat.temp 2027ProcessedTestData.dat


Run Qc2 RA_RR24 with triangle interpolation...

kvalobs=# select * from data where obstime>'2027-01-01' and obstime<'2028-01-01' and cfailed like '%TEST%Qc2%' \g | cat >> ./RA24-RESULTS-20100101-triangle.dat;

2027.doc

2027processedtestdata.dat.gz

generator.pl

#!/usr/bin/perl
#
# set missing values to -11111 at first and then update them all later
# NB we rely on the -32676 for counting good values ...
#
# Note: to run psql command from the command line:
#
#    psql kvalobs -c 'select count(*) from station'
#    psql kvalobs -c "select count(*) from station"
#    echo "select count(*) from station" > com.sql
#    psql kvalobs -f com.sql
#
# Example use:
#  ./generator.pl "2032-03-06 06:00:00" 4 75410 -308 110

system("touch RA_RA24_ScriptFile.sql");
system("/bin/rm RA_RA24_ScriptFile.sql");

open (SCRIPT_FILE, '>>RA_RA24_ScriptFile.sql');

$user_name= getpwuid $>;

if($#ARGV < 4) {
       &shouldo;
       exit;
}


$DATE=$ARGV[0];
$N=$ARGV[1];
$STID=$ARGV[2];
$TID=$ARGV[3];
$PID=$ARGV[4];

# substr(controlinfo,7,1)=3 || 1 || 2 --- dont forget need for this ???

$M=$N-1;

$UP1="UPDATE data SET original=((SELECT SUM(original) FROM data WHERE obstime BETWEEN date '$DATE' - interval '$M days' AND '$DATE' AND stationid=$STID AND typeid=$TID AND paramid=$PID AND original>-32767)+(SELECT COUNT(original) FROM data WHERE obstime BETWEEN date '$DATE' - interval '$M days' AND '$DATE' AND stationid=$STID AND typeid=$TID AND paramid=$PID AND original=-1)) WHERE obstime='$DATE' AND stationid=$STID AND typeid=$TID AND paramid=$PID AND original>-32767 AND (SELECT COUNT(original) FROM data WHERE obstime BETWEEN date '$DATE' - interval '$M days' AND '$DATE' AND stationid=$STID AND typeid=$TID AND paramid=$PID AND original>-32767)=$N;";

$UP2="UPDATE data SET controlinfo='1140000000002000' WHERE obstime='$DATE' AND stationid=$STID AND typeid=$TID AND paramid=$PID AND original>-32767 AND (SELECT COUNT(original) FROM data WHERE obstime BETWEEN date '$DATE' - interval '$M days' AND '$DATE' AND stationid=$STID AND typeid=$TID AND paramid=$PID AND original>-32767)=$N;"; 

print SCRIPT_FILE "\n";
print SCRIPT_FILE $UP1,"\n";
print SCRIPT_FILE "\n";
print SCRIPT_FILE $UP2,"\n";

for ($i=1; $i < $N; $i++){
          $j=$i-1;
          print SCRIPT_FILE "UPDATE data SET controlinfo='1110000000002000' WHERE obstime BETWEEN date '$DATE' -interval '$i days' AND date '$DATE' -interval '$j days' AND stationid=$STID AND typeid=$TID AND paramid=$PID AND original>-32767 AND (SELECT COUNT(original) FROM data WHERE obstime BETWEEN date '$DATE' - interval '$M days' AND '$DATE' AND stationid=$STID AND typeid=$TID AND paramid=$PID AND original>-32767)=$N;\n";
print SCRIPT_FILE "\n";
          print SCRIPT_FILE "UPDATE data SET original=-11111 WHERE obstime BETWEEN date '$DATE' -interval '$i days' AND date '$DATE' -interval '$j days' AND stationid=$STID AND typeid=$TID AND paramid=$PID AND original>-32767 AND (SELECT COUNT(original) FROM data WHERE obstime BETWEEN date '$DATE' - interval '$M days' AND '$DATE' AND stationid=$STID AND typeid=$TID AND paramid=$PID AND original>-32767)=$N;\n";
print SCRIPT_FILE "\n";
}

print SCRIPT_FILE "UPDATE data SET cfailed=cfailed||'TEST-DATA-RA_RR24 original='||corrected, original=-32767 WHERE obstime BETWEEN date '$DATE' - interval '$M days' AND '$DATE' AND stationid=$STID AND typeid=$TID AND paramid=$PID AND original=-11111;\n";
print SCRIPT_FILE "\n";
print SCRIPT_FILE "UPDATE data SET cfailed=cfailed||'TEST-DATA-RA_RR24 original='||corrected WHERE obstime='$DATE' AND stationid=$STID AND typeid=$TID AND paramid=$PID;\n";



close (SCRIPT_FILE); 

system("psql kvalobs -f RA_RA24_ScriptFile.sql");

## ----------------------------------------------------

sub shouldo {
  print
(" 
  Hi $user_name, usage is:
          $0 \"YYYY-MM-DD 06:00:00\" NUMBER_MISSING_POINTS StationID TypeID ParamID 

          e.g.  ./generator.pl \"2032-03-06 06:00:00\" 4 75410 -308 110

")
}

## ----------------------------------------------------

List of generator.pl calls to generate the test data.

It is possible to use the script generator.pl modify some real data and substitute missing values and an accummulated value, while also storing the original values. (NB copy and paste generator.pl from the listing above).