      program rank_histogram_summary_statistics
!
!     Calculation of all-station rank histogram summary statistics:
!
!      1) distribution of p-values (in nbins_p bins)
!      2) Average frequencies in NQUANT+1 = 100 quantile bins
!         (the quantile bin number is hard-coded!)        
!          
!     The calculation is made for the UTC hours defined by
!     the namelist parameters HOUR1, HOUR2, HOUR_STEP. It is assumed
!     that the input file (INFILE) only includes data for these  
!     hours. 
!
!     Jouni Räisänen, University of Helsinki, August 2023 
!
!-------------------------------------------------------------------------
!
!     INPUT FILE:      
!        
!     infile: a text file including the rank histogram statistics for all stations
!
!     The first line is a header. The other lines include:        
!
!     1) station code (as a 3-character string, if l_code_in_char =.true. Else as an integer)
!     2) longitude
!     3) latitude
!     4) UTC hour (24 for 'all-UTC' statistics
!     5) total number of observations for this UTC hour
!     6) Mean squared deviation (MSD) from a flat quantile frequency histogram
!     7) p-value for MSD (= probability for getting at least as large MSD by chance)
!     8-107) frequencies in quantile space: 0-1%, 1-2% ... 98-99%, 99-100%
!
!     Order of data assumed in the input file:
!
!     line 1: header
!     line 2: station 1, HOUR1
!     line 3: station 1, HOUR1 + HOUR_STEP
!     line n = 1 + (HOUR2-HOUR1)/HOUR_STEP + 1 : station 2, HOUR2
!     line n+1: station2, HOUR1   
!     etc.        
!--------------------------------------------------------------------------        
!        
!     OUTPUT FILES:            
!
!     outfile_p_values: p-values at individual stations (This far: only the all-UTC p-values!)
!        This file can be used for plotting the p-values on a map in python
!        
!     The first line is a header for eventual conversion to ODB. The other lines include
!
!     1) Station code as integer. If the original code was a 3-char string (GRUAN), 
!        the ordinal number of the station is written (for plotting in python)
!     2) longitude
!     3) latitude
!     4) p-value for MSD        
!----------------------------------------------------------------------------        
!     outfile_p_freq: frequency histogram of p-values (for different UTC hours separately)
!        This file can be used for plotting a histogram of the p-value frequencies in python!
!        
!     The first line is a header. The others include
!         
!     1) UTC hour
!     2-nbins_p+1) frequencies of p-values in nbins_p bins  
!-----------------------------------------------------------------------------
!     outfile_q = histogram of all-station-mean quantile bin frequencies (for different UTC hours separately)
!
!     The first line is a header. The others include
!         
!     1) UTC hour
!     2-101) quantile bin frequencies for 0-1%, 1-2% ... 99-100%.
!------------------------------------------------------------------------------        
!     outfile_grads (only if grads_out=.true.): p-value frequencies and quantile bin frequencies in GrADS binary format
!------------------------------------------------------------------------------
!     text_output: frequencies of selected small p-values and some other data in free-text format, for each UTC hour separately.
!-----------------------------------------------------------------------------------------------------------------------------
!
!     NAMELIST PARAMETERS:        
!
!     infile: see above 
!     outfile_p_values: see above
!     outfile_p_freq: see above
!     outfile_q: see above
!     outfile_grads: see above
!     text_output: see above
!     nbins_p: number of p-value bins in outfile_p_freq 
!     grads_out: if .true. (.false.), outfile_grads is written (not written)
!
!     hour1,hour2,hour_step: UTC hours in output = hour1, hour1+hour_step ... hour2
!
!     l_code_in_char: if .true., the station codes in infile are assumed to be 3-char string (else: integer)        
!     miss: missing value code. Default = -9.99e6. All values f with |f| >= |miss| are treated as missing.
       
     IMPLICIT NONE
      INTEGER :: I,J ! loop variables
      INTEGER,PARAMETER :: NHOUR=24 ! number of stations
      INTEGER,PARAMETER :: NQUANT=99 ! number of quantiles from 1% to 99%
      INTEGER,PARAMETER :: NPMAX=100 ! maximum bin number for frequency diagram of p-values  
      INTEGER,PARAMETER :: NSTATMAX=10000 ! maximum number of stations
      REAL :: frequency_q1(nquant+1) ! quantile frequencies at a single station and UTC hour
      REAL :: frequency_q(nquant+1,nhour+1) !all-station mean quantile frequencies for each UTC hour
      REAL :: frequency_p(npmax,nhour+1) !frequencies of p-values for each UTC hours
      REAL :: max_freq_p,max_freq_q ! maximum of p-value and quantile frequencies (for GrADS only)
      REAL :: frequency_p01(nhour+1) ! frequency of p_values <= 0.1 %
      REAL :: frequency_p1(nhour+1) ! same, <= 1 %
      REAL :: frequency_p5(nhour+1) ! same, <= 5 %     
      character*2 :: number_of_bin ! two-digit code for quantiles for outfile_q and outfile_p_values headers
      character*16 :: frequency_value  ! frequency written in F16.6 format
      character*160 :: infile,outfile_p_freq,outfile_p_values,outfile_q ! input and output files (see above)
      character*160 :: outfile_grads,text_output ! output files (see above)
      INTEGER,PARAMETER :: L_dataline=1700 ! maximum length of lines in text files
      character*1700 :: dataline,headerline,emptyline ! string variables for input and output
      
      INTEGER :: nbins_p ! number of p-value bins in outfile_p_freq
      INTEGER :: hour1,hour2,hour_step ! input / output UTC hours (see above)
      INTEGER :: hour ! UTC hour
      INTEGER :: n_line ! line count for lines read from infile
      INTEGER :: n_station ! station count
      INTEGER :: pbin ! index for p-value bin
      
      LOGICAL :: HOUR_USED(nhour+1) ! .true. for hours included in (hour1, hour1+hour_step,...,hour2)
      INTEGER :: nhour_used      ! number of UTC hours used      
      INTEGER :: station1_int ! the station code read from infile (RHARM, FMI)
      CHARACTER*3 :: station1_char ! the station code read from infile (GRUAN)
      LOGICAL :: l_code_in_char ! if .true., station codes in 3-char strings assumed
      REAL :: longitude1,latitude1 !longitude and latitude
      INTEGER :: station(nstatmax) !station codes of all stations
      REAL :: longitude(nstatmax),latitude(nstatmax) ! station longitudes and latitudes
      REAL :: p_value1, msd ! p-value and MSD read from infile
      REAL :: p_value(nhour+1,nstatmax) ! p-values at individual stations for map 
      REAL :: total_number ! number of observations read from infile
      
      LOGICAL :: GRADS_OUT  ! Output also as GrADS binaries, just for testing
      INTEGER :: IREC ! record number for GrADS output      
!
      REAL :: MISS ! Missing value code
      INTEGER :: valid_stations(nhour+1) ! stations with valid data
      
      NAMELIST/param/infile,outfile_p_values,outfile_p_freq,outfile_q,&
           outfile_grads,text_output,&
           nbins_p,grads_out,&
           hour1,hour2,hour_step,&
           l_code_in_char,miss

      miss=-9.99e6  ! default for missing value
      READ(*,NML=PARAM)
      valid_stations=0
!
!     Which UTC hours are used and what is their total number?
!      
      hour_used=.false.
      nhour_used=0.
      do i=hour1,hour2,hour_step
         hour_used(i+1)=.true.
         nhour_used=nhour_used+1
      enddo
!
!     Empty data line 
!
      do i=1,L_dataline
         emptyline(i:i)=' '
      enddo
!----------------------------------------------------------------------
!      
!     Initialize counters etc.
!
      frequency_p=0.
      frequency_q=0.
      frequency_p01=0.
      frequency_p1=0.
      frequency_p5=0.            
      n_station=0
      n_line=0
      p_value=miss
!
!     Read the contents of the input file
!      
      open(unit=1,form='formatted',status='old',file=infile)
      read(1,*) ! the first line in infile is a header
      write(*,*)'  Opened infile: ', infile
      
      do while(.true.) ! Loop continued until end of file reached
         if(l_code_in_char)then
 1         read(1,*,err=1,end=3)&     
            station1_char,longitude1,latitude1,hour,total_number,msd,p_value1,(frequency_q1(i),i=1,nquant+1)
         else
 2         read(1,*,err=2,end=3)&              
            station1_int,longitude1,latitude1,hour,total_number,msd,p_value1,(frequency_q1(i),i=1,nquant+1)
      endif
!      write(*,*)'n_station,Hour step:',n_station,hour_step      
         if(hour_used(hour+1))then    ! Only use the statistics for the UTC hours define by hour1, hour2, hour_step
            n_line=n_line+1         
            if(mod(n_line,nhour_used).eq.1)then ! This assumes that all the required UTC hours are always included in infile
               n_station=n_station+1
               if(l_code_in_char)then
                  station(n_station)=n_station                  
               else                  
                  station(n_station)=station1_int
               endif               
               longitude(n_station)=longitude1
               latitude(n_station)=latitude1               
            endif
           if(total_number.gt.0.and.abs(frequency_q1(1)).lt.abs(miss))then ! Only include stations with some valid data
               p_value(hour+1,n_station)=p_value1 ! for map of p-values           
               valid_stations(hour+1)=valid_stations(hour+1)+1
             do i=1,nquant+1
                frequency_q(i,hour+1)=frequency_q(i,hour+1)+frequency_q1(i) ! update the quantile bin frequencies
             enddo
             pbin=min(1+(p_value1*nbins_p),real(nbins_p))
             frequency_p(pbin,hour+1)=frequency_p(pbin,hour+1)+1  ! update the p-value bin frequencies
!             
!      Frequencies of small p-values:
!             
             if(p_value1.le.0.001.and.abs(p_value1).lt.abs(miss))then
                frequency_p01(hour+1)=frequency_p01(hour+1)+1.
             endif
             if(p_value1.le.0.01.and.abs(p_value1).lt.abs(miss))then
                frequency_p1(hour+1)=frequency_p1(hour+1)+1.
             endif
             if(p_value1.le.0.05.and.abs(p_value1).lt.abs(miss))then
                frequency_p5(hour+1)=frequency_p5(hour+1)+1.
             endif
          endif
          
          endif
      enddo       
3     continue
      close(1)
      write(*,*)'   Read infile: # of lines, # of stations: ',n_line,n_station

!      if(n_line.ne.n_station*nhour_used)then
!         write(*,*)'Something wrong !!!'
!         stop
!      endif      
!---------------------------------------------------------------------      
!     Divide all the frequencies by the number of stations 
!
      do hour=hour1,hour2,hour_step
         do i=1,nquant+1
            frequency_q(i,hour+1)=frequency_q(i,hour+1)/valid_stations(hour+1)
         enddo            
         do i=1,nbins_p   
            frequency_p(i,hour+1)=frequency_p(i,hour+1)/valid_stations(hour+1)            
         enddo
         frequency_p01(hour+1)=frequency_p01(hour+1)/valid_stations(hour+1)
         frequency_p1(hour+1)=frequency_p1(hour+1)/valid_stations(hour+1)
         frequency_p5(hour+1)=frequency_p5(hour+1)/valid_stations(hour+1)
      enddo
!
!     Find the normalized maximum frequencies for GrADS axis limits
!
      max_freq_p=0
      max_freq_q=0
      do i=1,nbins_p
         if(frequency_p(i,nhour+1).gt.max_freq_p/nbins_p)then
            max_freq_p=frequency_p(i,nhour+1)*nbins_p
         endif
      enddo      
      max_freq_p=1.02*max_freq_p

      do i=1,nquant+1
         if(frequency_q(i,nhour+1).gt.max_freq_q/(nquant+1))then
            max_freq_q=frequency_q(i,nhour+1)*(nquant+1)
         endif
      enddo      
      max_freq_q=1.02*max_freq_q     
!      
!---------------------------------------------------------------------      
!
!    Write the text output file:
!       1) number of stations      
!       2) frequencies of selected small p-values 
      
      open(unit=1,form='formatted',file=text_output)
      write(1,'(A20,I3)')'Number of stations: ',n_station
      write(1,'(A24,I3)')'Number of p-value bins: ',nbins_p
      if(grads_out)then
         write(1,'(A23,F6.3)')'Axis_limit_p_for_GrADS ',max_freq_p
         write(1,'(A23,F6.3)')'Axis_limit_q_for_GrADS ',max_freq_q
      endif      
      do hour=hour1,hour2,hour_step
         write(1,'(A22,I2.2,A6,F5.3)')'Frequency, p <= 0.001 ',hour,' UTC: ',&
              frequency_p01(hour+1)
         write(1,'(A22,I2.2,A6,F5.3)')'Frequency, p <= 0.01  ',hour,' UTC: ',&
              frequency_p1(hour+1)
         write(1,'(A22,I2.2,A6,F5.3)')'Frequency, p <= 0.05  ',hour,' UTC: ',&
              frequency_p5(hour+1)         
      enddo
      close(1)
!      
!    Write the average quantile bin frequencies and p-value bin frequencies  
!    to ODB compatible text files
      
!--------------------------------------------------------------------------
!
! Open the quantile bin frequency output file and write its first line.
!
    open(unit=2,form='formatted',status='unknown',file=outfile_q)      
    headerline=emptyline
    headerline='hour@hdr:integer '
    do j=0,nquant
       write(number_of_bin,'(i2.2)')j
       headerline=trim(headerline)//' f'//number_of_bin//'@body:real'
    enddo
    write(2,*)trim(headerline)
!
! Write the data lines for each UTC hour used
! 
    do hour=hour1,hour2,hour_step
       j=hour+1
       write(dataline,'(I6)')&
             hour
       do i=1,nquant+1
          write(frequency_value,'(F16.6)')frequency_q(i,j)
          dataline=trim(dataline)//frequency_value          
       enddo
       write(2,*)trim(dataline)
    enddo
    close(2)
!
! Write the file giving the p-values for individual stations  
!
    open(unit=2,form='formatted',status='unknown',file=outfile_p_values)
    headerline=' station@hdr:integer longitude@hdr:real	latitude@hdr:real p_value@body:real'
    write(2,*)trim(headerline)
    do i=1,n_station
       write(2,'(7X,I6,3F16.6)')&   ! this far: just fwrite the station ordinal number, in case
                                    ! the station code was a character string
            station(i),longitude(i),latitude(i),p_value(nhour+1,i)          
    enddo
    close(2)
!    
! Open the p_value frequency output file and write its first line.
!
    open(unit=2,form='formatted',status='unknown',file=outfile_p_freq)      
    headerline=emptyline
    headerline='hour@hdr:integer '
    do j=1,nbins_p
       write(number_of_bin,'(i2.2)')j-1
       headerline=trim(headerline)//' p'//number_of_bin//'@body:real'
    enddo
    write(2,*)trim(headerline)
!
! Write the data lines for each UTC hour used
! 
    do hour=hour1,hour2,hour_step
       j=hour+1
       write(dataline,'(I6)')&
             hour
       do i=1,nbins_p
          write(frequency_value,'(F16.6)')frequency_p(i,j)
          dataline=trim(dataline)//frequency_value          
       enddo
       write(2,*)trim(dataline)
    enddo
    close(2)
!      
!-------------------------------------------------------------------------------
!   
! Open the file for GrADS output and write its contents, for easier visualisation? 
! 
     if(grads_out)then
        open(unit=11,form='unformatted',file=outfile_grads,access='DIRECT',&
                 recl=4,status='unknown')
!
     do hour=hour1,hour2,hour_step
        j=(hour-hour1)/hour_step+1
        do i=1,nbins_p
           irec=(j-1)*(nbins_p+nquant+1)+i
           write(11,rec=irec)frequency_p(i,hour+1)
        enddo
        do i=1,nquant+1
           irec=(j-1)*(nbins_p+nquant+1)+nbins_p+i
           write(11,rec=irec)frequency_q(i,hour+1)
!           write(*,*)hour,i,frequency_q(i,hour+1)
        enddo
     enddo
     close(11)

  endif ! if (grads_out)      
         
  END program rank_histogram_summary_statistics
