pro rawsurface,site_code,year,current_year,shorto3
;this is a program to read in surface data and make files for reading into the other program.It just appends them.


; BAO site code = 348
  
ground_site_lookup,site_code,info_str
loc=info_str.loc
 datapath1=info_str.raw_data_loc
 dummy= info_str.processed_data_loc
stnid=info_str.code ;station ID
hagl=info_str.inlet_hgt ;Height above ground level
lt_adj=info_str.gmt_offset ; local time adjustment
datapath1=info_str.raw_data_loc ;where to find the raw files
year_start=info_str.year_started

years=indgen(abs(long(year_start)-long(current_year))+2)+long(year_start)

;read in the directory with the raw downloads
;can only read in one year of data at a time!
;first read in SRC

datapath= StrJoin(StrSplit(datapath1,'year_str',/extract,/REGEX),strmid(year,2,2))

IF !Version.OS_family EQ 'unix' THEN datafiles=file_search(datapath+'orig_data/*.src',Count=counts,FOLD_CASE=1)

IF !Version.OS_family EQ 'Windows' THEN datafiles=file_search(datapath+'orig_data\*.src')


; array of all data
ct=44640L ; how many 5-minutes in a month
data={var_units:['station_id','time','date','flags' ,'O3 (ppbv)'],model:'',$
    site_code:site_code,site_location:loc,stnid:stnid,hagl:hagl,$                                   
    time_string:strarr(31,24,60),date_string:strarr(31,24,60),flags:strarr(31,24,60),$
     o3ppbv:fltarr(31,24,60)*!Values.F_NaN}
    shorto3=REPLICATE(data,12,n_elements(years)) ; for each month 
    ;and then for two years to get the first 7 hours 
    ;of Jan of the next year for Local time calculation
   ct=0L
   mo=0
   
     ;calculate date of next year in LOCAL TIME
     ;calculate julian date and time     
     ;if time is negative
     ;if float(lt_adj) lt 0 then wrap_date_time=julday(1,1,float(ystr)+1,abs(Float(lt_adj))-1,59,59) 
     ;if float(lt_adj) gt 0 then wrap_date_time=julday(12,31,float(ystr)-1,23-(float(lt_adj)),59,59) ;would rarely use (for other side of GMT line)

for j=0,n_elements(datafiles) -1 do begin
openr,lun,datafiles(j),/get_lun
print, datafiles(j)
    data_str=''
      mo=0
      ct=0l
    for ij=0,1 do READF,lun,data_str
    data_string=strsplit(data_str,' ',/extract)
    year_head=data_string(n_elements(data_string)-1)
    for ij=0,1 do READF,lun,data_str
    strfields=str_sep(data_str,' ')
    model=STRUPCASE(strmid(strfields(WHERE(strlowcase(strfields) eq 'model')+1),0,3))
    for ij=0,1 DO READF,lun,data_str
   
    ;IF THE MODEL IS A 49 C 
    IF model EQ '49C' THEN BEGIN
    ;check the first and last data points for wrapped dates
      POINT_LUN,-lun, start_pos
      READF,lun,data_str

      data_string=STRSPLIT(data_str,' ',/extract)
     ; IF N_ELEMENTS(data_string) eq 1 then goto, skip1
      mi=long(strcompress(strmid(data_string(1),0,2),/remove))
      
      ;if long(data_string(1)) eq 2000 then goto, skip1
      WHILE ~EOF(lun) DO BEGIN      
      READF, lun, data_str
      if strcompress(data_str) EQ '' THEN BEGIN
      POINT_lun,lun,end_pos-32
      READF,lun, data_str
      goto, skipread
      ENDIF
      POINT_LUN,-lun, end_pos
      ENDWHILE
 skipread:
      ;ENDIF       
      data_string=STRSPLIT(data_str,' ',/extract)
      m_end=long(strcompress(strmid(data_string(1),0,2),/remove))
      ;WHEN You subtract the dates, do you get a negative number? 
      ;IF YES, READ ALL THE PREVIOUS YEAR'S VALUES INTO THE ARRAY   

      IF mi GT m_end THEN BEGIN
        ;READ IN THE DATA UNTIL IT REACHES THE NEW YEAR
        ;REWIND
        POINT_LUN,lun,start_pos
        ;HOW DO IT GET TO THE RIGHT INDEX FOR THE YEAR
        ;shorto3=REPLICATE(data,12,n_elements(years))
          mwas=mi
          WHILE ~EOF(lun) DO BEGIN
            READF, lun, data_str          
            data_string=STRSPLIT(data_str,' ',/extract)
            mi=long(strcompress(strmid(data_string(1),0,2),/remove))
            hi=long(strmid(data_string(0),0,2))
            mini=long(strmid(data_string(0),3,2))
            di=long(strmid(data_string(1),3,2))
            yi=LONG(year_head)-1
            ynst=STRMID(STRCOMPRESS(yi,/remove),2,2) 
            year_ind=where(years eq '20'+ynst)
            shorto3(mi-1,year_ind).date_string(di-1,hi,mini)=data_string(1)+'-'+ynst
            shorto3.stnid=stnid
            shorto3(mi-1,year_ind).model=model
            shorto3(mi-1,year_ind).time_string(di-1,hi,mini)=data_string(0)
            shorto3(mi-1,year_ind).flags(di-1,hi,mini)=data_string(2)
            shorto3(mi-1,year_ind).o3ppbv(di-1,hi,mini)=float(data_string(3))
            IF mi LT mwas THEN goto,start_read
            mwas=mi
           ENDWHILE      
      ENDIF ELSE POINT_LUN,lun,start_pos
      
    ;THEN BEGIN TO READ THE REST WITH THE YEAR
    ENDIF 
    
;FINISHED READING IN WRAPPED PART OF 49C FILES AND   
;FINISHED WITH HEADER NOW READING DATA    
start_read:
    WHILE ~ EOF(lun) DO BEGIN
      ;Read a line of text:
      READF,lun,data_str
      ;parse into fields
      data_string=STRSPLIT(data_str,' ',/extract)
      if n_elements(data_string) eq 1 then goto, skip1
      mi=long(strcompress(strmid(data_string(1),0,2),/remove))
      if long(data_string(1)) eq 2000 then goto, skip1

      if data_string(0) eq ';;' or data_string(0) eq 'Time' then goto, skip1
if mi ne mo then begin
ct=0L
mo=mi
endif

if model eq '49I' THEN BEGIN
yst=strmid(data_string(1),6,2)
yi=long(yst)
year_ind=where(years eq '20'+yst)
hi=long(strmid(data_string(0),0,2))
mini=long(strmid(data_string(0),3,2))
di=long(strmid(data_string(1),3,2))
shorto3(mi-1,year_ind).date_string(di-1,hi,mini)=data_string(1)
endif

    IF model EQ '49C' THEN BEGIN ;this is for 49C
      hi=long(strmid(data_string(0),0,2))
      mini=long(strmid(data_string(0),3,2))
      di=long(strmid(data_string(1),3,2))
      yst=strmid(year_head,2,2)
      yi=long(yst)
      year_ind=where(years eq '20'+yst)
;if year_ind eq 6 then stop
      shorto3(mi-1,year_ind).date_string(di-1,hi,mini)=data_string(1)+'-'+yst
    ENDIF
    
;if di eq '12' and mi eq '04' and j ne 7 then stop
      ;save in proper position in array
    
      shorto3.stnid=stnid
      shorto3(mi-1,year_ind).model=model
      shorto3(mi-1,year_ind).time_string(di-1,hi,mini)=data_string(0)
      shorto3(mi-1,year_ind).flags(di-1,hi,mini)=data_string(2)
      shorto3(mi-1,year_ind).o3ppbv(di-1,hi,mini)=float(data_string(3))
      ct=ct+1l
skip1:

    ENDWHILE
    skip4:
 free_lun,lun
close, lun

endfor ; end of each file

;read in long files data to fill the gap
rawsurface_longfiles,site_code,year,current_year,longo3

;check where there are values missing
for i=0,n_elements(shorto3(0,*)) -1 do begin ;for each year
for mi=0,11 do begin
IF STRMATCH(shorto3(mi,i).model,'49?',/FOLD_CASE) ne 1 THEN BEGIN
b=WHERE(STRMATCH(shorto3(*,i).model,'49?',/FOLD_CASE) eq 1,ctm)
IF ctm ge 1 THEN shorto3(mi,i).model=shorto3(b(0),i).model
ENDIF
sind=where(strcompress(shorto3(mi,i).o3ppbv,/remove) ne 'NaN',ctrs)
lind=where(strcompress(longo3(mi,i).o3ppbv,/remove) ne 'NaN',ctrl)

if ctrl lt 10 and ctrs lt 10 then goto, skiphere

for di=0,30 do begin
for hi=0,23 do begin
;how many values are there?
;if less than #? then will need to grab the long value
sind=where(strcompress(shorto3(mi,i).o3ppbv(di,hi,*),/remove) ne 'NaN',ctw)
if  ctw eq 0 then begin
lind=where(strcompress(longo3(mi,i).o3ppbv(di,hi,*),/remove)ne 'NaN',ctl)
 if ctl ne 0 then begin
 shorto3(mi,i).o3ppbv(di,hi,lind)=longo3(mi,i).o3ppbv(di,hi,lind)
      shorto3(mi,i).time_string(di,hi,lind)= longo3(mi,i).time_string(di,hi,lind)
   shorto3(mi,i).flags(di,hi,lind)=    longo3(mi,i).flags(di,hi,lind)
     shorto3(mi,i).date_string(di,hi,lind)=  longo3(mi,i).date_string(di,hi,lind)
endif

endif

endfor ;end of hi
endfor ;end of di

skiphere: ;skip to next month

endfor ; end of mi
endfor; end of year


;write out the files

for i=0,n_elements(years) -1 do begin
for mi=0,11 do begin
cind=where(strcompress(shorto3(mi,i).o3ppbv,/remove) ne 'NaN',ctrl)
if ctrl lt 10 then goto, skipout
if mi+1 lt 10 then outfile=datapath+STRLOWCASE(site_code)+'0'+ strcompress(mi+1,/remove)+strcompress(years(i),/remove)+'.src' $
else outfile=datapath+strlowcase(site_code)+ strcompress(mi+1,/remove)+strcompress(years(i),/remove)+'.src'
;if site_code eq 'MLO' and long(years(i)) le 2011 then outfile=strjoin(strsplit(outfile,'.src',/extract,/regex)+'_lst.src')

;if site_code eq 'BAO' then outfile='\\ccgg\ccg\towers\SharedFiles\RadonOzoneCeilometerData\BAOgroundBasedOzone\bao_'+dates(0)+'.src'
openw,lun,/get,outfile

IF mi NE 0 AND STRCOMPRESS(shorto3(mi,i).model,/remove) EQ '' THEN shorto3(mi,i).model=shorto3(mi-1,i).model

printf, lun,';;'
printf, lun,';; Stn: ',stnid,"  ", shorto3(mi,i).site_code
printf, lun,';;'
printf, lun,';; Model ', shorto3(mi,i).model
print, shorto3(mi,i).model,' model ', 'mi ',mi,' i ',years(i)
printf, lun,';;'
printf, lun,'Time   Date    Flags       o3 (ppbv)'

for di=0,30 do begin
for hi=0,23 do begin
for mini=0,59 do begin
if strcompress(shorto3(mi,i).o3ppbv(di,hi,mini),/remove) ne 'NaN' then begin
;if shorto3(mi,i).o3ppbv(di,hi,mini) gt -5000 and shorto3(mi,i).o3ppbv(di,hi,mini) lt 15000. then begin
printf,lun,format='(A5,"  ",A8," ",A8,"  ",F8.4)', shorto3(mi,i).time_string(di,hi,mini),shorto3(mi,i).date_string(di,hi,mini),shorto3(mi,i).flags(di,hi,mini),shorto3(mi,i).o3ppbv(di,hi,mini)
endif
endfor ;mini
endfor ; hi
endfor ;di

close, lun
free_lun,lun
skipout:
endfor
endfor

end