pro rawsurface_longfiles,site_code,year,current_year,rawlongo3 ;this is a program to read in surface data and make files for reading into the other program.It just appends them. ;just do one year at a time ;original files MUST be separated by year (i.e. don't let 12/31 roll to 01/01 because 49C does not put date in ;which year?? ; BAO site code = 348 ground_site_lookup,site_code,info_str loc=info_str.loc datapath1=info_str.raw_data_loc dummy= info_str.processed_data_loc stnid=info_str.code ;station ID hagl=info_str.inlet_hgt ;Height above ground level lt_adj=info_str.gmt_offset ; local time adjustment datapath1=info_str.raw_data_loc ;where to find the raw files year_start=info_str.year_started years=indgen(abs(long(year_start)-long(current_year))+2)+long(year_start) ;read in the directory with the raw downloads ;can only read in one year of data at a time! datapath= StrJoin(StrSplit(datapath1,'year_str',/extract,/REGEX),strmid(year,2,2)) IF !Version.OS_family EQ 'unix' THEN datafiles=file_search(datapath+'orig_data/*.lrc',FOLD_CASE=1) IF !Version.OS_family EQ 'Windows' THEN datafiles=file_search(datapath+'orig_data\*.lrc') ; array of all data ct=44640L ; how many 5-minutes in a month data={var_units:['station_id','time','date','flags' ,'O3 (ppbv)'],model:'',$ site_code:site_code,site_location:loc,stnid:stnid,hagl:hagl,$ time_string:strarr(31,24,60),date_string:strarr(31,24,60),flags:strarr(31,24,60),$ o3ppbv:fltarr(31,24,60)*!Values.F_NaN,intena:lonarr(31,24,60)*!Values.F_NaN,$ intenb:lonarr(31,24,60)*!Values.F_NaN,bencht:fltarr(31,24,60)*!Values.F_NaN,$ lampt:fltarr(31,24,60)*!Values.F_NaN,flowa:fltarr(31,24,60)*!Values.F_NaN,$ flowb:fltarr(31,24,60)*!Values.F_NaN,pres:fltarr(31,24,60)*!Values.F_NaN,o3lampt:fltarr(31,24,60)*!Values.F_NaN} rawlongo3=REPLICATE(data,12,n_elements(years)) ; for each month ;and then for two years to get the first 7 hours ;of Jan of the next year for Local time calculation ct=0L mo=0 ;calculate date of next year in LOCAL TIME ;calculate julian date and time ;if time is negative ;if float(lt_adj) lt 0 then wrap_date_time=julday(1,1,float(ystr)+1,abs(Float(lt_adj))-1,59,59) ;if float(lt_adj) gt 0 then wrap_date_time=julday(12,31,float(ystr)-1,23-(float(lt_adj)),59,59) ;would rarely use (for other side of GMT line) FOR j=0,n_elements(datafiles) -1 DO BEGIN openr,lun,datafiles(j),/get_lun ;print, datafiles(j) data_str='' mo=0 ct=0l for ij=0,1 do READF,lun,data_str data_string=strsplit(data_str,' ',/extract) year_head=data_string(n_elements(data_string)-1) print, year_head for ij=0,1 DO READF,lun,data_str strfields=str_sep(data_str,' ') model=STRUPCASE(strmid(strfields(WHERE(strlowcase(strfields) eq 'model')+1),0,3)) for ij=0,1 DO READF,lun,data_str colhdr=data_str ;IF THE MODEL IS A 49 C IF model EQ '49C' THEN BEGIN ;check the first and last data points for wrapped dates POINT_LUN,-lun, start_pos READF,lun,data_str ;PRINT, data_str data_string=STRSPLIT(data_str,' ',/extract) ;print, data_str ; IF N_ELEMENTS(data_string) eq 1 then goto, skip1 mi=long(strcompress(strmid(data_string(1),0,2),/remove)) ;if long(data_string(1)) eq 2000 then goto, skip1 WHILE ~EOF(lun) DO BEGIN READF, lun, data_str IF strcompress(data_str) EQ '' THEN BEGIN POINT_lun,lun,end_pos-111 READF,lun, data_str GOTO, skipread ENDIF POINT_LUN,-lun, end_pos ENDWHILE skipread: data_string=STRSPLIT(data_str,' ',/extract) m_end=long(strcompress(strmid(data_string(1),0,2),/remove)) POINT_LUN,-lun, end_pos ;WHEN You subtract the dates, do you get a negative number? ;IF YES, READ ALL THE PREVIOUS YEAR'S VALUES INTO THE ARRAY ;print, 'mi ',mi,' m_end ',m_end IF mi GT m_end THEN BEGIN ;READ IN THE DATA UNTIL IT REACHES THE NEW YEAR ;REWIND POINT_LUN,lun,start_pos ;HOW DO IT GET TO THE RIGHT INDEX FOR THE YEAR ;shorto3=REPLICATE(data,12,n_elements(years)) mwas=mi WHILE ~EOF(lun) DO BEGIN READF, lun, data_str data_string=STRSPLIT(data_str,' ',/extract) mi=long(strcompress(strmid(data_string(1),0,2),/remove)) hi=long(strmid(data_string(0),0,2)) mini=long(strmid(data_string(0),3,2)) di=long(strmid(data_string(1),3,2)) yi=LONG(year_head)-1 ynst=STRMID(STRCOMPRESS(yi,/remove),2,2) year_ind=where(years eq '20'+ynst) rawlongo3(mi-1,year_ind).model=model print, model rawlongo3(mi-1,year_ind).date_string(di-1,hi,mini)=data_string(1)+'-'+ynst rawlongo3(mi-1,year_ind).intena(di-1,hi,mini)=float(data_string(4)) rawlongo3(mi-1,year_ind).intenb(di-1,hi,mini)=float(data_string(5)) rawlongo3(mi-1,year_ind).bencht(di-1,hi,mini)=float(data_string(6)) rawlongo3(mi-1,year_ind).lampt(di-1,hi,mini)=float(data_string(7)) rawlongo3(mi-1,year_ind).o3lampt(di-1,hi,mini)=float(data_string(8)) rawlongo3(mi-1,year_ind).flowa(di-1,hi,mini)=float(data_string(9)) rawlongo3(mi-1,year_ind).flowb(di-1,hi,mini)=float(data_string(10)) rawlongo3(mi-1,year_ind).pres(di-1,hi,mini)=float(data_string(11)) rawlongo3.stnid=stnid rawlongo3(mi-1,year_ind).model=model rawlongo3(mi-1,year_ind).time_string(di-1,hi,mini)=data_string(0) rawlongo3(mi-1,year_ind).flags(di-1,hi,mini)=data_string(2) rawlongo3(mi-1,year_ind).o3ppbv(di-1,hi,mini)=float(data_string(3)) IF mi LT mwas THEN goto,start_read mwas=mi ;print, 'mi ',mi,' mwas ',mwas ENDWHILE ENDIF ELSE POINT_LUN,lun,start_pos ;THEN BEGIN TO READ THE REST WITH THE YEAR ENDIF ;FINISHED READING IN WRAPPED PART OF 49C FILES AND ;FINISHED WITH HEADER NOW READING DATA start_read: WHILE ~ EOF(lun) DO BEGIN ;Read a line of text: READF,lun,data_str ;parse into fields data_string=STRSPLIT(data_str,' ',/extract) if n_elements(data_string) eq 1 then goto, skip1 ;print, data_string(1) mi=long(strcompress(strmid(data_string(1),0,2),/remove)) ; print, mi if long(data_string(1)) eq 2000 then goto, skip1 if mi ne mo then begin ct=0L mo=mi endif ;49I++++++++++++++++++++++++++++ ;; ;; wkt_20090609.lrc Wed Jun 10 15:50:40 2009 ;; ;; Model 49I- (prog iSeries 49i 01.05.00.159) ;; ;Time Date Flags o3 hio3 cellai cellbi bncht lmpt o3lt flowa flowb pres if model eq '49I' THEN BEGIN hi=long(strmid(data_string(0),0,2)) mini=long(strmid(data_string(0),3,2)) di=long(strmid(data_string(1),3,2)) yst=strmid(data_string(1),6,2) yi=long(yst) year_ind=where(years eq '20'+yst) rawlongo3(mi-1,year_ind).model=model rawlongo3(mi-1,year_ind).date_string(di-1,hi,mini)=data_string(1) rawlongo3(mi-1,year_ind).intena(di-1,hi,mini)=float(data_string(5)) rawlongo3(mi-1,year_ind).intenb(di-1,hi,mini)=float(data_string(6)) rawlongo3(mi-1,year_ind).bencht(di-1,hi,mini)=float(data_string(7)) rawlongo3(mi-1,year_ind).lampt(di-1,hi,mini)=float(data_string(8)) rawlongo3(mi-1,year_ind).o3lampt(di-1,hi,mini)=float(data_string(9)) rawlongo3(mi-1,year_ind).flowa(di-1,hi,mini)=float(data_string(10)) rawlongo3(mi-1,year_ind).flowb(di-1,hi,mini)=float(data_string(11)) rawlongo3(mi-1,year_ind).pres(di-1,hi,mini)=float(data_string(12)) endif IF model EQ '49C' THEN BEGIN ;this is for 49C hi=long(strmid(data_string(0),0,2)) mini=long(strmid(data_string(0),3,2)) di=long(strmid(data_string(1),3,2)) yst=strmid(year_head,2,2) year_ind=where(years eq '20'+yst) rawlongo3(mi-1,year_ind).model=model rawlongo3(mi-1,year_ind).date_string(di-1,hi,mini)=data_string(1)+'-'+yst rawlongo3(mi-1,year_ind).intena(di-1,hi,mini)=float(data_string(4)) rawlongo3(mi-1,year_ind).intenb(di-1,hi,mini)=float(data_string(5)) rawlongo3(mi-1,year_ind).bencht(di-1,hi,mini)=float(data_string(6)) rawlongo3(mi-1,year_ind).lampt(di-1,hi,mini)=float(data_string(7)) rawlongo3(mi-1,year_ind).o3lampt(di-1,hi,mini)=float(data_string(8)) rawlongo3(mi-1,year_ind).flowa(di-1,hi,mini)=float(data_string(9)) rawlongo3(mi-1,year_ind).flowb(di-1,hi,mini)=float(data_string(10)) rawlongo3(mi-1,year_ind).pres(di-1,hi,mini)=float(data_string(11)) endif ;save in proper position in array rawlongo3.stnid=stnid rawlongo3(mi-1,year_ind).time_string(di-1,hi,mini)=data_string(0) rawlongo3(mi-1,year_ind).flags(di-1,hi,mini)=data_string(2) rawlongo3(mi-1,year_ind).o3ppbv(di-1,hi,mini)=float(data_string(3)) ct=ct+1l skip1: ENDWHILE skip4: free_lun,lun close, lun endfor ; end of each file ;read in long files data to fill the gap ; ;write out the files for i=0,n_elements(years) -1 do begin for mi=0,11 do begin cind=where(strcompress(rawlongo3(mi,i).o3ppbv,/remove) ne 'NaN',ctrl) if ctrl lt 10 then goto, skipout if mi+1 lt 10 then outfile=datapath+strlowcase(site_code)+'0'+ strcompress(mi+1,/remove)+strcompress(years(i),/remove)+'.lrc' $ else outfile=datapath+strlowcase(site_code)+ strcompress(mi+1,/remove)+strcompress(years(i),/remove)+'.lrc' ;if site_code eq 'MLO' and long(years(i)) le 2011 then outfile=strjoin(strsplit(outfile,'.lrc',/extract,/regex)+'_lst.lrc') ;if site_code eq 'BAO' then outfile='\\ccgg\ccg\towers\SharedFiles\RadonOzoneCeilometerData\BAOgroundBasedOzone\bao_'+dates(0)+'.src' openw,lun,/get,outfile printf, lun,';;' printf, lun,';; Stn: ',stnid," ", rawlongo3(mi,i).site_code printf, lun,';;' printf, lun,';; Model ', rawlongo3(mi,i).model printf, lun,';;' printf, lun,colhdr for di=0,30 do begin for hi=0,23 do begin for mini=0,59 do begin if strcompress(rawlongo3(mi,i).o3ppbv(di,hi,mini),/remove) ne 'NaN' then begin ;if rawlongo3(mi,i).o3ppbv(di,hi,mini) gt -5000 and rawlongo3(mi,i).o3ppbv(di,hi,mini) lt 15000. then begin printf,lun,format='(A5," ",A8," ",A8," ",F8.4," ",I8," ",I8," ",f8.4," ",f8.4," ",f8.4," ",f8.4," ",f8.4," ",f8.4)',$ rawlongo3(mi,i).time_string(di,hi,mini),rawlongo3(mi,i).date_string(di,hi,mini),rawlongo3(mi,i).flags(di,hi,mini),$ rawlongo3(mi,i).o3ppbv(di,hi,mini),rawlongo3(mi,i).intena(di,hi,mini),rawlongo3(mi,i).intenb(di,hi,mini),rawlongo3(mi,i).bencht(di,hi,mini),$ rawlongo3(mi,i).lampt(di,hi,mini),rawlongo3(mi,i).o3lampt(di,hi,mini),rawlongo3(mi,i).flowa(di,hi,mini),rawlongo3(mi,i).flowb(di,hi,mini),$ rawlongo3(mi,i).pres(di,hi,mini) endif endfor ;mini endfor ; hi endfor ;di close, lun free_lun,lun skipout: endfor endfor end