# Notes on using R to plot/analyze clap-psap data files

# create a merged data file
cd /aer/work/derek_work/clap_psap_v2/output/absorption
# add station ID to each file
for I in ???; do prepend text="${I}," first="STN," $I > $I.csv; done


# merge the csv files
mergeheaders *.csv > all.tmp

# edit header line to create harmonized variable names
head -1 tmp.csv | perl -pe 's/([sG])1/\1/g' | tr -d m > all.csv
# sort data, get rid of duplicate lines, delete spaces and quotes
grep -v EPOCH tmp.csv | sort -u -t, --key=3,3  >> all.csv
rm tmp.csv
rm 2016_*.csv



# R commands to create all.Rdata from all.csv
library(tidyverse)
library(readr)
library(stringr)
setwd("/aer/work/derek_work/clap_psap_v2/output")

da <- read.csv("all.csv")
da$EPOCH <- as.POSIXct(da$EPOCH, tz="GMT", origin=ISOdatetime(1970,1,1,0,0,0,tz="GMT"))


# calculate filter optical depths
da$OfB_A11=-log(da$IrB_A11)
da$OfB_A12=-log(da$IrB_A12)
da$OfB_A13=-log(da$IrB_A13)
da$OfB_A14=-log(da$IrB_A14)
da$OfG_A11=-log(da$IrG_A11)
da$OfG_A12=-log(da$IrG_A12)
da$OfG_A13=-log(da$IrG_A13)
da$OfG_A14=-log(da$IrG_A14)
da$OfR_A11=-log(da$IrR_A11)
da$OfR_A12=-log(da$IrR_A12)
da$OfR_A13=-log(da$IrR_A13)
da$OfR_A14=-log(da$IrR_A14)
# calculate reference filter optical depth
da$OaB_A11=da$OeB_A11-da$OsB_A11
da$OaB_A12=da$OeB_A12-da$OsB_A12
da$OaB_A13=da$OeB_A13-da$OsB_A13
da$OaB_A14=da$OeB_A14-da$OsB_A13
da$OaG_A11=da$OeG_A11-da$OsG_A11
da$OaG_A12=da$OeG_A12-da$OsG_A12
da$OaG_A13=da$OeG_A13-da$OsG_A13
da$OaG_A14=da$OeG_A14-da$OsG_A13
da$OaR_A11=da$OeR_A11-da$OsR_A11
da$OaR_A12=da$OeR_A12-da$OsR_A12
da$OaR_A13=da$OeR_A13-da$OsR_A13
da$OaR_A14=da$OeR_A14-da$OsR_A13
#save.image(file="all.Rdata")

#make a smaller data frame with just optical depth data, and gather the optical depths
od <- as_tibble(da) %>% select(runn, type, EPOCH, matches("^O[fa]"))
od <- od %>% gather(fVAR,Of,OfB_A11:OfR_A14)
od <- od %>% gather(aVAR,Oa,OaB_A11:OaR_A14)
od$fwl=str_match(od$fVAR,"^..(.)")[,2]
od$finst=str_match(od$fVAR,"^......(.)")[,2]
od$awl=str_match(od$aVAR,"^..(.)")[,2]
od$ainst=str_match(od$aVAR,"^......(.)")[,2]
# now just pick out rows where wavelengths and instruments match
od <- od %>% filter(fwl==awl,finst==ainst)
# rename wavelength and instrument codes, also type variable
od <- od %>% rename(wl=fwl, inst=finst, f_type=type, run=runn)
# delete variable names
od <- od %>% select(-fVAR, -aVAR, -awl, -ainst)
# convert wl and inst to factors
od$f_wl <- as.factor( od$wl )
od$f_inst <- as.factor( od$inst )
od$f_run <- as.factor( od$run )
od$f_filter=as.factor(if_else(od$f_inst=="1" & od$run < 60 | od$f_inst=="2" & od$run >= 60 | od$f_inst=="3" & od$run < 60 | od$f_inst=="4","Pall","Azumi"))
save.image(file="all.Rdata")