#!/bin/bash
#Sites with CPD3 CLAP data (clean archive)
#Use four years of data (2013-2016, inclusive)
#
#ARN A11 2013 2017
#BND A11 2013 2017
#BRW A12 2013 2014-08-21T14:24:00Z
#BRW A11 2014-08-21T14:24:00Z 2017
#GSN A11 2013 2016-06-20T00:00:00Z
#LLN A12 2013 2017
#MLO A12 2013 2013-07-18T20:21:10Z
#MLO A11 2013-07-18T20:21:10Z 2017
#SUM A12 2013 2015-08-17T17:43:02Z
#SUM A11 2015-08-17T17:43:02Z 2017
#THD A12 2013 2013-06-10T17:06:37Z
#THD A11 2013-06-10T17:06:37Z 2017

# get hourly-averaged SSA values
da.get arn 'B[sa][BGR]_(S11|A11)' 2013 2017 avgh \
 | da.calc.intensives \
 | da.select --include="ZSSA.*" > arn_c1.c3d

da.get bnd 'B[sa][BGR]_(S11|A11)' 2013 2017 avgh \
 | da.calc.intensives \
 | da.select --include="ZSSA.*" > bnd_c1.c3d

da.get brw 'B[sa][BGR]_(S11|A12)' 2013 2014-08-21T14:00:00Z avgh \
 | da.calc.intensives \
 | da.select --include="ZSSA.*" > brw_c2.c3d
da.get brw 'B[sa][BGR]_(S11|A11)' 2014-08-21T15:00:00Z 2017 avgh \
 | da.calc.intensives \
 | da.select --include="ZSSA.*" > brw_c1.c3d

da.get gsn 'B[sa][BGR]_(S11|A11)' 2013 2017 avgh \
 | da.calc.intensives \
 | da.select --include="ZSSA.*" > gsn_c1.c3d

da.get lln 'B[sa][BGR]_(S11|A12)' 2013 2017 avgh \
 | da.calc.intensives \
 | da.select --include="ZSSA.*" > lln_c2.c3d

da.get mlo 'B[sa][BGR]_(S11|A12)' 2013 2013-07-18T20:00:00Z avgh \
 | da.calc.intensives \
 | da.select --include="ZSSA.*" > mlo_c2.c3d
da.get mlo 'B[sa][BGR]_(S11|A11)' 2013-07-18T21:00:00Z 2017 avgh \
 | da.calc.intensives \
 | da.select --include="ZSSA.*" > mlo_c1.c3d

da.get sum 'B[sa][BGR]_(S11|A12)' 2013 2015-08-17T17:00:00Z avgh \
 | da.calc.intensives \
 | da.select --include="ZSSA.*" > sum_c2.c3d
da.get sum 'B[sa][BGR]_(S11|A11)' 2015-08-17T18:00:00Z 2017 avgh \
 | da.calc.intensives \
 | da.select --include="ZSSA.*" > sum_c1.c3d

da.get thd 'B[sa][BGR]_(S11|A12)' 2013 2013-06-10T17:00:00Z avgh \
 | da.calc.intensives \
 | da.select --include="ZSSA.*" > thd_c2.c3d
da.get thd 'B[sa][BGR]_(S11|A11)' 2013-06-10T18:00:00Z 2017 avgh \
 | da.calc.intensives \
 | da.select --include="ZSSA.*" > thd_c1.c3d

# get raw and clean data
da.get arn '(Ba|Ir)[BGR]_A11,F1_A11,L_A11' 2013 2017 'raw,clean' \
 | da.corr.removeundefined --remove=":raw:L_A11" --require=":clean:Ba[BGR]_A11" \
 | da.select --exclude=":raw:F1_A11" --set-apply=":clean:F1_A11" --set-archive=raw \
 | da.select --include=":raw:" --set-archive=avgh \
 | da.avg \
 | da.select --include="Ba.*" > arn_r1.c3d

da.get bnd '(Ba|Ir)[BGR]_A11,F1_A11,L_A11' 2013 2017 'raw,clean' \
 | da.corr.removeundefined --remove=":raw:L_A11" --require=":clean:Ba[BGR]_A11" \
 | da.select --exclude=":raw:F1_A11" --set-apply=":clean:F1_A11" --set-archive=raw \
 | da.select --include=":raw:" --set-archive=avgh \
 | da.avg \
 | da.select --include="Ba.*" > bnd_r1.c3d

da.get brw '(Ba|Ir)[BGR]_A12,F1_A12,L_A12' 2013 2014-08-21T14:00:00Z 'raw,clean' \
 | da.corr.removeundefined --remove=":raw:L_A12" --require=":clean:Ba[BGR]_A12" \
 | da.select --exclude=":raw:F1_A12" --set-apply=":clean:F1_A12" --set-archive=raw \
 | da.select --include=":raw:" --set-archive=avgh \
 | da.avg \
 | da.select --include="Ba.*" > brw_r2.c3d
da.get brw '(Ba|Ir)[BGR]_A11,F1_A11,L_A11' 2014-08-21T15:00:00Z 2017 'raw,clean' \
 | da.corr.removeundefined --remove=":raw:L_A11" --require=":clean:Ba[BGR]_A11" \
 | da.select --exclude=":raw:F1_A11" --set-apply=":clean:F1_A11" --set-archive=raw \
 | da.select --include=":raw:" --set-archive=avgh \
 | da.avg \
 | da.select --include="Ba.*" > brw_r1.c3d

da.get gsn '(Ba|Ir)[BGR]_A11,F1_A11,L_A11' 2013 2017 'raw,clean' \
 | da.corr.removeundefined --remove=":raw:L_A11" --require=":clean:Ba[BGR]_A11" \
 | da.select --exclude=":raw:F1_A11" --set-apply=":clean:F1_A11" --set-archive=raw \
 | da.select --include=":raw:" --set-archive=avgh \
 | da.avg \
 | da.select --include="Ba.*" > gsn_r1.c3d

da.get lln '(Ba|Ir)[BGR]_A12,F1_A12,L_A12' 2013 2017 'raw,clean' \
 | da.corr.removeundefined --remove=":raw:L_A12" --require=":clean:Ba[BGR]_A12" \
 | da.select --exclude=":raw:F1_A12" --set-apply=":clean:F1_A12" --set-archive=raw \
 | da.select --include=":raw:" --set-archive=avgh \
 | da.avg \
 | da.select --include="Ba.*" > lln_r2.c3d

da.get mlo '(Ba|Ir)[BGR]_A12,F1_A12,L_A12' 2013 2013-07-18T20:00:00Z 'raw,clean' \
 | da.corr.removeundefined --remove=":raw:L_A12" --require=":clean:Ba[BGR]_A12" \
 | da.select --exclude=":raw:F1_A12" --set-apply=":clean:F1_A12" --set-archive=raw \
 | da.select --include=":raw:" --set-archive=avgh \
 | da.avg \
 | da.select --include="Ba.*" > mlo_r2.c3d
da.get mlo '(Ba|Ir)[BGR]_A11,F1_A11,L_A11' 2013-07-18T21:00:00Z 2017 'raw,clean' \
 | da.corr.removeundefined --remove=":raw:L_A11" --require=":clean:Ba[BGR]_A11" \
 | da.select --exclude=":raw:F1_A12" --set-apply=":clean:F1_A12" --set-archive=raw \
 | da.select --include=":raw:" --set-archive=avgh \
 | da.avg \
 | da.select --include="Ba.*" > mlo_r1.c3d

da.get sum '(Ba|Ir)[BGR]_A12,F1_A12,L_A12' 2013 2015-08-17T17:00:00Z 'raw,clean' \
 | da.corr.removeundefined --remove=":raw:L_A12" --require=":clean:Ba[BGR]_A12" \
 | da.select --exclude=":raw:F1_A12" --set-apply=":clean:F1_A12" --set-archive=raw \
 | da.select --include=":raw:" --set-archive=avgh \
 | da.avg \
 | da.select --include="Ba.*" > sum_r2.c3d
da.get sum '(Ba|Ir)[BGR]_A11,F1_A11,L_A11' 2015-08-17T18:00:00Z 2017 'raw,clean' \
 | da.corr.removeundefined --remove=":raw:L_A11" --require=":clean:Ba[BGR]_A11" \
 | da.select --exclude=":raw:F1_A11" --set-apply=":clean:F1_A11" --set-archive=raw \
 | da.select --include=":raw:" --set-archive=avgh \
 | da.avg \
 | da.select --include="Ba.*" > sum_r1.c3d

da.get thd '(Ba|Ir)[BGR]_A12,F1_A12,L_A12' 2013 2013-06-10T17:00:00Z 'raw,clean' \
 | da.corr.removeundefined --remove=":raw:L_A12" --require=":clean:Ba[BGR]_A12" \
 | da.select --exclude=":raw:F1_A12" --set-apply=":clean:F1_A12" --set-archive=raw \
 | da.select --include=":raw:" --set-archive=avgh \
 | da.avg \
 | da.select --include="Ba.*" > thd_r2.c3d
da.get thd '(Ba|Ir)[BGR]_A11,F1_A11,L_A11' 2013-06-10T18:00:00Z 2017 'raw,clean' \
 | da.corr.removeundefined --remove=":raw:L_A11" --require=":clean:Ba[BGR]_A11" \
 | da.select --exclude=":raw:F1_A11" --set-apply=":clean:F1_A11" --set-archive=raw \
 | da.select --include=":raw:" --set-archive=avgh \
 | da.avg \
 | da.select --include="Ba.*" > thd_r1.c3d


# to rename A12 to A11
#da.select --capture-variable='(.*)_A12' --set-apply='.*_A12' --set-variable='${1}_A11'

da.multiplex arn_c1.c3d arn_r1.c3d > arn.c3d
da.multiplex bnd_c1.c3d bnd_r1.c3d > bnd.c3d
da.multiplex gsn_c1.c3d gsn_r1.c3d > gsn.c3d
da.multiplex brw_c1.c3d brw_r1.c3d \
 '| da.select --capture-variable="(.*)_A12" --set-apply=".*_A12" --set-variable="\${1}_A11" brw_c2.c3d' \
 '| da.select --capture-variable="(.*)_A12" --set-apply=".*_A12" --set-variable="\${1}_A11" brw_r2.c3d' \
 > brw.c3d
da.multiplex mlo_c1.c3d mlo_r1.c3d \
 '| da.select --capture-variable="(.*)_A12" --set-apply=".*_A12" --set-variable="\${1}_A11" mlo_c2.c3d' \
 '| da.select --capture-variable="(.*)_A12" --set-apply=".*_A12" --set-variable="\${1}_A11" mlo_r2.c3d' \
 > mlo.c3d
da.multiplex sum_c1.c3d sum_r1.c3d \
 '| da.select --capture-variable="(.*)_A12" --set-apply=".*_A12" --set-variable="\${1}_A11" sum_c2.c3d' \
 '| da.select --capture-variable="(.*)_A12" --set-apply=".*_A12" --set-variable="\${1}_A11" sum_r2.c3d' \
 > sum.c3d
da.multiplex thd_c1.c3d thd_r1.c3d \
 '| da.select --capture-variable="(.*)_A12" --set-apply=".*_A12" --set-variable="\${1}_A11" thd_c2.c3d' \
 '| da.select --capture-variable="(.*)_A12" --set-apply=".*_A12" --set-variable="\${1}_A11" thd_r2.c3d' \
 > thd.c3d
da.multiplex \
 '| da.select --capture-variable="(.*)_A12" --set-apply=".*_A12" --set-variable="\${1}_A11" lln_c2.c3d' \
 '| da.select --capture-variable="(.*)_A12" --set-apply=".*_A12" --set-variable="\${1}_A11" lln_r2.c3d' \
 > lln.c3d