% Processa dados de NO2-CAPS e salva no formato padrão do LFA (level 1, 2 e 3)
% Experimento Diadema Trailer Dez2016
%
% Functions used: export_LFA_GoAmazon_jan2015, fmedia_qualquer_coisa2
% Outputs: raw [MatlabTime(UTC),NO2(ppb),RawLoss(Mm-1),Pressure(Torr),Temperature(K),Signal(adim),Status,LastBaseline(Mm-1)]
% NO2_Level1_5min (dd-mmm-yyyy HH:MM; sec_since_1970; mean_no2; std_no2; median_no2; p25_no2; p75_no2; max_no2; min_no2; num_no2)
% NO2_Level1_30min
% Status: abcde, where: a)pump status(0=pump off, normal operation, 1=pump on, normal operation, 5=alarm); b)baseline status(0=normal operation, 1=baseline on (flush), 2=baseline on (measurement)); c)unused; d)unused(=0); e)Wavelength(=4)
% Criteria for Level1: Temp>315K; Pressure>700Torr; -10<NO2<200ppb; remove pump alarm and baseline data points
% Criteria for Level2: remove calibration and maintanace periods and eventually correct the baseline
%
% Obs1: os baselines são feitos manualmente, e não há flags para isso. É
% necessário cruzar o logbook com os dados para remover períodos de
% baseline. Nesses períodos, há um degrau na série temporal de pressão, o
% que facilita a identificação.
% Obs2: Apesar do manual indicar precisão de 0.1 ppb, o zero noise é de +/-1.0 ppb
% Obs3: Cada vez que o instrumento reinicia, leva cerca de 1 hora para
% estabilizar a temperatura
%
% ---------------- Parametros -----------------------
caminho='C:\A\Diadema2016\data_trailer_2016\CAPS Data\';
caminho1='C:\A\Diadema2016\processados\NO2_CAPS\';
fuso=0; % fuso horario para plotar hora local
tit='Diadema-2016';
Site='Diadema_2016';
datalimite=datenum(2017,12,31);
filtmanut=true;
basicplots=true;
lv1=false; % logical para gerar arquivos level1
lv2=false; % logical para gerar arquivos level2
%
% inicio e termino de periodos de manutencao (horario em utc): removidos das analises
% não foi necessário fazer ajuste de baseline
if(filtmanut)
C1=xlsread('C:\A\Diadema2016\logbook_trailer_Diadema2016.xlsx','todos','A4:M135');
C2=xlsread('C:\A\Diadema2016\logbook_trailer_Diadema2016.xlsx','no2','A4:O34');
manut(:,1)=cat(1,datenum(C1(:,1:6)),datenum(C2(:,1:6)));
manut(:,2)=cat(1,datenum(C1(:,7:12)),datenum(C2(:,7:12)));
manut(:,3)=cat(1,C1(:,13),C2(:,13));
manut(:,4:5)=cat(1,zeros(length(C1),2),C2(:,14:15));
end
%
%media=false; % logical para calcular médias de 30seg para processar os dados brutos (2-3seg)
% inicio e termino de periodo de zoom para fazer grafico
zoom1=[datenum(2016,11,22,0,0,0) datenum(2017,3,15,0,0,0)];
zoom2=[datenum(2017,03,05,0,0,0) datenum(2017,3,15,0,0,0)];%
% ---------------- Fim dos parametros --------------------
%
raw=zeros(1,8);
files=dir(caminho);
disp('-----Reading raw data files')
for ifile=3:size(files,1)
nome=getfield(files, {ifile,1}, 'name');
disp(nome)
fid=fopen(strcat(caminho,nome));
% --------- Retrive data from raw file ---------
textscan(fid,'%s',63); % skipping
header=textscan(fid,'%s',1);
C=textscan(fid,'%f%f%f%f%f%f%f%f%f%s','delimiter',',');
fclose(fid);
raw_=zeros(size(C{1,1},1),size(C,2)-1);
for j=1:size(C,2)-1 % somente numeros
raw_(:,j)=C{1,j};
end
raw_(:,1)=datenum(C{1,10},'yyyy/mm/dd HH:MM:SS');
raw_(:,7)=[];
%
if(mod(raw_(length(raw_),1),1)*24 < 23)
disp(['!!! Warning: File ',nome,' finished before 23:00 utc'])
end
%
raw=cat(1,raw,raw_);
clear C raw_
end
%
raw(1,:)=[];
raw(isnan(raw(:,2)),:)=[]; % remove lines with NO2=NaN
aux=sortrows(raw,1); % sort data based on day of year
clear raw
raw=aux;
clear aux
%
raw(raw(:,1)>datalimite,:)=[];
save([caminho1,tit,'_NO2_level0.mat'],'raw','-mat')
%
% Remove maintanance periods from dataset
if(filtmanut)
raworig=raw;
for i=1:size(manut,1)
aux=find(raw(:,1)>=manut(i,1) & raw(:,1)<=manut(i,2));
if(manut(i,3)==0)
raw(aux,:)=[]; % remove maintanance periods from dataset
else
% detrend baseline (ozone_detrend.m)
if(manut(i,4)==0)
raw(aux,2)=raw(aux,2)+(-polyval(manut(i,4:5),raw(aux,1)));
else
raw(aux,2)=raw(aux,2)+(polyval(manut(i,4:5),raw(aux(1),1))-polyval(manut(i,4:5),raw(aux,1)));
end
end
end
figure,plot(raworig(:,1),raworig(:,2),'r.'),hold on,grid on
plot(raw(:,1),raw(:,2),'b.')
dynamicDateTicks
clear aux
save([caminho1,tit,'_NO2_clean.mat'],'raw','-mat')
end

% ------ Basic plots ---------------------------
if(basicplots)
figure
subplot(2,1,1),plot(raw(:,1)+(fuso/24),raw(:,2),'b.')
title(tit,'FontSize',14)
ylabel('NO_2 (ppb)','FontSize',14),grid on, axis tight
%xlabel('UTC Time')
dynamicDateTicks
subplot(2,1,2),plot(mod(raw(:,1)+(fuso/24),1),raw(:,2),'b.')
ylabel('NO_2 (ppb)','FontSize',14),grid on,xlabel('UTC Time','FontSize',14), axis tight
set(gca,'XTick',[0.125 0.25 0.375 0.5 0.625 0.75 0.875 1.0])
set(gca,'XTickLabel',{'3:00';'6:00';'9:00';'12:00';'15:00';'18:00';'21:00';'0:00'})
%hgsave(gcf,strcat(caminho1,'NO2_conc'))
saveas(gcf,strcat(caminho1,tit,'_NO2_conc'),'jpg')
%
aux=find(raw(:,1)>=zoom1(1) & raw(:,1)<=zoom1(2));
figure
subplot(2,1,1),plot(raw(aux,1)+(fuso/24),raw(aux,2),'b.')
title(tit,'FontSize',14)
ylabel('NO_2 (ppb)','FontSize',14),grid on
%xlabel('UTC Time')
%dynamicDateTicks
set(gca,'XTick',zoom1(1):2:zoom1(2))
set(gca,'XTickLabel',datestr(zoom1(1):2:zoom1(2),'dd/mm'))
set(gca,'XLim',[zoom1(1) zoom1(2)])
subplot(2,1,2),plot(mod(raw(aux,1)+(fuso/24),1),raw(aux,2),'b.')
ylabel('NO_2 (ppb)','FontSize',14),grid on,xlabel('UTC Time','FontSize',14), axis tight
set(gca,'XTick',[0.125 0.25 0.375 0.5 0.625 0.75 0.875 1.0])
set(gca,'XTickLabel',{'3:00';'6:00';'9:00';'12:00';'15:00';'18:00';'21:00';'0:00'})
%hgsave(gcf,strcat(caminho1,tit,'-zoom1-conc'))
saveas(gcf,strcat(caminho1,tit,'_NO2_zoom1_conc'),'jpg')
%
aux=find(raw(:,1)>=zoom2(1) & raw(:,1)<=zoom2(2));
figure
subplot(2,1,1),plot(raw(aux,1)+(fuso/24),raw(aux,2),'b.')
title(tit,'FontSize',14)
ylabel('NO_2 (ppb)','FontSize',14),grid on
set(gca,'XTick',zoom2(1):1:zoom2(2))
set(gca,'XTickLabel',datestr(zoom2(1):1:zoom2(2),'dd/mm'))
set(gca,'XLim',[zoom2(1) zoom2(2)])
subplot(2,1,2),plot(mod(raw(aux,1)+(fuso/24),1),raw(aux,2),'b.')
ylabel('NO_2 (ppb)','FontSize',14),grid on,xlabel('UTC Time','FontSize',14), axis tight
set(gca,'XTick',[0.125 0.25 0.375 0.5 0.625 0.75 0.875 1.0])
set(gca,'XTickLabel',{'3:00';'6:00';'9:00';'12:00';'15:00';'18:00';'21:00';'0:00'})
%hgsave(gcf,strcat(caminho1,tit,'-zoom2-conc'))
saveas(gcf,strcat(caminho1,tit,'_NO2_zoom2_conc'),'jpg')
%
% Ciclo diurno médio
first = 0; % first averaging time
last = 23.5/24; % last averaging time
delta = 30/60/24; % 30min
mdata=cat(2,mod(raw(aux,1),1),raw(aux,2));
tol=0;
stat=fmedia_qualquer_coisa2(mdata,first,last,delta,tol);
figure
errorbar(stat(:,1),stat(:,2),stat(:,3),'b.')
dynamicDateTicks
set(gca,'XLim',[0 1])
ylabel('NO_2 (ppb)','FontSize',14),grid on,xlabel('UTC Time','FontSize',14)
title([tit,' Avg Diurnal Cycle'],'FontSize',14)
%hgsave(gcf,strcat(caminho1,tit,'-dcycle_alldata'))
saveas(gcf,strcat(caminho1,tit,'NO2_dcycle_level0'),'jpg')
end
%
if(lv1)
% ------ Filters for Level1 data ---------------------------
%raw [MatlabTime(UTC),NO2(ppb),RawLoss(Mm-1),Pressure(Torr),Temperature(K),Signal(adim),Status,LastBaseline(Mm-1)]
% Status: abcde, where: a)pump status(0=pump off, normal operation, 1=pump on, normal operation, 5=alarm); b)baseline status(0=normal operation, 1=baseline on (flush), 2=baseline on (measurement)); c)unused; d)unused(=0); e)Wavelength(=4)
C=load('C:\A\Diadema2016\processados\NO2_CAPS\Diadema-2016_NO2_clean.mat');
raw=C.raw;
C=load('C:\A\Diadema2016\processados\NO2_CAPS\Diadema-2016_NO2_level0.mat');
raw_level0=C.raw;
tit='Diadema-2016';
%
orig_size=size(raw_level0,1);
level1=raw;
level1(:,9)=floor(raw(:,7)/10000); % pump status
level1(:,10)=floor(raw(:,7)/1000)-10;%.*status(:,1); % baseline status
%
level1(level1(:,9)==5,:)=[]; % remove pump alarm data points
level1(level1(:,10)==1,:)=[]; % remove baseline flushing data points
level1(level1(:,4)<650,:)=[]; % remove Press<650 Torr
level1(level1(:,5)<315,:)=[]; % remove T<315 K
%level1(level1(:,6)<1.55E+05,:)=[]; % remove signal<1.55E+05
level1(level1(:,2)<-10,:)=[]; % remove NO2<-10 ppb
level1(level1(:,2)>200,:)=[]; % remove NO2>200 ppb
%
cor='brgkcbrgk';
var={'NO2','RawLoss','Pressure','Temperature','Signal','Status','LastBaseline','PumpStatus','BaselineStatus'};
for j=2:10
if(j==7);continue;end
figure
plot(level1(:,1),level1(:,j),['-',cor(j-1),'.'])
dynamicDateTicks
title(var{1,j-1}),grid on,hold on
end
figure,plot(raw_level0(:,1),raw_level0(:,2),'b.'),hold on,grid on, dynamicDateTicks
plot(level1(:,1),level1(:,2),'r.')
%
% Save Level 1 data
Instrument='CAPS';
Model='Aerodyne';
serial_num='114001';
Site=tit;
Val_level='1';
Proc_vers='1.0';
labels{1}='NO2';
units{1}='(ppb)';
export_LFA_GoAmazon_jan2015(labels,units,Instrument,Model,serial_num,Site,Val_level,Proc_vers,orig_size,level1(:,1),level1(:,2))
end
%
% -------------- Level 2 data --------------------------------------------
% Criteria for Level2: correct the baseline when needed
if(lv2)
% Lê o arquivo Level 1 5min que já salvou
aux=importdata('C:\A\Diadema2016\processados\LFA_standard\Diadema-2016_CAPS_Aerodyne_Level1_5min.csv');
level1_5min=aux.data(:,1:2);
clear aux level1
level1_5min(:,1)=level1_5min(:,1)/(24*60*60)+datenum(1970,1,1); % unix time to matlab time
orig_size=sum(~isnan(level1_5min(:,2)));
tit='Diadema-2016';
%
Instrument='CAPS';
Model='Aerodyne';
serial_num='114001';
Site=tit;
Val_level='2';
Proc_vers='1.0';
labels{1}='NO2';
units{1}='(ppb)';
export_LFA_GoAmazon_jan2015(labels,units,Instrument,Model,serial_num,Site,Val_level,Proc_vers,orig_size,level1_5min(:,1),level1_5min(:,2))
end
%
% Verificação final
% aux=importdata('C:\A\Diadema2016\processados\LFA_standard\Diadema-2016_CAPS_Aerodyne_Level1_5min.csv');
% level1_5min=aux.data(:,1:2);
% level1_5min(:,1)=level1_5min(:,1)/(24*60*60)+datenum(1970,1,1); % unix time to matlab time
% aux=importdata('C:\A\Diadema2016\processados\LFA_standard\Diadema-2016_CAPS_Aerodyne_Level2_5min.csv');
% level2_5min=aux.data(:,1:2);
% level2_5min(:,1)=level2_5min(:,1)/(24*60*60)+datenum(1970,1,1); % unix time to matlab time
% figure, plot(level1_5min(:,1),level1_5min(:,2),'b.'),grid on, hold on,dynamicDateTicks
% plot(level2_5min(:,1),level2_5min(:,2),'r.')