% Processa dados de NO2-CAPS do T1 e salva no formato padrão do LFA (level 1, 2 e 3)
%
% Functions used: export_LFA_GoAmazon_jan2015, fmedia_qualquer_coisa2
% Outputs: raw [MatlabTime(UTC),NO2(ppb),RawLoss(Mm-1),Pressure(Torr),Temperature(K),Signal(adim),Status,LastBaseline(Mm-1)]
%          NO2_Level1_5min (dd-mmm-yyyy HH:MM; sec_since_1970; mean_no2; std_no2; median_no2; p25_no2; p75_no2; max_no2; min_no2; num_no2)
%          NO2_Level1_30min
% Status: abcde, where: a)pump status(0=pump off, normal operation, 1=pump on, normal operation, 5=alarm); b)baseline status(0=normal operation, 1=baseline on (flush), 2=baseline on (measurement)); c)unused; d)unused(=0); e)Wavelength(=4)
% Criteria for Level1: Temp>315K; Pressure>700Torr; -10<NO2<200ppb; remove pump alarm and baseline data points  
% Criteria for Level2: remove calibration and maintanace periods and eventually correct the baseline
%
% Obs1: os baselines são feitos manualmente, e não há flags para isso. É
% necessário cruzar o logbook com os dados para remover períodos de
% baseline. Nesses períodos, há um degrau na série temporal de pressão, o
% que facilita a identificação.
% Obs2: Apesar do manual indicar precisão de 0.1 ppb, o zero noise é de +/-1.0 ppb
% Obs3: Cada vez que o instrumento reinicia, leva cerca de 1 hora para
% estabilizar a temperatura
%
% ---------------- Parametros -----------------------
caminho='C:\A\GOAMAZON\T1\NO2_114001sn\raw\';
caminho1='C:\A\GOAMAZON\T1\NO2_114001sn\';
fuso=-4; % fuso horario para plotar hora local
tit='T1_2015-2016_NO2';
datalimite=datenum(2017,1,1);
lv2=false; % logical para gerar arquivos level2
% Read maintanance file
% fid=fopen([caminho1,'NO2_manut.txt']); 
% textscan(fid,'%s',36);  % skipping
% C=textscan(fid,'%s%s%f%f%*s','delimiter',',');
% fclose(fid); 
% manut(:,1)=datenum(C{1,1},'yyyy/mm/dd HH:MM'); % maintanance start time (utc)
% manut(:,2)=datenum(C{1,2},'yyyy/mm/dd HH:MM'); % maintanance end time (utc)
% manut(:,3)=C{1,3}; % Remove/Adjust(0/1)
% manut(:,4)=C{1,4}; % Baseline Adjustment
% clear fid C
%media=false; % logical para calcular médias de 30seg para processar os dados brutos (2-3seg)
%
% ---------------- Fim dos parametros --------------------
%
raw=zeros(1,8);
files=dir(caminho);
disp('-----Reading raw data files')
for ifile=3:size(files,1)
    nome=getfield(files, {ifile,1}, 'name');
    disp(nome)
    fid=fopen(strcat(caminho,nome));
    % --------- Retrive data from raw file ---------
    textscan(fid,'%s',63);  % skipping
    header=textscan(fid,'%s',1);        
    C=textscan(fid,'%f%f%f%f%f%f%f%f%f%s','delimiter',',');       
    fclose(fid);    
    raw_=zeros(size(C{1,1},1),size(C,2)-1);
    for j=1:size(C,2)-1 % somente numeros
        raw_(:,j)=C{1,j};
    end    
    raw_(:,1)=datenum(C{1,10},'yyyy/mm/dd HH:MM:SS');
    raw_(:,7)=[];
    %
    if(mod(raw_(length(raw_),1),1)*24 < 23)
        disp(['!!! Warning: File ',nome,' finished before 23:00 utc'])
    end
    %
    raw=cat(1,raw,raw_);
    clear C raw_    
end
%
raw(1,:)=[];
raw(isnan(raw(:,2)),:)=[]; % remove lines with NO2=NaN
aux=sortrows(raw,1); % sort data based on day of year
clear raw
raw=aux;
clear aux
%
raw(raw(:,1)>datalimite,:)=[];
save([caminho1,tit,'_NO2_Level0.txt'],'raw','-ASCII')
%
% ------ Filters for Level1 data ---------------------------
raw(:,9)=floor(raw(:,7)/10000); % pump status
raw(:,10)=floor(raw(:,7)/1000)-10;%.*status(:,1); % baseline status
orig_size=size(raw,1);
level1=raw;
%
level1(level1(:,9)==5,:)=[]; % remove pump alarm data points
level1(level1(:,10)==1,:)=[]; % remove baseline flushing data points
level1(level1(:,4)<700,:)=[]; % remove Press<700 Torr
level1(level1(:,5)<315,:)=[]; % remove T<315 K
%level1(level1(:,6)<1.55E+05,:)=[]; % remove signal<1.55E+05
level1(level1(:,2)<-10,:)=[]; % remove NO2<-10 ppb
level1(level1(:,2)>200,:)=[]; % remove NO2>200 ppb
%
cor='bkgkcbkgk';
var={'NO2','RawLoss','Pressure','Temperature','Signal','Status','LastBaseline','PumpStatus','BaselineStatus'};
for j=2:10
    if(j==7);continue;end
    figure
    plot(raw(:,1),raw(:,j),'r.'),hold on
    plot(level1(:,1),level1(:,j),['-',cor(j-1),'.'])
    dynamicDateTicks
     title(var{1,j-1}),grid on,hold on
end
%
% ------- Additional periods to remove (specific for T1) ---------------
rem=[datenum(2015,12,01,18,00,00) datenum(2015,12,02,00,00,00);...
    datenum(2015,12,02,00,00,00) datenum(2015,12,08,19,25,00);... % optei por excluir os 6 primeiros dias, pois o zero estava muito fora, e com uma clara tendência de diminuição
    datenum(2015,12,08,19,05,00) datenum(2015,12,08,19,25,00);... % zero check (5ppb)
    datenum(2015,12,10,15,48,00) datenum(2015,12,10,15,57,00);... % zero check (-1.7ppb)
    datenum(2015,12,10,20,41,00) datenum(2015,12,10,20,44,00);...
    datenum(2015,12,13,11,48,00) datenum(2015,12,13,11,55,00);...
    datenum(2015,12,21,11,19,00) datenum(2015,12,21,11,22,00);...
    datenum(2015,12,23,22,20,00) datenum(2015,12,23,23,30,00);... % zero check (-9ppb)
    datenum(2015,12,28,12,15,00) datenum(2015,12,28,13,05,00);...
    datenum(2016,01,02,19,00,00) datenum(2016,01,02,21,00,00);...
    datenum(2016,01,11,22,15,00) datenum(2016,01,11,23,00,00);... % zero check (-5ppb)
    datenum(2016,01,18,20,32,00) datenum(2016,01,18,20,40,00);... % zero check (0ppb)
    datenum(2016,02,01,13,19,00) datenum(2016,02,01,13,41,00);... % zero check (0ppb)
    datenum(2016,02,15,15,45,00) datenum(2016,02,15,15,55,00);... % zero check (0ppb)
    datenum(2016,02,22,13,00,00) datenum(2016,02,22,13,44,00);... % inicializando
    datenum(2016,03,05,01,15,00) datenum(2016,03,05,01,30,00);... % zero check (0ppb)
    datenum(2016,03,22,13,32,00) datenum(2016,03,22,13,40,00);... % zero check (0ppb)
    datenum(2016,04,06,00,35,00) datenum(2016,04,06,00,45,00);... % zero check (0ppb)
    datenum(2016,04,19,13,15,00) datenum(2016,04,19,13,30,00)]; % zero check (0ppb)
for i=1:length(rem)
    level1(level1(:,1)>=rem(i,1) & level1(:,1)<=rem(i,2),2)=NaN; 
end
level1(isnan(level1(:,1)),2)=[];
figure,plot(raw(:,1),raw(:,2),'r.'),hold on,grid on
plot(level1(:,1),level1(:,2),'b.')
dynamicDateTicks
% ------------ Save Level 1 data -----------------------------------------
Instrument='CAPS';
Model='Aerodyne';
serial_num='114001';
Site='T1';
Val_level='1';
Proc_vers='1.0';
labels{1}='NO2';
units{1}='(ppb)';
export_LFA_GoAmazon_jan2015(labels,units,Instrument,Model,serial_num,Site,Val_level,Proc_vers,orig_size,level1(:,1),level1(:,2))
%
% -------------- Level 2 data --------------------------------------------
if(lv2)
    % Lê o arquivo Level 1 5min que acabou de salvar
    %fdir=['0_Ascii-files/Level',Val_level,'/'];
    %fname=[Site,'_',Instrument,'_',Model,'_Level',Val_level];
    aux=importdata('C:\A\GOAMAZON\T1\LFA_standard\T1_2015-2016_NO2_CAPS_Aerodyne_Level1_5min.csv');
    level1_5min=aux.data;
    clear aux level1
    level1_5min(:,1)=level1_5min(:,1)/(24*60*60)+datenum(1970,1,1); % unix time to matlab time
    orig_size=sum(~isnan(level1_5min(:,2))); 
    orig=level1_5min;
    tit='T1_2015-2016_NO2';
    %
    % Detrending baseline accoring to zero checks
    aux=find(level1_5min(:,1)>=datenum(2015,12,08,19,25,00) & level1_5min(:,1)<=datenum(2015,12,10,15,48,00));
    A=-1.7/(datenum(2015,12,10,15,48,00)-datenum(2015,12,08,19,25,00)); % ppb/day
    A=A/(24*12); % ppb/5min
    level1_5min(aux,2)=level1_5min(aux,2)-A*(0:length(aux)-1)';
    %
    aux=find(level1_5min(:,1)>=datenum(2015,12,10,15,57,00) & level1_5min(:,1)<=datenum(2015,12,23,22,20,00));
    A=-9/(datenum(2015,12,23,22,20,00)-datenum(2015,12,10,15,57,00)); % ppb/day
    A=A/(24*12); % ppb/5min
    level1_5min(aux,2)=level1_5min(aux,2)-A*(0:length(aux)-1)';
    %
    aux=find(level1_5min(:,1)>=datenum(2015,12,23,23,30,00) & level1_5min(:,1)<=datenum(2016,01,11,22,15,00));
    A=-5/(datenum(2016,01,11,22,15,00)-datenum(2015,12,23,23,30,00)); % ppb/day
    A=A/(24*12); % ppb/5min
    level1_5min(aux,2)=level1_5min(aux,2)-A*(0:length(aux)-1)';
    clear aux
    %
    figure
    plot(orig(:,1),orig(:,2),'r.'),grid on, hold on
    plot(level1_5min(:,1),level1_5min(:,2),'b.'),dynamicDateTicks
    %
    Instrument='CAPS';
    Model='Aerodyne';
    serial_num='114001';
    Site=tit;
    Val_level='2';
    Proc_vers='1.0';
    labels{1}='NO2';
    units{1}='(ppb)';
    export_LFA_GoAmazon_jan2015(labels,units,Instrument,Model,serial_num,Site,Val_level,Proc_vers,orig_size,level1_5min(:,1),level1_5min(:,2))
end


