% CO N2O Los Gatos data processing (LRizzo)
% Read and process raw data, saving output files according to the LFA standard
% Functions used: frepeated; export_LFA_GoAmazon_jan2015
% Output: alldata [tmat CO(ppb) N2O(ppb) H2O(ppb) GasP(torr) GasT(C) AmbT(C) LTC0(V) AIN5 AIN6]
% 
% Criteria for Level1: (Exclude records when) AIN5<0.67; AIN5>0.68; LTC0<=0.2; GasT<26; GasT>40; 
%                      CO<=-1ppm; CO>3ppm; N2O<=-1ppm; N2O>50ppm
%                      Correct for clock delays in comparison to MAAP and Benzene  
% Criteria for Level2: remove calibration and maintanace periods and eventually correct the baseline
%
% Parametros de entrada:
caminho='C:\A\GOAMAZON\T1\LosGatos_110062sn\extraidos\';
caminho1='C:\A\GOAMAZON\T1\LosGatos_110062sn\';
tit='T1_2015-2016_CO';
%media=false; % logical para calcular médias de 30seg para processar os dados brutos (1seg)
%
% ---------------- Fim dos parametros --------------------
%
files=dir(caminho);
alldata=zeros(1,11);
disp('-----Reading raw data files')
for ifile=3:size(files,1)
        nome=getfield(files, {ifile,1}, 'name');
        disp(nome)
        fid=fopen(strcat(caminho,nome));
        label=textscan(fid,'%s',25,'Delimiter',',');
        C=textscan(fid,'%s%f%*f %f%*f %f%*f %*f%*f%*f%*f %f%*f%f%*f%f%*f%f%*f%f%*f%f%*f%f%*s','Delimiter',',');
        fclose(fid);
        if(isnan(C{1,2}(size(C{1,2},1)))) % o arquivo tem um texto no final            
            aux=strfind(C{1,1}(:),'-----BEGIN'); % localiza end of data
            cade=true;eod=1;
            while(cade) % jeito tosco de procurar um elemento em uma cell...
                if (aux{eod,1}==1)
                    cade=false;
                else
                    eod=eod+1;
                    if(eod>size(aux,1));cade=false;end
                end
            end
            for j=1:size(C,2)
                C{1,j}(eod:size(C{1,j},1))=[]; % remove final text lines
            end
        end
        %
        vsize=zeros(1,size(C,2));
        for j=1:size(C,2)
            vsize(j)=size(C{1,j},1);
        end
        if(~isempty(find(abs(diff(vsize))>0)))
            for j=1:size(C,2)
                if(length(C{1,j})>min(vsize))
                    C{1,j}(length(C{1,j}))=[]; %remover linha sobrando
                end
            end            
        end 
        %
        num=zeros(size(C{1,1},1),size(C,2)-1);
        for j=2:size(C,2) % somente numeros
            num(:,j-1)=C{1,j};
        end   
        %disp('------ Convertendo data e hora para matriz de tempo ------')
        aux1=zeros(size(num,1),1);
        for i=1:size(num,1)
%             if(mod(i,100)==0)
%                 disp(i)
%             end            
            aux1(i)=datenum(C{1,1}(i),'mm/dd/yy HH:MM:SS');
        end    
        highres=cat(2,aux1,num);
        dados=highres; clear highres
%         if(media)
%             %   1 min averages (time stamp at the beginning of each period)
%             disp('Calculating 1 min averages')
%             first = floor(highres(1,1));   % first averaging time
%             last = ceil(highres(length(highres),1));  % last averaging time
%             delta = 1/60/24;     % 1min
%             tol=-1/60/60/24; %1sec
%             [dados]=fmedia_qualquer_coisa2(highres(:,1:4),first,last,delta,tol);
%             dados(:,3:size(dados,2))=[];
%             clear highres first last delta tol
%         else
%             dados=highres;
%             clear highres
%         end        
        alldata=cat(1,alldata,dados);              
        clear aux cade eod C num aux1 dados vsize
end
alldata(1,:)=[]; % [tmat CO(ppm) N2O(ppm) H2O(ppm) GasP(torr) GasT(C) AmbT(C) LTC0(V) AIN5 AIN6 fitflag]
alldata(:,11)=[]; % a variável fitflag é sempre igual a 3, ou seja, não traz informação relevante
%
aux=sortrows(alldata,1); % sort data based on day of year
clear alldata
alldata=aux;
clear aux
%
disp('----Checking for repeated data')
aux=find(diff(alldata(:,1))==0);
alldata(aux+1,:)=[];
save(strcat(caminho1,'T1_2015-2016_CO_LosGatos_level0.mat'),'alldata','-mat')
alldata(:,2:3)=alldata(:,2:3)*1000; % CO(ppb),N2O(ppb),H2O(ppm)
%
figure,subplot(3,1,1),plot(alldata(:,1),alldata(:,9),'-b.'),grid on,dynamicDateTicks
subplot(3,1,2),plot(alldata(:,1),alldata(:,8),'-g.'),grid on,dynamicDateTicks
subplot(3,1,3),plot(alldata(:,1),alldata(:,6),'-r.'),grid on,dynamicDateTicks
figure,subplot(3,1,1),plot(alldata(:,1),alldata(:,2),'-k.'),grid on,dynamicDateTicks
subplot(3,1,2),plot(alldata(:,1),alldata(:,3),'-r.'),grid on,dynamicDateTicks
subplot(3,1,3),plot(alldata(:,1),alldata(:,4),'-c.'),grid on,dynamicDateTicks
%
%
% Level1 data -------------------------------------------------------
origsize=size(alldata,1);
orig=alldata;
aux1=find(alldata(:,9)<0.67 | alldata(:,9)>0.68 | alldata(:,8)<=0.2 | alldata(:,6)<26 | alldata(:,6)>40);
aux2=find(alldata(:,2)<=-1 | alldata(:,2)>=4000 | alldata(:,3)<=-1 | alldata(:,3)>=5000);
rem=cat(1,aux1,aux2);
alldata(rem,:)=[];
% aux3=find(alldata(:,1)>=datenum(2014,8,17,13,0,0) & alldata(:,1)<=datenum(2014,11,27,12,0,0));
% alldata(aux3,1)=alldata(aux3,1)+55/60/24; % lag of 55 min in comparison to maap and benzene (co was late) 
clear aux1 aux2 aux3 rem
%
% Save Level 1 data 
Instrument='LosGatos';
Model='ICOS';
serial_num='110062';
Site=tit;
Val_level='1';
Proc_vers='1.0';
labels{1}='CO';labels{2}='N2O';labels{3}='H2O';
units{1}='ppb';units{2}='ppb';units{3}='ppm';
export_LFA_GoAmazon_jan2015(labels,units,Instrument,Model,serial_num,Site,Val_level,Proc_vers,origsize,alldata(:,1),alldata(:,2:4))
%
%
% Level2 data -------------------------------------------------------
% Lê os arquivos Level 1 
aux=importdata('C:\A\GOAMAZON\T1\LFA_standard\T1_2015-2016_CO_LosGatos_ICOS_Level1_5min.csv');
level1=cat(2,aux.data(:,1:2),aux.data(:,9),aux.data(:,16));
level1(:,1)=level1(:,1)/(24*60*60)+datenum(1970,1,1); % unix time to matlab time
origsize=sum(~isnan(level1(:,2)));
orig=level1;
tit='T1_2015-2016_CO';
%
aux=importdata('C:\A\GOAMAZON\T1\LFA_standard\T1_2015-2016_BC_Aethalometer_AE33_Level1_5min.csv');
bc=cat(2,aux.data(:,1),aux.data(:,37)); % BC880
bc(:,1)=bc(:,1)/(24*60*60)+datenum(1970,1,1); % unix time to matlab time
clear aux 
%
aux1=find(level1(:,1)<datenum(2015,12,8,20,0,0)); % antes do problema
aux2=find(level1(:,1)>=datenum(2015,12,11) & level1(:,1)<datenum(2016,2,20)); % durante o problema
aux3=find(level1(:,1)>=datenum(2016,2,26)); % depois do problema
figure
subplot(1,3,1),plot(mod(level1(aux1,1),1),level1(aux1,2),'b.'),grid on
ylabel('CO (ppb)')
subplot(1,3,2),plot(mod(level1(aux2,1),1),level1(aux2,2),'r.'),grid on,hold on
ylabel('CO (ppb)')
subplot(1,3,3),plot(mod(level1(aux3,1),1),level1(aux3,2),'g.'),grid on,hold on
ylabel('CO (ppb)')
figure
subplot(1,3,1),plot(bc(aux1,2),level1(aux1,2),'b.'),grid on,hold on
[a,b,rsquare]=flinreg(bc(aux1,2),level1(aux1,2));
x=[min(bc(aux1,2)) max(bc(aux1,2))];
plot(x,a*x+b,'-k')
legend('data',[num2str(a,'%1.2f'),'*x+',num2str(b,'%3.0f'),' R^2=',num2str(rsquare,'%1.2f')])  
ylabel('CO (ppb)'),xlabel('BC880 (ug/m3)')
subplot(1,3,2),plot(bc(aux2,2),level1(aux2,2),'r.'),grid on,hold on
[a,b,rsquare]=flinreg(bc(aux2,2),level1(aux2,2));
x=[min(bc(aux2,2)) max(bc(aux2,2))];
plot(x,a*x+b,'-k')
legend('data',[num2str(a,'%1.2f'),'*x+',num2str(b,'%3.0f'),' R^2=',num2str(rsquare,'%1.2f')])  
ylabel('CO (ppb)'),xlabel('BC880 (ug/m3)')
subplot(1,3,3),plot(bc(aux3,2),level1(aux3,2),'g.'),grid on,hold on
[a,b,rsquare]=flinreg(bc(aux3,2),level1(aux3,2));
x=[min(bc(aux3,2)) max(bc(aux3,2))];
plot(x,a*x+b,'-k')
legend('data',[num2str(a,'%1.2f'),'*x+',num2str(b,'%3.0f'),' R^2=',num2str(rsquare,'%1.2f')])  
ylabel('CO (ppb)'),xlabel('BC880 (ug/m3)')
%
% Correção para o período 2151211 a 20160220
level1(level1(:,1)>=datenum(2015,12,8,20,0,0) & level1(:,1)<=datenum(2015,12,11,00,0,0),2:4)=NaN;
level1(level1(:,1)>=datenum(2016,2,20,21,51,0) & level1(:,1)<=datenum(2016,2,25),2:4)=NaN;
aux2=find(level1(:,1)>=datenum(2015,12,11) & level1(:,1)<datenum(2016,2,20,21,50,0)); % durante o problema
level1(aux2,2)=-25+25*level1(aux2,2);
level1(level1(:,2)<50,2)=NaN;
%
figure,plot(orig(:,1),orig(:,2),'r.'),grid on, hold on, dynamicDateTicks
ylabel('CO (ppb)')
plot(level1(:,1),level1(:,2),'k.')
% Save Level 2 data 
% 
Instrument='LosGatos';
Model='ICOS';
serial_num='110062';
Site=tit;
Val_level='2';
Proc_vers='1.0';
labels{1}='CO';labels{2}='N2O';labels{3}='H2O';
units{1}='ppb';units{2}='ppb';units{3}='ppm';
clear aux
export_LFA_GoAmazon_jan2015(labels,units,Instrument,Model,serial_num,Site,Val_level,Proc_vers,origsize,level1(:,1),level1(:,2:4))




