clear
outdir   = '~/research/cruise/2019_Thompson_TN368_0703-0719/data/gps/';
tsgflag  = 1;
flrflag  = 1;

juldate  = 7; % 12:20; % The day in May 2019 to read
inidate  = 5; % the first day of the cruise in Jul 2019

gpsdir   = '/Volumes/tgt-data/scs/NAV/';
tsgdir   = '/Volumes/tgt-data/scs/SEAWATER/';
flrdir   = '/Volumes/tgt-data/scs/SEAWATER/';

% GPS file names (only 20190705 onwards)
gpsfiles = dir([gpsdir,'CNAV3050-GGA-RAW*.Raw']);
gpsfiles = vertcat(gpsfiles.name);

% TSG file names
tsgfiles = dir([tsgdir,'TSG-RAW_*.Raw']);
tsgfiles = vertcat(tsgfiles.name);

% FLUOR file names
flrfiles = dir([flrdir,'FLOUROMETER-RAW_*.Raw']);
flrfiles = vertcat(flrfiles.name);
% parameters to convert the counts ug/l (from the manual)
sfac     = 0.0074;  % (ug/l/count)
darkcount= 48;      % (counts)

% initialize the variables with [] for the later concatenation
congpslat   = [];
congpslon   = [];
congpsdate  = [];
if tsgflag
    % initialize the variables with [] for the later concatenation
    cont1       = [];
    conc1       = [];
    cons        = [];
    consv       = [];
    cont2       = [];
    contsgdate  = [];
    contsglat   = [];
    contsglon   = [];
end
if flrflag
    % initialize the variables with [] for the later concatenation
    conflr      = [];
    conflrdate  = [];
    conflrlat   = [];
    conflrlon   = [];
end

for iday = 1:length(juldate)
    if juldate(iday) < 10
        daystr   = ['0' num2str(juldate(iday))];
    else
        daystr   = num2str(juldate(iday));
    end
    gpsinds  = find(contains(gpsfiles,['201907' daystr]) == 1);
    gpsfile  = [gpsdir gpsfiles(gpsinds,:)];
    
    if tsgflag
        tsginds  = find(contains(tsgfiles,['201907' daystr]) == 1);
        tsgfile  = [tsgdir tsgfiles(tsginds,:)];
    end
    
    if flrflag
        flrinds  = find(contains(flrfiles,['201907' daystr]) == 1);
        flrfile  = [flrdir flrfiles(flrinds,:)];
    end
    
    clear gdate glon glat;
    date = juldate(iday);
    disp(['Processing data for July ' num2str(date)]);
    
    % load GPS data
    fid        = fopen(gpsfile);
    wpind      = 0;
    clear llstr
    while ~feof(fid) %Check every line in the file
        wpind  = wpind + 1;
        llstr(wpind,:) = fscanf(fid,'%d/%d/%d,%d:%d:%f,$GPGGA,%f,%f,N,%f,W,%d,%d,%f,%f,M,%f,M,%f,%d*%c%c\n',[1,18]);
    end
    fclose(fid);
    for iw = 1:wpind
        gdate(iw) =  datetime(llstr(iw,3),llstr(iw,1),llstr(iw,2),llstr(iw,4),llstr(iw,5),llstr(iw,6));
        glat(iw)  =  floor(llstr(iw,8)/100) -  (floor(llstr(iw,8)/100)*100 - llstr(iw,8))/60;
        glon(iw)  = -floor(llstr(iw,9)/100) + (floor(llstr(iw,9)/100)*100 - llstr(iw,9))/60;
    end
    % remove the repetitive gps time
    [gdate,unqind,repind] = unique(gdate);
    glat = glat(unqind);
    glon = glon(unqind);
    
    if tsgflag
        % load TSG data
        fid          = fopen(tsgfile);
        wpind      = 0;
        clear llstrtsg
        while ~feof(fid) %Check every line in the file
            wpind  = wpind + 1;
            llstrtsg(wpind,:) = fscanf(fid,'%d/%d/%d,%d:%d:%f, %f,  %f,  %f\n',[1,9]);
        end
        fclose(fid);
        
        clear tsd_day t1_day c1_day s_day;
        tsd_day = datetime([llstrtsg(:,3) llstrtsg(:,1) llstrtsg(:,2) llstrtsg(:,4) llstrtsg(:,5) llstrtsg(:,6)]);
        t1_day   = llstrtsg(:,7);
        c1_day   = llstrtsg(:,8);
        s_day    = llstrtsg(:,9);
        
        % obtain the lat-lon of the tsg data using interpolation
        tsginterp_lat        = interp1(gdate, glat, tsd_day,'linear');
        tsginterp_lon        = interp1(gdate, glon, tsd_day,'linear');
        nnind                = find(isnan(tsginterp_lat) | isnan(tsginterp_lon));
        tsginterp_lat(nnind) = [];
        tsginterp_lon(nnind) = [];
        tsd_day(nnind)       = [];
        t1_day(nnind)        = [];
        c1_day(nnind)        = [];
        s_day(nnind)         = [];
    end
    
    if flrflag
        clear flr_day ftime_day;
        
        % load Flr data
        fid        = fopen(flrfile);
        wpind      = 0;
        clear llstrflr;
        while ~feof(fid) %Check every line in the file
            wpind  = wpind + 1;
            tline  = fgetl(fid);
            tmon   = str2num(tline(1:2));
            tday   = str2num(tline(4:5));
            tyear  = str2num(tline(7:10));
            thour  = str2num(tline(12:13));
            tmin   = str2num(tline(15:16));
            tsec   = str2num(tline(18:23));
            ftime_day(wpind) = datetime([tyear tmon tday thour tmin tsec]);
            flr_day(wpind)   = str2num(tline(47:49));
        end
% This part of the code failed at Line 9089 for some strange reason       
%         while ~feof(fid) %Check every line in the file
%             wpind  = wpind + 1;
%             llstrflr(wpind,:) = fscanf(fid,'%d/%d/%d,%d:%d:%f,%d/%d/%d\t%d:%d:%d\t%d\t%d\t%d\n',[1,15]);
%         end
%        ftime_day =  datetime([llstrflr(:,3) llstrflr(:,1) llstrflr(:,2) llstrflr(:,4) llstrflr(:,5) llstrflr(:,6)]);
%        flr_day   =  double(llstrflr(:,14));
        fclose(fid);
        
        % obtain the lat-lon of the tsg data using interpolation
        flrinterp_lat        = interp1(gdate, glat, ftime_day,'linear');
        flrinterp_lon        = interp1(gdate, glon, ftime_day,'linear');
        nnind                = find(isnan(flrinterp_lat) | isnan(flrinterp_lon));
        flrinterp_lat(nnind) = [];
        flrinterp_lon(nnind) = [];
        ftime_day(nnind)     = [];
        flr_day(nnind)       = [];
        
        flr_day = sfac.*(flr_day - darkcount);
    end
    
    lat     = glat(:);
    lon     = glon(:);
    gpsdate = gdate(:);
    gpsoutfile = [outdir 'gps_' daystr '.mat'];
    save(gpsoutfile,'gpsdate', 'lat','lon')
    disp(['Saved file: ' gpsoutfile]);
    gpsmatrix = [year(gpsdate) month(gpsdate) day(gpsdate) hour(gpsdate) minute(gpsdate) second(gpsdate) lat lon];
    xlswrite([outdir 'gps_' daystr '.xls'],gpsmatrix);

    if tsgflag
        t1         = t1_day(:);
        c1         = c1_day(:);
        s          = s_day(:);
        date_tsg   = tsd_day(:);
        lat_tsg    = tsginterp_lat(:);
        lon_tsg    = tsginterp_lon(:);
        tsgoutfile = [outdir 'tsg_with_gps_' daystr '.mat'];
        save(tsgoutfile,'date_tsg', 'lat_tsg','lon_tsg', 't1','c1','s')
        disp(['Saved file: ' tsgoutfile]);
        tsgmatrix = [year(date_tsg) month(date_tsg) day(date_tsg) hour(date_tsg) ...
                     minute(date_tsg) second(date_tsg) lat_tsg lon_tsg t1 c1 s];
        xlswrite([outdir 'tsg_with_gps_' daystr '.xls'],tsgmatrix);
    end
    
    if flrflag
        flr        = flr_day(:);
        date_flr   = ftime_day(:);
        lat_flr    = flrinterp_lat(:);
        lon_flr    = flrinterp_lon(:);
        flroutfile = [outdir 'flr_with_gps_' daystr '.mat'];
        save(flroutfile,'date_flr', 'lat_flr','lon_flr', 'flr')
        disp(['Saved file: ' flroutfile]);
        flrmatrix = [year(date_flr) month(date_flr) day(date_flr) hour(date_flr) ...
                     minute(date_flr) second(date_flr) lat_flr lon_flr flr];
        xlswrite([outdir 'flr_with_gps_' daystr '.xls'],flrmatrix);
    end
    
    % concatenate the variable from each day
    congpslat   = [congpslat;  lat(:)    ];
    congpslon   = [congpslon;  lon(:)    ];
    congpsdate  = [congpsdate; gpsdate(:)];
    
    if tsgflag
        cont1       = [cont1;       t1(:)      ];
        conc1       = [conc1;       c1(:)      ];
        cons        = [cons;        s(:)       ];
        contsgdate  = [contsgdate;  date_tsg(:)];
        contsglat   = [contsglat;   lat_tsg(:) ];
        contsglon   = [contsglon;   lon_tsg(:) ];
    end
    if flrflag
        conflr      = [conflr;      flr(:)     ];
        conflrdate  = [conflrdate;  date_flr(:)];
        conflrlat   = [conflrlat;   lat_flr(:) ];
        conflrlon   = [conflrlon;   lon_flr(:) ];
    end
end

if juldate(1) > inidate
    % concatenate the variable with data from the previous days
    numpday = juldate-inidate;  % number of previous days to read
    for iday = 1:numpday
        if juldate(1)-iday < 10
            daystrm1   = ['0' num2str(juldate(1)-iday)];
        else
            daystrm1   = num2str(juldate(1)-iday);
        end
        
        pgpsfile    = [outdir 'gps_' daystrm1 '.mat'];
        pgps        = load(pgpsfile);
        congpslat   = [pgps.lat(:);     congpslat ];
        congpslon   = [pgps.lon(:);     congpslon ];
        congpsdate  = [pgps.gpsdate(:); congpsdate];
        
        if tsgflag
            ptsgfile    = [outdir 'tsg_with_gps_' daystrm1 '.mat'];
            ptsg        = load(ptsgfile);
            cont1       = [ptsg.t1(:);       cont1;       ];
            conc1       = [ptsg.c1(:);       conc1;       ];
            cons        = [ptsg.s(:);        cons;        ];
            contsgdate  = [ptsg.date_tsg(:); contsgdate;  ];
            contsglat   = [ptsg.lat_tsg(:);  contsglat;   ];
            contsglon   = [ptsg.lon_tsg(:);  contsglon;   ];
        end
        if flrflag
            pflrfile    = [outdir 'flr_with_gps_' daystrm1 '.mat'];
            pflr        = load(pflrfile);
            conflr      = [pflr.flr(:);      conflr;      ];
            conflrdate  = [pflr.date_flr(:); conflrdate;  ];
            conflrlat   = [pflr.lat_flr(:);  conflrlat;   ];
            conflrlon   = [pflr.lon_flr(:);  conflrlon;   ];
        end
    end
end

% save the concatenated files:

lat     = congpslat(:);
lon     = congpslon(:);
gpsdate = congpsdate(:);
gpsoutfile = [outdir 'gps.mat'];
save(gpsoutfile,'gpsdate', 'lat','lon')
disp(['Saved file: ' gpsoutfile]);
gpsmatrix = [year(gpsdate) month(gpsdate) day(gpsdate) hour(gpsdate) minute(gpsdate) second(gpsdate) lat lon];
xlswrite([outdir 'gps.xls'],gpsmatrix);

if tsgflag
    t1         = cont1(:);
    c1         = conc1(:);
    s          = cons(:);
    date_tsg   = contsgdate(:);
    lat_tsg    = contsglat(:);
    lon_tsg    = contsglon(:);
    tsgoutfile = [outdir 'tsg_with_gps.mat'];
    save(tsgoutfile,'date_tsg', 'lat_tsg','lon_tsg', 't1','c1','s')
    disp(['Saved file: ' tsgoutfile]);
    tsgmatrix = [year(date_tsg) month(date_tsg) day(date_tsg) hour(date_tsg) ...
        minute(date_tsg) second(date_tsg) lat_tsg lon_tsg t1 c1 s];
    xlswrite([outdir 'tsg_with_gps.xls'],tsgmatrix);
end

if flrflag
    flr        = conflr(:);
    date_flr   = conflrdate(:);
    lat_flr    = conflrlat(:);
    lon_flr    = conflrlon(:);
    flroutfile = [outdir 'flr_with_gps.mat'];
    save(flroutfile,'date_flr', 'lat_flr','lon_flr', 'flr')
    disp(['Saved file: ' flroutfile]);
    flrmatrix = [year(date_flr) month(date_flr) day(date_flr) hour(date_flr) ...
        minute(date_flr) second(date_flr) lat_flr lon_flr flr];
    xlswrite([outdir 'flr_with_gps.xls'],flrmatrix);
end

    