%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% M-file to process buoy m"x" metsys data.
% Simple bounds checks have already been
% applied to the data which have been output
% from the perl script.  This step is
% to concattenate the data and to create a
% much large files and do more sophisticated (sp?)
% processing of the data
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% define an array
fpath=['/data/span9/bahr/perlprogs/'];
mond=[0,31,59,90,120,151,181,212,243,273,304,334];
monn=[31,28,31,30,31,30,31,31,30,31,30,31];
monc=['jan',
      'feb',
      'mar',
      'apr',
      'may',
      'jun',
      'jul',
      'aug',
      'sep',
      'oct',
      'nov',
      'dec'];
dtt=10.0/60.0/24.0;
% open file with mooring number which we will be working on
eval(['load ',fpath,'mooringid.dat']);
eval(['mid=mooringid;']);
% grab time
eval(['load ',fpath,'time.dat']);
eval(['gtime=time;']);
year=gtime(1);
month=gtime(2);
day=gtime(3);
hour=gtime(4);
minu=gtime(5);
sec=0;
yday=gtime(6);
yearo=year;
% create filename for the current month
pathm1=['/data/span9/bahr/m',num2str(mid),'/adcp/',num2str(year),'/'];
patho=pathm1;
fnm1=['m',num2str(mid),'_adcp_bat_',monc(month,:),'_',num2str(year)];
% see if the file exists
fid=fopen([pathm1,fnm1,'.mat'],'r+');
% if the file exists then fid will be greater than 2.  If the file
% doesn't exist we want to create it but we also want to load
% the previous months file.  We also have to careful around the year
% change.
%# Note change to add a flag array for the data.  This new array will
%# contains flags defining how good the data used is for both plotting
%# and distribution.  The flag array will contain 1 less element than
%# the main array.  We will not flag the time, but pressure, temperature
%# conductivity and salinity will be flaged.
if fid > 2,
	fclose(fid);
	eval(['load ',pathm1,fnm1,'.mat']);
	eval(['b=batn;']);
else
	b=[];
end
% Check to see if we want to load the old file.  We assume that data
% from the oasis mooring will not be older than 7 days.
if day > 8,
	a=[];
else,
	if month==1,
		yearo=year-1;
		patho=['/data/span9/bahr/m',num2str(mid),'/adcp/',num2str(yearo),'/'];
		fnm1o=['m',num2str(mid),'_adcp_bat_dec_',num2str(yearo)];
	else,
		yearo=year;
		patho=pathm1;
		fnm1o=['m',num2str(mid),'_adcp_bat_',monc(month-1,:),'_',num2str(year)];
	end
% check to see if old file is there.  If it is load it.
	fido=fopen([patho,fnm1o,'.mat'],'r+');	
	if fido > 2,
		fclose(fido);
		eval(['load ',patho,fnm1o,'.mat']);
		eval(['a=batn;']);
	else
		a=[];
	end
end
% create one array of data.  We have to be careful doing this at the
% year change.  Since the year is not explicitely stated in the file.
% Do a trick to handle year changes.  If we have a year change yearo!=year
% then we subtract the max days from the previous year?
%# Note we use 366 as December 31 at 12pm is denoted 365.5 days.
maxd=366;
if yearo~=year,
	maxd=366;
	if rem(yearo,4)==0,
		maxd=maxd+1;
	end
	if isempty(a)==0,
		a(:,1)=a(:,1)-maxd;
	end
end
bat=[a
     b];
% we should now have the old matlab process ctd sys files loaded.
% The next step is to load the new files with data to be processed.
ff1=['m',num2str(mid),'_adcp_bat.dat'];
ff2=['m',num2str(mid),'_adcpo_bat.dat'];
if (month==1)&(day >=2),
	fod1=fopen([pathm1,ff1],'r');
	fod2=fopen([pathm1,ff2],'r');
else,
	fod1=fopen([pathm1,ff1],'r');
	fod2=fopen([patho,ff2],'r');
end
if fod1 > 2,
	fclose(fod1);
	eval(['load ',pathm1,'m',num2str(mid),'_adcp_bat.dat']);
	eval(['m1=m',num2str(mid),'_adcp_bat;']);
else,
	m1=[];
end
if fod2 > 2,
	fclose(fod2);
	if(month==1)&(day >=2),
		eval(['load ',pathm1,'m',num2str(mid),'_adcpo_bat.dat']);
	else,
		eval(['load ',patho,'m',num2str(mid),'_adcpo_bat.dat']);
	end
	eval(['m1o=m',num2str(mid),'_adcpo_bat;']);
else,
	m1o=[];
end
% we only have to worry about day ==1 on this part.  But that doesn't
% cover the posibility that data from before may be in one of the old
% files?
if ((day==1)&(month==1)&(size(m1o,1)>0)),
	m1o(:,1)=m1o(:,1)-maxd;
end
% we have a problem here at the year change.  We need to worry about.
% since the files only look at year day and not have the year in it.
m1=[m1o
    m1];

% we now have the data.  Now we need to check times and sort times?
% format of ctd files is time, press, temp, cond, salt
ydnow=mond(month)+day+hour/24+minu/(24*60);
if (rem(year,4)==0)&(month > 2),
	ydnow=ydnow+1;
end
ydnowl=ydnow-1;
if size(m1,1) > 0,
	dt=diff(m1(:,1));
%check for neg values to see if we have times out of order.
	[ineg]=find(dt < 0);
% we want to do all the time check stuff as vetor/matrix checks and
% try to avoid costly loops and redoing stuff that we don't need to
% do more than once.
	if isempty(ineg)==0,
% we need to do a sort of the file
		[newt1 idex1]=sort(m1(:,1));
		m1=[newt1 m1(idex1,2:size(m1,2))];
	end
	dt=diff(m1(:,1));
	idi=find(dt==0);
	m1(idi,:)=[];
% again below we have to worry about the year change.  This could
% be a recipe for disaster.  But try to get something that runs now
% and patch up the year bug later.
% data is now sorted.  Now we need to find which values are in the
% cleaned data file already and remove get send them to the nowhere.
% we only want to process what we have to.  If I can think up a vectorized
% method for doing this, then this should be done before the sort to make
% the sort faster (fewer points sort).
% compute current year day time
% find the minimum time in the currently captured files
	tmin=min(m1(:,1));
	if size(bat,1) > 0,
		cmin=min(bat(:,1));
		tmax=max(bat(:,1));
	else,
		cmin=370;
		tmax=0;
	end
% Note: Only m1 is beyond tmax of file.
% Now a hard check to do.  Look for equal times and remove them
% from the m1 file because we have already dealt with them.
	lm1=size(m1(:,1),1);
	im=zeros(lm1,1);
	if size(bat,1) > 0,
		for i=1:lm1,
			if m1(i,1) > tmax,
				i=lm1;
			end
			id=find(bat(:,1)==m1(i,1));
			if isempty(id)==0,
				im(i)=1;
			end
		end
	end
	m1(logical(im),:)=[];
end
if(isempty(m1) ==0),
% Remove points we already have in the metsys files.
% Now we need to stuff our values in.  We also want to keep a "binary"
% mask of where the values are put in for computing our filters later.
% compare against current day -1 and see how much of old file
% we need to retain before stuffing in our values in a slow loop.
% NOTE: ABOVE needs to be modified for the transition that occurs at
% the new year.
% now we have paired down the file somewhat.  The next step is to
% actually loop and insert the data.  We set up some indicies first
% before actually doing the loop.
%######################################################################
%# when we add points we will need to expand the flag array accordingly
%# here.  This will be the first step.  We might also change how bmask
%# is used.  The mask, the array, and the flag array will be necessary
%# for any filter to work properly.  The filter will have to know if
%# a point is new and if the points around it can be used in the filter
%# or if it will have to expand to look for other points.
	i=0; j=0;
	lbat=size(bat,1);
	bmask=zeros(lbat,1);
	lnew=size(m1,1);
	k=1;
	if(cmin==370),
		bat=m1;
		bmask=ones(lnew,1);
		k=lnew+1;
	else,
	   if(m1(k,1) < cmin),
		while(m1(k,1) < cmin),
			k=k+1;
		end
		bat=[m1(1:k-1,:); bat];
		bmask=[ones(length([1:k-1]),1)
		       bmask];
	  end
	end
	while(k <= lnew),
		lbat=size(bat,1);
		if lbat==0,
			bat=m1;
			k=lnew+1;
			bmask=[bmask
		       	       ones(lnew,1)];
		else,
			[index]=find(bat(:,1) < m1(k,1));
			idmz=max(index);
			if idmz==lbat,
				bat=[bat
                     	     	     m1([k:lnew],:)];
				bmask=[bmask
			               ones(length(k:lnew),1)];
			 	k=lnew;
			else
				bat=[bat([1:idmz],:)
		     	     	     m1(k,:)
		     	             bat([idmz+1:lbat],:)];
				bmask=[bmask([1:idmz])
				       1
			               bmask([idmz+1:lbat])];
			end
		end
		k=k+1;
	end
% If the above works, bmask should be a file with 1's where
% we have inserted data and should be the same length as
% the array met with the inserted data.
%# We should also have the flag array the same size with 0's
%# for the new values.  We assume all data is good at first.
%# we will then remove it as we progress.
%# Now to compute bounds checks on data.
%#
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Now we can apply our gradient filter to the data (?)
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% get the indecies of the insertion points
% Need to save the appropriate data here!
% we need to be aware of year and month breaks
% in our data if we want to save it correctly.
	if day > 8,
		batn=bat;
		eval(['save ',pathm1,fnm1,' batn']);
	else
		if month==1,
			ll=bat(:,1) < 0;
			batn=bat(ll,:);
			batn(:,1)=batn(:,1)+maxd;
			eval(['save ',patho,fnm1o,' batn']);
			clear batn;
			batn=bat(~ll,:);
			eval(['save ',pathm1,fnm1,' batn']);
		else
			dmax=mond(month)+1;
			if (month > 2)&(rem(year,4)==0),
				dmax=dmax+1;
			end
			ll=bat(:,1) < dmax;
			batn=bat(ll,:);
			eval(['save ',patho,fnm1o,' batn']);
			clear batn;
			batn=bat(~ll,:);
			eval(['save ',pathm1,fnm1,' batn']);
		end
	end
end % end of im m1 has data
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Now process the last days worth of data for display
% by various web pages.
% 1) compute 24 hour means and deviations along with
% current values.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
if(size(bat(:,1),1) > 0),
	ydp=ydnow-7;
	if ydp < 0,
		ydp=ydp-1;
	end
	lp=(bat(:,1) > ydp)&(bat(:,1) <= ydnow);
	batp=bat(lp,:);
% now to write out a simple table of numbers for a perl
% script to load and create an htlm table from
	bad=-99;
% we need to check for the output file if it exists we remove it
	batplot(batp(:,1),batp(:,2),year,mid,2);
end
% We could add plots of the data here and save them to standard images.
% matlab would be good for all but stick and contour plots.
%
quit
