import os.path import seawater as sw import numpy as np import csv from datetime import datetime from pytz import timezone import pytz import shutil #copying files import time #strptime for e-mail alarms from geopy.distance import geodesic import pandas as pd #for reading master spreadsheet xls file import json #for loading json file manually import wgFunctions as wg import warnings #----------------------------# #vehicle = 'wgTiny' #deployments = ['20250107','20250131','20250403','20250514','20250606','20250717','20250813','20250904','20251020','20251112','20251203'] vehicle = 'wgHansen' deployments = ['20250107','20250311','20250415','20250527','20250709','20250820','20251001','20251110'] #----------------------------# print("start at: " + time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.localtime())) for dep in deployments: #get cal/config info off master spreadsheet #define data directories #myDirWG = 'C:/Users/tmasek/Dropbox/Projects/MBARI/WaveGliderDisplay/WaveGlider/' myDirWG = '//atlas.shore.mbari.org/WaveGlider/' #myDirWeb = 'C:/Users/tmasek/Dropbox/Projects/MBARI/WaveGliderDisplay/WaveGlider/' + vehicle +'/data/' myDirWeb = '//atlas.shore.mbari.org/oasis_users/web/extern/bog/waveglider/' +vehicle + '/data/' #see if there is a active run and all calib lines are added #get cal/config info off master spreadsheet warnings.filterwarnings('ignore', category=UserWarning, module='openpyxl') xlInfo = pd.read_excel(myDirWG +'WG_cal_config.xlsx', engine='openpyxl', sheet_name='Info', na_values=['NaN','N/A'], keep_default_na=False, engine_kwargs={'read_only': True}) xlCCU = pd.read_excel(myDirWG +'WG_cal_config.xlsx', engine='openpyxl', sheet_name='CCU', na_values=['NaN','N/A'], keep_default_na=False, engine_kwargs={'read_only': True}) xlGPCTD= pd.read_excel(myDirWG +'WG_cal_config.xlsx', engine='openpyxl', sheet_name='GPCTD+DO', na_values=['NaN','N/A'], keep_default_na=False, engine_kwargs={'read_only': True}) xlBB = pd.read_excel(myDirWG +'WG_cal_config.xlsx', engine='openpyxl', sheet_name='BB', na_values=['NaN','N/A'], keep_default_na=False, engine_kwargs={'read_only': True}) xlpH = pd.read_excel(myDirWG +'WG_cal_config.xlsx', engine='openpyxl', sheet_name='pH', na_values=['NaN','N/A'], keep_default_na=False, engine_kwargs={'read_only': True}) xlCO2 = pd.read_excel(myDirWG +'WG_cal_config.xlsx', engine='openpyxl', sheet_name='CO2', na_values=['NaN','N/A'], keep_default_na=False, engine_kwargs={'read_only': True}) xlVR2C = pd.read_excel(myDirWG +'WG_cal_config.xlsx', engine='openpyxl', sheet_name='VR2C', na_values=['NaN','N/A'], keep_default_na=False, engine_kwargs={'read_only': True}) xlMET = pd.read_excel(myDirWG +'WG_cal_config.xlsx', engine='openpyxl', sheet_name='MET', na_values=['NaN','N/A'], keep_default_na=False, engine_kwargs={'read_only': True}) xlDeps = xlInfo['Deployment'] xlRowStart = xlDeps[xlDeps == vehicle].index[0] #print(xlInfo[xlRowStart + 1:int(xlInfo.shape[0])]) for index, row in xlInfo[xlRowStart + 1:int(xlInfo.shape[0])].iterrows(): #print(index) #print(dep) #print(int(row['Deployment'])) if row['Deployment'] == '': break if int(dep) == int(row['Deployment']): #xlRowInd = xlDeps[xlDeps == int(dep)].index[0] xlRowInd = index myDirRT = myDirWG +'deployment_data/' + vehicle +'/' + dep +'/realTime/' # tBeg, tEnd fmt = '%Y-%m-%dT%H:%M:%SZ' tBeg = datetime.strptime(xlInfo.loc[xlRowInd,'tBeg'], fmt).replace(tzinfo=pytz.utc) if(str(xlInfo.loc[xlRowInd,'tEnd']) != ''): tEnd = datetime.strptime(xlInfo.loc[xlRowInd,'tEnd'], fmt).replace(tzinfo=pytz.utc) else: tEnd = datetime.now(timezone('UTC')) if os.path.exists(myDirRT): shutil.rmtree(myDirRT) if os.path.exists(myDirWeb): shutil.rmtree(myDirWeb) os.makedirs(myDirRT) os.makedirs(myDirWeb) os.makedirs(myDirWeb + 'csv/') os.makedirs(myDirWeb + 'log/') # CCU vids CCUname = xlCCU.loc[xlRowInd,'CCU'] CCUnum = str(int(xlCCU.loc[xlRowInd,'VID'])) # CTD1 O2cal1 = xlGPCTD.loc[xlRowInd,:][7:14].tolist() O2cal2 = xlGPCTD.loc[xlRowInd,:][19:26].tolist() # Eco-Puck EPcal = [ xlBB.loc[xlRowInd,:][7:9].tolist(), xlBB.loc[xlRowInd,:][9:11].tolist(), xlBB.loc[xlRowInd,:][11:13].tolist() ] # pH try: pHcal = [ round( xlpH.loc[xlRowInd,:].iloc[12], 6), xlpH.loc[xlRowInd,:].iloc[11] ] except: pHcal = ['N/A', 'N/A'] # CO2 CO2cal = xlCO2.loc[xlRowInd,:][6:10].tolist() CO2stndKn = xlCO2.loc[xlRowInd,:].iloc[19] print(vehicle + " " + str(tBeg) + " " + str(tEnd)) #retrieve buoy data ptsur_data = wg.getBuoyData('PointSur', myDirRT, tBeg, tEnd) wmon_data = wg.getBuoyData('MontereyCanyon', myDirRT, tBeg, tEnd) #----------------------------# print(vehicle + " " + str(tBeg) + " " + str(tEnd)) print(str(CCUnum)+ " " + str(tBeg.isoformat())+ " " + str(tEnd.isoformat())) CTD1time, CTD1temp, CTD1sal, CTD1press, CTD1o2Hz, CTD1o2sol, CTD2time, CTD2temp, CTD2sal, CTD2press, CTD2o2Hz, CTD2o2sol = wg.getCTDDataRecord(CCUnum,tBeg,tEnd) print('CTD:' + str(len(CTD1time)) + " records") CTD1woO2time, CTD1woO2temp, CTD1woO2sal, CTD1woO2press, CTD1woO2o2Hz, CTD1woO2o2sol, CTD2woO2time, CTD2woO2temp, CTD2woO2sal, CTD2woO2press, CTD2woO2o2Hz, CTD2woO2o2sol = wg.getCTDwoO2DataRecord(CCUnum,tBeg,tEnd) print('CTDwoO2:' + str(len(CTD1woO2time)) + " records") METtime, METwindDir, METwindSpd, METairPress, METairTemp = wg.getWindRecord(CCUnum,tBeg,tEnd) print('Wind:' + str(len(METtime)) + " records") WGtime, WGlon, WGlat, WGdist, WGwaypoint, WGHead, WGDesHead, WGCOG = wg.getTelemRecord(CCUnum,tBeg,tEnd) print('Telemetry:' + str(len(WGtime)) + " records") PwrTime, PwrSolarPwrGen, PwrBatChargePwr, PwrOutPwrGen, PwrTotalBatPwr = wg.getPowermRecord(CCUnum,tBeg,tEnd) print('Power:' + str(len(PwrTime)) + " records") EPtime, EPcounts470, EPcounts650, EPcountsChl = wg.getEcoPuckRecord(CCUnum,tBeg,tEnd) print('EcoPuck:' + str(len(EPtime)) + " records") VR2Cstatus = wg.getVemcoStatRecord(CCUnum,tBeg,tEnd) print('VR2C Status:' + str(len(VR2Cstatus)) + " records") VR2Ctime, VR2CtagID, VR2CtagReading = wg.getVemcoTagRecord(CCUnum,tBeg,tEnd) print('VR2C Tag:' + str(len(VR2Ctime)) + " records") CO2time, CO2equilOffCO2, CO2equilOffTemp, CO2zeroOffCO2, CO2airOffCO2, CO2airOffTemp, pHtime, pHV1, pHtimeInc, CO2equil, CO2zero, CO2air, CO2stnd, CO2humEquilOff, CO2humAirOff, CO2humStndOff = wg.getCO2Record(CCUnum,tBeg,tEnd) print('C02:' + str(len(CO2time)) + " records") #----------------------------# # Convert your raw data #Eco-Puck EPbb470 = list() EPbb650 = list() EPchl = list() for i in range(0,len(EPtime)): EPchl.append( wg.convEP(EPcountsChl[i], EPcal[2]) ) diff = list() for j in range(0,len(CTD1time)): diff.append( CTD1time[j] - EPtime[i] ) if len(diff) > 0: ind = np.argmin(np.absolute(diff)) if np.absolute(diff[ind]) < 10*60*1000: EPbb470.append( wg.counts2bb( wg.convEP(EPcounts470[i], EPcal[0]), float(470), CTD1sal[ind] ) ) EPbb650.append( wg.counts2bb( wg.convEP(EPcounts650[i], EPcal[1]), float(650), CTD1sal[ind] ) ) else: # USE WITH CAUTION!! EPbb470.append(np.nan) EPbb650.append(np.nan) # EPbb470.append( counts2bb( convEP(EPcounts470[i], EPcal[0]), float(470), 33.5 ) ) # EPbb650.append( counts2bb( convEP(EPcounts650[i], EPcal[1]), float(650), 33.5 ) ) else: # USE WITH CAUTION!! EPbb470.append(np.nan) EPbb650.append(np.nan) # EPbb470.append( counts2bb( convEP(EPcounts470[i], EPcal[0]), float(470), 33.5 ) ) # EPbb650.append( counts2bb( convEP(EPcounts650[i], EPcal[1]), float(650), 33.5 ) ) CTD1oxygen = list() if len(CTD1time) > 0: CTD1sigt = (sw.dens(CTD1sal,CTD1temp,CTD1press)-1000).tolist() for i in range(0,len(CTD1time)): if len(CTD1o2sol) > 0: CTD1oxygen.append(wg.convO2(CTD1o2Hz[i], CTD1o2sol[i], O2cal1, CTD1press[i], CTD1temp[i], CTD1sigt[i])) else: CTD1oxygen.append(np.nan) #CTD2 CTD2oxygen = list() if len(CTD2time) > 0: CTD2sigt = (sw.dens(CTD2sal,CTD2temp,CTD2press)-1000).tolist() for i in range(0,len(CTD2time)): if len(CTD2o2sol) > 0: CTD2oxygen.append(wg.convO2(CTD2o2Hz[i], CTD2o2sol[i], O2cal2, CTD2press[i], CTD2temp[i], CTD2sigt[i])) else: CTD2oxygen.append(np.nan) #pH pH = list() for i in range(0,len(pHtime)): diff = list() for j in range(0,len(CTD1time)): diff.append( CTD1time[j] - pHtime[i] ) if len(diff) > 0: ind = np.argmin(np.absolute(diff)) if np.absolute(diff[ind]) < 60*60*1000: #**CHANGED TO 60 MIN B/C OF SOME UNRESOLVED ISSUE...** pH.append( wg.pHV12pH(pHV1[i], pHcal, CTD1temp[ind]) ) else: # USE WITH CAUTION!! pH.append(np.nan) # pH.append( pHV12pH(pHV1[i], pHcal, 11) ) else: pH.append(np.nan) # pH.append( pHV12pH(pHV1[i], pHcal, 11) ) #pCO2 #replace standard zeros with NaN CO2stndMeas = list() CO2stndTime = list() CO2stnd[CO2stnd==0] = np.nan for i in range(0, len(CO2humStndOff)): if CO2humStndOff[i]==0: CO2humStndOff[i] = np.nan #de-NaN and linear-interp stnd meas's for i in range(0, len(CO2time)): if not np.isnan(CO2stnd[i,5]): CO2stndMeas.append( CO2stnd[i,5] ) CO2stndTime.append( CO2time[i] ) if (len(CO2stndMeas)) == 0 and (len(CO2time) > 0): #if haven't got any stnd readings yet, assume is as known CO2stndMeas = [CO2stndKn, CO2stndKn] #this will overwrite once u do get one CO2stndTime = [CO2time[0], CO2time[-1]] if len(CO2time) > 0: CO2stndMeasInterp = np.interp(CO2time, CO2stndTime, CO2stndMeas).tolist() #apply calibration pCO2water = list() pCO2air = list() for i in range(0,len(CO2time)): #Air - just apply cal and u'r done pCO2air.append( wg.convCO2( CO2airOffCO2[i], CO2zeroOffCO2[i], CO2cal, CO2airOffTemp[i], CO2stndMeasInterp[i], CO2stndKn ) ) #Water - must also convert xCO2 to pCO2 xCO2waterConv = wg.convCO2( CO2equilOffCO2[i], CO2zeroOffCO2[i], CO2cal, CO2equilOffTemp[i], CO2stndMeasInterp[i], CO2stndKn ) diff = list() for j in range(0,len(CTD1time)): diff.append( CTD1time[j] - CO2time[i] ) if len(diff) > 0: ind = np.argmin(np.absolute(diff)) if np.absolute(diff[ind]) < 10*60*1000: pCO2water.append( wg.calcpCO2( xCO2waterConv, CTD1temp[ind], CTD1sal[ind] ) ) else: pCO2water.append(np.nan) else: pCO2water.append(np.nan) #clean-up LR distance ind1 = np.where(np.array(WGdist)<0)[0] ind2 = np.where(np.array(WGdist)>10000)[0] for i in range(0,len(ind1)): WGdist[ind1[i]] = 0 for i in range(0,len(ind2)): WGdist[ind2[i]] = 0 #other way of calc'ing dist using gps WGdist_v2 = list() WGdist_v2.append(0) for i in range(0,len(WGlon)-1): WGdist_v2.append( int( geodesic( (WGlat[i], WGlon[i]), (WGlat[i+1],WGlon[i+1]) ).meters ) ) #clean-up gps distance ind3 = np.where(np.array(WGdist_v2)<0)[0] ind4 = np.where(np.array(WGdist_v2)>10000)[0] for i in range(0,len(ind3)): WGdist_v2[ind3[i]] = 0 for i in range(0,len(ind4)): WGdist_v2[ind4[i]] = 0 #burst-avg CTD #CTD1 CTD1timeAvg, CTD1tempAvg = wg.burstAvgCTD(CTD1time, CTD1temp) null, CTD1salAvg = wg.burstAvgCTD(CTD1time, CTD1sal) null, CTD1oxygenAvg = wg.burstAvgCTD(CTD1time, CTD1oxygen) #CTD2 CTD2timeAvg, CTD2tempAvg = wg.burstAvgCTD(CTD2time, CTD2temp) null, CTD2salAvg = wg.burstAvgCTD(CTD2time, CTD2sal) null, CTD2oxygenAvg = wg.burstAvgCTD(CTD2time, CTD2oxygen) #align CTD2 to CTD1 CTD2tempAvga = wg.alignCTD( CTD2timeAvg, CTD2tempAvg, CTD1timeAvg) CTD2salAvga = wg.alignCTD( CTD2timeAvg, CTD2salAvg, CTD1timeAvg) CTD2oxygenAvga = wg.alignCTD( CTD2timeAvg, CTD2oxygenAvg, CTD1timeAvg) #calc difference CTDdTime = CTD1timeAvg CTDdTemp = np.subtract(CTD1tempAvg, CTD2tempAvga).tolist() CTDdSal = np.subtract(CTD1salAvg, CTD2salAvga).tolist() CTDdOxygen = np.subtract(CTD1oxygenAvg, CTD2oxygenAvga).tolist() # #use distance to calculate speed WGspdKnots=list() WGspdKnots.append(0) #initialize to keep length equal to time for i in range(0,len(WGdist_v2)-1): if WGdist_v2[i+1] != 0: try: WGspdKnots.append((WGdist_v2[i+1])/((WGtime[i+1]-WGtime[i])/1000)*3600/1852) #m/s to kn except: WGspdKnots.append(0) else: WGspdKnots.append(0) #keeps same length as WGtime indkn0=np.where(np.array(WGspdKnots)<0)[0] for i in range(0,len(indkn0)): WGspdKnots[indkn0[i]]=0 indknlarge=np.where(np.array(WGspdKnots)>4)[0] for i in range(0,len(indknlarge)): WGspdKnots[indknlarge[i]]=0 # #----------------------------# # #Crate Wave Energy dataFrame and write to csv #West Monterey and Pt Sur wave data and WG speed #extract data from each buoy and create a common time for buoy data -- sometimes buoy data skips a reading wmon=wmon_data[['wvht','apd','steepness','unix_time']] ptsur=ptsur_data[['wvht','apd','steepness','unix_time']] max1=np.max(np.max(ptsur['unix_time'])) max2=np.max(np.max(wmon['unix_time'])) maxval=np.max([max1,max2]) min1=np.min(np.min(ptsur['unix_time'])) min2=np.min(np.min(wmon['unix_time'])) minval=np.min([min1,min2]) #30 min time stamp buoytime=np.arange(minval,maxval+1000*30*60,1000*30*60) #emtpy array buoydata=pd.DataFrame(index=range(0,len(buoytime)), columns=['unix_time','Wmon_wvht','Wmon_apd','Wmon_steep','Psur_wvht','Psur_apd','Psur_steep']) buoydata['unix_time']=buoytime #assign data from each buoy to correct time stamp buoydata['Wmon_wvht']=buoydata['unix_time'].map(wmon.set_index('unix_time')['wvht']) buoydata['Wmon_apd']=buoydata['unix_time'].map(wmon.set_index('unix_time')['apd']) buoydata['Wmon_steep']=buoydata['unix_time'].map(wmon.set_index('unix_time')['steepness']) buoydata['Psur_wvht']=buoydata['unix_time'].map(ptsur.set_index('unix_time')['wvht']) buoydata['Psur_apd']=buoydata['unix_time'].map(ptsur.set_index('unix_time')['apd']) buoydata['Psur_steep']=buoydata['unix_time'].map(ptsur.set_index('unix_time')['steepness']) # #align 30-min wave data to 5-min WGtime WmonWvhtAligned = list() WmonApdAligned = list() WmonSteepAligned = list() PtSurWvhtAligned = list() PtSurApdAligned = list() PtSurSteepAligned = list() WGbuoytime = list() WGSpdAligned = list() #has to be reduced to match length of time of buoy data lastind=-1 #define try: buoy_last=buoydata['unix_time'][len(buoydata['unix_time'])-1] for j in range(0,len(WGtime)): diff=list() if WGtime[j] <= buoy_last+5*60*100: WGbuoytime.append(WGtime[j]) WGSpdAligned.append(WGspdKnots[j]) diff.append( WGtime[j] - buoydata['unix_time']) ind = np.argmin(np.absolute(diff)) if len(diff) > 0 and np.absolute(diff[0][ind]) < 5*60*1000 and ind!=lastind:# take all points within 5min of each other and remove consecutive duplicates #if np.absolute(diff[0][ind]) < 5*60*1000 and ind!=lastind:# take all points within 5min of each other and remove consecutive duplicates WmonWvhtAligned.append(buoydata['Wmon_wvht'][ind]) WmonApdAligned.append(buoydata['Wmon_apd'][ind]) WmonSteepAligned.append(buoydata['Wmon_steep'][ind]) PtSurWvhtAligned.append(buoydata['Psur_wvht'][ind]) PtSurApdAligned.append(buoydata['Psur_apd'][ind]) PtSurSteepAligned.append(buoydata['Psur_steep'][ind]) lastind=ind else: WmonWvhtAligned.append(np.nan) WmonApdAligned.append(np.nan) WmonSteepAligned.append(np.nan) PtSurWvhtAligned.append(np.nan) PtSurApdAligned.append(np.nan) PtSurSteepAligned.append(np.nan) # else: # WmonWvhtAligned.append(np.nan) # WmonApdAligned.append(np.nan) # WmonSteepAligned.append(np.nan) # PtSurWvhtAligned.append(np.nan) # PtSurApdAligned.append(np.nan) # PtSurSteepAligned.append(np.nan) #convert wave height to feet WmonWvhtAlignedFeet=list() PtSurWvhtAlignedFeet=list() for i in range (0,len(WGbuoytime)): WmonWvhtAlignedFeet.append(WmonWvhtAligned[i]*3937/1200) PtSurWvhtAlignedFeet.append(PtSurWvhtAligned[i]*3937/1200) #now, write aligned file for Wave Energy wg.writeFileWave(myDirWeb,'WaveEnergy',['VehicleSpeed','WMonWaveHeight','WMonPeriod','WMonSteepness','PtSurWaveHeight', 'PtSurPeriod','PtSurSteepness'], WGbuoytime,[WGSpdAligned, WmonWvhtAlignedFeet, WmonApdAligned, WmonSteepAligned,PtSurWvhtAlignedFeet,PtSurApdAligned, PtSurSteepAligned], [4,1,1,1,1,1,1]) shutil.copy2(myDirWeb +'csv/WaveEnergy.csv', myDirRT +'WaveEnergy' +'.csv') except: print('no deployment') #if no WG time exists, catches error #----------------------------# # Write .csv files #Blank data (all zeros) w/ 'Waveglider' time stamps blank = list() for i in range(0,len(WGtime)): blank.append(0) wg.writeFile(myDirWeb+'/csv/', tBeg,'Blank',['blank'],WGtime,[blank],[0]) #Position wg.writeFile(myDirWeb+'/csv/', tBeg,'Position',['lon','lat'], WGtime, [WGlon, WGlat], [5,5]) shutil.copy2(myDirWeb +'csv/Position.csv', myDirRT +'Position' +'.csv') #Target Waypoint wg.writeFile(myDirWeb+'/csv/', tBeg,'Waypoint', ['Waypoint'], WGtime, [WGwaypoint], [1]) shutil.copy2(myDirWeb +'csv/Waypoint.csv', myDirRT +'Waypoint' +'.csv') #CTD1 if len(CTD1time) > 0: wg.writeFile(myDirWeb+'/csv/', tBeg,'CTD',['temperature','salinity','oxygen'], CTD1time,[CTD1temp,CTD1sal,CTD1oxygen],[3,3,3]) shutil.copy2(myDirWeb +'csv/CTD.csv', myDirRT +'CTD' +'.csv') #CTD2 if len(CTD2time) > 0: wg.writeFile(myDirWeb+'/csv/', tBeg,'CTD2',['temperature','salinity','oxygen'], CTD2time,[CTD2temp,CTD2sal,CTD2oxygen],[3,3,3]) shutil.copy2(myDirWeb +'csv/CTD2.csv', myDirRT +'CTD2' +'.csv') #CTDd if len(CTD1time) > 0 and len(CTD2time) > 0: wg.writeFile(myDirWeb+'/csv/', tBeg,'CTDd',['dT','dS','dO2'], CTDdTime,[CTDdTemp,CTDdSal,CTDdOxygen],[3,3,3]) shutil.copy2(myDirWeb +'csv/CTDd.csv', myDirRT +'CTDd' +'.csv') #CO2 #first, align 30-min CO2 to 5-min pH pCO2waterAligned = list() pCO2airAligned = list() for i in range(0,len(CO2time)): #assuming pCO2 measurement occurs at same time as FIRST pH measurement...JS 8/31/16 for j in range(0,len(pHtimeInc)-1): pCO2waterAligned.append(np.nan) pCO2airAligned.append(np.nan) pCO2waterAligned.append(pCO2water[i]) pCO2airAligned.append(pCO2air[i]) #now, write aligned file wg.writeFile(myDirWeb+'/csv/', tBeg,'CO2',['pH','pCO2water','pCO2air'], pHtime,[pH, pCO2waterAligned, pCO2airAligned], [3,2,2]) shutil.copy2(myDirWeb +'csv/CO2.csv', myDirRT +'CO2' +'.csv') #Eco-Puck wg.writeFile(myDirWeb+'/csv/', tBeg,'EcoPuck',['fluor','bb470','bb650'], EPtime,[EPchl,EPbb470,EPbb650],[3,5,5]) shutil.copy2(myDirWeb +'csv/EcoPuck.csv', myDirRT +'EcoPuck' +'.csv') #MET wg.writeFile(myDirWeb+'/csv/', tBeg,'METwind',['windDir','windSpd'], METtime, [METwindDir, METwindSpd], [1,1]) shutil.copy2(myDirWeb +'csv/METwind.csv', myDirRT +'METwind' +'.csv') wg.writeFile(myDirWeb+'/csv/', tBeg,'MET',['airPress','airTemp'], METtime, [METairPress, METairTemp], [1,1]) shutil.copy2(myDirWeb +'csv/MET.csv', myDirRT +'MET' +'.csv') #VR2C wg.writeFile(myDirWeb+'/csv/', tBeg,'tagID',['tagID'], VR2Ctime, [VR2CtagID], [0]) shutil.copy2(myDirWeb +'csv/tagID.csv', myDirRT +'tagID' +'.csv') #CO2 diagnostics wg.writeFile(myDirWeb+'/csv/', tBeg,'CO2equil',['Temp-ON', 'Temp-OFF', 'Press-ON','Press-OFF', 'xCO2-ON','xCO2-OFF'], CO2time, [CO2equil[:,0].tolist(), CO2equil[:,1].tolist(), CO2equil[:,2].tolist(), CO2equil[:,3].tolist(), CO2equil[:,4].tolist(), CO2equil[:,5].tolist()], [2,2,2,2,2,2] ) shutil.copy2(myDirWeb +'csv/CO2equil.csv', myDirRT +'CO2equil' +'.csv') wg.writeFile(myDirWeb+'/csv/', tBeg,'CO2zero',['Temp-ON', 'Temp-OFF', 'Press-ON','Press-OFF', 'xCO2-ON','xCO2-OFF'], CO2time, [CO2zero[:,0].tolist(), CO2zero[:,1].tolist(), CO2zero[:,2].tolist(), CO2zero[:,3].tolist(), CO2zero[:,4].tolist(), CO2zero[:,5].tolist()], [2,2,2,2,2,2] ) shutil.copy2(myDirWeb +'csv/CO2zero.csv', myDirRT +'CO2zero' +'.csv') wg.writeFile(myDirWeb+'/csv/', tBeg,'CO2air',['Temp-ON', 'Temp-OFF', 'Press-ON','Press-OFF', 'xCO2-ON','xCO2-OFF'], CO2time, [CO2air[:,0].tolist(), CO2air[:,1].tolist(), CO2air[:,2].tolist(), CO2air[:,3].tolist(), CO2air[:,4].tolist(), CO2air[:,5].tolist()], [2,2,2,2,2,2] ) shutil.copy2(myDirWeb +'csv/CO2air.csv', myDirRT +'CO2air' +'.csv') wg.writeFile(myDirWeb+'/csv/', tBeg,'CO2stnd',['Temp-ON', 'Temp-OFF', 'Press-ON','Press-OFF', 'xCO2-ON','xCO2-OFF'], CO2time, [CO2stnd[:,0].tolist(), CO2stnd[:,1].tolist(), CO2stnd[:,2].tolist(), CO2stnd[:,3].tolist(), CO2stnd[:,4].tolist(), CO2stnd[:,5].tolist()], [2,2,2,2,2,2] ) shutil.copy2(myDirWeb +'csv/CO2stnd.csv', myDirRT +'CO2stnd' +'.csv') wg.writeFile(myDirWeb+'/csv/', tBeg,'CO2hum',['equilOff','airOff','stndOff'], CO2time, [CO2humEquilOff, CO2humAirOff, CO2humStndOff], [3,3,3]) shutil.copy2(myDirWeb +'csv/CO2hum.csv', myDirRT +'CO2hum' +'.csv') #Power wg.writeFile(myDirWeb+'/csv/', tBeg,'Power',['solarPowerGenerated','batteryChargingPower', 'outputPowerGenerated', 'totalBatteryPower'], PwrTime, [PwrSolarPwrGen, PwrBatChargePwr, PwrOutPwrGen, PwrTotalBatPwr], [3,3,3,3]) shutil.copy2(myDirWeb +'csv/Power.csv', myDirRT +'Power' +'.csv') #Nav wg.writeFile(myDirWeb+'/csv/', tBeg,'Nav',['COG','heading', 'desired heading'], WGtime, [WGCOG, WGHead, WGDesHead], [1, 1, 1]) shutil.copy2(myDirWeb +'csv/Nav.csv', myDirRT +'Nav' +'.csv') # #Speed # writeFile('Speed','Knots', # WGtime, WGspdKnots, 2) # shutil.copy2(myDirWeb +'csv/Speed.csv', myDirRT +'Speed' +'.csv') wg.writeFileXY(myDirWeb,'CO2_vs_pH', ['CO2','pH'], [pCO2waterAligned, pH], [2,3]) #----------------------------# # Sort .csv files by time files = sorted(os.listdir(myDirWeb +'csv/')) for i in range(0, len(files)): if ( files[i] != 'CO2_vs_pH.csv' and files[i] != 'CO2SYS_2100.csv' and files[i] != 'CO2SYS_2400.csv' ): #get .csv file contents myPath = myDirWeb +'csv/' +files[i] f = open(myPath, 'r') contents = f.readlines() #get time stamp only f.seek(0, 0) #return to beginning reader = csv.reader(f) tStamp = list() f.readline() #skip header line for row in reader: tStamp.append(int(row[0])) f.close() #sort by time ind = np.argsort(tStamp) + 1 #create new, sorted list contentsNew = list() contentsNew.append(contents[0][0:-1]) for j in range(0, len(ind)): if contents[ind[j]].find('\n')!=-1: contentsNew.append(contents[ind[j]][0:-1]) else: contentsNew.append(contents[ind[j]]) #replace old file f = open(myPath, 'w') contentsNewJoin = '\n'.join(contentsNew) f.write(contentsNewJoin) f.close() #----------------------------# #lonlat for leaflet map wg.writeLonLat(myDirWeb) #Distance wg.writeFile(myDirWeb+'/csv/', tBeg, 'Distance',['distance'], WGtime, [WGdist], [0]) wg.writeFile(myDirWeb+'/csv/', tBeg, 'Distance_v2',['distance'], WGtime, [WGdist_v2], [0]) year = datetime.now(timezone('UTC')).year #----------------------------# with open(myDirWeb +'../json/dataTable_preDepl.json', newline='\r\n') as data_file: contents = json.load(data_file) #enter current mission stats days = (tEnd - tBeg).days days = days + (tEnd - tBeg).seconds /60/60/24 dist = sum(WGdist_v2) / 1000 contents[0]['dist'] = round(dist) contents[0]['durDay'] = round(days,1) contents[1]['dist'] = round(contents[j]['dist']+contents[0]['dist'],1) contents[1]['durDay'] = round(contents[j]['durDay']+contents[0]['durDay'],1) with open(myDirWeb +'../json/dataTable.json', 'w', newline='\r\n') as outfile: json.dump(contents, outfile) # Update lastUpdated.txt f = open(myDirWeb +'log/lastUpdated.txt','w') f.write( datetime.now(timezone('UTC')).strftime('%Y-%m-%d %H:%M:%S') ) f.close() #----------------------------# print("ended at: " + time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.localtime()))