import os.path import seawater as sw import numpy as np import csv from datetime import datetime, timedelta from pytz import timezone import pytz import time #strptime for e-mail alarms from geopy.distance import geodesic import pandas as pd #for reading master spreadsheet xls file import json #for loading json file manually import wgFunctions as wg import warnings #----------------------------# vehicles = ['wgHansen','wgTiny','wgTest'] #vehicles = ['wgHansen'] #vehicles = ['wgTest'] #----------------------------# print("start at: " + time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.localtime())) for veh in vehicles: #get cal/config info off master spreadsheet #define data directories #myDirWG = 'C:/Users/tmasek/Dropbox/Projects/MBARI/WaveGliderDisplay/WaveGlider/' myDirWG = '//atlas.shore.mbari.org/WaveGlider/' #myDirWeb = 'C:/Users/tmasek/Dropbox/Projects/MBARI/WaveGliderDisplay/WaveGlider/' + vehicle +'/data/' myDirWeb = '//atlas.shore.mbari.org/oasis_users/web/extern/bog/waveglider/' +veh + '/data/' #see if there is a active run and all calib lines are added #get cal/config info off master spreadsheet warnings.filterwarnings('ignore', category=UserWarning, module='openpyxl') xlInfo = pd.read_excel(myDirWG +'WG_cal_config.xlsx', engine='openpyxl', sheet_name='Info', na_values=['NaN','N/A'], keep_default_na=False, engine_kwargs={'read_only': True}) xlCCU = pd.read_excel(myDirWG +'WG_cal_config.xlsx', engine='openpyxl', sheet_name='CCU', na_values=['NaN','N/A'], keep_default_na=False, engine_kwargs={'read_only': True}) xlGPCTD= pd.read_excel(myDirWG +'WG_cal_config.xlsx', engine='openpyxl', sheet_name='GPCTD+DO', na_values=['NaN','N/A'], keep_default_na=False, engine_kwargs={'read_only': True}) xlBB = pd.read_excel(myDirWG +'WG_cal_config.xlsx', engine='openpyxl', sheet_name='BB', na_values=['NaN','N/A'], keep_default_na=False, engine_kwargs={'read_only': True}) xlpH = pd.read_excel(myDirWG +'WG_cal_config.xlsx', engine='openpyxl', sheet_name='pH', na_values=['NaN','N/A'], keep_default_na=False, engine_kwargs={'read_only': True}) xlCO2 = pd.read_excel(myDirWG +'WG_cal_config.xlsx', engine='openpyxl', sheet_name='CO2', na_values=['NaN','N/A'], keep_default_na=False, engine_kwargs={'read_only': True}) xlVR2C = pd.read_excel(myDirWG +'WG_cal_config.xlsx', engine='openpyxl', sheet_name='VR2C', na_values=['NaN','N/A'], keep_default_na=False, engine_kwargs={'read_only': True}) xlMET = pd.read_excel(myDirWG +'WG_cal_config.xlsx', engine='openpyxl', sheet_name='MET', na_values=['NaN','N/A'], keep_default_na=False, engine_kwargs={'read_only': True}) dep = '' xlDeps = xlInfo['Deployment'] xlRowStart = xlDeps[xlDeps == veh].index[0] finished_run = False for index, row in xlInfo[xlRowStart + 1:int(xlInfo.shape[0])].iterrows(): if row['Deployment'] == '': break fmt = '%Y-%m-%dT%H:%M:%SZ' if row['tEnd'] != '': if (datetime.now(timezone('UTC')) - datetime.strptime(row['tEnd'], fmt).replace(tzinfo=pytz.utc)) < timedelta(days=400) : dep = str(row['Deployment']) myDirPC = myDirWG +'deployment_data/' +veh +'/' +dep +'/postCruise/' if not os.path.isdir(myDirPC): os.makedirs(myDirPC) finished_run = True xlRowInd = index # tBeg, tEnd fmt = '%Y-%m-%dT%H:%M:%SZ' tBeg = datetime.strptime(xlInfo.loc[xlRowInd,'tBeg'], fmt).replace(tzinfo=pytz.utc) tEnd = datetime.strptime(xlInfo.loc[xlRowInd,'tEnd'], fmt).replace(tzinfo=pytz.utc) #----------------------------# if finished_run: # CCU vids CCUname = xlCCU.loc[xlRowInd,'CCU'] CCUnum = str(int(xlCCU.loc[xlRowInd,'VID'])) # CTD1 O2cal1 = xlGPCTD.loc[xlRowInd,:][7:14].tolist() O2cal2 = xlGPCTD.loc[xlRowInd,:][19:26].tolist() # Eco-Puck EPcal = [ xlBB.loc[xlRowInd,:][7:9].tolist(), xlBB.loc[xlRowInd,:][9:11].tolist(), xlBB.loc[xlRowInd,:][11:13].tolist() ] # pH try: pHcal = [ round( xlpH.loc[xlRowInd,:].iloc[12], 6), xlpH.loc[xlRowInd,:].iloc[11] ] except: pHcal = ['N/A', 'N/A'] # CO2 CO2cal = xlCO2.loc[xlRowInd,:][6:10].tolist() CO2stndKn = xlCO2.loc[xlRowInd,:].iloc[19] print(veh + " " + str(tBeg) + " " + str(tEnd)) print(str(CCUnum)+ " " + str(tBeg.isoformat())+ " " + str(tEnd.isoformat())) CTD1time, CTD1temp, CTD1sal, CTD1press, CTD1o2Hz, CTD1o2sol, CTD2time, CTD2temp, CTD2sal, CTD2press, CTD2o2Hz, CTD2o2sol = wg.getCTDDataRecord(CCUnum,tBeg,tEnd) print('CTD:' + str(len(CTD1time)) + " records") CTD1woO2time, CTD1woO2temp, CTD1woO2sal, CTD1woO2press, CTD1woO2o2Hz, CTD1woO2o2sol, CTD2woO2time, CTD2woO2temp, CTD2woO2sal, CTD2woO2press, CTD2woO2o2Hz, CTD2woO2o2sol = wg.getCTDwoO2DataRecord(CCUnum,tBeg,tEnd) print('CTDwoO2:' + str(len(CTD1woO2time)) + " records") METtime, METwindDir, METwindSpd, METairPress, METairTemp = wg.getWindRecord(CCUnum,tBeg,tEnd) print('Wind:' + str(len(METtime)) + " records") WGtime, WGlon, WGlat, WGdist, WGwaypoint, WGHead, WGDesHead, WGCOG = wg.getTelemRecord(CCUnum,tBeg,tEnd) print('Telemetry:' + str(len(WGtime)) + " records") PwrTime, PwrSolarPwrGen, PwrBatChargePwr, PwrOutPwrGen, PwrTotalBatPwr = wg.getPowermRecord(CCUnum,tBeg,tEnd) print('Power:' + str(len(PwrTime)) + " records") EPtime, EPcounts470, EPcounts650, EPcountsChl = wg.getEcoPuckRecord(CCUnum,tBeg,tEnd) print('EcoPuck:' + str(len(EPtime)) + " records") VR2Cstatus = wg.getVemcoStatRecord(CCUnum,tBeg,tEnd) print('VR2C Status:' + str(len(VR2Cstatus)) + " records") VR2Ctime, VR2CtagID, VR2CtagReading = wg.getVemcoTagRecord(CCUnum,tBeg,tEnd) print('VR2C Tag:' + str(len(VR2Ctime)) + " records") CO2time, CO2equilOffCO2, CO2equilOffTemp, CO2zeroOffCO2, CO2airOffCO2, CO2airOffTemp, pHtime, pHV1, pHtimeInc, CO2equil, CO2zero, CO2air, CO2stnd, CO2humEquilOff, CO2humAirOff, CO2humStndOff = wg.getCO2Record(CCUnum,tBeg,tEnd) print('C02:' + str(len(CO2time)) + " records") #----------------------------# # Convert your raw data #Eco-Puck EPbb470 = list() EPbb650 = list() EPchl = list() for i in range(0,len(EPtime)): EPchl.append( wg.convEP(EPcountsChl[i], EPcal[2]) ) diff = list() for j in range(0,len(CTD1time)): diff.append( CTD1time[j] - EPtime[i] ) if len(diff) > 0: ind = np.argmin(np.absolute(diff)) if np.absolute(diff[ind]) < 10*60*1000: EPbb470.append( wg.counts2bb( wg.convEP(EPcounts470[i], EPcal[0]), float(470), CTD1sal[ind] ) ) EPbb650.append( wg.counts2bb( wg.convEP(EPcounts650[i], EPcal[1]), float(650), CTD1sal[ind] ) ) else: # USE WITH CAUTION!! EPbb470.append(np.nan) EPbb650.append(np.nan) # #if no CTD data, can use METtime instead (could also use CO2time) # for j in range(0,len(METtime)): else: # USE WITH CAUTION!! EPbb470.append(np.nan) EPbb650.append(np.nan) # diff.append( METtime[j] - EPtime[i] ) # EPbb650.append( wg.counts2bb( wg.convEP(EPcounts650[i], EPcal[1]), float(650), 33.5 ) ) CTD1oxygen = list() if len(CTD1time) > 0: CTD1sigt = (sw.dens(CTD1sal,CTD1temp,CTD1press)-1000).tolist() for i in range(0,len(CTD1time)): if len(CTD1o2sol) > 0: CTD1oxygen.append(wg.convO2(CTD1o2Hz[i], CTD1o2sol[i], O2cal1, CTD1press[i], CTD1temp[i], CTD1sigt[i])) else: CTD1oxygen.append(np.nan) #CTD2 CTD2oxygen = list() if len(CTD2time) > 0: CTD2sigt = (sw.dens(CTD2sal,CTD2temp,CTD2press)-1000).tolist() for i in range(0,len(CTD2time)): if len(CTD2o2sol) > 0: CTD2oxygen.append(wg.convO2(CTD2o2Hz[i], CTD2o2sol[i], O2cal2, CTD2press[i], CTD2temp[i], CTD2sigt[i])) else: CTD2oxygen.append(np.nan) #pH pH = list() for i in range(0,len(pHtime)): diff = list() for j in range(0,len(CTD1time)): diff.append( CTD1time[j] - pHtime[i] ) if len(diff) > 0: ind = np.argmin(np.absolute(diff)) if np.absolute(diff[ind]) < 60*60*1000: #**CHANGED TO 60 MIN B/C OF SOME UNRESOLVED ISSUE...** pH.append( wg.pHV12pH(pHV1[i], pHcal, CTD1temp[ind]) ) else: pH.append(np.nan) # pH.append( wg.pHV12pH(pHV1[i], pHcal, 11) ) else: pH.append(np.nan) # pH.append( wg.pHV12pH(pHV1[i], pHcal, 11) ) #pCO2 #replace standard zeros with NaN CO2stndMeas = list() CO2stndTime = list() CO2stnd[CO2stnd==0] = np.nan for i in range(0, len(CO2humStndOff)): if CO2humStndOff[i]==0: CO2humStndOff[i] = np.nan #de-NaN and linear-interp stnd meas's for i in range(0, len(CO2time)): if not np.isnan(CO2stnd[i,5]): CO2stndMeas.append( CO2stnd[i,5] ) CO2stndTime.append( CO2time[i] ) if (len(CO2stndMeas)) == 0 and (len(CO2time) > 0): #if haven't got any stnd readings yet, assume is as known CO2stndMeas = [CO2stndKn, CO2stndKn] #this will overwrite once u do get one CO2stndTime = [CO2time[0], CO2time[-1]] if len(CO2time) > 0: CO2stndMeasInterp = np.interp(CO2time, CO2stndTime, CO2stndMeas).tolist() #apply calibration pCO2water = list() pCO2air = list() for i in range(0,len(CO2time)): #Air - just apply cal and u'r done pCO2air.append( wg.convCO2( CO2airOffCO2[i], CO2zeroOffCO2[i], CO2cal, CO2airOffTemp[i], CO2stndMeasInterp[i], CO2stndKn ) ) #Water - must also convert xCO2 to pCO2 xCO2waterConv = wg.convCO2( CO2equilOffCO2[i], CO2zeroOffCO2[i], CO2cal, CO2equilOffTemp[i], CO2stndMeasInterp[i], CO2stndKn ) diff = list() for j in range(0,len(CTD1time)): diff.append( CTD1time[j] - CO2time[i] ) if len(diff) > 0: ind = np.argmin(np.absolute(diff)) if np.absolute(diff[ind]) < 10*60*1000: pCO2water.append( wg.calcpCO2( xCO2waterConv, CTD1temp[ind], CTD1sal[ind] ) ) else: pCO2water.append(np.nan) else: pCO2water.append(np.nan) #clean-up LR distance ind1 = np.where(np.array(WGdist)<0)[0] ind2 = np.where(np.array(WGdist)>10000)[0] for i in range(0,len(ind1)): WGdist[ind1[i]] = 0 for i in range(0,len(ind2)): WGdist[ind2[i]] = 0 #other way of calc'ing dist using gps WGdist_v2 = list() WGdist_v2.append(0) for i in range(0,len(WGlon)-1): WGdist_v2.append( int( geodesic( (WGlat[i], WGlon[i]), (WGlat[i+1],WGlon[i+1]) ).meters ) ) #clean-up gps distance ind3 = np.where(np.array(WGdist_v2)<0)[0] ind4 = np.where(np.array(WGdist_v2)>10000)[0] for i in range(0,len(ind3)): WGdist_v2[ind3[i]] = 0 for i in range(0,len(ind4)): WGdist_v2[ind4[i]] = 0 #burst-avg CTD #CTD1 CTD1timeAvg, CTD1tempAvg = wg.burstAvgCTD(CTD1time, CTD1temp) null, CTD1salAvg = wg.burstAvgCTD(CTD1time, CTD1sal) null, CTD1oxygenAvg = wg.burstAvgCTD(CTD1time, CTD1oxygen) #CTD2 CTD2timeAvg, CTD2tempAvg = wg.burstAvgCTD(CTD2time, CTD2temp) null, CTD2salAvg = wg.burstAvgCTD(CTD2time, CTD2sal) null, CTD2oxygenAvg = wg.burstAvgCTD(CTD2time, CTD2oxygen) #align CTD2 to CTD1 CTD2tempAvga = wg.alignCTD( CTD2timeAvg, CTD2tempAvg, CTD1timeAvg) CTD2salAvga = wg.alignCTD( CTD2timeAvg, CTD2salAvg, CTD1timeAvg) CTD2oxygenAvga = wg.alignCTD( CTD2timeAvg, CTD2oxygenAvg, CTD1timeAvg) #calc difference CTDdTime = CTD1timeAvg CTDdTemp = np.subtract(CTD1tempAvg, CTD2tempAvga).tolist() CTDdSal = np.subtract(CTD1salAvg, CTD2salAvga).tolist() CTDdOxygen = np.subtract(CTD1oxygenAvg, CTD2oxygenAvga).tolist() # #use distance to calculate speed WGspdKnots=list() WGspdKnots.append(0) #initialize to keep length equal to time for i in range(0,len(WGdist_v2)-1): if WGdist_v2[i+1] != 0: try: WGspdKnots.append((WGdist_v2[i+1])/((WGtime[i+1]-WGtime[i])/1000)*3600/1852) #m/s to kn except: WGspdKnots.append(0) else: WGspdKnots.append(0) #keeps same length as WGtime indkn0=np.where(np.array(WGspdKnots)<0)[0] for i in range(0,len(indkn0)): WGspdKnots[indkn0[i]]=0 indknlarge=np.where(np.array(WGspdKnots)>4)[0] for i in range(0,len(indknlarge)): WGspdKnots[indknlarge[i]]=0 #----------------------------# # Write .csv files #Blank data (all zeros) w/ 'Waveglider' time stamps blank = list() for i in range(0,len(WGtime)): blank.append(0) wg.writeFile(myDirPC, tBeg, 'Blank',['blank'],WGtime,[blank],[0]) #Position wg.writeFile(myDirPC, tBeg, 'Position',['lon','lat'], WGtime, [WGlon, WGlat], [5,5]) #Target Waypoint wg.writeFile(myDirPC, tBeg, 'Waypoint', ['Waypoint'], WGtime, [WGwaypoint], [1]) #CTD1 if len(CTD1time) > 0: wg.writeFile(myDirPC, tBeg, 'CTD',['temperature','salinity','oxygen'], CTD1time,[CTD1temp,CTD1sal,CTD1oxygen],[3,3,3]) #CTD2 if len(CTD2time) > 0: wg.writeFile(myDirPC, tBeg, 'CTD2',['temperature','salinity','oxygen'], CTD2time,[CTD2temp,CTD2sal,CTD2oxygen],[3,3,3]) #CTDd if len(CTD1time) > 0 and len(CTD2time) > 0: wg.writeFile(myDirPC, tBeg, 'CTDd',['dT','dS','dO2'], CTDdTime,[CTDdTemp,CTDdSal,CTDdOxygen],[3,3,3]) #CO2 #first, align hourly-CO2 to 10-min pH pCO2waterAligned = list() pCO2airAligned = list() for i in range(0,len(CO2time)): #assuming pCO2 measurement occurs at same time as FIRST pH measurement...JS 8/31/16 for j in range(0,len(pHtimeInc)-1): pCO2waterAligned.append(np.nan) pCO2airAligned.append(np.nan) pCO2waterAligned.append(pCO2water[i]) pCO2airAligned.append(pCO2air[i]) #now, write aligned file wg.writeFile(myDirPC, tBeg, 'CO2',['pH','pCO2water','pCO2air'], pHtime,[pH, pCO2waterAligned, pCO2airAligned], [3,2,2]) #Eco-Puck wg.writeFile(myDirPC, tBeg, 'EcoPuck',['chl','bb470','bb650'], EPtime,[EPchl,EPbb470,EPbb650],[3,5,5]) #MET wg.writeFile(myDirPC, tBeg, 'METwind',['windDir','windSpd'], METtime, [METwindDir, METwindSpd], [1,1]) wg.writeFile(myDirPC, tBeg, 'MET',['airPress','airTemp'], METtime, [METairPress, METairTemp], [1,1]) #Distance wg.writeFile(myDirPC, tBeg, 'Distance',['distance'], WGtime, [WGdist], [0]) #CO2 diagnostics if len(CO2time) > 0: wg.writeFile(myDirPC, tBeg, 'CO2equil',['Temp-ON', 'Temp-OFF', 'Press-ON','Press-OFF', 'xCO2-ON','xCO2-OFF'], CO2time, [CO2equil[:,0].tolist(), CO2equil[:,1].tolist(), CO2equil[:,2].tolist(), CO2equil[:,3].tolist(), CO2equil[:,4].tolist(), CO2equil[:,5].tolist()], [2,2,2,2,2,2] ) wg.writeFile(myDirPC, tBeg, 'CO2zero',['Temp-ON', 'Temp-OFF', 'Press-ON','Press-OFF', 'xCO2-ON','xCO2-OFF'], CO2time, [CO2zero[:,0].tolist(), CO2zero[:,1].tolist(), CO2zero[:,2].tolist(), CO2zero[:,3].tolist(), CO2zero[:,4].tolist(), CO2zero[:,5].tolist()], [2,2,2,2,2,2] ) wg.writeFile(myDirPC, tBeg, 'CO2air',['Temp-ON', 'Temp-OFF', 'Press-ON','Press-OFF', 'xCO2-ON','xCO2-OFF'], CO2time, [CO2air[:,0].tolist(), CO2air[:,1].tolist(), CO2air[:,2].tolist(), CO2air[:,3].tolist(), CO2air[:,4].tolist(), CO2air[:,5].tolist()], [2,2,2,2,2,2] ) wg.writeFile(myDirPC, tBeg, 'CO2stnd',['Temp-ON', 'Temp-OFF', 'Press-ON','Press-OFF', 'xCO2-ON','xCO2-OFF'], CO2time, [CO2stnd[:,0].tolist(), CO2stnd[:,1].tolist(), CO2stnd[:,2].tolist(), CO2stnd[:,3].tolist(), CO2stnd[:,4].tolist(), CO2stnd[:,5].tolist()], [2,2,2,2,2,2] ) wg.writeFile(myDirPC, tBeg, 'CO2hum',['equilOff','airOff','stndOff'], CO2time, [CO2humEquilOff, CO2humAirOff, CO2humStndOff], [3,3,3]) #Power wg.writeFile(myDirPC, tBeg, 'Power',['solarPowerGenerated','batteryChargingPower', 'outputPowerGenerated', 'totalBatteryPower'], PwrTime, [PwrSolarPwrGen, PwrBatChargePwr, PwrOutPwrGen, PwrTotalBatPwr], [3,3,3,3]) with open(myDirWeb +'../json/dataTable_preDepl.json', newline='\r\n') as data_file: contents = json.load(data_file) days = (datetime.now(timezone('UTC')) - tBeg).days days = days + ((datetime.now(timezone('UTC')) - tBeg).seconds /60/60/24 ) dist = sum(WGdist_v2) / 1000 contents[1]['dist'] = round(contents[1]['dist']+round(dist),1) contents[1]['durDay'] = round(contents[1]['durDay']+days,1) #write new table with open(myDirWeb +'../json/dataTable_preDepl.json', 'w', newline='\r\n') as outfile: json.dump(contents, outfile) #----------------------------# # Validate Data # Call to plotting function (assumed to be defined elsewhere) wg.qcWG_plotAll(myDirPC) print("ended at: " + time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.localtime()))