1 """Functions associated with the study as a whole
2
3
4 """
5 import FirePerimeters
6 import MapSeries
7 import Static
8 import TimeSeries
9 from GP_Util import *
10
12 """Object Representing the whole study.
13
14 The study object is the central component of all subsequent
15 analysis. It is the first argument passed to nearly every other high
16 level function in the toolset. In reality its just a central
17 placeholder to connect and store parameters and results from various
18 other functions.
19
20 Inputs
21 workingdir path to study directory to be created
22 title title
23
24 Procedure
25 Create / Overwrite main study directory
26 Name empty lists for source and output data series
27
28 Data
29 sourcenames list of source data names
30 infodict dict of source data parameters
31 serieslist list of output data series names
32 seriesdict dict of extract var {name:file}
33
34 """
35 - def __init__(self, workingdir="", title=""):
36 """Initialize the study object. Set the parameters, make output folders, and create new empty lists to store data """
37 self.title = title
38 self.workingdir = workingdir
39
40 MakeFolder(self.workingdir)
41 self.datafolder = os.path.abspath('StudyData')
42 MakeFolder(self.datafolder)
43 gp.Workspace = self.datafolder
44
45 self.sourcenames = []
46 self.infodict = {}
47 self.serieslist = []
48 self.seriesdict = {}
49
50 log.info('Create New Study: '+ self.workingdir)
51
53 """Save the study object to mainpickle"""
54 pickle.dump(self, open(os.path.join(self.workingdir,'mainpickle'),'w'))
55
57 """Add some new unique dates to the study dates"""
58 newdates = newdates[newdates > self.startdate]
59 newdates = newdates[newdates < self.enddate]
60
61 self.dates = num.unique( num.hstack((self.dates,newdates)) )
62 self.startdates = self.dates[:-1]
63 self.enddates = self.dates[1:]
64
65 TimeSeries.Import(self,'SDATE',invalues=self.startdates)
66 TimeSeries.Import(self,'EDATE',invalues=self.enddates)
67
69 """Add an Output Data Dict and update namelist"""
70 varname = vardict['name']
71 self.seriesdict[varname] = vardict
72 if varname not in self.serieslist:
73 self.serieslist.append(varname)
74 self.Save()
75
77 """Add a Source Data Dictionary and update namelist"""
78 self.infodict[varname] = vardict
79 if varname not in self.sourcenames:
80 self.sourcenames.append(varname)
81 self.Save()
82
83
84
86 """Load an Existing study by specifying the study directory"""
87 study = pickle.load(open(os.path.join(workingdir,'mainpickle'),'r'))
88 msg = 'Loaded Saved Study from '+os.path.basename(workingdir)
89 msg += '\n Data Vars: ' + str(study.infodict.keys() )
90 msg += '\n Extract Vars: ' + str(study.serieslist)
91 log.info(msg)
92 return study
93
94 -def CreatePointGrid(study, studyshp, cellsize, startdate, enddate, dateinterval):
95 """Creates the geographic base data for a study. These include a
96 study area polygon shapefile, a point grid and a raster each
97 representing the study area. The study sample dates are defined.
98 Index variables are initialized (via Static.Import and
99 TimeSeries.import for the PointID and dates respectivly)
100
101 Inputs
102 study study object
103 studyshp input study area shapefile
104 cellsize study cellsize
105 startdate starting date of the study
106 enddate ending date of the study
107 dateinterval max length of sampled intervals
108
109 Procedure
110 assign study parameters
111 copy and format study area polygon
112 create study area raster
113 Import index variables
114 PID point id number
115 SDATE sample interval start date
116 EDATE sample interval end date
117
118 Outputs
119 studyarea.shp study area shapefile
120 studyraster study area raster
121 pointgrid.shp study area point grid
122
123 """
124
125 os.chdir(study.datafolder)
126 gp.Workspace = study.datafolder
127
128
129 study.startdate = startdate
130 study.enddate = enddate
131 study.dateinterval= dateinterval
132 study.dates = num.arange(startdate, enddate+dateinterval, dateinterval)
133 study.startdates = study.dates[:-1]
134 study.enddates = study.dates[1:]
135
136 study.isFF = False
137 study.cellsize = cellsize
138
139 study.studyshp = os.path.abspath('studyarea.shp')
140 study.finstudyshp=os.path.abspath('finstudyarea.shp')
141 study.studyraster = os.path.abspath('studyraster')
142 study.pointgrid = os.path.abspath('pointgrid.shp')
143 study.mask = os.path.abspath('pointgrid.npy')
144 ascii = os.path.abspath('ascii.txt')
145
146
147 gp.copy(studyshp, study.studyshp)
148
149
150 gp.AddField_management(study.studyshp, "EVTDATE", "DOUBLE")
151 gp.CalculateField_management(study.studyshp, 'EVTDATE', study.startdate)
152 gp.AddField_management(study.studyshp, "STATE", "DOUBLE")
153 gp.CalculateField_management(study.studyshp, 'STATE', -9999)
154
155
156 gp.copy(study.studyshp, study.finstudyshp)
157 gp.CalculateField_management(study.finstudyshp, 'EVTDATE', study.enddate)
158
159
160 gp.PolygonToRaster_conversion(study.studyshp,'EVTDATE',study.studyraster,'#','#',study.cellsize)
161
162
163 gp.RasterToPoint_conversion(study.studyraster,study.pointgrid,'VALUE')
164 FilterFields(study.pointgrid,['EVTDATE','POINTID','STATE'])
165 gp.CalculateField_management(study.pointgrid, 'POINTID', ("!FID!"), "PYTHON")
166
167 gp.PointToRaster_conversion(study.pointgrid,"FID", study.studyraster, 'MAXIMUM', "NONE", study.cellsize)
168
169 study.height = gp.describe(study.studyraster).Height
170 study.width = gp.describe(study.studyraster).Width
171
172
173 gp.RasterToASCII_Conversion(study.studyraster,ascii)
174 mask = num.loadtxt(ascii,skiprows=6)
175 mask = mask != -9999
176 while sum(mask[0,:]) == 0:
177 mask = mask[1:,:]
178 log.info('droprow')
179 while sum(mask[:,-1]) == 0:
180 mask = mask[:,:-1]
181 log.info('dropcol')
182
183 study.rawpointcount = mask.shape[0] * mask.shape[1]
184 study.pointcount = num.sum(mask)
185
186 num.save(study.mask, mask)
187 os.remove(ascii)
188
189 Static.Import(study,'PID',study.studyraster)
190 TimeSeries.Import(study,'SDATE',invalues=study.startdates)
191 TimeSeries.Import(study,'EDATE',invalues=study.enddates)
192
193 log.info('Created Study Grid Points')
194 log.info('height: '+str(study.height) +', width: '+str(study.width))
195
196 study.Save()
197
199 """Call the type-specific Case History Extraction Functions"""
200
201 for varname in study.sourcenames:
202 datatype = study.infodict[varname]['type']
203 if datatype == 'fire':
204 FirePerimeters.Extract(study,varname)
205 if datatype == 'map':
206 MapSeries.Extract(study,varname)
207 if datatype == 'time':
208 TimeSeries.Extract(study,varname)
209 if datatype == 'static':
210 Static.Extract(study,varname)
211
213 """Create a randomly sampled subset of study points to export to
214 easily readable comma separated format (.csv).
215
216 Inputs
217 study study object (after a call to ExtractData() )
218 outdir subset folder to be created in the study directory
219 n number of sample points (large n for all points)
220
221 Outputs
222
223 Variable data tables ( [varname].csv )
224 For each study output variable a table is produced which is
225 organized with study sample points in rows and study sample
226 dates in columns.
227
228 Combined case history table ( casedata.csv )
229 Additionally, a second data table is produced which is a
230 combination of all the data in the variable data tables. In this
231 file each row corresponds to a single point at a single date
232 with each study variable in columns. This table is simply a
233 combined reorganization of the individual tables """
234
235 log.info('Extract Subset: '+outdir+' ('+str(n)+')')
236 subdir = MakeFolder(os.path.join(study.workingdir,outdir))
237
238
239 mask = num.load(study.mask).flatten()
240 maskidx = num.where(mask == True)[0]
241 sampidx = num.random.randint(low=0, high=maskidx.shape[0], size=n )
242 takelist = maskidx[sampidx]
243
244
245 filelist = [study.seriesdict[f]['file'] for f in study.serieslist]
246 outlist = [ os.path.basename(f) for f in filelist]
247 outlist = [ os.path.abspath(f) for f in outlist]
248 outlist = [ os.path.splitext(f)[0]+'.csv' for f in outlist]
249
250
251 datalist = [num.load(f)[takelist,:] for f in filelist]
252
253
254 for (data,outfile) in zip(datalist,outlist):
255 num.savetxt(outfile, data, fmt='%f',delimiter=',')
256
257
258 datalist = [data.flatten() for data in datalist]
259 combdata = num.vstack(datalist).transpose()
260 ArrayToCSV(os.path.abspath('casedata.csv'), combdata, study.serieslist)
261
262 if copy == True:
263 CopyResults(study.workingdir, subdir)
264
265
266
267 -def CopyResults(studydir, srcdir, dbox='C:\\My Dropbox\\Data'):
268 """Copy a results folder to the dropbox while making
269 necessary parent directories"""
270
271
272 newstudydir = os.path.join(dbox, os.path.basename(studydir))
273 if os.path.exists(newstudydir) == False:
274 MakeFolder(newstudydir)
275
276 destdir = os.path.join(newstudydir, os.path.basename(srcdir))
277 MakeFolder(destdir)
278
279
280 for name in os.listdir(srcdir):
281 srcpath = os.path.join(srcdir, name)
282 destpath = os.path.join(destdir, name)
283 shutil.copy(srcpath,destpath)
284