1
|
|
|
# |
2
|
|
|
# Published interface for awips.dataaccess package |
3
|
|
|
# |
4
|
|
|
# |
5
|
|
|
# SOFTWARE HISTORY |
6
|
|
|
# |
7
|
|
|
# Date Ticket# Engineer Description |
8
|
|
|
# ------------ ------- ---------- ------------------------- |
9
|
|
|
# 12/10/12 njensen Initial Creation. |
10
|
|
|
# Feb 14, 2013 1614 bsteffen refactor data access framework to use single request. |
11
|
|
|
# 04/10/13 1871 mnash move getLatLonCoords to JGridData and add default args |
12
|
|
|
# 05/29/13 2023 dgilling Hook up ThriftClientRouter. |
13
|
|
|
# 03/03/14 2673 bsteffen Add ability to query only ref times. |
14
|
|
|
# 07/22/14 3185 njensen Added optional/default args to newDataRequest |
15
|
|
|
# 07/30/14 3185 njensen Renamed valid identifiers to optional |
16
|
|
|
# Apr 26, 2015 4259 njensen Updated for new JEP API |
17
|
|
|
# Apr 13, 2016 5379 tgurney Add getIdentifierValues(), getRequiredIdentifiers(), |
18
|
|
|
# and getOptionalIdentifiers() |
19
|
|
|
# Oct 07, 2016 ---- mjames@ucar Added getForecastRun |
20
|
|
|
# Oct 18, 2016 5916 bsteffen Add setLazyLoadGridLatLon |
21
|
|
|
# Oct 11, 2018 ---- mjames@ucar Added getMetarObs() getSynopticObs() |
22
|
|
|
# |
23
|
|
|
|
24
|
|
|
import sys |
25
|
|
|
import warnings |
26
|
|
|
|
27
|
|
|
THRIFT_HOST = "edex" |
28
|
|
|
|
29
|
|
|
USING_NATIVE_THRIFT = False |
30
|
|
|
|
31
|
|
|
if 'jep' in sys.modules: |
32
|
|
|
# intentionally do not catch if this fails to import, we want it to |
33
|
|
|
# be obvious that something is configured wrong when running from within |
34
|
|
|
# Java instead of allowing false confidence and fallback behavior |
35
|
|
|
import JepRouter |
|
|
|
|
36
|
|
|
router = JepRouter |
37
|
|
|
else: |
38
|
|
|
from awips.dataaccess import ThriftClientRouter |
39
|
|
|
router = ThriftClientRouter.ThriftClientRouter(THRIFT_HOST) |
40
|
|
|
USING_NATIVE_THRIFT = True |
41
|
|
|
|
42
|
|
|
|
43
|
|
View Code Duplication |
def getRadarProductIDs(availableParms): |
|
|
|
|
44
|
|
|
""" |
|
|
|
|
45
|
|
|
Get only the numeric idetifiers for NEXRAD3 products. |
46
|
|
|
|
47
|
|
|
Args: |
48
|
|
|
availableParms: Full list of radar parameters |
49
|
|
|
|
50
|
|
|
Returns: |
51
|
|
|
List of filtered parameters |
52
|
|
|
""" |
53
|
|
|
productIDs = [] |
|
|
|
|
54
|
|
|
for p in list(availableParms): |
|
|
|
|
55
|
|
|
try: |
|
|
|
|
56
|
|
|
if isinstance(int(p), int): |
|
|
|
|
57
|
|
|
productIDs.append(str(p)) |
|
|
|
|
58
|
|
|
except ValueError: |
|
|
|
|
59
|
|
|
pass |
|
|
|
|
60
|
|
|
|
61
|
|
|
return productIDs |
|
|
|
|
62
|
|
|
|
63
|
|
|
|
64
|
|
|
def getRadarProductNames(availableParms): |
65
|
|
|
""" |
66
|
|
|
Get only the named idetifiers for NEXRAD3 products. |
67
|
|
|
|
68
|
|
|
Args: |
69
|
|
|
availableParms: Full list of radar parameters |
70
|
|
|
|
71
|
|
|
Returns: |
72
|
|
|
List of filtered parameters |
73
|
|
|
""" |
74
|
|
|
productNames = [] |
75
|
|
|
for p in list(availableParms): |
76
|
|
|
if len(p) > 3: |
77
|
|
|
productNames.append(p) |
78
|
|
|
|
79
|
|
|
return productNames |
80
|
|
|
|
81
|
|
|
|
82
|
|
View Code Duplication |
def getMetarObs(response): |
|
|
|
|
83
|
|
|
""" |
84
|
|
|
Processes a DataAccessLayer "obs" response into a dictionary, |
85
|
|
|
with special consideration for multi-value parameters |
86
|
|
|
"presWeather", "skyCover", and "skyLayerBase". |
87
|
|
|
|
88
|
|
|
Args: |
89
|
|
|
response: DAL getGeometry() list |
90
|
|
|
|
91
|
|
|
Returns: |
92
|
|
|
A dictionary of METAR obs |
93
|
|
|
""" |
94
|
|
|
from datetime import datetime |
95
|
|
|
single_val_params = ["timeObs", "stationName", "longitude", "latitude", |
96
|
|
|
"temperature", "dewpoint", "windDir", |
97
|
|
|
"windSpeed", "seaLevelPress"] |
98
|
|
|
multi_val_params = ["presWeather", "skyCover", "skyLayerBase"] |
99
|
|
|
params = single_val_params + multi_val_params |
|
|
|
|
100
|
|
|
station_names, pres_weather, sky_cov, sky_layer_base = [], [], [], [] |
101
|
|
|
obs = dict({params: [] for params in params}) |
102
|
|
|
for ob in response: |
|
|
|
|
103
|
|
|
avail_params = ob.getParameters() |
104
|
|
|
if "presWeather" in avail_params: |
105
|
|
|
pres_weather.append(ob.getString("presWeather")) |
106
|
|
|
elif "skyCover" in avail_params and "skyLayerBase" in avail_params: |
107
|
|
|
sky_cov.append(ob.getString("skyCover")) |
108
|
|
|
sky_layer_base.append(ob.getNumber("skyLayerBase")) |
109
|
|
|
else: |
110
|
|
|
# If we already have a record for this stationName, skip |
111
|
|
|
if ob.getString('stationName') not in station_names: |
112
|
|
|
station_names.append(ob.getString('stationName')) |
113
|
|
|
for param in single_val_params: |
114
|
|
|
if param in avail_params: |
115
|
|
|
if param == 'timeObs': |
116
|
|
|
obs[param].append(datetime.fromtimestamp(ob.getNumber(param) / 1000.0)) |
117
|
|
|
else: |
118
|
|
|
try: |
119
|
|
|
obs[param].append(ob.getNumber(param)) |
120
|
|
|
except TypeError: |
121
|
|
|
obs[param].append(ob.getString(param)) |
122
|
|
|
else: |
123
|
|
|
obs[param].append(None) |
124
|
|
|
|
125
|
|
|
obs['presWeather'].append(pres_weather) |
126
|
|
|
obs['skyCover'].append(sky_cov) |
127
|
|
|
obs['skyLayerBase'].append(sky_layer_base) |
128
|
|
|
pres_weather = [] |
129
|
|
|
sky_cov = [] |
130
|
|
|
sky_layer_base = [] |
131
|
|
|
return obs |
132
|
|
|
|
133
|
|
|
|
134
|
|
View Code Duplication |
def getSynopticObs(response): |
|
|
|
|
135
|
|
|
""" |
136
|
|
|
Processes a DataAccessLayer "sfcobs" response into a dictionary |
137
|
|
|
of available parameters. |
138
|
|
|
|
139
|
|
|
Args: |
140
|
|
|
response: DAL getGeometry() list |
141
|
|
|
|
142
|
|
|
Returns: |
143
|
|
|
A dictionary of synop obs |
144
|
|
|
""" |
145
|
|
|
from datetime import datetime |
146
|
|
|
station_names = [] |
147
|
|
|
params = response[0].getParameters() |
148
|
|
|
sfcobs = dict({params: [] for params in params}) |
149
|
|
|
for sfcob in response: |
150
|
|
|
# If we already have a record for this stationId, skip |
151
|
|
|
if sfcob.getString('stationId') not in station_names: |
152
|
|
|
station_names.append(sfcob.getString('stationId')) |
153
|
|
|
for param in params: |
154
|
|
|
if param == 'timeObs': |
155
|
|
|
sfcobs[param].append(datetime.fromtimestamp(sfcob.getNumber(param) / 1000.0)) |
156
|
|
|
else: |
157
|
|
|
try: |
158
|
|
|
sfcobs[param].append(sfcob.getNumber(param)) |
159
|
|
|
except TypeError: |
160
|
|
|
sfcobs[param].append(sfcob.getString(param)) |
161
|
|
|
|
162
|
|
|
return sfcobs |
163
|
|
|
|
164
|
|
|
|
165
|
|
View Code Duplication |
def getForecastRun(cycle, times): |
|
|
|
|
166
|
|
|
""" |
167
|
|
|
Get the latest forecast run (list of objects) from all |
168
|
|
|
all cycles and times returned from DataAccessLayer "grid" |
169
|
|
|
response. |
170
|
|
|
|
171
|
|
|
Args: |
172
|
|
|
cycle: Forecast cycle reference time |
173
|
|
|
times: All available times/cycles |
174
|
|
|
|
175
|
|
|
Returns: |
176
|
|
|
DataTime array for a single forecast run |
177
|
|
|
""" |
178
|
|
|
fcstRun = [] |
179
|
|
|
for t in times: |
180
|
|
|
if str(t)[:19] == str(cycle): |
181
|
|
|
fcstRun.append(t) |
182
|
|
|
return fcstRun |
183
|
|
|
|
184
|
|
|
|
185
|
|
|
def getAvailableTimes(request, refTimeOnly=False): |
186
|
|
|
""" |
187
|
|
|
Get the times of available data to the request. |
188
|
|
|
|
189
|
|
|
Args: |
190
|
|
|
request: the IDataRequest to get data for |
191
|
|
|
refTimeOnly: optional, use True if only unique refTimes should be |
192
|
|
|
returned (without a forecastHr) |
193
|
|
|
|
194
|
|
|
Returns: |
195
|
|
|
a list of DataTimes |
196
|
|
|
""" |
197
|
|
|
return router.getAvailableTimes(request, refTimeOnly) |
198
|
|
|
|
199
|
|
|
|
200
|
|
|
def getGridData(request, times=[]): |
201
|
|
|
""" |
202
|
|
|
Gets the grid data that matches the request at the specified times. Each |
203
|
|
|
combination of parameter, level, and dataTime will be returned as a |
204
|
|
|
separate IGridData. |
205
|
|
|
|
206
|
|
|
Args: |
207
|
|
|
request: the IDataRequest to get data for |
208
|
|
|
times: a list of DataTimes, a TimeRange, or None if the data is time |
209
|
|
|
agnostic |
210
|
|
|
|
211
|
|
|
Returns: |
212
|
|
|
a list of IGridData |
213
|
|
|
""" |
214
|
|
|
return router.getGridData(request, times) |
215
|
|
|
|
216
|
|
|
|
217
|
|
|
def getGeometryData(request, times=[]): |
218
|
|
|
""" |
219
|
|
|
Gets the geometry data that matches the request at the specified times. |
220
|
|
|
Each combination of geometry, level, and dataTime will be returned as a |
221
|
|
|
separate IGeometryData. |
222
|
|
|
|
223
|
|
|
Args: |
224
|
|
|
request: the IDataRequest to get data for |
225
|
|
|
times: a list of DataTimes, a TimeRange, or None if the data is time |
226
|
|
|
agnostic |
227
|
|
|
|
228
|
|
|
Returns: |
229
|
|
|
a list of IGeometryData |
230
|
|
|
""" |
231
|
|
|
return router.getGeometryData(request, times) |
232
|
|
|
|
233
|
|
|
|
234
|
|
|
def getAvailableLocationNames(request): |
235
|
|
|
""" |
236
|
|
|
Gets the available location names that match the request without actually |
237
|
|
|
requesting the data. |
238
|
|
|
|
239
|
|
|
Args: |
240
|
|
|
request: the request to find matching location names for |
241
|
|
|
|
242
|
|
|
Returns: |
243
|
|
|
a list of strings of available location names. |
244
|
|
|
""" |
245
|
|
|
return router.getAvailableLocationNames(request) |
246
|
|
|
|
247
|
|
|
|
248
|
|
|
def getAvailableParameters(request): |
249
|
|
|
""" |
250
|
|
|
Gets the available parameters names that match the request without actually |
251
|
|
|
requesting the data. |
252
|
|
|
|
253
|
|
|
Args: |
254
|
|
|
request: the request to find matching parameter names for |
255
|
|
|
|
256
|
|
|
Returns: |
257
|
|
|
a list of strings of available parameter names. |
258
|
|
|
""" |
259
|
|
|
return router.getAvailableParameters(request) |
260
|
|
|
|
261
|
|
|
|
262
|
|
|
def getAvailableLevels(request): |
263
|
|
|
""" |
264
|
|
|
Gets the available levels that match the request without actually |
265
|
|
|
requesting the data. |
266
|
|
|
|
267
|
|
|
Args: |
268
|
|
|
request: the request to find matching levels for |
269
|
|
|
|
270
|
|
|
Returns: |
271
|
|
|
a list of strings of available levels. |
272
|
|
|
""" |
273
|
|
|
return router.getAvailableLevels(request) |
274
|
|
|
|
275
|
|
|
|
276
|
|
|
def getRequiredIdentifiers(request): |
277
|
|
|
""" |
278
|
|
|
Gets the required identifiers for this request. These identifiers |
279
|
|
|
must be set on a request for the request of this datatype to succeed. |
280
|
|
|
|
281
|
|
|
Args: |
282
|
|
|
request: the request to find required identifiers for |
283
|
|
|
|
284
|
|
|
Returns: |
285
|
|
|
a list of strings of required identifiers |
286
|
|
|
""" |
287
|
|
|
if str(request) == request: |
288
|
|
|
warnings.warn("Use getRequiredIdentifiers(IDataRequest) instead", |
289
|
|
|
DeprecationWarning) |
290
|
|
|
return router.getRequiredIdentifiers(request) |
291
|
|
|
|
292
|
|
|
|
293
|
|
|
def getOptionalIdentifiers(request): |
294
|
|
|
""" |
295
|
|
|
Gets the optional identifiers for this request. |
296
|
|
|
|
297
|
|
|
Args: |
298
|
|
|
request: the request to find optional identifiers for |
299
|
|
|
|
300
|
|
|
Returns: |
301
|
|
|
a list of strings of optional identifiers |
302
|
|
|
""" |
303
|
|
|
if str(request) == request: |
304
|
|
|
warnings.warn("Use getOptionalIdentifiers(IDataRequest) instead", |
305
|
|
|
DeprecationWarning) |
306
|
|
|
return router.getOptionalIdentifiers(request) |
307
|
|
|
|
308
|
|
|
|
309
|
|
|
def getIdentifierValues(request, identifierKey): |
310
|
|
|
""" |
311
|
|
|
Gets the allowed values for a particular identifier on this datatype. |
312
|
|
|
|
313
|
|
|
Args: |
314
|
|
|
request: the request to find identifier values for |
315
|
|
|
identifierKey: the identifier to find values for |
316
|
|
|
|
317
|
|
|
Returns: |
318
|
|
|
a list of strings of allowed values for the specified identifier |
319
|
|
|
""" |
320
|
|
|
return router.getIdentifierValues(request, identifierKey) |
321
|
|
|
|
322
|
|
|
|
323
|
|
|
def newDataRequest(datatype=None, **kwargs): |
324
|
|
|
""" |
325
|
|
|
Creates a new instance of IDataRequest suitable for the runtime environment. |
326
|
|
|
All args are optional and exist solely for convenience. |
327
|
|
|
|
328
|
|
|
Args: |
329
|
|
|
datatype: the datatype to create a request for |
330
|
|
|
parameters: a list of parameters to set on the request |
331
|
|
|
levels: a list of levels to set on the request |
332
|
|
|
locationNames: a list of locationNames to set on the request |
333
|
|
|
envelope: an envelope to limit the request |
334
|
|
|
kwargs: any leftover kwargs will be set as identifiers |
335
|
|
|
|
336
|
|
|
Returns: |
337
|
|
|
a new IDataRequest |
338
|
|
|
""" |
339
|
|
|
return router.newDataRequest(datatype, **kwargs) |
340
|
|
|
|
341
|
|
|
|
342
|
|
|
def getSupportedDatatypes(): |
343
|
|
|
""" |
344
|
|
|
Gets the datatypes that are supported by the framework |
345
|
|
|
|
346
|
|
|
Returns: |
347
|
|
|
a list of strings of supported datatypes |
348
|
|
|
""" |
349
|
|
|
return router.getSupportedDatatypes() |
350
|
|
|
|
351
|
|
|
|
352
|
|
|
def changeEDEXHost(newHostName): |
353
|
|
|
""" |
354
|
|
|
Changes the EDEX host the Data Access Framework is communicating with. Only |
355
|
|
|
works if using the native Python client implementation, otherwise, this |
356
|
|
|
method will throw a TypeError. |
357
|
|
|
|
358
|
|
|
Args: |
359
|
|
|
newHostName: the EDEX host to connect to |
360
|
|
|
""" |
361
|
|
|
if USING_NATIVE_THRIFT: |
362
|
|
|
global THRIFT_HOST |
363
|
|
|
THRIFT_HOST = newHostName |
364
|
|
|
global router |
365
|
|
|
router = ThriftClientRouter.ThriftClientRouter(THRIFT_HOST) |
|
|
|
|
366
|
|
|
else: |
367
|
|
|
raise TypeError("Cannot call changeEDEXHost when using JepRouter.") |
368
|
|
|
|
369
|
|
|
|
370
|
|
|
def setLazyLoadGridLatLon(lazyLoadGridLatLon): |
371
|
|
|
""" |
372
|
|
|
Provide a hint to the Data Access Framework indicating whether to load the |
373
|
|
|
lat/lon data for a grid immediately or wait until it is needed. This is |
374
|
|
|
provided as a performance tuning hint and should not affect the way the |
375
|
|
|
Data Access Framework is used. Depending on the internal implementation of |
376
|
|
|
the Data Access Framework this hint might be ignored. Examples of when this |
377
|
|
|
should be set to True are when the lat/lon information is not used or when |
378
|
|
|
it is used only if certain conditions within the data are met. It could be |
379
|
|
|
set to False if it is guaranteed that all lat/lon information is needed and |
380
|
|
|
it would be better to get any performance overhead for generating the |
381
|
|
|
lat/lon data out of the way during the initial request. |
382
|
|
|
|
383
|
|
|
|
384
|
|
|
Args: |
385
|
|
|
lazyLoadGridLatLon: Boolean value indicating whether to lazy load. |
386
|
|
|
""" |
387
|
|
|
try: |
388
|
|
|
router.setLazyLoadGridLatLon(lazyLoadGridLatLon) |
389
|
|
|
except AttributeError: |
390
|
|
|
# The router is not required to support this capability. |
391
|
|
|
pass |
392
|
|
|
|