1
|
|
|
# Copyright (c) 2022 Stefan Bender |
2
|
|
|
# |
3
|
|
|
# This module is part of pyspaceweather. |
4
|
|
|
# pyspaceweather is free software: you can redistribute it or modify |
5
|
|
|
# it under the terms of the GNU General Public License as published |
6
|
|
|
# by the Free Software Foundation, version 2. |
7
|
|
|
# See accompanying COPYING.GPLv2 file or http://www.gnu.org/licenses/gpl-2.0.html. |
8
|
|
|
"""Python interface for OMNI space weather data |
9
|
|
|
|
10
|
|
|
Omni2 [#]_ space weather data interface for python. |
11
|
|
|
|
12
|
|
|
.. [#] https://omniweb.gsfc.nasa.gov/ow.html |
13
|
|
|
""" |
14
|
|
|
import os |
15
|
|
|
from pkg_resources import resource_filename |
16
|
|
|
import logging |
17
|
|
|
from warnings import warn |
18
|
|
|
|
19
|
|
|
import numpy as np |
20
|
|
|
import pandas as pd |
21
|
|
|
|
22
|
|
|
from .core import _assert_file_exists, _dl_file |
23
|
|
|
|
24
|
|
|
__all__ = [ |
25
|
|
|
"cache_omnie", |
26
|
|
|
"omnie_hourly", |
27
|
|
|
"read_omnie", |
28
|
|
|
] |
29
|
|
|
|
30
|
|
|
OMNI_URL_BASE = "https://spdf.gsfc.nasa.gov/pub/data/omni/low_res_omni/extended" |
31
|
|
|
OMNI_PREFIX, OMNI_EXT = "omni2", "dat" |
32
|
|
|
OMNI_SUBDIR = "omni_extended" |
33
|
|
|
LOCAL_PATH = resource_filename(__name__, os.path.join("data", OMNI_SUBDIR)) |
34
|
|
|
|
35
|
|
|
|
36
|
|
View Code Duplication |
def cache_omnie( |
|
|
|
|
37
|
|
|
year, |
38
|
|
|
prefix=None, |
39
|
|
|
ext=None, |
40
|
|
|
local_path=None, |
41
|
|
|
url_base=None, |
42
|
|
|
): |
43
|
|
|
"""Download OMNI2 data to local cache |
44
|
|
|
|
45
|
|
|
Downloads the OMNI2 (extended) data file to the local location. |
46
|
|
|
|
47
|
|
|
.. [#] https://spdf.gsfc.nasa.gov/pub/data/omni/low_res_omni/extended/ |
48
|
|
|
|
49
|
|
|
Parameters |
50
|
|
|
---------- |
51
|
|
|
year: int |
52
|
|
|
Year of the data. |
53
|
|
|
prefix: str, optional |
54
|
|
|
File prefix for constructing the file name as <prefix>_year.<ext>. |
55
|
|
|
Defaults to 'omni2'. |
56
|
|
|
ext: str, optional |
57
|
|
|
File extension for constructing the file name as <prefix>_year.<ext>. |
58
|
|
|
Defaults to 'dat'. |
59
|
|
|
local_path: str, optional |
60
|
|
|
Path to the locally stored data yearly files, defaults to the |
61
|
|
|
data location within the package. |
62
|
|
|
url_base: str, optional |
63
|
|
|
URL for the directory that contains the yearly files. |
64
|
|
|
|
65
|
|
|
Returns |
66
|
|
|
------- |
67
|
|
|
Nothing. |
68
|
|
|
""" |
69
|
|
|
prefix = prefix or OMNI_PREFIX |
70
|
|
|
ext = ext or OMNI_EXT |
71
|
|
|
local_path = local_path or LOCAL_PATH |
72
|
|
|
url_base = url_base or OMNI_URL_BASE |
73
|
|
|
|
74
|
|
|
basename = "{0}_{1:04d}.{2}".format(prefix, year, ext) |
75
|
|
|
|
76
|
|
|
if not os.path.exists(local_path): |
77
|
|
|
os.makedirs(local_path) |
78
|
|
|
|
79
|
|
|
omnie_file = os.path.join(local_path, basename) |
80
|
|
|
if not os.path.exists(omnie_file): |
81
|
|
|
url = os.path.join(url_base, basename) |
82
|
|
|
logging.info("%s not found, downloading from %s.", omnie_file, url) |
83
|
|
|
_dl_file(omnie_file, url) |
84
|
|
|
|
85
|
|
|
|
86
|
|
|
def read_omnie(omnie_file): |
87
|
|
|
"""Read and parse OMNI2 extended files [#]_ |
88
|
|
|
|
89
|
|
|
Parses the Omni2 extended data files, available at [#]_, |
90
|
|
|
into a :class:`pandas.DataFrame`. |
91
|
|
|
|
92
|
|
|
.. [#] https://omniweb.gsfc.nasa.gov/ow.html |
93
|
|
|
.. [#] https://spdf.gsfc.nasa.gov/pub/data/omni/low_res_omni/extended/ |
94
|
|
|
|
95
|
|
|
Parameters |
96
|
|
|
---------- |
97
|
|
|
omnie_file: str |
98
|
|
|
File to parse, absolute path or relative to the current dir. |
99
|
|
|
|
100
|
|
|
Returns |
101
|
|
|
------- |
102
|
|
|
sw_df: pandas.DataFrame |
103
|
|
|
The parsed OMNI2 space weather data (hourly values). |
104
|
|
|
Details in |
105
|
|
|
https://spdf.gsfc.nasa.gov/pub/data/omni/low_res_omni/extended/aareadme_extended |
106
|
|
|
|
107
|
|
|
Raises an ``IOError`` if the file is not found. |
108
|
|
|
|
109
|
|
|
The dataframe contains the following columns: |
110
|
|
|
|
111
|
|
|
year: |
112
|
|
|
The observation year |
113
|
|
|
doy: |
114
|
|
|
Day of the year |
115
|
|
|
hour: |
116
|
|
|
Hour of the day |
117
|
|
|
bsrn: |
118
|
|
|
Bartels Solar Rotation Number. |
119
|
|
|
id_imf: |
120
|
|
|
ID for IMF spacecraft |
121
|
|
|
id_sw: |
122
|
|
|
ID for SW plasma spacecraft |
123
|
|
|
n_imf: |
124
|
|
|
Number of points in IMF averages |
125
|
|
|
n_plasma: |
126
|
|
|
Numberof points in plasma averages |
127
|
|
|
B_mag_avg: |
128
|
|
|
Magnetic field magnitude average B |
129
|
|
|
B_mag: |
130
|
|
|
Magnetic field vector magnitude |
131
|
|
|
theta_B: |
132
|
|
|
Latitude angle of the magnetic field vector |
133
|
|
|
phi_B: |
134
|
|
|
Longitude angle of the magnetic field vector |
135
|
|
|
B_x: |
136
|
|
|
B_x GSE, GSM |
137
|
|
|
B_y_GSE: |
138
|
|
|
B_y GSE |
139
|
|
|
B_z_GSE: |
140
|
|
|
B_z GSE |
141
|
|
|
B_y_GSM: |
142
|
|
|
B_y GSM |
143
|
|
|
B_z_GSM: |
144
|
|
|
B_z GSM |
145
|
|
|
sigma_B_mag_avg: |
146
|
|
|
RMS standard deviation of B_mag_avg |
147
|
|
|
sigma_B_mag: |
148
|
|
|
RMS standard deviation of B_mag |
149
|
|
|
sigma_B_x_GSE: |
150
|
|
|
RMS standard deviation of B_x_GSE |
151
|
|
|
sigma_B_y_GSE: |
152
|
|
|
RMS standard deviation of B_y_GSE |
153
|
|
|
sigma_B_z_GSE: |
154
|
|
|
RMS standard deviation of B_z_GSE |
155
|
|
|
T_p: |
156
|
|
|
Proton temperature |
157
|
|
|
n_p: |
158
|
|
|
Proton density |
159
|
|
|
v_plasma: |
160
|
|
|
Plasma flow speed |
161
|
|
|
phi_v: |
162
|
|
|
Plasma flow longitude angle |
163
|
|
|
theta_v: |
164
|
|
|
Plasma flow latitude angle |
165
|
|
|
n_alpha_n_p: |
166
|
|
|
Alpha/Proton ratio |
167
|
|
|
p_flow: |
168
|
|
|
Flow pressure |
169
|
|
|
sigma_T: |
170
|
|
|
Standard deviation of T_p |
171
|
|
|
sigma_n: |
172
|
|
|
Standard deviation of n_p |
173
|
|
|
sigma_v: |
174
|
|
|
Standard deviation of v_plasma |
175
|
|
|
sigma_phi_v: |
176
|
|
|
Standard deviation of phi_v |
177
|
|
|
sigma_theta_v: |
178
|
|
|
Standard deviation of theta_v |
179
|
|
|
sigma_na_np: |
180
|
|
|
Standard deviation of n_alpha_n_p |
181
|
|
|
E: |
182
|
|
|
Electric field magnitude |
183
|
|
|
beta_plasma: |
184
|
|
|
Plasma beta |
185
|
|
|
mach: |
186
|
|
|
Alfvén Mach number |
187
|
|
|
Kp: |
188
|
|
|
Kp index value |
189
|
|
|
R: |
190
|
|
|
Sunspot number |
191
|
|
|
Dst: |
192
|
|
|
Dst index value |
193
|
|
|
AE: |
194
|
|
|
AE index value |
195
|
|
|
p_01MeV, p_02MeV, p_04MeV, p_10MeV, p_30MeV, p_60MeV: |
196
|
|
|
Proton fluxes >1 MeV, >2 MeV, >4 MeV, >10 MeV, >30 MeV, > 60 MeV |
197
|
|
|
flag: |
198
|
|
|
Flag (-1, ..., 6) |
199
|
|
|
Ap: |
200
|
|
|
Ap index value |
201
|
|
|
f107_adj: |
202
|
|
|
F10.7 radio flux at 1 AU |
203
|
|
|
PC: |
204
|
|
|
PC index value |
205
|
|
|
AL, AU: |
206
|
|
|
AL and AU index values |
207
|
|
|
mach_mag: |
208
|
|
|
Magnetosonic Mach number |
209
|
|
|
|
210
|
|
|
The extended dataset contains the addional columns: |
211
|
|
|
|
212
|
|
|
Lya: |
213
|
|
|
Solar Lyman-alpha irradiance |
214
|
|
|
QI_p: |
215
|
|
|
Proton QI |
216
|
|
|
""" |
217
|
|
|
_assert_file_exists(omnie_file) |
218
|
|
|
# FORMAT( |
219
|
|
|
# 2I4,I3,I5,2I3,2I4,14F6.1,F9.0,F6.1,F6.0,2F6.1,F6.3,F6.2, |
220
|
|
|
# F9.0,F6.1,F6.0,2F6.1,F6.3,2F7.2,F6.1,I3,I4,I6,I5,F10.2, |
221
|
|
|
# 5F9.2,I3,I4,2F6.1,2I6,F5.1,F9.6,F7.4 |
222
|
|
|
# ) |
223
|
|
|
sw = np.genfromtxt( |
224
|
|
|
omnie_file, |
225
|
|
|
skip_header=0, |
226
|
|
|
delimiter=[ |
227
|
|
|
# 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 |
228
|
|
|
# yy dd hr br i1 i2 n1 n2 B B' tB fB Bx By Bz By Bz sB sB sB |
229
|
|
|
4, 4, 3, 5, 3, 3, 4, 4, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, |
230
|
|
|
# 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 |
231
|
|
|
# sB sB Tp np v fv tv nr p sT sn sv sf st sr E bp M Kp R |
232
|
|
|
6, 6, 9, 6, 6, 6, 6, 6, 6, 9, 6, 6, 6, 6, 6, 7, 7, 6, 3, 4, |
233
|
|
|
# 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 |
234
|
|
|
# Ds AE p1 p2 p4p10p30p60 fl Apf10 PC AL AU Mm La QI |
235
|
|
|
6, 5,10, 9, 9, 9, 9, 9, 3, 4, 6, 6, 6, 6, 5, 9, 7, |
236
|
|
|
], |
237
|
|
|
dtype=( |
238
|
|
|
"i4,i4,i4,i4,i4,i4,i4,i4,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8," |
239
|
|
|
"f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,f8,i4,i4," |
240
|
|
|
"i4,i4,f8,f8,f8,f8,f8,f8,i4,i4,f8,f8,i4,i4,f8,f8,f8" |
241
|
|
|
), |
242
|
|
|
names=[ |
243
|
|
|
"year", "doy", "hour", "bsrn", "id_imf", "id_sw", "n_imf", "n_plasma", |
244
|
|
|
"B_mag_avg", "B_mag", "theta_B", "phi_B", |
245
|
|
|
"B_x", "B_y_GSE", "B_z_GSE", "B_y_GSM", "B_z_GSM", |
246
|
|
|
"sigma_B_mag_avg", "sigma_B_mag", |
247
|
|
|
"sigma_B_x_GSE", "sigma_B_y_GSE", "sigma_B_z_GSE", |
248
|
|
|
"T_p", "n_p", "v_plasma", "phi_v", "theta_v", "n_alpha_n_p", "p_flow", |
249
|
|
|
"sigma_T", "sigma_n", "sigma_v", |
250
|
|
|
"sigma_phi_v", "sigma_theta_v", "sigma_na_np", |
251
|
|
|
"E", "beta_plasma", "mach", "Kp", "R", "Dst", "AE", |
252
|
|
|
"p_01MeV", "p_02MeV", "p_04MeV", "p_10MeV", "p_30MeV", "p_60MeV", |
253
|
|
|
"flag", "Ap", "f107_adj", "PC", "AL", "AU", "mach_mag", "Lya", "QI_p", |
254
|
|
|
] |
255
|
|
|
) |
256
|
|
|
sw = sw[sw["year"] != -1] |
257
|
|
|
ts = pd.to_datetime( |
258
|
|
|
[ |
259
|
|
|
"{0:04d}.{1:03d} {2:02d}".format(yy, dd, hh) |
260
|
|
|
for yy, dd, hh in sw[["year", "doy", "hour"]] |
261
|
|
|
], |
262
|
|
|
format="%Y.%j %H", |
263
|
|
|
) |
264
|
|
|
sw_df = pd.DataFrame(sw, index=ts) |
265
|
|
|
# Adjust Kp to 0...9 |
266
|
|
|
sw_df["Kp"] = 0.1 * sw_df["Kp"] |
267
|
|
|
return sw_df |
268
|
|
|
|
269
|
|
|
|
270
|
|
View Code Duplication |
def omnie_hourly( |
|
|
|
|
271
|
|
|
year, |
272
|
|
|
prefix=None, |
273
|
|
|
ext=None, |
274
|
|
|
local_path=None, |
275
|
|
|
url_base=None, |
276
|
|
|
cache=False, |
277
|
|
|
): |
278
|
|
|
"""OMNI hourly data for year `year` |
279
|
|
|
|
280
|
|
|
Loads the OMNI hourly data for the given year, |
281
|
|
|
from the locally cached data. |
282
|
|
|
Use `local_path` to set a custom location if you |
283
|
|
|
have the omni data already available. |
284
|
|
|
|
285
|
|
|
Parameters: |
286
|
|
|
----------- |
287
|
|
|
year: int |
288
|
|
|
Year of the data. |
289
|
|
|
prefix: str, optional, default 'omni2' |
290
|
|
|
File prefix for constructing the file name as <prefix>_year.<ext>. |
291
|
|
|
ext: str, optional, default 'dat' |
292
|
|
|
File extension for constructing the file name as <prefix>_year.<ext>. |
293
|
|
|
local_path: str, optional |
294
|
|
|
Path to the locally stored data yearly files, defaults to the |
295
|
|
|
data location within the package. |
296
|
|
|
url_base: str, optional |
297
|
|
|
URL for the directory that contains the yearly files. |
298
|
|
|
cache: boolean, optional, default False |
299
|
|
|
Download files locally if they are not already available. |
300
|
|
|
|
301
|
|
|
Returns |
302
|
|
|
------- |
303
|
|
|
sw_df: pandas.DataFrame |
304
|
|
|
The parsed space weather data (hourly values). |
305
|
|
|
|
306
|
|
|
Raises an ``IOError`` if the file is not available. |
307
|
|
|
|
308
|
|
|
See Also |
309
|
|
|
-------- |
310
|
|
|
read_omnie |
311
|
|
|
""" |
312
|
|
|
prefix = prefix or OMNI_PREFIX |
313
|
|
|
ext = ext or OMNI_EXT |
314
|
|
|
local_path = local_path or LOCAL_PATH |
315
|
|
|
url_base = url_base or OMNI_URL_BASE |
316
|
|
|
|
317
|
|
|
basename = "{0}_{1:04d}.{2}".format(prefix, year, ext) |
318
|
|
|
omnie_file = os.path.join(local_path, basename) |
319
|
|
|
|
320
|
|
|
# ensure that the file exists |
321
|
|
|
if not os.path.exists(omnie_file): |
322
|
|
|
warn("Could not find OMNI2 data {0}.".format(omnie_file)) |
323
|
|
|
if cache: |
324
|
|
|
warn("Trying to cache to `{0}'".format(omnie_file)) |
325
|
|
|
_dl_file(omnie_file, os.path.join(url_base, basename)) |
326
|
|
|
else: |
327
|
|
|
warn( |
328
|
|
|
"Local data files not found, pass `cache=True` " |
329
|
|
|
"or run `sw.cache_omnie()` to download the file." |
330
|
|
|
) |
331
|
|
|
|
332
|
|
|
return read_omnie(omnie_file) |
333
|
|
|
|