1
|
|
|
from __future__ import print_function |
|
|
|
|
2
|
|
|
|
3
|
|
|
import copy |
4
|
|
|
import collections |
5
|
|
|
|
6
|
|
|
import numpy as np |
7
|
|
|
import healpy as hp |
8
|
|
|
import matplotlib.pyplot as plt |
9
|
|
|
from scipy.stats import poisson |
10
|
|
|
|
11
|
|
|
from astropy.convolution import Gaussian2DKernel |
12
|
|
|
from astropy.convolution import convolve_fft as convolve |
13
|
|
|
|
14
|
|
|
from threeML.plugin_prototype import PluginPrototype |
15
|
|
|
from threeML.plugins.gammaln import logfactorial |
16
|
|
|
from threeML.parallel import parallel_client |
17
|
|
|
from threeML.io.progress_bar import progress_bar |
18
|
|
|
|
19
|
|
|
from astromodels import Parameter |
20
|
|
|
|
21
|
|
|
from hawc_hal.maptree import map_tree_factory |
22
|
|
|
from hawc_hal.response import hawc_response_factory |
23
|
|
|
from hawc_hal.convolved_source import ConvolvedPointSource, \ |
24
|
|
|
ConvolvedExtendedSource3D, ConvolvedExtendedSource2D, ConvolvedSourcesContainer |
25
|
|
|
from hawc_hal.healpix_handling import FlatSkyToHealpixTransform |
26
|
|
|
from hawc_hal.healpix_handling import SparseHealpix |
27
|
|
|
from hawc_hal.healpix_handling import get_gnomonic_projection |
28
|
|
|
from hawc_hal.psf_fast import PSFConvolutor |
29
|
|
|
from hawc_hal.log_likelihood import log_likelihood |
30
|
|
|
from hawc_hal.util import ra_to_longitude |
31
|
|
|
|
32
|
|
|
|
33
|
|
|
class HAL(PluginPrototype): |
|
|
|
|
34
|
|
|
""" |
35
|
|
|
The HAWC Accelerated Likelihood plugin for 3ML. |
36
|
|
|
:param name: name for the plugin |
37
|
|
|
:param maptree: Map Tree (either ROOT or hdf5 format) |
38
|
|
|
:param response: Response of HAWC (either ROOT or hd5 format) |
39
|
|
|
:param roi: a ROI instance describing the Region Of Interest |
40
|
|
|
:param flat_sky_pixels_size: size of the pixel for the flat sky projection (Hammer Aitoff) |
41
|
|
|
""" |
42
|
|
|
|
43
|
|
|
def __init__(self, name, maptree, response_file, roi, flat_sky_pixels_size=0.17): |
|
|
|
|
44
|
|
|
|
45
|
|
|
# Store ROI |
46
|
|
|
self._roi = roi |
47
|
|
|
|
48
|
|
|
# Set up the flat-sky projection |
49
|
|
|
|
50
|
|
|
self._flat_sky_projection = roi.get_flat_sky_projection(flat_sky_pixels_size) |
51
|
|
|
|
52
|
|
|
# Read map tree (data) |
53
|
|
|
|
54
|
|
|
self._maptree = map_tree_factory(maptree, roi=roi) |
55
|
|
|
|
56
|
|
|
# Read detector response_file |
57
|
|
|
|
58
|
|
|
self._response = hawc_response_factory(response_file) |
59
|
|
|
|
60
|
|
|
# Use a renormalization of the background as nuisance parameter |
61
|
|
|
# NOTE: it is fixed to 1.0 unless the user explicitly sets it free (experimental) |
62
|
|
|
self._nuisance_parameters = collections.OrderedDict() |
63
|
|
|
self._nuisance_parameters['%s_bkg_renorm' % name] = Parameter('%s_bkg_renorm' % name, 1.0, |
64
|
|
|
min_value=0.5, max_value=1.5, |
65
|
|
|
delta=0.01, |
66
|
|
|
desc="Renormalization for background map", |
67
|
|
|
free=False, |
68
|
|
|
is_normalization=False) |
69
|
|
|
|
70
|
|
|
# Instance parent class |
71
|
|
|
|
72
|
|
|
super(HAL, self).__init__(name, self._nuisance_parameters) |
73
|
|
|
|
74
|
|
|
self._likelihood_model = None |
75
|
|
|
|
76
|
|
|
# These lists will contain the maps for the point sources |
77
|
|
|
self._convolved_point_sources = ConvolvedSourcesContainer() |
78
|
|
|
# and this one for extended sources |
79
|
|
|
self._convolved_ext_sources = ConvolvedSourcesContainer() |
80
|
|
|
|
81
|
|
|
# All energy/nHit bins are loaded in memory |
82
|
|
|
self._all_planes = list(self._maptree.analysis_bins_labels) |
83
|
|
|
|
84
|
|
|
# The active planes list always contains the list of *indexes* of the active planes |
85
|
|
|
self._active_planes = None |
86
|
|
|
|
87
|
|
|
# Set up the transformations from the flat-sky projection to Healpix, as well as the list of active pixels |
88
|
|
|
# (one for each energy/nHit bin). We make a separate transformation because different energy bins might have |
89
|
|
|
# different nsides |
90
|
|
|
self._active_pixels = collections.OrderedDict() |
91
|
|
|
self._flat_sky_to_healpix_transform = collections.OrderedDict() |
92
|
|
|
|
93
|
|
|
for bin_id in self._maptree: |
94
|
|
|
|
95
|
|
|
this_maptree = self._maptree[bin_id] |
96
|
|
|
this_nside = this_maptree.nside |
97
|
|
|
this_active_pixels = roi.active_pixels(this_nside) |
98
|
|
|
|
99
|
|
|
this_flat_sky_to_hpx_transform = FlatSkyToHealpixTransform(self._flat_sky_projection.wcs, |
100
|
|
|
'icrs', |
101
|
|
|
this_nside, |
102
|
|
|
this_active_pixels, |
103
|
|
|
(self._flat_sky_projection.npix_width, |
104
|
|
|
self._flat_sky_projection.npix_height), |
105
|
|
|
order='bilinear') |
106
|
|
|
|
107
|
|
|
self._active_pixels[bin_id] = this_active_pixels |
108
|
|
|
self._flat_sky_to_healpix_transform[bin_id] = this_flat_sky_to_hpx_transform |
109
|
|
|
|
110
|
|
|
# This will contain a list of PSF convolutors for extended sources, if there is any in the model |
111
|
|
|
|
112
|
|
|
self._psf_convolutors = None |
113
|
|
|
|
114
|
|
|
# Pre-compute the log-factorial factor in the likelihood, so we do not keep to computing it over and over |
115
|
|
|
# again. |
116
|
|
|
self._log_factorials = collections.OrderedDict() |
117
|
|
|
|
118
|
|
|
# We also apply a bias so that the numerical value of the log-likelihood stays small. This helps when |
119
|
|
|
# fitting with algorithms like MINUIT because the convergence criterium involves the difference between |
120
|
|
|
# two likelihood values, which would be affected by numerical precision errors if the two values are |
121
|
|
|
# too large |
122
|
|
|
self._saturated_model_like_per_maptree = collections.OrderedDict() |
123
|
|
|
|
124
|
|
|
# The actual computation is in a method so we can recall it on clone (see the get_simulated_dataset method) |
125
|
|
|
self._compute_likelihood_biases() |
126
|
|
|
|
127
|
|
|
# This will save a clone of self for simulations |
128
|
|
|
self._clone = None |
129
|
|
|
|
130
|
|
|
def _setup_psf_convolutors(self): |
131
|
|
|
|
132
|
|
|
central_response_bins = self._response.get_response_dec_bin(self._roi.ra_dec_center[1]) |
133
|
|
|
|
134
|
|
|
self._psf_convolutors = collections.OrderedDict() |
135
|
|
|
for bin_id in central_response_bins: |
136
|
|
|
self._psf_convolutors[bin_id] = PSFConvolutor(central_response_bins[bin_id].psf, self._flat_sky_projection) |
137
|
|
|
|
138
|
|
|
def _compute_likelihood_biases(self): |
139
|
|
|
|
140
|
|
|
for bin_label in self._maptree: |
141
|
|
|
|
142
|
|
|
data_analysis_bin = self._maptree[bin_label] |
143
|
|
|
|
144
|
|
|
this_log_factorial = np.sum(logfactorial(data_analysis_bin.observation_map.as_partial())) |
145
|
|
|
self._log_factorials[bin_label] = this_log_factorial |
146
|
|
|
|
147
|
|
|
# As bias we use the likelihood value for the saturated model |
148
|
|
|
obs = data_analysis_bin.observation_map.as_partial() |
149
|
|
|
bkg = data_analysis_bin.background_map.as_partial() |
150
|
|
|
|
151
|
|
|
sat_model = np.clip(obs - bkg, 1e-50, None).astype(np.float64) |
152
|
|
|
|
153
|
|
|
self._saturated_model_like_per_maptree[bin_label] = log_likelihood(obs, bkg, sat_model) - this_log_factorial |
154
|
|
|
|
155
|
|
|
def get_saturated_model_likelihood(self): |
156
|
|
|
""" |
157
|
|
|
Returns the likelihood for the saturated model (i.e. a model exactly equal to observation - background). |
158
|
|
|
|
159
|
|
|
:return: |
160
|
|
|
""" |
161
|
|
|
return sum(self._saturated_model_like_per_maptree.values()) |
162
|
|
|
|
163
|
|
|
def set_active_measurements(self, bin_id_min=None, bin_id_max=None, bin_list=None): |
164
|
|
|
""" |
165
|
|
|
Set the active analysis bins to use during the analysis. It can be used in two ways: |
166
|
|
|
|
167
|
|
|
- Specifying a range: if the response and the maptree allows it, you can specify a minimum id and a maximum id |
168
|
|
|
number. This only works if the analysis bins are numerical, like in the normal fHit analysis. For example: |
169
|
|
|
|
170
|
|
|
> set_active_measurement(bin_id_min=1, bin_id_max=9( |
171
|
|
|
|
172
|
|
|
- Specifying a list of bins as strings. This is more powerful, as allows to select any bins, even |
173
|
|
|
non-contiguous bins. For example: |
174
|
|
|
|
175
|
|
|
> set_active_measurement(bin_list=[list]) |
176
|
|
|
|
177
|
|
|
:param bin_id_min: minimum bin (only works for fHit analysis. For the others, use bin_list) |
178
|
|
|
:param bin_id_max: maximum bin (only works for fHit analysis. For the others, use bin_list) |
179
|
|
|
:param bin_list: a list of analysis bins to use |
180
|
|
|
:return: None |
181
|
|
|
""" |
182
|
|
|
|
183
|
|
|
# Check for legal input |
184
|
|
|
if bin_id_min is not None: |
185
|
|
|
|
186
|
|
|
assert bin_id_max is not None, "If you provide a minimum bin, you also need to provide a maximum bin" |
187
|
|
|
|
188
|
|
|
# Make sure they are integers |
189
|
|
|
bin_id_min = int(bin_id_min) |
190
|
|
|
bin_id_max = int(bin_id_max) |
191
|
|
|
|
192
|
|
|
self._active_planes = [] |
193
|
|
|
for this_bin in range(bin_id_min, bin_id_max + 1): |
194
|
|
|
this_bin = str(this_bin) |
195
|
|
|
if this_bin not in self._all_planes: |
196
|
|
|
|
197
|
|
|
raise ValueError("Bin %s it not contained in this response" % this_bin) |
198
|
|
|
|
199
|
|
|
self._active_planes.append(this_bin) |
200
|
|
|
|
201
|
|
|
else: |
202
|
|
|
|
203
|
|
|
assert bin_id_max is None, "If you provide a maximum bin, you also need to provide a minimum bin" |
204
|
|
|
|
205
|
|
|
assert bin_list is not None |
206
|
|
|
|
207
|
|
|
self._active_planes = [] |
208
|
|
|
|
209
|
|
|
for this_bin in bin_list: |
210
|
|
|
|
211
|
|
|
if not this_bin in self._all_planes: |
212
|
|
|
|
213
|
|
|
raise ValueError("Bin %s it not contained in this response" % this_bin) |
214
|
|
|
|
215
|
|
|
self._active_planes.append(this_bin) |
216
|
|
|
|
217
|
|
|
def display(self, verbose=False): |
218
|
|
|
""" |
219
|
|
|
Prints summary of the current object content. |
220
|
|
|
""" |
221
|
|
|
|
222
|
|
|
print("Region of Interest: ") |
223
|
|
|
print("--------------------\n") |
224
|
|
|
self._roi.display() |
225
|
|
|
|
226
|
|
|
print("") |
227
|
|
|
print("Flat sky projection: ") |
228
|
|
|
print("----------------------\n") |
229
|
|
|
|
230
|
|
|
print("Width x height: %s x %s px" % (self._flat_sky_projection.npix_width, |
231
|
|
|
self._flat_sky_projection.npix_height)) |
232
|
|
|
print("Pixel sizes: %s deg" % self._flat_sky_projection.pixel_size) |
233
|
|
|
|
234
|
|
|
print("") |
235
|
|
|
print("Response: ") |
236
|
|
|
print("----------\n") |
237
|
|
|
|
238
|
|
|
self._response.display(verbose) |
239
|
|
|
|
240
|
|
|
print("") |
241
|
|
|
print("Map Tree: ") |
242
|
|
|
print("----------\n") |
243
|
|
|
|
244
|
|
|
self._maptree.display() |
245
|
|
|
|
246
|
|
|
print("") |
247
|
|
|
print("Active energy/nHit planes: ") |
248
|
|
|
print("---------------------------\n") |
249
|
|
|
print(self._active_planes) |
250
|
|
|
|
251
|
|
|
def set_model(self, likelihood_model_instance): |
252
|
|
|
""" |
253
|
|
|
Set the model to be used in the joint minimization. Must be a LikelihoodModel instance. |
254
|
|
|
""" |
255
|
|
|
|
256
|
|
|
self._likelihood_model = likelihood_model_instance |
257
|
|
|
|
258
|
|
|
# Reset |
259
|
|
|
self._convolved_point_sources.reset() |
260
|
|
|
self._convolved_ext_sources.reset() |
261
|
|
|
|
262
|
|
|
# For each point source in the model, build the convolution class |
263
|
|
|
|
264
|
|
|
for source in self._likelihood_model.point_sources.values(): |
265
|
|
|
|
266
|
|
|
this_convolved_point_source = ConvolvedPointSource(source, self._response, self._flat_sky_projection) |
267
|
|
|
|
268
|
|
|
self._convolved_point_sources.append(this_convolved_point_source) |
269
|
|
|
|
270
|
|
|
# Samewise for extended sources |
271
|
|
|
|
272
|
|
|
ext_sources = self._likelihood_model.extended_sources.values() |
273
|
|
|
|
274
|
|
|
# NOTE: ext_sources evaluate to False if empty |
275
|
|
|
if ext_sources: |
276
|
|
|
|
277
|
|
|
# We will need to convolve |
278
|
|
|
|
279
|
|
|
self._setup_psf_convolutors() |
280
|
|
|
|
281
|
|
|
for source in ext_sources: |
282
|
|
|
|
283
|
|
|
if source.spatial_shape.n_dim == 2: |
284
|
|
|
|
285
|
|
|
this_convolved_ext_source = ConvolvedExtendedSource2D(source, |
286
|
|
|
self._response, |
287
|
|
|
self._flat_sky_projection) |
288
|
|
|
|
289
|
|
|
else: |
290
|
|
|
|
291
|
|
|
this_convolved_ext_source = ConvolvedExtendedSource3D(source, |
292
|
|
|
self._response, |
293
|
|
|
self._flat_sky_projection) |
294
|
|
|
|
295
|
|
|
self._convolved_ext_sources.append(this_convolved_ext_source) |
296
|
|
|
|
297
|
|
|
def display_spectrum(self): |
|
|
|
|
298
|
|
|
""" |
299
|
|
|
Make a plot of the current spectrum and its residuals (integrated over space) |
300
|
|
|
|
301
|
|
|
:return: a matplotlib.Figure |
302
|
|
|
""" |
303
|
|
|
|
304
|
|
|
n_point_sources = self._likelihood_model.get_number_of_point_sources() |
305
|
|
|
n_ext_sources = self._likelihood_model.get_number_of_extended_sources() |
306
|
|
|
|
307
|
|
|
total_counts = np.zeros(len(self._active_planes), dtype=float) |
308
|
|
|
total_model = np.zeros_like(total_counts) |
309
|
|
|
model_only = np.zeros_like(total_counts) |
310
|
|
|
net_counts = np.zeros_like(total_counts) |
311
|
|
|
yerr_low = np.zeros_like(total_counts) |
312
|
|
|
yerr_high = np.zeros_like(total_counts) |
313
|
|
|
|
314
|
|
|
for i, energy_id in enumerate(self._active_planes): |
315
|
|
|
|
316
|
|
|
data_analysis_bin = self._maptree[energy_id] |
317
|
|
|
|
318
|
|
|
this_model_map_hpx = self._get_expectation(data_analysis_bin, energy_id, n_point_sources, n_ext_sources) |
319
|
|
|
|
320
|
|
|
this_model_tot = np.sum(this_model_map_hpx) |
321
|
|
|
|
322
|
|
|
this_data_tot = np.sum(data_analysis_bin.observation_map.as_partial()) |
323
|
|
|
this_bkg_tot = np.sum(data_analysis_bin.background_map.as_partial()) |
324
|
|
|
|
325
|
|
|
total_counts[i] = this_data_tot |
326
|
|
|
net_counts[i] = this_data_tot - this_bkg_tot |
327
|
|
|
model_only[i] = this_model_tot |
328
|
|
|
|
329
|
|
|
this_wh_model = this_model_tot + this_bkg_tot |
330
|
|
|
total_model[i] = this_wh_model |
331
|
|
|
|
332
|
|
|
if this_data_tot >= 50.0: |
333
|
|
|
|
334
|
|
|
# Gaussian limit |
335
|
|
|
# Under the null hypothesis the data are distributed as a Gaussian with mu = model |
336
|
|
|
# and sigma = sqrt(model) |
337
|
|
|
# NOTE: since we neglect the background uncertainty, the background is part of the |
338
|
|
|
# model |
339
|
|
|
yerr_low[i] = np.sqrt(this_data_tot) |
340
|
|
|
yerr_high[i] = np.sqrt(this_data_tot) |
341
|
|
|
|
342
|
|
|
else: |
343
|
|
|
|
344
|
|
|
# Low-counts |
345
|
|
|
# Under the null hypothesis the data are distributed as a Poisson distribution with |
346
|
|
|
# mean = model, plot the 68% confidence interval (quantile=[0.16,1-0.16]). |
347
|
|
|
# NOTE: since we neglect the background uncertainty, the background is part of the |
348
|
|
|
# model |
349
|
|
|
quantile = 0.16 |
350
|
|
|
mean = this_wh_model |
351
|
|
|
y_low = poisson.isf(1-quantile, mu=mean) |
352
|
|
|
y_high = poisson.isf(quantile, mu=mean) |
353
|
|
|
yerr_low[i] = mean-y_low |
354
|
|
|
yerr_high[i] = y_high-mean |
355
|
|
|
|
356
|
|
|
residuals = (total_counts - total_model) / np.sqrt(total_model) |
357
|
|
|
residuals_err = [yerr_high / np.sqrt(total_model), |
358
|
|
|
yerr_low / np.sqrt(total_model)] |
359
|
|
|
|
360
|
|
|
yerr = [yerr_high, yerr_low] |
361
|
|
|
|
362
|
|
|
return self._plot_spectrum(net_counts, yerr, model_only, residuals, residuals_err) |
363
|
|
|
|
364
|
|
|
def _plot_spectrum(self, net_counts, yerr, model_only, residuals, residuals_err): |
|
|
|
|
365
|
|
|
|
366
|
|
|
fig, subs = plt.subplots(2, 1, gridspec_kw={'height_ratios': [2, 1], 'hspace': 0}) |
367
|
|
|
|
368
|
|
|
subs[0].errorbar(self._active_planes, net_counts, yerr=yerr, |
369
|
|
|
capsize=0, |
370
|
|
|
color='black', label='Net counts', fmt='.') |
371
|
|
|
|
372
|
|
|
subs[0].plot(self._active_planes, model_only, label='Convolved model') |
373
|
|
|
|
374
|
|
|
subs[0].legend(bbox_to_anchor=(1.0, 1.0), loc="upper right", |
375
|
|
|
numpoints=1) |
376
|
|
|
|
377
|
|
|
# Residuals |
378
|
|
|
subs[1].axhline(0, linestyle='--') |
379
|
|
|
|
380
|
|
|
subs[1].errorbar( |
381
|
|
|
self._active_planes, residuals, |
382
|
|
|
yerr=residuals_err, |
383
|
|
|
capsize=0, fmt='.' |
384
|
|
|
) |
385
|
|
|
|
386
|
|
|
y_limits = [min(net_counts[net_counts > 0]) / 2., max(net_counts) * 2.] |
387
|
|
|
|
388
|
|
|
subs[0].set_yscale("log", nonposy='clip') |
389
|
|
|
subs[0].set_ylabel("Counts per bin") |
390
|
|
|
subs[0].set_xticks([]) |
391
|
|
|
|
392
|
|
|
subs[1].set_xlabel("Analysis bin") |
393
|
|
|
subs[1].set_ylabel(r"$\frac{{cts - mod - bkg}}{\sqrt{mod + bkg}}$") |
394
|
|
|
subs[1].set_xticks(self._active_planes) |
395
|
|
|
subs[1].set_xticklabels(self._active_planes) |
396
|
|
|
|
397
|
|
|
subs[0].set_ylim(y_limits) |
398
|
|
|
|
399
|
|
|
return fig |
400
|
|
|
|
401
|
|
|
def get_log_like(self): |
402
|
|
|
""" |
403
|
|
|
Return the value of the log-likelihood with the current values for the |
404
|
|
|
parameters |
405
|
|
|
""" |
406
|
|
|
|
407
|
|
|
n_point_sources = self._likelihood_model.get_number_of_point_sources() |
408
|
|
|
n_ext_sources = self._likelihood_model.get_number_of_extended_sources() |
409
|
|
|
|
410
|
|
|
# Make sure that no source has been added since we filled the cache |
411
|
|
|
assert n_point_sources == self._convolved_point_sources.n_sources_in_cache and \ |
412
|
|
|
n_ext_sources == self._convolved_ext_sources.n_sources_in_cache, \ |
413
|
|
|
"The number of sources has changed. Please re-assign the model to the plugin." |
414
|
|
|
|
415
|
|
|
# This will hold the total log-likelihood |
416
|
|
|
total_log_like = 0 |
417
|
|
|
|
418
|
|
|
for bin_id in self._active_planes: |
419
|
|
|
|
420
|
|
|
data_analysis_bin = self._maptree[bin_id] |
421
|
|
|
|
422
|
|
|
this_model_map_hpx = self._get_expectation(data_analysis_bin, bin_id, n_point_sources, n_ext_sources) |
423
|
|
|
|
424
|
|
|
# Now compare with observation |
425
|
|
|
bkg_renorm = self._nuisance_parameters.values()[0].value |
426
|
|
|
|
427
|
|
|
obs = data_analysis_bin.observation_map.as_partial() # type: np.array |
428
|
|
|
bkg = data_analysis_bin.background_map.as_partial() * bkg_renorm # type: np.array |
429
|
|
|
|
430
|
|
|
this_pseudo_log_like = log_likelihood(obs, |
431
|
|
|
bkg, |
432
|
|
|
this_model_map_hpx) |
433
|
|
|
|
434
|
|
|
total_log_like += this_pseudo_log_like - self._log_factorials[bin_id] \ |
435
|
|
|
- self._saturated_model_like_per_maptree[bin_id] |
436
|
|
|
|
437
|
|
|
return total_log_like |
438
|
|
|
|
439
|
|
|
def write(self, response_file_name, map_tree_file_name): |
440
|
|
|
""" |
441
|
|
|
Write this dataset to disk in HDF format. |
442
|
|
|
|
443
|
|
|
:param response_file_name: filename for the response |
444
|
|
|
:param map_tree_file_name: filename for the map tree |
445
|
|
|
:return: None |
446
|
|
|
""" |
447
|
|
|
|
448
|
|
|
self._maptree.write(map_tree_file_name) |
449
|
|
|
self._response.write(response_file_name) |
450
|
|
|
|
451
|
|
|
def get_simulated_dataset(self, name): |
452
|
|
|
""" |
453
|
|
|
Return a simulation of this dataset using the current model with current parameters. |
454
|
|
|
|
455
|
|
|
:param name: new name for the new plugin instance |
456
|
|
|
:return: a HAL instance |
457
|
|
|
""" |
458
|
|
|
|
459
|
|
|
|
460
|
|
|
# First get expectation under the current model and store them, if we didn't do it yet |
461
|
|
|
|
462
|
|
|
if self._clone is None: |
463
|
|
|
|
464
|
|
|
n_point_sources = self._likelihood_model.get_number_of_point_sources() |
465
|
|
|
n_ext_sources = self._likelihood_model.get_number_of_extended_sources() |
466
|
|
|
|
467
|
|
|
expectations = [] |
468
|
|
|
|
469
|
|
|
for bin_id, data_analysis_bin in enumerate(self._maptree): |
470
|
|
|
|
471
|
|
|
if bin_id not in self._active_planes: |
472
|
|
|
|
473
|
|
|
expectations.append(None) |
474
|
|
|
|
475
|
|
|
else: |
476
|
|
|
|
477
|
|
|
expectations.append(self._get_expectation(data_analysis_bin, bin_id, |
478
|
|
|
n_point_sources, n_ext_sources) + |
479
|
|
|
data_analysis_bin.background_map.as_partial()) |
480
|
|
|
|
481
|
|
|
if parallel_client.is_parallel_computation_active(): |
482
|
|
|
|
483
|
|
|
# Do not clone, as the parallel environment already makes clones |
484
|
|
|
|
485
|
|
|
clone = self |
486
|
|
|
|
487
|
|
|
else: |
488
|
|
|
|
489
|
|
|
clone = copy.deepcopy(self) |
490
|
|
|
|
491
|
|
|
self._clone = (clone, expectations) |
492
|
|
|
|
493
|
|
|
# Substitute the observation and background for each data analysis bin |
494
|
|
|
for bin_id, data_analysis_bin in enumerate(self._clone[0]._maptree): |
|
|
|
|
495
|
|
|
|
496
|
|
|
if bin_id not in self._active_planes: |
497
|
|
|
|
498
|
|
|
continue |
499
|
|
|
|
500
|
|
|
else: |
501
|
|
|
|
502
|
|
|
# Active plane. Generate new data |
503
|
|
|
expectation = self._clone[1][bin_id] |
504
|
|
|
new_data = np.random.poisson(expectation, size=(1, expectation.shape[0])).flatten() |
505
|
|
|
|
506
|
|
|
# Substitute data |
507
|
|
|
data_analysis_bin.observation_map.set_new_values(new_data) |
508
|
|
|
|
509
|
|
|
# Now change name and return |
510
|
|
|
self._clone[0]._name = name |
|
|
|
|
511
|
|
|
# Adjust the name of the nuisance parameter |
512
|
|
|
old_name = self._clone[0]._nuisance_parameters.keys()[0] |
|
|
|
|
513
|
|
|
new_name = old_name.replace(self.name, name) |
514
|
|
|
self._clone[0]._nuisance_parameters[new_name] = self._clone[0]._nuisance_parameters.pop(old_name) |
|
|
|
|
515
|
|
|
|
516
|
|
|
# Recompute biases |
517
|
|
|
self._clone[0]._compute_likelihood_biases() |
|
|
|
|
518
|
|
|
|
519
|
|
|
return self._clone[0] |
520
|
|
|
|
521
|
|
|
def _get_expectation(self, data_analysis_bin, energy_bin_id, n_point_sources, n_ext_sources): |
522
|
|
|
|
523
|
|
|
# Compute the expectation from the model |
524
|
|
|
|
525
|
|
|
this_model_map = None |
526
|
|
|
|
527
|
|
|
for pts_id in range(n_point_sources): |
528
|
|
|
|
529
|
|
|
this_convolved_source = self._convolved_point_sources[pts_id] |
530
|
|
|
|
531
|
|
|
expectation_per_transit = this_convolved_source.get_source_map(energy_bin_id, tag=None) |
532
|
|
|
|
533
|
|
|
expectation_from_this_source = expectation_per_transit * data_analysis_bin.n_transits |
534
|
|
|
|
535
|
|
|
if this_model_map is None: |
536
|
|
|
|
537
|
|
|
# First addition |
538
|
|
|
|
539
|
|
|
this_model_map = expectation_from_this_source |
540
|
|
|
|
541
|
|
|
else: |
542
|
|
|
|
543
|
|
|
this_model_map += expectation_from_this_source |
544
|
|
|
|
545
|
|
|
# Now process extended sources |
546
|
|
|
if n_ext_sources > 0: |
547
|
|
|
|
548
|
|
|
this_ext_model_map = None |
549
|
|
|
|
550
|
|
|
for ext_id in range(n_ext_sources): |
551
|
|
|
|
552
|
|
|
this_convolved_source = self._convolved_ext_sources[ext_id] |
553
|
|
|
|
554
|
|
|
expectation_per_transit = this_convolved_source.get_source_map(energy_bin_id) |
555
|
|
|
|
556
|
|
|
if this_ext_model_map is None: |
557
|
|
|
|
558
|
|
|
# First addition |
559
|
|
|
|
560
|
|
|
this_ext_model_map = expectation_per_transit |
561
|
|
|
|
562
|
|
|
else: |
563
|
|
|
|
564
|
|
|
this_ext_model_map += expectation_per_transit |
565
|
|
|
|
566
|
|
|
# Now convolve with the PSF |
567
|
|
|
if this_model_map is None: |
568
|
|
|
|
|
|
|
|
569
|
|
|
# Only extended sources |
570
|
|
|
|
|
|
|
|
571
|
|
|
this_model_map = (self._psf_convolutors[energy_bin_id].extended_source_image(this_ext_model_map) * |
572
|
|
|
data_analysis_bin.n_transits) |
573
|
|
|
|
|
|
|
|
574
|
|
|
else: |
575
|
|
|
|
576
|
|
|
this_model_map += (self._psf_convolutors[energy_bin_id].extended_source_image(this_ext_model_map) * |
577
|
|
|
data_analysis_bin.n_transits) |
578
|
|
|
|
579
|
|
|
|
580
|
|
|
# Now transform from the flat sky projection to HEALPiX |
581
|
|
|
|
582
|
|
|
if this_model_map is not None: |
583
|
|
|
|
584
|
|
|
# First divide for the pixel area because we need to interpolate brightness |
585
|
|
|
this_model_map = this_model_map / self._flat_sky_projection.project_plane_pixel_area |
586
|
|
|
|
587
|
|
|
this_model_map_hpx = self._flat_sky_to_healpix_transform[energy_bin_id](this_model_map, fill_value=0.0) |
588
|
|
|
|
589
|
|
|
# Now multiply by the pixel area of the new map to go back to flux |
590
|
|
|
this_model_map_hpx *= hp.nside2pixarea(data_analysis_bin.nside, degrees=True) |
591
|
|
|
|
592
|
|
|
else: |
593
|
|
|
|
594
|
|
|
# No sources |
595
|
|
|
|
596
|
|
|
this_model_map_hpx = 0.0 |
597
|
|
|
|
598
|
|
|
return this_model_map_hpx |
599
|
|
|
|
600
|
|
|
@staticmethod |
601
|
|
|
def _represent_healpix_map(fig, hpx_map, longitude, latitude, xsize, resolution, smoothing_kernel_sigma): |
|
|
|
|
602
|
|
|
|
603
|
|
|
proj = get_gnomonic_projection(fig, hpx_map, |
604
|
|
|
rot=(longitude, latitude, 0.0), |
605
|
|
|
xsize=xsize, |
606
|
|
|
reso=resolution) |
607
|
|
|
|
608
|
|
|
if smoothing_kernel_sigma is not None: |
609
|
|
|
|
610
|
|
|
# Get the sigma in pixels |
611
|
|
|
sigma = smoothing_kernel_sigma * 60 / resolution |
612
|
|
|
|
613
|
|
|
proj = convolve(list(proj), |
614
|
|
|
Gaussian2DKernel(sigma), |
615
|
|
|
nan_treatment='fill', |
616
|
|
|
preserve_nan=True) |
617
|
|
|
|
618
|
|
|
return proj |
619
|
|
|
|
620
|
|
|
def display_fit(self, smoothing_kernel_sigma=0.1, display_colorbar=False): |
|
|
|
|
621
|
|
|
""" |
622
|
|
|
Make a figure containing 3 maps for each active analysis bins with respectively model, data and residuals. |
623
|
|
|
|
624
|
|
|
:param smoothing_kernel_sigma: sigma for the Gaussian smoothing kernel |
625
|
|
|
:param display_colorbar: whether or not to display the colorbar in the residuals |
626
|
|
|
:return: a matplotlib.Figure |
627
|
|
|
""" |
628
|
|
|
|
629
|
|
|
n_point_sources = self._likelihood_model.get_number_of_point_sources() |
630
|
|
|
n_ext_sources = self._likelihood_model.get_number_of_extended_sources() |
631
|
|
|
|
632
|
|
|
# This is the resolution (i.e., the size of one pixel) of the image |
633
|
|
|
resolution = 3.0 # arcmin |
634
|
|
|
|
635
|
|
|
# The image is going to cover the diameter plus 20% padding |
636
|
|
|
xsize = self._get_optimal_xsize(resolution) |
637
|
|
|
|
638
|
|
|
n_active_planes = len(self._active_planes) |
639
|
|
|
|
640
|
|
|
fig, subs = plt.subplots(n_active_planes, 3, figsize=(8, n_active_planes * 2)) |
641
|
|
|
|
642
|
|
|
with progress_bar(len(self._active_planes), title='Smoothing maps') as prog_bar: |
643
|
|
|
|
644
|
|
|
images = ['None'] * 3 |
645
|
|
|
|
646
|
|
|
for i, plane_id in enumerate(self._active_planes): |
647
|
|
|
|
648
|
|
|
data_analysis_bin = self._maptree[plane_id] |
649
|
|
|
|
650
|
|
|
# Get the center of the projection for this plane |
651
|
|
|
this_ra, this_dec = self._roi.ra_dec_center |
652
|
|
|
|
653
|
|
|
this_model_map_hpx = self._get_expectation(data_analysis_bin, plane_id, n_point_sources, n_ext_sources) |
654
|
|
|
|
655
|
|
|
# Make a full healpix map for a second |
656
|
|
|
whole_map = SparseHealpix(this_model_map_hpx, |
657
|
|
|
self._active_pixels[plane_id], |
658
|
|
|
data_analysis_bin.observation_map.nside).as_dense() |
659
|
|
|
|
660
|
|
|
# Healpix uses longitude between -180 and 180, while R.A. is between 0 and 360. We need to fix that: |
661
|
|
|
longitude = ra_to_longitude(this_ra) |
662
|
|
|
|
663
|
|
|
# Declination is already between -90 and 90 |
664
|
|
|
latitude = this_dec |
665
|
|
|
|
666
|
|
|
# Plot model |
667
|
|
|
|
668
|
|
|
proj_m = self._represent_healpix_map(fig, whole_map, |
669
|
|
|
longitude, latitude, |
670
|
|
|
xsize, resolution, smoothing_kernel_sigma) |
671
|
|
|
|
672
|
|
|
images[0] = subs[i][0].imshow(proj_m, origin='lower') |
673
|
|
|
|
674
|
|
|
# Remove numbers from axis |
675
|
|
|
subs[i][0].axis('off') |
676
|
|
|
|
677
|
|
|
# Plot data map |
678
|
|
|
# Here we removed the background otherwise nothing is visible |
679
|
|
|
# Get background (which is in a way "part of the model" since the uncertainties are neglected) |
680
|
|
|
background_map = data_analysis_bin.background_map.as_dense() |
681
|
|
|
bkg_subtracted = data_analysis_bin.observation_map.as_dense() - background_map |
682
|
|
|
|
683
|
|
|
proj_d = self._represent_healpix_map(fig, bkg_subtracted, |
684
|
|
|
longitude, latitude, |
685
|
|
|
xsize, resolution, smoothing_kernel_sigma) |
686
|
|
|
|
687
|
|
|
images[1] = subs[i][1].imshow(proj_d, origin='lower') |
688
|
|
|
|
689
|
|
|
# Remove numbers from axis |
690
|
|
|
subs[i][1].axis('off') |
691
|
|
|
|
692
|
|
|
# Now residuals |
693
|
|
|
res = proj_d - proj_m |
694
|
|
|
# proj_res = self._represent_healpix_map(fig, res, |
695
|
|
|
# longitude, latitude, |
696
|
|
|
# xsize, resolution, smoothing_kernel_sigma) |
697
|
|
|
images[2] = subs[i][2].imshow(res, origin='lower') |
698
|
|
|
|
699
|
|
|
# Remove numbers from axis |
700
|
|
|
subs[i][2].axis('off') |
701
|
|
|
|
702
|
|
|
subs[i][0].set_title('model, bin {}'.format(data_analysis_bin.name)) |
703
|
|
|
subs[i][1].set_title('excess, bin {}'.format(data_analysis_bin.name)) |
704
|
|
|
subs[i][2].set_title('residuals, bin {}'.format(data_analysis_bin.name)) |
705
|
|
|
|
706
|
|
|
if display_colorbar: |
707
|
|
|
for j, image in enumerate(images): |
708
|
|
|
plt.colorbar(image, ax=subs[i][j]) |
709
|
|
|
|
710
|
|
|
prog_bar.increase() |
711
|
|
|
|
712
|
|
|
fig.set_tight_layout(True) |
713
|
|
|
|
714
|
|
|
return fig |
715
|
|
|
|
716
|
|
|
def _get_optimal_xsize(self, resolution): |
717
|
|
|
|
718
|
|
|
return 2.2 * self._roi.data_radius.to("deg").value / (resolution / 60.0) |
719
|
|
|
|
720
|
|
|
def display_stacked_image(self, smoothing_kernel_sigma=0.5): |
|
|
|
|
721
|
|
|
""" |
722
|
|
|
Display a map with all active analysis bins stacked together. |
723
|
|
|
|
724
|
|
|
:param smoothing_kernel_sigma: sigma for the Gaussian smoothing kernel to apply |
725
|
|
|
:return: a matplotlib.Figure instance |
726
|
|
|
""" |
727
|
|
|
|
728
|
|
|
# This is the resolution (i.e., the size of one pixel) of the image in arcmin |
729
|
|
|
resolution = 3.0 |
730
|
|
|
|
731
|
|
|
# The image is going to cover the diameter plus 20% padding |
732
|
|
|
xsize = self._get_optimal_xsize(resolution) |
733
|
|
|
|
734
|
|
|
active_planes_bins = [self._maptree[x] for x in self._active_planes] |
735
|
|
|
|
736
|
|
|
# Get the center of the projection for this plane |
737
|
|
|
this_ra, this_dec = self._roi.ra_dec_center |
738
|
|
|
|
739
|
|
|
# Healpix uses longitude between -180 and 180, while R.A. is between 0 and 360. We need to fix that: |
740
|
|
|
longitude = ra_to_longitude(this_ra) |
741
|
|
|
|
742
|
|
|
# Declination is already between -90 and 90 |
743
|
|
|
latitude = this_dec |
744
|
|
|
|
745
|
|
|
total = None |
746
|
|
|
|
747
|
|
|
for i, data_analysis_bin in enumerate(active_planes_bins): |
748
|
|
|
|
749
|
|
|
# Plot data |
750
|
|
|
background_map = data_analysis_bin.background_map.as_dense() |
751
|
|
|
this_data = data_analysis_bin.observation_map.as_dense() - background_map |
752
|
|
|
idx = np.isnan(this_data) |
753
|
|
|
# this_data[idx] = hp.UNSEEN |
754
|
|
|
|
755
|
|
|
if i == 0: |
756
|
|
|
|
757
|
|
|
total = this_data |
758
|
|
|
|
759
|
|
|
else: |
760
|
|
|
|
761
|
|
|
# Sum only when there is no UNSEEN, so that the UNSEEN pixels will stay UNSEEN |
762
|
|
|
total[~idx] += this_data[~idx] |
763
|
|
|
|
764
|
|
|
delta_coord = (self._roi.data_radius.to("deg").value * 2.0) / 15.0 |
765
|
|
|
|
766
|
|
|
fig, sub = plt.subplots(1, 1) |
767
|
|
|
|
768
|
|
|
proj = self._represent_healpix_map(fig, total, longitude, latitude, xsize, resolution, smoothing_kernel_sigma) |
769
|
|
|
|
770
|
|
|
sub.imshow(proj, origin='lower') |
771
|
|
|
sub.axis('off') |
772
|
|
|
|
773
|
|
|
hp.graticule(delta_coord, delta_coord) |
774
|
|
|
|
775
|
|
|
return fig |
776
|
|
|
|
777
|
|
|
def inner_fit(self): |
778
|
|
|
""" |
779
|
|
|
This is used for the profile likelihood. Keeping fixed all parameters in the |
780
|
|
|
LikelihoodModel, this method minimize the logLike over the remaining nuisance |
781
|
|
|
parameters, i.e., the parameters belonging only to the model for this |
782
|
|
|
particular detector. If there are no nuisance parameters, simply return the |
783
|
|
|
logLike value. |
784
|
|
|
""" |
785
|
|
|
|
786
|
|
|
return self.get_log_like() |
787
|
|
|
|
788
|
|
|
def get_number_of_data_points(self): |
789
|
|
|
""" |
790
|
|
|
Return the number of active bins across all active analysis bins |
791
|
|
|
|
792
|
|
|
:return: number of active bins |
793
|
|
|
""" |
794
|
|
|
|
795
|
|
|
n_points = 0 |
796
|
|
|
|
797
|
|
|
for bin_id in self._maptree: |
798
|
|
|
n_points += self._maptree[bin_id].observation_map.as_partial().shape[0] |
799
|
|
|
|
800
|
|
|
return n_points |
801
|
|
|
|
This check looks for invalid names for a range of different identifiers.
You can set regular expressions to which the identifiers must conform if the defaults do not match your requirements.
If your project includes a Pylint configuration file, the settings contained in that file take precedence.
To find out more about Pylint, please refer to their site.