|
1
|
|
|
""" |
|
2
|
|
|
Calculations of concordance between annotation sets. |
|
3
|
|
|
""" |
|
4
|
|
|
import abc |
|
5
|
|
|
import enum |
|
6
|
|
|
import math |
|
7
|
|
|
import time |
|
8
|
|
|
from collections import defaultdict |
|
9
|
|
|
from typing import Collection, Sequence, Type, Union |
|
10
|
|
|
|
|
11
|
|
|
import decorateme |
|
|
|
|
|
|
12
|
|
|
import numpy as np |
|
|
|
|
|
|
13
|
|
|
import pandas as pd |
|
|
|
|
|
|
14
|
|
|
from pocketutils.core.chars import Chars |
|
|
|
|
|
|
15
|
|
|
from pocketutils.core.enums import CleverEnum |
|
|
|
|
|
|
16
|
|
|
from pocketutils.tools.unit_tools import UnitTools |
|
|
|
|
|
|
17
|
|
|
|
|
18
|
|
|
from mandos.analysis import AnalysisUtils as Au |
|
19
|
|
|
from mandos.analysis.io_defns import SimilarityDfLongForm, SimilarityDfShortForm |
|
20
|
|
|
from mandos.model.hits import AbstractHit |
|
21
|
|
|
|
|
22
|
|
|
# note that most of these math functions are much faster than their numpy counterparts |
|
23
|
|
|
# if we're not broadcasting, it's almost always better to use them |
|
24
|
|
|
# some are more accurate, too |
|
25
|
|
|
# e.g. we're using fsum rather than sum |
|
26
|
|
|
from mandos.model.utils.setup import logger |
|
27
|
|
|
|
|
28
|
|
|
|
|
29
|
|
|
@decorateme.auto_repr_str() |
|
|
|
|
|
|
30
|
|
|
class MatrixCalculator(metaclass=abc.ABCMeta): |
|
31
|
|
|
def calc_all(self, hits: Sequence[AbstractHit]) -> SimilarityDfLongForm: |
|
|
|
|
|
|
32
|
|
|
raise NotImplemented() |
|
|
|
|
|
|
33
|
|
|
|
|
34
|
|
|
|
|
35
|
|
|
class _Inf: |
|
36
|
|
|
def __init__(self, n: int): |
|
37
|
|
|
self.n = n |
|
|
|
|
|
|
38
|
|
|
self.used, self.t0, self.nonzeros = set(), time.monotonic(), 0 |
|
|
|
|
|
|
39
|
|
|
|
|
40
|
|
|
def is_used(self, c1: str, c2: str) -> bool: |
|
|
|
|
|
|
41
|
|
|
return (c1, c2) in self.used or (c2, c1) in self.used |
|
42
|
|
|
|
|
43
|
|
|
def got(self, c1: str, c2: str, z: float) -> None: |
|
|
|
|
|
|
44
|
|
|
self.used.add((c1, c2)) |
|
45
|
|
|
self.nonzeros += int(c1 != c2 and not np.isnan(z) and 0 < z < 1) |
|
46
|
|
|
if self.i % 50000 == 0: |
|
47
|
|
|
self.log("info") |
|
48
|
|
|
elif self.i % 5000 == 0: |
|
49
|
|
|
self.log("trace") |
|
50
|
|
|
|
|
51
|
|
|
@property |
|
52
|
|
|
def i(self) -> int: |
|
|
|
|
|
|
53
|
|
|
return len(self.used) |
|
54
|
|
|
|
|
55
|
|
|
def log(self, level: str) -> None: |
|
|
|
|
|
|
56
|
|
|
delta = UnitTools.delta_time_to_str(time.monotonic() - self.t0, space=Chars.narrownbsp) |
|
57
|
|
|
logger.log( |
|
58
|
|
|
level.upper(), |
|
59
|
|
|
f"Processed {self.i:,}/{self.n:,} pairs in {delta};" |
|
60
|
|
|
+ f" {self.nonzeros:,} ({self.nonzeros / self.i * 100:.1f}%) are nonzero", |
|
61
|
|
|
) |
|
62
|
|
|
|
|
63
|
|
|
def __repr__(self): |
|
64
|
|
|
return f"{self.__class__.__name__}({self.i}/{self.n})" |
|
65
|
|
|
|
|
66
|
|
|
def __str__(self): |
|
67
|
|
|
return repr(self) |
|
68
|
|
|
|
|
69
|
|
|
|
|
70
|
|
|
class JPrimeMatrixCalculator(MatrixCalculator): |
|
|
|
|
|
|
71
|
|
|
def calc_all(self, hits: Sequence[AbstractHit]) -> SimilarityDfLongForm: |
|
72
|
|
|
key_to_hit = Au.hit_multidict(hits, "search_key") |
|
73
|
|
|
logger.notice(f"Calculating J on {len(key_to_hit)} keys from {len(hits)} hits") |
|
74
|
|
|
dfs = [] |
|
75
|
|
|
for key, key_hits in key_to_hit.items(): |
|
76
|
|
|
df: SimilarityDfShortForm = self.calc_one(key, key_hits) |
|
|
|
|
|
|
77
|
|
|
df = df.to_long_form(kind="psi", key=key) |
|
|
|
|
|
|
78
|
|
|
dfs += [df] |
|
79
|
|
|
return SimilarityDfLongForm(pd.concat(dfs)) |
|
80
|
|
|
|
|
81
|
|
|
def calc_one(self, key: str, hits: Sequence[AbstractHit]) -> SimilarityDfShortForm: |
|
|
|
|
|
|
82
|
|
|
ik2hits = Au.hit_multidict(hits, "origin_inchikey") |
|
83
|
|
|
logger.info(f"Calculating J on {key} for {len(ik2hits)} compounds and {len(hits)} hits") |
|
84
|
|
|
data = defaultdict(dict) |
|
85
|
|
|
inf = _Inf(n=int(len(ik2hits) * (len(ik2hits) - 1) / 2)) |
|
86
|
|
|
for (c1, hits1) in ik2hits.items(): |
|
|
|
|
|
|
87
|
|
|
for (c2, hits2) in ik2hits.items(): |
|
|
|
|
|
|
88
|
|
|
if inf.is_used(c1, c2): |
|
89
|
|
|
continue |
|
90
|
|
|
z = 1 if c1 == c2 else self._j_prime(key, hits1, hits2) |
|
|
|
|
|
|
91
|
|
|
data[c1][c2] = z |
|
92
|
|
|
inf.got(c1, c2, z) |
|
93
|
|
|
inf.log("notice") |
|
94
|
|
|
return SimilarityDfShortForm.from_dict(data) |
|
95
|
|
|
|
|
96
|
|
|
def _j_prime( |
|
97
|
|
|
self, key: str, hits1: Collection[AbstractHit], hits2: Collection[AbstractHit] |
|
|
|
|
|
|
98
|
|
|
) -> float: |
|
99
|
|
|
if len(hits1) == 0 or len(hits2) == 0: |
|
100
|
|
|
return 0 |
|
101
|
|
|
sources = {h.data_source for h in hits1}.intersection({h.data_source for h in hits2}) |
|
102
|
|
|
if len(sources) == 0: |
|
103
|
|
|
return np.nan |
|
104
|
|
|
values = [ |
|
105
|
|
|
self._jx( |
|
106
|
|
|
key, |
|
107
|
|
|
[h for h in hits1 if h.data_source == source], |
|
108
|
|
|
[h for h in hits2 if h.data_source == source], |
|
109
|
|
|
) |
|
110
|
|
|
for source in sources |
|
111
|
|
|
] |
|
112
|
|
|
return float(math.fsum(values) / len(values)) |
|
113
|
|
|
|
|
114
|
|
|
def _jx( |
|
115
|
|
|
self, key: str, hits1: Collection[AbstractHit], hits2: Collection[AbstractHit] |
|
|
|
|
|
|
116
|
|
|
) -> float: |
|
117
|
|
|
# TODO -- for testing only |
|
|
|
|
|
|
118
|
|
|
# TODO: REMOVE ME! |
|
|
|
|
|
|
119
|
|
|
if key in ["core.chemidplus.effects", "extra.chemidplus.specific-effects"]: |
|
120
|
|
|
hits1 = [h.copy(weight=np.power(10, -h.weight)) for h in hits1] |
|
121
|
|
|
hits2 = [h.copy(weight=np.power(10, -h.weight)) for h in hits2] |
|
122
|
|
|
pair_to_weights = Au.weights_of_pairs(hits1, hits2) |
|
123
|
|
|
values = [self._wedge(ca, cb) / self._vee(ca, cb) for ca, cb in pair_to_weights.values()] |
|
124
|
|
|
return float(math.fsum(values) / len(values)) |
|
125
|
|
|
|
|
126
|
|
|
def _wedge(self, ca: float, cb: float) -> float: |
|
|
|
|
|
|
127
|
|
|
return math.sqrt(Au.elle(ca) * Au.elle(cb)) |
|
128
|
|
|
|
|
129
|
|
|
def _vee(self, ca: float, cb: float) -> float: |
|
|
|
|
|
|
130
|
|
|
return Au.elle(ca) + Au.elle(cb) - math.sqrt(Au.elle(ca) * Au.elle(cb)) |
|
131
|
|
|
|
|
132
|
|
|
|
|
133
|
|
|
class MatrixAlg(CleverEnum): |
|
|
|
|
|
|
134
|
|
|
j = enum.auto() |
|
135
|
|
|
|
|
136
|
|
|
@property |
|
137
|
|
|
def clazz(self) -> Type[MatrixCalculator]: |
|
|
|
|
|
|
138
|
|
|
return {MatrixAlg.j: JPrimeMatrixCalculator}[self] |
|
139
|
|
|
|
|
140
|
|
|
|
|
141
|
|
|
@decorateme.auto_utils() |
|
|
|
|
|
|
142
|
|
|
class MatrixCalculation: |
|
143
|
|
|
@classmethod |
|
144
|
|
|
def create(cls, algorithm: Union[str, MatrixAlg]) -> MatrixCalculator: |
|
|
|
|
|
|
145
|
|
|
return MatrixAlg.of(algorithm).clazz() |
|
146
|
|
|
|
|
147
|
|
|
|
|
148
|
|
|
__all__ = ["MatrixCalculator", "JPrimeMatrixCalculator", "MatrixCalculation", "MatrixAlg"] |
|
149
|
|
|
|
dmyersturnbull /
mandos
Loading history...