mandos.entry.tools.searchers.Searcher._search_one() - Code Metrics - Inspection of "fix: various" - dmyersturnbull/mandos - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Push — main ( 65730f...fad324 )

by Douglas

created 2021-10-28 02:08 UTC

one() C

↳ Parent: mandos.entry.tools.searchers.Searcher.search()

Complexity

Conditions

Size

Total Lines	53
Code Lines	39

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
cc	9
eloc	39
nop	4
dl	0
loc	53
rs	6.6106
c	0
b	0
f	0

How to fix Long Method

"""
Run searches and write files.
"""

from __future__ import annotations

import time
from dataclasses import dataclass
from datetime import timedelta
from pathlib import Path
from typing import Sequence

from pocketutils.core.exceptions import IllegalStateError

from typeddfs import Checksums, TypedDfs


from mandos.model import CompoundNotFoundError
from mandos.model.hit_dfs import HitDf
from mandos.model.hits import AbstractHit
from mandos.model.search_caches import SearchCache
from mandos.model.searches import Search, SearchError
from mandos.model.settings import SETTINGS
from mandos.model.utils.setup import logger


def _fix_cols(df):

    return df.rename(columns={s: s.lower() for s in df.columns})


InputCompoundsDf = (
    TypedDfs.typed("InputCompoundsDf")
    .require("inchikey")
    .reserve("inchi", "smiles", "compound_id", dtype=str)
    .post(_fix_cols)
    .strict(cols=False)
    .secure()
).build()


@dataclass(frozen=True, repr=True, order=True)

class SearchReturnInfo:
    n_kept: int
    n_processed: int
    n_errored: int
    time_taken: timedelta


@dataclass(frozen=True, repr=True)
class Searcher:
    """
    Executes one or more searches and saves the results.
    Create and use once.
    """

    what: Search
    input_df: InputCompoundsDf
    to: Path

    proceed: bool
    restart: bool

    def search(self) -> SearchReturnInfo:

        """
        Performs the search, and writes data.
        """
        inchikeys = self.input_df["inchikey"].unique()
        if self.is_complete:
            logger.info(f"{self.to} already complete")
            return SearchReturnInfo(
                n_kept=len(inchikeys), n_processed=0, n_errored=0, time_taken=timedelta(seconds=0)
            )
        logger.info(f"Will save every {SETTINGS.save_every} compounds")
        logger.info(f"Writing {self.what.key} to {self.to}")
        annotes = []
        compounds_run = set()
        cache = SearchCache(self.to, inchikeys, restart=self.restart, proceed=self.proceed)
        # refresh so we know it's (no longer) complete
        # this would only happen if we're forcing this -- which is not currently allowed
        (
            Checksums()
            .load_dirsum_of_file(self.to, missing_ok=True)
            .remove(self.to, missing_ok=True)
            .write(rm_if_empty=True)
        )
        t0, n0, n_proc, n_err, n_annot = time.monotonic(), cache.at, 0, 0, 0

        while True:
            try:
                compound = cache.next()

            except StopIteration:
                break
            try:
                with logger.contextualize(compound=compound):
                    x = self.what.find(compound)

                annotes.extend(x)
            except CompoundNotFoundError:
                logger.info(f"Compound {compound} not found for {self.what.key}")
                x = []

                n_err += 1
            except Exception:
                raise SearchError(
                    f"Failed {self.what.key} [{self.what.search_class}] on compound {compound}",
                    compound=compound,
                    search_key=self.what.key,
                    search_class=self.what.search_class,
                )
            compounds_run.add(compound)
            logger.debug(f"Found {len(x)} {self.what.search_name()} annotations for {compound}")
            n_annot += len(x)
            n_proc += 1
            # logging, caching, and such:
            on_nth = cache.at % SETTINGS.save_every == SETTINGS.save_every - 1
            is_last = cache.at == len(inchikeys) - 1
            if on_nth or is_last:
                logger.log(
                    "NOTICE" if is_last else "INFO",
                    f"Found {len(annotes)} {self.what.search_name()} annotations"
                    + f" for {cache.at} of {len(inchikeys)} compounds",
                )
                self._save(annotes, done=is_last)
            cache.save(*compounds_run)  # CRITICAL -- do this AFTER saving
        # done!
        i1, t1 = cache.at, time.monotonic()

        assert i1 == len(inchikeys)
        cache.kill()
        logger.success(f"Wrote {self.what.key} to {self.to}")
        return SearchReturnInfo(
            n_kept=n0, n_processed=n_proc, n_errored=n_err, time_taken=timedelta(seconds=t1 - t0)
        )

    @property
    def is_partial(self) -> bool:

        return self.to.exists() and not self.is_complete

    @property
    def is_complete(self) -> bool:

        done = self.to in Checksums().load_dirsum_of_file(self.to)
        if done and not self.to.exists():
            raise IllegalStateError(f"{self.to} marked complete but does not exist")
        return done

    def _save(self, hits: Sequence[AbstractHit], *, done: bool):
        df = HitDf.from_hits(hits)

        # keep all of the original extra columns from the input
        # e.g. if the user had 'inchi' or 'smiles' or 'pretty_name'
        for extra_col in [c for c in self.input_df.columns if c != "inchikey"]:
            extra_mp = self.input_df.set_index("inchikey")[extra_col].to_dict()
            df[extra_col] = df["origin_inchikey"].map(extra_mp.get)
        # write the file
        df: HitDf = HitDf.of(df)

        params = self.what.get_params()
        df = df.set_attrs(**params, key=self.what.key)

        df.write_file(self.to, mkdirs=True, attrs=True, dir_hash=done)
        logger.debug(f"Saved {len(df)} rows to {self.to}")


__all__ = ["Searcher", "InputCompoundsDf", "SearchReturnInfo"]


1			"""
2			Run searches and write files.
3			"""
4
5			from __future__ import annotations
6
7			import time
8			from dataclasses import dataclass
9			from datetime import timedelta
10			from pathlib import Path
11			from typing import Sequence
12
13			from pocketutils.core.exceptions import IllegalStateError
			0 ignored issues – show introduced 2021-08-02 23:39 UTC by Report Bug Copy Issue Report Unable to import 'pocketutils.core.exceptions' Loading history...
14			from typeddfs import Checksums, TypedDfs
			0 ignored issues – show introduced 2021-08-07 00:15 UTC by Report Bug Copy Issue Report Unable to import 'typeddfs' Loading history...
15
16			from mandos.model import CompoundNotFoundError
17			from mandos.model.hit_dfs import HitDf
18			from mandos.model.hits import AbstractHit
19			from mandos.model.search_caches import SearchCache
20			from mandos.model.searches import Search, SearchError
21			from mandos.model.settings import SETTINGS
22			from mandos.model.utils.setup import logger
23
24
25			def _fix_cols(df):
			0 ignored issues – show Coding Style Naming introduced 2021-08-02 23:39 UTC by Report Bug Copy Issue Report Argument name "df" doesn't conform to snake_case naming style ('([^\\W\\dA-Z][^\\WA-Z]2,\|_[^\\WA-Z]*\|__[^\\WA-Z\\d_][^\\WA-Z]+__)$' pattern) This check looks for invalid names for a range of different identifiers. You can set regular expressions to which the identifiers must conform if the defaults do not match your requirements. If your project includes a Pylint configuration file, the settings contained in that file take precedence. To find out more about Pylint, please refer to their site. Loading history...
26			return df.rename(columns={s: s.lower() for s in df.columns})
27
28
29			InputCompoundsDf = (
30			TypedDfs.typed("InputCompoundsDf")
31			.require("inchikey")
32			.reserve("inchi", "smiles", "compound_id", dtype=str)
33			.post(_fix_cols)
34			.strict(cols=False)
35			.secure()
36			).build()
37
38
39			@dataclass(frozen=True, repr=True, order=True)
			0 ignored issues – show introduced 2021-04-03 02:07 UTC by Report Bug Copy Issue Report Missing class docstring Loading history...
40			class SearchReturnInfo:
41			n_kept: int
42			n_processed: int
43			n_errored: int
44			time_taken: timedelta
45
46
47			@dataclass(frozen=True, repr=True)
48			class Searcher:
49			"""
50			Executes one or more searches and saves the results.
51			Create and use once.
52			"""
53
54			what: Search
55			input_df: InputCompoundsDf
56			to: Path
			0 ignored issues – show Coding Style Naming introduced 2021-10-28 02:14 UTC by Report Bug Copy Issue Report Attribute name "to" doesn't conform to snake_case naming style ('([^\\W\\dA-Z][^\\WA-Z]2,\|_[^\\WA-Z]*\|__[^\\WA-Z\\d_][^\\WA-Z]+__)$' pattern) This check looks for invalid names for a range of different identifiers. You can set regular expressions to which the identifiers must conform if the defaults do not match your requirements. If your project includes a Pylint configuration file, the settings contained in that file take precedence. To find out more about Pylint, please refer to their site. Loading history...
57			proceed: bool
58			restart: bool
59
60			def search(self) -> SearchReturnInfo:
			0 ignored issues – show Comprehensibility introduced 2021-10-28 02:14 UTC by Report Bug Copy Issue Report This function exceeds the maximum number of variables (16/15). Loading history...
61			"""
62			Performs the search, and writes data.
63			"""
64			inchikeys = self.input_df["inchikey"].unique()
65			if self.is_complete:
66			logger.info(f"{self.to} already complete")
67			return SearchReturnInfo(
68			n_kept=len(inchikeys), n_processed=0, n_errored=0, time_taken=timedelta(seconds=0)
69			)
70			logger.info(f"Will save every {SETTINGS.save_every} compounds")
71			logger.info(f"Writing {self.what.key} to {self.to}")
72			annotes = []
73			compounds_run = set()
74			cache = SearchCache(self.to, inchikeys, restart=self.restart, proceed=self.proceed)
75			# refresh so we know it's (no longer) complete
76			# this would only happen if we're forcing this -- which is not currently allowed
77			(
78			Checksums()
79			.load_dirsum_of_file(self.to, missing_ok=True)
80			.remove(self.to, missing_ok=True)
81			.write(rm_if_empty=True)
82			)
83			t0, n0, n_proc, n_err, n_annot = time.monotonic(), cache.at, 0, 0, 0
			0 ignored issues – show Coding Style Naming introduced 2021-10-28 02:14 UTC by Report Bug Copy Issue Report Variable name "n0" doesn't conform to snake_case naming style ('([^\\W\\dA-Z][^\\WA-Z]2,\|_[^\\WA-Z]\|__[^\\WA-Z\\d_][^\\WA-Z]+__)$' pattern) This check looks for invalid names for a range of different identifiers. You can set regular expressions to which the identifiers must conform if the defaults do not match your requirements. If your project includes a Pylint configuration file, the settings contained in that file take precedence. To find out more about Pylint, please refer to their site. Loading history... Coding Style Naming introduced 2021-10-28 02:14 UTC by Report Bug Copy Issue Report Variable name "t0" doesn't conform to snake_case naming style ('([^\\W\\dA-Z][^\\WA-Z]2,\|_[^\\WA-Z]\|__[^\\WA-Z\\d_][^\\WA-Z]+__)$' pattern) This check looks for invalid names for a range of different identifiers. You can set regular expressions to which the identifiers must conform if the defaults do not match your requirements. If your project includes a Pylint configuration file, the settings contained in that file take precedence. To find out more about Pylint, please refer to their site. Loading history...
84			while True:
85			try:
86			compound = cache.next()
			0 ignored issues – show Bug introduced 2021-09-29 03:10 UTC by Report Bug Copy Issue Report `cache.next` does not seem to be callable. Loading history...
87			except StopIteration:
88			break
89			try:
90			with logger.contextualize(compound=compound):
91			x = self.what.find(compound)
			0 ignored issues – show Coding Style Naming introduced 2021-09-29 03:10 UTC by Report Bug Copy Issue Report Variable name "x" doesn't conform to snake_case naming style ('([^\\W\\dA-Z][^\\WA-Z]2,\|_[^\\WA-Z]*\|__[^\\WA-Z\\d_][^\\WA-Z]+__)$' pattern) This check looks for invalid names for a range of different identifiers. You can set regular expressions to which the identifiers must conform if the defaults do not match your requirements. If your project includes a Pylint configuration file, the settings contained in that file take precedence. To find out more about Pylint, please refer to their site. Loading history...
92			annotes.extend(x)
93			except CompoundNotFoundError:
94			logger.info(f"Compound {compound} not found for {self.what.key}")
95			x = []
			0 ignored issues – show Coding Style Naming introduced 2021-09-29 16:53 UTC by Report Bug Copy Issue Report Variable name "x" doesn't conform to snake_case naming style ('([^\\W\\dA-Z][^\\WA-Z]2,\|_[^\\WA-Z]*\|__[^\\WA-Z\\d_][^\\WA-Z]+__)$' pattern) This check looks for invalid names for a range of different identifiers. You can set regular expressions to which the identifiers must conform if the defaults do not match your requirements. If your project includes a Pylint configuration file, the settings contained in that file take precedence. To find out more about Pylint, please refer to their site. Loading history...
96			n_err += 1
97			except Exception:
98			raise SearchError(
99			f"Failed {self.what.key} [{self.what.search_class}] on compound {compound}",
100			compound=compound,
101			search_key=self.what.key,
102			search_class=self.what.search_class,
103			)
104			compounds_run.add(compound)
105			logger.debug(f"Found {len(x)} {self.what.search_name()} annotations for {compound}")
106			n_annot += len(x)
107			n_proc += 1
108			# logging, caching, and such:
109			on_nth = cache.at % SETTINGS.save_every == SETTINGS.save_every - 1
110			is_last = cache.at == len(inchikeys) - 1
111			if on_nth or is_last:
112			logger.log(
113			"NOTICE" if is_last else "INFO",
114			f"Found {len(annotes)} {self.what.search_name()} annotations"
115			+ f" for {cache.at} of {len(inchikeys)} compounds",
116			)
117			self._save(annotes, done=is_last)
118			cache.save(*compounds_run) # CRITICAL -- do this AFTER saving
119			# done!
120			i1, t1 = cache.at, time.monotonic()
			0 ignored issues – show Coding Style Naming introduced 2021-10-28 02:14 UTC by Report Bug Copy Issue Report Variable name "i1" doesn't conform to snake_case naming style ('([^\\W\\dA-Z][^\\WA-Z]2,\|_[^\\WA-Z]\|__[^\\WA-Z\\d_][^\\WA-Z]+__)$' pattern) This check looks for invalid names for a range of different identifiers. You can set regular expressions to which the identifiers must conform if the defaults do not match your requirements. If your project includes a Pylint configuration file, the settings contained in that file take precedence. To find out more about Pylint, please refer to their site. Loading history... Coding Style Naming introduced 2021-10-28 02:14 UTC by Report Bug Copy Issue Report Variable name "t1" doesn't conform to snake_case naming style ('([^\\W\\dA-Z][^\\WA-Z]2,\|_[^\\WA-Z]\|__[^\\WA-Z\\d_][^\\WA-Z]+__)$' pattern) This check looks for invalid names for a range of different identifiers. You can set regular expressions to which the identifiers must conform if the defaults do not match your requirements. If your project includes a Pylint configuration file, the settings contained in that file take precedence. To find out more about Pylint, please refer to their site. Loading history...
121			assert i1 == len(inchikeys)
122			cache.kill()
123			logger.success(f"Wrote {self.what.key} to {self.to}")
124			return SearchReturnInfo(
125			n_kept=n0, n_processed=n_proc, n_errored=n_err, time_taken=timedelta(seconds=t1 - t0)
126			)
127
128			@property
129			def is_partial(self) -> bool:
			0 ignored issues – show introduced 2021-10-28 02:14 UTC by Report Bug Copy Issue Report Missing function or method docstring Loading history...
130			return self.to.exists() and not self.is_complete
131
132			@property
133			def is_complete(self) -> bool:
			0 ignored issues – show introduced 2021-10-28 02:14 UTC by Report Bug Copy Issue Report Missing function or method docstring Loading history...
134			done = self.to in Checksums().load_dirsum_of_file(self.to)
135			if done and not self.to.exists():
136			raise IllegalStateError(f"{self.to} marked complete but does not exist")
137			return done
138
139			def _save(self, hits: Sequence[AbstractHit], *, done: bool):
140			df = HitDf.from_hits(hits)
			0 ignored issues – show Coding Style Naming introduced 2021-03-21 02:08 UTC by Report Bug Copy Issue Report Variable name "df" doesn't conform to snake_case naming style ('([^\\W\\dA-Z][^\\WA-Z]2,\|_[^\\WA-Z]*\|__[^\\WA-Z\\d_][^\\WA-Z]+__)$' pattern) This check looks for invalid names for a range of different identifiers. You can set regular expressions to which the identifiers must conform if the defaults do not match your requirements. If your project includes a Pylint configuration file, the settings contained in that file take precedence. To find out more about Pylint, please refer to their site. Loading history...
141			# keep all of the original extra columns from the input
142			# e.g. if the user had 'inchi' or 'smiles' or 'pretty_name'
143			for extra_col in [c for c in self.input_df.columns if c != "inchikey"]:
144			extra_mp = self.input_df.set_index("inchikey")[extra_col].to_dict()
145			df[extra_col] = df["origin_inchikey"].map(extra_mp.get)
146			# write the file
147			df: HitDf = HitDf.of(df)
			0 ignored issues – show Coding Style Naming introduced 2021-10-12 03:37 UTC by Report Bug Copy Issue Report Variable name "df" doesn't conform to snake_case naming style ('([^\\W\\dA-Z][^\\WA-Z]2,\|_[^\\WA-Z]*\|__[^\\WA-Z\\d_][^\\WA-Z]+__)$' pattern) This check looks for invalid names for a range of different identifiers. You can set regular expressions to which the identifiers must conform if the defaults do not match your requirements. If your project includes a Pylint configuration file, the settings contained in that file take precedence. To find out more about Pylint, please refer to their site. Loading history...
148			params = self.what.get_params()
149			df = df.set_attrs(**params, key=self.what.key)
			0 ignored issues – show Coding Style Naming introduced 2021-10-28 02:14 UTC by Report Bug Copy Issue Report Variable name "df" doesn't conform to snake_case naming style ('([^\\W\\dA-Z][^\\WA-Z]2,\|_[^\\WA-Z]*\|__[^\\WA-Z\\d_][^\\WA-Z]+__)$' pattern) This check looks for invalid names for a range of different identifiers. You can set regular expressions to which the identifiers must conform if the defaults do not match your requirements. If your project includes a Pylint configuration file, the settings contained in that file take precedence. To find out more about Pylint, please refer to their site. Loading history...
150			df.write_file(self.to, mkdirs=True, attrs=True, dir_hash=done)
151			logger.debug(f"Saved {len(df)} rows to {self.to}")
152
153
154			__all__ = ["Searcher", "InputCompoundsDf", "SearchReturnInfo"]
155

dmyersturnbull / mandos

Push — main ( 65730f...fad324 )

one() C

Complexity

Size

Duplication

Importance

How to fix Long Method

Long Method

Duplication Side-by-Side

Filter issues like