richlewis42 / scikit-chem

skchem/data/converters/base.py

#! /usr/bin/env python
#
# Copyright (C) 2016 Rich Lewis <rl403@cam.ac.uk>
# License: 3-clause BSD

"""
# skchem.data.converters.base

Defines the base converter class.
"""

import warnings
import logging
import os
from collections import namedtuple

import numpy as np

The import numpy could not be resolved.

import pandas as pd

The import pandas could not be resolved.

import h5py

The import h5py could not be resolved.

from fuel.datasets import H5PYDataset

The import fuel.datasets could not be resolved.

from ... import forcefields
from ... import filters
from ... import features
from ... import standardizers
from ... import pipeline

logger = logging.getLogger(__name__)

The name logger does not conform to the constant naming conventions ((([A-Z_][A-Z0-9_]*)|(__.*__))$).

def default_pipeline():
    """ Return a default pipeline to be used for general datasets. """
    return pipeline.Pipeline([
        standardizers.ChemAxonStandardizer(keep_failed=True, warn_on_fail=False),
        forcefields.UFF(add_hs=True, warn_on_fail=False),
        filters.OrganicFilter(),
        filters.AtomNumberFilter(above=5, below=100, include_hydrogens=True),
        filters.MassFilter(below=1000)
    ])

DEFAULT_PYTABLES_KW = {
    'complib': 'bzip2',
    'complevel': 9
}

def contiguous_order(to_order, splits):
    """ Determine a contiguous order from non-overlapping splits, and put data in that order.

    Args:
        to_order (iterable<pd.Series, pd.DataFrame, pd.Panel>):
            The pandas objects to put in contiguous order.
        splits (iterable<pd.Series>):
            The non-overlapping splits, as boolean masks.

    Returns:
        iterable<pd.Series, pd.DataFrame, pd.Panel>: The data in contiguous order.
    """

    member = pd.Series(0, index=splits[0].index)
    for i, split in enumerate(splits):
        member[split] = i
    idx = member.sort_values().index
    return (order.reindex(idx) for order in to_order)

Feature = namedtuple('Feature', ['fper', 'key', 'axis_names'])


def default_features():

This function should have a docstring.

    return (
        Feature(fper=features.MorganFeaturizer(),
                key='X_morg',
                axis_names=['batch', 'features']),
        Feature(fper=features.PhysicochemicalFeaturizer(),
                key='X_pc',
                axis_names=['batch', 'features']),
        Feature(fper=features.AtomFeaturizer(max_atoms=100),

The keyword max_atoms does not seem to exist for the constructor call.

                key='A',
                axis_names=['batch', 'atom_idx', 'features']),
        Feature(fper=features.GraphDistanceTransformer(max_atoms=100),

The keyword max_atoms does not seem to exist for the constructor call.

                key='G',
                axis_names=['batch', 'atom_idx', 'atom_idx']),
        Feature(fper=features.SpacialDistanceTransformer(max_atoms=100),

The keyword max_atoms does not seem to exist for the constructor call.

                key='G_d',
                axis_names=['batch', 'atom_idx', 'atom_idx']),
        Feature(fper=features.ChemAxonFeaturizer(features='all'),
                key='X_cx',
                axis_names=['batch', 'features']),
        Feature(fper=features.ChemAxonAtomFeaturizer(features='all', max_atoms=100),

The keyword max_atoms does not seem to exist for the constructor call.

                key='A_cx',
                axis_names=['batch', 'atom_idx', 'features'])
    )


class Split(object):

This class should have a docstring.

    def __init__(self, mask, name, converter):
        self.mask = mask
        self.name = name
        self.converter = converter

    @property
    def contiguous(self):

This method should have a docstring.

        diff = np.ediff1d(self.mask.astype(int))
        if self.mask.iloc[0] != 0:
            diff[0] = 1
        if self.mask.iloc[-1] != 0:
            diff[-1] = -1
        return sum(diff == -1) == 1 or sum(diff == 1) == 1

    @property
    def indices(self):

This method should have a docstring.

        return np.nonzero(self.mask)[0]

    def save(self):

This method should have a docstring.

        self.converter.data_file[self.name + '_indices'] = self.indices
        with warnings.catch_warnings():
            warnings.simplefilter('ignore')
            self.mask.to_hdf(self.converter.data_file.filename, '/indices/' + self.name)

    @property
    def ref(self):

This method should have a docstring.

        return self.converter.data_file[self.name + '_indices'].ref

    def to_dict(self):

This method should have a docstring.

        idx = self.indices
        if self.contiguous:
            low, high = min(idx), max(idx)
            return {source: (low, high) for source in self.converter.source_names}
        else:
            return {source: (-1, -1, self.ref) for source in self.converter.source_names}


class Converter(object):
    """ Create a fuel dataset from molecules and targets. """

    def __init__(self, directory, output_directory, output_filename='default.h5'):

The argument directory seems to be unused.

The argument output_filename seems to be unused.

The argument output_directory seems to be unused.

        raise NotImplemented

NotImplemented raised - should raise NotImplementedError

    def run(self, ms, y, output_path, splits=None, features=None, pytables_kws=DEFAULT_PYTABLES_KW):

The default value DEFAULT_PYTABLES_KW (__builtin__.dict) might cause unintended side-effects.

The name ms does not conform to the argument naming conventions ([a-z_][a-z0-9_]{2,30}$).

The name y does not conform to the argument naming conventions ([a-z_][a-z0-9_]{2,30}$).

Too many arguments (7/5)

features is re-defining a name which is already available in the outer-scope (previously defined on line 24).

        """
           Args:
        ms (pd.Series):
            The molecules of the dataset.
        ys (pd.Series or pd.DataFrame):
            The target labels of the dataset.
        output_path (str):
            The path to which the dataset should be saved.
        features (list[Feature]):
            The features to calculate. Defaults are used if `None`.
        splits (iterable<(name, split)>):
            An iterable of name, split tuples. Splits are provided as boolean arrays of the whole data.

This line is too long as per the coding-style (103/100).

        """

        self.output_path = output_path

The attribute output_path was defined outside __init__.

        self.pytables_kws = pytables_kws

The attribute pytables_kws was defined outside __init__.

        self.features = features if features is not None else default_features()

The attribute features was defined outside __init__.

        self.feature_names = [feat.key for feat in self.features]

The attribute feature_names was defined outside __init__.

        self.task_names = ['y']

The attribute task_names was defined outside __init__.

        self.splits = [Split(split, name, self) for name, split in splits]

The attribute splits was defined outside __init__.

        self.create_file(output_path)

        self.save_splits()
        self.save_molecules(ms)
        self.save_targets(y)
        self.save_features(ms)

    @property
    def source_names(self):

This method should have a docstring.

        return self.feature_names + self.task_names

    @property
    def split_names(self):

This method should have a docstring.

        return self.splits

    def create_file(self, path):

This method should have a docstring.

        logger.info('Creating h5 file at %s...', self.output_path)
        self.data_file = h5py.File(path, 'w')

The attribute data_file was defined outside __init__.

        return self.data_file

    def save_molecules(self, mols):

        """ Save the molecules to the data file. """

        logger.info('Writing molecules to file...')
        logger.debug('Writing %s molecules to %s', len(mols), self.data_file.filename)
        with warnings.catch_warnings():
            warnings.simplefilter('ignore')
            mols.to_hdf(self.data_file.filename, 'structure', **self.pytables_kws)
            mols.apply(lambda m: m.to_smiles().encode('utf-8')).to_hdf(self.data_file.filename, 'smiles')

This line is too long as per the coding-style (105/100).

    def save_frame(self, data, name, prefix='targets'):

        """ Save the a frame to the data file. """

        logger.info('Writing %s', name)
        logger.debug('Writing data of shape %s to %s', data.shape, self.data_file.filename)

        with warnings.catch_warnings():
            warnings.simplefilter('ignore')
            if len(data.shape) > 2:
                data = data.transpose(2, 1, 0)  # panel serializes backwards for some reason...
            data.to_hdf(self.data_file.filename,
                        key='/{prefix}/{name}'.format(prefix=prefix, name=name),
                        **self.pytables_kws)

        if isinstance(data, pd.Series):
            self.data_file[name] = h5py.SoftLink('/{prefix}/{name}/values'.format(prefix=prefix, name=name))

This line is too long as per the coding-style (108/100).

            self.data_file[name].dims[0].label = data.index.name

        elif isinstance(data, pd.DataFrame):
            self.data_file[name] = h5py.SoftLink('/{prefix}/{name}/block0_values'.format(prefix=prefix, name=name))

This line is too long as per the coding-style (115/100).

            self.data_file[name].dims[0].label = data.index.name
            self.data_file[name].dims[1].label = data.columns.name

        elif isinstance(data, pd.Panel):
            self.data_file[name] = h5py.SoftLink('/{prefix}/{name}/block0_values'.format(prefix=prefix, name=name))

This line is too long as per the coding-style (115/100).

            self.data_file[name].dims[0].label = data.minor_axis.name # as panel serializes backwards

This line is too long as per the coding-style (101/100).

            self.data_file[name].dims[1].label = data.major_axis.name
            self.data_file[name].dims[2].label = data.items.name

    def save_targets(self, y):

The name y does not conform to the argument naming conventions ([a-z_][a-z0-9_]{2,30}$).

This method should have a docstring.

        self.save_frame(y, name='y', prefix='targets')

    def save_features(self, ms):

The name ms does not conform to the argument naming conventions ([a-z_][a-z0-9_]{2,30}$).

        """ Save all features for the dataset. """
        logger.debug('Saving features')
        for feat in self.features:
            self._save_feature(ms, feat)

    def _save_feature(self, ms, feat):

The name ms does not conform to the argument naming conventions ([a-z_][a-z0-9_]{2,30}$).

        """ Calculate and save a feature to the data file. """
        logger.info('Calculating %s', feat.key)

        fps = feat.fper.transform(ms)
        self.save_frame(fps, name=feat.key, prefix='feats')

    def save_splits(self):

        """ Save the splits to the data file. """

        logger.info('Producing dataset splits...')
        for split in self.splits:
            split.save()
        split_dict = {split.name: split.to_dict() for split in self.splits}
        splits = H5PYDataset.create_split_array(split_dict)
        logger.debug('split: %s', splits)
        logger.info('Saving splits...')
        with warnings.catch_warnings():
            warnings.simplefilter('ignore')
            self.data_file.attrs['split'] = splits

    @classmethod
    def convert(cls, **kwargs):

This method should have a docstring.

        kwargs.setdefault('directory', os.getcwd())
        kwargs.setdefault('output_directory', os.getcwd())

        return cls(**kwargs).output_path,

    @classmethod
    def fill_subparser(cls, subparser):

This method should have a docstring.

The argument subparser seems to be unused.

        return cls.convert