OdinAPI.extract_from_document() - Code Metrics - Inspection of "Version 3.0.0" - myedibleenso/py-processors - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — master ( 352884...922d0d )

by Gus

created 2017-03-26 10:12 UTC

OdinAPI.extract_from_document() B

↳ Parent: OdinAPI

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes	2
Bugs	0	Features	0

Metric	Value
cc	2
dl	0
loc	24
rs	8.9713
c	2
b	0
f	0

#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
#from pkg_resources import resource_filename
from six.moves.urllib.request import urlretrieve
from .utils import *
from .annotators import *
from .sentiment import SentimentAnalysisAPI
from .serialization import JSONSerializer
import os
import shlex
import subprocess as sp
import requests
import re
import time
import sys
import logging
import warnings


class ProcessorsAPI(object):

    """
    Manages a connection with the processors-server jar and provides an interface to the API.

    Parameters
    ----------
    port : int
        The port the server is running on or should be started on.  Default is 8886.
    hostname : str
        The host name to use for the server.  Default is "localhost".
    timeout : int
        The number of seconds to wait for the server to initialize.  Default is 120.
    jvm_mem : str
        The maximum amount of memory to allocate to the JVM for the server.  Default is "-Xmx3G".
    jar_path : str
        The path to the processors-server jar.  Default is the jar installed with the package.
    kee_alive : bool
        Whether or not to keep the server running when ProcessorsAPI instance goes out of scope.  Default is false (server is shut down).
    log_file: str
        The path for the log file.  Default is .py-processors.log in the user's home directory.

    Methods
    -------
    annotate(text)
        Produces a Document from the provided `text` using the default processor.
    fastnlp.annotate(text)
        Produces a Document from the provided `text` using FastNLPProcessor.
    bionlp.annotate(text)
        Produces a Document from the provided `text` using BioNLPProcessor.
    annotate_from_sentences(sentences)
        Produces a Document from `sentences` (a list of text split into sentences). Uses the default processor.
    fastnlp.annotate_from_sentences(sentences)
        Produces a Document from `sentences` (a list of text split into sentences). Uses FastNLPProcessor.
    bionlp.annotate_from_sentences(sentences)
        Produces a Document from `sentences` (a list of text split into sentences). Uses BioNLPProcessor.
    corenlp.sentiment.score_sentence(sentence)
        Produces a sentiment score for the provided `sentence` (an instance of Sentence).
    corenlp.sentiment.score_document(doc)
        Produces sentiment scores for the provided `doc` (an instance of Document).  One score is produced for each sentence.
    corenlp.sentiment.score_segmented_text(sentences)
        Produces sentiment scores for the provided `sentences` (a list of text segmented into sentences).  One score is produced for item in `sentences`.
    odin.extract_from_text(text, rules)
        Produces a list of Mentions for matches of the provided `rules` on the `text`.  `rules` can be a string of Odin rules, or a url ending in `.yml` or `.yaml`.
    odin.extract_from_document(doc, rules)
        Produces a list of Mentions for matches of the provided `rules` on the `doc` (an instance of Document).  `rules` can be a string of Odin rules, or a url ending in .yml or yaml.
    start_server(jar_path, **kwargs)
        Starts the server using the provided `jar_path`.  Optionally takes hostname, port, jvm_mem, and timeout.
    stop_server()
        Attempts to stop the server running at self.address.
    """

    PROC_VAR = 'PROCESSORS_SERVER'
    TIMEOUT = 120
    # save to lib loc
    DEFAULT_JAR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "processors-server.jar")
    PORT = 8886
    JVM_MEM = "-Xmx3G"
    HOST = "localhost"
    LOG = full_path(os.path.join("~", "py-processors.log"))
    #print(resource_filename(__name__, "processors-server.jar"))

    def __init__(self, **kwargs):

        self.hostname = kwargs.get("hostname", ProcessorsAPI.HOST)
        self.port = kwargs.get("port", ProcessorsAPI.PORT)
        self.make_address(self.hostname, self.port)
        self.timeout = kwargs.get("timeout", ProcessorsAPI.TIMEOUT)
        self.jvm_mem = kwargs.get("jvm_mem", ProcessorsAPI.JVM_MEM)
        self._start_command = "java {mem} -cp {jp} NLPServer --port {port} --host {host}" # mem, jar path, port, host
        # whether or not to stop the server when the object is destroyed
        self.keep_alive = kwargs.get("keep_alive", False)
        # how long to wait between requests
        self.wait_time = 2
        # processors
        self.default = Processor(self.address)
        self.fastnlp = FastNLPProcessor(self.address)
        self.bionlp = BioNLPProcessor(self.address)
        # sentiment
        self.sentiment = SentimentAnalysisAPI(self.address)
        # odin
        self.odin = OdinAPI(self.address)
        # use the os module's devnull for compatibility with python 2.7
        #self.DEVNULL = open(os.devnull, 'wb')
        self.logger = logging.getLogger(__name__)
        self.log_file = self._prepare_log_file(kwargs.get("log_file", ProcessorsAPI.LOG))
        self.jar_path = kwargs.get("jar_path", ProcessorsAPI.DEFAULT_JAR)
        # attempt to establish connection with server
        self.establish_connection()

    def _check_server_version(self):
        """
        Checks server version to see if it meets the recommendations
        """
        # avoid circular imports by delaying this import
        from .__init__ import __ps_rec__
        try:
            service_address = "{}/version".format(self.address)
            server_version = post_json(service_address, None)["version"]
            if float(__ps_rec__) != float(server_version):
                warnings.warn("Recommended server version is {}, but server version is {}".format(__ps_rec__, server_version))
            else:
                self.logger.info("Server version meets recommendations (v{})".format(__ps_rec__))
        except:
            warnings.warn("Unable to determine server version.  Recommended version is {}".format(__ps_rec__))


    def _prepare_log_file(self, lf):
        """
        Configure logger and return file path for logging
        """
        # log_file
        log_file = ProcessorsAPI.LOG if not lf else os.path.expanduser(lf)
        # configure logger
        self.logger.setLevel(logging.DEBUG)
        # create console handler and set level to info
        handler = logging.StreamHandler()
        handler.setLevel(logging.INFO)
        formatter = logging.Formatter("%(levelname)s - %(message)s")
        handler.setFormatter(formatter)
        self.logger.addHandler(handler)
        # create debug file handler and set level to debug
        handler = logging.FileHandler(log_file, "w")
        handler.setLevel(logging.DEBUG)
        formatter = logging.Formatter("%(levelname)s - %(message)s")
        handler.setFormatter(formatter)
        self.logger.addHandler(handler)
        return log_file

    def annotate(self, text):
        """
        Uses default processor (CoreNLP) to annotate text.  Included for backwards compatibility.
        """
        return self.default.annotate(text)

    def annotate_from_sentences(self, sentences):
        """
        Uses default processor (CoreNLP) to annotate a list of segmented sentences.
        """
        return self.default.annotate_from_sentences(sentences)

    def is_running(self):
        return True if self.annotate("Blah") else False

    def establish_connection(self):
        """
        Attempt to connect to a server (assumes server is running)
        """
        if self.is_running():
            self.logger.info("Connection with server established!")
            self._check_server_version()
        else:
            try:
                # resolve jar path if server is not already running
                self._resolve_jar_path(self.jar_path)
                # Attempt to start the server
                self._start_server()
            except Exception as e:
                self.logger.warn("Unable to start server. Please start the server manually with .start_server(jar_path=\"path/to/processors-server.jar\")")
                self.logger.warn("\n{}".format(e))

    def _resolve_jar_path(self, jar_path=None):
        """
        Attempts to preferentially set value of self.jar_path
        """
        # Preference 1: if a .jar is given, check to see if the path is valid
        if jar_path:
            jp = full_path(jar_path)
            # check if path is valid
            if os.path.exists(jp):
                self.jar_path = jp
        # Preference 2: if a PROCESSORS_SERVER environment variable is defined, check its validity
        elif ProcessorsAPI.PROC_VAR in os.environ:
            self.logger.info("Using path given via $PROCESSORS_SERVER")
            jp = full_path(os.environ[ProcessorsAPI.PROC_VAR])
            # check if path is valid
            if os.path.exists(jp):
                self.jar_path = jp
            else:
                self.jar_path = None
                self.logger.warn("WARNING: {0} path is invalid.  \nPlease verify this entry in your environment:\n\texport {0}=/path/to/processors-server.jar".format(ProcessorsAPI.PROC_VAR))
        # Preference 3: attempt to use the processors-sever.jar (download if not found)
        # check if jar exists
        if not self.jar_path or not os.path.exists(self.jar_path):
            self.logger.info("No jar found.  Downloading to {} ...".format(ProcessorsAPI.DEFAULT_JAR))
            ProcessorsAPI._download_jar()
            self.jar_path = ProcessorsAPI.DEFAULT_JAR

    def start_server(self, jar_path, **kwargs):
        """
        Starts processors-sever.jar
        """
        self.port = kwargs.get("port", self.port)
        self.hostname = kwargs.get("hostname", self.hostname)
        self.jvm_mem = kwargs.get("jvm_mem", self.jvm_mem)
        self.timeout = int(float(kwargs.get("timeout", self.jvm_mem))/2)
        jp = full_path(jar_path)
        if jp:
            self.jar_path = jp
            self._start_server()
        else:
            raise Exception("Please provide jar_path=\"path/to/processors-server.jar\"")

    def stop_server(self, port=None):
        """
        Sends a poison pill to the server and waits for shutdown response
        """
        port = port or self.port
        address = "http://{}:{}".format(self.hostname, port)
        shutdown_address = "{}/shutdown".format(address)
        # attempt shutdown
        try:
            response = requests.post(shutdown_address)
            if response:
                print(response.content.decode("utf-8"))
            return True
        # will fail if the server is already down
        except Exception as e:
            pass
        return False

    def _ensure_jar_path_exists(self):
        # check if jar exists
        if not os.path.exists(self.jar_path):
            raise Exception("jar not found at {}".format(self.jar_path))

    def _start_server(self, port=None):
        """
        "Private" method called by start_server()
        """

        # does the jar exist?
        self._ensure_jar_path_exists()

        if port:
            self.port = port
        # build the command
        cmd = self._start_command.format(mem=self.jvm_mem, jp=self.jar_path, port=self.port, host=self.hostname)
        self._process = sp.Popen(shlex.split(cmd),
                                 shell=False,
                                 stderr=open(self.log_file, 'wb'),
                                 stdout=open(self.log_file, 'wb'),
                                 universal_newlines=True)

        self.logger.info("Starting processors-server ({}) ...".format(cmd))
        print("\nWaiting for server...")

        progressbar_length = int(self.timeout/self.wait_time)
        for i in range(progressbar_length):
            try:
                success = self.annotate("blah")
                if success:
                    print("\n\nConnection with processors-server established ({})".format(self.address))
                    return True
                sys.stdout.write("\r[{:{}}]".format('='*i, progressbar_length))
                time.sleep(self.wait_time)
            except Exception as e:
                raise(e)

        # if the server still hasn't started, raise an Exception
        raise Exception("Couldn't connect to processors-server. Is the port in use?")

    def make_address(self, hostname, port):
        # update hostname
        self.hostname = hostname
        # update port
        self.port = port
        # update address
        self.address = "http://{}:{}".format(self.hostname, self.port)

    @staticmethod
    def _download_jar(jar_url="http://www.cs.arizona.edu/~hahnpowell/processors-server/current/processors-server.jar"):
        # download processors-server.jar
        ppjar = ProcessorsAPI.DEFAULT_JAR
        percent = 0
        def dlProgress(count, blockSize, totalSize):
            percent = int(count*blockSize*100/totalSize)
            sys.stdout.write("\r{}% complete".format(percent))
            sys.stdout.flush()

        print("Downloading {} from {} ...".format(ppjar, jar_url))
        urlretrieve(jar_url, ppjar, reporthook=dlProgress)
        print("\nDownload Complete! {}".format(ppjar))

    def __del__(self):
        """
        Stop server unless otherwise specified
        """
        if not self.keep_alive:
            try:
                self.stop_server()
                # close our file object
                #self.DEVNULL.close()
                print("Successfully shut down processors-server!")
            except Exception as e:
                self.logger.debug(e)
                print("Couldn't kill processors-server.  Was server started externally?")


class OdinAPI(object):
    """
    API for performing rule-based information extraction with Odin.

    Parameters
    ----------
    address : str
        The base address for the API (i.e., everything preceding `/api/..`)

    """

    validator = re.compile("^(https?|ftp):.+?\.?ya?ml$")

    def __init__(self, address):
        self._service = "{}/api/odin/extract".format(address)

    def _extract(self, json_data):
        try:
            mns_json = post_json(self._service, json_data)
            return JSONSerializer.mentions_from_JSON(mns_json)
        except Exception as e:
            print(e)
            return None

    @staticmethod
    def valid_rule_url(url):
        return True if OdinAPI.validator.match(url) else False

    def extract_from_text(self, text, rules):
        """
        Sends text to the server with rules for information extraction (IE).

        Parameters
        ----------
        text : str
            `rules` will be applied to this `text`.
        rules : str
            Either Odin rules provided as a `yaml` string, or a url pointing to a `yaml` file of rules.

        Returns
        -------
        [processors.odin.Mention] or None
            Rule matches produce a list of `processors.odin.Mention`.
        """
        if OdinAPI.valid_rule_url(rules):
            # this is actually a URL to a yaml file
            url = rules
            container = TextWithURL(text, url)
        else:
            container = TextWithRules(text, rules)
        return self._extract(container.to_JSON())

    def extract_from_document(self, doc, rules):
        """
        Sends a `processors.ds.Document` (`doc`) to the server with rules for information extraction (IE).

        Parameters
        ----------
        doc : processors.ds.Document
            `rules` will be applied to this `processors.ds.Document`.
        rules : str
            Either Odin rules provided as a `yaml` string, or a url pointing to a `yaml` file of rules.

        Returns
        -------
        [processors.odin.Mention] or None
            Rule matches produce a list of `processors.odin.Mention`.

        """
        if OdinAPI.valid_rule_url(rules):
            # this is actually a URL to a yaml file
            url = rules
            container = DocumentWithURL(doc, rules)
        else:
            container = DocumentWithRules(doc, rules)
        return self._extract(container.to_JSON())

#############################################
# Containers for Odin data
# transmitted to the server for processing
#############################################

class TextWithRules(object):

    def __init__(self, text, rules):
        self.text = text
        self.rules = rules

    def to_JSON_dict(self):
        jdict = dict()
        jdict["text"] = self.text
        jdict["rules"] = self.rules
        return jdict

    def to_JSON(self):
        return json.dumps(self.to_JSON_dict(), sort_keys=True, indent=4)

class TextWithURL(object):

    def __init__(self, text, url):
        self.text = text
        # TODO: throw exception if url is invalid
        self.url = url

    def to_JSON_dict(self):
        jdict = dict()
        jdict["text"] = self.text
        jdict["url"] = self.url
        return jdict

    def to_JSON(self):
        return json.dumps(self.to_JSON_dict(), sort_keys=True, indent=4)

class DocumentWithRules(object):

    def __init__(self, document, rules):
        # TODO: throw exception if isinstance(document, Document) is False
        self.document = document
        self.rules = rules

    def to_JSON_dict(self):
        jdict = dict()
        jdict["document"] = self.document.to_JSON_dict()
        jdict["rules"] = self.rules
        return jdict

    def to_JSON(self):
        return json.dumps(self.to_JSON_dict(), sort_keys=True, indent=4)

class DocumentWithURL(object):

    def __init__(self, document, url):
        # TODO: throw exception if isinstance(document, Document) is False
        self.document = document
        # TODO: throw exception if url is invalid
        self.url = url

    def to_JSON_dict(self):
        jdict = dict()
        jdict["document"] = self.document.to_JSON_dict()
        jdict["url"] = self.url
        return jdict

    def to_JSON(self):
        return json.dumps(self.to_JSON_dict(), sort_keys=True, indent=4)


1			#!/usr/bin/env python
2			# -- coding: utf-8 --
3			from __future__ import unicode_literals
4			#from pkg_resources import resource_filename
5			from six.moves.urllib.request import urlretrieve
6			from .utils import *
7			from .annotators import *
8			from .sentiment import SentimentAnalysisAPI
9			from .serialization import JSONSerializer
10			import os
11			import shlex
12			import subprocess as sp
13			import requests
14			import re
15			import time
16			import sys
17			import logging
18			import warnings
19
20
21			class ProcessorsAPI(object):
22
23			"""
24			Manages a connection with the processors-server jar and provides an interface to the API.
25
26			Parameters
27			----------
28			port : int
29			The port the server is running on or should be started on. Default is 8886.
30			hostname : str
31			The host name to use for the server. Default is "localhost".
32			timeout : int
33			The number of seconds to wait for the server to initialize. Default is 120.
34			jvm_mem : str
35			The maximum amount of memory to allocate to the JVM for the server. Default is "-Xmx3G".
36			jar_path : str
37			The path to the processors-server jar. Default is the jar installed with the package.
38			kee_alive : bool
39			Whether or not to keep the server running when ProcessorsAPI instance goes out of scope. Default is false (server is shut down).
40			log_file: str
41			The path for the log file. Default is .py-processors.log in the user's home directory.
42
43			Methods
44			-------
45			annotate(text)
46			Produces a Document from the provided `text` using the default processor.
47			fastnlp.annotate(text)
48			Produces a Document from the provided `text` using FastNLPProcessor.
49			bionlp.annotate(text)
50			Produces a Document from the provided `text` using BioNLPProcessor.
51			annotate_from_sentences(sentences)
52			Produces a Document from `sentences` (a list of text split into sentences). Uses the default processor.
53			fastnlp.annotate_from_sentences(sentences)
54			Produces a Document from `sentences` (a list of text split into sentences). Uses FastNLPProcessor.
55			bionlp.annotate_from_sentences(sentences)
56			Produces a Document from `sentences` (a list of text split into sentences). Uses BioNLPProcessor.
57			corenlp.sentiment.score_sentence(sentence)
58			Produces a sentiment score for the provided `sentence` (an instance of Sentence).
59			corenlp.sentiment.score_document(doc)
60			Produces sentiment scores for the provided `doc` (an instance of Document). One score is produced for each sentence.
61			corenlp.sentiment.score_segmented_text(sentences)
62			Produces sentiment scores for the provided `sentences` (a list of text segmented into sentences). One score is produced for item in `sentences`.
63			odin.extract_from_text(text, rules)
64			Produces a list of Mentions for matches of the provided `rules` on the `text`. `rules` can be a string of Odin rules, or a url ending in `.yml` or `.yaml`.
65			odin.extract_from_document(doc, rules)
66			Produces a list of Mentions for matches of the provided `rules` on the `doc` (an instance of Document). `rules` can be a string of Odin rules, or a url ending in .yml or yaml.
67			start_server(jar_path, **kwargs)
68			Starts the server using the provided `jar_path`. Optionally takes hostname, port, jvm_mem, and timeout.
69			stop_server()
70			Attempts to stop the server running at self.address.
71			"""
72
73			PROC_VAR = 'PROCESSORS_SERVER'
74			TIMEOUT = 120
75			# save to lib loc
76			DEFAULT_JAR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "processors-server.jar")
77			PORT = 8886
78			JVM_MEM = "-Xmx3G"
79			HOST = "localhost"
80			LOG = full_path(os.path.join("~", "py-processors.log"))
81			#print(resource_filename(__name__, "processors-server.jar"))
82
83			def __init__(self, **kwargs):
84
85			self.hostname = kwargs.get("hostname", ProcessorsAPI.HOST)
86			self.port = kwargs.get("port", ProcessorsAPI.PORT)
87			self.make_address(self.hostname, self.port)
88			self.timeout = kwargs.get("timeout", ProcessorsAPI.TIMEOUT)
89			self.jvm_mem = kwargs.get("jvm_mem", ProcessorsAPI.JVM_MEM)
90			self._start_command = "java {mem} -cp {jp} NLPServer --port {port} --host {host}" # mem, jar path, port, host
91			# whether or not to stop the server when the object is destroyed
92			self.keep_alive = kwargs.get("keep_alive", False)
93			# how long to wait between requests
94			self.wait_time = 2
95			# processors
96			self.default = Processor(self.address)
97			self.fastnlp = FastNLPProcessor(self.address)
98			self.bionlp = BioNLPProcessor(self.address)
99			# sentiment
100			self.sentiment = SentimentAnalysisAPI(self.address)
101			# odin
102			self.odin = OdinAPI(self.address)
103			# use the os module's devnull for compatibility with python 2.7
104			#self.DEVNULL = open(os.devnull, 'wb')
105			self.logger = logging.getLogger(__name__)
106			self.log_file = self._prepare_log_file(kwargs.get("log_file", ProcessorsAPI.LOG))
107			self.jar_path = kwargs.get("jar_path", ProcessorsAPI.DEFAULT_JAR)
108			# attempt to establish connection with server
109			self.establish_connection()
110
111			def _check_server_version(self):
112			"""
113			Checks server version to see if it meets the recommendations
114			"""
115			# avoid circular imports by delaying this import
116			from .__init__ import __ps_rec__
117			try:
118			service_address = "{}/version".format(self.address)
119			server_version = post_json(service_address, None)["version"]
120			if float(__ps_rec__) != float(server_version):
121			warnings.warn("Recommended server version is {}, but server version is {}".format(__ps_rec__, server_version))
122			else:
123			self.logger.info("Server version meets recommendations (v{})".format(__ps_rec__))
124			except:
125			warnings.warn("Unable to determine server version. Recommended version is {}".format(__ps_rec__))
126
127
128			def _prepare_log_file(self, lf):
129			"""
130			Configure logger and return file path for logging
131			"""
132			# log_file
133			log_file = ProcessorsAPI.LOG if not lf else os.path.expanduser(lf)
134			# configure logger
135			self.logger.setLevel(logging.DEBUG)
136			# create console handler and set level to info
137			handler = logging.StreamHandler()
138			handler.setLevel(logging.INFO)
139			formatter = logging.Formatter("%(levelname)s - %(message)s")
140			handler.setFormatter(formatter)
141			self.logger.addHandler(handler)
142			# create debug file handler and set level to debug
143			handler = logging.FileHandler(log_file, "w")
144			handler.setLevel(logging.DEBUG)
145			formatter = logging.Formatter("%(levelname)s - %(message)s")
146			handler.setFormatter(formatter)
147			self.logger.addHandler(handler)
148			return log_file
149
150			def annotate(self, text):
151			"""
152			Uses default processor (CoreNLP) to annotate text. Included for backwards compatibility.
153			"""
154			return self.default.annotate(text)
155
156			def annotate_from_sentences(self, sentences):
157			"""
158			Uses default processor (CoreNLP) to annotate a list of segmented sentences.
159			"""
160			return self.default.annotate_from_sentences(sentences)
161
162			def is_running(self):
163			return True if self.annotate("Blah") else False
164
165			def establish_connection(self):
166			"""
167			Attempt to connect to a server (assumes server is running)
168			"""
169			if self.is_running():
170			self.logger.info("Connection with server established!")
171			self._check_server_version()
172			else:
173			try:
174			# resolve jar path if server is not already running
175			self._resolve_jar_path(self.jar_path)
176			# Attempt to start the server
177			self._start_server()
178			except Exception as e:
179			self.logger.warn("Unable to start server. Please start the server manually with .start_server(jar_path=\"path/to/processors-server.jar\")")
180			self.logger.warn("\n{}".format(e))
181
182			def _resolve_jar_path(self, jar_path=None):
183			"""
184			Attempts to preferentially set value of self.jar_path
185			"""
186			# Preference 1: if a .jar is given, check to see if the path is valid
187			if jar_path:
188			jp = full_path(jar_path)
189			# check if path is valid
190			if os.path.exists(jp):
191			self.jar_path = jp
192			# Preference 2: if a PROCESSORS_SERVER environment variable is defined, check its validity
193			elif ProcessorsAPI.PROC_VAR in os.environ:
194			self.logger.info("Using path given via $PROCESSORS_SERVER")
195			jp = full_path(os.environ[ProcessorsAPI.PROC_VAR])
196			# check if path is valid
197			if os.path.exists(jp):
198			self.jar_path = jp
199			else:
200			self.jar_path = None
201			self.logger.warn("WARNING: {0} path is invalid. \nPlease verify this entry in your environment:\n\texport {0}=/path/to/processors-server.jar".format(ProcessorsAPI.PROC_VAR))
202			# Preference 3: attempt to use the processors-sever.jar (download if not found)
203			# check if jar exists
204			if not self.jar_path or not os.path.exists(self.jar_path):
205			self.logger.info("No jar found. Downloading to {} ...".format(ProcessorsAPI.DEFAULT_JAR))
206			ProcessorsAPI._download_jar()
207			self.jar_path = ProcessorsAPI.DEFAULT_JAR
208
209			def start_server(self, jar_path, **kwargs):
210			"""
211			Starts processors-sever.jar
212			"""
213			self.port = kwargs.get("port", self.port)
214			self.hostname = kwargs.get("hostname", self.hostname)
215			self.jvm_mem = kwargs.get("jvm_mem", self.jvm_mem)
216			self.timeout = int(float(kwargs.get("timeout", self.jvm_mem))/2)
217			jp = full_path(jar_path)
218			if jp:
219			self.jar_path = jp
220			self._start_server()
221			else:
222			raise Exception("Please provide jar_path=\"path/to/processors-server.jar\"")
223
224			def stop_server(self, port=None):
225			"""
226			Sends a poison pill to the server and waits for shutdown response
227			"""
228			port = port or self.port
229			address = "http://{}:{}".format(self.hostname, port)
230			shutdown_address = "{}/shutdown".format(address)
231			# attempt shutdown
232			try:
233			response = requests.post(shutdown_address)
234			if response:
235			print(response.content.decode("utf-8"))
236			return True
237			# will fail if the server is already down
238			except Exception as e:
239			pass
240			return False
241
242			def _ensure_jar_path_exists(self):
243			# check if jar exists
244			if not os.path.exists(self.jar_path):
245			raise Exception("jar not found at {}".format(self.jar_path))
246
247			def _start_server(self, port=None):
248			"""
249			"Private" method called by start_server()
250			"""
251
252			# does the jar exist?
253			self._ensure_jar_path_exists()
254
255			if port:
256			self.port = port
257			# build the command
258			cmd = self._start_command.format(mem=self.jvm_mem, jp=self.jar_path, port=self.port, host=self.hostname)
259			self._process = sp.Popen(shlex.split(cmd),
260			shell=False,
261			stderr=open(self.log_file, 'wb'),
262			stdout=open(self.log_file, 'wb'),
263			universal_newlines=True)
264
265			self.logger.info("Starting processors-server ({}) ...".format(cmd))
266			print("\nWaiting for server...")
267
268			progressbar_length = int(self.timeout/self.wait_time)
269			for i in range(progressbar_length):
270			try:
271			success = self.annotate("blah")
272			if success:
273			print("\n\nConnection with processors-server established ({})".format(self.address))
274			return True
275			sys.stdout.write("\r[{:{}}]".format('='*i, progressbar_length))
276			time.sleep(self.wait_time)
277			except Exception as e:
278			raise(e)
279
280			# if the server still hasn't started, raise an Exception
281			raise Exception("Couldn't connect to processors-server. Is the port in use?")
282
283			def make_address(self, hostname, port):
284			# update hostname
285			self.hostname = hostname
286			# update port
287			self.port = port
288			# update address
289			self.address = "http://{}:{}".format(self.hostname, self.port)
290
291			@staticmethod
292			def _download_jar(jar_url="http://www.cs.arizona.edu/~hahnpowell/processors-server/current/processors-server.jar"):
293			# download processors-server.jar
294			ppjar = ProcessorsAPI.DEFAULT_JAR
295			percent = 0
296			def dlProgress(count, blockSize, totalSize):
297			percent = int(countblockSize100/totalSize)
298			sys.stdout.write("\r{}% complete".format(percent))
299			sys.stdout.flush()
300
301			print("Downloading {} from {} ...".format(ppjar, jar_url))
302			urlretrieve(jar_url, ppjar, reporthook=dlProgress)
303			print("\nDownload Complete! {}".format(ppjar))
304
305			def __del__(self):
306			"""
307			Stop server unless otherwise specified
308			"""
309			if not self.keep_alive:
310			try:
311			self.stop_server()
312			# close our file object
313			#self.DEVNULL.close()
314			print("Successfully shut down processors-server!")
315			except Exception as e:
316			self.logger.debug(e)
317			print("Couldn't kill processors-server. Was server started externally?")
318
319
320			class OdinAPI(object):
321			"""
322			API for performing rule-based information extraction with Odin.
323
324			Parameters
325			----------
326			address : str
327			The base address for the API (i.e., everything preceding `/api/..`)
328
329			"""
330
331			validator = re.compile("^(https?\|ftp):.+?\.?ya?ml$")
332
333			def __init__(self, address):
334			self._service = "{}/api/odin/extract".format(address)
335
336			def _extract(self, json_data):
337			try:
338			mns_json = post_json(self._service, json_data)
339			return JSONSerializer.mentions_from_JSON(mns_json)
340			except Exception as e:
341			print(e)
342			return None
343
344			@staticmethod
345			def valid_rule_url(url):
346			return True if OdinAPI.validator.match(url) else False
347
348			def extract_from_text(self, text, rules):
349			"""
350			Sends text to the server with rules for information extraction (IE).
351
352			Parameters
353			----------
354			text : str
355			`rules` will be applied to this `text`.
356			rules : str
357			Either Odin rules provided as a `yaml` string, or a url pointing to a `yaml` file of rules.
358
359			Returns
360			-------
361			[processors.odin.Mention] or None
362			Rule matches produce a list of `processors.odin.Mention`.
363			"""
364			if OdinAPI.valid_rule_url(rules):
365			# this is actually a URL to a yaml file
366			url = rules
367			container = TextWithURL(text, url)
368			else:
369			container = TextWithRules(text, rules)
370			return self._extract(container.to_JSON())
371
372			def extract_from_document(self, doc, rules):
373			"""
374			Sends a `processors.ds.Document` (`doc`) to the server with rules for information extraction (IE).
375
376			Parameters
377			----------
378			doc : processors.ds.Document
379			`rules` will be applied to this `processors.ds.Document`.
380			rules : str
381			Either Odin rules provided as a `yaml` string, or a url pointing to a `yaml` file of rules.
382
383			Returns
384			-------
385			[processors.odin.Mention] or None
386			Rule matches produce a list of `processors.odin.Mention`.
387
388			"""
389			if OdinAPI.valid_rule_url(rules):
390			# this is actually a URL to a yaml file
391			url = rules
392			container = DocumentWithURL(doc, rules)
393			else:
394			container = DocumentWithRules(doc, rules)
395			return self._extract(container.to_JSON())
396
397			#############################################
398			# Containers for Odin data
399			# transmitted to the server for processing
400			#############################################
401
402			class TextWithRules(object):
403
404			def __init__(self, text, rules):
405			self.text = text
406			self.rules = rules
407
408			def to_JSON_dict(self):
409			jdict = dict()
410			jdict["text"] = self.text
411			jdict["rules"] = self.rules
412			return jdict
413
414			def to_JSON(self):
415			return json.dumps(self.to_JSON_dict(), sort_keys=True, indent=4)
416
417			class TextWithURL(object):
418
419			def __init__(self, text, url):
420			self.text = text
421			# TODO: throw exception if url is invalid
422			self.url = url
423
424			def to_JSON_dict(self):
425			jdict = dict()
426			jdict["text"] = self.text
427			jdict["url"] = self.url
428			return jdict
429
430			def to_JSON(self):
431			return json.dumps(self.to_JSON_dict(), sort_keys=True, indent=4)
432
433			class DocumentWithRules(object):
434
435			def __init__(self, document, rules):
436			# TODO: throw exception if isinstance(document, Document) is False
437			self.document = document
438			self.rules = rules
439
440			def to_JSON_dict(self):
441			jdict = dict()
442			jdict["document"] = self.document.to_JSON_dict()
443			jdict["rules"] = self.rules
444			return jdict
445
446			def to_JSON(self):
447			return json.dumps(self.to_JSON_dict(), sort_keys=True, indent=4)
448
449			class DocumentWithURL(object):
450
451			def __init__(self, document, url):
452			# TODO: throw exception if isinstance(document, Document) is False
453			self.document = document
454			# TODO: throw exception if url is invalid
455			self.url = url
456
457			def to_JSON_dict(self):
458			jdict = dict()
459			jdict["document"] = self.document.to_JSON_dict()
460			jdict["url"] = self.url
461			return jdict
462
463			def to_JSON(self):
464			return json.dumps(self.to_JSON_dict(), sort_keys=True, indent=4)
465

myedibleenso / py-processors

Push — master ( 352884...922d0d )

OdinAPI.extract_from_document() B

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like