ProcessorsAPI.dlProgress() - Code Metrics - Inspection of "Retrieve jar when first needed" - myedibleenso/py-processors - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — master ( 913a21...6ec523 )

by Gus

created 2016-06-30 20:11 UTC

ProcessorsAPI.dlProgress() A

↳ Parent: ProcessorsAPI.download_jar()

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
cc	1
c	0
b	0
f	0
dl	0
loc	4
rs	10

#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
#from pkg_resources import resource_filename
try:
    from urllib import urlretrieve
except:
    from urllib.request import urlretrieve
from .processors import *
from .sentiment import SentimentAnalysisAPI
from .odin import OdinAPI
import os
import shlex
import os
import subprocess as sp
import requests
import time
import sys
import logging


class ProcessorsAPI(object):

    PROC_VAR = 'PROCESSORS_SERVER'
    # save to lib loc
    DEFAULT_JAR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "processors-server.jar")
    #print(resource_filename(__name__, "processors-server.jar"))

    def __init__(self, port, hostname="localhost", timeout=120, jvm_mem="-Xmx3G", jar_path=None, keep_alive=False, log_file=None):

        self.hostname = hostname
        self.port = port
        self.make_address(hostname, port)
        self._start_command = "java {} -cp {} NLPServer --port {port} --host {host}" # mem, jar path, port, host
        self.timeout = timeout
        self.jvm_mem = jvm_mem
        # whether or not to stop the server when the object is destroyed
        self.keep_alive = keep_alive
        # how long to wait between requests
        self.wait_time = 2
        # processors
        self.default = Processor(self.address)
        self.fastnlp = FastNLPProcessor(self.address)
        self.bionlp = BioNLPProcessor(self.address)
        # sentiment
        self.sentiment = SentimentAnalysisAPI(self.address)
        # odin
        self.odin = OdinAPI(self.address)
        # use the os module's devnull for compatibility with python 2.7
        #self.DEVNULL = open(os.devnull, 'wb')
        self.logger = logging.getLogger(__name__)
        self.log_file = self.prepare_log_file(log_file)

        # resolve jar path
        self.resolve_jar_path(jar_path)
        # attempt to establish connection with server
        self.establish_connection()

    def prepare_log_file(self, lf):
        """
        Configure logger and return file path for logging
        """
        # log_file
        log_file = os.path.expanduser(os.path.join("~", ".py-processors.log")) if not lf else os.path.expanduser(lf)
        # configure logger
        self.logger.setLevel(logging.DEBUG)
        # create console handler and set level to info
        handler = logging.StreamHandler()
        handler.setLevel(logging.INFO)
        formatter = logging.Formatter("%(levelname)s - %(message)s")
        handler.setFormatter(formatter)
        self.logger.addHandler(handler)
        # create debug file handler and set level to debug
        handler = logging.FileHandler(log_file, "w")
        handler.setLevel(logging.DEBUG)
        formatter = logging.Formatter("%(levelname)s - %(message)s")
        handler.setFormatter(formatter)
        self.logger.addHandler(handler)
        return log_file

    def annotate(self, text):
        """
        Uses default processor (CoreNLP) to annotate text.  Included for backwards compatibility.
        """
        return self.default.annotate(text)

    def annotate_from_sentences(self, sentences):
        """
        Uses default processor (CoreNLP) to annotate a list of segmented sentences.
        """
        return self.default.annotate_from_sentences(sentences)

    def establish_connection(self):
        """
        Attempt to connect to a server (assumes server is running)
        """
        if self.annotate("Blah"):
            print("Connection with server established!")
        else:
            try:
                # Attempt to start the server
                self._start_server()
            except Exception as e:
                if not os.path.exists(self.jar_path):
                    print("\nprocessors-server.jar not found at {}.".format(self.jar_path))
                print("Unable to start server. Please start the server manually with .start_server(\"path/to/processors-server.jar\")")
                print("\n{}".format(e))

    def resolve_jar_path(self, jar_path):
        """
        Attempts to preferentially set value of self.jar_path
        """
        # Preference 1: if a .jar is given, check to see if the path is valid
        if jar_path:
            print("Using provided path")
            jp = os.path.expanduser(jar_path)
            # check if path is valid
            if os.path.exists(jp):
                self.jar_path = jp
        else:
            # Preference 2: if a PROCESSORS_SERVER environment variable is defined, check its validity
            if ProcessorsAPI.PROC_VAR in os.environ:
                print("Using path given via $PROCESSORS_SERVER")
                jp = os.path.expanduser(os.environ[ProcessorsAPI.PROC_VAR])
                # check if path is valid
                if os.path.exists(jp):
                    self.jar_path = jp
                else:
                    self.jar_path = None
                    print("WARNING: {0} path is invalid.  \nPlease verify this entry in your environment:\n\texport {0}=/path/to/processors-server.jar".format(ProcessorsAPI.PROC_VAR))
            # Preference 3: attempt to use the processors-sever.jar (download if not found)
            else:
                print("Using default")
                # check if jar exists
                if not os.path.exists(ProcessorsAPI.DEFAULT_JAR):
                    ProcessorsAPI.download_jar()
                self.jar_path = ProcessorsAPI.DEFAULT_JAR

    def start_server(self, jar_path=None, timeout=120):
        """
        Starts processors-sever.jar
        """
        self.timeout = int(float(timeout)/2)
        if jar_path:
            self.jar_path = jar_path
        self._start_server()

    def stop_server(self, port=None):
        """
        Sends a poison pill to the server and waits for shutdown response
        """
        port = port or self.port
        address = "http://{}:{}".format(self.hostname, port)
        shutdown_address = "{}/shutdown".format(address)
        # attempt shutdown
        try:
            response = requests.post(shutdown_address)
            if response:
                print(response.content.decode("utf-8"))
            return True
        # will fail if the server is already down
        except Exception as e:
            pass
        return False

    def _ensure_jar_path_exists(self):
        # check if jar exists
        if not os.path.exists(self.jar_path):
            raise Exception("jar not found at {}".format(self.jar_path))

    def _start_server(self, port=None):
        """
        "Private" method called by start_server()
        """

        self._ensure_jar_path_exists()

        if port:
            self.port = port
        # build the command
        cmd = self._start_command.format(self.jvm_mem, self.jar_path, port=self.port, host=self.hostname)
        self._process = sp.Popen(shlex.split(cmd),
                                 shell=False,
                                 stderr=open(self.log_file, 'wb'),
                                 stdout=open(self.log_file, 'wb'),
                                 universal_newlines=True)

        self.logger.info("Starting processors-server ({}) ...".format(cmd))
        print("\nWaiting for server...")

        progressbar_length = int(self.timeout/self.wait_time)
        for i in range(progressbar_length):
            try:
                success = self.annotate("blah")
                if success:
                    print("\n\nConnection with processors-server established ({})".format(self.address))
                    return True
                sys.stdout.write("\r[{:{}}]".format('='*i, progressbar_length))
                time.sleep(self.wait_time)
            except Exception as e:
                raise(e)

        # if the server still hasn't started, raise an Exception
        raise Exception("Couldn't connect to processors-server. Is the port in use?")

    def make_address(self, hostname, port):
        # update hostname
        self.hostname = hostname
        # update port
        self.port = port
        # update address
        self.address = "http://{}:{}".format(self.hostname, self.port)

    @staticmethod
    def download_jar(jar_url="http://www.cs.arizona.edu/~hahnpowell/processors-server/current/processors-server.jar"):
        # download processors-server.jar
        ppjar = ProcessorsAPI.DEFAULT_JAR
        percent = 0
        def dlProgress(count, blockSize, totalSize):
            percent = int(count*blockSize*100/totalSize)
            sys.stdout.write("\r{}% complete".format(percent))
            sys.stdout.flush()

        print("Downloading {} from {} ...".format(ppjar, jar_url))
        urlretrieve(jar_url, ppjar, reporthook=dlProgress)
        print("\nDownload Complete! {}".format(ppjar))


    def _get_path(self, p):
        """
        Expand a user-specified path.  Supports "~" shortcut.
        """
        return os.path.abspath(os.path.normpath(os.path.expanduser(p)))

    def __del__(self):
        """
        Stop server unless otherwise specified
        """
        if not self.keep_alive:
            try:
                self.stop_server()
                # close our file object
                #self.DEVNULL.close()
                print("Successfully shut down processors-server!")
            except Exception as e:
                self.logger.debug(e)
                print("Couldn't kill processors-server.  Was server started externally?")


1			#!/usr/bin/env python
2			# -- coding: utf-8 --
3			from __future__ import unicode_literals
4			#from pkg_resources import resource_filename
5			try:
6			from urllib import urlretrieve
7			except:
8			from urllib.request import urlretrieve
9			from .processors import *
10			from .sentiment import SentimentAnalysisAPI
11			from .odin import OdinAPI
12			import os
13			import shlex
14			import os
15			import subprocess as sp
16			import requests
17			import time
18			import sys
19			import logging
20
21
22			class ProcessorsAPI(object):
23
24			PROC_VAR = 'PROCESSORS_SERVER'
25			# save to lib loc
26			DEFAULT_JAR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "processors-server.jar")
27			#print(resource_filename(__name__, "processors-server.jar"))
28
29			def __init__(self, port, hostname="localhost", timeout=120, jvm_mem="-Xmx3G", jar_path=None, keep_alive=False, log_file=None):
30
31			self.hostname = hostname
32			self.port = port
33			self.make_address(hostname, port)
34			self._start_command = "java {} -cp {} NLPServer --port {port} --host {host}" # mem, jar path, port, host
35			self.timeout = timeout
36			self.jvm_mem = jvm_mem
37			# whether or not to stop the server when the object is destroyed
38			self.keep_alive = keep_alive
39			# how long to wait between requests
40			self.wait_time = 2
41			# processors
42			self.default = Processor(self.address)
43			self.fastnlp = FastNLPProcessor(self.address)
44			self.bionlp = BioNLPProcessor(self.address)
45			# sentiment
46			self.sentiment = SentimentAnalysisAPI(self.address)
47			# odin
48			self.odin = OdinAPI(self.address)
49			# use the os module's devnull for compatibility with python 2.7
50			#self.DEVNULL = open(os.devnull, 'wb')
51			self.logger = logging.getLogger(__name__)
52			self.log_file = self.prepare_log_file(log_file)
53
54			# resolve jar path
55			self.resolve_jar_path(jar_path)
56			# attempt to establish connection with server
57			self.establish_connection()
58
59			def prepare_log_file(self, lf):
60			"""
61			Configure logger and return file path for logging
62			"""
63			# log_file
64			log_file = os.path.expanduser(os.path.join("~", ".py-processors.log")) if not lf else os.path.expanduser(lf)
65			# configure logger
66			self.logger.setLevel(logging.DEBUG)
67			# create console handler and set level to info
68			handler = logging.StreamHandler()
69			handler.setLevel(logging.INFO)
70			formatter = logging.Formatter("%(levelname)s - %(message)s")
71			handler.setFormatter(formatter)
72			self.logger.addHandler(handler)
73			# create debug file handler and set level to debug
74			handler = logging.FileHandler(log_file, "w")
75			handler.setLevel(logging.DEBUG)
76			formatter = logging.Formatter("%(levelname)s - %(message)s")
77			handler.setFormatter(formatter)
78			self.logger.addHandler(handler)
79			return log_file
80
81			def annotate(self, text):
82			"""
83			Uses default processor (CoreNLP) to annotate text. Included for backwards compatibility.
84			"""
85			return self.default.annotate(text)
86
87			def annotate_from_sentences(self, sentences):
88			"""
89			Uses default processor (CoreNLP) to annotate a list of segmented sentences.
90			"""
91			return self.default.annotate_from_sentences(sentences)
92
93			def establish_connection(self):
94			"""
95			Attempt to connect to a server (assumes server is running)
96			"""
97			if self.annotate("Blah"):
98			print("Connection with server established!")
99			else:
100			try:
101			# Attempt to start the server
102			self._start_server()
103			except Exception as e:
104			if not os.path.exists(self.jar_path):
105			print("\nprocessors-server.jar not found at {}.".format(self.jar_path))
106			print("Unable to start server. Please start the server manually with .start_server(\"path/to/processors-server.jar\")")
107			print("\n{}".format(e))
108
109			def resolve_jar_path(self, jar_path):
110			"""
111			Attempts to preferentially set value of self.jar_path
112			"""
113			# Preference 1: if a .jar is given, check to see if the path is valid
114			if jar_path:
115			print("Using provided path")
116			jp = os.path.expanduser(jar_path)
117			# check if path is valid
118			if os.path.exists(jp):
119			self.jar_path = jp
120			else:
121			# Preference 2: if a PROCESSORS_SERVER environment variable is defined, check its validity
122			if ProcessorsAPI.PROC_VAR in os.environ:
123			print("Using path given via $PROCESSORS_SERVER")
124			jp = os.path.expanduser(os.environ[ProcessorsAPI.PROC_VAR])
125			# check if path is valid
126			if os.path.exists(jp):
127			self.jar_path = jp
128			else:
129			self.jar_path = None
130			print("WARNING: {0} path is invalid. \nPlease verify this entry in your environment:\n\texport {0}=/path/to/processors-server.jar".format(ProcessorsAPI.PROC_VAR))
131			# Preference 3: attempt to use the processors-sever.jar (download if not found)
132			else:
133			print("Using default")
134			# check if jar exists
135			if not os.path.exists(ProcessorsAPI.DEFAULT_JAR):
136			ProcessorsAPI.download_jar()
137			self.jar_path = ProcessorsAPI.DEFAULT_JAR
138
139			def start_server(self, jar_path=None, timeout=120):
140			"""
141			Starts processors-sever.jar
142			"""
143			self.timeout = int(float(timeout)/2)
144			if jar_path:
145			self.jar_path = jar_path
146			self._start_server()
147
148			def stop_server(self, port=None):
149			"""
150			Sends a poison pill to the server and waits for shutdown response
151			"""
152			port = port or self.port
153			address = "http://{}:{}".format(self.hostname, port)
154			shutdown_address = "{}/shutdown".format(address)
155			# attempt shutdown
156			try:
157			response = requests.post(shutdown_address)
158			if response:
159			print(response.content.decode("utf-8"))
160			return True
161			# will fail if the server is already down
162			except Exception as e:
163			pass
164			return False
165
166			def _ensure_jar_path_exists(self):
167			# check if jar exists
168			if not os.path.exists(self.jar_path):
169			raise Exception("jar not found at {}".format(self.jar_path))
170
171			def _start_server(self, port=None):
172			"""
173			"Private" method called by start_server()
174			"""
175
176			self._ensure_jar_path_exists()
177
178			if port:
179			self.port = port
180			# build the command
181			cmd = self._start_command.format(self.jvm_mem, self.jar_path, port=self.port, host=self.hostname)
182			self._process = sp.Popen(shlex.split(cmd),
183			shell=False,
184			stderr=open(self.log_file, 'wb'),
185			stdout=open(self.log_file, 'wb'),
186			universal_newlines=True)
187
188			self.logger.info("Starting processors-server ({}) ...".format(cmd))
189			print("\nWaiting for server...")
190
191			progressbar_length = int(self.timeout/self.wait_time)
192			for i in range(progressbar_length):
193			try:
194			success = self.annotate("blah")
195			if success:
196			print("\n\nConnection with processors-server established ({})".format(self.address))
197			return True
198			sys.stdout.write("\r[{:{}}]".format('='*i, progressbar_length))
199			time.sleep(self.wait_time)
200			except Exception as e:
201			raise(e)
202
203			# if the server still hasn't started, raise an Exception
204			raise Exception("Couldn't connect to processors-server. Is the port in use?")
205
206			def make_address(self, hostname, port):
207			# update hostname
208			self.hostname = hostname
209			# update port
210			self.port = port
211			# update address
212			self.address = "http://{}:{}".format(self.hostname, self.port)
213
214			@staticmethod
215			def download_jar(jar_url="http://www.cs.arizona.edu/~hahnpowell/processors-server/current/processors-server.jar"):
216			# download processors-server.jar
217			ppjar = ProcessorsAPI.DEFAULT_JAR
218			percent = 0
219			def dlProgress(count, blockSize, totalSize):
220			percent = int(countblockSize100/totalSize)
221			sys.stdout.write("\r{}% complete".format(percent))
222			sys.stdout.flush()
223
224			print("Downloading {} from {} ...".format(ppjar, jar_url))
225			urlretrieve(jar_url, ppjar, reporthook=dlProgress)
226			print("\nDownload Complete! {}".format(ppjar))
227
228
229			def _get_path(self, p):
230			"""
231			Expand a user-specified path. Supports "~" shortcut.
232			"""
233			return os.path.abspath(os.path.normpath(os.path.expanduser(p)))
234
235			def __del__(self):
236			"""
237			Stop server unless otherwise specified
238			"""
239			if not self.keep_alive:
240			try:
241			self.stop_server()
242			# close our file object
243			#self.DEVNULL.close()
244			print("Successfully shut down processors-server!")
245			except Exception as e:
246			self.logger.debug(e)
247			print("Couldn't kill processors-server. Was server started externally?")
248

myedibleenso / py-processors

Push — master ( 913a21...6ec523 )

ProcessorsAPI.dlProgress() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like