ContourMerged.__init__() - Code Metrics - Inspection of "create script to generate a merged/average contour..." - bartromgens/nsmaps - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — master ( 35f6e6...1a96f6 )

by Bart

created 2017-04-05 11:44 UTC

ContourMerged.init() A

↳ Parent: ContourMerged

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes	1
Bugs	0	Features	0

Metric	Value
cc	1
c	1
b	0
f	0
dl	0
loc	5
rs	9.4285

import sys
import os
import math
import fnmatch

from timeit import default_timer as timer
from multiprocessing import Process, Queue

import numpy
from scipy.spatial import KDTree

import matplotlib.pyplot as plt
from matplotlib.figure import Figure
from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas

import geojson
import geojsoncontour

import nsmaps
from nsmaps.logger import logger


def dotproduct(v1, v2):
    return sum((a * b) for a, b in zip(v1, v2))


def length(v):
    return math.sqrt(dotproduct(v, v))


def angle(v1, v2):
    return math.acos(dotproduct(v1, v2) / (length(v1) * length(v2)))


class ContourData(object):
    def __init__(self):
        self.Z = None
        self.index_begin = 0


class ContourPlotConfig(object):
    def __init__(self):
        self.stepsize_deg = 0.005
        self.n_processes = 4
        self.cycle_speed_kmh = 18.0
        self.n_nearest = 20
        self.lon_start = 3.0
        self.lat_start = 50.5
        self.delta_deg = 6.5
        self.lon_end = self.lon_start + self.delta_deg
        self.lat_end = self.lat_start + self.delta_deg / 2.0
        self.min_angle_between_segments = 7

    def print_bounding_box(self):
        print(
            '[[' + str(self.lon_start) + ',' + str(self.lat_start) + '],'
            '[' + str(self.lon_start) + ',' + str(self.lat_end) + '],'
            '[' + str(self.lon_end) + ',' + str(self.lat_end) + '],'
            '[' + str(self.lon_end) + ',' + str(self.lat_start) + '],'
            '[' + str(self.lon_start) + ',' + str(self.lat_start) + ']]'
        )


class TestConfig(ContourPlotConfig):
    def __init__(self):
        super().__init__()
        self.stepsize_deg = 0.005
        self.n_processes = 4
        self.lon_start = 4.8
        self.lat_start = 52.0
        self.delta_deg = 1.0
        self.lon_end = self.lon_start + self.delta_deg
        self.lat_end = self.lat_start + self.delta_deg / 2.0
        self.min_angle_between_segments = 7
        self.latrange = []
        self.lonrange = []
        self.Z = [[]]


class Contour(object):
    def __init__(self, departure_station, stations, config):
        self.departure_station = departure_station
        self.stations = stations
        self.config = config
        self.lonrange = numpy.arange(self.config.lon_start, self.config.lon_end, self.config.stepsize_deg)
        self.latrange = numpy.arange(self.config.lat_start, self.config.lat_end, self.config.stepsize_deg / 2.0)

    def create_contour_data(self):
        logger.info('BEGIN')
        if self.departure_station.has_travel_time_data():
            self.stations.travel_times_from_json(self.departure_station.get_travel_time_filepath())
        else:
            logger.error('Input file ' + self.departure_station.get_travel_time_filepath() + ' not found. Skipping station.')

        start = timer()
        numpy.set_printoptions(3, threshold=100, suppress=True)  # .3f

        altitude = 0.0
        self.Z = numpy.zeros((int(self.lonrange.shape[0]), int(self.latrange.shape[0])))
        gps = nsmaps.utilgeo.GPS()

        positions = []
        for station in self.stations:
            x, y, z = gps.lla2ecef([station.get_lat(), station.get_lon(), altitude])
            positions.append([x, y, z])

        logger.info('starting spatial interpolation')

        # tree to find nearest neighbors
        tree = KDTree(positions)

        queue = Queue()
        processes = []
        if self.config.n_nearest > len(self.stations):
            self.config.n_nearest = len(self.stations)
        latrange_per_process = int(len(self.latrange)/self.config.n_processes)
        for i in range(0, self.config.n_processes):
            begin = i * latrange_per_process
            end = (i+1) * latrange_per_process
            latrange_part = self.latrange[begin:end]
            process = Process(target=self.interpolate_travel_time, args=(queue, i, self.stations.stations, tree, gps, latrange_part,
                                                                         self.lonrange, altitude, self.config.n_nearest, self.config.cycle_speed_kmh))
            processes.append(process)

        for process in processes:
            process.start()

        # get from the queue and append the values
        for i in range(0, self.config.n_processes):
            data = queue.get()
            index_begin = data.index_begin
            begin = int(index_begin*len(self.latrange)/self.config.n_processes)
            end = int((index_begin+1)*len(self.latrange)/self.config.n_processes)
            self.Z[0:][begin:end] = data.Z

        for process in processes:
            process.join()

        end = timer()
        logger.info('finished spatial interpolation in ' + str(end - start) + ' [sec]')
        logger.info('END')

    @property
    def data_filename(self):
        return 'data/contour_data_' + self.departure_station.get_code() + '.npz'

    def load(self):
        with open(self.data_filename, 'rb') as filein:
            self.Z = numpy.load(filein)

    def save(self):
        with open(self.data_filename, 'wb') as fileout:
            numpy.save(fileout, self.Z)

    def create_geojson(self, filepath, stroke_width=1, levels=[], norm=None, overwrite=False):
        if not overwrite and os.path.exists(filepath):
            logger.error('Output file ' + filepath + ' already exists. Will not override.')
            return

        figure = Figure(frameon=False)
        FigureCanvas(figure)
        ax = figure.add_subplot(111)
        # contours = plt.contourf(lonrange, latrange, Z, levels=levels, cmap=plt.cm.plasma)
        contours = ax.contour(
            self.lonrange, self.latrange, self.Z,
            levels=levels,
            norm=norm,
            cmap=plt.cm.jet,
            linewidths=3
        )

        ndigits = len(str(int(1.0 / self.config.stepsize_deg))) + 1
        logger.info('converting contour to geojson file: ' + filepath)
        geojsoncontour.contour_to_geojson(
            contour=contours,
            geojson_filepath=filepath,
            contour_levels=levels,
            min_angle_deg=self.config.min_angle_between_segments,
            ndigits=ndigits,
            unit='min',
            stroke_width=stroke_width
        )

        cbar = figure.colorbar(contours, format='%d', orientation='horizontal')
        cbar.set_label('Travel time [minutes]')
        # cbar.set_ticks(self.config.colorbar_ticks)
        ax.set_visible(False)
        figure.savefig(
            filepath.replace('.geojson', '') + "_colorbar.png",
            dpi=90,
            bbox_inches='tight',
            pad_inches=0,
            transparent=True,
        )

    @staticmethod
    def interpolate_travel_time(q, position, stations, kdtree, gps, latrange, lonrange, altitude, n_nearest, cycle_speed_kmh):
        # n_nearest: check N nearest stations as best start for cycle route
        logger.info('interpolate_travel_time')
        Z = numpy.zeros((int(latrange.shape[0]), int(lonrange.shape[0])))
        min_per_m = 1.0 / (cycle_speed_kmh / 60.0 * 1000.0)
        for i, lat in enumerate(latrange):
            if i % (len(latrange) / 10) == 0:
                logger.debug(str(int(i / len(latrange) * 100)) + '%')

            for j, lon in enumerate(lonrange):
                x, y, z = gps.lla2ecef([lat, lon, altitude])
                distances, indexes = kdtree.query([x, y, z], n_nearest)
                min_travel_time = sys.float_info.max
                for distance, index in zip(distances, indexes):
                    if stations[index].travel_time_min is None:
                        continue
                    travel_time = stations[index].travel_time_min + distance * min_per_m
                    if travel_time < min_travel_time:
                        min_travel_time = travel_time
                Z[i][j] = min_travel_time
        data = ContourData()
        data.index_begin = position
        data.Z = Z
        q.put(data)
        logger.info('end interpolate_travel_time')
        return


class ContourMerged(object):

    def __init__(self, config):
        self.config = config
        self.lonrange = numpy.arange(self.config.lon_start, self.config.lon_end, self.config.stepsize_deg)
        self.latrange = numpy.arange(self.config.lat_start, self.config.lat_end, self.config.stepsize_deg / 2.0)
        self.Z = numpy.zeros((int(self.lonrange.shape[0]), int(self.latrange.shape[0])))

    def merge_grid_data(self, data_dir):
        counter = 0
        for file in os.listdir(data_dir):
            if fnmatch.fnmatch(file, '*.npz'):
                print(file)
                counter += 1
                with open(os.path.join(data_dir, file), 'rb') as filein:
                    self.Z += numpy.load(filein)
        self.Z = self.Z / counter

    def create_geojson(self, filepath, stroke_width=1, levels=[], norm=None):
        figure = Figure(frameon=False)
        FigureCanvas(figure)
        ax = figure.add_subplot(111)
        colormap = plt.cm.jet
        contours = ax.contourf(self.lonrange, self.latrange, self.Z, levels=levels, cmap=colormap)
        contours = ax.contour(
            self.lonrange, self.latrange, self.Z,
            levels=levels,
            norm=norm,
            cmap=colormap,
            linewidths=1
        )

        ndigits = len(str(int(1.0 / self.config.stepsize_deg))) + 1
        geojsoncontour.contour_to_geojson(
            contour=contours,
            geojson_filepath=filepath,
            contour_levels=levels,
            min_angle_deg=self.config.min_angle_between_segments,
            ndigits=ndigits,
            unit='min',
            stroke_width=stroke_width
        )

        # figure.savefig(
        #     "combined_map.png",
        #     dpi=300,
        #     bbox_inches='tight',
        #     pad_inches=0,
        #     transparent=True,
        # )

1			import sys
2			import os
3			import math
4			import fnmatch
5
6			from timeit import default_timer as timer
7			from multiprocessing import Process, Queue
8
9			import numpy
10			from scipy.spatial import KDTree
11
12			import matplotlib.pyplot as plt
13			from matplotlib.figure import Figure
14			from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
15
16			import geojson
17			import geojsoncontour
18
19			import nsmaps
20			from nsmaps.logger import logger
21
22
23			def dotproduct(v1, v2):
24			return sum((a * b) for a, b in zip(v1, v2))
25
26
27			def length(v):
28			return math.sqrt(dotproduct(v, v))
29
30
31			def angle(v1, v2):
32			return math.acos(dotproduct(v1, v2) / (length(v1) * length(v2)))
33
34
35			class ContourData(object):
36			def __init__(self):
37			self.Z = None
38			self.index_begin = 0
39
40
41			class ContourPlotConfig(object):
42			def __init__(self):
43			self.stepsize_deg = 0.005
44			self.n_processes = 4
45			self.cycle_speed_kmh = 18.0
46			self.n_nearest = 20
47			self.lon_start = 3.0
48			self.lat_start = 50.5
49			self.delta_deg = 6.5
50			self.lon_end = self.lon_start + self.delta_deg
51			self.lat_end = self.lat_start + self.delta_deg / 2.0
52			self.min_angle_between_segments = 7
53
54			def print_bounding_box(self):
55			print(
56			'[[' + str(self.lon_start) + ',' + str(self.lat_start) + '],'
57			'[' + str(self.lon_start) + ',' + str(self.lat_end) + '],'
58			'[' + str(self.lon_end) + ',' + str(self.lat_end) + '],'
59			'[' + str(self.lon_end) + ',' + str(self.lat_start) + '],'
60			'[' + str(self.lon_start) + ',' + str(self.lat_start) + ']]'
61			)
62
63
64			class TestConfig(ContourPlotConfig):
65			def __init__(self):
66			super().__init__()
67			self.stepsize_deg = 0.005
68			self.n_processes = 4
69			self.lon_start = 4.8
70			self.lat_start = 52.0
71			self.delta_deg = 1.0
72			self.lon_end = self.lon_start + self.delta_deg
73			self.lat_end = self.lat_start + self.delta_deg / 2.0
74			self.min_angle_between_segments = 7
75			self.latrange = []
76			self.lonrange = []
77			self.Z = [[]]
78
79
80			class Contour(object):
81			def __init__(self, departure_station, stations, config):
82			self.departure_station = departure_station
83			self.stations = stations
84			self.config = config
85			self.lonrange = numpy.arange(self.config.lon_start, self.config.lon_end, self.config.stepsize_deg)
86			self.latrange = numpy.arange(self.config.lat_start, self.config.lat_end, self.config.stepsize_deg / 2.0)
87
88			def create_contour_data(self):
89			logger.info('BEGIN')
90			if self.departure_station.has_travel_time_data():
91			self.stations.travel_times_from_json(self.departure_station.get_travel_time_filepath())
92			else:
93			logger.error('Input file ' + self.departure_station.get_travel_time_filepath() + ' not found. Skipping station.')
94
95			start = timer()
96			numpy.set_printoptions(3, threshold=100, suppress=True) # .3f
97
98			altitude = 0.0
99			self.Z = numpy.zeros((int(self.lonrange.shape[0]), int(self.latrange.shape[0])))
100			gps = nsmaps.utilgeo.GPS()
101
102			positions = []
103			for station in self.stations:
104			x, y, z = gps.lla2ecef([station.get_lat(), station.get_lon(), altitude])
105			positions.append([x, y, z])
106
107			logger.info('starting spatial interpolation')
108
109			# tree to find nearest neighbors
110			tree = KDTree(positions)
111
112			queue = Queue()
113			processes = []
114			if self.config.n_nearest > len(self.stations):
115			self.config.n_nearest = len(self.stations)
116			latrange_per_process = int(len(self.latrange)/self.config.n_processes)
117			for i in range(0, self.config.n_processes):
118			begin = i * latrange_per_process
119			end = (i+1) * latrange_per_process
120			latrange_part = self.latrange[begin:end]
121			process = Process(target=self.interpolate_travel_time, args=(queue, i, self.stations.stations, tree, gps, latrange_part,
122			self.lonrange, altitude, self.config.n_nearest, self.config.cycle_speed_kmh))
123			processes.append(process)
124
125			for process in processes:
126			process.start()
127
128			# get from the queue and append the values
129			for i in range(0, self.config.n_processes):
130			data = queue.get()
131			index_begin = data.index_begin
132			begin = int(index_begin*len(self.latrange)/self.config.n_processes)
133			end = int((index_begin+1)*len(self.latrange)/self.config.n_processes)
134			self.Z[0:][begin:end] = data.Z
135
136			for process in processes:
137			process.join()
138
139			end = timer()
140			logger.info('finished spatial interpolation in ' + str(end - start) + ' [sec]')
141			logger.info('END')
142
143			@property
144			def data_filename(self):
145			return 'data/contour_data_' + self.departure_station.get_code() + '.npz'
146
147			def load(self):
148			with open(self.data_filename, 'rb') as filein:
149			self.Z = numpy.load(filein)
150
151			def save(self):
152			with open(self.data_filename, 'wb') as fileout:
153			numpy.save(fileout, self.Z)
154
155			def create_geojson(self, filepath, stroke_width=1, levels=[], norm=None, overwrite=False):
156			if not overwrite and os.path.exists(filepath):
157			logger.error('Output file ' + filepath + ' already exists. Will not override.')
158			return
159
160			figure = Figure(frameon=False)
161			FigureCanvas(figure)
162			ax = figure.add_subplot(111)
163			# contours = plt.contourf(lonrange, latrange, Z, levels=levels, cmap=plt.cm.plasma)
164			contours = ax.contour(
165			self.lonrange, self.latrange, self.Z,
166			levels=levels,
167			norm=norm,
168			cmap=plt.cm.jet,
169			linewidths=3
170			)
171
172			ndigits = len(str(int(1.0 / self.config.stepsize_deg))) + 1
173			logger.info('converting contour to geojson file: ' + filepath)
174			geojsoncontour.contour_to_geojson(
175			contour=contours,
176			geojson_filepath=filepath,
177			contour_levels=levels,
178			min_angle_deg=self.config.min_angle_between_segments,
179			ndigits=ndigits,
180			unit='min',
181			stroke_width=stroke_width
182			)
183
184			cbar = figure.colorbar(contours, format='%d', orientation='horizontal')
185			cbar.set_label('Travel time [minutes]')
186			# cbar.set_ticks(self.config.colorbar_ticks)
187			ax.set_visible(False)
188			figure.savefig(
189			filepath.replace('.geojson', '') + "_colorbar.png",
190			dpi=90,
191			bbox_inches='tight',
192			pad_inches=0,
193			transparent=True,
194			)
195
196			@staticmethod
197			def interpolate_travel_time(q, position, stations, kdtree, gps, latrange, lonrange, altitude, n_nearest, cycle_speed_kmh):
198			# n_nearest: check N nearest stations as best start for cycle route
199			logger.info('interpolate_travel_time')
200			Z = numpy.zeros((int(latrange.shape[0]), int(lonrange.shape[0])))
201			min_per_m = 1.0 / (cycle_speed_kmh / 60.0 * 1000.0)
202			for i, lat in enumerate(latrange):
203			if i % (len(latrange) / 10) == 0:
204			logger.debug(str(int(i / len(latrange) * 100)) + '%')
205
206			for j, lon in enumerate(lonrange):
207			x, y, z = gps.lla2ecef([lat, lon, altitude])
208			distances, indexes = kdtree.query([x, y, z], n_nearest)
209			min_travel_time = sys.float_info.max
210			for distance, index in zip(distances, indexes):
211			if stations[index].travel_time_min is None:
212			continue
213			travel_time = stations[index].travel_time_min + distance * min_per_m
214			if travel_time < min_travel_time:
215			min_travel_time = travel_time
216			Z[i][j] = min_travel_time
217			data = ContourData()
218			data.index_begin = position
219			data.Z = Z
220			q.put(data)
221			logger.info('end interpolate_travel_time')
222			return
223
224
225			class ContourMerged(object):
226
227			def __init__(self, config):
228			self.config = config
229			self.lonrange = numpy.arange(self.config.lon_start, self.config.lon_end, self.config.stepsize_deg)
230			self.latrange = numpy.arange(self.config.lat_start, self.config.lat_end, self.config.stepsize_deg / 2.0)
231			self.Z = numpy.zeros((int(self.lonrange.shape[0]), int(self.latrange.shape[0])))
232
233			def merge_grid_data(self, data_dir):
234			counter = 0
235			for file in os.listdir(data_dir):
236			if fnmatch.fnmatch(file, '*.npz'):
237			print(file)
238			counter += 1
239			with open(os.path.join(data_dir, file), 'rb') as filein:
240			self.Z += numpy.load(filein)
241			self.Z = self.Z / counter
242
243			def create_geojson(self, filepath, stroke_width=1, levels=[], norm=None):
244			figure = Figure(frameon=False)
245			FigureCanvas(figure)
246			ax = figure.add_subplot(111)
247			colormap = plt.cm.jet
248			contours = ax.contourf(self.lonrange, self.latrange, self.Z, levels=levels, cmap=colormap)
249			contours = ax.contour(
250			self.lonrange, self.latrange, self.Z,
251			levels=levels,
252			norm=norm,
253			cmap=colormap,
254			linewidths=1
255			)
256
257			ndigits = len(str(int(1.0 / self.config.stepsize_deg))) + 1
258			geojsoncontour.contour_to_geojson(
259			contour=contours,
260			geojson_filepath=filepath,
261			contour_levels=levels,
262			min_angle_deg=self.config.min_angle_between_segments,
263			ndigits=ndigits,
264			unit='min',
265			stroke_width=stroke_width
266			)
267
268			# figure.savefig(
269			# "combined_map.png",
270			# dpi=300,
271			# bbox_inches='tight',
272			# pad_inches=0,
273			# transparent=True,
274			# )

bartromgens / nsmaps

Push — master ( 35f6e6...1a96f6 )

ContourMerged.__init__() A

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like

ContourMerged.init() A