isomer.ui.builder.update_frontends() - Code Metrics - Inspection of "Many core improvements and fixes for 1.1" - isomeric/isomer - Measure and Improve Code Quality continuously with Scrutinizer

Test Failed

Pull Request — master (#2)

by Heiko 'riot'

created 2020-10-02 13:08 UTC

isomer.ui.builder.update_frontends() F

↳ Parent: isomer.ui.builder

Complexity

Conditions

Size

Total Lines	123
Code Lines	68

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
cc	17
eloc	68
nop	3
dl	0
loc	123
rs	1.8
c	0
b	0
f	0

How to fix Long Method Complexity

#!/usr/bin/env python
# -*- coding: UTF-8 -*-

# Isomer - The distributed application framework
# ==============================================
# Copyright (C) 2011-2020 Heiko 'riot' Weinen <[email protected]> and others.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""
Frontend building process.

Since this involves a lot of javascript handling, it is best advised to not
directly use any of the functionality except `install_frontend` and maybe
`rebuild_frontend`.
"""

import json
import os
import shutil
from glob import glob
from shutil import copy

import pkg_resources
from isomer.logger import isolog, debug, verbose, warn, error, critical
from isomer.misc.path import get_path
from isomer.tool import run_process


def log(*args, **kwargs):
    """Log as builder emitter"""
    kwargs.update({"emitter": "BUILDER", "frame_ref": 2})
    isolog(*args, **kwargs)


# TODO: Move the copy resource/directory tree operations to a utility lib

def copy_directory_tree(root_src_dir: str, root_dst_dir: str, hardlink: bool = False,
                        move: bool = False):
    """Copies/links/moves a whole directory tree

    :param root_src_dir: Source filesystem location
    :param root_dst_dir: Target filesystem location
    :param hardlink: Create hardlinks instead of copying (experimental)
    :param move: Move whole directory
    """

    for src_dir, dirs, files in os.walk(root_src_dir):
        dst_dir = src_dir.replace(root_src_dir, root_dst_dir, 1)
        if not os.path.exists(dst_dir):
            os.makedirs(dst_dir)
        for file_ in files:
            src_file = os.path.join(src_dir, file_)
            dst_file = os.path.join(dst_dir, file_)
            try:
                if os.path.exists(dst_file):
                    if hardlink:
                        log("Removing destination:", dst_file, lvl=verbose)
                        os.remove(dst_file)
                    else:
                        log("Overwriting destination:", dst_file, lvl=verbose)
                else:
                    log("Destination not existing:", dst_file, lvl=verbose)
            except PermissionError as e:
                log("No permission to remove destination:", e, lvl=error)

            try:
                if hardlink:
                    log("Hardlinking ", src_file, dst_dir, lvl=verbose)
                    os.link(src_file, dst_file)
                elif move:
                    log("Moving", src_file, dst_dir)
                    shutil.move(src_file, dst_dir)
                else:
                    log("Copying ", src_file, dst_dir, lvl=verbose)
                    copy(src_file, dst_dir)
            except PermissionError as e:
                log(
                    " No permission to create destination %s for directory:"
                    % ("link" if hardlink else "copy/move"),
                    dst_dir,
                    e,
                    lvl=error,
                )
            except Exception as e:
                log("Error during copy_directory_tree operation:",
                    type(e), e, lvl=error)

            log("Done with tree:", root_dst_dir, lvl=verbose)


def copy_resource_tree(package: str, source: str, target: str):
    """Copies a whole resource tree

    :param package: Package object with resources
    :param source: Source folder inside package resources
    :param target: Filesystem destination
    """

    pkg = pkg_resources.Requirement.parse(package)

    log(
        "Copying component frontend tree for %s to %s (%s)" % (package, target, source),
        lvl=verbose,
    )

    if not os.path.exists(target):
        os.mkdir(target)

    for item in pkg_resources.resource_listdir(pkg, source):
        log("Handling resource item:", item, lvl=verbose)

        if item in ("__pycache__", "__init__.py"):
            continue

        target_name = os.path.join(
            target, source.split("frontend")[1].lstrip("/"), item
        )
        log("Would copy to:", target_name, lvl=verbose)

        if pkg_resources.resource_isdir(pkg, source + "/" + item):
            log("Creating subdirectory:", target_name, lvl=verbose)
            try:
                os.mkdir(target_name)
            except FileExistsError:
                log("Subdirectory already exists, ignoring", lvl=verbose)

            log("Recursing resource subdirectory:", source + "/" + item, lvl=verbose)
            copy_resource_tree(package, source + "/" + item, target)
        else:
            log("Copying resource file:", source + "/" + item, lvl=verbose)
            with open(target_name, "wb") as f:
                f.write(pkg_resources.resource_string(pkg, source + "/" + item))


def get_frontend_locations(development):
    """Determine the frontend target and root locations.
    The root is where the complete source code for the frontend will be
    assembled, whereas the target is its installation directory after
    building
    :param development: If True, uses the development frontend server location
    :return:
    """

    log("Checking frontend location", lvl=debug)

    if development is True:
        log("Using development frontend location", lvl=warn)
        root = os.path.realpath(
            os.path.dirname(os.path.realpath(__file__)) + "/../../frontend"
        )
        target = get_path("lib", "frontend-dev")
        if not os.path.exists(target):
            log("Creating development frontend folder", lvl=debug)
            try:
                os.makedirs(target)
            except PermissionError:
                log(
                    "Cannot create development frontend target! "
                    "Check permissions on",
                    target,
                )
                return None, None
    else:
        log("Using production frontend location", lvl=debug)
        root = get_path("lib", "repository/frontend")
        target = get_path("lib", "frontend")

    log("Frontend components located in", root, lvl=debug)

    return root, target


def generate_component_folders(folder):
    """If not existing, create the components' holding folder inside the
    frontend source tree

    :param folder: Target folder in the frontend's source, where frontend
    modules will be copied to
    """

    if not os.path.isdir(folder):
        log("Creating new components folder")
        os.makedirs(folder)
    else:
        log("Clearing components folder")
        for thing in os.listdir(folder):
            target = os.path.join(folder, thing)

            try:
                shutil.rmtree(target)
            except NotADirectoryError:
                os.unlink(target)
            except PermissionError:
                log(
                    "Cannot remove data in old components folder! "
                    "Check permissions in",
                    folder,
                    thing,
                    lvl=warn,
                )


def get_components(frontend_root):
    """Iterate over all installed isomer modules to find all the isomer
    components frontends and their dependencies
    :param frontend_root: Frontend source root directory
    :return:
    """

    def inspect_entry_point(component_entry_point):
        """Use pkg_tools to inspect an installed module for its metadata

        :param component_entry_point: A single entrypoint for an isomer module
        """

        name = component_entry_point.name
        package = component_entry_point.dist.project_name
        location = component_entry_point.dist.location
        loaded = component_entry_point.load()

        log("Package:", package, lvl=debug)

        log(
            "Entry point: ",
            component_entry_point,
            name,
            component_entry_point.resolve().__module__,
            lvl=debug,
        )
        component_name = component_entry_point.resolve().__module__.split(".")[1]

        log("Loaded: ", loaded, lvl=verbose)
        component = {
            "location": location,
            "version": str(component_entry_point.dist.parsed_version),
            "description": loaded.__doc__,
            "package": package,
        }

        try:
            pkg = pkg_resources.Requirement.parse(package)
            log("Checking component data resources", lvl=debug)
            try:
                resources = pkg_resources.resource_listdir(pkg, "frontend")
            except FileNotFoundError:
                log("Component does not have a frontend", lvl=debug)
                resources = []

            if len(resources) > 0:
                component["frontend"] = resources
                component["method"] = "resources"
        except ModuleNotFoundError:

            frontend = os.path.join(location, "frontend")
            log("Checking component data folders ", frontend, lvl=verbose)
            if os.path.isdir(frontend) and frontend != frontend_root:
                component["frontend"] = frontend
                component["method"] = "folder"

        if "frontend" not in component:
            log(
                "Component without frontend directory:",
                component,
                lvl=debug,
            )
            return None, None
        return component_name, component

    log("Updating frontend components")

    inspected_components = {}
    inspected_locations = []
    try:
        from pkg_resources import iter_entry_points

        entry_point_tuple = (
            iter_entry_points(group="isomer.sails", name=None),
            iter_entry_points(group="isomer.components", name=None),
        )

        for iterator in entry_point_tuple:
            for entry_point in iterator:
                try:
                    inspectable_package = entry_point.dist.project_name

                    if inspectable_package == "isomer":
                        log("Not inspecting base isomer package", lvl=debug)
                        continue

                    inspected_name, inspected_component = inspect_entry_point(
                        entry_point)

                    if inspected_name is not None and \
                            inspected_component is not None:
                        location = inspected_component['location']
                        if location not in inspected_locations:
                            inspected_locations.append(location)
                            inspected_components[inspected_name] = inspected_component
                except Exception as e:
                    log(
                        "Could not inspect entrypoint: ",
                        e,
                        type(e),
                        entry_point,
                        iterator,
                        lvl=error,
                        exc=True,
                    )

        # frontends = iter_entry_points(group='isomer.frontend', name=None)
        # for entrypoint in frontends:
        #     name = entrypoint.name
        #     location = entrypoint.dist.location
        #
        #     log('Frontend entrypoint:', name, location, entrypoint, lvl=hilight)

    except Exception as e:
        log("Error during frontend install: ", e, type(e), lvl=error, exc=True)

    component_list = list(inspected_components.keys())
    log("Components after lookup (%i):" % len(component_list),
        sorted(component_list))

    return inspected_components


def update_frontends(frontend_components: dict, frontend_root: str, install: bool):
    """Installs all found entrypoints and returns the list of all required
    dependencies

    :param frontend_root: Frontend source root directory
    :param install: If true, collect installable dependencies
    :param frontend_components: Dictionary with component names and metadata
    :return:
    """

    def get_component_dependencies(pkg_method: str, pkg_origin: str, pkg_name: str,
                                   pkg_object):
        """Inspect components resource or requirement strings to collect
        their dependencies

        :param pkg_method: Method how the dependencies are stored,
            either 'folder' or 'resources'. Folder expects a requirements.txt
            with javascript dependencies, whereas resources expects them
            inside the setup.py of the module
        :param pkg_origin: Folder with the module's frontend root
        :param pkg_name: Name of the entrypoint
        :param pkg_object: Entrypoint object
        """

        packages = []

        if pkg_method == "folder":
            requirements_file = os.path.join(pkg_origin, "requirements.txt")

            if os.path.exists(requirements_file):
                log(
                    "Adding package dependencies for",
                    pkg_name,
                    lvl=debug,
                )
                with open(requirements_file, "r") as f:
                    for requirements_line in f.readlines():
                        packages.append(requirements_line.replace("\n", ""))
        elif pkg_method == "resources":
            log("Getting resources:", pkg_object, lvl=debug)
            resource = pkg_resources.Requirement.parse(pkg_object)
            if pkg_resources.resource_exists(
                resource, "frontend/requirements.txt"
            ):
                resource_string = pkg_resources.resource_string(
                    resource, "frontend/requirements.txt"
                )

                # TODO: Not sure if decoding to ascii is a smart
                #  idea for npm package names.
                for resource_line in (
                    resource_string.decode("ascii").rstrip("\n").split("\n")
                ):
                    log("Resource string:", resource_line, lvl=debug)
                    packages.append(resource_line.replace("\n", ""))

        return packages

    def install_frontend_data(pkg_object, pkg_name: str):
        """Gather all frontend components' data files and while inspecting
        the components, collect their dependencies, as well, if frontend
        installation has been requested

        :param pkg_object: Setuptools entrypoint descriptor
        :param pkg_name: Name of the entrypoint
        """

        origin = pkg_object["frontend"]
        method = pkg_object["method"]
        package_object = pkg_object.get("package", None)
        target = os.path.join(frontend_root, "src", "components", pkg_name)
        target = os.path.normpath(target)

        if install:
            module_dependencies = get_component_dependencies(
                method, origin, pkg_name,
                package_object
            )
        else:
            module_dependencies = []

        log("Copying:", origin, target, lvl=debug)
        if method == "folder":
            copy_directory_tree(origin, target)
        elif method == "resources":
            copy_resource_tree(package_object, "frontend", target)

        for module_filename in glob(target + "/*.module.js"):
            module_name = os.path.basename(module_filename).split(".module.js")[
                0
            ]
            module_line = (
                u"import {s} from './components/{p}/{s}.module';\n"
                u"modules.push({s});\n".format(s=module_name, p=pkg_name)
            )

            yield module_dependencies, module_line, module_name

    log("Checking unique frontend locations: ", frontend_components, lvl=debug,
        pretty=True)

    importable_modules = []
    dependency_packages = []
    modules = []  # For checking if we already got it

    for package_name, package_component in frontend_components.items():
        if "frontend" in package_component:
            for dependencies, import_line, module in install_frontend_data(package_component, package_name):
                if module not in modules:
                    modules += module
                    if len(dependencies) > 0:
                        dependency_packages += dependencies
                    importable_modules.append(import_line)

        else:
            log("Module without frontend:", package_name, package_component,
                lvl=debug)

    log("Dependencies:", dependency_packages, "Component Imports:",
        importable_modules, pretty=True,
        lvl=debug)

    return dependency_packages, importable_modules


def get_sails_dependencies(root):
    """Get all core user interface (sails) dependencies

    :param root: Frontend source root directory
    """

    packages = []

    with open(os.path.join(root, 'package.json'), 'r') as f:
        package_json = json.load(f)

    log('Adding deployment packages', lvl=verbose)
    for package, version in package_json['dependencies'].items():
        packages.append("@".join([package, version]))

    log('Adding development packages', lvl=verbose)
    for package, version in package_json['devDependencies'].items():
        packages.append("@".join([package, version]))

    log('Found %i isomer base dependencies' % len(packages), lvl=debug)
    return packages


def install_dependencies(dependency_list: list, frontend_root: str):
    """Instruct npm to install a list of all dependencies

    :param frontend_root: Frontend source root directory
    :param dependency_list: List of javascript dependency packages
    """

    log("Installing dependencies:", dependency_list, lvl=debug)
    command_line = ["npm", "install", "--no-save"] + dependency_list

    log("Using npm in:", frontend_root, lvl=debug)
    success, installer = run_process(frontend_root, command_line)

    if success:
        log("Frontend installing done.", lvl=debug)
    else:
        log("Could not install dependencies:", installer)


def write_main(importable_modules: list, root: str):
    """With the gathered importable modules, populate the main frontend
    loader and write it to the frontend's root

    :param importable_modules: List of importable javascript module files
    :param root: Frontend source root directory
    """

    log("Writing main frontend loader", lvl=debug)
    with open(os.path.join(root, "src", "main.tpl.js"), "r") as f:
        main = "".join(f.readlines())

    parts = main.split("/* COMPONENT SECTION */")
    if len(parts) != 3:
        log("Frontend loader seems damaged! Please check!", lvl=critical)
        return

    try:
        with open(os.path.join(root, "src", "main.js"), "w") as f:
            f.write(parts[0])
            f.write("/* COMPONENT SECTION:BEGIN */\n")
            for line in importable_modules:
                f.write(line)
            f.write("/* COMPONENT SECTION:END */\n")
            f.write(parts[2])
    except Exception as write_exception:
        log(
            "Error during frontend package info writing. Check " "permissions! ",
            write_exception,
            lvl=error,
        )


def rebuild_frontend(root: str, target: str, build_type: str):
    """Instruct npm to rebuild the frontend

    :param root: Frontend source root directory
    :param target: frontend build target installation directory
    :param build_type: Type of frontend build, either 'dist' or 'build'
    :return:

    """

    log("Starting frontend build.", lvl=warn)

    # TODO: Switch to i.t.run_process
    log("Using npm in:", root, lvl=debug)
    command = ["npm", "run", build_type]
    success, builder_output = run_process(root, command)

    if success is False:
        log("Error during frontend build:", builder_output, lvl=error)
        return

    log("Frontend build done: ", builder_output, lvl=debug)

    try:
        copy_directory_tree(
            os.path.join(root, build_type), target, hardlink=False
        )
        copy_directory_tree(
            os.path.join(root, "assets"),
            os.path.join(target, "assets"),
            hardlink=False,
        )
    except PermissionError:
        log("No permission to change:", target, lvl=error)

    log("Frontend deployed")


def install_frontend(
    force_rebuild: bool = False,
    install: bool = True,
    development: bool = False,
    build_type: str = "dist",
):
    """Builds and installs the frontend.

    The process works like this:

    * Find the frontend locations (source root and target)
    * Generate the target component folders to copy modules' frontend sources to
    * Gather all component meta data
    * Collect all dependencies (when installing them is desired) and their module
      imports
    * If desired, install all dependencies
    * Write the frontend main loader with all module entrypoints
    * Run npm build `BUILD_TYPE` and copy all resulting files to the frontend target
      folder

    :param force_rebuild: Trigger a rebuild of the sources.
    :param install: Trigger installation of the frontend's dependencies
    :param development: Use development frontend server locations
    :param build_type: Type of frontend build, either 'dist' or 'build'
    """

    frontend_root, frontend_target = get_frontend_locations(development)

    if frontend_root is None or frontend_target is None:
        log("Cannot determine either frontend root or target, please inspect",
            lvl=error)
        return

    component_folder = os.path.join(frontend_root, "src", "components")
    generate_component_folders(component_folder)

    components = get_components(frontend_root)

    installation_packages, imports = update_frontends(
        components, frontend_root, install
    )

    if install:
        installation_packages += get_sails_dependencies(frontend_root)
        install_dependencies(installation_packages, frontend_root)

    write_main(imports, frontend_root)

    if force_rebuild:
        rebuild_frontend(frontend_root, frontend_target, build_type)

    log("Done: Install Frontend")

    # TODO: We have to find a way to detect if we need to rebuild (and
    #  possibly wipe) stuff. This maybe the case, when a frontend
    #  module has been updated/added/removed.


1			#!/usr/bin/env python
2			# -- coding: UTF-8 --
3
4			# Isomer - The distributed application framework
5			# ==============================================
6			# Copyright (C) 2011-2020 Heiko 'riot' Weinen <[email protected]> and others.
7			#
8			# This program is free software: you can redistribute it and/or modify
9			# it under the terms of the GNU Affero General Public License as published by
10			# the Free Software Foundation, either version 3 of the License, or
11			# (at your option) any later version.
12			#
13			# This program is distributed in the hope that it will be useful,
14			# but WITHOUT ANY WARRANTY; without even the implied warranty of
15			# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16			# GNU Affero General Public License for more details.
17			#
18			# You should have received a copy of the GNU Affero General Public License
19			# along with this program. If not, see <http://www.gnu.org/licenses/>.
20
21			"""
22			Frontend building process.
23
24			Since this involves a lot of javascript handling, it is best advised to not
25			directly use any of the functionality except `install_frontend` and maybe
26			`rebuild_frontend`.
27			"""
28
29			import json
30			import os
31			import shutil
32			from glob import glob
33			from shutil import copy
34
35			import pkg_resources
36			from isomer.logger import isolog, debug, verbose, warn, error, critical
37			from isomer.misc.path import get_path
38			from isomer.tool import run_process
39
40
41			def log(args, *kwargs):
42			"""Log as builder emitter"""
43			kwargs.update({"emitter": "BUILDER", "frame_ref": 2})
44			isolog(args, *kwargs)
45
46
47			# TODO: Move the copy resource/directory tree operations to a utility lib
48
49			def copy_directory_tree(root_src_dir: str, root_dst_dir: str, hardlink: bool = False,
50			move: bool = False):
51			"""Copies/links/moves a whole directory tree
52
53			:param root_src_dir: Source filesystem location
54			:param root_dst_dir: Target filesystem location
55			:param hardlink: Create hardlinks instead of copying (experimental)
56			:param move: Move whole directory
57			"""
58
59			for src_dir, dirs, files in os.walk(root_src_dir):
60			dst_dir = src_dir.replace(root_src_dir, root_dst_dir, 1)
61			if not os.path.exists(dst_dir):
62			os.makedirs(dst_dir)
63			for file_ in files:
64			src_file = os.path.join(src_dir, file_)
65			dst_file = os.path.join(dst_dir, file_)
66			try:
67			if os.path.exists(dst_file):
68			if hardlink:
69			log("Removing destination:", dst_file, lvl=verbose)
70			os.remove(dst_file)
71			else:
72			log("Overwriting destination:", dst_file, lvl=verbose)
73			else:
74			log("Destination not existing:", dst_file, lvl=verbose)
75			except PermissionError as e:
76			log("No permission to remove destination:", e, lvl=error)
77
78			try:
79			if hardlink:
80			log("Hardlinking ", src_file, dst_dir, lvl=verbose)
81			os.link(src_file, dst_file)
82			elif move:
83			log("Moving", src_file, dst_dir)
84			shutil.move(src_file, dst_dir)
85			else:
86			log("Copying ", src_file, dst_dir, lvl=verbose)
87			copy(src_file, dst_dir)
88			except PermissionError as e:
89			log(
90			" No permission to create destination %s for directory:"
91			% ("link" if hardlink else "copy/move"),
92			dst_dir,
93			e,
94			lvl=error,
95			)
96			except Exception as e:
97			log("Error during copy_directory_tree operation:",
98			type(e), e, lvl=error)
99
100			log("Done with tree:", root_dst_dir, lvl=verbose)
101
102
103			def copy_resource_tree(package: str, source: str, target: str):
104			"""Copies a whole resource tree
105
106			:param package: Package object with resources
107			:param source: Source folder inside package resources
108			:param target: Filesystem destination
109			"""
110
111			pkg = pkg_resources.Requirement.parse(package)
112
113			log(
114			"Copying component frontend tree for %s to %s (%s)" % (package, target, source),
115			lvl=verbose,
116			)
117
118			if not os.path.exists(target):
119			os.mkdir(target)
120
121			for item in pkg_resources.resource_listdir(pkg, source):
122			log("Handling resource item:", item, lvl=verbose)
123
124			if item in ("__pycache__", "__init__.py"):
125			continue
126
127			target_name = os.path.join(
128			target, source.split("frontend")[1].lstrip("/"), item
129			)
130			log("Would copy to:", target_name, lvl=verbose)
131
132			if pkg_resources.resource_isdir(pkg, source + "/" + item):
133			log("Creating subdirectory:", target_name, lvl=verbose)
134			try:
135			os.mkdir(target_name)
136			except FileExistsError:
137			log("Subdirectory already exists, ignoring", lvl=verbose)
138
139			log("Recursing resource subdirectory:", source + "/" + item, lvl=verbose)
140			copy_resource_tree(package, source + "/" + item, target)
141			else:
142			log("Copying resource file:", source + "/" + item, lvl=verbose)
143			with open(target_name, "wb") as f:
144			f.write(pkg_resources.resource_string(pkg, source + "/" + item))
145
146
147			def get_frontend_locations(development):
148			"""Determine the frontend target and root locations.
149			The root is where the complete source code for the frontend will be
150			assembled, whereas the target is its installation directory after
151			building
152			:param development: If True, uses the development frontend server location
153			:return:
154			"""
155
156			log("Checking frontend location", lvl=debug)
157
158			if development is True:
159			log("Using development frontend location", lvl=warn)
160			root = os.path.realpath(
161			os.path.dirname(os.path.realpath(__file__)) + "/../../frontend"
162			)
163			target = get_path("lib", "frontend-dev")
164			if not os.path.exists(target):
165			log("Creating development frontend folder", lvl=debug)
166			try:
167			os.makedirs(target)
168			except PermissionError:
169			log(
170			"Cannot create development frontend target! "
171			"Check permissions on",
172			target,
173			)
174			return None, None
175			else:
176			log("Using production frontend location", lvl=debug)
177			root = get_path("lib", "repository/frontend")
178			target = get_path("lib", "frontend")
179
180			log("Frontend components located in", root, lvl=debug)
181
182			return root, target
183
184
185			def generate_component_folders(folder):
186			"""If not existing, create the components' holding folder inside the
187			frontend source tree
188
189			:param folder: Target folder in the frontend's source, where frontend
190			modules will be copied to
191			"""
192
193			if not os.path.isdir(folder):
194			log("Creating new components folder")
195			os.makedirs(folder)
196			else:
197			log("Clearing components folder")
198			for thing in os.listdir(folder):
199			target = os.path.join(folder, thing)
200
201			try:
202			shutil.rmtree(target)
203			except NotADirectoryError:
204			os.unlink(target)
205			except PermissionError:
206			log(
207			"Cannot remove data in old components folder! "
208			"Check permissions in",
209			folder,
210			thing,
211			lvl=warn,
212			)
213
214
215			def get_components(frontend_root):
216			"""Iterate over all installed isomer modules to find all the isomer
217			components frontends and their dependencies
218			:param frontend_root: Frontend source root directory
219			:return:
220			"""
221
222			def inspect_entry_point(component_entry_point):
223			"""Use pkg_tools to inspect an installed module for its metadata
224
225			:param component_entry_point: A single entrypoint for an isomer module
226			"""
227
228			name = component_entry_point.name
229			package = component_entry_point.dist.project_name
230			location = component_entry_point.dist.location
231			loaded = component_entry_point.load()
232
233			log("Package:", package, lvl=debug)
234
235			log(
236			"Entry point: ",
237			component_entry_point,
238			name,
239			component_entry_point.resolve().__module__,
240			lvl=debug,
241			)
242			component_name = component_entry_point.resolve().__module__.split(".")[1]
243
244			log("Loaded: ", loaded, lvl=verbose)
245			component = {
246			"location": location,
247			"version": str(component_entry_point.dist.parsed_version),
248			"description": loaded.__doc__,
249			"package": package,
250			}
251
252			try:
253			pkg = pkg_resources.Requirement.parse(package)
254			log("Checking component data resources", lvl=debug)
255			try:
256			resources = pkg_resources.resource_listdir(pkg, "frontend")
257			except FileNotFoundError:
258			log("Component does not have a frontend", lvl=debug)
259			resources = []
260
261			if len(resources) > 0:
262			component["frontend"] = resources
263			component["method"] = "resources"
264			except ModuleNotFoundError:
			0 ignored issues – show Comprehensibility Best Practice introduced 2020-02-27 17:41 UTC by Report Bug Copy Issue Report The variable `ModuleNotFoundError` does not seem to be defined. Loading history...
265			frontend = os.path.join(location, "frontend")
266			log("Checking component data folders ", frontend, lvl=verbose)
267			if os.path.isdir(frontend) and frontend != frontend_root:
268			component["frontend"] = frontend
269			component["method"] = "folder"
270
271			if "frontend" not in component:
272			log(
273			"Component without frontend directory:",
274			component,
275			lvl=debug,
276			)
277			return None, None
278			return component_name, component
279
280			log("Updating frontend components")
281
282			inspected_components = {}
283			inspected_locations = []
284			try:
285			from pkg_resources import iter_entry_points
286
287			entry_point_tuple = (
288			iter_entry_points(group="isomer.sails", name=None),
289			iter_entry_points(group="isomer.components", name=None),
290			)
291
292			for iterator in entry_point_tuple:
293			for entry_point in iterator:
294			try:
295			inspectable_package = entry_point.dist.project_name
296
297			if inspectable_package == "isomer":
298			log("Not inspecting base isomer package", lvl=debug)
299			continue
300
301			inspected_name, inspected_component = inspect_entry_point(
302			entry_point)
303
304			if inspected_name is not None and \
305			inspected_component is not None:
306			location = inspected_component['location']
307			if location not in inspected_locations:
308			inspected_locations.append(location)
309			inspected_components[inspected_name] = inspected_component
310			except Exception as e:
311			log(
312			"Could not inspect entrypoint: ",
313			e,
314			type(e),
315			entry_point,
316			iterator,
317			lvl=error,
318			exc=True,
319			)
320
321			# frontends = iter_entry_points(group='isomer.frontend', name=None)
322			# for entrypoint in frontends:
323			# name = entrypoint.name
324			# location = entrypoint.dist.location
325			#
326			# log('Frontend entrypoint:', name, location, entrypoint, lvl=hilight)
327
328			except Exception as e:
329			log("Error during frontend install: ", e, type(e), lvl=error, exc=True)
330
331			component_list = list(inspected_components.keys())
332			log("Components after lookup (%i):" % len(component_list),
333			sorted(component_list))
334
335			return inspected_components
336
337
338			def update_frontends(frontend_components: dict, frontend_root: str, install: bool):
339			"""Installs all found entrypoints and returns the list of all required
340			dependencies
341
342			:param frontend_root: Frontend source root directory
343			:param install: If true, collect installable dependencies
344			:param frontend_components: Dictionary with component names and metadata
345			:return:
346			"""
347
348			def get_component_dependencies(pkg_method: str, pkg_origin: str, pkg_name: str,
349			pkg_object):
350			"""Inspect components resource or requirement strings to collect
351			their dependencies
352
353			:param pkg_method: Method how the dependencies are stored,
354			either 'folder' or 'resources'. Folder expects a requirements.txt
355			with javascript dependencies, whereas resources expects them
356			inside the setup.py of the module
357			:param pkg_origin: Folder with the module's frontend root
358			:param pkg_name: Name of the entrypoint
359			:param pkg_object: Entrypoint object
360			"""
361
362			packages = []
363
364			if pkg_method == "folder":
365			requirements_file = os.path.join(pkg_origin, "requirements.txt")
366
367			if os.path.exists(requirements_file):
368			log(
369			"Adding package dependencies for",
370			pkg_name,
371			lvl=debug,
372			)
373			with open(requirements_file, "r") as f:
374			for requirements_line in f.readlines():
375			packages.append(requirements_line.replace("\n", ""))
376			elif pkg_method == "resources":
377			log("Getting resources:", pkg_object, lvl=debug)
378			resource = pkg_resources.Requirement.parse(pkg_object)
379			if pkg_resources.resource_exists(
380			resource, "frontend/requirements.txt"
381			):
382			resource_string = pkg_resources.resource_string(
383			resource, "frontend/requirements.txt"
384			)
385
386			# TODO: Not sure if decoding to ascii is a smart
387			# idea for npm package names.
388			for resource_line in (
389			resource_string.decode("ascii").rstrip("\n").split("\n")
390			):
391			log("Resource string:", resource_line, lvl=debug)
392			packages.append(resource_line.replace("\n", ""))
393
394			return packages
395
396			def install_frontend_data(pkg_object, pkg_name: str):
397			"""Gather all frontend components' data files and while inspecting
398			the components, collect their dependencies, as well, if frontend
399			installation has been requested
400
401			:param pkg_object: Setuptools entrypoint descriptor
402			:param pkg_name: Name of the entrypoint
403			"""
404
405			origin = pkg_object["frontend"]
406			method = pkg_object["method"]
407			package_object = pkg_object.get("package", None)
408			target = os.path.join(frontend_root, "src", "components", pkg_name)
409			target = os.path.normpath(target)
410
411			if install:
412			module_dependencies = get_component_dependencies(
413			method, origin, pkg_name,
414			package_object
415			)
416			else:
417			module_dependencies = []
418
419			log("Copying:", origin, target, lvl=debug)
420			if method == "folder":
421			copy_directory_tree(origin, target)
422			elif method == "resources":
423			copy_resource_tree(package_object, "frontend", target)
424
425			for module_filename in glob(target + "/*.module.js"):
426			module_name = os.path.basename(module_filename).split(".module.js")[
427			0
428			]
429			module_line = (
430			u"import {s} from './components/{p}/{s}.module';\n"
431			u"modules.push({s});\n".format(s=module_name, p=pkg_name)
432			)
433
434			yield module_dependencies, module_line, module_name
435
436			log("Checking unique frontend locations: ", frontend_components, lvl=debug,
437			pretty=True)
438
439			importable_modules = []
440			dependency_packages = []
441			modules = [] # For checking if we already got it
442
443			for package_name, package_component in frontend_components.items():
444			if "frontend" in package_component:
445			for dependencies, import_line, module in install_frontend_data(package_component, package_name):
446			if module not in modules:
447			modules += module
448			if len(dependencies) > 0:
449			dependency_packages += dependencies
450			importable_modules.append(import_line)
451
452			else:
453			log("Module without frontend:", package_name, package_component,
454			lvl=debug)
455
456			log("Dependencies:", dependency_packages, "Component Imports:",
457			importable_modules, pretty=True,
458			lvl=debug)
459
460			return dependency_packages, importable_modules
461
462
463			def get_sails_dependencies(root):
464			"""Get all core user interface (sails) dependencies
465
466			:param root: Frontend source root directory
467			"""
468
469			packages = []
470
471			with open(os.path.join(root, 'package.json'), 'r') as f:
472			package_json = json.load(f)
473
474			log('Adding deployment packages', lvl=verbose)
475			for package, version in package_json['dependencies'].items():
476			packages.append("@".join([package, version]))
477
478			log('Adding development packages', lvl=verbose)
479			for package, version in package_json['devDependencies'].items():
480			packages.append("@".join([package, version]))
481
482			log('Found %i isomer base dependencies' % len(packages), lvl=debug)
483			return packages
484
485
486			def install_dependencies(dependency_list: list, frontend_root: str):
487			"""Instruct npm to install a list of all dependencies
488
489			:param frontend_root: Frontend source root directory
490			:param dependency_list: List of javascript dependency packages
491			"""
492
493			log("Installing dependencies:", dependency_list, lvl=debug)
494			command_line = ["npm", "install", "--no-save"] + dependency_list
495
496			log("Using npm in:", frontend_root, lvl=debug)
497			success, installer = run_process(frontend_root, command_line)
498
499			if success:
500			log("Frontend installing done.", lvl=debug)
501			else:
502			log("Could not install dependencies:", installer)
503
504
505			def write_main(importable_modules: list, root: str):
506			"""With the gathered importable modules, populate the main frontend
507			loader and write it to the frontend's root
508
509			:param importable_modules: List of importable javascript module files
510			:param root: Frontend source root directory
511			"""
512
513			log("Writing main frontend loader", lvl=debug)
514			with open(os.path.join(root, "src", "main.tpl.js"), "r") as f:
515			main = "".join(f.readlines())
516
517			parts = main.split("/* COMPONENT SECTION */")
518			if len(parts) != 3:
519			log("Frontend loader seems damaged! Please check!", lvl=critical)
520			return
521
522			try:
523			with open(os.path.join(root, "src", "main.js"), "w") as f:
524			f.write(parts[0])
525			f.write("/* COMPONENT SECTION:BEGIN */\n")
526			for line in importable_modules:
527			f.write(line)
528			f.write("/* COMPONENT SECTION:END */\n")
529			f.write(parts[2])
530			except Exception as write_exception:
531			log(
532			"Error during frontend package info writing. Check " "permissions! ",
533			write_exception,
534			lvl=error,
535			)
536
537
538			def rebuild_frontend(root: str, target: str, build_type: str):
539			"""Instruct npm to rebuild the frontend
540
541			:param root: Frontend source root directory
542			:param target: frontend build target installation directory
543			:param build_type: Type of frontend build, either 'dist' or 'build'
544			:return:
545
546			"""
547
548			log("Starting frontend build.", lvl=warn)
549
550			# TODO: Switch to i.t.run_process
551			log("Using npm in:", root, lvl=debug)
552			command = ["npm", "run", build_type]
553			success, builder_output = run_process(root, command)
554
555			if success is False:
556			log("Error during frontend build:", builder_output, lvl=error)
557			return
558
559			log("Frontend build done: ", builder_output, lvl=debug)
560
561			try:
562			copy_directory_tree(
563			os.path.join(root, build_type), target, hardlink=False
564			)
565			copy_directory_tree(
566			os.path.join(root, "assets"),
567			os.path.join(target, "assets"),
568			hardlink=False,
569			)
570			except PermissionError:
571			log("No permission to change:", target, lvl=error)
572
573			log("Frontend deployed")
574
575
576			def install_frontend(
577			force_rebuild: bool = False,
578			install: bool = True,
579			development: bool = False,
580			build_type: str = "dist",
581			):
582			"""Builds and installs the frontend.
583
584			The process works like this:
585
586			* Find the frontend locations (source root and target)
587			* Generate the target component folders to copy modules' frontend sources to
588			* Gather all component meta data
589			* Collect all dependencies (when installing them is desired) and their module
590			imports
591			* If desired, install all dependencies
592			* Write the frontend main loader with all module entrypoints
593			* Run npm build `BUILD_TYPE` and copy all resulting files to the frontend target
594			folder
595
596			:param force_rebuild: Trigger a rebuild of the sources.
597			:param install: Trigger installation of the frontend's dependencies
598			:param development: Use development frontend server locations
599			:param build_type: Type of frontend build, either 'dist' or 'build'
600			"""
601
602			frontend_root, frontend_target = get_frontend_locations(development)
603
604			if frontend_root is None or frontend_target is None:
605			log("Cannot determine either frontend root or target, please inspect",
606			lvl=error)
607			return
608
609			component_folder = os.path.join(frontend_root, "src", "components")
610			generate_component_folders(component_folder)
611
612			components = get_components(frontend_root)
613
614			installation_packages, imports = update_frontends(
615			components, frontend_root, install
616			)
617
618			if install:
619			installation_packages += get_sails_dependencies(frontend_root)
620			install_dependencies(installation_packages, frontend_root)
621
622			write_main(imports, frontend_root)
623
624			if force_rebuild:
625			rebuild_frontend(frontend_root, frontend_target, build_type)
626
627			log("Done: Install Frontend")
628
629			# TODO: We have to find a way to detect if we need to rebuild (and
630			# possibly wipe) stuff. This maybe the case, when a frontend
631			# module has been updated/added/removed.
632

isomeric / isomer

Pull Request — master (#2)

isomer.ui.builder.update_frontends() F

Complexity

Size

Duplication

Importance

How to fix Long Method Complexity

Long Method

Complexity

Duplication Side-by-Side

Filter issues like