isomer.ui.builder.generate_component_folders() - Code Metrics - Inspection of "removes (hopefully last ;) moved HFOS content" - isomeric/isomer - Measure and Improve Code Quality continuously with Scrutinizer

Test Failed

Push — master ( 1c66f3...c17ee3 )

by Heiko 'riot'

created 2020-12-20 19:23 UTC

isomer.ui.builder.generate_component_folders() B

↳ Parent: isomer.ui.builder

Complexity

Conditions

Size

Total Lines	37
Code Lines	24

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
cc	5
eloc	24
nop	1
dl	0
loc	37
rs	8.8373
c	0
b	0
f	0

#!/usr/bin/env python
# -*- coding: UTF-8 -*-

# Isomer - The distributed application framework
# ==============================================
# Copyright (C) 2011-2020 Heiko 'riot' Weinen <[email protected]> and others.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""
Frontend building process.

Since this involves a lot of javascript handling, it is best advised to not
directly use any of the functionality except `install_frontend` and maybe
`rebuild_frontend`.
"""

import json
import os
import shutil
from glob import glob
from shutil import copy

import pkg_resources
from isomer.logger import isolog, debug, verbose, warn, error, critical
from isomer.misc.path import get_path
from isomer.tool import run_process


def log(*args, **kwargs):
    """Log as builder emitter"""
    kwargs.update({"emitter": "BUILDER", "frame_ref": 2})
    isolog(*args, **kwargs)


# TODO: Move the copy resource/directory tree operations to a utility lib

def copy_directory_tree(root_src_dir: str, root_dst_dir: str, hardlink: bool = False,
                        move: bool = False):
    """Copies/links/moves a whole directory tree

    :param root_src_dir: Source filesystem location
    :param root_dst_dir: Target filesystem location
    :param hardlink: Create hardlinks instead of copying (experimental)
    :param move: Move whole directory
    """

    for src_dir, dirs, files in os.walk(root_src_dir):
        dst_dir = src_dir.replace(root_src_dir, root_dst_dir, 1)
        if not os.path.exists(dst_dir):
            os.makedirs(dst_dir)
        for file_ in files:
            src_file = os.path.join(src_dir, file_)
            dst_file = os.path.join(dst_dir, file_)
            try:
                if os.path.exists(dst_file):
                    if hardlink:
                        log("Removing destination:", dst_file, lvl=verbose)
                        os.remove(dst_file)
                    else:
                        log("Overwriting destination:", dst_file, lvl=verbose)
                else:
                    log("Destination not existing:", dst_file, lvl=verbose)
            except PermissionError as e:
                log("No permission to remove destination:", e, lvl=error)

            try:
                if hardlink:
                    log("Hardlinking ", src_file, dst_dir, lvl=verbose)
                    os.link(src_file, dst_file)
                elif move:
                    log("Moving", src_file, dst_dir)
                    shutil.move(src_file, dst_dir)
                else:
                    log("Copying ", src_file, dst_dir, lvl=verbose)
                    copy(src_file, dst_dir)
            except PermissionError as e:
                log(
                    " No permission to create destination %s for directory:"
                    % ("link" if hardlink else "copy/move"),
                    dst_dir,
                    e,
                    lvl=error,
                )
            except Exception as e:
                log("Error during copy_directory_tree operation:",
                    type(e), e, lvl=error)

            log("Done with tree:", root_dst_dir, lvl=verbose)


def copy_resource_tree(package: str, source: str, target: str):
    """Copies a whole resource tree

    :param package: Package object with resources
    :param source: Source folder inside package resources
    :param target: Filesystem destination
    """

    pkg = pkg_resources.Requirement.parse(package)

    log(
        "Copying component frontend tree for %s to %s (%s)" % (package, target, source),
        lvl=verbose,
    )

    if not os.path.exists(target):
        os.mkdir(target)

    for item in pkg_resources.resource_listdir(pkg, source):
        log("Handling resource item:", item, lvl=verbose)

        if item in ("__pycache__", "__init__.py"):
            continue

        target_name = os.path.join(
            target, source.split("frontend")[1].lstrip("/"), item
        )
        log("Would copy to:", target_name, lvl=verbose)

        if pkg_resources.resource_isdir(pkg, source + "/" + item):
            log("Creating subdirectory:", target_name, lvl=verbose)
            try:
                os.mkdir(target_name)
            except FileExistsError:
                log("Subdirectory already exists, ignoring", lvl=verbose)

            log("Recursing resource subdirectory:", source + "/" + item, lvl=verbose)
            copy_resource_tree(package, source + "/" + item, target)
        else:
            log("Copying resource file:", source + "/" + item, lvl=verbose)
            with open(target_name, "wb") as f:
                f.write(pkg_resources.resource_string(pkg, source + "/" + item))


def get_frontend_locations(development):
    """Determine the frontend target and root locations.
    The root is where the complete source code for the frontend will be
    assembled, whereas the target is its installation directory after
    building
    :param development: If True, uses the development frontend server location
    :return:
    """

    log("Checking frontend location", lvl=debug)

    if development is True:
        log("Using development frontend location", lvl=warn)
        root = os.path.realpath(
            os.path.dirname(os.path.realpath(__file__)) + "/../../frontend"
        )
        target = get_path("lib", "frontend-dev")
        if not os.path.exists(target):
            log("Creating development frontend folder", lvl=debug)
            try:
                os.makedirs(target)
            except PermissionError:
                log(
                    "Cannot create development frontend target! "
                    "Check permissions on",
                    target,
                )
                return None, None
    else:
        log("Using production frontend location", lvl=debug)
        root = get_path("lib", "repository/frontend")
        target = get_path("lib", "frontend")

    log("Frontend components located in", root, lvl=debug)

    return root, target


def generate_component_folders(folder):
    """If not existing, create the components' holding folder inside the
    frontend source tree

    :param folder: Target folder in the frontend's source, where frontend
    modules will be copied to
    """

    if not os.path.isdir(folder):
        log("Creating new components folder")
        os.makedirs(folder)
        return True
    else:
        log("Clearing components folder")
        for thing in os.listdir(folder):
            target = os.path.join(folder, thing)

            try:
                log("Deleting target '%s'", lvl=debug)
                shutil.rmtree(target)
            except NotADirectoryError:
                log("Target '%s' is a symbolical link, unlinking", lvl=debug)
                os.unlink(target)
            except PermissionError:
                log(
                    "Cannot remove data in old components folder! "
                    "Check permissions in",
                    folder,
                    thing,
                    lvl=warn,
                )
                return False
            finally:
                log("Cannot clear old components, giving up", lvl=error, exc=True)
                return False

    return True

def get_components(frontend_root):
    """Iterate over all installed isomer modules to find all the isomer
    components frontends and their dependencies
    :param frontend_root: Frontend source root directory
    :return:
    """

    def inspect_entry_point(component_entry_point):
        """Use pkg_tools to inspect an installed module for its metadata

        :param component_entry_point: A single entrypoint for an isomer module
        """

        name = component_entry_point.name
        package = component_entry_point.dist.project_name
        location = component_entry_point.dist.location
        loaded = component_entry_point.load()

        log("Package:", package, lvl=debug)

        log(
            "Entry point: ",
            component_entry_point,
            name,
            component_entry_point.resolve().__module__,
            lvl=debug,
        )
        component_name = component_entry_point.resolve().__module__.split(".")[1]

        log("Loaded: ", loaded, lvl=verbose)
        component = {
            "location": location,
            "version": str(component_entry_point.dist.parsed_version),
            "description": loaded.__doc__,
            "package": package,
        }

        try:
            pkg = pkg_resources.Requirement.parse(package)
            log("Checking component data resources", lvl=debug)
            try:
                resources = pkg_resources.resource_listdir(pkg, "frontend")
            except FileNotFoundError:
                log("Component does not have a frontend", lvl=debug)
                resources = []

            if len(resources) > 0:
                component["frontend"] = resources
                component["method"] = "resources"
        except ModuleNotFoundError:

            frontend = os.path.join(location, "frontend")
            log("Checking component data folders ", frontend, lvl=verbose)
            if os.path.isdir(frontend) and frontend != frontend_root:
                component["frontend"] = frontend
                component["method"] = "folder"

        if "frontend" not in component:
            log(
                "Component without frontend directory:",
                component,
                lvl=debug,
            )
            return None, None
        return component_name, component

    log("Updating frontend components")

    inspected_components = {}
    inspected_locations = []
    try:
        from pkg_resources import iter_entry_points

        entry_point_tuple = (
            iter_entry_points(group="isomer.sails", name=None),
            iter_entry_points(group="isomer.components", name=None),
        )

        for iterator in entry_point_tuple:
            for entry_point in iterator:
                try:
                    inspectable_package = entry_point.dist.project_name

                    if inspectable_package == "isomer":
                        log("Not inspecting base isomer package", lvl=debug)
                        continue

                    inspected_name, inspected_component = inspect_entry_point(
                        entry_point)

                    if inspected_name is not None and \
                            inspected_component is not None:
                        location = inspected_component['location']
                        if location not in inspected_locations:
                            inspected_locations.append(location)
                            inspected_components[inspected_name] = inspected_component
                except Exception as e:
                    log(
                        "Could not inspect entrypoint: ",
                        e,
                        type(e),
                        entry_point,
                        iterator,
                        lvl=error,
                        exc=True,
                    )

        # frontends = iter_entry_points(group='isomer.frontend', name=None)
        # for entrypoint in frontends:
        #     name = entrypoint.name
        #     location = entrypoint.dist.location
        #
        #     log('Frontend entrypoint:', name, location, entrypoint, lvl=hilight)

    except Exception as e:
        log("Error during frontend install: ", e, type(e), lvl=error, exc=True)
        return False

    component_list = list(inspected_components.keys())
    log("Components after lookup (%i):" % len(component_list),
        sorted(component_list))

    return inspected_components


def update_frontends(frontend_components: dict, frontend_root: str, install: bool):
    """Installs all found entrypoints and returns the list of all required
    dependencies

    :param frontend_root: Frontend source root directory
    :param install: If true, collect installable dependencies
    :param frontend_components: Dictionary with component names and metadata
    :return:
    """

    def get_component_dependencies(pkg_method: str, pkg_origin: str, pkg_name: str,
                                   pkg_object):
        """Inspect components resource or requirement strings to collect
        their dependencies

        :param pkg_method: Method how the dependencies are stored,
            either 'folder' or 'resources'. Folder expects a requirements.txt
            with javascript dependencies, whereas resources expects them
            inside the setup.py of the module
        :param pkg_origin: Folder with the module's frontend root
        :param pkg_name: Name of the entrypoint
        :param pkg_object: Entrypoint object
        """

        packages = []

        if pkg_method == "folder":
            requirements_file = os.path.join(pkg_origin, "requirements.txt")

            if os.path.exists(requirements_file):
                log(
                    "Adding package dependencies for",
                    pkg_name,
                    lvl=debug,
                )
                with open(requirements_file, "r") as f:
                    for requirements_line in f.readlines():
                        packages.append(requirements_line.replace("\n", ""))
        elif pkg_method == "resources":
            log("Getting resources:", pkg_object, lvl=debug)
            resource = pkg_resources.Requirement.parse(pkg_object)
            if pkg_resources.resource_exists(
                resource, "frontend/requirements.txt"
            ):
                resource_string = pkg_resources.resource_string(
                    resource, "frontend/requirements.txt"
                )

                # TODO: Not sure if decoding to ascii is a smart
                #  idea for npm package names.
                for resource_line in (
                    resource_string.decode("ascii").rstrip("\n").split("\n")
                ):
                    log("Resource string:", resource_line, lvl=debug)
                    packages.append(resource_line.replace("\n", ""))

        return packages

    def install_frontend_data(pkg_object, pkg_name: str):
        """Gather all frontend components' data files and while inspecting
        the components, collect their dependencies, as well, if frontend
        installation has been requested

        :param pkg_object: Setuptools entrypoint descriptor
        :param pkg_name: Name of the entrypoint
        """

        origin = pkg_object["frontend"]
        method = pkg_object["method"]
        package_object = pkg_object.get("package", None)
        target = os.path.join(frontend_root, "src", "components", pkg_name)
        target = os.path.normpath(target)

        if install:
            module_dependencies = get_component_dependencies(
                method, origin, pkg_name,
                package_object
            )
        else:
            module_dependencies = []

        log("Copying:", origin, target, lvl=debug)
        if method == "folder":
            copy_directory_tree(origin, target)
        elif method == "resources":
            copy_resource_tree(package_object, "frontend", target)

        for module_filename in glob(target + "/*.module.js"):
            module_name = os.path.basename(module_filename).split(".module.js")[
                0
            ]
            module_line = (
                u"import {s} from './components/{p}/{s}.module';\n"
                u"modules.push({s});\n".format(s=module_name, p=pkg_name)
            )

            yield module_dependencies, module_line, module_name

    log("Checking unique frontend locations: ", frontend_components, lvl=debug,
        pretty=True)

    importable_modules = []
    dependency_packages = []
    modules = []  # For checking if we already got it

    for package_name, package_component in frontend_components.items():
        if "frontend" in package_component:
            for dependencies, import_line, module in install_frontend_data(package_component, package_name):
                if module not in modules:
                    modules += module
                    if len(dependencies) > 0:
                        dependency_packages += dependencies
                    importable_modules.append(import_line)

        else:
            log("Module without frontend:", package_name, package_component,
                lvl=debug)

    log("Dependencies:", dependency_packages, "Component Imports:",
        importable_modules, pretty=True,
        lvl=debug)

    return dependency_packages, importable_modules


def get_sails_dependencies(root):
    """Get all core user interface (sails) dependencies

    :param root: Frontend source root directory
    """

    packages = []

    with open(os.path.join(root, 'package.json'), 'r') as f:
        package_json = json.load(f)

    log('Adding deployment packages', lvl=verbose)
    for package, version in package_json['dependencies'].items():
        packages.append("@".join([package, version]))

    log('Adding development packages', lvl=verbose)
    for package, version in package_json['devDependencies'].items():
        packages.append("@".join([package, version]))

    log('Found %i isomer base dependencies' % len(packages), lvl=debug)
    return packages


def install_dependencies(dependency_list: list, frontend_root: str):
    """Instruct npm to install a list of all dependencies

    :param frontend_root: Frontend source root directory
    :param dependency_list: List of javascript dependency packages
    """

    log("Installing dependencies:", dependency_list, lvl=debug)
    command_line = ["npm", "install", "--no-save"] + dependency_list

    log("Using npm in:", frontend_root, lvl=debug)
    success, installer = run_process(frontend_root, command_line)

    if success:
        log("Frontend installing done.", lvl=debug)
    else:
        log("Could not install dependencies:", installer)
        return False


def write_main(importable_modules: list, root: str):
    """With the gathered importable modules, populate the main frontend
    loader and write it to the frontend's root

    :param importable_modules: List of importable javascript module files
    :param root: Frontend source root directory
    """

    log("Writing main frontend loader", lvl=debug)
    with open(os.path.join(root, "src", "main.tpl.js"), "r") as f:
        main = "".join(f.readlines())

    parts = main.split("/* COMPONENT SECTION */")
    if len(parts) != 3:
        log("Frontend loader seems damaged! Please check!", lvl=critical)
        return False

    try:
        with open(os.path.join(root, "src", "main.js"), "w") as f:
            f.write(parts[0])
            f.write("/* COMPONENT SECTION:BEGIN */\n")
            for line in importable_modules:
                f.write(line)
            f.write("/* COMPONENT SECTION:END */\n")
            f.write(parts[2])
    except Exception as write_exception:
        log(
            "Error during frontend package info writing. Check " "permissions! ",
            write_exception,
            lvl=error,
        )
        return False


def rebuild_frontend(root: str, target: str, build_type: str):
    """Instruct npm to rebuild the frontend

    :param root: Frontend source root directory
    :param target: frontend build target installation directory
    :param build_type: Type of frontend build, either 'dist' or 'build'
    :return:

    """

    log("Starting frontend build.", lvl=warn)

    # TODO: Switch to i.t.run_process
    log("Using npm in:", root, lvl=debug)
    command = ["npm", "run", build_type]
    success, builder_output = run_process(root, command)

    if success is False:
        log("Error during frontend build:", builder_output, lvl=error)
        return False

    log("Frontend build done: ", builder_output, lvl=debug)

    try:
        copy_directory_tree(
            os.path.join(root, build_type), target, hardlink=False
        )
        copy_directory_tree(
            os.path.join(root, "assets"),
            os.path.join(target, "assets"),
            hardlink=False,
        )
    except PermissionError:
        log("No permission to change:", target, lvl=error)
        return False

    log("Frontend deployed")


def install_frontend(
    force_rebuild: bool = False,
    install: bool = True,
    development: bool = False,
    build_type: str = "dist",
):
    """Builds and installs the frontend.

    The process works like this:

    * Find the frontend locations (source root and target)
    * Generate the target component folders to copy modules' frontend sources to
    * Gather all component meta data
    * Collect all dependencies (when installing them is desired) and their module
      imports
    * If desired, install all dependencies
    * Write the frontend main loader with all module entrypoints
    * Run npm build `BUILD_TYPE` and copy all resulting files to the frontend target
      folder

    :param force_rebuild: Trigger a rebuild of the sources.
    :param install: Trigger installation of the frontend's dependencies
    :param development: Use development frontend server locations
    :param build_type: Type of frontend build, either 'dist' or 'build'
    """

    frontend_root, frontend_target = get_frontend_locations(development)

    if frontend_root is None or frontend_target is None:
        log("Cannot determine either frontend root or target",
            lvl=error)
        return False

    component_folder = os.path.join(frontend_root, "src", "components")
    if not generate_component_folders(component_folder):
        log("Cannot generate component folders",
            lvl=error)
        return False

    components = get_components(frontend_root)
    if components is False:
        log("Could not get components", lvl=error)
        return False

    installation_packages, imports = update_frontends(
        components, frontend_root, install
    )

    if install:
        installation_packages += get_sails_dependencies(frontend_root)
        installed = install_dependencies(installation_packages, frontend_root)
        if installed is False:
            log("Could not install dependencies", lvl=error)
            return False

    wrote_main = write_main(imports, frontend_root)

    if wrote_main is False:
        log("Could not write frontend loader", lvl=error)
        return False

    if force_rebuild:
        rebuilt = rebuild_frontend(frontend_root, frontend_target, build_type)
        if rebuilt is False:
            log("Frontend build failed", lvl=error)
            return False

    log("Done: Install Frontend")
    return True

    # TODO: We have to find a way to detect if we need to rebuild (and
    #  possibly wipe) stuff. This maybe the case, when a frontend
    #  module has been updated/added/removed.


1			#!/usr/bin/env python
2			# -- coding: UTF-8 --
3
4			# Isomer - The distributed application framework
5			# ==============================================
6			# Copyright (C) 2011-2020 Heiko 'riot' Weinen <[email protected]> and others.
7			#
8			# This program is free software: you can redistribute it and/or modify
9			# it under the terms of the GNU Affero General Public License as published by
10			# the Free Software Foundation, either version 3 of the License, or
11			# (at your option) any later version.
12			#
13			# This program is distributed in the hope that it will be useful,
14			# but WITHOUT ANY WARRANTY; without even the implied warranty of
15			# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16			# GNU Affero General Public License for more details.
17			#
18			# You should have received a copy of the GNU Affero General Public License
19			# along with this program. If not, see <http://www.gnu.org/licenses/>.
20
21			"""
22			Frontend building process.
23
24			Since this involves a lot of javascript handling, it is best advised to not
25			directly use any of the functionality except `install_frontend` and maybe
26			`rebuild_frontend`.
27			"""
28
29			import json
30			import os
31			import shutil
32			from glob import glob
33			from shutil import copy
34
35			import pkg_resources
36			from isomer.logger import isolog, debug, verbose, warn, error, critical
37			from isomer.misc.path import get_path
38			from isomer.tool import run_process
39
40
41			def log(args, *kwargs):
42			"""Log as builder emitter"""
43			kwargs.update({"emitter": "BUILDER", "frame_ref": 2})
44			isolog(args, *kwargs)
45
46
47			# TODO: Move the copy resource/directory tree operations to a utility lib
48
49			def copy_directory_tree(root_src_dir: str, root_dst_dir: str, hardlink: bool = False,
50			move: bool = False):
51			"""Copies/links/moves a whole directory tree
52
53			:param root_src_dir: Source filesystem location
54			:param root_dst_dir: Target filesystem location
55			:param hardlink: Create hardlinks instead of copying (experimental)
56			:param move: Move whole directory
57			"""
58
59			for src_dir, dirs, files in os.walk(root_src_dir):
60			dst_dir = src_dir.replace(root_src_dir, root_dst_dir, 1)
61			if not os.path.exists(dst_dir):
62			os.makedirs(dst_dir)
63			for file_ in files:
64			src_file = os.path.join(src_dir, file_)
65			dst_file = os.path.join(dst_dir, file_)
66			try:
67			if os.path.exists(dst_file):
68			if hardlink:
69			log("Removing destination:", dst_file, lvl=verbose)
70			os.remove(dst_file)
71			else:
72			log("Overwriting destination:", dst_file, lvl=verbose)
73			else:
74			log("Destination not existing:", dst_file, lvl=verbose)
75			except PermissionError as e:
76			log("No permission to remove destination:", e, lvl=error)
77
78			try:
79			if hardlink:
80			log("Hardlinking ", src_file, dst_dir, lvl=verbose)
81			os.link(src_file, dst_file)
82			elif move:
83			log("Moving", src_file, dst_dir)
84			shutil.move(src_file, dst_dir)
85			else:
86			log("Copying ", src_file, dst_dir, lvl=verbose)
87			copy(src_file, dst_dir)
88			except PermissionError as e:
89			log(
90			" No permission to create destination %s for directory:"
91			% ("link" if hardlink else "copy/move"),
92			dst_dir,
93			e,
94			lvl=error,
95			)
96			except Exception as e:
97			log("Error during copy_directory_tree operation:",
98			type(e), e, lvl=error)
99
100			log("Done with tree:", root_dst_dir, lvl=verbose)
101
102
103			def copy_resource_tree(package: str, source: str, target: str):
104			"""Copies a whole resource tree
105
106			:param package: Package object with resources
107			:param source: Source folder inside package resources
108			:param target: Filesystem destination
109			"""
110
111			pkg = pkg_resources.Requirement.parse(package)
112
113			log(
114			"Copying component frontend tree for %s to %s (%s)" % (package, target, source),
115			lvl=verbose,
116			)
117
118			if not os.path.exists(target):
119			os.mkdir(target)
120
121			for item in pkg_resources.resource_listdir(pkg, source):
122			log("Handling resource item:", item, lvl=verbose)
123
124			if item in ("__pycache__", "__init__.py"):
125			continue
126
127			target_name = os.path.join(
128			target, source.split("frontend")[1].lstrip("/"), item
129			)
130			log("Would copy to:", target_name, lvl=verbose)
131
132			if pkg_resources.resource_isdir(pkg, source + "/" + item):
133			log("Creating subdirectory:", target_name, lvl=verbose)
134			try:
135			os.mkdir(target_name)
136			except FileExistsError:
137			log("Subdirectory already exists, ignoring", lvl=verbose)
138
139			log("Recursing resource subdirectory:", source + "/" + item, lvl=verbose)
140			copy_resource_tree(package, source + "/" + item, target)
141			else:
142			log("Copying resource file:", source + "/" + item, lvl=verbose)
143			with open(target_name, "wb") as f:
144			f.write(pkg_resources.resource_string(pkg, source + "/" + item))
145
146
147			def get_frontend_locations(development):
148			"""Determine the frontend target and root locations.
149			The root is where the complete source code for the frontend will be
150			assembled, whereas the target is its installation directory after
151			building
152			:param development: If True, uses the development frontend server location
153			:return:
154			"""
155
156			log("Checking frontend location", lvl=debug)
157
158			if development is True:
159			log("Using development frontend location", lvl=warn)
160			root = os.path.realpath(
161			os.path.dirname(os.path.realpath(__file__)) + "/../../frontend"
162			)
163			target = get_path("lib", "frontend-dev")
164			if not os.path.exists(target):
165			log("Creating development frontend folder", lvl=debug)
166			try:
167			os.makedirs(target)
168			except PermissionError:
169			log(
170			"Cannot create development frontend target! "
171			"Check permissions on",
172			target,
173			)
174			return None, None
175			else:
176			log("Using production frontend location", lvl=debug)
177			root = get_path("lib", "repository/frontend")
178			target = get_path("lib", "frontend")
179
180			log("Frontend components located in", root, lvl=debug)
181
182			return root, target
183
184
185			def generate_component_folders(folder):
186			"""If not existing, create the components' holding folder inside the
187			frontend source tree
188
189			:param folder: Target folder in the frontend's source, where frontend
190			modules will be copied to
191			"""
192
193			if not os.path.isdir(folder):
194			log("Creating new components folder")
195			os.makedirs(folder)
196			return True
197			else:
198			log("Clearing components folder")
199			for thing in os.listdir(folder):
200			target = os.path.join(folder, thing)
201
202			try:
203			log("Deleting target '%s'", lvl=debug)
204			shutil.rmtree(target)
205			except NotADirectoryError:
206			log("Target '%s' is a symbolical link, unlinking", lvl=debug)
207			os.unlink(target)
208			except PermissionError:
209			log(
210			"Cannot remove data in old components folder! "
211			"Check permissions in",
212			folder,
213			thing,
214			lvl=warn,
215			)
216			return False
217			finally:
218			log("Cannot clear old components, giving up", lvl=error, exc=True)
219			return False
220
221			return True
222
223			def get_components(frontend_root):
224			"""Iterate over all installed isomer modules to find all the isomer
225			components frontends and their dependencies
226			:param frontend_root: Frontend source root directory
227			:return:
228			"""
229
230			def inspect_entry_point(component_entry_point):
231			"""Use pkg_tools to inspect an installed module for its metadata
232
233			:param component_entry_point: A single entrypoint for an isomer module
234			"""
235
236			name = component_entry_point.name
237			package = component_entry_point.dist.project_name
238			location = component_entry_point.dist.location
239			loaded = component_entry_point.load()
240
241			log("Package:", package, lvl=debug)
242
243			log(
244			"Entry point: ",
245			component_entry_point,
246			name,
247			component_entry_point.resolve().__module__,
248			lvl=debug,
249			)
250			component_name = component_entry_point.resolve().__module__.split(".")[1]
251
252			log("Loaded: ", loaded, lvl=verbose)
253			component = {
254			"location": location,
255			"version": str(component_entry_point.dist.parsed_version),
256			"description": loaded.__doc__,
257			"package": package,
258			}
259
260			try:
261			pkg = pkg_resources.Requirement.parse(package)
262			log("Checking component data resources", lvl=debug)
263			try:
264			resources = pkg_resources.resource_listdir(pkg, "frontend")
265			except FileNotFoundError:
266			log("Component does not have a frontend", lvl=debug)
267			resources = []
268
269			if len(resources) > 0:
270			component["frontend"] = resources
271			component["method"] = "resources"
272			except ModuleNotFoundError:
			0 ignored issues – show Comprehensibility Best Practice introduced 2020-02-27 17:41 UTC by Report Bug Copy Issue Report The variable `ModuleNotFoundError` does not seem to be defined. Loading history...
273			frontend = os.path.join(location, "frontend")
274			log("Checking component data folders ", frontend, lvl=verbose)
275			if os.path.isdir(frontend) and frontend != frontend_root:
276			component["frontend"] = frontend
277			component["method"] = "folder"
278
279			if "frontend" not in component:
280			log(
281			"Component without frontend directory:",
282			component,
283			lvl=debug,
284			)
285			return None, None
286			return component_name, component
287
288			log("Updating frontend components")
289
290			inspected_components = {}
291			inspected_locations = []
292			try:
293			from pkg_resources import iter_entry_points
294
295			entry_point_tuple = (
296			iter_entry_points(group="isomer.sails", name=None),
297			iter_entry_points(group="isomer.components", name=None),
298			)
299
300			for iterator in entry_point_tuple:
301			for entry_point in iterator:
302			try:
303			inspectable_package = entry_point.dist.project_name
304
305			if inspectable_package == "isomer":
306			log("Not inspecting base isomer package", lvl=debug)
307			continue
308
309			inspected_name, inspected_component = inspect_entry_point(
310			entry_point)
311
312			if inspected_name is not None and \
313			inspected_component is not None:
314			location = inspected_component['location']
315			if location not in inspected_locations:
316			inspected_locations.append(location)
317			inspected_components[inspected_name] = inspected_component
318			except Exception as e:
319			log(
320			"Could not inspect entrypoint: ",
321			e,
322			type(e),
323			entry_point,
324			iterator,
325			lvl=error,
326			exc=True,
327			)
328
329			# frontends = iter_entry_points(group='isomer.frontend', name=None)
330			# for entrypoint in frontends:
331			# name = entrypoint.name
332			# location = entrypoint.dist.location
333			#
334			# log('Frontend entrypoint:', name, location, entrypoint, lvl=hilight)
335
336			except Exception as e:
337			log("Error during frontend install: ", e, type(e), lvl=error, exc=True)
338			return False
339
340			component_list = list(inspected_components.keys())
341			log("Components after lookup (%i):" % len(component_list),
342			sorted(component_list))
343
344			return inspected_components
345
346
347			def update_frontends(frontend_components: dict, frontend_root: str, install: bool):
348			"""Installs all found entrypoints and returns the list of all required
349			dependencies
350
351			:param frontend_root: Frontend source root directory
352			:param install: If true, collect installable dependencies
353			:param frontend_components: Dictionary with component names and metadata
354			:return:
355			"""
356
357			def get_component_dependencies(pkg_method: str, pkg_origin: str, pkg_name: str,
358			pkg_object):
359			"""Inspect components resource or requirement strings to collect
360			their dependencies
361
362			:param pkg_method: Method how the dependencies are stored,
363			either 'folder' or 'resources'. Folder expects a requirements.txt
364			with javascript dependencies, whereas resources expects them
365			inside the setup.py of the module
366			:param pkg_origin: Folder with the module's frontend root
367			:param pkg_name: Name of the entrypoint
368			:param pkg_object: Entrypoint object
369			"""
370
371			packages = []
372
373			if pkg_method == "folder":
374			requirements_file = os.path.join(pkg_origin, "requirements.txt")
375
376			if os.path.exists(requirements_file):
377			log(
378			"Adding package dependencies for",
379			pkg_name,
380			lvl=debug,
381			)
382			with open(requirements_file, "r") as f:
383			for requirements_line in f.readlines():
384			packages.append(requirements_line.replace("\n", ""))
385			elif pkg_method == "resources":
386			log("Getting resources:", pkg_object, lvl=debug)
387			resource = pkg_resources.Requirement.parse(pkg_object)
388			if pkg_resources.resource_exists(
389			resource, "frontend/requirements.txt"
390			):
391			resource_string = pkg_resources.resource_string(
392			resource, "frontend/requirements.txt"
393			)
394
395			# TODO: Not sure if decoding to ascii is a smart
396			# idea for npm package names.
397			for resource_line in (
398			resource_string.decode("ascii").rstrip("\n").split("\n")
399			):
400			log("Resource string:", resource_line, lvl=debug)
401			packages.append(resource_line.replace("\n", ""))
402
403			return packages
404
405			def install_frontend_data(pkg_object, pkg_name: str):
406			"""Gather all frontend components' data files and while inspecting
407			the components, collect their dependencies, as well, if frontend
408			installation has been requested
409
410			:param pkg_object: Setuptools entrypoint descriptor
411			:param pkg_name: Name of the entrypoint
412			"""
413
414			origin = pkg_object["frontend"]
415			method = pkg_object["method"]
416			package_object = pkg_object.get("package", None)
417			target = os.path.join(frontend_root, "src", "components", pkg_name)
418			target = os.path.normpath(target)
419
420			if install:
421			module_dependencies = get_component_dependencies(
422			method, origin, pkg_name,
423			package_object
424			)
425			else:
426			module_dependencies = []
427
428			log("Copying:", origin, target, lvl=debug)
429			if method == "folder":
430			copy_directory_tree(origin, target)
431			elif method == "resources":
432			copy_resource_tree(package_object, "frontend", target)
433
434			for module_filename in glob(target + "/*.module.js"):
435			module_name = os.path.basename(module_filename).split(".module.js")[
436			0
437			]
438			module_line = (
439			u"import {s} from './components/{p}/{s}.module';\n"
440			u"modules.push({s});\n".format(s=module_name, p=pkg_name)
441			)
442
443			yield module_dependencies, module_line, module_name
444
445			log("Checking unique frontend locations: ", frontend_components, lvl=debug,
446			pretty=True)
447
448			importable_modules = []
449			dependency_packages = []
450			modules = [] # For checking if we already got it
451
452			for package_name, package_component in frontend_components.items():
453			if "frontend" in package_component:
454			for dependencies, import_line, module in install_frontend_data(package_component, package_name):
455			if module not in modules:
456			modules += module
457			if len(dependencies) > 0:
458			dependency_packages += dependencies
459			importable_modules.append(import_line)
460
461			else:
462			log("Module without frontend:", package_name, package_component,
463			lvl=debug)
464
465			log("Dependencies:", dependency_packages, "Component Imports:",
466			importable_modules, pretty=True,
467			lvl=debug)
468
469			return dependency_packages, importable_modules
470
471
472			def get_sails_dependencies(root):
473			"""Get all core user interface (sails) dependencies
474
475			:param root: Frontend source root directory
476			"""
477
478			packages = []
479
480			with open(os.path.join(root, 'package.json'), 'r') as f:
481			package_json = json.load(f)
482
483			log('Adding deployment packages', lvl=verbose)
484			for package, version in package_json['dependencies'].items():
485			packages.append("@".join([package, version]))
486
487			log('Adding development packages', lvl=verbose)
488			for package, version in package_json['devDependencies'].items():
489			packages.append("@".join([package, version]))
490
491			log('Found %i isomer base dependencies' % len(packages), lvl=debug)
492			return packages
493
494
495			def install_dependencies(dependency_list: list, frontend_root: str):
496			"""Instruct npm to install a list of all dependencies
497
498			:param frontend_root: Frontend source root directory
499			:param dependency_list: List of javascript dependency packages
500			"""
501
502			log("Installing dependencies:", dependency_list, lvl=debug)
503			command_line = ["npm", "install", "--no-save"] + dependency_list
504
505			log("Using npm in:", frontend_root, lvl=debug)
506			success, installer = run_process(frontend_root, command_line)
507
508			if success:
509			log("Frontend installing done.", lvl=debug)
510			else:
511			log("Could not install dependencies:", installer)
512			return False
513
514
515			def write_main(importable_modules: list, root: str):
516			"""With the gathered importable modules, populate the main frontend
517			loader and write it to the frontend's root
518
519			:param importable_modules: List of importable javascript module files
520			:param root: Frontend source root directory
521			"""
522
523			log("Writing main frontend loader", lvl=debug)
524			with open(os.path.join(root, "src", "main.tpl.js"), "r") as f:
525			main = "".join(f.readlines())
526
527			parts = main.split("/* COMPONENT SECTION */")
528			if len(parts) != 3:
529			log("Frontend loader seems damaged! Please check!", lvl=critical)
530			return False
531
532			try:
533			with open(os.path.join(root, "src", "main.js"), "w") as f:
534			f.write(parts[0])
535			f.write("/* COMPONENT SECTION:BEGIN */\n")
536			for line in importable_modules:
537			f.write(line)
538			f.write("/* COMPONENT SECTION:END */\n")
539			f.write(parts[2])
540			except Exception as write_exception:
541			log(
542			"Error during frontend package info writing. Check " "permissions! ",
543			write_exception,
544			lvl=error,
545			)
546			return False
547
548
549			def rebuild_frontend(root: str, target: str, build_type: str):
550			"""Instruct npm to rebuild the frontend
551
552			:param root: Frontend source root directory
553			:param target: frontend build target installation directory
554			:param build_type: Type of frontend build, either 'dist' or 'build'
555			:return:
556
557			"""
558
559			log("Starting frontend build.", lvl=warn)
560
561			# TODO: Switch to i.t.run_process
562			log("Using npm in:", root, lvl=debug)
563			command = ["npm", "run", build_type]
564			success, builder_output = run_process(root, command)
565
566			if success is False:
567			log("Error during frontend build:", builder_output, lvl=error)
568			return False
569
570			log("Frontend build done: ", builder_output, lvl=debug)
571
572			try:
573			copy_directory_tree(
574			os.path.join(root, build_type), target, hardlink=False
575			)
576			copy_directory_tree(
577			os.path.join(root, "assets"),
578			os.path.join(target, "assets"),
579			hardlink=False,
580			)
581			except PermissionError:
582			log("No permission to change:", target, lvl=error)
583			return False
584
585			log("Frontend deployed")
586
587
588			def install_frontend(
589			force_rebuild: bool = False,
590			install: bool = True,
591			development: bool = False,
592			build_type: str = "dist",
593			):
594			"""Builds and installs the frontend.
595
596			The process works like this:
597
598			* Find the frontend locations (source root and target)
599			* Generate the target component folders to copy modules' frontend sources to
600			* Gather all component meta data
601			* Collect all dependencies (when installing them is desired) and their module
602			imports
603			* If desired, install all dependencies
604			* Write the frontend main loader with all module entrypoints
605			* Run npm build `BUILD_TYPE` and copy all resulting files to the frontend target
606			folder
607
608			:param force_rebuild: Trigger a rebuild of the sources.
609			:param install: Trigger installation of the frontend's dependencies
610			:param development: Use development frontend server locations
611			:param build_type: Type of frontend build, either 'dist' or 'build'
612			"""
613
614			frontend_root, frontend_target = get_frontend_locations(development)
615
616			if frontend_root is None or frontend_target is None:
617			log("Cannot determine either frontend root or target",
618			lvl=error)
619			return False
620
621			component_folder = os.path.join(frontend_root, "src", "components")
622			if not generate_component_folders(component_folder):
623			log("Cannot generate component folders",
624			lvl=error)
625			return False
626
627			components = get_components(frontend_root)
628			if components is False:
629			log("Could not get components", lvl=error)
630			return False
631
632			installation_packages, imports = update_frontends(
633			components, frontend_root, install
634			)
635
636			if install:
637			installation_packages += get_sails_dependencies(frontend_root)
638			installed = install_dependencies(installation_packages, frontend_root)
639			if installed is False:
640			log("Could not install dependencies", lvl=error)
641			return False
642
643			wrote_main = write_main(imports, frontend_root)
644
645			if wrote_main is False:
646			log("Could not write frontend loader", lvl=error)
647			return False
648
649			if force_rebuild:
650			rebuilt = rebuild_frontend(frontend_root, frontend_target, build_type)
651			if rebuilt is False:
652			log("Frontend build failed", lvl=error)
653			return False
654
655			log("Done: Install Frontend")
656			return True
657
658			# TODO: We have to find a way to detect if we need to rebuild (and
659			# possibly wipe) stuff. This maybe the case, when a frontend
660			# module has been updated/added/removed.
661

isomeric / isomer

Push — master ( 1c66f3...c17ee3 )

isomer.ui.builder.generate_component_folders() B

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like