ui.builder

Complexity

Total Complexity

86

Size/Duplication

Total Lines	655
Duplicated Lines	0 %

Importance

Changes

0

12 Functions

Rating	Name	Duplication	Size	Complexity
D	copy_directory_tree()	0	52	12
A	log()	0	4	1
B	write_main()	0	32	6
A	get_sails_dependencies()	0	21	4
A	install_dependencies()	0	18	2
F	get_components()	0	122	15
F	update_frontends()	0	123	17
A	get_frontend_locations()	0	36	4
B	generate_component_folders()	0	34	5
C	install_frontend()	0	69	10
B	copy_resource_tree()	0	42	7
A	rebuild_frontend()	0	37	3

#!/usr/bin/env python
# -*- coding: UTF-8 -*-

# Isomer - The distributed application framework
# ==============================================
# Copyright (C) 2011-2020 Heiko 'riot' Weinen <riot@c-base.org> and others.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""
Frontend building process.

Since this involves a lot of javascript handling, it is best advised to not
directly use any of the functionality except `install_frontend` and maybe
`rebuild_frontend`.
"""

import json
import os
import shutil
from glob import glob
from shutil import copy

import pkg_resources
from isomer.logger import isolog, debug, verbose, warn, error, critical
from isomer.misc.path import get_path
from isomer.tool import run_process


def log(*args, **kwargs):
    """Log as builder emitter"""
    kwargs.update({"emitter": "BUILDER", "frame_ref": 2})
    isolog(*args, **kwargs)


# TODO: Move the copy resource/directory tree operations to a utility lib

def copy_directory_tree(root_src_dir: str, root_dst_dir: str, hardlink: bool = False,
                        move: bool = False):
    """Copies/links/moves a whole directory tree

    :param root_src_dir: Source filesystem location
    :param root_dst_dir: Target filesystem location
    :param hardlink: Create hardlinks instead of copying (experimental)
    :param move: Move whole directory
    """

    for src_dir, dirs, files in os.walk(root_src_dir):
        dst_dir = src_dir.replace(root_src_dir, root_dst_dir, 1)
        if not os.path.exists(dst_dir):
            os.makedirs(dst_dir)
        for file_ in files:
            src_file = os.path.join(src_dir, file_)
            dst_file = os.path.join(dst_dir, file_)
            try:
                if os.path.exists(dst_file):
                    if hardlink:
                        log("Removing destination:", dst_file, lvl=verbose)
                        os.remove(dst_file)
                    else:
                        log("Overwriting destination:", dst_file, lvl=verbose)
                else:
                    log("Destination not existing:", dst_file, lvl=verbose)
            except PermissionError as e:
                log("No permission to remove destination:", e, lvl=error)

            try:
                if hardlink:
                    log("Hardlinking ", src_file, dst_dir, lvl=verbose)
                    os.link(src_file, dst_file)
                elif move:
                    log("Moving", src_file, dst_dir)
                    shutil.move(src_file, dst_dir)
                else:
                    log("Copying ", src_file, dst_dir, lvl=verbose)
                    copy(src_file, dst_dir)
            except PermissionError as e:
                log(
                    " No permission to create destination %s for directory:"
                    % ("link" if hardlink else "copy/move"),
                    dst_dir,
                    e,
                    lvl=error,
                )
            except Exception as e:
                log("Error during copy_directory_tree operation:",
                    type(e), e, lvl=error)

            log("Done with tree:", root_dst_dir, lvl=verbose)


def copy_resource_tree(package: str, source: str, target: str):
    """Copies a whole resource tree

    :param package: Package object with resources
    :param source: Source folder inside package resources
    :param target: Filesystem destination
    """

    pkg = pkg_resources.Requirement.parse(package)

    log(
        "Copying component frontend tree for %s to %s (%s)" % (package, target, source),
        lvl=verbose,
    )

    if not os.path.exists(target):
        os.mkdir(target)

    for item in pkg_resources.resource_listdir(pkg, source):
        log("Handling resource item:", item, lvl=verbose)

        if item in ("__pycache__", "__init__.py"):
            continue

        target_name = os.path.join(
            target, source.split("frontend")[1].lstrip("/"), item
        )
        log("Would copy to:", target_name, lvl=verbose)

        if pkg_resources.resource_isdir(pkg, source + "/" + item):
            log("Creating subdirectory:", target_name, lvl=verbose)
            try:
                os.mkdir(target_name)
            except FileExistsError:
                log("Subdirectory already exists, ignoring", lvl=verbose)

            log("Recursing resource subdirectory:", source + "/" + item, lvl=verbose)
            copy_resource_tree(package, source + "/" + item, target)
        else:
            log("Copying resource file:", source + "/" + item, lvl=verbose)
            with open(target_name, "wb") as f:
                f.write(pkg_resources.resource_string(pkg, source + "/" + item))


def get_frontend_locations(development):
    """Determine the frontend target and root locations.
    The root is where the complete source code for the frontend will be
    assembled, whereas the target is its installation directory after
    building
    :param development: If True, uses the development frontend server location
    :return:
    """

    log("Checking frontend location", lvl=debug)

    if development is True:
        log("Using development frontend location", lvl=warn)
        root = os.path.realpath(
            os.path.dirname(os.path.realpath(__file__)) + "/../../frontend"
        )
        target = get_path("lib", "frontend-dev")
        if not os.path.exists(target):
            log("Creating development frontend folder", lvl=debug)
            try:
                os.makedirs(target)
            except PermissionError:
                log(
                    "Cannot create development frontend target! "
                    "Check permissions on",
                    target,
                )
                return None, None
    else:
        log("Using production frontend location", lvl=debug)
        root = get_path("lib", "repository/frontend")
        target = get_path("lib", "frontend")

    log("Frontend components located in", root, lvl=debug)

    return root, target


def generate_component_folders(folder):
    """If not existing, create the components' holding folder inside the
    frontend source tree

    :param folder: Target folder in the frontend's source, where frontend
    modules will be copied to
    """

    if not os.path.isdir(folder):
        log("Creating new components folder")
        os.makedirs(folder)
        return True
    else:
        log("Clearing components folder")
        for thing in os.listdir(folder):
            target = os.path.join(folder, thing)

            try:
                log("Deleting target '%s'", lvl=debug)
                shutil.rmtree(target)
            except NotADirectoryError:
                log("Target '%s' is a symbolical link, unlinking", lvl=debug)
                os.unlink(target)
            except PermissionError:
                log(
                    "Cannot remove data in old components folder! "
                    "Check permissions in",
                    folder,
                    thing,
                    lvl=warn,
                )
                return False

    return True


def get_components(frontend_root):
    """Iterate over all installed isomer modules to find all the isomer
    components frontends and their dependencies
    :param frontend_root: Frontend source root directory
    :return:
    """

    def inspect_entry_point(component_entry_point):
        """Use pkg_tools to inspect an installed module for its metadata

        :param component_entry_point: A single entrypoint for an isomer module
        """

        name = component_entry_point.name
        package = component_entry_point.dist.project_name
        location = component_entry_point.dist.location
        loaded = component_entry_point.load()

        log("Package:", package, lvl=debug)

        log(
            "Entry point: ",
            component_entry_point,
            name,
            component_entry_point.resolve().__module__,
            lvl=debug,
        )
        component_name = component_entry_point.resolve().__module__.split(".")[1]

        log("Loaded: ", loaded, lvl=verbose)
        component = {
            "location": location,
            "version": str(component_entry_point.dist.parsed_version),
            "description": loaded.__doc__,
            "package": package,
        }

        try:
            pkg = pkg_resources.Requirement.parse(package)
            log("Checking component data resources", lvl=debug)
            try:
                resources = pkg_resources.resource_listdir(pkg, "frontend")
            except FileNotFoundError:
                log("Component does not have a frontend", lvl=debug)
                resources = []

            if len(resources) > 0:
                component["frontend"] = resources
                component["method"] = "resources"
        except ModuleNotFoundError:

The variable ModuleNotFoundError does not seem to be defined.

            frontend = os.path.join(location, "frontend")
            log("Checking component data folders ", frontend, lvl=verbose)
            if os.path.isdir(frontend) and frontend != frontend_root:
                component["frontend"] = frontend
                component["method"] = "folder"

        if "frontend" not in component:
            log(
                "Component without frontend directory:",
                component,
                lvl=debug,
            )
            return None, None
        return component_name, component

    log("Updating frontend components")

    inspected_components = {}
    inspected_locations = []
    try:
        from pkg_resources import iter_entry_points

        entry_point_tuple = (
            iter_entry_points(group="isomer.sails", name=None),
            iter_entry_points(group="isomer.components", name=None),
        )

        for iterator in entry_point_tuple:
            for entry_point in iterator:
                try:
                    inspectable_package = entry_point.dist.project_name

                    if inspectable_package == "isomer":
                        log("Not inspecting base isomer package", lvl=debug)
                        continue

                    inspected_name, inspected_component = inspect_entry_point(
                        entry_point)

                    if inspected_name is not None and \
                            inspected_component is not None:
                        location = inspected_component['location']
                        if location not in inspected_locations:
                            inspected_locations.append(location)
                            inspected_components[inspected_name] = inspected_component
                except Exception as e:
                    log(
                        "Could not inspect entrypoint: ",
                        e,
                        type(e),
                        entry_point,
                        iterator,
                        lvl=error,
                        exc=True,
                    )

        # frontends = iter_entry_points(group='isomer.frontend', name=None)
        # for entrypoint in frontends:
        #     name = entrypoint.name
        #     location = entrypoint.dist.location
        #
        #     log('Frontend entrypoint:', name, location, entrypoint, lvl=hilight)

    except Exception as e:
        log("Error during frontend install: ", e, type(e), lvl=error, exc=True)
        return False

    component_list = list(inspected_components.keys())
    log("Components after lookup (%i):" % len(component_list),
        sorted(component_list))

    return inspected_components


def update_frontends(frontend_components: dict, frontend_root: str, install: bool):
    """Installs all found entrypoints and returns the list of all required
    dependencies

    :param frontend_root: Frontend source root directory
    :param install: If true, collect installable dependencies
    :param frontend_components: Dictionary with component names and metadata
    :return:
    """

    def get_component_dependencies(pkg_method: str, pkg_origin: str, pkg_name: str,
                                   pkg_object):
        """Inspect components resource or requirement strings to collect
        their dependencies

        :param pkg_method: Method how the dependencies are stored,
            either 'folder' or 'resources'. Folder expects a requirements.txt
            with javascript dependencies, whereas resources expects them
            inside the setup.py of the module
        :param pkg_origin: Folder with the module's frontend root
        :param pkg_name: Name of the entrypoint
        :param pkg_object: Entrypoint object
        """

        packages = []

        if pkg_method == "folder":
            requirements_file = os.path.join(pkg_origin, "requirements.txt")

            if os.path.exists(requirements_file):
                log(
                    "Adding package dependencies for",
                    pkg_name,
                    lvl=debug,
                )
                with open(requirements_file, "r") as f:
                    for requirements_line in f.readlines():
                        packages.append(requirements_line.replace("\n", ""))
        elif pkg_method == "resources":
            log("Getting resources:", pkg_object, lvl=debug)
            resource = pkg_resources.Requirement.parse(pkg_object)
            if pkg_resources.resource_exists(
                resource, "frontend/requirements.txt"
            ):
                resource_string = pkg_resources.resource_string(
                    resource, "frontend/requirements.txt"
                )

                # TODO: Not sure if decoding to ascii is a smart
                #  idea for npm package names.
                for resource_line in (
                    resource_string.decode("ascii").rstrip("\n").split("\n")
                ):
                    log("Resource string:", resource_line, lvl=debug)
                    packages.append(resource_line.replace("\n", ""))

        return packages

    def install_frontend_data(pkg_object, pkg_name: str):
        """Gather all frontend components' data files and while inspecting
        the components, collect their dependencies, as well, if frontend
        installation has been requested

        :param pkg_object: Setuptools entrypoint descriptor
        :param pkg_name: Name of the entrypoint
        """

        origin = pkg_object["frontend"]
        method = pkg_object["method"]
        package_object = pkg_object.get("package", None)
        target = os.path.join(frontend_root, "src", "components", pkg_name)
        target = os.path.normpath(target)

        if install:
            module_dependencies = get_component_dependencies(
                method, origin, pkg_name,
                package_object
            )
        else:
            module_dependencies = []

        log("Copying:", origin, target, lvl=debug)
        if method == "folder":
            copy_directory_tree(origin, target)
        elif method == "resources":
            copy_resource_tree(package_object, "frontend", target)

        for module_filename in glob(target + "/*.module.js"):
            module_name = os.path.basename(module_filename).split(".module.js")[
                0
            ]
            module_line = (
                u"import {s} from './components/{p}/{s}.module';\n"
                u"modules.push({s});\n".format(s=module_name, p=pkg_name)
            )

            yield module_dependencies, module_line, module_name

    log("Checking unique frontend locations: ", frontend_components, lvl=debug,
        pretty=True)

    importable_modules = []
    dependency_packages = []
    modules = []  # For checking if we already got it

    for package_name, package_component in frontend_components.items():
        if "frontend" in package_component:
            for dependencies, import_line, module in install_frontend_data(package_component, package_name):
                if module not in modules:
                    modules += module
                    if len(dependencies) > 0:
                        dependency_packages += dependencies
                    importable_modules.append(import_line)

        else:
            log("Module without frontend:", package_name, package_component,
                lvl=debug)

    log("Dependencies:", dependency_packages, "Component Imports:",
        importable_modules, pretty=True,
        lvl=debug)

    return dependency_packages, importable_modules


def get_sails_dependencies(root):
    """Get all core user interface (sails) dependencies

    :param root: Frontend source root directory
    """

    packages = []

    with open(os.path.join(root, 'package.json'), 'r') as f:
        package_json = json.load(f)

    log('Adding deployment packages', lvl=verbose)
    for package, version in package_json['dependencies'].items():
        packages.append("@".join([package, version]))

    log('Adding development packages', lvl=verbose)
    for package, version in package_json['devDependencies'].items():
        packages.append("@".join([package, version]))

    log('Found %i isomer base dependencies' % len(packages), lvl=debug)
    return packages


def install_dependencies(dependency_list: list, frontend_root: str):
    """Instruct npm to install a list of all dependencies

    :param frontend_root: Frontend source root directory
    :param dependency_list: List of javascript dependency packages
    """

    log("Installing dependencies:", dependency_list, lvl=debug)
    command_line = ["npm", "install", "--no-save"] + dependency_list

    log("Using npm in:", frontend_root, lvl=debug)
    success, installer = run_process(frontend_root, command_line)

    if success:
        log("Frontend installing done.", lvl=debug)
    else:
        log("Could not install dependencies:", installer)
        return False


def write_main(importable_modules: list, root: str):
    """With the gathered importable modules, populate the main frontend
    loader and write it to the frontend's root

    :param importable_modules: List of importable javascript module files
    :param root: Frontend source root directory
    """

    log("Writing main frontend loader", lvl=debug)
    with open(os.path.join(root, "src", "main.tpl.js"), "r") as f:
        main = "".join(f.readlines())

    parts = main.split("/* COMPONENT SECTION */")
    if len(parts) != 3:
        log("Frontend loader seems damaged! Please check!", lvl=critical)
        return False

    try:
        with open(os.path.join(root, "src", "main.js"), "w") as f:
            f.write(parts[0])
            f.write("/* COMPONENT SECTION:BEGIN */\n")
            for line in importable_modules:
                f.write(line)
            f.write("/* COMPONENT SECTION:END */\n")
            f.write(parts[2])
    except Exception as write_exception:
        log(
            "Error during frontend package info writing. Check " "permissions! ",
            write_exception,
            lvl=error,
        )
        return False


def rebuild_frontend(root: str, target: str, build_type: str):
    """Instruct npm to rebuild the frontend

    :param root: Frontend source root directory
    :param target: frontend build target installation directory
    :param build_type: Type of frontend build, either 'dist' or 'build'
    :return:

    """

    log("Starting frontend build.", lvl=warn)

    # TODO: Switch to i.t.run_process
    log("Using npm in:", root, lvl=debug)
    command = ["npm", "run", build_type]
    success, builder_output = run_process(root, command)

    if success is False:
        log("Error during frontend build:", builder_output, lvl=error)
        return False

    log("Frontend build done: ", builder_output, lvl=debug)

    try:
        copy_directory_tree(
            os.path.join(root, build_type), target, hardlink=False
        )
        copy_directory_tree(
            os.path.join(root, "assets"),
            os.path.join(target, "assets"),
            hardlink=False,
        )
    except PermissionError:
        log("No permission to change:", target, lvl=error)
        return False

    log("Frontend deployed")


def install_frontend(
    force_rebuild: bool = False,
    install: bool = True,
    development: bool = False,
    build_type: str = "dist",
):
    """Builds and installs the frontend.

    The process works like this:

    * Find the frontend locations (source root and target)
    * Generate the target component folders to copy modules' frontend sources to
    * Gather all component meta data
    * Collect all dependencies (when installing them is desired) and their module
      imports
    * If desired, install all dependencies
    * Write the frontend main loader with all module entrypoints
    * Run npm build `BUILD_TYPE` and copy all resulting files to the frontend target
      folder

    :param force_rebuild: Trigger a rebuild of the sources.
    :param install: Trigger installation of the frontend's dependencies
    :param development: Use development frontend server locations
    :param build_type: Type of frontend build, either 'dist' or 'build'
    """

    frontend_root, frontend_target = get_frontend_locations(development)

    if frontend_root is None or frontend_target is None:
        log("Cannot determine either frontend root or target",
            lvl=error)
        return False

    component_folder = os.path.join(frontend_root, "src", "components")
    if not generate_component_folders(component_folder):
        log("Cannot generate component folders",
            lvl=error)
        return False

    components = get_components(frontend_root)
    if components is False:
        log("Could not get components", lvl=error)
        return False

    installation_packages, imports = update_frontends(
        components, frontend_root, install
    )

    if install:
        installation_packages += get_sails_dependencies(frontend_root)
        installed = install_dependencies(installation_packages, frontend_root)
        if installed is False:
            log("Could not install dependencies", lvl=error)
            return False

    wrote_main = write_main(imports, frontend_root)

    if wrote_main is False:
        log("Could not write frontend loader", lvl=error)
        return False

    if force_rebuild:
        rebuilt = rebuild_frontend(frontend_root, frontend_target, build_type)
        if rebuilt is False:
            log("Frontend build failed", lvl=error)
            return False

    log("Done: Install Frontend")
    return True

    # TODO: We have to find a way to detect if we need to rebuild (and
    #  possibly wipe) stuff. This maybe the case, when a frontend
    #  module has been updated/added/removed.