ocrd.mets_server.ClientSideOcrdMets.add_file() - Code Metrics - Inspection of "Utilize processing server proxy to mets servers" - OCR-D/core - Measure and Improve Code Quality continuously with Scrutinizer

Passed

Pull Request — master (#1220)

unknown

created 2024-06-03 12:52 UTC

ocrd.mets_server.ClientSideOcrdMets.add_file() A

↳ Parent: ocrd.mets_server

Complexity

Conditions

Size

Total Lines	22
Code Lines	17

Duplication

Lines	0
Ratio	0 %

Importance

Changes

Metric	Value
eloc	17
dl	0
loc	22
rs	9.55
c	0
b	0
f	0
cc	4
nop	9

How to fix Many Parameters

"""
# METS server functionality
"""
import re
from os import _exit, chmod
from typing import Dict, Optional, Union, List, Tuple
from pathlib import Path
from urllib.parse import urlparse
import socket
import atexit

from fastapi import FastAPI, Request, Form, Response
from fastapi.responses import JSONResponse
from requests import Session as requests_session
from requests.exceptions import ConnectionError
from requests_unixsocket import Session as requests_unixsocket_session
from pydantic import BaseModel, Field, ValidationError

import uvicorn

from ocrd_models import OcrdFile, ClientSideOcrdFile, OcrdAgent, ClientSideOcrdAgent
from ocrd_utils import getLogger, deprecated_alias


#
# Models
#


class OcrdFileModel(BaseModel):
    file_grp: str = Field()
    file_id: str = Field()
    mimetype: str = Field()
    page_id: Optional[str] = Field()
    url: Optional[str] = Field()
    local_filename: Optional[str] = Field()

    @staticmethod
    def create(
        file_grp: str, file_id: str, page_id: Optional[str], url: Optional[str],
        local_filename: Optional[Union[str, Path]], mimetype: str
    ):
        return OcrdFileModel(
            file_grp=file_grp, file_id=file_id, page_id=page_id, mimetype=mimetype, url=url,
            local_filename=str(local_filename)
        )


class OcrdAgentModel(BaseModel):
    name: str = Field()
    type: str = Field()
    role: str = Field()
    otherrole: Optional[str] = Field()
    othertype: str = Field()
    notes: Optional[List[Tuple[Dict[str, str], Optional[str]]]] = Field()

    @staticmethod
    def create(
        name: str, _type: str, role: str, otherrole: str, othertype: str,
        notes: List[Tuple[Dict[str, str], Optional[str]]]
    ):
        return OcrdAgentModel(name=name, type=_type, role=role, otherrole=otherrole, othertype=othertype, notes=notes)


class OcrdFileListModel(BaseModel):
    files: List[OcrdFileModel] = Field()

    @staticmethod
    def create(files: List[OcrdFile]):
        ret = OcrdFileListModel(
            files=[
                OcrdFileModel.create(
                    file_grp=f.fileGrp, file_id=f.ID, mimetype=f.mimetype, page_id=f.pageId, url=f.url,
                    local_filename=f.local_filename
                ) for f in files
            ]
        )
        return ret


class OcrdFileGroupListModel(BaseModel):
    file_groups: List[str] = Field()

    @staticmethod
    def create(file_groups: List[str]):
        return OcrdFileGroupListModel(file_groups=file_groups)


class OcrdAgentListModel(BaseModel):
    agents: List[OcrdAgentModel] = Field()

    @staticmethod
    def create(agents: List[OcrdAgent]):
        return OcrdAgentListModel(
            agents=[
                OcrdAgentModel.create(
                    name=a.name, _type=a.type, role=a.role, otherrole=a.otherrole, othertype=a.othertype, notes=a.notes
                ) for a in agents
            ]
        )


#
# Client
#


class ClientSideOcrdMets:
    """
    Partial substitute for :py:class:`ocrd_models.ocrd_mets.OcrdMets` which provides for
    :py:meth:`ocrd_models.ocrd_mets.OcrdMets.find_files`,
    :py:meth:`ocrd_models.ocrd_mets.OcrdMets.find_all_files`, and
    :py:meth:`ocrd_models.ocrd_mets.OcrdMets.add_agent`,
    :py:meth:`ocrd_models.ocrd_mets.OcrdMets.agents`,
    :py:meth:`ocrd_models.ocrd_mets.OcrdMets.add_file` to query via HTTP a
    :py:class:`ocrd.mets_server.OcrdMetsServer`.
    """

    def __init__(self, url, workspace_path: Optional[str] = None):
        self.protocol = "tcp" if url.startswith("http://") else "uds"
        self.log = getLogger(f"ocrd.mets_client[{url}]")
        self.url = url if self.protocol == "tcp" else f'http+unix://{url.replace("/", "%2F")}'
        self.ws_dir_path = workspace_path if workspace_path else None

        if self.protocol == "tcp" and "tcp_mets" in self.url:
            self.multiplexing_mode = True
            if not self.ws_dir_path:
                # Must be set since this path is the way to multiplex among multiple workspaces on the PS side
                raise ValueError("ClientSideOcrdMets runs in multiplexing mode but the workspace dir path is not set!")
        else:
            self.multiplexing_mode = False

    @property
    def session(self) -> Union[requests_session, requests_unixsocket_session]:
        return requests_session() if self.protocol == "tcp" else requests_unixsocket_session()

    def __getattr__(self, name):
        raise NotImplementedError(f"ClientSideOcrdMets has no access to '{name}' - try without METS server")

    def __str__(self):
        return f"<ClientSideOcrdMets[url={self.url}]>"

    def save(self):
        """
        Request writing the changes to the file system
        """
        if not self.multiplexing_mode:
            self.session.request("PUT", url=self.url)
        else:
            self.session.request(
                "POST",
                self.url,
                json=MpxReq.save(self.ws_dir_path)
            )

    def stop(self):
        """
        Request stopping the mets server
        """
        try:
            if not self.multiplexing_mode:
                self.session.request("DELETE", self.url)
                return
            else:
                self.session.request(
                    "POST",
                    self.url,
                    json=MpxReq.stop(self.ws_dir_path)
                )
        except ConnectionError:
            # Expected because we exit the process without returning
            pass

    def reload(self):
        """
        Request reloading of the mets file from the file system
        """
        if not self.multiplexing_mode:
            return self.session.request("POST", f"{self.url}/reload").text
        else:
            return self.session.request(
                "POST",
                self.url,
                json=MpxReq.reload(self.ws_dir_path)
            ).json()["text"]

    @property
    def unique_identifier(self):
        if not self.multiplexing_mode:
            return self.session.request("GET", f"{self.url}/unique_identifier").text
        else:
            return self.session.request(
                "POST",
                self.url,
                json=MpxReq.unique_identifier(self.ws_dir_path)
            ).json()["text"]

    @property
    def workspace_path(self):
        if not self.multiplexing_mode:
            self.ws_dir_path = self.session.request("GET", f"{self.url}/workspace_path").text
            return self.ws_dir_path
        else:
            self.ws_dir_path = self.session.request(
                "POST",
                self.url,
                json=MpxReq.workspace_path(self.ws_dir_path)
            ).json()["text"]
            return self.ws_dir_path

    @property
    def file_groups(self):
        if not self.multiplexing_mode:
            return self.session.request("GET", f"{self.url}/file_groups").json()["file_groups"]
        else:
            return self.session.request(
                "POST",
                self.url,
                json=MpxReq.file_groups(self.ws_dir_path)
            ).json()["file_groups"]

    @property
    def agents(self):
        if not self.multiplexing_mode:
            agent_dicts = self.session.request("GET", f"{self.url}/agent").json()["agents"]
        else:
            agent_dicts = self.session.request(
                "POST",
                self.url,
                json=MpxReq.agents(self.ws_dir_path)
            ).json()["agents"]

        for agent_dict in agent_dicts:
            agent_dict["_type"] = agent_dict.pop("type")
        return [ClientSideOcrdAgent(None, **agent_dict) for agent_dict in agent_dicts]

    def add_agent(self, *args, **kwargs):
        if not self.multiplexing_mode:
            return self.session.request("POST", f"{self.url}/agent", json=OcrdAgentModel.create(**kwargs).dict())
        else:
            self.session.request(
                "POST",
                self.url,
                json=MpxReq.add_agent(self.ws_dir_path, OcrdAgentModel.create(**kwargs).dict())
            ).json()
            return OcrdAgentModel.create(**kwargs)

    @deprecated_alias(ID="file_id")
    @deprecated_alias(pageId="page_id")
    @deprecated_alias(fileGrp="file_grp")
    def find_files(self, **kwargs):
        self.log.debug("find_files(%s)", kwargs)
        if "pageId" in kwargs:
            kwargs["page_id"] = kwargs.pop("pageId")
        if "ID" in kwargs:
            kwargs["file_id"] = kwargs.pop("ID")
        if "fileGrp" in kwargs:
            kwargs["file_grp"] = kwargs.pop("fileGrp")

        if not self.multiplexing_mode:
            r = self.session.request(method="GET", url=f"{self.url}/file", params={**kwargs})
        else:
            r = self.session.request(
                "POST",
                self.url,
                json=MpxReq.find_files(self.ws_dir_path, {**kwargs})
            )

        for f in r.json()["files"]:
            yield ClientSideOcrdFile(
                None, ID=f["file_id"], pageId=f["page_id"], fileGrp=f["file_grp"], url=f["url"],
                local_filename=f["local_filename"], mimetype=f["mimetype"]
            )

    def find_all_files(self, *args, **kwargs):
        return list(self.find_files(*args, **kwargs))

    @deprecated_alias(pageId="page_id")
    @deprecated_alias(ID="file_id")
    def add_file(
        self, file_grp, content=None, file_id=None, url=None, local_filename=None, mimetype=None, page_id=None, **kwargs
    ):
        data = OcrdFileModel.create(
            file_id=file_id, file_grp=file_grp, page_id=page_id, mimetype=mimetype, url=url,
            local_filename=local_filename
        )

        if not self.multiplexing_mode:
            r = self.session.request("POST", f"{self.url}/file", data=data.dict())
            if not r:
                raise RuntimeError("Add file failed. Please check provided parameters")
        else:
            r = self.session.request("POST", self.url, json=MpxReq.add_file(self.ws_dir_path, data.dict()))
            if "error" in r:
                raise RuntimeError(f"Add file failed: Msg: {r['error']}")

        return ClientSideOcrdFile(
            None, ID=file_id, fileGrp=file_grp, url=url, pageId=page_id, mimetype=mimetype,
            local_filename=local_filename
        )


class MpxReq:
    """This class wrapps the request bodies needed for the tcp forwarding

    For every mets-server-call like find_files or workspace_path a special request_body is
    needed to call `MetsServerProxy.forward_tcp_request`. These are created by this functions.

    Reason to put this to a separate class is to allow easier testing
    """
    @staticmethod
    def save(ws_dir_path: str) -> Dict:
        return {
            "workspace_path": ws_dir_path,
            "method_type": "PUT",
            "response_type": "empty",
            "request_url": "",
            "request_data": {}
        }

    @staticmethod
    def stop(ws_dir_path: str) -> Dict:
        return {
            "workspace_path": ws_dir_path,
            "method_type": "DELETE",
            "response_type": "empty",
            "request_url": "",
            "request_data": {}
        }

    @staticmethod
    def reload(ws_dir_path: str) -> Dict:
        return {
            "workspace_path": ws_dir_path,
            "method_type": "POST",
            "response_type": "text",
            "request_url": "reload",
            "request_data": {}
        }

    @staticmethod
    def unique_identifier(ws_dir_path: str) -> Dict:
        return {
            "workspace_path": ws_dir_path,
            "method_type": "GET",
            "response_type": "text",
            "request_url": "unique_identifier",
            "request_data": {}
        }

    @staticmethod
    def workspace_path(ws_dir_path: str) -> Dict:
        return {
            "workspace_path": ws_dir_path,
            "method_type": "GET",
            "response_type": "text",
            "request_url": "workspace_path",
            "request_data": {}
        }

    @staticmethod
    def file_groups(ws_dir_path: str) -> Dict:
        return {
            "workspace_path": ws_dir_path,
            "method_type": "GET",
            "response_type": "dict",
            "request_url": "file_groups",
            "request_data": {}
        }

    @staticmethod
    def agents(ws_dir_path: str) -> Dict:
        return {
            "workspace_path": ws_dir_path,
            "method_type": "GET",
            "response_type": "class",
            "request_url": "agent",
            "request_data": {}
        }

    @staticmethod
    def add_agent(ws_dir_path: str, agent_model: Dict) -> Dict:
        return {
            "workspace_path": ws_dir_path,
            "method_type": "POST",
            "response_type": "class",
            "request_url": "agent",
            "request_data": {
                "class": agent_model
            }
        }

    @staticmethod
    def find_files(ws_dir_path: str, params: Dict) -> Dict:
        return {
            "workspace_path": ws_dir_path,
            "method_type": "GET",
            "response_type": "class",
            "request_url": "file",
            "request_data": {
                "params": params
            }
        }

    @staticmethod
    def add_file(ws_dir_path: str, data: Dict) -> Dict:
        return {
            "workspace_path": ws_dir_path,
            "method_type": "POST",
            "response_type": "class",
            "request_url": "file",
            "request_data": {
                "form": data
            }
        }

#
# Server
#


class OcrdMetsServer:
    def __init__(self, workspace, url):
        self.workspace = workspace
        self.url = url
        self.is_uds = not (url.startswith('http://') or url.startswith('https://'))
        self.log = getLogger(f'ocrd.mets_server[{self.url}]')

    def shutdown(self):
        if self.is_uds:
            if Path(self.url).exists():
                self.log.warning(f'UDS socket {self.url} still exists, removing it')
                Path(self.url).unlink()
        # os._exit because uvicorn catches SystemExit raised by sys.exit
        _exit(0)

    def startup(self):
        self.log.info("Starting up METS server")

        workspace = self.workspace

        app = FastAPI(
            title="OCR-D METS Server",
            description="Providing simultaneous write-access to mets.xml for OCR-D",
        )

        @app.exception_handler(ValidationError)
        async def exception_handler_validation_error(request: Request, exc: ValidationError):
            return JSONResponse(status_code=400, content=exc.errors())

        @app.exception_handler(FileExistsError)
        async def exception_handler_file_exists(request: Request, exc: FileExistsError):
            return JSONResponse(status_code=400, content=str(exc))

        @app.exception_handler(re.error)
        async def exception_handler_invalid_regex(request: Request, exc: re.error):
            return JSONResponse(status_code=400, content=f'invalid regex: {exc}')

        @app.put(path='/')
        def save():
            """
            Write current changes to the file system
            """
            return workspace.save_mets()

        @app.delete(path='/')
        async def stop():
            """
            Stop the mets server
            """
            getLogger('ocrd.models.ocrd_mets').info(f'Shutting down METS Server {self.url}')
            workspace.save_mets()
            self.shutdown()

        @app.post(path='/reload')
        async def workspace_reload_mets():
            """
            Reload mets file from the file system
            """
            workspace.reload_mets()
            return Response(content=f'Reloaded from {workspace.directory}', media_type="text/plain")

        @app.get(path='/unique_identifier', response_model=str)
        async def unique_identifier():
            return Response(content=workspace.mets.unique_identifier, media_type='text/plain')

        @app.get(path='/workspace_path', response_model=str)
        async def workspace_path():
            return Response(content=workspace.directory, media_type="text/plain")

        @app.get(path='/file_groups', response_model=OcrdFileGroupListModel)
        async def file_groups():
            return {'file_groups': workspace.mets.file_groups}

        @app.get(path='/agent', response_model=OcrdAgentListModel)
        async def agents():
            return OcrdAgentListModel.create(workspace.mets.agents)

        @app.post(path='/agent', response_model=OcrdAgentModel)
        async def add_agent(agent: OcrdAgentModel):
            kwargs = agent.dict()
            kwargs['_type'] = kwargs.pop('type')
            workspace.mets.add_agent(**kwargs)
            return agent

        @app.get(path="/file", response_model=OcrdFileListModel)
        async def find_files(
            file_grp: Optional[str] = None,
            file_id: Optional[str] = None,
            page_id: Optional[str] = None,
            mimetype: Optional[str] = None,
            local_filename: Optional[str] = None,
            url: Optional[str] = None
        ):
            """
            Find files in the mets
            """
            found = workspace.mets.find_all_files(
                fileGrp=file_grp, ID=file_id, pageId=page_id, mimetype=mimetype, local_filename=local_filename, url=url
            )
            return OcrdFileListModel.create(found)

        @app.post(path='/file', response_model=OcrdFileModel)
        async def add_file(
            file_grp: str = Form(),
            file_id: str = Form(),
            page_id: Optional[str] = Form(),
            mimetype: str = Form(),
            url: Optional[str] = Form(None),
            local_filename: Optional[str] = Form(None)
        ):
            """
            Add a file
            """
            # Validate
            file_resource = OcrdFileModel.create(
                file_grp=file_grp, file_id=file_id, page_id=page_id, mimetype=mimetype, url=url,
                local_filename=local_filename
            )
            # Add to workspace
            kwargs = file_resource.dict()
            workspace.add_file(**kwargs)
            return file_resource

        # ------------- #

        if self.is_uds:
            # Create socket and change to world-readable and -writable to avoid permission errors
            self.log.debug(f"chmod 0o677 {self.url}")
            server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
            server.bind(self.url)  # creates the socket file
            atexit.register(self.shutdown)
            server.close()
            chmod(self.url, 0o666)
            uvicorn_kwargs = {'uds': self.url}
        else:
            parsed = urlparse(self.url)
            uvicorn_kwargs = {'host': parsed.hostname, 'port': parsed.port}

        self.log.debug("Starting uvicorn")
        uvicorn.run(app, **uvicorn_kwargs)


1			"""
2			# METS server functionality
3			"""
4			import re
5			from os import _exit, chmod
6			from typing import Dict, Optional, Union, List, Tuple
7			from pathlib import Path
8			from urllib.parse import urlparse
9			import socket
10			import atexit
11
12			from fastapi import FastAPI, Request, Form, Response
13			from fastapi.responses import JSONResponse
14			from requests import Session as requests_session
15			from requests.exceptions import ConnectionError
16			from requests_unixsocket import Session as requests_unixsocket_session
17			from pydantic import BaseModel, Field, ValidationError
18
19			import uvicorn
20
21			from ocrd_models import OcrdFile, ClientSideOcrdFile, OcrdAgent, ClientSideOcrdAgent
22			from ocrd_utils import getLogger, deprecated_alias
23
24
25			#
26			# Models
27			#
28
29
30			class OcrdFileModel(BaseModel):
31			file_grp: str = Field()
32			file_id: str = Field()
33			mimetype: str = Field()
34			page_id: Optional[str] = Field()
35			url: Optional[str] = Field()
36			local_filename: Optional[str] = Field()
37
38			@staticmethod
39			def create(
40			file_grp: str, file_id: str, page_id: Optional[str], url: Optional[str],
41			local_filename: Optional[Union[str, Path]], mimetype: str
42			):
43			return OcrdFileModel(
44			file_grp=file_grp, file_id=file_id, page_id=page_id, mimetype=mimetype, url=url,
45			local_filename=str(local_filename)
46			)
47
48
49			class OcrdAgentModel(BaseModel):
50			name: str = Field()
51			type: str = Field()
52			role: str = Field()
53			otherrole: Optional[str] = Field()
54			othertype: str = Field()
55			notes: Optional[List[Tuple[Dict[str, str], Optional[str]]]] = Field()
56
57			@staticmethod
58			def create(
59			name: str, _type: str, role: str, otherrole: str, othertype: str,
60			notes: List[Tuple[Dict[str, str], Optional[str]]]
61			):
62			return OcrdAgentModel(name=name, type=_type, role=role, otherrole=otherrole, othertype=othertype, notes=notes)
63
64
65			class OcrdFileListModel(BaseModel):
66			files: List[OcrdFileModel] = Field()
67
68			@staticmethod
69			def create(files: List[OcrdFile]):
70			ret = OcrdFileListModel(
71			files=[
72			OcrdFileModel.create(
73			file_grp=f.fileGrp, file_id=f.ID, mimetype=f.mimetype, page_id=f.pageId, url=f.url,
74			local_filename=f.local_filename
75			) for f in files
76			]
77			)
78			return ret
79
80
81			class OcrdFileGroupListModel(BaseModel):
82			file_groups: List[str] = Field()
83
84			@staticmethod
85			def create(file_groups: List[str]):
86			return OcrdFileGroupListModel(file_groups=file_groups)
87
88
89			class OcrdAgentListModel(BaseModel):
90			agents: List[OcrdAgentModel] = Field()
91
92			@staticmethod
93			def create(agents: List[OcrdAgent]):
94			return OcrdAgentListModel(
95			agents=[
96			OcrdAgentModel.create(
97			name=a.name, _type=a.type, role=a.role, otherrole=a.otherrole, othertype=a.othertype, notes=a.notes
98			) for a in agents
99			]
100			)
101
102
103			#
104			# Client
105			#
106
107
108			class ClientSideOcrdMets:
109			"""
110			Partial substitute for :py:class:`ocrd_models.ocrd_mets.OcrdMets` which provides for
111			:py:meth:`ocrd_models.ocrd_mets.OcrdMets.find_files`,
112			:py:meth:`ocrd_models.ocrd_mets.OcrdMets.find_all_files`, and
113			:py:meth:`ocrd_models.ocrd_mets.OcrdMets.add_agent`,
114			:py:meth:`ocrd_models.ocrd_mets.OcrdMets.agents`,
115			:py:meth:`ocrd_models.ocrd_mets.OcrdMets.add_file` to query via HTTP a
116			:py:class:`ocrd.mets_server.OcrdMetsServer`.
117			"""
118
119			def __init__(self, url, workspace_path: Optional[str] = None):
120			self.protocol = "tcp" if url.startswith("http://") else "uds"
121			self.log = getLogger(f"ocrd.mets_client[{url}]")
122			self.url = url if self.protocol == "tcp" else f'http+unix://{url.replace("/", "%2F")}'
123			self.ws_dir_path = workspace_path if workspace_path else None
124
125			if self.protocol == "tcp" and "tcp_mets" in self.url:
126			self.multiplexing_mode = True
127			if not self.ws_dir_path:
128			# Must be set since this path is the way to multiplex among multiple workspaces on the PS side
129			raise ValueError("ClientSideOcrdMets runs in multiplexing mode but the workspace dir path is not set!")
130			else:
131			self.multiplexing_mode = False
132
133			@property
134			def session(self) -> Union[requests_session, requests_unixsocket_session]:
135			return requests_session() if self.protocol == "tcp" else requests_unixsocket_session()
136
137			def __getattr__(self, name):
138			raise NotImplementedError(f"ClientSideOcrdMets has no access to '{name}' - try without METS server")
139
140			def __str__(self):
141			return f"<ClientSideOcrdMets[url={self.url}]>"
142
143			def save(self):
144			"""
145			Request writing the changes to the file system
146			"""
147			if not self.multiplexing_mode:
148			self.session.request("PUT", url=self.url)
149			else:
150			self.session.request(
151			"POST",
152			self.url,
153			json=MpxReq.save(self.ws_dir_path)
154			)
155
156			def stop(self):
157			"""
158			Request stopping the mets server
159			"""
160			try:
161			if not self.multiplexing_mode:
162			self.session.request("DELETE", self.url)
163			return
164			else:
165			self.session.request(
166			"POST",
167			self.url,
168			json=MpxReq.stop(self.ws_dir_path)
169			)
170			except ConnectionError:
171			# Expected because we exit the process without returning
172			pass
173
174			def reload(self):
175			"""
176			Request reloading of the mets file from the file system
177			"""
178			if not self.multiplexing_mode:
179			return self.session.request("POST", f"{self.url}/reload").text
180			else:
181			return self.session.request(
182			"POST",
183			self.url,
184			json=MpxReq.reload(self.ws_dir_path)
185			).json()["text"]
186
187			@property
188			def unique_identifier(self):
189			if not self.multiplexing_mode:
190			return self.session.request("GET", f"{self.url}/unique_identifier").text
191			else:
192			return self.session.request(
193			"POST",
194			self.url,
195			json=MpxReq.unique_identifier(self.ws_dir_path)
196			).json()["text"]
197
198			@property
199			def workspace_path(self):
200			if not self.multiplexing_mode:
201			self.ws_dir_path = self.session.request("GET", f"{self.url}/workspace_path").text
202			return self.ws_dir_path
203			else:
204			self.ws_dir_path = self.session.request(
205			"POST",
206			self.url,
207			json=MpxReq.workspace_path(self.ws_dir_path)
208			).json()["text"]
209			return self.ws_dir_path
210
211			@property
212			def file_groups(self):
213			if not self.multiplexing_mode:
214			return self.session.request("GET", f"{self.url}/file_groups").json()["file_groups"]
215			else:
216			return self.session.request(
217			"POST",
218			self.url,
219			json=MpxReq.file_groups(self.ws_dir_path)
220			).json()["file_groups"]
221
222			@property
223			def agents(self):
224			if not self.multiplexing_mode:
225			agent_dicts = self.session.request("GET", f"{self.url}/agent").json()["agents"]
226			else:
227			agent_dicts = self.session.request(
228			"POST",
229			self.url,
230			json=MpxReq.agents(self.ws_dir_path)
231			).json()["agents"]
232
233			for agent_dict in agent_dicts:
234			agent_dict["_type"] = agent_dict.pop("type")
235			return [ClientSideOcrdAgent(None, **agent_dict) for agent_dict in agent_dicts]
236
237			def add_agent(self, args, *kwargs):
238			if not self.multiplexing_mode:
239			return self.session.request("POST", f"{self.url}/agent", json=OcrdAgentModel.create(**kwargs).dict())
240			else:
241			self.session.request(
242			"POST",
243			self.url,
244			json=MpxReq.add_agent(self.ws_dir_path, OcrdAgentModel.create(**kwargs).dict())
245			).json()
246			return OcrdAgentModel.create(**kwargs)
247
248			@deprecated_alias(ID="file_id")
249			@deprecated_alias(pageId="page_id")
250			@deprecated_alias(fileGrp="file_grp")
251			def find_files(self, **kwargs):
252			self.log.debug("find_files(%s)", kwargs)
253			if "pageId" in kwargs:
254			kwargs["page_id"] = kwargs.pop("pageId")
255			if "ID" in kwargs:
256			kwargs["file_id"] = kwargs.pop("ID")
257			if "fileGrp" in kwargs:
258			kwargs["file_grp"] = kwargs.pop("fileGrp")
259
260			if not self.multiplexing_mode:
261			r = self.session.request(method="GET", url=f"{self.url}/file", params={**kwargs})
262			else:
263			r = self.session.request(
264			"POST",
265			self.url,
266			json=MpxReq.find_files(self.ws_dir_path, {**kwargs})
267			)
268
269			for f in r.json()["files"]:
270			yield ClientSideOcrdFile(
271			None, ID=f["file_id"], pageId=f["page_id"], fileGrp=f["file_grp"], url=f["url"],
272			local_filename=f["local_filename"], mimetype=f["mimetype"]
273			)
274
275			def find_all_files(self, args, *kwargs):
276			return list(self.find_files(args, *kwargs))
277
278			@deprecated_alias(pageId="page_id")
279			@deprecated_alias(ID="file_id")
280			def add_file(
281			self, file_grp, content=None, file_id=None, url=None, local_filename=None, mimetype=None, page_id=None, **kwargs
282			):
283			data = OcrdFileModel.create(
284			file_id=file_id, file_grp=file_grp, page_id=page_id, mimetype=mimetype, url=url,
285			local_filename=local_filename
286			)
287
288			if not self.multiplexing_mode:
289			r = self.session.request("POST", f"{self.url}/file", data=data.dict())
290			if not r:
291			raise RuntimeError("Add file failed. Please check provided parameters")
292			else:
293			r = self.session.request("POST", self.url, json=MpxReq.add_file(self.ws_dir_path, data.dict()))
294			if "error" in r:
295			raise RuntimeError(f"Add file failed: Msg: {r['error']}")
296
297			return ClientSideOcrdFile(
298			None, ID=file_id, fileGrp=file_grp, url=url, pageId=page_id, mimetype=mimetype,
299			local_filename=local_filename
300			)
301
302
303			class MpxReq:
304			"""This class wrapps the request bodies needed for the tcp forwarding
305
306			For every mets-server-call like find_files or workspace_path a special request_body is
307			needed to call `MetsServerProxy.forward_tcp_request`. These are created by this functions.
308
309			Reason to put this to a separate class is to allow easier testing
310			"""
311			@staticmethod
312			def save(ws_dir_path: str) -> Dict:
313			return {
314			"workspace_path": ws_dir_path,
315			"method_type": "PUT",
316			"response_type": "empty",
317			"request_url": "",
318			"request_data": {}
319			}
320
321			@staticmethod
322			def stop(ws_dir_path: str) -> Dict:
323			return {
324			"workspace_path": ws_dir_path,
325			"method_type": "DELETE",
326			"response_type": "empty",
327			"request_url": "",
328			"request_data": {}
329			}
330
331			@staticmethod
332			def reload(ws_dir_path: str) -> Dict:
333			return {
334			"workspace_path": ws_dir_path,
335			"method_type": "POST",
336			"response_type": "text",
337			"request_url": "reload",
338			"request_data": {}
339			}
340
341			@staticmethod
342			def unique_identifier(ws_dir_path: str) -> Dict:
343			return {
344			"workspace_path": ws_dir_path,
345			"method_type": "GET",
346			"response_type": "text",
347			"request_url": "unique_identifier",
348			"request_data": {}
349			}
350
351			@staticmethod
352			def workspace_path(ws_dir_path: str) -> Dict:
353			return {
354			"workspace_path": ws_dir_path,
355			"method_type": "GET",
356			"response_type": "text",
357			"request_url": "workspace_path",
358			"request_data": {}
359			}
360
361			@staticmethod
362			def file_groups(ws_dir_path: str) -> Dict:
363			return {
364			"workspace_path": ws_dir_path,
365			"method_type": "GET",
366			"response_type": "dict",
367			"request_url": "file_groups",
368			"request_data": {}
369			}
370
371			@staticmethod
372			def agents(ws_dir_path: str) -> Dict:
373			return {
374			"workspace_path": ws_dir_path,
375			"method_type": "GET",
376			"response_type": "class",
377			"request_url": "agent",
378			"request_data": {}
379			}
380
381			@staticmethod
382			def add_agent(ws_dir_path: str, agent_model: Dict) -> Dict:
383			return {
384			"workspace_path": ws_dir_path,
385			"method_type": "POST",
386			"response_type": "class",
387			"request_url": "agent",
388			"request_data": {
389			"class": agent_model
390			}
391			}
392
393			@staticmethod
394			def find_files(ws_dir_path: str, params: Dict) -> Dict:
395			return {
396			"workspace_path": ws_dir_path,
397			"method_type": "GET",
398			"response_type": "class",
399			"request_url": "file",
400			"request_data": {
401			"params": params
402			}
403			}
404
405			@staticmethod
406			def add_file(ws_dir_path: str, data: Dict) -> Dict:
407			return {
408			"workspace_path": ws_dir_path,
409			"method_type": "POST",
410			"response_type": "class",
411			"request_url": "file",
412			"request_data": {
413			"form": data
414			}
415			}
416
417			#
418			# Server
419			#
420
421
422			class OcrdMetsServer:
423			def __init__(self, workspace, url):
424			self.workspace = workspace
425			self.url = url
426			self.is_uds = not (url.startswith('http://') or url.startswith('https://'))
427			self.log = getLogger(f'ocrd.mets_server[{self.url}]')
428
429			def shutdown(self):
430			if self.is_uds:
431			if Path(self.url).exists():
432			self.log.warning(f'UDS socket {self.url} still exists, removing it')
433			Path(self.url).unlink()
434			# os._exit because uvicorn catches SystemExit raised by sys.exit
435			_exit(0)
436
437			def startup(self):
438			self.log.info("Starting up METS server")
439
440			workspace = self.workspace
441
442			app = FastAPI(
443			title="OCR-D METS Server",
444			description="Providing simultaneous write-access to mets.xml for OCR-D",
445			)
446
447			@app.exception_handler(ValidationError)
448			async def exception_handler_validation_error(request: Request, exc: ValidationError):
449			return JSONResponse(status_code=400, content=exc.errors())
450
451			@app.exception_handler(FileExistsError)
452			async def exception_handler_file_exists(request: Request, exc: FileExistsError):
453			return JSONResponse(status_code=400, content=str(exc))
454
455			@app.exception_handler(re.error)
456			async def exception_handler_invalid_regex(request: Request, exc: re.error):
457			return JSONResponse(status_code=400, content=f'invalid regex: {exc}')
458
459			@app.put(path='/')
460			def save():
461			"""
462			Write current changes to the file system
463			"""
464			return workspace.save_mets()
465
466			@app.delete(path='/')
467			async def stop():
468			"""
469			Stop the mets server
470			"""
471			getLogger('ocrd.models.ocrd_mets').info(f'Shutting down METS Server {self.url}')
472			workspace.save_mets()
473			self.shutdown()
474
475			@app.post(path='/reload')
476			async def workspace_reload_mets():
477			"""
478			Reload mets file from the file system
479			"""
480			workspace.reload_mets()
481			return Response(content=f'Reloaded from {workspace.directory}', media_type="text/plain")
482
483			@app.get(path='/unique_identifier', response_model=str)
484			async def unique_identifier():
485			return Response(content=workspace.mets.unique_identifier, media_type='text/plain')
486
487			@app.get(path='/workspace_path', response_model=str)
488			async def workspace_path():
489			return Response(content=workspace.directory, media_type="text/plain")
490
491			@app.get(path='/file_groups', response_model=OcrdFileGroupListModel)
492			async def file_groups():
493			return {'file_groups': workspace.mets.file_groups}
494
495			@app.get(path='/agent', response_model=OcrdAgentListModel)
496			async def agents():
497			return OcrdAgentListModel.create(workspace.mets.agents)
498
499			@app.post(path='/agent', response_model=OcrdAgentModel)
500			async def add_agent(agent: OcrdAgentModel):
501			kwargs = agent.dict()
502			kwargs['_type'] = kwargs.pop('type')
503			workspace.mets.add_agent(**kwargs)
504			return agent
505
506			@app.get(path="/file", response_model=OcrdFileListModel)
507			async def find_files(
508			file_grp: Optional[str] = None,
509			file_id: Optional[str] = None,
510			page_id: Optional[str] = None,
511			mimetype: Optional[str] = None,
512			local_filename: Optional[str] = None,
513			url: Optional[str] = None
514			):
515			"""
516			Find files in the mets
517			"""
518			found = workspace.mets.find_all_files(
519			fileGrp=file_grp, ID=file_id, pageId=page_id, mimetype=mimetype, local_filename=local_filename, url=url
520			)
521			return OcrdFileListModel.create(found)
522
523			@app.post(path='/file', response_model=OcrdFileModel)
524			async def add_file(
525			file_grp: str = Form(),
526			file_id: str = Form(),
527			page_id: Optional[str] = Form(),
528			mimetype: str = Form(),
529			url: Optional[str] = Form(None),
530			local_filename: Optional[str] = Form(None)
531			):
532			"""
533			Add a file
534			"""
535			# Validate
536			file_resource = OcrdFileModel.create(
537			file_grp=file_grp, file_id=file_id, page_id=page_id, mimetype=mimetype, url=url,
538			local_filename=local_filename
539			)
540			# Add to workspace
541			kwargs = file_resource.dict()
542			workspace.add_file(**kwargs)
543			return file_resource
544
545			# ------------- #
546
547			if self.is_uds:
548			# Create socket and change to world-readable and -writable to avoid permission errors
549			self.log.debug(f"chmod 0o677 {self.url}")
550			server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
551			server.bind(self.url) # creates the socket file
552			atexit.register(self.shutdown)
553			server.close()
554			chmod(self.url, 0o666)
555			uvicorn_kwargs = {'uds': self.url}
556			else:
557			parsed = urlparse(self.url)
558			uvicorn_kwargs = {'host': parsed.hostname, 'port': parsed.port}
559
560			self.log.debug("Starting uvicorn")
561			uvicorn.run(app, **uvicorn_kwargs)
562

OCR-D / core

Pull Request — master (#1220)

ocrd.mets_server.ClientSideOcrdMets.add_file() A

Complexity

Size

Duplication

Importance

How to fix Many Parameters

Many Parameters

Duplication Side-by-Side

Filter issues like