|
1
|
|
|
""" |
|
2
|
|
|
OCR-D CLI: management of processor resources |
|
3
|
|
|
|
|
4
|
|
|
.. click:: ocrd.cli.resmgr:resmgr_cli |
|
5
|
|
|
:prog: ocrd resmgr |
|
6
|
|
|
:nested: full |
|
7
|
|
|
""" |
|
8
|
|
|
import sys |
|
9
|
|
|
from pathlib import Path |
|
10
|
|
|
from shutil import which |
|
11
|
|
|
from yaml import safe_load, safe_dump |
|
12
|
|
|
|
|
13
|
|
|
import requests |
|
14
|
|
|
import click |
|
15
|
|
|
|
|
16
|
|
|
from ocrd_utils import ( |
|
17
|
|
|
directory_size, |
|
18
|
|
|
getLogger, |
|
19
|
|
|
get_moduledir, |
|
20
|
|
|
get_ocrd_tool_json, |
|
21
|
|
|
initLogging, |
|
22
|
|
|
RESOURCE_LOCATIONS, |
|
23
|
|
|
RESOURCE_TYPES |
|
24
|
|
|
) |
|
25
|
|
|
from ocrd.constants import RESOURCE_USER_LIST_COMMENT |
|
26
|
|
|
|
|
27
|
|
|
from ..resource_manager import OcrdResourceManager |
|
28
|
|
|
|
|
29
|
|
|
|
|
30
|
|
|
def print_resources(executable, reslist, resmgr): |
|
31
|
|
|
print(f"{executable}") |
|
32
|
|
|
for resdict in reslist: |
|
33
|
|
|
res_loc = resmgr.resource_dir_to_location(resdict['path']) if 'path' in resdict else '' |
|
34
|
|
|
print(f"- {resdict['name']} @ {res_loc} ({resdict['url']})\n {resdict['description']}") |
|
35
|
|
|
print() |
|
36
|
|
|
|
|
37
|
|
|
|
|
38
|
|
|
@click.group("resmgr") |
|
39
|
|
|
def resmgr_cli(): |
|
40
|
|
|
""" |
|
41
|
|
|
Managing processor resources |
|
42
|
|
|
""" |
|
43
|
|
|
initLogging() |
|
44
|
|
|
|
|
45
|
|
|
|
|
46
|
|
|
@resmgr_cli.command('list-available') |
|
47
|
|
|
@click.option('-D', '--no-dynamic', is_flag=True, default=False, |
|
48
|
|
|
help="Whether to skip looking into each processor's --dump-{json,module-dir} for module-level resources") |
|
49
|
|
|
@click.option('-e', '--executable', metavar='EXEC', default='ocrd-*', |
|
50
|
|
|
help='Show only resources for executable beginning with EXEC', ) |
|
51
|
|
|
def list_available(executable, no_dynamic): |
|
52
|
|
|
""" |
|
53
|
|
|
List available resources |
|
54
|
|
|
""" |
|
55
|
|
|
resmgr = OcrdResourceManager() |
|
56
|
|
|
for executable, reslist in resmgr.list_available(executable=executable, dynamic=not no_dynamic): |
|
57
|
|
|
print_resources(executable, reslist, resmgr) |
|
58
|
|
|
|
|
59
|
|
|
|
|
60
|
|
|
@resmgr_cli.command('list-installed') |
|
61
|
|
|
@click.option('-e', '--executable', help='Show only resources for executable EXEC', metavar='EXEC') |
|
62
|
|
|
def list_installed(executable=None): |
|
63
|
|
|
""" |
|
64
|
|
|
List installed resources |
|
65
|
|
|
""" |
|
66
|
|
|
resmgr = OcrdResourceManager() |
|
67
|
|
|
for executable, reslist in resmgr.list_installed(executable): |
|
68
|
|
|
print_resources(executable, reslist, resmgr) |
|
69
|
|
|
|
|
70
|
|
|
|
|
71
|
|
|
@resmgr_cli.command('download') |
|
72
|
|
|
@click.option('-n', '--any-url', default='', help='URL of unregistered resource to download/copy from') |
|
73
|
|
|
@click.option('-D', '--no-dynamic', default=False, is_flag=True, |
|
74
|
|
|
help="Skip looking into each processor's --dump-{json,module-dir} module-registered resources") |
|
75
|
|
|
@click.option('-t', '--resource-type', type=click.Choice(RESOURCE_TYPES), default='file', |
|
76
|
|
|
help='Type of resource (when unregistered or incomplete)',) |
|
77
|
|
|
@click.option('-P', '--path-in-archive', default='.', help='Path to extract in case of archive type (when unregistered or incomplete)') |
|
78
|
|
|
@click.option('-a', '--allow-uninstalled', is_flag=True, |
|
79
|
|
|
help="Allow installing resources for not installed processors",) |
|
80
|
|
|
@click.option('-o', '--overwrite', help='Overwrite existing resources', is_flag=True) |
|
81
|
|
|
@click.option('-l', '--location', type=click.Choice(RESOURCE_LOCATIONS), |
|
82
|
|
|
help="Where to store resources - defaults to first location in processor's 'resource_locations' " |
|
83
|
|
|
"list, i.e. usually 'data'") |
|
84
|
|
|
@click.argument('executable', required=True) |
|
85
|
|
|
@click.argument('name', required=False) |
|
86
|
|
|
def download(any_url, no_dynamic, resource_type, path_in_archive, allow_uninstalled, overwrite, location, executable, |
|
87
|
|
|
name): |
|
88
|
|
|
""" |
|
89
|
|
|
Download resource NAME for processor EXECUTABLE. |
|
90
|
|
|
|
|
91
|
|
|
NAME is the name of the resource made available by downloading or copying. |
|
92
|
|
|
|
|
93
|
|
|
If NAME is '*' (asterisk), then download all known registered resources for this processor. |
|
94
|
|
|
|
|
95
|
|
|
If ``--any-url=URL`` or ``-n URL`` is given, then URL is accepted regardless of registered resources for ``NAME``. |
|
96
|
|
|
(This can be used for unknown resources or for replacing registered resources.) |
|
97
|
|
|
|
|
98
|
|
|
If ``--resource-type`` is set to `archive`, then that archive gets unpacked after download, |
|
99
|
|
|
and its ``--path-in-archive`` will subsequently be renamed to NAME. |
|
100
|
|
|
""" |
|
101
|
|
|
log = getLogger('ocrd.cli.resmgr') |
|
102
|
|
|
resmgr = OcrdResourceManager() |
|
103
|
|
|
if executable != '*' and not name: |
|
104
|
|
|
log.error(f"Unless EXECUTABLE ('{executable}') is the '*' wildcard, NAME is required") |
|
105
|
|
|
sys.exit(1) |
|
106
|
|
|
elif executable == '*': |
|
107
|
|
|
executable = None |
|
108
|
|
|
if name == '*': |
|
109
|
|
|
name = None |
|
110
|
|
|
if executable and not which(executable): |
|
111
|
|
|
if not allow_uninstalled: |
|
112
|
|
|
log.error(f"Executable '{executable}' is not installed. " |
|
113
|
|
|
f"To download resources anyway, use the -a/--allow-uninstalled flag") |
|
114
|
|
|
sys.exit(1) |
|
115
|
|
|
else: |
|
116
|
|
|
log.info(f"Executable '{executable}' is not installed, but downloading resources anyway") |
|
117
|
|
|
reslist = resmgr.list_available(executable=executable, dynamic=not no_dynamic, name=name) |
|
118
|
|
|
if not any(r[1] for r in reslist): |
|
119
|
|
|
log.info(f"No resources {name} found in registry for executable {executable}") |
|
120
|
|
|
if executable and name: |
|
121
|
|
|
reslist = [(executable, [{ |
|
122
|
|
|
'url': any_url or '???', |
|
123
|
|
|
'name': name, |
|
124
|
|
|
'type': resource_type, |
|
125
|
|
|
'path_in_archive': path_in_archive}] |
|
126
|
|
|
)] |
|
127
|
|
|
for this_executable, this_reslist in reslist: |
|
128
|
|
|
resource_locations = get_ocrd_tool_json(this_executable)['resource_locations'] |
|
129
|
|
|
if not location: |
|
130
|
|
|
location = resource_locations[0] |
|
131
|
|
|
elif location not in resource_locations: |
|
132
|
|
|
log.warning(f"The selected --location {location} is not in the {this_executable}'s resource search path, " |
|
133
|
|
|
f"refusing to install to invalid location. Instead installing to: {resource_locations[0]}") |
|
134
|
|
|
res_dest_dir = resmgr.build_resource_dest_dir(location=location, executable=this_executable) |
|
135
|
|
|
for res_dict in this_reslist: |
|
136
|
|
|
try: |
|
137
|
|
|
fpath = resmgr.handle_resource( |
|
138
|
|
|
res_dict=res_dict, |
|
139
|
|
|
executable=this_executable, |
|
140
|
|
|
dest_dir=res_dest_dir, |
|
141
|
|
|
any_url=any_url, |
|
142
|
|
|
overwrite=overwrite, |
|
143
|
|
|
resource_type=resource_type, |
|
144
|
|
|
path_in_archive=path_in_archive |
|
145
|
|
|
) |
|
146
|
|
|
if not fpath: |
|
147
|
|
|
continue |
|
148
|
|
|
except FileExistsError as exc: |
|
149
|
|
|
log.info(str(exc)) |
|
150
|
|
|
usage = res_dict.get('parameter_usage', 'as-is') |
|
151
|
|
|
log.info(f"Use in parameters as '{resmgr.parameter_usage(res_dict['name'], usage)}'") |
|
152
|
|
|
|
|
153
|
|
|
|
|
154
|
|
|
@resmgr_cli.command('migrate') |
|
155
|
|
|
@click.argument('migration', type=click.Choice(['2.37.0'])) |
|
156
|
|
|
def migrate(migration): |
|
157
|
|
|
""" |
|
158
|
|
|
Update the configuration after updating core to MIGRATION |
|
159
|
|
|
""" |
|
160
|
|
|
resmgr = OcrdResourceManager(skip_init=True) |
|
161
|
|
|
log = getLogger('ocrd.resmgr.migrate') |
|
162
|
|
|
if not resmgr.user_list.exists(): |
|
163
|
|
|
log.info(f'No configuration file found at {resmgr.user_list}, nothing to do') |
|
164
|
|
|
if migration == '2.37.0': |
|
165
|
|
|
backup_file = resmgr.user_list.with_suffix(f'.yml.before-{migration}') |
|
166
|
|
|
yaml_in_str = resmgr.user_list.read_text() |
|
167
|
|
|
log.info(f'Backing {resmgr.user_list} to {backup_file}') |
|
168
|
|
|
backup_file.write_text(yaml_in_str) |
|
169
|
|
|
log.info(f'Applying migration {migration} to {resmgr.user_list}') |
|
170
|
|
|
yaml_in = safe_load(yaml_in_str) |
|
171
|
|
|
yaml_out = {} |
|
172
|
|
|
for executable, reslist_in in yaml_in.items(): |
|
173
|
|
|
yaml_out[executable] = [] |
|
174
|
|
|
for resdict_in in reslist_in: |
|
175
|
|
|
resdict_out = {} |
|
176
|
|
|
for k_in, v_in in resdict_in.items(): |
|
177
|
|
|
k_out, v_out = k_in, v_in |
|
178
|
|
|
if k_in == 'type' and v_in in ['github-dir', 'tarball']: |
|
179
|
|
|
if v_in == 'github-dir': |
|
180
|
|
|
v_out = 'directory' |
|
181
|
|
|
elif v_in == 'tarball': |
|
182
|
|
|
v_out = 'directory' |
|
183
|
|
|
resdict_out[k_out] = v_out |
|
184
|
|
|
yaml_out[executable].append(resdict_out) |
|
185
|
|
|
resmgr.user_list.write_text( |
|
186
|
|
|
RESOURCE_USER_LIST_COMMENT + '\n# migrated with ocrd resmgr migrate {migration}\n' + safe_dump(yaml_out)) |
|
187
|
|
|
log.info(f'Applied migration {migration} to {resmgr.user_list}') |
|
188
|
|
|
|
OCR-D /
core
