|
1
|
|
|
""" |
|
2
|
|
|
OCR-D CLI: management of processor resources |
|
3
|
|
|
|
|
4
|
|
|
.. click:: ocrd.cli.resmgr:resmgr_cli |
|
5
|
|
|
:prog: ocrd resmgr |
|
6
|
|
|
:nested: full |
|
7
|
|
|
""" |
|
8
|
|
|
import sys |
|
9
|
|
|
from os import environ |
|
10
|
|
|
from pathlib import Path |
|
11
|
|
|
from distutils.spawn import find_executable as which |
|
12
|
|
|
from yaml import safe_load, safe_dump |
|
13
|
|
|
|
|
14
|
|
|
import requests |
|
15
|
|
|
import click |
|
16
|
|
|
|
|
17
|
|
|
from ocrd_utils import ( |
|
18
|
|
|
initLogging, |
|
19
|
|
|
directory_size, |
|
20
|
|
|
getLogger, |
|
21
|
|
|
RESOURCE_LOCATIONS, |
|
22
|
|
|
) |
|
23
|
|
|
from ocrd.constants import RESOURCE_USER_LIST_COMMENT |
|
24
|
|
|
|
|
25
|
|
|
from ..resource_manager import OcrdResourceManager |
|
26
|
|
|
|
|
27
|
|
|
def print_resources(executable, reslist, resmgr): |
|
28
|
|
|
print('%s' % executable) |
|
29
|
|
|
for resdict in reslist: |
|
30
|
|
|
print('- %s %s (%s)\n %s' % ( |
|
31
|
|
|
resdict['name'], |
|
32
|
|
|
'@ %s' % resmgr.resource_dir_to_location(resdict['path']) if 'path' in resdict else '', |
|
33
|
|
|
resdict['url'], |
|
34
|
|
|
resdict['description'] |
|
35
|
|
|
)) |
|
36
|
|
|
print() |
|
37
|
|
|
|
|
38
|
|
|
@click.group("resmgr") |
|
39
|
|
|
def resmgr_cli(): |
|
40
|
|
|
""" |
|
41
|
|
|
Managing processor resources |
|
42
|
|
|
""" |
|
43
|
|
|
initLogging() |
|
44
|
|
|
|
|
45
|
|
|
@resmgr_cli.command('list-available') |
|
46
|
|
|
@click.option('-D', '--no-dynamic', is_flag=True, default=False, help="Whether to skip looking into each processor's --dump-json for module-level resources") |
|
47
|
|
|
@click.option('-e', '--executable', help='Show only resources for executable beginning with EXEC', metavar='EXEC', default='ocrd-*') |
|
48
|
|
|
def list_available(executable, no_dynamic): |
|
49
|
|
|
""" |
|
50
|
|
|
List available resources |
|
51
|
|
|
""" |
|
52
|
|
|
resmgr = OcrdResourceManager() |
|
53
|
|
|
for executable, reslist in resmgr.list_available(executable=executable, dynamic=not no_dynamic): |
|
54
|
|
|
print_resources(executable, reslist, resmgr) |
|
55
|
|
|
|
|
56
|
|
|
@resmgr_cli.command('list-installed') |
|
57
|
|
|
@click.option('-e', '--executable', help='Show only resources for executable EXEC', metavar='EXEC') |
|
58
|
|
|
def list_installed(executable=None): |
|
59
|
|
|
""" |
|
60
|
|
|
List installed resources |
|
61
|
|
|
""" |
|
62
|
|
|
resmgr = OcrdResourceManager() |
|
63
|
|
|
for executable, reslist in resmgr.list_installed(executable): |
|
64
|
|
|
print_resources(executable, reslist, resmgr) |
|
65
|
|
|
|
|
66
|
|
|
@resmgr_cli.command('download') |
|
67
|
|
|
@click.option('-n', '--any-url', help='URL of unregistered resource to download/copy from', default='') |
|
68
|
|
|
@click.option('-t', '--resource-type', help='Type of resource', type=click.Choice(['file', 'directory', 'archive']), default='file') |
|
69
|
|
|
@click.option('-P', '--path-in-archive', help='Path to extract in case of archive type', default='.') |
|
70
|
|
|
@click.option('-a', '--allow-uninstalled', help="Allow installing resources for uninstalled processors", is_flag=True) |
|
71
|
|
|
@click.option('-o', '--overwrite', help='Overwrite existing resources', is_flag=True) |
|
72
|
|
|
@click.option('-l', '--location', help='Where to store resources', type=click.Choice(RESOURCE_LOCATIONS), default='data', show_default=True) |
|
73
|
|
|
@click.argument('executable', required=True) |
|
74
|
|
|
@click.argument('name', required=False) |
|
75
|
|
|
def download(any_url, resource_type, path_in_archive, allow_uninstalled, overwrite, location, executable, name): |
|
76
|
|
|
""" |
|
77
|
|
|
Download resource NAME for processor EXECUTABLE. |
|
78
|
|
|
|
|
79
|
|
|
NAME is the name of the resource made available by downloading or copying. |
|
80
|
|
|
|
|
81
|
|
|
If NAME is '*' (asterisk), then download all known registered resources for this processor. |
|
82
|
|
|
|
|
83
|
|
|
If ``--any-url=URL`` or ``-n URL`` is given, then URL is accepted regardless of registered resources for ``NAME``. |
|
84
|
|
|
(This can be used for unknown resources or for replacing registered resources.) |
|
85
|
|
|
|
|
86
|
|
|
If ``--resource-type`` is set to `archive`, then that archive gets unpacked after download, |
|
87
|
|
|
and its ``--path-in-archive`` will subsequently be renamed to NAME. |
|
88
|
|
|
""" |
|
89
|
|
|
log = getLogger('ocrd.cli.resmgr') |
|
90
|
|
|
resmgr = OcrdResourceManager() |
|
91
|
|
|
basedir = resmgr.location_to_resource_dir(location) |
|
92
|
|
|
if executable != '*' and not name: |
|
93
|
|
|
log.error("Unless EXECUTABLE ('%s') is the '*' wildcard, NAME is required" % executable) |
|
94
|
|
|
sys.exit(1) |
|
95
|
|
|
elif executable == '*': |
|
96
|
|
|
executable = None |
|
97
|
|
|
if name == '*': |
|
98
|
|
|
name = None |
|
99
|
|
|
is_url = (any_url.startswith('https://') or any_url.startswith('http://')) if any_url else False |
|
100
|
|
|
is_filename = Path(any_url).exists() if any_url else False |
|
101
|
|
|
if executable and not which(executable): |
|
102
|
|
|
if not allow_uninstalled: |
|
103
|
|
|
log.error("Executable '%s' is not installed. " \ |
|
104
|
|
|
"To download resources anyway, use the -a/--allow-uninstalled flag", executable) |
|
105
|
|
|
sys.exit(1) |
|
106
|
|
|
else: |
|
107
|
|
|
log.info("Executable %s is not installed, but " \ |
|
108
|
|
|
"downloading resources anyway", executable) |
|
109
|
|
|
reslist = resmgr.find_resources(executable=executable, name=name) |
|
110
|
|
|
if not reslist: |
|
111
|
|
|
log.info("No resources found in registry") |
|
112
|
|
|
if executable and name: |
|
113
|
|
|
reslist = [(executable, {'url': any_url or '???', 'name': name, |
|
114
|
|
|
'type': resource_type, |
|
115
|
|
|
'path_in_archive': path_in_archive})] |
|
116
|
|
|
for executable, resdict in reslist: |
|
117
|
|
|
if 'size' in resdict: |
|
118
|
|
|
registered = "registered" |
|
119
|
|
|
else: |
|
120
|
|
|
registered = "unregistered" |
|
121
|
|
|
if any_url: |
|
122
|
|
|
resdict['url'] = any_url |
|
123
|
|
|
if resdict['url'] == '???': |
|
124
|
|
|
log.warning("Cannot download user resource %s", resdict['name']) |
|
125
|
|
|
continue |
|
126
|
|
|
if resdict['url'].startswith('https://') or resdict['url'].startswith('http://'): |
|
127
|
|
|
log.info("Downloading %s resource '%s' (%s)", registered, resdict['name'], resdict['url']) |
|
128
|
|
|
with requests.get(resdict['url'], stream=True) as r: |
|
129
|
|
|
resdict['size'] = int(r.headers.get('content-length')) |
|
130
|
|
|
else: |
|
131
|
|
|
log.info("Copying %s resource '%s' (%s)", registered, resdict['name'], resdict['url']) |
|
132
|
|
|
urlpath = Path(resdict['url']) |
|
133
|
|
|
resdict['url'] = str(urlpath.resolve()) |
|
134
|
|
|
if Path(urlpath).is_dir(): |
|
135
|
|
|
resdict['size'] = directory_size(urlpath) |
|
136
|
|
|
else: |
|
137
|
|
|
resdict['size'] = urlpath.stat().st_size |
|
138
|
|
|
with click.progressbar(length=resdict['size']) as bar: |
|
139
|
|
|
fpath = resmgr.download( |
|
140
|
|
|
executable, |
|
141
|
|
|
resdict['url'], |
|
142
|
|
|
name=resdict['name'], |
|
143
|
|
|
resource_type=resdict.get('type', resource_type), |
|
144
|
|
|
path_in_archive=resdict.get('path_in_archive', path_in_archive), |
|
145
|
|
|
overwrite=overwrite, |
|
146
|
|
|
size=resdict['size'], |
|
147
|
|
|
no_subdir=location == 'cwd', |
|
148
|
|
|
basedir=basedir, |
|
149
|
|
|
progress_cb=lambda delta: bar.update(delta) |
|
|
|
|
|
|
150
|
|
|
) |
|
151
|
|
|
if registered == 'unregistered': |
|
152
|
|
|
log.info("%s resource '%s' (%s) not a known resource, creating stub in %s'", executable, name, any_url, resmgr.user_list) |
|
153
|
|
|
resmgr.add_to_user_database(executable, fpath, url=any_url) |
|
154
|
|
|
resmgr.save_user_list() |
|
155
|
|
|
log.info("Installed resource %s under %s", resdict['url'], fpath) |
|
156
|
|
|
log.info("Use in parameters as '%s'", resmgr.parameter_usage(resdict['name'], usage=resdict.get('parameter_usage', 'as-is'))) |
|
157
|
|
|
|
|
158
|
|
|
@resmgr_cli.command('migrate') |
|
159
|
|
|
@click.argument('migration', type=click.Choice(['2.37.0'])) |
|
160
|
|
|
def migrate(migration): |
|
161
|
|
|
""" |
|
162
|
|
|
Update the configuration after updating core to MIGRATION |
|
163
|
|
|
""" |
|
164
|
|
|
resmgr = OcrdResourceManager(skip_init=True) |
|
165
|
|
|
log = getLogger('ocrd.resmgr.migrate') |
|
166
|
|
|
if not resmgr.user_list.exists(): |
|
167
|
|
|
log.info(f'No configuration file found at {resmgr.user_list}, nothing to do') |
|
168
|
|
|
if migration == '2.37.0': |
|
169
|
|
|
backup_file = resmgr.user_list.with_suffix(f'.yml.before-{migration}') |
|
170
|
|
|
yaml_in_str = resmgr.user_list.read_text() |
|
171
|
|
|
log.info(f'Backing {resmgr.user_list} to {backup_file}') |
|
172
|
|
|
backup_file.write_text(yaml_in_str) |
|
173
|
|
|
log.info(f'Applying migration {migration} to {resmgr.user_list}') |
|
174
|
|
|
yaml_in = safe_load(yaml_in_str) |
|
175
|
|
|
yaml_out = {} |
|
176
|
|
|
for executable, reslist_in in yaml_in.items(): |
|
177
|
|
|
yaml_out[executable] = [] |
|
178
|
|
|
for resdict_in in reslist_in: |
|
179
|
|
|
resdict_out = {} |
|
180
|
|
|
for k_in, v_in in resdict_in.items(): |
|
181
|
|
|
k_out, v_out = k_in, v_in |
|
182
|
|
|
if k_in == 'type' and v_in in ['github-dir', 'tarball']: |
|
183
|
|
|
if v_in == 'github-dir': |
|
184
|
|
|
v_out = 'directory' |
|
185
|
|
|
elif v_in == 'tarball': |
|
186
|
|
|
v_out = 'directory' |
|
187
|
|
|
resdict_out[k_out] = v_out |
|
188
|
|
|
yaml_out[executable].append(resdict_out) |
|
189
|
|
|
resmgr.user_list.write_text(RESOURCE_USER_LIST_COMMENT + |
|
190
|
|
|
'\n# migrated with ocrd resmgr migrate {migration}\n' + |
|
191
|
|
|
safe_dump(yaml_out)) |
|
192
|
|
|
log.info(f'Applied migration {migration} to {resmgr.user_list}') |
|
193
|
|
|
|
OCR-D /
core
Loading history...