test_fetch_metadata_function_with_indices() - Code Metrics - Inspection of "fixed file storage observer" - IDSIA/sacred - Measure and Improve Code Quality continuously with Scrutinizer

Completed

Push — master ( dbc38f...56accc )

by Klaus

created 2017-01-16 09:53 UTC

test_fetch_metadata_function_with_indices() D

↳ Parent: Project

Complexity

Conditions

Size

Total Lines

Duplication

Lines	0
Ratio	0 %

Importance

Changes	2
Bugs	0	Features	0

Metric	Value
cc	8
dl	0
loc	38
rs	4
c	2
b	0
f	0

#!/usr/bin/env python
# coding=utf-8
from __future__ import (division, print_function, unicode_literals,
                        absolute_import)

import datetime
import tempfile
import io

import pytest

tinydb = pytest.importorskip("tinydb")
hashfs = pytest.importorskip("hashfs")

from tinydb import Query

from sacred.dependencies import get_digest
from sacred.observers.tinydb_hashfs import TinyDbObserver, TinyDbReader


# Utilities and fixtures
@pytest.fixture
def sample_run():

    T1 = datetime.datetime(1999, 5, 4, 3, 2, 1, 0)

    exp = {
        'name': 'test_exp',
        'sources': [],
        'doc': '',
        'base_dir': '/tmp',
        'dependencies': ['sacred==0.7b0']
    }
    host = {'hostname': 'test_host', 'cpu_count': 1, 'python_version': '3.4'}
    config = {'config': 'True', 'foo': 'bar', 'answer': 42}
    command = 'run'
    meta_info = {'comment': 'test run'}
    sample_run = {
        '_id': 'FED235DA13',
        'ex_info': exp,
        'command': command,
        'host_info': host,
        'start_time': T1,
        'config': config,
        'meta_info': meta_info,
    }

    filename = 'setup.py'
    md5 = get_digest(filename)
    sample_run['ex_info']['sources'] = [[filename, md5]]

    return sample_run


def run_test_experiment(exp_name, exp_id, root_dir):

    T2 = datetime.datetime(1999, 5, 5, 5, 5, 5, 5)
    T3 = datetime.datetime(1999, 5, 5, 6, 6, 6, 6)

    run_date = sample_run()
    run_date['ex_info']['name'] = exp_name
    run_date['_id'] = exp_id

    # Create
    tinydb_obs = TinyDbObserver.create(path=root_dir)

    # Start exp 1
    tinydb_obs.started_event(**run_date)

    # Heartbeat
    info = {'my_info': [1, 2, 3], 'nr': 7}
    outp = 'some output'
    tinydb_obs.heartbeat_event(info=info, captured_out=outp, beat_time=T2)
    # Add Artifact
    filename = "sacred/__about__.py"
    name = 'about'
    tinydb_obs.artifact_event(name, filename)

    # Add Resource
    filename = "sacred/__init__.py"
    tinydb_obs.resource_event(filename)

    # Complete
    tinydb_obs.completed_event(stop_time=T3, result=42)

    return tinydb_obs


def strip_file_handles(results):
    """Return a database result set with all file handle objects removed.abs

    Utility function to aid comparison of database entries. As file handles are
    created newly each object, these are always different so can be excluded.
    """

    if not isinstance(results, (list, tuple)):
        results = [results]

    cleaned_results = []
    for result in results:
        sources = result['experiment']['sources']
        artifacts = result['artifacts']
        resources = result['resources']
        if sources:
            for src in sources:
                if isinstance(src[-1], io.BufferedReader):
                    del src[-1]
        if artifacts:
            for art in artifacts:
                if isinstance(art[-1], io.BufferedReader):
                    del art[-1]
        if resources:
            for res in resources:
                if isinstance(res[-1], io.BufferedReader):
                    del res[-1]
        cleaned_results.append(result)

    return cleaned_results


# TinyDbReader Tests
def test_tinydb_reader_loads_db_and_fs(tmpdir):

    root = tmpdir.strpath
    tinydb_obs = run_test_experiment(exp_name='exp1', exp_id='1234', root_dir=root)
    tinydb_reader = TinyDbReader(root)

    assert tinydb_obs.fs.root == tinydb_reader.fs.root
    # Different file handles are used in each object so compar based on str
    # representation
    assert str(tinydb_obs.runs.all()[0]) == str(tinydb_reader.runs.all()[0])


def test_tinydb_reader_raises_exceptions(tmpdir):
    with pytest.raises(IOError):
        TinyDbReader('foo')


def test_fetch_metadata_function_with_indices(tmpdir, sample_run):

    # Setup and run three experiments
    root = tmpdir.strpath
    tinydb_obs = run_test_experiment(exp_name='experiment 1 alpha',
                                     exp_id='1234', root_dir=root)
    tinydb_obs = run_test_experiment(exp_name='experiment 2 beta',
                                     exp_id='5678', root_dir=root)
    tinydb_obs = run_test_experiment(exp_name='experiment 3 alpha',
                                     exp_id='9990', root_dir=root)

    tinydb_reader = TinyDbReader(root)

    # Test fetch by indices
    res = tinydb_reader.fetch_metadata(indices=-1)
    res2 = tinydb_reader.fetch_metadata(indices=[-1])
    assert strip_file_handles(res) == strip_file_handles(res2)
    res3 = tinydb_reader.fetch_metadata(indices=[0, -1])
    assert len(res3) == 2

    exp1_res = tinydb_reader.fetch_metadata(indices=0)
    assert len(exp1_res) == 1
    assert exp1_res[0]['experiment']['name'] == 'experiment 1 alpha'
    assert exp1_res[0]['_id'] == '1234'

    # Test Exception
    with pytest.raises(ValueError):
        tinydb_reader.fetch_metadata(indices=4)

    # Test returned values
    exp1 = strip_file_handles(exp1_res)[0]

    sample_run['ex_info']['name'] = 'experiment 1 alpha'
    sample_run['ex_info']['sources'] = [
        ['setup.py', get_digest('setup.py')]
    ]

    assert exp1 == {
        '_id': '1234',
        'experiment': sample_run['ex_info'],
        'format': tinydb_obs.VERSION,
        'command': sample_run['command'],
        'host': sample_run['host_info'],
        'start_time': sample_run['start_time'],
        'heartbeat': datetime.datetime(1999, 5, 5, 5, 5, 5, 5),
        'info': {'my_info': [1, 2, 3], 'nr': 7},
        'captured_out': 'some output',
        'artifacts': [
            ['about', 'sacred/__about__.py', get_digest('sacred/__about__.py')]
        ],
        'config': sample_run['config'],
        'meta': sample_run['meta_info'],
        'status': 'COMPLETED',
        'resources': [
            ['sacred/__init__.py', get_digest('sacred/__init__.py')]
        ],
        'result': 42,
        'stop_time': datetime.datetime(1999, 5, 5, 6, 6, 6, 6)
    }


def test_fetch_metadata_function_with_exp_name(tmpdir):

    # Setup and run three experiments
    root = tmpdir.strpath
    run_test_experiment(exp_name='experiment 1 alpha',
                        exp_id='1234', root_dir=root)
    run_test_experiment(exp_name='experiment 2 beta',
                        exp_id='5678', root_dir=root)
    run_test_experiment(exp_name='experiment 3 alpha',
                        exp_id='9990', root_dir=root)

    tinydb_reader = TinyDbReader(root)

    # Test Fetch by exp name
    res1 = tinydb_reader.fetch_metadata(exp_name='alpha')
    assert len(res1) == 2
    res2 = tinydb_reader.fetch_metadata(exp_name='experiment 1')
    assert len(res2) == 1
    assert res2[0]['experiment']['name'] == 'experiment 1 alpha'
    res2 = tinydb_reader.fetch_metadata(exp_name='foo')
    assert len(res2) == 0


def test_fetch_metadata_function_with_querry(tmpdir):

    # Setup and run three experiments
    root = tmpdir.strpath
    run_test_experiment(exp_name='experiment 1 alpha',
                        exp_id='1234', root_dir=root)
    run_test_experiment(exp_name='experiment 2 beta',
                        exp_id='5678', root_dir=root)
    run_test_experiment(exp_name='experiment 3 alpha beta',
                        exp_id='9990', root_dir=root)

    tinydb_reader = TinyDbReader(root)

    record = Query()

    exp1_query = record.experiment.name.matches('.*alpha$')

    exp3_query = (
        (record.experiment.name.search('alpha')) &
        (record._id == '9990')
    )

    # Test Fetch by Tinydb Query
    res1 = tinydb_reader.fetch_metadata(query=exp1_query)
    assert len(res1) == 1
    assert res1[0]['experiment']['name'] == 'experiment 1 alpha'

    res2 = tinydb_reader.fetch_metadata(
        query=record.experiment.name.search('experiment [23]'))
    assert len(res2) == 2

    res3 = tinydb_reader.fetch_metadata(query=exp3_query)
    assert len(res3) == 1
    assert res3[0]['experiment']['name'] == 'experiment 3 alpha beta'

    # Test Exception
    with pytest.raises(ValueError):
        tinydb_reader.fetch_metadata()


def test_search_function(tmpdir):

    # Setup and run three experiments
    root = tmpdir.strpath
    run_test_experiment(exp_name='experiment 1 alpha',
                        exp_id='1234', root_dir=root)
    run_test_experiment(exp_name='experiment 2 beta',
                        exp_id='5678', root_dir=root)
    run_test_experiment(exp_name='experiment 3 alpha beta',
                        exp_id='9990', root_dir=root)

    tinydb_reader = TinyDbReader(root)

    # Test Fetch by Tinydb Query in search function
    record = Query()
    q = record.experiment.name.search('experiment [23]')

    res = tinydb_reader.search(q)
    assert len(res) == 2
    res2 = tinydb_reader.fetch_metadata(query=q)
    assert strip_file_handles(res) == strip_file_handles(res2)


def test_fetch_files_function(tmpdir):
    # Setup and run three experiments
    root = tmpdir.strpath
    run_test_experiment(exp_name='experiment 1 alpha',
                        exp_id='1234', root_dir=root)
    run_test_experiment(exp_name='experiment 2 beta',
                        exp_id='5678', root_dir=root)
    run_test_experiment(exp_name='experiment 3 alpha beta',
                        exp_id='9990', root_dir=root)

    tinydb_reader = TinyDbReader(root)

    res = tinydb_reader.fetch_files(indices=0)
    assert len(res) == 1
    assert list(res[0]['artifacts'].keys()) == ['about']
    assert isinstance(res[0]['artifacts']['about'], io.BufferedReader)
    assert res[0]['date'] == datetime.datetime(1999, 5, 4, 3, 2, 1)
    assert res[0]['exp_id'] == '1234'
    assert res[0]['exp_name'] == 'experiment 1 alpha'
    assert list(res[0]['resources'].keys()) == ['sacred/__init__.py']
    assert isinstance(res[0]['resources']['sacred/__init__.py'], io.BufferedReader)
    assert list(res[0]['sources'].keys()) == ['setup.py']
    assert isinstance(res[0]['sources']['setup.py'], io.BufferedReader)


def test_fetch_report_function(tmpdir):

    # Setup and run three experiments
    root = tmpdir.strpath
    run_test_experiment(exp_name='experiment 1 alpha',
                        exp_id='1234', root_dir=root)
    run_test_experiment(exp_name='experiment 2 beta',
                        exp_id='5678', root_dir=root)
    run_test_experiment(exp_name='experiment 3 alpha beta',
                        exp_id='9990', root_dir=root)

    tinydb_reader = TinyDbReader(root)

    res = tinydb_reader.fetch_report(indices=0)

    target = """
-------------------------------------------------
Experiment: experiment 1 alpha
-------------------------------------------------
ID: 1234
Date: Tue 04 May 1999    Duration: 27:04:05.0

Parameters:
    answer: 42
    config: True
    foo: bar

Result:
    42

Dependencies:
    sacred==0.7b0

Resources:
    sacred/__init__.py

Source Files:
    setup.py

Outputs:
    about
"""

    assert res[0] == target


1			#!/usr/bin/env python
2			# coding=utf-8
3			from __future__ import (division, print_function, unicode_literals,
4			absolute_import)
5
6			import datetime
7			import tempfile
8			import io
9
10			import pytest
11
12			tinydb = pytest.importorskip("tinydb")
13			hashfs = pytest.importorskip("hashfs")
14
15			from tinydb import Query
16
17			from sacred.dependencies import get_digest
18			from sacred.observers.tinydb_hashfs import TinyDbObserver, TinyDbReader
19
20
21			# Utilities and fixtures
22			@pytest.fixture
23			def sample_run():
24
25			T1 = datetime.datetime(1999, 5, 4, 3, 2, 1, 0)
26
27			exp = {
28			'name': 'test_exp',
29			'sources': [],
30			'doc': '',
31			'base_dir': '/tmp',
32			'dependencies': ['sacred==0.7b0']
33			}
34			host = {'hostname': 'test_host', 'cpu_count': 1, 'python_version': '3.4'}
35			config = {'config': 'True', 'foo': 'bar', 'answer': 42}
36			command = 'run'
37			meta_info = {'comment': 'test run'}
38			sample_run = {
39			'_id': 'FED235DA13',
40			'ex_info': exp,
41			'command': command,
42			'host_info': host,
43			'start_time': T1,
44			'config': config,
45			'meta_info': meta_info,
46			}
47
48			filename = 'setup.py'
49			md5 = get_digest(filename)
50			sample_run['ex_info']['sources'] = [[filename, md5]]
51
52			return sample_run
53
54
55			def run_test_experiment(exp_name, exp_id, root_dir):
56
57			T2 = datetime.datetime(1999, 5, 5, 5, 5, 5, 5)
58			T3 = datetime.datetime(1999, 5, 5, 6, 6, 6, 6)
59
60			run_date = sample_run()
61			run_date['ex_info']['name'] = exp_name
62			run_date['_id'] = exp_id
63
64			# Create
65			tinydb_obs = TinyDbObserver.create(path=root_dir)
66
67			# Start exp 1
68			tinydb_obs.started_event(**run_date)
69
70			# Heartbeat
71			info = {'my_info': [1, 2, 3], 'nr': 7}
72			outp = 'some output'
73			tinydb_obs.heartbeat_event(info=info, captured_out=outp, beat_time=T2)
74			# Add Artifact
75			filename = "sacred/__about__.py"
76			name = 'about'
77			tinydb_obs.artifact_event(name, filename)
78
79			# Add Resource
80			filename = "sacred/__init__.py"
81			tinydb_obs.resource_event(filename)
82
83			# Complete
84			tinydb_obs.completed_event(stop_time=T3, result=42)
85
86			return tinydb_obs
87
88
89			def strip_file_handles(results):
90			"""Return a database result set with all file handle objects removed.abs
91
92			Utility function to aid comparison of database entries. As file handles are
93			created newly each object, these are always different so can be excluded.
94			"""
95
96			if not isinstance(results, (list, tuple)):
97			results = [results]
98
99			cleaned_results = []
100			for result in results:
101			sources = result['experiment']['sources']
102			artifacts = result['artifacts']
103			resources = result['resources']
104			if sources:
105			for src in sources:
106			if isinstance(src[-1], io.BufferedReader):
107			del src[-1]
108			if artifacts:
109			for art in artifacts:
110			if isinstance(art[-1], io.BufferedReader):
111			del art[-1]
112			if resources:
113			for res in resources:
114			if isinstance(res[-1], io.BufferedReader):
115			del res[-1]
116			cleaned_results.append(result)
117
118			return cleaned_results
119
120
121			# TinyDbReader Tests
122			def test_tinydb_reader_loads_db_and_fs(tmpdir):
123
124			root = tmpdir.strpath
125			tinydb_obs = run_test_experiment(exp_name='exp1', exp_id='1234', root_dir=root)
126			tinydb_reader = TinyDbReader(root)
127
128			assert tinydb_obs.fs.root == tinydb_reader.fs.root
129			# Different file handles are used in each object so compar based on str
130			# representation
131			assert str(tinydb_obs.runs.all()[0]) == str(tinydb_reader.runs.all()[0])
132
133
134			def test_tinydb_reader_raises_exceptions(tmpdir):
135			with pytest.raises(IOError):
136			TinyDbReader('foo')
137
138
139			def test_fetch_metadata_function_with_indices(tmpdir, sample_run):
140
141			# Setup and run three experiments
142			root = tmpdir.strpath
143			tinydb_obs = run_test_experiment(exp_name='experiment 1 alpha',
144			exp_id='1234', root_dir=root)
145			tinydb_obs = run_test_experiment(exp_name='experiment 2 beta',
146			exp_id='5678', root_dir=root)
147			tinydb_obs = run_test_experiment(exp_name='experiment 3 alpha',
148			exp_id='9990', root_dir=root)
149
150			tinydb_reader = TinyDbReader(root)
151
152			# Test fetch by indices
153			res = tinydb_reader.fetch_metadata(indices=-1)
154			res2 = tinydb_reader.fetch_metadata(indices=[-1])
155			assert strip_file_handles(res) == strip_file_handles(res2)
156			res3 = tinydb_reader.fetch_metadata(indices=[0, -1])
157			assert len(res3) == 2
158
159			exp1_res = tinydb_reader.fetch_metadata(indices=0)
160			assert len(exp1_res) == 1
161			assert exp1_res[0]['experiment']['name'] == 'experiment 1 alpha'
162			assert exp1_res[0]['_id'] == '1234'
163
164			# Test Exception
165			with pytest.raises(ValueError):
166			tinydb_reader.fetch_metadata(indices=4)
167
168			# Test returned values
169			exp1 = strip_file_handles(exp1_res)[0]
170
171			sample_run['ex_info']['name'] = 'experiment 1 alpha'
172			sample_run['ex_info']['sources'] = [
173			['setup.py', get_digest('setup.py')]
174			]
175
176			assert exp1 == {
177			'_id': '1234',
178			'experiment': sample_run['ex_info'],
179			'format': tinydb_obs.VERSION,
180			'command': sample_run['command'],
181			'host': sample_run['host_info'],
182			'start_time': sample_run['start_time'],
183			'heartbeat': datetime.datetime(1999, 5, 5, 5, 5, 5, 5),
184			'info': {'my_info': [1, 2, 3], 'nr': 7},
185			'captured_out': 'some output',
186			'artifacts': [
187			['about', 'sacred/__about__.py', get_digest('sacred/__about__.py')]
188			],
189			'config': sample_run['config'],
190			'meta': sample_run['meta_info'],
191			'status': 'COMPLETED',
192			'resources': [
193			['sacred/__init__.py', get_digest('sacred/__init__.py')]
194			],
195			'result': 42,
196			'stop_time': datetime.datetime(1999, 5, 5, 6, 6, 6, 6)
197			}
198
199
200			def test_fetch_metadata_function_with_exp_name(tmpdir):
201
202			# Setup and run three experiments
203			root = tmpdir.strpath
204			run_test_experiment(exp_name='experiment 1 alpha',
205			exp_id='1234', root_dir=root)
206			run_test_experiment(exp_name='experiment 2 beta',
207			exp_id='5678', root_dir=root)
208			run_test_experiment(exp_name='experiment 3 alpha',
209			exp_id='9990', root_dir=root)
210
211			tinydb_reader = TinyDbReader(root)
212
213			# Test Fetch by exp name
214			res1 = tinydb_reader.fetch_metadata(exp_name='alpha')
215			assert len(res1) == 2
216			res2 = tinydb_reader.fetch_metadata(exp_name='experiment 1')
217			assert len(res2) == 1
218			assert res2[0]['experiment']['name'] == 'experiment 1 alpha'
219			res2 = tinydb_reader.fetch_metadata(exp_name='foo')
220			assert len(res2) == 0
221
222
223			def test_fetch_metadata_function_with_querry(tmpdir):
224
225			# Setup and run three experiments
226			root = tmpdir.strpath
227			run_test_experiment(exp_name='experiment 1 alpha',
228			exp_id='1234', root_dir=root)
229			run_test_experiment(exp_name='experiment 2 beta',
230			exp_id='5678', root_dir=root)
231			run_test_experiment(exp_name='experiment 3 alpha beta',
232			exp_id='9990', root_dir=root)
233
234			tinydb_reader = TinyDbReader(root)
235
236			record = Query()
237
238			exp1_query = record.experiment.name.matches('.*alpha$')
239
240			exp3_query = (
241			(record.experiment.name.search('alpha')) &
242			(record._id == '9990')
243			)
244
245			# Test Fetch by Tinydb Query
246			res1 = tinydb_reader.fetch_metadata(query=exp1_query)
247			assert len(res1) == 1
248			assert res1[0]['experiment']['name'] == 'experiment 1 alpha'
249
250			res2 = tinydb_reader.fetch_metadata(
251			query=record.experiment.name.search('experiment [23]'))
252			assert len(res2) == 2
253
254			res3 = tinydb_reader.fetch_metadata(query=exp3_query)
255			assert len(res3) == 1
256			assert res3[0]['experiment']['name'] == 'experiment 3 alpha beta'
257
258			# Test Exception
259			with pytest.raises(ValueError):
260			tinydb_reader.fetch_metadata()
261
262
263			def test_search_function(tmpdir):
264
265			# Setup and run three experiments
266			root = tmpdir.strpath
267			run_test_experiment(exp_name='experiment 1 alpha',
268			exp_id='1234', root_dir=root)
269			run_test_experiment(exp_name='experiment 2 beta',
270			exp_id='5678', root_dir=root)
271			run_test_experiment(exp_name='experiment 3 alpha beta',
272			exp_id='9990', root_dir=root)
273
274			tinydb_reader = TinyDbReader(root)
275
276			# Test Fetch by Tinydb Query in search function
277			record = Query()
278			q = record.experiment.name.search('experiment [23]')
279
280			res = tinydb_reader.search(q)
281			assert len(res) == 2
282			res2 = tinydb_reader.fetch_metadata(query=q)
283			assert strip_file_handles(res) == strip_file_handles(res2)
284
285
286			def test_fetch_files_function(tmpdir):
287			# Setup and run three experiments
288			root = tmpdir.strpath
289			run_test_experiment(exp_name='experiment 1 alpha',
290			exp_id='1234', root_dir=root)
291			run_test_experiment(exp_name='experiment 2 beta',
292			exp_id='5678', root_dir=root)
293			run_test_experiment(exp_name='experiment 3 alpha beta',
294			exp_id='9990', root_dir=root)
295
296			tinydb_reader = TinyDbReader(root)
297
298			res = tinydb_reader.fetch_files(indices=0)
299			assert len(res) == 1
300			assert list(res[0]['artifacts'].keys()) == ['about']
301			assert isinstance(res[0]['artifacts']['about'], io.BufferedReader)
302			assert res[0]['date'] == datetime.datetime(1999, 5, 4, 3, 2, 1)
303			assert res[0]['exp_id'] == '1234'
304			assert res[0]['exp_name'] == 'experiment 1 alpha'
305			assert list(res[0]['resources'].keys()) == ['sacred/__init__.py']
306			assert isinstance(res[0]['resources']['sacred/__init__.py'], io.BufferedReader)
307			assert list(res[0]['sources'].keys()) == ['setup.py']
308			assert isinstance(res[0]['sources']['setup.py'], io.BufferedReader)
309
310
311			def test_fetch_report_function(tmpdir):
312
313			# Setup and run three experiments
314			root = tmpdir.strpath
315			run_test_experiment(exp_name='experiment 1 alpha',
316			exp_id='1234', root_dir=root)
317			run_test_experiment(exp_name='experiment 2 beta',
318			exp_id='5678', root_dir=root)
319			run_test_experiment(exp_name='experiment 3 alpha beta',
320			exp_id='9990', root_dir=root)
321
322			tinydb_reader = TinyDbReader(root)
323
324			res = tinydb_reader.fetch_report(indices=0)
325
326			target = """
327			-------------------------------------------------
328			Experiment: experiment 1 alpha
329			-------------------------------------------------
330			ID: 1234
331			Date: Tue 04 May 1999 Duration: 27:04:05.0
332
333			Parameters:
334			answer: 42
335			config: True
336			foo: bar
337
338			Result:
339			42
340
341			Dependencies:
342			sacred==0.7b0
343
344			Resources:
345			sacred/__init__.py
346
347			Source Files:
348			setup.py
349
350			Outputs:
351			about
352			"""
353
354			assert res[0] == target
355

IDSIA / sacred

Push — master ( dbc38f...56accc )

test_fetch_metadata_function_with_indices() D

Complexity

Size

Duplication

Importance

Duplication Side-by-Side

Filter issues like