Completed
Pull Request — master (#858)
by Eddie
10:07 queued 01:13
created

to_dict()   B

Complexity

Conditions 4

Size

Total Lines 26

Duplication

Lines 0
Ratio 0 %
Metric Value
cc 4
dl 0
loc 26
rs 8.5806
1
#
2
# Copyright 2015 Quantopian, Inc.
3
#
4
# Licensed under the Apache License, Version 2.0 (the "License");
5
# you may not use this file except in compliance with the License.
6
# You may obtain a copy of the License at
7
#
8
#     http://www.apache.org/licenses/LICENSE-2.0
9
#
10
# Unless required by applicable law or agreed to in writing, software
11
# distributed under the License is distributed on an "AS IS" BASIS,
12
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
# See the License for the specific language governing permissions and
14
# limitations under the License.
15
16
"""
17
18
Performance Tracking
19
====================
20
21
    +-----------------+----------------------------------------------------+
22
    | key             | value                                              |
23
    +=================+====================================================+
24
    | period_start    | The beginning of the period to be tracked. datetime|
25
    |                 | in pytz.utc timezone. Will always be 0:00 on the   |
26
    |                 | date in UTC. The fact that the time may be on the  |
27
    |                 | prior day in the exchange's local time is ignored  |
28
    +-----------------+----------------------------------------------------+
29
    | period_end      | The end of the period to be tracked. datetime      |
30
    |                 | in pytz.utc timezone. Will always be 23:59 on the  |
31
    |                 | date in UTC. The fact that the time may be on the  |
32
    |                 | next day in the exchange's local time is ignored   |
33
    +-----------------+----------------------------------------------------+
34
    | progress        | percentage of test completed                       |
35
    +-----------------+----------------------------------------------------+
36
    | capital_base    | The initial capital assumed for this tracker.      |
37
    +-----------------+----------------------------------------------------+
38
    | cumulative_perf | A dictionary representing the cumulative           |
39
    |                 | performance through all the events delivered to    |
40
    |                 | this tracker. For details see the comments on      |
41
    |                 | :py:meth:`PerformancePeriod.to_dict`               |
42
    +-----------------+----------------------------------------------------+
43
    | todays_perf     | A dictionary representing the cumulative           |
44
    |                 | performance through all the events delivered to    |
45
    |                 | this tracker with datetime stamps between last_open|
46
    |                 | and last_close. For details see the comments on    |
47
    |                 | :py:meth:`PerformancePeriod.to_dict`               |
48
    |                 | TODO: adding this because we calculate it. May be  |
49
    |                 | overkill.                                          |
50
    +-----------------+----------------------------------------------------+
51
    | cumulative_risk | A dictionary representing the risk metrics         |
52
    | _metrics        | calculated based on the positions aggregated       |
53
    |                 | through all the events delivered to this tracker.  |
54
    |                 | For details look at the comments for               |
55
    |                 | :py:meth:`zipline.finance.risk.RiskMetrics.to_dict`|
56
    +-----------------+----------------------------------------------------+
57
58
"""
59
60
from __future__ import division
61
import logbook
62
from six import iteritems
63
from datetime import datetime
64
65
import pandas as pd
66
from pandas.tseries.tools import normalize_date
67
from zipline.finance.performance.period import PerformancePeriod
68
69
import zipline.finance.risk as risk
70
71
from zipline.utils.serialization_utils import (
72
    VERSION_LABEL
73
)
74
from . position_tracker import PositionTracker
75
76
log = logbook.Logger('Performance')
77
78
79
class PerformanceTracker(object):
80
    """
81
    Tracks the performance of the algorithm.
82
    """
83
    def __init__(self, sim_params, env, data_portal):
84
        self.sim_params = sim_params
85
        self.env = env
86
87
        self.period_start = self.sim_params.period_start
88
        self.period_end = self.sim_params.period_end
89
        self.last_close = self.sim_params.last_close
90
        first_open = self.sim_params.first_open.tz_convert(
91
            self.env.exchange_tz
92
        )
93
        self.day = pd.Timestamp(datetime(first_open.year, first_open.month,
94
                                         first_open.day), tz='UTC')
95
        self.market_open, self.market_close = env.get_open_and_close(self.day)
96
        self.total_days = self.sim_params.days_in_period
97
        self.capital_base = self.sim_params.capital_base
98
        self.emission_rate = sim_params.emission_rate
99
100
        all_trading_days = env.trading_days
101
        mask = ((all_trading_days >= normalize_date(self.period_start)) &
102
                (all_trading_days <= normalize_date(self.period_end)))
103
104
        self.trading_days = all_trading_days[mask]
105
106
        self._data_portal = data_portal
107
        if data_portal is not None:
108
            self._adjustment_reader = data_portal._adjustment_reader
109
        else:
110
            self._adjustment_reader = None
111
112
        self.position_tracker = PositionTracker(
113
            asset_finder=env.asset_finder,
114
            data_portal=data_portal,
115
            data_frequency=self.sim_params.data_frequency)
116
117
        if self.emission_rate == 'daily':
118
            self.all_benchmark_returns = pd.Series(
119
                index=self.trading_days)
120
            self.cumulative_risk_metrics = \
121
                risk.RiskMetricsCumulative(self.sim_params, self.env)
122
        elif self.emission_rate == 'minute':
123
            self.all_benchmark_returns = pd.Series(index=pd.date_range(
124
                self.sim_params.first_open, self.sim_params.last_close,
125
                freq='Min'))
126
127
            self.cumulative_risk_metrics = \
128
                risk.RiskMetricsCumulative(self.sim_params, self.env,
129
                                           create_first_day_stats=True)
130
131
        # this performance period will span the entire simulation from
132
        # inception.
133
        self.cumulative_performance = PerformancePeriod(
134
            # initial cash is your capital base.
135
            starting_cash=self.capital_base,
136
            data_frequency=self.sim_params.data_frequency,
137
            data_portal=data_portal,
138
            # the cumulative period will be calculated over the entire test.
139
            period_open=self.period_start,
140
            period_close=self.period_end,
141
            # don't save the transactions for the cumulative
142
            # period
143
            keep_transactions=False,
144
            keep_orders=False,
145
            # don't serialize positions for cumulative period
146
            serialize_positions=False,
147
            asset_finder=self.env.asset_finder,
148
            name="Cumulative"
149
        )
150
        self.cumulative_performance.position_tracker = self.position_tracker
151
152
        # this performance period will span just the current market day
153
        self.todays_performance = PerformancePeriod(
154
            # initial cash is your capital base.
155
            starting_cash=self.capital_base,
156
            data_frequency=self.sim_params.data_frequency,
157
            data_portal=data_portal,
158
            # the daily period will be calculated for the market day
159
            period_open=self.market_open,
160
            period_close=self.market_close,
161
            keep_transactions=True,
162
            keep_orders=True,
163
            serialize_positions=True,
164
            asset_finder=self.env.asset_finder,
165
            name="Daily"
166
        )
167
        self.todays_performance.position_tracker = self.position_tracker
168
169
        self.saved_dt = self.period_start
170
        # one indexed so that we reach 100%
171
        self.day_count = 0.0
172
        self.txn_count = 0
173
174
        self.account_needs_update = True
175
        self._account = None
176
177
    def __repr__(self):
178
        return "%s(%r)" % (
179
            self.__class__.__name__,
180
            {'simulation parameters': self.sim_params})
181
182
    @property
183
    def progress(self):
184
        if self.emission_rate == 'minute':
185
            # Fake a value
186
            return 1.0
187
        elif self.emission_rate == 'daily':
188
            return self.day_count / self.total_days
189
190
    def set_date(self, date):
191
        if self.emission_rate == 'minute':
192
            self.saved_dt = date
193
            self.todays_performance.period_close = self.saved_dt
194
195
    def get_portfolio(self, performance_needs_update, dt):
196
        if performance_needs_update:
197
            self.position_tracker.sync_last_sale_prices(dt)
198
            self.update_performance()
199
            self.account_needs_update = True
200
        return self.cumulative_performance.as_portfolio()
201
202
    def update_performance(self):
203
        # calculate performance as of last trade
204
        self.cumulative_performance.calculate_performance()
205
        self.todays_performance.calculate_performance()
206
207
    def get_account(self, performance_needs_update, dt):
208
        if performance_needs_update:
209
            self.position_tracker.sync_last_sale_prices(dt)
210
            self.update_performance()
211
            self.account_needs_update = True
212
        if self.account_needs_update:
213
            self._update_account()
214
        return self._account
215
216
    def _update_account(self):
217
        self._account = self.cumulative_performance.as_account()
218
        self.account_needs_update = False
219
220
    def to_dict(self, emission_type=None):
221
        """
222
        Creates a dictionary representing the state of this tracker.
223
        Returns a dict object of the form described in header comments.
224
        """
225
        # Default to the emission rate of this tracker if no type is provided
226
        if emission_type is None:
227
            emission_type = self.emission_rate
228
229
        _dict = {
230
            'period_start': self.period_start,
231
            'period_end': self.period_end,
232
            'capital_base': self.capital_base,
233
            'cumulative_perf': self.cumulative_performance.to_dict(),
234
            'progress': self.progress,
235
            'cumulative_risk_metrics': self.cumulative_risk_metrics.to_dict()
236
        }
237
        if emission_type == 'daily':
238
            _dict['daily_perf'] = self.todays_performance.to_dict()
239
        elif emission_type == 'minute':
240
            _dict['minute_perf'] = self.todays_performance.to_dict(
241
                self.saved_dt)
242
        else:
243
            raise ValueError("Invalid emission type: %s" % emission_type)
244
245
        return _dict
246
247
    def copy_state_from(self, other_perf_tracker):
248
        self.all_benchmark_returns = other_perf_tracker.all_benchmark_returns
249
250
        if other_perf_tracker.position_tracker:
251
            self.position_tracker._unpaid_dividends = \
252
                other_perf_tracker.position_tracker._unpaid_dividends
253
254
            self.position_tracker._unpaid_stock_dividends = \
255
                other_perf_tracker.position_tracker._unpaid_stock_dividends
256
257
    def process_transaction(self, transaction):
258
        self.txn_count += 1
259
        self.cumulative_performance.handle_execution(transaction)
260
        self.todays_performance.handle_execution(transaction)
261
        self.position_tracker.execute_transaction(transaction)
262
263
    def handle_splits(self, splits):
264
        leftover_cash = self.position_tracker.handle_splits(splits)
265
        if leftover_cash > 0:
266
            self.cumulative_performance.handle_cash_payment(leftover_cash)
267
            self.todays_performance.handle_cash_payment(leftover_cash)
268
269
    def process_order(self, event):
270
        self.cumulative_performance.record_order(event)
271
        self.todays_performance.record_order(event)
272
273
    def process_commission(self, commission):
274
        sid = commission['sid']
275
        cost = commission['cost']
276
277
        self.position_tracker.handle_commission(sid, cost)
278
        self.cumulative_performance.handle_commission(cost)
279
        self.todays_performance.handle_commission(cost)
280
281
    def process_close_position(self, event):
282
        txn = self.position_tracker.\
283
            maybe_create_close_position_transaction(event)
284
        if txn:
285
            self.process_transaction(txn)
286
287
    def check_upcoming_dividends(self, next_trading_day):
288
        """
289
        Check if we currently own any stocks with dividends whose ex_date is
290
        the next trading day.  Track how much we should be payed on those
291
        dividends' pay dates.
292
293
        Then check if we are owed cash/stock for any dividends whose pay date
294
        is the next trading day.  Apply all such benefits, then recalculate
295
        performance.
296
        """
297
        if self._adjustment_reader is None:
298
            return
299
        position_tracker = self.position_tracker
300
        held_sids = set(position_tracker.positions)
301
        # Dividends whose ex_date is the next trading day.  We need to check if
302
        # we own any of these stocks so we know to pay them out when the pay
303
        # date comes.
304
        if held_sids:
305
            dividends_earnable = self._adjustment_reader.\
306
                get_dividends_with_ex_date(held_sids, next_trading_day)
307
            stock_dividends = self._adjustment_reader.\
308
                get_stock_dividends_with_ex_date(held_sids, next_trading_day)
309
            position_tracker.earn_dividends(dividends_earnable,
310
                                            stock_dividends)
311
312
        net_cash_payment = position_tracker.pay_dividends(next_trading_day)
313
        if not net_cash_payment:
314
            return
315
316
        self.cumulative_performance.handle_dividends_paid(net_cash_payment)
317
        self.todays_performance.handle_dividends_paid(net_cash_payment)
318
319
    def check_asset_auto_closes(self, next_trading_day):
320
        """
321
        Check if the position tracker currently owns any Assets with an
322
        auto-close date that is the next trading day.  Close those positions.
323
324
        Parameters
325
        ----------
326
        next_trading_day : pandas.Timestamp
327
            The next trading day of the simulation
328
        """
329
        auto_close_events = self.position_tracker.auto_close_position_events(
330
            next_trading_day=next_trading_day
331
        )
332
        for event in auto_close_events:
333
            self.process_close_position(event)
334
335
    def handle_minute_close(self, dt):
336
        """
337
        Handles the close of the given minute. This includes handling
338
        market-close functions if the given minute is the end of the market
339
        day.
340
341
        Parameters
342
        __________
343
        dt : Timestamp
344
            The minute that is ending
345
346
        Returns
347
        _______
348
        (dict, dict/None)
349
            A tuple of the minute perf packet and daily perf packet.
350
            If the market day has not ended, the daily perf packet is None.
351
        """
352
        self.position_tracker.sync_last_sale_prices(dt)
353
        self.update_performance()
354
        todays_date = normalize_date(dt)
355
        account = self.get_account(False, dt)
356
357
        bench_returns = self.all_benchmark_returns.loc[todays_date:dt]
358
        # cumulative returns
359
        bench_since_open = (1. + bench_returns).prod() - 1
360
361
        self.cumulative_risk_metrics.update(todays_date,
362
                                            self.todays_performance.returns,
363
                                            bench_since_open,
364
                                            account.leverage)
365
366
        minute_packet = self.to_dict(emission_type='minute')
367
368
        # if this is the close, update dividends for the next day.
369
        # Return the performance tuple
370
        if dt == self.market_close:
371
            return minute_packet, self._handle_market_close(todays_date)
372
        else:
373
            return minute_packet, None
374
375
    def handle_market_close_daily(self, dt):
376
        """
377
        Function called after handle_data when running with daily emission
378
        rate.
379
        """
380
        self.position_tracker.sync_last_sale_prices(dt)
381
        self.update_performance()
382
        completed_date = self.day
383
        account = self.get_account(False, dt)
384
385
        benchmark_value = self.all_benchmark_returns[completed_date]
386
387
        self.cumulative_risk_metrics.update(
388
            completed_date,
389
            self.todays_performance.returns,
390
            benchmark_value,
391
            account.leverage)
392
393
        daily_packet = self._handle_market_close(completed_date)
394
395
        return daily_packet
396
397
    def _handle_market_close(self, completed_date):
398
399
        # increment the day counter before we move markers forward.
400
        self.day_count += 1.0
401
402
        # Get the next trading day and, if it is past the bounds of this
403
        # simulation, return the daily perf packet
404
        next_trading_day = self.env.next_trading_day(completed_date)
405
406
        # Check if any assets need to be auto-closed before generating today's
407
        # perf period
408
        if next_trading_day:
409
            self.check_asset_auto_closes(next_trading_day=next_trading_day)
410
411
        # Take a snapshot of our current performance to return to the
412
        # browser.
413
        daily_update = self.to_dict(emission_type='daily')
414
415
        # On the last day of the test, don't create tomorrow's performance
416
        # period.  We may not be able to find the next trading day if we're at
417
        # the end of our historical data
418
        if self.market_close >= self.last_close:
419
            return daily_update
420
421
        # move the market day markers forward
422
        self.market_open, self.market_close = \
423
            self.env.next_open_and_close(self.day)
424
        self.day = self.env.next_trading_day(self.day)
425
426
        # Roll over positions to current day.
427
        self.todays_performance.rollover()
428
        self.todays_performance.period_open = self.market_open
429
        self.todays_performance.period_close = self.market_close
430
431
        # If the next trading day is irrelevant, then return the daily packet
432
        if (next_trading_day is None) or (next_trading_day >= self.last_close):
433
            return daily_update
434
435
        # Check for any dividends and auto-closes, then return the daily perf
436
        # packet
437
        self.check_upcoming_dividends(next_trading_day=next_trading_day)
438
        return daily_update
439
440
    def handle_simulation_end(self):
441
        """
442
        When the simulation is complete, run the full period risk report
443
        and send it out on the results socket.
444
        """
445
446
        log_msg = "Simulated {n} trading days out of {m}."
447
        log.info(log_msg.format(n=int(self.day_count), m=self.total_days))
448
        log.info("first open: {d}".format(
449
            d=self.sim_params.first_open))
450
        log.info("last close: {d}".format(
451
            d=self.sim_params.last_close))
452
453
        bms = pd.Series(
454
            index=self.cumulative_risk_metrics.cont_index,
455
            data=self.cumulative_risk_metrics.benchmark_returns_cont)
456
        ars = pd.Series(
457
            index=self.cumulative_risk_metrics.cont_index,
458
            data=self.cumulative_risk_metrics.algorithm_returns_cont)
459
        acl = self.cumulative_risk_metrics.algorithm_cumulative_leverages
460
        self.risk_report = risk.RiskReport(
461
            ars,
462
            self.sim_params,
463
            benchmark_returns=bms,
464
            algorithm_leverages=acl,
465
            env=self.env)
466
467
        risk_dict = self.risk_report.to_dict()
468
        return risk_dict
469
470
    def __getstate__(self):
471
        state_dict = \
472
            {k: v for k, v in iteritems(self.__dict__)
473
                if not k.startswith('_')}
474
475
        STATE_VERSION = 4
476
        state_dict[VERSION_LABEL] = STATE_VERSION
477
478
        return state_dict
479
480
    def __setstate__(self, state):
481
482
        OLDEST_SUPPORTED_STATE = 4
483
        version = state.pop(VERSION_LABEL)
484
485
        if version < OLDEST_SUPPORTED_STATE:
486
            raise BaseException("PerformanceTracker saved state is too old.")
487
488
        self.__dict__.update(state)
489