1
|
|
|
# |
2
|
|
|
# Copyright 2015 Quantopian, Inc. |
3
|
|
|
# |
4
|
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); |
5
|
|
|
# you may not use this file except in compliance with the License. |
6
|
|
|
# You may obtain a copy of the License at |
7
|
|
|
# |
8
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0 |
9
|
|
|
# |
10
|
|
|
# Unless required by applicable law or agreed to in writing, software |
11
|
|
|
# distributed under the License is distributed on an "AS IS" BASIS, |
12
|
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
13
|
|
|
# See the License for the specific language governing permissions and |
14
|
|
|
# limitations under the License. |
15
|
|
|
|
16
|
|
|
from __future__ import division |
17
|
|
|
|
18
|
|
|
import logbook |
19
|
|
|
import numpy as np |
20
|
|
|
import pandas as pd |
21
|
|
|
from pandas.lib import checknull |
22
|
|
|
from collections import namedtuple |
23
|
|
|
try: |
24
|
|
|
# optional cython based OrderedDict |
25
|
|
|
from cyordereddict import OrderedDict |
26
|
|
|
except ImportError: |
27
|
|
|
from collections import OrderedDict |
28
|
|
|
from six import iteritems, itervalues |
29
|
|
|
|
30
|
|
|
from zipline.protocol import Event, DATASOURCE_TYPE |
31
|
|
|
from zipline.finance.transaction import Transaction |
32
|
|
|
from zipline.utils.serialization_utils import ( |
33
|
|
|
VERSION_LABEL |
34
|
|
|
) |
35
|
|
|
|
36
|
|
|
import zipline.protocol as zp |
37
|
|
|
from zipline.assets import ( |
38
|
|
|
Equity, Future |
39
|
|
|
) |
40
|
|
|
from zipline.errors import PositionTrackerMissingAssetFinder |
41
|
|
|
from . position import positiondict |
42
|
|
|
|
43
|
|
|
log = logbook.Logger('Performance') |
44
|
|
|
|
45
|
|
|
|
46
|
|
|
PositionStats = namedtuple('PositionStats', |
47
|
|
|
['net_exposure', |
48
|
|
|
'gross_value', |
49
|
|
|
'gross_exposure', |
50
|
|
|
'short_value', |
51
|
|
|
'short_exposure', |
52
|
|
|
'shorts_count', |
53
|
|
|
'long_value', |
54
|
|
|
'long_exposure', |
55
|
|
|
'longs_count', |
56
|
|
|
'net_value']) |
57
|
|
|
|
58
|
|
|
|
59
|
|
|
def calc_position_values(amounts, |
60
|
|
|
last_sale_prices, |
61
|
|
|
value_multipliers): |
62
|
|
|
iter_amount_price_multiplier = zip( |
63
|
|
|
amounts, |
64
|
|
|
last_sale_prices, |
65
|
|
|
itervalues(value_multipliers), |
66
|
|
|
) |
67
|
|
|
return [ |
68
|
|
|
price * amount * multiplier for |
69
|
|
|
price, amount, multiplier in iter_amount_price_multiplier |
70
|
|
|
] |
71
|
|
|
|
72
|
|
|
|
73
|
|
|
def calc_net(values): |
74
|
|
|
# Returns 0.0 if there are no values. |
75
|
|
|
return sum(values, np.float64()) |
76
|
|
|
|
77
|
|
|
|
78
|
|
|
def calc_position_exposures(amounts, |
79
|
|
|
last_sale_prices, |
80
|
|
|
exposure_multipliers): |
81
|
|
|
iter_amount_price_multiplier = zip( |
82
|
|
|
amounts, |
83
|
|
|
last_sale_prices, |
84
|
|
|
itervalues(exposure_multipliers), |
85
|
|
|
) |
86
|
|
|
return [ |
87
|
|
|
price * amount * multiplier for |
88
|
|
|
price, amount, multiplier in iter_amount_price_multiplier |
89
|
|
|
] |
90
|
|
|
|
91
|
|
|
|
92
|
|
|
def calc_long_value(position_values): |
93
|
|
|
return sum(i for i in position_values if i > 0) |
94
|
|
|
|
95
|
|
|
|
96
|
|
|
def calc_short_value(position_values): |
97
|
|
|
return sum(i for i in position_values if i < 0) |
98
|
|
|
|
99
|
|
|
|
100
|
|
|
def calc_long_exposure(position_exposures): |
101
|
|
|
return sum(i for i in position_exposures if i > 0) |
102
|
|
|
|
103
|
|
|
|
104
|
|
|
def calc_short_exposure(position_exposures): |
105
|
|
|
return sum(i for i in position_exposures if i < 0) |
106
|
|
|
|
107
|
|
|
|
108
|
|
|
def calc_longs_count(position_exposures): |
109
|
|
|
return sum(1 for i in position_exposures if i > 0) |
110
|
|
|
|
111
|
|
|
|
112
|
|
|
def calc_shorts_count(position_exposures): |
113
|
|
|
return sum(1 for i in position_exposures if i < 0) |
114
|
|
|
|
115
|
|
|
|
116
|
|
|
def calc_gross_exposure(long_exposure, short_exposure): |
117
|
|
|
return long_exposure + abs(short_exposure) |
118
|
|
|
|
119
|
|
|
|
120
|
|
|
def calc_gross_value(long_value, short_value): |
121
|
|
|
return long_value + abs(short_value) |
122
|
|
|
|
123
|
|
|
|
124
|
|
|
class PositionTracker(object): |
125
|
|
|
|
126
|
|
|
def __init__(self, asset_finder): |
127
|
|
|
self.asset_finder = asset_finder |
128
|
|
|
|
129
|
|
|
# sid => position object |
130
|
|
|
self.positions = positiondict() |
131
|
|
|
# Arrays for quick calculations of positions value |
132
|
|
|
self._position_value_multipliers = OrderedDict() |
133
|
|
|
self._position_exposure_multipliers = OrderedDict() |
134
|
|
|
self._unpaid_dividends = pd.DataFrame( |
135
|
|
|
columns=zp.DIVIDEND_PAYMENT_FIELDS, |
136
|
|
|
) |
137
|
|
|
self._positions_store = zp.Positions() |
138
|
|
|
|
139
|
|
|
# Dict, keyed on dates, that contains lists of close position events |
140
|
|
|
# for any Assets in this tracker's positions |
141
|
|
|
self._auto_close_position_sids = {} |
142
|
|
|
|
143
|
|
|
def _update_asset(self, sid): |
144
|
|
|
try: |
145
|
|
|
self._position_value_multipliers[sid] |
146
|
|
|
self._position_exposure_multipliers[sid] |
147
|
|
|
except KeyError: |
148
|
|
|
# Check if there is an AssetFinder |
149
|
|
|
if self.asset_finder is None: |
150
|
|
|
raise PositionTrackerMissingAssetFinder() |
151
|
|
|
|
152
|
|
|
# Collect the value multipliers from applicable sids |
153
|
|
|
asset = self.asset_finder.retrieve_asset(sid) |
154
|
|
|
if isinstance(asset, Equity): |
155
|
|
|
self._position_value_multipliers[sid] = 1 |
156
|
|
|
self._position_exposure_multipliers[sid] = 1 |
157
|
|
|
if isinstance(asset, Future): |
158
|
|
|
self._position_value_multipliers[sid] = 0 |
159
|
|
|
self._position_exposure_multipliers[sid] = \ |
160
|
|
|
asset.contract_multiplier |
161
|
|
|
# Futures auto-close timing is controlled by the Future's |
162
|
|
|
# auto_close_date property |
163
|
|
|
self._insert_auto_close_position_date( |
164
|
|
|
dt=asset.auto_close_date, |
165
|
|
|
sid=sid |
166
|
|
|
) |
167
|
|
|
|
168
|
|
|
def _insert_auto_close_position_date(self, dt, sid): |
169
|
|
|
""" |
170
|
|
|
Inserts the given SID in to the list of positions to be auto-closed by |
171
|
|
|
the given dt. |
172
|
|
|
|
173
|
|
|
Parameters |
174
|
|
|
---------- |
175
|
|
|
dt : pandas.Timestamp |
176
|
|
|
The date before-which the given SID will be auto-closed |
177
|
|
|
sid : int |
178
|
|
|
The SID of the Asset to be auto-closed |
179
|
|
|
""" |
180
|
|
|
if dt is not None: |
181
|
|
|
self._auto_close_position_sids.setdefault(dt, set()).add(sid) |
182
|
|
|
|
183
|
|
|
def auto_close_position_events(self, next_trading_day): |
184
|
|
|
""" |
185
|
|
|
Generates CLOSE_POSITION events for any SIDs whose auto-close date is |
186
|
|
|
before or equal to the given date. |
187
|
|
|
|
188
|
|
|
Parameters |
189
|
|
|
---------- |
190
|
|
|
next_trading_day : pandas.Timestamp |
191
|
|
|
The time before-which certain Assets need to be closed |
192
|
|
|
|
193
|
|
|
Yields |
194
|
|
|
------ |
195
|
|
|
Event |
196
|
|
|
A close position event for any sids that should be closed before |
197
|
|
|
the next_trading_day parameter |
198
|
|
|
""" |
199
|
|
|
past_asset_end_dates = set() |
200
|
|
|
|
201
|
|
|
# Check the auto_close_position_dates dict for SIDs to close |
202
|
|
|
for date, sids in self._auto_close_position_sids.items(): |
203
|
|
|
if date > next_trading_day: |
204
|
|
|
continue |
205
|
|
|
past_asset_end_dates.add(date) |
206
|
|
|
|
207
|
|
|
for sid in sids: |
208
|
|
|
# Yield a CLOSE_POSITION event |
209
|
|
|
event = Event({ |
210
|
|
|
'dt': date, |
211
|
|
|
'type': DATASOURCE_TYPE.CLOSE_POSITION, |
212
|
|
|
'sid': sid, |
213
|
|
|
}) |
214
|
|
|
yield event |
215
|
|
|
|
216
|
|
|
# Clear out past dates |
217
|
|
|
while past_asset_end_dates: |
218
|
|
|
self._auto_close_position_sids.pop(past_asset_end_dates.pop()) |
219
|
|
|
|
220
|
|
|
def update_last_sale(self, event): |
221
|
|
|
# NOTE, PerformanceTracker already vetted as TRADE type |
222
|
|
|
sid = event.sid |
223
|
|
|
if sid not in self.positions: |
224
|
|
|
return 0 |
225
|
|
|
|
226
|
|
|
price = event.price |
227
|
|
|
|
228
|
|
|
if checknull(price): |
229
|
|
|
return 0 |
230
|
|
|
|
231
|
|
|
pos = self.positions[sid] |
232
|
|
|
pos.last_sale_date = event.dt |
233
|
|
|
pos.last_sale_price = price |
234
|
|
|
|
235
|
|
|
def update_positions(self, positions): |
236
|
|
|
# update positions in batch |
237
|
|
|
self.positions.update(positions) |
238
|
|
|
for sid, pos in iteritems(positions): |
239
|
|
|
self._update_asset(sid) |
240
|
|
|
|
241
|
|
|
def update_position(self, sid, amount=None, last_sale_price=None, |
242
|
|
|
last_sale_date=None, cost_basis=None): |
243
|
|
|
pos = self.positions[sid] |
244
|
|
|
|
245
|
|
|
if amount is not None: |
246
|
|
|
pos.amount = amount |
247
|
|
|
self._update_asset(sid=sid) |
248
|
|
|
if last_sale_price is not None: |
249
|
|
|
pos.last_sale_price = last_sale_price |
250
|
|
|
if last_sale_date is not None: |
251
|
|
|
pos.last_sale_date = last_sale_date |
252
|
|
|
if cost_basis is not None: |
253
|
|
|
pos.cost_basis = cost_basis |
254
|
|
|
|
255
|
|
|
def execute_transaction(self, txn): |
256
|
|
|
# Update Position |
257
|
|
|
# ---------------- |
258
|
|
|
sid = txn.sid |
259
|
|
|
position = self.positions[sid] |
260
|
|
|
position.update(txn) |
261
|
|
|
self._update_asset(sid) |
262
|
|
|
|
263
|
|
|
def handle_commission(self, sid, cost): |
264
|
|
|
# Adjust the cost basis of the stock if we own it |
265
|
|
|
if sid in self.positions: |
266
|
|
|
self.positions[sid].adjust_commission_cost_basis(sid, cost) |
267
|
|
|
|
268
|
|
|
def handle_split(self, split): |
269
|
|
|
if split.sid in self.positions: |
270
|
|
|
# Make the position object handle the split. It returns the |
271
|
|
|
# leftover cash from a fractional share, if there is any. |
272
|
|
|
position = self.positions[split.sid] |
273
|
|
|
leftover_cash = position.handle_split(split.sid, split.ratio) |
274
|
|
|
self._update_asset(split.sid) |
275
|
|
|
return leftover_cash |
276
|
|
|
|
277
|
|
|
def _maybe_earn_dividend(self, dividend): |
278
|
|
|
""" |
279
|
|
|
Take a historical dividend record and return a Series with fields in |
280
|
|
|
zipline.protocol.DIVIDEND_FIELDS (plus an 'id' field) representing |
281
|
|
|
the cash/stock amount we are owed when the dividend is paid. |
282
|
|
|
""" |
283
|
|
|
if dividend['sid'] in self.positions: |
284
|
|
|
return self.positions[dividend['sid']].earn_dividend(dividend) |
285
|
|
|
else: |
286
|
|
|
return zp.dividend_payment() |
287
|
|
|
|
288
|
|
|
def earn_dividends(self, dividend_frame): |
289
|
|
|
""" |
290
|
|
|
Given a frame of dividends whose ex_dates are all the next trading day, |
291
|
|
|
calculate and store the cash and/or stock payments to be paid on each |
292
|
|
|
dividend's pay date. |
293
|
|
|
""" |
294
|
|
|
earned = dividend_frame.apply(self._maybe_earn_dividend, axis=1)\ |
295
|
|
|
.dropna(how='all') |
296
|
|
|
if len(earned) > 0: |
297
|
|
|
# Store the earned dividends so that they can be paid on the |
298
|
|
|
# dividends' pay_dates. |
299
|
|
|
self._unpaid_dividends = pd.concat( |
300
|
|
|
[self._unpaid_dividends, earned], |
301
|
|
|
) |
302
|
|
|
|
303
|
|
|
def _maybe_pay_dividend(self, dividend): |
304
|
|
|
""" |
305
|
|
|
Take a historical dividend record, look up any stored record of |
306
|
|
|
cash/stock we are owed for that dividend, and return a Series |
307
|
|
|
with fields drawn from zipline.protocol.DIVIDEND_PAYMENT_FIELDS. |
308
|
|
|
""" |
309
|
|
|
try: |
310
|
|
|
unpaid_dividend = self._unpaid_dividends.loc[dividend['id']] |
311
|
|
|
return unpaid_dividend |
312
|
|
|
except KeyError: |
313
|
|
|
return zp.dividend_payment() |
314
|
|
|
|
315
|
|
|
def pay_dividends(self, dividend_frame): |
316
|
|
|
""" |
317
|
|
|
Given a frame of dividends whose pay_dates are all the next trading |
318
|
|
|
day, grant the cash and/or stock payments that were calculated on the |
319
|
|
|
given dividends' ex dates. |
320
|
|
|
""" |
321
|
|
|
payments = dividend_frame.apply(self._maybe_pay_dividend, axis=1)\ |
322
|
|
|
.dropna(how='all') |
323
|
|
|
|
324
|
|
|
# Mark these dividends as paid by dropping them from our unpaid |
325
|
|
|
# table. |
326
|
|
|
self._unpaid_dividends.drop(payments.index) |
327
|
|
|
|
328
|
|
|
# Add stock for any stock dividends paid. Again, the values here may |
329
|
|
|
# be negative in the case of short positions. |
330
|
|
|
stock_payments = payments[payments['payment_sid'].notnull()] |
331
|
|
|
for _, row in stock_payments.iterrows(): |
332
|
|
|
stock = row['payment_sid'] |
333
|
|
|
share_count = row['share_count'] |
334
|
|
|
# note we create a Position for stock dividend if we don't |
335
|
|
|
# already own the asset |
336
|
|
|
position = self.positions[stock] |
337
|
|
|
|
338
|
|
|
position.amount += share_count |
339
|
|
|
self._update_asset(stock) |
340
|
|
|
|
341
|
|
|
# Add cash equal to the net cash payed from all dividends. Note that |
342
|
|
|
# "negative cash" is effectively paid if we're short an asset, |
343
|
|
|
# representing the fact that we're required to reimburse the owner of |
344
|
|
|
# the stock for any dividends paid while borrowing. |
345
|
|
|
net_cash_payment = payments['cash_amount'].fillna(0).sum() |
346
|
|
|
return net_cash_payment |
347
|
|
|
|
348
|
|
|
def maybe_create_close_position_transaction(self, event): |
349
|
|
|
try: |
350
|
|
|
pos = self.positions[event.sid] |
351
|
|
|
amount = pos.amount |
352
|
|
|
if amount == 0: |
353
|
|
|
return None |
354
|
|
|
except KeyError: |
355
|
|
|
return None |
356
|
|
|
if 'price' in event: |
357
|
|
|
price = event.price |
358
|
|
|
else: |
359
|
|
|
price = pos.last_sale_price |
360
|
|
|
txn = Transaction( |
361
|
|
|
sid=event.sid, |
362
|
|
|
amount=(-1 * pos.amount), |
363
|
|
|
dt=event.dt, |
364
|
|
|
price=price, |
365
|
|
|
commission=0, |
366
|
|
|
order_id=0 |
367
|
|
|
) |
368
|
|
|
return txn |
369
|
|
|
|
370
|
|
|
def get_positions(self): |
371
|
|
|
|
372
|
|
|
positions = self._positions_store |
373
|
|
|
|
374
|
|
|
for sid, pos in iteritems(self.positions): |
375
|
|
|
|
376
|
|
|
if pos.amount == 0: |
377
|
|
|
# Clear out the position if it has become empty since the last |
378
|
|
|
# time get_positions was called. Catching the KeyError is |
379
|
|
|
# faster than checking `if sid in positions`, and this can be |
380
|
|
|
# potentially called in a tight inner loop. |
381
|
|
|
try: |
382
|
|
|
del positions[sid] |
383
|
|
|
except KeyError: |
384
|
|
|
pass |
385
|
|
|
continue |
386
|
|
|
|
387
|
|
|
# Note that this will create a position if we don't currently have |
388
|
|
|
# an entry |
389
|
|
|
position = positions[sid] |
390
|
|
|
position.amount = pos.amount |
391
|
|
|
position.cost_basis = pos.cost_basis |
392
|
|
|
position.last_sale_price = pos.last_sale_price |
393
|
|
|
return positions |
394
|
|
|
|
395
|
|
|
def get_positions_list(self): |
396
|
|
|
positions = [] |
397
|
|
|
for sid, pos in iteritems(self.positions): |
398
|
|
|
if pos.amount != 0: |
399
|
|
|
positions.append(pos.to_dict()) |
400
|
|
|
return positions |
401
|
|
|
|
402
|
|
|
def stats(self): |
403
|
|
|
amounts = [] |
404
|
|
|
last_sale_prices = [] |
405
|
|
|
for pos in itervalues(self.positions): |
406
|
|
|
amounts.append(pos.amount) |
407
|
|
|
last_sale_prices.append(pos.last_sale_price) |
408
|
|
|
|
409
|
|
|
position_values = calc_position_values( |
410
|
|
|
amounts, |
411
|
|
|
last_sale_prices, |
412
|
|
|
self._position_value_multipliers |
413
|
|
|
) |
414
|
|
|
|
415
|
|
|
position_exposures = calc_position_exposures( |
416
|
|
|
amounts, |
417
|
|
|
last_sale_prices, |
418
|
|
|
self._position_exposure_multipliers |
419
|
|
|
) |
420
|
|
|
|
421
|
|
|
long_value = calc_long_value(position_values) |
422
|
|
|
short_value = calc_short_value(position_values) |
423
|
|
|
gross_value = calc_gross_value(long_value, short_value) |
424
|
|
|
long_exposure = calc_long_exposure(position_exposures) |
425
|
|
|
short_exposure = calc_short_exposure(position_exposures) |
426
|
|
|
gross_exposure = calc_gross_exposure(long_exposure, short_exposure) |
427
|
|
|
net_exposure = calc_net(position_exposures) |
428
|
|
|
longs_count = calc_longs_count(position_exposures) |
429
|
|
|
shorts_count = calc_shorts_count(position_exposures) |
430
|
|
|
net_value = calc_net(position_values) |
431
|
|
|
|
432
|
|
|
return PositionStats( |
433
|
|
|
long_value=long_value, |
434
|
|
|
gross_value=gross_value, |
435
|
|
|
short_value=short_value, |
436
|
|
|
long_exposure=long_exposure, |
437
|
|
|
short_exposure=short_exposure, |
438
|
|
|
gross_exposure=gross_exposure, |
439
|
|
|
net_exposure=net_exposure, |
440
|
|
|
longs_count=longs_count, |
441
|
|
|
shorts_count=shorts_count, |
442
|
|
|
net_value=net_value |
443
|
|
|
) |
444
|
|
|
|
445
|
|
|
def __getstate__(self): |
446
|
|
|
state_dict = {} |
447
|
|
|
|
448
|
|
|
state_dict['asset_finder'] = self.asset_finder |
449
|
|
|
state_dict['positions'] = dict(self.positions) |
450
|
|
|
state_dict['unpaid_dividends'] = self._unpaid_dividends |
451
|
|
|
state_dict['auto_close_position_sids'] = self._auto_close_position_sids |
452
|
|
|
|
453
|
|
|
STATE_VERSION = 3 |
454
|
|
|
state_dict[VERSION_LABEL] = STATE_VERSION |
455
|
|
|
return state_dict |
456
|
|
|
|
457
|
|
|
def __setstate__(self, state): |
458
|
|
|
OLDEST_SUPPORTED_STATE = 3 |
459
|
|
|
version = state.pop(VERSION_LABEL) |
460
|
|
|
|
461
|
|
|
if version < OLDEST_SUPPORTED_STATE: |
462
|
|
|
raise BaseException("PositionTracker saved state is too old.") |
463
|
|
|
|
464
|
|
|
self.asset_finder = state['asset_finder'] |
465
|
|
|
self.positions = positiondict() |
466
|
|
|
# note that positions_store is temporary and gets regened from |
467
|
|
|
# .positions |
468
|
|
|
self._positions_store = zp.Positions() |
469
|
|
|
|
470
|
|
|
self._unpaid_dividends = state['unpaid_dividends'] |
471
|
|
|
self._auto_close_position_sids = state['auto_close_position_sids'] |
472
|
|
|
|
473
|
|
|
# Arrays for quick calculations of positions value |
474
|
|
|
self._position_value_multipliers = OrderedDict() |
475
|
|
|
self._position_exposure_multipliers = OrderedDict() |
476
|
|
|
|
477
|
|
|
# Update positions is called without a finder |
478
|
|
|
self.update_positions(state['positions']) |
479
|
|
|
|