first commit

core/budfox/budfox.js

@@ -0,0 +1,78 @@
+// Budfox is the realtime market for Gekko!
+//
+// Read more here:
+// @link https://github.com/askmike/gekko/blob/stable/docs/internals/budfox.md
+//
+// > [getting up] I don't know. I guess I realized that I'm just Bud Fox.
+// > As much as I wanted to be Gordon Gekko, I'll *always* be Bud Fox.
+// > [tosses back the handkerchief and walks away]
+
+var _ = require('lodash');
+var async = require('async');
+
+var util = require(__dirname + '/../util');
+var dirs = util.dirs();
+
+var Heart = require(dirs.budfox + 'heart');
+var MarketDataProvider =  require(dirs.budfox + 'marketDataProvider');
+var CandleManager = require(dirs.budfox + 'candleManager');
+
+var BudFox = function(config) {
+  _.bindAll(this);
+
+  Readable.call(this, {objectMode: true});
+
+  // BudFox internal modules:
+  
+  this.heart = new Heart;
+  this.marketDataProvider = new MarketDataProvider(config);
+  this.candleManager = new CandleManager;
+
+  //    BudFox data flow:
+
+  // relay a marketUpdate event
+  this.marketDataProvider.on(
+    'marketUpdate',
+    e => this.emit('marketUpdate', e)
+  );
+
+  // relay a marketStart event
+  this.marketDataProvider.on(
+    'marketStart',
+    e => this.emit('marketStart', e)
+  );
+
+  // Output the candles
+  this.candleManager.on(
+    'candles',
+    this.pushCandles
+  );
+
+  // on every `tick` retrieve trade data
+  this.heart.on(
+    'tick',
+    this.marketDataProvider.retrieve
+  );
+
+  // on new trade data create candles
+  this.marketDataProvider.on(
+    'trades',
+    this.candleManager.processTrades
+  );
+
+  this.heart.pump();
+}
+
+var Readable = require('stream').Readable;
+
+BudFox.prototype = Object.create(Readable.prototype, {
+  constructor: { value: BudFox }
+});
+
+BudFox.prototype._read = function noop() {}
+
+BudFox.prototype.pushCandles = function(candles) {
+  _.each(candles, this.push);
+}
+
+module.exports = BudFox;

core/budfox/candleCreator.js

@@ -0,0 +1,200 @@
+// The CandleCreator creates one minute candles based on trade batches. Note
+// that it also adds empty candles to fill gaps with no volume.
+//
+// Expects trade batches to be written like:
+//
+// {
+//   amount: x,
+//   start: (moment),
+//   end: (moment),
+//   first: (trade),
+//   last: (trade),
+//   timespan: x,
+//   all: [
+//      // batch of new trades with
+//      // moments instead of timestamps
+//   ]
+// }
+//
+// Emits 'new candles' event with:
+//
+// [
+//     {
+//       start: (moment),
+//       end: (moment),
+//       high: (float),
+//       open: (float),
+//       low: (float),
+//       close: (float)
+//       volume: (float)
+//       vwp: (float) // volume weighted price
+//    },
+//    {
+//       start: (moment), // + 1
+//       end: (moment),
+//       high: (float),
+//       open: (float),
+//      low: (float),
+//      close: (float)
+//       volume: (float)
+//       vwp: (float) // volume weighted price
+//    }
+//    // etc.
+// ]
+//
+
+var _ = require('lodash');
+var moment = require('moment');
+
+var util = require(__dirname + '/../util');
+
+var CandleCreator = function() {
+  _.bindAll(this);
+
+  // TODO: remove fixed date
+  this.threshold = moment("1970-01-01", "YYYY-MM-DD");
+
+  // This also holds the leftover between fetches
+  this.buckets = {};
+}
+
+util.makeEventEmitter(CandleCreator);
+
+CandleCreator.prototype.write = function(batch) {
+  var trades = batch.data;
+
+  if(_.isEmpty(trades))
+    return;
+
+  trades = this.filter(trades);
+  this.fillBuckets(trades);
+  var candles = this.calculateCandles();
+
+  candles = this.addEmptyCandles(candles);
+
+  if(_.isEmpty(candles))
+    return;  
+
+  // the last candle is not complete
+  this.threshold = candles.pop().start;
+
+  this.emit('candles', candles);
+}
+
+CandleCreator.prototype.filter = function(trades) {
+  // make sure we only include trades more recent
+  // than the previous emitted candle
+  return _.filter(trades, function(trade) {
+    return trade.date > this.threshold;
+  }, this);
+}
+
+// put each trade in a per minute bucket
+CandleCreator.prototype.fillBuckets = function(trades) {
+  _.each(trades, function(trade) {
+    var minute = trade.date.format('YYYY-MM-DD HH:mm');
+
+    if(!(minute in this.buckets))
+      this.buckets[minute] = [];
+
+    this.buckets[minute].push(trade);
+  }, this);
+
+  this.lastTrade = _.last(trades);
+}
+
+// convert each bucket into a candle
+CandleCreator.prototype.calculateCandles = function() {
+  var minutes = _.size(this.buckets);
+
+  // catch error from high volume getTrades
+  if (this.lastTrade !== undefined)
+    // create a string referencing the minute this trade happened in
+    var lastMinute = this.lastTrade.date.format('YYYY-MM-DD HH:mm');
+
+  var candles = _.map(this.buckets, function(bucket, name) {
+    var candle = this.calculateCandle(bucket);
+
+    // clean all buckets, except the last one:
+    // this candle is not complete
+    if(name !== lastMinute)
+      delete this.buckets[name];
+
+    return candle;
+  }, this);
+
+  return candles;
+}
+
+CandleCreator.prototype.calculateCandle = function(trades) {
+  var first = _.first(trades);
+
+  var f = parseFloat;
+
+  var candle = {
+    start: first.date.clone().startOf('minute'),
+    open: f(first.price),
+    high: f(first.price),
+    low: f(first.price),
+    close: f(_.last(trades).price),
+    vwp: 0,
+    volume: 0,
+    trades: _.size(trades)
+  };
+
+  _.each(trades, function(trade) {
+    candle.high = _.max([candle.high, f(trade.price)]);
+    candle.low = _.min([candle.low, f(trade.price)]);
+    candle.volume += f(trade.amount);
+    candle.vwp += f(trade.price) * f(trade.amount);
+  });
+
+  candle.vwp /= candle.volume;
+
+  return candle;
+}
+
+// Gekko expects a candle every minute, if nothing happened
+// during a particilar minute Gekko will add empty candles with:
+//
+// - open, high, close, low, vwp are the same as the close of the previous candle.
+// - trades, volume are 0
+CandleCreator.prototype.addEmptyCandles = function(candles) {
+  var amount = _.size(candles);
+  if(!amount)
+    return candles;
+
+  // iterator
+  var start = _.first(candles).start.clone();
+  var end = _.last(candles).start;
+  var i, j = -1;
+
+  var minutes = _.map(candles, function(candle) {
+    return +candle.start;
+  });
+
+  while(start < end) {
+    start.add(1, 'm');
+    i = +start;
+    j++;
+
+    if(_.contains(minutes, i))
+      continue; // we have a candle for this minute
+
+    var lastPrice = candles[j].close;
+
+    candles.splice(j + 1, 0, {
+      start: start.clone(),
+      open: lastPrice,
+      high: lastPrice,
+      low: lastPrice,
+      close: lastPrice,
+      vwp: lastPrice,
+      volume: 0,
+      trades: 0
+    });
+  }
+  return candles;
+}
+
+module.exports = CandleCreator;

core/budfox/candleManager.js

@@ -0,0 +1,34 @@
+// The candleManager consumes trades and emits:
+// - `candles`: array of minutly candles.
+// - `candle`: the most recent candle after a fetch Gekko.
+
+var _ = require('lodash');
+var moment = require('moment');
+var fs = require('fs');
+
+var util = require(__dirname + '/../util');
+var dirs = util.dirs();
+var config = util.getConfig();
+var log = require(dirs.core + 'log');
+
+var CandleCreator = require(dirs.budfox + 'candleCreator');
+
+var Manager = function() {
+  _.bindAll(this);
+
+  this.candleCreator = new CandleCreator;
+
+  this.candleCreator
+    .on('candles', this.relayCandles);
+};
+
+util.makeEventEmitter(Manager);
+Manager.prototype.processTrades = function(tradeBatch) {
+  this.candleCreator.write(tradeBatch);
+}
+
+Manager.prototype.relayCandles = function(candles) {
+  this.emit('candles', candles);
+}
+
+module.exports = Manager;

core/budfox/heart.js

@@ -0,0 +1,51 @@
+// The heart schedules and emit ticks every 20 seconds.
+
+var util = require(__dirname + '/../util');
+var log = require(util.dirs().core + 'log');
+
+var _ = require('lodash');
+var moment = require('moment');
+
+if (util.getConfig().watch.tickrate)
+  var TICKRATE = util.getConfig().watch.tickrate;
+else if(util.getConfig().watch.exchange === 'okcoin')
+  var TICKRATE = 2;
+else
+  var TICKRATE = 20;
+
+var Heart = function() {
+  this.lastTick = false;
+
+  _.bindAll(this);
+}
+
+util.makeEventEmitter(Heart);
+
+Heart.prototype.pump = function() {
+  log.debug('scheduling ticks');
+  this.scheduleTicks();
+}
+
+Heart.prototype.tick = function() {
+  if(this.lastTick) {
+    // make sure the last tick happened not to lang ago
+    // @link https://github.com/askmike/gekko/issues/514
+    if(this.lastTick < moment().unix() - TICKRATE * 3)
+      util.die('Failed to tick in time, see https://github.com/askmike/gekko/issues/514 for details', true);
+  }
+
+  this.lastTick = moment().unix();
+  this.emit('tick');
+}
+
+Heart.prototype.scheduleTicks = function() {
+  setInterval(
+    this.tick,
+    +moment.duration(TICKRATE, 's')
+  );
+
+  // start!
+  _.defer(this.tick);
+}
+
+module.exports = Heart;

core/budfox/marketDataProvider.js

@@ -0,0 +1,45 @@
+// 
+// The market data provider will fetch data from a datasource on tick. It emits:
+// 
+// - `trades`: batch of newly detected trades
+// - `trade`: after Gekko fetched new trades, this
+//   will be the most recent one.
+
+const _ = require('lodash');
+const util = require(__dirname + '/../util');
+
+const MarketFetcher = require('./marketFetcher');
+const dirs = util.dirs();
+
+const Manager = function(config) {
+
+  _.bindAll(this);
+
+  // fetch trades
+  this.source = new MarketFetcher(config);
+
+  // relay newly fetched trades
+  this.source
+    .on('trades batch', this.relayTrades);
+}
+
+util.makeEventEmitter(Manager);
+
+// HANDLERS
+Manager.prototype.retrieve = function() {
+  this.source.fetch();
+}
+
+
+Manager.prototype.relayTrades = function(batch) {
+  this.sendMarketStart(batch);
+  this.emit('marketUpdate', batch.last.date);
+
+  this.emit('trades', batch);
+}
+
+Manager.prototype.sendMarketStart = _.once(function(batch) {
+  this.emit('marketStart', batch.first.date);
+});
+
+module.exports = Manager;

core/budfox/marketFetcher.js

@@ -0,0 +1,109 @@
+//
+// The fetcher is responsible for fetching new
+// market data at the exchange on interval. It will emit
+// the following events:
+//
+// - `trades batch` - all new trades.
+// - `trade` - the most recent trade after every fetch
+
+const _ = require('lodash');
+const moment = require('moment');
+const utc = moment.utc;
+const util = require(__dirname + '/../util');
+const dirs = util.dirs();
+
+const config = util.getConfig();
+const log = require(dirs.core + 'log');
+const exchangeChecker = require(dirs.gekko + 'exchange/exchangeChecker');
+
+const TradeBatcher = require(util.dirs().budfox + 'tradeBatcher');
+
+const Fetcher = function(config) {
+  if(!_.isObject(config))
+    throw new Error('TradeFetcher expects a config');
+
+  const exchangeName = config.watch.exchange.toLowerCase();
+  const DataProvider = require(util.dirs().gekko + 'exchange/wrappers/' + exchangeName);
+  _.bindAll(this);
+
+  // Create a public dataProvider object which can retrieve live
+  // trade information from an exchange.
+  this.exchangeTrader = new DataProvider(config.watch);
+
+  this.exchange = exchangeChecker.settings(config.watch);
+
+  var requiredHistory = config.tradingAdvisor.candleSize * config.tradingAdvisor.historySize;
+
+  // If the trading adviser is enabled we might need a very specific fetch since
+  // to line up [local db, trading method, and fetching]
+  if(config.tradingAdvisor.enabled && config.tradingAdvisor.firstFetchSince) {
+    this.firstSince = config.tradingAdvisor.firstFetchSince;
+
+    if(this.exchange.providesHistory === 'date') {
+      this.firstSince = moment.unix(this.firstSince).utc();
+    }
+  }
+
+  this.batcher = new TradeBatcher(this.exchange.tid);
+
+  this.pair = [
+    config.watch.asset,
+    config.watch.currency
+  ].join('/');
+
+  log.info('Starting to watch the market:',
+    this.exchange.name,
+    this.pair
+  );
+
+  // if the exchange returns an error
+  // we will keep on retrying until next
+  // scheduled fetch.
+  this.tries = 0;
+  this.limit = 20; // [TODO]
+
+  this.firstFetch = true;
+
+  this.batcher.on('new batch', this.relayTrades);
+}
+
+util.makeEventEmitter(Fetcher);
+
+Fetcher.prototype._fetch = function(since) {
+  if(++this.tries >= this.limit)
+    return;
+
+  this.exchangeTrader.getTrades(since, this.processTrades, false);
+}
+
+Fetcher.prototype.fetch = function() {
+  var since = false;
+  if(this.firstFetch) {
+    since = this.firstSince;
+    this.firstFetch = false;
+  } else
+    since = false;
+
+  this.tries = 0;
+  log.debug('Requested', this.pair, 'trade data from', this.exchange.name, '...');
+  this._fetch(since);
+}
+
+Fetcher.prototype.processTrades = function(err, trades) {
+  if(err || _.isEmpty(trades)) {
+    if(err) {
+      log.warn(this.exchange.name, 'returned an error while fetching trades:', err);
+      log.debug('refetching...');
+    } else
+      log.debug('Trade fetch came back empty, refetching...');
+    setTimeout(this._fetch, +moment.duration('s', 1));
+    return;
+  }
+  this.batcher.write(trades);
+}
+
+Fetcher.prototype.relayTrades = function(batch) {
+  this.emit('trades batch', batch);
+}
+
+module.exports = Fetcher;

core/budfox/tradeBatcher.js

@@ -0,0 +1,124 @@
+// 
+// Small wrapper that only propagates new trades.
+// 
+// Expects trade batches to be written like:
+// [
+//  {
+//    tid: x, // tid is preferred, but if none available date will also work
+//    price: x,
+//    date: (timestamp),
+//    amount: x
+//  },
+//  {
+//    tid: x + 1,
+//    price: x,
+//    date: (timestamp),
+//    amount: x
+//  }
+// ]
+// 
+// Emits 'new trades' event with:
+// {
+//   amount: x,
+//   start: (moment),
+//   end: (moment),
+//   first: (trade),
+//   last: (trade)
+//   data: [
+//      // batch of new trades with 
+//      // moments instead of timestamps
+//   ]
+// }
+
+var _ = require('lodash');
+var moment = require('moment');
+var util = require('../util');
+var log = require('../log');
+
+var TradeBatcher = function(tid) {
+  if(!_.isString(tid))
+    throw new Error('tid is not a string');
+
+  _.bindAll(this);
+  this.tid = tid;
+  this.last = -1;
+}
+
+util.makeEventEmitter(TradeBatcher);
+
+TradeBatcher.prototype.write = function(batch) {
+
+  if(!_.isArray(batch))
+    throw new Error('batch is not an array');
+
+  if(_.isEmpty(batch))
+    return log.debug('Trade fetch came back empty.');
+
+  var filterBatch = this.filter(batch);
+
+  var amount = _.size(filterBatch);
+  if(!amount)
+    return log.debug('No new trades.');
+
+  var momentBatch = this.convertDates(filterBatch);
+
+  var last = _.last(momentBatch);
+  var first = _.first(momentBatch);
+
+  log.debug(
+    'Processing', amount, 'new trades.',
+    'From',
+    first.date.format('YYYY-MM-DD HH:mm:ss'),
+    'UTC to',
+    last.date.format('YYYY-MM-DD HH:mm:ss'),
+    'UTC.',
+    '(' + first.date.from(last.date, true) + ')'
+  );
+
+  this.emit('new batch', {
+    amount: amount,
+    start: first.date,
+    end: last.date,
+    last: last,
+    first: first,
+    data: momentBatch
+  });
+
+  this.last = last[this.tid];
+
+  // we overwrote those, get unix ts back
+  if(this.tid === 'date')
+    this.last = this.last.unix();
+
+}
+
+TradeBatcher.prototype.filter = function(batch) {
+  // make sure we're not trying to count
+  // beyond infinity
+  var lastTid = _.last(batch)[this.tid];
+  if(lastTid === lastTid + 1)
+    util.die('trade tid is max int, Gekko can\'t process..');
+
+  // remove trades that have zero amount
+  // see @link
+  // https://github.com/askmike/gekko/issues/486
+  batch = _.filter(batch, function(trade) {
+    return trade.amount > 0;
+  });
+
+  // weed out known trades
+  // TODO: optimize by stopping as soon as the
+  // first trade is too old (reverse first)
+  return _.filter(batch, function(trade) {
+    return this.last < trade[this.tid];
+  }, this);
+}
+
+TradeBatcher.prototype.convertDates = function(batch) {
+  return _.map(_.cloneDeep(batch), function(trade) {
+    trade.date = moment.unix(trade.date).utc();
+    return trade;
+  });
+}
+
+module.exports = TradeBatcher;