first commit

core/budfox/budfox.js

@@ -0,0 +1,78 @@
+// Budfox is the realtime market for Gekko!
+//
+// Read more here:
+// @link https://github.com/askmike/gekko/blob/stable/docs/internals/budfox.md
+//
+// > [getting up] I don't know. I guess I realized that I'm just Bud Fox.
+// > As much as I wanted to be Gordon Gekko, I'll *always* be Bud Fox.
+// > [tosses back the handkerchief and walks away]
+
+var _ = require('lodash');
+var async = require('async');
+
+var util = require(__dirname + '/../util');
+var dirs = util.dirs();
+
+var Heart = require(dirs.budfox + 'heart');
+var MarketDataProvider =  require(dirs.budfox + 'marketDataProvider');
+var CandleManager = require(dirs.budfox + 'candleManager');
+
+var BudFox = function(config) {
+  _.bindAll(this);
+
+  Readable.call(this, {objectMode: true});
+
+  // BudFox internal modules:
+  
+  this.heart = new Heart;
+  this.marketDataProvider = new MarketDataProvider(config);
+  this.candleManager = new CandleManager;
+
+  //    BudFox data flow:
+
+  // relay a marketUpdate event
+  this.marketDataProvider.on(
+    'marketUpdate',
+    e => this.emit('marketUpdate', e)
+  );
+
+  // relay a marketStart event
+  this.marketDataProvider.on(
+    'marketStart',
+    e => this.emit('marketStart', e)
+  );
+
+  // Output the candles
+  this.candleManager.on(
+    'candles',
+    this.pushCandles
+  );
+
+  // on every `tick` retrieve trade data
+  this.heart.on(
+    'tick',
+    this.marketDataProvider.retrieve
+  );
+
+  // on new trade data create candles
+  this.marketDataProvider.on(
+    'trades',
+    this.candleManager.processTrades
+  );
+
+  this.heart.pump();
+}
+
+var Readable = require('stream').Readable;
+
+BudFox.prototype = Object.create(Readable.prototype, {
+  constructor: { value: BudFox }
+});
+
+BudFox.prototype._read = function noop() {}
+
+BudFox.prototype.pushCandles = function(candles) {
+  _.each(candles, this.push);
+}
+
+module.exports = BudFox;

core/budfox/candleCreator.js

@@ -0,0 +1,200 @@
+// The CandleCreator creates one minute candles based on trade batches. Note
+// that it also adds empty candles to fill gaps with no volume.
+//
+// Expects trade batches to be written like:
+//
+// {
+//   amount: x,
+//   start: (moment),
+//   end: (moment),
+//   first: (trade),
+//   last: (trade),
+//   timespan: x,
+//   all: [
+//      // batch of new trades with
+//      // moments instead of timestamps
+//   ]
+// }
+//
+// Emits 'new candles' event with:
+//
+// [
+//     {
+//       start: (moment),
+//       end: (moment),
+//       high: (float),
+//       open: (float),
+//       low: (float),
+//       close: (float)
+//       volume: (float)
+//       vwp: (float) // volume weighted price
+//    },
+//    {
+//       start: (moment), // + 1
+//       end: (moment),
+//       high: (float),
+//       open: (float),
+//      low: (float),
+//      close: (float)
+//       volume: (float)
+//       vwp: (float) // volume weighted price
+//    }
+//    // etc.
+// ]
+//
+
+var _ = require('lodash');
+var moment = require('moment');
+
+var util = require(__dirname + '/../util');
+
+var CandleCreator = function() {
+  _.bindAll(this);
+
+  // TODO: remove fixed date
+  this.threshold = moment("1970-01-01", "YYYY-MM-DD");
+
+  // This also holds the leftover between fetches
+  this.buckets = {};
+}
+
+util.makeEventEmitter(CandleCreator);
+
+CandleCreator.prototype.write = function(batch) {
+  var trades = batch.data;
+
+  if(_.isEmpty(trades))
+    return;
+
+  trades = this.filter(trades);
+  this.fillBuckets(trades);
+  var candles = this.calculateCandles();
+
+  candles = this.addEmptyCandles(candles);
+
+  if(_.isEmpty(candles))
+    return;  
+
+  // the last candle is not complete
+  this.threshold = candles.pop().start;
+
+  this.emit('candles', candles);
+}
+
+CandleCreator.prototype.filter = function(trades) {
+  // make sure we only include trades more recent
+  // than the previous emitted candle
+  return _.filter(trades, function(trade) {
+    return trade.date > this.threshold;
+  }, this);
+}
+
+// put each trade in a per minute bucket
+CandleCreator.prototype.fillBuckets = function(trades) {
+  _.each(trades, function(trade) {
+    var minute = trade.date.format('YYYY-MM-DD HH:mm');
+
+    if(!(minute in this.buckets))
+      this.buckets[minute] = [];
+
+    this.buckets[minute].push(trade);
+  }, this);
+
+  this.lastTrade = _.last(trades);
+}
+
+// convert each bucket into a candle
+CandleCreator.prototype.calculateCandles = function() {
+  var minutes = _.size(this.buckets);
+
+  // catch error from high volume getTrades
+  if (this.lastTrade !== undefined)
+    // create a string referencing the minute this trade happened in
+    var lastMinute = this.lastTrade.date.format('YYYY-MM-DD HH:mm');
+
+  var candles = _.map(this.buckets, function(bucket, name) {
+    var candle = this.calculateCandle(bucket);
+
+    // clean all buckets, except the last one:
+    // this candle is not complete
+    if(name !== lastMinute)
+      delete this.buckets[name];
+
+    return candle;
+  }, this);
+
+  return candles;
+}
+
+CandleCreator.prototype.calculateCandle = function(trades) {
+  var first = _.first(trades);
+
+  var f = parseFloat;
+
+  var candle = {
+    start: first.date.clone().startOf('minute'),
+    open: f(first.price),
+    high: f(first.price),
+    low: f(first.price),
+    close: f(_.last(trades).price),
+    vwp: 0,
+    volume: 0,
+    trades: _.size(trades)
+  };
+
+  _.each(trades, function(trade) {
+    candle.high = _.max([candle.high, f(trade.price)]);
+    candle.low = _.min([candle.low, f(trade.price)]);
+    candle.volume += f(trade.amount);
+    candle.vwp += f(trade.price) * f(trade.amount);
+  });
+
+  candle.vwp /= candle.volume;
+
+  return candle;
+}
+
+// Gekko expects a candle every minute, if nothing happened
+// during a particilar minute Gekko will add empty candles with:
+//
+// - open, high, close, low, vwp are the same as the close of the previous candle.
+// - trades, volume are 0
+CandleCreator.prototype.addEmptyCandles = function(candles) {
+  var amount = _.size(candles);
+  if(!amount)
+    return candles;
+
+  // iterator
+  var start = _.first(candles).start.clone();
+  var end = _.last(candles).start;
+  var i, j = -1;
+
+  var minutes = _.map(candles, function(candle) {
+    return +candle.start;
+  });
+
+  while(start < end) {
+    start.add(1, 'm');
+    i = +start;
+    j++;
+
+    if(_.contains(minutes, i))
+      continue; // we have a candle for this minute
+
+    var lastPrice = candles[j].close;
+
+    candles.splice(j + 1, 0, {
+      start: start.clone(),
+      open: lastPrice,
+      high: lastPrice,
+      low: lastPrice,
+      close: lastPrice,
+      vwp: lastPrice,
+      volume: 0,
+      trades: 0
+    });
+  }
+  return candles;
+}
+
+module.exports = CandleCreator;

core/budfox/candleManager.js

@@ -0,0 +1,34 @@
+// The candleManager consumes trades and emits:
+// - `candles`: array of minutly candles.
+// - `candle`: the most recent candle after a fetch Gekko.
+
+var _ = require('lodash');
+var moment = require('moment');
+var fs = require('fs');
+
+var util = require(__dirname + '/../util');
+var dirs = util.dirs();
+var config = util.getConfig();
+var log = require(dirs.core + 'log');
+
+var CandleCreator = require(dirs.budfox + 'candleCreator');
+
+var Manager = function() {
+  _.bindAll(this);
+
+  this.candleCreator = new CandleCreator;
+
+  this.candleCreator
+    .on('candles', this.relayCandles);
+};
+
+util.makeEventEmitter(Manager);
+Manager.prototype.processTrades = function(tradeBatch) {
+  this.candleCreator.write(tradeBatch);
+}
+
+Manager.prototype.relayCandles = function(candles) {
+  this.emit('candles', candles);
+}
+
+module.exports = Manager;

core/budfox/heart.js

@@ -0,0 +1,51 @@
+// The heart schedules and emit ticks every 20 seconds.
+
+var util = require(__dirname + '/../util');
+var log = require(util.dirs().core + 'log');
+
+var _ = require('lodash');
+var moment = require('moment');
+
+if (util.getConfig().watch.tickrate)
+  var TICKRATE = util.getConfig().watch.tickrate;
+else if(util.getConfig().watch.exchange === 'okcoin')
+  var TICKRATE = 2;
+else
+  var TICKRATE = 20;
+
+var Heart = function() {
+  this.lastTick = false;
+
+  _.bindAll(this);
+}
+
+util.makeEventEmitter(Heart);
+
+Heart.prototype.pump = function() {
+  log.debug('scheduling ticks');
+  this.scheduleTicks();
+}
+
+Heart.prototype.tick = function() {
+  if(this.lastTick) {
+    // make sure the last tick happened not to lang ago
+    // @link https://github.com/askmike/gekko/issues/514
+    if(this.lastTick < moment().unix() - TICKRATE * 3)
+      util.die('Failed to tick in time, see https://github.com/askmike/gekko/issues/514 for details', true);
+  }
+
+  this.lastTick = moment().unix();
+  this.emit('tick');
+}
+
+Heart.prototype.scheduleTicks = function() {
+  setInterval(
+    this.tick,
+    +moment.duration(TICKRATE, 's')
+  );
+
+  // start!
+  _.defer(this.tick);
+}
+
+module.exports = Heart;

core/budfox/marketDataProvider.js

@@ -0,0 +1,45 @@
+// 
+// The market data provider will fetch data from a datasource on tick. It emits:
+// 
+// - `trades`: batch of newly detected trades
+// - `trade`: after Gekko fetched new trades, this
+//   will be the most recent one.
+
+const _ = require('lodash');
+const util = require(__dirname + '/../util');
+
+const MarketFetcher = require('./marketFetcher');
+const dirs = util.dirs();
+
+const Manager = function(config) {
+
+  _.bindAll(this);
+
+  // fetch trades
+  this.source = new MarketFetcher(config);
+
+  // relay newly fetched trades
+  this.source
+    .on('trades batch', this.relayTrades);
+}
+
+util.makeEventEmitter(Manager);
+
+// HANDLERS
+Manager.prototype.retrieve = function() {
+  this.source.fetch();
+}
+
+
+Manager.prototype.relayTrades = function(batch) {
+  this.sendMarketStart(batch);
+  this.emit('marketUpdate', batch.last.date);
+
+  this.emit('trades', batch);
+}
+
+Manager.prototype.sendMarketStart = _.once(function(batch) {
+  this.emit('marketStart', batch.first.date);
+});
+
+module.exports = Manager;

core/budfox/marketFetcher.js

@@ -0,0 +1,109 @@
+//
+// The fetcher is responsible for fetching new
+// market data at the exchange on interval. It will emit
+// the following events:
+//
+// - `trades batch` - all new trades.
+// - `trade` - the most recent trade after every fetch
+
+const _ = require('lodash');
+const moment = require('moment');
+const utc = moment.utc;
+const util = require(__dirname + '/../util');
+const dirs = util.dirs();
+
+const config = util.getConfig();
+const log = require(dirs.core + 'log');
+const exchangeChecker = require(dirs.gekko + 'exchange/exchangeChecker');
+
+const TradeBatcher = require(util.dirs().budfox + 'tradeBatcher');
+
+const Fetcher = function(config) {
+  if(!_.isObject(config))
+    throw new Error('TradeFetcher expects a config');
+
+  const exchangeName = config.watch.exchange.toLowerCase();
+  const DataProvider = require(util.dirs().gekko + 'exchange/wrappers/' + exchangeName);
+  _.bindAll(this);
+
+  // Create a public dataProvider object which can retrieve live
+  // trade information from an exchange.
+  this.exchangeTrader = new DataProvider(config.watch);
+
+  this.exchange = exchangeChecker.settings(config.watch);
+
+  var requiredHistory = config.tradingAdvisor.candleSize * config.tradingAdvisor.historySize;
+
+  // If the trading adviser is enabled we might need a very specific fetch since
+  // to line up [local db, trading method, and fetching]
+  if(config.tradingAdvisor.enabled && config.tradingAdvisor.firstFetchSince) {
+    this.firstSince = config.tradingAdvisor.firstFetchSince;
+
+    if(this.exchange.providesHistory === 'date') {
+      this.firstSince = moment.unix(this.firstSince).utc();
+    }
+  }
+
+  this.batcher = new TradeBatcher(this.exchange.tid);
+
+  this.pair = [
+    config.watch.asset,
+    config.watch.currency
+  ].join('/');
+
+  log.info('Starting to watch the market:',
+    this.exchange.name,
+    this.pair
+  );
+
+  // if the exchange returns an error
+  // we will keep on retrying until next
+  // scheduled fetch.
+  this.tries = 0;
+  this.limit = 20; // [TODO]
+
+  this.firstFetch = true;
+
+  this.batcher.on('new batch', this.relayTrades);
+}
+
+util.makeEventEmitter(Fetcher);
+
+Fetcher.prototype._fetch = function(since) {
+  if(++this.tries >= this.limit)
+    return;
+
+  this.exchangeTrader.getTrades(since, this.processTrades, false);
+}
+
+Fetcher.prototype.fetch = function() {
+  var since = false;
+  if(this.firstFetch) {
+    since = this.firstSince;
+    this.firstFetch = false;
+  } else
+    since = false;
+
+  this.tries = 0;
+  log.debug('Requested', this.pair, 'trade data from', this.exchange.name, '...');
+  this._fetch(since);
+}
+
+Fetcher.prototype.processTrades = function(err, trades) {
+  if(err || _.isEmpty(trades)) {
+    if(err) {
+      log.warn(this.exchange.name, 'returned an error while fetching trades:', err);
+      log.debug('refetching...');
+    } else
+      log.debug('Trade fetch came back empty, refetching...');
+    setTimeout(this._fetch, +moment.duration('s', 1));
+    return;
+  }
+  this.batcher.write(trades);
+}
+
+Fetcher.prototype.relayTrades = function(batch) {
+  this.emit('trades batch', batch);
+}
+
+module.exports = Fetcher;

core/budfox/tradeBatcher.js

@@ -0,0 +1,124 @@
+// 
+// Small wrapper that only propagates new trades.
+// 
+// Expects trade batches to be written like:
+// [
+//  {
+//    tid: x, // tid is preferred, but if none available date will also work
+//    price: x,
+//    date: (timestamp),
+//    amount: x
+//  },
+//  {
+//    tid: x + 1,
+//    price: x,
+//    date: (timestamp),
+//    amount: x
+//  }
+// ]
+// 
+// Emits 'new trades' event with:
+// {
+//   amount: x,
+//   start: (moment),
+//   end: (moment),
+//   first: (trade),
+//   last: (trade)
+//   data: [
+//      // batch of new trades with 
+//      // moments instead of timestamps
+//   ]
+// }
+
+var _ = require('lodash');
+var moment = require('moment');
+var util = require('../util');
+var log = require('../log');
+
+var TradeBatcher = function(tid) {
+  if(!_.isString(tid))
+    throw new Error('tid is not a string');
+
+  _.bindAll(this);
+  this.tid = tid;
+  this.last = -1;
+}
+
+util.makeEventEmitter(TradeBatcher);
+
+TradeBatcher.prototype.write = function(batch) {
+
+  if(!_.isArray(batch))
+    throw new Error('batch is not an array');
+
+  if(_.isEmpty(batch))
+    return log.debug('Trade fetch came back empty.');
+
+  var filterBatch = this.filter(batch);
+
+  var amount = _.size(filterBatch);
+  if(!amount)
+    return log.debug('No new trades.');
+
+  var momentBatch = this.convertDates(filterBatch);
+
+  var last = _.last(momentBatch);
+  var first = _.first(momentBatch);
+
+  log.debug(
+    'Processing', amount, 'new trades.',
+    'From',
+    first.date.format('YYYY-MM-DD HH:mm:ss'),
+    'UTC to',
+    last.date.format('YYYY-MM-DD HH:mm:ss'),
+    'UTC.',
+    '(' + first.date.from(last.date, true) + ')'
+  );
+
+  this.emit('new batch', {
+    amount: amount,
+    start: first.date,
+    end: last.date,
+    last: last,
+    first: first,
+    data: momentBatch
+  });
+
+  this.last = last[this.tid];
+
+  // we overwrote those, get unix ts back
+  if(this.tid === 'date')
+    this.last = this.last.unix();
+
+}
+
+TradeBatcher.prototype.filter = function(batch) {
+  // make sure we're not trying to count
+  // beyond infinity
+  var lastTid = _.last(batch)[this.tid];
+  if(lastTid === lastTid + 1)
+    util.die('trade tid is max int, Gekko can\'t process..');
+
+  // remove trades that have zero amount
+  // see @link
+  // https://github.com/askmike/gekko/issues/486
+  batch = _.filter(batch, function(trade) {
+    return trade.amount > 0;
+  });
+
+  // weed out known trades
+  // TODO: optimize by stopping as soon as the
+  // first trade is too old (reverse first)
+  return _.filter(batch, function(trade) {
+    return this.last < trade[this.tid];
+  }, this);
+}
+
+TradeBatcher.prototype.convertDates = function(batch) {
+  return _.map(_.cloneDeep(batch), function(trade) {
+    trade.date = moment.unix(trade.date).utc();
+    return trade;
+  });
+}
+
+module.exports = TradeBatcher;

core/candleBatcher.js

@@ -0,0 +1,93 @@
+// internally we only use 1m
+// candles, this can easily
+// convert them to any desired
+// size.
+
+// Acts as ~fake~ stream: takes
+// 1m candles as input and emits
+// bigger candles.
+// 
+// input are transported candles.
+
+var _ = require('lodash');
+var util = require(__dirname + '/util');
+
+var CandleBatcher = function(candleSize) {
+  if(!_.isNumber(candleSize))
+    throw new Error('candleSize is not a number');
+
+  this.candleSize = candleSize;
+  this.smallCandles = [];
+  this.calculatedCandles = [];
+
+  _.bindAll(this);
+}
+
+util.makeEventEmitter(CandleBatcher);
+
+CandleBatcher.prototype.write = function(candles) {
+  if(!_.isArray(candles)) {
+    throw new Error('candles is not an array');
+  }
+
+  this.emitted = 0;
+
+  _.each(candles, function(candle) {
+    this.smallCandles.push(candle);
+    this.check();
+  }, this);
+
+  return this.emitted;
+}
+
+CandleBatcher.prototype.check = function() {
+  if(_.size(this.smallCandles) % this.candleSize !== 0)
+    return;
+
+  this.emitted++;
+  this.calculatedCandles.push(this.calculate());
+  this.smallCandles = [];
+}
+
+CandleBatcher.prototype.flush = function() {
+  _.each(
+    this.calculatedCandles,
+    candle => this.emit('candle', candle)
+  );
+
+  this.calculatedCandles = [];
+}
+
+CandleBatcher.prototype.calculate = function() {
+  // remove the id property of the small candle
+  var { id, ...first } = this.smallCandles.shift();
+
+  first.vwp = first.vwp * first.volume;
+
+  var candle = _.reduce(
+    this.smallCandles,
+    function(candle, m) {
+      candle.high = _.max([candle.high, m.high]);
+      candle.low = _.min([candle.low, m.low]);
+      candle.close = m.close;
+      candle.volume += m.volume;
+      candle.vwp += m.vwp * m.volume;
+      candle.trades += m.trades;
+      return candle;
+    },
+    first
+  );
+
+  if(candle.volume)
+    // we have added up all prices (relative to volume)
+    // now divide by volume to get the Volume Weighted Price
+    candle.vwp /= candle.volume;
+  else
+    // empty candle
+    candle.vwp = candle.open;
+
+  candle.start = first.start;
+  return candle;
+}
+
+module.exports = CandleBatcher;

core/emitter.js

@@ -0,0 +1,34 @@
+// Gekko uses a custom event emitter within the GekkoStream (the plugins) to guarantee
+// the correct order of events that are triggered by eachother. Turns sync events from
+// LIFO into a FIFO stack based model.
+//
+// More details here: https://forum.gekko.wizb.it/thread-56579.html
+
+const util = require('util');
+const events = require('events');
+const NativeEventEmitter = events.EventEmitter;
+
+const GekkoEventEmitter = function() {
+  NativeEventEmitter.call(this);
+  this.defferedEvents = [];
+}
+
+util.inherits(GekkoEventEmitter, NativeEventEmitter);
+
+// push to stack
+GekkoEventEmitter.prototype.deferredEmit = function(name, payload) {
+  this.defferedEvents.push({name, payload});
+}
+
+// resolve FIFO
+GekkoEventEmitter.prototype.broadcastDeferredEmit = function() {
+  if(this.defferedEvents.length === 0)
+    return false;
+
+  const event = this.defferedEvents.shift();
+
+  this.emit(event.name, event.payload);
+  return true;
+}
+
+module.exports = GekkoEventEmitter;

core/gekkoStream.js

@@ -0,0 +1,121 @@
+// Small writable stream wrapper that
+// passes data to all `candleConsumers`.
+
+const Writable = require('stream').Writable;
+const _ = require('lodash');
+const async = require('async');
+const moment = require('moment');
+
+const util = require('./util');
+const env = util.gekkoEnv();
+const mode = util.gekkoMode();
+const config = util.getConfig();
+const log = require(util.dirs().core + 'log');
+
+var Gekko = function(plugins) {
+  this.plugins = plugins;
+  this.candleConsumers = plugins
+    .filter(plugin => plugin.processCandle);
+  Writable.call(this, {objectMode: true});
+
+  this.producers = this.plugins
+    .filter(p => p.meta.emits);
+
+  this.finalize = _.bind(this.finalize, this);
+}
+
+Gekko.prototype = Object.create(Writable.prototype, {
+  constructor: { value: Gekko }
+});
+
+if(config.debug && mode !== 'importer') {
+  // decorate with more debug information
+  Gekko.prototype._write = function(chunk, encoding, _done) {
+
+    if(chunk.isFinished) {
+      return this.finalize();
+    }
+
+    const start = moment();
+    var relayed = false;
+    var at = null;
+
+    const timer = setTimeout(() => {
+      if(!relayed)
+        log.error([
+          `The plugin "${at}" has not processed a candle for 1 second.`,
+          `This will cause Gekko to slow down or stop working completely.`
+        ].join(' '));
+    }, 1000);
+
+    const flushEvents = _.after(this.candleConsumers.length, () => {
+      relayed = true;
+      clearInterval(timer);
+      this.flushDefferedEvents();
+      _done();
+    });
+    _.each(this.candleConsumers, function(c) {
+      at = c.meta.name;
+      c.processCandle(chunk, flushEvents);
+    }, this);
+  }
+} else {
+  // skip decoration
+  Gekko.prototype._write = function(chunk, encoding, _done) {
+    if(chunk.isFinished) {
+      return this.finalize();
+    }
+
+    const flushEvents = _.after(this.candleConsumers.length, () => {
+      this.flushDefferedEvents();
+      _done();
+    });
+    _.each(this.candleConsumers, function(c) {
+      c.processCandle(chunk, flushEvents);
+    }, this);
+  }
+}
+
+Gekko.prototype.flushDefferedEvents = function() {
+  const broadcasted = _.find(
+    this.producers,
+    producer => producer.broadcastDeferredEmit()
+  );
+
+  // If we braodcasted anything we might have
+  // triggered more events, recurse until we
+  // have fully broadcasted everything.
+  if(broadcasted)
+    this.flushDefferedEvents();
+}
+
+Gekko.prototype.finalize = function() {
+  var tradingMethod = _.find(
+    this.candleConsumers,
+    c => c.meta.name === 'Trading Advisor'
+  );
+
+  if(!tradingMethod)
+    return this.shutdown();
+
+  tradingMethod.finish(this.shutdown.bind(this));
+}
+
+Gekko.prototype.shutdown = function() {
+  this.end();
+  async.eachSeries(
+    this.plugins,
+    function(c, callback) {
+      if (c.finalize) c.finalize(callback);
+      else callback();
+    },
+    () => {
+      // If we are a child process, we signal to the parent to kill the child once it is done
+      // so that is has time to process all remaining events (and send report data)
+      if (env === 'child-process') process.send('done');
+      else process.exit(0);
+    }
+  );
+};
+
+module.exports = Gekko;

core/log.js

@@ -0,0 +1,81 @@
+/*
+
+  Lightweight logger, print everything that is send to error, warn
+  and messages to stdout (the terminal). If config.debug is set in config
+  also print out everything send to debug.
+
+*/
+
+var moment = require('moment');
+var fmt = require('util').format;
+var _ = require('lodash');
+var util = require('./util');
+var config = util.getConfig();
+var debug = config.debug;
+var silent = config.silent;
+
+var sendToParent = function() {
+  var send = method => (...args) => {
+    process.send({log: method, message: args.join(' ')});
+  }
+
+  return {
+    error: send('error'),
+    warn: send('warn'),
+    info: send('info'),
+    write: send('write')
+  }
+}
+
+var Log = function() {
+  _.bindAll(this);
+  this.env = util.gekkoEnv();
+
+  if(this.env === 'standalone')
+    this.output = console;
+  else if(this.env === 'child-process')
+    this.output = sendToParent();
+};
+
+Log.prototype = {
+  _write: function(method, args, name) {
+    if(!name)
+      name = method.toUpperCase();
+
+    var message = moment().format('YYYY-MM-DD HH:mm:ss');
+    message += ' (' + name + '):\t';
+    message += fmt.apply(null, args);
+
+    this.output[method](message);
+  },
+  error: function() {
+    this._write('error', arguments);
+  },
+  warn: function() {
+    this._write('warn', arguments);
+  },
+  info: function() {
+    this._write('info', arguments);
+  },
+  write: function() {
+    var args = _.toArray(arguments);
+    var message = fmt.apply(null, args);
+    this.output.info(message);
+  }
+}
+
+if(debug)
+  Log.prototype.debug = function() {
+    this._write('info', arguments, 'DEBUG');  
+  }
+else
+  Log.prototype.debug = _.noop;
+
+if(silent) {
+  Log.prototype.debug = _.noop;
+  Log.prototype.info = _.noop;
+  Log.prototype.warn = _.noop;
+  Log.prototype.write = _.noop;
+}
+
+module.exports = new Log;

core/markets/backtest.js

@@ -0,0 +1,108 @@
+var _ = require('lodash');
+var util = require('../util');
+var config = util.getConfig();
+var dirs = util.dirs();
+var log = require(dirs.core + 'log');
+var moment = require('moment');
+
+var adapter = config[config.adapter];
+var Reader = require(dirs.gekko + adapter.path + '/reader');
+var daterange = config.backtest.daterange;
+
+var to = moment.utc(daterange.to);
+var from = moment.utc(daterange.from);
+
+if(to <= from)
+  util.die('This daterange does not make sense.')
+
+if(!from.isValid())
+  util.die('invalid `from`');
+
+if(!to.isValid())
+  util.die('invalid `to`');
+
+var Market = function() {
+
+  _.bindAll(this);
+  this.pushing = false;
+  this.ended = false;
+  this.closed = false;
+
+  Readable.call(this, {objectMode: true});
+
+  log.write('');
+  log.info('\tWARNING: BACKTESTING FEATURE NEEDS PROPER TESTING');
+  log.info('\tWARNING: ACT ON THESE NUMBERS AT YOUR OWN RISK!');
+  log.write('');
+
+  this.reader = new Reader();
+  this.batchSize = config.backtest.batchSize;
+  this.iterator = {
+    from: from.clone(),
+    to: from.clone().add(this.batchSize, 'm').subtract(1, 's')
+  }
+}
+
+var Readable = require('stream').Readable;
+Market.prototype = Object.create(Readable.prototype, {
+  constructor: { value: Market }
+});
+
+Market.prototype._read = _.once(function() {
+  this.get();
+});
+
+Market.prototype.get = function() {
+  if(this.iterator.to >= to) {
+    this.iterator.to = to;
+    this.ended = true;
+  }
+
+  this.reader.get(
+    this.iterator.from.unix(),
+    this.iterator.to.unix(),
+    'full',
+    this.processCandles
+  )
+}
+
+Market.prototype.processCandles = function(err, candles) {
+  this.pushing = true;
+  var amount = _.size(candles);
+
+  if(amount === 0) {
+    if(this.ended) {
+      this.closed = true;
+      this.reader.close();
+      this.push({isFinished: true});
+    } else {
+      util.die('Query returned no candles (do you have local data for the specified range?)');
+    }
+  }
+
+  if(!this.ended && amount < this.batchSize) {
+    var d = function(ts) {
+      return moment.unix(ts).utc().format('YYYY-MM-DD HH:mm:ss');
+    }
+    var from = d(_.first(candles).start);
+    var to = d(_.last(candles).start);
+    log.warn(`Simulation based on incomplete market data (${this.batchSize - amount} missing between ${from} and ${to}).`);
+  }
+
+  _.each(candles, function(c, i) {
+    c.start = moment.unix(c.start);
+    this.push(c);
+  }, this);
+
+  this.pushing = false;
+
+  this.iterator = {
+    from: this.iterator.from.clone().add(this.batchSize, 'm'),
+    to: this.iterator.from.clone().add(this.batchSize * 2, 'm').subtract(1, 's')
+  }
+
+  if(!this.closed)
+    this.get();
+}
+
+module.exports = Market;

core/markets/importer.js

@@ -0,0 +1,117 @@
+var _ = require('lodash');
+var util = require('../util');
+var config = util.getConfig();
+var dirs = util.dirs();
+var log = require(dirs.core + 'log');
+var moment = require('moment');
+var gekkoEnv = util.gekkoEnv();
+
+var adapter = config[config.adapter];
+var daterange = config.importer.daterange;
+
+var from = moment.utc(daterange.from);
+
+if(daterange.to) {
+  var to = moment.utc(daterange.to);
+} else {
+  var to = moment().utc();
+  log.debug(
+    'No end date specified for importing, setting to',
+    to.format()
+  );
+}
+log.debug(to.format());
+
+if(!from.isValid())
+  util.die('invalid `from`');
+
+if(!to.isValid())
+  util.die('invalid `to`');
+
+var TradeBatcher = require(dirs.budfox + 'tradeBatcher');
+var CandleManager = require(dirs.budfox + 'candleManager');
+var exchangeChecker = require(dirs.gekko + 'exchange/exchangeChecker');
+
+var error = exchangeChecker.cantFetchFullHistory(config.watch);
+if(error)
+  util.die(error, true);
+
+var fetcher = require(dirs.importers + config.watch.exchange);
+
+if(to <= from)
+  util.die('This daterange does not make sense.')
+
+var Market = function() {
+  _.bindAll(this);
+  this.exchangeSettings = exchangeChecker.settings(config.watch);
+
+  this.tradeBatcher = new TradeBatcher(this.exchangeSettings.tid);
+  this.candleManager = new CandleManager;
+  this.fetcher = fetcher({
+    to: to,
+    from: from
+  });
+
+  this.done = false;
+
+  this.fetcher.bus.on(
+    'trades',
+    this.processTrades
+  );
+
+  this.fetcher.bus.on(
+    'done',
+    function() {
+      this.done = true;
+    }.bind(this)
+  )
+
+  this.tradeBatcher.on(
+    'new batch',
+    this.candleManager.processTrades
+  );
+
+  this.candleManager.on(
+    'candles',
+    this.pushCandles
+  );
+
+  Readable.call(this, {objectMode: true});
+
+  this.get();
+}
+
+var Readable = require('stream').Readable;
+Market.prototype = Object.create(Readable.prototype, {
+  constructor: { value: Market }
+});
+
+Market.prototype._read = _.noop;
+
+Market.prototype.pushCandles = function(candles) {
+  _.each(candles, this.push);
+}
+
+Market.prototype.get = function() {
+  this.fetcher.fetch();
+}
+
+Market.prototype.processTrades = function(trades) {
+  this.tradeBatcher.write(trades);
+
+  if(this.done) {
+    log.info('Done importing!');
+    this.emit('end');
+    return;
+  }
+
+  if(_.size(trades) && gekkoEnv === 'child-process') {
+    let lastAtTS = _.last(trades).date;
+    let lastAt = moment.unix(lastAtTS).utc().format();
+    process.send({event: 'marketUpdate', payload: lastAt});
+  }
+
+  setTimeout(this.get, 1000);
+}
+
+module.exports = Market;

core/markets/leech.js

@@ -0,0 +1,91 @@
+// a leech market is "semi-realtime" and pulls out candles of a
+// database (which is expected to be updated regularly, like with a
+// realtime market running in parallel).
+
+const _ = require('lodash');
+const moment = require('moment');
+
+const util = require('../util');
+const dirs = util.dirs();
+const config = util.getConfig();
+
+const exchangeChecker = require(dirs.gekko + 'exchange/exchangeChecker');
+
+const adapter = config[config.adapter];
+const Reader = require(dirs.gekko + adapter.path + '/reader');
+
+const TICKINTERVAL = 20 * 1000; // 20 seconds
+
+const slug = config.watch.exchange.toLowerCase();
+const exchange = exchangeChecker.getExchangeCapabilities(slug);
+
+if(!exchange)
+  util.die(`Unsupported exchange: ${slug}`)
+
+const error = exchangeChecker.cantMonitor(config.watch);
+if(error)
+  util.die(error, true);
+
+if(config.market.from)
+  var fromTs = moment.utc(config.market.from).unix();
+else
+  var fromTs = moment().startOf('minute').unix();
+
+
+var Market = function() {
+
+  _.bindAll(this);
+
+  Readable.call(this, {objectMode: true});
+
+  this.reader = new Reader();
+  this.latestTs = fromTs;
+
+  setInterval(
+    this.get,
+    TICKINTERVAL
+  );
+}
+
+var Readable = require('stream').Readable;
+Market.prototype = Object.create(Readable.prototype, {
+  constructor: { value: Market }
+});
+
+Market.prototype._read = _.once(function() {
+  this.get();
+});
+
+Market.prototype.get = function() {
+  var future = moment().add(1, 'minute').unix();
+
+  this.reader.get(
+    this.latestTs,
+    future,
+    'full',
+    this.processCandles
+  )
+}
+
+Market.prototype.processCandles = function(err, candles) {
+  var amount = _.size(candles);
+  if(amount === 0) {
+    // no new candles!
+    return;
+  }
+
+  // TODO:
+  // verify that the correct amount of candles was passed:
+  //
+  // if `this.latestTs` was at 10:00 and we receive 3 candles with the latest at 11:00
+  // we know we are missing 57 candles...
+
+  _.each(candles, function(c, i) {
+    c.start = moment.unix(c.start).utc();
+    this.push(c);
+  }, this);
+
+  this.latestTs = _.last(candles).start.unix() + 1;
+}
+
+module.exports = Market;

core/markets/realtime.js

@@ -0,0 +1,19 @@
+const _ = require('lodash');
+
+const util = require('../util');
+const dirs = util.dirs();
+
+const exchangeChecker = require(dirs.gekko + 'exchange/exchangeChecker');
+const config = util.getConfig();
+
+const slug = config.watch.exchange.toLowerCase();
+const exchange = exchangeChecker.getExchangeCapabilities(slug);
+
+if(!exchange)
+  util.die(`Unsupported exchange: ${slug}`)
+
+const error = exchangeChecker.cantMonitor(config.watch);
+if(error)
+  util.die(error, true);
+
+module.exports = require(dirs.budfox + 'budfox');

core/pipeline.js

@@ -0,0 +1,252 @@
+/*
+
+  A pipeline implements a full Gekko Flow based on a config and 
+  a mode. The mode is an abstraction that tells Gekko what market
+  to load (realtime, backtesting or importing) while making sure
+  all enabled plugins are actually supported by that market.
+
+  Read more here:
+  https://gekko.wizb.it/docs/internals/architecture.html
+
+*/
+
+
+var util = require('./util');
+var dirs = util.dirs();
+
+var _ = require('lodash');
+var async = require('async');
+
+var log = require(dirs.core + 'log');
+
+var pipeline = (settings) => {
+
+  var mode = settings.mode;
+  var config = settings.config;
+
+  // prepare a GekkoStream
+  var GekkoStream = require(dirs.core + 'gekkoStream');
+
+  // all plugins
+  var plugins = [];
+  // all emitting plugins
+  var emitters = {};
+  // all plugins interested in candles
+  var candleConsumers = [];
+
+  // utility to check and load plugins.
+  var pluginHelper = require(dirs.core + 'pluginUtil');
+
+  // meta information about every plugin that tells Gekko
+  // something about every available plugin
+  var pluginParameters = require(dirs.gekko + 'plugins');
+  // meta information about the events plugins can broadcast
+  // and how they should hooked up to consumers.
+  var subscriptions = require(dirs.gekko + 'subscriptions');
+
+  var market;
+
+  // Instantiate each enabled plugin
+  var loadPlugins = function(next) {
+    // load all plugins
+    async.mapSeries(
+      pluginParameters,
+      pluginHelper.load,
+      function(error, _plugins) {
+        if(error)
+          return util.die(error, true);
+
+        plugins = _.compact(_plugins);
+        next();
+      }
+    );
+  };
+
+  // Some plugins emit their own events, store
+  // a reference to those plugins.
+  var referenceEmitters = function(next) {
+
+    _.each(plugins, function(plugin) {
+      if(plugin.meta.emits)
+        emitters[plugin.meta.slug] = plugin;
+    });
+
+    next();
+  }
+
+  // Subscribe all plugins to other emitting plugins
+  var subscribePlugins = function(next) {
+    // events broadcasted by plugins
+    var pluginSubscriptions = _.filter(
+      subscriptions,
+      sub => sub.emitter !== 'market'
+    );
+
+    // some events can be broadcasted by different
+    // plugins, however the pipeline only allows a single
+    // emitting plugin for each event to be enabled.
+    _.each(
+      pluginSubscriptions.filter(s => _.isArray(s.emitter)),
+      subscription => {
+        // cache full list
+        subscription.emitters = subscription.emitter;
+        var singleEventEmitters = subscription.emitter
+          .filter(
+            s => _.size(plugins.filter(p => p.meta.slug === s))
+          );
+
+        if(_.size(singleEventEmitters) > 1) {
+          var error = `Multiple plugins are broadcasting`;
+          error += ` the event "${subscription.event}" (${singleEventEmitters.join(',')}).`;
+          error += 'This is unsupported.'
+          util.die(error);
+        } else {
+          subscription.emitter = _.first(singleEventEmitters);
+        }
+      }
+    );
+
+    // subscribe interested plugins to
+    // emitting plugins
+    _.each(plugins, function(plugin) {
+      _.each(pluginSubscriptions, function(sub) {
+
+        if(plugin[sub.handler]) {
+          // if a plugin wants to listen
+          // to something disabled
+          if(!emitters[sub.emitter]) {
+            if(!plugin.meta.greedy) {
+
+              let emitterMessage;
+              if(sub.emitters) {
+                emitterMessage = 'all of the emitting plugins [ ';
+                emitterMessage += sub.emitters.join(', ');
+                emitterMessage += ' ] are disabled.';
+              } else {
+                emitterMessage += 'the emitting plugin (' + sub.emitter;
+                emitterMessage += ')is disabled.'
+              }
+
+              log.error([
+                plugin.meta.name,
+                'wanted to listen to event',
+                sub.event + ',',
+                'however',
+                emitterMessage,
+                plugin.meta.name + ' might malfunction because of it.'
+              ].join(' '));
+            }
+            return;
+          }
+
+          // attach handler
+          emitters[sub.emitter]
+            .on(sub.event,
+              plugin[
+                sub.handler
+              ])
+        }
+
+      });
+    });
+
+    // events broadcasted by the market
+    var marketSubscriptions = _.filter(
+      subscriptions,
+      {emitter: 'market'}
+    );
+
+    // subscribe plugins to the market
+    _.each(plugins, function(plugin) {
+      _.each(marketSubscriptions, function(sub) {
+
+        if(plugin[sub.handler]) {
+          if(sub.event === 'candle')
+            candleConsumers.push(plugin);
+        }
+
+      });
+    });
+
+    next();
+  }
+
+  var prepareMarket = function(next) {
+    if(mode === 'backtest' && config.backtest.daterange === 'scan')
+      require(dirs.core + 'prepareDateRange')(next);
+    else
+      next();
+  }
+
+  var setupMarket = function(next) {
+    // load a market based on the config (or fallback to mode)
+    let marketType;
+    if(config.market)
+      marketType = config.market.type;
+    else
+      marketType = mode;
+
+    var Market = require(dirs.markets + marketType);
+
+    market = new Market(config);
+
+    next();
+  }
+
+  var subscribePluginsToMarket = function(next) {
+
+    // events broadcasted by the market
+    var marketSubscriptions = _.filter(
+      subscriptions,
+      {emitter: 'market'}
+    );
+
+    _.each(plugins, function(plugin) {
+      _.each(marketSubscriptions, function(sub) {
+
+        if(sub.event === 'candle')
+          // these are handled via the market stream
+          return;
+
+        if(plugin[sub.handler]) {
+          market.on(sub.event, plugin[sub.handler]);
+        }
+
+      });
+    });
+
+    next();
+
+  }
+
+  log.info('Setting up Gekko in', mode, 'mode');
+  log.info('');
+
+  async.series(
+    [
+      loadPlugins,
+      referenceEmitters,
+      subscribePlugins,
+      prepareMarket,
+      setupMarket,
+      subscribePluginsToMarket
+    ],
+    function() {
+
+      var gekkoStream = new GekkoStream(plugins);
+
+      market
+        .pipe(gekkoStream)
+
+        // convert JS objects to JSON string
+        // .pipe(new require('stringify-stream')())
+        // output to standard out
+        // .pipe(process.stdout);
+
+      market.on('end', gekkoStream.finalize);
+    }
+  );
+
+}
+
+module.exports = pipeline;

core/pluginUtil.js

@@ -0,0 +1,120 @@
+var _ = require('lodash');
+var async = require('async');
+var Emitter = require('./emitter');
+
+var util = require(__dirname + '/util');
+
+var log = require(util.dirs().core + 'log');
+
+var config = util.getConfig();
+var pluginDir = util.dirs().plugins;
+var gekkoMode = util.gekkoMode();
+var inherits = require('util').inherits;
+
+var pluginHelper = {
+  // Checks whether we can load a module
+
+  // @param Object plugin
+  //    plugin config object
+  // @return String
+  //    error message if we can't
+  //    use the module.
+  cannotLoad: function(plugin) {
+
+    // verify plugin dependencies are installed
+    if(_.has(plugin, 'dependencies'))
+      var error = false;
+
+      _.each(plugin.dependencies, function(dep) {
+        try {
+          var a = require(dep.module);
+        }
+        catch(e) {
+          log.error('ERROR LOADING DEPENDENCY', dep.module);
+
+          if(!e.message) {
+            log.error(e);
+            util.die();
+          }
+
+          if(!e.message.startsWith('Cannot find module'))
+            return util.die(e);
+
+          error = [
+            'The plugin',
+            plugin.slug,
+            'expects the module',
+            dep.module,
+            'to be installed.',
+            'However it is not, install',
+            'it by running: \n\n',
+            '\tnpm install',
+            dep.module + '@' + dep.version
+          ].join(' ');
+        }
+      });
+
+    return error;
+  },
+  // loads a plugin
+  // 
+  // @param Object plugin
+  //    plugin config object
+  // @param Function next
+  //    callback
+  load: function(plugin, next) {
+
+    plugin.config = config[plugin.slug];
+
+    if(!plugin.config || !plugin.config.enabled)
+      return next();
+
+    if(!_.contains(plugin.modes, gekkoMode)) {
+      log.warn(
+        'The plugin',
+        plugin.name,
+        'does not support the mode',
+        gekkoMode + '.',
+        'It has been disabled.'
+      )
+      return next();
+    }
+
+    log.info('Setting up:');
+    log.info('\t', plugin.name);
+    log.info('\t', plugin.description);
+
+    var cannotLoad = pluginHelper.cannotLoad(plugin);
+    if(cannotLoad)
+      return next(cannotLoad);
+
+    if(plugin.path)
+      var Constructor = require(pluginDir + plugin.path(config));
+    else
+      var Constructor = require(pluginDir + plugin.slug);
+
+    if(plugin.async) {
+      inherits(Constructor, Emitter);
+      var instance = new Constructor(util.defer(function(err) {
+        next(err, instance);
+      }), plugin);
+      Emitter.call(instance);
+
+      instance.meta = plugin;
+    } else {
+      inherits(Constructor, Emitter);
+      var instance = new Constructor(plugin);
+      Emitter.call(instance);
+
+      instance.meta = plugin;
+      _.defer(function() {
+        next(null, instance); 
+      });
+    }
+
+    if(!plugin.silent)
+      log.info('\n');
+  }
+}
+
+module.exports = pluginHelper;

core/prepareDateRange.js

@@ -0,0 +1,67 @@
+var _ = require('lodash');
+var prompt = require('prompt-lite');
+var moment = require('moment');
+
+var util = require('./util');
+var config = util.getConfig();
+var dirs = util.dirs();
+var log = require(dirs.core + 'log');
+
+var scan = require(dirs.tools + 'dateRangeScanner');
+
+// helper to store the evenutally detected
+// daterange.
+var setDateRange = function(from, to) {
+  config.backtest.daterange = {
+    from: moment.unix(from).utc().format(),
+    to: moment.unix(to).utc().format(),
+  };
+  util.setConfig(config);
+}
+
+
+module.exports = function(done) {
+  scan((err, ranges) => {
+
+    if(_.size(ranges) === 0)
+      util.die('No history found for this market', true);
+
+    if(_.size(ranges) === 1) {
+      var r = _.first(ranges);
+      log.info('Gekko was able to find a single daterange in the locally stored history:');
+      log.info('\t', 'from:', moment.unix(r.from).utc().format('YYYY-MM-DD HH:mm:ss'));
+      log.info('\t', 'to:', moment.unix(r.to).utc().format('YYYY-MM-DD HH:mm:ss'));
+
+      
+      setDateRange(r.from, r.to);
+      return done();
+    }
+
+    log.info(
+      'Gekko detected multiple dateranges in the locally stored history.',
+      'Please pick the daterange you are interested in testing:'
+    );
+
+    _.each(ranges, (range, i) => {
+      log.info('\t\t', `OPTION ${i + 1}:`);
+      log.info('\t', 'from:', moment.unix(range.from).utc().format('YYYY-MM-DD HH:mm:ss'));
+      log.info('\t', 'to:', moment.unix(range.to).utc().format('YYYY-MM-DD HH:mm:ss'));
+    });
+
+    prompt.get({name: 'option'}, (err, result) => {
+
+      var option = parseInt(result.option);
+      if(option === NaN)
+        util.die('Not an option..', true);
+
+      var range = ranges[option - 1];
+
+      if(!range)
+        util.die('Not an option..', true);
+
+      setDateRange(range.from, range.to);
+      return done();
+    });
+
+  });
+}

core/stats.js

@@ -0,0 +1,17 @@
+const stats = require('stats-lite');
+const lodash = require('lodash');
+
+
+// simply monkey patch the stats with other stuff we
+// need and pass on.
+
+// sharpe ratio
+//
+// @param returns (array - list of returns)
+// @param rfreturn (number - risk free return)
+// 
+stats.sharpe = (returns, rfreturn) => {
+  return (stats.mean(returns) - rfreturn) / stats.stdev(returns);
+}
+
+module.exports = stats;

core/tools/candleLoader.js

@@ -0,0 +1,100 @@
+// TODO: properly handle a daterange for which no data is available.
+
+const batchSize = 1000;
+
+const _ = require('lodash');
+const fs = require('fs');
+const moment = require('moment');
+
+const util = require('../../core/util');
+const config = util.getConfig();
+const dirs = util.dirs();
+const log = require(dirs.core + '/log');
+
+const adapter = config[config.adapter];
+const Reader = require(dirs.gekko + adapter.path + '/reader');
+const daterange = config.daterange;
+
+const CandleBatcher = require(dirs.core + 'candleBatcher');
+
+const to = moment.utc(daterange.to).startOf('minute');
+const from = moment.utc(daterange.from).startOf('minute');
+const toUnix = to.unix();
+
+if(to <= from)
+  util.die('This daterange does not make sense.')
+
+if(!from.isValid())
+  util.die('invalid `from`');
+
+if(!to.isValid())
+  util.die('invalid `to`');
+
+let iterator = {
+  from: from.clone(),
+  to: from.clone().add(batchSize, 'm').subtract(1, 's')
+}
+
+var DONE = false;
+
+var result = [];
+var reader = new Reader();
+var batcher;
+var next;
+var doneFn = () => {
+  process.nextTick(() => {
+    next(result);
+  })
+};
+
+module.exports = function(candleSize, _next) {
+  next = _.once(_next);
+
+  batcher = new CandleBatcher(candleSize)
+    .on('candle', handleBatchedCandles);
+
+  getBatch();
+}
+
+const getBatch = () => {
+  reader.get(
+    iterator.from.unix(),
+    iterator.to.unix(),
+    'full',
+    handleCandles
+  )
+}
+
+const shiftIterator = () => {
+  iterator = {
+    from: iterator.from.clone().add(batchSize, 'm'),
+    to: iterator.from.clone().add(batchSize * 2, 'm').subtract(1, 's')
+  }
+}
+
+const handleCandles = (err, data) => {
+  if(err) {
+    console.error(err);
+    util.die('Encountered an error..')
+  }
+
+  if(_.size(data) && _.last(data).start >= toUnix || iterator.from.unix() >= toUnix)
+    DONE = true;
+
+  batcher.write(data);
+  batcher.flush();
+
+  if(DONE) {
+    reader.close();
+
+    setTimeout(doneFn, 100);
+
+  } else {
+    shiftIterator();
+    getBatch();
+  }
+}
+
+const handleBatchedCandles = candle => {
+  result.push(candle);
+}

core/tools/configBuilder.js

@@ -0,0 +1,42 @@
+const fs = require('fs');
+const _ = require('lodash');
+const toml = require('toml');
+
+const util = require('../util');
+const dirs = util.dirs();
+
+const getTOML = function(fileName) {
+  var raw = fs.readFileSync(fileName);
+  return toml.parse(raw);
+}
+
+// build a config object out of a directory of TOML files
+module.exports = function() {
+  const configDir = util.dirs().config;
+
+  let _config = getTOML(configDir + 'general.toml');
+  fs.readdirSync(configDir + 'plugins').forEach(function(pluginFile) {
+    let pluginName = _.first(pluginFile.split('.'))
+    _config[pluginName] = getTOML(configDir + 'plugins/' + pluginFile);
+  });
+
+  // attach the proper adapter
+  let adapter = _config.adapter;
+  _config[adapter] = getTOML(configDir + 'adapters/' + adapter + '.toml');
+
+  if(_config.tradingAdvisor.enabled) {
+    // also load the strat
+    let strat = _config.tradingAdvisor.method;
+    let stratFile = configDir + 'strategies/' + strat + '.toml';
+    if(!fs.existsSync(stratFile))
+      util.die('Cannot find the strategy config file for ' + strat);
+    _config[strat] = getTOML(stratFile);
+  }
+
+  const mode = util.gekkoMode();
+
+  if(mode === 'backtest')
+    _config.backtest = getTOML(configDir + 'backtest.toml');
+
+  return _config;
+}

core/tools/dataStitcher.js

@@ -0,0 +1,233 @@
+var _ = require('lodash');
+var fs = require('fs');
+var moment = require('moment');
+
+var util = require('../util');
+var config = util.getConfig();
+var dirs = util.dirs();
+var log = require(dirs.core + '/log');
+
+var Stitcher = function(batcher) {
+  this.batcher = batcher;
+}
+
+Stitcher.prototype.ago = function(ts) {
+  var now = moment().utc();
+  var then = moment.unix(ts).utc();
+  return now.diff(then, 'minutes') + ' minutes ago';
+}
+
+Stitcher.prototype.verifyExchange = function() {
+  require(dirs.gekko + 'exchange/dependencyCheck');
+  const exchangeChecker = require(dirs.gekko + 'exchange/exchangeChecker');
+  const slug = config.watch.exchange.toLowerCase();
+  let exchange;
+  try {
+    exchange = exchangeChecker.getExchangeCapabilities(slug);
+  } catch(e) {
+    util.die(e.message);
+  }
+
+  if(!exchange)
+    util.die(`Unsupported exchange: ${slug}`);
+
+  var error = exchangeChecker.cantMonitor(config.watch);
+  if(error)
+    util.die(error, true);
+}
+
+Stitcher.prototype.prepareHistoricalData = function(done) {
+  this.verifyExchange();
+
+  // - step 1: check most recent stored candle window
+  // - step 2: check oldest trade reachable by API
+  // - step 3: see if overlap
+  // - step 4: feed candle stream into CandleBatcher
+
+  if(config.tradingAdvisor.historySize === 0)
+    return done();
+
+  var requiredHistory = config.tradingAdvisor.candleSize * config.tradingAdvisor.historySize;
+  var Reader = require(dirs.plugins + config.adapter + '/reader');
+  
+  this.reader = new Reader;
+
+  log.info(
+    '\tThe trading method requests',
+    requiredHistory,
+    'minutes of historic data. Checking availablity..'
+  );
+
+  var endTime = moment().utc().startOf('minute');
+  var idealStartTime = endTime.clone().subtract(requiredHistory, 'm');
+  
+  this.reader.mostRecentWindow(idealStartTime, endTime, function(localData) {
+    // now we know what data is locally available, what
+    // data would we need from the exchange?
+    
+    if(!localData) {
+      log.info('\tNo usable local data available, trying to get as much as possible from the exchange..');
+      var idealExchangeStartTime = idealStartTime.clone();
+      var idealExchangeStartTimeTS = idealExchangeStartTime.unix();
+    }
+    else if (idealStartTime.unix() < localData.from) {
+      log.info('\tLocal data is still too recent, trying to get as much as possible from the exchange');
+      var idealExchangeStartTime = idealStartTime.clone();
+      var idealExchangeStartTimeTS = idealExchangeStartTime.unix();
+    }
+    else {
+      log.debug('\tAvailable local data:');
+      log.debug('\t\tfrom:', this.ago(localData.from));
+      log.debug('\t\tto:', this.ago(localData.to));
+
+      log.info('\tUsable local data available, trying to match with exchange data..')
+      // local data is available, we need the next minute
+
+
+      // make sure we grab back in history far enough
+      var secondsOverlap = 60 * 15; // 15 minutes
+      var idealExchangeStartTimeTS = localData.to - secondsOverlap;
+      var idealExchangeStartTime = moment.unix(idealExchangeStartTimeTS).utc();
+
+      // already set the
+      util.setConfigProperty(
+        'tradingAdvisor',
+        'firstFetchSince',
+        idealExchangeStartTimeTS
+      );
+    }
+
+    // Limit the history Gekko can try to get from the exchange.
+    var minutesAgo = endTime.diff(idealExchangeStartTime, 'minutes');
+    var maxMinutesAgo = 4 * 60; // 4 hours
+    if(minutesAgo > maxMinutesAgo) {
+      log.info('\tPreventing Gekko from requesting', minutesAgo, 'minutes of history.');
+      idealExchangeStartTime = endTime.clone().subtract(maxMinutesAgo, 'minutes');
+      idealExchangeStartTimeTS = idealExchangeStartTime.unix();
+    } 
+
+    log.debug('\tFetching exchange data since', this.ago(idealExchangeStartTimeTS))
+    this.checkExchangeTrades(idealExchangeStartTime, function(err, exchangeData) {
+      log.debug('\tAvailable exchange data:');
+      log.debug('\t\tfrom:', this.ago(exchangeData.from));
+      log.debug('\t\tto:', this.ago(exchangeData.to));
+
+      // in case we have limited local data, and the
+      // exchange would offer more than we have: ignore
+      // the local data..
+      if(
+        localData &&
+        exchangeData &&
+        exchangeData.from < localData.from
+      ) {
+        log.debug('\tExchange offered more data than locally available. Ignoring local data');
+        localData = false;
+      }
+
+      var stitchable = localData && exchangeData.from <= localData.to;
+      if(stitchable) {
+        log.debug('\tStitching datasets');
+
+        // we can combine local data with exchange data
+        if(idealStartTime.unix() >= localData.from) {
+          log.info(
+            '\tFull history locally available.',
+            'Seeding the trading method with all required historical candles.'
+          );
+
+        } else {
+
+          // stitchable but not enough
+
+          log.info(
+            '\tPartial history locally available, but',
+            Math.round((localData.from - idealStartTime.unix()) / 60),
+            'minutes are missing.')
+          log.info('\tSeeding the trading method with',
+            'partial historical data (Gekko needs more time before',
+            'it can give advice).'
+          );
+        }
+
+        // seed all historic data up to the point the exchange can provide.
+        var from = localData.from;
+        var to = moment.unix(exchangeData.from).utc()
+          .startOf('minute')
+          .subtract(1, 'minute')
+          .unix();
+
+        log.debug('\tSeeding with:');
+        log.debug('\t\tfrom:', this.ago(from));
+        log.debug('\t\tto:', this.ago(to));
+        return this.seedLocalData(from, to, done);
+
+      } else if(!stitchable) {
+        log.debug('\tUnable to stitch datasets.')
+        // we cannot use any local data..
+        log.info(
+          '\tNot seeding locally available data to the trading method.'
+        );
+
+        if(exchangeData.from < idealExchangeStartTimeTS) {
+          log.info('\tHowever the exchange returned enough data anyway!');
+        } else if(localData) {
+          log.info(
+            '\tThe exchange does not return enough data.',
+            Math.round((localData.from - idealStartTime.unix()) / 60),
+            'minutes are still missing.'
+          );
+        }
+      }
+
+      done();
+
+    }.bind(this));
+  }.bind(this));
+}
+
+Stitcher.prototype.checkExchangeTrades = function(since, next) {
+  var provider = config.watch.exchange.toLowerCase();
+  var DataProvider = require(util.dirs().gekko + 'exchange/wrappers/' + provider);
+
+  var exchangeConfig = config.watch;
+
+  // include trader config if trading is enabled
+  if (_.isObject(config.trader) && config.trader.enabled) {
+    exchangeConfig = _.extend(config.watch, config.trader);
+  }
+
+  var watcher = new DataProvider(exchangeConfig);
+  watcher.getTrades(since, function(e, d) {
+    if(e) {
+      util.die(e.message);
+    }
+
+    if(_.isEmpty(d))
+      return util.die(
+        `Gekko tried to retrieve data since ${since.format('YYYY-MM-DD HH:mm:ss')}, however
+        ${provider} did not return any trades.`
+      );
+
+    next(e, {
+      from: _.first(d).date,
+      to: _.last(d).date
+    })
+  });
+}
+
+Stitcher.prototype.seedLocalData = function(from, to, next) {
+  this.reader.get(from, to, 'full', function(err, rows) {
+    rows = _.map(rows, row => {
+      row.start = moment.unix(row.start);
+      return row;
+    });
+
+    this.batcher.write(rows);
+    this.batcher.flush();
+    this.reader.close();
+    next();
+
+  }.bind(this));
+}
+
+module.exports = Stitcher;

core/tools/dateRangeScanner.js

@@ -0,0 +1,138 @@
+var BATCH_SIZE = 60; // minutes
+var MISSING_CANDLES_ALLOWED = 3; // minutes, per batch
+
+var _ = require('lodash');
+var moment = require('moment');
+var async = require('async');
+
+var util = require('../util');
+var config = util.getConfig();
+var dirs = util.dirs();
+var log = require(dirs.core + 'log');
+
+var adapter = config[config.adapter];
+var Reader = require(dirs.gekko + adapter.path + '/reader');
+
+var reader = new Reader();
+
+// todo: rewrite with generators or async/await..
+var scan = function(done) {
+  log.info('Scanning local history for backtestable dateranges.');
+
+  reader.tableExists('candles', (err, exists) => {
+
+    if(err)
+      return done(err, null, reader);
+
+    if(!exists)
+      return done(null, [], reader);
+
+    async.parallel({
+      boundry: reader.getBoundry,
+      available: reader.countTotal
+    }, (err, res) => {
+
+      var first = res.boundry.first;
+      var last = res.boundry.last;
+
+      var optimal = (last - first) / 60;
+
+      log.debug('Available', res.available);
+      log.debug('Optimal', optimal);
+
+      // There is a candle for every minute
+      if(res.available === optimal + 1) {
+        log.info('Gekko is able to fully use the local history.');
+        return done(false, [{
+          from: first,
+          to: last
+        }], reader);
+      }
+
+      // figure out where the gaps are..
+
+      var missing = optimal - res.available + 1;
+
+      log.info(`The database has ${missing} candles missing, Figuring out which ones...`);
+      
+      var iterator = {
+        from: last - (BATCH_SIZE * 60),
+        to: last
+      }
+
+      var batches = [];
+
+      // loop through all candles we have
+      // in batches and track whether they
+      // are complete
+      async.whilst(
+          () => {
+            return iterator.from > first
+          },
+          next => {
+            var from = iterator.from;
+            var to = iterator.to;
+            reader.count(
+              from,
+              iterator.to,
+              (err, count) => {
+                var complete = count + MISSING_CANDLES_ALLOWED > BATCH_SIZE;
+
+                if(complete)
+                  batches.push({
+                    to: to,
+                    from: from
+                  });
+
+                next();
+              }
+            );
+
+            iterator.from -= BATCH_SIZE * 60;
+            iterator.to -= BATCH_SIZE * 60;
+          },
+          () => {
+            if(batches.length === 0) {
+              return done(null, [], reader);
+            }
+
+            // batches is now a list like
+            // [ {from: unix, to: unix } ]
+
+            var ranges = [ batches.shift() ];
+
+            _.each(batches, batch => {
+              var curRange = _.last(ranges);
+              if(batch.to === curRange.from)
+                curRange.from = batch.from;
+              else
+                ranges.push( batch );
+            })
+
+            // we have been counting chronologically reversed
+            // (backwards, from now into the past), flip definitions
+            ranges = ranges.reverse();
+
+            _.map(ranges, r => {
+              return {
+                from: r.to,
+                to: r.from
+              }
+            });
+
+
+            // ranges is now a list like
+            // [ {from: unix, to: unix } ]
+            //
+            // it contains all valid dataranges available for the
+            // end user.
+
+            return done(false, ranges, reader);
+          }
+        )
+    });
+
+  });
+}
+
+module.exports = scan;

core/util.js

@@ -0,0 +1,189 @@
+var moment = require('moment');
+var _ = require('lodash');
+var path = require('path');
+var fs = require('fs');
+var semver = require('semver');
+var program = require('commander');
+
+var startTime = moment();
+
+var _config = false;
+var _package = false;
+var _nodeVersion = false;
+var _gekkoMode = false;
+var _gekkoEnv = false;
+
+var _args = false;
+
+// helper functions
+var util = {
+  getConfig: function() {
+    // cache
+    if(_config)
+      return _config;
+
+    if(!program.config)
+        util.die('Please specify a config file.', true);
+
+    if(!fs.existsSync(util.dirs().gekko + program.config))
+      util.die('Cannot find the specified config file.', true);
+
+    _config = require(util.dirs().gekko + program.config);
+    return _config;
+  },
+  // overwrite the whole config
+  setConfig: function(config) {
+    _config = config;
+  },
+  setConfigProperty: function(parent, key, value) {
+    if(parent)
+      _config[parent][key] = value;
+    else
+      _config[key] = value;
+  },
+  getVersion: function() {
+    return util.getPackage().version;
+  },
+  getPackage: function() {
+    if(_package)
+      return _package;
+
+
+    _package = JSON.parse( fs.readFileSync(__dirname + '/../package.json', 'utf8') );
+    return _package;
+  },
+  getRequiredNodeVersion: function() {
+    return util.getPackage().engines.node;
+  },
+  recentNode: function() {
+    var required = util.getRequiredNodeVersion();
+    return semver.satisfies(process.version, required);
+  },
+  // check if two moments are corresponding
+  // to the same time
+  equals: function(a, b) {
+    return !(a < b || a > b)
+  },
+  minToMs: function(min) {
+    return min * 60 * 1000;
+  },
+  defer: function(fn) {
+    return function(args) {
+      var args = _.toArray(arguments);
+      return _.defer(function() { fn.apply(this, args) });
+    }
+  },
+  logVersion: function() {
+    return  `Gekko version: v${util.getVersion()}`
+    + `\nNodejs version: ${process.version}`;
+  },
+  die: function(m, soft) {
+
+    if(_gekkoEnv === 'child-process') {
+      return process.send({type: 'error', error: '\n ERROR: ' + m + '\n'});
+    }
+
+    var log = console.log.bind(console);
+
+    if(m) {
+      if(soft) {
+        log('\n ERROR: ' + m + '\n\n');
+      } else {
+        log(`\nGekko encountered an error and can\'t continue`);
+        log('\nError:\n');
+        log(m, '\n\n');
+        log('\nMeta debug info:\n');
+        log(util.logVersion());
+        log('');
+      }
+    }
+    process.exit(1);
+  },
+  dirs: function() {
+    var ROOT = __dirname + '/../';
+
+    return {
+      gekko: ROOT,
+      core: ROOT + 'core/',
+      markets: ROOT + 'core/markets/',
+      exchanges: ROOT + 'exchange/wrappers/',
+      plugins: ROOT + 'plugins/',
+      methods: ROOT + 'strategies/',
+      indicators: ROOT + 'strategies/indicators/',
+      budfox: ROOT + 'core/budfox/',
+      importers: ROOT + 'importers/exchanges/',
+      tools: ROOT + 'core/tools/',
+      workers: ROOT + 'core/workers/',
+      web: ROOT + 'web/',
+      config: ROOT + 'config/',
+      broker: ROOT + 'exchange/'
+    }
+  },
+  inherit: function(dest, source) {
+    require('util').inherits(
+      dest,
+      source
+    );
+  },
+  makeEventEmitter: function(dest) {
+    util.inherit(dest, require('events').EventEmitter);
+  },
+  setGekkoMode: function(mode) {
+    _gekkoMode = mode;
+  },
+  gekkoMode: function() {
+    if(_gekkoMode)
+      return _gekkoMode;
+
+    if(program['import'])
+      return 'importer';
+    else if(program.backtest)
+      return 'backtest';
+    else
+      return 'realtime';
+  },
+  gekkoModes: function() {
+    return [
+      'importer',
+      'backtest',
+      'realtime'
+    ]
+  },
+  setGekkoEnv: function(env) {
+    _gekkoEnv = env;
+  },
+  gekkoEnv: function() {
+    return _gekkoEnv || 'standalone';
+  },
+  launchUI: function() {
+    if(program['ui'])
+      return true;
+    else
+      return false;
+  },
+  getStartTime: function() {
+    return startTime;
+  },
+}
+
+// NOTE: those options are only used
+// in stand alone mode
+program
+  .version(util.logVersion())
+  .option('-c, --config <file>', 'Config file')
+  .option('-b, --backtest', 'backtesting mode')
+  .option('-i, --import', 'importer mode')
+  .option('--ui', 'launch a web UI')
+  .parse(process.argv);
+
+// make sure the current node version is recent enough
+if(!util.recentNode())
+  util.die([
+    'Your local version of Node.js is too old. ',
+    'You have ',
+    process.version,
+    ' and you need atleast ',
+    util.getRequiredNodeVersion()
+  ].join(''), true);
+
+module.exports = util;

core/workers/datasetScan/parent.js

@@ -0,0 +1,54 @@
+var _ = require('lodash');
+var moment = require('moment');
+var async = require('async');
+var os = require('os');
+
+var util = require('../../util');
+var dirs = util.dirs();
+
+var dateRangeScan = require('../dateRangeScan/parent');
+
+module.exports = function(config, done) {
+
+  util.setConfig(config);
+
+  var adapter = config[config.adapter];
+  var scan = require(dirs.gekko + adapter.path + '/scanner');
+
+  scan((err, markets) => {
+
+    if(err)
+      return done(err);
+
+      let numCPUCores = os.cpus().length;
+      if(numCPUCores === undefined)
+         numCPUCores = 1;
+      async.eachLimit(markets, numCPUCores, (market, next) => {
+
+      let marketConfig = _.clone(config);
+      marketConfig.watch = market;
+
+      dateRangeScan(marketConfig, (err, ranges) => {
+        if(err)
+          return next();
+
+        market.ranges = ranges;
+
+        next();
+      });
+
+    }, err => {
+      let resp = {
+        datasets: [],
+        errors: []
+      }
+      markets.forEach(market => {
+        if(market.ranges)
+          resp.datasets.push(market);
+        else
+          resp.errors.push(market);
+      })
+      done(err, resp);
+    })
+  });
+}

core/workers/dateRangeScan/child.js

@@ -0,0 +1,26 @@
+var util = require(__dirname + '/../../util');
+
+var dirs = util.dirs();
+var ipc = require('relieve').IPCEE(process);
+
+ipc.on('start', config => {
+
+  // force correct gekko env
+  util.setGekkoEnv('child-process');
+
+  // force disable debug
+  config.debug = false;
+
+  // persist config
+  util.setConfig(config);
+
+  var scan = require(dirs.tools + 'dateRangeScanner');
+  scan(
+    (err, ranges, reader) => {
+      reader.close();
+      ipc.send('ranges', ranges);
+      process.exit(0);
+    }
+  );
+});
+

core/workers/dateRangeScan/parent.js

@@ -0,0 +1,21 @@
+var ForkTask = require('relieve').tasks.ForkTask;
+var fork = require('child_process').fork;
+
+module.exports = function(config, done) {
+  var debug = typeof v8debug === 'object';
+  if (debug) {
+    process.execArgv = [];
+  }
+
+  task = new ForkTask(fork(__dirname + '/child'));
+
+  task.send('start', config);
+
+  task.once('ranges', ranges => {
+    return done(false, ranges);
+  });
+  task.on('exit', code => {
+    if(code !== 0)
+      done('ERROR, unable to scan dateranges, please check the console.');
+  });
+}

core/workers/loadCandles/child.js

@@ -0,0 +1,28 @@
+var start = (config, candleSize, daterange) => {
+  var util = require(__dirname + '/../../util');
+
+  // force correct gekko env
+  util.setGekkoEnv('child-process');
+
+  // force disable debug
+  config.debug = false;
+  util.setConfig(config);
+
+  var dirs = util.dirs();
+
+  var load = require(dirs.tools + 'candleLoader');
+  load(config.candleSize, candles => {
+    process.send(candles);
+  })
+}
+
+process.send('ready');
+
+process.on('message', (m) => {
+  if(m.what === 'start')
+    start(m.config, m.candleSize, m.daterange);
+});
+
+process.on('disconnect', function() {
+  process.exit(0);
+})

core/workers/loadCandles/parent.js

@@ -0,0 +1,68 @@
+// example usage:
+
+// let config = {
+//   watch: {
+//     exchange: 'poloniex',
+//     currency: 'USDT',
+//     asset: 'BTC'
+//   },
+//   daterange: {
+//     from: '2016-05-22 11:22',
+//     to: '2016-06-03 19:56'
+//   },
+//   adapter: 'sqlite',
+//   sqlite: {
+//     path: 'plugins/sqlite',
+
+//     dataDirectory: 'history',
+//     version: 0.1,
+
+//     dependencies: [{
+//       module: 'sqlite3',
+//       version: '3.1.4'
+//     }]
+//   },
+//   candleSize: 100
+// }
+
+// module.exports(config, function(err, data) {
+//   console.log('FINAL CALLBACK');
+//   console.log('err', err);
+//   console.log('data', data.length);
+// })
+
+
+const fork = require('child_process').fork;
+const _ = require('lodash');
+
+module.exports = (config, callback) => {
+  var debug = typeof v8debug === 'object';
+  if (debug) {
+    process.execArgv = [];
+  }
+
+  const child = fork(__dirname + '/child');
+
+  const message = {
+    what: 'start',
+    config
+  }
+
+  const done = _.once(callback);
+
+  child.on('message', function(m) {
+    if(m === 'ready')
+      return child.send(message);
+
+    // else we are done and have candles!
+    done(null, m);
+    if (this.connected) {
+      this.disconnect();
+    }
+  });
+
+  child.on('exit', code => {
+    if(code !== 0)
+      done('ERROR, unable to load candles, please check the console.');
+  });
+}

core/workers/pipeline/child.js

@@ -0,0 +1,66 @@
+/*
+
+  Gekko is a Bitcoin trading bot for popular Bitcoin exchanges written 
+  in node, it features multiple trading methods using technical analysis.
+
+  If you are interested in how Gekko works, read more about Gekko's 
+  architecture here:
+
+  https://github.com/askmike/gekko/blob/stable/docs/internals/architecture.md
+
+  Disclaimer:
+
+  USE AT YOUR OWN RISK!
+
+  The author of this project is NOT responsible for any damage or loss caused 
+  by this software. There can be bugs and the bot may not perform as expected 
+  or specified. Please consider testing it first with paper trading and/or 
+  backtesting on historical data. Also look at the code to see what how 
+  it is working.
+
+*/
+
+var start = (mode, config) => {
+  var util = require(__dirname + '/../../util');
+
+  // force correct gekko env
+  util.setGekkoEnv('child-process');
+
+  var dirs = util.dirs();
+
+  // force correct gekko mode & config
+  util.setGekkoMode(mode);
+  util.setConfig(config);
+
+  var pipeline = require(dirs.core + 'pipeline');
+  pipeline({
+    config: config,
+    mode: mode
+  });
+}
+
+process.send('ready');
+
+process.on('message', function(m) {
+  if(m.what === 'start')
+    start(m.mode, m.config);
+
+  if(m.what === 'exit')
+    process.exit(0);
+});
+
+process.on('disconnect', function() {
+  console.log('disconnect');
+  process.exit(-1);
+})
+
+process
+  .on('unhandledRejection', (message, p) => {
+    console.error('unhandledRejection', message);
+    process.send({type: 'error', message: message});
+  })
+  .on('uncaughtException', err => {
+    console.error('uncaughtException', err);
+    process.send({type: 'error', error: err});
+    process.exit(1);
+  });

core/workers/pipeline/messageHandlers/backtestHandler.js

@@ -0,0 +1,25 @@
+// Relay the backtest message it when it comes in.
+
+module.exports = done => {
+  let backtest;
+
+  return {
+    message: message => {
+      if(message.type === 'error') {
+        done(message.error);
+      }
+
+      if(message.backtest) {
+        done(null, message.backtest);
+      }
+    },
+    exit: status => {
+      if(status !== 0) {
+        if(backtest)
+          console.error('Child process died after finishing backtest');
+        else
+          done('Child process has died.');
+      }
+    }
+  }
+}

core/workers/pipeline/messageHandlers/importerHandler.js

@@ -0,0 +1,26 @@
+module.exports = cb => {
+
+  return {
+    message: message => {
+
+      if(message.event === 'marketUpdate')
+        cb(null, {
+          done: false,
+          latest: message.payload
+        })
+
+      else if(message.type === 'error') {
+        cb(message.error);
+      }
+
+      else if(message.type === 'log')
+        console.log(message.log);
+    },
+    exit: status => {
+      if(status !== 0)
+        return cb('Child process has died.');
+      else
+        cb(null, { done: true });
+    }
+  }
+}

core/workers/pipeline/messageHandlers/realtimeHandler.js

@@ -0,0 +1,24 @@
+// pass back all messages as is
+// (except for errors and logs)
+
+module.exports = cb => {
+
+  return {
+    message: message => {
+
+      if(message.type === 'error') {
+        cb(message.error);
+      }
+
+      else
+        cb(null, message);
+
+    },
+    exit: status => {
+      if(status !== 0)
+        cb('Child process has died.');
+      else
+        cb(null, { done: true });
+    }
+  }
+}

core/workers/pipeline/parent.js

@@ -0,0 +1,34 @@
+var fork = require('child_process').fork;
+
+module.exports = (mode, config, callback) => {
+  var debug = typeof v8debug === 'object';
+  if (debug) {
+    process.execArgv = [];
+  }
+
+  var child = fork(__dirname + '/child');
+
+  // How we should handle client messages depends
+  // on the mode of the Pipeline that is being ran.
+  var handle = require('./messageHandlers/' + mode + 'Handler')(callback);
+
+  var message = {
+    what: 'start',
+    mode: mode,
+    config: config
+  };
+
+  child.on('message', function(m) {
+    if(m === 'ready')
+      return child.send(message);
+
+    if(m === 'done')
+      return child.send({what: 'exit'});
+
+    handle.message(m);
+  });
+
+  child.on('exit', handle.exit);
+
+  return child;
+}