first commit

importers/exchanges/binance.js

@@ -0,0 +1,60 @@
+const moment = require('moment');
+const util = require('../../core/util.js');
+const _ = require('lodash');
+const log = require('../../core/log');
+
+var config = util.getConfig();
+var dirs = util.dirs();
+
+var Fetcher = require(dirs.exchanges + 'binance');
+
+util.makeEventEmitter(Fetcher);
+
+var end = false;
+var done = false;
+var from = false;
+
+var fetcher = new Fetcher(config.watch);
+
+var fetch = () => {
+  fetcher.import = true;
+  fetcher.getTrades(from, handleFetch);
+};
+
+var handleFetch = (err, trades) => {
+  if (err) {
+    log.error(`There was an error importing from Binance ${err}`);
+    fetcher.emit('done');
+    return fetcher.emit('trades', []);
+}
+
+  if (trades.length > 0) {
+    var last = moment.unix(_.last(trades).date).utc();
+    // Conversion to milliseconds epoch time means we have to compensate for possible leap seconds
+    var next = from.clone().add(1, 'h').subtract(1, 's');
+  } else {
+    // Conversion to milliseconds epoch time means we have to compensate for possible leap seconds
+    var next = from.clone().add(1, 'h').subtract(1, 's');
+    log.debug('Import step returned no results, moving to the next 1h period');
+  }
+
+  if (from.add(1, 'h') >= end) {
+    fetcher.emit('done');
+
+    var endUnix = end.unix();
+    trades = _.filter(trades, t => t.date <= endUnix);
+  }
+
+  from = next.clone();
+  fetcher.emit('trades', trades);
+};
+
+module.exports = function(daterange) {
+  from = daterange.from.clone().utc();
+  end = daterange.to.clone().utc();
+
+  return {
+    bus: fetcher,
+    fetch: fetch,
+  };
+};

importers/exchanges/bitfinex.js

@@ -0,0 +1,148 @@
+const Bitfinex = require('bitfinex-api-node');
+const util = require('../../core/util.js');
+const _ = require('lodash');
+const moment = require('moment');
+const log = require('../../core/log');
+
+const config = util.getConfig();
+
+const dirs = util.dirs();
+
+const Fetcher = require(dirs.exchanges + 'bitfinex');
+const retry = require(dirs.exchanges + '../exchangeUtils').retry;
+
+Fetcher.prototype.getTrades = function(upto, callback, descending) {
+  const handle = (err, data) => {
+    if (err) return callback(err);
+
+    var trades = [];
+    if (_.isArray(data)) {
+      trades = _.map(data, function(trade) {
+        return {
+          tid: trade.ID,
+          date: moment(trade.MTS).format('X'),
+          price: +trade.PRICE,
+          amount: +Math.abs(trade.AMOUNT),
+        };
+      });
+    }
+
+    callback(null, descending ? trades : trades.reverse());
+  };
+
+  let path = 'trades/t' + this.pair + '/hist';
+  if (upto) {
+    let start = moment(upto).subtract(1, 'd').valueOf();
+    let end = moment(upto).valueOf();
+    path += `?limit=1000&start=${start}&end=${end}`;
+  }
+
+  log.debug('Querying trades with: ' + path);
+  const fetch = cb => this.bitfinex.makePublicRequest(path, this.handleResponse('getTrades', cb));
+  retry(null, fetch, handle);
+};
+
+util.makeEventEmitter(Fetcher);
+
+var end = false;
+var done = false;
+var from = false;
+
+var lastTimestamp = false;
+var lastId = false;
+
+var batch = [];
+var batch_start = false;
+var batch_end = false;
+var batch_last = false;
+
+const SCANNING_STRIDE = 24;
+const ITERATING_STRIDE = 2;
+var stride = ITERATING_STRIDE;
+
+var fetcher = new Fetcher(config.watch);
+fetcher.bitfinex = new Bitfinex(null, null, { version: 2, transform: true }).rest;
+
+var retryCritical = {
+  retries: 10,
+  factor: 1.2,
+  minTimeout: 70 * 1000,
+  maxTimeout: 120 * 1000,
+};
+
+var fetch = () => {
+  fetcher.import = true;
+
+  if (lastTimestamp) {
+    // We need to slow this down to prevent hitting the rate limits
+    setTimeout(() => {
+
+      // make sure we fetch with overlap from last batch
+      const since = lastTimestamp - 1000;
+      fetcher.getTrades(since, handleFetch);
+    }, 2500);
+  } else {
+    lastTimestamp = from.valueOf();
+    batch_start = moment(from);
+    batch_end = moment(from).add(stride, 'h');
+
+    fetcher.getTrades(batch_end, handleFetch);
+  }
+};
+
+var handleFetch = (err, trades) => {
+  if (err) {
+    log.error(`There was an error importing from Bitfinex ${err}`);
+    fetcher.emit('done');
+    return fetcher.emit('trades', []);
+  }
+
+  trades = _.filter(trades, t => !lastId || t.tid < lastId);
+
+  if (trades.length) {
+    stride = ITERATING_STRIDE;
+    batch = trades.concat(batch);
+    var last = moment.unix(_.first(trades).date);
+    lastTimestamp = last.valueOf();
+    lastId = _.first(trades).tid;
+  } else {
+    stride = SCANNING_STRIDE;
+    lastTimestamp = moment(lastTimestamp)
+      .subtract(stride, 'h')
+      .valueOf();
+  }
+
+  // if we're not done the batch we need to refetch
+  if (trades.length && moment(lastTimestamp) >= batch_start) {
+    return fetch();
+  }
+
+  var lastBatch = batch;
+
+  // in this case we've finished the last batch and are complete
+  if (batch_end.isSame(end)) {
+    fetcher.emit('done');
+  } else {
+    // the batch if complete, lets advance to the next set
+    lastId = false;
+    batch = [];
+    batch_start = moment(batch_end);
+    batch_end = moment(batch_end).add(stride, 'h');
+
+    if (batch_end > end) batch_end = moment(end);
+
+    lastTimestamp = batch_end.valueOf();
+  }
+
+  fetcher.emit('trades', lastBatch);
+};
+
+module.exports = function(daterange) {
+  from = daterange.from.clone();
+  end = daterange.to.clone();
+
+  return {
+    bus: fetcher,
+    fetch: fetch,
+  };
+};

importers/exchanges/btcc.js

@@ -0,0 +1,89 @@
+var BTCChina = require('btc-china-fork');
+var util = require('../../core/util.js');
+var _ = require('lodash');
+var moment = require('moment');
+var log = require('../../core/log');
+
+var config = util.getConfig();
+
+var dirs = util.dirs();
+
+var Fetcher = require(dirs.exchanges + 'btcc');
+
+// patch getTrades..
+Fetcher.prototype.getTrades = function(fromTid, sinceTime, callback) {
+  var args = _.toArray(arguments);
+  var process = function(err, result) {
+    if(err)
+      return this.retry(this.getTrades, args);
+
+    callback(result);
+  }.bind(this);
+
+  if(sinceTime)
+    var params = {
+      limit: 1,
+      sincetype: 'time',
+      since: sinceTime
+    }
+
+  else if(fromTid)
+    var params = {
+      limit: 5000,
+      since: fromTid
+    }
+
+  this.btcc.getHistoryData(process, params);
+}
+
+util.makeEventEmitter(Fetcher);
+
+var iterator = false;
+var end = false;
+var done = false;
+var from = false;
+
+var fetcher = new Fetcher(config.watch);
+
+var fetch = () => {
+  if(!iterator)
+    fetcher.getTrades(false, from, handleFirstFetch);
+  else
+    fetcher.getTrades(iterator, false, handleFetch);
+}
+
+// we use the first fetch to figure out 
+// the tid of the moment we want data from
+var handleFirstFetch = trades => {
+  iterator = _.first(trades).tid;
+  fetch();
+}
+
+var handleFetch = trades => {
+
+  iterator = _.last(trades).tid;
+  var last = moment.unix(_.last(trades).date);
+
+  if(last > end) {
+    fetcher.emit('done');
+
+    var endUnix = end.unix();
+    trades = _.filter(
+      trades,
+      t => t.date <= endUnix
+    );
+  }
+
+  fetcher.emit('trades', trades);
+}
+
+module.exports = function (daterange) {
+  from = daterange.from.unix();
+  end = daterange.to.clone();
+
+  return {
+    bus: fetcher,
+    fetch: fetch
+  }
+}
+

importers/exchanges/coinfalcon.js

@@ -0,0 +1,53 @@
+const moment = require('moment');
+const util = require('../../core/util.js');
+const _ = require('lodash');
+const log = require('../../core/log');
+
+var config = util.getConfig();
+var dirs = util.dirs();
+
+var Fetcher = require(dirs.exchanges + 'coinfalcon');
+
+util.makeEventEmitter(Fetcher);
+
+var end = false;
+var done = false;
+var from = false;
+
+var fetcher = new Fetcher(config.watch);
+
+var fetch = () => {
+  fetcher.import = true;
+  log.debug('[CoinFalcon] Getting trades from: ', from);
+  fetcher.getTrades(from, handleFetch, true);
+};
+
+var handleFetch = (unk, trades) => {
+  if (trades.length > 0) {
+    var last = moment.unix(_.last(trades).date).utc();
+    var next = last.clone();
+  } else {
+    var next = from.clone().add(1, 'h');
+    log.debug('Import step returned no results, moving to the next 1h period');
+  }
+
+  if (from.add(1, 'h') >= end) {
+    fetcher.emit('done');
+
+    var endUnix = end.unix();
+    trades = _.filter(trades, t => t.date <= endUnix);
+  }
+
+  from = next.clone();
+  fetcher.emit('trades', trades);
+};
+
+module.exports = function(daterange) {
+  from = daterange.from.clone().utc();
+  end = daterange.to.clone().utc();
+
+  return {
+    bus: fetcher,
+    fetch: fetch,
+  };
+};

importers/exchanges/gdax.js

@@ -0,0 +1,168 @@
+const util = require('../../core/util.js');
+const _ = require('lodash');
+const moment = require('moment');
+const log = require('../../core/log');
+
+const config = util.getConfig();
+
+const dirs = util.dirs();
+
+const QUERY_DELAY = 350;
+const BATCH_SIZE = 100;
+const SCAN_ITER_SIZE = 50000;
+const BATCH_ITER_SIZE = BATCH_SIZE * 10;
+
+const Fetcher = require(dirs.exchanges + 'gdax');
+const retry = require(dirs.exchanges + '../exchangeUtils').retry;
+
+Fetcher.prototype.getTrades = function(sinceTid, callback) {
+  let lastScan = 0;
+
+  const handle = (err, data) => {
+    if (err) return callback(err);
+
+    let result = _.map(data, function(trade) {
+      return {
+        tid: trade.trade_id,
+        amount: parseFloat(trade.size),
+        date: moment.utc(trade.time).format("X"),
+        price: parseFloat(trade.price)
+      };
+    });
+
+    callback(null, result.reverse());
+  };
+
+  const fetch = cb => this.gdax_public.getProductTrades(this.pair, { after: sinceTid, limit: BATCH_SIZE }, this.processResponse('getTrades', cb));
+  retry(null, fetch, handle);
+};
+
+Fetcher.prototype.findFirstTrade = function(sinceTs, callback) {
+  let currentId = 0;
+  let sinceM = moment(sinceTs).utc();
+
+  log.info(`Scanning for the first trade ID to start batching requests, may take a few minutes ...`);
+
+  const handle = (err, data) => {
+    if (err) return callback(err);
+
+    let m = moment.utc(_.first(data).time);
+    let ts = m.valueOf();
+    if (ts < sinceTs) {
+      log.info(`First trade ID for batching found ${currentId - SCAN_ITER_SIZE}`);
+      return callback(undefined, currentId - SCAN_ITER_SIZE);
+    }
+
+    currentId = _.first(data).trade_id;
+    log.debug(`Have trade id ${currentId} for date ${_.first(data).time} ${sinceM.from(m, true)} to scan`);
+
+    let nextScanId = currentId - SCAN_ITER_SIZE;
+    if (nextScanId <= SCAN_ITER_SIZE) {
+      currentId = BATCH_ITER_SIZE;
+      log.info(`First trade ID for batching found ${currentId}`);
+      return callback(undefined, currentId);
+    }
+
+    setTimeout(() => {
+      const fetch = cb => this.gdax_public.getProductTrades(this.pair, { after: nextScanId, limit: 1 }, this.processResponse('getTrades', cb));
+      retry(null, fetch, handle);
+    }, QUERY_DELAY);
+  }
+
+  const fetch = cb => this.gdax_public.getProductTrades(this.pair, { limit: 1 }, this.processResponse('getTrades', cb));
+  retry(null, fetch, handle);
+}
+
+util.makeEventEmitter(Fetcher);
+
+let end = false;
+let done = false;
+let from = false;
+
+let batch = [];
+let batchId = false; // Lowest ID for the current a batch
+
+let lastId = false;
+
+let latestId = false;
+let latestMoment = false;
+
+let fetcher = new Fetcher(config.watch);
+
+let retryForever = {
+  forever: true,
+  factor: 1.2,
+  minTimeout: 10 * 1000,
+  maxTimeout: 120 * 1000
+};
+
+let fetch = () => {
+  fetcher.import = true;
+
+  // We are in the sub-iteration step for a given batch
+  if (lastId) {
+    setTimeout(() => {
+      fetcher.getTrades(lastId, handleFetch);
+    }, QUERY_DELAY);
+  }
+  // We are running the first query, and need to find the starting batch
+  else {
+    let process = (err, firstBatchId) => {
+      if (err) return handleFetch(err);
+
+      batchId = firstBatchId;
+      fetcher.getTrades(batchId + 1, handleFetch);
+    }
+    fetcher.findFirstTrade(from.valueOf(), process);
+  }
+}
+
+let handleFetch = (err, trades) => {
+  if (err) {
+    log.error(`There was an error importing from GDAX ${err}`);
+    fetcher.emit('done');
+    return fetcher.emit('trades', []);
+  }
+
+  if (trades.length) {
+    batch = trades.concat(batch);
+
+    let last = moment.unix(_.first(trades).date).utc();
+    lastId = _.first(trades).tid
+
+    let latestTrade = _.last(trades);
+    if (!latestId || latestTrade.tid > latestId) {
+      latestId = latestTrade.tid;
+      latestMoment = moment.unix(latestTrade.date).utc();
+    }
+
+    // still doing sub-iteration in the batch
+    if (lastId >= (batchId - BATCH_ITER_SIZE) && last >= from)
+      return fetch();
+  }
+
+  batchId += BATCH_ITER_SIZE;
+  lastId = batchId + 1;
+
+  if (latestMoment >= end) {
+    fetcher.emit('done');
+  }
+
+  let endUnix = end.unix();
+  let startUnix = from.unix();
+  batch = _.filter(batch, t => t.date >= startUnix && t.date <= endUnix);
+
+  fetcher.emit('trades', batch);
+  batch = [];
+}
+
+module.exports = function (daterange) {
+
+  from = daterange.from.utc().clone();
+  end = daterange.to.utc().clone();
+
+  return {
+    bus: fetcher,
+    fetch: fetch
+  }
+}

importers/exchanges/kraken.js

@@ -0,0 +1,81 @@
+var _ = require('lodash');
+var moment = require('moment');
+
+var util = require('../../core/util.js');
+var log = require('../../core/log');
+
+var config = util.getConfig();
+
+var dirs = util.dirs();
+
+var Fetcher = require(dirs.exchanges + 'kraken');
+
+util.makeEventEmitter(Fetcher);
+
+var end = false;
+var done = false;
+var from = false;
+
+var lastId = false;
+var prevLastId = false;
+
+var fetcher = new Fetcher(config.watch);
+
+var fetch = () => {
+    fetcher.import = true;
+
+    if (lastId) {
+        var tidAsTimestamp = lastId / 1000000;
+        setTimeout(() => {
+            fetcher.getTrades(tidAsTimestamp, handleFetch)
+        }, 500);
+    }
+    else
+        fetcher.getTrades(from, handleFetch);
+}
+
+var handleFetch = (err, trades) => {
+    if(!err && !trades.length) {
+        console.log('no trades');
+        err = 'No trades';
+    }
+
+    if (err) {
+        log.error(`There was an error importing from Kraken ${err}`);
+        fetcher.emit('done');
+        return fetcher.emit('trades', []);
+    }
+
+    var last = moment.unix(_.last(trades).date).utc();
+    lastId = _.last(trades).tid
+    if(last < from) {
+        log.debug('Skipping data, they are before from date', last.format());
+        return fetch();
+    }
+
+    if  (last > end || lastId === prevLastId) {
+        fetcher.emit('done');
+
+        var endUnix = end.unix();
+        trades = _.filter(
+            trades,
+            t => t.date <= endUnix
+        )
+    }
+
+    prevLastId = lastId
+    fetcher.emit('trades', trades);
+}
+
+module.exports = function (daterange) {
+
+    from = daterange.from.clone();
+    end = daterange.to.clone();
+
+    return {
+        bus: fetcher,
+        fetch: fetch
+    }
+}
+
+

importers/exchanges/luno.js

@@ -0,0 +1,111 @@
+const moment = require('moment');
+const util = require('../../core/util.js');
+const _ = require('lodash');
+const retry = require('../../exchange/exchangeUtils').retry;
+
+const config = util.getConfig();
+const dirs = util.dirs();
+const Fetcher = require(dirs.exchanges + 'luno');
+
+util.makeEventEmitter(Fetcher);
+
+var end = false;
+var from = false;
+const REQUEST_INTERVAL = 5 * 1000;
+
+Fetcher.prototype.getTrades = function(since, callback, descending) {
+  const recoverableErrors = [
+    'SOCKETTIMEDOUT',
+    'TIMEDOUT',
+    'CONNRESET',
+    'CONNREFUSED',
+    'NOTFOUND'
+  ];
+
+  const processResponse = function(funcName, callback) {
+    return (error, body) => {
+      if (!error && !body) {
+        error = new Error('Empty response');
+      }
+
+      if (error) {
+        console.log(funcName, 'processResponse received ERROR:', error.message);
+        if (includes(error.message, recoverableErrors)) {
+          error.notFatal = true;
+        }
+
+        if (includes(error.message, ['error 429'])) {
+          error.notFatal = true;
+          error.backoffDelay = 10000;
+        }
+
+        return callback(error, undefined);
+      }
+
+      return callback(undefined, body);
+    }
+  };
+
+  const process = (err, result) => {
+    if (err) {
+      console.log('Error importing trades:', err);
+      return;
+    }
+    trades = _.map(result.trades, function(t) {
+      return {
+        price: t.price,
+        date: Math.round(t.timestamp / 1000),
+        amount: t.volume,
+        tid: t.timestamp
+      };
+    });
+    callback(null, trades.reverse());
+  }
+
+  if (moment.isMoment(since)) since = since.valueOf();
+  (_.isNumber(since) && since > 0) ? since: since = 0;
+
+  console.log('importer getting trades from Luno since', moment.utc(since).format('YYYY-MM-DD HH:mm:ss'), 'UTC');
+
+  const handler = cb => this.luno.getTrades({ since: since, pair: this.pair }, processResponse('getTrades', cb));
+  retry(null, handler, process);
+}
+
+const fetcher = new Fetcher(config.watch);
+
+const fetch = () => {
+  fetcher.import = true;
+  setTimeout(() => fetcher.getTrades(from, handleFetch), REQUEST_INTERVAL);
+};
+
+const handleFetch = (err, trades) => {
+  if (err) {
+    console.log(`There was an error importing from Luno: ${err}`);
+    fetcher.emit('done');
+    return fetcher.emit('trades', []);
+  }
+
+  if (trades.length > 0) {
+    from = moment.utc(_.last(trades).tid + 1).clone();
+  } else {
+    fetcher.emit('done');
+  }
+
+  if (from >= end) {
+    fetcher.emit('done');
+    const endUnix = end.unix();
+    trades = _.filter(trades, t => t.date <= endUnix);
+  }
+
+  fetcher.emit('trades', trades);
+};
+
+module.exports = function(daterange) {
+  from = daterange.from.clone().utc();
+  end = daterange.to.clone().utc();
+
+  return {
+    bus: fetcher,
+    fetch: fetch,
+  };
+};

importers/exchanges/poloniex.js

@@ -0,0 +1,126 @@
+const util = require('../../core/util.js');
+const _ = require('lodash');
+const moment = require('moment');
+const log = require('../../core/log');
+const retry = require('../../exchange/exchangeUtils').retry;
+
+const config = util.getConfig();
+
+const dirs = util.dirs();
+
+const Fetcher = require(dirs.exchanges + 'poloniex');
+
+const batchSize = 60 * 2; // 2 hour
+const overlapSize = 10; // 10 minutes
+
+// Helper methods
+function joinCurrencies(currencyA, currencyB){
+    return currencyA + '_' + currencyB;
+}
+
+// patch getTrades..
+Fetcher.prototype.getTrades = function(range, callback) {
+  const handle = (err, result) => {
+    if(err) {
+      return callback(err);
+    }
+
+    if(_.size(result) === 50000) {
+      // to many trades..
+      util.die('too many trades..');
+    }
+
+    result = _.map(result, function(trade) {
+      return {
+        tid: trade.tradeID,
+        amount: +trade.amount,
+        date: moment.utc(trade.date).format('X'),
+        price: +trade.rate
+      };
+    });
+
+    callback(result.reverse());
+  };
+
+  const params = {
+    currencyPair: joinCurrencies(this.currency, this.asset)
+  }
+
+  params.start = range.from.unix();
+  params.end = range.to.unix();
+
+  const fetch = next => this.poloniex._public('returnTradeHistory', params, this.processResponse(next));
+  retry(null, fetch, handle);
+}
+
+util.makeEventEmitter(Fetcher);
+
+var iterator = false;
+var end = false;
+var done = false;
+
+var fetcher = new Fetcher(config.watch);
+
+var fetch = () => {
+  log.info(
+    config.watch.currency,
+    config.watch.asset,
+    'Requesting data from',
+    iterator.from.format('YYYY-MM-DD HH:mm:ss') + ',',
+    'to',
+    iterator.to.format('YYYY-MM-DD HH:mm:ss')
+  );
+
+  if(util.gekkoEnv === 'child-process') {
+    let msg = ['Requesting data from',
+      iterator.from.format('YYYY-MM-DD HH:mm:ss') + ',',
+      'to',
+      iterator.to.format('YYYY-MM-DD HH:mm:ss')].join('');
+    process.send({type: 'log', log: msg});
+  }
+  fetcher.getTrades(iterator, handleFetch);
+}
+
+var handleFetch = trades => {
+  iterator.from.add(batchSize, 'minutes').subtract(overlapSize, 'minutes');
+  iterator.to.add(batchSize, 'minutes').subtract(overlapSize, 'minutes');
+
+  if(!_.size(trades)) {
+    // fix https://github.com/askmike/gekko/issues/952
+    if(iterator.to.clone().add(batchSize * 4, 'minutes') > end) {
+      fetcher.emit('done');
+    }
+
+    return fetcher.emit('trades', []);
+  }
+
+  var last = moment.unix(_.last(trades).date);
+
+  if(last > end) {
+    fetcher.emit('done');
+
+    var endUnix = end.unix();
+    trades = _.filter(
+      trades,
+      t => t.date <= endUnix
+    );
+  }
+
+  fetcher.emit('trades', trades);
+}
+
+module.exports = function (daterange) {
+  iterator = {
+    from: daterange.from.clone(),
+    to: daterange.from.clone().add(batchSize, 'minutes')
+  }
+  end = daterange.to.clone();
+
+  return {
+    bus: fetcher,
+    fetch: fetch
+  }
+}
+
+
+

importers/exchanges/therocktrading.js

@@ -0,0 +1,91 @@
+const moment = require('moment');
+const util = require('../../core/util');
+const log = require('../../core/log');
+const _ = require('lodash');
+const retry = require('../../exchange/exchangeUtils').retry;
+
+const config = util.getConfig();
+const dirs = util.dirs();
+const Fetcher = require(dirs.exchanges + 'therocktrading');
+
+Fetcher.prototype.getTrades = function(since, to, page, callback, descending) {
+  const handle = (err, data) => {
+    if (err) return callback(err);
+    var trades = [];
+    if (_.isArray(data.trades)) {
+      trades = _.map(data.trades, function(trade) {
+        return {
+          tid: trade.id,
+          price: trade.price,
+          amount: trade.amount,
+          date: moment.utc(trade.date).format('X')
+        };
+      });
+    }
+
+    callback(null, descending ? trades : trades.reverse());
+  };
+
+  if (moment.isMoment(since)) since = since.format();
+  if (moment.isMoment(to)) to = to.format();
+
+  let options = {
+    before: to,
+    after: since,
+    page: page,
+    order: "ASC",
+    per_page: 200
+  }
+  const fetch = cb => this.therocktrading.trades(this.pair, options, this.processResponse('getTrades', cb));
+  retry(null, fetch, handle);
+};
+
+util.makeEventEmitter(Fetcher);
+
+var end = false;
+var done = false;
+var from = false;
+
+var page = 1;
+
+var fetcher = new Fetcher(config.watch);
+
+var fetch = () => {
+    fetcher.import = true;
+
+    log.debug("IMPORTER: starting fetcher")
+
+    fetcher.getTrades(from, end, page, handleFetch);
+}
+
+var handleFetch = (err, trades) => {
+    if(!err && !trades.length) {
+        console.log('no more trades');
+        fetcher.emit('done');
+    }
+
+    if (err) {
+        log.error(`There was an error importing from TheRockTrading ${err}`);
+        fetcher.emit('done');
+        return fetcher.emit('trades', []);
+    }
+
+    if (trades.length > 0) {
+        page = page + 1;
+    }
+
+    fetcher.emit('trades', trades);
+}
+
+module.exports = function (daterange) {
+
+    from = daterange.from.clone();
+    end = daterange.to.clone();
+
+    return {
+        bus: fetcher,
+        fetch: fetch
+    }
+}
+
+