first commit

importers/exchanges/bitfinex.js

@@ -0,0 +1,148 @@
+const Bitfinex = require('bitfinex-api-node');
+const util = require('../../core/util.js');
+const _ = require('lodash');
+const moment = require('moment');
+const log = require('../../core/log');
+
+const config = util.getConfig();
+
+const dirs = util.dirs();
+
+const Fetcher = require(dirs.exchanges + 'bitfinex');
+const retry = require(dirs.exchanges + '../exchangeUtils').retry;
+
+Fetcher.prototype.getTrades = function(upto, callback, descending) {
+  const handle = (err, data) => {
+    if (err) return callback(err);
+
+    var trades = [];
+    if (_.isArray(data)) {
+      trades = _.map(data, function(trade) {
+        return {
+          tid: trade.ID,
+          date: moment(trade.MTS).format('X'),
+          price: +trade.PRICE,
+          amount: +Math.abs(trade.AMOUNT),
+        };
+      });
+    }
+
+    callback(null, descending ? trades : trades.reverse());
+  };
+
+  let path = 'trades/t' + this.pair + '/hist';
+  if (upto) {
+    let start = moment(upto).subtract(1, 'd').valueOf();
+    let end = moment(upto).valueOf();
+    path += `?limit=1000&start=${start}&end=${end}`;
+  }
+
+  log.debug('Querying trades with: ' + path);
+  const fetch = cb => this.bitfinex.makePublicRequest(path, this.handleResponse('getTrades', cb));
+  retry(null, fetch, handle);
+};
+
+util.makeEventEmitter(Fetcher);
+
+var end = false;
+var done = false;
+var from = false;
+
+var lastTimestamp = false;
+var lastId = false;
+
+var batch = [];
+var batch_start = false;
+var batch_end = false;
+var batch_last = false;
+
+const SCANNING_STRIDE = 24;
+const ITERATING_STRIDE = 2;
+var stride = ITERATING_STRIDE;
+
+var fetcher = new Fetcher(config.watch);
+fetcher.bitfinex = new Bitfinex(null, null, { version: 2, transform: true }).rest;
+
+var retryCritical = {
+  retries: 10,
+  factor: 1.2,
+  minTimeout: 70 * 1000,
+  maxTimeout: 120 * 1000,
+};
+
+var fetch = () => {
+  fetcher.import = true;
+
+  if (lastTimestamp) {
+    // We need to slow this down to prevent hitting the rate limits
+    setTimeout(() => {
+
+      // make sure we fetch with overlap from last batch
+      const since = lastTimestamp - 1000;
+      fetcher.getTrades(since, handleFetch);
+    }, 2500);
+  } else {
+    lastTimestamp = from.valueOf();
+    batch_start = moment(from);
+    batch_end = moment(from).add(stride, 'h');
+
+    fetcher.getTrades(batch_end, handleFetch);
+  }
+};
+
+var handleFetch = (err, trades) => {
+  if (err) {
+    log.error(`There was an error importing from Bitfinex ${err}`);
+    fetcher.emit('done');
+    return fetcher.emit('trades', []);
+  }
+
+  trades = _.filter(trades, t => !lastId || t.tid < lastId);
+
+  if (trades.length) {
+    stride = ITERATING_STRIDE;
+    batch = trades.concat(batch);
+    var last = moment.unix(_.first(trades).date);
+    lastTimestamp = last.valueOf();
+    lastId = _.first(trades).tid;
+  } else {
+    stride = SCANNING_STRIDE;
+    lastTimestamp = moment(lastTimestamp)
+      .subtract(stride, 'h')
+      .valueOf();
+  }
+
+  // if we're not done the batch we need to refetch
+  if (trades.length && moment(lastTimestamp) >= batch_start) {
+    return fetch();
+  }
+
+  var lastBatch = batch;
+
+  // in this case we've finished the last batch and are complete
+  if (batch_end.isSame(end)) {
+    fetcher.emit('done');
+  } else {
+    // the batch if complete, lets advance to the next set
+    lastId = false;
+    batch = [];
+    batch_start = moment(batch_end);
+    batch_end = moment(batch_end).add(stride, 'h');
+
+    if (batch_end > end) batch_end = moment(end);
+
+    lastTimestamp = batch_end.valueOf();
+  }
+
+  fetcher.emit('trades', lastBatch);
+};
+
+module.exports = function(daterange) {
+  from = daterange.from.clone();
+  end = daterange.to.clone();
+
+  return {
+    bus: fetcher,
+    fetch: fetch,
+  };
+};