first commit

plugins/mongodb/handle.js

@@ -0,0 +1,47 @@
+var mongojs = require('mongojs');
+var mongoUtil = require('./util');
+
+var util = require('../../core/util.js');
+var config = util.getConfig();
+var dirs = util.dirs();
+
+// verify the correct dependencies are installed
+var pluginHelper = require(`${dirs.core}pluginUtil`);
+var pluginMock = {
+  slug: 'mongodb adapter',
+  dependencies: config.mongodb.dependencies
+}
+
+// exit if plugin couldn't be loaded
+var cannotLoad = pluginHelper.cannotLoad(pluginMock);
+if (cannotLoad) {
+  util.die(cannotLoad);
+}
+
+var mode = util.gekkoMode();
+
+var collections = [
+  mongoUtil.settings.historyCollection,
+  mongoUtil.settings.adviceCollection
+]
+
+var connection = mongojs(config.mongodb.connectionString, collections);
+var collection = connection.collection(mongoUtil.settings.historyCollection);
+
+if (mode === 'backtest') {
+  var pair = mongoUtil.settings.pair.join('_');
+
+  collection.find({ pair }).toArray((err, docs) => { // check if we've got any records
+    if (err) {
+      util.die(err);
+    }
+    if (docs.length === 0) {
+      util.die(`History table for ${config.watch.exchange} with pair ${pair} is empty.`);
+    }
+  })
+}
+
+if(mongoUtil.settings.exchange) {
+    collection.createIndex({start: 1, pair: 1}, {unique: true}); // create unique index on "time" and "pair"
+}
+module.exports = connection;

plugins/mongodb/reader.js

@@ -0,0 +1,120 @@
+var _ = require('lodash');
+var util = require('../../core/util.js');
+var log = require(`${util.dirs().core}log`);
+
+var handle = require('./handle');
+var mongoUtil = require('./util');
+
+var Reader = function Reader () {
+  _.bindAll(this);
+  this.db = handle;
+  this.collection = this.db.collection(mongoUtil.settings.historyCollection);
+  this.pair = mongoUtil.settings.pair.join('_');
+}
+
+// returns the furtherst point (up to `from`) in time we have valid data from
+Reader.prototype.mostRecentWindow = function mostRecentWindow (from, to, next) {
+  var mFrom = from.unix();
+  var mTo = to.unix();
+
+  var maxAmount = mTo - mFrom + 1;
+
+  // Find some documents
+  this.collection.find({ pair: this.pair, start: { $gte: mFrom, $lte: mTo } }).sort({ start: -1 }, (err, docs) => {
+    if (err) {
+      return util.die('DB error at `mostRecentWindow`');
+    }
+    // no candles are available
+    if (docs.length === 0) {
+      log.debug('no candles are available');
+      return next(false);
+    }
+
+    if (docs.length === maxAmount) {
+      // full history is available!
+      log.debug('full history is available!');
+      return next({
+        mFrom,
+        mTo
+      });
+    }
+
+    // we have at least one gap, figure out where
+    var mostRecent = _.first(docs).start;
+
+    var gapIndex = _.findIndex(docs, (r, i) => r.start !== mostRecent - i * 60);
+
+    // if there was no gap in the records, but
+    // there were not enough records.
+    if (gapIndex === -1) {
+      var leastRecent = _.last(docs).start;
+      return next({
+        from: leastRecent,
+        to: mostRecent
+      });
+    }
+
+    // else return mostRecent and the
+    // the minute before the gap
+    return next({
+      from: docs[gapIndex - 1].start,
+      to: mostRecent
+    });
+  })
+}
+
+Reader.prototype.get = function get (from, to, what, next) { // returns all fields in general
+  // Find some documents
+  this.collection.find({ pair: this.pair, start: { $gte: from, $lte: to } }).sort({ start: 1 }, (err, docs) => {
+    if (err) {
+      return util.die('DB error at `get`');
+    }
+    return next(null, docs);
+  });
+}
+
+Reader.prototype.count = function fCount (from, to, next) {
+  this.collection.count({ pair: this.pair, start: { $gte: from, $lte: to } }, (err, count) => {
+    if (err) {
+      return util.die('DB error at `count`');
+    }
+    return next(null, count);
+  })
+}
+
+Reader.prototype.countTotal = function countTotal (next) {
+  this.collection.count({ pair: this.pair }, (err, count) => {
+    if (err) {
+      return util.die('DB error at `countTotal`');
+    }
+    return next(null, count);
+  })
+}
+
+Reader.prototype.getBoundry = function getBoundry (next) {
+  this.collection.find({ pair: this.pair }, { start: 1 }).sort({ start: 1 }).limit(1, (err, docs) => {
+    if (err) {
+      return util.die('DB error at `getBoundry`');
+    }
+    var start = _.first(docs).start;
+
+    this.collection.find({ pair: this.pair }, { start: 1 }).sort({ start: -1 }).limit(1, (err2, docs2) => {
+      if (err2) {
+        return util.die('DB error at `getBoundry`');
+      }
+      var end = _.first(docs2).start;
+      return next(null, { first: start, last: end });
+    });
+    return null;
+  });
+}
+
+Reader.prototype.tableExists = function(name, next) {
+  return next(null, true); // Return true for backtest
+}
+
+Reader.prototype.close = function close () {
+  this.db = null;
+}
+
+module.exports = Reader;

plugins/mongodb/scanner.js

@@ -0,0 +1,40 @@
+const async = require('async');
+var _ = require('lodash');
+var util = require('../../core/util.js');
+var log = require(`${util.dirs().core}log`);
+
+var handle = require('./handle');
+
+module.exports = done => {
+    this.db = handle;
+
+    let markets = [];
+    async.waterfall([
+        (cb) => {
+            handle.getCollectionNames(cb)
+        },
+        (collections, cb) => {
+            async.each(collections, (collection, cb) => {
+                let [exchange, type] = collection.split('_');
+                if (type === 'candles') {
+                    handle.collection(collection).distinct('pair', {}, (err, pairs) => {
+                        console.log(exchange);
+                        pairs.forEach((pair) => {
+                            pair = pair.split('_');
+                            markets.push({
+                                exchange: exchange,
+                                currency: _.first(pair),
+                                asset: _.last(pair)
+                            });
+                        });
+                        cb();
+                    })
+                } else {
+                    cb();
+                }
+            }, () => {
+                cb(null, markets)
+            })
+        }
+    ], done)
+}

plugins/mongodb/util.js

@@ -0,0 +1,15 @@
+var config = require('../../core/util.js').getConfig();
+
+var watch = config.watch;
+var exchangeLowerCase = watch ? watch.exchange.toLowerCase() : watch = {}; // Do not crash on this, not needed to read from db
+
+var settings = {
+  exchange: watch.exchange,
+  pair: [watch.currency, watch.asset],
+  historyCollection: `${exchangeLowerCase}_candles`,
+  adviceCollection: `${exchangeLowerCase}_advices`
+};
+
+module.exports = {
+  settings
+};

plugins/mongodb/writer.js

@@ -0,0 +1,115 @@
+var _ = require('lodash');
+var config = require('../../core/util.js').getConfig();
+
+var moment = require('moment');
+var util = require('../../core/util.js');
+var log = require(`${util.dirs().core}log`)
+
+var handle = require('./handle');
+var mongoUtil = require('./util');
+
+var Store = function Store (done) {
+  _.bindAll(this);
+  this.done = done;
+  this.db = handle;
+  this.historyCollection = this.db.collection(mongoUtil.settings.historyCollection);
+  this.adviceCollection = this.db.collection(mongoUtil.settings.adviceCollection);
+
+  this.candleCache = [];
+
+  this.pair = mongoUtil.settings.pair.join('_');
+
+  this.price = 'N/A';
+  this.marketTime = 'N/A';
+
+  done();
+}
+
+Store.prototype.writeCandles = function writeCandles () {
+  if (_.isEmpty(this.candleCache)) { // nothing to do
+    return;
+  }
+
+  var candles = [];
+  _.each(this.candleCache, candle => {
+    var mCandle = {
+      time: moment().unix(),
+      start: candle.start.unix(),
+      open: candle.open,
+      high: candle.high,
+      low: candle.low,
+      close: candle.close,
+      vwp: candle.vwp,
+      volume: candle.volume,
+      trades: candle.trades,
+      pair: this.pair
+    };
+    candles.push(mCandle);
+  });
+
+  // Fix error when whole batch would failed to insert if one or more duplicate candle
+  this.historyCollection.insert(candles, { ordered: false }, e => {
+    if (e) {
+      let msg = 'mongojs insert() ' + e.writeErrors.length + ' of ' + candles.length + ' failed.';
+      _.forEach(_.countBy(e.writeErrors, 'code'), (c, k) => {
+        msg += ' Code: E' + k + ' count: ' + c;
+      });
+      log.debug(msg);
+    }
+  });
+
+  this.candleCache = [];
+}
+
+var processCandle = function processCandle (candle, done) {
+  // because we might get a lot of candles
+  // in the same tick, we rather batch them
+  // up and insert them at once at next tick.
+  this.price = candle.close; // used in adviceWriter
+  this.marketTime = candle.start;
+
+  this.candleCache.push(candle);
+  if (this.candleCache.length >= 100)
+    this.writeCandles();
+  done();
+}
+
+var finalize = function(done) {
+  this.writeCandles();
+  // Fix connection closed before all candles was written to db
+  setTimeout( () => {
+    this.db = null;
+    done();
+  }, 1000);
+}
+
+var processAdvice = function processAdvice (advice) {
+  if (config.candleWriter.muteSoft && advice.recommendation === 'soft') {
+    return;
+  }
+
+  log.debug(`Writing advice '${advice.recommendation}' to database.`);
+  var mAdvice = {
+    time: moment().unix(),
+    marketTime: this.marketTime,
+    pair: this.pair,
+    recommendation: advice.recommendation,
+    price: this.price,
+    portfolio: advice.portfolio
+  };
+
+  this.adviceCollection.insert(mAdvice);
+}
+
+if (config.adviceWriter.enabled) {
+  log.debug('Enabling adviceWriter.');
+  Store.prototype.processAdvice = processAdvice;
+}
+
+if (config.candleWriter.enabled) {
+  log.debug('Enabling candleWriter.');
+  Store.prototype.processCandle = processCandle;
+  Store.prototype.finalize = finalize;
+}
+
+module.exports = Store;