first commit

plugins/sqlite/handle.js

@@ -0,0 +1,58 @@
+var _ = require('lodash');
+var fs = require('fs');
+
+var util = require('../../core/util.js');
+var config = util.getConfig();
+var dirs = util.dirs();
+
+var adapter = config.sqlite;
+
+// verify the correct dependencies are installed
+var pluginHelper = require(dirs.core + 'pluginUtil');
+var pluginMock = {
+  slug: 'sqlite adapter',
+  dependencies: adapter.dependencies,
+};
+
+var cannotLoad = pluginHelper.cannotLoad(pluginMock);
+if (cannotLoad) util.die(cannotLoad);
+
+// should be good now
+if (config.debug) var sqlite3 = require('sqlite3').verbose();
+else var sqlite3 = require('sqlite3');
+
+var plugins = require(util.dirs().gekko + 'plugins');
+
+var version = adapter.version;
+
+var dbName = config.watch.exchange.toLowerCase() + '_' + version + '.db';
+var dir = dirs.gekko + adapter.dataDirectory;
+
+var fullPath = [dir, dbName].join('/');
+
+var mode = util.gekkoMode();
+if (mode === 'realtime' || mode === 'importer') {
+  if (!fs.existsSync(dir)) fs.mkdirSync(dir);
+} else if (mode === 'backtest') {
+  if (!fs.existsSync(dir)) util.die('History directory does not exist.');
+
+  if (!fs.existsSync(fullPath))
+    util.die(
+      `History database does not exist for exchange ${
+        config.watch.exchange
+      } at version ${version}.`
+    );
+}
+
+module.exports = {
+  initDB: () => {
+    var journalMode = config.sqlite.journalMode || 'PERSIST';
+    var syncMode = journalMode === 'WAL' ? 'NORMAL' : 'FULL';
+  
+    var db = new sqlite3.Database(fullPath);
+    db.run('PRAGMA synchronous = ' + syncMode);
+    db.run('PRAGMA journal_mode = ' + journalMode);
+    db.configure('busyTimeout', 10000);
+    return db;
+  }
+};

plugins/sqlite/reader.js

@@ -0,0 +1,169 @@
+var _ = require('lodash');
+var util = require('../../core/util.js');
+var config = util.getConfig();
+var log = require(util.dirs().core + 'log');
+
+var sqlite = require('./handle');
+var sqliteUtil = require('./util');
+
+var Reader = function() {
+  _.bindAll(this);
+  this.db = sqlite.initDB(true);
+}
+
+
+// returns the most recent window complete candle
+// windows within `from` and `to`
+Reader.prototype.mostRecentWindow = function(from, to, next) {
+  to = to.unix();
+  from = from.unix();
+
+  var maxAmount = to - from + 1;
+
+  this.db.all(`
+    SELECT start from ${sqliteUtil.table('candles')}
+    WHERE start <= ${to} AND start >= ${from}
+    ORDER BY start DESC
+  `, function(err, rows) {
+    if(err) {
+
+      // bail out if the table does not exist
+      if(err.message.split(':')[1] === ' no such table')
+        return next(false);
+
+      log.error(err);
+      return util.die('DB error while reading mostRecentWindow');
+    }
+
+    // no candles are available
+    if(rows.length === 0) {
+      return next(false);
+    }
+
+    if(rows.length === maxAmount) {
+
+      // full history is available!
+
+      return next({
+        from: from,
+        to: to
+      });
+    }
+
+    // we have at least one gap, figure out where
+    var mostRecent = _.first(rows).start;
+
+    var gapIndex = _.findIndex(rows, function(r, i) {
+      return r.start !== mostRecent - i * 60;
+    });
+
+    // if there was no gap in the records, but
+    // there were not enough records.
+    if(gapIndex === -1) {
+      var leastRecent = _.last(rows).start;
+      return next({
+        from: leastRecent,
+        to: mostRecent
+      });
+    }
+
+    // else return mostRecent and the
+    // the minute before the gap
+    return next({
+      from: rows[ gapIndex - 1 ].start,
+      to: mostRecent
+    });
+
+  })
+}
+
+Reader.prototype.tableExists = function(name, next) {
+
+  this.db.all(`
+    SELECT name FROM sqlite_master WHERE type='table' AND name='${sqliteUtil.table(name)}';
+  `, function(err, rows) {
+    if(err) {
+      console.error(err);
+      return util.die('DB error at `get`');
+    }
+
+    next(null, rows.length === 1);
+  });
+}
+
+Reader.prototype.get = function(from, to, what, next) {
+  if(what === 'full')
+    what = '*';
+
+  this.db.all(`
+    SELECT ${what} from ${sqliteUtil.table('candles')}
+    WHERE start <= ${to} AND start >= ${from}
+    ORDER BY start ASC
+  `, function(err, rows) {
+    if(err) {
+      console.error(err);
+      return util.die('DB error at `get`');
+    }
+
+    next(null, rows);
+  });
+}
+
+Reader.prototype.count = function(from, to, next) {
+  this.db.all(`
+    SELECT COUNT(*) as count from ${sqliteUtil.table('candles')}
+    WHERE start <= ${to} AND start >= ${from}
+  `, function(err, res) {
+    if(err) {
+      console.error(err);
+      return util.die('DB error at `get`');
+    }
+
+    next(null, _.first(res).count);
+  });
+}
+
+Reader.prototype.countTotal = function(next) {
+  this.db.all(`
+    SELECT COUNT(*) as count from ${sqliteUtil.table('candles')}
+  `, function(err, res) {
+    if(err) {
+      console.error(err);
+      return util.die('DB error at `get`');
+    }
+
+    next(null, _.first(res).count);
+  });
+}
+
+Reader.prototype.getBoundry = function(next) {
+
+  this.db.all(`
+    SELECT
+    (
+      SELECT start
+      FROM ${sqliteUtil.table('candles')}
+      ORDER BY start LIMIT 1
+    ) as 'first',
+    (
+      SELECT start
+      FROM ${sqliteUtil.table('candles')}
+      ORDER BY start DESC
+      LIMIT 1
+    ) as 'last'
+  `, function(err, rows) {
+    if(err) {
+      console.error(err);
+      return util.die('DB error at `get`');
+    }
+
+    next(null, _.first(rows));
+  });
+}
+
+Reader.prototype.close = function() {
+  this.db.close();
+  this.db = null;
+}
+
+module.exports = Reader;

plugins/sqlite/scanner.js

@@ -0,0 +1,64 @@
+const _ = require('lodash');
+const async = require('async');
+const fs = require('fs');
+
+const util = require('../../core/util.js');
+const config = util.getConfig();
+const dirs = util.dirs();
+
+const sqlite3 = require('sqlite3');
+
+// todo: rewrite with generators or async/await..
+module.exports = done => {
+  const dbDirectory = dirs.gekko + config.sqlite.dataDirectory
+
+  if(!fs.existsSync(dbDirectory))
+    return done(null, []);
+
+  const files = fs.readdirSync(dbDirectory);
+
+  const dbs = files
+    .filter(f => {
+      let parts = f.split('.');
+      if(_.last(parts) === 'db')
+        return true;
+    })
+
+  if(!_.size(dbs))
+    return done(null, []);
+
+  let markets = [];
+
+  async.each(dbs, (db, next) => {
+
+    const exchange = _.first(db.split('_'));
+    const handle = new sqlite3.Database(dbDirectory + '/' + db, sqlite3.OPEN_READONLY, err => {
+      if(err)
+        return next(err);
+
+      handle.all(`SELECT name FROM sqlite_master WHERE type='table'`, (err, tables) => {
+        if(err)
+          return next(err);
+        
+        _.each(tables, table => {
+          let parts = table.name.split('_');
+          let first = parts.shift();
+          if(first === 'candles') 
+            markets.push({
+              exchange: exchange,
+              currency: _.first(parts),
+              asset: _.last(parts)
+            });
+        });
+
+        next();
+      });
+    });
+
+
+  },
+  // got all tables!
+  err => {
+    done(err, markets);
+  });
+}

plugins/sqlite/util.js

@@ -0,0 +1,15 @@
+var config = require('../../core/util.js').getConfig();
+
+var watch = config.watch;
+var settings = {
+  exchange: watch.exchange,
+  pair: [watch.currency, watch.asset],
+  historyPath: config.sqlite.dataDirectory
+}
+
+module.exports = {
+  settings: settings,
+  table: function(name) {
+    return [name, settings.pair.join('_')].join('_');
+  }
+}

plugins/sqlite/writer.js

@@ -0,0 +1,128 @@
+var _ = require('lodash');
+var config = require('../../core/util.js').getConfig();
+
+var sqlite = require('./handle');
+var sqliteUtil = require('./util');
+var util = require('../../core/util');
+var log = require('../../core/log');
+
+var Store = function(done, pluginMeta) {
+  _.bindAll(this);
+  this.done = done;
+
+  this.db = sqlite.initDB(false);
+  this.db.serialize(this.upsertTables);
+
+  this.cache = [];
+  this.buffered = util.gekkoMode() === "importer";
+}
+
+Store.prototype.upsertTables = function() {
+  var createQueries = [
+    `
+      CREATE TABLE IF NOT EXISTS
+      ${sqliteUtil.table('candles')} (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        start INTEGER UNIQUE,
+        open REAL NOT NULL,
+        high REAL NOT NULL,
+        low REAL NOT NULL,
+        close REAL NOT NULL,
+        vwp REAL NOT NULL,
+        volume REAL NOT NULL,
+        trades INTEGER NOT NULL
+      );
+    `,
+
+    // TODO: create trades
+    // ``
+
+    // TODO: create advices
+    // ``
+  ];
+
+  var next = _.after(_.size(createQueries), this.done);
+
+  _.each(createQueries, function(q) {
+    this.db.run(q, next);
+  }, this);
+}
+
+Store.prototype.writeCandles = function() {
+  if(_.isEmpty(this.cache))
+    return;
+
+  const transaction = () => {
+    this.db.run("BEGIN TRANSACTION");
+
+    var stmt = this.db.prepare(`
+      INSERT OR IGNORE INTO ${sqliteUtil.table('candles')}
+      VALUES (?,?,?,?,?,?,?,?,?)
+    `, function(err, rows) {
+        if(err) {
+          log.error(err);
+          return util.die('DB error at INSERT: '+ err);
+        }
+      });
+
+    _.each(this.cache, candle => {
+      stmt.run(
+        null,
+        candle.start.unix(),
+        candle.open,
+        candle.high,
+        candle.low,
+        candle.close,
+        candle.vwp,
+        candle.volume,
+        candle.trades
+      );
+    });
+
+    stmt.finalize();
+    this.db.run("COMMIT");
+    // TEMP: should fix https://forum.gekko.wizb.it/thread-57279-post-59194.html#pid59194
+    this.db.run("pragma wal_checkpoint;");
+    
+    this.cache = [];
+  }
+
+  this.db.serialize(transaction);
+}
+
+var processCandle = function(candle, done) {
+  this.cache.push(candle);
+  if (!this.buffered || this.cache.length > 1000) 
+    this.writeCandles();
+
+  done();
+};
+
+var finalize = function(done) {
+  this.writeCandles();
+  this.db.close(() => { done(); });
+  this.db = null;
+}
+
+if(config.candleWriter.enabled) {
+  Store.prototype.processCandle = processCandle;
+  Store.prototype.finalize = finalize;
+}
+
+// TODO: add storing of trades / advice?
+
+// var processTrades = function(candles) {
+//   util.die('NOT IMPLEMENTED');
+// }
+
+// var processAdvice = function(candles) {
+//   util.die('NOT IMPLEMENTED');
+// }
+
+// if(config.tradeWriter.enabled)
+//  Store.prototype.processTrades = processTrades;
+
+// if(config.adviceWriter.enabled)
+//   Store.prototype.processAdvice = processAdvice;
+
+module.exports = Store;