first commit

plugins/postgresql/handle.js

@@ -0,0 +1,114 @@
+const _ = require('lodash');
+const fs = require('fs');
+
+const util = require('../../core/util.js');
+const config = util.getConfig();
+const dirs = util.dirs();
+
+const log = require(util.dirs().core + 'log');
+const postgresUtil = require('./util');
+
+const adapter = config.postgresql;
+
+// verify the correct dependencies are installed
+const pluginHelper = require(dirs.core + 'pluginUtil');
+const pluginMock = {
+  slug: 'postgresql adapter',
+  dependencies: config.postgresql.dependencies
+}
+
+const cannotLoad = pluginHelper.cannotLoad(pluginMock);
+if(cannotLoad) {
+  util.die(cannotLoad);
+}
+
+const pg = require('pg');
+
+const version = adapter.version;
+
+const dbName = postgresUtil.database();
+
+const mode = util.gekkoMode();
+
+const connectionString = config.postgresql.connectionString;
+
+const checkClient = new pg.Pool({
+  connectionString: connectionString + '/postgres',
+});
+const pool = new pg.Pool({
+  connectionString: connectionString + '/' + dbName,
+});
+
+// We need to check if the db exists first.
+// This requires connecting to the default
+// postgres database first. Your postgres
+// user will need appropriate rights.
+checkClient.connect((err, client, done) => {
+  if(err) {
+    util.die(err);
+  }
+
+  log.debug("Check database exists: " + dbName);
+  client.query("select count(*) from pg_catalog.pg_database where datname = $1", [dbName],
+    (err, res) => {
+      if(err) {
+        util.die(err);
+      }
+
+      if(res.rows[0].count !== '0') {
+        // database exists
+        log.debug("Database exists: " + dbName);
+        log.debug("Postgres connection pool is ready, db " + dbName);
+        upsertTables();
+        done();
+        return;
+      }
+
+      // database dot NOT exist
+
+      if(mode === 'backtest') {
+        // no point in trying to backtest with
+        // non existing data.
+        util.die(`History does not exist for exchange ${config.watch.exchange}.`);
+      }
+
+      createDatabase(client, done);
+    });
+});
+
+const createDatabase = (client, done) => {
+  client.query("CREATE DATABASE " + dbName, err => {
+    if(err) {
+      util.die(err);
+    }
+
+    log.debug("Postgres connection pool is ready, db " + dbName);
+    done();
+    upsertTables();
+  });
+}
+
+const upsertTables = () => {
+  const upsertQuery =
+    `CREATE TABLE IF NOT EXISTS
+    ${postgresUtil.table('candles')} (
+      id BIGSERIAL PRIMARY KEY,
+      start integer UNIQUE,
+      open double precision NOT NULL,
+      high double precision NOT NULL,
+      low double precision NOT NULL,
+      close double precision NOT NULL,
+      vwp double precision NOT NULL,
+      volume double precision NOT NULL,
+      trades INTEGER NOT NULL
+    );`;
+
+  pool.query(upsertQuery, (err) => {
+    if(err) {
+      util.die(err);
+    }
+  });
+}
+
+
+module.exports = pool;

plugins/postgresql/reader.js

@@ -0,0 +1,209 @@
+var _ = require('lodash');
+var util = require('../../core/util.js');
+var config = util.getConfig();
+var log = require(util.dirs().core + 'log');
+
+var handle = require('./handle');
+var postgresUtil = require('./util');
+
+const { Query } = require('pg');
+
+var Reader = function() {
+  _.bindAll(this);
+  this.db = handle;
+}
+
+// returns the furthest point (up to `from`) in time we have valid data from
+Reader.prototype.mostRecentWindow = function(from, to, next) {
+  to = to.unix();
+  from = from.unix();
+
+  var maxAmount = to - from + 1;
+  
+  this.db.connect((err, client, done) => {
+
+    if(err) {
+      log.error(err);
+      return util.die(err.message);
+    }
+
+    var query = client.query(new Query(`
+      SELECT start from ${postgresUtil.table('candles')}
+      WHERE start <= ${to} AND start >= ${from}
+      ORDER BY start DESC
+    `), function (err, result) {
+      if (err) {
+        // bail out if the table does not exist
+        if (err.message.indexOf(' does not exist') !== -1)
+          return next(false);
+
+        log.error(err);
+        return util.die('DB error while reading mostRecentWindow');
+      }
+    });
+
+    var rows = [];
+    query.on('row', function(row) {
+      rows.push(row);
+    });
+
+    // After all data is returned, close connection and return results
+    query.on('end', function() {
+      done();
+      // no candles are available
+      if(rows.length === 0) {
+        return next(false);
+      }
+
+      if(rows.length === maxAmount) {
+
+        // full history is available!
+
+        return next({
+          from: from,
+          to: to
+        });
+      }
+
+      // we have at least one gap, figure out where
+      var mostRecent = _.first(rows).start;
+
+      var gapIndex = _.findIndex(rows, function(r, i) {
+        return r.start !== mostRecent - i * 60;
+      });
+
+      // if there was no gap in the records, but
+      // there were not enough records.
+      if(gapIndex === -1) {
+        var leastRecent = _.last(rows).start;
+        return next({
+          from: leastRecent,
+          to: mostRecent
+        });
+      }
+
+      // else return mostRecent and the
+      // the minute before the gap
+      return next({
+        from: rows[ gapIndex - 1 ].start,
+        to: mostRecent
+      });
+    });
+  });  
+}
+
+Reader.prototype.tableExists = function (name, next) {
+  this.db.connect((err,client,done) => {
+    client.query(`
+      SELECT table_name
+      FROM information_schema.tables
+      WHERE table_schema='${postgresUtil.schema()}'
+        AND table_name='${postgresUtil.table(name)}';
+    `, function(err, result) {
+      done();
+      if (err) {
+        return util.die('DB error at `tableExists`');
+      }
+
+      next(null, result.rows.length === 1);
+    });
+  });  
+}
+
+Reader.prototype.get = function(from, to, what, next) {
+  if(what === 'full'){
+    what = '*';
+  }
+  
+  this.db.connect((err,client,done) => {
+    var query = client.query(new Query(`
+    SELECT ${what} from ${postgresUtil.table('candles')}
+    WHERE start <= ${to} AND start >= ${from}
+    ORDER BY start ASC
+    `));
+
+    var rows = [];
+    query.on('row', function(row) {
+      rows.push(row);
+    });
+
+    query.on('end',function(){
+      done();
+      next(null, rows);
+    });
+  });  
+}
+
+Reader.prototype.count = function(from, to, next) {
+  this.db.connect((err,client,done) => {
+    if(err) {
+      log.error(err);
+      return util.die(err.message);
+    }
+
+    var query = client.query(new Query(`
+      SELECT COUNT(*) as count from ${postgresUtil.table('candles')}
+      WHERE start <= ${to} AND start >= ${from}
+    `));
+    var rows = [];
+    query.on('row', function(row) {
+      rows.push(row);
+    });
+
+    query.on('end',function(){
+      done();
+      next(null, _.first(rows).count);
+    });
+  });  
+}
+
+Reader.prototype.countTotal = function(next) {
+  this.db.connect((err,client,done) => {
+    var query = client.query(new Query(`
+    SELECT COUNT(*) as count from ${postgresUtil.table('candles')}
+    `));
+    var rows = [];
+    query.on('row', function(row) {
+      rows.push(row);
+    });
+
+    query.on('end',function(){
+      done();
+      next(null, _.first(rows).count);
+    });
+  });  
+}
+
+Reader.prototype.getBoundry = function(next) {
+  this.db.connect((err,client,done) => {
+    var query = client.query(new Query(`
+    SELECT (
+      SELECT start
+      FROM ${postgresUtil.table('candles')}
+      ORDER BY start LIMIT 1
+    ) as first,
+    (
+      SELECT start
+      FROM ${postgresUtil.table('candles')}
+      ORDER BY start DESC
+      LIMIT 1
+    ) as last
+    `));
+    var rows = [];
+    query.on('row', function(row) {
+      rows.push(row);
+    });
+
+    query.on('end',function(){
+      done();
+      next(null, _.first(rows));
+    });
+  });  
+}
+
+Reader.prototype.close = function() {
+  //obsolete due to connection pooling
+  //this.db.end();
+}
+
+module.exports = Reader;

plugins/postgresql/scanner.js

@@ -0,0 +1,93 @@
+const _ = require('lodash');
+const async = require('async');
+var pg = require('pg');
+
+const util = require('../../core/util.js');
+const config = util.getConfig();
+const dirs = util.dirs();
+var postgresUtil = require('./util');
+
+var connectionString = config.postgresql.connectionString;
+
+
+module.exports = done => {
+  var scanClient = new pg.Client(connectionString+"/postgres");
+
+  let markets = [];
+
+  scanClient.connect(function (err) {
+    if(err){
+      util.die(err);
+    }
+
+    var sql = "select datname from pg_database";
+
+    // In single DB setup we don't need to go look into other DBs
+    if (postgresUtil.useSingleDatabase()) {
+      sql = "select datname from pg_database where datname='" + postgresUtil.database() + "'";
+    }
+
+    var query = scanClient.query(sql, function (err, result) {
+
+      async.each(result.rows, (dbRow, next) => {
+
+        var scanTablesClient = new pg.Client(connectionString + "/" + dbRow.datname);
+        var dbName = dbRow.datname;
+
+        scanTablesClient.connect(function (err) {
+          if (err) {
+            return next();
+          }
+
+          var query = scanTablesClient.query(`
+            SELECT table_name
+            FROM information_schema.tables
+            WHERE table_schema='${postgresUtil.schema()}';
+          `, function(err, result) {
+            if (err) {
+              return util.die('DB error at `scanning tables`');
+            }
+
+            _.each(result.rows, table => {
+              let parts = table.table_name.split('_');
+              let first = parts.shift();
+              let exchangeName = dbName;
+
+              /**
+               * If using single database, we need to strip
+               * exchange from table name. See here how tables
+               * are named:
+               *
+               * - in single database setup: poloniex_candles_usdt_btc
+               * - in multi db setup: db = poloniex, table = candles_usdt_btc
+               */
+              if (postgresUtil.useSingleDatabase()) {
+                exchangeName = first;
+                first = parts.shift();
+              }
+
+              if(first === 'candles')
+                markets.push({
+                  exchange: exchangeName,
+                  currency: _.first(parts),
+                  asset: _.last(parts)
+                });
+            });
+
+            scanTablesClient.end();
+
+            next();
+          });
+        });
+
+      },
+      // got all tables!
+      err => {
+        scanClient.end();
+        done(err, markets);
+      });
+
+    });
+
+  });
+}

plugins/postgresql/util.js

@@ -0,0 +1,66 @@
+var config = require('../../core/util.js').getConfig();
+
+var watch = config.watch;
+if(watch) {
+  var settings = {
+    exchange: watch.exchange,
+    pair: [watch.currency, watch.asset]
+  }
+}
+
+/**
+ * Returns true if we use single database where
+ * all our tables are stored. The default is to store
+ * every exchange into it's own db.
+ *
+ * Set config.postgresql.database to use single db setup
+ */
+function useSingleDatabase() {
+    return !!config.postgresql.database;
+}
+
+/**
+ * Postgres has tables in lowercase if you don't
+ * escape their names. Which we don't and so let's
+ * just lowercase them.
+ */
+function useLowerCaseTableNames() {
+  return !config.postgresql.noLowerCaseTableName;
+}
+
+module.exports = {
+  settings: settings,
+
+  // true if we have single db setup (see postrgesql.database config key)
+  useSingleDatabase: useSingleDatabase,
+
+  // returns DB name (depends on single db setup)
+  database: function () {
+    return useSingleDatabase() ?
+      config.postgresql.database :
+      config.watch.exchange.toLowerCase().replace(/\-/g,'');
+  },
+
+  // returns table name which can be different if we use
+  // single or multiple db setup.
+  table: function (name) {
+    if (useSingleDatabase()) {
+      name = watch.exchange.replace(/\-/g,'') + '_' + name;
+    }
+    var fullName = [name, settings.pair.join('_')].join('_');
+    return useLowerCaseTableNames() ? fullName.toLowerCase() : fullName;
+  },
+
+  startconstraint: function (name) {
+    if (useSingleDatabase()) {
+      name = watch.exchange.replace(/\-/g,'') + '_' + name;
+    }
+    var fullName = [name, settings.pair.join('_')].join('_');
+    return useLowerCaseTableNames() ? fullName.toLowerCase() + '_start_key' : fullName + '_start_key';
+  },
+
+  // postgres schema name. defaults to 'public'
+  schema: function () {
+    return config.postgresql.schema ? config.postgresql.schema : 'public';
+  }
+}

plugins/postgresql/writer.js

@@ -0,0 +1,73 @@
+var _ = require('lodash');
+const log = require('../../core/log');
+const util = require('../../core/util');
+const config = util.getConfig();
+
+var handle = require('./handle');
+var postgresUtil = require('./util');
+
+var Store = function(done, pluginMeta) {
+  _.bindAll(this);
+  this.done = done;
+  this.db = handle;
+  this.cache = [];
+  done();
+}
+
+Store.prototype.writeCandles = function() {
+  if(_.isEmpty(this.cache)){
+    return;
+  }
+
+  //log.debug('Writing candles to DB!');
+  _.each(this.cache, candle => {
+    var stmt =  `
+    BEGIN; 
+    LOCK TABLE ${postgresUtil.table('candles')} IN SHARE ROW EXCLUSIVE MODE; 
+    INSERT INTO ${postgresUtil.table('candles')} 
+    (start, open, high,low, close, vwp, volume, trades) 
+    VALUES 
+    (${candle.start.unix()}, ${candle.open}, ${candle.high}, ${candle.low}, ${candle.close}, ${candle.vwp}, ${candle.volume}, ${candle.trades}) 
+    ON CONFLICT ON CONSTRAINT ${postgresUtil.startconstraint('candles')} 
+    DO NOTHING; 
+    COMMIT; 
+    `;
+
+    this.db.connect((err,client,done) => {
+      if(err) {
+        util.die(err);
+      }
+      client.query(stmt, (err, res) => {
+        done();
+        if (err) {
+          log.debug(err.stack)
+        } else {
+          //log.debug(res)
+        }
+      });
+    });
+  });
+
+  this.cache = [];
+}
+
+var processCandle = function(candle, done) {
+  this.cache.push(candle);
+  if (this.cache.length > 1)
+    this.writeCandles();
+
+  done();
+};
+
+var finalize = function(done) {
+  this.writeCandles();
+  this.db = null;
+  done();
+}
+
+if(config.candleWriter.enabled) {
+  Store.prototype.processCandle = processCandle;
+  Store.prototype.finalize = finalize;
+}
+
+module.exports = Store;