first commit

core/workers/datasetScan/parent.js

@@ -0,0 +1,54 @@
+var _ = require('lodash');
+var moment = require('moment');
+var async = require('async');
+var os = require('os');
+
+var util = require('../../util');
+var dirs = util.dirs();
+
+var dateRangeScan = require('../dateRangeScan/parent');
+
+module.exports = function(config, done) {
+
+  util.setConfig(config);
+
+  var adapter = config[config.adapter];
+  var scan = require(dirs.gekko + adapter.path + '/scanner');
+
+  scan((err, markets) => {
+
+    if(err)
+      return done(err);
+
+      let numCPUCores = os.cpus().length;
+      if(numCPUCores === undefined)
+         numCPUCores = 1;
+      async.eachLimit(markets, numCPUCores, (market, next) => {
+
+      let marketConfig = _.clone(config);
+      marketConfig.watch = market;
+
+      dateRangeScan(marketConfig, (err, ranges) => {
+        if(err)
+          return next();
+
+        market.ranges = ranges;
+
+        next();
+      });
+
+    }, err => {
+      let resp = {
+        datasets: [],
+        errors: []
+      }
+      markets.forEach(market => {
+        if(market.ranges)
+          resp.datasets.push(market);
+        else
+          resp.errors.push(market);
+      })
+      done(err, resp);
+    })
+  });
+}

core/workers/dateRangeScan/child.js

@@ -0,0 +1,26 @@
+var util = require(__dirname + '/../../util');
+
+var dirs = util.dirs();
+var ipc = require('relieve').IPCEE(process);
+
+ipc.on('start', config => {
+
+  // force correct gekko env
+  util.setGekkoEnv('child-process');
+
+  // force disable debug
+  config.debug = false;
+
+  // persist config
+  util.setConfig(config);
+
+  var scan = require(dirs.tools + 'dateRangeScanner');
+  scan(
+    (err, ranges, reader) => {
+      reader.close();
+      ipc.send('ranges', ranges);
+      process.exit(0);
+    }
+  );
+});
+

core/workers/dateRangeScan/parent.js

@@ -0,0 +1,21 @@
+var ForkTask = require('relieve').tasks.ForkTask;
+var fork = require('child_process').fork;
+
+module.exports = function(config, done) {
+  var debug = typeof v8debug === 'object';
+  if (debug) {
+    process.execArgv = [];
+  }
+
+  task = new ForkTask(fork(__dirname + '/child'));
+
+  task.send('start', config);
+
+  task.once('ranges', ranges => {
+    return done(false, ranges);
+  });
+  task.on('exit', code => {
+    if(code !== 0)
+      done('ERROR, unable to scan dateranges, please check the console.');
+  });
+}

core/workers/loadCandles/child.js

@@ -0,0 +1,28 @@
+var start = (config, candleSize, daterange) => {
+  var util = require(__dirname + '/../../util');
+
+  // force correct gekko env
+  util.setGekkoEnv('child-process');
+
+  // force disable debug
+  config.debug = false;
+  util.setConfig(config);
+
+  var dirs = util.dirs();
+
+  var load = require(dirs.tools + 'candleLoader');
+  load(config.candleSize, candles => {
+    process.send(candles);
+  })
+}
+
+process.send('ready');
+
+process.on('message', (m) => {
+  if(m.what === 'start')
+    start(m.config, m.candleSize, m.daterange);
+});
+
+process.on('disconnect', function() {
+  process.exit(0);
+})

core/workers/loadCandles/parent.js

@@ -0,0 +1,68 @@
+// example usage:
+
+// let config = {
+//   watch: {
+//     exchange: 'poloniex',
+//     currency: 'USDT',
+//     asset: 'BTC'
+//   },
+//   daterange: {
+//     from: '2016-05-22 11:22',
+//     to: '2016-06-03 19:56'
+//   },
+//   adapter: 'sqlite',
+//   sqlite: {
+//     path: 'plugins/sqlite',
+
+//     dataDirectory: 'history',
+//     version: 0.1,
+
+//     dependencies: [{
+//       module: 'sqlite3',
+//       version: '3.1.4'
+//     }]
+//   },
+//   candleSize: 100
+// }
+
+// module.exports(config, function(err, data) {
+//   console.log('FINAL CALLBACK');
+//   console.log('err', err);
+//   console.log('data', data.length);
+// })
+
+
+const fork = require('child_process').fork;
+const _ = require('lodash');
+
+module.exports = (config, callback) => {
+  var debug = typeof v8debug === 'object';
+  if (debug) {
+    process.execArgv = [];
+  }
+
+  const child = fork(__dirname + '/child');
+
+  const message = {
+    what: 'start',
+    config
+  }
+
+  const done = _.once(callback);
+
+  child.on('message', function(m) {
+    if(m === 'ready')
+      return child.send(message);
+
+    // else we are done and have candles!
+    done(null, m);
+    if (this.connected) {
+      this.disconnect();
+    }
+  });
+
+  child.on('exit', code => {
+    if(code !== 0)
+      done('ERROR, unable to load candles, please check the console.');
+  });
+}

core/workers/pipeline/child.js

@@ -0,0 +1,66 @@
+/*
+
+  Gekko is a Bitcoin trading bot for popular Bitcoin exchanges written 
+  in node, it features multiple trading methods using technical analysis.
+
+  If you are interested in how Gekko works, read more about Gekko's 
+  architecture here:
+
+  https://github.com/askmike/gekko/blob/stable/docs/internals/architecture.md
+
+  Disclaimer:
+
+  USE AT YOUR OWN RISK!
+
+  The author of this project is NOT responsible for any damage or loss caused 
+  by this software. There can be bugs and the bot may not perform as expected 
+  or specified. Please consider testing it first with paper trading and/or 
+  backtesting on historical data. Also look at the code to see what how 
+  it is working.
+
+*/
+
+var start = (mode, config) => {
+  var util = require(__dirname + '/../../util');
+
+  // force correct gekko env
+  util.setGekkoEnv('child-process');
+
+  var dirs = util.dirs();
+
+  // force correct gekko mode & config
+  util.setGekkoMode(mode);
+  util.setConfig(config);
+
+  var pipeline = require(dirs.core + 'pipeline');
+  pipeline({
+    config: config,
+    mode: mode
+  });
+}
+
+process.send('ready');
+
+process.on('message', function(m) {
+  if(m.what === 'start')
+    start(m.mode, m.config);
+
+  if(m.what === 'exit')
+    process.exit(0);
+});
+
+process.on('disconnect', function() {
+  console.log('disconnect');
+  process.exit(-1);
+})
+
+process
+  .on('unhandledRejection', (message, p) => {
+    console.error('unhandledRejection', message);
+    process.send({type: 'error', message: message});
+  })
+  .on('uncaughtException', err => {
+    console.error('uncaughtException', err);
+    process.send({type: 'error', error: err});
+    process.exit(1);
+  });

core/workers/pipeline/messageHandlers/backtestHandler.js

@@ -0,0 +1,25 @@
+// Relay the backtest message it when it comes in.
+
+module.exports = done => {
+  let backtest;
+
+  return {
+    message: message => {
+      if(message.type === 'error') {
+        done(message.error);
+      }
+
+      if(message.backtest) {
+        done(null, message.backtest);
+      }
+    },
+    exit: status => {
+      if(status !== 0) {
+        if(backtest)
+          console.error('Child process died after finishing backtest');
+        else
+          done('Child process has died.');
+      }
+    }
+  }
+}

core/workers/pipeline/messageHandlers/importerHandler.js

@@ -0,0 +1,26 @@
+module.exports = cb => {
+
+  return {
+    message: message => {
+
+      if(message.event === 'marketUpdate')
+        cb(null, {
+          done: false,
+          latest: message.payload
+        })
+
+      else if(message.type === 'error') {
+        cb(message.error);
+      }
+
+      else if(message.type === 'log')
+        console.log(message.log);
+    },
+    exit: status => {
+      if(status !== 0)
+        return cb('Child process has died.');
+      else
+        cb(null, { done: true });
+    }
+  }
+}

core/workers/pipeline/messageHandlers/realtimeHandler.js

@@ -0,0 +1,24 @@
+// pass back all messages as is
+// (except for errors and logs)
+
+module.exports = cb => {
+
+  return {
+    message: message => {
+
+      if(message.type === 'error') {
+        cb(message.error);
+      }
+
+      else
+        cb(null, message);
+
+    },
+    exit: status => {
+      if(status !== 0)
+        cb('Child process has died.');
+      else
+        cb(null, { done: true });
+    }
+  }
+}

core/workers/pipeline/parent.js

@@ -0,0 +1,34 @@
+var fork = require('child_process').fork;
+
+module.exports = (mode, config, callback) => {
+  var debug = typeof v8debug === 'object';
+  if (debug) {
+    process.execArgv = [];
+  }
+
+  var child = fork(__dirname + '/child');
+
+  // How we should handle client messages depends
+  // on the mode of the Pipeline that is being ran.
+  var handle = require('./messageHandlers/' + mode + 'Handler')(callback);
+
+  var message = {
+    what: 'start',
+    mode: mode,
+    config: config
+  };
+
+  child.on('message', function(m) {
+    if(m === 'ready')
+      return child.send(message);
+
+    if(m === 'done')
+      return child.send({what: 'exit'});
+
+    handle.message(m);
+  });
+
+  child.on('exit', handle.exit);
+
+  return child;
+}