first commit

web/state/cache.js

@@ -0,0 +1,14 @@
+const _ = require('lodash');
+
+const cache = {};
+
+module.exports = {
+  set: (name, val) => {
+    cache[name] = val;
+    return true;
+  },
+  get: name => {
+    if(_.has(cache, name))
+      return cache[name];
+  }
+}

web/state/gekkoManager.js

@@ -0,0 +1,254 @@
+const _ = require('lodash');
+const moment = require('moment');
+
+const broadcast = require('./cache').get('broadcast');
+const Logger = require('./logger');
+const pipelineRunner = require('../../core/workers/pipeline/parent');
+const reduceState = require('./reduceState.js');
+const now = () => moment().format('YYYY-MM-DD HH:mm');
+
+const GekkoManager = function() {
+  this.gekkos = {};
+  this.instances = {};
+  this.loggers = {};
+
+  this.archivedGekkos = {};
+}
+
+GekkoManager.prototype.add = function({mode, config}) {
+  // set type
+  let type;
+  if(mode === 'realtime') {
+    if(config.market && config.market.type)
+      type = config.market.type;
+    else
+      type = 'watcher';
+  } else {
+    type = '';
+  }
+
+  let logType = type;
+  if(logType === 'leech') {
+    if(config.trader && config.trader.enabled)
+      logType = 'tradebot';
+    else
+      logType = 'papertrader';
+  }
+
+  const date = now().replace(' ', '-').replace(':', '-');
+  const n = (Math.random() + '').slice(3);
+  const id = `${date}-${logType}-${n}`;
+
+  // make sure we catch events happening inside te gekko instance
+  config.childToParent.enabled = true;
+
+  const state = {
+    mode,
+    config,
+    id,
+    type,
+    logType,
+    active: true,
+    stopped: false,
+    errored: false,
+    errorMessage: false,
+    events: {
+      initial: {},
+      latest: {}
+    },
+    start: moment()
+  }
+
+  this.gekkos[id] = state;
+
+  this.loggers[id] = new Logger(id);
+
+  // start the actual instance
+  this.instances[id] = pipelineRunner(mode, config, this.handleRawEvent(id));
+
+  // after passing API credentials to the actual instance we mask them
+  if(logType === 'trader') {
+    config.trader.key = '[REDACTED]';
+    config.trader.secret = '[REDACTED]';
+  }
+
+  console.log(`${now()} Gekko ${id} started.`);
+
+  broadcast({
+    type: 'gekko_new',
+    id,
+    state
+  });
+
+  return state;
+}
+
+GekkoManager.prototype.handleRawEvent = function(id) {
+  const logger = this.loggers[id];
+
+  return (err, event) => {
+    if(err) {
+      return this.handleFatalError(id, err);
+    }
+
+    if(!event) {
+      return;
+    }
+
+    if(event.log) {
+      return logger.write(event.message);
+    }
+
+    if(event.type) {
+      this.handleGekkoEvent(id, event);
+    }
+  }
+}
+
+GekkoManager.prototype.handleGekkoEvent = function(id, event) {
+  this.gekkos[id] = reduceState(this.gekkos[id], event);
+  broadcast({
+    type: 'gekko_event',
+    id,
+    event
+  });
+}
+
+GekkoManager.prototype.handleFatalError = function(id, err) {
+  const state = this.gekkos[id];
+
+  if(!state || state.errored || state.stopped)
+    return;
+
+  state.errored = true;
+  state.errorMessage = err;
+  console.error('RECEIVED ERROR IN GEKKO INSTANCE', id);
+  console.error(err);
+  broadcast({
+    type: 'gekko_error',
+    id,
+    error: err
+  });
+
+  this.archive(id);
+
+  if(state.logType === 'watcher') {
+    this.handleWatcherError(state, id);
+  }
+}
+
+// There might be leechers depending on this watcher, if so
+// figure out it we can safely start a new watcher without
+// the leechers noticing.
+GekkoManager.prototype.handleWatcherError = function(state, id) {
+  console.log(`${now()} A gekko watcher crashed.`);
+  if(!state.events.latest.candle) {
+    console.log(`${now()} was unable to start.`);
+  }
+
+  let latestCandleTime = moment.unix(0);
+  if(state.events.latest && state.events.latest.candle) {
+    latestCandleTime = state.events.latest.candle.start;
+  }
+  const leechers = _.values(this.gekkos)
+    .filter(gekko => {
+      if(gekko.type !== 'leech') {
+        return false;
+      }
+
+      if(_.isEqual(gekko.config.watch, state.config.watch)) {
+        return true;
+      }
+    });
+
+  if(leechers.length) {
+    console.log(`${now()} ${leechers.length} leecher(s) were depending on this watcher.`);
+    if(moment().diff(latestCandleTime, 'm') < 60) {
+      console.log(`${now()} Watcher had recent data, starting a new one in a minute.`);
+      // after a minute try to start a new one again..
+      setTimeout(() => {
+        const mode = 'realtime';
+        const config = state.config;
+        this.add({mode, config});
+      }, 1000 * 60);
+    } else {
+      console.log(`${now()} Watcher did not have recent data, killing its leechers.`);
+      leechers.forEach(leecher => this.stop(leecher.id));
+    }
+
+  }
+}
+
+GekkoManager.prototype.stop = function(id) {
+  if(!this.gekkos[id])
+    return false;
+
+  console.log(`${now()} stopping Gekko ${id}`);
+
+  this.gekkos[id].stopped = true;
+  this.gekkos[id].active = false;
+
+  // todo: graceful shutdown (via gekkoStream's
+  // finish function).
+  this.instances[id].kill();
+
+  broadcast({
+    type: 'gekko_stopped',
+    id
+  });
+
+  this.archive(id);
+
+  return true;
+}
+
+GekkoManager.prototype.archive = function(id) {
+  this.archivedGekkos[id] = this.gekkos[id];
+  this.archivedGekkos[id].stopped = true;
+  this.archivedGekkos[id].active = false;
+  delete this.gekkos[id];
+
+  broadcast({
+    type: 'gekko_archived',
+    id
+  });
+}
+
+GekkoManager.prototype.delete = function(id) {
+  if(this.gekkos[id]) {
+    throw new Error('Cannot delete a running Gekko, stop it first.');
+  }
+
+  if(!this.archivedGekkos[id]) {
+    throw new Error('Cannot delete unknown Gekko.');
+  }
+
+  console.log(`${now()} deleting Gekko ${id}`);
+
+  broadcast({
+    type: 'gekko_deleted',
+    id
+  });
+
+  delete this.archivedGekkos[id];
+
+  return true;
+}
+
+GekkoManager.prototype.archive = function(id) {
+  this.archivedGekkos[id] = this.gekkos[id];
+  this.archivedGekkos[id].stopped = true;
+  this.archivedGekkos[id].active = false;
+  delete this.gekkos[id];
+
+  broadcast({
+    type: 'gekko_archived',
+    id
+  });
+}
+
+GekkoManager.prototype.list = function() {
+  return { live: this.gekkos, archive: this.archivedGekkos };
+}
+
+module.exports = GekkoManager;

web/state/listManager.js

@@ -0,0 +1,59 @@
+// manages a list of things that change over time
+// used for:
+// - The currently running imports
+// - etc..
+const _ = require('lodash');
+
+var ListManager = function() {
+  this._list = [];
+}
+
+// add an item to the list
+ListManager.prototype.add = function(obj) {
+  if(!obj.id)
+    return false;
+  this._list.push(_.clone(obj));
+  return true;
+}
+
+// update some properties on an item
+ListManager.prototype.update = function(id, updates) {
+  let item = this._list.find(i => i.id === id);
+  if(!item)
+    return false;
+  _.merge(item, updates);
+  return true;
+}
+
+// get an item from the list
+ListManager.prototype.get = function(id) {
+  return this._list.find(i => i.id === id);
+}
+
+// push a value to a array property of an item
+ListManager.prototype.push = function(id, prop, value) {
+  let item = this._list.find(i => i.id === id);
+  if(!item)
+    return false;
+
+  item[prop].push(value);
+  return true;
+}
+
+// delete an item from the list
+ListManager.prototype.delete = function(id) {
+  let wasThere = this._list.find(i => i.id === id);
+  this._list = this._list.filter(i => i.id !== id);
+
+  if(wasThere)
+    return true;
+  else
+    return false;
+}
+
+// getter
+ListManager.prototype.list = function() {
+  return this._list;
+}
+
+module.exports = ListManager;

web/state/logger.js

@@ -0,0 +1,38 @@
+const fs = require('fs');
+const _ = require('lodash');
+
+const BASEPATH = __dirname + '/../../logs/';
+
+const Logger = function(id) {
+
+  this.fileName = `${id}.log`;
+
+  this.writing = false;
+  this.queue = [];
+
+  _.bindAll(this);
+}
+
+Logger.prototype.write = function(line) {
+  if(!this.writing) {
+    this.writing = true;
+    fs.appendFile(
+      BASEPATH + this.fileName,
+      line + '\n',
+      this.handleWriteCallback
+    );
+  } else
+    this.queue.push(line);
+}
+
+Logger.prototype.handleWriteCallback = function(err) {
+  if(err)
+    console.error(`ERROR WRITING LOG FILE ${this.fileName}:`, err);
+
+  this.writing = false;
+
+  if(_.size(this.queue))
+    this.write(this.queue.shift())
+}
+
+module.exports = Logger;

web/state/reduceState.js

@@ -0,0 +1,67 @@
+// Redux/vuex inspired reducer, reduces an event into a gekko state.
+// NOTE: this is used by the backend as well as the frontend.
+
+const skipInitialEvents = ['marketUpdate'];
+const skipLatestEvents = ['marketStart', 'stratWarmupCompleted'];
+const trackAllEvents = ['tradeCompleted', 'advice', 'roundtrip'];
+
+const reduce = (state, event) => {
+  const type = event.type;
+  const payload = event.payload;
+
+  state = {
+    ...state,
+    latestUpdate: new Date()
+  }
+
+  if(trackAllEvents.includes(type)) {
+    if(!state.events[type]) {
+      state = {
+        ...state,
+        events: {
+          ...state.events,
+          [type]: [ payload ]
+        }
+      }
+    } else {
+      state = {
+        ...state,
+        events: {
+          ...state.events,
+          [type]: [ ...state.events[type], payload ]
+        }
+      }
+    }
+  }
+
+  if(!state.events.initial[type] && !skipInitialEvents.includes(type)) {
+    state = {
+      ...state,
+      events: {
+        ...state.events,
+        initial: {
+          ...state.events.initial,
+          [type]: payload
+        }
+      }
+    }
+  }
+
+  if(!skipLatestEvents.includes(type)) {
+    state = {
+      ...state,
+      events: {
+        ...state.events,
+        latest: {
+          ...state.events.latest,
+          [type]: payload
+        }
+      }
+    }
+  }
+
+  return state;
+}
+
+// export default reduce;
+module.exports = reduce;