first commit
This commit is contained in:
Francois Klieber committed
commit
aa7869eb33
386 files changed
+69375
No files matched your search
@@ -0,0 +1,100 @@
|
||||
// TODO: properly handle a daterange for which no data is available.
|
||||
|
||||
const batchSize = 1000;
|
||||
|
||||
const _ = require('lodash');
|
||||
const fs = require('fs');
|
||||
const moment = require('moment');
|
||||
|
||||
const util = require('../../core/util');
|
||||
const config = util.getConfig();
|
||||
const dirs = util.dirs();
|
||||
const log = require(dirs.core + '/log');
|
||||
|
||||
const adapter = config[config.adapter];
|
||||
const Reader = require(dirs.gekko + adapter.path + '/reader');
|
||||
const daterange = config.daterange;
|
||||
|
||||
const CandleBatcher = require(dirs.core + 'candleBatcher');
|
||||
|
||||
const to = moment.utc(daterange.to).startOf('minute');
|
||||
const from = moment.utc(daterange.from).startOf('minute');
|
||||
const toUnix = to.unix();
|
||||
|
||||
if(to <= from)
|
||||
util.die('This daterange does not make sense.')
|
||||
|
||||
if(!from.isValid())
|
||||
util.die('invalid `from`');
|
||||
|
||||
if(!to.isValid())
|
||||
util.die('invalid `to`');
|
||||
|
||||
let iterator = {
|
||||
from: from.clone(),
|
||||
to: from.clone().add(batchSize, 'm').subtract(1, 's')
|
||||
}
|
||||
|
||||
var DONE = false;
|
||||
|
||||
var result = [];
|
||||
var reader = new Reader();
|
||||
var batcher;
|
||||
var next;
|
||||
var doneFn = () => {
|
||||
process.nextTick(() => {
|
||||
next(result);
|
||||
})
|
||||
};
|
||||
|
||||
module.exports = function(candleSize, _next) {
|
||||
next = _.once(_next);
|
||||
|
||||
batcher = new CandleBatcher(candleSize)
|
||||
.on('candle', handleBatchedCandles);
|
||||
|
||||
getBatch();
|
||||
}
|
||||
|
||||
const getBatch = () => {
|
||||
reader.get(
|
||||
iterator.from.unix(),
|
||||
iterator.to.unix(),
|
||||
'full',
|
||||
handleCandles
|
||||
)
|
||||
}
|
||||
|
||||
const shiftIterator = () => {
|
||||
iterator = {
|
||||
from: iterator.from.clone().add(batchSize, 'm'),
|
||||
to: iterator.from.clone().add(batchSize * 2, 'm').subtract(1, 's')
|
||||
}
|
||||
}
|
||||
|
||||
const handleCandles = (err, data) => {
|
||||
if(err) {
|
||||
console.error(err);
|
||||
util.die('Encountered an error..')
|
||||
}
|
||||
|
||||
if(_.size(data) && _.last(data).start >= toUnix || iterator.from.unix() >= toUnix)
|
||||
DONE = true;
|
||||
|
||||
batcher.write(data);
|
||||
batcher.flush();
|
||||
|
||||
if(DONE) {
|
||||
reader.close();
|
||||
|
||||
setTimeout(doneFn, 100);
|
||||
|
||||
} else {
|
||||
shiftIterator();
|
||||
getBatch();
|
||||
}
|
||||
}
|
||||
|
||||
const handleBatchedCandles = candle => {
|
||||
result.push(candle);
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
const fs = require('fs');
|
||||
const _ = require('lodash');
|
||||
const toml = require('toml');
|
||||
|
||||
const util = require('../util');
|
||||
const dirs = util.dirs();
|
||||
|
||||
const getTOML = function(fileName) {
|
||||
var raw = fs.readFileSync(fileName);
|
||||
return toml.parse(raw);
|
||||
}
|
||||
|
||||
// build a config object out of a directory of TOML files
|
||||
module.exports = function() {
|
||||
const configDir = util.dirs().config;
|
||||
|
||||
let _config = getTOML(configDir + 'general.toml');
|
||||
fs.readdirSync(configDir + 'plugins').forEach(function(pluginFile) {
|
||||
let pluginName = _.first(pluginFile.split('.'))
|
||||
_config[pluginName] = getTOML(configDir + 'plugins/' + pluginFile);
|
||||
});
|
||||
|
||||
// attach the proper adapter
|
||||
let adapter = _config.adapter;
|
||||
_config[adapter] = getTOML(configDir + 'adapters/' + adapter + '.toml');
|
||||
|
||||
if(_config.tradingAdvisor.enabled) {
|
||||
// also load the strat
|
||||
let strat = _config.tradingAdvisor.method;
|
||||
let stratFile = configDir + 'strategies/' + strat + '.toml';
|
||||
if(!fs.existsSync(stratFile))
|
||||
util.die('Cannot find the strategy config file for ' + strat);
|
||||
_config[strat] = getTOML(stratFile);
|
||||
}
|
||||
|
||||
const mode = util.gekkoMode();
|
||||
|
||||
if(mode === 'backtest')
|
||||
_config.backtest = getTOML(configDir + 'backtest.toml');
|
||||
|
||||
return _config;
|
||||
}
|
||||
@@ -0,0 +1,233 @@
|
||||
var _ = require('lodash');
|
||||
var fs = require('fs');
|
||||
var moment = require('moment');
|
||||
|
||||
var util = require('../util');
|
||||
var config = util.getConfig();
|
||||
var dirs = util.dirs();
|
||||
var log = require(dirs.core + '/log');
|
||||
|
||||
var Stitcher = function(batcher) {
|
||||
this.batcher = batcher;
|
||||
}
|
||||
|
||||
Stitcher.prototype.ago = function(ts) {
|
||||
var now = moment().utc();
|
||||
var then = moment.unix(ts).utc();
|
||||
return now.diff(then, 'minutes') + ' minutes ago';
|
||||
}
|
||||
|
||||
Stitcher.prototype.verifyExchange = function() {
|
||||
require(dirs.gekko + 'exchange/dependencyCheck');
|
||||
const exchangeChecker = require(dirs.gekko + 'exchange/exchangeChecker');
|
||||
const slug = config.watch.exchange.toLowerCase();
|
||||
let exchange;
|
||||
try {
|
||||
exchange = exchangeChecker.getExchangeCapabilities(slug);
|
||||
} catch(e) {
|
||||
util.die(e.message);
|
||||
}
|
||||
|
||||
if(!exchange)
|
||||
util.die(`Unsupported exchange: ${slug}`);
|
||||
|
||||
var error = exchangeChecker.cantMonitor(config.watch);
|
||||
if(error)
|
||||
util.die(error, true);
|
||||
}
|
||||
|
||||
Stitcher.prototype.prepareHistoricalData = function(done) {
|
||||
this.verifyExchange();
|
||||
|
||||
// - step 1: check most recent stored candle window
|
||||
// - step 2: check oldest trade reachable by API
|
||||
// - step 3: see if overlap
|
||||
// - step 4: feed candle stream into CandleBatcher
|
||||
|
||||
if(config.tradingAdvisor.historySize === 0)
|
||||
return done();
|
||||
|
||||
var requiredHistory = config.tradingAdvisor.candleSize * config.tradingAdvisor.historySize;
|
||||
var Reader = require(dirs.plugins + config.adapter + '/reader');
|
||||
|
||||
this.reader = new Reader;
|
||||
|
||||
log.info(
|
||||
'\tThe trading method requests',
|
||||
requiredHistory,
|
||||
'minutes of historic data. Checking availablity..'
|
||||
);
|
||||
|
||||
var endTime = moment().utc().startOf('minute');
|
||||
var idealStartTime = endTime.clone().subtract(requiredHistory, 'm');
|
||||
|
||||
this.reader.mostRecentWindow(idealStartTime, endTime, function(localData) {
|
||||
// now we know what data is locally available, what
|
||||
// data would we need from the exchange?
|
||||
|
||||
if(!localData) {
|
||||
log.info('\tNo usable local data available, trying to get as much as possible from the exchange..');
|
||||
var idealExchangeStartTime = idealStartTime.clone();
|
||||
var idealExchangeStartTimeTS = idealExchangeStartTime.unix();
|
||||
}
|
||||
else if (idealStartTime.unix() < localData.from) {
|
||||
log.info('\tLocal data is still too recent, trying to get as much as possible from the exchange');
|
||||
var idealExchangeStartTime = idealStartTime.clone();
|
||||
var idealExchangeStartTimeTS = idealExchangeStartTime.unix();
|
||||
}
|
||||
else {
|
||||
log.debug('\tAvailable local data:');
|
||||
log.debug('\t\tfrom:', this.ago(localData.from));
|
||||
log.debug('\t\tto:', this.ago(localData.to));
|
||||
|
||||
log.info('\tUsable local data available, trying to match with exchange data..')
|
||||
// local data is available, we need the next minute
|
||||
|
||||
|
||||
// make sure we grab back in history far enough
|
||||
var secondsOverlap = 60 * 15; // 15 minutes
|
||||
var idealExchangeStartTimeTS = localData.to - secondsOverlap;
|
||||
var idealExchangeStartTime = moment.unix(idealExchangeStartTimeTS).utc();
|
||||
|
||||
// already set the
|
||||
util.setConfigProperty(
|
||||
'tradingAdvisor',
|
||||
'firstFetchSince',
|
||||
idealExchangeStartTimeTS
|
||||
);
|
||||
}
|
||||
|
||||
// Limit the history Gekko can try to get from the exchange.
|
||||
var minutesAgo = endTime.diff(idealExchangeStartTime, 'minutes');
|
||||
var maxMinutesAgo = 4 * 60; // 4 hours
|
||||
if(minutesAgo > maxMinutesAgo) {
|
||||
log.info('\tPreventing Gekko from requesting', minutesAgo, 'minutes of history.');
|
||||
idealExchangeStartTime = endTime.clone().subtract(maxMinutesAgo, 'minutes');
|
||||
idealExchangeStartTimeTS = idealExchangeStartTime.unix();
|
||||
}
|
||||
|
||||
log.debug('\tFetching exchange data since', this.ago(idealExchangeStartTimeTS))
|
||||
this.checkExchangeTrades(idealExchangeStartTime, function(err, exchangeData) {
|
||||
log.debug('\tAvailable exchange data:');
|
||||
log.debug('\t\tfrom:', this.ago(exchangeData.from));
|
||||
log.debug('\t\tto:', this.ago(exchangeData.to));
|
||||
|
||||
// in case we have limited local data, and the
|
||||
// exchange would offer more than we have: ignore
|
||||
// the local data..
|
||||
if(
|
||||
localData &&
|
||||
exchangeData &&
|
||||
exchangeData.from < localData.from
|
||||
) {
|
||||
log.debug('\tExchange offered more data than locally available. Ignoring local data');
|
||||
localData = false;
|
||||
}
|
||||
|
||||
var stitchable = localData && exchangeData.from <= localData.to;
|
||||
if(stitchable) {
|
||||
log.debug('\tStitching datasets');
|
||||
|
||||
// we can combine local data with exchange data
|
||||
if(idealStartTime.unix() >= localData.from) {
|
||||
log.info(
|
||||
'\tFull history locally available.',
|
||||
'Seeding the trading method with all required historical candles.'
|
||||
);
|
||||
|
||||
} else {
|
||||
|
||||
// stitchable but not enough
|
||||
|
||||
log.info(
|
||||
'\tPartial history locally available, but',
|
||||
Math.round((localData.from - idealStartTime.unix()) / 60),
|
||||
'minutes are missing.')
|
||||
log.info('\tSeeding the trading method with',
|
||||
'partial historical data (Gekko needs more time before',
|
||||
'it can give advice).'
|
||||
);
|
||||
}
|
||||
|
||||
// seed all historic data up to the point the exchange can provide.
|
||||
var from = localData.from;
|
||||
var to = moment.unix(exchangeData.from).utc()
|
||||
.startOf('minute')
|
||||
.subtract(1, 'minute')
|
||||
.unix();
|
||||
|
||||
log.debug('\tSeeding with:');
|
||||
log.debug('\t\tfrom:', this.ago(from));
|
||||
log.debug('\t\tto:', this.ago(to));
|
||||
return this.seedLocalData(from, to, done);
|
||||
|
||||
} else if(!stitchable) {
|
||||
log.debug('\tUnable to stitch datasets.')
|
||||
// we cannot use any local data..
|
||||
log.info(
|
||||
'\tNot seeding locally available data to the trading method.'
|
||||
);
|
||||
|
||||
if(exchangeData.from < idealExchangeStartTimeTS) {
|
||||
log.info('\tHowever the exchange returned enough data anyway!');
|
||||
} else if(localData) {
|
||||
log.info(
|
||||
'\tThe exchange does not return enough data.',
|
||||
Math.round((localData.from - idealStartTime.unix()) / 60),
|
||||
'minutes are still missing.'
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
done();
|
||||
|
||||
}.bind(this));
|
||||
}.bind(this));
|
||||
}
|
||||
|
||||
Stitcher.prototype.checkExchangeTrades = function(since, next) {
|
||||
var provider = config.watch.exchange.toLowerCase();
|
||||
var DataProvider = require(util.dirs().gekko + 'exchange/wrappers/' + provider);
|
||||
|
||||
var exchangeConfig = config.watch;
|
||||
|
||||
// include trader config if trading is enabled
|
||||
if (_.isObject(config.trader) && config.trader.enabled) {
|
||||
exchangeConfig = _.extend(config.watch, config.trader);
|
||||
}
|
||||
|
||||
var watcher = new DataProvider(exchangeConfig);
|
||||
watcher.getTrades(since, function(e, d) {
|
||||
if(e) {
|
||||
util.die(e.message);
|
||||
}
|
||||
|
||||
if(_.isEmpty(d))
|
||||
return util.die(
|
||||
`Gekko tried to retrieve data since ${since.format('YYYY-MM-DD HH:mm:ss')}, however
|
||||
${provider} did not return any trades.`
|
||||
);
|
||||
|
||||
next(e, {
|
||||
from: _.first(d).date,
|
||||
to: _.last(d).date
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
Stitcher.prototype.seedLocalData = function(from, to, next) {
|
||||
this.reader.get(from, to, 'full', function(err, rows) {
|
||||
rows = _.map(rows, row => {
|
||||
row.start = moment.unix(row.start);
|
||||
return row;
|
||||
});
|
||||
|
||||
this.batcher.write(rows);
|
||||
this.batcher.flush();
|
||||
this.reader.close();
|
||||
next();
|
||||
|
||||
}.bind(this));
|
||||
}
|
||||
|
||||
module.exports = Stitcher;
|
||||
@@ -0,0 +1,138 @@
|
||||
var BATCH_SIZE = 60; // minutes
|
||||
var MISSING_CANDLES_ALLOWED = 3; // minutes, per batch
|
||||
|
||||
var _ = require('lodash');
|
||||
var moment = require('moment');
|
||||
var async = require('async');
|
||||
|
||||
var util = require('../util');
|
||||
var config = util.getConfig();
|
||||
var dirs = util.dirs();
|
||||
var log = require(dirs.core + 'log');
|
||||
|
||||
var adapter = config[config.adapter];
|
||||
var Reader = require(dirs.gekko + adapter.path + '/reader');
|
||||
|
||||
var reader = new Reader();
|
||||
|
||||
// todo: rewrite with generators or async/await..
|
||||
var scan = function(done) {
|
||||
log.info('Scanning local history for backtestable dateranges.');
|
||||
|
||||
reader.tableExists('candles', (err, exists) => {
|
||||
|
||||
if(err)
|
||||
return done(err, null, reader);
|
||||
|
||||
if(!exists)
|
||||
return done(null, [], reader);
|
||||
|
||||
async.parallel({
|
||||
boundry: reader.getBoundry,
|
||||
available: reader.countTotal
|
||||
}, (err, res) => {
|
||||
|
||||
var first = res.boundry.first;
|
||||
var last = res.boundry.last;
|
||||
|
||||
var optimal = (last - first) / 60;
|
||||
|
||||
log.debug('Available', res.available);
|
||||
log.debug('Optimal', optimal);
|
||||
|
||||
// There is a candle for every minute
|
||||
if(res.available === optimal + 1) {
|
||||
log.info('Gekko is able to fully use the local history.');
|
||||
return done(false, [{
|
||||
from: first,
|
||||
to: last
|
||||
}], reader);
|
||||
}
|
||||
|
||||
// figure out where the gaps are..
|
||||
|
||||
var missing = optimal - res.available + 1;
|
||||
|
||||
log.info(`The database has ${missing} candles missing, Figuring out which ones...`);
|
||||
|
||||
var iterator = {
|
||||
from: last - (BATCH_SIZE * 60),
|
||||
to: last
|
||||
}
|
||||
|
||||
var batches = [];
|
||||
|
||||
// loop through all candles we have
|
||||
// in batches and track whether they
|
||||
// are complete
|
||||
async.whilst(
|
||||
() => {
|
||||
return iterator.from > first
|
||||
},
|
||||
next => {
|
||||
var from = iterator.from;
|
||||
var to = iterator.to;
|
||||
reader.count(
|
||||
from,
|
||||
iterator.to,
|
||||
(err, count) => {
|
||||
var complete = count + MISSING_CANDLES_ALLOWED > BATCH_SIZE;
|
||||
|
||||
if(complete)
|
||||
batches.push({
|
||||
to: to,
|
||||
from: from
|
||||
});
|
||||
|
||||
next();
|
||||
}
|
||||
);
|
||||
|
||||
iterator.from -= BATCH_SIZE * 60;
|
||||
iterator.to -= BATCH_SIZE * 60;
|
||||
},
|
||||
() => {
|
||||
if(batches.length === 0) {
|
||||
return done(null, [], reader);
|
||||
}
|
||||
|
||||
// batches is now a list like
|
||||
// [ {from: unix, to: unix } ]
|
||||
|
||||
var ranges = [ batches.shift() ];
|
||||
|
||||
_.each(batches, batch => {
|
||||
var curRange = _.last(ranges);
|
||||
if(batch.to === curRange.from)
|
||||
curRange.from = batch.from;
|
||||
else
|
||||
ranges.push( batch );
|
||||
})
|
||||
|
||||
// we have been counting chronologically reversed
|
||||
// (backwards, from now into the past), flip definitions
|
||||
ranges = ranges.reverse();
|
||||
|
||||
_.map(ranges, r => {
|
||||
return {
|
||||
from: r.to,
|
||||
to: r.from
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
// ranges is now a list like
|
||||
// [ {from: unix, to: unix } ]
|
||||
//
|
||||
// it contains all valid dataranges available for the
|
||||
// end user.
|
||||
|
||||
return done(false, ranges, reader);
|
||||
}
|
||||
)
|
||||
});
|
||||
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = scan;
|
||||
Reference in new issue
Block a user